fix(#2961): windows: escape brackets and parentheses when opening file (#2962)

* Revert "fix(#2862): windows path replaces backslashes with forward slashes (#2903)"

This reverts commit 45a93d9.

* fix the case when '()' and '[]' are both in file path

* remove debug messages

* remove unnecessary comments

* add is_windows feature flag when normalizing path

* add is_windows flag for filename change

* Revert "add is_windows flag for filename change"

This reverts commit ada77cb.

---------

Co-authored-by: Alexander Courtis <alex@courtis.org>

Author: ljie-PI

lua/nvim-tree/actions/node/open-file.lua

@@ -331,9 +331,9 @@ local function open_in_new_window(filename, mode)
 
   local fname
   if M.relative_path then
-    fname = vim.fn.fnameescape(utils.path_relative(filename, vim.fn.getcwd()))
+    fname = utils.escape_special_chars(vim.fn.fnameescape(utils.path_relative(filename, vim.fn.getcwd())))
   else
-    fname = vim.fn.fnameescape(filename)
+    fname = utils.escape_special_chars(vim.fn.fnameescape(filename))
   end
 
   local command
@@ -370,36 +370,35 @@ end
 ---@param mode string
 ---@param filename string
 function M.fn(mode, filename)
-  local fname = utils.escape_special_chars(filename)
   if type(mode) ~= "string" then
     mode = ""
   end
 
   if mode == "tabnew" then
-    return open_file_in_tab(fname)
+    return open_file_in_tab(filename)
   end
 
   if mode == "drop" then
-    return drop(fname)
+    return drop(filename)
   end
 
   if mode == "tab_drop" then
-    return tab_drop(fname)
+    return tab_drop(filename)
   end
 
   if mode == "edit_in_place" then
-    return edit_in_current_buf(fname)
+    return edit_in_current_buf(filename)
   end
 
-  local buf_loaded = is_already_loaded(fname)
+  local buf_loaded = is_already_loaded(filename)
 
   local found_win = utils.get_win_buf_from_path(filename)
   if found_win and (mode == "preview" or mode == "preview_no_picker") then
     return
   end
 
   if not found_win then
-    open_in_new_window(fname, mode)
+    open_in_new_window(filename, mode)
   else
     vim.api.nvim_set_current_win(found_win)
     vim.bo.bufhidden = ""

lua/nvim-tree/utils.lua

@@ -59,6 +59,17 @@ function M.path_basename(path)
   return path:sub(i + 1, #path)
 end
 
+--- Check if there are parentheses before brackets, it causes problems for windows.
+--- Refer to issue #2862 and #2961 for more details.
+local function has_parentheses_and_brackets(path)
+  local _, i_parentheses = path:find("(", 1, true)
+  local _, i_brackets = path:find("[", 1, true)
+  if i_parentheses and i_brackets then
+    return true
+  end
+  return false
+end
+
 --- Get a path relative to another path.
 ---@param path string
 ---@param relative_to string|nil
@@ -68,13 +79,18 @@ function M.path_relative(path, relative_to)
     return path
   end
 
-  local _, r = path:find(M.path_add_trailing(relative_to), 1, true)
-  local p = path
+  local norm_path = path
+  if M.is_windows and has_parentheses_and_brackets(path) then
+    norm_path = path:gsub("/", "\\")
+  end
+
+  local _, r = norm_path:find(M.path_add_trailing(relative_to), 1, true)
+  local p = norm_path
   if r then
     -- take the relative path starting after '/'
     -- if somehow given a completely matching path,
     -- returns ""
-    p = path:sub(r + 1)
+    p = norm_path:sub(r + 1)
   end
   return p
 end
@@ -272,14 +288,22 @@ function M.canonical_path(path)
   return path
 end
 
+--- Escapes special characters in string for windows, refer to issue #2862 and #2961 for more details.
+local function escape_special_char_for_windows(path)
+  if has_parentheses_and_brackets(path) then
+    return path:gsub("\\", "/"):gsub("/ ", "\\ ")
+  end
+  return path:gsub("%(", "\\("):gsub("%)", "\\)")
+end
+
 --- Escapes special characters in string if windows else returns unmodified string.
 ---@param path string
 ---@return string|nil
 function M.escape_special_chars(path)
   if path == nil then
     return path
   end
-  return M.is_windows and path:gsub("\\", "/") or path
+  return M.is_windows and escape_special_char_for_windows(path) or path
 end
 
 --- Create empty sub-tables if not present