feat(files): Use TS string parser for non-nightly versions (#1060)

* feat(files): Use TS string parser for non-nightly versions

* feat(files): Update changed metadata after updating lines

Author: kristijanhusak

lua/orgmode/colors/highlighter/markup/init.lua

@@ -169,7 +169,7 @@ end
 ---@return OrgMarkupPreparedHighlight[]
 function OrgMarkup:get_prepared_headline_highlights(headline)
   local highlights =
-    self:get_node_highlights(headline:node(), headline.file:bufnr(), select(1, headline:node():range()))
+    self:get_node_highlights(headline:node(), headline.file:get_source(), select(1, headline:node():range()))
 
   local result = {}
 

lua/orgmode/files/file.lua

@@ -9,6 +9,7 @@ local Hyperlink = require('orgmode.org.links.hyperlink')
 local Range = require('orgmode.files.elements.range')
 local Footnote = require('orgmode.objects.footnote')
 local Memoize = require('orgmode.utils.memoize')
+local is_nightly = vim.fn.has('nvim-0.12') > 0
 
 ---@class OrgFileMetadata
 ---@field mtime number File modified time in nanoseconds
@@ -17,13 +18,15 @@ local Memoize = require('orgmode.utils.memoize')
 
 ---@class OrgFileOpts
 ---@field filename string
+---@field lines? string[]
 ---@field buf? number
 
 ---@class OrgFile
 ---@field filename string
 ---@field buf number
 ---@field index number
 ---@field lines string[]
+---@field content string
 ---@field metadata OrgFileMetadata
 ---@field parser vim.treesitter.LanguageTree
 ---@field root TSNode
@@ -45,18 +48,19 @@ function OrgFile:new(opts)
     filename = opts.filename,
     index = 0,
     buf = opts.buf or -1,
-    lines = {},
+    lines = opts.lines or {},
+    content = table.concat(opts.lines or {}, '\n'),
     metadata = {
       mtime = stat and stat.mtime.nsec or 0,
       mtime_sec = stat and stat.mtime.sec or 0,
       changedtick = opts.buf and vim.api.nvim_buf_get_changedtick(opts.buf) or 0,
     },
   }
-  if data.buf > 0 then
-    data.lines = self:_get_lines(data.buf)
+  local this = setmetatable(data, self)
+  if this.buf > 0 then
+    this:_update_lines(this:_get_lines(this.buf))
   end
-  setmetatable(data, self)
-  return data
+  return this
 end
 
 ---Load the file
@@ -75,12 +79,23 @@ function OrgFile.load(filename)
     return Promise.resolve(false)
   end
 
-  bufnr = OrgFile._load_buffer(filename)
+  -- TODO: Remove once Neovim adds string parser back
+  -- See: https://github.com/nvim-orgmode/orgmode/issues/1049
+  if is_nightly then
+    bufnr = OrgFile._load_buffer(filename)
 
-  return Promise.resolve(OrgFile:new({
-    filename = filename,
-    buf = bufnr,
-  }))
+    return Promise.resolve(OrgFile:new({
+      filename = filename,
+      buf = bufnr,
+    }))
+  end
+
+  return utils.readfile(filename, { schedule = true }):next(function(lines)
+    return OrgFile:new({
+      filename = filename,
+      lines = lines,
+    })
+  end)
 end
 
 ---Reload the file if it has been modified
@@ -94,12 +109,12 @@ function OrgFile:reload()
   local buf_changed = false
   local file_changed = false
 
-  if bufnr then
+  if bufnr > -1 then
     local new_changedtick = vim.api.nvim_buf_get_changedtick(bufnr)
     buf_changed = self.metadata.changedtick ~= new_changedtick
-    self.metadata.changedtick = new_changedtick
     if buf_changed then
-      self.lines = self:_get_lines(bufnr)
+      self:_update_lines(self:_get_lines(bufnr))
+      self.metadata.changedtick = new_changedtick
     end
   end
   local stat = vim.uv.fs_stat(self.filename)
@@ -108,13 +123,15 @@ function OrgFile:reload()
     local new_mtime_sec = stat.mtime.sec
     file_changed = (new_mtime_nsec > 0 and self.metadata.mtime ~= new_mtime_nsec)
       or self.metadata.mtime_sec ~= new_mtime_sec
-    self.metadata.mtime = new_mtime_nsec
-    self.metadata.mtime_sec = new_mtime_sec
   end
 
   if file_changed and not buf_changed then
     return utils.readfile(self.filename, { schedule = true }):next(function(lines)
-      self.lines = lines
+      self:_update_lines(lines)
+      if stat then
+        self.metadata.mtime = stat.mtime.nsec
+        self.metadata.mtime_sec = stat.mtime.sec
+      end
       return self
     end)
   end
@@ -184,7 +201,7 @@ function OrgFile:parse(skip_if_not_modified)
   if skip_if_not_modified and self.root and not self:is_modified() then
     return self.root
   end
-  self.parser = ts.get_parser(self:bufnr(), 'org', {})
+  self.parser = self:_get_parser()
   local trees = self.parser:parse()
   self.root = trees[1]:root()
   return self.root
@@ -203,7 +220,7 @@ function OrgFile:get_ts_matches(query, parent_node)
   local ts_query = ts_utils.get_query(query)
   local matches = {}
 
-  for _, match, _ in ts_query:iter_matches(parent_node, self:bufnr(), nil, nil, { all = true }) do
+  for _, match, _ in ts_query:iter_matches(parent_node, self:get_source(), nil, nil, { all = true }) do
     local items = {}
     for id, nodes in pairs(match) do
       local name = ts_query.captures[id]
@@ -233,7 +250,7 @@ function OrgFile:get_ts_captures(query, node)
   local ts_query = ts_utils.get_query(query)
   local matches = {}
 
-  for _, match in ts_query:iter_captures(node, self:bufnr()) do
+  for _, match in ts_query:iter_captures(node, self:get_source()) do
     table.insert(matches, match)
   end
   return matches
@@ -489,13 +506,13 @@ function OrgFile:get_node_text(node, range)
     return ''
   end
   if range then
-    return ts.get_node_text(node, self:bufnr(), {
+    return ts.get_node_text(node, self:get_source(), {
       metadata = {
         range = range,
       },
     })
   end
-  return ts.get_node_text(node, self:bufnr())
+  return ts.get_node_text(node, self:get_source())
 end
 
 ---@param node? TSNode
@@ -557,15 +574,27 @@ end
 
 ---@return number
 function OrgFile:bufnr()
-  local bufnr = self.buf
+  -- TODO: Remove once Neovim adds string parser back
+  -- See: https://github.com/nvim-orgmode/orgmode/issues/1049
+  if is_nightly then
+    local bufnr = self.buf
+    -- Do not consider unloaded buffers as valid
+    -- Treesitter is not working in them
+    if bufnr > -1 and vim.api.nvim_buf_is_loaded(bufnr) then
+      return bufnr
+    end
+    local new_bufnr = self._load_buffer(self.filename)
+    self.buf = new_bufnr
+    return new_bufnr
+  end
+
+  local bufnr = utils.get_buffer_by_filename(self.filename)
   -- Do not consider unloaded buffers as valid
   -- Treesitter is not working in them
   if bufnr > -1 and vim.api.nvim_buf_is_loaded(bufnr) then
     return bufnr
   end
-  local new_bufnr = self._load_buffer(self.filename)
-  self.buf = new_bufnr
-  return new_bufnr
+  return -1
 end
 
 ---@private
@@ -819,7 +848,7 @@ function OrgFile:get_links()
     (link_desc) @link
   ]])
 
-  local source = self:bufnr()
+  local source = self:get_source()
   for _, node in ipairs(matches) do
     table.insert(links, Hyperlink.from_node(node, source))
   end
@@ -840,7 +869,7 @@ function OrgFile:get_footnote_references()
 
   local footnotes = {}
   local processed_lines = {}
-  for _, match in ts_query:iter_captures(self.root, self:bufnr()) do
+  for _, match in ts_query:iter_captures(self.root, self:get_source()) do
     local line_start, _, line_end = match:range()
     if not processed_lines[line_start] then
       if line_start == line_end then
@@ -947,6 +976,13 @@ function OrgFile:_get_directive(directive_name, all_matches)
   return nil
 end
 
+function OrgFile:_update_lines(lines)
+  self.lines = lines
+  self.content = table.concat(lines, '\n')
+  self:parse()
+  return self
+end
+
 ---@private
 ---Get all buffer lines, ensure empty buffer returns empty table
 ---@return string[]
@@ -958,4 +994,34 @@ function OrgFile:_get_lines(bufnr)
   return lines
 end
 
+---@private
+---@return vim.treesitter.LanguageTree
+function OrgFile:_get_parser()
+  local bufnr = self:bufnr()
+
+  if bufnr > -1 then
+    -- Always get the fresh parser for the buffer
+    return ts.get_parser(bufnr, 'org', {})
+  end
+
+  -- In case the buffer got unloaded, go back to string parser
+  if not self.parser or self:is_modified() or type(self.parser:source()) == 'number' then
+    return ts.get_string_parser(self.content, 'org', {})
+  end
+
+  return self.parser
+end
+
+--- Get the ts source for the file
+--- If there is a buffer, return buffer number
+--- Otherwise, return the string content
+---@return integer | string
+function OrgFile:get_source()
+  local bufnr = self:bufnr()
+  if bufnr > -1 then
+    return bufnr
+  end
+  return self.content
+end
+
 return OrgFile

lua/orgmode/files/headline.lua

@@ -738,7 +738,7 @@ function Headline:get_plan_dates()
     if name ~= 'NONE' then
       has_plan_dates = true
     end
-    dates[name:upper()] = Date.from_node(timestamp, self.file:bufnr(), {
+    dates[name:upper()] = Date.from_node(timestamp, self.file:get_source(), {
       type = name:upper(),
     })
     dates_nodes[name:upper()] = node
@@ -792,7 +792,7 @@ function Headline:get_non_plan_dates()
   end
 
   local all_dates = {}
-  local source = self.file:bufnr()
+  local source = self.file:get_source()
   for _, match in ipairs(matches) do
     local dates = Date.from_node(match, source)
     vim.list_extend(all_dates, dates)

tests/plenary/files/file_spec.lua

@@ -1,6 +1,9 @@
 local OrgFile = require('orgmode.files.file')
 local config = require('orgmode.config')
 local Range = require('orgmode.files.elements.range')
+-- TODO: Remove once Neovim adds string parser back
+-- See: https://github.com/nvim-orgmode/orgmode/issues/1049
+local is_nightly = vim.fn.has('nvim-0.12') > 0
 
 describe('OrgFile', function()
   ---@return OrgFile
@@ -20,7 +23,11 @@ describe('OrgFile', function()
       local stat = vim.uv.fs_stat(filename) or {}
       assert.are.same(stat.mtime.nsec, file.metadata.mtime)
       assert.are.same(stat.mtime.sec, file.metadata.mtime_sec)
-      assert.are.same(2, file.metadata.changedtick)
+      if is_nightly then
+        assert.are.same(2, file.metadata.changedtick)
+      else
+        assert.are.same(0, file.metadata.changedtick)
+      end
     end)
 
     it('should not load a file that is not an org file', function()
@@ -39,10 +46,18 @@ describe('OrgFile', function()
       local stat = vim.uv.fs_stat(filename) or {}
       assert.are.same(stat.mtime.nsec, file.metadata.mtime)
       assert.are.same(stat.mtime.sec, file.metadata.mtime_sec)
-      assert.are.same(2, file.metadata.changedtick)
+      if is_nightly then
+        assert.are.same(2, file.metadata.changedtick)
+      else
+        assert.are.same(0, file.metadata.changedtick)
+      end
       vim.cmd('write!')
       file:reload_sync()
-      assert.are.same(2, file.metadata.changedtick)
+      if is_nightly then
+        assert.are.same(2, file.metadata.changedtick)
+      else
+        assert.are.same(4, file.metadata.changedtick)
+      end
     end)
 
     it('should load files with special characters in filename from buffer', function()
@@ -477,6 +492,20 @@ describe('OrgFile', function()
   end)
 
   describe('set_node_text', function()
+    if not is_nightly then
+      it('should throw an error if file is not loaded in buffer', function()
+        local file = load_file_sync({
+          '* Headline 1 :TAG:',
+          '  The content',
+          '  Multi line',
+        })
+        local paragraph_node = file:get_node_at_cursor():parent()
+        assert.is.error_matches(function()
+          return file:set_node_text(paragraph_node, 'New Text')
+        end, '%[orgmode%] No valid buffer for file ' .. file.filename .. ' to edit')
+      end)
+    end
+
     it('should set node text', function()
       local file = load_file_sync({
         '* Headline 1 :TAG:',
@@ -530,6 +559,30 @@ describe('OrgFile', function()
   end)
 
   describe('bufnr', function()
+    if not is_nightly then
+      it('should return -1 if there is no buffer', function()
+        local file = load_file_sync({
+          '* Headline 1 :TAG:',
+          '  The content',
+          '  Multi line',
+        })
+        assert.are.same(-1, file:bufnr())
+      end)
+
+      it('should return -1 if file is loaded in buffer but buffer is not loaded', function()
+        local file = load_file_sync({
+          '* Headline 1 :TAG:',
+          '  The content',
+          '  Multi line',
+        })
+        vim.cmd('edit ' .. file.filename)
+        assert.is.True(file:bufnr() > 0)
+        vim.cmd('bdelete')
+        assert.are.same(-1, file:bufnr())
+        assert.is.True(vim.fn.bufnr(file.filename) > 0)
+      end)
+    end
+
     it('should return buffer number if file is loaded', function()
       local file = load_file_sync({
         '* Headline 1 :TAG:',