Merge pull request #190 from glennsarti/spike-extract-bolt-info

(GH-94) Extract Bolt module metadata and use within Plans

Author: jpogran

lib/puppet-languageserver/manifest/completion_provider.rb

@@ -27,7 +27,7 @@ def self.complete(content, line_num, char_num, options = {})
           # Add resources
           all_resources { |x| items << x }
 
-          all_functions { |x| items << x }
+          all_functions(options[:tasks_mode]) { |x| items << x }
 
           response = LSP::CompletionList.new
           response.items = items
@@ -53,6 +53,14 @@ def self.complete(content, line_num, char_num, options = {})
           # Add resources
           all_resources { |x| items << x }
 
+        when 'Puppet::Pops::Model::PlanDefinition'
+          # We are in the root of a `plan` statement
+
+          # Add resources
+          all_resources { |x| items << x }
+
+          all_functions(options[:tasks_mode]) { |x| items << x }
+
         when 'Puppet::Pops::Model::ResourceExpression'
           # We are inside a resource definition.  Should display all available
           # properities and parameters.
@@ -176,8 +184,8 @@ def self.all_resources(&block)
         end
       end
 
-      def self.all_functions(&block)
-        PuppetLanguageServer::PuppetHelper.function_names.each do |name|
+      def self.all_functions(tasks_mode, &block)
+        PuppetLanguageServer::PuppetHelper.function_names(tasks_mode).each do |name|
           item = LSP::CompletionItem.new(
             'label'  => name.to_s,
             'kind'   => LSP::CompletionItemKind::FUNCTION,
@@ -234,7 +242,8 @@ def self.resolve(completion_item)
           end
 
         when 'function'
-          item_type = PuppetLanguageServer::PuppetHelper.function(data['name'])
+          # We don't know if this resolution is coming from a plan or not, so just assume it is
+          item_type = PuppetLanguageServer::PuppetHelper.function(data['name'], true)
           return result if item_type.nil?
           result.documentation = item_type.doc unless item_type.doc.nil?
           unless item_type.nil? || item_type.signatures.count.zero?

lib/puppet-languageserver/manifest/hover_provider.rb

@@ -33,7 +33,7 @@ def self.resolve(content, line_num, char_num, options = {})
 
           content = get_hover_content_for_access_expression(path, expr)
         when 'Puppet::Pops::Model::CallNamedFunctionExpression'
-          content = get_call_named_function_expression_content(item)
+          content = get_call_named_function_expression_content(item, options[:tasks_mode])
         when 'Puppet::Pops::Model::AttributeOperation'
           # Get the parent resource class
           distance_up_ast = -1
@@ -66,7 +66,7 @@ def self.resolve(content, line_num, char_num, options = {})
           # https://github.com/puppetlabs/puppet-specifications/blob/master/language/names.md#names
           # Datatypes have to start with uppercase and can be fully qualified
           if item.cased_value =~ /^[A-Z][a-zA-Z:0-9]*$/ # rubocop:disable Style/GuardClause
-            content = get_puppet_datatype_content(item)
+            content = get_puppet_datatype_content(item, options[:tasks_mode])
           else
             raise "#{item.cased_value} is an unknown QualifiedReference"
           end
@@ -143,10 +143,10 @@ def self.get_attribute_class_parameter_content(item_class, param)
         content
       end
 
-      def self.get_call_named_function_expression_content(item)
+      def self.get_call_named_function_expression_content(item, tasks_mode)
         func_name = item.functor_expr.value
 
-        func_info = PuppetLanguageServer::PuppetHelper.function(func_name)
+        func_info = PuppetLanguageServer::PuppetHelper.function(func_name, tasks_mode)
         raise "Function #{func_name} does not exist" if func_info.nil?
 
         content = "**#{func_name}** Function"
@@ -189,8 +189,8 @@ def self.get_puppet_class_content(item_class)
       end
       private_class_method :get_puppet_class_content
 
-      def self.get_puppet_datatype_content(item)
-        dt_info = PuppetLanguageServer::PuppetHelper.datatype(item.cased_value)
+      def self.get_puppet_datatype_content(item, tasks_mode)
+        dt_info = PuppetLanguageServer::PuppetHelper.datatype(item.cased_value, tasks_mode)
         raise "DataType #{item.cased_value} does not exist" if dt_info.nil?
 
         content = "**#{item.cased_value}** Data Type"

lib/puppet-languageserver/puppet_helper.rb

@@ -65,6 +65,48 @@ def self.get_puppet_resource(typename, title, local_workspace)
       sidecar_queue.execute_sync('resource_list', args)
     end
 
+    # Static data
+    def self.static_data_loaded?
+      @static_data_loaded.nil? ? false : @static_data_loaded
+    end
+
+    def self.load_static_data
+      raise('Puppet Helper Cache has not been configured') if @inmemory_cache.nil?
+      @static_data_loaded = false
+
+      bolt_static_data = PuppetLanguageServer::Sidecar::Protocol::AggregateMetadata.new
+      Dir.glob(File.join(PuppetLanguageServer.static_data_dir, 'bolt-*.json')) do |path|
+        PuppetLanguageServer.log_message(:debug, "Importing static data file #{path}...")
+        # No need to catch errors here. As this is static data and is tested in rspec
+        # Sure, we could have corrupt/missing files on disk, but then we have bigger issues
+        data = PuppetLanguageServer::Sidecar::Protocol::AggregateMetadata.new.from_json!(File.open(path, 'rb:UTF-8') { |f| f.read })
+        data.each_list { |_, list| bolt_static_data.concat!(list) }
+      end
+
+      @inmemory_cache.import_sidecar_list!(bolt_static_data.classes,   :class,    :bolt)
+      @inmemory_cache.import_sidecar_list!(bolt_static_data.datatypes, :datatype, :bolt)
+      @inmemory_cache.import_sidecar_list!(bolt_static_data.functions, :function, :bolt)
+      @inmemory_cache.import_sidecar_list!(bolt_static_data.types,     :type,     :bolt)
+
+      bolt_static_data.each_list do |k, v|
+        if v.nil?
+          PuppetLanguageServer.log_message(:debug, "Static bolt data returned no #{k}")
+        else
+          PuppetLanguageServer.log_message(:debug, "Static bolt data returned #{v.count} #{k}")
+        end
+      end
+
+      @static_data_loaded = true
+    end
+
+    def self.load_static_data_async
+      raise('Puppet Helper Cache has not been configured') if @inmemory_cache.nil?
+      @static_data_loaded = false
+      Thread.new do
+        load_static_data
+      end
+    end
+
     # Types
     def self.default_types_loaded?
       @default_types_loaded.nil? ? false : @default_types_loaded
@@ -131,18 +173,25 @@ def self.filtered_function_names(&block)
       result
     end
 
-    def self.function(name)
+    def self.function(name, tasks_mode = false)
       return nil if @default_functions_loaded == false
       raise('Puppet Helper Cache has not been configured') if @inmemory_cache.nil?
       load_default_functions unless @default_functions_loaded
-      @inmemory_cache.object_by_name(:function, name)
+      exclude_origins = tasks_mode ? [] : [:bolt]
+      @inmemory_cache.object_by_name(
+        :function,
+        name,
+        :fuzzy_match     => true,
+        :exclude_origins => exclude_origins
+      )
     end
 
-    def self.function_names
+    def self.function_names(tasks_mode = false)
       return [] if @default_functions_loaded == false
       raise('Puppet Helper Cache has not been configured') if @inmemory_cache.nil?
       load_default_functions if @default_functions_loaded.nil?
-      @inmemory_cache.object_names_by_section(:function).map(&:to_s)
+      exclude_origins = tasks_mode ? [] : [:bolt]
+      @inmemory_cache.object_names_by_section(:function, :exclude_origins => exclude_origins).map(&:to_s)
     end
 
     # Classes and Defined Types
@@ -200,11 +249,18 @@ def self.load_default_datatypes_async
       sidecar_queue.enqueue('default_datatypes', [])
     end
 
-    def self.datatype(name)
-      return nil if @default_functions_loaded == false
+    def self.datatype(name, tasks_mode = false)
+      return nil if @default_datatypes_loaded == false
       raise('Puppet Helper Cache has not been configured') if @inmemory_cache.nil?
-      load_default_datatypes unless @default_functions_loaded
-      @inmemory_cache.object_by_name(:datatype, name)
+      load_default_datatypes unless @default_datatypes_loaded
+      load_static_data if tasks_mode && !static_data_loaded?
+      exclude_origins = tasks_mode ? [] : [:bolt]
+      @inmemory_cache.object_by_name(
+        :datatype,
+        name,
+        :fuzzy_match     => true,
+        :exclude_origins => exclude_origins
+      )
     end
 
     def self.cache

lib/puppet-languageserver/puppet_helper/cache.rb

@@ -4,7 +4,7 @@ module PuppetLanguageServer
   module PuppetHelper
     class Cache
       SECTIONS = %i[class type function datatype].freeze
-      ORIGINS = %i[default workspace].freeze
+      ORIGINS = %i[default workspace bolt].freeze
 
       def initialize(_options = {})
         @cache_lock = Mutex.new
@@ -36,27 +36,60 @@ def remove_section!(section, origin = nil)
       end
 
       # section => <Type of object in the file :function, :type, :class, :datatype>
-      def object_by_name(section, name)
+      def object_by_name(section, name, options = {})
+        # options[:exclude_origins]
+        # options[:fuzzy_match]
+        options = {
+          :exclude_origins => [],
+          :fuzzy_match     => false
+        }.merge(options)
+
         name = name.intern if name.is_a?(String)
         return nil if section.nil?
         @cache_lock.synchronize do
-          @inmemory_cache.each do |_, sections|
+          @inmemory_cache.each do |origin, sections|
             next if sections[section].nil? || sections[section].empty?
+            next if options[:exclude_origins].include?(origin)
             sections[section].each do |item|
-              return item if item.key == name
+              match = options[:fuzzy_match] ? fuzzy_match?(item.key, name) : item.key == name
+              return item if match
             end
           end
         end
         nil
       end
 
+      # Performs fuzzy text matching of Puppet Language Type names
+      #  e.g 'TargetSpec' in 'Boltlib::TargetSpec'
+      # @api private
+      def fuzzy_match?(obj, test_obj)
+        value = obj.is_a?(String) ? obj.dup : obj.to_s
+        test_string = test_obj.is_a?(String) ? test_obj.dup : test_obj.to_s
+
+        # Test for equality
+        return true if value == test_string
+
+        # Test for a shortname
+        unless test_string.start_with?('::')
+          # e.g 'TargetSpec' in 'Boltlib::TargetSpec'
+          return true if value.end_with?('::' + test_string)
+        end
+
+        false
+      end
+
       # section => <Type of object in the file :function, :type, :class, :datatype>
-      def object_names_by_section(section)
+      # options[:exclude_origins]
+      def object_names_by_section(section, options = {})
+        options = {
+          :exclude_origins => []
+        }.merge(options)
         result = []
         return result if section.nil?
         @cache_lock.synchronize do
-          @inmemory_cache.each do |_, sections|
+          @inmemory_cache.each do |origin, sections|
             next if sections[section].nil? || sections[section].empty?
+            next if options[:exclude_origins].include?(origin)
             result.concat(sections[section].map { |i| i.key })
           end
         end

lib/puppet-languageserver/sidecar_protocol.rb

@@ -17,11 +17,11 @@ class ActionParams < Hash
         include Base
 
         def to_json(*options)
-          JSON.generate(to_h, options)
+          ::JSON.generate(to_h, options)
         end
 
         def from_json!(json_string)
-          obj = JSON.parse(json_string)
+          obj = ::JSON.parse(json_string)
           obj.each do |key, value|
             self[key] = value
           end
@@ -37,7 +37,7 @@ def to_json(*options)
         end
 
         def from_json!(json_string)
-          from_h!(JSON.parse(json_string))
+          from_h!(::JSON.parse(json_string))
         end
 
         def ==(other)
@@ -128,7 +128,7 @@ def to_json(*options)
         end
 
         def from_json!(json_string)
-          obj = JSON.parse(json_string)
+          obj = ::JSON.parse(json_string)
           obj.each do |child_hash|
             child = child_type.new
             self << child.from_h!(child_hash)
@@ -153,7 +153,7 @@ def to_json(*options)
         end
 
         def from_json!(json_string)
-          obj = JSON.parse(json_string)
+          obj = ::JSON.parse(json_string)
           self.dot_content = obj['dot_content']
           self.error_content = obj['error_content']
           self
@@ -421,7 +421,7 @@ def to_json(*options)
         end
 
         def from_json!(json_string)
-          from_h!(JSON.parse(json_string))
+          from_h!(::JSON.parse(json_string))
         end
       end
 
@@ -470,7 +470,7 @@ def to_json(*options)
         end
 
         def from_json!(json_string)
-          obj = JSON.parse(json_string)
+          obj = ::JSON.parse(json_string)
           obj.each do |key, value|
             info = METADATA_LIST[key.intern]
             next if info.nil?

lib/puppet-languageserver/static_data/bolt-aggregate.json

@@ -0,0 +1 @@
+{"functions":[{"key":"aggregate::count","calling_source":null,"source":null,"line":null,"char":null,"length":null,"doc":"Aggregates the key/value pairs in the results of a ResultSet into a hash\nmapping the keys to a hash of each distinct value and how many nodes returned\nthat value for the key.","function_version":4,"signatures":[{"key":"aggregate::count(ResultSet $resultset)","doc":"Aggregates the key/value pairs in the results of a ResultSet into a hash\nmapping the keys to a hash of each distinct value and how many nodes returned\nthat value for the key.","return_types":["Any"],"parameters":[{"name":"resultset","doc":"","types":["ResultSet"],"signature_key_offset":27,"signature_key_length":10}]}]},{"key":"aggregate::nodes","calling_source":null,"source":null,"line":null,"char":null,"length":null,"doc":"Aggregates the key/value pairs in the results of a ResultSet into a hash\nmapping the keys to a hash of each distinct value and the list of nodes\nreturning that value for the key.","function_version":4,"signatures":[{"key":"aggregate::nodes(ResultSet $resultset)","doc":"Aggregates the key/value pairs in the results of a ResultSet into a hash\nmapping the keys to a hash of each distinct value and the list of nodes\nreturning that value for the key.","return_types":["Any"],"parameters":[{"name":"resultset","doc":"","types":["ResultSet"],"signature_key_offset":27,"signature_key_length":10}]}]}]}

lib/puppet-languageserver/static_data/bolt-boltlib.json

@@ -0,0 +1 @@
+{"functions":[{"key":"canary::merge","calling_source":null,"source":null,"line":null,"char":null,"length":null,"doc":"Merges two ResultSets into a new ResultSet","function_version":4,"signatures":[{"key":"canary::merge(ResultSet $merger, ResultSet $mergee)","doc":"Merges two ResultSets into a new ResultSet","return_types":["Any"],"parameters":[{"name":"merger","doc":"","types":["ResultSet"],"signature_key_offset":24,"signature_key_length":7},{"name":"mergee","doc":"","types":["ResultSet"],"signature_key_offset":43,"signature_key_length":7}]}]},{"key":"canary::random_split","calling_source":null,"source":null,"line":null,"char":null,"length":null,"doc":"Splits an array into two groups, where the 1st group is a randomly selected\nsample of the input (of the specified size) and the 2nd group is the remainder.\n\nThis function takes 2 parameters:\n* The array to split (Array)\n* The number of items to sample from the array (Integer)\n\nReturns an array of [<sample>, <remainder>].","function_version":4,"signatures":[{"key":"canary::random_split(Array $arr, Integer $size)","doc":"Splits an array into two groups, where the 1st group is a randomly selected\nsample of the input (of the specified size) and the 2nd group is the remainder.\n\nThis function takes 2 parameters:\n* The array to split (Array)\n* The number of items to sample from the array (Integer)\n\nReturns an array of [<sample>, <remainder>].","return_types":["Any"],"parameters":[{"name":"arr","doc":"","types":["Array"],"signature_key_offset":27,"signature_key_length":4},{"name":"size","doc":"","types":["Integer"],"signature_key_offset":41,"signature_key_length":5}]}]},{"key":"canary::skip","calling_source":null,"source":null,"line":null,"char":null,"length":null,"doc":"Returns a ResultSet with canary/skipped-node errors for each Target provided.\n\nThis function takes a single parameter:\n* List of nodes (Array[Variant[Target,String]])\n\nReturns a ResultSet.","function_version":4,"signatures":[{"key":"canary::skip(Array[Variant[Target,String]] $nodes)","doc":"Returns a ResultSet with canary/skipped-node errors for each Target provided.\n\nThis function takes a single parameter:\n* List of nodes (Array[Variant[Target,String]])\n\nReturns a ResultSet.","return_types":["Any"],"parameters":[{"name":"nodes","doc":"","types":["Array[Variant[Target,String]]"],"signature_key_offset":43,"signature_key_length":6}]}]}]}

lib/puppet-languageserver/static_data/bolt-canary.json

@@ -0,0 +1 @@
+{"functions":[{"key":"ctrl::do_until","calling_source":null,"source":null,"line":null,"char":null,"length":null,"doc":"Repeat the block until it returns a truthy value. Returns the value.","function_version":4,"signatures":[{"key":"ctrl::do_until(Optional[Hash[String[1], Any]] $options, Callable &$block)","doc":"Repeat the block until it returns a truthy value. Returns the value.","return_types":["Any"],"parameters":[{"name":"options","doc":"Additional options: 'until'","types":["Optional[Hash[String[1], Any]]"],"signature_key_offset":46,"signature_key_length":8},{"name":"&block","doc":"","types":["Callable"],"signature_key_offset":65,"signature_key_length":7}]}]},{"key":"ctrl::sleep","calling_source":null,"source":null,"line":null,"char":null,"length":null,"doc":"Sleeps for specified number of seconds.","function_version":4,"signatures":[{"key":"ctrl::sleep(Numeric $period)","doc":"Sleeps for specified number of seconds.","return_types":["Undef"],"parameters":[{"name":"period","doc":"Time to sleep (in seconds)","types":["Numeric"],"signature_key_offset":20,"signature_key_length":7}]}]}]}

lib/puppet-languageserver/static_data/bolt-ctrl.json

@@ -0,0 +1 @@
+{"functions":[{"key":"file::exists","calling_source":null,"source":null,"line":null,"char":null,"length":null,"doc":"check if a file exists","function_version":4,"signatures":[{"key":"file::exists(String $filename)","doc":"check if a file exists","return_types":["Boolean"],"parameters":[{"name":"filename","doc":"Absolute path or Puppet file path.","types":["String"],"signature_key_offset":20,"signature_key_length":9}]}]},{"key":"file::read","calling_source":null,"source":null,"line":null,"char":null,"length":null,"doc":"Read a file and return its contents.","function_version":4,"signatures":[{"key":"file::read(String $filename)","doc":"Read a file and return its contents.","return_types":["String"],"parameters":[{"name":"filename","doc":"Absolute path or Puppet file path.","types":["String"],"signature_key_offset":18,"signature_key_length":9}]}]},{"key":"file::readable","calling_source":null,"source":null,"line":null,"char":null,"length":null,"doc":"check if a file is readable","function_version":4,"signatures":[{"key":"file::readable(String $filename)","doc":"check if a file is readable","return_types":["Boolean"],"parameters":[{"name":"filename","doc":"Absolute path or Puppet file path.","types":["String"],"signature_key_offset":22,"signature_key_length":9}]}]},{"key":"file::write","calling_source":null,"source":null,"line":null,"char":null,"length":null,"doc":"Write a string to a file.","function_version":4,"signatures":[{"key":"file::write(String $filename, String $content)","doc":"Write a string to a file.","return_types":["Undef"],"parameters":[{"name":"filename","doc":"Absolute path.","types":["String"],"signature_key_offset":19,"signature_key_length":9},{"name":"content","doc":"File content to write.","types":["String"],"signature_key_offset":37,"signature_key_length":8}]}]}]}