Move flay to do pre-filtering, not post-filtering. (#313)

* Bump flay and sexp_processor for new pre-filtering option.

This also removes json, as that ships with every ruby these days. I
had trouble building it, which seems unnecessary to even do.

I *think* I manually updated this lockfile properly. I don't have
/home/app/codeclimate-parser-client so I don't know how you're
supposed to use bundler properly. I basically updated and then carved
out some of the diffs until everything ran clean.

* [WIP] roll back #206

This has been merged upstream.

* [WIP] remove `delete_comments!` in favor of using `filters`

* Improved filtering of "false positives" for various languages.

+ Fixed filtering for go. It was too narrow.
+ Improved engine_conf in most specs to allow for variable mass thresholds.
+ Ensured that filter specs have actual bodies so we're not wiping everything.

* Bump flay, ruby_parser, and sexp_processor.

Author: zenspider

Gemfile

@@ -2,9 +2,8 @@
 source "https://rubygems.org"
 
 gem "concurrent-ruby", "~> 1.0.0"
-gem "flay", "~> 2.10"
-gem "json"
-gem "sexp_processor", "~> 4.10"
+gem "flay", "~> 2.12"
+gem "sexp_processor", "~> 4.11"
 
 gem "codeclimate-parser-client",
   path: "/home/app/codeclimate-parser-client"

Gemfile.lock

@@ -14,14 +14,13 @@ GEM
     diff-lcs (1.2.5)
     erubis (2.7.0)
     excon (0.57.1)
-    flay (2.10.0)
+    flay (2.12.0)
       erubis (~> 2.7.0)
       path_expander (~> 1.0)
       ruby_parser (~> 3.0)
       sexp_processor (~> 4.0)
-    json (1.8.3)
     method_source (0.8.2)
-    path_expander (1.0.2)
+    path_expander (1.0.3)
     pry (0.10.3)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
@@ -40,9 +39,9 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.3.0)
     rspec-support (3.3.0)
-    ruby_parser (3.10.0)
+    ruby_parser (3.11.0)
       sexp_processor (~> 4.9)
-    sexp_processor (4.10.0)
+    sexp_processor (4.11.0)
     slop (3.6.0)
 
 PLATFORMS
@@ -51,12 +50,11 @@ PLATFORMS
 DEPENDENCIES
   codeclimate-parser-client!
   concurrent-ruby (~> 1.0.0)
-  flay (~> 2.10)
-  json
+  flay (~> 2.12)
   pry
   rake
   rspec
-  sexp_processor (~> 4.10)
+  sexp_processor (~> 4.11)
 
 BUNDLED WITH
    1.10.6

lib/cc/engine/analyzers/go/main.rb

@@ -14,14 +14,13 @@ class Main < CC::Engine::Analyzers::Base
           PATTERNS = ["**/*.go"].freeze
           DEFAULT_MASS_THRESHOLD = 100
           DEFAULT_FILTERS = [
-            "(ImportSpec ___)",
+            "(GenDecl _ (specs (ImportSpec ___)) _)",
+            "(comments ___)",
           ].freeze
           POINTS_PER_OVERAGE = 10_000
           REQUEST_PATH = "/go"
-          COMMENT_MATCHER = Sexp::Matcher.parse("(_ (comments ___) ___)")
 
           def transform_sexp(sexp)
-            delete_comments!(sexp)
             sexp.delete_if { |node| node[0] == :name }
           end
 
@@ -38,10 +37,6 @@ def process_file(file)
           def default_filters
             DEFAULT_FILTERS.map { |filter| Sexp::Matcher.parse filter }
           end
-
-          def delete_comments!(sexp)
-            sexp.search_each(COMMENT_MATCHER) { |node| node.delete_at(1) }
-          end
         end
       end
     end

lib/cc/engine/analyzers/php/main.rb

@@ -14,14 +14,10 @@ class Main < CC::Engine::Analyzers::Base
           DEFAULT_MASS_THRESHOLD = 75
           DEFAULT_FILTERS = [
             "(Stmt_Use ___)",
+            "(comments ___)",
           ].freeze
           POINTS_PER_OVERAGE = 35_000
           REQUEST_PATH = "/php"
-          COMMENT_MATCHER = Sexp::Matcher.parse("(_ (comments ___) ___)")
-
-          def transform_sexp(sexp)
-            delete_comments!(sexp)
-          end
 
           def use_sexp_lines?
             false
@@ -36,10 +32,6 @@ def process_file(file)
           def default_filters
             DEFAULT_FILTERS.map { |filter| Sexp::Matcher.parse filter }
           end
-
-          def delete_comments!(sexp)
-            sexp.search_each(COMMENT_MATCHER) { |node| node.delete_at(1) }
-          end
         end
       end
     end

lib/ccflay.rb

@@ -3,7 +3,6 @@
 require "flay"
 require "concurrent"
 require "digest"
-require "zlib"
 
 ##
 # A thread-safe and stable hash subclass of Flay.
@@ -21,16 +20,6 @@ def initialize(option = nil)
   end
 end
 
-# Overwrite `NODE_NAMES` from Flay to assign all values on-demand instead of
-# using a predefined registry.
-Sexp::NODE_NAMES.delete_if { true }
-
-Sexp::NODE_NAMES.default_proc = lambda do |hash, key|
-  # Use CRC checksums so hash values are order-independent (i.e. consistent
-  # between runs).
-  hash[key] = Zlib.crc32(key.to_s)
-end
-
 class Sexp
   attr_writer :mass
 

spec/cc/engine/analyzers/go/main_spec.rb

@@ -123,7 +123,7 @@ module CC::Engine::Analyzers
           }
         EOGO
 
-        issues = run_engine(engine_conf).strip.split("\0")
+        issues = run_engine(engine_conf 25).strip.split("\0")
         expect(issues).to be_empty
       end
 
@@ -218,7 +218,7 @@ module CC::Engine::Analyzers
         expect(run_engine(engine_conf)).to be_empty
       end
 
-      def engine_conf
+      def engine_conf mass = 10
         CC::Engine::Analyzers::EngineConfig.new({
           'config' => {
             'checks' => {
@@ -231,7 +231,7 @@ def engine_conf
             },
             'languages' => {
               'go' => {
-                'mass_threshold' => 10,
+                'mass_threshold' => mass,
               },
             },
           },

spec/cc/engine/analyzers/javascript/main_spec.rb

@@ -159,16 +159,18 @@
       b = require('bar'),
       c = require('baz'),
       d = require('bam');
+    a + b + c + d;
     EOJS
 
     create_source_file("bar.js", <<~EOJS)
     const a = require('foo'),
       b = require('bar'),
       c = require('baz'),
       d = require('bam');
+    print(a);
     EOJS
 
-    issues = run_engine(engine_conf).strip.split("\0")
+    issues = run_engine(engine_conf 3).strip.split("\0")
     expect(issues).to be_empty
   end
 
@@ -201,7 +203,7 @@
     })
   end
 
-  def engine_conf
+  def engine_conf mass = 1
     CC::Engine::Analyzers::EngineConfig.new({
       'config' => {
         'checks' => {
@@ -214,7 +216,7 @@ def engine_conf
         },
         'languages' => {
           'javascript' => {
-            'mass_threshold' => 1,
+            'mass_threshold' => mass,
           },
         },
       },

spec/cc/engine/analyzers/php/main_spec.rb

@@ -177,6 +177,7 @@
       use KeepClear\\Traits\\Controllers\\ApiFilter;
       use KeepClear\\Traits\\Controllers\\ApiParseBody;
       use KeepClear\\Traits\\Controllers\\ApiException;
+      a / b;
       EOPHP
 
       create_source_file("bar.php", <<~EOPHP)
@@ -190,9 +191,10 @@
       use KeepClear\\Traits\\Controllers\\ApiFilter;
       use KeepClear\\Traits\\Controllers\\ApiParseBody;
       use KeepClear\\Traits\\Controllers\\ApiException;
+      a + b;
       EOPHP
 
-      issues = run_engine(engine_conf).strip.split("\0")
+      issues = run_engine(engine_conf 6).strip.split("\0")
       expect(issues).to be_empty
     end
 
@@ -275,12 +277,12 @@
     end
   end
 
-  def engine_conf
+  def engine_conf mass = 5
     CC::Engine::Analyzers::EngineConfig.new({
       'config' => {
         'languages' => {
           'php' => {
-            'mass_threshold' => 5,
+            'mass_threshold' => mass,
           },
         },
       },

spec/cc/engine/analyzers/typescript/main_spec.rb

@@ -140,16 +140,18 @@
       b = require('bar'),
       c = require('baz'),
       d = require('bam');
+    print(a);
     EOTS
 
     create_source_file("bar.ts", <<~EOTS)
     const a = require('foo'),
       b = require('bar'),
       c = require('baz'),
       d = require('bam');
+    c * d;
     EOTS
 
-    issues = run_engine(engine_conf).strip.split("\0")
+    issues = run_engine(engine_conf 3).strip.split("\0")
     expect(issues).to be_empty
   end
 
@@ -209,7 +211,7 @@
     expect(json["fingerprint"]).to eq("d8f0315c3c4e9ba81003a7ec6c823fb0")
   end
 
-  def engine_conf
+  def engine_conf mass = 1
     CC::Engine::Analyzers::EngineConfig.new({
       'config' => {
         'checks' => {
@@ -222,7 +224,7 @@ def engine_conf
         },
         'languages' => {
           'typescript' => {
-            'mass_threshold' => 1,
+            'mass_threshold' => mass,
           },
         },
       },