Merge pull request #6756 from erik-krogh/extractBigReg

JS: extract regexp literals for string concatenations

Author: erik-krogh

javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java

@@ -3,6 +3,8 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 import java.util.Stack;
@@ -332,17 +334,28 @@ private static class Context {
     private final Label parent;
     private final int childIndex;
     private final IdContext idcontext;
+    private final boolean binopOperand;
 
     public Context(Label parent, int childIndex, IdContext idcontext) {
+      this(parent, childIndex, idcontext, false);
+    }
+
+    public Context(Label parent, int childIndex, IdContext idcontext, boolean binopOperand) {
       this.parent = parent;
       this.childIndex = childIndex;
       this.idcontext = idcontext;
+      this.binopOperand = binopOperand;
     }
 
     /** True if the visited AST node occurs as part of a type annotation. */
     public boolean isInsideType() {
       return idcontext.isInsideType();
     }
+
+    /** True if the visited AST node occurs as one of the operands of a binary operation. */
+    public boolean isBinopOperand() {
+      return binopOperand;
+    }
   }
 
   private class V extends DefaultVisitor<Context, Label> {
@@ -358,16 +371,24 @@ public V(Platform platform, SourceType sourceType) {
     }
 
     private Label visit(INode child, Label parent, int childIndex) {
-      return visit(child, parent, childIndex, IdContext.VAR_BIND);
+      return visit(child, parent, childIndex, IdContext.VAR_BIND, false);
     }
 
     private Label visitAll(List<? extends INode> children, Label parent) {
       return visitAll(children, parent, IdContext.VAR_BIND, 0);
     }
 
     private Label visit(INode child, Label parent, int childIndex, IdContext idContext) {
+      return visit(child, parent, childIndex, idContext, false);
+    }
+
+    private Label visit(INode child, Label parent, int childIndex, boolean binopOperand) {
+      return visit(child, parent, childIndex, IdContext.VAR_BIND, binopOperand);
+    }
+
+    private Label visit(INode child, Label parent, int childIndex, IdContext idContext, boolean binopOperand) {
       if (child == null) return null;
-      return child.accept(this, new Context(parent, childIndex, idContext));
+      return child.accept(this, new Context(parent, childIndex, idContext, binopOperand));
     }
 
     private Label visitAll(
@@ -379,7 +400,7 @@ private Label visitAll(
         List<? extends INode> children, Label parent, IdContext idContext, int index, int step) {
       Label res = null;
       for (INode child : children) {
-        res = visit(child, parent, index, idContext);
+        res = visit(child, parent, index, idContext, false);
         index += step;
       }
       return res;
@@ -567,12 +588,17 @@ public Label visit(Literal nd, Context c) {
       String valueString = nd.getStringValue();
 
       trapwriter.addTuple("literals", valueString, source, key);
+      Position start = nd.getLoc().getStart();
+      com.semmle.util.locations.Position startPos = new com.semmle.util.locations.Position(start.getLine(), start.getColumn() + 1 /* Convert from 0-based to 1-based. */, start.getOffset());
+      
       if (nd.isRegExp()) {
         OffsetTranslation offsets = new OffsetTranslation();
         offsets.set(0, 1); // skip the initial '/'
-        regexpExtractor.extract(source.substring(1, source.lastIndexOf('/')), offsets, nd, false);
-      } else if (nd.isStringLiteral() && !c.isInsideType() && nd.getRaw().length() < 1000) {
-        regexpExtractor.extract(valueString, makeStringLiteralOffsets(nd.getRaw()), nd, true);
+        SourceMap sourceMap = SourceMap.legacyWithStartPos(SourceMap.fromString(nd.getRaw()).offsetBy(0, offsets), startPos);
+        regexpExtractor.extract(source.substring(1, source.lastIndexOf('/')), sourceMap, nd, false);
+      } else if (nd.isStringLiteral() && !c.isInsideType() && nd.getRaw().length() < 1000 && !c.isBinopOperand()) {
+        SourceMap sourceMap = SourceMap.legacyWithStartPos(SourceMap.fromString(nd.getRaw()).offsetBy(0, makeStringLiteralOffsets(nd.getRaw())), startPos);
+        regexpExtractor.extract(valueString, sourceMap, nd, true);
 
         // Scan the string for template tags, if we're in a context where such tags are relevant.
         if (scopeManager.isInTemplateFile()) {
@@ -593,6 +619,38 @@ private boolean isOctalDigit(char ch) {
       return '0' <= ch && ch <= '7';
     }
 
+    /**
+     * Constant-folds simple string concatenations in `exp` while keeping an offset translation
+     * that tracks back to the original source.
+     */ 
+    private Pair<String, OffsetTranslation> getStringConcatResult(Expression exp) {
+      if (exp instanceof BinaryExpression) {
+        BinaryExpression be = (BinaryExpression) exp;
+        if (be.getOperator().equals("+")) {
+          Pair<String, OffsetTranslation> left = getStringConcatResult(be.getLeft());
+          Pair<String, OffsetTranslation> right = getStringConcatResult(be.getRight());
+          if (left == null || right == null) {
+            return null;
+          }
+          String str = left.fst() + right.fst();
+          if (str.length() > 1000) {
+            return null;
+          }
+            
+          int delta = be.getRight().getLoc().getStart().getOffset() - be.getLeft().getLoc().getStart().getOffset();
+          int offset = left.fst().length();
+          return Pair.make(str, left.snd().append(right.snd(), offset, delta));
+        }
+      } else if (exp instanceof Literal) {
+        Literal lit = (Literal) exp;
+        if (!lit.isStringLiteral()) {
+          return null;
+        }
+        return Pair.make(lit.getStringValue(), makeStringLiteralOffsets(lit.getRaw()));
+      }
+      return null;
+    }
+
     /**
      * Builds a translation from offsets in a string value back to its original raw literal text
      * (including quotes).
@@ -789,11 +847,32 @@ public Label visit(AssignmentExpression nd, Context c) {
     @Override
     public Label visit(BinaryExpression nd, Context c) {
       Label key = super.visit(nd, c);
-      visit(nd.getLeft(), key, 0);
-      visit(nd.getRight(), key, 1);
+      visit(nd.getLeft(), key, 0, true);
+      visit(nd.getRight(), key, 1, true);
+      extractRegxpFromBinop(nd, c);
       return key;
     }
 
+    private void extractRegxpFromBinop(BinaryExpression nd, Context c) {
+      if (c.isBinopOperand()) {
+        return;
+      }
+      Pair<String, OffsetTranslation> concatResult = getStringConcatResult(nd);
+      if (concatResult == null) {
+        return;
+      }
+      String foldedString = concatResult.fst();
+      if (foldedString.length() > 1000 && !foldedString.trim().isEmpty()) {
+        return;
+      }
+      OffsetTranslation offsets = concatResult.snd();
+      Position start = nd.getLoc().getStart();
+      com.semmle.util.locations.Position startPos = new com.semmle.util.locations.Position(start.getLine(), start.getColumn() + 1 /* Convert from 0-based to 1-based. */, start.getOffset());
+      SourceMap sourceMap = SourceMap.legacyWithStartPos(SourceMap.fromString(nd.getLoc().getSource()).offsetBy(0, offsets), startPos);
+      regexpExtractor.extract(foldedString, sourceMap, nd, true);
+      return;
+    }
+
     @Override
     public Label visit(ComprehensionBlock nd, Context c) {
       Label key = super.visit(nd, c);

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -43,7 +43,7 @@ public class Main {
    * A version identifier that should be updated every time the extractor changes in such a way that
    * it may produce different tuples for the same file under the same {@link ExtractorConfig}.
    */
-  public static final String EXTRACTOR_VERSION = "2021-10-25";
+  public static final String EXTRACTOR_VERSION = "2021-10-28";
 
   public static final Pattern NEWLINE = Pattern.compile("\n");
 

javascript/extractor/src/com/semmle/js/extractor/RegExpExtractor.java

@@ -43,7 +43,7 @@
 import com.semmle.js.ast.regexp.ZeroWidthPositiveLookbehind;
 import com.semmle.js.parser.RegExpParser;
 import com.semmle.js.parser.RegExpParser.Result;
-import com.semmle.util.locations.OffsetTranslation;
+import com.semmle.util.locations.SourceMap;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.TrapWriter.Label;
 
@@ -52,8 +52,7 @@ public class RegExpExtractor {
   private final TrapWriter trapwriter;
   private final LocationManager locationManager;
   private final RegExpParser parser = new RegExpParser();
-  private Position literalStart;
-  private OffsetTranslation offsets;
+  private SourceMap sourceMap;
 
   public RegExpExtractor(TrapWriter trapwriter, LocationManager locationManager) {
     this.trapwriter = trapwriter;
@@ -122,17 +121,16 @@ private Label extractTerm(RegExpTerm term, Label parent, int idx) {
   }
 
   public void emitLocation(SourceElement term, Label lbl) {
-    int col = literalStart.getColumn();
-    int sl, sc, el, ec;
-    sl = el = literalStart.getLine();
-    sc = col + offsets.get(term.getLoc().getStart().getColumn());
-    ec = col + offsets.get(term.getLoc().getEnd().getColumn());
-    sc += 1; // convert to 1-based
-    ec += 1; // convert to 1-based
-    ec -= 1; // convert to inclusive
+    int start = term.getLoc().getStart().getColumn();
+    int sl = sourceMap.getStart(start).getLine();
+    int sc = sourceMap.getStart(start).getColumn();
+    int end = term.getLoc().getEnd().getColumn();
+    int el = sourceMap.getStart(end).getLine();
+    int ec = sourceMap.getStart(end).getColumn() - 1; // convert to inclusive
     locationManager.emitSnippetLocation(lbl, sl, sc, el, ec);
   }
 
+
   private class V implements Visitor {
     private Label parent;
     private int idx;
@@ -348,16 +346,13 @@ public void visit(CharacterClassRange nd) {
     }
   }
 
-  public void extract(
-      String src, OffsetTranslation offsets, Node parent, boolean isSpeculativeParsing) {
+  public void extract(String src, SourceMap sourceMap, Node parent, boolean isSpeculativeParsing) {
     Result res = parser.parse(src);
-
     if (isSpeculativeParsing && res.getErrors().size() > 0) {
       return;
     }
 
-    this.literalStart = parent.getLoc().getStart();
-    this.offsets = offsets;
+    this.sourceMap = sourceMap;
     RegExpTerm ast = res.getAST();
     new V().visit(ast, trapwriter.localID(parent), 0);