Merge remote-tracking branch 'upstream/master' into csharp/dataflow/arrays

Author: hvitved

change-notes/1.25/analysis-javascript.md

@@ -6,6 +6,7 @@
   - [Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise)
   - [bluebird](http://bluebirdjs.com/)
   - [express](https://www.npmjs.com/package/express)
+  - [fancy-log](https://www.npmjs.com/package/fancy-log)
   - [fastify](https://www.npmjs.com/package/fastify)
   - [fstream](https://www.npmjs.com/package/fstream)
   - [jGrowl](https://github.com/stanlemon/jGrowl)
@@ -22,6 +23,7 @@
   - [sqlite](https://www.npmjs.com/package/sqlite)
   - [ssh2-streams](https://www.npmjs.com/package/ssh2-streams)
   - [ssh2](https://www.npmjs.com/package/ssh2)
+  - [vue](https://www.npmjs.com/package/vue)
   - [yargs](https://www.npmjs.com/package/yargs)
   - [webpack-dev-server](https://www.npmjs.com/package/webpack-dev-server)
 

config/opcode-qldoc.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+import os
+import re
+path = os.path
+
+needs_an_re = re.compile(r'^(?!Unary)[AEIOU]')  # Name requiring "an" instead of "a".
+start_qldoc_re = re.compile(r'^\s*/\*\*')  # Start of a QLDoc comment
+end_qldoc_re = re.compile(r'\*/\s*$')  # End of a QLDoc comment
+blank_qldoc_line_re = re.compile(r'^\s*\*\s*$')  # A line in a QLDoc comment with only the '*'
+instruction_class_re = re.compile(r'^class (?P<name>[A-aa-z0-9]+)Instruction\s')  # Declaration of an `Instruction` class
+opcode_base_class_re = re.compile(r'^abstract class (?P<name>[A-aa-z0-9]+)Opcode\s')  # Declaration of an `Opcode` base class
+opcode_class_re = re.compile(r'^  class (?P<name>[A-aa-z0-9]+)\s')  # Declaration of an `Opcode` class
+
+script_dir = path.realpath(path.dirname(__file__))
+instruction_path = path.realpath(path.join(script_dir, '../cpp/ql/src/semmle/code/cpp/ir/implementation/raw/Instruction.qll'))
+opcode_path = path.realpath(path.join(script_dir, '../cpp/ql/src/semmle/code/cpp/ir/implementation/Opcode.qll'))
+
+# Scan `Instruction.qll`, keeping track of the QLDoc comment attached to each declaration of a class
+# whose name ends with `Instruction`.
+instruction_comments = {}
+in_qldoc = False
+saw_blank_line_in_qldoc = False
+qldoc_lines = []
+with open(instruction_path, 'r', encoding='utf-8') as instr:
+    for line in instr:
+        if in_qldoc:
+            if end_qldoc_re.search(line):
+                qldoc_lines.append(line)
+                in_qldoc = False
+            elif blank_qldoc_line_re.search(line):
+                # We're going to skip any lines after the first blank line, to avoid duplicating all
+                # of the verbose description.
+                saw_blank_line_in_qldoc = True
+            elif not saw_blank_line_in_qldoc:
+                qldoc_lines.append(line)
+        else:
+            if start_qldoc_re.search(line):
+                # Starting a new QLDoc comment.
+                saw_blank_line_in_qldoc = False
+                qldoc_lines.append(line)
+                if not end_qldoc_re.search(line):
+                    in_qldoc = True
+            else:
+                instruction_match = instruction_class_re.search(line)
+                if instruction_match:
+                    # Found the declaration of an `Instruction` class. Record the QLDoc comments.
+                    instruction_comments[instruction_match.group('name')] = qldoc_lines
+                qldoc_lines = []
+
+# Scan `Opcode.qll`. Whenever we see the declaration of an `Opcode` class for which we have a
+# corresponding `Instruction` class, we'll attach a copy of the `Instruction`'s QLDoc comment.
+in_qldoc = False
+qldoc_lines = []
+output_lines = []
+with open(opcode_path, 'r', encoding='utf-8') as opcode:
+    for line in opcode:
+        if in_qldoc:
+            qldoc_lines.append(line)
+            if end_qldoc_re.search(line):
+                in_qldoc = False
+        else:
+            if start_qldoc_re.search(line):
+                qldoc_lines.append(line)
+                if not end_qldoc_re.search(line):
+                    in_qldoc = True
+            else:
+                name_without_suffix = None
+                name = None
+                indent = ''
+                opcode_base_match = opcode_base_class_re.search(line)
+                if opcode_base_match:
+                    name_without_suffix = opcode_base_match.group('name')
+                    name = name_without_suffix + 'Opcode'
+                else:
+                    opcode_match = opcode_class_re.search(line)
+                    if opcode_match:
+                        name_without_suffix = opcode_match.group('name')
+                        name = name_without_suffix
+                        # Indent by two additional spaces, since opcodes are declared in the
+                        # `Opcode` module.
+                        indent = '  '
+                
+                if name_without_suffix:
+                    # Found an `Opcode` that matches a known `Instruction`. Replace the QLDoc with
+                    # a copy of the one from the `Instruction`.
+                    if instruction_comments.get(name_without_suffix):
+                        article = 'an' if needs_an_re.search(name_without_suffix) else 'a'
+                        qldoc_lines = [
+                            indent + '/**\n',
+                            indent + ' * The `Opcode` for ' + article + ' `' + name_without_suffix + 'Instruction`.\n',
+                            indent + ' *\n',
+                            indent + ' * See the `' + name_without_suffix + 'Instruction` documentation for more details.\n',
+                            indent + ' */\n'
+                        ]
+                output_lines.extend(qldoc_lines)
+                qldoc_lines = []
+                output_lines.append(line)
+
+# Write out the updated `Opcode.qll`
+with open(opcode_path, 'w', encoding='utf-8') as opcode:
+    opcode.writelines(output_lines)

cpp/ql/src/Security/CWE/CWE-327/BrokenCryptoAlgorithm.ql

@@ -18,7 +18,7 @@ abstract class InsecureCryptoSpec extends Locatable {
 }
 
 Function getAnInsecureFunction() {
-  result.getName().regexpMatch(algorithmBlacklistRegex()) and
+  result.getName().regexpMatch(getInsecureAlgorithmRegex()) and
   exists(result.getACallToThisFunction())
 }
 
@@ -33,7 +33,7 @@ class InsecureFunctionCall extends InsecureCryptoSpec, FunctionCall {
 }
 
 Macro getAnInsecureMacro() {
-  result.getName().regexpMatch(algorithmBlacklistRegex()) and
+  result.getName().regexpMatch(getInsecureAlgorithmRegex()) and
   exists(result.getAnInvocation())
 }
 

cpp/ql/src/semmle/code/cpp/Class.qll

@@ -33,7 +33,7 @@ private import semmle.code.cpp.internal.ResolveClass
 class Class extends UserType {
   Class() { isClass(underlyingElement(this)) }
 
-  override string getCanonicalQLClass() { result = "Class" }
+  override string getAPrimaryQlClass() { result = "Class" }
 
   /** Gets a child declaration of this class, struct or union. */
   override Declaration getADeclaration() { result = this.getAMember() }
@@ -768,7 +768,7 @@ class ClassDerivation extends Locatable, @derivation {
    */
   Class getBaseClass() { result = getBaseType().getUnderlyingType() }
 
-  override string getCanonicalQLClass() { result = "ClassDerivation" }
+  override string getAPrimaryQlClass() { result = "ClassDerivation" }
 
   /**
    * Gets the type from which we are deriving, without resolving any
@@ -849,9 +849,7 @@ class ClassDerivation extends Locatable, @derivation {
 class LocalClass extends Class {
   LocalClass() { isLocal() }
 
-  override string getCanonicalQLClass() {
-    not this instanceof LocalStruct and result = "LocalClass"
-  }
+  override string getAPrimaryQlClass() { not this instanceof LocalStruct and result = "LocalClass" }
 
   override Function getEnclosingAccessHolder() { result = this.getEnclosingFunction() }
 }
@@ -872,7 +870,7 @@ class LocalClass extends Class {
 class NestedClass extends Class {
   NestedClass() { this.isMember() }
 
-  override string getCanonicalQLClass() {
+  override string getAPrimaryQlClass() {
     not this instanceof NestedStruct and result = "NestedClass"
   }
 
@@ -893,7 +891,7 @@ class NestedClass extends Class {
 class AbstractClass extends Class {
   AbstractClass() { exists(PureVirtualFunction f | this.getAMemberFunction() = f) }
 
-  override string getCanonicalQLClass() { result = "AbstractClass" }
+  override string getAPrimaryQlClass() { result = "AbstractClass" }
 }
 
 /**
@@ -934,7 +932,7 @@ class TemplateClass extends Class {
     exists(result.getATemplateArgument())
   }
 
-  override string getCanonicalQLClass() { result = "TemplateClass" }
+  override string getAPrimaryQlClass() { result = "TemplateClass" }
 }
 
 /**
@@ -955,7 +953,7 @@ class ClassTemplateInstantiation extends Class {
 
   ClassTemplateInstantiation() { tc.getAnInstantiation() = this }
 
-  override string getCanonicalQLClass() { result = "ClassTemplateInstantiation" }
+  override string getAPrimaryQlClass() { result = "ClassTemplateInstantiation" }
 
   /**
    * Gets the class template from which this instantiation was instantiated.
@@ -996,7 +994,7 @@ abstract class ClassTemplateSpecialization extends Class {
       count(int i | exists(result.getTemplateArgument(i)))
   }
 
-  override string getCanonicalQLClass() { result = "ClassTemplateSpecialization" }
+  override string getAPrimaryQlClass() { result = "ClassTemplateSpecialization" }
 }
 
 /**
@@ -1025,7 +1023,7 @@ class FullClassTemplateSpecialization extends ClassTemplateSpecialization {
     not this instanceof ClassTemplateInstantiation
   }
 
-  override string getCanonicalQLClass() { result = "FullClassTemplateSpecialization" }
+  override string getAPrimaryQlClass() { result = "FullClassTemplateSpecialization" }
 }
 
 /**
@@ -1064,7 +1062,7 @@ class PartialClassTemplateSpecialization extends ClassTemplateSpecialization {
       count(int i | exists(getTemplateArgument(i)))
   }
 
-  override string getCanonicalQLClass() { result = "PartialClassTemplateSpecialization" }
+  override string getAPrimaryQlClass() { result = "PartialClassTemplateSpecialization" }
 }
 
 /**
@@ -1089,7 +1087,7 @@ deprecated class Interface extends Class {
     )
   }
 
-  override string getCanonicalQLClass() { result = "Interface" }
+  override string getAPrimaryQlClass() { result = "Interface" }
 }
 
 /**
@@ -1104,7 +1102,7 @@ deprecated class Interface extends Class {
 class VirtualClassDerivation extends ClassDerivation {
   VirtualClassDerivation() { hasSpecifier("virtual") }
 
-  override string getCanonicalQLClass() { result = "VirtualClassDerivation" }
+  override string getAPrimaryQlClass() { result = "VirtualClassDerivation" }
 }
 
 /**
@@ -1124,7 +1122,7 @@ class VirtualClassDerivation extends ClassDerivation {
 class VirtualBaseClass extends Class {
   VirtualBaseClass() { exists(VirtualClassDerivation cd | cd.getBaseClass() = this) }
 
-  override string getCanonicalQLClass() { result = "VirtualBaseClass" }
+  override string getAPrimaryQlClass() { result = "VirtualBaseClass" }
 
   /** A virtual class derivation of which this class/struct is the base. */
   VirtualClassDerivation getAVirtualDerivation() { result.getBaseClass() = this }
@@ -1146,7 +1144,7 @@ class VirtualBaseClass extends Class {
 class ProxyClass extends UserType {
   ProxyClass() { usertypes(underlyingElement(this), _, 9) }
 
-  override string getCanonicalQLClass() { result = "ProxyClass" }
+  override string getAPrimaryQlClass() { result = "ProxyClass" }
 
   /** Gets the location of the proxy class. */
   override Location getLocation() { result = getTemplateParameter().getDefinitionLocation() }

cpp/ql/src/semmle/code/cpp/Declaration.qll

@@ -124,7 +124,7 @@ class Declaration extends Locatable, @declaration {
    * To test whether this declaration has a particular name in the global
    * namespace, use `hasGlobalName`.
    */
-  abstract string getName();
+  string getName() { none() } // overridden in subclasses
 
   /** Holds if this declaration has the given name. */
   predicate hasName(string name) { name = this.getName() }
@@ -140,7 +140,7 @@ class Declaration extends Locatable, @declaration {
   }
 
   /** Gets a specifier of this declaration. */
-  abstract Specifier getASpecifier();
+  Specifier getASpecifier() { none() } // overridden in subclasses
 
   /** Holds if this declaration has a specifier with the given name. */
   predicate hasSpecifier(string name) { this.getASpecifier().hasName(name) }
@@ -156,7 +156,7 @@ class Declaration extends Locatable, @declaration {
    * Gets the location of a declaration entry corresponding to this
    * declaration.
    */
-  abstract Location getADeclarationLocation();
+  Location getADeclarationLocation() { none() } // overridden in subclasses
 
   /**
    * Gets the declaration entry corresponding to this declaration that is a
@@ -165,7 +165,7 @@ class Declaration extends Locatable, @declaration {
   DeclarationEntry getDefinition() { none() }
 
   /** Gets the location of the definition, if any. */
-  abstract Location getDefinitionLocation();
+  Location getDefinitionLocation() { none() } // overridden in subclasses
 
   /** Holds if the declaration has a definition. */
   predicate hasDefinition() { exists(this.getDefinition()) }
@@ -289,6 +289,8 @@ class Declaration extends Locatable, @declaration {
   }
 }
 
+private class TDeclarationEntry = @var_decl or @type_decl or @fun_decl;
+
 /**
  * A C/C++ declaration entry. For example the following code contains five
  * declaration entries:
@@ -304,9 +306,9 @@ class Declaration extends Locatable, @declaration {
  * See the comment above `Declaration` for an explanation of the relationship
  * between `Declaration` and `DeclarationEntry`.
  */
-abstract class DeclarationEntry extends Locatable {
+class DeclarationEntry extends Locatable, TDeclarationEntry {
   /** Gets a specifier associated with this declaration entry. */
-  abstract string getASpecifier();
+  string getASpecifier() { none() } // overridden in subclasses
 
   /**
    * Gets the name associated with the corresponding definition (where
@@ -329,10 +331,10 @@ abstract class DeclarationEntry extends Locatable {
    *  `I.getADeclarationEntry()` returns `D`
    *  but `D.getDeclaration()` only returns `C`
    */
-  abstract Declaration getDeclaration();
+  Declaration getDeclaration() { none() } // overridden in subclasses
 
   /** Gets the name associated with this declaration entry, if any. */
-  abstract string getName();
+  string getName() { none() } // overridden in subclasses
 
   /**
    * Gets the type associated with this declaration entry.
@@ -341,7 +343,7 @@ abstract class DeclarationEntry extends Locatable {
    * For function declarations, get the return type of the function.
    * For type declarations, get the type being declared.
    */
-  abstract Type getType();
+  Type getType() { none() } // overridden in subclasses
 
   /**
    * Gets the type associated with this declaration entry after specifiers
@@ -359,7 +361,7 @@ abstract class DeclarationEntry extends Locatable {
   predicate hasSpecifier(string specifier) { getASpecifier() = specifier }
 
   /** Holds if this declaration entry is a definition. */
-  abstract predicate isDefinition();
+  predicate isDefinition() { none() } // overridden in subclasses
 
   override string toString() {
     if isDefinition()
@@ -371,6 +373,8 @@ abstract class DeclarationEntry extends Locatable {
   }
 }
 
+private class TAccessHolder = @function or @usertype;
+
 /**
  * A declaration that can potentially have more C++ access rights than its
  * enclosing element. This comprises `Class` (they have access to their own
@@ -392,7 +396,7 @@ abstract class DeclarationEntry extends Locatable {
  * the informal phrase "_R_ occurs in a member or friend of class C", where
  * `AccessHolder` corresponds to this _R_.
  */
-abstract class AccessHolder extends Declaration {
+class AccessHolder extends Declaration, TAccessHolder {
   /**
    * Holds if `this` can access private members of class `c`.
    *
@@ -410,7 +414,7 @@ abstract class AccessHolder extends Declaration {
   /**
    * Gets the nearest enclosing `AccessHolder`.
    */
-  abstract AccessHolder getEnclosingAccessHolder();
+  AccessHolder getEnclosingAccessHolder() { none() } // overridden in subclasses
 
   /**
    * Holds if a base class `base` of `derived` _is accessible at_ `this` (N4140