Merge pull request #9884 from tausbn/ql-untangle-parsers

QL: Untangle the various parsers

Author: tausbn

ql/Cargo.lock

@@ -9,8 +9,10 @@ edition = "2018"
 [dependencies]
 flate2 = "1.0"
 node-types = { path = "../node-types" }
-tree-sitter = "0.19"
-tree-sitter-ql = { git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "343cc5873e20510586ade803659ef8ce153bd603" }
+tree-sitter = ">= 0.20, < 0.21"
+tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "4b8078c7fdcce9d4ca06ce3cfec3a61e8c3f4555"}
+tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"}
+tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
 clap = "2.33"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }

ql/extractor/Cargo.toml

@@ -85,7 +85,13 @@ fn main() -> std::io::Result<()> {
     let file_list = fs::File::open(file_list)?;
 
     let language = tree_sitter_ql::language();
+    let dbscheme = tree_sitter_ql_dbscheme::language();
+    let yaml = tree_sitter_ql_yaml::language();
     let schema = node_types::read_node_types_str("ql", tree_sitter_ql::NODE_TYPES)?;
+    let dbscheme_schema =
+        node_types::read_node_types_str("dbscheme", tree_sitter_ql_dbscheme::NODE_TYPES)?;
+    let yaml_schema = node_types::read_node_types_str("yaml", tree_sitter_ql_yaml::NODE_TYPES)?;
+
     let lines: std::io::Result<Vec<String>> = std::io::BufReader::new(file_list).lines().collect();
     let lines = lines?;
     lines
@@ -105,15 +111,37 @@ fn main() -> std::io::Result<()> {
             let source = std::fs::read(&path)?;
             let code_ranges = vec![];
             let mut trap_writer = trap::Writer::new();
-            extractor::extract(
-                language,
-                "ql",
-                &schema,
-                &mut trap_writer,
-                &path,
-                &source,
-                &code_ranges,
-            )?;
+            if line.ends_with(".dbscheme") {
+                extractor::extract(
+                    dbscheme,
+                    "dbscheme",
+                    &dbscheme_schema,
+                    &mut trap_writer,
+                    &path,
+                    &source,
+                    &code_ranges,
+                )?
+            } else if line.ends_with("qlpack.yml") {
+                extractor::extract(
+                    yaml,
+                    "yaml",
+                    &yaml_schema,
+                    &mut trap_writer,
+                    &path,
+                    &source,
+                    &code_ranges,
+                )?
+            } else {
+                extractor::extract(
+                    language,
+                    "ql",
+                    &schema,
+                    &mut trap_writer,
+                    &path,
+                    &source,
+                    &code_ranges,
+                )?
+            }
             std::fs::create_dir_all(&src_archive_file.parent().unwrap())?;
             std::fs::copy(&path, &src_archive_file)?;
             write_trap(&trap_dir, path, &trap_writer, trap_compression)

ql/extractor/src/main.rs

@@ -11,4 +11,6 @@ clap = "2.33"
 node-types = { path = "../node-types" }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
-tree-sitter-ql = { git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "343cc5873e20510586ade803659ef8ce153bd603" }
+tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "4b8078c7fdcce9d4ca06ce3cfec3a61e8c3f4555"}
+tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"}
+tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}

ql/generator/Cargo.toml

@@ -564,10 +564,20 @@ fn main() -> std::io::Result<()> {
     let ql_library_path = matches.value_of("library").expect("missing --library");
     let ql_library_path = PathBuf::from(ql_library_path);
 
-    let languages = vec![Language {
-        name: "QL".to_owned(),
-        node_types: tree_sitter_ql::NODE_TYPES,
-    }];
+    let languages = vec![
+        Language {
+            name: "QL".to_owned(),
+            node_types: tree_sitter_ql::NODE_TYPES,
+        },
+        Language {
+            name: "Dbscheme".to_owned(),
+            node_types: tree_sitter_ql_dbscheme::NODE_TYPES,
+        },
+        Language {
+            name: "Yaml".to_owned(),
+            node_types: tree_sitter_ql_yaml::NODE_TYPES,
+        },
+    ];
     let mut dbscheme_writer = LineWriter::new(File::create(dbscheme_path)?);
     write!(
         dbscheme_writer,

ql/generator/src/main.rs

@@ -32,6 +32,10 @@ class AstNode extends TAstNode {
       node = toQL(this) and
       result = node.getLocation()
     )
+    or
+    result = toGenerateYaml(this).getLocation()
+    or
+    result = toDbscheme(this).getLocation()
   }
 
   predicate hasLocationInfo(
@@ -111,7 +115,7 @@ class TopLevel extends TTopLevel, AstNode {
   ModuleMember getAMember() { result = this.getMember(_) }
 
   /** Gets the `i`'th member of this top-level module. */
-  ModuleMember getMember(int i) { toQL(result) = file.getChild(i).(QL::ModuleMember).getChild(_) }
+  ModuleMember getMember(int i) { toQL(result) = file.getChild(i).getChild(_) }
 
   /** Gets a top-level import in this module. */
   Import getAnImport() { result = this.getAMember() }
@@ -349,7 +353,7 @@ class Predicate extends TPredicate, AstNode, PredicateOrBuiltin, Declaration {
  * A relation in the database.
  */
 class Relation extends TDBRelation, AstNode, Declaration {
-  QL::DbTable table;
+  Dbscheme::Table table;
 
   Relation() { this = TDBRelation(table) }
 
@@ -358,9 +362,9 @@ class Relation extends TDBRelation, AstNode, Declaration {
    */
   override string getName() { result = table.getTableName().getChild().getValue() }
 
-  private QL::DbColumn getColumn(int i) {
+  private Dbscheme::Column getColumn(int i) {
     result =
-      rank[i + 1](QL::DbColumn column, int child |
+      rank[i + 1](Dbscheme::Column column, int child |
         table.getChild(child) = column
       |
         column order by child
@@ -373,7 +377,7 @@ class Relation extends TDBRelation, AstNode, Declaration {
   /** Gets the `i`th parameter type */
   string getParameterType(int i) {
     // TODO: This is just using the name of the type, not the actual type. Checkout Type.qll
-    result = this.getColumn(i).getColType().getChild().(QL::Token).getValue()
+    result = this.getColumn(i).getColType().getChild().(Dbscheme::Token).getValue()
   }
 
   /**
@@ -1171,13 +1175,27 @@ class Import extends TImport, ModuleMember, TypeRef {
   string importedAs() { result = imp.getChild(1).(QL::ModuleName).getChild().getValue() }
 
   /**
-   * Gets the `i`th selected name from the imported module.
+   * Gets the qualified name of the module selected in the import statement.
    * E.g. for
    * `import foo.bar::Baz::Qux`
-   * It is true that `getSelectionName(0) = "Baz"` and `getSelectionName(1) = "Qux"`.
+   * It is true that `getSelectionName() = "Baz::Qux"`.
+   *
+   * Does NOT include type arguments!
+   */
+  string getSelectionName() { result = this.getModuleExpr().getQualifiedName() }
+
+  /**
+   * Gets the module expression selected in the import statement.
+   * E.g. for
+   * `import foo.Bar::Baz::Qux`
+   * The module expression is the `Bar::Baz::Qux` part.
    */
-  string getSelectionName(int i) {
-    result = imp.getChild(0).(QL::ImportModuleExpr).getName(i).getValue()
+  ModuleExpr getModuleExpr() { toQL(result) = imp.getChild(0).(QL::ImportModuleExpr).getChild() }
+
+  override AstNode getAChild(string pred) {
+    result = super.getAChild(pred)
+    or
+    pred = directMember("getModuleExpr") and result = this.getModuleExpr()
   }
 
   /**
@@ -1187,27 +1205,25 @@ class Import extends TImport, ModuleMember, TypeRef {
    * It is true that `getQualifiedName(0) = "foo"` and `getQualifiedName(1) = "bar"`.
    */
   string getQualifiedName(int i) {
-    result = imp.getChild(0).(QL::ImportModuleExpr).getChild().getName(i).getValue()
+    result = imp.getChild(0).(QL::ImportModuleExpr).getQualName(i).getValue()
   }
 
   /**
-   * Gets the full string specifying the module being imported.
+   * Gets a full string specifying the module being imported.
+   *
+   * Does NOT include type arguments!
    */
   string getImportString() {
-    exists(string selec |
-      not exists(this.getSelectionName(_)) and selec = ""
+    exists(string qual |
+      not exists(this.getQualifiedName(_)) and qual = ""
       or
-      selec =
-        "::" + strictconcat(int i, string q | q = this.getSelectionName(i) | q, "::" order by i)
+      qual = strictconcat(int i, string q | q = this.getQualifiedName(i) | q, "." order by i) + "."
     |
-      result =
-        strictconcat(int i, string q | q = this.getQualifiedName(i) | q, "." order by i) + selec
+      result = qual + this.getSelectionName()
     )
   }
 
-  override Type getResolvedType() {
-    exists(FileOrModule mod | resolve(this, mod) | result = mod.toType())
-  }
+  override Type getResolvedType() { result = this.getModuleExpr().getResolvedType() }
 }
 
 /** A formula, such as `x = 6 and y < 5`. */
@@ -2282,6 +2298,19 @@ class ModuleExpr extends TModuleExpr, TypeRef {
   SignatureExpr getArgument(int i) {
     result.toQL() = me.getAFieldOrChild().(QL::ModuleInstantiation).getChild(i)
   }
+
+  /**
+   * Gets the qualified name for this module expression, which does not include the type arguments.
+   */
+  string getQualifiedName() {
+    exists(string qual |
+      not exists(this.getQualifier()) and qual = ""
+      or
+      qual = this.getQualifier().getQualifiedName() + "::"
+    |
+      result = qual + this.getName()
+    )
+  }
 }
 
 /** A signature expression, either a `PredicateExpr` or a `TypeExpr`. */
@@ -2426,11 +2455,13 @@ module YAML {
   class YAMLNode extends TYamlNode, AstNode {
     /** Holds if the predicate is a root node (has no parent) */
     predicate isRoot() { not exists(this.getParent()) }
+
+    override AstNode getParent() { toGenerateYaml(result) = toGenerateYaml(this).getParent() }
   }
 
   /** A YAML comment. */
   class YamlComment extends TYamlCommemt, YAMLNode {
-    QL::YamlComment yamlcomment;
+    Yaml::Comment yamlcomment;
 
     YamlComment() { this = TYamlCommemt(yamlcomment) }
 
@@ -2442,23 +2473,23 @@ module YAML {
 
   /** A YAML entry. */
   class YamlEntry extends TYamlEntry, YAMLNode {
-    QL::YamlEntry yamle;
+    Yaml::Entry yamle;
 
     YamlEntry() { this = TYamlEntry(yamle) }
 
     /** Gets the key of this YAML entry. */
     YamlKey getKey() {
-      exists(QL::YamlKeyvaluepair pair |
+      exists(Yaml::Keyvaluepair pair |
         pair.getParent() = yamle and
         result = TYamlKey(pair.getKey())
       )
     }
 
-    YamlListItem getListItem() { toQL(result).getParent() = yamle }
+    YamlListItem getListItem() { toGenerateYaml(result).getParent() = yamle }
 
     /** Gets the value of this YAML entry. */
-    YAMLValue getValue() {
-      exists(QL::YamlKeyvaluepair pair |
+    YamlValue getValue() {
+      exists(Yaml::Keyvaluepair pair |
         pair.getParent() = yamle and
         result = TYamlValue(pair.getValue())
       )
@@ -2472,15 +2503,15 @@ module YAML {
 
   /** A YAML key. */
   class YamlKey extends TYamlKey, YAMLNode {
-    QL::YamlKey yamlkey;
+    Yaml::Key yamlkey;
 
     YamlKey() { this = TYamlKey(yamlkey) }
 
     /**
      * Gets the value of this YAML key.
      */
-    YAMLValue getValue() {
-      exists(QL::YamlKeyvaluepair pair |
+    YamlValue getValue() {
+      exists(Yaml::Keyvaluepair pair |
         pair.getKey() = yamlkey and result = TYamlValue(pair.getValue())
       )
     }
@@ -2489,7 +2520,7 @@ module YAML {
 
     /** Gets the value of this YAML value. */
     string getNamePart(int i) {
-      i = 0 and result = yamlkey.getChild(0).(QL::SimpleId).getValue()
+      i = 0 and result = yamlkey.getChild(0).(Yaml::SimpleId).getValue()
       or
       exists(YamlKey child |
         child = TYamlKey(yamlkey.getChild(1)) and
@@ -2511,14 +2542,14 @@ module YAML {
 
   /** A YAML list item. */
   class YamlListItem extends TYamlListitem, YAMLNode {
-    QL::YamlListitem yamllistitem;
+    Yaml::Listitem yamllistitem;
 
     YamlListItem() { this = TYamlListitem(yamllistitem) }
 
     /**
      * Gets the value of this YAML list item.
      */
-    YAMLValue getValue() { result = TYamlValue(yamllistitem.getChild()) }
+    YamlValue getValue() { result = TYamlValue(yamllistitem.getChild()) }
 
     override string getAPrimaryQlClass() { result = "YamlListItem" }
   }
@@ -2527,12 +2558,12 @@ module YAML {
   deprecated class YAMLListItem = YamlListItem;
 
   /** A YAML value. */
-  class YAMLValue extends TYamlValue, YAMLNode {
-    QL::YamlValue yamlvalue;
+  class YamlValue extends TYamlValue, YAMLNode {
+    Yaml::Value yamlvalue;
 
-    YAMLValue() { this = TYamlValue(yamlvalue) }
+    YamlValue() { this = TYamlValue(yamlvalue) }
 
-    override string getAPrimaryQlClass() { result = "YAMLValue" }
+    override string getAPrimaryQlClass() { result = "YamlValue" }
 
     /** Gets the value of this YAML value. */
     string getValue() { result = yamlvalue.getValue() }