Merge pull request #11527 from michaelnebel/java/regeneratemodels

Java/C#: Delete old model generator scripts and update Java model re-generator script.

Author: michaelnebel

.github/workflows/mad_modelDiff.yml

@@ -11,7 +11,7 @@ on:
     branches:
       - main
     paths:
-      - "java/ql/src/utils/model-generator/**/*.*"
+      # - "java/ql/src/utils/model-generator/**/*.*"
       - ".github/workflows/mad_modelDiff.yml"
 
 permissions:
@@ -61,8 +61,8 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/model-generator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $MODELS/${SHORTNAME}.qll
-            mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
+            python java/ql/src/utils/model-generator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $MODELS/${SHORTNAME}.model.yml
+            mv $MODELS/${SHORTNAME}.model.yml $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.model.yml
             cd ..
           }
 
@@ -85,16 +85,16 @@ jobs:
           set -x
           MODELS=`pwd`/tmp-models
           ls -1 tmp-models/
-          for m in $MODELS/*_main.qll ; do
+          for m in $MODELS/*_main.model.yml ; do
             t="${m/main/"pr"}"
             basename=`basename $m`
-            name="diff_${basename/_main.qll/""}"
+            name="diff_${basename/_main.model.yml/""}"
             (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
           done
       - uses: actions/upload-artifact@v3
         with:
           name: models
-          path: tmp-models/*.qll
+          path: tmp-models/*.model.yml
           retention-days: 20
       - uses: actions/upload-artifact@v3
         with:

.github/workflows/mad_regenerate-models.yml

@@ -53,7 +53,7 @@ jobs:
           java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
       - name: Stage changes
         run: |
-          find java -name "*.qll" -print0 | xargs -0 git add
+          find java -name "*.model.yml" -print0 | xargs -0 git add
           git status
           git diff --cached > models.patch
       - uses: actions/upload-artifact@v3

csharp/ql/src/utils/model-generator/GenerateFlowModelExtensions.py

@@ -5,19 +5,17 @@
 from pathlib import Path
 import json
 import os
-import requests
 import shutil
 import subprocess
 import tempfile
 import sys
 
 
-defaultModelPath = "java/ql/lib/semmle/code/java/frameworks"
 lgtmSlugToModelFile = {
     # "apache/commons-beanutils": "apache/BeanUtilsGenerated.qll",
     # "apache/commons-codec": "apache/CodecGenerated.qll",
     # "apache/commons-lang": "apache/Lang3Generated.qll",
-    "apache/commons-io": "apache/IOGenerated.qll",
+    "apache/commons-io": "org.apache.commons.io",
 }
 
 
@@ -36,13 +34,12 @@ def regenerateModel(lgtmSlug, extractedDb):
         print("ERROR: slug " + lgtmSlug +
               " is not mapped to a model file in script " + sys.argv[0])
         sys.exit(1)
-    modelFile = defaultModelPath + "/" + lgtmSlugToModelFile[lgtmSlug]
+    modelFile = lgtmSlugToModelFile[lgtmSlug]
     codeQlRoot = findGitRoot()
-    targetModel = codeQlRoot + "/" + modelFile
     subprocess.check_call([codeQlRoot + "/java/ql/src/utils/model-generator/GenerateFlowModel.py",
-                           "--with-summaries", "--with-sinks",
-                           extractedDb, targetModel])
-    print("Regenerated " + targetModel)
+                           "--with-summaries", "--with-sinks", "--with-negative-summaries",
+                           extractedDb, modelFile])
+    print("Regenerated " + modelFile)
     shutil.rmtree(tmpDir)