Merge pull request #10894 from igfoo/igfoo/psi

Kotlin: Refactor PSI handling

Author: igfoo

java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt

@@ -2,6 +2,7 @@ package com.github.codeql
 
 import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
 import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.ir.IrElement
@@ -139,6 +140,8 @@ class KotlinExtractorExtension(
             logger.flush()
             logger.info("Extraction for invocation TRAP file $invocationTrapFile")
             logger.flush()
+            logger.info("Kotlin version ${KotlinCompilerVersion.getVersion()}")
+            logger.flush()
             logPeakMemoryUsage(logger, "before extractor")
             if (System.getenv("CODEQL_EXTRACTOR_JAVA_KOTLIN_DUMP") == "true") {
                 logger.info("moduleFragment:\n" + moduleFragment.dump())

java/kotlin-extractor/src/main/kotlin/LinesOfCode.kt

@@ -1,9 +1,10 @@
 package com.github.codeql
 
-import com.github.codeql.utils.versions.Psi2Ir
+import com.github.codeql.utils.versions.getPsi2Ir
 import com.intellij.psi.PsiComment
 import com.intellij.psi.PsiElement
 import com.intellij.psi.PsiWhiteSpace
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.kdoc.psi.api.KDocElement
@@ -15,9 +16,16 @@ class LinesOfCode(
     val tw: FileTrapWriter,
     val file: IrFile
 ) {
-    val psi2Ir = Psi2Ir(logger)
+    val psi2Ir = getPsi2Ir(logger).also {
+        if (it == null) {
+            logger.warn("Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
+        }
+    }
 
     fun linesOfCodeInFile(id: Label<DbFile>) {
+        if (psi2Ir == null) {
+            return
+        }
         val ktFile = psi2Ir.getKtFile(file)
         if (ktFile == null) {
             return
@@ -26,6 +34,9 @@ class LinesOfCode(
     }
 
     fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label<out DbSourceline>) {
+        if (psi2Ir == null) {
+            return
+        }
         val p = psi2Ir.findPsiElement(d, file)
         if (p == null) {
             return

java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt

@@ -3,9 +3,11 @@ package com.github.codeql.comments
 import com.github.codeql.*
 import com.github.codeql.utils.IrVisitorLookup
 import com.github.codeql.utils.isLocalFunction
-import com.github.codeql.utils.versions.Psi2Ir
+import com.github.codeql.utils.versions.getPsi2Ir
+import com.github.codeql.utils.versions.Psi2IrFacade
 import com.intellij.psi.PsiComment
 import com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.expressions.IrBody
@@ -21,18 +23,23 @@ import org.jetbrains.kotlin.psi.psiUtil.startOffset
 class CommentExtractor(private val fileExtractor: KotlinFileExtractor, private val file: IrFile, private val fileLabel: Label<out DbFile>) {
     private val tw = fileExtractor.tw
     private val logger = fileExtractor.logger
-    private val psi2Ir = Psi2Ir(logger)
-    private val ktFile = psi2Ir.getKtFile(file)
 
     fun extract() {
+        val psi2Ir = getPsi2Ir(logger)
+        if (psi2Ir == null) {
+            logger.warn("Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
+            return
+        }
+        val ktFile = psi2Ir.getKtFile(file)
         if (ktFile == null) {
             logger.warn("Comments are not being processed in ${file.path}.")
-        } else {
-            ktFile.accept(commentVisitor)
+            return
         }
+        val commentVisitor = mkCommentVisitor(psi2Ir)
+        ktFile.accept(commentVisitor)
     }
 
-    private val commentVisitor =
+    private fun mkCommentVisitor(psi2Ir: Psi2IrFacade): KtVisitor<Unit, Unit> =
         object : KtVisitor<Unit, Unit>() {
             override fun visitElement(element: PsiElement) {
                 element.acceptChildren(this)

java/kotlin-extractor/src/main/kotlin/utils/IrVisitorLookup.kt

@@ -1,14 +1,14 @@
 package com.github.codeql.utils
 
-import com.github.codeql.utils.versions.Psi2Ir
+import com.github.codeql.utils.versions.Psi2IrFacade
 import com.intellij.psi.PsiElement
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrDeclaration
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.util.isFakeOverride
 import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
 
-class IrVisitorLookup(private val psi2Ir: Psi2Ir, private val psi: PsiElement, private val file: IrFile) :
+class IrVisitorLookup(private val psi2Ir: Psi2IrFacade, private val psi: PsiElement, private val file: IrFile) :
     IrElementVisitor<Unit, MutableCollection<IrElement>> {
     private val location = psi.getLocation()
 

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4_32/Psi2Ir.kt

@@ -1,19 +1,5 @@
 package com.github.codeql.utils.versions
 
 import com.github.codeql.FileLogger
-import com.intellij.psi.PsiElement
-import org.jetbrains.kotlin.ir.IrElement
-import org.jetbrains.kotlin.ir.declarations.IrFile
-import org.jetbrains.kotlin.psi.KtFile
 
-class Psi2Ir(private val logger: FileLogger) : Psi2IrFacade {
-    override fun getKtFile(irFile: IrFile): KtFile? {
-        logger.warn("Comment extraction is not supported for Kotlin < 1.5.20")
-        return null
-    }
-
-    override fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement? {
-        logger.error("Attempted comment extraction for Kotlin < 1.5.20")
-        return null
-    }
-}
+fun getPsi2Ir(@Suppress("UNUSED_PARAMETER") logger: FileLogger): Psi2IrFacade? = null

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_20/Psi2Ir.kt

@@ -8,7 +8,9 @@ import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.psi.KtFile
 
-class Psi2Ir(private val logger: FileLogger): Psi2IrFacade {
+fun getPsi2Ir(logger: FileLogger): Psi2IrFacade? = Psi2Ir(logger)
+
+private class Psi2Ir(private val logger: FileLogger): Psi2IrFacade {
     override fun getKtFile(irFile: IrFile): KtFile? {
         return irFile.getKtFile()
     }

java/ql/integration-tests/posix-only/kotlin/logs/index_logs.py

@@ -32,6 +32,7 @@ def write_line(origin, kind, msg):
                     j = json.loads(line)
                     msg = j['message']
                     msg = re.sub('(?<=Extraction for invocation TRAP file ).*/kt-db/trap/java/invocations/kotlin\..*\.trap', '<FILENAME>', msg)
+                    msg = re.sub('(?<=Kotlin version )[0-9.]+', '<VERSION>', msg)
                     if msg.startswith('Peak memory: '):
                         # Peak memory information varies from run to run, so just ignore it
                         continue

java/ql/integration-tests/posix-only/kotlin/logs/logs.expected

@@ -1,5 +1,6 @@
 | 1 | 1 | Test script | Log file | 1 |
 | 1 | 2 | CodeQL Kotlin extractor | INFO | Extraction started |
 | 1 | 3 | CodeQL Kotlin extractor | INFO | Extraction for invocation TRAP file <FILENAME> |
-| 1 | 4 | CodeQL Kotlin extractor | INFO | Extracting file test.kt |
-| 1 | 5 | CodeQL Kotlin extractor | INFO | Extraction completed |
+| 1 | 4 | CodeQL Kotlin extractor | INFO | Kotlin version <VERSION> |
+| 1 | 5 | CodeQL Kotlin extractor | INFO | Extracting file test.kt |
+| 1 | 6 | CodeQL Kotlin extractor | INFO | Extraction completed |