Skip to content

Commit 9f9a516

Browse files
smowtonsmowton
authored
Merge pull request github#11510 from smowton/smowton/fix/kotlin-populate-source-class-files
Kotlin: stub trap .class files when extracting a class from Kotlin source
2 parents 170c9af + c526020 commit 9f9a516

File tree

19 files changed

+228
-229
lines changed

19 files changed

+228
-229
lines changed

‎java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java‎

Lines changed: 34 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@
1919
import java.util.zip.ZipFile;
2020

2121
import com.github.codeql.Logger;
22-
import static com.github.codeql.ClassNamesKt.getIrDeclBinaryName;
22+
import static com.github.codeql.ClassNamesKt.getIrElementBinaryName;
2323
import static com.github.codeql.ClassNamesKt.getIrClassVirtualFile;
2424

25+
import org.jetbrains.kotlin.ir.IrElement;
2526
import org.jetbrains.kotlin.ir.declarations.IrClass;
2627

2728
import com.intellij.openapi.vfs.VirtualFile;
@@ -212,20 +213,19 @@ private File trapFileFor(File file) {
212213
PathTransformer.std().fileAsDatabaseString(file) + ".trap.gz");
213214
}
214215

215-
private File getTrapFileForDecl(IrDeclaration sym, String signature) {
216+
private File getTrapFileForDecl(IrElement sym, String signature) {
216217
if (currentSpecFileEntry == null)
217218
return null;
218219
return trapFileForDecl(sym, signature);
219220
}
220221

221-
private File trapFileForDecl(IrDeclaration sym, String signature) {
222+
private File trapFileForDecl(IrElement sym, String signature) {
222223
return FileUtil.fileRelativeTo(currentSpecFileEntry.getTrapFolder(),
223224
trapFilePathForDecl(sym, signature));
224225
}
225226

226-
private String trapFilePathForDecl(IrDeclaration sym, String signature) {
227-
String binaryName = getIrDeclBinaryName(sym);
228-
String binaryNameWithSignature = binaryName + signature;
227+
private String trapFilePathForDecl(IrElement sym, String signature) {
228+
String binaryName = getIrElementBinaryName(sym);
229229
// TODO: Reinstate this?
230230
//if (getTrackClassOrigins())
231231
// classId += "-" + StringDigestor.digest(sym.getSourceFileId());
@@ -241,7 +241,7 @@ private String trapFilePathForDecl(IrDeclaration sym, String signature) {
241241
* Deletion of existing trap files.
242242
*/
243243

244-
private void deleteTrapFileAndDependencies(IrDeclaration sym, String signature) {
244+
private void deleteTrapFileAndDependencies(IrElement sym, String signature) {
245245
File trap = trapFileForDecl(sym, signature);
246246
if (trap.exists()) {
247247
trap.delete();
@@ -269,7 +269,7 @@ private void deleteTrapFileAndDependencies(IrDeclaration sym, String signature)
269269
* Any unique suffix needed to distinguish `sym` from other declarations with the same name.
270270
* For functions for example, this means its parameter signature.
271271
*/
272-
private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, TrapClassVersion trapFileVersion, IrDeclaration sym, String signature) {
272+
private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, TrapClassVersion trapFileVersion, IrElement sym, String signature) {
273273
if (use_trap_locking) {
274274
TrapClassVersion currVersion = TrapClassVersion.fromSymbol(sym, log);
275275
String shortName = sym instanceof IrDeclarationWithName ? ((IrDeclarationWithName)sym).getName().asString() : "(name unknown)";
@@ -326,7 +326,7 @@ private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, Tr
326326
return trapWriter(trap, sym, signature);
327327
}
328328

329-
private TrapFileManager trapWriter(File trapFile, IrDeclaration sym, String signature) {
329+
private TrapFileManager trapWriter(File trapFile, IrElement sym, String signature) {
330330
if (!trapFile.getName().endsWith(".trap.gz"))
331331
throw new CatastrophicError("OdasaOutput only supports writing to compressed trap files");
332332
String relative = FileUtil.relativePath(trapFile, currentSpecFileEntry.getTrapFolder());
@@ -335,7 +335,7 @@ private TrapFileManager trapWriter(File trapFile, IrDeclaration sym, String sign
335335
return concurrentWriter(trapFile, relative, log, sym, signature);
336336
}
337337

338-
private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrDeclaration sym, String signature) {
338+
private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrElement sym, String signature) {
339339
if (trapFile.exists())
340340
return null;
341341
return new TrapFileManager(trapFile, relative, true, log, sym, signature);
@@ -345,11 +345,11 @@ public class TrapFileManager implements AutoCloseable {
345345

346346
private TrapDependencies trapDependenciesForClass;
347347
private File trapFile;
348-
private IrDeclaration sym;
348+
private IrElement sym;
349349
private String signature;
350350
private boolean hasError = false;
351351

352-
private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrDeclaration sym, String signature) {
352+
private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrElement sym, String signature) {
353353
trapDependenciesForClass = new TrapDependencies(relative);
354354
this.trapFile = trapFile;
355355
this.sym = sym;
@@ -360,7 +360,7 @@ public File getFile() {
360360
return trapFile;
361361
}
362362

363-
public void addDependency(IrDeclaration dep, String signature) {
363+
public void addDependency(IrElement dep, String signature) {
364364
trapDependenciesForClass.addDependency(trapFilePathForDecl(dep, signature));
365365
}
366366

@@ -422,7 +422,7 @@ public void setHasError() {
422422
* previously set by a call to {@link OdasaOutput#setCurrentSourceFile(File)}.
423423
*/
424424
public TrapLocker getTrapLockerForCurrentSourceFile() {
425-
return new TrapLocker((IrClass)null, null);
425+
return new TrapLocker((IrClass)null, null, true);
426426
}
427427

428428
/**
@@ -460,19 +460,19 @@ public TrapLocker getTrapLockerForModule(String moduleName) {
460460
*
461461
* @return a {@link TrapLocker} for the trap file corresponding to the given class symbol.
462462
*/
463-
public TrapLocker getTrapLockerForDecl(IrDeclaration sym, String signature) {
464-
return new TrapLocker(sym, signature);
463+
public TrapLocker getTrapLockerForDecl(IrElement sym, String signature, boolean fromSource) {
464+
return new TrapLocker(sym, signature, fromSource);
465465
}
466466

467467
public class TrapLocker implements AutoCloseable {
468-
private final IrDeclaration sym;
468+
private final IrElement sym;
469469
private final File trapFile;
470470
// trapFileBase is used when doing lockless TRAP file writing.
471471
// It is trapFile without the #metadata.trap.gz suffix.
472472
private File trapFileBase = null;
473473
private TrapClassVersion trapFileVersion = null;
474474
private final String signature;
475-
private TrapLocker(IrDeclaration decl, String signature) {
475+
private TrapLocker(IrElement decl, String signature, boolean fromSource) {
476476
this.sym = decl;
477477
this.signature = signature;
478478
if (sym==null) {
@@ -485,7 +485,10 @@ private TrapLocker(IrDeclaration decl, String signature) {
485485
} else {
486486
// We encode the metadata into the filename, so that the
487487
// TRAP filenames for different metadatas don't overlap.
488-
trapFileVersion = TrapClassVersion.fromSymbol(sym, log);
488+
if (fromSource)
489+
trapFileVersion = new TrapClassVersion(0, 0, 0, "kotlin");
490+
else
491+
trapFileVersion = TrapClassVersion.fromSymbol(sym, log);
489492
String baseName = normalTrapFile.getName().replace(".trap.gz", "");
490493
// If a class has lots of inner classes, then we get lots of files
491494
// in a single directory. This makes our directory listings later slow.
@@ -717,11 +720,18 @@ private static long getVirtualFileTimeStamp(VirtualFile vf, Logger log) {
717720
return vf.getTimeStamp();
718721
}
719722

720-
private static TrapClassVersion fromSymbol(IrDeclaration sym, Logger log) {
721-
VirtualFile vf = sym instanceof IrClass ? getIrClassVirtualFile((IrClass)sym) :
722-
sym.getParent() instanceof IrClass ? getIrClassVirtualFile((IrClass)sym.getParent()) :
723-
null;
724-
if(vf == null)
723+
private static VirtualFile getVirtualFileIfClass(IrElement e) {
724+
if (e instanceof IrClass)
725+
return getIrClassVirtualFile((IrClass)e);
726+
else
727+
return null;
728+
}
729+
730+
private static TrapClassVersion fromSymbol(IrElement sym, Logger log) {
731+
VirtualFile vf = getVirtualFileIfClass(sym);
732+
if (vf == null && sym instanceof IrDeclaration)
733+
vf = getVirtualFileIfClass(((IrDeclaration)sym).getParent());
734+
if (vf == null)
725735
return new TrapClassVersion(-1, 0, 0, null);
726736

727737
final int[] versionStore = new int[1];

‎java/kotlin-extractor/src/main/kotlin/ExternalDeclExtractor.kt‎

Lines changed: 78 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,14 @@
11
package com.github.codeql
22

3-
import com.github.codeql.utils.isExternalDeclaration
43
import com.github.codeql.utils.isExternalFileClassMember
54
import com.semmle.extractor.java.OdasaOutput
65
import com.semmle.util.data.StringDigestor
76
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
7+
import org.jetbrains.kotlin.ir.IrElement
88
import org.jetbrains.kotlin.ir.declarations.*
9-
import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable
109
import org.jetbrains.kotlin.ir.util.isFileClass
1110
import org.jetbrains.kotlin.ir.util.packageFqName
12-
import org.jetbrains.kotlin.ir.util.parentClassOrNull
13-
import org.jetbrains.kotlin.name.FqName
11+
import java.io.BufferedWriter
1412
import java.io.File
1513
import java.util.ArrayList
1614
import java.util.HashSet
@@ -25,87 +23,103 @@ class ExternalDeclExtractor(val logger: FileLogger, val invocationTrapFile: Stri
2523
val propertySignature = ";property"
2624
val fieldSignature = ";field"
2725

26+
val output = OdasaOutput(false, logger).also {
27+
it.setCurrentSourceFile(File(sourceFilePath))
28+
}
29+
2830
fun extractLater(d: IrDeclarationWithName, signature: String): Boolean {
2931
if (d !is IrClass && !isExternalFileClassMember(d)) {
3032
logger.errorElement("External declaration is neither a class, nor a top-level declaration", d)
3133
return false
3234
}
33-
val declBinaryName = declBinaryNames.getOrPut(d) { getIrDeclBinaryName(d) }
35+
val declBinaryName = declBinaryNames.getOrPut(d) { getIrElementBinaryName(d) }
3436
val ret = externalDeclsDone.add(Pair(declBinaryName, signature))
3537
if (ret) externalDeclWorkList.add(Pair(d, signature))
3638
return ret
3739
}
3840
fun extractLater(c: IrClass) = extractLater(c, "")
3941

42+
fun writeStubTrapFile(e: IrElement, signature: String = "") {
43+
extractElement(e, signature, true) { trapFileBW, _, _ ->
44+
trapFileBW.write("// Trap file stubbed because this declaration was extracted from source in $sourceFilePath\n")
45+
trapFileBW.write("// Part of invocation $invocationTrapFile\n")
46+
}
47+
}
48+
49+
private fun extractElement(element: IrElement, possiblyLongSignature: String, fromSource: Boolean, extractorFn: (BufferedWriter, String, OdasaOutput.TrapFileManager) -> Unit) {
50+
// In order to avoid excessively long signatures which can lead to trap file names longer than the filesystem
51+
// limit, we truncate and add a hash to preserve uniqueness if necessary.
52+
val signature = if (possiblyLongSignature.length > 100) {
53+
possiblyLongSignature.substring(0, 92) + "#" + StringDigestor.digest(possiblyLongSignature).substring(0, 8)
54+
} else { possiblyLongSignature }
55+
output.getTrapLockerForDecl(element, signature, fromSource).useAC { locker ->
56+
locker.trapFileManager.useAC { manager ->
57+
val shortName = when(element) {
58+
is IrDeclarationWithName -> element.name.asString()
59+
is IrFile -> element.name
60+
else -> "(unknown name)"
61+
}
62+
if (manager == null) {
63+
logger.info("Skipping extracting external decl $shortName")
64+
} else {
65+
val trapFile = manager.file
66+
val trapTmpFile = File.createTempFile("${trapFile.nameWithoutExtension}.", ".${trapFile.extension}.tmp", trapFile.parentFile)
67+
try {
68+
GZIPOutputStream(trapTmpFile.outputStream()).bufferedWriter().use {
69+
extractorFn(it, signature, manager)
70+
}
71+
72+
if (!trapTmpFile.renameTo(trapFile)) {
73+
logger.error("Failed to rename $trapTmpFile to $trapFile")
74+
}
75+
} catch (e: Exception) {
76+
manager.setHasError()
77+
logger.error("Failed to extract '$shortName'. Partial TRAP file location is $trapTmpFile", e)
78+
}
79+
}
80+
}
81+
}
82+
}
83+
4084
fun extractExternalClasses() {
41-
val output = OdasaOutput(false, logger)
42-
output.setCurrentSourceFile(File(sourceFilePath))
4385
do {
4486
val nextBatch = ArrayList(externalDeclWorkList)
4587
externalDeclWorkList.clear()
4688
nextBatch.forEach { workPair ->
4789
val (irDecl, possiblyLongSignature) = workPair
48-
// In order to avoid excessively long signatures which can lead to trap file names longer than the filesystem
49-
// limit, we truncate and add a hash to preserve uniqueness if necessary.
50-
val signature = if (possiblyLongSignature.length > 100) {
51-
possiblyLongSignature.substring(0, 92) + "#" + StringDigestor.digest(possiblyLongSignature).substring(0, 8)
52-
} else { possiblyLongSignature }
53-
output.getTrapLockerForDecl(irDecl, signature).useAC { locker ->
54-
locker.trapFileManager.useAC { manager ->
55-
val shortName = when(irDecl) {
56-
is IrDeclarationWithName -> irDecl.name.asString()
57-
else -> "(unknown name)"
58-
}
59-
if(manager == null) {
60-
logger.info("Skipping extracting external decl $shortName")
61-
} else {
62-
val trapFile = manager.file
63-
val trapTmpFile = File.createTempFile("${trapFile.nameWithoutExtension}.", ".${trapFile.extension}.tmp", trapFile.parentFile)
64-
65-
val containingClass = getContainingClassOrSelf(irDecl)
66-
if (containingClass == null) {
67-
logger.errorElement("Unable to get containing class", irDecl)
68-
return
69-
}
70-
val binaryPath = getIrClassBinaryPath(containingClass)
71-
try {
72-
GZIPOutputStream(trapTmpFile.outputStream()).bufferedWriter().use { trapFileBW ->
73-
// We want our comments to be the first thing in the file,
74-
// so start off with a mere TrapWriter
75-
val tw = TrapWriter(logger.loggerBase, TrapLabelManager(), trapFileBW, diagnosticTrapWriter)
76-
tw.writeComment("Generated by the CodeQL Kotlin extractor for external dependencies")
77-
tw.writeComment("Part of invocation $invocationTrapFile")
78-
if (signature != possiblyLongSignature) {
79-
tw.writeComment("Function signature abbreviated; full signature is: $possiblyLongSignature")
80-
}
81-
// Now elevate to a SourceFileTrapWriter, and populate the
82-
// file information if needed:
83-
val ftw = tw.makeFileTrapWriter(binaryPath, true)
90+
extractElement(irDecl, possiblyLongSignature, false) { trapFileBW, signature, manager ->
91+
val containingClass = getContainingClassOrSelf(irDecl)
92+
if (containingClass == null) {
93+
logger.errorElement("Unable to get containing class", irDecl)
94+
} else {
95+
val binaryPath = getIrClassBinaryPath(containingClass)
8496

85-
val fileExtractor = KotlinFileExtractor(logger, ftw, null, binaryPath, manager, this, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState)
97+
// We want our comments to be the first thing in the file,
98+
// so start off with a mere TrapWriter
99+
val tw = TrapWriter(logger.loggerBase, TrapLabelManager(), trapFileBW, diagnosticTrapWriter)
100+
tw.writeComment("Generated by the CodeQL Kotlin extractor for external dependencies")
101+
tw.writeComment("Part of invocation $invocationTrapFile")
102+
if (signature != possiblyLongSignature) {
103+
tw.writeComment("Function signature abbreviated; full signature is: $possiblyLongSignature")
104+
}
105+
// Now elevate to a SourceFileTrapWriter, and populate the
106+
// file information if needed:
107+
val ftw = tw.makeFileTrapWriter(binaryPath, true)
86108

87-
if (irDecl is IrClass) {
88-
// Populate a location and compilation-unit package for the file. This is similar to
89-
// the beginning of `KotlinFileExtractor.extractFileContents` but without an `IrFile`
90-
// to start from.
91-
val pkg = irDecl.packageFqName?.asString() ?: ""
92-
val pkgId = fileExtractor.extractPackage(pkg)
93-
ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation())
94-
ftw.writeCupackage(ftw.fileId, pkgId)
109+
val fileExtractor = KotlinFileExtractor(logger, ftw, null, binaryPath, manager, this, primitiveTypeMapping, pluginContext, KotlinFileExtractor.DeclarationStack(), globalExtensionState)
95110

96-
fileExtractor.extractClassSource(irDecl, extractDeclarations = !irDecl.isFileClass, extractStaticInitializer = false, extractPrivateMembers = false, extractFunctionBodies = false)
97-
} else {
98-
fileExtractor.extractDeclaration(irDecl, extractPrivateMembers = false, extractFunctionBodies = false)
99-
}
100-
}
111+
if (irDecl is IrClass) {
112+
// Populate a location and compilation-unit package for the file. This is similar to
113+
// the beginning of `KotlinFileExtractor.extractFileContents` but without an `IrFile`
114+
// to start from.
115+
val pkg = irDecl.packageFqName?.asString() ?: ""
116+
val pkgId = fileExtractor.extractPackage(pkg)
117+
ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation())
118+
ftw.writeCupackage(ftw.fileId, pkgId)
101119

102-
if (!trapTmpFile.renameTo(trapFile)) {
103-
logger.error("Failed to rename $trapTmpFile to $trapFile")
104-
}
105-
} catch (e: Exception) {
106-
manager.setHasError()
107-
logger.error("Failed to extract '$shortName'. Partial TRAP file location is $trapTmpFile", e)
108-
}
120+
fileExtractor.extractClassSource(irDecl, extractDeclarations = !irDecl.isFileClass, extractStaticInitializer = false, extractPrivateMembers = false, extractFunctionBodies = false)
121+
} else {
122+
fileExtractor.extractDeclaration(irDecl, extractPrivateMembers = false, extractFunctionBodies = false)
109123
}
110124
}
111125
}

0 commit comments

Comments
 (0)