diff --git a/lib/analyze-action-post.js b/lib/analyze-action-post.js index 64bd028666..4cabbab03a 100644 --- a/lib/analyze-action-post.js +++ b/lib/analyze-action-post.js @@ -119927,6 +119927,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 04fac76278..b8804700e1 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -88172,6 +88172,7 @@ var fs6 = __toESM(require("fs")); var path6 = __toESM(require("path")); // src/caching-utils.ts +var crypto = __toESM(require("crypto")); var core6 = __toESM(require_core()); async function getTotalCacheSize(paths, logger, quiet = false) { const sizes = await Promise.all( @@ -88182,6 +88183,11 @@ async function getTotalCacheSize(paths, logger, quiet = false) { function shouldStoreCache(kind) { return kind === "full" /* Full */ || kind === "store" /* Store */; } +var cacheKeyHashLength = 16; +function createCacheKeyHash(components) { + const componentsJson = JSON.stringify(components); + return crypto.createHash("sha256").update(componentsJson).digest("hex").substring(0, cacheKeyHashLength); +} // src/config/db-config.ts var jsonschema = __toESM(require_lib3()); @@ -88207,7 +88213,6 @@ var bundleVersion = "codeql-bundle-v2.23.3"; var cliVersion = "2.23.3"; // src/overlay-database-utils.ts -var crypto = __toESM(require("crypto")); var fs3 = __toESM(require("fs")); var path3 = __toESM(require("path")); var actionsCache = __toESM(require_cache3()); @@ -88619,10 +88624,6 @@ async function getCacheRestoreKeyPrefix(config, codeQlVersion) { const componentsHash = createCacheKeyHash(cacheKeyComponents); return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`; } -function createCacheKeyHash(components) { - const componentsJson = JSON.stringify(components); - return crypto.createHash("sha256").update(componentsJson).digest("hex").substring(0, 16); -} // src/tools-features.ts var semver3 = __toESM(require_semver2()); @@ -88656,6 +88657,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", @@ -91052,66 +91058,107 @@ var CODEQL_DEPENDENCY_CACHE_VERSION = 1; function getJavaTempDependencyDir() { return (0, import_path.join)(getTemporaryDirectory(), "codeql_java", "repository"); } -function getDefaultCacheConfig() { - return { - java: { - paths: [ - // Maven - (0, import_path.join)(os3.homedir(), ".m2", "repository"), - // Gradle - (0, import_path.join)(os3.homedir(), ".gradle", "caches"), - // CodeQL Java build-mode: none - getJavaTempDependencyDir() - ], - hash: [ - // Maven - "**/pom.xml", - // Gradle - "**/*.gradle*", - "**/gradle-wrapper.properties", - "buildSrc/**/Versions.kt", - "buildSrc/**/Dependencies.kt", - "gradle/*.versions.toml", - "**/versions.properties" - ] - }, - csharp: { - paths: [(0, import_path.join)(os3.homedir(), ".nuget", "packages")], - hash: [ - // NuGet - "**/packages.lock.json", - // Paket - "**/paket.lock" - ] - }, - go: { - paths: [(0, import_path.join)(os3.homedir(), "go", "pkg", "mod")], - hash: ["**/go.sum"] - } - }; +function getJavaDependencyDirs() { + return [ + // Maven + (0, import_path.join)(os3.homedir(), ".m2", "repository"), + // Gradle + (0, import_path.join)(os3.homedir(), ".gradle", "caches"), + // CodeQL Java build-mode: none + getJavaTempDependencyDir() + ]; } +async function makePatternCheck(patterns) { + const globber = await makeGlobber(patterns); + if ((await globber.glob()).length === 0) { + return void 0; + } + return patterns; +} +var CSHARP_BASE_PATTERNS = [ + // NuGet + "**/packages.lock.json", + // Paket + "**/paket.lock" +]; +var CSHARP_EXTRA_PATTERNS = [ + "**/*.csproj", + "**/packages.config", + "**/nuget.config" +]; +async function getCsharpHashPatterns(codeql, features) { + const basePatterns = await internal.makePatternCheck(CSHARP_BASE_PATTERNS); + if (basePatterns !== void 0) { + return basePatterns; + } + if (await features.getValue("csharp_new_cache_key" /* CsharpNewCacheKey */, codeql)) { + return internal.makePatternCheck(CSHARP_EXTRA_PATTERNS); + } + return void 0; +} +var defaultCacheConfigs = { + java: { + getDependencyPaths: getJavaDependencyDirs, + getHashPatterns: async () => internal.makePatternCheck([ + // Maven + "**/pom.xml", + // Gradle + "**/*.gradle*", + "**/gradle-wrapper.properties", + "buildSrc/**/Versions.kt", + "buildSrc/**/Dependencies.kt", + "gradle/*.versions.toml", + "**/versions.properties" + ]) + }, + csharp: { + getDependencyPaths: () => [(0, import_path.join)(os3.homedir(), ".nuget", "packages")], + getHashPatterns: getCsharpHashPatterns + }, + go: { + getDependencyPaths: () => [(0, import_path.join)(os3.homedir(), "go", "pkg", "mod")], + getHashPatterns: async () => internal.makePatternCheck(["**/go.sum"]) + } +}; async function makeGlobber(patterns) { return glob.create(patterns.join("\n")); } -async function uploadDependencyCaches(config, logger, minimizeJavaJars) { +async function checkHashPatterns(codeql, features, language, cacheConfig, checkType, logger) { + const patterns = await cacheConfig.getHashPatterns(codeql, features); + if (patterns === void 0) { + logger.info( + `Skipping ${checkType} of dependency cache for ${language} as we cannot calculate a hash for the cache key.` + ); + } + return patterns; +} +async function uploadDependencyCaches(codeql, features, config, logger) { const status = []; for (const language of config.languages) { - const cacheConfig = getDefaultCacheConfig()[language]; + const cacheConfig = defaultCacheConfigs[language]; if (cacheConfig === void 0) { logger.info( `Skipping upload of dependency cache for ${language} as we have no caching configuration for it.` ); continue; } - const globber = await makeGlobber(cacheConfig.hash); - if ((await globber.glob()).length === 0) { + const patterns = await checkHashPatterns( + codeql, + features, + language, + cacheConfig, + "upload", + logger + ); + if (patterns === void 0) { status.push({ language, result: "no-hash" /* NoHash */ }); - logger.info( - `Skipping upload of dependency cache for ${language} as we cannot calculate a hash for the cache key.` - ); continue; } - const size = await getTotalCacheSize(cacheConfig.paths, logger, true); + const size = await getTotalCacheSize( + cacheConfig.getDependencyPaths(), + logger, + true + ); if (size === 0) { status.push({ language, result: "empty" /* Empty */ }); logger.info( @@ -91119,13 +91166,13 @@ async function uploadDependencyCaches(config, logger, minimizeJavaJars) { ); continue; } - const key = await cacheKey2(language, cacheConfig, minimizeJavaJars); + const key = await cacheKey2(codeql, features, language, patterns); logger.info( `Uploading cache of size ${size} for ${language} with key ${key}...` ); try { const start = performance.now(); - await actionsCache3.saveCache(cacheConfig.paths, key); + await actionsCache3.saveCache(cacheConfig.getDependencyPaths(), key); const upload_duration_ms = Math.round(performance.now() - start); status.push({ language, @@ -91147,22 +91194,50 @@ async function uploadDependencyCaches(config, logger, minimizeJavaJars) { } return status; } -async function cacheKey2(language, cacheConfig, minimizeJavaJars = false) { - const hash2 = await glob.hashFiles(cacheConfig.hash.join("\n")); - return `${await cachePrefix2(language, minimizeJavaJars)}${hash2}`; +async function cacheKey2(codeql, features, language, patterns) { + const hash2 = await glob.hashFiles(patterns.join("\n")); + return `${await cachePrefix2(codeql, features, language)}${hash2}`; } -async function cachePrefix2(language, minimizeJavaJars) { +async function getFeaturePrefix(codeql, features, language) { + const enabledFeatures = []; + const addFeatureIfEnabled = async (feature) => { + if (await features.getValue(feature, codeql)) { + enabledFeatures.push(feature); + } + }; + if (language === "java" /* java */) { + const minimizeJavaJars = await features.getValue( + "java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, + codeql + ); + if (minimizeJavaJars) { + return "minify-"; + } + } else if (language === "csharp" /* csharp */) { + await addFeatureIfEnabled("csharp_new_cache_key" /* CsharpNewCacheKey */); + } + if (enabledFeatures.length > 0) { + return `${createCacheKeyHash(enabledFeatures)}-`; + } + return ""; +} +async function cachePrefix2(codeql, features, language) { const runnerOs = getRequiredEnvParam("RUNNER_OS"); const customPrefix = process.env["CODEQL_ACTION_DEPENDENCY_CACHE_PREFIX" /* DEPENDENCY_CACHING_PREFIX */]; let prefix = CODEQL_DEPENDENCY_CACHE_PREFIX; if (customPrefix !== void 0 && customPrefix.length > 0) { prefix = `${prefix}-${customPrefix}`; } - if (language === "java" /* java */ && minimizeJavaJars) { - prefix = `minify-${prefix}`; + const featurePrefix = await getFeaturePrefix(codeql, features, language); + if (featurePrefix === "minify-") { + return `${featurePrefix}${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; + } else { + return `${prefix}-${featurePrefix}${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; } - return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; } +var internal = { + makePatternCheck +}; // src/diagnostics.ts var import_fs = require("fs"); @@ -93975,14 +94050,11 @@ async function run() { logger ); if (shouldStoreCache(config.dependencyCachingEnabled)) { - const minimizeJavaJars = await features.getValue( - "java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, - codeql - ); dependencyCacheResults = await uploadDependencyCaches( + codeql, + features, config, - logger, - minimizeJavaJars + logger ); } if (isInTestMode()) { diff --git a/lib/autobuild-action.js b/lib/autobuild-action.js index 94f3208a99..8811fabda7 100644 --- a/lib/autobuild-action.js +++ b/lib/autobuild-action.js @@ -83976,6 +83976,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/init-action-post.js b/lib/init-action-post.js index c8733b2efb..59f2f8dbb4 100644 --- a/lib/init-action-post.js +++ b/lib/init-action-post.js @@ -123308,6 +123308,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/init-action.js b/lib/init-action.js index 9cd3723250..d1e5f331ef 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -85233,6 +85233,7 @@ function wrapApiConfigurationError(e) { } // src/caching-utils.ts +var crypto = __toESM(require("crypto")); var core6 = __toESM(require_core()); async function getTotalCacheSize(paths, logger, quiet = false) { const sizes = await Promise.all( @@ -85265,6 +85266,11 @@ function getCachingKind(input) { return "none" /* None */; } } +var cacheKeyHashLength = 16; +function createCacheKeyHash(components) { + const componentsJson = JSON.stringify(components); + return crypto.createHash("sha256").update(componentsJson).digest("hex").substring(0, cacheKeyHashLength); +} function getDependencyCachingEnabled() { const dependencyCaching = getOptionalInput("dependency-caching") || process.env["CODEQL_ACTION_DEPENDENCY_CACHING" /* DEPENDENCY_CACHING */]; if (dependencyCaching !== void 0) return getCachingKind(dependencyCaching); @@ -85631,7 +85637,6 @@ var bundleVersion = "codeql-bundle-v2.23.3"; var cliVersion = "2.23.3"; // src/overlay-database-utils.ts -var crypto = __toESM(require("crypto")); var fs3 = __toESM(require("fs")); var path4 = __toESM(require("path")); var actionsCache = __toESM(require_cache3()); @@ -86033,10 +86038,6 @@ async function getCacheRestoreKeyPrefix(config, codeQlVersion) { const componentsHash = createCacheKeyHash(cacheKeyComponents); return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`; } -function createCacheKeyHash(components) { - const componentsJson = JSON.stringify(components); - return crypto.createHash("sha256").update(componentsJson).digest("hex").substring(0, 16); -} // src/tools-features.ts var semver3 = __toESM(require_semver2()); @@ -86070,6 +86071,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", @@ -87239,68 +87245,105 @@ var CODEQL_DEPENDENCY_CACHE_VERSION = 1; function getJavaTempDependencyDir() { return (0, import_path.join)(getTemporaryDirectory(), "codeql_java", "repository"); } -function getDefaultCacheConfig() { - return { - java: { - paths: [ - // Maven - (0, import_path.join)(os2.homedir(), ".m2", "repository"), - // Gradle - (0, import_path.join)(os2.homedir(), ".gradle", "caches"), - // CodeQL Java build-mode: none - getJavaTempDependencyDir() - ], - hash: [ - // Maven - "**/pom.xml", - // Gradle - "**/*.gradle*", - "**/gradle-wrapper.properties", - "buildSrc/**/Versions.kt", - "buildSrc/**/Dependencies.kt", - "gradle/*.versions.toml", - "**/versions.properties" - ] - }, - csharp: { - paths: [(0, import_path.join)(os2.homedir(), ".nuget", "packages")], - hash: [ - // NuGet - "**/packages.lock.json", - // Paket - "**/paket.lock" - ] - }, - go: { - paths: [(0, import_path.join)(os2.homedir(), "go", "pkg", "mod")], - hash: ["**/go.sum"] - } - }; +function getJavaDependencyDirs() { + return [ + // Maven + (0, import_path.join)(os2.homedir(), ".m2", "repository"), + // Gradle + (0, import_path.join)(os2.homedir(), ".gradle", "caches"), + // CodeQL Java build-mode: none + getJavaTempDependencyDir() + ]; +} +async function makePatternCheck(patterns) { + const globber = await makeGlobber(patterns); + if ((await globber.glob()).length === 0) { + return void 0; + } + return patterns; +} +var CSHARP_BASE_PATTERNS = [ + // NuGet + "**/packages.lock.json", + // Paket + "**/paket.lock" +]; +var CSHARP_EXTRA_PATTERNS = [ + "**/*.csproj", + "**/packages.config", + "**/nuget.config" +]; +async function getCsharpHashPatterns(codeql, features) { + const basePatterns = await internal.makePatternCheck(CSHARP_BASE_PATTERNS); + if (basePatterns !== void 0) { + return basePatterns; + } + if (await features.getValue("csharp_new_cache_key" /* CsharpNewCacheKey */, codeql)) { + return internal.makePatternCheck(CSHARP_EXTRA_PATTERNS); + } + return void 0; } +var defaultCacheConfigs = { + java: { + getDependencyPaths: getJavaDependencyDirs, + getHashPatterns: async () => internal.makePatternCheck([ + // Maven + "**/pom.xml", + // Gradle + "**/*.gradle*", + "**/gradle-wrapper.properties", + "buildSrc/**/Versions.kt", + "buildSrc/**/Dependencies.kt", + "gradle/*.versions.toml", + "**/versions.properties" + ]) + }, + csharp: { + getDependencyPaths: () => [(0, import_path.join)(os2.homedir(), ".nuget", "packages")], + getHashPatterns: getCsharpHashPatterns + }, + go: { + getDependencyPaths: () => [(0, import_path.join)(os2.homedir(), "go", "pkg", "mod")], + getHashPatterns: async () => internal.makePatternCheck(["**/go.sum"]) + } +}; async function makeGlobber(patterns) { return glob.create(patterns.join("\n")); } -async function downloadDependencyCaches(languages, logger, minimizeJavaJars) { +async function checkHashPatterns(codeql, features, language, cacheConfig, checkType, logger) { + const patterns = await cacheConfig.getHashPatterns(codeql, features); + if (patterns === void 0) { + logger.info( + `Skipping ${checkType} of dependency cache for ${language} as we cannot calculate a hash for the cache key.` + ); + } + return patterns; +} +async function downloadDependencyCaches(codeql, features, languages, logger) { const status = []; for (const language of languages) { - const cacheConfig = getDefaultCacheConfig()[language]; + const cacheConfig = defaultCacheConfigs[language]; if (cacheConfig === void 0) { logger.info( `Skipping download of dependency cache for ${language} as we have no caching configuration for it.` ); continue; } - const globber = await makeGlobber(cacheConfig.hash); - if ((await globber.glob()).length === 0) { + const patterns = await checkHashPatterns( + codeql, + features, + language, + cacheConfig, + "download", + logger + ); + if (patterns === void 0) { status.push({ language, hit_kind: "no-hash" /* NoHash */ }); - logger.info( - `Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.` - ); continue; } - const primaryKey = await cacheKey2(language, cacheConfig, minimizeJavaJars); + const primaryKey = await cacheKey2(codeql, features, language, patterns); const restoreKeys = [ - await cachePrefix2(language, minimizeJavaJars) + await cachePrefix2(codeql, features, language) ]; logger.info( `Downloading cache for ${language} with key ${primaryKey} and restore keys ${restoreKeys.join( @@ -87309,7 +87352,7 @@ async function downloadDependencyCaches(languages, logger, minimizeJavaJars) { ); const start = performance.now(); const hitKey = await actionsCache3.restoreCache( - cacheConfig.paths, + cacheConfig.getDependencyPaths(), primaryKey, restoreKeys ); @@ -87325,22 +87368,50 @@ async function downloadDependencyCaches(languages, logger, minimizeJavaJars) { } return status; } -async function cacheKey2(language, cacheConfig, minimizeJavaJars = false) { - const hash = await glob.hashFiles(cacheConfig.hash.join("\n")); - return `${await cachePrefix2(language, minimizeJavaJars)}${hash}`; +async function cacheKey2(codeql, features, language, patterns) { + const hash = await glob.hashFiles(patterns.join("\n")); + return `${await cachePrefix2(codeql, features, language)}${hash}`; } -async function cachePrefix2(language, minimizeJavaJars) { +async function getFeaturePrefix(codeql, features, language) { + const enabledFeatures = []; + const addFeatureIfEnabled = async (feature) => { + if (await features.getValue(feature, codeql)) { + enabledFeatures.push(feature); + } + }; + if (language === "java" /* java */) { + const minimizeJavaJars = await features.getValue( + "java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, + codeql + ); + if (minimizeJavaJars) { + return "minify-"; + } + } else if (language === "csharp" /* csharp */) { + await addFeatureIfEnabled("csharp_new_cache_key" /* CsharpNewCacheKey */); + } + if (enabledFeatures.length > 0) { + return `${createCacheKeyHash(enabledFeatures)}-`; + } + return ""; +} +async function cachePrefix2(codeql, features, language) { const runnerOs = getRequiredEnvParam("RUNNER_OS"); const customPrefix = process.env["CODEQL_ACTION_DEPENDENCY_CACHE_PREFIX" /* DEPENDENCY_CACHING_PREFIX */]; let prefix = CODEQL_DEPENDENCY_CACHE_PREFIX; if (customPrefix !== void 0 && customPrefix.length > 0) { prefix = `${prefix}-${customPrefix}`; } - if (language === "java" /* java */ && minimizeJavaJars) { - prefix = `minify-${prefix}`; + const featurePrefix = await getFeaturePrefix(codeql, features, language); + if (featurePrefix === "minify-") { + return `${featurePrefix}${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; + } else { + return `${prefix}-${featurePrefix}${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; } - return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; } +var internal = { + makePatternCheck +}; // src/diagnostics.ts var import_fs = require("fs"); @@ -89704,7 +89775,7 @@ async function getWorkflowAbsolutePath(logger) { async function checkWorkflow(logger, codeql) { if (!isDynamicWorkflow() && process.env["CODEQL_ACTION_SKIP_WORKFLOW_VALIDATION" /* SKIP_WORKFLOW_VALIDATION */] !== "true") { core12.startGroup("Validating workflow"); - const validateWorkflowResult = await internal.validateWorkflow( + const validateWorkflowResult = await internal2.validateWorkflow( codeql, logger ); @@ -89718,7 +89789,7 @@ async function checkWorkflow(logger, codeql) { core12.endGroup(); } } -var internal = { +var internal2 = { validateWorkflow }; @@ -90067,15 +90138,12 @@ exec ${goBinaryPath} "$@"` core13.exportVariable(envVar, "false"); } } - const minimizeJavaJars = await features.getValue( - "java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, - codeql - ); if (shouldRestoreCache(config.dependencyCachingEnabled)) { dependencyCachingResults = await downloadDependencyCaches( + codeql, + features, config.languages, - logger, - minimizeJavaJars + logger ); } if (await codeQlVersionAtLeast(codeql, "2.17.1")) { @@ -90113,7 +90181,7 @@ exec ${goBinaryPath} "$@"` logger.debug( `${"CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */} is already set to '${process.env["CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */]}', so the Action will not override it.` ); - } else if (minimizeJavaJars && config.dependencyCachingEnabled && config.buildMode === "none" /* None */ && config.languages.includes("java" /* java */)) { + } else if (await features.getValue("java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, codeql) && config.dependencyCachingEnabled && config.buildMode === "none" /* None */ && config.languages.includes("java" /* java */)) { core13.exportVariable( "CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */, "true" diff --git a/lib/resolve-environment-action.js b/lib/resolve-environment-action.js index 4f6d48f3f9..99bce348a7 100644 --- a/lib/resolve-environment-action.js +++ b/lib/resolve-environment-action.js @@ -83967,6 +83967,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/setup-codeql-action.js b/lib/setup-codeql-action.js index 18b293b2a8..e070664fe8 100644 --- a/lib/setup-codeql-action.js +++ b/lib/setup-codeql-action.js @@ -83595,14 +83595,17 @@ var fs3 = __toESM(require("fs")); var path3 = __toESM(require("path")); var actionsCache = __toESM(require_cache3()); -// src/git-utils.ts +// src/caching-utils.ts var core6 = __toESM(require_core()); + +// src/git-utils.ts +var core7 = __toESM(require_core()); var toolrunner2 = __toESM(require_toolrunner()); var io3 = __toESM(require_io2()); var runGitCommand = async function(workingDirectory, args, customErrorMessage) { let stdout = ""; let stderr = ""; - core6.debug(`Running git command: git ${args.join(" ")}`); + core7.debug(`Running git command: git ${args.join(" ")}`); try { await new toolrunner2.ToolRunner(await io3.which("git", true), args, { silent: true, @@ -83622,7 +83625,7 @@ var runGitCommand = async function(workingDirectory, args, customErrorMessage) { if (stderr.includes("not a git repository")) { reason = "The checkout path provided to the action does not appear to be a git repository."; } - core6.info(`git call failed. ${customErrorMessage} Error: ${reason}`); + core7.info(`git call failed. ${customErrorMessage} Error: ${reason}`); throw error4; } }; @@ -83733,7 +83736,7 @@ async function getRef() { ) !== head; if (hasChangedRef) { const newRef = ref.replace(pull_ref_regex, "refs/pull/$1/head"); - core6.debug( + core7.debug( `No longer on merge commit, rewriting ref from ${ref} to ${newRef}.` ); return newRef; @@ -83759,16 +83762,16 @@ async function isAnalyzingDefaultBranch() { } // src/logging.ts -var core7 = __toESM(require_core()); +var core8 = __toESM(require_core()); function getActionsLogger() { return { - debug: core7.debug, - info: core7.info, - warning: core7.warning, - error: core7.error, - isDebug: core7.isDebug, - startGroup: core7.startGroup, - endGroup: core7.endGroup + debug: core8.debug, + info: core8.info, + warning: core8.warning, + error: core8.error, + isDebug: core8.isDebug, + startGroup: core8.startGroup, + endGroup: core8.endGroup }; } function formatDuration(durationMs) { @@ -83876,6 +83879,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", @@ -84561,9 +84569,6 @@ var AnalysisKind = /* @__PURE__ */ ((AnalysisKind2) => { })(AnalysisKind || {}); var supportedAnalysisKinds = new Set(Object.values(AnalysisKind)); -// src/caching-utils.ts -var core8 = __toESM(require_core()); - // src/config/db-config.ts var jsonschema = __toESM(require_lib4()); var semver4 = __toESM(require_semver2()); diff --git a/lib/start-proxy-action-post.js b/lib/start-proxy-action-post.js index 590556cc8d..2a3aa57833 100644 --- a/lib/start-proxy-action-post.js +++ b/lib/start-proxy-action-post.js @@ -119333,6 +119333,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/start-proxy-action.js b/lib/start-proxy-action.js index 322609fc07..c368419521 100644 --- a/lib/start-proxy-action.js +++ b/lib/start-proxy-action.js @@ -99995,6 +99995,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/upload-lib.js b/lib/upload-lib.js index 6057b1298b..0d40d0f58b 100644 --- a/lib/upload-lib.js +++ b/lib/upload-lib.js @@ -87033,6 +87033,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/upload-sarif-action-post.js b/lib/upload-sarif-action-post.js index c853dd8b65..09e2941167 100644 --- a/lib/upload-sarif-action-post.js +++ b/lib/upload-sarif-action-post.js @@ -119499,6 +119499,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/upload-sarif-action.js b/lib/upload-sarif-action.js index f927d490f9..f61289fa81 100644 --- a/lib/upload-sarif-action.js +++ b/lib/upload-sarif-action.js @@ -86511,14 +86511,17 @@ var fs3 = __toESM(require("fs")); var path3 = __toESM(require("path")); var actionsCache = __toESM(require_cache3()); -// src/git-utils.ts +// src/caching-utils.ts var core6 = __toESM(require_core()); + +// src/git-utils.ts +var core7 = __toESM(require_core()); var toolrunner2 = __toESM(require_toolrunner()); var io3 = __toESM(require_io2()); var runGitCommand = async function(workingDirectory, args, customErrorMessage) { let stdout = ""; let stderr = ""; - core6.debug(`Running git command: git ${args.join(" ")}`); + core7.debug(`Running git command: git ${args.join(" ")}`); try { await new toolrunner2.ToolRunner(await io3.which("git", true), args, { silent: true, @@ -86538,7 +86541,7 @@ var runGitCommand = async function(workingDirectory, args, customErrorMessage) { if (stderr.includes("not a git repository")) { reason = "The checkout path provided to the action does not appear to be a git repository."; } - core6.info(`git call failed. ${customErrorMessage} Error: ${reason}`); + core7.info(`git call failed. ${customErrorMessage} Error: ${reason}`); throw error4; } }; @@ -86683,7 +86686,7 @@ async function getRef() { ) !== head; if (hasChangedRef) { const newRef = ref.replace(pull_ref_regex, "refs/pull/$1/head"); - core6.debug( + core7.debug( `No longer on merge commit, rewriting ref from ${ref} to ${newRef}.` ); return newRef; @@ -86709,16 +86712,16 @@ async function isAnalyzingDefaultBranch() { } // src/logging.ts -var core7 = __toESM(require_core()); +var core8 = __toESM(require_core()); function getActionsLogger() { return { - debug: core7.debug, - info: core7.info, - warning: core7.warning, - error: core7.error, - isDebug: core7.isDebug, - startGroup: core7.startGroup, - endGroup: core7.endGroup + debug: core8.debug, + info: core8.info, + warning: core8.warning, + error: core8.error, + isDebug: core8.isDebug, + startGroup: core8.startGroup, + endGroup: core8.endGroup }; } function formatDuration(durationMs) { @@ -86826,6 +86829,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", @@ -87261,9 +87269,6 @@ var core9 = __toESM(require_core()); var fs6 = __toESM(require("fs")); var path6 = __toESM(require("path")); -// src/caching-utils.ts -var core8 = __toESM(require_core()); - // src/config/db-config.ts var jsonschema = __toESM(require_lib4()); var semver4 = __toESM(require_semver2()); diff --git a/src/analyze-action.ts b/src/analyze-action.ts index ff24dd4007..3ab1dd1321 100644 --- a/src/analyze-action.ts +++ b/src/analyze-action.ts @@ -438,14 +438,11 @@ async function run() { // Store dependency cache(s) if dependency caching is enabled. if (shouldStoreCache(config.dependencyCachingEnabled)) { - const minimizeJavaJars = await features.getValue( - Feature.JavaMinimizeDependencyJars, - codeql, - ); dependencyCacheResults = await uploadDependencyCaches( + codeql, + features, config, logger, - minimizeJavaJars, ); } diff --git a/src/caching-utils.ts b/src/caching-utils.ts index 9da7978016..33dac7cfb4 100644 --- a/src/caching-utils.ts +++ b/src/caching-utils.ts @@ -1,3 +1,5 @@ +import * as crypto from "crypto"; + import * as core from "@actions/core"; import { getOptionalInput, isDefaultSetup } from "./actions-util"; @@ -71,6 +73,33 @@ export function getCachingKind(input: string | undefined): CachingKind { } } +// The length to which `createCacheKeyHash` truncates hash strings. +export const cacheKeyHashLength = 16; + +/** + * Creates a SHA-256 hash of the cache key components to ensure uniqueness + * while keeping the cache key length manageable. + * + * @param components Object containing all components that should influence cache key uniqueness + * @returns A short SHA-256 hash (first 16 characters) of the components + */ +export function createCacheKeyHash(components: Record): string { + // From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify + // + // "Properties are visited using the same algorithm as Object.keys(), which + // has a well-defined order and is stable across implementations. For example, + // JSON.stringify on the same object will always produce the same string, and + // JSON.parse(JSON.stringify(obj)) would produce an object with the same key + // ordering as the original (assuming the object is completely + // JSON-serializable)." + const componentsJson = JSON.stringify(components); + return crypto + .createHash("sha256") + .update(componentsJson) + .digest("hex") + .substring(0, cacheKeyHashLength); +} + /** Determines whether dependency caching is enabled. */ export function getDependencyCachingEnabled(): CachingKind { // If the workflow specified something always respect that diff --git a/src/dependency-caching.test.ts b/src/dependency-caching.test.ts new file mode 100644 index 0000000000..eefb8504cd --- /dev/null +++ b/src/dependency-caching.test.ts @@ -0,0 +1,389 @@ +import * as fs from "fs"; +import path from "path"; + +import * as actionsCache from "@actions/cache"; +import * as glob from "@actions/glob"; +import test from "ava"; +import * as sinon from "sinon"; + +import { cacheKeyHashLength } from "./caching-utils"; +import { createStubCodeQL } from "./codeql"; +import { + CacheConfig, + checkHashPatterns, + getCsharpHashPatterns, + getFeaturePrefix, + makePatternCheck, + internal, + CSHARP_BASE_PATTERNS, + CSHARP_EXTRA_PATTERNS, + downloadDependencyCaches, + CacheHitKind, + cacheKey, +} from "./dependency-caching"; +import { Feature } from "./feature-flags"; +import { KnownLanguage } from "./languages"; +import { + setupTests, + createFeatures, + getRecordingLogger, + checkExpectedLogMessages, + LoggedMessage, +} from "./testing-utils"; +import { withTmpDir } from "./util"; + +setupTests(test); + +function makeAbsolutePatterns(tmpDir: string, patterns: string[]): string[] { + return patterns.map((pattern) => path.join(tmpDir, pattern)); +} + +test("makePatternCheck - returns undefined if no patterns match", async (t) => { + await withTmpDir(async (tmpDir) => { + fs.writeFileSync(path.join(tmpDir, "test.java"), ""); + const result = await makePatternCheck( + makeAbsolutePatterns(tmpDir, ["**/*.cs"]), + ); + t.is(result, undefined); + }); +}); + +test("makePatternCheck - returns all patterns if any pattern matches", async (t) => { + await withTmpDir(async (tmpDir) => { + fs.writeFileSync(path.join(tmpDir, "test.java"), ""); + const patterns = makeAbsolutePatterns(tmpDir, ["**/*.cs", "**/*.java"]); + const result = await makePatternCheck(patterns); + t.deepEqual(result, patterns); + }); +}); + +test("getCsharpHashPatterns - returns base patterns if any pattern matches", async (t) => { + const codeql = createStubCodeQL({}); + const features = createFeatures([]); + const makePatternCheckStub = sinon.stub(internal, "makePatternCheck"); + + makePatternCheckStub + .withArgs(CSHARP_BASE_PATTERNS) + .resolves(CSHARP_BASE_PATTERNS); + makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).rejects(); + + await t.notThrowsAsync(async () => { + const result = await getCsharpHashPatterns(codeql, features); + t.deepEqual(result, CSHARP_BASE_PATTERNS); + }); +}); + +test("getCsharpHashPatterns - returns base patterns if any base pattern matches and CsharpNewCacheKey is enabled", async (t) => { + const codeql = createStubCodeQL({}); + const features = createFeatures([Feature.CsharpNewCacheKey]); + const makePatternCheckStub = sinon.stub(internal, "makePatternCheck"); + + makePatternCheckStub + .withArgs(CSHARP_BASE_PATTERNS) + .resolves(CSHARP_BASE_PATTERNS); + makePatternCheckStub + .withArgs(CSHARP_EXTRA_PATTERNS) + .resolves(CSHARP_EXTRA_PATTERNS); + + await t.notThrowsAsync(async () => { + const result = await getCsharpHashPatterns(codeql, features); + t.deepEqual(result, CSHARP_BASE_PATTERNS); + }); +}); + +test("getCsharpHashPatterns - returns extra patterns if any extra pattern matches and CsharpNewCacheKey is enabled", async (t) => { + const codeql = createStubCodeQL({}); + const features = createFeatures([Feature.CsharpNewCacheKey]); + const makePatternCheckStub = sinon.stub(internal, "makePatternCheck"); + + makePatternCheckStub.withArgs(CSHARP_BASE_PATTERNS).resolves(undefined); + makePatternCheckStub + .withArgs(CSHARP_EXTRA_PATTERNS) + .resolves(CSHARP_EXTRA_PATTERNS); + + await t.notThrowsAsync(async () => { + const result = await getCsharpHashPatterns(codeql, features); + t.deepEqual(result, CSHARP_EXTRA_PATTERNS); + }); +}); + +test("getCsharpHashPatterns - returns undefined if neither base nor extra patterns match", async (t) => { + const codeql = createStubCodeQL({}); + const features = createFeatures([Feature.CsharpNewCacheKey]); + const makePatternCheckStub = sinon.stub(internal, "makePatternCheck"); + + makePatternCheckStub.withArgs(CSHARP_BASE_PATTERNS).resolves(undefined); + makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined); + + await t.notThrowsAsync(async () => { + const result = await getCsharpHashPatterns(codeql, features); + t.deepEqual(result, undefined); + }); +}); + +test("checkHashPatterns - logs when no patterns match", async (t) => { + const codeql = createStubCodeQL({}); + const features = createFeatures([]); + const messages: LoggedMessage[] = []; + const config: CacheConfig = { + getDependencyPaths: () => [], + getHashPatterns: async () => undefined, + }; + + const result = await checkHashPatterns( + codeql, + features, + KnownLanguage.csharp, + config, + "download", + getRecordingLogger(messages), + ); + + t.is(result, undefined); + checkExpectedLogMessages(t, messages, [ + "Skipping download of dependency cache", + ]); +}); + +test("checkHashPatterns - returns patterns when patterns match", async (t) => { + await withTmpDir(async (tmpDir) => { + const codeql = createStubCodeQL({}); + const features = createFeatures([]); + const messages: LoggedMessage[] = []; + const patterns = makeAbsolutePatterns(tmpDir, ["**/*.cs", "**/*.java"]); + + fs.writeFileSync(path.join(tmpDir, "test.java"), ""); + + const config: CacheConfig = { + getDependencyPaths: () => [], + getHashPatterns: async () => makePatternCheck(patterns), + }; + + const result = await checkHashPatterns( + codeql, + features, + KnownLanguage.csharp, + config, + "upload", + getRecordingLogger(messages), + ); + + t.deepEqual(result, patterns); + t.deepEqual(messages, []); + }); +}); + +type RestoreCacheFunc = ( + paths: string[], + primaryKey: string, + restoreKeys: string[] | undefined, +) => Promise; + +/** + * Constructs a function that `actionsCache.restoreCache` can be stubbed with. + * + * @param mockCacheKeys The keys of caches that we want to exist in the Actions cache. + * + * @returns Returns a function that `actionsCache.restoreCache` can be stubbed with. + */ +function makeMockCacheCheck(mockCacheKeys: string[]): RestoreCacheFunc { + return async ( + _paths: string[], + primaryKey: string, + restoreKeys: string[] | undefined, + ) => { + // The behaviour here mirrors what the real `restoreCache` would do: + // - Starting with the primary restore key, check all caches for a match: + // even for the primary restore key, this only has to be a prefix match. + // - If the primary restore key doesn't prefix-match any cache, then proceed + // in the same way for each restore key in turn. + for (const restoreKey of [primaryKey, ...(restoreKeys || [])]) { + for (const mockCacheKey of mockCacheKeys) { + if (mockCacheKey.startsWith(restoreKey)) { + return mockCacheKey; + } + } + } + // Only if no restore key matches any cache key prefix, there is no matching + // cache and we return `undefined`. + return undefined; + }; +} + +test("downloadDependencyCaches - does not restore caches with feature keys if no features are enabled", async (t) => { + process.env["RUNNER_OS"] = "Linux"; + + const codeql = createStubCodeQL({}); + const messages: LoggedMessage[] = []; + const logger = getRecordingLogger(messages); + + sinon.stub(glob, "hashFiles").resolves("abcdef"); + + const keyWithFeature = await cacheKey( + codeql, + createFeatures([Feature.CsharpNewCacheKey]), + KnownLanguage.csharp, + // Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above. + [], + ); + + const restoreCacheStub = sinon + .stub(actionsCache, "restoreCache") + .callsFake(makeMockCacheCheck([keyWithFeature])); + + const makePatternCheckStub = sinon.stub(internal, "makePatternCheck"); + makePatternCheckStub + .withArgs(CSHARP_BASE_PATTERNS) + .resolves(CSHARP_BASE_PATTERNS); + makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined); + + const results = await downloadDependencyCaches( + codeql, + createFeatures([]), + [KnownLanguage.csharp], + logger, + ); + t.is(results.length, 1); + t.is(results[0].language, KnownLanguage.csharp); + t.is(results[0].hit_kind, CacheHitKind.Miss); + t.assert(restoreCacheStub.calledOnce); +}); + +test("downloadDependencyCaches - restores caches with feature keys if features are enabled", async (t) => { + process.env["RUNNER_OS"] = "Linux"; + + const codeql = createStubCodeQL({}); + const messages: LoggedMessage[] = []; + const logger = getRecordingLogger(messages); + const features = createFeatures([Feature.CsharpNewCacheKey]); + + sinon.stub(glob, "hashFiles").resolves("abcdef"); + + const keyWithFeature = await cacheKey( + codeql, + features, + KnownLanguage.csharp, + // Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above. + [], + ); + + const restoreCacheStub = sinon + .stub(actionsCache, "restoreCache") + .callsFake(makeMockCacheCheck([keyWithFeature])); + + const makePatternCheckStub = sinon.stub(internal, "makePatternCheck"); + makePatternCheckStub + .withArgs(CSHARP_BASE_PATTERNS) + .resolves(CSHARP_BASE_PATTERNS); + makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined); + + const results = await downloadDependencyCaches( + codeql, + features, + [KnownLanguage.csharp], + logger, + ); + t.is(results.length, 1); + t.is(results[0].language, KnownLanguage.csharp); + t.is(results[0].hit_kind, CacheHitKind.Exact); + t.assert(restoreCacheStub.calledOnce); +}); + +test("downloadDependencyCaches - restores caches with feature keys if features are enabled for partial matches", async (t) => { + process.env["RUNNER_OS"] = "Linux"; + + const codeql = createStubCodeQL({}); + const messages: LoggedMessage[] = []; + const logger = getRecordingLogger(messages); + const features = createFeatures([Feature.CsharpNewCacheKey]); + + const hashFilesStub = sinon.stub(glob, "hashFiles"); + hashFilesStub.onFirstCall().resolves("abcdef"); + hashFilesStub.onSecondCall().resolves("123456"); + + const keyWithFeature = await cacheKey( + codeql, + features, + KnownLanguage.csharp, + // Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above. + [], + ); + + const restoreCacheStub = sinon + .stub(actionsCache, "restoreCache") + .callsFake(makeMockCacheCheck([keyWithFeature])); + + const makePatternCheckStub = sinon.stub(internal, "makePatternCheck"); + makePatternCheckStub + .withArgs(CSHARP_BASE_PATTERNS) + .resolves(CSHARP_BASE_PATTERNS); + makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined); + + const results = await downloadDependencyCaches( + codeql, + features, + [KnownLanguage.csharp], + logger, + ); + t.is(results.length, 1); + t.is(results[0].language, KnownLanguage.csharp); + t.is(results[0].hit_kind, CacheHitKind.Partial); + t.assert(restoreCacheStub.calledOnce); +}); + +test("getFeaturePrefix - returns empty string if no features are enabled", async (t) => { + const codeql = createStubCodeQL({}); + const features = createFeatures([]); + + for (const knownLanguage of Object.values(KnownLanguage)) { + const result = await getFeaturePrefix(codeql, features, knownLanguage); + t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`); + } +}); + +test("getFeaturePrefix - Java - returns 'minify-' if JavaMinimizeDependencyJars is enabled", async (t) => { + const codeql = createStubCodeQL({}); + const features = createFeatures([Feature.JavaMinimizeDependencyJars]); + + const result = await getFeaturePrefix(codeql, features, KnownLanguage.java); + t.deepEqual(result, "minify-"); +}); + +test("getFeaturePrefix - non-Java - returns '' if JavaMinimizeDependencyJars is enabled", async (t) => { + const codeql = createStubCodeQL({}); + const features = createFeatures([Feature.JavaMinimizeDependencyJars]); + + for (const knownLanguage of Object.values(KnownLanguage)) { + // Skip Java since we expect a result for it, which is tested in the previous test. + if (knownLanguage === KnownLanguage.java) { + continue; + } + const result = await getFeaturePrefix(codeql, features, knownLanguage); + t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`); + } +}); + +test("getFeaturePrefix - C# - returns prefix if CsharpNewCacheKey is enabled", async (t) => { + const codeql = createStubCodeQL({}); + const features = createFeatures([Feature.CsharpNewCacheKey]); + + const result = await getFeaturePrefix(codeql, features, KnownLanguage.csharp); + t.notDeepEqual(result, ""); + t.assert(result.endsWith("-")); + // Check the length of the prefix, which should correspond to `cacheKeyHashLength` + 1 for the trailing `-`. + t.is(result.length, cacheKeyHashLength + 1); +}); + +test("getFeaturePrefix - non-C# - returns '' if CsharpNewCacheKey is enabled", async (t) => { + const codeql = createStubCodeQL({}); + const features = createFeatures([Feature.CsharpNewCacheKey]); + + for (const knownLanguage of Object.values(KnownLanguage)) { + // Skip C# since we expect a result for it, which is tested in the previous test. + if (knownLanguage === KnownLanguage.csharp) { + continue; + } + const result = await getFeaturePrefix(codeql, features, knownLanguage); + t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`); + } +}); diff --git a/src/dependency-caching.ts b/src/dependency-caching.ts index 5c7a4a4e88..220f1d5bab 100644 --- a/src/dependency-caching.ts +++ b/src/dependency-caching.ts @@ -6,9 +6,11 @@ import * as glob from "@actions/glob"; import { getTemporaryDirectory } from "./actions-util"; import { listActionsCaches } from "./api-client"; -import { getTotalCacheSize } from "./caching-utils"; +import { createCacheKeyHash, getTotalCacheSize } from "./caching-utils"; +import { CodeQL } from "./codeql"; import { Config } from "./config-utils"; import { EnvVar } from "./environment"; +import { Feature, FeatureEnablement } from "./feature-flags"; import { KnownLanguage, Language } from "./languages"; import { Logger } from "./logging"; import { getErrorMessage, getRequiredEnvParam } from "./util"; @@ -16,15 +18,21 @@ import { getErrorMessage, getRequiredEnvParam } from "./util"; /** * Caching configuration for a particular language. */ -interface CacheConfig { - /** The paths of directories on the runner that should be included in the cache. */ - paths: string[]; +export interface CacheConfig { + /** Gets the paths of directories on the runner that should be included in the cache. */ + getDependencyPaths: () => string[]; /** - * Patterns for the paths of files whose contents affect which dependencies are used - * by a project. We find all files which match these patterns, calculate a hash for - * their contents, and use that hash as part of the cache key. + * Gets an array of glob patterns for the paths of files whose contents affect which dependencies are used + * by a project. This function also checks whether there are any matching files and returns + * `undefined` if no files match. + * + * The glob patterns are intended to be used for cache keys, where we find all files which match these + * patterns, calculate a hash for their contents, and use that hash as part of the cache key. */ - hash: string[]; + getHashPatterns: ( + codeql: CodeQL, + features: FeatureEnablement, + ) => Promise; } const CODEQL_DEPENDENCY_CACHE_PREFIX = "codeql-dependencies"; @@ -39,21 +47,105 @@ export function getJavaTempDependencyDir(): string { return join(getTemporaryDirectory(), "codeql_java", "repository"); } +/** + * Returns an array of paths of directories on the runner that should be included in a dependency cache + * for a Java analysis. It is important that this is a function, because we call `getTemporaryDirectory` + * which would otherwise fail in tests if we haven't had a chance to initialise `RUNNER_TEMP`. + * + * @returns The paths of directories on the runner that should be included in a dependency cache + * for a Java analysis. + */ +export function getJavaDependencyDirs(): string[] { + return [ + // Maven + join(os.homedir(), ".m2", "repository"), + // Gradle + join(os.homedir(), ".gradle", "caches"), + // CodeQL Java build-mode: none + getJavaTempDependencyDir(), + ]; +} + +/** + * Checks that there are files which match `patterns`. If there are matching files for any of the patterns, + * this function returns all `patterns`. Otherwise, `undefined` is returned. + * + * @param patterns The glob patterns to find matching files for. + * @returns The array of glob patterns if there are matching files, or `undefined` otherwise. + */ +export async function makePatternCheck( + patterns: string[], +): Promise { + const globber = await makeGlobber(patterns); + + if ((await globber.glob()).length === 0) { + return undefined; + } + + return patterns; +} + +/** These files contain accurate information about dependencies, including the exact versions + * that the relevant package manager has determined for the project. Using these gives us + * stable hashes unless the dependencies change. + */ +export const CSHARP_BASE_PATTERNS = [ + // NuGet + "**/packages.lock.json", + // Paket + "**/paket.lock", +]; + +/** These are less accurate for use in cache key calculations, because they: + * + * - Don't contain the exact versions used. They may only contain version ranges or none at all. + * - They contain information unrelated to dependencies, which we don't care about. + * + * As a result, the hash we compute from these files may change, even if + * the dependencies haven't changed. + */ +export const CSHARP_EXTRA_PATTERNS = [ + "**/*.csproj", + "**/packages.config", + "**/nuget.config", +]; + +/** + * Returns the list of glob patterns that should be used to calculate the cache key hash + * for a C# dependency cache. This will try to use `CSHARP_BASE_PATTERNS` whenever possible. + * As a fallback, it will also use `CSHARP_EXTRA_PATTERNS` if the corresponding FF is enabled. + * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. + * @returns A list of glob patterns to use for hashing. + */ +export async function getCsharpHashPatterns( + codeql: CodeQL, + features: FeatureEnablement, +): Promise { + const basePatterns = await internal.makePatternCheck(CSHARP_BASE_PATTERNS); + + if (basePatterns !== undefined) { + return basePatterns; + } + + if (await features.getValue(Feature.CsharpNewCacheKey, codeql)) { + return internal.makePatternCheck(CSHARP_EXTRA_PATTERNS); + } + + // If we get to this point, we didn't find any files with `CSHARP_BASE_PATTERNS`, + // and `Feature.CsharpNewCacheKey` is not enabled. + return undefined; +} + /** * Default caching configurations per language. */ -function getDefaultCacheConfig(): { [language: string]: CacheConfig } { - return { - java: { - paths: [ - // Maven - join(os.homedir(), ".m2", "repository"), - // Gradle - join(os.homedir(), ".gradle", "caches"), - // CodeQL Java build-mode: none - getJavaTempDependencyDir(), - ], - hash: [ +const defaultCacheConfigs: { [language: string]: CacheConfig } = { + java: { + getDependencyPaths: getJavaDependencyDirs, + getHashPatterns: async () => + internal.makePatternCheck([ // Maven "**/pom.xml", // Gradle @@ -63,23 +155,17 @@ function getDefaultCacheConfig(): { [language: string]: CacheConfig } { "buildSrc/**/Dependencies.kt", "gradle/*.versions.toml", "**/versions.properties", - ], - }, - csharp: { - paths: [join(os.homedir(), ".nuget", "packages")], - hash: [ - // NuGet - "**/packages.lock.json", - // Paket - "**/paket.lock", - ], - }, - go: { - paths: [join(os.homedir(), "go", "pkg", "mod")], - hash: ["**/go.sum"], - }, - }; -} + ]), + }, + csharp: { + getDependencyPaths: () => [join(os.homedir(), ".nuget", "packages")], + getHashPatterns: getCsharpHashPatterns, + }, + go: { + getDependencyPaths: () => [join(os.homedir(), "go", "pkg", "mod")], + getHashPatterns: async () => internal.makePatternCheck(["**/go.sum"]), + }, +}; async function makeGlobber(patterns: string[]): Promise { return glob.create(patterns.join("\n")); @@ -107,23 +193,57 @@ export interface DependencyCacheRestoreStatus { /** An array of `DependencyCacheRestoreStatus` objects for each analysed language with a caching configuration. */ export type DependencyCacheRestoreStatusReport = DependencyCacheRestoreStatus[]; +/** + * A wrapper around `cacheConfig.getHashPatterns` which logs when there are no files to calculate + * a hash for the cache key from. + * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. + * @param language The language the `CacheConfig` is for. For use in the log message. + * @param cacheConfig The caching configuration to call `getHashPatterns` on. + * @param checkType Whether we are checking the patterns for a download or upload. + * @param logger The logger to write the log message to if there is an error. + * @returns An array of glob patterns to use for hashing files, or `undefined` if there are no matching files. + */ +export async function checkHashPatterns( + codeql: CodeQL, + features: FeatureEnablement, + language: Language, + cacheConfig: CacheConfig, + checkType: "download" | "upload", + logger: Logger, +): Promise { + const patterns = await cacheConfig.getHashPatterns(codeql, features); + + if (patterns === undefined) { + logger.info( + `Skipping ${checkType} of dependency cache for ${language} as we cannot calculate a hash for the cache key.`, + ); + } + + return patterns; +} + /** * Attempts to restore dependency caches for the languages being analyzed. * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. * @param languages The languages being analyzed. * @param logger A logger to record some informational messages to. - * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. + * * @returns An array of `DependencyCacheRestoreStatus` objects for each analysed language with a caching configuration. */ export async function downloadDependencyCaches( + codeql: CodeQL, + features: FeatureEnablement, languages: Language[], logger: Logger, - minimizeJavaJars: boolean, ): Promise { const status: DependencyCacheRestoreStatusReport = []; for (const language of languages) { - const cacheConfig = getDefaultCacheConfig()[language]; + const cacheConfig = defaultCacheConfigs[language]; if (cacheConfig === undefined) { logger.info( @@ -134,19 +254,22 @@ export async function downloadDependencyCaches( // Check that we can find files to calculate the hash for the cache key from, so we don't end up // with an empty string. - const globber = await makeGlobber(cacheConfig.hash); - - if ((await globber.glob()).length === 0) { + const patterns = await checkHashPatterns( + codeql, + features, + language, + cacheConfig, + "download", + logger, + ); + if (patterns === undefined) { status.push({ language, hit_kind: CacheHitKind.NoHash }); - logger.info( - `Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.`, - ); continue; } - const primaryKey = await cacheKey(language, cacheConfig, minimizeJavaJars); + const primaryKey = await cacheKey(codeql, features, language, patterns); const restoreKeys: string[] = [ - await cachePrefix(language, minimizeJavaJars), + await cachePrefix(codeql, features, language), ]; logger.info( @@ -157,7 +280,7 @@ export async function downloadDependencyCaches( const start = performance.now(); const hitKey = await actionsCache.restoreCache( - cacheConfig.paths, + cacheConfig.getDependencyPaths(), primaryKey, restoreKeys, ); @@ -203,20 +326,22 @@ export type DependencyCacheUploadStatusReport = DependencyCacheUploadStatus[]; /** * Attempts to store caches for the languages that were analyzed. * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. * @param config The configuration for this workflow. * @param logger A logger to record some informational messages to. - * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. * * @returns An array of `DependencyCacheUploadStatus` objects for each analysed language with a caching configuration. */ export async function uploadDependencyCaches( + codeql: CodeQL, + features: FeatureEnablement, config: Config, logger: Logger, - minimizeJavaJars: boolean, ): Promise { const status: DependencyCacheUploadStatusReport = []; for (const language of config.languages) { - const cacheConfig = getDefaultCacheConfig()[language]; + const cacheConfig = defaultCacheConfigs[language]; if (cacheConfig === undefined) { logger.info( @@ -227,13 +352,16 @@ export async function uploadDependencyCaches( // Check that we can find files to calculate the hash for the cache key from, so we don't end up // with an empty string. - const globber = await makeGlobber(cacheConfig.hash); - - if ((await globber.glob()).length === 0) { + const patterns = await checkHashPatterns( + codeql, + features, + language, + cacheConfig, + "upload", + logger, + ); + if (patterns === undefined) { status.push({ language, result: CacheStoreResult.NoHash }); - logger.info( - `Skipping upload of dependency cache for ${language} as we cannot calculate a hash for the cache key.`, - ); continue; } @@ -247,7 +375,11 @@ export async function uploadDependencyCaches( // use the cache quota that we compete with. In that case, we do not wish to use up all of the quota // with the dependency caches. For this, we could use the Cache API to check whether other workflows // are using the quota and how full it is. - const size = await getTotalCacheSize(cacheConfig.paths, logger, true); + const size = await getTotalCacheSize( + cacheConfig.getDependencyPaths(), + logger, + true, + ); // Skip uploading an empty cache. if (size === 0) { @@ -258,7 +390,7 @@ export async function uploadDependencyCaches( continue; } - const key = await cacheKey(language, cacheConfig, minimizeJavaJars); + const key = await cacheKey(codeql, features, language, patterns); logger.info( `Uploading cache of size ${size} for ${language} with key ${key}...`, @@ -266,7 +398,7 @@ export async function uploadDependencyCaches( try { const start = performance.now(); - await actionsCache.saveCache(cacheConfig.paths, key); + await actionsCache.saveCache(cacheConfig.getDependencyPaths(), key); const upload_duration_ms = Math.round(performance.now() - start); status.push({ @@ -299,31 +431,86 @@ export async function uploadDependencyCaches( /** * Computes a cache key for the specified language. * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. * @param language The language being analyzed. - * @param cacheConfig The cache configuration for the language. - * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. + * @param patterns The file patterns to hash. + * * @returns A cache key capturing information about the project(s) being analyzed in the specified language. */ -async function cacheKey( +export async function cacheKey( + codeql: CodeQL, + features: FeatureEnablement, language: Language, - cacheConfig: CacheConfig, - minimizeJavaJars: boolean = false, + patterns: string[], ): Promise { - const hash = await glob.hashFiles(cacheConfig.hash.join("\n")); - return `${await cachePrefix(language, minimizeJavaJars)}${hash}`; + const hash = await glob.hashFiles(patterns.join("\n")); + return `${await cachePrefix(codeql, features, language)}${hash}`; +} + +/** + * If experimental features which the cache contents depend on are enabled for the current language, + * this function returns a prefix that uniquely identifies the set of enabled features. The purpose of + * this is to avoid restoring caches whose contents depended on experimental features, if those + * experimental features are later disabled. + * + * @param codeql The CodeQL instance. + * @param features Information about enabled features. + * @param language The language we are creating the key for. + * + * @returns A cache key prefix identifying the enabled, experimental features that the cache depends on. + */ +export async function getFeaturePrefix( + codeql: CodeQL, + features: FeatureEnablement, + language: Language, +): Promise { + const enabledFeatures: Feature[] = []; + + const addFeatureIfEnabled = async (feature: Feature) => { + if (await features.getValue(feature, codeql)) { + enabledFeatures.push(feature); + } + }; + + if (language === KnownLanguage.java) { + // To ensure a safe rollout of JAR minimization, we change the key when the feature is enabled. + const minimizeJavaJars = await features.getValue( + Feature.JavaMinimizeDependencyJars, + codeql, + ); + + // To maintain backwards compatibility with this, we return "minify-" instead of a hash. + if (minimizeJavaJars) { + return "minify-"; + } + } else if (language === KnownLanguage.csharp) { + await addFeatureIfEnabled(Feature.CsharpNewCacheKey); + } + + // If any features that affect the cache are enabled, return a feature prefix by + // computing a hash of the feature array. + if (enabledFeatures.length > 0) { + return `${createCacheKeyHash(enabledFeatures)}-`; + } + + // No feature prefix. + return ""; } /** * Constructs a prefix for the cache key, comprised of a CodeQL-specific prefix, a version number that * can be changed to invalidate old caches, the runner's operating system, and the specified language name. * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. * @param language The language being analyzed. - * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. * @returns The prefix that identifies what a cache is for. */ async function cachePrefix( + codeql: CodeQL, + features: FeatureEnablement, language: Language, - minimizeJavaJars: boolean, ): Promise { const runnerOs = getRequiredEnvParam("RUNNER_OS"); const customPrefix = process.env[EnvVar.DEPENDENCY_CACHING_PREFIX]; @@ -333,12 +520,18 @@ async function cachePrefix( prefix = `${prefix}-${customPrefix}`; } - // To ensure a safe rollout of JAR minimization, we change the key when the feature is enabled. - if (language === KnownLanguage.java && minimizeJavaJars) { - prefix = `minify-${prefix}`; + // Calculate the feature prefix for the cache, if any. This is a hash that identifies + // experimental features that affect the cache contents. + const featurePrefix = await getFeaturePrefix(codeql, features, language); + + // Assemble the cache key. For backwards compatibility with the JAR minification experiment's existing + // feature prefix usage, we add that feature prefix at the start. Other feature prefixes are inserted + // after the general CodeQL dependency cache prefix. + if (featurePrefix === "minify-") { + return `${featurePrefix}${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; + } else { + return `${prefix}-${featurePrefix}${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; } - - return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; } /** Represents information about our overall cache usage for CodeQL dependency caches. */ @@ -371,3 +564,7 @@ export async function getDependencyCacheUsage( return undefined; } + +export const internal = { + makePatternCheck, +}; diff --git a/src/feature-flags.ts b/src/feature-flags.ts index e92341df20..1334969795 100644 --- a/src/feature-flags.ts +++ b/src/feature-flags.ts @@ -47,6 +47,7 @@ export enum Feature { AnalyzeUseNewUpload = "analyze_use_new_upload", CleanupTrapCaches = "cleanup_trap_caches", CppDependencyInstallation = "cpp_dependency_installation_enabled", + CsharpNewCacheKey = "csharp_new_cache_key", DiffInformedQueries = "diff_informed_queries", DisableCsharpBuildless = "disable_csharp_buildless", DisableJavaBuildlessEnabled = "disable_java_buildless_enabled", @@ -132,6 +133,11 @@ export const featureConfig: Record< legacyApi: true, minimumVersion: "2.15.0", }, + [Feature.CsharpNewCacheKey]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: undefined, + }, [Feature.DiffInformedQueries]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/src/init-action.ts b/src/init-action.ts index 97cb6b784e..3512520c2c 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -578,15 +578,12 @@ async function run() { } // Restore dependency cache(s), if they exist. - const minimizeJavaJars = await features.getValue( - Feature.JavaMinimizeDependencyJars, - codeql, - ); if (shouldRestoreCache(config.dependencyCachingEnabled)) { dependencyCachingResults = await downloadDependencyCaches( + codeql, + features, config.languages, logger, - minimizeJavaJars, ); } @@ -648,7 +645,7 @@ async function run() { `${EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS} is already set to '${process.env[EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS]}', so the Action will not override it.`, ); } else if ( - minimizeJavaJars && + (await features.getValue(Feature.JavaMinimizeDependencyJars, codeql)) && config.dependencyCachingEnabled && config.buildMode === BuildMode.None && config.languages.includes(KnownLanguage.java) diff --git a/src/overlay-database-utils.ts b/src/overlay-database-utils.ts index 50a093a895..ebb020ba86 100644 --- a/src/overlay-database-utils.ts +++ b/src/overlay-database-utils.ts @@ -1,4 +1,3 @@ -import * as crypto from "crypto"; import * as fs from "fs"; import * as path from "path"; @@ -11,6 +10,7 @@ import { getWorkflowRunID, } from "./actions-util"; import { getAutomationID } from "./api-client"; +import { createCacheKeyHash } from "./caching-utils"; import { type CodeQL } from "./codeql"; import { type Config } from "./config-utils"; import { getCommitOid, getFileOidsUnderPath } from "./git-utils"; @@ -514,27 +514,3 @@ export async function getCacheRestoreKeyPrefix( // easier to debug and understand the cache key structure. return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`; } - -/** - * Creates a SHA-256 hash of the cache key components to ensure uniqueness - * while keeping the cache key length manageable. - * - * @param components Object containing all components that should influence cache key uniqueness - * @returns A short SHA-256 hash (first 16 characters) of the components - */ -function createCacheKeyHash(components: Record): string { - // From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify - // - // "Properties are visited using the same algorithm as Object.keys(), which - // has a well-defined order and is stable across implementations. For example, - // JSON.stringify on the same object will always produce the same string, and - // JSON.parse(JSON.stringify(obj)) would produce an object with the same key - // ordering as the original (assuming the object is completely - // JSON-serializable)." - const componentsJson = JSON.stringify(components); - return crypto - .createHash("sha256") - .update(componentsJson) - .digest("hex") - .substring(0, 16); -}