99outputs :
1010 cache-dir :
1111 description : " The directory where the cache was stored"
12- value : ${{ steps.fill -compilation-dir.outputs.compdir }}
12+ value : ${{ steps.output -compilation-dir.outputs.compdir }}
1313
1414runs :
1515 using : composite
2727 if : ${{ github.event_name == 'pull_request' }}
2828 uses : actions/cache/restore@v3
2929 with :
30- path : ' **/.cache'
30+ path : |
31+ **/.cache
32+ ~/.codeql/compile-cache
3133key : codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
3234 restore-keys : |
3335 codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
@@ -37,18 +39,111 @@ runs:
3739 if : ${{ github.event_name != 'pull_request' }}
3840 uses : actions/cache@v3
3941 with :
40- path : ' **/.cache'
42+ path : |
43+ **/.cache
44+ ~/.codeql/compile-cache
4145key : codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
4246 restore-keys : | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
4347 codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
4448 codeql-compile-${{ inputs.key }}-main-
45- name : Fill compilation cache directory
46- id : fill -compilation-dir
49+ name : Output-compilationdir
50+ id : output -compilation-dir
4751 shell : bash
4852 run : |
49- # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
50- node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
51-
5253 echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
5354env :
5455 COMBINED_CACHE_DIR : ${{ runner.temp }}/compilation-dir
56+ - name : Fill compilation cache directory
57+ id : fill-compilation-dir
58+ uses : actions/github-script@v6
59+ env :
60+ COMBINED_CACHE_DIR : ${{ runner.temp }}/compilation-dir
61+ with :
62+ script : |
63+ // # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
64+ // mkdir -p ${COMBINED_CACHE_DIR}
65+ // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
66+ // # copy the contents of the .cache folders into the combined cache folder.
67+ // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
68+ // # clean up the .cache folders
69+ // rm -rf **/.cache/*
70+
71+ const fs = require("fs");
72+ const path = require("path");
73+ const os = require("os");
74+
75+ // the first argv is the cache folder to create.
76+ const COMBINED_CACHE_DIR = process.env.COMBINED_CACHE_DIR;
77+
78+ function* walkCaches(dir) {
79+ const files = fs.readdirSync(dir, { withFileTypes: true });
80+ for (const file of files) {
81+ if (file.isDirectory()) {
82+ const filePath = path.join(dir, file.name);
83+ yield* walkCaches(filePath);
84+ if (file.name === ".cache") {
85+ yield filePath;
86+ }
87+ }
88+ }
89+ }
90+
91+ async function copyDir(src, dest) {
92+ for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
93+ const srcPath = path.join(src, file.name);
94+ const destPath = path.join(dest, file.name);
95+ if (file.isDirectory()) {
96+ if (!fs.existsSync(destPath)) {
97+ fs.mkdirSync(destPath);
98+ }
99+ await copyDir(srcPath, destPath);
100+ } else {
101+ await fs.promises.copyFile(srcPath, destPath);
102+ }
103+ }
104+ }
105+
106+ async function main() {
107+ const cacheDirs = [...walkCaches(".")];
108+
109+ for (const dir of cacheDirs) {
110+ console.log(`Found .cache dir at ${dir}`);
111+ }
112+
113+ const globalCacheDir = path.join(os.homedir(), ".codeql", "compile-cache");
114+ if (fs.existsSync(globalCacheDir)) {
115+ console.log("Found global home dir: " + globalCacheDir);
116+ cacheDirs.push(globalCacheDir);
117+ }
118+
119+ if (cacheDirs.length === 0) {
120+ console.log("No cache dirs found");
121+ return;
122+ }
123+
124+ // mkdir -p ${COMBINED_CACHE_DIR}
125+ fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
126+
127+ // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
128+ await Promise.all(
129+ cacheDirs.map((cacheDir) =>
130+ (async function () {
131+ await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
132+ await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
133+ })()
134+ )
135+ );
136+
137+ // # copy the contents of the .cache folders into the combined cache folder.
138+ // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
139+ await Promise.all(
140+ cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
141+ );
142+
143+ // # clean up the .cache folders
144+ // rm -rf **/.cache/*
145+ await Promise.all(
146+ cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
147+ );
148+ }
149+ main();
0 commit comments