From 510fafd34da81c7817a459a7bc124847205ee80e Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 10 Dec 2024 17:34:17 -0500 Subject: [PATCH 1/9] Add libre-tube for kotlin snapshot. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index 37d17350cd..77665940fe 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -12,4 +12,5 @@ jazzer,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7 tinydb,https://github.com/msiemens/tinydb.git,python,"3.9.20,3.10.15,3.11.10,3.12.7","pyproject.toml,poetry.lock",poetry,,,poetry install,,10644a0e07ad180c5b756aba272ee6b0dbd12df8 funcy,https://github.com/Suor/funcy.git,python,"3.9.20,3.10.15",setup.py,pip,,rm test_requirements.txt,python setup.py install,,859056d039adea75c1c3550286437ce0b612fe92 numpy,https://github.com/numpy/numpy.git,python-c,"3.9.20,3.10.15,3.11.10,3.12.7","setup.py,pyproject.toml",pip,,git submodule update --init,,,93fdebfcb4bc4cd53c959ccd0117a612d5f13f1a -requests,https://github.com/psf/requests.git,python,"3.9.20,3.10.15,3.11.10,3.12.7","setup.py",,,,pip install .,,23540c93cac97c763fe59e843a08fa2825aa80fd \ No newline at end of file +requests,https://github.com/psf/requests.git,python,"3.9.20,3.10.15,3.11.10,3.12.7","setup.py",,,,pip install .,,23540c93cac97c763fe59e843a08fa2825aa80fd +libretube,https://github.com/libre-tube/LibreTube.git,java17,,gradle.kts,gradle,,,,GRADLE_ARGS='clean assembleDebug -PjavacRelease=17' ANDROID_HOME=~/.android/android_sdk,bd855037be3127ba908a8e0f7763d6196961b7c4 From 83b0b1a90be4f29091a6c86de7da299bf8b43e01 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 10 Dec 2024 20:34:15 -0500 Subject: [PATCH 2/9] Add itflow for PHP. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index 77665940fe..f9d3e10e4e 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -14,3 +14,4 @@ funcy,https://github.com/Suor/funcy.git,python,"3.9.20,3.10.15",setup.py,pip,,rm numpy,https://github.com/numpy/numpy.git,python-c,"3.9.20,3.10.15,3.11.10,3.12.7","setup.py,pyproject.toml",pip,,git submodule update --init,,,93fdebfcb4bc4cd53c959ccd0117a612d5f13f1a requests,https://github.com/psf/requests.git,python,"3.9.20,3.10.15,3.11.10,3.12.7","setup.py",,,,pip install .,,23540c93cac97c763fe59e843a08fa2825aa80fd libretube,https://github.com/libre-tube/LibreTube.git,java17,,gradle.kts,gradle,,,,GRADLE_ARGS='clean assembleDebug -PjavacRelease=17' ANDROID_HOME=~/.android/android_sdk,bd855037be3127ba908a8e0f7763d6196961b7c4 +itflow,https://github.com/itflow-org/itflow.git,php,,,,,"wget https://raw.githubusercontent.com/composer/getcomposer.org/4a4a3856e3a706f3e4fed7f1ae6f84681c29ef69/web/installer -O - -q | php -- --quiet",,,e24c79fbaf16ee274d205c1b31758d07ace31c47 From 820baedb5ae4aed91b67b580767a03d481e9fe51 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 10 Dec 2024 21:39:40 -0500 Subject: [PATCH 3/9] Add dependabot for ruby. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index f9d3e10e4e..4b6ca77128 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -15,3 +15,4 @@ numpy,https://github.com/numpy/numpy.git,python-c,"3.9.20,3.10.15,3.11.10,3.12.7 requests,https://github.com/psf/requests.git,python,"3.9.20,3.10.15,3.11.10,3.12.7","setup.py",,,,pip install .,,23540c93cac97c763fe59e843a08fa2825aa80fd libretube,https://github.com/libre-tube/LibreTube.git,java17,,gradle.kts,gradle,,,,GRADLE_ARGS='clean assembleDebug -PjavacRelease=17' ANDROID_HOME=~/.android/android_sdk,bd855037be3127ba908a8e0f7763d6196961b7c4 itflow,https://github.com/itflow-org/itflow.git,php,,,,,"wget https://raw.githubusercontent.com/composer/getcomposer.org/4a4a3856e3a706f3e4fed7f1ae6f84681c29ef69/web/installer -O - -q | php -- --quiet",,,e24c79fbaf16ee274d205c1b31758d07ace31c47 +dependabot,https://github.com/dependabot/dependabot-core.git,ruby,,,,,,,,e1024fb85f0d9e3396feaab33ee44941bb6cc68a From e396aa80eef73d25fba6963edd82ddf61c948d16 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Sat, 14 Dec 2024 20:50:02 -0500 Subject: [PATCH 4/9] Replace itflow with symfony. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index 4b6ca77128..21f3abf937 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -14,5 +14,5 @@ funcy,https://github.com/Suor/funcy.git,python,"3.9.20,3.10.15",setup.py,pip,,rm numpy,https://github.com/numpy/numpy.git,python-c,"3.9.20,3.10.15,3.11.10,3.12.7","setup.py,pyproject.toml",pip,,git submodule update --init,,,93fdebfcb4bc4cd53c959ccd0117a612d5f13f1a requests,https://github.com/psf/requests.git,python,"3.9.20,3.10.15,3.11.10,3.12.7","setup.py",,,,pip install .,,23540c93cac97c763fe59e843a08fa2825aa80fd libretube,https://github.com/libre-tube/LibreTube.git,java17,,gradle.kts,gradle,,,,GRADLE_ARGS='clean assembleDebug -PjavacRelease=17' ANDROID_HOME=~/.android/android_sdk,bd855037be3127ba908a8e0f7763d6196961b7c4 -itflow,https://github.com/itflow-org/itflow.git,php,,,,,"wget https://raw.githubusercontent.com/composer/getcomposer.org/4a4a3856e3a706f3e4fed7f1ae6f84681c29ef69/web/installer -O - -q | php -- --quiet",,,e24c79fbaf16ee274d205c1b31758d07ace31c47 +symfony,https://github.com/symfony/symfony.git,php,,,,,,,,95c43e38811a159c82112a13e631ee7121b83e54 dependabot,https://github.com/dependabot/dependabot-core.git,ruby,,,,,,,,e1024fb85f0d9e3396feaab33ee44941bb6cc68a From 75f96b4f3ab4a486fe6b534175c6cf7aaa9bed3e Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Sat, 14 Dec 2024 20:51:03 -0500 Subject: [PATCH 5/9] Bugfix for undefined in mergeDependencies. Signed-off-by: Caroline Russell --- lib/cli/index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/cli/index.js b/lib/cli/index.js index 7f162a0619..405d1d1493 100644 --- a/lib/cli/index.js +++ b/lib/cli/index.js @@ -5618,6 +5618,9 @@ export function mergeDependencies( } if (adep["dependsOn"]) { for (const eachDepends of adep["dependsOn"]) { + if (!eachDepends){ + continue + } if (parentRef) { if (eachDepends.toLowerCase() !== parentRef.toLowerCase()) { deps_map[adep.ref].add(eachDepends); @@ -5630,6 +5633,9 @@ export function mergeDependencies( if (adep["provides"]) { providesFound = true; for (const eachProvides of adep["provides"]) { + if (!eachProvides){ + continue + } if ( parentRef && eachProvides.toLowerCase() !== parentRef.toLowerCase() From 4fd5e6c5b94d2140e128fde369c24fc55c595961 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Sat, 14 Dec 2024 21:31:25 -0500 Subject: [PATCH 6/9] Add typescript + angular snapshots. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index 21f3abf937..b41a02a788 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -16,3 +16,7 @@ requests,https://github.com/psf/requests.git,python,"3.9.20,3.10.15,3.11.10,3.12 libretube,https://github.com/libre-tube/LibreTube.git,java17,,gradle.kts,gradle,,,,GRADLE_ARGS='clean assembleDebug -PjavacRelease=17' ANDROID_HOME=~/.android/android_sdk,bd855037be3127ba908a8e0f7763d6196961b7c4 symfony,https://github.com/symfony/symfony.git,php,,,,,,,,95c43e38811a159c82112a13e631ee7121b83e54 dependabot,https://github.com/dependabot/dependabot-core.git,ruby,,,,,,,,e1024fb85f0d9e3396feaab33ee44941bb6cc68a +ng-select,https://github.com/ng-select/ng-select.git,typescript,,,yarn,,,corepack enable yarn;yarn install,,ba14f813135e6f910e2c39114e53ba291b943742 +ngx-bootstrap,https://github.com/valor-software/ngx-bootstrap.git,typescript,,,npm,,,npm install.,,c31c3caf63011743d4ce1adb185cf319a37fcc79 +ngx-bootstrap,https://github.com/valor-software/ngx-bootstrap.git,typescript,,,npm,,,npm install .,,c31c3caf63011743d4ce1adb185cf319a37fcc79 + From 90e42ff66d155ef53df854f930a74e645f22d52e Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Sat, 14 Dec 2024 22:11:21 -0500 Subject: [PATCH 7/9] Add typescript + react snapshots. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index b41a02a788..d554af4dc4 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -17,6 +17,7 @@ libretube,https://github.com/libre-tube/LibreTube.git,java17,,gradle.kts,gradle, symfony,https://github.com/symfony/symfony.git,php,,,,,,,,95c43e38811a159c82112a13e631ee7121b83e54 dependabot,https://github.com/dependabot/dependabot-core.git,ruby,,,,,,,,e1024fb85f0d9e3396feaab33ee44941bb6cc68a ng-select,https://github.com/ng-select/ng-select.git,typescript,,,yarn,,,corepack enable yarn;yarn install,,ba14f813135e6f910e2c39114e53ba291b943742 -ngx-bootstrap,https://github.com/valor-software/ngx-bootstrap.git,typescript,,,npm,,,npm install.,,c31c3caf63011743d4ce1adb185cf319a37fcc79 ngx-bootstrap,https://github.com/valor-software/ngx-bootstrap.git,typescript,,,npm,,,npm install .,,c31c3caf63011743d4ce1adb185cf319a37fcc79 +plate,https://github.com/udecode/plate.git,typescript,,,yarn,,rm -rf templates,corepack enable yarn;yarn install,,24a683100edb6357fc45f043b29533e579ad1e19 +chartdb,https://github.com/chartdb/chartdb.git,typescript,,,npm,,,npm install .,,2b6b73326155f18d6d56779c0657a3506e2d2cde From 19422c54d594dc0538e85780bb9b1309d1c5866d Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Mon, 16 Dec 2024 14:44:14 -0500 Subject: [PATCH 8/9] Fix prebuild command timing. Signed-off-by: Caroline Russell --- test/diff/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/diff/generate.py b/test/diff/generate.py index 16d935b12a..ac6ac318a0 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -25,7 +25,7 @@ def build_args(): parser.add_argument( '--clone-dir', type=Path, - default=Path(f'{os.getenv("GITHUB_WORKSPACE")}/src_repos'), + default=f'{os.getenv("GITHUB_WORKSPACE")}/src_repos', help='Path to src_repos', dest='clone_dir' ) @@ -460,7 +460,7 @@ def run_pre_builds(repo_data, output_dir, debug_cmds, sdkman_sh): [ cmds.extend(row['pre_build_cmd'].split(';')) for row in repo_data - if row['pre_build_cmd'] + if row['pre_build_cmd'] and row['pre_build_cmd'].startswith('sdk use ') ] cmds = [cmd.lstrip().rstrip() for cmd in cmds] cmds = set(cmds) From 43b658ff1e168a08844fea17351d5887614753d2 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Mon, 16 Dec 2024 14:44:53 -0500 Subject: [PATCH 9/9] Move cdxgen checkout to address yarn vs pnpm issue. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 18 +++++++++++------- test/diff/generate.py | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index faf18f7b40..f5d85e5ad1 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -22,9 +22,8 @@ jobs: steps: - uses: actions/checkout@v4 - - - name: Set up pnpm - uses: pnpm/action-setup@v4 + with: + path: cdxgen_src - name: cdxgen, custom-json-diff installs shell: bash @@ -32,16 +31,21 @@ jobs: SHELL: bash run: | rm -rf original_snapshots new_snapshots src_repos + cd cdxgen_src + corepack enable pnpm cdxgen_tarball=$(pnpm pack | tail -1) npm install -g "$cdxgen_tarball" - git clone https://github.com/appthreat/cdxgen-samples.git original_snapshots + cd .. python3.12 -m venv .venv - source .venv/bin/activate && pip install -r test/diff/requirements.txt + source .venv/bin/activate && pip install -r cdxgen_src/test/diff/requirements.txt + git clone https://github.com/appthreat/cdxgen-samples.git original_snapshots + cd original_snapshots + git checkout feature/expand_snapshots_3 - name: Generate scripts run: | source .venv/bin/activate - python test/diff/generate.py + python cdxgen_src/test/diff/generate.py - name: Upload shell scripts generated as artifact uses: actions/upload-artifact@v4 @@ -65,7 +69,7 @@ jobs: - name: Test BOMs run: | source .venv/bin/activate - python test/diff/diff_tests.py --migrate-legacy + python cdxgen_src/test/diff/diff_tests.py --migrate-legacy if test -f new_snapshots/diffs.json; then echo "status=FAILED" >> "$GITHUB_ENV" fi diff --git a/test/diff/generate.py b/test/diff/generate.py index ac6ac318a0..9eeb499263 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -18,7 +18,7 @@ def build_args(): parser.add_argument( '--repo-csv', type=Path, - default='test/diff/repos.csv', + default='cdxgen_src/test/diff/repos.csv', help='Path to sources.csv', dest='repo_csv' )