Skip to content

Commit e901ca3

Browse files
committed
Update parsing logic
1 parent 6de9c73 commit e901ca3

File tree

2 files changed

+230
-68
lines changed

2 files changed

+230
-68
lines changed

condaParser.test.ts

Lines changed: 129 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,135 @@ test('Gets files', async () => {
55
expect(files.length).toEqual(1);
66
});
77

8-
test('Parses manifests', async() => {
8+
function roundTripJSON(obj: any): object {
9+
return JSON.parse(JSON.stringify(obj))
10+
}
11+
12+
test('Parses manifests', async () => {
913
var files = conda.searchFiles("test", "environment.yaml");
1014
var manifests = conda.getManifestsFromEnvironmentFiles(files);
1115
expect(manifests.length).toEqual(1);
12-
})
16+
expect(roundTripJSON(manifests[0])).toEqual(
17+
{
18+
"resolved": {
19+
"pkg:conda/python@3.8": {
20+
"package_url": "pkg:conda/python@3.8",
21+
"relationship": "direct",
22+
"dependencies": []
23+
},
24+
"pkg:conda/pytorch@1.10": {
25+
"package_url": "pkg:conda/pytorch@1.10",
26+
"relationship": "direct",
27+
"dependencies": []
28+
},
29+
"pkg:conda/torchvision": {
30+
"package_url": "pkg:conda/torchvision",
31+
"relationship": "direct",
32+
"dependencies": []
33+
},
34+
"pkg:conda/cudatoolkit@11.0": {
35+
"package_url": "pkg:conda/cudatoolkit@11.0",
36+
"relationship": "direct",
37+
"dependencies": []
38+
},
39+
"pkg:conda/pip": {
40+
"package_url": "pkg:conda/pip",
41+
"relationship": "direct",
42+
"dependencies": []
43+
},
44+
"pkg:pypi/pytorch-lightning@1.5.2": {
45+
"package_url": "pkg:pypi/pytorch-lightning@1.5.2",
46+
"relationship": "direct",
47+
"dependencies": []
48+
}, "pkg:pypi/einops@0.3.2": {
49+
"package_url": "pkg:pypi/einops@0.3.2",
50+
"relationship": "direct",
51+
"dependencies": []
52+
},
53+
"pkg:pypi/kornia@0.6.1": {
54+
"package_url": "pkg:pypi/kornia@0.6.1",
55+
"relationship": "direct",
56+
"dependencies": []
57+
},
58+
"pkg:pypi/opencv-python@4.5.4.58": {
59+
"package_url": "pkg:pypi/opencv-python@4.5.4.58",
60+
"relationship": "direct",
61+
"dependencies": []
62+
},
63+
"pkg:pypi/matplotlib@3.5.0": {
64+
"package_url": "pkg:pypi/matplotlib@3.5.0",
65+
"relationship": "direct",
66+
"dependencies": []
67+
},
68+
"pkg:pypi/imageio@2.10.4": {
69+
"package_url": "pkg:pypi/imageio@2.10.4",
70+
"relationship": "direct",
71+
"dependencies": []
72+
},
73+
"pkg:pypi/imageio-ffmpeg@0.4.5": {
74+
"package_url": "pkg:pypi/imageio-ffmpeg@0.4.5",
75+
"relationship": "direct",
76+
"dependencies": []
77+
},
78+
"pkg:pypi/torch-optimizer@0.3.0": {
79+
"package_url": "pkg:pypi/torch-optimizer@0.3.0",
80+
"relationship": "direct",
81+
"dependencies": []
82+
},
83+
"pkg:pypi/setuptools@58.2.0": {
84+
"package_url": "pkg:pypi/setuptools@58.2.0",
85+
"relationship": "direct",
86+
"dependencies": []
87+
},
88+
"pkg:pypi/pymcubes@0.1.2": {
89+
"package_url": "pkg:pypi/pymcubes@0.1.2",
90+
"relationship": "direct",
91+
"dependencies": []
92+
},
93+
"pkg:pypi/pycollada@0.7.1": {
94+
"package_url": "pkg:pypi/pycollada@0.7.1",
95+
"relationship": "direct",
96+
"dependencies": []
97+
},
98+
"pkg:pypi/trimesh@3.9.1": {
99+
"package_url": "pkg:pypi/trimesh@3.9.1",
100+
"relationship": "direct",
101+
"dependencies": []
102+
},
103+
"pkg:pypi/pyglet@1.5.10": {
104+
"package_url": "pkg:pypi/pyglet@1.5.10"
105+
, "relationship": "direct",
106+
"dependencies": []
107+
},
108+
"pkg:pypi/networkx@2.5": {
109+
"package_url": "pkg:pypi/networkx@2.5",
110+
"relationship": "direct",
111+
"dependencies": []
112+
},
113+
"pkg:pypi/plyfile@0.7.2": {
114+
"package_url": "pkg:pypi/plyfile@0.7.2",
115+
"relationship": "direct",
116+
"dependencies": []
117+
},
118+
"pkg:pypi/open3d@0.13.0": {
119+
"package_url": "pkg:pypi/open3d@0.13.0",
120+
"relationship": "direct",
121+
"dependencies": []
122+
},
123+
"pkg:pypi/configargparse@1.5.3": {
124+
"package_url": "pkg:pypi/configargparse@1.5.3",
125+
"relationship": "direct",
126+
"dependencies": []
127+
},
128+
"pkg:pypi/ninja": {
129+
"package_url": "pkg:pypi/ninja",
130+
"relationship": "direct",
131+
"dependencies": []
132+
}
133+
},
134+
"name": "test",
135+
"file": {
136+
"source_location": "test/environment.yaml"
137+
}
138+
})
139+
});

condaParser.ts

Lines changed: 101 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -14,73 +14,108 @@ import {
1414
} from '@github/dependency-submission-toolkit'
1515
import { YAMLMap } from 'yaml';
1616

17-
/**getManifestFromEnvironmentFile(document, fileName) {
18-
core.debug(`getManifestFromEnvironmentFile processing ${fileName}`);
19-
20-
let manifest = new Manifest("Environment", fileName);
21-
22-
23-
/**
24-
let manifest = new Manifest(document.name, fileName);
25-
26-
core.debug(`Processing ${document.packages?.length} packages`);
27-
28-
document.packages?.forEach(pkg => {
29-
let packageName = pkg.name;
30-
let packageVersion = pkg.packageVersion;
31-
let referenceLocator = pkg.externalRefs?.find(ref => ref.referenceCategory === "PACKAGE-MANAGER" && ref.referenceType === "purl")?.referenceLocator;
32-
let genericPurl = `pkg:generic/${packageName}@${packageVersion}`;
33-
// SPDX 2.3 defines a purl field
34-
let purl;
35-
if (pkg.purl != undefined) {
36-
purl = pkg.purl;
37-
} else if (referenceLocator != undefined) {
38-
purl = referenceLocator;
39-
} else {
40-
purl = genericPurl;
41-
}
42-
43-
// Working around weird encoding issues from an SBOM generator
44-
// Find the last instance of %40 and replace it with @
45-
purl = replaceVersionEscape(purl);
46-
47-
let relationships = document.relationships?.find(rel => rel.relatedSpdxElement == pkg.SPDXID && rel.relationshipType == "DEPENDS_ON" && rel.spdxElementId != "SPDXRef-RootPackage");
48-
if (relationships != null && relationships.length > 0) {
49-
manifest.addIndirectDependency(new Package(purl));
50-
} else {
17+
/**getManifestFromEnvironmentFile(document, fileName) {
18+
core.debug(`getManifestFromEnvironmentFile processing ${fileName}`);
19+
20+
let manifest = new Manifest("Environment", fileName);
21+
22+
23+
/**
24+
let manifest = new Manifest(document.name, fileName);
25+
26+
core.debug(`Processing ${document.packages?.length} packages`);
27+
28+
document.packages?.forEach(pkg => {
29+
let packageName = pkg.name;
30+
let packageVersion = pkg.packageVersion;
31+
let referenceLocator = pkg.externalRefs?.find(ref => ref.referenceCategory === "PACKAGE-MANAGER" && ref.referenceType === "purl")?.referenceLocator;
32+
let genericPurl = `pkg:generic/${packageName}@${packageVersion}`;
33+
// SPDX 2.3 defines a purl field
34+
let purl;
35+
if (pkg.purl != undefined) {
36+
purl = pkg.purl;
37+
} else if (referenceLocator != undefined) {
38+
purl = referenceLocator;
39+
} else {
40+
purl = genericPurl;
41+
}
42+
43+
// Working around weird encoding issues from an SBOM generator
44+
// Find the last instance of %40 and replace it with @
45+
purl = replaceVersionEscape(purl);
46+
47+
let relationships = document.relationships?.find(rel => rel.relatedSpdxElement == pkg.SPDXID && rel.relationshipType == "DEPENDS_ON" && rel.spdxElementId != "SPDXRef-RootPackage");
48+
if (relationships != null && relationships.length > 0) {
49+
manifest.addIndirectDependency(new Package(purl));
50+
} else {
51+
manifest.addDirectDependency(new Package(purl));
52+
}
53+
});
54+
return manifest;
55+
}*/
56+
57+
/***/
58+
59+
export default class CondaParser {
60+
61+
static searchFiles(filePath = "", filePattern = "") {
62+
if (filePath == "") {
63+
let filePath = core.getInput('filePath');
64+
}
65+
if (filePattern == "") {
66+
let filePattern = core.getInput('filePattern');
67+
}
68+
69+
return glob.sync(`${filePath}/${filePattern}`, {});
70+
}
71+
72+
static getManifestsFromEnvironmentFiles(files: string[]) {
73+
core.debug(`Processing ${files.length} files`);
74+
let manifests: any[] = [];
75+
files?.forEach(filePath => {
76+
core.debug(`Processing ${filePath}`);
77+
const contents = fs.readFileSync(filePath, 'utf8')
78+
manifests.push(CondaParser.getManifestFromYaml(yaml.parse(contents), filePath));
79+
});
80+
return manifests;
81+
}
82+
83+
// Gets a Manifest object from an environment.yaml
84+
static getManifestFromYaml(yaml: any, filePath: string) {
85+
core.debug(`getManifestFromEnvironmentFile processing ${yaml}`);
86+
87+
let manifest = new Manifest(yaml.name, filePath);
88+
yaml.dependencies?.forEach((dependency: any) => {
89+
let purl = "";
90+
// If it's an object with the collection `pip`, then these are PyPI dependencies
91+
if (dependency instanceof Object && dependency.pip != null) {
92+
dependency.pip.forEach((pipDependency:string) => {
93+
purl = this.getPurlFromDependency(pipDependency, "pypi");
94+
manifest.addDirectDependency(new Package(purl));
95+
});
96+
} else {
97+
purl = this.getPurlFromDependency(dependency, "conda");
5198
manifest.addDirectDependency(new Package(purl));
52-
}
99+
}
53100
});
54101
return manifest;
55-
}*/
56-
57-
/***/
58-
59-
export default class CondaParser {
60-
61-
static searchFiles(filePath = "", filePattern = "") {
62-
if (filePath == "") {
63-
let filePath = core.getInput('filePath');
64-
}
65-
if (filePattern == "") {
66-
let filePattern = core.getInput('filePattern');
67-
}
68-
69-
return glob.sync(`${filePath}/${filePattern}`, {});
70-
}
71-
72-
static getManifestsFromEnvironmentFiles(files:string[]) {
73-
core.debug(`Processing ${files.length} files`);
74-
let manifests: any[] = [];
75-
files?.forEach(filePath => {
76-
core.debug(`Processing ${filePath}`);
77-
const contents = fs.readFileSync(filePath, 'utf8')
78-
manifests.push(yaml.parse(contents));
79-
});
80-
return manifests;
81-
}
82-
83-
static getManifestFromYaml(yaml:any) {
84-
85-
}
102+
}
103+
104+
// Gets a purl string from an environment file's list of dependencies
105+
static getPurlFromDependency(dependency: string, ecosystem: string) {
106+
// Versions look like "python=3.8" or if nested in a pip section like "pytorch==1.0"
107+
// Split on the '=' to separate the packageName and version
108+
let delimiter = (ecosystem == "pypi") ? "==" : "=";
109+
let split = dependency.split(delimiter);
110+
let packageName = split[0];
111+
// If there is a version specified, use it, otherwise, leave it off.
112+
let version = (split.length > 1) ? `@${split[1]}` : "";
113+
114+
// If it's a PyPI dependency, then normalize the package name
115+
if (ecosystem == "pypi") {
116+
packageName = packageName.toLowerCase().replace("_", "-");
117+
}
118+
119+
return `pkg:${ecosystem}/${packageName}${version}`;
120+
}
86121
}

0 commit comments

Comments
 (0)