Skip to content

Commit edae83b

Browse files
vishalbolluvishalbollu
authored
Prevent users from installing conflicting packages
1 parent 6a53a8c commit edae83b

File tree

14 files changed

+50
-103
lines changed

14 files changed

+50
-103
lines changed

‎docs/applications/implementations/aggregators.md‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -37,20 +37,6 @@ def aggregate_spark(data, columns, args):
3737
return discretizer.getSplits()
3838
```
3939

40-
## Pre-installed Packages
41-
42-
The following packages have been pre-installed and can be used in your implementations:
43-
44-
```text
45-
pyspark==2.4.0
46-
numpy==1.15.4
47-
pandas==0.23.4
48-
scipy==1.2.0
49-
sympy==1.3
50-
statsmodels==0.9.0
51-
python-dateutil==2.7.5
52-
six==1.11.0
53-
wrapt==1.11.0
54-
```
40+
## Customization
5541

56-
You can install additional PyPI packages and import your own Python packages. See [Python Packages](../advanced/python-packages.md) for more details.
42+
You can import PyPI packages or your own Python packages to help create more complex aggregators. See [Python Packages](../advanced/python-packages.md) for more details.

‎docs/applications/implementations/models.md‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -46,23 +46,6 @@ def create_estimator(run_config, model_config):
4646
)
4747
```
4848

49-
## Pre-installed Packages
49+
## Customization
5050

51-
The following packages have been pre-installed and can be used in your implementations:
52-
53-
```text
54-
tensorflow==1.12.0
55-
numpy==1.15.4
56-
pandas==0.23.4
57-
scipy==1.2.0
58-
sympy==1.3
59-
statsmodels==0.9.0
60-
python-dateutil==2.7.5
61-
six==1.11.0
62-
wrapt==1.11.0
63-
requests==2.21.0
64-
oauthlib==3.0.0
65-
httplib2==0.12.0
66-
```
67-
68-
You can install additional PyPI packages and import your own Python packages. See [Python Packages](../advanced/python-packages.md) for more details.
51+
You can import PyPI packages or your own Python packages to help create more complex models. See [Python Packages](../advanced/python-packages.md) for more details.

‎docs/applications/implementations/transformers.md‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -81,20 +81,6 @@ def reverse_transform_python(transformed_value, args):
8181
return args["mean"] + (transformed_value * args["stddev"])
8282
```
8383

84-
## Pre-installed Packages
85-
86-
The following packages have been pre-installed and can be used in your implementations:
87-
88-
```text
89-
pyspark==2.4.0
90-
numpy==1.15.4
91-
pandas==0.23.4
92-
scipy==1.2.0
93-
sympy==1.3
94-
statsmodels==0.9.0
95-
python-dateutil==2.7.5
96-
six==1.11.0
97-
wrapt==1.11.0
98-
```
84+
## Customization
9985

100-
You can install additional PyPI packages and import your own Python packages. See [Python Packages](../advanced/python-packages.md) for more details.
86+
You can import PyPI packages or your own Python packages to help create more complex transformers. See [Python Packages](../advanced/python-packages.md) for more details.

‎images/python-packager/Dockerfile‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,9 @@ RUN apt-get update -qq && apt-get install -y -q \
2626
COPY pkg/workloads/consts.py /src/
2727
COPY pkg/workloads/lib /src/lib
2828

29-
COPY pkg/workloads/spark_job/requirements.txt /src/spark_job/requirements.txt
3029
COPY pkg/workloads/tf_api/requirements.txt /src/tf_api/requirements.txt
31-
COPY pkg/workloads/tf_train/requirements.txt /src/tf_train/requirements.txt
3230

3331
RUN pip3 install -r /src/lib/requirements.txt && \
34-
pip3 install -r /src/tf_train/requirements.txt && \
35-
pip3 install -r /src/spark_job/requirements.txt && \
3632
pip3 install -r /src/tf_api/requirements.txt && \
3733
rm -rf /root/.cache/pip*
3834

‎images/spark/Dockerfile‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,7 @@ RUN sed -i "/^set -ex$/c\set -e" /opt/entrypoint.sh
3636
ENV PYTHONPATH="/src:${PYTHONPATH}"
3737

3838
COPY pkg/workloads/lib/requirements.txt /src/lib/requirements.txt
39-
COPY pkg/workloads/spark_job/requirements.txt /src/spark_job/requirements.txt
4039
RUN pip3 install -r /src/lib/requirements.txt && \
41-
pip3 install -r /src/spark_job/requirements.txt && \
4240
rm -rf /root/.cache/pip*
4341

4442
COPY pkg/workloads/consts.py /src/

‎images/tf-train-gpu/Dockerfile‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@ FROM cortexlabs/tf-base-gpu
33
ENV PYTHONPATH="/src:${PYTHONPATH}"
44

55
COPY pkg/workloads/lib/requirements.txt /src/lib/requirements.txt
6-
COPY pkg/workloads/tf_train/requirements.txt /src/tf_train/requirements.txt
76
RUN pip3 install -r /src/lib/requirements.txt && \
8-
pip3 install -r /src/tf_train/requirements.txt && \
97
rm -rf /root/.cache/pip*
108

119
COPY pkg/workloads/consts.py /src/

‎images/tf-train/Dockerfile‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@ FROM cortexlabs/tf-base
33
ENV PYTHONPATH="/src:${PYTHONPATH}"
44

55
COPY pkg/workloads/lib/requirements.txt /src/lib/requirements.txt
6-
COPY pkg/workloads/tf_train/requirements.txt /src/tf_train/requirements.txt
76
RUN pip3 install -r /src/lib/requirements.txt && \
8-
pip3 install -r /src/tf_train/requirements.txt && \
97
rm -rf /root/.cache/pip*
108

119
COPY pkg/workloads/consts.py /src/

‎pkg/operator/context/python_packages.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ func loadPythonPackages(files map[string][]byte) (context.PythonPackages, error)
9292
id := hash.Bytes(buf.Bytes())
9393
pythonPackage := context.PythonPackage{
9494
ResourceConfigFields: userconfig.ResourceConfigFields{
95-
Name: consts.RequirementsTxt,
95+
Name: packageName,
9696
},
9797
ComputedResourceFields: &context.ComputedResourceFields{
9898
ResourceFields: &context.ResourceFields{

‎pkg/operator/telemetry/telemetry.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,6 @@ func (client *Client) sendErrorEvent(operatorID string, err error) {
124124

125125
func ReportEvent(name string) {
126126
if cc.EnableTelemetry {
127-
fmt.Println(cc.EnableTelemetry)
128127
go getDefaultClient().sendUsageEvent(aws.HashedAccountID, name)
129128
}
130129
}

‎pkg/workloads/lib/package.py‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,19 @@
1515
import os
1616
import sys
1717
import argparse
18+
import glob
1819
from subprocess import run
1920

2021
from lib import util, aws
2122
from lib.context import Context
2223
from lib.log import get_logger
2324
from lib.exceptions import UserException, CortexException
2425

26+
import requirements
27+
2528
logger = get_logger()
2629

27-
LOCAL_PACKAGE_PATH = "/src/package"
30+
LOCAL_PACKAGE_PATH = "/packages"
2831
WHEELHOUSE_PATH = "/wheelhouse"
2932

3033

@@ -35,6 +38,16 @@ def get_build_order(python_packages):
3538
return build_order + sorted([name for name in python_packages if name != "requirements.txt"])
3639

3740

41+
def get_restricted_packages():
42+
cortex_packages = {"pyspark": "2.4.0", "tensorflow": "1.12.0"}
43+
req_files = glob.glob("/src/**/requirements.txt", recursive=True)
44+
for req_file in req_files:
45+
with open(req_file) as f:
46+
for req in requirements.parse(f):
47+
cortex_packages[req.name] = req.specs[0][1]
48+
return cortex_packages
49+
50+
3851
def build_packages(python_packages, bucket):
3952
cmd_partial = {}
4053
build_order = get_build_order(python_packages)
@@ -50,22 +63,35 @@ def build_packages(python_packages, bucket):
5063

5164
logger.info("Setting up packages")
5265

66+
restricted_packages = get_restricted_packages()
67+
5368
for package_name in build_order:
69+
package_wheel_path = os.path.join(WHEELHOUSE_PATH, package_name)
5470
requirement = cmd_partial[package_name]
55-
logger.info("Building package {}".format(package_name))
71+
logger.info("Building: {}".format(package_name))
5672
completed_process = run(
57-
"pip3 wheel -w {} {}".format(
58-
os.path.join(WHEELHOUSE_PATH, package_name), requirement
59-
).split()
73+
"pip3 wheel -w {} {}".format(package_wheel_path, requirement).split()
6074
)
75+
6176
if completed_process.returncode != 0:
6277
raise UserException("creating wheels", package_name)
6378

79+
for wheelname in os.listdir(package_wheel_path):
80+
name_split = wheelname.split("-")
81+
dist_name, version = name_split[0], name_split[1]
82+
expected_version = restricted_packages.get(dist_name, None)
83+
if expected_version is not None and version != expected_version:
84+
raise UserException(
85+
"when installing {}, found {}=={} but cortex requires {}=={}".format(
86+
package_name, dist_name, version, dist_name, expected_version
87+
)
88+
)
89+
6490
logger.info("Validating packages")
6591

6692
for package_name in build_order:
6793
requirement = cmd_partial[package_name]
68-
logger.info("Installing package {}".format(package_name))
94+
logger.info("Installing: {}".format(package_name))
6995
completed_process = run(
7096
"pip3 install --no-index --find-links={} {}".format(
7197
os.path.join(WHEELHOUSE_PATH, package_name), requirement
@@ -94,6 +120,10 @@ def build(args):
94120
try:
95121
build_packages(python_packages, ctx.bucket)
96122
util.log_job_finished(ctx.workload_id)
123+
except CortexException as e:
124+
e.wrap("error")
125+
logger.exception(e)
126+
ctx.upload_resource_status_failed(*python_packages_list)
97127
except Exception as e:
98128
logger.exception(e)
99129
ctx.upload_resource_status_failed(*python_packages_list)

0 commit comments

Comments
 (0)