Skip to content

Commit c29bd5d

Browse files
author
Ziqun Ye
committed
use arm base image when local machine is detected as m# machine
1 parent 1f18adc commit c29bd5d

File tree

18 files changed

+560
-112
lines changed

18 files changed

+560
-112
lines changed

‎ads/opctl/conda/cmds.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -602,7 +602,8 @@ def _publish(
602602
)
603603
except Exception:
604604
raise RuntimeError(f"Could not pack environment {conda_slug}.")
605-
605+
if "/" in conda_slug:
606+
raise ValueError("Invalid conda_slug. found `/` in slug name. Please use a different slug name.")
606607
pack_file = os.path.join(pack_folder_path, f"{conda_slug}.tar.gz")
607608
if not os.path.exists(pack_file):
608609
raise RuntimeError(f"Pack {pack_file} was not created.")

‎ads/opctl/conda/pack.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,12 @@ def main(pack_folder_path):
3131
stderr=subprocess.PIPE,
3232
)
3333
stdout, stderr = process.communicate()
34-
# import pdb; pdb.set_trace()
35-
# if stderr:
36-
# print(stderr)
37-
# raise Exception(
38-
# f"Error export environment information from {pack_folder_path}"
39-
# )
34+
35+
if process.returncode and stderr:
36+
print(stderr)
37+
raise Exception(
38+
f"Error export environment information from {pack_folder_path}"
39+
)
4040
try:
4141
new_env_info = yaml.safe_load(StringIO(stdout.decode("utf-8")))
4242
except Exception as e:

‎ads/opctl/docker/Dockerfile.job‎

Lines changed: 46 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
# Used OL8 because miniconda required a higher version of glibc that was unavoidable
2-
FROM ocr-docker-remote.artifactory.oci.oraclecorp.com/os/oraclelinux:8-slim-fips
1+
# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
3+
4+
FROM ghcr.io/oracle/oraclelinux:7-slim
35

46
# Configure environment
57
ENV DATASCIENCE_USER datascience
@@ -9,139 +11,76 @@ ENV DATASCIENCE_INSTALL_DIR /etc/datascience
911
ENV LOGS_DIRECTORY /logs
1012

1113
ARG release=19
12-
ARG update=10
13-
ARG target=cpu
14-
15-
# Move this arg around the file as needed when you want to use cache for local builds/testing to speed up the process
16-
# This will allow you to set success points where you know previous code in the dockerfile is working
17-
# ARG RAND
18-
19-
COPY common /etc/datascience
20-
COPY etc/yum.repos.d /etc/yum.repos.d
21-
22-
# source: https://confluence.oraclecorp.com/confluence/pages/viewpage.action?spaceKey=ESEDoc&title=Artifactory+FAQ
23-
ADD Symantec_Private_SSL_SHA256_CA.cer /etc/pki/ca-trust/source/anchors
24-
ADD Symantec_Private_SSL_SHA256_Root.cer /etc/pki/ca-trust/source/anchors
25-
26-
RUN update-ca-trust
27-
28-
RUN microdnf install yum yum-utils && yum clean all && rm -rf /var/cache/yum
14+
ARG update=13
2915

3016
RUN \
31-
yum install -y \
32-
oracle-release-el8 \
33-
oraclelinux-developer-release-el8 \
34-
oracle-epel-release-el8 \
35-
https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
36-
yum-config-manager --enable ol8_addons --enable ol8_oracle_instantclient > /dev/null && yum -y update && \
37-
yum clean all && \
38-
rm -rf /var/cache/yum/*
39-
40-
RUN \
41-
yum install -y \
42-
--setopt=skip_missing_names_on_install=False \
17+
yum -y -q install oracle-release-el7 && \
18+
yum-config-manager --add-repo http://yum.oracle.com/repo/OracleLinux/OL7/developer_EPEL/x86_64 && \
19+
yum-config-manager --enable ol7_optional_latest --enable ol7_addons --enable ol7_software_collections --enable ol7_oracle_instantclient > /dev/null && \
20+
yum groupinstall -y -q 'Development Tools' && \
21+
yum update -y && \
22+
yum install -y --setopt=skip_missing_names_on_install=False \
4323
bzip2 \
24+
curl \
4425
git \
45-
httpd \
26+
gcc-gfortran \
4627
libcurl-devel \
4728
libxml2-devel \
48-
java-11-openjdk-headless \
4929
oracle-instantclient${release}.${update}-basic \
5030
oracle-instantclient${release}.${update}-sqlplus \
5131
openssl \
5232
openssl-devel \
5333
patch \
54-
snappy \
55-
gcc-c++ \
5634
sudo \
5735
unzip \
58-
vi \
5936
zip \
37+
gcc-c++ \
6038
wget \
61-
jq \
6239
gcc \
63-
htop \
64-
git-lfs \
65-
pcsc-lite-libs \
66-
perl-Env \
67-
perl-CPAN \
68-
perl-devel \
69-
curl-devel \
70-
expat-devel \
71-
asciidoc \
72-
xmlto \
73-
supervisor && \
74-
yum clean all && \
75-
rm -rf /var/cache/yum/*
76-
77-
RUN \
78-
yum groupinstall -y -q "development tools" \
7940
&& yum clean all \
8041
&& rm -rf /var/cache/yum/*
8142

8243
# setup user
8344
RUN \
84-
mkdir -p /home/$DATASCIENCE_USER && \
85-
useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \
86-
chown $DATASCIENCE_USER /home/$DATASCIENCE_USER && \
87-
chown -R $DATASCIENCE_USER:users /usr/local/ && \
88-
touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \
89-
mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR
90-
91-
# COPY build-tool-entrypoint.sh /etc/datascience/build-tool-entrypoint.sh
92-
# COPY install-capture-entrypoint.sh /etc/datascience/install-capture-entrypoint.sh
93-
45+
mkdir -p /home/$DATASCIENCE_USER && \
46+
useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \
47+
chown -R $DATASCIENCE_USER /home/$DATASCIENCE_USER && \
48+
chown -R $DATASCIENCE_USER:users /usr/local/ && \
49+
touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \
50+
mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR && \
51+
mkdir -p $LOGS_DIRECTORY && chown -R $DATASCIENCE_USER:users $LOGS_DIRECTORY && \
52+
mkdir -p $LOGS_DIRECTORY/harness && chown -R $DATASCIENCE_USER:users $LOGS_DIRECTORY/harness
53+
54+
RUN mkdir -p /etc/datascience/build
55+
RUN mkdir -p $DATASCIENCE_INSTALL_DIR/{pre-build-ds,post-build-ds,pre-run-ds,pre-run-user}
56+
57+
#conda
9458
# set a default language for localization. necessary for oci cli
9559
ARG LANG=en_US.utf8
9660
ENV LANG=$LANG
9761
ENV SHELL=/bin/bash
9862

99-
############### SETUP ROOT (BASE) ENVIRONMENT WITH CONDA ######################
100-
ARG PYTHON_MAJOR='3'
101-
ARG PYTHON_MINOR='8'
102-
ARG PYTHON_PATCH=''
103-
ARG MINICONDA_VER=py${PYTHON_MAJOR}${PYTHON_MINOR}_23.5.2-0
104-
ARG MAMBAFORGE_VER=23.3.1-1
105-
10663
# set /opt folder permissions for $DATASCIENCE_USER. Conda is going to live in this folder.
107-
RUN chown $DATASCIENCE_USER /opt
64+
RUN chown -R $DATASCIENCE_USER /opt
10865

10966
USER $DATASCIENCE_USER
11067
WORKDIR /home/datascience
68+
RUN wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /home/datascience/miniconda.sh \
69+
&& /bin/bash /home/datascience/miniconda.sh -f -b -p /opt/conda \
70+
&& rm /home/datascience/miniconda.sh \
71+
&& /opt/conda/bin/conda clean -yaf
11172

112-
# Note in order to run sudo commands as a non root user, you must specify --credential yes if using qemu static to build the image
113-
RUN wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Miniconda3-${MINICONDA_VER}-$(uname)-$(uname -m).sh \
114-
-O /home/datascience/Miniconda3.sh \
115-
&& /bin/bash /home/datascience/Miniconda3.sh -f -b -p /opt/conda \
116-
&& rm /home/datascience/Miniconda3.sh \
117-
&& wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Mambaforge-${MAMBAFORGE_VER}-$(uname)-$(uname -m).sh \
118-
-O /home/datascience/Mambaforge3.sh \
119-
&& /bin/bash /home/datascience/Mambaforge3.sh -f -b -p /opt/conda \
120-
&& rm /home/datascience/Mambaforge3.sh \
121-
&& ls /opt/**/*
122-
123-
# Adding conda to the path
124-
ENV PATH="/opt/conda/bin:${PATH}"
125-
126-
# Set activate conda by default
73+
WORKDIR /
12774
USER root
12875
RUN printf "#!/bin/bash\nsource /opt/conda/bin/activate\n" > /etc/profile.d/enableconda.sh \
12976
&& chmod +x /etc/profile.d/enableconda.sh
13077

13178
USER $DATASCIENCE_USER
79+
ENV PATH="/opt/conda/bin:${PATH}"
13280
WORKDIR /home/datascience
13381

134-
COPY base-env-arm.yaml /opt/base-env.yaml
135-
COPY jupyter-arm.yaml /opt/jupyter.yaml
136-
137-
# Needs to be set to prevent a hang with mamba installations if this dockerfile
138-
# is being built by running qemu-user-static to emulate an ARM machine
139-
# <https://github.com/mamba-org/mamba/issues/1611>
140-
ENV G_SLICE=always-malloc
141-
142-
# Replaced conda with mamba. Conda wasn't able to resolve conflicts
143-
RUN mamba env update -n root -f /opt/jupyter.yaml && mamba clean -yaf && rm -rf /home/datascience/.cache/pip
144-
82+
COPY docker/base-env.yaml /opt/base-env.yaml
83+
RUN conda env update -q -n root -f /opt/base-env.yaml && conda clean -yaf && rm -rf /home/datascience/.cache/pip
14584

14685
USER $DATASCIENCE_USER
14786

@@ -157,3 +96,13 @@ ARG PIP_INDEX_URL
15796

15897
############# Setup Conda environment tools ###########################
15998
ARG RAND=1
99+
100+
ARG RUN_WORKING_DIR="/home/datascience"
101+
WORKDIR $RUN_WORKING_DIR
102+
103+
# clean tmp folder
104+
RUN rm -rf /tmp/*
105+
106+
RUN mkdir -p /etc/datascience/operators
107+
108+
USER datascience

‎ads/opctl/docker/Dockerfile_arm.job‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
# Copyright (c) 2023 Oracle and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
3+
4+
# Used OL8 because miniconda required a higher version of glibc that was unavoidable
5+
FROM ocr-docker-remote.artifactory.oci.oraclecorp.com/os/oraclelinux:8-slim-fips
6+
7+
# Configure environment
8+
ENV DATASCIENCE_USER datascience
9+
ENV DATASCIENCE_UID 1000
10+
ENV HOME /home/$DATASCIENCE_USER
11+
ENV DATASCIENCE_INSTALL_DIR /etc/datascience
12+
ENV LOGS_DIRECTORY /logs
13+
14+
ARG release=19
15+
ARG update=10
16+
ARG target=cpu
17+
18+
# Move this arg around the file as needed when you want to use cache for local builds/testing to speed up the process
19+
# This will allow you to set success points where you know previous code in the dockerfile is working
20+
# ARG RAND
21+
22+
COPY common /etc/datascience
23+
COPY etc/yum.repos.d /etc/yum.repos.d
24+
25+
# source: https://confluence.oraclecorp.com/confluence/pages/viewpage.action?spaceKey=ESEDoc&title=Artifactory+FAQ
26+
ADD Symantec_Private_SSL_SHA256_CA.cer /etc/pki/ca-trust/source/anchors
27+
ADD Symantec_Private_SSL_SHA256_Root.cer /etc/pki/ca-trust/source/anchors
28+
29+
RUN update-ca-trust
30+
31+
RUN microdnf install yum yum-utils && yum clean all && rm -rf /var/cache/yum
32+
33+
RUN \
34+
yum install -y \
35+
oracle-release-el8 \
36+
oraclelinux-developer-release-el8 \
37+
oracle-epel-release-el8 \
38+
https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
39+
yum-config-manager --enable ol8_addons --enable ol8_oracle_instantclient > /dev/null && yum -y update && \
40+
yum clean all && \
41+
rm -rf /var/cache/yum/*
42+
43+
RUN \
44+
yum install -y \
45+
--setopt=skip_missing_names_on_install=False \
46+
bzip2 \
47+
git \
48+
httpd \
49+
libcurl-devel \
50+
libxml2-devel \
51+
java-11-openjdk-headless \
52+
oracle-instantclient${release}.${update}-basic \
53+
oracle-instantclient${release}.${update}-sqlplus \
54+
openssl \
55+
openssl-devel \
56+
patch \
57+
snappy \
58+
gcc-c++ \
59+
sudo \
60+
unzip \
61+
vi \
62+
zip \
63+
wget \
64+
jq \
65+
gcc \
66+
htop \
67+
git-lfs \
68+
pcsc-lite-libs \
69+
perl-Env \
70+
perl-CPAN \
71+
perl-devel \
72+
curl-devel \
73+
expat-devel \
74+
asciidoc \
75+
xmlto \
76+
supervisor && \
77+
yum clean all && \
78+
rm -rf /var/cache/yum/*
79+
80+
RUN \
81+
yum groupinstall -y -q "development tools" \
82+
&& yum clean all \
83+
&& rm -rf /var/cache/yum/*
84+
85+
# setup user
86+
RUN \
87+
mkdir -p /home/$DATASCIENCE_USER && \
88+
useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \
89+
chown $DATASCIENCE_USER /home/$DATASCIENCE_USER && \
90+
chown -R $DATASCIENCE_USER:users /usr/local/ && \
91+
touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \
92+
mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR
93+
94+
# COPY build-tool-entrypoint.sh /etc/datascience/build-tool-entrypoint.sh
95+
# COPY install-capture-entrypoint.sh /etc/datascience/install-capture-entrypoint.sh
96+
97+
# set a default language for localization. necessary for oci cli
98+
ARG LANG=en_US.utf8
99+
ENV LANG=$LANG
100+
ENV SHELL=/bin/bash
101+
102+
############### SETUP ROOT (BASE) ENVIRONMENT WITH CONDA ######################
103+
ARG PYTHON_MAJOR='3'
104+
ARG PYTHON_MINOR='8'
105+
ARG PYTHON_PATCH=''
106+
ARG MINICONDA_VER=py${PYTHON_MAJOR}${PYTHON_MINOR}_23.5.2-0
107+
ARG MAMBAFORGE_VER=23.3.1-1
108+
109+
# set /opt folder permissions for $DATASCIENCE_USER. Conda is going to live in this folder.
110+
RUN chown $DATASCIENCE_USER /opt
111+
112+
USER $DATASCIENCE_USER
113+
WORKDIR /home/datascience
114+
115+
# Note in order to run sudo commands as a non root user, you must specify --credential yes if using qemu static to build the image
116+
RUN wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Miniconda3-${MINICONDA_VER}-$(uname)-$(uname -m).sh \
117+
-O /home/datascience/Miniconda3.sh \
118+
&& /bin/bash /home/datascience/Miniconda3.sh -f -b -p /opt/conda \
119+
&& rm /home/datascience/Miniconda3.sh \
120+
&& wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Mambaforge-${MAMBAFORGE_VER}-$(uname)-$(uname -m).sh \
121+
-O /home/datascience/Mambaforge3.sh \
122+
&& /bin/bash /home/datascience/Mambaforge3.sh -f -b -p /opt/conda \
123+
&& rm /home/datascience/Mambaforge3.sh \
124+
&& ls /opt/**/*
125+
126+
# Adding conda to the path
127+
ENV PATH="/opt/conda/bin:${PATH}"
128+
129+
# Set activate conda by default
130+
USER root
131+
RUN printf "#!/bin/bash\nsource /opt/conda/bin/activate\n" > /etc/profile.d/enableconda.sh \
132+
&& chmod +x /etc/profile.d/enableconda.sh
133+
134+
USER $DATASCIENCE_USER
135+
WORKDIR /home/datascience
136+
137+
COPY base-env-arm.yaml /opt/base-env.yaml
138+
COPY jupyter-arm.yaml /opt/jupyter.yaml
139+
140+
# Needs to be set to prevent a hang with mamba installations if this dockerfile
141+
# is being built by running qemu-user-static to emulate an ARM machine
142+
# <https://github.com/mamba-org/mamba/issues/1611>
143+
ENV G_SLICE=always-malloc
144+
145+
# Replaced conda with mamba. Conda wasn't able to resolve conflicts
146+
RUN mamba env update -n root -f /opt/jupyter.yaml && mamba clean -yaf && rm -rf /home/datascience/.cache/pip
147+
148+
149+
USER $DATASCIENCE_USER
150+
151+
####### WRAP UP ###############################
152+
RUN python -c 'import sys; assert(sys.version_info[:2]) == (3, 8), "Python 3.8 is not detected"'
153+
WORKDIR /
154+
155+
RUN conda list
156+
157+
USER root
158+
159+
ARG PIP_INDEX_URL
160+
161+
############# Setup Conda environment tools ###########################
162+
ARG RAND=1

0 commit comments

Comments
 (0)