Skip to content

Commit 1f18adc

Browse files
author
Ziqun Ye
committed
first working version
1 parent c12ffd9 commit 1f18adc

File tree

2 files changed

+104
-52
lines changed

2 files changed

+104
-52
lines changed

‎ads/opctl/conda/pack.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,20 @@ def main(pack_folder_path):
2323
manifest_path = glob.glob(os.path.join(pack_folder_path, "*_manifest.yaml"))[0]
2424
with open(manifest_path) as f:
2525
env = yaml.safe_load(f.read())
26-
26+
2727
with tempfile.TemporaryDirectory() as td:
2828
process = subprocess.Popen(
2929
["conda", "env", "export", "--prefix", pack_folder_path],
3030
stdout=subprocess.PIPE,
3131
stderr=subprocess.PIPE,
3232
)
3333
stdout, stderr = process.communicate()
34-
if stderr:
35-
print(stderr)
36-
raise Exception(
37-
f"Error export environment information from {pack_folder_path}"
38-
)
34+
# import pdb; pdb.set_trace()
35+
# if stderr:
36+
# print(stderr)
37+
# raise Exception(
38+
# f"Error export environment information from {pack_folder_path}"
39+
# )
3940
try:
4041
new_env_info = yaml.safe_load(StringIO(stdout.decode("utf-8")))
4142
except Exception as e:

‎ads/opctl/docker/Dockerfile.job‎

Lines changed: 97 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
1-
# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
2-
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
3-
4-
FROM ghcr.io/oracle/oraclelinux:7-slim
1+
# Used OL8 because miniconda required a higher version of glibc that was unavoidable
2+
FROM ocr-docker-remote.artifactory.oci.oraclecorp.com/os/oraclelinux:8-slim-fips
53

64
# Configure environment
75
ENV DATASCIENCE_USER datascience
@@ -11,76 +9,139 @@ ENV DATASCIENCE_INSTALL_DIR /etc/datascience
119
ENV LOGS_DIRECTORY /logs
1210

1311
ARG release=19
14-
ARG update=13
12+
ARG update=10
13+
ARG target=cpu
14+
15+
# Move this arg around the file as needed when you want to use cache for local builds/testing to speed up the process
16+
# This will allow you to set success points where you know previous code in the dockerfile is working
17+
# ARG RAND
18+
19+
COPY common /etc/datascience
20+
COPY etc/yum.repos.d /etc/yum.repos.d
21+
22+
# source: https://confluence.oraclecorp.com/confluence/pages/viewpage.action?spaceKey=ESEDoc&title=Artifactory+FAQ
23+
ADD Symantec_Private_SSL_SHA256_CA.cer /etc/pki/ca-trust/source/anchors
24+
ADD Symantec_Private_SSL_SHA256_Root.cer /etc/pki/ca-trust/source/anchors
25+
26+
RUN update-ca-trust
27+
28+
RUN microdnf install yum yum-utils && yum clean all && rm -rf /var/cache/yum
1529

1630
RUN \
17-
yum -y -q install oracle-release-el7 && \
18-
yum-config-manager --add-repo http://yum.oracle.com/repo/OracleLinux/OL7/developer_EPEL/x86_64 && \
19-
yum-config-manager --enable ol7_optional_latest --enable ol7_addons --enable ol7_software_collections --enable ol7_oracle_instantclient > /dev/null && \
20-
yum groupinstall -y -q 'Development Tools' && \
21-
yum update -y && \
22-
yum install -y --setopt=skip_missing_names_on_install=False \
31+
yum install -y \
32+
oracle-release-el8 \
33+
oraclelinux-developer-release-el8 \
34+
oracle-epel-release-el8 \
35+
https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
36+
yum-config-manager --enable ol8_addons --enable ol8_oracle_instantclient > /dev/null && yum -y update && \
37+
yum clean all && \
38+
rm -rf /var/cache/yum/*
39+
40+
RUN \
41+
yum install -y \
42+
--setopt=skip_missing_names_on_install=False \
2343
bzip2 \
24-
curl \
2544
git \
26-
gcc-gfortran \
45+
httpd \
2746
libcurl-devel \
2847
libxml2-devel \
48+
java-11-openjdk-headless \
2949
oracle-instantclient${release}.${update}-basic \
3050
oracle-instantclient${release}.${update}-sqlplus \
3151
openssl \
3252
openssl-devel \
3353
patch \
54+
snappy \
55+
gcc-c++ \
3456
sudo \
3557
unzip \
58+
vi \
3659
zip \
37-
gcc-c++ \
3860
wget \
61+
jq \
3962
gcc \
63+
htop \
64+
git-lfs \
65+
pcsc-lite-libs \
66+
perl-Env \
67+
perl-CPAN \
68+
perl-devel \
69+
curl-devel \
70+
expat-devel \
71+
asciidoc \
72+
xmlto \
73+
supervisor && \
74+
yum clean all && \
75+
rm -rf /var/cache/yum/*
76+
77+
RUN \
78+
yum groupinstall -y -q "development tools" \
4079
&& yum clean all \
4180
&& rm -rf /var/cache/yum/*
4281

4382
# setup user
4483
RUN \
45-
mkdir -p /home/$DATASCIENCE_USER && \
46-
useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \
47-
chown -R $DATASCIENCE_USER /home/$DATASCIENCE_USER && \
48-
chown -R $DATASCIENCE_USER:users /usr/local/ && \
49-
touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \
50-
mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR && \
51-
mkdir -p $LOGS_DIRECTORY && chown -R $DATASCIENCE_USER:users $LOGS_DIRECTORY && \
52-
mkdir -p $LOGS_DIRECTORY/harness && chown -R $DATASCIENCE_USER:users $LOGS_DIRECTORY/harness
53-
54-
RUN mkdir -p /etc/datascience/build
55-
RUN mkdir -p $DATASCIENCE_INSTALL_DIR/{pre-build-ds,post-build-ds,pre-run-ds,pre-run-user}
56-
57-
#conda
84+
mkdir -p /home/$DATASCIENCE_USER && \
85+
useradd -m -s /bin/bash -N -u $DATASCIENCE_UID $DATASCIENCE_USER && \
86+
chown $DATASCIENCE_USER /home/$DATASCIENCE_USER && \
87+
chown -R $DATASCIENCE_USER:users /usr/local/ && \
88+
touch /etc/sudoers.d/$DATASCIENCE_USER && echo "$DATASCIENCE_USER ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/$DATASCIENCE_USER && \
89+
mkdir -p $DATASCIENCE_INSTALL_DIR && chown $DATASCIENCE_USER $DATASCIENCE_INSTALL_DIR
90+
91+
# COPY build-tool-entrypoint.sh /etc/datascience/build-tool-entrypoint.sh
92+
# COPY install-capture-entrypoint.sh /etc/datascience/install-capture-entrypoint.sh
93+
5894
# set a default language for localization. necessary for oci cli
5995
ARG LANG=en_US.utf8
6096
ENV LANG=$LANG
6197
ENV SHELL=/bin/bash
6298

99+
############### SETUP ROOT (BASE) ENVIRONMENT WITH CONDA ######################
100+
ARG PYTHON_MAJOR='3'
101+
ARG PYTHON_MINOR='8'
102+
ARG PYTHON_PATCH=''
103+
ARG MINICONDA_VER=py${PYTHON_MAJOR}${PYTHON_MINOR}_23.5.2-0
104+
ARG MAMBAFORGE_VER=23.3.1-1
105+
63106
# set /opt folder permissions for $DATASCIENCE_USER. Conda is going to live in this folder.
64-
RUN chown -R $DATASCIENCE_USER /opt
107+
RUN chown $DATASCIENCE_USER /opt
65108

66109
USER $DATASCIENCE_USER
67110
WORKDIR /home/datascience
68-
RUN wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /home/datascience/miniconda.sh \
69-
&& /bin/bash /home/datascience/miniconda.sh -f -b -p /opt/conda \
70-
&& rm /home/datascience/miniconda.sh \
71-
&& /opt/conda/bin/conda clean -yaf
72111

73-
WORKDIR /
112+
# Note in order to run sudo commands as a non root user, you must specify --credential yes if using qemu static to build the image
113+
RUN wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Miniconda3-${MINICONDA_VER}-$(uname)-$(uname -m).sh \
114+
-O /home/datascience/Miniconda3.sh \
115+
&& /bin/bash /home/datascience/Miniconda3.sh -f -b -p /opt/conda \
116+
&& rm /home/datascience/Miniconda3.sh \
117+
&& wget -nv https://artifactory.oci.oraclecorp.com/odsc-dev-generic-local/datascience/Mambaforge-${MAMBAFORGE_VER}-$(uname)-$(uname -m).sh \
118+
-O /home/datascience/Mambaforge3.sh \
119+
&& /bin/bash /home/datascience/Mambaforge3.sh -f -b -p /opt/conda \
120+
&& rm /home/datascience/Mambaforge3.sh \
121+
&& ls /opt/**/*
122+
123+
# Adding conda to the path
124+
ENV PATH="/opt/conda/bin:${PATH}"
125+
126+
# Set activate conda by default
74127
USER root
75128
RUN printf "#!/bin/bash\nsource /opt/conda/bin/activate\n" > /etc/profile.d/enableconda.sh \
76129
&& chmod +x /etc/profile.d/enableconda.sh
77130

78131
USER $DATASCIENCE_USER
79-
ENV PATH="/opt/conda/bin:${PATH}"
80132
WORKDIR /home/datascience
81133

82-
COPY docker/base-env.yaml /opt/base-env.yaml
83-
RUN conda env update -q -n root -f /opt/base-env.yaml && conda clean -yaf && rm -rf /home/datascience/.cache/pip
134+
COPY base-env-arm.yaml /opt/base-env.yaml
135+
COPY jupyter-arm.yaml /opt/jupyter.yaml
136+
137+
# Needs to be set to prevent a hang with mamba installations if this dockerfile
138+
# is being built by running qemu-user-static to emulate an ARM machine
139+
# <https://github.com/mamba-org/mamba/issues/1611>
140+
ENV G_SLICE=always-malloc
141+
142+
# Replaced conda with mamba. Conda wasn't able to resolve conflicts
143+
RUN mamba env update -n root -f /opt/jupyter.yaml && mamba clean -yaf && rm -rf /home/datascience/.cache/pip
144+
84145

85146
USER $DATASCIENCE_USER
86147

@@ -96,13 +157,3 @@ ARG PIP_INDEX_URL
96157

97158
############# Setup Conda environment tools ###########################
98159
ARG RAND=1
99-
100-
ARG RUN_WORKING_DIR="/home/datascience"
101-
WORKDIR $RUN_WORKING_DIR
102-
103-
# clean tmp folder
104-
RUN rm -rf /tmp/*
105-
106-
RUN mkdir -p /etc/datascience/operators
107-
108-
USER datascience

0 commit comments

Comments
 (0)