Skip to content

Commit a2162d8

Browse files
sfc-gh-zyaosfc-gh-turbaszek
sfc-gh-zyao
and
sfc-gh-turbaszek
authored
SNOW-2483517 sproc continuous integration (#2629)
Co-authored-by: Tomasz Urbaszek <tomasz.urbaszek@snowflake.com>
1 parent 11803c5 commit a2162d8

File tree

10 files changed

+462
-32
lines changed

10 files changed

+462
-32
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,11 @@ repos:
2020
src/snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/.*\.h|
2121
)$
2222
- id: check-yaml
23-
exclude: .github/repo_meta.yaml
23+
exclude: >
24+
(?x)^(
25+
.github/repo_meta.yaml|
26+
ci/anaconda/recipe/meta.yaml|
27+
)$
2428
- id: debug-statements
2529
- id: check-ast
2630
- repo: https://github.com/asottile/yesqa

‎ci/anaconda/bld.bat‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎ci/anaconda/build.sh‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎ci/anaconda/conda_build.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Check https://snow-external.slack.com/archives/C02D68R4D0D/p1678899446863299 for context about using --numpy
2+
conda install conda-build
3+
conda install conda-verify
4+
conda install diffutils
5+
conda build ci/anaconda/recipe/ --python 3.9
6+
conda build ci/anaconda/recipe/ --python 3.10
7+
conda build ci/anaconda/recipe/ --python 3.11
8+
conda build ci/anaconda/recipe/ --python 3.12
9+
conda build ci/anaconda/recipe/ --python 3.13

‎ci/anaconda/meta.yaml‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎ci/anaconda/package_builder.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
export SNOWFLAKE_CONNECTOR_PYTHON_DIR=/repo/snowflake-connector-python
4+
export CONDA_BLD_PATH=/repo/conda-bld
5+
6+
mkdir -p $CONDA_BLD_PATH
7+
cd "$SNOWFLAKE_CONNECTOR_PYTHON_DIR"
8+
# Build with .tar.bz2 (pkg_format = 1) and .conda (pkg_format = 2).
9+
conda config --set conda_build.pkg_format 1
10+
bash ./ci/anaconda/conda_build.sh
11+
conda config --set conda_build.pkg_format 2
12+
bash ./ci/anaconda/conda_build.sh
13+
conda config --remove-key conda_build.pkg_format
14+
conda build purge
15+
cd $CONDA_BLD_PATH
16+
conda index .
17+
chmod -R o+w,g+w $CONDA_BLD_PATH

‎ci/anaconda/recipe/meta.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
{% set name = "snowflake-connector-python" %}
2+
{% set version = os.environ.get('SNOWFLAKE_CONNECTOR_PYTHON_VERSION', 0) %}
3+
{% set build_number = os.environ.get('PUBLIC_CONNECTOR_BUILD_NUMBER', 0) %}
4+
5+
package:
6+
name: {{ name|lower }}
7+
version: {{ version }}
8+
9+
source:
10+
path: ../../..
11+
12+
build:
13+
number: {{ build_number }}
14+
string: "py{{ py }}_{{ build_number }}"
15+
script:
16+
- export SF_NO_COPY_ARROW_LIB=1 # [unix]
17+
- export SF_ARROW_LIBDIR="${PREFIX}/lib" # [unix]
18+
- export ENABLE_EXT_MODULES=true # [unix]
19+
- {{ PYTHON }} -m pip install . --no-use-pep517 --no-deps -vvv
20+
entry_points:
21+
- snowflake-dump-ocsp-response = snowflake.connector.tool.dump_ocsp_response:main
22+
- snowflake-dump-ocsp-response-cache = snowflake.connector.tool.dump_ocsp_response_cache:main
23+
- snowflake-dump-certs = snowflake.connector.tool.dump_certs:main
24+
25+
requirements:
26+
build:
27+
- {{ compiler("c") }}
28+
- {{ compiler("cxx") }}
29+
- libgcc-ng
30+
- libstdcxx-ng
31+
- patch # [not win]
32+
host:
33+
- setuptools >=40.6.0
34+
- wheel
35+
- cython
36+
- python {{ python }}
37+
run:
38+
{% if py == 39 %}
39+
- python >=3.9,<3.10.0a0
40+
{% elif py == 310 %}
41+
- python >=3.10,<3.11.0a0
42+
{% elif py == 311 %}
43+
- python >=3.11,<3.12.0a0
44+
{% elif py == 312 %}
45+
- python >=3.12,<3.13.0a0
46+
{% elif py == 313 %}
47+
- python >=3.13,<3.14.0a0
48+
{% else %}
49+
- python
50+
{% endif %}
51+
- asn1crypto >0.24.0,<2.0.0
52+
- cryptography >=44.0.1
53+
- pyOpenSSL >=24.0.0,<26.0.0
54+
- pyjwt >=2.10.1,<3.0.0
55+
- pytz
56+
- requests >=2.32.4,<3.0.0
57+
- packaging
58+
- charset-normalizer >=2,<4
59+
- idna >=3.7,<4
60+
- urllib3 >=1.26.5,<2.0.0 # [py<310]
61+
- certifi >=2024.7.4
62+
- typing_extensions >=4.3,<5
63+
- filelock >=3.5,<4
64+
- sortedcontainers >=2.4.0
65+
- platformdirs >=2.6.0,<5.0.0
66+
- tomlkit
67+
- boto3 >=1.24
68+
- botocore >=1.24
69+
test:
70+
requires:
71+
- pip
72+
imports:
73+
- snowflake
74+
- snowflake.connector
75+
- snowflake.connector.nanoarrow_arrow_iterator # [unix]
76+
commands:
77+
- pip check
78+
79+
about:
80+
home: https://github.com/snowflakedb/snowflake-connector-python
81+
summary: Snowflake Connector for Python

‎ci/anaconda/run.sh‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
2+
# Before manual running, do something similar to the following in command line.
3+
# export WORKSPACE=/home/jdoe/my_workspace;
4+
# export PUBLIC_CONNECTOR_BUILD_NUMBER=321;
5+
# export aarch64_base_image=<location_of_centos8_aarch64_base_image>;
6+
# export x86_base_image=<location_of_centos8_x86_base_image>;
7+
8+
# Here miniconda-install.sh is just a installer that I downloaded from Anaconda official site,
9+
# https://repo.anaconda.com/miniconda/
10+
11+
12+
if [[ -z $WORKSPACE ]]; then
13+
# Development on dev machine
14+
WORKSPACE=$HOME
15+
fi
16+
17+
# ===== Build docker image =====
18+
cd $WORKSPACE
19+
20+
# Validate dependency sync before building
21+
python3 -m venv tmp_validate_env
22+
source tmp_validate_env/bin/activate
23+
pip install pyyaml
24+
python3 $WORKSPACE/snowflake-connector-python/ci/anaconda/validate_deps_sync.py
25+
if [[ $? -ne 0 ]]; then
26+
echo "[FAILURE] setup.cfg and meta.yaml dependencies are not in sync"
27+
deactivate
28+
rm -rf tmp_validate_env
29+
exit 1
30+
fi
31+
deactivate
32+
rm -rf tmp_validate_env
33+
34+
docker build \
35+
--build-arg ARCH=$(uname -m) \
36+
--build-arg AARCH64_BASE_IMAGE="${aarch64_base_image}" \
37+
--build-arg X86_BASE_IMAGE="${x86_base_image}" \
38+
-t snowflake_connector_python_image \
39+
-f - . <<'DOCKERFILE'
40+
# Use different base images based on target platform
41+
42+
ARG ARCH
43+
ARG AARCH64_BASE_IMAGE=artifactory.int.snowflakecomputing.com/development-docker-virtual/arm64v8/centos:8
44+
ARG X86_BASE_IMAGE=artifactory.int.snowflakecomputing.com/development-docker-virtual/centos:8
45+
46+
FROM ${AARCH64_BASE_IMAGE} AS base-aarch64
47+
48+
FROM ${X86_BASE_IMAGE} AS base-x86_64
49+
50+
51+
52+
# Select the appropriate base image based on target architecture
53+
54+
FROM base-${ARCH} AS base
55+
56+
COPY miniconda-install.sh .
57+
58+
59+
60+
RUN chmod 0755 miniconda-install.sh
61+
62+
63+
64+
RUN mkdir -p /etc/miniconda && bash miniconda-install.sh -b -u -p /etc/miniconda/
65+
66+
67+
68+
RUN ln -s /etc/miniconda/bin/conda /usr/bin/conda && rm miniconda-install.sh
69+
DOCKERFILE
70+
71+
# Go back to the original directory
72+
cd $WORKSPACE
73+
74+
75+
# Check to make sure repos exist to build conda packages
76+
if [[ -d $WORKSPACE/snowflake-connector-python ]]; then
77+
echo "Check snowflake-connector-python repo exists - PASSED"
78+
else
79+
echo "[FAILURE] Please clone snowflake-connector-python repo at $WORKSPACE/snowflake-connector-python"
80+
fi
81+
82+
# Extract connector version if not provided
83+
if [[ -z "$SNOWFLAKE_CONNECTOR_PYTHON_VERSION" ]]; then
84+
VERSION_FILE="$WORKSPACE/snowflake-connector-python/src/snowflake/connector/version.py"
85+
if [[ -f "$VERSION_FILE" ]]; then
86+
SNOWFLAKE_CONNECTOR_PYTHON_VERSION=$( \
87+
grep -Eo 'VERSION\s*=\s*\([^)]*\)' "$VERSION_FILE" \
88+
| grep -Eo '[0-9]+' \
89+
| paste -sd '.' - \
90+
)
91+
export SNOWFLAKE_CONNECTOR_PYTHON_VERSION
92+
fi
93+
fi
94+
95+
# Run packager in docker image
96+
docker run \
97+
-v $WORKSPACE/snowflake-connector-python/:/repo/snowflake-connector-python \
98+
-v $WORKSPACE/conda-bld:/repo/conda-bld \
99+
-e SNOWFLAKE_CONNECTOR_PYTHON_VERSION=${SNOWFLAKE_CONNECTOR_PYTHON_VERSION} \
100+
-e PUBLIC_CONNECTOR_BUILD_NUMBER=${PUBLIC_CONNECTOR_BUILD_NUMBER} \
101+
snowflake_connector_python_image \
102+
/repo/snowflake-connector-python/ci/anaconda/package_builder.sh
103+
104+
# Cleanup image for disk space
105+
docker container prune -f
106+
docker rmi snowflake_connector_python_image

0 commit comments

Comments
 (0)