Skip to content

Commit ccc7262

Browse files
committed
Support Python's build-in module venv for virtual environments
1 parent a9cc1d8 commit ccc7262

File tree

7 files changed

+314
-17
lines changed

7 files changed

+314
-17
lines changed
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
name: (DRAFT) Java Code Structure Graph Analysis using venv
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
# Ignore changes in documentation, general configuration and reports for push events
8+
paths-ignore:
9+
- '**/*.md'
10+
- '**/*.txt'
11+
- '**/*.css'
12+
- '**/*.html'
13+
- '**/*.js'
14+
- '.gitignore'
15+
- '.gitattributes'
16+
- 'renovate.json'
17+
- 'renovate-presets/**'
18+
- 'changelogTemplate.mustache'
19+
- '**.code-workspace'
20+
- '.github/workflows/internal-typescript-code-analysis.yml'
21+
- '.github/workflows/*documentation.yml'
22+
pull_request:
23+
branches:
24+
- main
25+
# Ignore changes in documentation, general configuration and reports for pull request events
26+
paths-ignore:
27+
- '**/*.md'
28+
- '**/*.txt'
29+
- '**/*.css'
30+
- '**/*.html'
31+
- '**/*.js'
32+
- '.gitignore'
33+
- '.gitattributes'
34+
- 'renovate.json'
35+
- 'renovate-presets/**'
36+
- 'changelogTemplate.mustache'
37+
- '**.code-workspace'
38+
- '.github/workflows/internal-typescript-code-analysis.yml'
39+
- '.github/workflows/*documentation.yml'
40+
41+
# Requires the secret NEO4J_INITIAL_PASSWORD to be configured
42+
jobs:
43+
prepare-code-to-analyze:
44+
runs-on: ubuntu-latest
45+
outputs:
46+
analysis-name: ${{ steps.set-analysis-name.outputs.analysis-name }}
47+
sources-upload-name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }}
48+
artifacts-upload-name: ${{ steps.set-artifacts-upload-name.outputs.artifacts-upload-name }}
49+
50+
env:
51+
PROJECT_NAME: AxonFramework
52+
# Version variable names matches renovate.json configuration entry
53+
AXON_FRAMEWORK_VERSION: 4.11.2
54+
55+
steps:
56+
- name: Checkout GIT Repository
57+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
58+
59+
- name: Run script tests
60+
id: script-tests
61+
run: ./scripts/runTests.sh
62+
63+
- name: Set Set output variable 'analysis-name'
64+
id: set-analysis-name
65+
run: echo "analysis-name=${{ env.PROJECT_NAME }}-${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT"
66+
67+
- name: Setup temp directory if missing
68+
run: mkdir -p ./temp
69+
70+
- name: Download ${{ steps.set-analysis-name.outputs.analysis-name }}
71+
working-directory: temp
72+
run: |
73+
mkdir -p ${{ steps.set-analysis-name.outputs.analysis-name }}
74+
cd ${{ steps.set-analysis-name.outputs.analysis-name }}
75+
echo "Working directory: $( pwd -P )"
76+
./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }}
77+
78+
- name: Debug folder structure in temp directory
79+
if: runner.debug == '1'
80+
working-directory: temp
81+
run: |
82+
ls -R | grep ":$" | sed -e 's/:$//' -e 's/[^-][^\/]*\//--/g' -e 's/^/ /' -e 's/-/|/'
83+
84+
- name: (Prepare Code to Analyze) Generate ARTIFACT_UPLOAD_ID
85+
run: echo "ARTIFACT_UPLOAD_ID=$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 10)" >> $GITHUB_ENV
86+
87+
- name: (Prepare Code to Analyze) Set sources-upload-name
88+
id: set-sources-upload-name
89+
run: echo "sources-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-sources_input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
90+
91+
- name: (Prepare Code to Analyze) Set output variable 'artifacts-upload-name'
92+
id: set-artifacts-upload-name
93+
run: echo "artifacts-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-artifacts-input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
94+
95+
- name: (Prepare Code to Analyze) Upload sources to analyze
96+
if: success()
97+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
98+
with:
99+
name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }}
100+
path: ./temp/${{ steps.set-analysis-name.outputs.analysis-name }}/source
101+
include-hidden-files: true
102+
if-no-files-found: error
103+
retention-days: 1
104+
105+
- name: (Prepare Code to Analyze) Upload artifacts to analyze
106+
if: success()
107+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
108+
with:
109+
name: ${{ steps.set-artifacts-upload-name.outputs.artifacts-upload-name }}
110+
path: ./temp/${{ steps.set-analysis-name.outputs.analysis-name }}/artifacts
111+
if-no-files-found: error
112+
retention-days: 1
113+
114+
115+
116+
analyze-code-graph:
117+
needs: [prepare-code-to-analyze]
118+
uses: ./.github/workflows/public-analyze-code-graph.yml
119+
with:
120+
analysis-name: ${{ needs.prepare-code-to-analyze.outputs.analysis-name }}
121+
artifacts-upload-name: ${{ needs.prepare-code-to-analyze.outputs.artifacts-upload-name }}
122+
sources-upload-name: ${{ needs.prepare-code-to-analyze.outputs.sources-upload-name }}
123+
jupyter-pdf: "false"
124+
use-venv_virtual_python_environment: "true"

.github/workflows/public-analyze-code-graph.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@ on:
6161
required: false
6262
type: string
6363
default: 'true'
64+
use-venv_virtual_python_environment:
65+
description: >
66+
Use venv for virtual Python environments instead of Conda ("true") or not ("false", default).
67+
required: false
68+
type: string
69+
default: 'false'
6470
outputs:
6571
uploaded-analysis-results:
6672
description: >
@@ -103,6 +109,7 @@ jobs:
103109

104110
# "Setup Python" can be skipped if jupyter notebook analysis-results aren't needed
105111
- name: (Python Setup) Use version ${{ matrix.python }} with Conda package manager Miniforge
112+
if: inputs.use-venv_virtual_python_environment == 'false'
106113
id: prepare-conda-environment
107114
uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3
108115
with:
@@ -112,7 +119,15 @@ jobs:
112119
environment-file: ./conda-environment.yml
113120
auto-activate-base: false
114121
show-channel-urls: true
122+
123+
- name: (Python Setup) Use version ${{ matrix.python }} with venv environment management module
124+
if: inputs.use-venv_virtual_python_environment == 'true'
125+
uses: actions/setup-python@v5
126+
with:
127+
python-version: ${{ matrix.python }}
128+
115129
- name: (Python Setup) Conda environment info
130+
if: inputs.use-venv_virtual_python_environment == 'false'
116131
shell: bash -el {0}
117132
run: |
118133
conda info
@@ -168,6 +183,7 @@ jobs:
168183
ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: ${{ inputs.jupyter-pdf }}
169184
IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "" # Options: "none", "aggregated", "full". default = "plugin" or ""
170185
PREPARE_CONDA_ENVIRONMENT: "false" # Had already been done in step with id "prepare-conda-environment".
186+
USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV: ${{ inputs.use-venv_virtual_python_environment }}
171187
run: |
172188
TYPESCRIPT_SCAN_HEAP_MEMORY=${{ inputs.typescript-scan-heap-memory }} ./../../scripts/analysis/analyze.sh ${{ inputs.analysis-arguments }}
173189

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,8 @@ __pycache__/
9898

9999
# Python environments
100100
.conda
101+
.venv/
102+
*.pyc
101103

102104
# Optuna (and other) Database data
103105
*.db

requirements.txt

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# --- Core Python version ---
2+
# NOTE: Python version must be >= 3.12 for compatibility
3+
# This should be enforced by the user/environment, not pip
4+
5+
# --- Core tools ---
6+
jupyter==1.1.*
7+
matplotlib==3.10.*
8+
nbconvert[webpdf]==7.16.*
9+
numpy==1.26.*
10+
pandas==2.2.*
11+
pip==25.0.*
12+
setuptools==75.8.* # opentsne uses sklearn.base uses joblib uses distutils missing in Python >= 12 (TODO use native openTSNE?)
13+
typing-extensions==4.12.* # Needed for opentsne and Python >= 3.12
14+
15+
# --- Visualization ---
16+
wordcloud==1.9.*
17+
monotonic==1.*
18+
plotly[kaleido]==6.2.*
19+
seaborn==0.13 # To visualize clustering results
20+
21+
# --- Machine Learning / Optimization ---
22+
scikit-learn==1.6.*
23+
optuna==4.3.*
24+
umap-learn==0.5.* # Dimensionality reduction to visualize node embeddings in 2D
25+
26+
# --- Database connector ---
27+
neo4j==5.23.*
28+
29+
# --- Native/scientific packages (may require compilation) ---
30+
# These are included but may cause install errors in pip/venv
31+
opentsne==1.0.* # Dimensionality reduction to visualize node embeddings in 2D. Might get replaced by umap.
32+
shap==0.48.* # For e.g. explaining anomaly detection results

scripts/activateCondaEnvironment.sh

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,33 @@
11
#!/usr/bin/env bash
22

3-
# Activates the Conda (Python package manager) environment "codegraph" with all packages needed to execute the Jupyter Notebooks.
3+
# Activates the Conda (Python package manager) environment "codegraph" with all packages needed to run the included Jupyter Notebooks and Python scripts.
44

55
# Note: This script uses the conda environment defined in CODEGRAPH_CONDA_ENVIRONMENT (defaults to "codegraph").
6-
# If the environment hadn't been created yet, it will use "conda-environment.yml" from the root directory
7-
# in the same directory as the given jupyter notebook ipynb file
8-
# to create the environment.
6+
# If the environment hadn't been created yet, it will use "conda-environment.yml" from the root directory to create the environment.
97

108
# Requires operatingSystemFunctions.sh
119

1210
# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
1311
set -o errexit -o pipefail
1412

13+
PREPARE_CONDA_ENVIRONMENT=${PREPARE_CONDA_ENVIRONMENT:-"true"} # Wether to prepare a Python environment with Conda if needed (default, "true") or use an already prepared Conda environment ("false")
14+
15+
if [ "${PREPARE_CONDA_ENVIRONMENT}" = "false" ]; then
16+
echo "activateCondaEnvironment: Skipping activation. An already activated environment and installed dependencies are expected (PREPARE_CONDA_ENVIRONMENT=false)."
17+
# "return" needs to be used here instead of "exit".
18+
# This script is included in another script by using "source".
19+
# "exit" would end the main script, "return" just ends this sub script.
20+
return 0
21+
fi
22+
23+
if [ "${USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV}" = "true" ]; then
24+
echo "activateCondaEnvironment: Skipping activation. venv will be used instead of conda (USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV=true)."
25+
# "return" needs to be used here instead of "exit".
26+
# This script is included in another script by using "source".
27+
# "exit" would end the main script, "return" just ends this sub script.
28+
return 0
29+
fi
30+
1531
## Get this "scripts" directory if not already set
1632
# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution.
1733
# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
@@ -37,16 +53,6 @@ echo "activateCondaEnvironment: CONDA_PREFIX=${CONDA_PREFIX}"
3753
echo "activateCondaEnvironment: Current conda environment=${CONDA_DEFAULT_ENV}"
3854
echo "activateCondaEnvironment: Target conda environment=${CODEGRAPH_CONDA_ENVIRONMENT}"
3955

40-
PREPARE_CONDA_ENVIRONMENT=${PREPARE_CONDA_ENVIRONMENT:-"true"} # Wether to prepare a Python environment with Conda if needed (default, "true") or use an already prepared Conda environment ("false")
41-
42-
if [ "${PREPARE_CONDA_ENVIRONMENT}" = "false" ]; then
43-
echo "activateCondaEnvironment: Skipping activation. ${PREPARE_CONDA_ENVIRONMENT} is set to false."
44-
# "return" needs to be used here instead of "exit".
45-
# This script is included in another script by using "source".
46-
# "exit" would end the main script, "return" just ends this sub script.
47-
return 0
48-
fi
49-
5056
# Include operation system function to for example detect Windows.
5157
source "${SCRIPTS_DIR}/operatingSystemFunctions.sh"
5258

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#!/usr/bin/env bash
2+
3+
# Activates the .venv environment (Python build-in virtual environments) with all packages necessary to run the included Jupyter Notebooks and Python scripts.
4+
5+
# Note: If the environment hadn't been created yet, it will use "requirements.txt" from the root directory to create the environment.
6+
7+
# Requires operatingSystemFunctions.sh
8+
9+
# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
10+
set -o errexit -o pipefail
11+
12+
exit_failed() {
13+
case "$0" in
14+
*/sh) return 1 ;; # Script is sourced
15+
*) exit 1 ;; # Script is executed directly
16+
esac
17+
}
18+
19+
exit_successful() {
20+
case "$0" in
21+
*/sh) return 0 ;; # Script is sourced
22+
*) exit 0 ;; # Script is executed directly
23+
esac
24+
}
25+
26+
USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV=${USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV:-"false"} # Use "venv" for virtual Python environments ("true") or use an already prepared (e.g. conda) environment (default, "false").
27+
28+
if [ "${USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV}" = "false" ]; then
29+
echo "activatePythonEnvironment: Skipping activation. An already activated environment and installed dependencies are expected e.g. by using conda (USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV=false)."
30+
# "return" needs to be used here instead of "exit".
31+
# This script is included in another script by using "source".
32+
# "exit" would end the main script, "return" just ends this sub script.
33+
exit_successful
34+
fi
35+
36+
## Get this "scripts" directory if not already set
37+
# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution.
38+
# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
39+
# This way non-standard tools like readlink aren't needed.
40+
SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
41+
echo "activatePythonEnvironment: SCRIPTS_DIR=${SCRIPTS_DIR}"
42+
43+
# Get the root directory by taking the path of this script and going one directory up.
44+
ROOT_DIRECTORY=${ROOT_DIRECTORY:-"$(dirname "${SCRIPTS_DIR}")"} # Repository directory containing the Jupyter Notebooks
45+
echo "activatePythonEnvironment: ROOT_DIRECTORY=${ROOT_DIRECTORY}"
46+
47+
# Get the file name of the environment description file for the conda package and environment manager
48+
# that contains all dependencies and their versions.
49+
PYTHON_ENVIRONMENT_FILE=${PYTHON_ENVIRONMENT_FILE:-"${ROOT_DIRECTORY}/requirements.txt"} # Pip (package manager for Python) environment file path
50+
if [ ! -f "${PYTHON_ENVIRONMENT_FILE}" ] ; then
51+
echo "activatePythonEnvironment: Couldn't find environment file ${PYTHON_ENVIRONMENT_FILE}."
52+
exit_failed
53+
fi
54+
55+
deactivate_conda_if_necessary() {
56+
# Include operation system function to for example detect Windows.
57+
source "${SCRIPTS_DIR}/operatingSystemFunctions.sh"
58+
59+
# Determine the path to "conda"
60+
if [ -n "${CONDA}" ]; then
61+
if isWindows; then
62+
pathToConda="${CONDA}\\Scripts\\" # the trailing backslash character is required
63+
else
64+
pathToConda="${CONDA}/bin/" # the trailing slash character is required
65+
fi
66+
else
67+
pathToConda=""
68+
fi
69+
echo "activatePythonEnvironment: pathToConda=${pathToConda} (for deactivation)"
70+
71+
scriptExtension=$(ifWindows ".bat" "")
72+
echo "activatePythonEnvironment: scriptExtension=${scriptExtension}"
73+
74+
if "${pathToConda}conda" deactivate >/dev/null 2>&1; then
75+
# Call "deactivate" a second time to also deactivate the "base" environment
76+
"${pathToConda}conda" deactivate >/dev/null 2>&1;
77+
echo "activatePythonEnvironment: Conda deactivated"
78+
else
79+
echo "activatePythonEnvironment: Conda not found. Deactivation skipped"
80+
fi
81+
}
82+
83+
VENV_DIRECTORY=".venv"
84+
85+
# Create the virtual environment if needed
86+
if [ ! -d "${ROOT_DIRECTORY}/${VENV_DIRECTORY}" ]; then
87+
deactivate_conda_if_necessary
88+
echo "activatePythonEnvironment: Creating ${VENV_DIRECTORY} environment..."
89+
python3 -m venv "${VENV_DIRECTORY}"
90+
else
91+
echo "activatePythonEnvironment: Already created ${VENV_DIRECTORY} environment."
92+
fi
93+
94+
# Activate the virtual environment if needed
95+
if [ "${VIRTUAL_ENV}" != "${ROOT_DIRECTORY}/${VENV_DIRECTORY}" ]; then
96+
echo "activatePythonEnvironment: Activate ${VENV_DIRECTORY} environment..."
97+
source "${VENV_DIRECTORY}/bin/activate"
98+
else
99+
echo "activatePythonEnvironment: Already activated ${VENV_DIRECTORY} environment."
100+
fi
101+
102+
# Install the virtual environment if needed
103+
if pip install --dry-run --no-deps --requirement "${PYTHON_ENVIRONMENT_FILE}" 2>/dev/null | grep -q "Would install"; then
104+
echo "activatePythonEnvironment: Install environment dependencies..."
105+
pip install -q --requirement "${PYTHON_ENVIRONMENT_FILE}"
106+
else
107+
echo "activatePythonEnvironment: Already installed environment dependencies."
108+
fi
109+
110+
echo "activatePythonEnvironment: Python installation: $(which python)"

0 commit comments

Comments
 (0)