Skip to content

Commit 04487eb

Browse files
pearupearu
authored
Use miniforge for conda and enable windows unit-testing in CI (#4085)
1 parent 2e30055 commit 04487eb

File tree

8 files changed

+127
-131
lines changed

8 files changed

+127
-131
lines changed

‎.github/scripts/unittest-windows/environment.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ dependencies:
88
- scipy >= 1.4.1
99
- pip
1010
- pip:
11-
- PySoundFile
1211
- future
1312
- parameterized
1413
- dataclasses

‎.github/scripts/unittest-windows/install.sh‎

Lines changed: 10 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,17 @@ conda activate "${env_dir}"
2020

2121
source "$this_dir/set_cuda_envs.sh"
2222

23+
printf "* Installing torch import-time dependencies\n"
24+
pip install numpy
25+
2326
# 1. Install PyTorch
2427
if [ -z "${CUDA_VERSION:-}" ] ; then
25-
cudatoolkit="cpuonly"
26-
version="cpu"
28+
wheel="cpu"
2729
else
28-
version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
29-
cudatoolkit="pytorch-cuda=${version}"
30+
wheel="cu$(python -c "print(''.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
3031
fi
31-
printf "Installing PyTorch with %s\n" "${cudatoolkit}"
32-
conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia pytorch "${cudatoolkit}" pytest pybind11
32+
printf "Installing PyTorch\n"
33+
pip install --pre torch --index-url https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${wheel}
3334

3435
torch_cuda=$(python -c "import torch; print(torch.cuda.is_available())")
3536
echo torch.cuda.is_available is $torch_cuda
@@ -42,30 +43,17 @@ if [ ! -z "${CUDA_VERSION:-}" ] ; then
4243
fi
4344

4445
# 2. Install torchaudio
45-
printf "* Installing fsspec\n"
46+
printf "* Installing fsspec\n" # TODO: is this required for torchaudio??
4647
pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org fsspec
4748

4849
printf "* Installing torchaudio\n"
4950
"$root_dir/packaging/vc_env_helper.bat" pip install . -v --no-build-isolation
5051

5152
# 3. Install Test tools
5253
printf "* Installing test tools\n"
53-
NUMBA_DEV_CHANNEL=""
5454
SENTENCEPIECE_DEPENDENCY="sentencepiece"
55-
case "$(python --version)" in
56-
*3.9*)
57-
# Numba isn't available for Python 3.9 except on the numba dev channel and building from source fails
58-
# See https://github.com/librosa/librosa/issues/1270#issuecomment-759065048
59-
NUMBA_DEV_CHANNEL="-c numba/label/dev"
60-
;;
61-
*3.10*)
62-
# Don't install sentencepiece, no python 3.10 dependencies available for windows yet
63-
SENTENCEPIECE_DEPENDENCY=""
64-
NUMBA_DEV_CHANNEL="-c numba/label/dev"
65-
;;
66-
esac
6755
(
68-
conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} parameterized 'requests>=2.20'
56+
conda install -y -c conda-forge parameterized 'requests>=2.20'
6957
# Need to disable shell check since this'll fail out if SENTENCEPIECE_DEPENDENCY is empty
7058
# shellcheck disable=SC2086
7159
pip install \
@@ -77,18 +65,5 @@ esac
7765
inflect \
7866
pytest \
7967
pytest-cov \
80-
pytorch-lightning \
81-
'scipy==1.7.3' \
82-
unidecode \
83-
'protobuf<4.21.0' \
84-
demucs \
85-
tinytag \
86-
pyroomacoustics \
87-
flashlight-text \
88-
git+https://github.com/kpu/kenlm/
68+
scipy
8969
)
90-
# Install fairseq
91-
git clone https://github.com/pytorch/fairseq
92-
cd fairseq
93-
git checkout e47a4c8
94-
pip install .

‎.github/scripts/unittest-windows/run_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ python -m torch.utils.collect_env
1212
env | grep TORCHAUDIO || true
1313

1414
cd test
15-
pytest --continue-on-collection-errors --cov=torchaudio --junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml -v --durations 20 torchaudio_unittest
15+
pytest --continue-on-collection-errors --cov=torchaudio --junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml -v --durations 20 torchaudio_unittest -k "not torchscript and not fairseq and not demucs and not librosa"
1616
coverage html

‎.github/scripts/unittest-windows/setup_env.sh‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ if [ ! -d "${conda_dir}" ]; then
1919
printf "* Installing conda\n"
2020
export tmp_conda="$(echo $conda_dir | tr '/' '\\')"
2121
export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe"
22-
curl --silent --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O
22+
# not using miniconda because its installation will be incomplete
23+
# despite using `start /wait` in the install batch file
24+
curl -L --silent --output miniconda.exe https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Windows-x86_64.exe -O
2325
"$this_dir/install_conda.bat"
2426
unset tmp_conda
2527
unset miniconda_exe
@@ -34,5 +36,4 @@ fi
3436
conda activate "${env_dir}"
3537

3638
# 3. Install minimal build tools
37-
pip --quiet install cmake ninja
38-
conda install --quiet -y 'ffmpeg>=4.1'
39+
conda install -y -c conda-forge cmake ninja

‎.github/workflows/unittest-linux-cpu.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ jobs:
3131
3232
# Set up Environment Variables
3333
export PYTHON_VERSION="${{ matrix.python_version }}"
34+
export PIP_PROGRESS_BAR=off
35+
export CONDA_QUIET=1
3436
export CUDA_TESTS_ONLY="0"
3537
unset CUDA_VERSION
3638

‎.github/workflows/unittest-linux-gpu.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ jobs:
3232
set -ex
3333
# Set up Environment Variables
3434
export PYTHON_VERSION="${{ matrix.python_version }}"
35+
export PIP_PROGRESS_BAR=off
36+
export CONDA_QUIET=1
3537
export CU_VERSION="${{ matrix.cuda_arch_version }}"
3638
export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}"
3739
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true

‎.github/workflows/unittest-windows-cpu.yml‎

Lines changed: 53 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,60 @@
11
# name: Unittests on Windows CPU
22

3-
# on:
4-
# pull_request:
5-
# push:
6-
# branches:
7-
# - nightly
8-
# - main
9-
# - release/*
10-
# workflow_dispatch:
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- nightly
8+
- main
9+
- release/*
10+
workflow_dispatch:
1111

12-
# jobs:
13-
# unittests-windows-cpu:
14-
# uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
15-
# with:
16-
# repository: pytorch/audio
17-
# runner: windows.4xlarge
18-
# timeout: 180
19-
# script: |
20-
# # Mark Build Directory Safe
21-
# git config --global --add safe.directory /__w/audio/audio
12+
jobs:
13+
unittests-windows-cpu:
14+
uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
15+
with:
16+
repository: pytorch/audio
17+
runner: windows.4xlarge
18+
timeout: 180
19+
script: |
20+
# Mark Build Directory Safe
21+
git config --global --add safe.directory /__w/audio/audio
2222
23-
# # Set up Environment Variables
24-
# export PYTHON_VERSION="3.9"
25-
# unset CUDA_VERSION
23+
# Set up Environment Variables
24+
export PYTHON_VERSION="3.10"
25+
export PIP_PROGRESS_BAR=off
26+
export CONDA_QUIET=1
27+
unset CUDA_VERSION
2628
27-
# # Set CHANNEL
28-
# if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
29-
# export UPLOAD_CHANNEL=test
30-
# else
31-
# export UPLOAD_CHANNEL=nightly
32-
# fi
29+
# Set CHANNEL
30+
if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
31+
export UPLOAD_CHANNEL=test
32+
else
33+
export UPLOAD_CHANNEL=nightly
34+
fi
3335
34-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING=true
35-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS=true
36-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS=true
37-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS=true
38-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS=true
39-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_SOX=true
40-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CUDA=true
41-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_HW_ACCEL=true
42-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_KALDI=true
43-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX=true
44-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
45-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
46-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
47-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
48-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
36+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING=true
37+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS=true
38+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS=true
39+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS=true
40+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS=true
41+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_SOX=true
42+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CUDA=true
43+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_HW_ACCEL=true
44+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_KALDI=true
45+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX=true
46+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
47+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
48+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
49+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
50+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
51+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_QUANTIZATION=true
52+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CTC_DECODER=true
53+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_unidecode=true
54+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_inflect=true
55+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_pytorch_lightning=true
56+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
4957
50-
# .github/scripts/unittest-windows/setup_env.sh
51-
# .github/scripts/unittest-windows/install.sh
52-
# .github/scripts/unittest-windows/run_test.sh
58+
.github/scripts/unittest-windows/setup_env.sh
59+
.github/scripts/unittest-windows/install.sh
60+
.github/scripts/unittest-windows/run_test.sh

‎.github/workflows/unittest-windows-gpu.yml‎

Lines changed: 55 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,61 @@
1-
# name: Unittests on Windows GPU
1+
name: Unittests on Windows GPU
22

3-
# on:
4-
# pull_request:
5-
# push:
6-
# branches:
7-
# - nightly
8-
# - main
9-
# - release/*
10-
# workflow_dispatch:
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- nightly
8+
- main
9+
- release/*
10+
workflow_dispatch:
1111

12-
# jobs:
13-
# unittests-windows-gpu:
14-
# uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
15-
# with:
16-
# repository: pytorch/audio
17-
# runner: windows.g5.4xlarge.nvidia.gpu
18-
# timeout: 360
19-
# script: |
20-
# # Mark Build Directory Safe
21-
# git config --global --add safe.directory /__w/audio/audio
12+
jobs:
13+
unittests-windows-gpu:
14+
uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
15+
with:
16+
repository: pytorch/audio
17+
runner: windows.g5.4xlarge.nvidia.gpu
18+
timeout: 360
19+
script: |
20+
# Mark Build Directory Safe
21+
git config --global --add safe.directory /__w/audio/audio
2222
23-
# # Set up Environment Variables
24-
# export PYTHON_VERSION="3.9"
25-
# export CUDA_VERSION=12.6
23+
# Set up Environment Variables
24+
export PYTHON_VERSION="3.10"
25+
export CUDA_VERSION=12.6
26+
export PIP_PROGRESS_BAR=off
27+
export CONDA_QUIET=1
28+
export USE_CUDA=1
29+
export CUBLAS_WORKSPACE_CONFIG=:4096:8
2630
27-
# # Set CHANNEL
28-
# if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
29-
# export UPLOAD_CHANNEL=test
30-
# else
31-
# export UPLOAD_CHANNEL=nightly
32-
# fi
31+
# Set CHANNEL
32+
if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
33+
export UPLOAD_CHANNEL=test
34+
else
35+
export UPLOAD_CHANNEL=nightly
36+
fi
3337
34-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING=true
35-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS=true
36-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS=true
37-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS=true
38-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS=true
39-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_SOX=true
40-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_KALDI=true
41-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX=true
42-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
43-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
44-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
45-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
46-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true
47-
# export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
38+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING=true
39+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS=true
40+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS=true
41+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS=true
42+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS=true
43+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_SOX=true
44+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_KALDI=true
45+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX=true
46+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
47+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
48+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_AUDIO_OUT_DEVICE=true
49+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MACOS=true
50+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true
51+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
52+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_QUANTIZATION=true
53+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CTC_DECODER=true
54+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_unidecode=true
55+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_inflect=true
56+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_pytorch_lightning=true
57+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece=true
4858
49-
50-
# .github/scripts/unittest-windows/setup_env.sh
51-
# .github/scripts/unittest-windows/install.sh
52-
# .github/scripts/unittest-windows/run_test.sh
59+
.github/scripts/unittest-windows/setup_env.sh
60+
.github/scripts/unittest-windows/install.sh
61+
.github/scripts/unittest-windows/run_test.sh

0 commit comments

Comments
 (0)