diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..b780bbb6c --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,101 @@ +version: 2.1 + +parameters: + REF: + type: string + default: '' + description: Optional tag to build + +jobs: + arm-wheels: + parameters: + build: + type: string + image: + type: string + + machine: + image: ubuntu-2404:current + resource_class: arm.large # 4 vCPUs + + environment: + CIBW_ARCHS: "aarch64" + CIBW_MANYLINUX_AARCH64_IMAGE: "<< parameters.image >>" + CIBW_MUSLLINUX_AARCH64_IMAGE: "<< parameters.image >>" + CIBW_BUILD: "<< parameters.build >>" + + steps: + - checkout + - when: + condition: << pipeline.parameters.REF >> + steps: + - run: + name: Checkout branch/tag << pipeline.parameters.REF >> + command: | + echo "Switching to branch/tag << pipeline.parameters.REF >> if it exists" + git checkout << pipeline.parameters.REF >> || true + git pull origin << pipeline.parameters.REF >> || true + - run: + name: install cibuildwheel and other build reqs + command: | + python3 -m pip install --upgrade pip setuptools setuptools_scm[toml] + python3 -m pip install -rcibw-requirements.txt + + - run: + name: pip freeze + command: | + python3 -m pip freeze + + - run: + name: list wheels + command: | + python3 -m cibuildwheel . --print-build-identifiers + + - run: + name: cibuildwheel + command: | + python3 -m cibuildwheel . + + - store_test_results: + path: test-results/ + + - store_artifacts: + path: wheelhouse/ + + # - when: + # condition: + # or: + # - matches: + # pattern: ".+" + # value: "<< pipeline.git.tag >>" + # - << pipeline.parameters.REF >> + # steps: + # - run: + # environment: + # TWINE_NONINTERACTIVE: "1" + # command: | + # python3 -m pip install twine + # python3 -m twine upload --verbose --skip-existing wheelhouse/* + +workflows: + wheels: # This is the name of the workflow, feel free to change it to better match your workflow. + # Inside the workflow, you define the jobs you want to run. + jobs: + - arm-wheels: + name: arm-wheels-manylinux_2_28 + filters: + branches: + only: main + tags: + only: /.*/ + build: "*manylinux*" + image: quay.io/pypa/manylinux_2_28_aarch64 + - arm-wheels: + name: arm-wheels-musllinux_1_2 + filters: + branches: + only: main + tags: + only: /.*/ + build: "*musllinux*" + image: quay.io/pypa/musllinux_1_2_aarch64 diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1530ab92b..d40fd1b1f 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -32,7 +32,7 @@ jobs: strategy: matrix: py-ver-major: [3] - py-ver-minor: [8, 9, 10, 11, 12, 13] + py-ver-minor: [10, 11, 12, 13, 14] step: [lint, unit, bandit, mypy] env: @@ -40,22 +40,22 @@ jobs: TOXENV: ${{ format('py{0}{1}-{2}', matrix.py-ver-major, matrix.py-ver-minor, matrix.step) }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - - name: Set up Singularity + - name: Set up Singularity and environment-modules if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} run: sudo usermod -c 'CI Runner' "$(whoami)" - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ env.py-semver }} allow-prereleases: true @@ -64,6 +64,9 @@ jobs: requirements.txt tox.ini + - name: install dev libraries + run: sudo apt-get install -y libxml2-dev libxslt-dev + - name: Upgrade setuptools and install tox run: | pip install -U pip setuptools wheel @@ -81,7 +84,7 @@ jobs: - name: Upload coverage to Codecov if: ${{ matrix.step == 'unit' }} - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: @@ -89,23 +92,23 @@ jobs: tox-style: name: Linters - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: matrix: step: [lintreadme, shellcheck, pydocstyle] env: - py-semver: "3.12" - TOXENV: ${{ format('py312-{0}', matrix.step) }} + py-semver: "3.13" + TOXENV: ${{ format('py313-{0}', matrix.step) }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ env.py-semver }} cache: pip @@ -126,22 +129,22 @@ jobs: name: No leftovers runs-on: ubuntu-22.04 env: - py-semver: "3.12" + py-semver: "3.13" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - - name: Set up Singularity + - name: Set up Singularity and environment-modules run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. run: sudo usermod -c 'CI Runner' "$(whoami)" - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ env.py-semver }} cache: pip @@ -157,7 +160,7 @@ jobs: chmod a-w . - name: run tests - run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test + run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test PYTEST_EXTRA=-vvv conformance_tests: @@ -165,6 +168,7 @@ jobs: runs-on: ubuntu-22.04 strategy: + fail-fast: false matrix: cwl-version: [v1.0, v1.1, v1.2] container: [docker, singularity, podman] @@ -177,13 +181,13 @@ jobs: extras: "--relax-path-checks" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - - name: Set up Singularity + - name: Set up Singularity and environment-modules if: ${{ matrix.container == 'singularity' }} run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Singularity cache if: ${{ matrix.container == 'singularity' }} @@ -197,9 +201,9 @@ jobs: run: sudo rm -f /usr/bin/docker ; sudo apt-get install -y podman - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: 3.12 + python-version: 3.13 cache: pip - name: "Test CWL ${{ matrix.cwl-version }} conformance" @@ -210,13 +214,13 @@ jobs: CWLTOOL_OPTIONS: ${{ matrix.cwl-version == 'v1.2' && '--relax-path-checks' || '' }} ${{ matrix.extras }} run: ./conformance-test.sh - name: Archive test results - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: cwl-${{ matrix.cwl-version }}-${{ matrix.container }}${{ matrix.extras }}-conformance-results path: | **/cwltool_conf*.xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: @@ -226,17 +230,17 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - - name: Set up Singularity + - name: Set up Singularity and environment-modules run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: 3.12 + python-version: 3.13 cache: pip cache-dependency-path: | requirements.txt @@ -258,7 +262,7 @@ jobs: build_test_container: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: record cwltool version @@ -268,17 +272,17 @@ jobs: macos: name: Test on macos-latest - runs-on: macos-13 # not latest, that is now an Apple Silicon M1, for which seqtk is not yet built on bioconda + runs-on: macos-latest env: - TOXENV: py312-unit + TOXENV: py313-unit steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: 3.12 + python-version: 3.13 cache: pip cache-dependency-path: | requirements.txt @@ -301,7 +305,7 @@ jobs: - name: Test with tox run: tox - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3b84a185b..d4e0a37c2 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,13 +23,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: python - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/quay-publish.yml b/.github/workflows/quay-publish.yml index 16652ef44..4278a927b 100644 --- a/.github/workflows/quay-publish.yml +++ b/.github/workflows/quay-publish.yml @@ -8,7 +8,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Get image tags diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 000000000..2b0c42ccd --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,128 @@ +name: Python package build and publish + +on: + release: + types: [published] + workflow_dispatch: {} + repository_dispatch: {} + push: + branches: + - main + +concurrency: + group: wheels-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build_wheels: + name: ${{ matrix.image }} wheels + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - image: manylinux_2_28_x86_64 + build: "*manylinux*" + - image: musllinux_1_2_x86_64 + build: "*musllinux*" + + steps: + - uses: actions/checkout@v5 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v5 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + # - name: Set up QEMU + # if: runner.os == 'Linux' + # uses: docker/setup-qemu-action@v2 + # with: + # platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v3.2 + env: + CIBW_BUILD: ${{ matrix.build }} + CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} + CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} + # configure cibuildwheel to build native 64-bit archs ('auto64'), and some + # emulated ones + # Linux arm64 wheels are built on circleci + CIBW_ARCHS_LINUX: auto64 # ppc64le s390x + + - uses: actions/upload-artifact@v5 + with: + name: artifact-${{ matrix.image }} + path: ./wheelhouse/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v5 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v5 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + - name: Build sdist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v5 + with: + name: artifact-source + path: dist/*.tar.gz + + build_wheels_macos: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + # macos-13 is an intel runner, macos-14 is apple silicon + os: [macos-13, macos-14] + steps: + - uses: actions/checkout@v5 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v5 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + - name: Build wheels + uses: pypa/cibuildwheel@v3.2 + + - uses: actions/upload-artifact@v5 + with: + name: artifact-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + + # upload_pypi: + # needs: [build_wheels, build_sdist] + # runs-on: ubuntu-24.04 + # environment: deploy + # permissions: + # id-token: write + # if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'repository_dispatch' && github.event.client_payload.publish_wheel == true) + # steps: + # - uses: actions/download-artifact@v4 + # with: + # # unpacks default artifact into dist/ + # pattern: artifact-* + # merge-multiple: true + # path: dist + + # - name: Publish package distributions to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # skip-existing: true diff --git a/.gitignore b/.gitignore index fbe4b24fc..b4cab0e66 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ eggs/ *.egg .tox/ .pytest_cache +*.so # Editor Temps .*.sw? diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd9e9105d..bee0e6947 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ Style guide: - PEP-8 (as implemented by the `black` code formatting tool) -- Python 3.8+ compatible code +- Python 3.10+ compatible code - PEP-484 type hints The development is done using `git`, we encourage you to get familiar with it. diff --git a/MANIFEST.in b/MANIFEST.in index 187d19bea..83b8c3fa5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -11,6 +11,7 @@ include tests/loop-ext/* include tests/tmp1/tmp2/tmp3/.gitkeep include tests/tmp4/alpha/* include tests/wf/* +include tests/wf/inp-filelist.txt include tests/wf/operation/* include tests/override/* include tests/reloc/*.cwl @@ -19,6 +20,7 @@ include tests/reloc/dir2/* include tests/checker_wf/* include tests/subgraph/* include tests/input_deps/* +recursive-include tests/test_deps_env include tests/trs/* include tests/wf/generator/* include cwltool/py.typed diff --git a/Makefile b/Makefile index 1b08f4290..28d9b8219 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. -PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py) setup.py +PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py tests/cwl-conformance/*.py) setup.py DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv auto-walrus \ isort wheel autoflake pyupgrade bandit -rlint-requirements.txt\ -rtest-requirements.txt -rmypy-requirements.txt -rdocs/requirements.txt @@ -160,11 +160,11 @@ diff-cover.html: coverage.xml ## test : run the cwltool test suite test: $(PYSOURCES) - python3 -m pytest -rsfE ${PYTEST_EXTRA} + python3 -m pytest ${PYTEST_EXTRA} ## testcov : run the cwltool test suite and collect coverage testcov: $(PYSOURCES) - python3 -m pytest -rsfE --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA} + python3 -m pytest --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA} sloccount.sc: $(PYSOURCES) Makefile sloccount --duplicates --wide --details $^ > $@ @@ -183,14 +183,14 @@ mypy: $(PYSOURCES) mypyc: $(PYSOURCES) MYPYPATH=mypy-stubs CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \ - && pytest -rsfE -vv ${PYTEST_EXTRA} + && pytest -vv ${PYTEST_EXTRA} shellcheck: FORCE shellcheck build-cwltool-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \ cwltool-in-docker.sh pyupgrade: $(PYSOURCES) - pyupgrade --exit-zero-even-if-changed --py38-plus $^ + pyupgrade --exit-zero-even-if-changed --py310-plus $^ auto-walrus $^ release-test: FORCE diff --git a/README.rst b/README.rst index db40d0420..9dc97d62a 100644 --- a/README.rst +++ b/README.rst @@ -52,7 +52,7 @@ and provide comprehensive validation of CWL files as well as provide other tools related to working with CWL. ``cwltool`` is written and tested for -`Python `_ ``3.x {x = 6, 8, 9, 10, 11}`` +`Python `_ ``3.x {x = 9, 10, 11, 12, 13}`` The reference implementation consists of two packages. The ``cwltool`` package is the primary Python module containing the reference implementation in the @@ -189,6 +189,22 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``:: $ cwl-runner --tmp-outdir-prefix=/Users/username/project --tmpdir-prefix=/Users/username/project wc-tool.cwl wc-job.json + +Docker default platform on macOS with Apple Silicon +=================================================== + +If macOS users want to run CWL tools/workflows using ``cwltool`` with Docker and their software containers only have support for amd64 (64-bit x86) CPUs, but they have an Apple Silicon (aarch64/arm64) CPU, +they run into the error: + + WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested. + +To fix this, export the following environment variable before executing `cwltool`: + +``export DOCKER_DEFAULT_PLATFORM=linux/amd64`` + +To automatically have this variable set in the future, add the same command to ones respective shell profile (e.g. ``~/.zshrc``, ``~/.bash_profile``). + + Using uDocker ============= @@ -826,7 +842,7 @@ executor :: executor(tool, job_order_object, runtimeContext, logger) - (Process, Dict[Text, Any], RuntimeContext) -> Tuple[Dict[Text, Any], Text] + (Process, Dict[str, Any], RuntimeContext) -> Tuple[Dict[str, Any], str] An implementation of the top-level workflow execution loop should synchronously run a process object to completion and return the @@ -836,7 +852,7 @@ versionfunc :: () - () -> Text + () -> str Return version string. @@ -863,7 +879,7 @@ resolver :: resolver(document_loader, document) - (Loader, Union[Text, dict[Text, Any]]) -> Text + (Loader, str | dict[str, Any]) -> str Resolve a relative document identifier to an absolute one that can be fetched. @@ -874,7 +890,7 @@ construct_tool_object :: construct_tool_object(toolpath_object, loadingContext) - (MutableMapping[Text, Any], LoadingContext) -> Process + (MutableMapping[str, Any], LoadingContext) -> Process Hook to construct a Process object (eg CommandLineTool) object from a document. @@ -882,7 +898,7 @@ select_resources :: selectResources(request) - (Dict[str, int], RuntimeContext) -> Dict[Text, int] + (Dict[str, int], RuntimeContext) -> Dict[str, int] Take a resource request and turn it into a concrete resource assignment. @@ -890,7 +906,7 @@ make_fs_access :: make_fs_access(basedir) - (Text) -> StdFsAccess + (str) -> StdFsAccess Return a file system access object. @@ -908,6 +924,6 @@ Workflow.make_workflow_step :: make_workflow_step(toolpath_object, pos, loadingContext, parentworkflowProv) - (Dict[Text, Any], int, LoadingContext, Optional[ProvenanceProfile]) -> WorkflowStep + (Dict[str, Any], int, LoadingContext, Optional[ProvenanceProfile]) -> WorkflowStep Create and return a workflow step object. diff --git a/build-cwltool-docker.sh b/build-cwltool-docker.sh index a70fdf4df..3dfa7bb66 100755 --- a/build-cwltool-docker.sh +++ b/build-cwltool-docker.sh @@ -8,4 +8,4 @@ ${engine} run -t -v /var/run/docker.sock:/var/run/docker.sock \ -v /tmp:/tmp \ -v "$PWD":/tmp/cwltool \ quay.io/commonwl/cwltool_module /bin/sh -c \ - "apk add gcc bash git && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset or test_no_compute_chcksum or test_packed_workflow_execution[tests/wf/count-lines1-wf.cwl-tests/wf/wc-job.json-False] or test_sequential_workflow or test_single_process_subwf_subwf_inline_step)' --ignore-glob '*test_udocker.py' -n 0 -v -rs --pyargs cwltool" + "apk add gcc bash git && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset or test_no_compute_chcksum or test_packed_workflow_execution[tests/wf/count-lines1-wf.cwl-tests/wf/wc-job.json-False] or test_sequential_workflow or test_single_process_subwf_subwf_inline_step or test_cache_dockerreq_hint_instead_of_req or test_workflow_job_step_name_callback)' --ignore-glob '*test_udocker.py' -n 0 -v -rs --pyargs cwltool" diff --git a/cibw-requirements.txt b/cibw-requirements.txt new file mode 100644 index 000000000..6d9de094f --- /dev/null +++ b/cibw-requirements.txt @@ -0,0 +1 @@ +cibuildwheel>=3.1 diff --git a/conformance-test.sh b/conformance-test.sh index 36ea23b17..6df14a63c 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -97,19 +97,7 @@ if [[ "$VERSION" = *dev* ]] then CWLTOOL_OPTIONS+=" --enable-dev" fi -if [[ "$CONTAINER" = "singularity" ]]; then - CWLTOOL_OPTIONS+=" --singularity" - # This test fails because Singularity and Docker have - # different views on how to deal with this. - exclusions+=(docker_entrypoint) - if [[ "${VERSION}" = "v1.1" ]]; then - # This fails because of a difference (in Singularity vs Docker) in - # the way filehandles are passed to processes in the container and - # wc can tell somehow. - # See issue #1440 - exclusions+=(stdin_shorcut) - fi -elif [[ "$CONTAINER" = "podman" ]]; then +if [[ "$CONTAINER" = "podman" ]]; then CWLTOOL_OPTIONS+=" --podman" fi diff --git a/cwltool/argparser.py b/cwltool/argparser.py index efced5386..08001c4e7 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -3,19 +3,11 @@ import argparse import os import urllib -from typing import ( - Any, - Callable, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sequence, - Type, - Union, - cast, -) +from collections.abc import Callable, MutableSequence, Sequence +from typing import Any, cast + +import rich.markup +from rich_argparse import HelpPreviewAction, RichHelpFormatter from .loghandler import _logger from .process import Process, shortname @@ -25,9 +17,11 @@ def arg_parser() -> argparse.ArgumentParser: + RichHelpFormatter.group_name_formatter = str parser = argparse.ArgumentParser( + formatter_class=RichHelpFormatter, description="Reference executor for Common Workflow Language standards. " - "Not for production use." + "Not for production use.", ) parser.add_argument("--basedir", type=str) parser.add_argument( @@ -37,23 +31,15 @@ def arg_parser() -> argparse.ArgumentParser: help="Output directory. The default is the current directory.", ) - parser.add_argument( - "--log-dir", - type=str, - default="", - help="Log your tools stdout/stderr to this location outside of container " - "This will only log stdout/stderr if you specify stdout/stderr in their " - "respective fields or capture it as an output", - ) - parser.add_argument( "--parallel", action="store_true", default=False, help="Run jobs in parallel. ", ) - envgroup = parser.add_mutually_exclusive_group() - envgroup.add_argument( + envgroup = parser.add_argument_group(title="Control environment variables") + env_exclusive = envgroup.add_mutually_exclusive_group() + env_exclusive.add_argument( "--preserve-environment", type=str, action="append", @@ -64,7 +50,7 @@ def arg_parser() -> argparse.ArgumentParser: default=[], dest="preserve_environment", ) - envgroup.add_argument( + env_exclusive.add_argument( "--preserve-entire-environment", action="store_true", help="Preserve all environment variables when running CommandLineTools " @@ -73,54 +59,10 @@ def arg_parser() -> argparse.ArgumentParser: dest="preserve_entire_environment", ) - containergroup = parser.add_mutually_exclusive_group() - containergroup.add_argument( - "--rm-container", - action="store_true", - default=True, - help="Delete Docker container used by jobs after they exit (default)", - dest="rm_container", - ) - - containergroup.add_argument( - "--leave-container", - action="store_false", - default=True, - help="Do not delete Docker container used by jobs after they exit", - dest="rm_container", - ) - - cidgroup = parser.add_argument_group( - "Options for recording the Docker container identifier into a file." - ) - cidgroup.add_argument( - # Disabled as containerid is now saved by default - "--record-container-id", - action="store_true", - default=False, - help=argparse.SUPPRESS, - dest="record_container_id", - ) - - cidgroup.add_argument( - "--cidfile-dir", - type=str, - help="Store the Docker container ID into a file in the specified directory.", - default=None, - dest="cidfile_dir", - ) - - cidgroup.add_argument( - "--cidfile-prefix", - type=str, - help="Specify a prefix to the container ID filename. " - "Final file name will be followed by a timestamp. " - "The default is no prefix.", - default=None, - dest="cidfile_prefix", + files_group = parser.add_argument_group( + title="Manage intermediate, temporary, or final output files" ) - - parser.add_argument( + files_group.add_argument( "--tmpdir-prefix", type=str, help="Path prefix for temporary directories. If --tmpdir-prefix is not " @@ -130,7 +72,7 @@ def arg_parser() -> argparse.ArgumentParser: default=DEFAULT_TMP_PREFIX, ) - intgroup = parser.add_mutually_exclusive_group() + intgroup = files_group.add_mutually_exclusive_group() intgroup.add_argument( "--tmp-outdir-prefix", type=str, @@ -148,7 +90,7 @@ def arg_parser() -> argparse.ArgumentParser: "troubleshooting of CWL documents.", ) - tmpgroup = parser.add_mutually_exclusive_group() + tmpgroup = files_group.add_mutually_exclusive_group() tmpgroup.add_argument( "--rm-tmpdir", action="store_true", @@ -165,7 +107,7 @@ def arg_parser() -> argparse.ArgumentParser: dest="rm_tmpdir", ) - outgroup = parser.add_mutually_exclusive_group() + outgroup = files_group.add_mutually_exclusive_group() outgroup.add_argument( "--move-outputs", action="store_const", @@ -195,30 +137,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="move_outputs", ) - pullgroup = parser.add_mutually_exclusive_group() - pullgroup.add_argument( - "--enable-pull", - default=True, - action="store_true", - help="Try to pull Docker images", - dest="pull_image", - ) - - pullgroup.add_argument( - "--disable-pull", - default=True, - action="store_false", - help="Do not try to pull Docker images", - dest="pull_image", - ) - - parser.add_argument( - "--rdf-serializer", - help="Output RDF serialization format used by --print-rdf (one of " - "turtle (default), n3, nt, xml)", - default="turtle", - ) - parser.add_argument( "--eval-timeout", help="Time to wait for a Javascript expression to evaluate before giving " @@ -227,9 +145,7 @@ def arg_parser() -> argparse.ArgumentParser: default=60, ) - provgroup = parser.add_argument_group( - "Options for recording provenance information of the execution" - ) + provgroup = parser.add_argument_group("Recording provenance information of the execution") provgroup.add_argument( "--provenance", help="Save provenance to specified folder as a " @@ -287,7 +203,8 @@ def arg_parser() -> argparse.ArgumentParser: type=str, ) - printgroup = parser.add_mutually_exclusive_group() + non_exec_group = parser.add_argument_group(title="Non-execution options") + printgroup = non_exec_group.add_mutually_exclusive_group() printgroup.add_argument( "--print-rdf", action="store_true", @@ -335,6 +252,15 @@ def arg_parser() -> argparse.ArgumentParser: printgroup.add_argument( "--make-template", action="store_true", help="Generate a template input object" ) + non_exec_group.add_argument( + "--rdf-serializer", + help="Output RDF serialization format used by --print-rdf (one of " + "turtle (default), n3, nt, xml)", + default="turtle", + ) + non_exec_group.add_argument( + "--tool-help", action="store_true", help="Print command line help for tool" + ) strictgroup = parser.add_mutually_exclusive_group() strictgroup.add_argument( @@ -376,11 +302,27 @@ def arg_parser() -> argparse.ArgumentParser: dest="doc_cache", ) - volumegroup = parser.add_mutually_exclusive_group() - volumegroup.add_argument("--verbose", action="store_true", help="Default logging") - volumegroup.add_argument("--no-warnings", action="store_true", help="Only print errors.") - volumegroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.") - volumegroup.add_argument("--debug", action="store_true", help="Print even more logging") + volumegroup = parser.add_argument_group(title="Configure logging") + volume_exclusive = volumegroup.add_mutually_exclusive_group() + volume_exclusive.add_argument("--verbose", action="store_true", help="Default logging") + volume_exclusive.add_argument("--no-warnings", action="store_true", help="Only print errors.") + volume_exclusive.add_argument( + "--quiet", action="store_true", help="Only print warnings and errors." + ) + volume_exclusive.add_argument("--debug", action="store_true", help="Print even more logging") + volumegroup.add_argument( + "--log-dir", + type=str, + default="", + help="Log your tools stdout/stderr to this location outside of container " + "This will only log stdout/stderr if you specify stdout/stderr in their " + "respective fields or capture it as an output", + ) + volumegroup.add_argument( + "--timestamps", + action="store_true", + help="Add timestamps to the errors, warnings, and notifications.", + ) parser.add_argument( "--write-summary", @@ -391,30 +333,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="write_summary", ) - parser.add_argument( - "--strict-memory-limit", - action="store_true", - help="When running with " - "software containers and the Docker engine, pass either the " - "calculated memory allocation from ResourceRequirements or the " - "default of 1 gigabyte to Docker's --memory option.", - ) - - parser.add_argument( - "--strict-cpu-limit", - action="store_true", - help="When running with " - "software containers and the Docker engine, pass either the " - "calculated cpu allocation from ResourceRequirements or the " - "default of 1 core to Docker's --cpu option. " - "Requires docker version >= v1.13.", - ) - - parser.add_argument( - "--timestamps", - action="store_true", - help="Add timestamps to the errors, warnings, and notifications.", - ) parser.add_argument( "--js-console", action="store_true", help="Enable javascript console output" ) @@ -429,7 +347,103 @@ def arg_parser() -> argparse.ArgumentParser: help="File of options to pass to jshint. " 'This includes the added option "includewarnings". ', ) - dockergroup = parser.add_mutually_exclusive_group() + container_group = parser.add_argument_group( + title="Software container engine selection and configuration" + ) + pullgroup = container_group.add_mutually_exclusive_group() + pullgroup.add_argument( + "--enable-pull", + default=True, + action="store_true", + help="Try to pull Docker images", + dest="pull_image", + ) + + pullgroup.add_argument( + "--disable-pull", + default=True, + action="store_false", + help="Do not try to pull Docker images", + dest="pull_image", + ) + container_group.add_argument( + "--force-docker-pull", + action="store_true", + default=False, + help="Pull latest software container image even if it is locally present", + dest="force_docker_pull", + ) + container_group.add_argument( + "--no-read-only", + action="store_true", + default=False, + help="Do not set root directory in the container as read-only", + dest="no_read_only", + ) + + container_group.add_argument( + "--default-container", + help="Specify a default software container to use for any " + "CommandLineTool without a DockerRequirement.", + ) + container_group.add_argument( + "--no-match-user", + action="store_true", + help="Disable passing the current uid to `docker run --user`", + ) + container_group.add_argument( + "--custom-net", + type=str, + help="Passed to `docker run` as the `--net` parameter when " + "NetworkAccess is true, which is its default setting.", + ) + + container_cleanup = container_group.add_mutually_exclusive_group() + container_cleanup.add_argument( + "--rm-container", + action="store_true", + default=True, + help="Delete Docker container used by jobs after they exit (default)", + dest="rm_container", + ) + + container_cleanup.add_argument( + "--leave-container", + action="store_false", + default=True, + help="Do not delete Docker container used by jobs after they exit", + dest="rm_container", + ) + + cidgroup = parser.add_argument_group("Recording the software container identifier into a file") + cidgroup.add_argument( + # Disabled as containerid is now saved by default + "--record-container-id", + action="store_true", + default=False, + help=argparse.SUPPRESS, + dest="record_container_id", + ) + + cidgroup.add_argument( + "--cidfile-dir", + type=str, + help="Store the software container ID into a file in the specified directory.", + default=None, + dest="cidfile_dir", + ) + + cidgroup.add_argument( + "--cidfile-prefix", + type=str, + help="Specify a prefix to the software container ID filename. " + "Final file name will be followed by a timestamp. " + "The default is no prefix.", + default=None, + dest="cidfile_prefix", + ) + + dockergroup = container_group.add_mutually_exclusive_group() dockergroup.add_argument( "--user-space-docker-cmd", metavar="CMD", @@ -469,6 +483,24 @@ def arg_parser() -> argparse.ArgumentParser: "is specified under `hints`.", dest="use_container", ) + container_group.add_argument( + "--strict-memory-limit", + action="store_true", + help="When running with " + "software containers and the Docker engine, pass either the " + "calculated memory allocation from ResourceRequirements or the " + "default of 1 gigabyte to Docker's --memory option.", + ) + + container_group.add_argument( + "--strict-cpu-limit", + action="store_true", + help="When running with " + "software containers and the Docker engine, pass either the " + "calculated cpu allocation from ResourceRequirements or the " + "default of 1 core to Docker's --cpu option. " + "Requires docker version >= v1.13.", + ) dependency_resolvers_configuration_help = argparse.SUPPRESS dependencies_directory_help = argparse.SUPPRESS @@ -478,7 +510,7 @@ def arg_parser() -> argparse.ArgumentParser: if SOFTWARE_REQUIREMENTS_ENABLED: dependency_resolvers_configuration_help = ( "Dependency resolver " - "configuration file describing how to adapt 'SoftwareRequirement' " + "configuration file describing how to adapt `SoftwareRequirement` " "packages to current system." ) dependencies_directory_help = ( @@ -487,7 +519,7 @@ def arg_parser() -> argparse.ArgumentParser: use_biocontainers_help = ( "Use biocontainers for tools without an " "explicitly annotated Docker container." ) - conda_dependencies = "Short cut to use Conda to resolve 'SoftwareRequirement' packages." + conda_dependencies = "Short cut to use Conda to resolve `SoftwareRequirement` packages." parser.add_argument( "--beta-dependency-resolvers-configuration", @@ -510,8 +542,6 @@ def arg_parser() -> argparse.ArgumentParser: action="store_true", ) - parser.add_argument("--tool-help", action="store_true", help="Print command line help for tool") - parser.add_argument( "--relative-deps", choices=["primary", "cwd"], @@ -530,7 +560,7 @@ def arg_parser() -> argparse.ArgumentParser: parser.add_argument( "--enable-ext", action="store_true", - help="Enable loading and running 'cwltool:' extensions to the CWL standards.", + help="Enable loading and running `cwltool:` extensions to the CWL standards.", default=False, ) @@ -548,22 +578,6 @@ def arg_parser() -> argparse.ArgumentParser: help="Disable colored logging (default false)", ) - parser.add_argument( - "--default-container", - help="Specify a default software container to use for any " - "CommandLineTool without a DockerRequirement.", - ) - parser.add_argument( - "--no-match-user", - action="store_true", - help="Disable passing the current uid to `docker run --user`", - ) - parser.add_argument( - "--custom-net", - type=str, - help="Passed to `docker run` as the '--net' parameter when " - "NetworkAccess is true, which is its default setting.", - ) parser.add_argument( "--disable-validate", dest="do_validate", @@ -606,9 +620,9 @@ def arg_parser() -> argparse.ArgumentParser: parser.add_argument( "--on-error", - help="Desired workflow behavior when a step fails. One of 'stop' (do " - "not submit any more steps) or 'continue' (may submit other steps that " - "are not downstream from the error). Default is 'stop'.", + help="Desired workflow behavior when a step fails. One of `stop` (do " + "not submit any more steps) or `continue` (may submit other steps that " + "are not downstream from the error). Default is `stop`.", default="stop", choices=("stop", "continue"), ) @@ -636,21 +650,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="relax_path_checks", ) - parser.add_argument( - "--force-docker-pull", - action="store_true", - default=False, - help="Pull latest software container image even if it is locally present", - dest="force_docker_pull", - ) - parser.add_argument( - "--no-read-only", - action="store_true", - default=False, - help="Do not set root directory in the container as read-only", - dest="no_read_only", - ) - parser.add_argument( "--overrides", type=str, @@ -658,7 +657,8 @@ def arg_parser() -> argparse.ArgumentParser: help="Read process requirement overrides from file.", ) - subgroup = parser.add_mutually_exclusive_group() + target_group = parser.add_argument_group(title="Target selection (optional)") + subgroup = target_group.add_mutually_exclusive_group() subgroup.add_argument( "--target", "-t", @@ -679,8 +679,8 @@ def arg_parser() -> argparse.ArgumentParser: default=None, help="Only executes the underlying Process (CommandLineTool, " "ExpressionTool, or sub-Workflow) for the given step in a workflow. " - "This will not include any step-level processing: 'scatter', 'when'; " - "and there will be no processing of step-level 'default', or 'valueFrom' " + "This will not include any step-level processing: `scatter`, `when`; " + "and there will be no processing of step-level `default`, or `valueFrom` " "input modifiers. However, requirements/hints from the step or parent " "workflow(s) will be inherited as usual." "The input object must match that Process's inputs.", @@ -714,11 +714,15 @@ def arg_parser() -> argparse.ArgumentParser: "formatted description of the required input values for the given " "`cwl_document`.", ) - + parser.add_argument( + "--generate-help-preview", + action=HelpPreviewAction, + path="help-preview.svg", # (optional) or "help-preview.html" or "help-preview.txt" + ) return parser -def get_default_args() -> Dict[str, Any]: +def get_default_args() -> dict[str, Any]: """Get default values of cwltool's command line options.""" ap = arg_parser() args = ap.parse_args([]) @@ -728,11 +732,11 @@ def get_default_args() -> Dict[str, Any]: class FSAction(argparse.Action): """Base action for our custom actions.""" - objclass: Optional[str] = None + objclass: str | None = None def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, @@ -750,8 +754,8 @@ def __call__( self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[str] = None, + values: str | Sequence[Any] | None, + option_string: str | None = None, ) -> None: setattr( namespace, @@ -766,11 +770,11 @@ def __call__( class FSAppendAction(argparse.Action): """Appending version of the base action for our custom actions.""" - objclass: Optional[str] = None + objclass: str | None = None def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, @@ -788,8 +792,8 @@ def __call__( self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[str] = None, + values: str | Sequence[Any] | None, + option_string: str | None = None, ) -> None: g = getattr(namespace, self.dest) if not g: @@ -804,19 +808,19 @@ def __call__( class FileAction(FSAction): - objclass: Optional[str] = "File" + objclass: str | None = "File" class DirectoryAction(FSAction): - objclass: Optional[str] = "Directory" + objclass: str | None = "Directory" class FileAppendAction(FSAppendAction): - objclass: Optional[str] = "File" + objclass: str | None = "File" class DirectoryAppendAction(FSAppendAction): - objclass: Optional[str] = "Directory" + objclass: str | None = "Directory" class AppendAction(argparse.Action): @@ -827,7 +831,7 @@ class AppendAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, **kwargs: Any, @@ -840,8 +844,8 @@ def __call__( self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[str] = None, + values: str | Sequence[Any] | None, + option_string: str | None = None, ) -> None: g = getattr(namespace, self.dest, None) if g is None: @@ -859,13 +863,14 @@ def add_argument( toolparser: argparse.ArgumentParser, name: str, inptype: Any, - records: List[str], + records: list[str], description: str = "", default: Any = None, input_required: bool = True, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, base_uri: str = "", ) -> None: + description = rich.markup.escape(description) if len(name) == 1: flag = "-" else: @@ -888,59 +893,55 @@ def add_argument( return None ahelp = description.replace("%", "%%") - action: Optional[Union[Type[argparse.Action], str]] = None - atype: Optional[Any] = None - typekw: Dict[str, Any] = {} - - if inptype == "File": - action = FileAction - elif inptype == "Directory": - action = DirectoryAction - elif isinstance(inptype, MutableMapping) and inptype["type"] == "array": - if inptype["items"] == "File": + action: type[argparse.Action] | str | None = None + atype: Any | None = None + typekw: dict[str, Any] = {} + + match inptype: + case "File": + action = FileAction + case "Directory": + action = DirectoryAction + case {"type": "array", "items": "File"}: action = FileAppendAction - elif inptype["items"] == "Directory": + case {"type": "array", "items": "Directory"}: action = DirectoryAppendAction - else: + case {"type": "array", "items": str(items)}: action = AppendAction - items = inptype["items"] - if items == "int" or items == "long": - atype = int - elif items == "double" or items == "float": - atype = float - elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum": - atype = str - elif isinstance(inptype, MutableMapping) and inptype["type"] == "record": - records.append(name) - for field in inptype["fields"]: - fieldname = name + "." + shortname(field["name"]) - fieldtype = field["type"] - fielddescription = field.get("doc", "") - add_argument( - toolparser, - fieldname, - fieldtype, - records, - fielddescription, - default=default.get(shortname(field["name"]), None) if default else None, - input_required=required, - ) - return - elif inptype == "string": - atype = str - elif inptype == "int": - atype = int - elif inptype == "long": - atype = int - elif inptype == "double": - atype = float - elif inptype == "float": - atype = float - elif inptype == "boolean": - action = "store_true" - else: - _logger.debug("Can't make command line argument from %s", inptype) - return None + match items: + case "int" | "long": + atype = int + case "double" | "float": + atype = float + case {"type": "enum"}: + atype = str + case {"type": "record", "fields": list(fields)}: + records.append(name) + for field in fields: + fieldname = name + "." + shortname(field["name"]) + fieldtype = field["type"] + fielddescription = field.get("doc", "") + add_argument( + toolparser, + fieldname, + fieldtype, + records, + fielddescription, + default=default.get(shortname(field["name"]), None) if default else None, + input_required=required, + ) + return + case "string": + atype = str + case "int" | "long": + atype = int + case "double" | "float": + atype = float + case "boolean": + action = "store_true" + case _: + _logger.debug("Can't make command line argument from %s", inptype) + return None if action in (FileAction, DirectoryAction, FileAppendAction, DirectoryAppendAction): typekw["urljoin"] = urljoin @@ -962,8 +963,8 @@ def add_argument( def generate_parser( toolparser: argparse.ArgumentParser, tool: Process, - namemap: Dict[str, str], - records: List[str], + namemap: dict[str, str], + records: list[str], input_required: bool = True, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, base_uri: str = "", @@ -991,4 +992,10 @@ def generate_parser( base_uri, ) + toolparser.add_argument( + "--generate-help-preview", + action=HelpPreviewAction, + path="help-preview.svg", # (optional) or "help-preview.html" or "help-preview.txt" + ) + return toolparser diff --git a/cwltool/builder.py b/cwltool/builder.py index 2ba1e6543..cf6d6d5ac 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -3,21 +3,9 @@ import copy import logging import math +from collections.abc import Callable, MutableMapping, MutableSequence from decimal import Decimal -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Type, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, Optional, cast from cwl_utils import expression from cwl_utils.file_formats import check_format @@ -35,7 +23,6 @@ from .errors import WorkflowException from .loghandler import _logger from .mutation import MutationManager -from .software_requirements import DependenciesConfiguration from .stdfsaccess import StdFsAccess from .utils import ( CONTENT_LIMIT, @@ -54,8 +41,9 @@ ProvenanceProfile, # pylint: disable=unused-import ) from .pathmapper import PathMapper + from .software_requirements import DependenciesConfiguration -INPUT_OBJ_VOCAB: Dict[str, str] = { +INPUT_OBJ_VOCAB: dict[str, str] = { "Any": "https://w3id.org/cwl/salad#Any", "File": "https://w3id.org/cwl/cwl#File", "Directory": "https://w3id.org/cwl/cwl#Directory", @@ -107,18 +95,18 @@ class Builder(HasReqsHints): def __init__( self, job: CWLObjectType, - files: List[CWLObjectType], - bindings: List[CWLObjectType], + files: list[CWLObjectType], + bindings: list[CWLObjectType], schemaDefs: MutableMapping[str, CWLObjectType], names: Names, - requirements: List[CWLObjectType], - hints: List[CWLObjectType], - resources: Dict[str, Union[int, float]], - mutation_manager: Optional[MutationManager], - formatgraph: Optional[Graph], - make_fs_access: Type[StdFsAccess], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], + resources: dict[str, int | float], + mutation_manager: MutationManager | None, + formatgraph: Graph | None, + make_fs_access: type[StdFsAccess], fs_access: StdFsAccess, - job_script_provider: Optional[DependenciesConfiguration], + job_script_provider: Optional["DependenciesConfiguration"], timeout: float, debug: bool, js_console: bool, @@ -169,22 +157,27 @@ def __init__( self.pathmapper: Optional["PathMapper"] = None self.prov_obj: Optional["ProvenanceProfile"] = None - self.find_default_container: Optional[Callable[[], str]] = None + self.find_default_container: Callable[[], str] | None = None self.container_engine = container_engine - def build_job_script(self, commands: List[str]) -> Optional[str]: + def build_job_script(self, commands: list[str]) -> str | None: + """Use the job_script_provider to turn the commands into a job script.""" if self.job_script_provider is not None: return self.job_script_provider.build_job_script(self, commands) return None + def _capture_files(self, f: CWLObjectType) -> CWLObjectType: + self.files.append(f) + return f + def bind_input( self, schema: CWLObjectType, - datum: Union[CWLObjectType, List[CWLObjectType]], + datum: CWLObjectType | list[CWLObjectType], discover_secondaryFiles: bool, - lead_pos: Optional[Union[int, List[int]]] = None, - tail_pos: Optional[Union[str, List[int]]] = None, - ) -> List[MutableMapping[str, Union[str, List[int]]]]: + lead_pos: int | list[int] | None = None, + tail_pos: str | list[int] | None = None, + ) -> list[MutableMapping[str, str | list[int]]]: """ Bind an input object to the command line. @@ -200,8 +193,8 @@ def bind_input( if lead_pos is None: lead_pos = [] - bindings: List[MutableMapping[str, Union[str, List[int]]]] = [] - binding: Union[MutableMapping[str, Union[str, List[int]]], CommentedMap] = {} + bindings: list[MutableMapping[str, str | list[int]]] = [] + binding: MutableMapping[str, str | list[int]] | CommentedMap = {} value_from_expression = False if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping): binding = CommentedMap(schema["inputBinding"].items()) @@ -237,7 +230,7 @@ def bind_input( if isinstance(schema["type"], MutableSequence): bound_input = False for t in schema["type"]: - avsc: Optional[Schema] = None + avsc: Schema | None = None if isinstance(t, str) and self.names.has_name(t, None): avsc = self.names.get_name(t, None) elif ( @@ -324,7 +317,7 @@ def bind_input( if schema["type"] == "record": datum = cast(CWLObjectType, datum) - for f in cast(List[CWLObjectType], schema["fields"]): + for f in cast(list[CWLObjectType], schema["fields"]): name = cast(str, f["name"]) if name in datum and datum[name] is not None: bindings.extend( @@ -371,9 +364,9 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: datum = cast(CWLObjectType, datum) self.files.append(datum) - loadContents_sourceline: Union[ - None, MutableMapping[str, Union[str, List[int]]], CWLObjectType - ] = None + loadContents_sourceline: ( + None | MutableMapping[str, str | list[int]] | CWLObjectType + ) = None if binding and binding.get("loadContents"): loadContents_sourceline = binding elif schema.get("loadContents"): @@ -513,7 +506,7 @@ def addsf( if "format" in schema: eval_format: Any = self.do_eval(schema["format"]) if isinstance(eval_format, str): - evaluated_format: Union[str, List[str]] = eval_format + evaluated_format: str | list[str] = eval_format elif isinstance(eval_format, MutableSequence): for index, entry in enumerate(eval_format): message = None @@ -541,7 +534,7 @@ def addsf( raise SourceLine( schema["format"], index, WorkflowException, debug ).makeError(message) - evaluated_format = cast(List[str], eval_format) + evaluated_format = cast(list[str], eval_format) else: raise SourceLine(schema, "format", WorkflowException, debug).makeError( "An expression in the 'format' field must " @@ -566,7 +559,7 @@ def addsf( visit_class( datum.get("secondaryFiles", []), ("File", "Directory"), - _capture_files, + self._capture_files, ) if schema["type"] == "org.w3id.cwl.cwl.Directory": @@ -581,44 +574,43 @@ def addsf( self.files.append(datum) if schema["type"] == "Any": - visit_class(datum, ("File", "Directory"), _capture_files) + visit_class(datum, ("File", "Directory"), self._capture_files) # Position to front of the sort key if binding: for bi in bindings: - bi["position"] = cast(List[int], binding["position"]) + cast( - List[int], bi["position"] + bi["position"] = cast(list[int], binding["position"]) + cast( + list[int], bi["position"] ) bindings.append(binding) return bindings - def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str: + def tostr(self, value: MutableMapping[str, str] | Any) -> str: """ Represent an input parameter as a string. :raises WorkflowException: if the item is a File or Directory and the "path" is missing. """ - if isinstance(value, MutableMapping) and value.get("class") in ( - "File", - "Directory", - ): - if "path" not in value: - raise WorkflowException( - '{} object missing "path": {}'.format(value["class"], value) - ) - return value["path"] - elif isinstance(value, ScalarFloat): - rep = RoundTripRepresenter() - dec_value = Decimal(rep.represent_scalar_float(value).value) - if "E" in str(dec_value): - return str(dec_value.quantize(1)) - return str(dec_value) - else: - return str(value) - - def generate_arg(self, binding: CWLObjectType) -> List[str]: + match value: + case {"class": "File" | "Directory" as class_name, **rest}: + if "path" not in rest: + raise WorkflowException( + '{} object missing "path": {}'.format(class_name, value) + ) + return str(rest["path"]) + case ScalarFloat(): + rep = RoundTripRepresenter() + dec_value = Decimal(rep.represent_scalar_float(value).value) + if "E" in str(dec_value): + return str(dec_value.quantize(1)) + return str(dec_value) + case _: + return str(value) + + def generate_arg(self, binding: CWLObjectType) -> list[str]: + """Convert an input binding to a list of command line arguments.""" value = binding.get("datum") debug = _logger.isEnabledFor(logging.DEBUG) if "valueFrom" in binding: @@ -642,30 +634,28 @@ def generate_arg(self, binding: CWLObjectType) -> List[str]: raise WorkflowException("'separate' option can not be specified without prefix") argl: MutableSequence[CWLOutputType] = [] - if isinstance(value, MutableSequence): - if binding.get("itemSeparator") and value: - itemSeparator = cast(str, binding["itemSeparator"]) - argl = [itemSeparator.join([self.tostr(v) for v in value])] - elif binding.get("valueFrom"): - value = [self.tostr(v) for v in value] - return cast(List[str], ([prefix] if prefix else [])) + cast(List[str], value) - elif prefix and value: + match value: + case MutableSequence(): + if binding.get("itemSeparator") and value: + itemSeparator = cast(str, binding["itemSeparator"]) + argl = [itemSeparator.join([self.tostr(v) for v in value])] + elif binding.get("valueFrom"): + value = [self.tostr(v) for v in value] + return cast(list[str], ([prefix] if prefix else [])) + cast(list[str], value) + elif prefix and value: + return [prefix] + else: + return [] + case {"class": "File" | "Directory"}: + argl = [cast(CWLOutputType, value)] + case MutableMapping(): + return [prefix] if prefix else [] + case True if prefix: return [prefix] - else: + case bool() | None: return [] - elif isinstance(value, MutableMapping) and value.get("class") in ( - "File", - "Directory", - ): - argl = cast(MutableSequence[CWLOutputType], [value]) - elif isinstance(value, MutableMapping): - return [prefix] if prefix else [] - elif value is True and prefix: - return [prefix] - elif value is False or value is None or (value is True and not prefix): - return [] - else: - argl = [value] + case _: + argl = [value] args = [] for j in argl: @@ -678,11 +668,11 @@ def generate_arg(self, binding: CWLObjectType) -> List[str]: def do_eval( self, - ex: Optional[CWLOutputType], - context: Optional[Any] = None, + ex: CWLOutputType | None, + context: Any | None = None, recursive: bool = False, strip_whitespace: bool = True, - ) -> Optional[CWLOutputType]: + ) -> CWLOutputType | None: if recursive: if isinstance(ex, MutableMapping): return {k: self.do_eval(v, context, recursive) for k, v in ex.items()} diff --git a/cwltool/checker.py b/cwltool/checker.py index 676245698..d81642e1d 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -1,19 +1,7 @@ """Static checking of CWL workflow connectivity.""" -from collections import namedtuple -from typing import ( - Any, - Dict, - Iterator, - List, - Literal, - MutableMapping, - MutableSequence, - Optional, - Sized, - Union, - cast, -) +from collections.abc import Iterator, MutableMapping, MutableSequence, Sized +from typing import Any, Literal, NamedTuple, Optional, Union, cast from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno @@ -25,8 +13,7 @@ from .utils import CWLObjectType, CWLOutputType, SinkType, aslist -def _get_type(tp): - # type: (Any) -> Any +def _get_type(tp: Any) -> Any: if isinstance(tp, MutableMapping): if tp.get("type") not in ("array", "record", "enum"): return tp["type"] @@ -36,9 +23,9 @@ def _get_type(tp): def check_types( srctype: SinkType, sinktype: SinkType, - linkMerge: Optional[str], - valueFrom: Optional[str], -) -> Union[Literal["pass"], Literal["warning"], Literal["exception"]]: + linkMerge: str | None, + valueFrom: str | None, +) -> Literal["pass"] | Literal["warning"] | Literal["exception"]: """ Check if the source and sink types are correct. @@ -46,34 +33,40 @@ def check_types( """ if valueFrom is not None: return "pass" - if linkMerge is None: - if can_assign_src_to_sink(srctype, sinktype, strict=True): - return "pass" - if can_assign_src_to_sink(srctype, sinktype, strict=False): - return "warning" - return "exception" - if linkMerge == "merge_nested": - return check_types( - {"items": _get_type(srctype), "type": "array"}, - _get_type(sinktype), - None, - None, - ) - if linkMerge == "merge_flattened": - return check_types(merge_flatten_type(_get_type(srctype)), _get_type(sinktype), None, None) - raise WorkflowException(f"Unrecognized linkMerge enum {linkMerge!r}") + match linkMerge: + case None: + if can_assign_src_to_sink(srctype, sinktype, strict=True): + return "pass" + if can_assign_src_to_sink(srctype, sinktype, strict=False): + return "warning" + return "exception" + case "merge_nested": + return check_types( + {"items": _get_type(srctype), "type": "array"}, + _get_type(sinktype), + None, + None, + ) + case "merge_flattened": + return check_types( + merge_flatten_type(_get_type(srctype)), _get_type(sinktype), None, None + ) + case _: + raise WorkflowException(f"Unrecognized linkMerge enum {linkMerge!r}") def merge_flatten_type(src: SinkType) -> CWLOutputType: """Return the merge flattened type of the source type.""" - if isinstance(src, MutableSequence): - return [merge_flatten_type(cast(SinkType, t)) for t in src] - if isinstance(src, MutableMapping) and src.get("type") == "array": - return src - return {"items": src, "type": "array"} + match src: + case MutableSequence(): + return [merge_flatten_type(t) for t in src] + case {"type": "array"}: + return src + case _: + return {"items": src, "type": "array"} -def can_assign_src_to_sink(src: SinkType, sink: Optional[SinkType], strict: bool = False) -> bool: +def can_assign_src_to_sink(src: SinkType, sink: SinkType | None, strict: bool = False) -> bool: """ Check for identical type specifications, ignoring extra keys like inputBinding. @@ -98,31 +91,29 @@ def can_assign_src_to_sink(src: SinkType, sink: Optional[SinkType], strict: bool if src["type"] == "record" and sink["type"] == "record": return _compare_records(src, sink, strict) if src["type"] == "File" and sink["type"] == "File": - for sinksf in cast(List[CWLObjectType], sink.get("secondaryFiles", [])): + for sinksf in cast(list[CWLObjectType], sink.get("secondaryFiles", [])): if not [ 1 - for srcsf in cast(List[CWLObjectType], src.get("secondaryFiles", [])) + for srcsf in cast(list[CWLObjectType], src.get("secondaryFiles", [])) if sinksf == srcsf ]: if strict: return False return True - return can_assign_src_to_sink( - cast(SinkType, src["type"]), cast(Optional[SinkType], sink["type"]), strict - ) + return can_assign_src_to_sink(src["type"], sink["type"], strict) if isinstance(src, MutableSequence): if strict: for this_src in src: - if not can_assign_src_to_sink(cast(SinkType, this_src), sink): + if not can_assign_src_to_sink(this_src, sink): return False return True for this_src in src: - if this_src != "null" and can_assign_src_to_sink(cast(SinkType, this_src), sink): + if this_src != "null" and can_assign_src_to_sink(this_src, sink): return True return False if isinstance(sink, MutableSequence): for this_sink in sink: - if can_assign_src_to_sink(src, cast(SinkType, this_sink)): + if can_assign_src_to_sink(src, this_sink): return True return False return bool(src == sink) @@ -167,7 +158,8 @@ def _rec_fields(rec: MutableMapping[str, Any]) -> MutableMapping[str, Any]: return True -def missing_subset(fullset: List[Any], subset: List[Any]) -> List[Any]: +def missing_subset(fullset: list[Any], subset: list[Any]) -> list[Any]: + """Calculate the items missing from the fullset given the subset.""" missing = [] for i in subset: if i not in fullset: @@ -176,11 +168,11 @@ def missing_subset(fullset: List[Any], subset: List[Any]) -> List[Any]: def static_checker( - workflow_inputs: List[CWLObjectType], + workflow_inputs: list[CWLObjectType], workflow_outputs: MutableSequence[CWLObjectType], step_inputs: MutableSequence[CWLObjectType], - step_outputs: List[CWLObjectType], - param_to_step: Dict[str, CWLObjectType], + step_outputs: list[CWLObjectType], + param_to_step: dict[str, CWLObjectType], ) -> None: """ Check if all source and sink types of a workflow are compatible before run time. @@ -191,12 +183,12 @@ def static_checker( # sink parameters: step_inputs and workflow_outputs # make a dictionary of source parameters, indexed by the "id" field - src_dict: Dict[str, CWLObjectType] = {} + src_dict: dict[str, CWLObjectType] = {} for param in workflow_inputs + step_outputs: src_dict[cast(str, param["id"])] = param - step_inputs_val = check_all_types(src_dict, step_inputs, "source", param_to_step) - workflow_outputs_val = check_all_types( + step_inputs_val = _check_all_types(src_dict, step_inputs, "source", param_to_step) + workflow_outputs_val = _check_all_types( src_dict, workflow_outputs, "outputSource", param_to_step ) @@ -210,27 +202,34 @@ def static_checker( sink = warning.sink linkMerge = warning.linkMerge sinksf = sorted( - p["pattern"] for p in sink.get("secondaryFiles", []) if p.get("required", True) + cast(str, p["pattern"]) + for p in cast(MutableSequence[CWLObjectType], sink.get("secondaryFiles", [])) + if p.get("required", True) + ) + srcsf = sorted( + cast(str, p["pattern"]) + for p in cast(MutableSequence[CWLObjectType], src.get("secondaryFiles", [])) ) - srcsf = sorted(p["pattern"] for p in src.get("secondaryFiles", [])) # Every secondaryFile required by the sink, should be declared # by the source missing = missing_subset(srcsf, sinksf) + src_name = shortname(cast(str, src["id"])) + sink_id = cast(str, sink["id"]) + sink_name = shortname(sink_id) if missing: msg1 = "Parameter '{}' requires secondaryFiles {} but".format( - shortname(sink["id"]), + sink_name, missing, ) msg3 = SourceLine(src, "id").makeError( - "source '%s' does not provide those secondaryFiles." % (shortname(src["id"])) + "source '%s' does not provide those secondaryFiles." % (src_name) ) msg4 = SourceLine(src.get("_tool_entry", src), "secondaryFiles").makeError( "To resolve, add missing secondaryFiles patterns to definition of '%s' or" - % (shortname(src["id"])) + % (src_name) ) msg5 = SourceLine(sink.get("_tool_entry", sink), "secondaryFiles").makeError( - "mark missing secondaryFiles in definition of '%s' as optional." - % shortname(sink["id"]) + "mark missing secondaryFiles in definition of '%s' as optional." % (sink_name) ) msg = SourceLine(sink).makeError( "{}\n{}".format(msg1, bullets([msg3, msg4, msg5], " ")) @@ -240,13 +239,13 @@ def static_checker( msg = SourceLine(sink, "type").makeError( "'%s' is not an input parameter of %s, expected %s" % ( - shortname(sink["id"]), - param_to_step[sink["id"]]["run"], + sink_name, + param_to_step[sink_id]["run"], ", ".join( shortname(cast(str, s["id"])) for s in cast( - List[Dict[str, Union[str, bool]]], - param_to_step[sink["id"]]["inputs"], + list[dict[str, Union[str, bool]]], + param_to_step[sink_id]["inputs"], ) if not s.get("not_connected") ), @@ -258,12 +257,11 @@ def static_checker( msg = ( SourceLine(src, "type").makeError( "Source '%s' of type %s may be incompatible" - % (shortname(src["id"]), json_dumps(src["type"])) + % (src_name, json_dumps(src["type"])) ) + "\n" + SourceLine(sink, "type").makeError( - " with sink '%s' of type %s" - % (shortname(sink["id"]), json_dumps(sink["type"])) + " with sink '{}' of type {}".format(sink_name, json_dumps(sink["type"])) ) ) if linkMerge is not None: @@ -285,12 +283,12 @@ def static_checker( msg = ( SourceLine(src, "type").makeError( "Source '%s' of type %s is incompatible" - % (shortname(src["id"]), json_dumps(src["type"])) + % (shortname(cast(str, src["id"])), json_dumps(src["type"])) ) + "\n" + SourceLine(sink, "type").makeError( " with sink '{}' of type {}".format( - shortname(sink["id"]), json_dumps(sink["type"]) + shortname(cast(str, sink["id"])), json_dumps(sink["type"]) ) ) ) @@ -302,16 +300,17 @@ def static_checker( exception_msgs.append(msg) for sink in step_inputs: + sink_type = cast(Union[str, list[str], list[CWLObjectType], CWLObjectType], sink["type"]) if ( - "null" != sink["type"] - and "null" not in sink["type"] + "null" != sink_type + and "null" not in sink_type and "source" not in sink and "default" not in sink and "valueFrom" not in sink ): msg = SourceLine(sink).makeError( "Required parameter '%s' does not have source, default, or valueFrom expression" - % shortname(sink["id"]) + % shortname(cast(str, sink["id"])) ) exception_msgs.append(msg) @@ -324,15 +323,21 @@ def static_checker( raise ValidationException(all_exception_msg) -SrcSink = namedtuple("SrcSink", ["src", "sink", "linkMerge", "message"]) +class _SrcSink(NamedTuple): + """An error or warning message about a connection between two points of the workflow graph.""" + src: CWLObjectType + sink: CWLObjectType + linkMerge: str | None + message: str | None -def check_all_types( - src_dict: Dict[str, CWLObjectType], + +def _check_all_types( + src_dict: dict[str, CWLObjectType], sinks: MutableSequence[CWLObjectType], - sourceField: Union[Literal["source"], Literal["outputSource"]], - param_to_step: Dict[str, CWLObjectType], -) -> Dict[str, List[SrcSink]]: + sourceField: Literal["source"] | Literal["outputSource"], + param_to_step: dict[str, CWLObjectType], +) -> dict[str, list[_SrcSink]]: """ Given a list of sinks, check if their types match with the types of their sources. @@ -340,7 +345,7 @@ def check_all_types( (from :py:func:`check_types`) :raises ValidationException: if a sourceField is missing """ - validation = {"warning": [], "exception": []} # type: Dict[str, List[SrcSink]] + validation: dict[str, list[_SrcSink]] = {"warning": [], "exception": []} for sink in sinks: if sourceField in sink: valueFrom = cast(Optional[str], sink.get("valueFrom")) @@ -351,23 +356,23 @@ def check_all_types( extra_message = "pickValue is: %s" % pickValue if isinstance(sink[sourceField], MutableSequence): - linkMerge = cast( + linkMerge: str | None = cast( Optional[str], sink.get( "linkMerge", ("merge_nested" if len(cast(Sized, sink[sourceField])) > 1 else None), ), - ) # type: Optional[str] + ) if pickValue in ["first_non_null", "the_only_non_null"]: linkMerge = None - srcs_of_sink = [] # type: List[CWLObjectType] + srcs_of_sink: list[CWLObjectType] = [] for parm_id in cast(MutableSequence[str], sink[sourceField]): srcs_of_sink += [src_dict[parm_id]] if is_conditional_step(param_to_step, parm_id) and pickValue is None: validation["warning"].append( - SrcSink( + _SrcSink( src_dict[parm_id], sink, linkMerge, @@ -391,7 +396,7 @@ def check_all_types( if pickValue is not None: validation["warning"].append( - SrcSink( + _SrcSink( src_dict[parm_id], sink, linkMerge, @@ -404,13 +409,13 @@ def check_all_types( snk_typ = sink["type"] if "null" not in src_typ: - src_typ = ["null"] + cast(List[Any], src_typ) + src_typ = ["null"] + cast(list[Any], src_typ) if "null" not in cast( - Union[List[str], CWLObjectType], snk_typ + Union[list[str], CWLObjectType], snk_typ ): # Given our type names this works even if not a list validation["warning"].append( - SrcSink( + _SrcSink( src_dict[parm_id], sink, linkMerge, @@ -430,17 +435,17 @@ def check_all_types( check_result = check_types(src, sink, linkMerge, valueFrom) if check_result == "warning": validation["warning"].append( - SrcSink(src, sink, linkMerge, message=extra_message) + _SrcSink(src, sink, linkMerge, message=extra_message) ) elif check_result == "exception": validation["exception"].append( - SrcSink(src, sink, linkMerge, message=extra_message) + _SrcSink(src, sink, linkMerge, message=extra_message) ) return validation -def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: +def circular_dependency_checker(step_inputs: list[CWLObjectType]) -> None: """ Check if a workflow has circular dependency. @@ -448,8 +453,8 @@ def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: """ adjacency = get_dependency_tree(step_inputs) vertices = adjacency.keys() - processed: List[str] = [] - cycles: List[List[str]] = [] + processed: list[str] = [] + cycles: list[list[str]] = [] for vertex in vertices: if vertex not in processed: traversal_path = [vertex] @@ -461,7 +466,7 @@ def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: raise ValidationException(exception_msg) -def get_dependency_tree(step_inputs: List[CWLObjectType]) -> Dict[str, List[str]]: +def get_dependency_tree(step_inputs: list[CWLObjectType]) -> dict[str, list[str]]: """Get the dependency tree in the form of adjacency list.""" adjacency = {} # adjacency list of the dependency tree for step_input in step_inputs: @@ -482,10 +487,10 @@ def get_dependency_tree(step_inputs: List[CWLObjectType]) -> Dict[str, List[str] def processDFS( - adjacency: Dict[str, List[str]], - traversal_path: List[str], - processed: List[str], - cycles: List[List[str]], + adjacency: dict[str, list[str]], + traversal_path: list[str], + processed: list[str], + cycles: list[list[str]], ) -> None: """Perform depth first search.""" tip = traversal_path[-1] @@ -509,16 +514,17 @@ def get_step_id(field_id: str) -> str: return step_id -def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: +def is_conditional_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: + """Return True if the step given by the parm_id is a conditional step.""" if (source_step := param_to_step.get(parm_id)) is not None: if source_step.get("when") is not None: return True return False -def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: +def is_all_output_method_loop_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: """Check if a step contains a `loop` directive with `all_iterations` outputMethod.""" - source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id) + source_step: MutableMapping[str, Any] | None = param_to_step.get(parm_id) if source_step is not None: if ( source_step.get("loop") is not None diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 7a4e8ff71..54913b47f 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -7,31 +7,23 @@ import logging import os import re +import shlex import shutil import threading import urllib import urllib.parse -from enum import Enum -from functools import cmp_to_key, partial -from typing import ( - TYPE_CHECKING, - Any, - Dict, +from collections.abc import ( Generator, - List, + Iterable, Mapping, MutableMapping, MutableSequence, - Optional, - Pattern, - Set, - TextIO, - Type, - Union, - cast, ) +from enum import Enum +from functools import cmp_to_key, partial +from re import Pattern +from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast -import shellescape from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import Schema @@ -162,11 +154,11 @@ def __init__( self, builder: Builder, script: str, - output_callback: Optional[OutputCallbackType], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], - outdir: Optional[str] = None, - tmpdir: Optional[str] = None, + output_callback: OutputCallbackType | None, + requirements: list[CWLObjectType], + hints: list[CWLObjectType], + outdir: str | None = None, + tmpdir: str | None = None, ) -> None: """Initialize this ExpressionJob.""" self.builder = builder @@ -181,7 +173,7 @@ def __init__( def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: try: normalizeFilesDirs(self.builder.job) @@ -215,7 +207,7 @@ class ExpressionTool(Process): def job( self, job_order: CWLObjectType, - output_callbacks: Optional[OutputCallbackType], + output_callbacks: OutputCallbackType | None, runtimeContext: RuntimeContext, ) -> Generator[ExpressionJob, None, None]: builder = self._init_job(job_order, runtimeContext) @@ -235,18 +227,19 @@ class AbstractOperation(Process): def job( self, job_order: CWLObjectType, - output_callbacks: Optional[OutputCallbackType], + output_callbacks: OutputCallbackType | None, runtimeContext: RuntimeContext, ) -> JobsGeneratorType: raise WorkflowException("Abstract operation cannot be executed.") -def remove_path(f): # type: (CWLObjectType) -> None +def remove_path(f: CWLObjectType) -> None: + """Remove any 'path' property, if present.""" if "path" in f: del f["path"] -def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> Optional[CWLObjectType]: +def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> CWLObjectType | None: """ Remap a file from internal path to external path. @@ -289,7 +282,7 @@ def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> Optional[CWL ) revmap_f = builder.pathmapper.reversemap(path) - if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"): + if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"): # type: ignore[union-attr] f["location"] = revmap_f[1] elif ( uripath == outdir @@ -325,7 +318,7 @@ class CallbackJob: def __init__( self, job: "CommandLineTool", - output_callback: Optional[OutputCallbackType], + output_callback: OutputCallbackType | None, cachebuilder: Builder, jobcache: str, ) -> None: @@ -334,12 +327,12 @@ def __init__( self.output_callback = output_callback self.cachebuilder = cachebuilder self.outdir = jobcache - self.prov_obj = None # type: Optional[ProvenanceProfile] + self.prov_obj: ProvenanceProfile | None = None def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: if self.output_callback: self.output_callback( @@ -392,7 +385,7 @@ def check_valid_locations(fs_access: StdFsAccess, ob: CWLObjectType) -> None: raise ValidationException("Does not exist or is not a Directory: '%s'" % location) -OutputPortsType = Dict[str, Optional[CWLOutputType]] +OutputPortsType = dict[str, Optional[CWLOutputType]] class ParameterOutputWorkflowException(WorkflowException): @@ -411,13 +404,14 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext """Initialize this CommandLineTool.""" super().__init__(toolpath_object, loadingContext) self.prov_obj = loadingContext.prov_obj - self.path_check_mode = ( + self.path_check_mode: PathCheckingMode = ( PathCheckingMode.RELAXED if loadingContext.relax_path_checks else PathCheckingMode.STRICT - ) # type: PathCheckingMode + ) - def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: + def make_job_runner(self, runtimeContext: RuntimeContext) -> type[JobBase]: + """Return the correct CommandLineJob class given the container settings.""" dockerReq, dockerRequired = self.get_requirement("DockerRequirement") mpiReq, mpiRequired = self.get_requirement(MPIRequirementName) @@ -443,28 +437,6 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: return SingularityCommandLineJob elif runtimeContext.user_space_docker_cmd: return UDockerCommandLineJob - if mpiReq is not None: - if mpiRequired: - if dockerRequired: - raise UnsupportedRequirement( - "No support for Docker and MPIRequirement both being required" - ) - else: - _logger.warning( - "MPI has been required while Docker is hinted, discarding Docker hint(s)" - ) - self.hints = [h for h in self.hints if h["class"] != "DockerRequirement"] - return CommandLineJob - else: - if dockerRequired: - _logger.warning( - "Docker has been required while MPI is hinted, discarding MPI hint(s)" - ) - self.hints = [h for h in self.hints if h["class"] != MPIRequirementName] - else: - raise UnsupportedRequirement( - "Both Docker and MPI have been hinted - don't know what to do" - ) if runtimeContext.podman: return PodmanCommandLineJob return DockerCommandLineJob @@ -477,7 +449,7 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: @staticmethod def make_path_mapper( - reffiles: List[CWLObjectType], + reffiles: list[CWLObjectType], stagedir: str, runtimeContext: RuntimeContext, separateDirs: bool, @@ -499,12 +471,19 @@ def updatePathmap(self, outdir: str, pathmap: PathMapper, fn: CWLObjectType) -> ("Writable" if fn.get("writable") else "") + cast(str, fn["class"]), False, ) - for sf in cast(List[CWLObjectType], fn.get("secondaryFiles", [])): + for sf in cast(list[CWLObjectType], fn.get("secondaryFiles", [])): self.updatePathmap(outdir, pathmap, sf) - for ls in cast(List[CWLObjectType], fn.get("listing", [])): + for ls in cast(list[CWLObjectType], fn.get("listing", [])): self.updatePathmap(os.path.join(outdir, cast(str, fn["basename"])), pathmap, ls) - def _initialworkdir(self, j: JobBase, builder: Builder) -> None: + def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: + """ + Test and initialize the working directory. + + :param j: A :py:class:`~cwltool.job.CommandLineJob` or a + specialized container-based job. + If 'None', then only tests will be performed, no setup. + """ initialWorkdir, _ = self.get_requirement("InitialWorkDirRequirement") if initialWorkdir is None: return @@ -517,7 +496,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: cwl_version ) < ORDERED_VERSIONS.index("v1.1.0-dev1") - ls = [] # type: List[CWLObjectType] + ls: list[CWLObjectType] = [] if isinstance(initialWorkdir["listing"], str): # "listing" is just a string (must be an expression) so # just evaluate it and use the result as if it was in @@ -528,7 +507,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: if not isinstance(ls_evaluated, MutableSequence): fail = ls_evaluated else: - ls_evaluated2 = cast(MutableSequence[Union[None, CWLOutputType]], ls_evaluated) + ls_evaluated2 = ls_evaluated for entry in ls_evaluated2: if entry == None: # noqa if classic_dirent: @@ -587,7 +566,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError( message ) - ls = cast(List[CWLObjectType], ls_evaluated) + ls = cast(list[CWLObjectType], ls_evaluated) else: # "listing" is an array of either expressions or Dirent so # evaluate each item @@ -634,14 +613,20 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: for e in entry: ec = cast(CWLObjectType, e) ec["writable"] = t.get("writable", False) - ls.extend(cast(List[CWLObjectType], entry)) + ls.extend(cast(list[CWLObjectType], entry)) continue - et = {} # type: CWLObjectType + et: CWLObjectType = {} + writable = t.get("writable", False) + et["writable"] = writable if isinstance(entry, Mapping) and entry.get("class") in ( "File", "Directory", ): + if writable and "secondaryFiles" in entry: + secFiles = cast(MutableSequence[CWLObjectType], entry["secondaryFiles"]) + for sf in secFiles: + sf["writable"] = writable et["entry"] = cast(CWLOutputType, entry) else: if isinstance(entry, str): @@ -677,7 +662,6 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: et["entryname"] = entryname_field else: et["entryname"] = None - et["writable"] = t.get("writable", False) ls.append(et) else: # Expression, must return a Dirent, File, Directory @@ -686,7 +670,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: if not initwd_item: continue if isinstance(initwd_item, MutableSequence): - ls.extend(cast(List[CWLObjectType], initwd_item)) + ls.extend(cast(list[CWLObjectType], initwd_item)) else: ls.append(cast(CWLObjectType, initwd_item)) @@ -727,13 +711,14 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: ) if t2.get("entryname") or t2.get("writable"): - t2 = copy.deepcopy(t2) - t2entry = cast(CWLObjectType, t2["entry"]) + t2copy = copy.deepcopy(t2) + t2entry = cast(CWLObjectType, t2copy["entry"]) if t2.get("entryname"): - t2entry["basename"] = t2["entryname"] - t2entry["writable"] = t2.get("writable") + t2entry["basename"] = t2copy["entryname"] + t2entry["writable"] = t2copy.get("writable") + t2["entry"] = t2entry - ls[i] = cast(CWLObjectType, t2["entry"]) + ls[i] = t2["entry"] for i, t3 in enumerate(ls): if t3.get("class") not in ("File", "Directory"): @@ -772,19 +757,29 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: "is in 'requirements'." ) + if j is None: + return # Only testing + with SourceLine(initialWorkdir, "listing", WorkflowException, debug): j.generatefiles["listing"] = ls for entry in ls: if "basename" in entry: basename = cast(str, entry["basename"]) - entry["dirname"] = os.path.join(builder.outdir, os.path.dirname(basename)) + dirname = os.path.join(builder.outdir, os.path.dirname(basename)) + entry["dirname"] = dirname entry["basename"] = os.path.basename(basename) + if "secondaryFiles" in entry: + for sec_file in cast( + MutableSequence[CWLObjectType], entry["secondaryFiles"] + ): + sec_file["dirname"] = dirname normalizeFilesDirs(entry) self.updatePathmap( cast(Optional[str], entry.get("dirname")) or builder.outdir, cast(PathMapper, builder.pathmapper), entry, ) + if "listing" in entry: def remove_dirname(d: CWLObjectType) -> None: @@ -808,7 +803,7 @@ def job( job_order: CWLObjectType, output_callbacks: OutputCallbackType, runtimeContext: RuntimeContext, - ) -> Generator[Union[JobBase, CallbackJob], None, None]: + ) -> Generator[JobBase | CallbackJob, None, None]: workReuse, _ = self.get_requirement("WorkReuse") enableReuse = workReuse.get("enableReuse", True) if workReuse else True @@ -819,10 +814,10 @@ def job( cachecontext.tmpdir = "/tmp" # nosec cachecontext.stagedir = "/stage" cachebuilder = self._init_job(job_order, cachecontext) - cachebuilder.pathmapper = PathMapper( + cachebuilder.pathmapper = self.make_path_mapper( cachebuilder.files, - runtimeContext.basedir, cachebuilder.stagedir, + runtimeContext, separateDirs=False, ) _check_adjust = partial(check_adjust, self.path_check_mode.value, cachebuilder) @@ -836,6 +831,7 @@ def job( _check_adjust, ) visit_class([cachebuilder.files, cachebuilder.bindings], ("File"), _checksum) + self._initialworkdir(None, cachebuilder) # test the initial working directory cmdline = flatten(list(map(cachebuilder.generate_arg, cachebuilder.bindings))) docker_req, _ = self.get_requirement("DockerRequirement") @@ -850,15 +846,13 @@ def job( cmdline = ["docker", "run", dockerimg] + cmdline # not really run using docker, just for hashing purposes - keydict = { - "cmdline": cmdline - } # type: Dict[str, Union[MutableSequence[Union[str, int]], CWLObjectType]] + keydict: dict[str, MutableSequence[str | int] | CWLObjectType] = {"cmdline": cmdline} for shortcut in ["stdin", "stdout", "stderr"]: if shortcut in self.tool: keydict[shortcut] = self.tool[shortcut] - def calc_checksum(location: str) -> Optional[str]: + def calc_checksum(location: str) -> str | None: for e in cachebuilder.files: if ( "location" in e @@ -870,8 +864,7 @@ def calc_checksum(location: str) -> Optional[str]: return None def remove_prefix(s: str, prefix: str) -> str: - # replace with str.removeprefix when Python 3.9+ - return s[len(prefix) :] if s.startswith(prefix) else s + return s.removeprefix(prefix) for location, fobj in cachebuilder.pathmapper.items(): if fobj.type == "File": @@ -899,6 +892,14 @@ def remove_prefix(s: str, prefix: str) -> str: if cls in interesting and cls not in keydict: keydict[cls] = r + # If there are environmental variables to preserve, add it to the key + env_var_requirement = cast(dict[str, str], keydict.get("EnvVarRequirement", {})) + env_def = self.get_requirement("EnvVarRequirement")[0] or {} + if runtimeContext.preserve_environment is not None: + env_def.update(JobBase.extract_environment(runtimeContext, env_var_requirement)) + + if env_def: + keydict["EnvVarRequirement"] = env_def keydictstr = json_dumps(keydict, separators=(",", ":"), sort_keys=True) cachekey = hashlib.md5(keydictstr.encode("utf-8")).hexdigest() # nosec @@ -946,7 +947,7 @@ def remove_prefix(s: str, prefix: str) -> str: def update_status_output_callback( output_callbacks: OutputCallbackType, jobcachelock: TextIO, - outputs: Optional[CWLObjectType], + outputs: CWLObjectType | None, processStatus: str, ) -> None: # save status to the lockfile then release the lock @@ -1071,8 +1072,8 @@ def update_status_output_callback( j.inplace_update = cast(bool, inplaceUpdateReq["inplaceUpdate"]) normalizeFilesDirs(j.generatefiles) - readers = {} # type: Dict[str, CWLObjectType] - muts = set() # type: Set[str] + readers: dict[str, CWLObjectType] = {} + muts: set[str] = set() if builder.mutation_manager is not None: @@ -1103,7 +1104,7 @@ def register_reader(f: CWLObjectType) -> None: timelimit, _ = self.get_requirement("ToolTimeLimit") if timelimit is not None: with SourceLine(timelimit, "timelimit", ValidationException, debug): - limit_field = cast(Dict[str, Union[str, int]], timelimit)["timelimit"] + limit_field = cast(dict[str, Union[str, int]], timelimit)["timelimit"] if isinstance(limit_field, str): timelimit_eval = builder.do_eval(limit_field) if timelimit_eval and not isinstance(timelimit_eval, int): @@ -1142,7 +1143,7 @@ def register_reader(f: CWLObjectType) -> None: required_env = {} evr, _ = self.get_requirement("EnvVarRequirement") if evr is not None: - for eindex, t3 in enumerate(cast(List[Dict[str, str]], evr["envDef"])): + for eindex, t3 in enumerate(cast(list[dict[str, str]], evr["envDef"])): env_value_field = t3["envValue"] if "${" in env_value_field or "$(" in env_value_field: env_value_eval = builder.do_eval(env_value_field) @@ -1160,11 +1161,11 @@ def register_reader(f: CWLObjectType) -> None: shellcmd, _ = self.get_requirement("ShellCommandRequirement") if shellcmd is not None: - cmd = [] # type: List[str] + cmd: list[str] = [] for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): - arg = [shellescape.quote(a) for a in aslist(arg)] + arg = [shlex.quote(a) for a in aslist(arg)] cmd.extend(aslist(arg)) j.command_line = ["/bin/sh", "-c", " ".join(cmd)] else: @@ -1201,15 +1202,15 @@ def register_reader(f: CWLObjectType) -> None: def collect_output_ports( self, - ports: Union[CommentedSeq, Set[CWLObjectType]], + ports: CommentedSeq | set[CWLObjectType], builder: Builder, outdir: str, rcode: int, compute_checksum: bool = True, jobname: str = "", - readers: Optional[MutableMapping[str, CWLObjectType]] = None, + readers: MutableMapping[str, CWLObjectType] | None = None, ) -> OutputPortsType: - ret = {} # type: OutputPortsType + ret: OutputPortsType = {} debug = _logger.isEnabledFor(logging.DEBUG) cwl_version = self.metadata.get(ORIGINAL_CWLVERSION, None) if cwl_version != "v1.0": @@ -1283,17 +1284,17 @@ def collect_output( outdir: str, fs_access: StdFsAccess, compute_checksum: bool = True, - ) -> Optional[CWLOutputType]: - r = [] # type: List[CWLOutputType] + ) -> CWLOutputType | None: + r: list[CWLOutputType] = [] empty_and_optional = False debug = _logger.isEnabledFor(logging.DEBUG) - result: Optional[CWLOutputType] = None + result: CWLOutputType | None = None if "outputBinding" in schema: binding = cast( - MutableMapping[str, Union[bool, str, List[str]]], + MutableMapping[str, Union[bool, str, list[str]]], schema["outputBinding"], ) - globpatterns = [] # type: List[str] + globpatterns: list[str] = [] revmap = partial(revmap_file, builder, outdir) @@ -1332,34 +1333,37 @@ def collect_output( key=cmp_to_key(locale.strcoll), ) r.extend( - [ - { - "location": g, - "path": fs_access.join( - builder.outdir, - urllib.parse.unquote(g[len(prefix[0]) + 1 :]), - ), - "basename": decoded_basename, - "nameroot": os.path.splitext(decoded_basename)[0], - "nameext": os.path.splitext(decoded_basename)[1], - "class": "File" if fs_access.isfile(g) else "Directory", - } - for g, decoded_basename in zip( - sorted_glob_result, - map( - lambda x: os.path.basename(urllib.parse.unquote(x)), + cast( + Iterable[CWLOutputType], + [ + { + "location": g, + "path": fs_access.join( + builder.outdir, + urllib.parse.unquote(g[len(prefix[0]) + 1 :]), + ), + "basename": decoded_basename, + "nameroot": os.path.splitext(decoded_basename)[0], + "nameext": os.path.splitext(decoded_basename)[1], + "class": "File" if fs_access.isfile(g) else "Directory", + } + for g, decoded_basename in zip( sorted_glob_result, - ), - ) - ] + map( + lambda x: os.path.basename(urllib.parse.unquote(x)), + sorted_glob_result, + ), + ) + ], + ) ) except OSError as e: - _logger.warning(str(e)) + _logger.warning(str(e), exc_info=builder.debug) except Exception: _logger.error("Unexpected error from fs_access", exc_info=True) raise - for files in cast(List[Dict[str, Optional[CWLOutputType]]], r): + for files in cast(list[dict[str, Optional[CWLOutputType]]], r): rfile = files.copy() revmap(rfile) if files["class"] == "Directory": @@ -1384,13 +1388,14 @@ def collect_output( optional = False single = False - if isinstance(schema["type"], MutableSequence): - if "null" in schema["type"]: - optional = True - if "File" in schema["type"] or "Directory" in schema["type"]: + match schema["type"]: + case MutableSequence(): + if "null" in schema["type"]: + optional = True + if "File" in schema["type"] or "Directory" in schema["type"]: + single = True + case "File" | "Directory": single = True - elif schema["type"] == "File" or schema["type"] == "Directory": - single = True if "outputEval" in binding: with SourceLine(binding, "outputEval", WorkflowException, debug): @@ -1412,7 +1417,7 @@ def collect_output( "Multiple matches for output item that is a single file." ) else: - result = cast(CWLOutputType, result[0]) + result = result[0] if "secondaryFiles" in schema: with SourceLine(schema, "secondaryFiles", WorkflowException, debug): @@ -1515,7 +1520,7 @@ def collect_output( and schema["type"]["type"] == "record" ): out = {} - for field in cast(List[CWLObjectType], schema["type"]["fields"]): + for field in cast(list[CWLObjectType], schema["type"]["fields"]): out[shortname(cast(str, field["name"]))] = self.collect_output( field, builder, outdir, fs_access, compute_checksum=compute_checksum ) diff --git a/cwltool/context.py b/cwltool/context.py index 1e82ecc4a..4e106ac48 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -5,20 +5,8 @@ import shutil import tempfile import threading -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Literal, - Optional, - TextIO, - Tuple, - Union, -) +from collections.abc import Callable, Iterable +from typing import IO, TYPE_CHECKING, Any, Literal, Optional, TextIO, Union from ruamel.yaml.comments import CommentedMap from schema_salad.avro.schema import Names @@ -31,6 +19,7 @@ from .utils import DEFAULT_TMP_PREFIX, CWLObjectType, HasReqsHints, ResolverType if TYPE_CHECKING: + from _typeshed import SupportsWrite from cwl_utils.parser.cwl_v1_2 import LoadingOptions from .builder import Builder @@ -40,12 +29,13 @@ from .process import Process from .secrets import SecretStore from .software_requirements import DependenciesConfiguration + from .workflow_job import WorkflowJobStep class ContextBase: """Shared kwargs based initializer for :py:class:`RuntimeContext` and :py:class:`LoadingContext`.""" - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: dict[str, Any] | None = None) -> None: """Initialize.""" if kwargs: for k, v in kwargs.items(): @@ -64,8 +54,8 @@ def make_tool_notimpl(toolpath_object: CommentedMap, loadingContext: "LoadingCon def log_handler( outdir: str, base_path_logs: str, - stdout_path: Optional[str], - stderr_path: Optional[str], + stdout_path: str | None, + stderr_path: str | None, ) -> None: """Move logs from log location to final output.""" if outdir != base_path_logs: @@ -86,37 +76,37 @@ def set_log_dir(outdir: str, log_dir: str, subdir_name: str) -> str: class LoadingContext(ContextBase): - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: dict[str, Any] | None = None) -> None: """Initialize the LoadingContext from the kwargs.""" self.debug: bool = False self.metadata: CWLObjectType = {} - self.requirements: Optional[List[CWLObjectType]] = None - self.hints: Optional[List[CWLObjectType]] = None - self.overrides_list: List[CWLObjectType] = [] - self.loader: Optional[Loader] = None - self.avsc_names: Optional[Names] = None + self.requirements: list[CWLObjectType] | None = None + self.hints: list[CWLObjectType] | None = None + self.overrides_list: list[CWLObjectType] = [] + self.loader: Loader | None = None + self.avsc_names: Names | None = None self.disable_js_validation: bool = False - self.js_hint_options_file: Optional[str] = None + self.js_hint_options_file: str | None = None self.do_validate: bool = True self.enable_dev: bool = False self.strict: bool = True - self.resolver: Optional[ResolverType] = None - self.fetcher_constructor: Optional[FetcherCallableType] = None + self.resolver: ResolverType | None = None + self.fetcher_constructor: FetcherCallableType | None = None self.construct_tool_object = default_make_tool - self.research_obj: Optional[ResearchObject] = None + self.research_obj: ResearchObject | None = None self.orcid: str = "" self.cwl_full_name: str = "" self.host_provenance: bool = False self.user_provenance: bool = False self.prov_obj: Optional["ProvenanceProfile"] = None - self.do_update: Optional[bool] = None - self.jobdefaults: Optional[CommentedMap] = None + self.do_update: bool | None = None + self.jobdefaults: CommentedMap | None = None self.doc_cache: bool = True self.relax_path_checks: bool = False self.singularity: bool = False self.podman: bool = False self.eval_timeout: float = 60 - self.codegen_idx: Dict[str, Tuple[Any, "LoadingOptions"]] = {} + self.codegen_idx: dict[str, tuple[Any, "LoadingOptions"]] = {} self.fast_parser = False self.skip_resolve_all = False self.skip_schemas = False @@ -129,24 +119,24 @@ def copy(self) -> "LoadingContext": class RuntimeContext(ContextBase): - outdir: Optional[str] = None + outdir: str | None = None tmpdir: str = "" tmpdir_prefix: str = DEFAULT_TMP_PREFIX tmp_outdir_prefix: str = "" stagedir: str = "" - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: dict[str, Any] | None = None) -> None: """Initialize the RuntimeContext from the kwargs.""" select_resources_callable = Callable[ - [Dict[str, Union[int, float]], RuntimeContext], - Dict[str, Union[int, float]], + [dict[str, Union[int, float]], RuntimeContext], + dict[str, Union[int, float]], ] - self.user_space_docker_cmd: Optional[str] = None + self.user_space_docker_cmd: str | None = None self.secret_store: Optional["SecretStore"] = None self.no_read_only: bool = False - self.custom_net: Optional[str] = None + self.custom_net: str | None = None self.no_match_user: bool = False - self.preserve_environment: Optional[Iterable[str]] = None + self.preserve_environment: Iterable[str] | None = None self.preserve_entire_environment: bool = False self.use_container: bool = True self.force_docker_pull: bool = False @@ -154,7 +144,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.rm_tmpdir: bool = True self.pull_image: bool = True self.rm_container: bool = True - self.move_outputs: Union[Literal["move"], Literal["leave"], Literal["copy"]] = "move" + self.move_outputs: Literal["move"] | Literal["leave"] | Literal["copy"] = "move" self.log_dir: str = "" self.set_log_dir = set_log_dir self.log_dir_handler = log_handler @@ -165,9 +155,9 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.debug: bool = False self.compute_checksum: bool = True self.name: str = "" - self.default_container: Optional[str] = "" - self.find_default_container: Optional[Callable[[HasReqsHints], Optional[str]]] = None - self.cachedir: Optional[str] = None + self.default_container: str | None = "" + self.find_default_container: Callable[[HasReqsHints], str | None] | None = None + self.cachedir: str | None = None self.part_of: str = "" self.basedir: str = "" self.toplevel: bool = False @@ -179,27 +169,33 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.docker_tmpdir: str = "" self.docker_stagedir: str = "" self.js_console: bool = False - self.job_script_provider: Optional[DependenciesConfiguration] = None - self.select_resources: Optional[select_resources_callable] = None + self.job_script_provider: DependenciesConfiguration | None = None + self.select_resources: select_resources_callable | None = None self.eval_timeout: float = 60 - self.postScatterEval: Optional[Callable[[CWLObjectType], Optional[CWLObjectType]]] = None - self.on_error: Union[Literal["stop"], Literal["continue"]] = "stop" + self.postScatterEval: Callable[[CWLObjectType], CWLObjectType | None] | None = None + self.on_error: Literal["stop"] | Literal["continue"] = "stop" self.strict_memory_limit: bool = False self.strict_cpu_limit: bool = False - self.cidfile_dir: Optional[str] = None - self.cidfile_prefix: Optional[str] = None + self.cidfile_dir: str | None = None + self.cidfile_prefix: str | None = None - self.workflow_eval_lock: Optional[threading.Condition] = None - self.research_obj: Optional[ResearchObject] = None + self.workflow_eval_lock: Union[threading.Condition, None] = None + self.research_obj: ResearchObject | None = None self.orcid: str = "" self.cwl_full_name: str = "" - self.process_run_id: Optional[str] = None - self.prov_obj: Optional[ProvenanceProfile] = None + self.process_run_id: str | None = None + self.prov_host: bool = False + self.prov_user: bool = False + self.prov_obj: ProvenanceProfile | None = None self.mpi_config: MpiConfig = MpiConfig() - self.default_stdout: Optional[Union[IO[bytes], TextIO]] = None - self.default_stderr: Optional[Union[IO[bytes], TextIO]] = None + self.default_stdout: IO[bytes] | TextIO | None = None + self.default_stderr: IO[bytes] | TextIO | None = None self.validate_only: bool = False - self.validate_stdout: Optional[Union[IO[bytes], TextIO, IO[str]]] = None + self.validate_stdout: Optional["SupportsWrite[str]"] = None + self.workflow_job_step_name_callback: None | ( + Callable[[WorkflowJobStep, CWLObjectType], str] + ) = None + super().__init__(kwargs) if self.tmp_outdir_prefix == "": self.tmp_outdir_prefix = self.tmpdir_prefix diff --git a/cwltool/cuda.py b/cwltool/cuda.py index 719bfd867..86dcb3cdd 100644 --- a/cwltool/cuda.py +++ b/cwltool/cuda.py @@ -2,18 +2,17 @@ import subprocess # nosec import xml.dom.minidom # nosec -from typing import Tuple from .loghandler import _logger from .utils import CWLObjectType -def cuda_version_and_device_count() -> Tuple[str, int]: +def cuda_version_and_device_count() -> tuple[str, int]: """Determine the CUDA version and number of attached CUDA GPUs.""" try: - out = subprocess.check_output(["nvidia-smi", "-q", "-x"]) # nosec + out: str | bytes = subprocess.check_output(["nvidia-smi", "-q", "-x"]) # nosec except Exception as e: - _logger.warning("Error checking CUDA version with nvidia-smi: %s", e) + _logger.debug("Error checking CUDA version with nvidia-smi: %s", e, exc_info=e) return ("", 0) dm = xml.dom.minidom.parseString(out) # nosec @@ -63,5 +62,5 @@ def cuda_check(cuda_req: CWLObjectType, requestCount: int) -> int: return 0 return requestCount except Exception as e: - _logger.warning("Error checking CUDA requirements: %s", e) + _logger.warning("Error checking CUDA requirements: %s", e, exc_info=e) return 0 diff --git a/cwltool/cwlprov/__init__.py b/cwltool/cwlprov/__init__.py index b8ff8d14d..a9ca171e1 100644 --- a/cwltool/cwlprov/__init__.py +++ b/cwltool/cwlprov/__init__.py @@ -5,11 +5,12 @@ import pwd import re import uuid +from collections.abc import Callable from getpass import getuser -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, TypedDict, Union +from typing import IO, Any, Optional, TypedDict, Union -def _whoami() -> Tuple[str, str]: +def _whoami() -> tuple[str, str]: """Return the current operating system account as (username, fullname).""" username = getuser() try: @@ -48,7 +49,7 @@ def _check_mod_11_2(numeric_string: str) -> bool: return nums[-1].upper() == checkdigit -def _valid_orcid(orcid: Optional[str]) -> str: +def _valid_orcid(orcid: str | None) -> str: """ Ensure orcid is a valid ORCID identifier. @@ -106,8 +107,8 @@ def _valid_orcid(orcid: Optional[str]) -> str: { "uri": str, "about": str, - "content": Optional[Union[str, List[str]]], - "oa:motivatedBy": Dict[str, str], + "content": Optional[Union[str, list[str]]], + "oa:motivatedBy": dict[str, str], }, ) @@ -115,27 +116,27 @@ def _valid_orcid(orcid: Optional[str]) -> str: class Aggregate(TypedDict, total=False): """RO Aggregate class.""" - uri: Optional[str] - bundledAs: Optional[Dict[str, Any]] - mediatype: Optional[str] - conformsTo: Optional[Union[str, List[str]]] - createdOn: Optional[str] - createdBy: Optional[Dict[str, str]] + uri: str | None + bundledAs: dict[str, Any] | None + mediatype: str | None + conformsTo: str | list[str] | None + createdOn: str | None + createdBy: dict[str, str] | None # Aggregate.bundledAs is actually type Aggregate, but cyclic definitions are not supported class AuthoredBy(TypedDict, total=False): """RO AuthoredBy class.""" - orcid: Optional[str] - name: Optional[str] - uri: Optional[str] + orcid: str | None + name: str | None + uri: str | None def checksum_copy( src_file: IO[Any], - dst_file: Optional[IO[Any]] = None, - hasher: Optional[Callable[[], "hashlib._Hash"]] = None, + dst_file: IO[Any] | None = None, + hasher: Callable[[], "hashlib._Hash"] | None = None, buffersize: int = 1024 * 1024, ) -> str: """Compute checksums while copying a file.""" diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index ce8d63ad4..c0aa5ec5b 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -3,22 +3,10 @@ import logging import urllib import uuid +from collections.abc import MutableMapping, MutableSequence, Sequence from io import BytesIO from pathlib import PurePath, PurePosixPath -from socket import getfqdn -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sequence, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, cast from prov.identifier import Identifier, QualifiedName from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity @@ -35,12 +23,10 @@ ACCOUNT_UUID, CWLPROV, ENCODING, - FOAF, METADATA, ORE, PROVENANCE, RO, - SCHEMA, SHA1, SHA256, TEXT_PLAIN, @@ -55,7 +41,7 @@ from .ro import ResearchObject -def copy_job_order(job: Union[Process, JobsType], job_order_object: CWLObjectType) -> CWLObjectType: +def copy_job_order(job: Process | JobsType, job_order_object: CWLObjectType) -> CWLObjectType: """Create copy of job object for provenance.""" if not isinstance(job, WorkflowJob): # direct command line tool execution @@ -92,7 +78,7 @@ def __init__( user_provenance: bool, orcid: str, fsaccess: StdFsAccess, - run_uuid: Optional[uuid.UUID] = None, + run_uuid: uuid.UUID | None = None, ) -> None: """Initialize the provenance profile.""" self.fsaccess = fsaccess @@ -117,27 +103,8 @@ def __str__(self) -> str: """Represent this Provenvance profile as a string.""" return f"ProvenanceProfile <{self.workflow_run_uri}> in <{self.research_object}>" - def generate_prov_doc(self) -> Tuple[str, ProvDocument]: + def generate_prov_doc(self) -> tuple[str, ProvDocument]: """Add basic namespaces.""" - - def host_provenance(document: ProvDocument) -> None: - """Record host provenance.""" - document.add_namespace(CWLPROV) - document.add_namespace(UUID) - document.add_namespace(FOAF) - - hostname = getfqdn() - # won't have a foaf:accountServiceHomepage for unix hosts, but - # we can at least provide hostname - document.agent( - ACCOUNT_UUID, - { - PROV_TYPE: FOAF["OnlineAccount"], - "prov:location": hostname, - CWLPROV["hostname"]: hostname, - }, - ) - self.cwltool_version = f"cwltool {versionstring().split()[-1]}" self.document.add_namespace("wfprov", "http://purl.org/wf4ever/wfprov#") # document.add_namespace('prov', 'http://www.w3.org/ns/prov#') @@ -176,25 +143,10 @@ def host_provenance(document: ProvDocument) -> None: # .. but we always know cwltool was launched (directly or indirectly) # by a user account, as cwltool is a command line tool account = self.document.agent(ACCOUNT_UUID) - if self.orcid or self.full_name: - person: Dict[Union[str, Identifier], Any] = { - PROV_TYPE: PROV["Person"], - "prov:type": SCHEMA["Person"], - } - if self.full_name: - person["prov:label"] = self.full_name - person["foaf:name"] = self.full_name - person["schema:name"] = self.full_name - else: - # TODO: Look up name from ORCID API? - pass - agent = self.document.agent(self.orcid or uuid.uuid4().urn, person) - self.document.actedOnBehalfOf(account, agent) - else: - if self.host_provenance: - host_provenance(self.document) - if self.user_provenance: - self.research_object.user_provenance(self.document) + if self.host_provenance: + self.research_object.host_provenance(self.document) + if self.user_provenance or self.orcid or self.full_name: + self.research_object.user_provenance(self.document) # The execution of cwltool wfengine = self.document.agent( self.engine_uuid, @@ -246,8 +198,8 @@ def evaluate( self.used_artefacts(customised_job, self.workflow_run_uri) def record_process_start( - self, process: Process, job: JobsType, process_run_id: Optional[str] = None - ) -> Optional[str]: + self, process: Process, job: JobsType, process_run_id: str | None = None + ) -> str | None: if not hasattr(process, "steps"): process_run_id = self.workflow_run_uri elif not hasattr(job, "workflow"): @@ -263,7 +215,7 @@ def start_process( self, process_name: str, when: datetime.datetime, - process_run_id: Optional[str] = None, + process_run_id: str | None = None, ) -> str: """Record the start of each Process.""" if process_run_id is None: @@ -285,17 +237,18 @@ def record_process_end( self, process_name: str, process_run_id: str, - outputs: Union[CWLObjectType, MutableSequence[CWLObjectType], None], + outputs: CWLObjectType | MutableSequence[CWLObjectType] | None, when: datetime.datetime, ) -> None: self.generate_output_prov(outputs, process_run_id, process_name) self.document.wasEndedBy(process_run_id, None, self.workflow_run_uri, when) - def declare_file(self, value: CWLObjectType) -> Tuple[ProvEntity, ProvEntity, str]: + def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, str]: + """Construct a FileEntity for the given CWL File object.""" if value["class"] != "File": raise ValueError("Must have class:File: %s" % value) # Need to determine file hash aka RO filename - entity: Optional[ProvEntity] = None + entity: ProvEntity | None = None checksum = None if "checksum" in value: csum = cast(str, value["checksum"]) @@ -399,10 +352,10 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: # dir_bundle.identifier, {PROV["type"]: ORE["ResourceMap"], # ORE["describes"]: coll_b.identifier}) - coll_attribs: List[Tuple[Union[str, Identifier], Any]] = [ + coll_attribs: list[tuple[str | Identifier, Any]] = [ (ORE["isDescribedBy"], dir_bundle.identifier) ] - coll_b_attribs: List[Tuple[Union[str, Identifier], Any]] = [] + coll_b_attribs: list[tuple[str | Identifier, Any]] = [] # FIXME: .listing might not be populated yet - hopefully # a later call to this method will sort that @@ -469,7 +422,7 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: self.research_object.add_uri(coll.identifier.uri) return coll - def declare_string(self, value: str) -> Tuple[ProvEntity, str]: + def declare_string(self, value: str) -> tuple[ProvEntity, str]: """Save as string in UTF-8.""" byte_s = BytesIO(str(value).encode(ENCODING)) data_file = self.research_object.add_data_file(byte_s, content_type=TEXT_PLAIN) @@ -483,127 +436,126 @@ def declare_string(self, value: str) -> Tuple[ProvEntity, str]: def declare_artefact(self, value: Any) -> ProvEntity: """Create data artefact entities for all file objects.""" - if value is None: - # FIXME: If this can happen in CWL, we'll - # need a better way to represent this in PROV - return self.document.entity(CWLPROV["None"], {PROV_LABEL: "None"}) - - if isinstance(value, (bool, int, float)): - # Typically used in job documents for flags - - # FIXME: Make consistent hash URIs for these - # that somehow include the type - # (so "1" != 1 != "1.0" != true) - entity = self.document.entity(uuid.uuid4().urn, {PROV_VALUE: value}) - self.research_object.add_uri(entity.identifier.uri) - return entity - - if isinstance(value, str): - (entity, _) = self.declare_string(value) - return entity - - if isinstance(value, bytes): - # If we got here then we must be in Python 3 - byte_s = BytesIO(value) - data_file = self.research_object.add_data_file(byte_s) - # FIXME: Don't naively assume add_data_file uses hash in filename! - data_id = f"data:{PurePosixPath(data_file).stem}" - return self.document.entity( - data_id, - {PROV_TYPE: WFPROV["Artifact"], PROV_VALUE: str(value)}, - ) + if isinstance(value, MutableMapping) and "@id" in value: + # Already processed this value, but it might not be in this PROV + entities = self.document.get_record(value["@id"]) + if entities: + return cast(list[ProvEntity], entities)[0] + # else, unknown in PROV, re-add below as if it's fresh + + match value: + case None: + # FIXME: If this can happen in CWL, we'll + # need a better way to represent this in PROV + return self.document.entity(CWLPROV["None"], {PROV_LABEL: "None"}) + + case bool() | int() | float(): + # Typically used in job documents for flags + + # FIXME: Make consistent hash URIs for these + # that somehow include the type + # (so "1" != 1 != "1.0" != true) + entity = self.document.entity(uuid.uuid4().urn, {PROV_VALUE: value}) + self.research_object.add_uri(entity.identifier.uri) + return entity - if isinstance(value, MutableMapping): - if "@id" in value: - # Already processed this value, but it might not be in this PROV - entities = self.document.get_record(value["@id"]) - if entities: - return cast(List[ProvEntity], entities)[0] - # else, unknown in PROV, re-add below as if it's fresh + case str(val): + return self.declare_string(val)[0] + + case bytes(val): + # If we got here then we must be in Python 3 + byte_s = BytesIO(val) + data_file = self.research_object.add_data_file(byte_s) + # FIXME: Don't naively assume add_data_file uses hash in filename! + data_id = f"data:{PurePosixPath(data_file).stem}" + return self.document.entity( + data_id, + {PROV_TYPE: WFPROV["Artifact"], PROV_VALUE: str(val)}, + ) # Base case - we found a File we need to update - if value.get("class") == "File": - (entity, _, _) = self.declare_file(value) + case {"class": "File"}: + entity = self.declare_file(value)[0] value["@id"] = entity.identifier.uri return entity - - if value.get("class") == "Directory": + case {"class": "Directory"}: entity = self.declare_directory(value) value["@id"] = entity.identifier.uri return entity - coll_id = value.setdefault("@id", uuid.uuid4().urn) - # some other kind of dictionary? - # TODO: also Save as JSON - coll = self.document.entity( - coll_id, - [ - (PROV_TYPE, WFPROV["Artifact"]), - (PROV_TYPE, PROV["Collection"]), - (PROV_TYPE, PROV["Dictionary"]), - ], - ) + case {**rest}: + coll_id = value.setdefault("@id", uuid.uuid4().urn) + # some other kind of dictionary? + # TODO: also Save as JSON + coll = self.document.entity( + coll_id, + [ + (PROV_TYPE, WFPROV["Artifact"]), + (PROV_TYPE, PROV["Collection"]), + (PROV_TYPE, PROV["Dictionary"]), + ], + ) - if value.get("class"): - _logger.warning("Unknown data class %s.", value["class"]) - # FIXME: The class might be "http://example.com/somethingelse" - coll.add_asserted_type(CWLPROV[value["class"]]) - - # Let's iterate and recurse - coll_attribs: List[Tuple[Union[str, Identifier], Any]] = [] - for key, val in value.items(): - v_ent = self.declare_artefact(val) - self.document.membership(coll, v_ent) - m_entity = self.document.entity(uuid.uuid4().urn) - # Note: only support PROV-O style dictionary - # https://www.w3.org/TR/prov-dictionary/#dictionary-ontological-definition - # as prov.py do not easily allow PROV-N extensions - m_entity.add_asserted_type(PROV["KeyEntityPair"]) - m_entity.add_attributes({PROV["pairKey"]: str(key), PROV["pairEntity"]: v_ent}) - coll_attribs.append((PROV["hadDictionaryMember"], m_entity)) - coll.add_attributes(coll_attribs) - self.research_object.add_uri(coll.identifier.uri) - return coll - - # some other kind of Collection? - # TODO: also save as JSON - try: - members = [] - for each_input_obj in iter(value): - # Recurse and register any nested objects - e = self.declare_artefact(each_input_obj) - members.append(e) - - # If we reached this, then we were allowed to iterate - coll = self.document.entity( - uuid.uuid4().urn, - [ - (PROV_TYPE, WFPROV["Artifact"]), - (PROV_TYPE, PROV["Collection"]), - ], - ) - if not members: - coll.add_asserted_type(PROV["EmptyCollection"]) - else: - for member in members: - # FIXME: This won't preserve order, for that - # we would need to use PROV.Dictionary - # with numeric keys - self.document.membership(coll, member) - self.research_object.add_uri(coll.identifier.uri) - # FIXME: list value does not support adding "@id" - return coll - except TypeError: - _logger.warning("Unrecognized type %s of %r", type(value), value) - # Let's just fall back to Python repr() - entity = self.document.entity(uuid.uuid4().urn, {PROV_LABEL: repr(value)}) - self.research_object.add_uri(entity.identifier.uri) - return entity + if rest.get("class"): + _logger.warning("Unknown data class %s.", rest["class"]) + # FIXME: The class might be "http://example.com/somethingelse" + coll.add_asserted_type(CWLPROV[str(rest["class"])]) + + # Let's iterate and recurse + coll_attribs: list[tuple[str | Identifier, Any]] = [] + for key, kval in rest.items(): + v_ent = self.declare_artefact(kval) + self.document.membership(coll, v_ent) + m_entity = self.document.entity(uuid.uuid4().urn) + # Note: only support PROV-O style dictionary + # https://www.w3.org/TR/prov-dictionary/#dictionary-ontological-definition + # as prov.py do not easily allow PROV-N extensions + m_entity.add_asserted_type(PROV["KeyEntityPair"]) + m_entity.add_attributes({PROV["pairKey"]: str(key), PROV["pairEntity"]: v_ent}) + coll_attribs.append((PROV["hadDictionaryMember"], m_entity)) + coll.add_attributes(coll_attribs) + self.research_object.add_uri(coll.identifier.uri) + return coll + + case _: # some other kind of Collection? + # TODO: also save as JSON + try: + members = [] + for each_input_obj in iter(value): + # Recurse and register any nested objects + e = self.declare_artefact(each_input_obj) + members.append(e) + + # If we reached this, then we were allowed to iterate + coll = self.document.entity( + uuid.uuid4().urn, + [ + (PROV_TYPE, WFPROV["Artifact"]), + (PROV_TYPE, PROV["Collection"]), + ], + ) + if not members: + coll.add_asserted_type(PROV["EmptyCollection"]) + else: + for member in members: + # FIXME: This won't preserve order, for that + # we would need to use PROV.Dictionary + # with numeric keys + self.document.membership(coll, member) + self.research_object.add_uri(coll.identifier.uri) + # FIXME: list value does not support adding "@id" + return coll + except TypeError: + _logger.warning("Unrecognized type %s of %r", type(value), value, exc_info=True) + # Let's just fall back to Python repr() + entity = self.document.entity(uuid.uuid4().urn, {PROV_LABEL: repr(value)}) + self.research_object.add_uri(entity.identifier.uri) + return entity def used_artefacts( self, - job_order: Union[CWLObjectType, List[CWLObjectType]], + job_order: CWLObjectType | list[CWLObjectType], process_run_id: str, - name: Optional[str] = None, + name: str | None = None, ) -> None: """Add used() for each data artefact.""" if isinstance(job_order, list): @@ -630,9 +582,9 @@ def used_artefacts( def generate_output_prov( self, - final_output: Union[CWLObjectType, MutableSequence[CWLObjectType], None], - process_run_id: Optional[str], - name: Optional[str], + final_output: CWLObjectType | MutableSequence[CWLObjectType] | None, + process_run_id: str | None, + name: str | None, ) -> None: """Call wasGeneratedBy() for each output,copy the files into the RO.""" if isinstance(final_output, MutableSequence): @@ -704,7 +656,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) """Add http://www.w3.org/TR/prov-aq/ relations to nested PROV files.""" # NOTE: The below will only work if the corresponding metadata/provenance arcp URI # is a pre-registered namespace in the PROV Document - attribs: List[Tuple[Union[str, Identifier], Any]] = [ + attribs: list[tuple[str | Identifier, Any]] = [ (PROV["has_provenance"], prov_id) for prov_id in prov_ids ] self.document.activity(activity, other_attributes=attribs) @@ -713,7 +665,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) uris = [i.uri for i in prov_ids] self.research_object.add_annotation(activity, uris, PROV["has_provenance"].uri) - def finalize_prov_profile(self, name: Optional[str]) -> List[QualifiedName]: + def finalize_prov_profile(self, name: str | None) -> list[QualifiedName]: """Transfer the provenance related files to the RO.""" # NOTE: Relative posix path if name is None: diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index 7c6eaf5d6..28b7c86df 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -7,23 +7,13 @@ import tempfile import urllib import uuid +from collections.abc import MutableMapping, MutableSequence from pathlib import Path, PurePosixPath -from typing import ( - IO, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from socket import getfqdn +from typing import IO, TYPE_CHECKING, Any, Optional, cast import prov.model as provM -from prov.model import PROV, ProvDocument +from prov.model import ProvDocument from ..loghandler import _logger from ..stdfsaccess import StdFsAccess @@ -38,6 +28,7 @@ from . import Aggregate, Annotation, AuthoredBy, _valid_orcid, _whoami, checksum_copy from .provenance_constants import ( ACCOUNT_UUID, + CWLPROV, CWLPROV_VERSION, DATA, ENCODING, @@ -46,6 +37,7 @@ METADATA, ORCID, PROVENANCE, + SCHEMA, SHA1, SHA256, SHA512, @@ -57,6 +49,9 @@ Hasher, ) +if TYPE_CHECKING: + from .provenance_profile import ProvenanceProfile # pylint: disable=unused-import + class ResearchObject: """CWLProv Research Object.""" @@ -75,12 +70,12 @@ def __init__( self.folder = create_tmp_dir(temp_prefix_ro) self.closed = False # map of filename "data/de/alsdklkas": 12398123 bytes - self.bagged_size: Dict[str, int] = {} - self.tagfiles: Set[str] = set() - self._file_provenance: Dict[str, Aggregate] = {} - self._external_aggregates: List[Aggregate] = [] - self.annotations: List[Annotation] = [] - self._content_types: Dict[str, str] = {} + self.bagged_size: dict[str, int] = {} + self.tagfiles: set[str] = set() + self._file_provenance: dict[str, Aggregate] = {} + self._external_aggregates: list[Aggregate] = [] + self.annotations: list[Annotation] = [] + self._content_types: dict[str, str] = {} self.fsaccess = fsaccess # These should be replaced by generate_prov_doc when workflow/run IDs are known: self.engine_uuid = f"urn:uuid:{uuid.uuid4()}" @@ -93,6 +88,34 @@ def __init__( self._initialize() _logger.debug("[provenance] Temporary research object: %s", self.folder) + def initialize_provenance( + self, + full_name: str, + host_provenance: bool, + user_provenance: bool, + orcid: str, + fsaccess: StdFsAccess, + run_uuid: uuid.UUID | None = None, + ) -> "ProvenanceProfile": + """ + Provide a provenance profile initialization hook function. + + Allows overriding the default strategy to define the + provenance profile concepts and associations to extend + details as needed. + """ + from .provenance_profile import ProvenanceProfile + + return ProvenanceProfile( + research_object=self, + full_name=full_name, + host_provenance=host_provenance, + user_provenance=user_provenance, + orcid=orcid, + fsaccess=fsaccess, + run_uuid=run_uuid, + ) + def self_check(self) -> None: """Raise ValueError if this RO is closed.""" if self.closed: @@ -128,10 +151,22 @@ def _initialize_bagit(self) -> None: bag_it_file.write("BagIt-Version: 0.97\n") bag_it_file.write(f"Tag-File-Character-Encoding: {ENCODING}\n") + def resolve_user(self) -> tuple[str, str]: + """ + Provide a user provenance hook function. + + Allows overriding the default strategy to retrieve user provenance + in case the calling code can provide a better resolution. + The function must return a tuple of the (username, fullname) + that identifies the user. This user will be applied on top + to any provided ORCID or fullname by agent association. + """ + return _whoami() + def user_provenance(self, document: ProvDocument) -> None: """Add the user provenance.""" self.self_check() - (username, fullname) = _whoami() + (username, fullname) = self.resolve_user() if not self.full_name: self.full_name = fullname @@ -143,19 +178,21 @@ def user_provenance(self, document: ProvDocument) -> None: ACCOUNT_UUID, { provM.PROV_TYPE: FOAF["OnlineAccount"], - "prov:label": username, + provM.PROV_LABEL: username, FOAF["accountName"]: username, }, ) user = document.agent( self.orcid or USER_UUID, - { - provM.PROV_TYPE: PROV["Person"], - "prov:label": self.full_name, - FOAF["name"]: self.full_name, - FOAF["account"]: account, - }, + [ + (provM.PROV_TYPE, SCHEMA["Person"]), + (provM.PROV_TYPE, provM.PROV["Person"]), + (provM.PROV_LABEL, self.full_name), + (FOAF["name"], self.full_name), + (FOAF["account"], account), + (SCHEMA["name"], self.full_name), + ], ) # cwltool may be started on the shell (directly by user), # by shell script (indirectly by user) @@ -167,7 +204,36 @@ def user_provenance(self, document: ProvDocument) -> None: # get their name wrong!) document.actedOnBehalfOf(account, user) - def add_tagfile(self, path: str, timestamp: Optional[datetime.datetime] = None) -> None: + def resolve_host(self) -> tuple[str, str]: + """ + Provide a host provenance hook function. + + Allows overriding the default strategy to retrieve host provenance + in case the calling code can provide a better resolution. + The function must return a tuple of the (fqdn, uri) that identifies the host. + """ + fqdn = getfqdn() + return fqdn, fqdn # allow for (fqdn, uri) to be distinct, but the same by default + + def host_provenance(self, document: ProvDocument) -> None: + """Record host provenance.""" + document.add_namespace(CWLPROV) + document.add_namespace(UUID) + document.add_namespace(FOAF) + + hostname, uri = self.resolve_host() + # won't have a foaf:accountServiceHomepage for unix hosts, but + # we can at least provide hostname + document.agent( + ACCOUNT_UUID, + { + provM.PROV_TYPE: FOAF["OnlineAccount"], + provM.PROV_LOCATION: uri, + CWLPROV["hostname"]: hostname, + }, + ) + + def add_tagfile(self, path: str, timestamp: datetime.datetime | None = None) -> None: """Add tag files to our research object.""" self.self_check() checksums = {} @@ -202,14 +268,14 @@ def add_tagfile(self, path: str, timestamp: Optional[datetime.datetime] = None) "conformsTo": None, } - def _ro_aggregates(self) -> List[Aggregate]: + def _ro_aggregates(self) -> list[Aggregate]: """Gather dictionary of files to be added to the manifest.""" def guess_mediatype( rel_path: str, - ) -> Tuple[Optional[str], Optional[Union[str, List[str]]]]: + ) -> tuple[str | None, str | list[str] | None]: """Return the mediatypes.""" - media_types: Dict[Union[str, None], str] = { + media_types: dict[str | None, str] = { # Adapted from # https://w3id.org/bundle/2014-11-05/#media-types "txt": TEXT_PLAIN, @@ -223,12 +289,12 @@ def guess_mediatype( "provn": 'text/provenance-notation; charset="UTF-8"', "nt": "application/n-triples", } - conforms_to: Dict[Union[str, None], str] = { + conforms_to: dict[str | None, str] = { "provn": "http://www.w3.org/TR/2013/REC-prov-n-20130430/", "cwl": "https://w3id.org/cwl/", } - prov_conforms_to: Dict[str, str] = { + prov_conforms_to: dict[str, str] = { "provn": "http://www.w3.org/TR/2013/REC-prov-n-20130430/", "rdf": "http://www.w3.org/TR/2013/REC-prov-o-20130430/", "ttl": "http://www.w3.org/TR/2013/REC-prov-o-20130430/", @@ -238,13 +304,13 @@ def guess_mediatype( "json": "http://www.w3.org/Submission/2013/SUBM-prov-json-20130424/", } - extension: Optional[str] = rel_path.rsplit(".", 1)[-1].lower() + extension: str | None = rel_path.rsplit(".", 1)[-1].lower() if extension == rel_path: # No ".", no extension extension = None - mediatype: Optional[str] = media_types.get(extension, None) - conformsTo: Optional[Union[str, List[str]]] = conforms_to.get(extension, None) + mediatype: str | None = media_types.get(extension, None) + conformsTo: str | list[str] | None = conforms_to.get(extension, None) # TODO: Open CWL file to read its declared "cwlVersion", e.g. # cwlVersion = "v1.0" @@ -261,7 +327,7 @@ def guess_mediatype( conformsTo = prov_conforms_to[extension] return (mediatype, conformsTo) - aggregates: List[Aggregate] = [] + aggregates: list[Aggregate] = [] for path in self.bagged_size.keys(): temp_path = PurePosixPath(path) folder = temp_path.parent @@ -291,7 +357,7 @@ def guess_mediatype( bundledAs.update(self._file_provenance[path]) else: aggregate_dict["bundledAs"] = cast( - Optional[Dict[str, Any]], self._file_provenance[path] + Optional[dict[str, Any]], self._file_provenance[path] ) else: # Probably made outside wf run, part of job object? @@ -335,7 +401,8 @@ def guess_mediatype( aggregates.extend(self._external_aggregates) return aggregates - def add_uri(self, uri: str, timestamp: Optional[datetime.datetime] = None) -> Aggregate: + def add_uri(self, uri: str, timestamp: datetime.datetime | None = None) -> Aggregate: + """Add the given URI to this RO.""" self.self_check() aggr: Aggregate = {"uri": uri} aggr["createdOn"], aggr["createdBy"] = self._self_made(timestamp=timestamp) @@ -343,7 +410,7 @@ def add_uri(self, uri: str, timestamp: Optional[datetime.datetime] = None) -> Ag return aggr def add_annotation( - self, about: str, content: List[str], motivated_by: str = "oa:describing" + self, about: str, content: list[str], motivated_by: str = "oa:describing" ) -> str: """Cheap URI relativize for current directory and /.""" self.self_check() @@ -359,9 +426,9 @@ def add_annotation( self.annotations.append(ann) return uri - def _ro_annotations(self) -> List[Annotation]: + def _ro_annotations(self) -> list[Annotation]: """Append base RO and provenance annotations to the list of annotations.""" - annotations: List[Annotation] = [] + annotations: list[Annotation] = [] annotations.append( { "uri": uuid.uuid4().urn, @@ -414,7 +481,7 @@ def _ro_annotations(self) -> List[Annotation]: annotations.extend(self.annotations) return annotations - def _authored_by(self) -> Optional[AuthoredBy]: + def _authored_by(self) -> AuthoredBy | None: authored_by: AuthoredBy = {} """Returns the authoredBy metadata if it was supplied on CLI""" if self.orcid: @@ -469,8 +536,8 @@ def has_data_file(self, sha1hash: str) -> bool: def add_data_file( self, from_fp: IO[Any], - timestamp: Optional[datetime.datetime] = None, - content_type: Optional[str] = None, + timestamp: datetime.datetime | None = None, + content_type: str | None = None, ) -> str: """Copy inputs to data/ folder.""" self.self_check() @@ -510,8 +577,8 @@ def add_data_file( return rel_path def _self_made( - self, timestamp: Optional[datetime.datetime] = None - ) -> Tuple[str, Dict[str, str]]: # createdOn, createdBy + self, timestamp: datetime.datetime | None = None + ) -> tuple[str, dict[str, str]]: # createdOn, createdBy if timestamp is None: timestamp = datetime.datetime.now() return ( @@ -519,7 +586,7 @@ def _self_made( {"uri": self.engine_uuid, "name": self.cwltool_version}, ) - def add_to_manifest(self, rel_path: str, checksums: Dict[str, str]) -> None: + def add_to_manifest(self, rel_path: str, checksums: dict[str, str]) -> None: """Add files to the research object manifest.""" self.self_check() if PurePosixPath(rel_path).is_absolute(): @@ -568,7 +635,7 @@ def _add_to_bagit(self, rel_path: str, **checksums: str) -> None: def _relativise_files( self, - structure: Union[CWLObjectType, CWLOutputType, MutableSequence[CWLObjectType]], + structure: CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType], ) -> None: """Save any file objects into the RO and update the local paths.""" # Base case - we found a File we need to update @@ -576,7 +643,7 @@ def _relativise_files( if isinstance(structure, MutableMapping): if structure.get("class") == "File": - relative_path: Optional[Union[str, PurePosixPath]] = None + relative_path: str | PurePosixPath | None = None if "checksum" in structure: raw_checksum = cast(str, structure["checksum"]) alg, checksum = raw_checksum.split("$") @@ -602,13 +669,13 @@ def _relativise_files( del structure["path"] if structure.get("class") == "Directory": - # TODO: Generate anonymoys Directory with a "listing" + # TODO: Generate anonymous Directory with a "listing" # pointing to the hashed files del structure["location"] for val in structure.values(): try: - self._relativise_files(cast(CWLOutputType, val)) + self._relativise_files(val) except OSError: pass return @@ -616,4 +683,4 @@ def _relativise_files( if isinstance(structure, MutableSequence): for obj in structure: # Recurse and rewrite any nested File objects - self._relativise_files(cast(CWLOutputType, obj)) + self._relativise_files(obj) diff --git a/cwltool/cwlprov/writablebagfile.py b/cwltool/cwlprov/writablebagfile.py index d5ff3c731..ecd6463d6 100644 --- a/cwltool/cwlprov/writablebagfile.py +++ b/cwltool/cwlprov/writablebagfile.py @@ -8,10 +8,11 @@ import uuid from array import array from collections import OrderedDict +from collections.abc import MutableMapping from io import FileIO, TextIOWrapper from mmap import mmap from pathlib import Path, PurePosixPath -from typing import Any, BinaryIO, Dict, MutableMapping, Optional, Union, cast +from typing import Any, BinaryIO, cast from schema_salad.utils import json_dumps @@ -97,7 +98,7 @@ def readable(self) -> bool: """Return False, reading is not supported.""" return False - def truncate(self, size: Optional[int] = None) -> int: + def truncate(self, size: int | None = None) -> int: """Resize the stream, only if we haven't started writing.""" # FIXME: This breaks contract IOBase, # as it means we would have to recalculate the hash @@ -107,8 +108,8 @@ def truncate(self, size: Optional[int] = None) -> int: def write_bag_file( - research_object: "ResearchObject", path: str, encoding: Optional[str] = ENCODING -) -> Union[TextIOWrapper, WritableBagFile]: + research_object: "ResearchObject", path: str, encoding: str | None = ENCODING +) -> TextIOWrapper | WritableBagFile: """Write the bag file into our research object.""" research_object.self_check() # For some reason below throws BlockingIOError @@ -122,7 +123,7 @@ def write_bag_file( def open_log_file_for_activity( research_object: "ResearchObject", uuid_uri: str -) -> Union[TextIOWrapper, WritableBagFile]: +) -> TextIOWrapper | WritableBagFile: """Begin the per-activity log.""" research_object.self_check() # Ensure valid UUID for safe filenames @@ -195,7 +196,7 @@ def _finalize(research_object: "ResearchObject") -> None: (Path(research_object.folder) / "manifest-sha1.txt").touch() -def close_ro(research_object: "ResearchObject", save_to: Optional[str] = None) -> None: +def close_ro(research_object: "ResearchObject", save_to: str | None = None) -> None: """Close the Research Object, optionally saving to specified folder. Closing will remove any temporary files used by this research object. @@ -246,7 +247,7 @@ def create_job( relativised_input_objecttemp: CWLObjectType = {} research_object._relativise_files(copied) - def jdefault(o: Any) -> Dict[Any, Any]: + def jdefault(o: Any) -> dict[Any, Any]: return dict(o) if is_output: diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index dbe9e2f97..0b844df4a 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -1,9 +1,8 @@ -import urllib -from codecs import StreamWriter -from typing import IO, Any, Dict, Iterator, Optional, TextIO, Union, cast +"""RDF output.""" + +from typing import IO, Any from rdflib import Graph -from rdflib.query import ResultRow from ruamel.yaml.comments import CommentedMap from schema_salad.jsonld_context import makerdf from schema_salad.utils import ContextType @@ -37,172 +36,6 @@ def lastpart(uri: Any) -> str: return uri2 -def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: - qres = cast( - Iterator[ResultRow], - g.query( - """SELECT ?step ?run ?runtype - WHERE { - ?step cwl:run ?run . - ?run rdf:type ?runtype . - }""" - ), - ) # ResultRow because the query is of type SELECT - - for step, run, _ in qres: - stdout.write( - '"{}" [label="{}"]\n'.format(lastpart(step), f"{lastpart(step)} ({lastpart(run)})") - ) - - qres = cast( - Iterator[ResultRow], - g.query( - """SELECT ?step ?inp ?source - WHERE { - ?wf Workflow:steps ?step . - ?step cwl:inputs ?inp . - ?inp cwl:source ?source . - }""" - ), - ) # ResultRow because the query is of type SELECT - - for step, inp, source in qres: - stdout.write('"%s" [shape=box]\n' % (lastpart(inp))) - stdout.write('"{}" -> "{}" [label="{}"]\n'.format(lastpart(source), lastpart(inp), "")) - stdout.write('"{}" -> "{}" [label="{}"]\n'.format(lastpart(inp), lastpart(step), "")) - - qres = cast( - Iterator[ResultRow], - g.query( - """SELECT ?step ?out - WHERE { - ?wf Workflow:steps ?step . - ?step cwl:outputs ?out . - }""" - ), - ) # ResultRow because the query is of type SELECT - - for step, out in qres: - stdout.write('"%s" [shape=box]\n' % (lastpart(out))) - stdout.write('"{}" -> "{}" [label="{}"]\n'.format(lastpart(step), lastpart(out), "")) - - qres = cast( - Iterator[ResultRow], - g.query( - """SELECT ?out ?source - WHERE { - ?wf cwl:outputs ?out . - ?out cwl:source ?source . - }""" - ), - ) # ResultRow because the query is of type SELECT - - for out, source in qres: - stdout.write('"%s" [shape=octagon]\n' % (lastpart(out))) - stdout.write('"{}" -> "{}" [label="{}"]\n'.format(lastpart(source), lastpart(out), "")) - - qres = cast( - Iterator[ResultRow], - g.query( - """SELECT ?inp - WHERE { - ?wf rdf:type cwl:Workflow . - ?wf cwl:inputs ?inp . - }""" - ), - ) # ResultRow because the query is of type SELECT - - for (inp,) in qres: - stdout.write('"%s" [shape=octagon]\n' % (lastpart(inp))) - - -def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: - dotname: Dict[str, str] = {} - clusternode = {} - - stdout.write("compound=true\n") - - subworkflows = set() - qres = cast( - Iterator[ResultRow], - g.query( - """SELECT ?run - WHERE { - ?wf rdf:type cwl:Workflow . - ?wf Workflow:steps ?step . - ?step cwl:run ?run . - ?run rdf:type cwl:Workflow . - } ORDER BY ?wf""" - ), - ) # ResultRow because the query is of type SELECT - for (run,) in qres: - subworkflows.add(run) - - qres = cast( - Iterator[ResultRow], - g.query( - """SELECT ?wf ?step ?run ?runtype - WHERE { - ?wf rdf:type cwl:Workflow . - ?wf Workflow:steps ?step . - ?step cwl:run ?run . - ?run rdf:type ?runtype . - } ORDER BY ?wf""" - ), - ) # ResultRow because the query is of type SELECT - - currentwf: Optional[str] = None - for wf, step, _run, runtype in qres: - if step not in dotname: - dotname[step] = lastpart(step) - - if wf != currentwf: - if currentwf is not None: - stdout.write("}\n") - if wf in subworkflows: - if wf not in dotname: - dotname[wf] = "cluster_" + lastpart(wf) - stdout.write(f'subgraph "{dotname[wf]}" {{ label="{lastpart(wf)}"\n') # noqa: B907 - currentwf = wf - clusternode[wf] = step - else: - currentwf = None - - if str(runtype) != "https://w3id.org/cwl/cwl#Workflow": - stdout.write( - f'"{dotname[step]}" [label="{urllib.parse.urldefrag(str(step))[1]}"]\n' # noqa: B907 - ) - - if currentwf is not None: - stdout.write("}\n") - - qres = cast( - Iterator[ResultRow], - g.query( - """SELECT DISTINCT ?src ?sink ?srcrun ?sinkrun - WHERE { - ?wf1 Workflow:steps ?src . - ?wf2 Workflow:steps ?sink . - ?src cwl:out ?out . - ?inp cwl:source ?out . - ?sink cwl:in ?inp . - ?src cwl:run ?srcrun . - ?sink cwl:run ?sinkrun . - }""" - ), - ) # ResultRow because the query is of type SELECT - - for src, sink, srcrun, sinkrun in qres: - attr = "" - if srcrun in clusternode: - attr += 'ltail="%s"' % dotname[srcrun] - src = clusternode[srcrun] - if sinkrun in clusternode: - attr += ' lhead="%s"' % dotname[sinkrun] - sink = clusternode[sinkrun] - stdout.write(f'"{dotname[src]}" -> "{dotname[sink]}" [{attr}]\n') # noqa: B907 - - def printdot( wf: Process, ctx: ContextType, diff --git a/cwltool/cwlviewer.py b/cwltool/cwlviewer.py index e544a568e..db6b50358 100644 --- a/cwltool/cwlviewer.py +++ b/cwltool/cwlviewer.py @@ -1,17 +1,34 @@ """Visualize a CWL workflow.""" -from pathlib import Path -from typing import Iterator, List, cast +from collections.abc import Iterator +from importlib.resources import files +from typing import cast from urllib.parse import urlparse import pydot import rdflib +from packaging.version import Version -_queries_dir = (Path(__file__).parent / "rdfqueries").resolve() -_get_inner_edges_query_path = _queries_dir / "get_inner_edges.sparql" -_get_input_edges_query_path = _queries_dir / "get_input_edges.sparql" -_get_output_edges_query_path = _queries_dir / "get_output_edges.sparql" -_get_root_query_path = _queries_dir / "get_root.sparql" +if Version(pydot.__version__) > Version("3.0"): + quote_id_if_necessary = pydot.quote_id_if_necessary +else: + quote_id_if_necessary = pydot.quote_if_necessary # type: ignore[attr-defined] + + +def _get_inner_edges_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_inner_edges.sparql").read_text() + + +def _get_input_edges_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_input_edges.sparql").read_text() + + +def _get_output_edges_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_output_edges.sparql").read_text() + + +def _get_root_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_root.sparql").read_text() class CWLViewer: @@ -32,8 +49,7 @@ def _load_cwl_graph(self, rdf_description: str) -> rdflib.graph.Graph: return rdf_graph def _set_inner_edges(self) -> None: - with open(_get_inner_edges_query_path) as f: - get_inner_edges_query = f.read() + get_inner_edges_query = _get_inner_edges_query() inner_edges = cast( Iterator[rdflib.query.ResultRow], self._rdf_graph.query( @@ -89,18 +105,16 @@ def _set_inner_edges(self) -> None: self._dot_graph.add_node(n) self._dot_graph.add_edge( pydot.Edge( - str(inner_edge_row["source_step"]), - str(inner_edge_row["target_step"]), + quote_id_if_necessary(str(inner_edge_row["source_step"])), + quote_id_if_necessary(str(inner_edge_row["target_step"])), ) ) def _set_input_edges(self) -> None: - with open(_get_input_edges_query_path) as f: - get_input_edges_query = f.read() + get_input_edges_query = _get_input_edges_query() inputs_subgraph = pydot.Subgraph(graph_name="cluster_inputs") self._dot_graph.add_subgraph(inputs_subgraph) inputs_subgraph.set("rank", "same") - inputs_subgraph.create_attribute_methods(["style"]) inputs_subgraph.set("style", "dashed") inputs_subgraph.set("label", "Workflow Inputs") @@ -120,15 +134,18 @@ def _set_input_edges(self) -> None: ) n.set_name(str(input_row["input"])) inputs_subgraph.add_node(n) - self._dot_graph.add_edge(pydot.Edge(str(input_row["input"]), str(input_row["step"]))) + self._dot_graph.add_edge( + pydot.Edge( + quote_id_if_necessary(str(input_row["input"])), + quote_id_if_necessary(str(input_row["step"])), + ) + ) def _set_output_edges(self) -> None: - with open(_get_output_edges_query_path) as f: - get_output_edges = f.read() + get_output_edges = _get_output_edges_query() outputs_graph = pydot.Subgraph(graph_name="cluster_outputs") self._dot_graph.add_subgraph(outputs_graph) outputs_graph.set("rank", "same") - outputs_graph.create_attribute_methods(["style"]) outputs_graph.set("style", "dashed") outputs_graph.set("label", "Workflow Outputs") outputs_graph.set("labelloc", "b") @@ -148,13 +165,17 @@ def _set_output_edges(self) -> None: ) n.set_name(str(output_edge_row["output"])) outputs_graph.add_node(n) - self._dot_graph.add_edge(pydot.Edge(output_edge_row["step"], output_edge_row["output"])) + self._dot_graph.add_edge( + pydot.Edge( + quote_id_if_necessary(output_edge_row["step"]), + quote_id_if_necessary(output_edge_row["output"]), + ) + ) def _get_root_graph_uri(self) -> rdflib.term.Identifier: - with open(_get_root_query_path) as f: - get_root_query = f.read() + get_root_query = _get_root_query() root = cast( - List[rdflib.query.ResultRow], + list[rdflib.query.ResultRow], list( self._rdf_graph.query( get_root_query, diff --git a/cwltool/docker.py b/cwltool/docker.py index d0f628b15..69fecc830 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -9,8 +9,9 @@ import subprocess # nosec import sys import threading +from collections.abc import Callable, MutableMapping from io import StringIO # pylint: disable=redefined-builtin -from typing import Callable, Dict, List, MutableMapping, Optional, Set, Tuple, cast +from typing import Optional, cast import requests @@ -23,13 +24,13 @@ from .pathmapper import MapperEnt, PathMapper from .utils import CWLObjectType, create_tmp_dir, ensure_writable -_IMAGES: Set[str] = set() +_IMAGES: set[str] = set() _IMAGES_LOCK = threading.Lock() -__docker_machine_mounts: Optional[List[str]] = None +__docker_machine_mounts: list[str] | None = None __docker_machine_mounts_lock = threading.Lock() -def _get_docker_machine_mounts() -> List[str]: +def _get_docker_machine_mounts() -> list[str]: global __docker_machine_mounts if __docker_machine_mounts is None: with __docker_machine_mounts_lock: @@ -53,7 +54,7 @@ def _get_docker_machine_mounts() -> List[str]: return __docker_machine_mounts -def _check_docker_machine_path(path: Optional[str]) -> None: +def _check_docker_machine_path(path: str | None) -> None: if path is None: return mounts = _get_docker_machine_mounts() @@ -83,9 +84,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize a command line builder using the Docker software container engine.""" @@ -94,7 +95,7 @@ def __init__( def get_image( self, - docker_requirement: Dict[str, str], + docker_requirement: dict[str, str], pull_image: bool, force_pull: bool, tmp_outdir_prefix: str, @@ -127,7 +128,7 @@ def get_image( except (OSError, subprocess.CalledProcessError, UnicodeError): pass - cmd: List[str] = [] + cmd: list[str] = [] if "dockerFile" in docker_requirement: dockerfile_dir = create_tmp_dir(tmp_outdir_prefix) with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile: @@ -200,17 +201,17 @@ def get_from_requirements( pull_image: bool, force_pull: bool, tmp_outdir_prefix: str, - ) -> Optional[str]: + ) -> str | None: if not shutil.which(self.docker_exec): raise WorkflowException(f"{self.docker_exec} executable is not available") - if self.get_image(cast(Dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix): + if self.get_image(cast(dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix): return cast(Optional[str], r["dockerImageId"]) raise WorkflowException("Docker image %s not found" % r["dockerImageId"]) @staticmethod def append_volume( - runtime: List[str], + runtime: list[str], source: str, target: str, writable: bool = False, @@ -233,7 +234,7 @@ def append_volume( os.makedirs(source) def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: str | None ) -> None: """Append volume a file/dir mapping to the runtime option list.""" if not volume.resolved.startswith("_:"): @@ -242,9 +243,9 @@ def add_file_or_directory_volume( def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: """Append a writable file mapping to the runtime option list.""" @@ -266,9 +267,9 @@ def add_writable_file_volume( def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: """Append a writable directory mapping to the runtime option list.""" @@ -295,7 +296,7 @@ def add_writable_directory_volume( shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: # spec currently says "HOME must be set to the designated output # directory." but spec might change to designated temp directory. # runtime.append("--env=HOME=/tmp") @@ -306,7 +307,7 @@ def _required_env(self) -> Dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtimeContext: RuntimeContext - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], str | None]: any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False user_space_docker_cmd = runtimeContext.user_space_docker_cmd if user_space_docker_cmd: @@ -373,7 +374,7 @@ def create_runtime( if self.builder.resources.get("cudaDeviceCount"): runtime.append("--gpus=" + str(self.builder.resources["cudaDeviceCount"])) - cidfile_path: Optional[str] = None + cidfile_path: str | None = None # add parameters to docker to write a container ID file if runtimeContext.user_space_docker_cmd is None: if runtimeContext.cidfile_dir: @@ -445,9 +446,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize a command line builder using the Podman software container engine.""" diff --git a/cwltool/docker_id.py b/cwltool/docker_id.py index bb436b2cb..2f93934cc 100644 --- a/cwltool/docker_id.py +++ b/cwltool/docker_id.py @@ -1,10 +1,9 @@ """Helper functions for docker.""" import subprocess # nosec -from typing import List, Optional, Tuple -def docker_vm_id() -> Tuple[Optional[int], Optional[int]]: +def docker_vm_id() -> tuple[int | None, int | None]: """ Return the User ID and Group ID of the default docker user inside the VM. @@ -21,7 +20,7 @@ def docker_vm_id() -> Tuple[Optional[int], Optional[int]]: return (None, None) -def check_output_and_strip(cmd: List[str]) -> Optional[str]: +def check_output_and_strip(cmd: list[str]) -> str | None: """ Pass a command list to :py:func:`subprocess.check_output`. @@ -39,7 +38,7 @@ def check_output_and_strip(cmd: List[str]) -> Optional[str]: return None -def docker_machine_name() -> Optional[str]: +def docker_machine_name() -> str | None: """ Get the machine name of the active docker-machine machine. @@ -48,7 +47,7 @@ def docker_machine_name() -> Optional[str]: return check_output_and_strip(["docker-machine", "active"]) -def cmd_output_matches(check_cmd: List[str], expected_status: str) -> bool: +def cmd_output_matches(check_cmd: list[str], expected_status: str) -> bool: """ Run a command and compares output to expected. @@ -80,7 +79,7 @@ def docker_machine_running() -> bool: return cmd_output_matches(["docker-machine", "status", machine_name], "Running") -def cmd_output_to_int(cmd: List[str]) -> Optional[int]: +def cmd_output_to_int(cmd: list[str]) -> int | None: """ Run the provided command and returns the integer value of the result. @@ -97,7 +96,7 @@ def cmd_output_to_int(cmd: List[str]) -> Optional[int]: return None -def boot2docker_id() -> Tuple[Optional[int], Optional[int]]: +def boot2docker_id() -> tuple[int | None, int | None]: """ Get the UID and GID of the docker user inside a running boot2docker vm. @@ -108,7 +107,7 @@ def boot2docker_id() -> Tuple[Optional[int], Optional[int]]: return (uid, gid) -def docker_machine_id() -> Tuple[Optional[int], Optional[int]]: +def docker_machine_id() -> tuple[int | None, int | None]: """ Ask docker-machine for active machine and gets the UID of the docker user. diff --git a/cwltool/env_to_stdout.py b/cwltool/env_to_stdout.py index 33b832479..0309fe08f 100644 --- a/cwltool/env_to_stdout.py +++ b/cwltool/env_to_stdout.py @@ -11,10 +11,9 @@ """ import os -from typing import Dict -def deserialize_env(data: str) -> Dict[str, str]: +def deserialize_env(data: str) -> dict[str, str]: """Deserialize the output of `env -0` to dictionary.""" result = {} for item in data.strip("\0").split("\0"): diff --git a/cwltool/errors.py b/cwltool/errors.py index 045b9b383..2b7e50aed 100644 --- a/cwltool/errors.py +++ b/cwltool/errors.py @@ -11,6 +11,13 @@ from cwl_utils.errors import GraphTargetMissingException as GraphTargetMissingException from cwl_utils.errors import WorkflowException as WorkflowException +__all__ = ( + "GraphTargetMissingException", + "WorkflowException", + "UnsupportedRequirement", + "ArgumentException", +) + class UnsupportedRequirement(WorkflowException): pass diff --git a/cwltool/executors.py b/cwltool/executors.py index bfc87f9c7..090e4a676 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -7,18 +7,9 @@ import os import threading from abc import ABCMeta, abstractmethod +from collections.abc import Iterable, MutableSequence from threading import Lock -from typing import ( - Dict, - Iterable, - List, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from typing import Optional, cast import psutil from mypy_extensions import mypyc_attr @@ -28,7 +19,6 @@ from .command_line_tool import CallbackJob, ExpressionJob from .context import RuntimeContext, getdefault from .cuda import cuda_version_and_device_count -from .cwlprov.provenance_profile import ProvenanceProfile from .errors import WorkflowException from .job import JobBase from .loghandler import _logger @@ -49,9 +39,9 @@ class JobExecutor(metaclass=ABCMeta): def __init__(self) -> None: """Initialize.""" - self.final_output: MutableSequence[Optional[CWLObjectType]] = [] - self.final_status: List[str] = [] - self.output_dirs: Set[str] = set() + self.final_output: MutableSequence[CWLObjectType | None] = [] + self.final_status: list[str] = [] + self.output_dirs: set[str] = set() def __call__( self, @@ -59,10 +49,10 @@ def __call__( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> Tuple[Optional[CWLObjectType], str]: + ) -> tuple[CWLObjectType | None, str]: return self.execute(process, job_order_object, runtime_context, logger) - def output_callback(self, out: Optional[CWLObjectType], process_status: str) -> None: + def output_callback(self, out: CWLObjectType | None, process_status: str) -> None: """Collect the final status and outputs.""" self.final_status.append(process_status) self.final_output.append(out) @@ -83,7 +73,7 @@ def execute( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> Tuple[Union[Optional[CWLObjectType]], str]: + ) -> tuple[CWLObjectType | None, str]: """Execute the process.""" self.final_output = [] @@ -112,7 +102,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: runtime_context.toplevel = True runtime_context.workflow_eval_lock = threading.Condition(threading.RLock()) - job_reqs: Optional[List[CWLObjectType]] = None + job_reqs: list[CWLObjectType] | None = None if "https://w3id.org/cwl/cwl#requirements" in job_order_object: if process.metadata.get(ORIGINAL_CWLVERSION) == "v1.0": raise WorkflowException( @@ -121,7 +111,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: "can set the cwlVersion to v1.1" ) job_reqs = cast( - List[CWLObjectType], + list[CWLObjectType], job_order_object["https://w3id.org/cwl/cwl#requirements"], ) elif "cwl:defaults" in process.metadata and "https://w3id.org/cwl/cwl#requirements" in cast( @@ -134,7 +124,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: "can set the cwlVersion to v1.1" ) job_reqs = cast( - Optional[List[CWLObjectType]], + Optional[list[CWLObjectType]], cast(CWLObjectType, process.metadata["cwl:defaults"])[ "https://w3id.org/cwl/cwl#requirements" ], @@ -174,7 +164,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: and isinstance(process, (JobBase, Process, WorkflowJobStep, WorkflowJob)) and process.parent_wf ): - process_run_id: Optional[str] = None + process_run_id: str | None = None name = "primary" process.parent_wf.generate_output_prov(self.final_output[0], process_run_id, name) process.parent_wf.document.wasEndedBy( @@ -199,15 +189,17 @@ def run_jobs( logger: logging.Logger, runtime_context: RuntimeContext, ) -> None: - process_run_id: Optional[str] = None + process_run_id: str | None = None # define provenance profile for single commandline tool if not isinstance(process, Workflow) and runtime_context.research_obj is not None: - process.provenance_object = ProvenanceProfile( - runtime_context.research_obj, + process.provenance_object = runtime_context.research_obj.initialize_provenance( full_name=runtime_context.cwl_full_name, - host_provenance=False, - user_provenance=False, + # following are only set from main when directly command line tool + # when nested in a workflow, they should be disabled since they would + # already have been provided/initialized by the parent workflow prov-obj + host_provenance=runtime_context.prov_host, + user_provenance=runtime_context.prov_user, orcid=runtime_context.orcid, # single tool execution, so RO UUID = wf UUID = tool UUID run_uuid=runtime_context.research_obj.ro_uuid, @@ -277,22 +269,22 @@ class MultithreadedJobExecutor(JobExecutor): def __init__(self) -> None: """Initialize.""" super().__init__() - self.exceptions: List[WorkflowException] = [] - self.pending_jobs: List[JobsType] = [] + self.exceptions: list[WorkflowException] = [] + self.pending_jobs: list[JobsType] = [] self.pending_jobs_lock = threading.Lock() self.max_ram = int(psutil.virtual_memory().available / 2**20) - self.max_cores = float(psutil.cpu_count()) + self.max_cores = float(psutil.cpu_count() or 1) self.max_cuda = cuda_version_and_device_count()[1] self.allocated_ram = float(0) self.allocated_cores = float(0) self.allocated_cuda: int = 0 def select_resources( - self, request: Dict[str, Union[int, float]], runtime_context: RuntimeContext - ) -> Dict[str, Union[int, float]]: # pylint: disable=unused-argument + self, request: dict[str, int | float], runtime_context: RuntimeContext + ) -> dict[str, int | float]: # pylint: disable=unused-argument """NaĂ¯ve check for available cpu cores and memory.""" - result: Dict[str, Union[int, float]] = {} + result: dict[str, int | float] = {} maxrsc = {"cores": self.max_cores, "ram": self.max_ram} resources_types = {"cores", "ram"} if "cudaDeviceCountMin" in request or "cudaDeviceCountMax" in request: @@ -317,7 +309,7 @@ def select_resources( def _runner( self, - job: Union[JobBase, WorkflowJob, CallbackJob, ExpressionJob], + job: JobBase | WorkflowJob | CallbackJob | ExpressionJob, runtime_context: RuntimeContext, TMPDIR_LOCK: threading.Lock, ) -> None: @@ -334,7 +326,10 @@ def _runner( self.exceptions.append(err) except Exception as err: # pylint: disable=broad-except _logger.exception(f"Got workflow error: {err}") - self.exceptions.append(WorkflowException(str(err))) + wf_exc = WorkflowException(str(err)) + wf_exc.__cause__ = err + wf_exc.__suppress_context__ = True + self.exceptions.append(wf_exc) finally: if runtime_context.workflow_eval_lock: with runtime_context.workflow_eval_lock: @@ -351,7 +346,7 @@ def _runner( def run_job( self, - job: Optional[JobsType], + job: JobsType | None, runtime_context: RuntimeContext, ) -> None: """Execute a single Job in a separate thread.""" @@ -438,7 +433,7 @@ def run_jobs( logger: logging.Logger, runtime_context: RuntimeContext, ) -> None: - self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), psutil.cpu_count()) + self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), int(math.ceil(self.max_cores))) try: jobiter = process.job(job_order_object, self.output_callback, runtime_context) @@ -490,6 +485,6 @@ def execute( process: Process, job_order_object: CWLObjectType, runtime_context: RuntimeContext, - logger: Optional[logging.Logger] = None, - ) -> Tuple[Optional[CWLObjectType], str]: + logger: logging.Logger | None = None, + ) -> tuple[CWLObjectType | None, str]: return {}, "success" diff --git a/cwltool/extensions-v1.2.yml b/cwltool/extensions-v1.2.yml index c39b15d07..ae371c671 100644 --- a/cwltool/extensions-v1.2.yml +++ b/cwltool/extensions-v1.2.yml @@ -236,7 +236,7 @@ $graph: name: LoopOutputModes symbols: [ last, all ] default: last - doc: + doc: | - Specify the desired method of dealing with loop outputs - Default. Propagates only the last computed element to the subsequent steps when the loop terminates. - Propagates a single array with all output values to the subsequent steps when the loop terminates. diff --git a/cwltool/factory.py b/cwltool/factory.py index 85d7344e6..fc00eb061 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -1,5 +1,5 @@ import os -from typing import Any, Dict, Optional, Union +from typing import Any from . import load_tool from .context import LoadingContext, RuntimeContext @@ -10,7 +10,7 @@ class WorkflowStatus(Exception): - def __init__(self, out: Optional[CWLObjectType], status: str) -> None: + def __init__(self, out: CWLObjectType | None, status: str) -> None: """Signaling exception for the status of a Workflow.""" super().__init__("Completed %s" % status) self.out = out @@ -25,8 +25,8 @@ def __init__(self, t: Process, factory: "Factory") -> None: self.t = t self.factory = factory - def __call__(self, **kwargs): - # type: (**Any) -> Union[str, Optional[CWLObjectType]] + def __call__(self, **kwargs: Any) -> str | CWLObjectType | None: + """Invoke the tool.""" runtime_context = self.factory.runtime_context.copy() runtime_context.basedir = os.getcwd() out, status = self.factory.executor(self.t, kwargs, runtime_context) @@ -44,9 +44,9 @@ class Factory: def __init__( self, - executor: Optional[JobExecutor] = None, - loading_context: Optional[LoadingContext] = None, - runtime_context: Optional[RuntimeContext] = None, + executor: JobExecutor | None = None, + loading_context: LoadingContext | None = None, + runtime_context: RuntimeContext | None = None, ) -> None: if executor is None: executor = SingleJobExecutor() @@ -62,7 +62,7 @@ def __init__( else: self.loading_context = loading_context - def make(self, cwl: Union[str, Dict[str, Any]]) -> Callable: + def make(self, cwl: str | dict[str, Any]) -> Callable: """Instantiate a CWL object from a CWl document.""" load = load_tool.load_tool(cwl, self.loading_context) if isinstance(load, int): diff --git a/cwltool/flatten.py b/cwltool/flatten.py index 420d90d04..5694f6e11 100644 --- a/cwltool/flatten.py +++ b/cwltool/flatten.py @@ -1,12 +1,18 @@ -from typing import Any, Callable, List, cast +""" +Our version of the popular flatten() method. -# http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html +http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html +""" +from collections.abc import Callable +from typing import Any, cast -def flatten(thing, ltypes=(list, tuple)): - # type: (Any, Any) -> List[Any] + +def flatten(thing: Any) -> list[Any]: + """Flatten a list without recursion problems.""" if thing is None: return [] + ltypes = (list, tuple) if not isinstance(thing, ltypes): return [thing] @@ -22,4 +28,4 @@ def flatten(thing, ltypes=(list, tuple)): else: thing_list[i : i + 1] = thing_list[i] i += 1 - return cast(Callable[[Any], List[Any]], ltype)(thing_list) + return cast(Callable[[Any], list[Any]], ltype)(thing_list) diff --git a/cwltool/job.py b/cwltool/job.py index 817cb04c0..c46778b35 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -5,6 +5,7 @@ import math import os import re +import shlex import shutil import signal import stat @@ -15,27 +16,12 @@ import time import uuid from abc import ABCMeta, abstractmethod +from collections.abc import Callable, Iterable, Mapping, MutableMapping, MutableSequence +from re import Match from threading import Timer -from typing import ( - IO, - TYPE_CHECKING, - Callable, - Dict, - Iterable, - List, - Mapping, - Match, - MutableMapping, - MutableSequence, - Optional, - TextIO, - Tuple, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Optional, TextIO, Union, cast import psutil -import shellescape from prov.model import PROV from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dump, json_dumps @@ -113,7 +99,8 @@ def relink_initialworkdir( pass -def neverquote(string: str, pos: int = 0, endpos: int = 0) -> Optional[Match[str]]: +def neverquote(string: str, pos: int = 0, endpos: int = 0) -> Match[str] | None: + """No-op.""" return None @@ -122,32 +109,32 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize the job object.""" super().__init__() self.builder = builder self.joborder = joborder - self.stdin: Optional[str] = None - self.stderr: Optional[str] = None - self.stdout: Optional[str] = None + self.stdin: str | None = None + self.stderr: str | None = None + self.stdout: str | None = None self.successCodes: Iterable[int] = [] self.temporaryFailCodes: Iterable[int] = [] self.permanentFailCodes: Iterable[int] = [] self.requirements = requirements self.hints = hints self.name = name - self.command_line: List[str] = [] + self.command_line: list[str] = [] self.pathmapper = PathMapper([], "", "") self.make_path_mapper = make_path_mapper - self.generatemapper: Optional[PathMapper] = None + self.generatemapper: PathMapper | None = None # set in CommandLineTool.job(i) self.collect_outputs = cast("CollectOutputsType", None) - self.output_callback: Optional[OutputCallbackType] = None + self.output_callback: OutputCallbackType | None = None self.outdir = "" self.tmpdir = "" @@ -157,13 +144,13 @@ def __init__( "listing": [], "basename": "", } - self.stagedir: Optional[str] = None + self.stagedir: str | None = None self.inplace_update = False - self.prov_obj: Optional[ProvenanceProfile] = None - self.parent_wf: Optional[ProvenanceProfile] = None - self.timelimit: Optional[int] = None + self.prov_obj: ProvenanceProfile | None = None + self.parent_wf: ProvenanceProfile | None = None + self.timelimit: int | None = None self.networkaccess: bool = False - self.mpi_procs: Optional[int] = None + self.mpi_procs: int | None = None def __repr__(self) -> str: """Represent this Job object.""" @@ -173,7 +160,7 @@ def __repr__(self) -> str: def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: pass @@ -228,10 +215,10 @@ def is_streamable(file: str) -> bool: def _execute( self, - runtime: List[str], + runtime: list[str], env: MutableMapping[str, str], runtimeContext: RuntimeContext, - monitor_function: Optional[Callable[["subprocess.Popen[str]"], None]] = None, + monitor_function: Callable[["subprocess.Popen[str]"], None] | None = None, ) -> None: """Execute the tool, either directly or via script. @@ -271,7 +258,7 @@ def _execute( self.outdir, " \\\n ".join( [ - shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) + shlex.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line) ] ), @@ -306,8 +293,8 @@ def _execute( stdin_path = rmap[1] def stderr_stdout_log_path( - base_path_logs: str, stderr_or_stdout: Optional[str] - ) -> Optional[str]: + base_path_logs: str, stderr_or_stdout: str | None + ) -> str | None: if stderr_or_stdout is not None: abserr = os.path.join(base_path_logs, stderr_or_stdout) dnerr = os.path.dirname(abserr) @@ -321,7 +308,7 @@ def stderr_stdout_log_path( commands = [str(x) for x in runtime + self.command_line] if runtimeContext.secret_store is not None: commands = cast( - List[str], + list[str], runtimeContext.secret_store.retrieve(cast(CWLOutputType, commands)), ) env = cast( @@ -329,8 +316,8 @@ def stderr_stdout_log_path( runtimeContext.secret_store.retrieve(cast(CWLOutputType, env)), ) - job_script_contents: Optional[str] = None - builder: Optional[Builder] = getattr(self, "builder", None) + job_script_contents: str | None = None + builder: Builder | None = getattr(self, "builder", None) if builder is not None: job_script_contents = builder.build_job_script(commands) rcode = _job_popen( @@ -390,17 +377,30 @@ def stderr_stdout_log_path( except OSError as e: if e.errno == 2: if runtime: - _logger.error("'%s' not found: %s", runtime[0], str(e)) + _logger.error( + "'%s' not found: %s", runtime[0], str(e), exc_info=runtimeContext.debug + ) else: - _logger.error("'%s' not found: %s", self.command_line[0], str(e)) + _logger.error( + "'%s' not found: %s", + self.command_line[0], + str(e), + exc_info=runtimeContext.debug, + ) else: - _logger.exception("Exception while running job") + _logger.exception( + "Exception while running job: %s", str(e), exc_info=runtimeContext.debug + ) processStatus = "permanentFail" except WorkflowException as err: - _logger.error("[job %s] Job error:\n%s", self.name, str(err)) + _logger.error( + "[job %s] Job error:\n%s", self.name, str(err), exc_info=runtimeContext.debug + ) processStatus = "permanentFail" - except Exception: - _logger.exception("Exception while running job") + except Exception as err: + _logger.exception( + "Exception while running job: %s.", str(err), exc_info=runtimeContext.debug + ) processStatus = "permanentFail" if ( runtimeContext.research_obj is not None @@ -456,7 +456,7 @@ def stderr_stdout_log_path( shutil.rmtree(self.tmpdir, True) @abstractmethod - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: """Variables required by the CWL spec (HOME, TMPDIR, etc). Note that with containers, the paths will (likely) be those from @@ -464,12 +464,35 @@ def _required_env(self) -> Dict[str, str]: """ def _preserve_environment_on_containers_warning( - self, varname: Optional[Iterable[str]] = None + self, varname: Iterable[str] | None = None ) -> None: """When running in a container, issue a warning.""" # By default, don't do anything; ContainerCommandLineJob below # will issue a warning. + @staticmethod + def extract_environment( + runtimeContext: RuntimeContext, envVarReq: Mapping[str, str] + ) -> Mapping[str, str]: + """Extract environment variables that should be preserved.""" + # Start empty + env: dict[str, str] = {} + # Preserve any env vars + if runtimeContext.preserve_entire_environment: + env.update(os.environ) + elif runtimeContext.preserve_environment: + for key in runtimeContext.preserve_environment: + try: + env[key] = os.environ[key] + except KeyError: + _logger.warning( + f"Attempting to preserve environment variable {key!r} which is not present" + ) + # Apply EnvVarRequirement + env.update(envVarReq) + + return env + def prepare_environment( self, runtimeContext: RuntimeContext, envVarReq: Mapping[str, str] ) -> None: @@ -481,28 +504,17 @@ def prepare_environment( applied (in that order). """ # Start empty - env: Dict[str, str] = {} - - # Preserve any env vars + env: dict[str, str] = {} if runtimeContext.preserve_entire_environment: self._preserve_environment_on_containers_warning() - env.update(os.environ) elif runtimeContext.preserve_environment: self._preserve_environment_on_containers_warning(runtimeContext.preserve_environment) - for key in runtimeContext.preserve_environment: - try: - env[key] = os.environ[key] - except KeyError: - _logger.warning( - f"Attempting to preserve environment variable {key!r} which is not present" - ) + + env.update(self.extract_environment(runtimeContext, envVarReq)) # Set required env vars env.update(self._required_env()) - # Apply EnvVarRequirement - env.update(envVarReq) - # Set on ourselves self.environment = env @@ -510,11 +522,11 @@ def process_monitor(self, sproc: "subprocess.Popen[str]") -> None: """Watch a process, logging its max memory usage.""" monitor = psutil.Process(sproc.pid) # Value must be list rather than integer to utilise pass-by-reference in python - memory_usage: MutableSequence[Optional[int]] = [None] + memory_usage: MutableSequence[int | None] = [None] mem_tm: "Optional[Timer]" = None - def get_tree_mem_usage(memory_usage: MutableSequence[Optional[int]]) -> None: + def get_tree_mem_usage(memory_usage: MutableSequence[int | None]) -> None: nonlocal mem_tm try: with monitor.oneshot(): @@ -553,7 +565,7 @@ class CommandLineJob(JobBase): def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: if tmpdir_lock: with tmpdir_lock: @@ -589,7 +601,7 @@ def run( self._execute([], self.environment, runtimeContext, monitor_function) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: env = {} env["HOME"] = self.outdir env["TMPDIR"] = self.tmpdir @@ -615,7 +627,7 @@ def get_from_requirements( pull_image: bool, force_pull: bool, tmp_outdir_prefix: str, - ) -> Optional[str]: + ) -> str | None: pass @abstractmethod @@ -623,26 +635,26 @@ def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext, - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], str | None]: """Return the list of commands to run the selected container engine.""" @staticmethod @abstractmethod - def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None: + def append_volume(runtime: list[str], source: str, target: str, writable: bool = False) -> None: """Add binding arguments to the runtime list.""" @abstractmethod def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: str | None ) -> None: """Append volume a file/dir mapping to the runtime option list.""" @abstractmethod def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: """Append a writable file mapping to the runtime option list.""" @@ -650,15 +662,15 @@ def add_writable_file_volume( @abstractmethod def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: """Append a writable directory mapping to the runtime option list.""" def _preserve_environment_on_containers_warning( - self, varnames: Optional[Iterable[str]] = None + self, varnames: Iterable[str] | None = None ) -> None: """When running in a container, issue a warning.""" if varnames is None: @@ -674,10 +686,10 @@ def _preserve_environment_on_containers_warning( def create_file_and_add_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], - secret_store: Optional[SecretStore], + host_outdir_tgt: str | None, + secret_store: SecretStore | None, tmpdir_prefix: str, ) -> str: """Create the file and add a mapping.""" @@ -706,15 +718,15 @@ def create_file_and_add_volume( def add_volumes( self, pathmapper: PathMapper, - runtime: List[str], + runtime: list[str], tmpdir_prefix: str, - secret_store: Optional[SecretStore] = None, + secret_store: SecretStore | None = None, any_path_okay: bool = False, ) -> None: """Append volume mappings to the runtime option list.""" container_outdir = self.builder.outdir for key, vol in (itm for itm in pathmapper.items() if itm[1].staged): - host_outdir_tgt: Optional[str] = None + host_outdir_tgt: str | None = None if vol.target.startswith(container_outdir + "/"): host_outdir_tgt = os.path.join(self.outdir, vol.target[len(container_outdir) + 1 :]) if not host_outdir_tgt and not any_path_okay: @@ -723,22 +735,23 @@ def add_volumes( "the designated output directory, also know as " "$(runtime.outdir): {}".format(vol) ) - if vol.type in ("File", "Directory"): - self.add_file_or_directory_volume(runtime, vol, host_outdir_tgt) - elif vol.type == "WritableFile": - self.add_writable_file_volume(runtime, vol, host_outdir_tgt, tmpdir_prefix) - elif vol.type == "WritableDirectory": - self.add_writable_directory_volume(runtime, vol, host_outdir_tgt, tmpdir_prefix) - elif vol.type in ["CreateFile", "CreateWritableFile"]: - new_path = self.create_file_and_add_volume( - runtime, vol, host_outdir_tgt, secret_store, tmpdir_prefix - ) - pathmapper.update(key, new_path, vol.target, vol.type, vol.staged) + match vol.type: + case "File" | "Directory": + self.add_file_or_directory_volume(runtime, vol, host_outdir_tgt) + case "WritableFile": + self.add_writable_file_volume(runtime, vol, host_outdir_tgt, tmpdir_prefix) + case "WritableDirectory": + self.add_writable_directory_volume(runtime, vol, host_outdir_tgt, tmpdir_prefix) + case "CreateFile" | "CreateWritableFile": + new_path = self.create_file_and_add_volume( + runtime, vol, host_outdir_tgt, secret_store, tmpdir_prefix + ) + pathmapper.update(key, new_path, vol.target, vol.type, vol.staged) def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: debug = runtimeContext.debug if tmpdir_lock: @@ -751,7 +764,7 @@ def run( (docker_req, docker_is_req) = self.get_requirement("DockerRequirement") self.prov_obj = runtimeContext.prov_obj - img_id = None + img_id: str | None = None user_space_docker_cmd = runtimeContext.user_space_docker_cmd if docker_req is not None and user_space_docker_cmd: # For user-space docker implementations, a local image name or ID @@ -809,7 +822,7 @@ def run( ) except Exception as err: container = "Singularity" if runtimeContext.singularity else "Docker" - _logger.debug("%s error", container, exc_info=True) + _logger.debug("%s error", container, exc_info=runtimeContext.debug) if docker_is_req: raise UnsupportedRequirement( f"{container} is required to run this tool: {str(err)}" @@ -856,7 +869,7 @@ def docker_monitor( # to stdout, but the container is frozen, thus allowing us to start the # monitoring process without dealing with the cidfile or too-fast # container execution - cid: Optional[str] = None + cid: str | None = None while cid is None: time.sleep(1) # This is needed to avoid a race condition where the job @@ -918,34 +931,30 @@ def docker_monitor( def _job_popen( - commands: List[str], - stdin_path: Optional[str], - stdout_path: Optional[str], - stderr_path: Optional[str], + commands: list[str], + stdin_path: str | None, + stdout_path: str | None, + stderr_path: str | None, env: Mapping[str, str], cwd: str, make_job_dir: Callable[[], str], - job_script_contents: Optional[str] = None, - timelimit: Optional[int] = None, - name: Optional[str] = None, - monitor_function: Optional[Callable[["subprocess.Popen[str]"], None]] = None, - default_stdout: Optional[Union[IO[bytes], TextIO]] = None, - default_stderr: Optional[Union[IO[bytes], TextIO]] = None, + job_script_contents: str | None = None, + timelimit: int | None = None, + name: str | None = None, + monitor_function: Callable[["subprocess.Popen[str]"], None] | None = None, + default_stdout: IO[bytes] | TextIO | None = None, + default_stderr: IO[bytes] | TextIO | None = None, ) -> int: if job_script_contents is None and not FORCE_SHELLED_POPEN: - stdin: Union[IO[bytes], int] = subprocess.PIPE + stdin: IO[bytes] | int = subprocess.PIPE if stdin_path is not None: stdin = open(stdin_path, "rb") - stdout = ( - default_stdout if default_stdout is not None else sys.stderr - ) # type: Union[IO[bytes], TextIO] + stdout: IO[bytes] | TextIO = default_stdout if default_stdout is not None else sys.stderr if stdout_path is not None: stdout = open(stdout_path, "wb") - stderr = ( - default_stderr if default_stderr is not None else sys.stderr - ) # type: Union[IO[bytes], TextIO] + stderr: IO[bytes] | TextIO = default_stderr if default_stderr is not None else sys.stderr if stderr_path is not None: stderr = open(stderr_path, "wb") @@ -968,7 +977,7 @@ def _job_popen( tm = None if timelimit is not None and timelimit > 0: - def terminate(): # type: () -> None + def terminate() -> None: try: _logger.warning( "[job %s] exceeded time limit of %d seconds and will be terminated", @@ -1044,7 +1053,7 @@ def terminate(): # type: () -> None tm = None if timelimit is not None and timelimit > 0: - def terminate(): # type: () -> None + def terminate() -> None: try: _logger.warning( "[job %s] exceeded time limit of %d seconds and will be terminated", diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index d6352f918..853c203be 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -7,18 +7,9 @@ import re import urllib import uuid +from collections.abc import MutableMapping, MutableSequence from functools import partial -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from typing import Any, Union, cast from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -76,7 +67,7 @@ def default_loader( - fetcher_constructor: Optional[FetcherCallableType] = None, + fetcher_constructor: FetcherCallableType | None = None, enable_dev: bool = False, doc_cache: bool = True, ) -> Loader: @@ -90,11 +81,11 @@ def default_loader( def resolve_tool_uri( argsworkflow: str, - resolver: Optional[ResolverType] = None, - fetcher_constructor: Optional[FetcherCallableType] = None, - document_loader: Optional[Loader] = None, -) -> Tuple[str, str]: - uri = None # type: Optional[str] + resolver: ResolverType | None = None, + fetcher_constructor: FetcherCallableType | None = None, + document_loader: Loader | None = None, +) -> tuple[str, str]: + uri: str | None = None split = urllib.parse.urlsplit(argsworkflow) # In case of Windows path, urlsplit misjudge Drive letters as scheme, here we are skipping that if split.scheme and split.scheme in ["http", "https", "file"]: @@ -115,9 +106,9 @@ def resolve_tool_uri( def fetch_document( - argsworkflow: Union[str, CWLObjectType], - loadingContext: Optional[LoadingContext] = None, -) -> Tuple[LoadingContext, CommentedMap, str]: + argsworkflow: str | CWLObjectType, + loadingContext: LoadingContext | None = None, +) -> tuple[LoadingContext, CommentedMap, str]: """Retrieve a CWL document.""" if loadingContext is None: loadingContext = LoadingContext() @@ -144,27 +135,27 @@ def fetch_document( return loadingContext, workflowobj, uri if isinstance(argsworkflow, MutableMapping): uri = cast(str, argsworkflow["id"]) if argsworkflow.get("id") else "_:" + str(uuid.uuid4()) - workflowobj = cast(CommentedMap, cmap(cast(Dict[str, Any], argsworkflow), fn=uri)) + workflowobj = cast(CommentedMap, cmap(cast(dict[str, Any], argsworkflow), fn=uri)) loadingContext.loader.idx[uri] = workflowobj return loadingContext, workflowobj, uri raise ValidationException("Must be URI or object: '%s'" % argsworkflow) def _convert_stdstreams_to_files( - workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str, int]], str] + workflowobj: CWLObjectType | MutableSequence[CWLObjectType | str | int] | str, ) -> None: - if isinstance(workflowobj, MutableMapping): - if workflowobj.get("class") == "CommandLineTool": + match workflowobj: + case {"class": "CommandLineTool", **rest} if isinstance(workflowobj, MutableMapping): with SourceLine( workflowobj, "outputs", ValidationException, _logger.isEnabledFor(logging.DEBUG), ): - outputs = workflowobj.get("outputs", []) + outputs = rest.get("outputs", []) if not isinstance(outputs, CommentedSeq): raise ValidationException('"outputs" section is not ' "valid.") - for out in cast(MutableSequence[CWLObjectType], workflowobj.get("outputs", [])): + for out in cast(MutableSequence[CWLObjectType], outputs): if not isinstance(out, CommentedMap): raise ValidationException(f"Output {out!r} is not a valid OutputParameter.") for streamtype in ["stdout", "stderr"]: @@ -174,8 +165,8 @@ def _convert_stdstreams_to_files( "Not allowed to specify outputBinding when" " using %s shortcut." % streamtype ) - if streamtype in workflowobj: - filename = workflowobj[streamtype] + if streamtype in rest: + filename = rest[streamtype] else: filename = str( hashlib.sha1( # nosec @@ -185,13 +176,13 @@ def _convert_stdstreams_to_files( workflowobj[streamtype] = filename out["type"] = "File" out["outputBinding"] = cmap({"glob": filename}) - for inp in cast(MutableSequence[CWLObjectType], workflowobj.get("inputs", [])): + for inp in cast(MutableSequence[CWLObjectType], rest.get("inputs", [])): if inp.get("type") == "stdin": if "inputBinding" in inp: raise ValidationException( "Not allowed to specify inputBinding when" " using stdin shortcut." ) - if "stdin" in workflowobj: + if "stdin" in rest: raise ValidationException( "Not allowed to specify stdin path when" " using stdin type shortcut." ) @@ -201,7 +192,7 @@ def _convert_stdstreams_to_files( % cast(str, inp["id"]).rpartition("#")[2].split("/")[-1] ) inp["type"] = "File" - else: + case MutableMapping(): for entry in workflowobj.values(): _convert_stdstreams_to_files( cast( @@ -213,22 +204,22 @@ def _convert_stdstreams_to_files( entry, ) ) - if isinstance(workflowobj, MutableSequence): - for entry in workflowobj: - _convert_stdstreams_to_files( - cast( - Union[ - CWLObjectType, - MutableSequence[Union[CWLObjectType, str, int]], - str, - ], - entry, + case MutableSequence(): + for entry in workflowobj: + _convert_stdstreams_to_files( + cast( + Union[ + CWLObjectType, + MutableSequence[Union[CWLObjectType, str, int]], + str, + ], + entry, + ) ) - ) def _add_blank_ids( - workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str]]] + workflowobj: CWLObjectType | MutableSequence[CWLObjectType | str], ) -> None: if isinstance(workflowobj, MutableMapping): if ( @@ -256,20 +247,21 @@ def _add_blank_ids( def _fast_parser_convert_stdstreams_to_files( - processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]] + processobj: cwl_v1_2.Process | MutableSequence[cwl_v1_2.Process], ) -> None: - if isinstance(processobj, cwl_v1_2.CommandLineTool): - cwl_v1_2_utils.convert_stdstreams_to_files(processobj) - elif isinstance(processobj, cwl_v1_2.Workflow): - for st in processobj.steps: - _fast_parser_convert_stdstreams_to_files(st.run) - elif isinstance(processobj, MutableSequence): - for p in processobj: - _fast_parser_convert_stdstreams_to_files(p) + match processobj: + case cwl_v1_2.CommandLineTool(): + cwl_v1_2_utils.convert_stdstreams_to_files(processobj) + case cwl_v1_2.Workflow(steps=steps): + for st in steps: + _fast_parser_convert_stdstreams_to_files(st.run) + case MutableSequence(): + for p in processobj: + _fast_parser_convert_stdstreams_to_files(p) def _fast_parser_expand_hint_class( - hints: Optional[Any], loadingOptions: cwl_v1_2.LoadingOptions + hints: Any | None, loadingOptions: cwl_v1_2.LoadingOptions ) -> None: if isinstance(hints, MutableSequence): for h in hints: @@ -280,19 +272,19 @@ def _fast_parser_expand_hint_class( def _fast_parser_handle_hints( - processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]], + processobj: cwl_v1_2.Process | MutableSequence[cwl_v1_2.Process], loadingOptions: cwl_v1_2.LoadingOptions, ) -> None: if isinstance(processobj, (cwl_v1_2.CommandLineTool, cwl_v1_2.Workflow)): _fast_parser_expand_hint_class(processobj.hints, loadingOptions) - - if isinstance(processobj, cwl_v1_2.Workflow): - for st in processobj.steps: - _fast_parser_expand_hint_class(st.hints, loadingOptions) - _fast_parser_handle_hints(st.run, loadingOptions) - elif isinstance(processobj, MutableSequence): - for p in processobj: - _fast_parser_handle_hints(p, loadingOptions) + match processobj: + case cwl_v1_2.Workflow(steps=steps): + for st in steps: + _fast_parser_expand_hint_class(st.hints, loadingOptions) + _fast_parser_handle_hints(st.run, loadingOptions) + case MutableSequence(): + for p in processobj: + _fast_parser_handle_hints(p, loadingOptions) def update_index(document_loader: Loader, pr: CommentedMap) -> None: @@ -301,12 +293,12 @@ def update_index(document_loader: Loader, pr: CommentedMap) -> None: def fast_parser( - workflowobj: Union[CommentedMap, CommentedSeq, None], - fileuri: Optional[str], + workflowobj: CommentedMap | CommentedSeq | None, + fileuri: str | None, uri: str, loadingContext: LoadingContext, fetcher: Fetcher, -) -> Tuple[Union[CommentedMap, CommentedSeq], CommentedMap]: +) -> tuple[CommentedMap | CommentedSeq, CommentedMap]: lopt = cwl_v1_2.LoadingOptions(idx=loadingContext.codegen_idx, fileuri=fileuri, fetcher=fetcher) if uri not in loadingContext.codegen_idx: @@ -322,11 +314,11 @@ def fast_parser( _fast_parser_convert_stdstreams_to_files(objects) _fast_parser_handle_hints(objects, loadopt) - processobj: Union[MutableMapping[str, Any], MutableSequence[Any], float, str, None] + processobj: MutableMapping[str, Any] | MutableSequence[Any] | float | str | None processobj = cwl_v1_2.save(objects, relative_uris=False) - metadata: Dict[str, Any] = {} + metadata: dict[str, Any] = {} metadata["id"] = loadopt.fileuri if loadopt.namespaces: @@ -353,7 +345,7 @@ def fast_parser( objects, loadopt = loadingContext.codegen_idx[nofrag] fileobj = cmap( cast( - Union[int, float, str, Dict[str, Any], List[Any], None], + Union[int, float, str, dict[str, Any], list[Any], None], cwl_v1_2.save(objects, relative_uris=False), ) ) @@ -370,16 +362,16 @@ def fast_parser( return cast( Union[CommentedMap, CommentedSeq], - cmap(cast(Union[Dict[str, Any], List[Any]], processobj)), + cmap(cast(Union[dict[str, Any], list[Any]], processobj)), ), cast(CommentedMap, cmap(metadata)) def resolve_and_validate_document( loadingContext: LoadingContext, - workflowobj: Union[CommentedMap, CommentedSeq], + workflowobj: CommentedMap | CommentedSeq, uri: str, preprocess_only: bool = False, -) -> Tuple[LoadingContext, str]: +) -> tuple[LoadingContext, str]: """Validate a CWL document.""" if not loadingContext.loader: raise ValueError("loadingContext must have a loader.") @@ -394,7 +386,7 @@ def resolve_and_validate_document( if "cwl:tool" in workflowobj: jobobj, _ = loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) - del cast(Dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] + del cast(dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] @@ -440,24 +432,25 @@ def resolve_and_validate_document( "\n{}".format("\n".join(versions)) ) - if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj: - loadingContext.overrides_list.extend(resolve_overrides(jobobj, uri, uri)) - del jobobj["http://commonwl.org/cwltool#overrides"] - - if isinstance(jobobj, CommentedMap) and "https://w3id.org/cwl/cwl#requirements" in jobobj: - if cwlVersion not in ("v1.1.0-dev1", "v1.1"): - raise ValidationException( - "`cwl:requirements` in the input object is not part of CWL " - "v1.0. You can adjust to use `cwltool:overrides` instead; or you " - "can set the cwlVersion to v1.1 or greater." + if isinstance(jobobj, CommentedMap): + if "http://commonwl.org/cwltool#overrides" in jobobj: + loadingContext.overrides_list.extend(resolve_overrides(jobobj, uri, uri)) + del jobobj["http://commonwl.org/cwltool#overrides"] + + if "https://w3id.org/cwl/cwl#requirements" in jobobj: + if cwlVersion not in ("v1.1.0-dev1", "v1.1"): + raise ValidationException( + "`cwl:requirements` in the input object is not part of CWL " + "v1.0. You can adjust to use `cwltool:overrides` instead; or you " + "can set the cwlVersion to v1.1 or greater." + ) + loadingContext.overrides_list.append( + { + "overrideTarget": uri, + "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"], + } ) - loadingContext.overrides_list.append( - { - "overrideTarget": uri, - "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"], - } - ) - del jobobj["https://w3id.org/cwl/cwl#requirements"] + del jobobj["https://w3id.org/cwl/cwl#requirements"] (sch_document_loader, avsc_names) = process.get_schema(cwlVersion)[:2] @@ -560,14 +553,12 @@ def resolve_and_validate_document( return loadingContext, uri -def make_tool( - uri: Union[str, CommentedMap, CommentedSeq], loadingContext: LoadingContext -) -> Process: +def make_tool(uri: str | CommentedMap | CommentedSeq, loadingContext: LoadingContext) -> Process: """Make a Python CWL object.""" if loadingContext.loader is None: raise ValueError("loadingContext must have a loader") - resolveduri: Union[float, str, CommentedMap, CommentedSeq, None] + resolveduri: float | str | CommentedMap | CommentedSeq | None metadata: CWLObjectType if loadingContext.fast_parser and isinstance(uri, str) and not loadingContext.skip_resolve_all: @@ -578,21 +569,24 @@ def make_tool( resolveduri, metadata = loadingContext.loader.resolve_ref(uri) processobj = None - if isinstance(resolveduri, MutableSequence): - for obj in resolveduri: - if obj["id"].endswith("#main"): - processobj = obj - break - if not processobj: - raise GraphTargetMissingException( - "Tool file contains graph of multiple objects, must specify " - "one of #%s" - % ", #".join(urllib.parse.urldefrag(i["id"])[1] for i in resolveduri if "id" in i) - ) - elif isinstance(resolveduri, MutableMapping): - processobj = resolveduri - else: - raise Exception("Must resolve to list or dict") + match resolveduri: + case MutableSequence(): + for obj in resolveduri: + if obj["id"].endswith("#main"): + processobj = obj + break + if not processobj: + raise GraphTargetMissingException( + "Tool file contains graph of multiple objects, must specify " + "one of #%s" + % ", #".join( + urllib.parse.urldefrag(i["id"])[1] for i in resolveduri if "id" in i + ) + ) + case MutableMapping(): + processobj = resolveduri + case _: + raise Exception(f"Must resolve to list or dict: {resolveduri}") tool = loadingContext.construct_tool_object(processobj, loadingContext) @@ -606,8 +600,8 @@ def make_tool( def load_tool( - argsworkflow: Union[str, CWLObjectType], - loadingContext: Optional[LoadingContext] = None, + argsworkflow: str | CWLObjectType, + loadingContext: LoadingContext | None = None, ) -> Process: loadingContext, workflowobj, uri = fetch_document(argsworkflow, loadingContext) @@ -624,27 +618,28 @@ def resolve_overrides( ov: IdxResultType, ov_uri: str, baseurl: str, -) -> List[CWLObjectType]: +) -> list[CWLObjectType]: ovloader = Loader(overrides_ctx) ret, _ = ovloader.resolve_all(ov, baseurl) if not isinstance(ret, CommentedMap): raise Exception("Expected CommentedMap, got %s" % type(ret)) cwl_docloader = get_schema("v1.0")[0] cwl_docloader.resolve_all(ret, ov_uri) - return cast(List[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"]) + return cast(list[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"]) -def load_overrides(ov: str, base_url: str) -> List[CWLObjectType]: +def load_overrides(ov: str, base_url: str) -> list[CWLObjectType]: + """Load and resolve any overrides.""" ovloader = Loader(overrides_ctx) return resolve_overrides(ovloader.fetch(ov), ov, base_url) def recursive_resolve_and_validate_document( loadingContext: LoadingContext, - workflowobj: Union[CommentedMap, CommentedSeq], + workflowobj: CommentedMap | CommentedSeq, uri: str, preprocess_only: bool = False, -) -> Tuple[LoadingContext, str, Process]: +) -> tuple[LoadingContext, str, Process]: """Validate a CWL document, checking that a tool object can be built.""" loadingContext, uri = resolve_and_validate_document( loadingContext, diff --git a/cwltool/loghandler.py b/cwltool/loghandler.py index 76daa8be9..c76830816 100644 --- a/cwltool/loghandler.py +++ b/cwltool/loghandler.py @@ -11,7 +11,7 @@ def configure_logging( - stderr_handler: logging.Handler, + err_handler: logging.Handler, no_warnings: bool, quiet: bool, debug: bool, @@ -21,25 +21,29 @@ def configure_logging( ) -> None: """Configure logging.""" rdflib_logger = logging.getLogger("rdflib.term") - rdflib_logger.addHandler(stderr_handler) + rdflib_logger.addHandler(err_handler) rdflib_logger.setLevel(logging.ERROR) deps_logger = logging.getLogger("galaxy.tool_util.deps") - deps_logger.addHandler(stderr_handler) + deps_logger.addHandler(err_handler) ss_logger = logging.getLogger("salad") - ss_logger.addHandler(stderr_handler) if no_warnings: - stderr_handler.setLevel(logging.ERROR) - if quiet: + err_handler.setLevel(logging.ERROR) + ss_logger.setLevel(logging.ERROR) + elif quiet: # Silence STDERR, not an eventual provenance log file - stderr_handler.setLevel(logging.WARN) + err_handler.setLevel(logging.WARN) + ss_logger.setLevel(logging.WARN) + else: + err_handler.setLevel(logging.INFO) + ss_logger.setLevel(logging.INFO) if debug: # Increase to debug for both stderr and provenance log file base_logger.setLevel(logging.DEBUG) - stderr_handler.setLevel(logging.DEBUG) + err_handler.setLevel(logging.DEBUG) rdflib_logger.setLevel(logging.DEBUG) deps_logger.setLevel(logging.DEBUG) fmtclass = coloredlogs.ColoredFormatter if enable_color else logging.Formatter formatter = fmtclass("%(levelname)s %(message)s") if timestamps: formatter = fmtclass("[%(asctime)s] %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S") - stderr_handler.setFormatter(formatter) + err_handler.setFormatter(formatter) diff --git a/cwltool/main.py b/cwltool/main.py index 30f299f09..4cbf356c7 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -15,26 +15,15 @@ import urllib import warnings from codecs import getwriter -from typing import ( - IO, - Any, - Callable, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Sized, - Tuple, - Union, - cast, -) +from collections.abc import Callable, Mapping, MutableMapping, MutableSequence, Sized +from importlib.resources import files +from typing import IO, Any, Union, cast import argcomplete import coloredlogs import requests import ruamel.yaml +from rich_argparse import RichHelpFormatter from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml.main import YAML from schema_salad.exceptions import ValidationException @@ -95,10 +84,6 @@ from .procgenerator import ProcessGenerator from .resolver import ga4gh_tool_registries, tool_resolver from .secrets import SecretStore -from .software_requirements import ( - DependenciesConfiguration, - get_container_from_software_requirements, -) from .stdfsaccess import StdFsAccess from .subgraph import get_process, get_step, get_subgraph from .update import ALLUPDATES, UPDATES @@ -108,7 +93,6 @@ CWLOutputType, HasReqsHints, adjustDirObjs, - files, normalizeFilesDirs, processes_to_kill, trim_listing, @@ -131,7 +115,6 @@ def _terminate_processes() -> None: continuing to execute while it kills the processes that they've spawned. This may occasionally lead to unexpected behaviour. """ - global docker_exe # It's possible that another thread will spawn a new task while # we're executing, so it's not safe to use a for loop here. while processes_to_kill: @@ -183,13 +166,13 @@ def append_word_to_default_user_agent(word: str) -> None: def generate_example_input( - inptype: Optional[CWLOutputType], - default: Optional[CWLOutputType], -) -> Tuple[Any, str]: + inptype: CWLOutputType | None, + default: CWLOutputType | None, +) -> tuple[Any, str]: """Convert a single input schema into an example.""" example = None comment = "" - defaults = { + defaults: CWLObjectType = { "null": "null", "Any": "null", "boolean": False, @@ -202,40 +185,40 @@ def generate_example_input( "Directory": ruamel.yaml.comments.CommentedMap( [("class", "Directory"), ("path", "a/directory/path")] ), - } # type: CWLObjectType - if isinstance(inptype, MutableSequence): - optional = False - if "null" in inptype: - inptype.remove("null") - optional = True - if len(inptype) == 1: - example, comment = generate_example_input(inptype[0], default) - if optional: - if comment: + } + match inptype: + case MutableSequence(): + optional = False + if "null" in inptype: + inptype.remove("null") + optional = True + if len(inptype) == 1: + example, comment = generate_example_input(inptype[0], default) + if optional: + if comment: + comment = f"{comment} (optional)" + else: + comment = "optional" + else: + example, comment = generate_example_input(inptype[0], default) + type_names = [] + for entry in inptype: + value, e_comment = generate_example_input(entry, default) + if e_comment: + type_names.append(e_comment) + comment = "one of " + ", ".join(type_names) + if optional: comment = f"{comment} (optional)" - else: - comment = "optional" - else: - example, comment = generate_example_input(inptype[0], default) - type_names = [] - for entry in inptype: - value, e_comment = generate_example_input(entry, default) - if e_comment: - type_names.append(e_comment) - comment = "one of " + ", ".join(type_names) - if optional: - comment = f"{comment} (optional)" - elif isinstance(inptype, Mapping) and "type" in inptype: - if inptype["type"] == "array": - first_item = cast(MutableSequence[CWLObjectType], inptype["items"])[0] - items_len = len(cast(Sized, inptype["items"])) + case {"type": "array", "items": list(items)}: + first_item = cast(CWLObjectType, items[0]) + items_len = len(items) if items_len == 1 and "type" in first_item and first_item["type"] == "enum": # array of just an enum then list all the options example = first_item["symbols"] if "name" in first_item: comment = 'array of type "{}".'.format(first_item["name"]) else: - value, comment = generate_example_input(inptype["items"], None) + value, comment = generate_example_input(items, None) comment = "array of " + comment if items_len == 1: example = [value] @@ -243,46 +226,44 @@ def generate_example_input( example = value if default is not None: example = default - elif inptype["type"] == "enum": - symbols = cast(List[str], inptype["symbols"]) + case {"type": "enum", "symbols": list(symbols), **rest}: if default is not None: example = default - elif "default" in inptype: - example = inptype["default"] - elif len(cast(Sized, inptype["symbols"])) == 1: + elif "default" in rest: + example = rest["default"] + elif len(symbols) == 1: example = symbols[0] else: - example = "{}_enum_value".format(inptype.get("name", "valid")) + example = "{}_enum_value".format(rest.get("name", "valid")) comment = 'enum; valid values: "{}"'.format('", "'.join(symbols)) - elif inptype["type"] == "record": + case {"type": "record", "fields": list(fields), **rest}: example = ruamel.yaml.comments.CommentedMap() - if "name" in inptype: - comment = '"{}" record type.'.format(inptype["name"]) + if "name" in rest: + comment = '"{}" record type.'.format(rest["name"]) else: comment = "Anonymous record type." - for field in cast(List[CWLObjectType], inptype["fields"]): + for field in cast(list[CWLObjectType], fields): value, f_comment = generate_example_input(field["type"], None) example.insert(0, shortname(cast(str, field["name"])), value, f_comment) - elif "default" in inptype: - example = inptype["default"] - comment = f"default value of type {inptype['type']!r}" - else: - example = defaults.get(cast(str, inptype["type"]), str(inptype)) - comment = f"type {inptype['type']!r}" - else: - if not default: + case {"type": str(inp_type), "default": default}: + example = default + comment = f"default value of type {inp_type!r}" + case {"type": str(inp_type)}: + example = defaults.get(inp_type, str(inptype)) + comment = f"type {inp_type!r}" + case object() if not default: example = defaults.get(str(inptype), str(inptype)) comment = f"type {inptype!r}" - else: + case _: example = default comment = f"default value of type {inptype!r}." return example, comment def realize_input_schema( - input_types: MutableSequence[Union[str, CWLObjectType]], + input_types: MutableSequence[str | CWLObjectType], schema_defs: MutableMapping[str, CWLObjectType], -) -> MutableSequence[Union[str, CWLObjectType]]: +) -> MutableSequence[str | CWLObjectType]: """Replace references to named typed with the actual types.""" for index, entry in enumerate(input_types): if isinstance(entry, str): @@ -317,7 +298,7 @@ def realize_input_schema( if isinstance(entry["type"], Mapping): entry["type"] = cast( CWLOutputType, - realize_input_schema([cast(CWLObjectType, entry["type"])], schema_defs), + realize_input_schema([entry["type"]], schema_defs), ) if entry["type"] == "array": items = entry["items"] if not isinstance(entry["items"], str) else [entry["items"]] @@ -343,10 +324,10 @@ def generate_input_template(tool: Process) -> CWLObjectType: """Generate an example input object for the given CWL process.""" template = ruamel.yaml.comments.CommentedMap() for inp in cast( - List[MutableMapping[str, str]], + list[CWLObjectType], realize_input_schema(tool.tool["inputs"], tool.schemaDefs), ): - name = shortname(inp["id"]) + name = shortname(cast(str, inp["id"])) value, comment = generate_example_input(inp["type"], inp.get("default", None)) template.insert(0, name, value, comment) return template @@ -355,10 +336,10 @@ def generate_input_template(tool: Process) -> CWLObjectType: def load_job_order( args: argparse.Namespace, stdin: IO[Any], - fetcher_constructor: Optional[FetcherCallableType], - overrides_list: List[CWLObjectType], + fetcher_constructor: FetcherCallableType | None, + overrides_list: list[CWLObjectType], tool_file_uri: str, -) -> Tuple[Optional[CWLObjectType], str, Loader]: +) -> tuple[CWLObjectType | None, str, Loader]: job_order_object = None job_order_file = None @@ -386,7 +367,10 @@ def load_job_order( content_types=CWL_CONTENT_TYPES, ) - if job_order_object is not None and "http://commonwl.org/cwltool#overrides" in job_order_object: + if ( + isinstance(job_order_object, CommentedMap) + and "http://commonwl.org/cwltool#overrides" in job_order_object + ): ov_uri = file_uri(job_order_file or input_basedir) overrides_list.extend(resolve_overrides(job_order_object, ov_uri, tool_file_uri)) del job_order_object["http://commonwl.org/cwltool#overrides"] @@ -408,7 +392,7 @@ def load_job_order( def init_job_order( - job_order_object: Optional[CWLObjectType], + job_order_object: CWLObjectType | None, args: argparse.Namespace, process: Process, loader: Loader, @@ -417,16 +401,19 @@ def init_job_order( relative_deps: str = "primary", make_fs_access: Callable[[str], StdFsAccess] = StdFsAccess, input_basedir: str = "", - secret_store: Optional[SecretStore] = None, + secret_store: SecretStore | None = None, input_required: bool = True, - runtime_context: Optional[RuntimeContext] = None, + runtime_context: RuntimeContext | None = None, ) -> CWLObjectType: secrets_req, _ = process.get_requirement("http://commonwl.org/cwltool#Secrets") if job_order_object is None: - namemap = {} # type: Dict[str, str] - records = [] # type: List[str] + namemap: dict[str, str] = {} + records: list[str] = [] toolparser = generate_parser( - argparse.ArgumentParser(prog=args.workflow), + argparse.ArgumentParser( + prog=args.workflow, + formatter_class=RichHelpFormatter, + ), process, namemap, records, @@ -463,7 +450,7 @@ def init_job_order( if secret_store and secrets_req: secret_store.store( - [shortname(sc) for sc in cast(List[str], secrets_req["secrets"])], + [shortname(sc) for sc in cast(list[str], secrets_req["secrets"])], job_order_object, ) @@ -486,7 +473,7 @@ def path_to_loc(p: CWLObjectType) -> None: p["location"] = p["path"] del p["path"] - ns = {} # type: ContextType + ns: ContextType = {} ns.update(cast(ContextType, job_order_object.get("$namespaces", {}))) ns.update(cast(ContextType, process.metadata.get("$namespaces", {}))) ld = Loader(ns) @@ -514,7 +501,7 @@ def expand_formats(p: CWLObjectType) -> None: builder.bind_input( process.inputs_record_schema, job_order_object, discover_secondaryFiles=True ) - basedir: Optional[str] = None + basedir: str | None = None uri = cast(str, job_order_object[jobloader_id_name]) if uri == args.workflow: basedir = os.path.dirname(uri) @@ -532,7 +519,7 @@ def expand_formats(p: CWLObjectType) -> None: if secret_store and secrets_req: secret_store.store( - [shortname(sc) for sc in cast(List[str], secrets_req["secrets"])], + [shortname(sc) for sc in cast(list[str], secrets_req["secrets"])], job_order_object, ) @@ -560,7 +547,7 @@ def printdeps( stdout: IO[str], relative_deps: str, uri: str, - basedir: Optional[str] = None, + basedir: str | None = None, nestdirs: bool = True, ) -> None: """Print a JSON representation of the dependencies of the CWL document.""" @@ -577,13 +564,13 @@ def prov_deps( obj: CWLObjectType, document_loader: Loader, uri: str, - basedir: Optional[str] = None, + basedir: str | None = None, ) -> CWLObjectType: deps = find_deps(obj, document_loader, uri, basedir=basedir) def remove_non_cwl(deps: CWLObjectType) -> None: if "secondaryFiles" in deps: - sec_files = cast(List[CWLObjectType], deps["secondaryFiles"]) + sec_files = cast(list[CWLObjectType], deps["secondaryFiles"]) for index, entry in enumerate(sec_files): if not ("format" in entry and entry["format"] == CWL_IANA): del sec_files[index] @@ -598,17 +585,17 @@ def find_deps( obj: CWLObjectType, document_loader: Loader, uri: str, - basedir: Optional[str] = None, + basedir: str | None = None, nestdirs: bool = True, ) -> CWLObjectType: """Find the dependencies of the CWL document.""" - deps = { + deps: CWLObjectType = { "class": "File", "location": uri, "format": CWL_IANA, - } # type: CWLObjectType + } - def loadref(base: str, uri: str) -> Union[CommentedMap, CommentedSeq, str, None]: + def loadref(base: str, uri: str) -> CommentedMap | CommentedSeq | str | None: return document_loader.fetch(document_loader.fetcher.urljoin(base, uri)) sfs = scandeps( @@ -638,7 +625,8 @@ def print_pack( return json_dumps(target, indent=4, default=str) -def supported_cwl_versions(enable_dev: bool) -> List[str]: +def supported_cwl_versions(enable_dev: bool) -> list[str]: + """Return a list of currently supported CWL versions.""" # ALLUPDATES and UPDATES are dicts if enable_dev: versions = list(ALLUPDATES) @@ -649,7 +637,7 @@ def supported_cwl_versions(enable_dev: bool) -> List[str]: def setup_schema( - args: argparse.Namespace, custom_schema_callback: Optional[Callable[[], None]] + args: argparse.Namespace, custom_schema_callback: Callable[[], None] | None ) -> None: if custom_schema_callback is not None: custom_schema_callback() @@ -679,7 +667,7 @@ def __init__(self) -> None: """Use the default formatter with our custom formatstring.""" super().__init__("[%(asctime)sZ] %(message)s") - def formatTime(self, record: logging.LogRecord, datefmt: Optional[str] = None) -> str: + def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str: """Override the default formatTime to include the timezone.""" formatted_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(float(record.created))) with_msecs = f"{formatted_time},{record.msecs:03f}" @@ -692,8 +680,8 @@ def formatTime(self, record: logging.LogRecord, datefmt: Optional[str] = None) - def setup_provenance( args: argparse.Namespace, runtimeContext: RuntimeContext, - argsl: Optional[List[str]] = None, -) -> Tuple[ProvOut, "logging.StreamHandler[ProvOut]"]: + argsl: list[str] | None = None, +) -> tuple[ProvOut, "logging.StreamHandler[ProvOut]"]: if not args.compute_checksum: _logger.error("--provenance incompatible with --no-compute-checksum") raise ArgumentException() @@ -718,7 +706,7 @@ def setup_provenance( def setup_loadingContext( - loadingContext: Optional[LoadingContext], + loadingContext: LoadingContext | None, runtimeContext: RuntimeContext, args: argparse.Namespace, ) -> LoadingContext: @@ -793,7 +781,7 @@ def choose_target( args: argparse.Namespace, tool: Process, loading_context: LoadingContext, -) -> Optional[Process]: +) -> Process | None: """Walk the Workflow, extract the subset matches all the args.targets.""" if loading_context.loader is None: raise Exception("loading_context.loader cannot be None") @@ -829,7 +817,7 @@ def choose_step( args: argparse.Namespace, tool: Process, loading_context: LoadingContext, -) -> Optional[Process]: +) -> Process | None: """Walk the given Workflow and extract just args.single_step.""" if loading_context.loader is None: raise Exception("loading_context.loader cannot be None") @@ -861,7 +849,7 @@ def choose_process( args: argparse.Namespace, tool: Process, loadingContext: LoadingContext, -) -> Optional[Process]: +) -> Process | None: """Walk the given Workflow and extract just args.single_process.""" if loadingContext.loader is None: raise Exception("loadingContext.loader cannot be None") @@ -893,7 +881,7 @@ def choose_process( def check_working_directories( runtimeContext: RuntimeContext, -) -> Optional[int]: +) -> int | None: """Make any needed working directories.""" for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"): if ( @@ -940,7 +928,7 @@ def print_targets( _logger.info("%s steps targets:", prefix[:-1]) for t in tool.tool["steps"]: print(f" {prefix}{shortname(t['id'])}", file=stdout) - run: Union[str, Process, Dict[str, Any]] = t["run"] + run: str | Process | dict[str, Any] = t["run"] if isinstance(run, str): process = make_tool(run, loading_context) elif isinstance(run, dict): @@ -951,18 +939,18 @@ def print_targets( def main( - argsl: Optional[List[str]] = None, - args: Optional[argparse.Namespace] = None, - job_order_object: Optional[CWLObjectType] = None, + argsl: list[str] | None = None, + args: argparse.Namespace | None = None, + job_order_object: CWLObjectType | None = None, stdin: IO[Any] = sys.stdin, - stdout: Optional[IO[str]] = None, + stdout: IO[str] | None = None, stderr: IO[Any] = sys.stderr, versionfunc: Callable[[], str] = versionstring, - logger_handler: Optional[logging.Handler] = None, - custom_schema_callback: Optional[Callable[[], None]] = None, - executor: Optional[JobExecutor] = None, - loadingContext: Optional[LoadingContext] = None, - runtimeContext: Optional[RuntimeContext] = None, + logger_handler: logging.Handler | None = None, + custom_schema_callback: Callable[[], None] | None = None, + executor: JobExecutor | None = None, + loadingContext: LoadingContext | None = None, + runtimeContext: RuntimeContext | None = None, input_required: bool = True, ) -> int: if stdout is None: # force UTF-8 even if the console is configured differently @@ -979,14 +967,8 @@ def main( stdout = cast(IO[str], stdout) _logger.removeHandler(defaultStreamHandler) - stderr_handler = logger_handler - if stderr_handler is not None: - _logger.addHandler(stderr_handler) - else: - coloredlogs.install(logger=_logger, stream=stderr) - stderr_handler = _logger.handlers[-1] workflowobj = None - prov_log_handler: Optional[logging.StreamHandler[ProvOut]] = None + prov_log_handler: logging.StreamHandler[ProvOut] | None = None global docker_exe user_agent = "cwltool" @@ -994,11 +976,12 @@ def main( user_agent += f" {progname}" # append the real program name as well append_word_to_default_user_agent(user_agent) + err_handler: logging.Handler = defaultStreamHandler try: if args is None: if argsl is None: argsl = sys.argv[1:] - addl = [] # type: List[str] + addl: list[str] = [] if "CWLTOOL_OPTIONS" in os.environ: c_opts = os.environ["CWLTOOL_OPTIONS"].split(" ") addl = [x for x in c_opts if x != ""] @@ -1009,6 +992,13 @@ def main( if not args.cidfile_dir: args.cidfile_dir = os.getcwd() del args.record_container_id + if logger_handler is not None: + err_handler = logger_handler + _logger.addHandler(err_handler) + else: + coloredlogs.install(logger=_logger, stream=stdout if args.validate else stderr) + err_handler = _logger.handlers[-1] + logging.getLogger("salad").handlers = _logger.handlers if runtimeContext is None: runtimeContext = RuntimeContext(vars(args)) @@ -1027,7 +1017,7 @@ def main( setattr(args, key, val) configure_logging( - stderr_handler, + err_handler, args.no_warnings, args.quiet, runtimeContext.debug, @@ -1062,7 +1052,7 @@ def main( setup_schema(args, custom_schema_callback) - prov_log_stream: Optional[Union[io.TextIOWrapper, WritableBagFile]] = None + prov_log_stream: io.TextIOWrapper | WritableBagFile | None = None if args.provenance: try: prov_log_stream, prov_log_handler = setup_provenance(args, runtimeContext, argsl) @@ -1071,6 +1061,11 @@ def main( loadingContext = setup_loadingContext(loadingContext, runtimeContext, args) + if loadingContext.research_obj: + # early forward parameters required for a single command line tool + runtimeContext.prov_host = loadingContext.host_provenance + runtimeContext.prov_user = loadingContext.user_provenance + uri, tool_file_uri = resolve_tool_uri( args.workflow, resolver=loadingContext.resolver, @@ -1250,7 +1245,7 @@ def main( if args.parallel: temp_executor = MultithreadedJobExecutor() runtimeContext.select_resources = temp_executor.select_resources - real_executor = temp_executor # type: JobExecutor + real_executor: JobExecutor = temp_executor else: real_executor = SingleJobExecutor() else: @@ -1260,7 +1255,7 @@ def main( runtimeContext.basedir = input_basedir if isinstance(tool, ProcessGenerator): - tfjob_order = {} # type: CWLObjectType + tfjob_order: CWLObjectType = {} if loadingContext.jobdefaults: tfjob_order.update(loadingContext.jobdefaults) if job_order_object: @@ -1290,7 +1285,7 @@ def main( if isinstance(err.code, int): return err.code else: - _logger.debug("Non-integer SystemExit: %s", err.code) + _logger.debug("Non-integer SystemExit: %s", err.code, exc_info=args.debug) return 1 del args.workflow @@ -1300,6 +1295,8 @@ def main( use_conda_dependencies = getattr(args, "beta_conda_dependencies", None) # str if conf_file or use_conda_dependencies: + from .software_requirements import DependenciesConfiguration + runtimeContext.job_script_provider = DependenciesConfiguration(args) else: runtimeContext.find_default_container = functools.partial( @@ -1425,19 +1422,20 @@ def loc_to_path(obj: CWLObjectType) -> None: # public API for logging.StreamHandler prov_log_handler.close() close_ro(research_obj, args.provenance) - - _logger.removeHandler(stderr_handler) + _logger.removeHandler(err_handler) _logger.addHandler(defaultStreamHandler) def find_default_container( builder: HasReqsHints, - default_container: Optional[str] = None, - use_biocontainers: Optional[bool] = None, - container_image_cache_path: Optional[str] = None, -) -> Optional[str]: + default_container: str | None = None, + use_biocontainers: bool | None = None, + container_image_cache_path: str | None = None, +) -> str | None: """Find a container.""" if not default_container and use_biocontainers: + from .software_requirements import get_container_from_software_requirements + default_container = get_container_from_software_requirements( use_biocontainers, builder, container_image_cache_path ) diff --git a/cwltool/mpi.py b/cwltool/mpi.py index 2cc1122c6..cea53e2f7 100644 --- a/cwltool/mpi.py +++ b/cwltool/mpi.py @@ -3,7 +3,8 @@ import inspect import os import re -from typing import List, Mapping, MutableMapping, Optional, Type, TypeVar, Union +from collections.abc import Mapping, MutableMapping +from typing import TypeVar from schema_salad.utils import yaml_no_ts @@ -17,11 +18,11 @@ def __init__( self, runner: str = "mpirun", nproc_flag: str = "-n", - default_nproc: Union[int, str] = 1, - extra_flags: Optional[List[str]] = None, - env_pass: Optional[List[str]] = None, - env_pass_regex: Optional[List[str]] = None, - env_set: Optional[Mapping[str, str]] = None, + default_nproc: int | str = 1, + extra_flags: list[str] | None = None, + env_pass: list[str] | None = None, + env_pass_regex: list[str] | None = None, + env_set: Mapping[str, str] | None = None, ) -> None: """ Initialize from the argument mapping. @@ -46,7 +47,7 @@ def __init__( self.env_set = env_set or {} @classmethod - def load(cls: Type[MpiConfigT], config_file_name: str) -> MpiConfigT: + def load(cls: type[MpiConfigT], config_file_name: str) -> MpiConfigT: """Create the MpiConfig object from the contents of a YAML file. The file must contain exactly one object, whose attributes must diff --git a/cwltool/mutation.py b/cwltool/mutation.py index 077b92cb7..622807ec6 100644 --- a/cwltool/mutation.py +++ b/cwltool/mutation.py @@ -1,10 +1,16 @@ -from collections import namedtuple -from typing import Dict, cast +"""Support for InplaceUpdateRequirement.""" + +from typing import NamedTuple, cast from .errors import WorkflowException from .utils import CWLObjectType -MutationState = namedtuple("MutationState", ["generation", "readers", "stepname"]) + +class _MutationState(NamedTuple): + generation: int + readers: list[str] + stepname: str + _generation = "http://commonwl.org/cwltool#generation" @@ -20,11 +26,11 @@ class MutationManager: def __init__(self) -> None: """Initialize.""" - self.generations: Dict[str, MutationState] = {} + self.generations: dict[str, _MutationState] = {} def register_reader(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj_generation = obj.get(_generation, 0) if obj_generation != current.generation: @@ -40,7 +46,7 @@ def register_reader(self, stepname: str, obj: CWLObjectType) -> None: def release_reader(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj_generation = obj.get(_generation, 0) if obj_generation != current.generation: @@ -55,7 +61,7 @@ def release_reader(self, stepname: str, obj: CWLObjectType) -> None: def register_mutation(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj_generation = obj.get(_generation, 0) if len(current.readers) > 0: @@ -73,11 +79,11 @@ def register_mutation(self, stepname: str, obj: CWLObjectType) -> None: ) ) - self.generations[loc] = MutationState(current.generation + 1, current.readers, stepname) + self.generations[loc] = _MutationState(current.generation + 1, current.readers, stepname) def set_generation(self, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj[_generation] = current.generation def unset_generation(self, obj: CWLObjectType) -> None: diff --git a/cwltool/pack.py b/cwltool/pack.py index c9fbc4e04..8c250c51b 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -2,17 +2,8 @@ import copy import urllib -from typing import ( - Any, - Callable, - Dict, - MutableMapping, - MutableSequence, - Optional, - Set, - Union, - cast, -) +from collections.abc import Callable, MutableMapping, MutableSequence +from typing import Any, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.ref_resolver import Loader, SubLoader @@ -28,9 +19,9 @@ def find_run( - d: Union[CWLObjectType, ResolveType], + d: CWLObjectType | ResolveType, loadref: LoadRefType, - runs: Set[str], + runs: set[str], ) -> None: if isinstance(d, MutableSequence): for s in d: @@ -41,12 +32,12 @@ def find_run( runs.add(d["run"]) find_run(loadref(None, d["run"]), loadref, runs) for s in d.values(): - find_run(s, loadref, runs) + find_run(cast(Union[CWLObjectType, ResolveType], s), loadref, runs) def find_ids( - d: Union[CWLObjectType, CWLOutputType, MutableSequence[CWLObjectType], None], - ids: Set[str], + d: CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType] | None, + ids: set[str], ) -> None: if isinstance(d, MutableSequence): for s in d: @@ -56,10 +47,11 @@ def find_ids( if i in d and isinstance(d[i], str): ids.add(cast(str, d[i])) for s2 in d.values(): - find_ids(cast(CWLOutputType, s2), ids) + find_ids(s2, ids) -def replace_refs(d: Any, rewrite: Dict[str, str], stem: str, newstem: str) -> None: +def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> None: + """Replace references with the actual value.""" if isinstance(d, MutableSequence): for s, v in enumerate(d): if isinstance(v, str): @@ -87,12 +79,12 @@ def replace_refs(d: Any, rewrite: Dict[str, str], stem: str, newstem: str) -> No def import_embed( - d: Union[MutableSequence[CWLObjectType], CWLObjectType, CWLOutputType], - seen: Set[str], + d: MutableSequence[CWLObjectType] | CWLObjectType | CWLOutputType, + seen: set[str], ) -> None: if isinstance(d, MutableSequence): for v in d: - import_embed(cast(CWLOutputType, v), seen) + import_embed(v, seen) elif isinstance(d, MutableMapping): for n in ("id", "name"): if n in d: @@ -108,14 +100,14 @@ def import_embed( break for k in sorted(d.keys()): - import_embed(cast(CWLOutputType, d[k]), seen) + import_embed(d[k], seen) def pack( loadingContext: LoadingContext, uri: str, - rewrite_out: Optional[Dict[str, str]] = None, - loader: Optional[Loader] = None, + rewrite_out: dict[str, str] | None = None, + loader: Loader | None = None, ) -> CWLObjectType: # The workflow document we have in memory right now may have been # updated to the internal CWL version. We need to reload the @@ -153,9 +145,9 @@ def pack( document_loader.idx[po["id"]] = CommentedMap(po.items()) document_loader.idx[metadata["id"]] = CommentedMap(metadata.items()) - found_versions = {cast(str, loadingContext.metadata["cwlVersion"])} # type: Set[str] + found_versions: set[str] = {cast(str, loadingContext.metadata["cwlVersion"])} - def loadref(base: Optional[str], lr_uri: str) -> ResolveType: + def loadref(base: str | None, lr_uri: str) -> ResolveType: lr_loadingContext = loadingContext.copy() lr_loadingContext.metadata = {} lr_loadingContext, lr_workflowobj, lr_uri = fetch_document(lr_uri, lr_loadingContext) @@ -167,15 +159,15 @@ def loadref(base: Optional[str], lr_uri: str) -> ResolveType: raise Exception("loader should not be None") return lr_loadingContext.loader.resolve_ref(lr_uri, base_url=base)[0] - input_ids: Set[str] = set() - output_ids: Set[str] = set() + input_ids: set[str] = set() + output_ids: set[str] = set() if isinstance(processobj, MutableSequence): mainobj = processobj[0] else: mainobj = processobj - find_ids(cast(Dict[str, Any], mainobj)["inputs"], input_ids) - find_ids(cast(Dict[str, Any], mainobj)["outputs"], output_ids) + find_ids(cast(dict[str, Any], mainobj)["inputs"], input_ids) + find_ids(cast(dict[str, Any], mainobj)["outputs"], output_ids) runs = {uri} find_run(processobj, loadref, runs) @@ -190,15 +182,15 @@ def loadref(base: Optional[str], lr_uri: str) -> ResolveType: for f in runs: find_ids(document_loader.resolve_ref(f)[0], input_ids) - input_names: Set[str] = set() - output_names: Set[str] = set() + input_names: set[str] = set() + output_names: set[str] = set() - rewrite_inputs: Dict[str, str] = {} - rewrite_outputs: Dict[str, str] = {} + rewrite_inputs: dict[str, str] = {} + rewrite_outputs: dict[str, str] = {} mainpath, _ = urllib.parse.urldefrag(uri) - def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) -> None: + def rewrite_id(r: str, mainuri: str, rewrite: dict[str, str], names: set[str]) -> None: if r == mainuri: rewrite[r] = "#main" elif r.startswith(mainuri) and r[len(mainuri)] in ("#", "/"): @@ -225,7 +217,7 @@ def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) - packed = CommentedMap((("$graph", CommentedSeq()), ("cwlVersion", update_to_version))) namespaces = metadata.get("$namespaces", None) - schemas: Set[str] = set() + schemas: set[str] = set() if "$schemas" in metadata: for each_schema in metadata["$schemas"]: schemas.add(each_schema) @@ -261,7 +253,7 @@ def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) - "Operation", ): continue - dc = cast(Dict[str, Any], copy.deepcopy(dcr)) + dc = cast(dict[str, Any], copy.deepcopy(dcr)) v = rewrite_inputs[r] dc["id"] = v for n in ("name", "cwlVersion", "$namespaces", "$schemas"): diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 0a06eb47b..0cf5f7086 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -1,20 +1,10 @@ -import collections import logging import os import stat import urllib import uuid -from typing import ( - Dict, - ItemsView, - Iterable, - Iterator, - KeysView, - List, - Optional, - Tuple, - cast, -) +from collections.abc import ItemsView, Iterable, Iterator, KeysView +from typing import NamedTuple, Optional, cast from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException @@ -25,31 +15,24 @@ from .stdfsaccess import abspath from .utils import CWLObjectType, dedup, downloadHttpFile -MapperEnt = collections.namedtuple("MapperEnt", ["resolved", "target", "type", "staged"]) -""" Mapper entries. -.. py:attribute:: resolved - :type: str +class MapperEnt(NamedTuple): + """Mapper entries.""" - The "real" path on the local file system (after resolving relative paths - and traversing symlinks - -.. py:attribute:: target - :type: str - - The path on the target file system (under stagedir) - -.. py:attribute:: type - :type: str - - The object type. One of "File", "Directory", "CreateFile", "WritableFile", - or "CreateWritableFile". - -.. py:attribute:: staged - :type: bool - - If the File has been staged yet -""" + resolved: str + """ + The "real" path on the local file system (after resolving relative paths + and traversing symlinks + """ + target: str + """The path on the target file system (under stagedir)""" + type: str | None + """ + The object type. One of "File", "Directory", "CreateFile", "WritableFile", + or "CreateWritableFile". + """ + staged: bool | None + """If the File has been staged yet.""" @mypyc_attr(allow_interpreted_subclasses=True) @@ -92,20 +75,20 @@ class PathMapper: def __init__( self, - referenced_files: List[CWLObjectType], + referenced_files: list[CWLObjectType], basedir: str, stagedir: str, separateDirs: bool = True, ) -> None: """Initialize the PathMapper.""" - self._pathmap: Dict[str, MapperEnt] = {} + self._pathmap: dict[str, MapperEnt] = {} self.stagedir = stagedir self.separateDirs = separateDirs self.setup(dedup(referenced_files), basedir) def visitlisting( self, - listing: List[CWLObjectType], + listing: list[CWLObjectType], stagedir: str, basedir: str, copy: bool = False, @@ -128,13 +111,13 @@ def visit( copy: bool = False, staged: bool = False, ) -> None: + if obj["location"] in self._pathmap: + return stagedir = cast(Optional[str], obj.get("dirname")) or stagedir tgt = os.path.join( stagedir, cast(str, obj["basename"]), ) - if obj["location"] in self._pathmap: - return if obj["class"] == "Directory": location = cast(str, obj["location"]) if location.startswith("file://"): @@ -147,7 +130,7 @@ def visit( if location.startswith("file://"): staged = False self.visitlisting( - cast(List[CWLObjectType], obj.get("listing", [])), + cast(list[CWLObjectType], obj.get("listing", [])), tgt, basedir, copy=copy, @@ -158,7 +141,7 @@ def visit( ab = abspath(path, basedir) if "contents" in obj and path.startswith("_:"): self._pathmap[path] = MapperEnt( - obj["contents"], + cast(str, obj["contents"]), tgt, "CreateWritableFile" if copy else "CreateFile", staged, @@ -189,16 +172,19 @@ def visit( deref, tgt, "WritableFile" if copy else "File", staged ) self.visitlisting( - cast(List[CWLObjectType], obj.get("secondaryFiles", [])), + cast(list[CWLObjectType], obj.get("secondaryFiles", [])), stagedir, basedir, copy=copy, staged=staged, ) - def setup(self, referenced_files: List[CWLObjectType], basedir: str) -> None: - # Go through each file and set the target to its own directory along - # with any secondary files. + def setup(self, referenced_files: list[CWLObjectType], basedir: str) -> None: + """ + For each file, set the target to its own directory. + + Also processes secondary files into that same directory. + """ stagedir = self.stagedir for fob in referenced_files: if self.separateDirs: @@ -246,15 +232,17 @@ def parents(path: str) -> Iterable[str]: def reversemap( self, target: str, - ) -> Optional[Tuple[str, str]]: + ) -> tuple[str, str] | None: """Find the (source, resolved_path) for the given target, if any.""" for k, v in self._pathmap.items(): if v[1] == target: return (k, v[0]) return None - def update(self, key: str, resolved: str, target: str, ctype: str, stage: bool) -> MapperEnt: - """Update an existine entry.""" + def update( + self, key: str, resolved: str, target: str, ctype: str | None, stage: bool | None + ) -> MapperEnt: + """Update an existing entry.""" m = MapperEnt(resolved, target, ctype, stage) self._pathmap[key] = m return m diff --git a/cwltool/process.py b/cwltool/process.py index bde035118..75069b859 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -13,25 +13,17 @@ import textwrap import urllib.parse import uuid -from os import scandir -from typing import ( - TYPE_CHECKING, - Any, +from collections.abc import ( Callable, - Dict, Iterable, Iterator, - List, MutableMapping, MutableSequence, - Optional, - Set, Sized, - Tuple, - Type, - Union, - cast, ) +from importlib.resources import files +from os import scandir +from typing import TYPE_CHECKING, Any, Optional, Union, cast from cwl_utils import expression from mypy_extensions import mypyc_attr @@ -70,7 +62,6 @@ aslist, cmp_like_py2, ensure_writable, - files, get_listing, normalizeFilesDirs, random_outdir, @@ -161,14 +152,12 @@ def filter(self, record: logging.LogRecord) -> bool: "vocab_res_proc.yml", ) -SCHEMA_CACHE: Dict[ - str, Tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader] -] = {} -SCHEMA_FILE: Optional[CWLObjectType] = None -SCHEMA_DIR: Optional[CWLObjectType] = None -SCHEMA_ANY: Optional[CWLObjectType] = None +SCHEMA_CACHE: dict[str, tuple[Loader, Names | SchemaParseException, CWLObjectType, Loader]] = {} +SCHEMA_FILE: CWLObjectType | None = None +SCHEMA_DIR: CWLObjectType | None = None +SCHEMA_ANY: CWLObjectType | None = None -custom_schemas: Dict[str, Tuple[str, str]] = {} +custom_schemas: dict[str, tuple[str, str]] = {} def use_standard_schema(version: str) -> None: @@ -186,11 +175,11 @@ def use_custom_schema(version: str, name: str, text: str) -> None: def get_schema( version: str, -) -> Tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader]: +) -> tuple[Loader, Names | SchemaParseException, CWLObjectType, Loader]: if version in SCHEMA_CACHE: return SCHEMA_CACHE[version] - cache: Dict[str, Union[str, Graph, bool]] = {} + cache: dict[str, str | Graph | bool] = {} version = version.split("#")[-1] if ".dev" in version: version = ".".join(version.split(".")[:-1]) @@ -232,10 +221,10 @@ def shortname(inputid: str) -> str: def stage_files( pathmapper: PathMapper, - stage_func: Optional[Callable[[str, str], None]] = None, + stage_func: Callable[[str, str], None] | None = None, ignore_writable: bool = False, symlink: bool = True, - secret_store: Optional[SecretStore] = None, + secret_store: SecretStore | None = None, fix_conflicts: bool = False, ) -> None: """ @@ -244,9 +233,9 @@ def stage_files( :raises WorkflowException: if there is a file staging conflict """ items = pathmapper.items() if not symlink else pathmapper.items_exclude_children() - targets: Dict[str, MapperEnt] = {} + targets: dict[str, MapperEnt] = {} for key, entry in list(items): - if "File" not in entry.type: + if entry.type is None or "File" not in entry.type: continue if entry.target not in targets: targets[entry.target] = entry @@ -273,47 +262,45 @@ def stage_files( continue if not os.path.exists(os.path.dirname(entry.target)): os.makedirs(os.path.dirname(entry.target)) - if entry.type in ("File", "Directory") and os.path.exists(entry.resolved): - if symlink: # Use symlink func if allowed - os.symlink(entry.resolved, entry.target) - elif stage_func is not None: + match entry.type: + case "File" | "Directory" if os.path.exists(entry.resolved) and symlink: + os.symlink(entry.resolved, entry.target) # Use symlink func if allowed + case "File" | "Directory" if os.path.exists(entry.resolved) and stage_func is not None: stage_func(entry.resolved, entry.target) - elif ( - entry.type == "Directory" - and not os.path.exists(entry.target) - and entry.resolved.startswith("_:") - ): - os.makedirs(entry.target) - elif entry.type == "WritableFile" and not ignore_writable: - shutil.copy(entry.resolved, entry.target) - ensure_writable(entry.target) - elif entry.type == "WritableDirectory" and not ignore_writable: - if entry.resolved.startswith("_:"): + case "Directory" if not os.path.exists(entry.target) and entry.resolved.startswith( + "_:" + ): os.makedirs(entry.target) - else: - shutil.copytree(entry.resolved, entry.target) - ensure_writable(entry.target, include_root=True) - elif entry.type == "CreateFile" or entry.type == "CreateWritableFile": - with open(entry.target, "w") as new: - if secret_store is not None: - new.write(cast(str, secret_store.retrieve(entry.resolved))) - else: - new.write(entry.resolved) - if entry.type == "CreateFile": - os.chmod(entry.target, stat.S_IRUSR) # Read only - else: # it is a "CreateWritableFile" + case "WritableFile" if not ignore_writable: + shutil.copy(entry.resolved, entry.target) ensure_writable(entry.target) - pathmapper.update(key, entry.target, entry.target, entry.type, entry.staged) + case "WritableDirectory" if not ignore_writable: + if entry.resolved.startswith("_:"): + os.makedirs(entry.target) + else: + shutil.copytree(entry.resolved, entry.target) + ensure_writable(entry.target, include_root=True) + case "CreateFile" | "CreateWritableFile" as etype: + with open(entry.target, "w") as new: + if secret_store is not None: + new.write(cast(str, secret_store.retrieve(entry.resolved))) + else: + new.write(entry.resolved) + if etype == "CreateFile": + os.chmod(entry.target, stat.S_IRUSR) # Read only + else: # it is a "CreateWritableFile" + ensure_writable(entry.target) + pathmapper.update(key, entry.target, entry.target, entry.type, entry.staged) def relocateOutputs( outputObj: CWLObjectType, destination_path: str, - source_directories: Set[str], + source_directories: set[str], action: str, fs_access: StdFsAccess, compute_checksum: bool = True, - path_mapper: Type[PathMapper] = PathMapper, + path_mapper: type[PathMapper] = PathMapper, ) -> CWLObjectType: adjustDirObjs(outputObj, functools.partial(get_listing, fs_access, recursive=True)) @@ -321,7 +308,7 @@ def relocateOutputs( return outputObj def _collectDirEntries( - obj: Union[CWLObjectType, MutableSequence[CWLObjectType], None] + obj: CWLObjectType | MutableSequence[CWLObjectType] | None, ) -> Iterator[CWLObjectType]: if isinstance(obj, dict): if obj.get("class") in ("File", "Directory"): @@ -414,7 +401,7 @@ def add_sizes(fsaccess: StdFsAccess, obj: CWLObjectType) -> None: def fill_in_defaults( - inputs: List[CWLObjectType], + inputs: list[CWLObjectType], job: CWLObjectType, fsaccess: StdFsAccess, ) -> None: @@ -440,31 +427,31 @@ def fill_in_defaults( def avroize_type( - field_type: Union[CWLObjectType, MutableSequence[Any], CWLOutputType, None], + field_type: CWLObjectType | MutableSequence[Any] | CWLOutputType | None, name_prefix: str = "", -) -> Union[CWLObjectType, MutableSequence[Any], CWLOutputType, None]: +) -> CWLObjectType | MutableSequence[Any] | CWLOutputType | None: """Add missing information to a type so that CWL types are valid.""" - if isinstance(field_type, MutableSequence): - for i, field in enumerate(field_type): - field_type[i] = avroize_type(field, name_prefix) - elif isinstance(field_type, MutableMapping): - if field_type["type"] in ("enum", "record"): - if "name" not in field_type: - field_type["name"] = name_prefix + str(uuid.uuid4()) - if field_type["type"] == "record": - field_type["fields"] = avroize_type( - cast(MutableSequence[CWLOutputType], field_type["fields"]), name_prefix - ) - elif field_type["type"] == "array": - field_type["items"] = avroize_type( - cast(MutableSequence[CWLOutputType], field_type["items"]), name_prefix + match field_type: + case MutableSequence(): + for i, field in enumerate(field_type): + field_type[i] = avroize_type(field, name_prefix) + case {"type": "enum" | "record" as f_type, **rest}: + if "name" not in rest: + cast(CWLObjectType, field_type)["name"] = name_prefix + str(uuid.uuid4()) + if f_type == "record": + cast(CWLObjectType, field_type)["fields"] = avroize_type( + cast(MutableSequence[CWLOutputType], rest["fields"]), name_prefix + ) + case {"type": "array", "items": items}: + cast(CWLObjectType, field_type)["items"] = avroize_type( + cast(MutableSequence[CWLOutputType], items), name_prefix ) - else: - field_type["type"] = avroize_type(cast(CWLOutputType, field_type["type"]), name_prefix) - elif field_type == "File": - return "org.w3id.cwl.cwl.File" - elif field_type == "Directory": - return "org.w3id.cwl.cwl.Directory" + case {"type": f_type}: + cast(CWLObjectType, field_type)["type"] = avroize_type(f_type, name_prefix) + case "File": + return "org.w3id.cwl.cwl.File" + case "Directory": + return "org.w3id.cwl.cwl.Directory" return field_type @@ -491,8 +478,8 @@ def get_overrides(overrides: MutableSequence[CWLObjectType], toolid: str) -> CWL def var_spool_cwl_detector( obj: CWLOutputType, - item: Optional[Any] = None, - obj_key: Optional[Any] = None, + item: Any | None = None, + obj_key: Any | None = None, ) -> bool: """Detect any textual reference to /var/spool/cwl.""" r = False @@ -506,30 +493,34 @@ def var_spool_cwl_detector( r = True elif isinstance(obj, MutableMapping): for mkey, mvalue in obj.items(): - r = var_spool_cwl_detector(cast(CWLOutputType, mvalue), obj, mkey) or r + r = var_spool_cwl_detector(mvalue, obj, mkey) or r elif isinstance(obj, MutableSequence): for lkey, lvalue in enumerate(obj): - r = var_spool_cwl_detector(cast(CWLOutputType, lvalue), obj, lkey) or r + r = var_spool_cwl_detector(lvalue, obj, lkey) or r return r -def eval_resource( - builder: Builder, resource_req: Union[str, int, float] -) -> Optional[Union[str, int, float]]: +def eval_resource(builder: Builder, resource_req: str | int | float) -> str | int | float | None: + """Evaluate any CWL expressions inside a ResourceRequirement.""" if isinstance(resource_req, str) and expression.needs_parsing(resource_req): result = builder.do_eval(resource_req) - if isinstance(result, float): - if ORDERED_VERSIONS.index(builder.cwlVersion) >= ORDERED_VERSIONS.index("v1.2.0-dev4"): + match result: + case float(f_result) if ORDERED_VERSIONS.index( + builder.cwlVersion + ) >= ORDERED_VERSIONS.index("v1.2.0-dev4"): + return f_result + case float(): + raise WorkflowException( + "Floats are not valid in resource requirement expressions prior " + f"to CWL v1.2: {resource_req} returned {result}." + ) + case str() | int() | None: return result - raise WorkflowException( - "Floats are not valid in resource requirement expressions prior " - f"to CWL v1.2: {resource_req} returned {result}." - ) - if isinstance(result, (str, int)) or result is None: - return result - raise WorkflowException( - f"Got incorrect return type {type(result)} from resource expression evaluation of {resource_req}." - ) + case _: + raise WorkflowException( + f"Got incorrect return type {type(result)} from resource " + "expression evaluation of {resource_req}." + ) return resource_req @@ -578,7 +569,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext self.tool["id"] = "_:" + str(uuid.uuid4()) self.requirements.extend( cast( - List[CWLObjectType], + list[CWLObjectType], get_overrides(getdefault(loadingContext.overrides_list, []), self.tool["id"]).get( "requirements", [] ), @@ -597,7 +588,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext self.doc_loader = loadingContext.loader self.doc_schema = loadingContext.avsc_names - self.formatgraph: Optional[Graph] = None + self.formatgraph: Graph | None = None if self.doc_loader is not None: self.formatgraph = self.doc_loader.graph @@ -617,7 +608,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext avroize_type(cast(MutableSequence[CWLOutputType], sdtypes)) av = make_valid_avro( sdtypes, - {cast(str, t["name"]): cast(Dict[str, Any], t) for t in sdtypes}, + {cast(str, t["name"]): cast(dict[str, Any], t) for t in sdtypes}, set(), vocab=INPUT_OBJ_VOCAB, ) @@ -655,9 +646,9 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext c["type"] = avroize_type(c["type"], c["name"]) if key == "inputs": - cast(List[CWLObjectType], self.inputs_record_schema["fields"]).append(c) + cast(list[CWLObjectType], self.inputs_record_schema["fields"]).append(c) elif key == "outputs": - cast(List[CWLObjectType], self.outputs_record_schema["fields"]).append(c) + cast(list[CWLObjectType], self.outputs_record_schema["fields"]).append(c) with SourceLine(toolpath_object, "inputs", ValidationException, debug): self.inputs_record_schema = cast( @@ -681,7 +672,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext if toolpath_object.get("class") is not None and not getdefault( loadingContext.disable_js_validation, False ): - validate_js_options: Optional[Dict[str, Union[List[str], str, int]]] = None + validate_js_options: dict[str, list[str] | str | int] | None = None if loadingContext.js_hint_options_file is not None: try: with open(loadingContext.js_hint_options_file) as options_file: @@ -784,7 +775,7 @@ def _init_job(self, joborder: CWLObjectType, runtime_context: RuntimeContext) -> v = job[k] dircount = [0] - def inc(d: List[int]) -> None: + def inc(d: list[int]) -> None: d[0] += 1 visit_class(v, ("Directory",), lambda x: inc(dircount)) # noqa: B023 @@ -820,18 +811,53 @@ def inc(d: List[int]) -> None: except (ValidationException, WorkflowException) as err: raise WorkflowException("Invalid job input record:\n" + str(err)) from err - files: List[CWLObjectType] = [] + files: list[CWLObjectType] = [] bindings = CommentedSeq() outdir = "" tmpdir = "" stagedir = "" - docker_req, _ = self.get_requirement("DockerRequirement") + docker_req, docker_required = self.get_requirement("DockerRequirement") default_docker = None + mpi_req, mpi_required = self.get_requirement(MPIRequirementName) if docker_req is None and runtime_context.default_container: default_docker = runtime_context.default_container + if ( + docker_req is not None + and runtime_context.use_container + and not runtime_context.singularity + and not runtime_context.user_space_docker_cmd + and mpi_req is not None + ): + if mpi_required: + if docker_required: + raise UnsupportedRequirement( + "No support for DockerRequirement and MPIRequirement " + "both being required, unless Singularity or uDocker is being used." + ) + else: + _logger.warning( + "MPI has been required while DockerRequirement is hinted " + "and neither Singularity nor uDocker is being used, discarding Docker hint(s)." + ) + self.hints = [h for h in self.hints if h["class"] != "DockerRequirement"] + docker_req = None + docker_required = False + else: + if docker_required: + _logger.warning( + "Docker has been required (and neither Singularity nor " + "uDocker is being used) while MPI is hinted, discarding MPI hint(s)/" + ) + self.hints = [h for h in self.hints if h["class"] != MPIRequirementName] + else: + raise UnsupportedRequirement( + "Both Docker and MPI have been hinted and neither " + "Singularity nor uDocker are being used - don't know what to do." + ) + if (docker_req or default_docker) and runtime_context.use_container: if docker_req is not None: # Check if docker output directory is absolute @@ -947,7 +973,7 @@ def inc(d: List[int]) -> None: def evalResources( self, builder: Builder, runtimeContext: RuntimeContext - ) -> Dict[str, Union[int, float]]: + ) -> dict[str, int | float]: resourceReq, _ = self.get_requirement("ResourceRequirement") if resourceReq is None: resourceReq = {} @@ -957,7 +983,7 @@ def evalResources( ram = 1024 else: ram = 256 - request: Dict[str, Union[int, float, str]] = { + request: dict[str, int | float | str] = { "coresMin": 1, "coresMax": 1, "ramMin": ram, @@ -982,8 +1008,8 @@ def evalResources( ): if rsc is None: continue - mn: Optional[Union[int, float]] = None - mx: Optional[Union[int, float]] = None + mn: int | float | None = None + mx: int | float | None = None if rsc.get(a + "Min"): with SourceLine(rsc, f"{a}Min", WorkflowException, runtimeContext.debug): mn = cast( @@ -1005,7 +1031,7 @@ def evalResources( request[a + "Min"] = mn request[a + "Max"] = cast(Union[int, float], mx) - request_evaluated = cast(Dict[str, Union[int, float]], request) + request_evaluated = cast(dict[str, Union[int, float]], request) if runtimeContext.select_resources is not None: # Call select resources hook return runtimeContext.select_resources(request_evaluated, runtimeContext) @@ -1022,7 +1048,7 @@ def evalResources( def checkRequirements( self, - rec: Union[MutableSequence[CWLObjectType], CWLObjectType, CWLOutputType, None], + rec: MutableSequence[CWLObjectType] | CWLObjectType | CWLOutputType | None, supported_process_requirements: Iterable[str], ) -> None: """Check the presence of unsupported requirements.""" @@ -1038,7 +1064,7 @@ def checkRequirements( f"Unsupported requirement {entry['class']}." ) - def validate_hints(self, avsc_names: Names, hints: List[CWLObjectType], strict: bool) -> None: + def validate_hints(self, avsc_names: Names, hints: list[CWLObjectType], strict: bool) -> None: """Process the hints field.""" if self.doc_loader is None: return @@ -1085,11 +1111,11 @@ def __str__(self) -> str: return f"{type(self).__name__}: {self.tool['id']}" -_names: Set[str] = set() +_names: set[str] = set() -def uniquename(stem: str, names: Optional[Set[str]] = None) -> str: - global _names +def uniquename(stem: str, names: set[str] | None = None) -> str: + """Construct a thread-unique name using the given stem as a prefix.""" if names is None: names = _names c = 1 @@ -1123,8 +1149,8 @@ def nestdir(base: str, deps: CWLObjectType) -> CWLObjectType: def mergedirs( listing: MutableSequence[CWLObjectType], ) -> MutableSequence[CWLObjectType]: - r: List[CWLObjectType] = [] - ents: Dict[str, CWLObjectType] = {} + r: list[CWLObjectType] = [] + ents: dict[str, CWLObjectType] = {} for e in listing: basename = cast(str, e["basename"]) if basename not in ents: @@ -1138,14 +1164,14 @@ def mergedirs( if e.get("listing"): # name already in entries # merge it into the existing listing - cast(List[CWLObjectType], ents[basename].setdefault("listing", [])).extend( - cast(List[CWLObjectType], e["listing"]) + cast(list[CWLObjectType], ents[basename].setdefault("listing", [])).extend( + cast(list[CWLObjectType], e["listing"]) ) for e in ents.values(): if e["class"] == "Directory" and "listing" in e: e["listing"] = cast( MutableSequence[CWLOutputType], - mergedirs(cast(List[CWLObjectType], e["listing"])), + mergedirs(cast(list[CWLObjectType], e["listing"])), ) r.extend(ents.values()) return r @@ -1156,10 +1182,10 @@ def mergedirs( def scandeps( base: str, - doc: Union[CWLObjectType, MutableSequence[CWLObjectType]], - reffields: Set[str], - urlfields: Set[str], - loadref: Callable[[str, str], Union[CommentedMap, CommentedSeq, str, None]], + doc: CWLObjectType | MutableSequence[CWLObjectType], + reffields: set[str], + urlfields: set[str], + loadref: Callable[[str, str], CommentedMap | CommentedSeq | str | None], urljoin: Callable[[str, str], str] = urllib.parse.urljoin, nestdirs: bool = True, ) -> MutableSequence[CWLObjectType]: @@ -1200,52 +1226,54 @@ def scandeps( } if "basename" in doc: deps["basename"] = doc["basename"] - if doc["class"] == "Directory" and "listing" in doc: - deps["listing"] = doc["listing"] - if doc["class"] == "File" and "secondaryFiles" in doc: - deps["secondaryFiles"] = cast( - CWLOutputType, - scandeps( - base, - cast( - Union[CWLObjectType, MutableSequence[CWLObjectType]], - doc["secondaryFiles"], + match doc: + case {"class": "Directory", "listing": listing}: + deps["listing"] = listing + case {"class": "File", "secondaryFiles": sec_files}: + deps["secondaryFiles"] = cast( + CWLOutputType, + scandeps( + base, + cast( + Union[CWLObjectType, MutableSequence[CWLObjectType]], + sec_files, + ), + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, ), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, - ), - ) + ) if nestdirs: deps = nestdir(base, deps) r.append(deps) else: - if doc["class"] == "Directory" and "listing" in doc: - r.extend( - scandeps( - base, - cast(MutableSequence[CWLObjectType], doc["listing"]), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, + match doc: + case {"class": "Directory", "listing": listing}: + r.extend( + scandeps( + base, + cast(MutableSequence[CWLObjectType], listing), + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, + ) ) - ) - elif doc["class"] == "File" and "secondaryFiles" in doc: - r.extend( - scandeps( - base, - cast(MutableSequence[CWLObjectType], doc["secondaryFiles"]), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, + case {"class": "File", "secondaryFiles": sec_files}: + r.extend( + scandeps( + base, + cast(MutableSequence[CWLObjectType], sec_files), + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, + ) ) - ) for k, v in doc.items(): if k in reffields: diff --git a/cwltool/procgenerator.py b/cwltool/procgenerator.py index 34c1e650f..eb7eca076 100644 --- a/cwltool/procgenerator.py +++ b/cwltool/procgenerator.py @@ -1,5 +1,5 @@ import copy -from typing import Dict, Optional, Tuple, cast +from typing import cast from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException @@ -19,10 +19,10 @@ class ProcessGeneratorJob: def __init__(self, procgenerator: "ProcessGenerator") -> None: """Create a ProccessGenerator Job.""" self.procgenerator = procgenerator - self.jobout = None # type: Optional[CWLObjectType] - self.processStatus = None # type: Optional[str] + self.jobout: CWLObjectType | None = None + self.processStatus: str | None = None - def receive_output(self, jobout: Optional[CWLObjectType], processStatus: str) -> None: + def receive_output(self, jobout: CWLObjectType | None, processStatus: str) -> None: """Process the results.""" self.jobout = jobout self.processStatus = processStatus @@ -57,7 +57,7 @@ def job( except WorkflowException: raise except Exception as exc: - _logger.exception("Unexpected exception") + _logger.exception("Unexpected exception", exc_info=runtimeContext.debug) raise WorkflowException(str(exc)) from exc @@ -69,18 +69,18 @@ def __init__( ) -> None: """Create a ProcessGenerator from the given dictionary and context.""" super().__init__(toolpath_object, loadingContext) - self.loadingContext = loadingContext # type: LoadingContext + self.loadingContext: LoadingContext = loadingContext try: if isinstance(toolpath_object["run"], CommentedMap): - self.embedded_tool = loadingContext.construct_tool_object( + self.embedded_tool: Process = loadingContext.construct_tool_object( toolpath_object["run"], loadingContext - ) # type: Process + ) else: loadingContext.metadata = {} self.embedded_tool = load_tool(toolpath_object["run"], loadingContext) except ValidationException as vexc: if loadingContext.debug: - _logger.exception("Validation exception") + _logger.exception("Validation exception", exc_info=loadingContext.debug) raise WorkflowException( "Tool definition %s failed validation:\n%s" % (toolpath_object["run"], indent(str(vexc))) @@ -99,16 +99,16 @@ def result( job_order: CWLObjectType, jobout: CWLObjectType, runtimeContext: RuntimeContext, - ) -> Tuple[Process, CWLObjectType]: + ) -> tuple[Process, CWLObjectType]: try: loadingContext = self.loadingContext.copy() loadingContext.metadata = {} embedded_tool = load_tool( - cast(Dict[str, str], jobout["runProcess"])["location"], loadingContext + cast(dict[str, str], jobout["runProcess"])["location"], loadingContext ) except ValidationException as vexc: if runtimeContext.debug: - _logger.exception("Validation exception") + _logger.exception("Validation exception", exc_info=runtimeContext.debug) raise WorkflowException( "Tool definition %s failed validation:\n%s" % (jobout["runProcess"], indent(str(vexc))) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index e48957f26..ed8c1b410 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -3,20 +3,20 @@ import os import urllib from pathlib import Path -from typing import Optional from schema_salad.ref_resolver import Loader from .loghandler import _logger -def resolve_local(document_loader: Optional[Loader], uri: str) -> Optional[str]: +def resolve_local(document_loader: Loader | None, uri: str) -> str | None: + """Use the local resolver to find the target of the URI.""" pathpart, frag = urllib.parse.urldefrag(uri) try: pathobj = Path(pathpart).resolve() - except OSError: - _logger.debug("local resolver could not resolve %s", uri) + except OSError as exc: + _logger.debug("local resolver could not resolve %s due to %s", uri, str(exc)) return None if pathobj.is_file(): @@ -40,7 +40,8 @@ def resolve_local(document_loader: Optional[Loader], uri: str) -> Optional[str]: return None -def tool_resolver(document_loader: Loader, uri: str) -> Optional[str]: +def tool_resolver(document_loader: Loader, uri: str) -> str | None: + """Try both the local resolver and the GA4GH TRS resolver, in that order.""" for r in [resolve_local, resolve_ga4gh_tool]: ret = r(document_loader, uri) if ret is not None: @@ -62,7 +63,8 @@ def tool_resolver(document_loader: Loader, uri: str) -> Optional[str]: GA4GH_TRS_PRIMARY_DESCRIPTOR = "{0}/api/ga4gh/v2/tools/{1}/versions/{2}/plain-CWL/descriptor/{3}" -def resolve_ga4gh_tool(document_loader: Loader, uri: str) -> Optional[str]: +def resolve_ga4gh_tool(document_loader: Loader, uri: str) -> str | None: + """Use the GA4GH TRS API to resolve a tool reference.""" path, version = uri.partition(":")[::2] if not version: version = "latest" diff --git a/cwltool/run_job.py b/cwltool/run_job.py index 307872f7a..d2cb42e99 100644 --- a/cwltool/run_job.py +++ b/cwltool/run_job.py @@ -4,10 +4,10 @@ import os import subprocess # nosec import sys -from typing import BinaryIO, Dict, List, Optional, TextIO, Union +from typing import BinaryIO, TextIO -def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[str, str]: +def handle_software_environment(cwl_env: dict[str, str], script: str) -> dict[str, str]: """Update the provided environment dict by running the script.""" exec_env = cwl_env.copy() exec_env["_CWLTOOL"] = "1" @@ -29,7 +29,7 @@ def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[st return env -def main(argv: List[str]) -> int: +def main(argv: list[str]) -> int: """ Read in the configuration JSON and execute the commands. @@ -55,20 +55,20 @@ def main(argv: List[str]) -> int: stdout_path = popen_description["stdout_path"] stderr_path = popen_description["stderr_path"] if stdin_path is not None: - stdin: Union[BinaryIO, int] = open(stdin_path, "rb") + stdin: BinaryIO | int = open(stdin_path, "rb") else: stdin = subprocess.PIPE if stdout_path is not None: - stdout: Union[BinaryIO, TextIO] = open(stdout_path, "wb") + stdout: BinaryIO | TextIO = open(stdout_path, "wb") else: stdout = sys.stderr if stderr_path is not None: - stderr: Union[BinaryIO, TextIO] = open(stderr_path, "wb") + stderr: BinaryIO | TextIO = open(stderr_path, "wb") else: stderr = sys.stderr try: - env_script: Optional[str] = argv[2] + env_script: str | None = argv[2] except IndexError: env_script = None if env_script is not None: diff --git a/cwltool/schemas/v1.1.0-dev1/cwl-runner.cwl b/cwltool/schemas/v1.1.0-dev1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.1/cwl-runner.cwl b/cwltool/schemas/v1.1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev1/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev2/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev2/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev3/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev3/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev4/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev4/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev5/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev5/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.3.0-dev1/cwl-runner.cwl b/cwltool/schemas/v1.3.0-dev1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/secrets.py b/cwltool/secrets.py index f35f24c37..6a39231e4 100644 --- a/cwltool/secrets.py +++ b/cwltool/secrets.py @@ -1,7 +1,7 @@ """Minimal in memory storage of secrets.""" import uuid -from typing import Dict, List, MutableMapping, MutableSequence, Optional, cast +from collections.abc import MutableMapping, MutableSequence from .utils import CWLObjectType, CWLOutputType @@ -11,9 +11,9 @@ class SecretStore: def __init__(self) -> None: """Initialize the secret store.""" - self.secrets: Dict[str, str] = {} + self.secrets: dict[str, str] = {} - def add(self, value: Optional[CWLOutputType]) -> Optional[CWLOutputType]: + def add(self, value: CWLOutputType | None) -> CWLOutputType | None: """ Add the given value to the store. @@ -28,7 +28,7 @@ def add(self, value: Optional[CWLOutputType]) -> Optional[CWLOutputType]: return placeholder return value - def store(self, secrets: List[str], job: CWLObjectType) -> None: + def store(self, secrets: list[str], job: CWLObjectType) -> None: """Sanitize the job object of any of the given secrets.""" for j in job: if j in secrets: @@ -36,28 +36,30 @@ def store(self, secrets: List[str], job: CWLObjectType) -> None: def has_secret(self, value: CWLOutputType) -> bool: """Test if the provided document has any of our secrets.""" - if isinstance(value, str): - for k in self.secrets: - if k in value: - return True - elif isinstance(value, MutableMapping): - for this_value in value.values(): - if self.has_secret(cast(CWLOutputType, this_value)): - return True - elif isinstance(value, MutableSequence): - for this_value in value: - if self.has_secret(cast(CWLOutputType, this_value)): - return True + match value: + case str(val): + for k in self.secrets: + if k in val: + return True + case MutableMapping() as v_dict: + for this_value in v_dict.values(): + if self.has_secret(this_value): + return True + case MutableSequence() as seq: + for this_value in seq: + if self.has_secret(this_value): + return True return False def retrieve(self, value: CWLOutputType) -> CWLOutputType: """Replace placeholders with their corresponding secrets.""" - if isinstance(value, str): - for key, this_value in self.secrets.items(): - value = value.replace(key, this_value) - return value - elif isinstance(value, MutableMapping): - return {k: self.retrieve(cast(CWLOutputType, v)) for k, v in value.items()} - elif isinstance(value, MutableSequence): - return [self.retrieve(cast(CWLOutputType, v)) for v in value] + match value: + case str(val): + for key, this_value in self.secrets.items(): + val = val.replace(key, this_value) + return val + case MutableMapping() as v_dict: + return {k: self.retrieve(v) for k, v in v_dict.items()} + case MutableSequence() as seq: + return [self.retrieve(v) for v in seq] return value diff --git a/cwltool/singularity.py b/cwltool/singularity.py index c43183ac7..76f1fc488 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -6,8 +6,9 @@ import re import shutil import sys +from collections.abc import Callable, MutableMapping from subprocess import check_call, check_output # nosec -from typing import Callable, Dict, List, MutableMapping, Optional, Tuple, cast +from typing import cast from schema_salad.sourceline import SourceLine from spython.main import Client @@ -28,13 +29,13 @@ # This is a list containing major and minor versions as integer. # (The number of minor version digits can vary among different distributions, # therefore we need a list here.) -_SINGULARITY_VERSION: Optional[List[int]] = None +_SINGULARITY_VERSION: list[int] | None = None # Cached flavor / distribution of singularity # Can be singularity, singularity-ce or apptainer _SINGULARITY_FLAVOR: str = "" -def get_version() -> Tuple[List[int], str]: +def get_version() -> tuple[list[int], str]: """ Parse the output of 'singularity --version' to determine the flavor and version. @@ -74,6 +75,14 @@ def is_apptainer_1_or_newer() -> bool: return v[0][0] >= 1 +def is_apptainer_1_1_or_newer() -> bool: + """Check if apptainer singularity distribution is version 1.1 or higher.""" + v = get_version() + if v[1] != "apptainer": + return False + return v[0][0] >= 2 or (v[0][0] >= 1 and v[0][1] >= 1) + + def is_version_2_6() -> bool: """ Check if this singularity version is exactly version 2.6. @@ -118,11 +127,21 @@ def is_version_3_9_or_newer() -> bool: return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 9) +def is_version_3_10_or_newer() -> bool: + """Detect if Singularity v3.10+ is available.""" + v = get_version() + return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 10) + + def _normalize_image_id(string: str) -> str: + if ":" not in string: + string += "_latest" return string.replace("/", "_") + ".img" def _normalize_sif_id(string: str) -> str: + if ":" not in string: + string += "_latest" return string.replace("/", "_") + ".sif" @@ -131,9 +150,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Builder for invoking the Singularty software container engine.""" @@ -141,7 +160,7 @@ def __init__( @staticmethod def get_image( - dockerRequirement: Dict[str, str], + dockerRequirement: dict[str, str], pull_image: bool, tmp_outdir_prefix: str, force_pull: bool = False, @@ -247,7 +266,7 @@ def get_image( dockerRequirement["dockerImageId"] = path found = True if (force_pull or not found) and pull_image: - cmd = [] # type: List[str] + cmd: list[str] = [] if "dockerPull" in dockerRequirement: if cache_folder: env = os.environ.copy() @@ -329,7 +348,7 @@ def get_from_requirements( pull_image: bool, force_pull: bool, tmp_outdir_prefix: str, - ) -> Optional[str]: + ) -> str | None: """ Return the filename of the Singularity image. @@ -338,7 +357,7 @@ def get_from_requirements( if not bool(shutil.which("singularity")): raise WorkflowException("singularity executable is not available") - if not self.get_image(cast(Dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull): + if not self.get_image(cast(dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull): raise WorkflowException("Container image {} not found".format(r["dockerImageId"])) if "CWL_SINGULARITY_CACHE" in os.environ: @@ -350,7 +369,7 @@ def get_from_requirements( return os.path.abspath(img_path) @staticmethod - def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None: + def append_volume(runtime: list[str], source: str, target: str, writable: bool = False) -> None: """Add binding arguments to the runtime list.""" if is_version_3_9_or_newer(): DockerCommandLineJob.append_volume(runtime, source, target, writable, skip_mkdirs=True) @@ -364,7 +383,7 @@ def append_volume(runtime: List[str], source: str, target: str, writable: bool = runtime.append(vol) def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: str | None ) -> None: if not volume.resolved.startswith("_:"): if host_outdir_tgt is not None and not is_version_3_4_or_newer(): @@ -380,9 +399,9 @@ def add_file_or_directory_volume( def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: if host_outdir_tgt is not None and not is_version_3_4_or_newer(): @@ -417,9 +436,9 @@ def add_writable_file_volume( def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: if volume.resolved.startswith("_:"): @@ -452,7 +471,7 @@ def add_writable_directory_volume( shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: return { "TMPDIR": self.CONTAINER_TMPDIR, "HOME": self.builder.outdir, @@ -460,23 +479,27 @@ def _required_env(self) -> Dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], str | None]: """Return the Singularity runtime list of commands and options.""" any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False + runtime = [ "singularity", "--quiet", - "exec", + "run" if is_apptainer_1_1_or_newer() or is_version_3_10_or_newer() else "exec", "--contain", "--ipc", "--cleanenv", ] + if is_apptainer_1_1_or_newer() or is_version_3_10_or_newer(): + runtime.append("--no-eval") + if singularity_supports_userns(): runtime.append("--userns") else: runtime.append("--pid") - container_HOME: Optional[str] = None + container_HOME: str | None = None if is_version_3_1_or_newer(): # Remove HOME, as passed in a special way (restore it below) container_HOME = self.environment.pop("HOME") diff --git a/cwltool/singularity_utils.py b/cwltool/singularity_utils.py index e4cc88918..7f69eb2fc 100644 --- a/cwltool/singularity_utils.py +++ b/cwltool/singularity_utils.py @@ -2,10 +2,9 @@ import os import os.path -from subprocess import DEVNULL, PIPE, Popen, TimeoutExpired # nosec -from typing import Optional +import subprocess # nosec -_USERNS: Optional[bool] = None +_USERNS: bool | None = None def singularity_supports_userns() -> bool: @@ -14,17 +13,17 @@ def singularity_supports_userns() -> bool: if _USERNS is None: try: hello_image = os.path.join(os.path.dirname(__file__), "hello.simg") - result = Popen( # nosec + result = subprocess.run( # nosec ["singularity", "exec", "--userns", hello_image, "true"], - stderr=PIPE, - stdout=DEVNULL, - universal_newlines=True, - ).communicate(timeout=60)[1] + capture_output=True, + timeout=60, + text=True, + ).stderr _USERNS = ( "No valid /bin/sh" in result or "/bin/sh doesn't exist in container" in result or "executable file not found in" in result ) - except TimeoutExpired: + except subprocess.TimeoutExpired: _USERNS = False return _USERNS diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index ec99bda05..307e98fd8 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -8,35 +8,24 @@ """ import argparse +import importlib.metadata import os import string -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import TYPE_CHECKING, Any, Union, cast from .utils import HasReqsHints if TYPE_CHECKING: + from galaxy.tool_util.deps.requirements import ToolRequirements + from .builder import Builder try: - from galaxy.tool_util import deps - from galaxy.tool_util.deps.requirements import ToolRequirement, ToolRequirements -except ImportError: - ToolRequirement = None # type: ignore - ToolRequirements = None # type: ignore - deps = None # type: ignore - - -SOFTWARE_REQUIREMENTS_ENABLED = deps is not None + importlib.metadata.Distribution.from_name("galaxy-tool-util") + SOFTWARE_REQUIREMENTS_ENABLED = True +except ModuleNotFoundError: + SOFTWARE_REQUIREMENTS_ENABLED = False COMMAND_WITH_DEPENDENCIES_TEMPLATE = string.Template( """#!/bin/bash @@ -59,6 +48,8 @@ class DependenciesConfiguration: def __init__(self, args: argparse.Namespace) -> None: """Initialize.""" + self.tool_dependency_dir: str | None = None + self.dependency_resolvers_config_file: str | None = None conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) tool_dependency_dir = getattr(args, "beta_dependencies_directory", None) conda_dependencies = getattr(args, "beta_conda_dependencies", None) @@ -79,7 +70,10 @@ def __init__(self, args: argparse.Namespace) -> None: if self.tool_dependency_dir and not os.path.exists(self.tool_dependency_dir): os.makedirs(self.tool_dependency_dir) - def build_job_script(self, builder: "Builder", command: List[str]) -> str: + def build_job_script(self, builder: "Builder", command: list[str]) -> str: + """Use the galaxy-tool-util library to construct a build script.""" + from galaxy.tool_util import deps + ensure_galaxy_lib_available() resolution_config_dict = { "use": self.use_tool_dependencies, @@ -103,14 +97,18 @@ def build_job_script(self, builder: "Builder", command: List[str]) -> str: ) ) - template_kwds: Dict[str, str] = dict(handle_dependencies=handle_dependencies) + template_kwds: dict[str, str] = dict(handle_dependencies=handle_dependencies) job_script = COMMAND_WITH_DEPENDENCIES_TEMPLATE.substitute(template_kwds) return job_script -def get_dependencies(builder: HasReqsHints) -> ToolRequirements: +def get_dependencies( + builder: HasReqsHints, +) -> "ToolRequirements": (software_requirement, _) = builder.get_requirement("SoftwareRequirement") - dependencies: List[Union["ToolRequirement", Dict[str, Any]]] = [] + from galaxy.tool_util.deps.requirements import ToolRequirement, ToolRequirements + + dependencies: list[Union["ToolRequirement", dict[str, Any]]] = [] if software_requirement and software_requirement.get("packages"): packages = cast( MutableSequence[MutableMapping[str, Union[str, MutableSequence[str]]]], @@ -139,8 +137,8 @@ def get_dependencies(builder: HasReqsHints) -> ToolRequirements: def get_container_from_software_requirements( - use_biocontainers: bool, builder: HasReqsHints, container_image_cache_path: Optional[str] = "." -) -> Optional[str]: + use_biocontainers: bool, builder: HasReqsHints, container_image_cache_path: str | None = "." +) -> str | None: if use_biocontainers: ensure_galaxy_lib_available() from galaxy.tool_util.deps.container_classes import DOCKER_CONTAINER_TYPE diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index 069289111..761b83526 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -3,7 +3,7 @@ import glob import os import urllib -from typing import IO, Any, List +from typing import IO, Any from schema_salad.ref_resolver import file_uri, uri_file_path @@ -31,7 +31,8 @@ def __init__(self, basedir: str) -> None: def _abs(self, p: str) -> str: return abspath(p, self.basedir) - def glob(self, pattern: str) -> List[str]: + def glob(self, pattern: str) -> list[str]: + """Return a possibly empty list of absolute URI paths that match pathname.""" return [file_uri(str(self._abs(line))) for line in glob.glob(self._abs(pattern))] def open(self, fn: str, mode: str) -> IO[Any]: @@ -49,10 +50,12 @@ def isfile(self, fn: str) -> bool: def isdir(self, fn: str) -> bool: return os.path.isdir(self._abs(fn)) - def listdir(self, fn: str) -> List[str]: + def listdir(self, fn: str) -> list[str]: + """Return a list containing the absolute path URLs of the entries in the directory given by path.""" return [abspath(urllib.parse.quote(entry), fn) for entry in os.listdir(self._abs(fn))] - def join(self, path, *paths): # type: (str, *str) -> str + def join(self, path: str, *paths: str) -> str: + """Join one or more path segments intelligently.""" return os.path.join(path, *paths) def realpath(self, path: str) -> str: diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index f6df7e69f..8bad72a47 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -1,28 +1,21 @@ import urllib -from collections import namedtuple -from typing import ( - Any, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence +from typing import Any, NamedTuple, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from .context import LoadingContext from .load_tool import load_tool, make_tool -from .process import Process from .utils import CWLObjectType, aslist from .workflow import Workflow, WorkflowStep -Node = namedtuple("Node", ("up", "down", "type")) + +class _Node(NamedTuple): + up: list[str] + down: list[str] + type: str | None + + UP = "up" DOWN = "down" INPUT = "input" @@ -30,10 +23,10 @@ STEP = "step" -def subgraph_visit( +def _subgraph_visit( current: str, - nodes: MutableMapping[str, Node], - visited: Set[str], + nodes: MutableMapping[str, _Node], + visited: set[str], direction: str, ) -> None: if current in visited: @@ -45,65 +38,71 @@ def subgraph_visit( if direction == UP: d = nodes[current].up for c in d: - subgraph_visit(c, nodes, visited, direction) + _subgraph_visit(c, nodes, visited, direction) -def declare_node(nodes: Dict[str, Node], nodeid: str, tp: Optional[str]) -> Node: +def _declare_node(nodes: dict[str, _Node], nodeid: str, tp: str | None) -> _Node: + """ + Record the given nodeid in the graph. + + If the nodeid is already present, but its type is unset, set it. + :returns: The Node tuple (even if already present in the graph). + """ if nodeid in nodes: n = nodes[nodeid] if n.type is None: - nodes[nodeid] = Node(n.up, n.down, tp) + nodes[nodeid] = _Node(n.up, n.down, tp) else: - nodes[nodeid] = Node([], [], tp) + nodes[nodeid] = _Node([], [], tp) return nodes[nodeid] def find_step( - steps: List[WorkflowStep], stepid: str, loading_context: LoadingContext -) -> Tuple[Optional[CWLObjectType], Optional[WorkflowStep]]: + steps: list[WorkflowStep], stepid: str, loading_context: LoadingContext +) -> tuple[CWLObjectType | None, WorkflowStep | None]: """Find the step (raw dictionary and WorkflowStep) for a given step id.""" for st in steps: st_tool_id = st.tool["id"] if st_tool_id == stepid: return st.tool, st if stepid.startswith(st_tool_id): - run: Union[str, Process, CWLObjectType] = st.tool["run"] - if isinstance(run, Workflow): - result, st2 = find_step( - run.steps, stepid[len(st.tool["id"]) + 1 :], loading_context - ) - if result: - return result, st2 - elif isinstance(run, CommentedMap) and run["class"] == "Workflow": - process = make_tool(run, loading_context) - if isinstance(process, Workflow): - suffix = stepid[len(st.tool["id"]) + 1 :] - prefix = process.tool["id"] - if "#" in prefix: - sep = "/" - else: - sep = "#" - adj_stepid = f"{prefix}{sep}{suffix}" - result2, st3 = find_step( - process.steps, - adj_stepid, - loading_context, + match st.tool["run"]: + case Workflow(steps=steps): + result, st2 = find_step( + steps, stepid[len(st.tool["id"]) + 1 :], loading_context ) - if result2: - return result2, st3 - elif isinstance(run, str): - process = load_tool(run, loading_context) - if isinstance(process, Workflow): - suffix = stepid[len(st.tool["id"]) + 1 :] - prefix = process.tool["id"] - if "#" in prefix: - sep = "/" - else: - sep = "#" - adj_stepid = f"{prefix}{sep}{suffix}" - result3, st4 = find_step(process.steps, adj_stepid, loading_context) - if result3: - return result3, st4 + if result: + return result, st2 + case {"class": "Workflow"}: + process = make_tool(st.tool["run"], loading_context) + if isinstance(process, Workflow): + suffix = stepid[len(st.tool["id"]) + 1 :] + prefix = process.tool["id"] + if "#" in prefix: + sep = "/" + else: + sep = "#" + adj_stepid = f"{prefix}{sep}{suffix}" + result2, st3 = find_step( + process.steps, + adj_stepid, + loading_context, + ) + if result2: + return result2, st3 + case str(run_line): + process = load_tool(run_line, loading_context) + if isinstance(process, Workflow): + suffix = stepid[len(st.tool["id"]) + 1 :] + prefix = process.tool["id"] + if "#" in prefix: + sep = "/" + else: + sep = "#" + adj_stepid = f"{prefix}{sep}{suffix}" + result3, st4 = find_step(process.steps, adj_stepid, loading_context) + if result3: + return result3, st4 return None, None @@ -114,22 +113,22 @@ def get_subgraph( if tool.tool["class"] != "Workflow": raise Exception("Can only extract subgraph from workflow") - nodes: Dict[str, Node] = {} + nodes: dict[str, _Node] = {} for inp in tool.tool["inputs"]: - declare_node(nodes, inp["id"], INPUT) + _declare_node(nodes, inp["id"], INPUT) for out in tool.tool["outputs"]: - declare_node(nodes, out["id"], OUTPUT) + _declare_node(nodes, out["id"], OUTPUT) for i in aslist(out.get("outputSource", CommentedSeq)): # source is upstream from output (dependency) nodes[out["id"]].up.append(i) # output is downstream from source - declare_node(nodes, i, None) + _declare_node(nodes, i, None) nodes[i].down.append(out["id"]) for st in tool.tool["steps"]: - step = declare_node(nodes, st["id"], STEP) + step = _declare_node(nodes, st["id"], STEP) for i in st["in"]: if "source" not in i: continue @@ -137,7 +136,7 @@ def get_subgraph( # source is upstream from step (dependency) step.up.append(src) # step is downstream from source - declare_node(nodes, src, None) + _declare_node(nodes, src, None) nodes[src].down.append(st["id"]) for out in st["out"]: if isinstance(out, Mapping) and "id" in out: @@ -145,20 +144,20 @@ def get_subgraph( # output is downstream from step step.down.append(out) # step is upstream from output - declare_node(nodes, out, None) + _declare_node(nodes, out, None) nodes[out].up.append(st["id"]) # Find all the downstream nodes from the starting points - visited_down: Set[str] = set() + visited_down: set[str] = set() for r in roots: if nodes[r].type == OUTPUT: - subgraph_visit(r, nodes, visited_down, UP) + _subgraph_visit(r, nodes, visited_down, UP) else: - subgraph_visit(r, nodes, visited_down, DOWN) + _subgraph_visit(r, nodes, visited_down, DOWN) # Now make sure all the nodes are connected to upstream inputs - visited: Set[str] = set() - rewire: Dict[str, Tuple[str, CWLObjectType]] = {} + visited: set[str] = set() + rewire: dict[str, tuple[str, CWLObjectType]] = {} for v in visited_down: visited.add(v) if nodes[v].type in (STEP, OUTPUT): @@ -221,7 +220,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C extracted["inputs"] = CommentedSeq() extracted["outputs"] = CommentedSeq() - for in_port in cast(List[CWLObjectType], step["in"]): + for in_port in cast(list[CWLObjectType], step["in"]): name = "#" + cast(str, in_port["id"]).split("#")[-1].split("/")[-1] inp: CWLObjectType = {"id": name, "type": "Any"} if "default" in in_port: @@ -231,7 +230,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C if "linkMerge" in in_port: del in_port["linkMerge"] - for outport in cast(List[Union[str, Mapping[str, Any]]], step["out"]): + for outport in cast(list[Union[str, Mapping[str, Any]]], step["out"]): if isinstance(outport, Mapping): outport_id = cast(str, outport["id"]) else: @@ -256,7 +255,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C def get_process( tool: Workflow, step_id: str, loading_context: LoadingContext -) -> Tuple[Any, WorkflowStep]: +) -> tuple[Any, WorkflowStep]: """Find the underlying Process for a given Workflow step id.""" if loading_context.loader is None: raise Exception("loading_context.loader cannot be None") @@ -264,7 +263,7 @@ def get_process( if raw_step is None or step is None: raise Exception(f"Step {step_id} was not found") - run: Union[str, Any] = raw_step["run"] + run: str | Any = raw_step["run"] if isinstance(run, str): process = loading_context.loader.idx[run] diff --git a/cwltool/task_queue.py b/cwltool/task_queue.py index 59b1609e9..e86172a9a 100644 --- a/cwltool/task_queue.py +++ b/cwltool/task_queue.py @@ -5,7 +5,8 @@ import queue import threading -from typing import Callable, Optional +from collections.abc import Callable +from typing import Union from .loghandler import _logger @@ -36,12 +37,12 @@ class TaskQueue: def __init__(self, lock: threading.Lock, thread_count: int): """Create a new task queue using the specified lock and number of threads.""" self.thread_count = thread_count - self.task_queue: queue.Queue[Optional[Callable[[], None]]] = queue.Queue( + self.task_queue: queue.Queue[Callable[[], None] | None] = queue.Queue( maxsize=self.thread_count ) self.task_queue_threads = [] self.lock = lock - self.error: Optional[BaseException] = None + self.error: BaseException | None = None for _r in range(0, self.thread_count): t = threading.Thread(target=self._task_queue_func) @@ -65,8 +66,8 @@ def _task_queue_func(self) -> None: def add( self, task: Callable[[], None], - unlock: Optional[threading.Condition] = None, - check_done: Optional[threading.Event] = None, + unlock: Union[threading.Condition, None] = None, + check_done: Union[threading.Event, None] = None, ) -> None: """ Add your task to the queue. diff --git a/cwltool/udocker.py b/cwltool/udocker.py index 6598d6a7c..ea3fc78ca 100644 --- a/cwltool/udocker.py +++ b/cwltool/udocker.py @@ -1,7 +1,5 @@ """Enables Docker software containers via the udocker runtime.""" -from typing import List - from .docker import DockerCommandLineJob @@ -10,7 +8,7 @@ class UDockerCommandLineJob(DockerCommandLineJob): @staticmethod def append_volume( - runtime: List[str], + runtime: list[str], source: str, target: str, writable: bool = False, diff --git a/cwltool/update.py b/cwltool/update.py index 4fd66b37a..91a63f496 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -1,15 +1,7 @@ import copy +from collections.abc import Callable, MutableMapping, MutableSequence from functools import partial -from typing import ( - Callable, - Dict, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from typing import cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException @@ -20,7 +12,7 @@ from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field -def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> Tuple[CommentedMap, str]: +def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> tuple[CommentedMap, str]: """Public updater for v1.2 to v1.3.0-dev1.""" doc = copy.deepcopy(doc) @@ -78,11 +70,11 @@ def rewrite_loop_requirements(t: CWLObjectType) -> None: def v1_1to1_2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.1 to v1.2.""" doc = copy.deepcopy(doc) - upd: Union[CommentedSeq, CommentedMap] = doc + upd: CommentedSeq | CommentedMap = doc if isinstance(upd, MutableMapping) and "$graph" in upd: upd = upd["$graph"] for proc in aslist(upd): @@ -94,7 +86,7 @@ def v1_1to1_2( def v1_0to1_1( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.0 to v1.1.""" doc = copy.deepcopy(doc) @@ -133,14 +125,14 @@ def rewrite_requirements(t: CWLObjectType) -> None: def update_secondaryFiles( t: CWLOutputType, top: bool = False - ) -> Union[MutableSequence[MutableMapping[str, str]], MutableMapping[str, str]]: + ) -> MutableSequence[MutableMapping[str, str]] | MutableMapping[str, str]: if isinstance(t, CommentedSeq): new_seq = copy.deepcopy(t) for index, entry in enumerate(t): new_seq[index] = update_secondaryFiles(entry) return new_seq elif isinstance(t, MutableSequence): - return CommentedSeq([update_secondaryFiles(cast(CWLOutputType, p)) for p in t]) + return CommentedSeq([update_secondaryFiles(p) for p in t]) elif isinstance(t, MutableMapping): return cast(MutableMapping[str, str], t) elif top: @@ -166,7 +158,7 @@ def fix_inputBinding(t: CWLObjectType) -> None: visit_class(doc, ("ExpressionTool", "Workflow"), fix_inputBinding) visit_field(doc, "secondaryFiles", partial(update_secondaryFiles, top=True)) - upd: Union[CommentedMap, CommentedSeq] = doc + upd: CommentedMap | CommentedSeq = doc if isinstance(upd, MutableMapping) and "$graph" in upd: upd = upd["$graph"] for proc in aslist(upd): @@ -195,21 +187,21 @@ def fix_inputBinding(t: CWLObjectType) -> None: def v1_1_0dev1to1_1( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.1.0-dev1 to v1.1.""" return (doc, "v1.1") def v1_2_0dev1todev2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev1 to v1.2.0-dev2.""" return (doc, "v1.2.0-dev2") def v1_2_0dev2todev3( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev2 to v1.2.0-dev3.""" doc = copy.deepcopy(doc) @@ -221,7 +213,7 @@ def update_pickvalue(t: CWLObjectType) -> None: inp["pickValue"] = "the_only_non_null" visit_class(doc, "Workflow", update_pickvalue) - upd: Union[CommentedSeq, CommentedMap] = doc + upd: CommentedSeq | CommentedMap = doc if isinstance(upd, MutableMapping) and "$graph" in upd: upd = upd["$graph"] for proc in aslist(upd): @@ -232,21 +224,21 @@ def update_pickvalue(t: CWLObjectType) -> None: def v1_2_0dev3todev4( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev3 to v1.2.0-dev4.""" return (doc, "v1.2.0-dev4") def v1_2_0dev4todev5( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev4 to v1.2.0-dev5.""" return (doc, "v1.2.0-dev5") def v1_2_0dev5to1_2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev5 to v1.2.""" return (doc, "v1.2") @@ -264,13 +256,13 @@ def v1_2_0dev5to1_2( "v1.3.0-dev1", ] -UPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { +UPDATES: dict[str, Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]] | None] = { "v1.0": v1_0to1_1, "v1.1": v1_1to1_2, "v1.2": v1_2to1_3dev1, } -DEVUPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { +DEVUPDATES: dict[str, Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]] | None] = { "v1.1.0-dev1": v1_1_0dev1to1_1, "v1.2.0-dev1": v1_2_0dev1todev2, "v1.2.0-dev2": v1_2_0dev2todev3, @@ -291,21 +283,21 @@ def v1_2_0dev5to1_2( def identity( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Do-nothing, CWL document upgrade function.""" return (doc, cast(str, doc["cwlVersion"])) def checkversion( - doc: Union[CommentedSeq, CommentedMap], + doc: CommentedSeq | CommentedMap, metadata: CommentedMap, enable_dev: bool, -) -> Tuple[CommentedMap, str]: +) -> tuple[CommentedMap, str]: """Check the validity of the version of the give CWL document. Returns the document and the validated version string. """ - cdoc: Optional[CommentedMap] = None + cdoc: CommentedMap | None = None if isinstance(doc, CommentedSeq): if not isinstance(metadata, CommentedMap): raise Exception("Expected metadata to be CommentedMap") @@ -351,12 +343,12 @@ def checkversion( def update( - doc: Union[CommentedSeq, CommentedMap], + doc: CommentedSeq | CommentedMap, loader: Loader, baseuri: str, enable_dev: bool, metadata: CommentedMap, - update_to: Optional[str] = None, + update_to: str | None = None, ) -> CommentedMap: """Update a CWL document to 'update_to' (if provided) or INTERNAL_VERSION.""" if update_to is None: @@ -365,7 +357,7 @@ def update( (cdoc, version) = checkversion(doc, metadata, enable_dev) originalversion = copy.copy(version) - nextupdate: Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]] = identity + nextupdate: Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]] | None = identity while version != update_to and nextupdate: (cdoc, version) = nextupdate(cdoc, loader, baseuri) diff --git a/cwltool/utils.py b/cwltool/utils.py index c8620994a..05913bf63 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -19,6 +19,13 @@ import tempfile import urllib import uuid +from collections.abc import ( + Callable, + Generator, + Iterable, + MutableMapping, + MutableSequence, +) from datetime import datetime from email.utils import parsedate_to_datetime from functools import partial @@ -29,19 +36,11 @@ IO, TYPE_CHECKING, Any, - Callable, Deque, - Dict, - Generator, - Iterable, - List, Literal, - MutableMapping, - MutableSequence, NamedTuple, Optional, - Set, - Tuple, + TypeAlias, TypedDict, Union, cast, @@ -52,22 +51,16 @@ from cachecontrol.caches import FileCache from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException -from schema_salad.ref_resolver import Loader - -if sys.version_info >= (3, 9): - from importlib.resources import as_file, files -else: - from importlib_resources import as_file, files if TYPE_CHECKING: + from schema_salad.ref_resolver import Loader + from .command_line_tool import CallbackJob, ExpressionJob from .job import CommandLineJob, JobBase from .stdfsaccess import StdFsAccess from .workflow_job import WorkflowJob -__all__ = ["files", "as_file"] - -__random_outdir: Optional[str] = None +__random_outdir: str | None = None CONTENT_LIMIT = 64 * 1024 @@ -75,7 +68,7 @@ processes_to_kill: Deque["subprocess.Popen[str]"] = collections.deque() -CWLOutputType = Union[ +CWLOutputType: TypeAlias = Union[ None, bool, str, @@ -84,33 +77,35 @@ MutableSequence["CWLOutputType"], MutableMapping[str, "CWLOutputType"], ] -CWLObjectType = MutableMapping[str, Optional[CWLOutputType]] +CWLObjectType: TypeAlias = MutableMapping[str, Optional[CWLOutputType]] """Typical raw dictionary found in lightly parsed CWL.""" -JobsType = Union["CommandLineJob", "JobBase", "WorkflowJob", "ExpressionJob", "CallbackJob"] -JobsGeneratorType = Generator[Optional[JobsType], None, None] -OutputCallbackType = Callable[[Optional[CWLObjectType], str], None] -ResolverType = Callable[["Loader", str], Optional[str]] -DestinationsType = MutableMapping[str, Optional[CWLOutputType]] -ScatterDestinationsType = MutableMapping[str, List[Optional[CWLOutputType]]] -ScatterOutputCallbackType = Callable[[Optional[ScatterDestinationsType], str], None] -SinkType = Union[CWLOutputType, CWLObjectType] +JobsType: TypeAlias = Union[ + "CommandLineJob", "JobBase", "WorkflowJob", "ExpressionJob", "CallbackJob" +] +JobsGeneratorType: TypeAlias = Generator[Optional[JobsType], None, None] +OutputCallbackType: TypeAlias = Callable[[Optional[CWLObjectType], str], None] +ResolverType: TypeAlias = Callable[["Loader", str], Optional[str]] +DestinationsType: TypeAlias = MutableMapping[str, Optional[CWLOutputType]] +ScatterDestinationsType: TypeAlias = MutableMapping[str, list[Optional[CWLOutputType]]] +ScatterOutputCallbackType: TypeAlias = Callable[[Optional[ScatterDestinationsType], str], None] +SinkType: TypeAlias = Union[CWLOutputType, CWLObjectType] DirectoryType = TypedDict( - "DirectoryType", {"class": str, "listing": List[CWLObjectType], "basename": str} + "DirectoryType", {"class": str, "listing": list[CWLObjectType], "basename": str} ) -JSONType = Union[Dict[str, "JSONType"], List["JSONType"], str, int, float, bool, None] +JSONType: TypeAlias = Union[dict[str, "JSONType"], list["JSONType"], str, int, float, bool, None] class WorkflowStateItem(NamedTuple): """Workflow state item.""" parameter: CWLObjectType - value: Optional[CWLOutputType] + value: CWLOutputType | None success: str -ParametersType = List[CWLObjectType] -StepType = CWLObjectType # WorkflowStep +ParametersType: TypeAlias = list[CWLObjectType] +StepType: TypeAlias = CWLObjectType # WorkflowStep LoadListingType = Union[Literal["no_listing"], Literal["shallow_listing"], Literal["deep_listing"]] @@ -143,7 +138,7 @@ def copytree_with_merge(src: str, dst: str) -> None: shutil.copy2(spath, dpath) -def cmp_like_py2(dict1: Dict[str, Any], dict2: Dict[str, Any]) -> int: +def cmp_like_py2(dict1: dict[str, Any], dict2: dict[str, Any]) -> int: """ Compare in the same manner as Python2. @@ -178,8 +173,8 @@ def cmp_like_py2(dict1: Dict[str, Any], dict2: Dict[str, Any]) -> int: def bytes2str_in_dicts( - inp: Union[MutableMapping[str, Any], MutableSequence[Any], Any], -) -> Union[str, MutableSequence[Any], MutableMapping[str, Any]]: + inp: MutableMapping[str, Any] | MutableSequence[Any] | Any, +) -> str | MutableSequence[Any] | MutableMapping[str, Any]: """ Convert any present byte string to unicode string, inplace. @@ -259,20 +254,20 @@ def adjustDirObjs(rec: Any, op: Union[Callable[..., Any], "partial[Any]"]) -> No visit_class(rec, ("Directory",), op) -def dedup(listing: List[CWLObjectType]) -> List[CWLObjectType]: +def dedup(listing: list[CWLObjectType]) -> list[CWLObjectType]: marksub = set() - def mark(d: Dict[str, str]) -> None: + def mark(d: dict[str, str]) -> None: marksub.add(d["location"]) for entry in listing: if entry["class"] == "Directory": - for e in cast(List[CWLObjectType], entry.get("listing", [])): + for e in cast(list[CWLObjectType], entry.get("listing", [])): adjustFileObjs(e, mark) adjustDirObjs(e, mark) dd = [] - markdup: Set[str] = set() + markdup: set[str] = set() for r in listing: if r["location"] not in marksub and r["location"] not in markdup: dd.append(r) @@ -284,14 +279,14 @@ def mark(d: Dict[str, str]) -> None: def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = True) -> None: """Expand, recursively, any 'listing' fields in a Directory.""" if rec.get("class") != "Directory": - finddirs: List[CWLObjectType] = [] + finddirs: list[CWLObjectType] = [] visit_class(rec, ("Directory",), finddirs.append) for f in finddirs: get_listing(fs_access, f, recursive=recursive) return if "listing" in rec: return - listing: List[CWLOutputType] = [] + listing: list[CWLOutputType] = [] loc = cast(str, rec["location"]) for ld in fs_access.listdir(loc): parse = urllib.parse.urlparse(ld) @@ -310,7 +305,7 @@ def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = rec["listing"] = listing -def trim_listing(obj: Dict[str, Any]) -> None: +def trim_listing(obj: dict[str, Any]) -> None: """ Remove 'listing' field from Directory objects that are file references. @@ -322,7 +317,7 @@ def trim_listing(obj: Dict[str, Any]) -> None: del obj["listing"] -def downloadHttpFile(httpurl: str) -> Tuple[str, Optional[datetime]]: +def downloadHttpFile(httpurl: str) -> tuple[str, datetime | None]: """ Download a remote file, possibly using a locally cached copy. @@ -350,8 +345,8 @@ def downloadHttpFile(httpurl: str) -> Tuple[str, Optional[datetime]]: f.write(chunk) r.close() - date_raw: Optional[str] = r.headers.get("Last-Modified", None) - date: Optional[datetime] = parsedate_to_datetime(date_raw) if date_raw else None + date_raw: str | None = r.headers.get("Last-Modified", None) + date: datetime | None = parsedate_to_datetime(date_raw) if date_raw else None if date: date_epoch = date.timestamp() os.utime(f.name, (date_epoch, date_epoch)) @@ -406,15 +401,11 @@ def ensure_non_writable(path: str) -> None: def normalizeFilesDirs( - job: Optional[ - Union[ - MutableSequence[MutableMapping[str, Any]], - MutableMapping[str, Any], - DirectoryType, - ] - ] + job: None | ( + MutableSequence[MutableMapping[str, Any]] | MutableMapping[str, Any] | DirectoryType + ), ) -> None: - def addLocation(d: Dict[str, Any]) -> None: + def addLocation(d: dict[str, Any]) -> None: if "location" not in d: if d["class"] == "File" and ("contents" not in d): raise ValidationException( @@ -484,10 +475,10 @@ class HasReqsHints: def __init__(self) -> None: """Initialize this reqs decorator.""" - self.requirements: List[CWLObjectType] = [] - self.hints: List[CWLObjectType] = [] + self.requirements: list[CWLObjectType] = [] + self.hints: list[CWLObjectType] = [] - def get_requirement(self, feature: str) -> Tuple[Optional[CWLObjectType], Optional[bool]]: + def get_requirement(self, feature: str) -> tuple[CWLObjectType | None, bool | None]: """Retrieve the named feature from the requirements field, or the hints field.""" for item in reversed(self.requirements): if item["class"] == feature: diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index de4adaa14..4aafc497b 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -2,18 +2,9 @@ import itertools import json import logging -from collections import namedtuple -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from importlib.resources import files +from typing import Any, NamedTuple, cast from cwl_utils.errors import SubstitutionError from cwl_utils.expression import scanner as scan_expression @@ -32,10 +23,9 @@ from .errors import WorkflowException from .loghandler import _logger -from .utils import files -def is_expression(tool: Any, schema: Optional[Schema]) -> bool: +def is_expression(tool: Any, schema: Schema | None) -> bool: """Test a field/schema combo to see if it is a CWL Expression.""" return ( isinstance(schema, EnumSchema) @@ -60,72 +50,75 @@ def filter(self, record: logging.LogRecord) -> bool: def get_expressions( - tool: Union[CommentedMap, str, CommentedSeq], - schema: Optional[Union[Schema, ArraySchema]], - source_line: Optional[SourceLine] = None, -) -> List[Tuple[str, Optional[SourceLine]]]: + tool: CommentedMap | str | CommentedSeq, + schema: Schema | ArraySchema | None, + source_line: SourceLine | None = None, +) -> list[tuple[str, SourceLine | None]]: debug = _logger.isEnabledFor(logging.DEBUG) if is_expression(tool, schema): return [(cast(str, tool), source_line)] - elif isinstance(schema, UnionSchema): - valid_schema = None - - for possible_schema in schema.schemas: - if is_expression(tool, possible_schema): - return [(cast(str, tool), source_line)] - elif validate_ex( - possible_schema, - tool, - raise_ex=False, - logger=_logger_validation_warnings, - vocab={}, - ): - valid_schema = possible_schema - - return get_expressions(tool, valid_schema, source_line) - elif isinstance(schema, ArraySchema): - if not isinstance(tool, MutableSequence): - return [] - - return list( - itertools.chain( - *map( - lambda x: get_expressions( - x[1], getattr(schema, "items"), SourceLine(tool, x[0]) # noqa: B009 - ), - enumerate(tool), + match schema: + case UnionSchema(schemas=schemas): + valid_schema = None + + for possible_schema in schemas: + if is_expression(tool, possible_schema): + return [(cast(str, tool), source_line)] + elif validate_ex( + possible_schema, + tool, + raise_ex=False, + logger=_logger_validation_warnings, + vocab={}, + ): + valid_schema = possible_schema + + return get_expressions(tool, valid_schema, source_line) + case ArraySchema(items=items): + if not isinstance(tool, MutableSequence): + return [] + + return list( + itertools.chain( + *map( + lambda x: get_expressions( + x[1], items, SourceLine(tool, x[0]) # noqa: B009 + ), + enumerate(tool), + ) ) ) - ) - - elif isinstance(schema, RecordSchema): - if not isinstance(tool, MutableMapping): - return [] - - expression_nodes = [] - - for schema_field in schema.fields: - if schema_field.name in tool: - expression_nodes.extend( - get_expressions( - tool[schema_field.name], - schema_field.type, - SourceLine(tool, schema_field.name, include_traceback=debug), + case RecordSchema(fields=fields): + if not isinstance(tool, MutableMapping): + return [] + + expression_nodes = [] + + for schema_field in fields: + if schema_field.name in tool: + expression_nodes.extend( + get_expressions( + tool[schema_field.name], + schema_field.type, + SourceLine(tool, schema_field.name, include_traceback=debug), + ) ) - ) + return expression_nodes + case _: + return [] - return expression_nodes - else: - return [] +class JSHintJSReturn(NamedTuple): + """List of errors and the final values of the globals from running javascript.""" -JSHintJSReturn = namedtuple("JSHintJSReturn", ["errors", "globals"]) + errors: list[str] + globals: list[str] def jshint_js( js_text: str, - globals: Optional[List[str]] = None, - options: Optional[Dict[str, Union[List[str], str, int]]] = None, + globals: list[str] | None = None, + options: dict[str, list[str] | str | int] | None = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> JSHintJSReturn: @@ -177,7 +170,7 @@ def dump_jshint_error() -> None: except ValueError: dump_jshint_error() - jshint_errors: List[str] = [] + jshint_errors: list[str] = [] js_text_lines = js_text.split("\n") @@ -193,7 +186,7 @@ def dump_jshint_error() -> None: return JSHintJSReturn(jshint_errors, jshint_json.get("globals", [])) -def print_js_hint_messages(js_hint_messages: List[str], source_line: Optional[SourceLine]) -> None: +def print_js_hint_messages(js_hint_messages: list[str], source_line: SourceLine | None) -> None: """Log the message from JSHint, using the line number.""" if source_line is not None: for js_hint_message in js_hint_messages: @@ -203,7 +196,7 @@ def print_js_hint_messages(js_hint_messages: List[str], source_line: Optional[So def validate_js_expressions( tool: CommentedMap, schema: Schema, - jshint_options: Optional[Dict[str, Union[List[str], str, int]]] = None, + jshint_options: dict[str, list[str] | str | int] | None = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> None: diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 982ec7e70..fe34c75ce 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -3,16 +3,8 @@ import functools import logging import random -from typing import ( - Callable, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - cast, -) +from collections.abc import Callable, Mapping, MutableMapping, MutableSequence +from typing import cast from uuid import UUID from mypy_extensions import mypyc_attr @@ -41,24 +33,25 @@ def default_make_tool(toolpath_object: CommentedMap, loadingContext: LoadingContext) -> Process: """Instantiate the given CWL Process.""" - if not isinstance(toolpath_object, MutableMapping): - raise WorkflowException("Not a dict: '%s'" % toolpath_object) - if "class" in toolpath_object: - if toolpath_object["class"] == "CommandLineTool": + match toolpath_object: + case {"class": "CommandLineTool"}: return command_line_tool.CommandLineTool(toolpath_object, loadingContext) - if toolpath_object["class"] == "ExpressionTool": + case {"class": "ExpressionTool"}: return command_line_tool.ExpressionTool(toolpath_object, loadingContext) - if toolpath_object["class"] == "Workflow": + case {"class": "Workflow"}: return Workflow(toolpath_object, loadingContext) - if toolpath_object["class"] == "ProcessGenerator": + case {"class": "ProcessGenerator"}: return procgenerator.ProcessGenerator(toolpath_object, loadingContext) - if toolpath_object["class"] == "Operation": + case {"class": "Operation"}: return command_line_tool.AbstractOperation(toolpath_object, loadingContext) - - raise WorkflowException( - "Missing or invalid 'class' field in " - "%s, expecting one of: CommandLineTool, ExpressionTool, Workflow" % toolpath_object["id"] - ) + case MutableMapping(): + raise WorkflowException( + "Missing or invalid 'class' field in " + f"{toolpath_object['id']}, expecting one of: CommandLineTool, " + "ExpressionTool, Workflow" + ) + case _: + raise WorkflowException("Not a dict: '%s'" % toolpath_object) context.default_make_tool = default_make_tool @@ -73,15 +66,14 @@ def __init__( ) -> None: """Initialize this Workflow.""" super().__init__(toolpath_object, loadingContext) - self.provenance_object: Optional[ProvenanceProfile] = None + self.provenance_object: ProvenanceProfile | None = None if loadingContext.research_obj is not None: - run_uuid: Optional[UUID] = None + run_uuid: UUID | None = None is_main = not loadingContext.prov_obj # Not yet set if is_main: run_uuid = loadingContext.research_obj.ro_uuid - self.provenance_object = ProvenanceProfile( - loadingContext.research_obj, + self.provenance_object = loadingContext.research_obj.initialize_provenance( full_name=loadingContext.cwl_full_name, host_provenance=loadingContext.host_provenance, user_provenance=loadingContext.user_provenance, @@ -98,7 +90,7 @@ def __init__( loadingContext.requirements = self.requirements loadingContext.hints = self.hints - self.steps: List[WorkflowStep] = [] + self.steps: list[WorkflowStep] = [] validation_errors = [] for index, step in enumerate(self.tool.get("steps", [])): try: @@ -119,9 +111,9 @@ def __init__( workflow_inputs = self.tool["inputs"] workflow_outputs = self.tool["outputs"] - step_inputs: List[CWLObjectType] = [] - step_outputs: List[CWLObjectType] = [] - param_to_step: Dict[str, CWLObjectType] = {} + step_inputs: list[CWLObjectType] = [] + step_outputs: list[CWLObjectType] = [] + param_to_step: dict[str, CWLObjectType] = {} for step in self.steps: step_inputs.extend(step.tool["inputs"]) step_outputs.extend(step.tool["outputs"]) @@ -146,7 +138,7 @@ def make_workflow_step( toolpath_object: CommentedMap, pos: int, loadingContext: LoadingContext, - parentworkflowProv: Optional[ProvenanceProfile] = None, + parentworkflowProv: ProvenanceProfile | None = None, ) -> "WorkflowStep": return WorkflowStep(toolpath_object, pos, loadingContext, parentworkflowProv) @@ -196,7 +188,7 @@ def __init__( toolpath_object: CommentedMap, pos: int, loadingContext: LoadingContext, - parentworkflowProv: Optional[ProvenanceProfile] = None, + parentworkflowProv: ProvenanceProfile | None = None, ) -> None: """Initialize this WorkflowStep.""" debug = loadingContext.debug @@ -220,7 +212,7 @@ def __init__( loadingContext.requirements.append(parent_req) loadingContext.requirements.extend( cast( - List[CWLObjectType], + list[CWLObjectType], get_overrides(getdefault(loadingContext.overrides_list, []), self.id).get( "requirements", [] ), @@ -394,7 +386,7 @@ def __init__( oparam["type"] = {"type": "array", "items": oparam["type"]} self.tool["inputs"] = inputparms self.tool["outputs"] = outputparms - self.prov_obj: Optional[ProvenanceProfile] = None + self.prov_obj: ProvenanceProfile | None = None if loadingContext.research_obj is not None: self.prov_obj = parentworkflowProv if self.embedded_tool.tool["class"] == "Workflow": @@ -451,7 +443,9 @@ def job( runtimeContext, ) except WorkflowException: - _logger.error("Exception on step '%s'", runtimeContext.name) + _logger.error( + "Exception on step '%s'", runtimeContext.name, exc_info=runtimeContext.debug + ) raise except Exception as exc: _logger.exception("Unexpected exception") diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 2e69ca70c..a85dc81f8 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -3,18 +3,8 @@ import functools import logging import threading -from typing import ( - TYPE_CHECKING, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sized, - Tuple, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence, Sized +from typing import TYPE_CHECKING, Optional, Union, cast from cwl_utils import expression from schema_salad.sourceline import SourceLine @@ -56,7 +46,7 @@ def __init__(self, step: "WorkflowStep") -> None: self.tool = step.tool self.id = step.id self.submitted = False - self.iterable: Optional[JobsGeneratorType] = None + self.iterable: JobsGeneratorType | None = None self.completed = False self.name = uniquename("step %s" % shortname(self.id)) self.prov_obj = step.prov_obj @@ -70,7 +60,12 @@ def job( ) -> JobsGeneratorType: runtimeContext = runtimeContext.copy() runtimeContext.part_of = self.name - runtimeContext.name = shortname(self.id) + + if runtimeContext.workflow_job_step_name_callback is not None: + vfinputs = {shortname(k): v for k, v in joborder.items()} + runtimeContext.name = runtimeContext.workflow_job_step_name_callback(self, vfinputs) + else: + runtimeContext.name = shortname(self.id) _logger.info("[%s] start", self.name) @@ -88,11 +83,16 @@ def __init__( ) -> None: """Initialize.""" self.dest = dest - self.completed = 0 + self._completed: set[int] = set() self.processStatus = "success" self.total = total self.output_callback = output_callback - self.steps: List[Optional[JobsGeneratorType]] = [] + self.steps: list[JobsGeneratorType | None] = [] + + @property + def completed(self) -> int: + """The number of completed internal jobs.""" + return len(self._completed) def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatus: str) -> None: """Record the results of a scatter operation.""" @@ -108,15 +108,16 @@ def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatu if self.processStatus != "permanentFail": self.processStatus = processStatus - self.completed += 1 + if index not in self._completed: + self._completed.add(index) - if self.completed == self.total: - self.output_callback(self.dest, self.processStatus) + if self.completed == self.total: + self.output_callback(self.dest, self.processStatus) def setTotal( self, total: int, - steps: List[Optional[JobsGeneratorType]], + steps: list[JobsGeneratorType | None], ) -> None: """ Set the total number of expected outputs along with the steps. @@ -130,7 +131,7 @@ def setTotal( def parallel_steps( - steps: List[Optional[JobsGeneratorType]], + steps: list[JobsGeneratorType | None], rc: ReceiveScatterOutput, runtimeContext: RuntimeContext, ) -> JobsGeneratorType: @@ -180,9 +181,9 @@ def nested_crossproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[JobsGeneratorType | None] = [] for index in range(0, jobl): - sjob: Optional[CWLObjectType] = copy.copy(joborder) + sjob: CWLObjectType | None = copy.copy(joborder) assert sjob is not None # nosec sjob[scatter_key] = cast(MutableMapping[int, CWLObjectType], joborder[scatter_key])[index] @@ -247,14 +248,14 @@ def _flat_crossproduct_scatter( callback: ReceiveScatterOutput, startindex: int, runtimeContext: RuntimeContext, -) -> Tuple[List[Optional[JobsGeneratorType]], int]: +) -> tuple[list[JobsGeneratorType | None], int]: """Inner loop.""" scatter_key = scatter_keys[0] jobl = len(cast(Sized, joborder[scatter_key])) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[JobsGeneratorType | None] = [] put = startindex for index in range(0, jobl): - sjob: Optional[CWLObjectType] = copy.copy(joborder) + sjob: CWLObjectType | None = copy.copy(joborder) assert sjob is not None # nosec sjob[scatter_key] = cast(MutableMapping[int, CWLObjectType], joborder[scatter_key])[index] @@ -285,7 +286,7 @@ def dotproduct_scatter( output_callback: ScatterOutputCallbackType, runtimeContext: RuntimeContext, ) -> JobsGeneratorType: - jobl: Optional[int] = None + jobl: int | None = None for key in scatter_keys: if jobl is None: jobl = len(cast(Sized, joborder[key])) @@ -302,9 +303,9 @@ def dotproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[JobsGeneratorType | None] = [] for index in range(0, jobl): - sjobo: Optional[CWLObjectType] = copy.copy(joborder) + sjobo: CWLObjectType | None = copy.copy(joborder) assert sjobo is not None # nosec for key in scatter_keys: sjobo[key] = cast(MutableMapping[int, CWLObjectType], joborder[key])[index] @@ -323,12 +324,12 @@ def dotproduct_scatter( def match_types( - sinktype: Optional[SinkType], + sinktype: SinkType | None, src: WorkflowStateItem, iid: str, inputobj: CWLObjectType, - linkMerge: Optional[str], - valueFrom: Optional[str], + linkMerge: str | None, + valueFrom: str | None, ) -> bool: if isinstance(sinktype, MutableSequence): # Sink is union type @@ -350,7 +351,7 @@ def match_types( elif linkMerge: if iid not in inputobj: inputobj[iid] = [] - sourceTypes = cast(List[Optional[CWLOutputType]], inputobj[iid]) + sourceTypes = cast(list[Optional[CWLOutputType]], inputobj[iid]) if linkMerge == "merge_nested": sourceTypes.append(src.value) elif linkMerge == "merge_flattened": @@ -373,13 +374,13 @@ def match_types( def object_from_state( - state: Dict[str, Optional[WorkflowStateItem]], + state: dict[str, WorkflowStateItem | None], params: ParametersType, frag_only: bool, supportsMultipleInput: bool, sourceField: str, incomplete: bool = False, -) -> Optional[CWLObjectType]: +) -> CWLObjectType | None: inputobj: CWLObjectType = {} for inp in params: iid = original_id = cast(str, inp["id"]) @@ -410,7 +411,7 @@ def object_from_state( ("merge_nested" if len(connections) > 1 else None), ), ), - valueFrom=cast(str, inp.get("valueFrom")), + valueFrom=cast(Optional[str], inp.get("valueFrom")), ): raise WorkflowException( "Type mismatch between source '%s' (%s) and " @@ -473,17 +474,17 @@ class WorkflowJob: def __init__(self, workflow: "Workflow", runtimeContext: RuntimeContext) -> None: """Initialize this WorkflowJob.""" self.workflow = workflow - self.prov_obj: Optional[ProvenanceProfile] = None - self.parent_wf: Optional[ProvenanceProfile] = None + self.prov_obj: ProvenanceProfile | None = None + self.parent_wf: ProvenanceProfile | None = None self.tool = workflow.tool if runtimeContext.research_obj is not None: self.prov_obj = workflow.provenance_object self.parent_wf = workflow.parent_wf self.steps = [WorkflowJobStep(s) for s in workflow.steps] - self.state: Dict[str, Optional[WorkflowStateItem]] = {} + self.state: dict[str, WorkflowStateItem | None] = {} self.processStatus = "" self.did_callback = False - self.made_progress: Optional[bool] = None + self.made_progress: bool | None = None self.outdir = runtimeContext.get_outdir() self.name = uniquename( @@ -506,7 +507,7 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: self.workflow.get_requirement("MultipleInputFeatureRequirement")[0] ) - wo: Optional[CWLObjectType] = None + wo: CWLObjectType | None = None try: wo = object_from_state( self.state, @@ -524,7 +525,7 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: and self.parent_wf and self.prov_obj.workflow_run_uri != self.parent_wf.workflow_run_uri ): - process_run_id: Optional[str] = None + process_run_id: str | None = None self.prov_obj.generate_output_prov(wo or {}, process_run_id, self.name) self.prov_obj.document.wasEndedBy( self.prov_obj.workflow_run_uri, @@ -547,7 +548,7 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: def receive_output( self, step: WorkflowJobStep, - outputparms: List[CWLObjectType], + outputparms: list[CWLObjectType], final_output_callback: OutputCallbackType, jobout: CWLObjectType, processStatus: str, @@ -627,7 +628,7 @@ def try_make_job( "Workflow step contains valueFrom but StepInputExpressionRequirement not in requirements" ) - def postScatterEval(io: CWLObjectType) -> Optional[CWLObjectType]: + def postScatterEval(io: CWLObjectType) -> CWLObjectType | None: shortio = cast(CWLObjectType, {shortname(k): v for k, v in io.items()}) fs_access = getdefault(runtimeContext.make_fs_access, StdFsAccess)("") @@ -638,7 +639,7 @@ def postScatterEval(io: CWLObjectType) -> Optional[CWLObjectType]: with fs_access.open(cast(str, val["location"]), "rb") as f: val["contents"] = content_limit_respected_read(f) - def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType]: + def valueFromFunc(k: str, v: CWLOutputType | None) -> CWLOutputType | None: if k in valueFrom: adjustDirObjs(v, functools.partial(get_listing, fs_access, recursive=True)) @@ -694,7 +695,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] return psio if "scatter" in step.tool: - scatter = cast(List[str], aslist(step.tool["scatter"])) + scatter = cast(list[str], aslist(step.tool["scatter"])) method = step.tool.get("scatterMethod") if method is None and len(scatter) != 1: raise WorkflowException( @@ -711,17 +712,17 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] step.name, "', '".join(emptyscatter), ) - - if method == "dotproduct" or method is None: - jobs = dotproduct_scatter(step, inputobj, scatter, callback, runtimeContext) - elif method == "nested_crossproduct": - jobs = nested_crossproduct_scatter( - step, inputobj, scatter, callback, runtimeContext - ) - elif method == "flat_crossproduct": - jobs = flat_crossproduct_scatter( - step, inputobj, scatter, callback, runtimeContext - ) + match method: + case "dotproduct" | None: + jobs = dotproduct_scatter(step, inputobj, scatter, callback, runtimeContext) + case "nested_crossproduct": + jobs = nested_crossproduct_scatter( + step, inputobj, scatter, callback, runtimeContext + ) + case "flat_crossproduct": + jobs = flat_crossproduct_scatter( + step, inputobj, scatter, callback, runtimeContext + ) else: if _logger.isEnabledFor(logging.DEBUG): _logger.debug("[%s] job input %s", step.name, json_dumps(inputobj, indent=4)) @@ -765,7 +766,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: """Log the start of each workflow.""" _logger.info("[%s] start", self.name) @@ -862,12 +863,12 @@ def __init__(self, step: WorkflowJobStep, container_engine: str): """Initialize this WorkflowJobLoopStep.""" self.step: WorkflowJobStep = step self.container_engine: str = container_engine - self.joborder: Optional[CWLObjectType] = None + self.joborder: CWLObjectType | None = None self.processStatus: str = "success" self.iteration: int = 0 self.output_buffer: MutableMapping[ str, - Union[MutableSequence[Optional[CWLOutputType]], Optional[CWLOutputType]], + MutableSequence[CWLOutputType | None] | CWLOutputType | None, ] = {} def _set_empty_output(self, outputMethod: str) -> None: @@ -954,7 +955,7 @@ def loop_callback( try: loop = cast(MutableSequence[CWLObjectType], self.step.tool.get("loop", [])) outputMethod = self.step.tool.get("outputMethod", "last_iteration") - state: Dict[str, Optional[WorkflowStateItem]] = {} + state: dict[str, WorkflowStateItem | None] = {} for i in self.step.tool["outputs"]: if "id" in i: iid = cast(str, i["id"]) diff --git a/docs/cli.rst b/docs/cli.rst index d569f5586..1c6021bf4 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -3,4 +3,4 @@ cwltool Command Line Options .. autoprogram:: cwltool.argparser:arg_parser() :prog: cwltool - + :groups: diff --git a/docs/conf.py b/docs/conf.py index 6e04b5d64..e476f4191 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,8 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3", None), "schema_salad": ("https://schema-salad.readthedocs.io/en/stable/", None), - "rdflib": ("https://rdflib.readthedocs.io/en/6.2.0/", None), + "rdflib": ("https://rdflib.readthedocs.io/en/stable/", None), + "cwl_utils": ("https://cwl-utils.readthedocs.io/en/stable/", None), # "ruamel.yaml": ("https://yaml.readthedocs.io/en/stable/", None), } diff --git a/docs/requirements.txt b/docs/requirements.txt index 40a2eefbf..e9d89216c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,5 @@ sphinx >= 2.2 -sphinx-rtd-theme==2.0.0 +sphinx-rtd-theme==3.0.2 sphinx-autoapi sphinx-autodoc-typehints sphinxcontrib-autoprogram -importlib_resources;python_version<'3.9' diff --git a/lint-requirements.txt b/lint-requirements.txt index b0d7944eb..a2c6768cf 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ -flake8-bugbear<24.9 -black~=24.8 +flake8-bugbear<24.13 +black==25.* codespell diff --git a/mypy-requirements.txt b/mypy-requirements.txt index a7d0dacb8..5cb8e6fe9 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,9 +1,11 @@ -mypy==1.11.2 # also update pyproject.toml +mypy==1.18.2 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 +cwltest types-requests types-setuptools types-psutil types-mock -galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2 -galaxy-util<24.2 +galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<25.1 +galaxy-util<25.1 +pydantic>=2.12.0; python_version >= "3.14" diff --git a/mypy-stubs/arcp/parse.pyi b/mypy-stubs/arcp/parse.pyi index 560671663..3ff469307 100644 --- a/mypy-stubs/arcp/parse.pyi +++ b/mypy-stubs/arcp/parse.pyi @@ -22,4 +22,4 @@ class ARCPParseResult(ParseResult): def nih_uri(self) -> str | None: ... def ni_well_known(self, base: str = ...) -> str | None: ... @property - def hash(self) -> Tuple[str, str] | None: ... + def hash(self) -> tuple[str, str] | None: ... diff --git a/mypy-stubs/bagit.pyi b/mypy-stubs/bagit.pyi index 2b0f5b9f2..ff9a7cfbc 100644 --- a/mypy-stubs/bagit.pyi +++ b/mypy-stubs/bagit.pyi @@ -1,5 +1,6 @@ import argparse -from typing import Any, Dict, Iterator, List, Optional, Tuple +from collections.abc import Iterator +from typing import Any, Dict, List, Optional, Tuple from _typeshed import Incomplete @@ -19,28 +20,28 @@ class Bag: valid_files: Any = ... valid_directories: Any = ... tags: Any = ... - info: Dict[str, str] = ... + info: dict[str, str] = ... entries: Any = ... normalized_filesystem_names: Any = ... normalized_manifest_names: Any = ... algorithms: Any = ... tag_file_name: Any = ... path: Any = ... - def __init__(self, path: Optional[Any] = ...) -> None: ... + def __init__(self, path: Any | None = ...) -> None: ... @property - def algs(self) -> List[str]: ... + def algs(self) -> list[str]: ... @property def version(self) -> str: ... def manifest_files(self) -> Iterator[str]: ... def tagmanifest_files(self) -> None: ... - def compare_manifests_with_fs(self) -> Tuple[List[str], List[str]]: ... - def compare_fetch_with_fs(self) -> List[str]: ... + def compare_manifests_with_fs(self) -> tuple[list[str], list[str]]: ... + def compare_fetch_with_fs(self) -> list[str]: ... def payload_files(self) -> Iterator[str]: ... - def payload_entries(self) -> Dict[str, str]: ... + def payload_entries(self) -> dict[str, str]: ... def save(self, processes: int = ..., manifests: bool = ...) -> None: ... - def tagfile_entries(self) -> Dict[str, str]: ... + def tagfile_entries(self) -> dict[str, str]: ... def missing_optional_tagfiles(self) -> Iterator[str]: ... - def fetch_entries(self) -> Iterator[Tuple[str, str, str]]: ... + def fetch_entries(self) -> Iterator[tuple[str, str, str]]: ... def files_to_be_fetched(self) -> Iterator[str]: ... def has_oxum(self) -> bool: ... def validate( diff --git a/mypy-stubs/black/__init__.pyi b/mypy-stubs/black/__init__.pyi deleted file mode 100644 index f741ef771..000000000 --- a/mypy-stubs/black/__init__.pyi +++ /dev/null @@ -1,26 +0,0 @@ -import asyncio -from concurrent.futures import Executor -from enum import Enum -from pathlib import Path -from typing import ( - Any, - Iterator, - List, - MutableMapping, - Optional, - Pattern, - Set, - Sized, - Tuple, - Union, -) - -from black.mode import Mode as Mode -from black.mode import TargetVersion as TargetVersion - -FileContent = str -Encoding = str -NewLine = str -FileMode = Mode - -def format_str(src_contents: str, mode: Mode) -> FileContent: ... diff --git a/mypy-stubs/mistune.pyi b/mypy-stubs/mistune.pyi deleted file mode 100644 index 3778c9195..000000000 --- a/mypy-stubs/mistune.pyi +++ /dev/null @@ -1,197 +0,0 @@ -__author__ = "Aleksandr Slepchenkov" -__email__ = "Sl.aleksandr28@gmail.com" - -from typing import ( - Any, - Dict, - Iterable, - List, - Match, - Optional, - Pattern, - Sequence, - Tuple, - Type, -) - -Tokens = List[Dict[str, Any]] -# There are too much levels of optional unions of lists of text in cell and align 385 and 396 lines in mistune - -def escape(text: str, quote: bool = ..., smart_amp: bool = ...) -> str: ... - -class BlockGrammar: - def_links: Pattern[str] - def_footnotes: Pattern[str] - newline: Pattern[str] - block_code: Pattern[str] - fences: Pattern[str] - hrule: Pattern[str] - heading: Pattern[str] - lheading: Pattern[str] - block_quote: Pattern[str] - list_block: Pattern[str] - list_item: Pattern[str] - list_bullet: Pattern[str] - paragraph: Pattern[str] - block_html: Pattern[str] - table: Pattern[str] - nptable: Pattern[str] - text: Pattern[str] - -class BlockLexer: - grammar_class: Type[BlockGrammar] - default_rules: List[str] - list_rules: Tuple[str] - footnote_rules: Tuple[str] - tokens: Tokens - def_links: Dict[str, Dict[str, str]] - def_footnotes: Dict[str, int] - rules = ... # type: BlockGrammar - def __init__(self, rules: Optional[BlockGrammar] = ..., **kwargs: Any) -> None: ... - def __call__(self, text: str, rules: Optional[Sequence[str]] = ...) -> Tokens: ... - def parse(self, text: str, rules: Optional[Sequence[str]] = ...) -> Tokens: ... - def parse_newline(self, m: Match[str]) -> None: ... - def parse_block_code(self, m: Match[str]) -> None: ... - def parse_fences(self, m: Match[str]) -> None: ... - def parse_heading(self, m: Match[str]) -> None: ... - def parse_lheading(self, m: Match[str]) -> None: ... - def parse_hrule(self, m: Match[str]) -> None: ... - def parse_list_block(self, m: Match[str]) -> None: ... - def parse_block_quote(self, m: Match[str]) -> None: ... - def parse_def_links(self, m: Match[str]) -> None: ... - def parse_def_footnotes(self, m: Match[str]) -> None: ... - def parse_table(self, m: Match[str]) -> None: ... - def parse_nptable(self, m: Match[str]) -> None: ... - def parse_block_html(self, m: Match[str]) -> None: ... - def parse_paragraph(self, m: Match[str]) -> None: ... - def parse_text(self, m: Match[str]) -> None: ... - -class InlineGrammar: - escape: Pattern[str] - inline_html: Pattern[str] - autolink: Pattern[str] - link: Pattern[str] - reflink: Pattern[str] - nolink: Pattern[str] - url: Pattern[str] - double_emphasis: Pattern[str] - emphasis: Pattern[str] - code: Pattern[str] - linebreak: Pattern[str] - strikethrough: Pattern[str] - footnote: Pattern[str] - text: Pattern[str] - def hard_wrap(self) -> None: ... - -class InlineLexer: - grammar_class: Type[InlineGrammar] - default_rules: List[str] - inline_html_rules: List[str] - renderer: Renderer - links: Dict[str, Dict[str, str]] - footnotes: Dict[str, int] - footnote_index: int - _in_link: bool - _in_footnote: bool - _parse_inline_html: bool - rules: InlineGrammar - def __init__( - self, renderer: Renderer, rules: Optional[InlineGrammar] = ..., **kwargs: Any - ) -> None: ... - def __call__(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def setup( - self, - links: Optional[Dict[str, Dict[str, str]]], - footnotes: Optional[Dict[str, int]], - ) -> None: ... - line_match: Match[str] - line_started: bool - def output(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def output_escape(self, m: Match[str]) -> str: ... - def output_autolink(self, m: Match[str]) -> str: ... - def output_url(self, m: Match[str]) -> str: ... - def output_inline_html(self, m: Match[str]) -> str: ... - def output_footnote(self, m: Match[str]) -> Optional[str]: ... - def output_link(self, m: Match[str]) -> str: ... - def output_reflink(self, m: Match[str]) -> Optional[str]: ... - def output_nolink(self, m: Match[str]) -> Optional[str]: ... - def output_double_emphasis(self, m: Match[str]) -> str: ... - def output_emphasis(self, m: Match[str]) -> str: ... - def output_code(self, m: Match[str]) -> str: ... - def output_linebreak(self, m: Match[str]) -> str: ... - def output_strikethrough(self, m: Match[str]) -> str: ... - def output_text(self, m: Match[str]) -> str: ... - -class Renderer: - options: Dict[str, str] - def __init__(self, **kwargs: Any) -> None: ... - def placeholder(self) -> str: ... - def block_code( - self, code: str, lang: Any = ... - ) -> str: ... # It seems that lang should be string, however other types are valid as well - def block_quote(self, text: str) -> str: ... - def block_html(self, html: str) -> str: ... - def header(self, text: str, level: int, raw: Optional[str] = ...) -> str: ... - def hrule(self) -> str: ... - def list( - self, body: Any, ordered: bool = ... - ) -> str: ... # body - same reason as for lang above, and for other Any in this class - def list_item(self, text: Any) -> str: ... - def paragraph(self, text: str) -> str: ... - def table(self, header: Any, body: Any) -> str: ... - def table_row(self, content: Any) -> str: ... - def table_cell(self, content: Any, **flags: Dict[str, Any]) -> str: ... - def double_emphasis(self, text: Any) -> str: ... - def emphasis(self, text: Any) -> str: ... - def codespan(self, text: str) -> str: ... - def linebreak(self) -> str: ... - def strikethrough(self, text: Any) -> str: ... - def text(self, text: Any) -> str: ... - def escape(self, text: Any) -> str: ... - def autolink(self, link: Any, is_email: bool = ...) -> str: ... - def link(self, link: Any, title: Any, text: Any) -> str: ... - def image(self, src: Any, title: Any, text: Any) -> str: ... - def inline_html(self, html: Any) -> str: ... - def newline(self) -> str: ... - def footnote_ref(self, key: Any, index: int) -> str: ... - def footnote_item(self, key: Any, text: str) -> str: ... - def footnotes(self, text: Any) -> str: ... - -class Markdown: - renderer = ... # type: Renderer - inline = ... # type: InlineLexer - block = ... # type: BlockLexer - footnotes = ... # type: List[Dict[str, Any]] - tokens = ... # type: Tokens - def __init__( - self, - renderer: Optional[Renderer] = ..., - inline: Optional[InlineLexer] = ..., - block: Optional[BlockLexer] = ..., - **kwargs: Any, - ) -> None: ... - def __call__(self, text: str) -> str: ... - def render(self, text: str) -> str: ... - def parse(self, text: str) -> str: ... - token = ... # type: Dict[str, Any] - def pop(self) -> Optional[Dict[str, Any]]: ... - def peek(self) -> Optional[Dict[str, Any]]: ... - def output(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def tok(self) -> str: ... - def tok_text(self) -> str: ... - def output_newline(self) -> str: ... - def output_hrule(self) -> str: ... - def output_heading(self) -> str: ... - def output_code(self) -> str: ... - def output_table(self) -> str: ... - def output_block_quote(self) -> str: ... - def output_list(self) -> str: ... - def output_list_item(self) -> str: ... - def output_loose_item(self) -> str: ... - def output_footnote(self) -> str: ... - def output_close_html(self) -> str: ... - def output_open_html(self) -> str: ... - def output_paragraph(self) -> str: ... - def output_text(self) -> str: ... - -def markdown(text: str, escape: bool = ..., **kwargs: Any) -> str: ... diff --git a/mypy-stubs/prov/constants.py b/mypy-stubs/prov/constants.py index 314224a63..67980a67e 100644 --- a/mypy-stubs/prov/constants.py +++ b/mypy-stubs/prov/constants.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function, unicode_literals - from prov.identifier import Namespace __author__ = "Trung Dong Huynh" @@ -170,13 +168,9 @@ PROV_ATTRIBUTES = PROV_ATTRIBUTE_QNAMES | PROV_ATTRIBUTE_LITERALS PROV_RECORD_ATTRIBUTES = list((attr, str(attr)) for attr in PROV_ATTRIBUTES) -PROV_RECORD_IDS_MAP = dict((PROV_N_MAP[rec_type_id], rec_type_id) for rec_type_id in PROV_N_MAP) -PROV_ID_ATTRIBUTES_MAP = dict( - (prov_id, attribute) for (prov_id, attribute) in PROV_RECORD_ATTRIBUTES -) -PROV_ATTRIBUTES_ID_MAP = dict( - (attribute, prov_id) for (prov_id, attribute) in PROV_RECORD_ATTRIBUTES -) +PROV_RECORD_IDS_MAP = {PROV_N_MAP[rec_type_id]: rec_type_id for rec_type_id in PROV_N_MAP} +PROV_ID_ATTRIBUTES_MAP = {prov_id: attribute for (prov_id, attribute) in PROV_RECORD_ATTRIBUTES} +PROV_ATTRIBUTES_ID_MAP = {attribute: prov_id for (prov_id, attribute) in PROV_RECORD_ATTRIBUTES} # Extra definition for convenience PROV_TYPE = PROV["type"] diff --git a/mypy-stubs/prov/identifier.pyi b/mypy-stubs/prov/identifier.pyi index c0d79d912..c344128cd 100644 --- a/mypy-stubs/prov/identifier.pyi +++ b/mypy-stubs/prov/identifier.pyi @@ -23,8 +23,8 @@ class Namespace: def uri(self) -> str: ... @property def prefix(self) -> str: ... - def contains(self, identifier: Union[Identifier, str]) -> bool: ... - def qname(self, identifier: Union[Identifier, str]) -> QualifiedName: ... + def contains(self, identifier: Identifier | str) -> bool: ... + def qname(self, identifier: Identifier | str) -> QualifiedName: ... def __eq__(self, other: Any) -> bool: ... def __ne__(self, other: Any) -> bool: ... def __hash__(self) -> int: ... diff --git a/mypy-stubs/prov/model.pyi b/mypy-stubs/prov/model.pyi index 19a13bcf0..e1147872a 100644 --- a/mypy-stubs/prov/model.pyi +++ b/mypy-stubs/prov/model.pyi @@ -1,5 +1,6 @@ +from collections.abc import Iterable from datetime import datetime -from typing import IO, Any, Dict, Iterable, List, Set, Tuple +from typing import IO, Any, Dict, List, Set, Tuple from _typeshed import Incomplete from prov.constants import * @@ -16,7 +17,7 @@ XSD_DATATYPE_PARSERS: Incomplete # def first(a_set): ... -_attributes_type = Dict[str | Identifier, Any] | List[Tuple[str | Identifier, Any]] +_attributes_type = dict[str | Identifier, Any] | list[tuple[str | Identifier, Any]] class ProvRecord: FORMAL_ATTRIBUTES: Incomplete @@ -24,24 +25,24 @@ class ProvRecord: self, bundle: str, identifier: Identifier | str, - attributes: Dict[str, str] | None = ..., + attributes: dict[str, str] | None = ..., ) -> None: ... def __hash__(self) -> int: ... def copy(self) -> ProvRecord: ... def get_type(self) -> str: ... - def get_asserted_types(self) -> Set[str]: ... + def get_asserted_types(self) -> set[str]: ... def add_asserted_type(self, type_identifier: str | QualifiedName) -> None: ... - def get_attribute(self, attr_name: str) -> Set[str]: ... + def get_attribute(self, attr_name: str) -> set[str]: ... @property def identifier(self) -> Identifier: ... @property - def attributes(self) -> List[Tuple[str, str]]: ... + def attributes(self) -> list[tuple[str, str]]: ... @property - def args(self) -> Tuple[str, ...]: ... + def args(self) -> tuple[str, ...]: ... @property - def formal_attributes(self) -> Tuple[Tuple[str, str], ...]: ... + def formal_attributes(self) -> tuple[tuple[str, str], ...]: ... @property - def extra_attributes(self) -> Tuple[Tuple[str, str], ...]: ... + def extra_attributes(self) -> tuple[tuple[str, str], ...]: ... @property def bundle(self) -> "ProvBundle": ... @property @@ -203,7 +204,7 @@ class ProvBundle: document: ProvDocument | None = ..., ) -> None: ... @property - def namespaces(self) -> Set[Namespace]: ... + def namespaces(self) -> set[Namespace]: ... @property def default_ns_uri(self) -> str | None: ... @property @@ -211,7 +212,7 @@ class ProvBundle: @property def identifier(self) -> str | None | QualifiedName: ... @property - def records(self) -> List[ProvRecord]: ... + def records(self) -> list[ProvRecord]: ... def set_default_namespace(self, uri: Namespace) -> None: ... def get_default_namespace(self) -> Namespace: ... def add_namespace( @@ -219,15 +220,15 @@ class ProvBundle: ) -> Namespace: ... def get_registered_namespaces(self) -> Iterable[Namespace]: ... def valid_qualified_name( - self, identifier: QualifiedName | Tuple[str, Identifier] + self, identifier: QualifiedName | tuple[str, Identifier] ) -> QualifiedName | None: ... def get_records( self, class_or_type_or_tuple: ( - type | type[int | str] | Tuple[type | type[int | str] | Tuple[Any, ...], ...] | None + type | type[int | str] | tuple[type | type[int | str] | tuple[Any, ...], ...] | None ) = ..., - ) -> List[ProvRecord]: ... - def get_record(self, identifier: Identifier | None) -> ProvRecord | List[ProvRecord] | None: ... + ) -> list[ProvRecord]: ... + def get_record(self, identifier: Identifier | None) -> ProvRecord | list[ProvRecord] | None: ... def is_document(self) -> bool: ... def is_bundle(self) -> bool: ... def has_bundles(self) -> bool: ... @@ -452,4 +453,4 @@ class ProvDocument(ProvBundle): **args: Any, ) -> ProvDocument: ... -def sorted_attributes(element: ProvElement, attributes: List[str]) -> List[str]: ... +def sorted_attributes(element: ProvElement, attributes: list[str]) -> list[str]: ... diff --git a/mypy-stubs/pydot.pyi b/mypy-stubs/pydot.pyi deleted file mode 100644 index bd0ab3147..000000000 --- a/mypy-stubs/pydot.pyi +++ /dev/null @@ -1,150 +0,0 @@ -from typing import Any, Dict, List, Sequence, Union - -PY3: Any -str_type = str -GRAPH_ATTRIBUTES: Any -EDGE_ATTRIBUTES: Any -NODE_ATTRIBUTES: Any -CLUSTER_ATTRIBUTES: Any -DEFAULT_PROGRAMS: Any - -def is_windows() -> bool: ... -def is_anaconda() -> bool: ... -def get_executable_extension() -> str: ... -def graph_from_dot_data(s: str) -> List["Dot"]: ... - -class Common: - def set_parent_graph(self, parent_graph: "Graph") -> None: ... - def get_parent_graph(self) -> "Graph": ... - def set(self, name: str, value: str) -> None: ... - def get(self, name: str) -> str: ... - def get_attributes(self) -> Dict[str, str]: ... - def set_sequence(self, seq: str) -> None: ... - def get_sequence(self) -> str: ... - def create_attribute_methods(self, obj_attributes: List[str]) -> None: ... - -class Error(Exception): - value: Any - def __init__(self, value: str) -> None: ... - -class InvocationException(Exception): - value: Any - def __init__(self, value: str) -> None: ... - -class Node(Common): - obj_dict: Any - def __init__(self, name: str = ..., obj_dict: Any | None = ..., **attrs: str) -> None: ... - def set_name(self, node_name: str) -> None: ... - def get_name(self) -> str: ... - def get_port(self) -> str: ... - def add_style(self, style: str) -> None: ... - def to_string(self) -> str: ... - -class Edge(Common): - obj_dict: Any - def __init__( - self, - src: str = ..., - dst: str = ..., - obj_dict: Any | None = ..., - **attrs: Dict[str, str], - ) -> None: ... - def get_source(self) -> str: ... - def get_destination(self) -> str: ... - def __hash__(self) -> int: ... - def __eq__(self, edge: Any) -> bool: ... - def parse_node_ref(self, node_str: str) -> str: ... - def to_string(self) -> str: ... - -class Graph(Common): - obj_dict: Any - def __init__( - self, - graph_name: str = ..., - obj_dict: Any | None = ..., - graph_type: str = ..., - strict: bool = ..., - suppress_disconnected: bool = ..., - simplify: bool = ..., - **attrs: Dict[str, str], - ) -> None: ... - def get_graph_type(self) -> str: ... - def get_top_graph_type(self) -> str: ... - def set_graph_defaults(self, **attrs: Dict[str, str]) -> None: ... - def get_graph_defaults(self, **attrs: Dict[str, str]) -> Dict[str, str]: ... - def set_node_defaults(self, **attrs: Dict[str, str]) -> None: ... - def get_node_defaults(self, **attrs: Dict[str, str]) -> Dict[str, str]: ... - def set_edge_defaults(self, **attrs: Dict[str, str]) -> None: ... - def get_edge_defaults(self, **attrs: Dict[str, str]) -> Dict[str, str]: ... - def set_simplify(self, simplify: bool) -> None: ... - def get_simplify(self) -> bool: ... - def set_type(self, graph_type: str) -> None: ... - def get_type(self) -> str: ... - def set_name(self, graph_name: str) -> None: ... - def get_name(self) -> str: ... - def set_strict(self, val: bool) -> None: ... - def get_strict(self, val: Any) -> bool: ... - def set_suppress_disconnected(self, val: bool) -> None: ... - def get_suppress_disconnected(self, val: Any) -> None: ... - def get_next_sequence_number(self) -> int: ... - def add_node(self, graph_node: Node) -> None: ... - def del_node(self, name: Union[str, Node], index: int | None = ...) -> bool: ... - def get_node(self, name: str) -> Node: ... - def get_nodes(self) -> List[Node]: ... - def get_node_list(self) -> List[Node]: ... - def add_edge(self, graph_edge: Edge) -> None: ... - def del_edge( - self, - src_or_list: Union[Sequence[Node], Node], - dst: str | int | None = ..., - index: int | None = ..., - ) -> bool: ... - def get_edge( - self, src_or_list: Union[Sequence[Node], Node], dst: Any | None = ... - ) -> List[Edge]: ... - def get_edges(self) -> List[Edge]: ... - def get_edge_list(self) -> List[Edge]: ... - def add_subgraph(self, sgraph: Union["Subgraph", "Cluster"]) -> None: ... - def get_subgraph(self, name: str) -> List["Subgraph"]: ... - def get_subgraphs(self) -> List["Subgraph"]: ... - def get_subgraph_list(self) -> List["Subgraph"]: ... - def set_parent_graph(self, parent_graph: "Graph") -> None: ... - def to_string(self) -> str: ... - -class Subgraph(Graph): - def __init__( - self, - graph_name: str = ..., - obj_dict: Any | Dict[str, str] = ..., - suppress_disconnected: bool = ..., - simplify: bool = ..., - **attrs: Dict[str, str], - ) -> None: ... - -class Cluster(Graph): - def __init__( - self, - graph_name: str = ..., - obj_dict: Any | Dict[str, str] = ..., - suppress_disconnected: bool = ..., - simplify: bool = ..., - **attrs: Dict[str, str], - ) -> None: ... - -class Dot(Graph): - shape_files: Any - formats: Any - prog: str - def __init__(self, *argsl: Any, **argsd: Dict[str, str]): ... - def set_shape_files(self, file_paths: Union[str, Sequence[str]]) -> None: ... - def set_prog(self, prog: str) -> None: ... - def write( - self, - path: str, - prog: Any | str = ..., - format: str = ..., - encoding: Any | str = ..., - ) -> bool: ... - def create( - self, prog: Any | str = ..., format: str = ..., encoding: Any | str = ... - ) -> bytes: ... diff --git a/mypy-stubs/rdflib/collection.pyi b/mypy-stubs/rdflib/collection.pyi index 0bee98429..a04649b6a 100644 --- a/mypy-stubs/rdflib/collection.pyi +++ b/mypy-stubs/rdflib/collection.pyi @@ -1,4 +1,5 @@ -from typing import Any, Iterator +from collections.abc import Iterator +from typing import Any from rdflib.graph import Graph from rdflib.term import Node diff --git a/mypy-stubs/rdflib/graph.pyi b/mypy-stubs/rdflib/graph.pyi index d3e6f2f54..93f3e6eae 100644 --- a/mypy-stubs/rdflib/graph.pyi +++ b/mypy-stubs/rdflib/graph.pyi @@ -1,16 +1,7 @@ import pathlib -from typing import ( - IO, - Any, - Iterable, - Iterator, - List, - Optional, - Set, - Tuple, - Union, - overload, -) +from builtins import set as _set +from collections.abc import Iterable, Iterator +from typing import IO, Any, List, Optional, Tuple, Union, overload from rdflib import query from rdflib.collection import Collection @@ -26,9 +17,9 @@ class Graph(Node): def __init__( self, store: str = ..., - identifier: Optional[Any] = ..., - namespace_manager: Optional[Any] = ..., - base: Optional[Any] = ..., + identifier: Any | None = ..., + namespace_manager: Any | None = ..., + base: Any | None = ..., ) -> None: ... store: Any = ... identifier: Any = ... @@ -44,59 +35,53 @@ class Graph(Node): def remove(self, triple: Any) -> None: ... def triples( self, - triple: Tuple[ - Optional[Union[str, Identifier]], - Optional[Union[str, Identifier]], - Optional[Identifier], + triple: tuple[ + str | Identifier | None, + str | Identifier | None, + Identifier | None, ], - ) -> Iterator[Tuple[Identifier, Identifier, Identifier]]: ... + ) -> Iterator[tuple[Identifier, Identifier, Identifier]]: ... def __getitem__( self, item: slice | Path | Node ) -> Iterator[ - Tuple[Identifier, Identifier, Identifier] | Tuple[Identifier, identifier] | Node + tuple[Identifier, Identifier, Identifier] | tuple[Identifier, identifier] | Node ]: ... def __contains__(self, triple: Any) -> bool: ... def __add__(self, other: Any) -> Graph: ... def set(self, triple: Any) -> None: ... - def subjects( - self, predicate: Optional[Any] = ..., object: Optional[Any] = ... - ) -> Iterable[Node]: ... - def predicates( - self, subject: Optional[Any] = ..., object: Optional[Any] = ... - ) -> Iterable[Node]: ... + def subjects(self, predicate: Any | None = ..., object: Any | None = ...) -> Iterable[Node]: ... + def predicates(self, subject: Any | None = ..., object: Any | None = ...) -> Iterable[Node]: ... def objects( - self, subject: Optional[Any] = ..., predicate: Optional[Any] = ... + self, subject: Any | None = ..., predicate: Any | None = ... ) -> Iterable[Identifier]: ... - def subject_predicates(self, object: Optional[Any] = ...) -> None: ... - def subject_objects(self, predicate: Optional[Any] = ...) -> None: ... - def predicate_objects(self, subject: Optional[Any] = ...) -> None: ... - def triples_choices(self, triple: Any, context: Optional[Any] = ...) -> None: ... + def subject_predicates(self, object: Any | None = ...) -> None: ... + def subject_objects(self, predicate: Any | None = ...) -> None: ... + def predicate_objects(self, subject: Any | None = ...) -> None: ... + def triples_choices(self, triple: Any, context: Any | None = ...) -> None: ... def value( self, - subject: Optional[Any] = ..., + subject: Any | None = ..., predicate: Any = ..., - object: Optional[Any] = ..., - default: Optional[Any] = ..., + object: Any | None = ..., + default: Any | None = ..., any: bool = ..., ) -> Any: ... def label(self, subject: Any, default: str = ...) -> Any: ... def preferredLabel( self, subject: Any, - lang: Optional[Any] = ..., - default: Optional[Any] = ..., + lang: Any | None = ..., + default: Any | None = ..., labelProperties: Any = ..., - ) -> List[Tuple[Any, Any]]: ... + ) -> list[tuple[Any, Any]]: ... def comment(self, subject: Any, default: str = ...) -> Any: ... def items(self, list: Any) -> Iterator[Any]: ... - def transitiveClosure( - self, func: Any, arg: Any, seen: Optional[Any] = ... - ) -> Iterator[Any]: ... + def transitiveClosure(self, func: Any, arg: Any, seen: Any | None = ...) -> Iterator[Any]: ... def transitive_objects( - self, subject: Any, property: Any, remember: Optional[Any] = ... + self, subject: Any, property: Any, remember: Any | None = ... ) -> Iterator[Any]: ... def transitive_subjects( - self, predicate: Any, object: Any, remember: Optional[Any] = ... + self, predicate: Any, object: Any, remember: Any | None = ... ) -> Iterator[Any]: ... def seq(self, subject: Any) -> Seq | None: ... def qname(self, uri: Any) -> Any: ... @@ -104,7 +89,7 @@ class Graph(Node): def bind( self, prefix: Any, namespace: Any, override: bool = ..., replace: bool = ... ) -> Any: ... - def namespaces(self) -> Iterator[Tuple[Any, Any]]: ... + def namespaces(self) -> Iterator[tuple[Any, Any]]: ... def absolutize(self, uri: Any, defrag: int = ...) -> Any: ... # no destination and non-None positional encoding @@ -113,7 +98,7 @@ class Graph(Node): self, destination: None, format: str, - base: Optional[str], + base: str | None, encoding: str, **args: Any, ) -> bytes: ... @@ -124,7 +109,7 @@ class Graph(Node): self, destination: None = ..., format: str = ..., - base: Optional[str] = ..., + base: str | None = ..., *, encoding: str, **args: Any, @@ -136,7 +121,7 @@ class Graph(Node): self, destination: None = ..., format: str = ..., - base: Optional[str] = ..., + base: str | None = ..., encoding: None = ..., **args: Any, ) -> str: ... @@ -145,10 +130,10 @@ class Graph(Node): @overload def serialize( self, - destination: Union[str, pathlib.PurePath, IO[bytes]], + destination: str | pathlib.PurePath | IO[bytes], format: str = ..., - base: Optional[str] = ..., - encoding: Optional[str] = ..., + base: str | None = ..., + encoding: str | None = ..., **args: Any, ) -> "Graph": ... @@ -156,30 +141,30 @@ class Graph(Node): @overload def serialize( self, - destination: Optional[Union[str, pathlib.PurePath, IO[bytes]]] = ..., + destination: str | pathlib.PurePath | IO[bytes] | None = ..., format: str = ..., - base: Optional[str] = ..., - encoding: Optional[str] = ..., + base: str | None = ..., + encoding: str | None = ..., **args: Any, ) -> Union[bytes, str, "Graph"]: ... def parse( self, - source: Optional[Any] = ..., - publicID: Optional[Any] = ..., - format: Optional[str] = ..., - location: Optional[Any] = ..., - file: Optional[Any] = ..., - data: Optional[Any] = ..., + source: Any | None = ..., + publicID: Any | None = ..., + format: str | None = ..., + location: Any | None = ..., + file: Any | None = ..., + data: Any | None = ..., **args: Any, ) -> "Graph": ... - def load(self, source: Any, publicID: Optional[Any] = ..., format: str = ...) -> "Graph": ... + def load(self, source: Any, publicID: Any | None = ..., format: str = ...) -> "Graph": ... def query( self, query_object: Any, processor: str = ..., result: str = ..., - initNs: Optional[Any] = ..., - initBindings: Optional[Any] = ..., + initNs: Any | None = ..., + initBindings: Any | None = ..., use_store_provided: bool = ..., **kwargs: Any, ) -> query.Result: ... @@ -187,27 +172,25 @@ class Graph(Node): self, update_object: Any, processor: str = ..., - initNs: Optional[Any] = ..., - initBindings: Optional[Any] = ..., + initNs: Any | None = ..., + initBindings: Any | None = ..., use_store_provided: bool = ..., **kwargs: Any, ) -> Any: ... def n3(self) -> str: ... def isomorphic(self, other: Any) -> bool: ... def connected(self) -> bool: ... - def all_nodes(self) -> Set[Any]: ... + def all_nodes(self) -> _set[Node]: ... def collection(self, identifier: Any) -> Collection: ... def resource(self, identifier: Any) -> Resource: ... def skolemize( self, - new_graph: Optional[Any] = ..., - bnode: Optional[Any] = ..., - authority: Optional[Any] = ..., - basepath: Optional[Any] = ..., - ) -> Graph: ... - def de_skolemize( - self, new_graph: Optional[Any] = ..., uriref: Optional[Any] = ... + new_graph: Any | None = ..., + bnode: Any | None = ..., + authority: Any | None = ..., + basepath: Any | None = ..., ) -> Graph: ... + def de_skolemize(self, new_graph: Any | None = ..., uriref: Any | None = ...) -> Graph: ... class ConjunctiveGraph(Graph): context_aware: bool = ... @@ -216,32 +199,32 @@ class ConjunctiveGraph(Graph): def __init__( self, store: str = ..., - identifier: Optional[Any] = ..., - default_graph_base: Optional[Any] = ..., + identifier: Any | None = ..., + default_graph_base: Any | None = ..., ) -> None: ... def add(self, triple_or_quad: Any) -> None: ... def addN(self, quads: Any) -> None: ... def remove(self, triple_or_quad: Any) -> None: ... # def triples(self, triple_or_quad: Tuple[Optional[Union[str, BNode]], Optional[Union[str, BNode]], Optional[BNode]], context: Tuple[Optional[Union[str, BNode]], Optional[Union[str, BNode]], Optional[BNode]]) -> Iterator[Tuple[Identifier, Identifier, Identifier]]: ... - def quads(self, triple_or_quad: Optional[Any] = ...) -> None: ... - def triples_choices(self, triple: Any, context: Optional[Any] = ...) -> None: ... - def contexts(self, triple: Optional[Any] = ...) -> None: ... + def quads(self, triple_or_quad: Any | None = ...) -> None: ... + def triples_choices(self, triple: Any, context: Any | None = ...) -> None: ... + def contexts(self, triple: Any | None = ...) -> None: ... def get_context( self, identifier: Node | str | None, quoted: bool = ..., - base: Optional[str] = ..., + base: str | None = ..., ) -> Graph: ... def remove_context(self, context: Any) -> None: ... - def context_id(self, uri: Any, context_id: Optional[Any] = ...) -> Any: ... + def context_id(self, uri: Any, context_id: Any | None = ...) -> Any: ... def parse( self, - source: Optional[Any] = ..., - publicID: Optional[Any] = ..., - format: Optional[str] = ..., - location: Optional[Any] = ..., - file: Optional[Any] = ..., - data: Optional[Any] = ..., + source: Any | None = ..., + publicID: Any | None = ..., + format: str | None = ..., + location: Any | None = ..., + file: Any | None = ..., + data: Any | None = ..., **args: Any, ) -> Graph: ... diff --git a/mypy-stubs/rdflib/namespace/__init__.pyi b/mypy-stubs/rdflib/namespace/__init__.pyi index 63af92977..6f1b96f9d 100644 --- a/mypy-stubs/rdflib/namespace/__init__.pyi +++ b/mypy-stubs/rdflib/namespace/__init__.pyi @@ -59,7 +59,7 @@ SPLIT_START_CATEGORIES = NAME_START_CATEGORIES + ["Nd"] XMLNS = "http://www.w3.org/XML/1998/namespace" -def split_uri(uri: Any, split_start: Any = ...) -> Tuple[str, str]: ... +def split_uri(uri: Any, split_start: Any = ...) -> tuple[str, str]: ... from rdflib.namespace._CSVW import CSVW from rdflib.namespace._DC import DC diff --git a/mypy-stubs/rdflib/paths.pyi b/mypy-stubs/rdflib/paths.pyi index 9bea17956..621909877 100644 --- a/mypy-stubs/rdflib/paths.pyi +++ b/mypy-stubs/rdflib/paths.pyi @@ -1,5 +1,5 @@ -from collections.abc import Generator -from typing import Any, Callable, Union +from collections.abc import Callable, Generator +from typing import Any, Union from rdflib.term import Node as Node from rdflib.term import URIRef as URIRef diff --git a/mypy-stubs/rdflib/plugin.pyi b/mypy-stubs/rdflib/plugin.pyi index 9b94e99f3..cef006f0c 100644 --- a/mypy-stubs/rdflib/plugin.pyi +++ b/mypy-stubs/rdflib/plugin.pyi @@ -6,5 +6,5 @@ def register(name: str, kind: Any, module_path: str, class_name: str) -> None: . PluginT = TypeVar("PluginT") -def get(name: str, kind: Type[PluginT]) -> Type[PluginT]: ... +def get(name: str, kind: type[PluginT]) -> type[PluginT]: ... def plugins(name: Any | None = ..., kind: Any | None = ...) -> None: ... diff --git a/mypy-stubs/rdflib/query.pyi b/mypy-stubs/rdflib/query.pyi index 981fe12d2..a72bf711e 100644 --- a/mypy-stubs/rdflib/query.pyi +++ b/mypy-stubs/rdflib/query.pyi @@ -1,12 +1,12 @@ -from typing import IO, Any, Dict, Iterator, List, Mapping, Optional, Tuple, overload +from collections.abc import Iterator, Mapping +from typing import IO, Any, Dict, List, Optional, SupportsIndex, Tuple, overload from rdflib import URIRef, Variable from rdflib.term import Identifier -from typing_extensions import SupportsIndex -class ResultRow(Tuple["Identifier", ...]): +class ResultRow(tuple["Identifier", ...]): def __new__( - cls, values: Mapping[Variable, Identifier], labels: List[Variable] + cls, values: Mapping[Variable, Identifier], labels: list[Variable] ) -> ResultRow: ... def __getattr__(self, name: str) -> Identifier: ... @overload @@ -14,9 +14,9 @@ class ResultRow(Tuple["Identifier", ...]): @overload def __getitem__(self, __x: SupportsIndex) -> Identifier: ... @overload - def __getitem__(self, __x: slice) -> Tuple[Identifier, ...]: ... + def __getitem__(self, __x: slice) -> tuple[Identifier, ...]: ... def get(self, name: str, default: Any | None = ...) -> Identifier: ... - def asdict(self) -> Dict[str, Identifier]: ... + def asdict(self) -> dict[str, Identifier]: ... class Result: type: Any @@ -39,4 +39,4 @@ class Result: encoding: str = ..., format: str = ..., **args: Any, - ) -> Optional[bytes]: ... + ) -> bytes | None: ... diff --git a/mypy-stubs/rdflib/resource.pyi b/mypy-stubs/rdflib/resource.pyi index 0dd3b988e..d9e16dc55 100644 --- a/mypy-stubs/rdflib/resource.pyi +++ b/mypy-stubs/rdflib/resource.pyi @@ -1,4 +1,5 @@ -from typing import Any, Iterable, Iterator, Tuple +from collections.abc import Iterable, Iterator +from typing import Any, Tuple from _typeshed import Incomplete from rdflib.graph import Graph, Seq @@ -14,9 +15,9 @@ class Resource: def subjects(self, predicate: Any | None = ...) -> Iterable[Node]: ... def predicates(self, o: Incomplete | None = ...) -> Iterable[Node]: ... def objects(self, predicate: Any | None = ...) -> Iterable[Node]: ... - def subject_predicates(self) -> Iterator[Tuple[Node, Node]]: ... - def subject_objects(self) -> Iterator[Tuple[Node, Node]]: ... - def predicate_objects(self) -> Iterator[Tuple[Node, Node]]: ... + def subject_predicates(self) -> Iterator[tuple[Node, Node]]: ... + def subject_objects(self) -> Iterator[tuple[Node, Node]]: ... + def predicate_objects(self) -> Iterator[tuple[Node, Node]]: ... def value( self, p: Node, o: Node | None = ..., default: Any | None = ..., any: bool = ... ) -> Any: ... diff --git a/mypy-stubs/rdflib/term.pyi b/mypy-stubs/rdflib/term.pyi index 0830bdc29..83a595b54 100644 --- a/mypy-stubs/rdflib/term.pyi +++ b/mypy-stubs/rdflib/term.pyi @@ -1,9 +1,10 @@ -from typing import Any, Callable, Union +from collections.abc import Callable +from typing import Any, Union class Node: ... class Identifier(Node, str): - def __new__(cls, value: Union[Any, str, None]) -> "Identifier": ... + def __new__(cls, value: Any | str | None) -> "Identifier": ... def eq(self, other: Any) -> bool: ... def neq(self, other: Any) -> bool: ... diff --git a/mypy-stubs/shellescape/__init__.pyi b/mypy-stubs/shellescape/__init__.pyi deleted file mode 100644 index 621241e5e..000000000 --- a/mypy-stubs/shellescape/__init__.pyi +++ /dev/null @@ -1,5 +0,0 @@ -# Stubs for shellescape (Python 2) -# -# NOTE: This dynamically typed stub was automatically generated by stubgen. - -from .main import quote as quote diff --git a/mypy-stubs/shellescape/main.pyi b/mypy-stubs/shellescape/main.pyi deleted file mode 100644 index 69eade63e..000000000 --- a/mypy-stubs/shellescape/main.pyi +++ /dev/null @@ -1,5 +0,0 @@ -# Stubs for shellescape.main (Python 2) - -from typing import AnyStr - -def quote(s: AnyStr) -> AnyStr: ... diff --git a/mypy-stubs/spython/main/__init__.pyi b/mypy-stubs/spython/main/__init__.pyi index adced1f3a..ac22f83cb 100644 --- a/mypy-stubs/spython/main/__init__.pyi +++ b/mypy-stubs/spython/main/__init__.pyi @@ -1,4 +1,5 @@ -from typing import Iterator, Optional +from collections.abc import Iterator +from typing import Optional from .base import Client as _BaseClient from .build import build as base_build diff --git a/mypy-stubs/spython/main/build.pyi b/mypy-stubs/spython/main/build.pyi index 098ba3436..e2b295cc1 100644 --- a/mypy-stubs/spython/main/build.pyi +++ b/mypy-stubs/spython/main/build.pyi @@ -1,23 +1,24 @@ -from typing import Iterator, Optional +from collections.abc import Iterator +from typing import Optional from .base import Client def build( self: Client, - recipe: Optional[str] = ..., - image: Optional[str] = ..., - isolated: Optional[bool] = ..., - sandbox: Optional[bool] = ..., - writable: Optional[bool] = ..., - build_folder: Optional[str] = ..., - robot_name: Optional[bool] = ..., - ext: Optional[str] = ..., - sudo: Optional[bool] = ..., - stream: Optional[bool] = ..., - force: Optional[bool] = ..., - options: Optional[list[str]] | None = ..., - quiet: Optional[bool] = ..., - return_result: Optional[bool] = ..., - sudo_options: Optional[str | list[str]] = ..., - singularity_options: Optional[list[str]] = ..., + recipe: str | None = ..., + image: str | None = ..., + isolated: bool | None = ..., + sandbox: bool | None = ..., + writable: bool | None = ..., + build_folder: str | None = ..., + robot_name: bool | None = ..., + ext: str | None = ..., + sudo: bool | None = ..., + stream: bool | None = ..., + force: bool | None = ..., + options: list[str] | None | None = ..., + quiet: bool | None = ..., + return_result: bool | None = ..., + sudo_options: str | list[str] | None = ..., + singularity_options: list[str] | None = ..., ) -> tuple[str, Iterator[str]]: ... diff --git a/mypy-stubs/spython/main/parse/recipe.pyi b/mypy-stubs/spython/main/parse/recipe.pyi index dabd4ebc5..21378549b 100644 --- a/mypy-stubs/spython/main/parse/recipe.pyi +++ b/mypy-stubs/spython/main/parse/recipe.pyi @@ -1,19 +1,19 @@ from typing import Optional class Recipe: - cmd: Optional[str] + cmd: str | None comments: list[str] - entrypoint: Optional[str] + entrypoint: str | None environ: list[str] files: list[str] layer_files: dict[str, str] install: list[str] labels: list[str] ports: list[str] - test: Optional[str] + test: str | None volumes: list[str] - workdir: Optional[str] + workdir: str | None layer: int - fromHeader: Optional[str] - source: Optional[Recipe] - def __init__(self, recipe: Optional[Recipe] = ..., layer: int = ...) -> None: ... + fromHeader: str | None + source: Recipe | None + def __init__(self, recipe: Recipe | None = ..., layer: int = ...) -> None: ... diff --git a/mypy-stubs/spython/main/parse/writers/singularity.pyi b/mypy-stubs/spython/main/parse/writers/singularity.pyi index c80198461..b3011d7ea 100644 --- a/mypy-stubs/spython/main/parse/writers/singularity.pyi +++ b/mypy-stubs/spython/main/parse/writers/singularity.pyi @@ -5,6 +5,6 @@ from .base import WriterBase as WriterBase class SingularityWriter(WriterBase): name: str - def __init__(self, recipe: Optional[dict[str, Recipe]] = ...) -> None: ... + def __init__(self, recipe: dict[str, Recipe] | None = ...) -> None: ... def validate(self) -> None: ... def convert(self, runscript: str = ..., force: bool = ...) -> str: ... diff --git a/mypy.ini b/mypy.ini index bac992869..02545dce5 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,7 @@ show_column_numbers = true show_error_codes = true pretty = true warn_unreachable = True +local_partial_types = true [mypy-galaxy.tool_util.*] ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 05a2b82f7..11fd829bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,22 +1,37 @@ [build-system] requires = [ "setuptools>=45", - "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.11.2", # also update mypy-requirements.txt + "setuptools_scm[toml]>=8.0.4,<10", + "mypy==1.18.2", # also update mypy-requirements.txt "types-requests", "types-psutil", - "importlib_resources>=1.4;python_version<'3.9'", - "ruamel.yaml>=0.16.0,<0.18", - "schema-salad>=8.7,<9", + "ruamel.yaml>=0.16.0,<0.19", + "schema-salad>=8.9,<9", "cwl-utils>=0.32", "toml", "argcomplete>=1.12.0", + "rich-argparse", + "pydot >= 1.4.1" ] build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "cwltool/_version.py" +[tool.cibuildwheel] +test-command = "python -m pytest --ignore cwltool/schemas -n logical --dist worksteal --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool" +test-requires = "-r test-requirements.txt" +build-verbosity = 1 +environment = { CWLTOOL_USE_MYPYC="1", MYPYPATH="$(pwd)/mypy-stubs" } +# Disable building PyPy wheels on all platforms +# Skip free-threaded builds, as mypyc does not support them yet +# https://mypy.readthedocs.io/en/stable/changelog.html#mypyc-partial-unsafe-support-for-free-threading +skip = ["pp*", "cp31?t-*"] + +# Install system library +[tool.cibuildwheel.linux] +before-all = "apk add libxml2-dev libxslt-dev nodejs || yum install -y libxml2-devel libxslt-devel nodejs environment-modules || apt-get install -y --no-install-recommends libxml2-dev libxslt-dev nodejs environment-modules" + [tool.black] line-length = 100 -target-version = [ "py38" ] +target-version = [ "py310" ] diff --git a/requirements.txt b/requirements.txt index df82a6f43..c29798889 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,14 @@ requests>=2.6.1 ruamel.yaml>=0.16.0,<0.19 -rdflib>=4.2.2,<7.1 -shellescape>=3.4.1,<3.9 -schema-salad>=8.7,<9 +rdflib>=4.2.2,<7.5 +schema-salad>=8.9,<9 prov==1.5.1 mypy-extensions psutil>=5.6.6 -importlib_resources>=1.4;python_version<'3.9' coloredlogs -pydot>=1.4.1,<3 +pydot>=1.4.1 argcomplete>=1.12.0 pyparsing!=3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 cwl-utils>=0.32 spython>=0.3.0 +rich-argparse diff --git a/setup.py b/setup.py index 9980276e5..bdf5f975d 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,15 @@ #!/usr/bin/env python3 """Setup for the reference implementation of the CWL standards.""" +import glob import os import sys import warnings +from typing import TYPE_CHECKING, Any -from setuptools import setup +from setuptools import Extension, setup + +if TYPE_CHECKING: + from typing import TypeGuard if os.name == "nt": warnings.warn( @@ -20,6 +25,31 @@ stacklevel=1, ) + +def _is_list_of_setuptools_extension(items: list[Any]) -> "TypeGuard[list[Extension]]": + return all(isinstance(item, Extension) for item in items) + + +def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> list[str]: + """ + Find all interesting data files, for setup(package_data=). + + Arguments: + root: The directory to search in. + globs: A list of glob patterns to accept files. + """ + rv_dirs = [root for root, dirs, files in os.walk(base)] + rv = [] + for rv_dir in rv_dirs: + files = [] + for pat in globs: + files += glob.glob(os.path.join(rv_dir, pat)) + if not files: + continue + rv.extend([os.path.relpath(f, root) for f in files]) + return rv + + SETUP_DIR = os.path.dirname(__file__) README = os.path.join(SETUP_DIR, "README.rst") @@ -34,55 +64,51 @@ USE_MYPYC = True if USE_MYPYC: - mypyc_targets = [ - "cwltool/argparser.py", - "cwltool/builder.py", - "cwltool/checker.py", - "cwltool/command_line_tool.py", - # "cwltool/context.py", # monkeypatching - "cwltool/cwlrdf.py", - "cwltool/docker_id.py", - "cwltool/docker.py", - "cwltool/udocker.py", - "cwltool/errors.py", - "cwltool/executors.py", - "cwltool/factory.py", - "cwltool/flatten.py", - # "cwltool/__init__.py", - "cwltool/job.py", - "cwltool/load_tool.py", - # "cwltool/loghandler.py", # so we can monkeypatch the logger from tests - # "cwltool/__main__.py", - "cwltool/main.py", - "cwltool/mutation.py", - "cwltool/pack.py", - "cwltool/pathmapper.py", - "cwltool/process.py", - "cwltool/procgenerator.py", - # "cwltool/cwlprov/__init__.py", - "cwltool/cwlprov/provenance_constants.py", - "cwltool/cwlprov/provenance_profile.py", - "cwltool/cwlprov/ro.py", - # "cwltool/cwlprov/writablebagfile.py", # WritableBag is having issues - "cwltool/resolver.py", - "cwltool/secrets.py", - "cwltool/singularity.py", - "cwltool/software_requirements.py", - # "cwltool/stdfsaccess.py", # StdFsAccess needs to be subclassable - "cwltool/subgraph.py", - "cwltool/update.py", - "cwltool/utils.py", - "cwltool/validate_js.py", - "cwltool/workflow.py", + mypyc_skiplist = tuple( + os.path.join("cwltool", x) + for x in ( + "context.py", # monkeypatching + "__init__.py", + "loghandler.py", # so we can monkeypatch the logger from tests + "__main__.py", + "cwlprov/__init__.py", + "cuda.py", # for monkeypatch + "run_job.py", + "procgenerator.py", # for ProcessGenerator + "cwlprov/writablebagfile.py", # WritableBag is having issues + "stdfsaccess.py", # StdFsAccess needs to be subclassable + ) + ) + + everything = [os.path.join("cwltool", x) for x in _find_package_data("cwltool", ["*.py"])] + # Start with all the .py files + all_real_pys = [ + x for x in everything if not x.startswith(os.path.join("mypy", "typeshed") + os.sep) ] + # Strip out anything in our skiplist + mypyc_targets = [x for x in all_real_pys if x not in mypyc_skiplist] + + # Strip out any test code + mypyc_targets = [x for x in mypyc_targets if not x.startswith("tests" + os.sep)] - from mypyc.build import mypycify # type: ignore[import-untyped] + mypyc_targets.sort() + + from mypyc.build import mypycify opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") - ext_modules = mypycify(mypyc_targets, opt_level=opt_level) + debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "1") + force_multifile = os.getenv("MYPYC_MULTI_FILE", "") == "1" + ext_modules = mypycify( + mypyc_targets, + opt_level=opt_level, + debug_level=debug_level, + multi_file=force_multifile, + ) else: ext_modules = [] +assert _is_list_of_setuptools_extension(ext_modules), "Expected mypycify to use setuptools" + setup( name="cwltool", description="Common workflow language reference implementation", @@ -121,37 +147,36 @@ package_dir={"cwltool.tests": "tests"}, include_package_data=True, install_requires=[ - "setuptools", "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.16, < 0.19", - "rdflib >= 4.2.2, < 7.1.0", - "shellescape >= 3.4.1, < 3.9", - "schema-salad >= 8.7, < 9", + "rdflib >= 4.2.2, < 7.5.0", + "schema-salad >= 8.9, < 9", "prov == 1.5.1", "mypy-extensions", "psutil >= 5.6.6", - "importlib_resources>=1.4;python_version<'3.9'", "coloredlogs", - "pydot >= 1.4.1, <3", - "argcomplete", + "pydot >= 1.4.1", + "argcomplete >= 1.12.0", "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "cwl-utils >= 0.32", "spython >= 0.3.0", + "rich-argparse", ], extras_require={ "deps": [ - "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2", - "galaxy-util <24.2", + "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<25.1", + "galaxy-util <25.1", + "pillow", # workaround for https://github.com/galaxyproject/galaxy/pull/20525 ], }, - python_requires=">=3.8, <3.14", + python_requires=">=3.10, <3.15", use_scm_version=True, - setup_requires=PYTEST_RUNNER + ["setuptools_scm>=8.0.4,<9"], + setup_requires=PYTEST_RUNNER + ["setuptools_scm>=8.0.4,<10"], test_suite="tests", tests_require=[ - "bagit >= 1.6.4, < 1.9", - "pytest >= 6.2, < 8.4", + "bagit >= 1.6.4, < 1.10", + "pytest >= 6.2, < 8.5", "mock >= 2.0.0", "pytest-mock >= 1.10.0", "pytest-httpserver", @@ -171,12 +196,11 @@ "Operating System :: POSIX", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Bio-Informatics", "Topic :: Scientific/Engineering :: Astronomy", diff --git a/test-requirements.txt b/test-requirements.txt index e545ee65a..6b87c3db6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,13 +1,14 @@ -bagit>=1.6.4,<1.9 -pytest>= 6.2,< 8.4 +bagit>=1.6.4,<1.10 +pytest>= 6.2,< 8.5 pytest-xdist>=3.2.0 # for the worksteal scheduler psutil # enhances pytest-xdist to allow "-n logical" pytest-httpserver -pytest-retry;python_version>'3.9' +pytest-retry mock>=2.0.0 pytest-mock>=1.10.0 pytest-cov arcp>=0.2.0 -r requirements.txt -galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2 -galaxy-util<24.2 +galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<25.1 +galaxy-util<25.1 +pydantic>=2.12.0; python_version >= "3.14" diff --git a/tests/CometAdapter.cwl b/tests/CometAdapter.cwl new file mode 100755 index 000000000..3d7083560 --- /dev/null +++ b/tests/CometAdapter.cwl @@ -0,0 +1,229 @@ +#!/usr/bin/env cwl-runner +# Copyright (c) 2002-present, The OpenMS Team -- EKU Tuebingen, ETH Zurich, and FU Berlin +# SPDX-License-Identifier: Apache-2.0 +label: CometAdapter +doc: Annotates MS/MS spectra using Comet. +inputs: + in: + doc: Input file + type: File + out: + doc: Output file + type: string + database: + doc: FASTA file + type: File + comet_executable: + doc: The Comet executable. Provide a full or relative path, or make sure it can be found in your PATH environment. + type: File + pin_out: + doc: Output file - for Percolator input + type: string? + default_params_file: + doc: Default Comet params file. All parameters of this take precedence. A template file can be generated using 'comet.exe -p' + type: File? + precursor_mass_tolerance: + doc: "Precursor monoisotopic mass tolerance (Comet parameter: peptide_mass_tolerance). See also precursor_error_units to set the unit." + type: double? + precursor_error_units: + doc: "Unit of precursor monoisotopic mass tolerance for parameter precursor_mass_tolerance (Comet parameter: peptide_mass_units)" + type: string? + isotope_error: + doc: This parameter controls whether the peptide_mass_tolerance takes into account possible isotope errors in the precursor mass measurement. Use -8/-4/0/4/8 only for SILAC. + type: string? + fragment_mass_tolerance: + doc: "This is half the bin size, which is used to segment the MS/MS spectrum. Thus, the value should be a bit higher than for other search engines, since the bin might not be centered around the peak apex (see 'fragment_bin_offset').CAUTION: Low tolerances have heavy impact on RAM usage (since Comet uses a lot of bins in this case). Consider using use_sparse_matrix and/or spectrum_batch_size." + type: double? + fragment_error_units: + doc: Fragment monoisotopic mass error units + type: string? + fragment_bin_offset: + doc: "Offset of fragment bins. Recommended by Comet: low-res: 0.4, high-res: 0.0" + type: double? + instrument: + doc: "Comets theoretical_fragment_ions parameter: theoretical fragment ion peak representation, high-res: sum of intensities plus flanking bins, ion trap (low-res) ms/ms: sum of intensities of central M bin only" + type: string? + use_A_ions: + doc: use A ions for PSM + type: boolean? + use_B_ions: + doc: use B ions for PSM + type: boolean? + use_C_ions: + doc: use C ions for PSM + type: boolean? + use_X_ions: + doc: use X ions for PSM + type: boolean? + use_Y_ions: + doc: use Y ions for PSM + type: boolean? + use_Z_ions: + doc: use Z ions for PSM + type: boolean? + use_NL_ions: + doc: use neutral loss (NH3, H2O) ions from b/y for PSM + type: boolean? + enzyme: + doc: The enzyme used for peptide digestion. + type: string? + second_enzyme: + doc: Additional enzyme used for peptide digestion. + type: string? + num_enzyme_termini: + doc: Specify the termini where the cleavage rule has to match + type: string? + missed_cleavages: + doc: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. + type: long? + min_peptide_length: + doc: Minimum peptide length to consider. + type: long? + max_peptide_length: + doc: Maximum peptide length to consider. + type: long? + num_hits: + doc: Number of peptide hits (PSMs) per spectrum in output file + type: long? + precursor_charge: + doc: "Precursor charge range to search (if spectrum is not annotated with a charge or if override_charge!=keep any known): 0:[num] == search all charges, 2:6 == from +2 to +6, 3:3 == +3" + type: string? + override_charge: + doc: "_keep any known_: keep any precursor charge state (from input), _ignore known_: ignore known precursor charge state and use precursor_charge parameter, _ignore outside range_: ignore precursor charges outside precursor_charge range, _keep known search unknown_: keep any known precursor charge state. For unknown charge states, search as singly charged if there is no signal above the precursor m/z or use the precursor_charge range" + type: string? + ms_level: + doc: MS level to analyze, valid are levels 2 (default) or 3 + type: long? + activation_method: + doc: If not ALL, only searches spectra of the given method + type: string? + digest_mass_range: + doc: MH+ peptide mass range to analyze + type: string? + max_fragment_charge: + doc: Set maximum fragment charge state to analyze as long as still lower than precursor charge - 1. (Allowed max 5) + type: long? + max_precursor_charge: + doc: set maximum precursor charge state to analyze (allowed max 9) + type: long? + clip_nterm_methionine: + doc: If set to true, also considers the peptide sequence w/o N-term methionine separately and applies appropriate N-term mods to it + type: boolean? + spectrum_batch_size: + doc: max. number of spectra to search at a time; use 0 to search the entire scan range in one batch + type: long? + mass_offsets: + doc: One or more mass offsets to search (values subtracted from deconvoluted precursor mass). Has to include 0.0 if you want the default mass to be searched. + type: double[]? + minimum_peaks: + doc: Required minimum number of peaks in spectrum to search (default 10) + type: long? + minimum_intensity: + doc: Minimum intensity value to read in + type: double? + remove_precursor_peak: + doc: no = no removal, yes = remove all peaks around precursor m/z, charge_reduced = remove all charge reduced precursor peaks (for ETD/ECD). phosphate_loss = remove the HPO3 (-80) and H3PO4 (-98) precursor phosphate neutral loss peaks. See also remove_precursor_tolerance + type: string? + remove_precursor_tolerance: + doc: one-sided tolerance for precursor removal in Thompson + type: double? + clear_mz_range: + doc: for iTRAQ/TMT type data; will clear out all peaks in the specified m/z range, if not 0:0 + type: string? + fixed_modifications: + doc: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + type: string[]? + variable_modifications: + doc: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + type: string[]? + binary_modifications: + doc: "List of modification group indices. Indices correspond to the binary modification index used by comet to group individually searched lists of variable modifications.\nNote: if set, both variable_modifications and binary_modifications need to have the same number of entries as the N-th entry corresponds to the N-th variable_modification.\n if left empty (default), all entries are internally set to 0 generating all permutations of modified and unmodified residues.\n For a detailed explanation please see the parameter description in the Comet help." + type: long[]? + max_variable_mods_in_peptide: + doc: Set a maximum number of variable modifications per peptide + type: long? + require_variable_mod: + doc: If true, requires at least one variable modification per peptide + type: boolean? + reindex: + doc: Recalculate peptide to protein association using OpenMS. Annotates target-decoy information. + type: string? + log: + doc: Name of log file (created only when specified) + type: string? + debug: + doc: Sets the debug level + type: long? + threads: + doc: Sets the number of threads allowed to be used by the TOPP tool + type: long? + no_progress: + doc: Disables progress logging to command line + type: boolean? + force: + doc: Overrides tool-specific checks + type: boolean? + test: + doc: Enables the test mode (needed for internal use only) + type: boolean? + PeptideIndexing__decoy_string: + doc: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions in the protein database to indicate decoy proteins. If empty (default), it's determined automatically (checking for common terms, both as prefix and suffix). + type: string? + PeptideIndexing__decoy_string_position: + doc: Is the 'decoy_string' prepended (prefix) or appended (suffix) to the protein accession? (ignored if decoy_string is empty) + type: string? + PeptideIndexing__missing_decoy_action: + doc: "Action to take if NO peptide was assigned to a decoy protein (which indicates wrong database or decoy string): 'error' (exit with error, no output), 'warn' (exit with success, warning message), 'silent' (no action is taken, not even a warning)" + type: string? + PeptideIndexing__write_protein_sequence: + doc: If set, the protein sequences are stored as well. + type: boolean? + PeptideIndexing__write_protein_description: + doc: If set, the protein description is stored as well. + type: boolean? + PeptideIndexing__keep_unreferenced_proteins: + doc: If set, protein hits which are not referenced by any peptide are kept. + type: boolean? + PeptideIndexing__unmatched_action: + doc: "If peptide sequences cannot be matched to any protein: 1) raise an error; 2) warn (unmatched PepHits will miss target/decoy annotation with downstream problems); 3) remove the hit." + type: string? + PeptideIndexing__aaa_max: + doc: Maximal number of ambiguous amino acids (AAAs) allowed when matching to a protein database with AAAs. AAAs are 'B', 'J', 'Z' and 'X'. + type: long? + PeptideIndexing__mismatches_max: + doc: Maximal number of mismatched (mm) amino acids allowed when matching to a protein database. The required runtime is exponential in the number of mm's; apply with care. MM's are allowed in addition to AAA's. + type: long? + PeptideIndexing__IL_equivalent: + doc: Treat the isobaric amino acids isoleucine ('I') and leucine ('L') as equivalent (indistinguishable). Also occurrences of 'J' will be treated as 'I' thus avoiding ambiguous matching. + type: boolean? + PeptideIndexing__allow_nterm_protein_cleavage: + doc: Allow the protein N-terminus amino acid to clip. + type: string? + PeptideIndexing__enzyme__name: + doc: "Enzyme which determines valid cleavage sites - e.g. trypsin cleaves after lysine (K) or arginine (R), but not before proline (P). Default: deduce from input" + type: string? + PeptideIndexing__enzyme__specificity: + doc: "Specificity of the enzyme. Default: deduce from input.\n 'full': both internal cleavage sites must match.\n 'semi': one of two internal cleavage sites must match.\n 'none': allow all peptide hits no matter their context (enzyme is irrelevant)." + type: string? +outputs: + out: + type: File + outputBinding: + glob: $(inputs.out) + pin_out: + type: File? + outputBinding: + glob: $(inputs.pin_out) +cwlVersion: v1.2 +class: CommandLineTool +baseCommand: + - CometAdapter +requirements: + InlineJavascriptRequirement: {} + InitialWorkDirRequirement: + listing: + - entryname: cwl_inputs.json + entry: $(JSON.stringify(inputs)) +arguments: + - -ini + - cwl_inputs.json diff --git a/tests/arg-empty-prefix-separate-false.cwl b/tests/arg-empty-prefix-separate-false.cwl old mode 100644 new mode 100755 diff --git a/tests/cache_environment.yml b/tests/cache_environment.yml new file mode 100644 index 000000000..97cfe97a2 --- /dev/null +++ b/tests/cache_environment.yml @@ -0,0 +1,5 @@ +cwl:requirements: + - class: EnvVarRequirement + envDef: + - envName: TEST_ENV + envValue: value1 diff --git a/tests/cache_environment2.yml b/tests/cache_environment2.yml new file mode 100644 index 000000000..95f3d656b --- /dev/null +++ b/tests/cache_environment2.yml @@ -0,0 +1,5 @@ +cwl:requirements: + - class: EnvVarRequirement + envDef: + - envName: TEST_ENV + envValue: value2 diff --git a/tests/cache_environment_tool.cwl b/tests/cache_environment_tool.cwl new file mode 100644 index 000000000..fd1b80552 --- /dev/null +++ b/tests/cache_environment_tool.cwl @@ -0,0 +1,8 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + [] +outputs: + [] + +baseCommand: [bash, "-c", "echo $TEST_ENV"] diff --git a/tests/checker_wf/optional_array_mismatch.cwl b/tests/checker_wf/optional_array_mismatch.cwl old mode 100644 new mode 100755 index d6f052757..a557927f2 --- a/tests/checker_wf/optional_array_mismatch.cwl +++ b/tests/checker_wf/optional_array_mismatch.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: Workflow inputs: diff --git a/tests/cwl-conformance/cwltool-conftest.py b/tests/cwl-conformance/cwltool-conftest.py index 3e2b83990..da2f5d138 100644 --- a/tests/cwl-conformance/cwltool-conftest.py +++ b/tests/cwl-conformance/cwltool-conftest.py @@ -6,20 +6,20 @@ import json from io import StringIO -from typing import Any, Dict, List, Optional, Tuple +from typing import Any from cwltest import utils def pytest_cwl_execute_test( - config: utils.CWLTestConfig, processfile: str, jobfile: Optional[str] -) -> Tuple[int, Optional[Dict[str, Any]]]: + config: utils.CWLTestConfig, processfile: str, jobfile: str | None +) -> tuple[int, dict[str, Any] | None]: """Use the CWL reference runner (cwltool) to execute tests.""" from cwltool import main from cwltool.errors import WorkflowException stdout = StringIO() - argsl: List[str] = [f"--outdir={config.outdir}"] + argsl: list[str] = [f"--outdir={config.outdir}"] if config.runner_quiet: argsl.append("--quiet") elif config.verbose: diff --git a/tests/default_values_list.cwl b/tests/default_values_list.cwl old mode 100644 new mode 100755 diff --git a/tests/echo-badposition-expr.cwl b/tests/echo-badposition-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/echo-position-expr.cwl b/tests/echo-position-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/echo-stdout-log-dir.cwl b/tests/echo-stdout-log-dir.cwl old mode 100644 new mode 100755 diff --git a/tests/env.cwl b/tests/env.cwl old mode 100644 new mode 100755 diff --git a/tests/env2.cwl b/tests/env2.cwl old mode 100644 new mode 100755 diff --git a/tests/env3.cwl b/tests/env3.cwl old mode 100644 new mode 100755 diff --git a/tests/env4.cwl b/tests/env4.cwl old mode 100644 new mode 100755 diff --git a/tests/input_deps/docker-array-secondaryfiles.cwl b/tests/input_deps/docker-array-secondaryfiles.cwl old mode 100644 new mode 100755 diff --git a/tests/iwdr_bad_expr.cwl b/tests/iwdr_bad_expr.cwl old mode 100644 new mode 100755 diff --git a/tests/iwdr_dir_literal_real_file.cwl b/tests/iwdr_dir_literal_real_file.cwl old mode 100644 new mode 100755 diff --git a/tests/iwdr_writable_secfile-inputs.json b/tests/iwdr_writable_secfile-inputs.json new file mode 100644 index 000000000..f49270436 --- /dev/null +++ b/tests/iwdr_writable_secfile-inputs.json @@ -0,0 +1,12 @@ +{ + "example_file_with_secondary": { + "class": "File", + "location": "2.fasta", + "secondaryFiles": [ + { + "class": "File", + "location": "2.fastq" + } + ] + } +} diff --git a/tests/iwdr_writable_secfile.cwl b/tests/iwdr_writable_secfile.cwl new file mode 100644 index 000000000..98e72eac4 --- /dev/null +++ b/tests/iwdr_writable_secfile.cwl @@ -0,0 +1,23 @@ +cwlVersion: v1.2 +class: CommandLineTool + +requirements: + InlineJavascriptRequirement: {} + InitialWorkDirRequirement: + listing: + - entry: $(inputs.example_file_with_secondary) + writable: true +inputs: + example_file_with_secondary: + type: File + secondaryFiles: + - pattern: "^.fastq" + required: true +outputs: + same_file: + type: stdout + +baseCommand: cat +arguments: + - $(inputs.example_file_with_secondary.path) + - $(inputs.example_file_with_secondary.path.split('.')[0]+'.fastq') diff --git a/tests/load_contents-array.cwl b/tests/load_contents-array.cwl old mode 100644 new mode 100755 index f6b786ec6..eaad62436 --- a/tests/load_contents-array.cwl +++ b/tests/load_contents-array.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: "v1.2" class: CommandLineTool baseCommand: echo diff --git a/tests/loop-ext/all-output-loop-no-iteration.cwl b/tests/loop-ext/all-output-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/all-output-loop.cwl b/tests/loop-ext/all-output-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/default-value-loop.cwl b/tests/loop-ext/default-value-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-command-line-tool.cwl b/tests/loop-ext/invalid-loop-command-line-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-expression-tool.cwl b/tests/loop-ext/invalid-loop-expression-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-hint.cwl b/tests/loop-ext/invalid-loop-hint.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-scatter.cwl b/tests/loop-ext/invalid-loop-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-when-exception.cwl b/tests/loop-ext/invalid-loop-when-exception.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-when.cwl b/tests/loop-ext/invalid-loop-when.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-workflow.cwl b/tests/loop-ext/invalid-loop-workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-multi-source-loop-no-requirement.cwl b/tests/loop-ext/invalid-multi-source-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-no-loopWhen.cwl b/tests/loop-ext/invalid-no-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-non-boolean-loopWhen.cwl b/tests/loop-ext/invalid-non-boolean-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-non-boolean-loopWhen2.cwl b/tests/loop-ext/invalid-non-boolean-loopWhen2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-value-from-loop-no-requirement.cwl b/tests/loop-ext/invalid-value-from-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/loop-inside-loop-all.cwl b/tests/loop-ext/loop-inside-loop-all.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/loop-inside-loop.cwl b/tests/loop-ext/loop-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/loop-inside-scatter.cwl b/tests/loop-ext/loop-inside-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/multi-source-loop.cwl b/tests/loop-ext/multi-source-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/opt-var-loop.cwl b/tests/loop-ext/opt-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/scatter-inside-loop.cwl b/tests/loop-ext/scatter-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/single-var-loop-no-iteration.cwl b/tests/loop-ext/single-var-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/single-var-loop.cwl b/tests/loop-ext/single-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/two-vars-loop-2.cwl b/tests/loop-ext/two-vars-loop-2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/two-vars-loop.cwl b/tests/loop-ext/two-vars-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/value-from-loop.cwl b/tests/loop-ext/value-from-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/all-output-loop-no-iteration.cwl b/tests/loop/all-output-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/all-output-loop.cwl b/tests/loop/all-output-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/default-value-loop.cwl b/tests/loop/default-value-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-loop-scatter.cwl b/tests/loop/invalid-loop-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-loop-when-exception.cwl b/tests/loop/invalid-loop-when-exception.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-loop-when-exception2.cwl b/tests/loop/invalid-loop-when-exception2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-multi-source-loop-no-requirement.cwl b/tests/loop/invalid-multi-source-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-no-loopWhen.cwl b/tests/loop/invalid-no-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-non-boolean-loopWhen.cwl b/tests/loop/invalid-non-boolean-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-non-boolean-loopWhen2.cwl b/tests/loop/invalid-non-boolean-loopWhen2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-value-from-loop-no-requirement.cwl b/tests/loop/invalid-value-from-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/loop-inside-loop-all.cwl b/tests/loop/loop-inside-loop-all.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/loop-inside-loop.cwl b/tests/loop/loop-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/loop-inside-scatter.cwl b/tests/loop/loop-inside-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/multi-source-loop.cwl b/tests/loop/multi-source-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/opt-var-loop.cwl b/tests/loop/opt-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/scatter-inside-loop.cwl b/tests/loop/scatter-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/single-var-loop-no-iteration.cwl b/tests/loop/single-var-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/single-var-loop.cwl b/tests/loop/single-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/two-vars-loop-2.cwl b/tests/loop/two-vars-loop-2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/two-vars-loop.cwl b/tests/loop/two-vars-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/value-from-loop.cwl b/tests/loop/value-from-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/nested-array.cwl b/tests/nested-array.cwl old mode 100644 new mode 100755 index 8272614fc..6aa3650b8 --- a/tests/nested-array.cwl +++ b/tests/nested-array.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool baseCommand: echo diff --git a/tests/non_portable.cwl b/tests/non_portable.cwl old mode 100644 new mode 100755 diff --git a/tests/non_portable2.cwl b/tests/non_portable2.cwl old mode 100644 new mode 100755 diff --git a/tests/override/env-tool.cwl b/tests/override/env-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/override/env-tool_v1.1.0-dev1.cwl b/tests/override/env-tool_v1.1.0-dev1.cwl old mode 100644 new mode 100755 diff --git a/tests/override/env-tool_v1.1.cwl b/tests/override/env-tool_v1.1.cwl old mode 100644 new mode 100755 diff --git a/tests/portable.cwl b/tests/portable.cwl old mode 100644 new mode 100755 diff --git a/tests/reloc/test.cwl b/tests/reloc/test.cwl old mode 100644 new mode 100755 index 41d37474e..c88cf48da --- a/tests/reloc/test.cwl +++ b/tests/reloc/test.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool inputs: diff --git a/tests/scatter_numbers.cwl b/tests/scatter_numbers.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-bad.cwl b/tests/secondary-files-bad.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-required-container.cwl b/tests/secondary-files-required-container.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-required-missing-container.cwl b/tests/secondary-files-required-missing-container.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-string-v1.cwl b/tests/secondary-files-string-v1.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files.cwl b/tests/secondary-files.cwl old mode 100644 new mode 100755 diff --git a/tests/sing_pullfolder_test.cwl b/tests/sing_pullfolder_test.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/1432.cwl b/tests/subgraph/1432.cwl old mode 100644 new mode 100755 index 54d553875..41283abfe --- a/tests/subgraph/1432.cwl +++ b/tests/subgraph/1432.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Workflow cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-tool2.cwl b/tests/subgraph/env-tool2.cwl old mode 100644 new mode 100755 index f568e5450..14c8a6826 --- a/tests/subgraph/env-tool2.cwl +++ b/tests/subgraph/env-tool2.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: CommandLineTool cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-tool2_no_env.cwl b/tests/subgraph/env-tool2_no_env.cwl old mode 100644 new mode 100755 index c4a6f6327..2b3711b77 --- a/tests/subgraph/env-tool2_no_env.cwl +++ b/tests/subgraph/env-tool2_no_env.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: CommandLineTool cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-tool2_req.cwl b/tests/subgraph/env-tool2_req.cwl old mode 100644 new mode 100755 index 96b0bc3d1..943108056 --- a/tests/subgraph/env-tool2_req.cwl +++ b/tests/subgraph/env-tool2_req.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: CommandLineTool cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-wf2.cwl b/tests/subgraph/env-wf2.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_hint_collision.cwl b/tests/subgraph/env-wf2_hint_collision.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_hint_req_collision.cwl b/tests/subgraph/env-wf2_hint_req_collision.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_long.cwl b/tests/subgraph/env-wf2_long.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_only_hint.cwl b/tests/subgraph/env-wf2_only_hint.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_req_collision.cwl b/tests/subgraph/env-wf2_req_collision.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_subwf-packed.cwl b/tests/subgraph/env-wf2_subwf-packed.cwl old mode 100644 new mode 100755 index ed119f0f2..e1c09649d --- a/tests/subgraph/env-wf2_subwf-packed.cwl +++ b/tests/subgraph/env-wf2_subwf-packed.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "$graph": [ { @@ -127,4 +128,4 @@ } ], "cwlVersion": "v1.2" -} \ No newline at end of file +} diff --git a/tests/subgraph/env-wf2_subwf.cwl b/tests/subgraph/env-wf2_subwf.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_subwf_b.cwl b/tests/subgraph/env-wf2_subwf_b.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/steplevel-resreq.cwl b/tests/subgraph/steplevel-resreq.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/timelimit2-wf.cwl b/tests/subgraph/timelimit2-wf.cwl old mode 100644 new mode 100755 diff --git a/tests/test_caching.py b/tests/test_caching.py new file mode 100644 index 000000000..db3bf0246 --- /dev/null +++ b/tests/test_caching.py @@ -0,0 +1,172 @@ +import re +from pathlib import Path + +import pytest + +from .util import get_data, get_main_output, needs_docker + +test_factors = [(""), ("--parallel"), ("--debug"), ("--parallel --debug")] + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_wf_without_container(tmp_path: Path, factor: str) -> None: + """Confirm that we can run a workflow without a container.""" + test_file = "hello-workflow.cwl" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--cachedir", + cache_dir, + "--outdir", + str(tmp_path / "outdir"), + get_data("tests/wf/" + test_file), + "--usermessage", + "hello", + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_issue_740_fixed(tmp_path: Path, factor: str) -> None: + """Confirm that re-running a particular workflow with caching succeeds.""" + test_file = "cache_test_workflow.cwl" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + commands = factor.split() + commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "Output of job will be cached in" not in stderr + assert error_code == 0, stderr + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_cache_relative_paths(tmp_path: Path, factor: str) -> None: + """Confirm that re-running a particular workflow with caching succeeds.""" + test_file = "secondary-files.cwl" + test_job_file = "secondary-files-job.yml" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--out", + str(tmp_path / "out"), + "--cachedir", + cache_dir, + get_data(f"tests/{test_file}"), + get_data(f"tests/{test_job_file}"), + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + commands = factor.split() + commands.extend( + [ + "--out", + str(tmp_path / "out2"), + "--cachedir", + cache_dir, + get_data(f"tests/{test_file}"), + get_data(f"tests/{test_job_file}"), + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "Output of job will be cached in" not in stderr + assert error_code == 0, stderr + + assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists() + + +@pytest.mark.parametrize("factor", test_factors) +def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None: + """Confirm that running a CLT with a default literal file with caching succeeds.""" + test_file = "tests/wf/extract_region_specs.cwl" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--out", + str(tmp_path / "out"), + "--cachedir", + cache_dir, + get_data(test_file), + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_cache_dockerreq_hint_instead_of_req(tmp_path: Path, factor: str) -> None: + """The cache must not be checked when there is an invalid use of an absolute path in iwdr.listing.""" + cache_dir = str(tmp_path / "cwltool_cache") + test_job_file = "tests/wf/loadContents-input.yml" + # First, run the iwd-container-entryname1 conformance tests with caching turned on + test1_file = "tests/wf/iwd-container-entryname1.cwl" + commands1 = factor.split() + commands1.extend( + [ + "--out", + str(tmp_path / "out1"), + "--cachedir", + cache_dir, + get_data(test1_file), + get_data(test_job_file), + ] + ) + error_code1, _, stderr1 = get_main_output(commands1) + + stderr1 = re.sub(r"\s\s+", " ", stderr1) + assert "completed success" in stderr1 + assert error_code1 == 0 + # Second, run the iwd-container-entryname3 test, which should fail + # even though it would be a cache hit, except that its DockerRequirement is + # in `hints` instead of `requirements` and one of the initial working directory + # items has an absolute path starting with `/`. + test2_file = "tests/wf/iwd-container-entryname3.cwl" + commands2 = factor.split() + commands2.extend( + [ + "--out", + str(tmp_path / "out2"), + "--cachedir", + cache_dir, + get_data(test2_file), + get_data(test_job_file), + ] + ) + error_code2, _, stderr2 = get_main_output(commands2) + + stderr2 = re.sub(r"\s\s+", " ", stderr2) + assert ( + "at index 0 of listing is invalid, name can only start with '/' " + "when DockerRequirement is in 'requirements" in stderr2 + ) + assert error_code2 == 1 diff --git a/tests/test_context.py b/tests/test_context.py index acb7dfbca..505dfd635 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,10 +1,17 @@ +import logging import subprocess import sys +from collections.abc import MutableMapping +from io import StringIO +from pathlib import Path +from typing import cast from cwltool.context import RuntimeContext from cwltool.factory import Factory +from cwltool.utils import CWLObjectType +from cwltool.workflow_job import WorkflowJobStep -from .util import get_data +from .util import get_data, needs_docker def test_replace_default_stdout_stderr() -> None: @@ -26,3 +33,59 @@ def test_replace_default_stdout_stderr() -> None: assert echo(inp="foo") == {"out": "foo\n"} sys.stdout = original_stdout sys.stderr = original_stderr + + +@needs_docker +def test_workflow_job_step_name_callback() -> None: + """Test ability to hook custom workflow step naming""" + + stream = StringIO() + streamhandler = logging.StreamHandler(stream) + _logger = logging.getLogger("cwltool") + _logger.addHandler(streamhandler) + + try: + runtime_context = RuntimeContext() + + def step_name_hook(step: WorkflowJobStep, job: CWLObjectType) -> str: + j1 = cast(MutableMapping[str, CWLObjectType], job) + inp = cast(MutableMapping[str, str], j1.get("revtool_input", j1.get("sorted_input"))) + return "{} on {}".format( + step.name, + inp.get("basename"), + ) + + runtime_context.workflow_job_step_name_callback = step_name_hook + + factory = Factory(None, None, runtime_context) + revsort = factory.make(get_data("tests/wf/revsort.cwl")) + + result = revsort( + workflow_input={ + "class": "File", + "location": Path(get_data("tests/wf/whale.txt")).as_uri(), + "format": "https://www.iana.org/assignments/media-types/text/plain", + } + ) + + result = cast(CWLObjectType, result) + + sorted_out = cast(MutableMapping[str, str], result["sorted_output"]) + loc = sorted_out["location"] + + assert result == { + "sorted_output": { + "basename": "output.txt", + "checksum": "sha1$b9214658cc453331b62c2282b772a5c063dbd284", + "class": "File", + "http://commonwl.org/cwltool#generation": 0, + "nameext": ".txt", + "nameroot": "output", + "size": 1111, + "location": loc, + }, + } + + assert "rev on whale.txt" in stream.getvalue() + finally: + _logger.removeHandler(streamhandler) diff --git a/tests/test_cuda.py b/tests/test_cuda.py index 27dfae39d..93b1c7a47 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -287,7 +287,7 @@ def test_cuda_eval_resource_range() -> None: with open(get_data("extensions-v1.1.yml")) as res: use_custom_schema("v1.2", "http://commonwl.org/cwltool", res.read()) - joborder = {} # type: CWLObjectType + joborder: CWLObjectType = {} loadingContext = LoadingContext({"do_update": True}) runtime_context = RuntimeContext({}) @@ -304,7 +304,7 @@ def test_cuda_eval_resource_max() -> None: with open(get_data("extensions-v1.1.yml")) as res: use_custom_schema("v1.2", "http://commonwl.org/cwltool", res.read()) - joborder = {} # type: CWLObjectType + joborder: CWLObjectType = {} loadingContext = LoadingContext({"do_update": True}) runtime_context = RuntimeContext({}) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index ae18a41ae..ac932ea94 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -6,7 +6,6 @@ from pathlib import Path from shutil import which from types import ModuleType -from typing import Optional, Tuple import pytest @@ -16,7 +15,7 @@ from .util import get_data, get_main_output, get_tool_env, needs_docker -deps: Optional[ModuleType] = None +deps: ModuleType | None = None try: from galaxy.tool_util import deps except ImportError: @@ -25,6 +24,7 @@ @needs_docker @pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") +@pytest.mark.skip(reason="too flaky") @pytest.mark.flaky(retries=3) def test_biocontainers(tmp_path: Path) -> None: wflow = get_data("tests/seqtk_seq.cwl") @@ -46,6 +46,7 @@ def test_biocontainers(tmp_path: Path) -> None: @needs_docker @pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") +@pytest.mark.skip(reason="too flaky") def test_biocontainers_resolution(tmp_path: Path) -> None: """Confirm expected container name for --beta-use-biocontainers.""" tool = load_tool(get_data("tests/seqtk_seq.cwl"), LoadingContext()) @@ -56,7 +57,7 @@ def test_biocontainers_resolution(tmp_path: Path) -> None: @pytest.fixture(scope="session") -def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]: +def bioconda_setup(request: pytest.FixtureRequest) -> tuple[int | None, str]: """ Caches the conda environment created for seqtk_seq.cwl. @@ -108,7 +109,7 @@ def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]: @pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") -def test_bioconda(bioconda_setup: Tuple[Optional[int], str]) -> None: +def test_bioconda(bioconda_setup: tuple[int | None, str]) -> None: error_code, stderr = bioconda_setup assert error_code == 0, stderr @@ -119,15 +120,15 @@ def test_modules(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Do a basic smoke test using environment modules to satisfy a SoftwareRequirement.""" wflow = get_data("tests/random_lines.cwl") job = get_data("tests/random_lines_job.json") - monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles")) + monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles")) error_code, _, stderr = get_main_output( [ "--outdir", str(tmp_path / "out"), - "--beta-dependency-resolvers-configuration", "--beta-dependencies-directory", str(tmp_path / "deps"), - "tests/test_deps_env_modules_resolvers_conf.yml", + "--beta-dependency-resolvers-configuration", + get_data("tests/test_deps_env_modules_resolvers_conf.yml"), "--debug", wflow, job, @@ -145,7 +146,7 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Do so by by running `env` as the tool and parsing its output. """ - monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles")) + monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles")) tool_env = get_tool_env( tmp_path, [ @@ -155,6 +156,6 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> get_data("tests/env_with_software_req.yml"), ) - assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space " + assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space ", tool_env tool_path = tool_env["PATH"].split(":") assert get_data("tests/test_deps_env/random-lines/1.0/scripts") in tool_path diff --git a/tests/test_environment.py b/tests/test_environment.py index ba87041b3..b9fa24579 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -2,10 +2,12 @@ import os from abc import ABC, abstractmethod +from collections.abc import Callable, Mapping from pathlib import Path -from typing import Any, Callable, Dict, List, Mapping, Union +from typing import Union import pytest +from packaging.version import Version from cwltool.singularity import get_version @@ -17,7 +19,7 @@ # TODO: maybe add regex? Env = Mapping[str, str] CheckerTypes = Union[None, str, Callable[[str, Env], bool]] -EnvChecks = Dict[str, CheckerTypes] +EnvChecks = dict[str, CheckerTypes] def assert_envvar_matches(check: CheckerTypes, k: str, env: Mapping[str, str]) -> None: @@ -66,7 +68,7 @@ def checks(tmp_prefix: str) -> EnvChecks: """Return a mapping from environment variable names to how to check for correctness.""" # Any flags to pass to cwltool to force use of the correct container - flags: List[str] + flags: list[str] # Does the env tool (maybe in our container) accept a `-0` flag? env_accepts_null: bool @@ -132,32 +134,33 @@ def PWD(v: str, env: Env) -> bool: } # Singularity variables appear to be in flux somewhat. - version = get_version()[0] - vmajor = version[0] - assert vmajor == 3, "Tests only work for Singularity 3" - vminor = version[1] + version = Version(".".join(map(str, get_version()[0]))) + assert version >= Version("3"), "Tests only work for Singularity 3+" sing_vars: EnvChecks = { "SINGULARITY_CONTAINER": None, "SINGULARITY_NAME": None, } - if vminor < 5: + if version < Version("3.5"): sing_vars["SINGULARITY_APPNAME"] = None - if vminor >= 5: + if (version >= Version("3.5")) and (version < Version("3.6")): + sing_vars["SINGULARITY_INIT"] = "1" + if version >= Version("3.5"): sing_vars["PROMPT_COMMAND"] = None sing_vars["SINGULARITY_ENVIRONMENT"] = None - if vminor == 5: - sing_vars["SINGULARITY_INIT"] = "1" - elif vminor > 5: + if version >= Version("3.6"): sing_vars["SINGULARITY_COMMAND"] = "exec" - if vminor >= 7: - if vminor > 9: - sing_vars["SINGULARITY_BIND"] = "" - else: + if version >= Version("3.7"): + if version > Version("3.9"): + sing_vars["SINGULARITY_BIND"] = "" + else: - def BIND(v: str, env: Env) -> bool: - return v.startswith(tmp_prefix) and v.endswith(":/tmp") + def BIND(v: str, env: Env) -> bool: + return v.startswith(tmp_prefix) and v.endswith(":/tmp") - sing_vars["SINGULARITY_BIND"] = BIND + sing_vars["SINGULARITY_BIND"] = BIND + if version >= Version("3.10"): + sing_vars["SINGULARITY_COMMAND"] = "run" + sing_vars["SINGULARITY_NO_EVAL"] = None result.update(sing_vars) @@ -194,14 +197,14 @@ def BIND(v: str, env: Env) -> bool: @CRT_PARAMS -def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test that basic env vars (only) show up.""" tmp_prefix = str(tmp_path / "canary") extra_env = { "USEDVAR": "VARVAL", "UNUSEDVAR": "VARVAL", } - args = crt_params.flags + [f"--tmpdir-prefix={tmp_prefix}"] + args = crt_params.flags + [f"--tmpdir-prefix={tmp_prefix}", "--debug"] env = get_tool_env( tmp_path, args, @@ -214,7 +217,9 @@ def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> Non @CRT_PARAMS -def test_preserve_single(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_preserve_single( + crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """Test that preserving a single env var works.""" tmp_prefix = str(tmp_path / "canary") extra_env = { @@ -238,7 +243,36 @@ def test_preserve_single(crt_params: CheckHolder, tmp_path: Path, monkeypatch: A @CRT_PARAMS -def test_preserve_all(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_preserve_single_missing( + crt_params: CheckHolder, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test that attempting to preserve an unset env var produces a warning.""" + tmp_prefix = str(tmp_path / "canary") + args = crt_params.flags + [ + f"--tmpdir-prefix={tmp_prefix}", + "--preserve-environment=RANDOMVAR", + ] + env = get_tool_env( + tmp_path, + args, + monkeypatch=monkeypatch, + runtime_env_accepts_null=crt_params.env_accepts_null, + ) + checks = crt_params.checks(tmp_prefix) + assert "RANDOMVAR" not in checks + assert ( + "Attempting to preserve environment variable " "'RANDOMVAR' which is not present" + ) in caplog.text + assert_env_matches(checks, env) + + +@CRT_PARAMS +def test_preserve_all( + crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """Test that preserving all works.""" tmp_prefix = str(tmp_path / "canary") extra_env = { diff --git a/tests/test_examples.py b/tests/test_examples.py index 4d479e313..f371f45ae 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -9,7 +9,7 @@ import urllib.parse from io import StringIO from pathlib import Path -from typing import Any, Dict, List, Union, cast +from typing import Any, cast import cwl_utils.expression as expr import pydot @@ -69,7 +69,7 @@ def test_expression_match(expression: str, expected: bool) -> None: assert (match is not None) == expected -interpolate_input = { +interpolate_input: dict[str, Any] = { "foo": { "bar": {"baz": "zab1"}, "b ar": {"baz": 2}, @@ -77,7 +77,7 @@ def test_expression_match(expression: str, expected: bool) -> None: 'b"ar': {"baz": None}, }, "lst": ["A", "B"], -} # type: Dict[str, Any] +} interpolate_parameters = [ ("$(foo)", interpolate_input["foo"]), @@ -403,14 +403,14 @@ def test_scandeps() -> None: } def loadref( - base: str, p: Union[CommentedMap, CommentedSeq, str, None] - ) -> Union[CommentedMap, CommentedSeq, str, None]: + base: str, p: CommentedMap | CommentedSeq | str | None + ) -> CommentedMap | CommentedSeq | str | None: if isinstance(p, dict): return p raise Exception("test case can't load things") scanned_deps = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( cast(str, obj["id"]), obj, @@ -473,7 +473,7 @@ def loadref( assert scanned_deps == expected_deps scanned_deps2 = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( cast(str, obj["id"]), obj, @@ -508,14 +508,14 @@ def test_scandeps_samedirname() -> None: } def loadref( - base: str, p: Union[CommentedMap, CommentedSeq, str, None] - ) -> Union[CommentedMap, CommentedSeq, str, None]: + base: str, p: CommentedMap | CommentedSeq | str | None + ) -> CommentedMap | CommentedSeq | str | None: if isinstance(p, dict): return p raise Exception("test case can't load things") scanned_deps = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( "", obj, @@ -576,7 +576,7 @@ def test_scandeps_defaults_with_secondaryfiles() -> None: def test_dedupe() -> None: - not_deduped = [ + not_deduped: list[CWLObjectType] = [ {"class": "File", "location": "file:///example/a"}, {"class": "File", "location": "file:///example/a"}, {"class": "File", "location": "file:///example/d"}, @@ -585,7 +585,7 @@ def test_dedupe() -> None: "location": "file:///example/c", "listing": [{"class": "File", "location": "file:///example/d"}], }, - ] # type: List[CWLObjectType] + ] expected = [ {"class": "File", "location": "file:///example/a"}, @@ -649,7 +649,7 @@ def test_dedupe() -> None: @pytest.mark.parametrize("name, source, sink, expected", source_to_sink) def test_compare_types( - name: str, source: Dict[str, Any], sink: Dict[str, Any], expected: bool + name: str, source: dict[str, Any], sink: dict[str, Any], expected: bool ) -> None: assert can_assign_src_to_sink(source, sink) == expected, name @@ -675,7 +675,7 @@ def test_compare_types( @pytest.mark.parametrize("name, source, sink, expected", source_to_sink_strict) def test_compare_types_strict( - name: str, source: Dict[str, Any], sink: Dict[str, Any], expected: bool + name: str, source: dict[str, Any], sink: dict[str, Any], expected: bool ) -> None: assert can_assign_src_to_sink(source, sink, strict=True) == expected, name @@ -995,8 +995,10 @@ def test_var_spool_cwl_checker3() -> None: def test_print_dot() -> None: # print Workflow cwl_path = get_data("tests/wf/three_step_color.cwl") - expected_dot = pydot.graph_from_dot_data( - """ + expected_dot = cast( + list[pydot.core.Dot], + pydot.graph_from_dot_data( + """ digraph {{ graph [bgcolor="#eeeeee", clusterrank=local, @@ -1041,10 +1043,11 @@ def test_print_dot() -> None: "command_line_tool" -> "file_output"; }} """.format() + ), )[0] stdout = StringIO() assert main(["--debug", "--print-dot", cwl_path], stdout=stdout) == 0 - computed_dot = pydot.graph_from_dot_data(stdout.getvalue())[0] + computed_dot = cast(list[pydot.core.Dot], pydot.graph_from_dot_data(stdout.getvalue()))[0] computed_edges = sorted((source, target) for source, target in computed_dot.obj_dict["edges"]) expected_edges = sorted((source, target) for source, target in expected_dot.obj_dict["edges"]) assert computed_edges == expected_edges @@ -1118,7 +1121,7 @@ def test_cid_file_dir_arg_is_file_instead_of_dir(tmp_path: Path, factor: str) -> @needs_docker @pytest.mark.parametrize("factor", test_factors) def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None: - """Test that --cachedir with a bad path should produce a specific error.""" + """Test that --cidefile-dir with a bad path should produce a specific error.""" test_file = "cache_test_workflow.cwl" bad_cidfile_dir = tmp_path / "cidfile-dir-badpath" commands = factor.split() @@ -1233,11 +1236,11 @@ def test_secondary_files_v1_0(tmp_path: Path, factor: str) -> None: assert error_code == 0 -@needs_docker @pytest.mark.parametrize("factor", test_factors) -def test_wf_without_container(tmp_path: Path, factor: str) -> None: - """Confirm that we can run a workflow without a container.""" - test_file = "hello-workflow.cwl" +def test_cache_environment_variable(tmp_path: Path, factor: str) -> None: + """Ensure that changing the environment variables will result in different cache keys""" + test_file = "cache_environment_tool.cwl" + test_job_file = "cache_environment.yml" cache_dir = str(tmp_path / "cwltool_cache") commands = factor.split() commands.extend( @@ -1246,106 +1249,39 @@ def test_wf_without_container(tmp_path: Path, factor: str) -> None: cache_dir, "--outdir", str(tmp_path / "outdir"), - get_data("tests/wf/" + test_file), - "--usermessage", - "hello", + get_data("tests/" + test_file), + get_data(f"tests/{test_job_file}"), ] ) error_code, _, stderr = get_main_output(commands) stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 - - -@needs_docker -@pytest.mark.parametrize("factor", test_factors) -def test_issue_740_fixed(tmp_path: Path, factor: str) -> None: - """Confirm that re-running a particular workflow with caching succeeds.""" - test_file = "cache_test_workflow.cwl" - cache_dir = str(tmp_path / "cwltool_cache") - commands = factor.split() - commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 + assert "Output of job will be cached in" in stderr + assert error_code == 0, stderr - commands = factor.split() - commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) assert "Output of job will be cached in" not in stderr assert error_code == 0, stderr - -@needs_docker -@pytest.mark.parametrize("factor", test_factors) -def test_cache_relative_paths(tmp_path: Path, factor: str) -> None: - """Confirm that re-running a particular workflow with caching succeeds.""" - test_file = "secondary-files.cwl" - test_job_file = "secondary-files-job.yml" - cache_dir = str(tmp_path / "cwltool_cache") commands = factor.split() + test_job_file = "cache_environment2.yml" commands.extend( [ - "--out", - str(tmp_path / "out"), "--cachedir", cache_dir, - get_data(f"tests/{test_file}"), + "--outdir", + str(tmp_path / "outdir"), + get_data("tests/" + test_file), get_data(f"tests/{test_job_file}"), ] ) - error_code, _, stderr = get_main_output(commands) - stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 - - commands = factor.split() - commands.extend( - [ - "--out", - str(tmp_path / "out2"), - "--cachedir", - cache_dir, - get_data(f"tests/{test_file}"), - get_data(f"tests/{test_job_file}"), - ] - ) error_code, _, stderr = get_main_output(commands) stderr = re.sub(r"\s\s+", " ", stderr) - assert "Output of job will be cached in" not in stderr + assert "Output of job will be cached in" in stderr assert error_code == 0, stderr - assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists() - - -@pytest.mark.parametrize("factor", test_factors) -def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None: - """Confirm that running a CLT with a default literal file with caching succeeds.""" - test_file = "tests/wf/extract_region_specs.cwl" - cache_dir = str(tmp_path / "cwltool_cache") - commands = factor.split() - commands.extend( - [ - "--out", - str(tmp_path / "out"), - "--cachedir", - cache_dir, - get_data(test_file), - ] - ) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 - def test_write_summary(tmp_path: Path) -> None: """Test --write-summary.""" @@ -1682,7 +1618,7 @@ def test_arguments_self() -> None: else: factory.runtime_context.use_container = False check = factory.make(get_data("tests/wf/paramref_arguments_self.cwl")) - outputs = cast(Dict[str, Any], check()) + outputs = cast(dict[str, Any], check()) assert "self_review" in outputs assert len(outputs) == 1 assert outputs["self_review"]["checksum"] == "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1" @@ -1820,9 +1756,9 @@ def test_validate_optional_src_with_mandatory_sink() -> None: ["--validate", get_data("tests/wf/optional_src_mandatory_sink.cwl")] ) assert exit_code == 0 - stderr = re.sub(r"\s\s+", " ", stderr) - assert 'Source \'opt_file\' of type ["null", "File"] may be incompatible' in stderr - assert "with sink 'r' of type \"File\"" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert 'Source \'opt_file\' of type ["null", "File"] may be incompatible' in stdout + assert "with sink 'r' of type \"File\"" in stdout def test_res_req_expr_float_1_0() -> None: @@ -1875,12 +1811,11 @@ def test_invalid_nested_array() -> None: ] ) assert exit_code == 1, stderr - stderr = re.sub(r"\n\s+", " ", stderr) - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Tool definition failed validation:" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert "Tool definition failed validation:" in stdout assert ( - "tests/nested-array.cwl:6:5: Field 'type' references unknown identifier 'string[][]'" - ) in stderr + "tests/nested-array.cwl:7:5: Field 'type' references unknown identifier 'string[][]'" + ) in stdout def test_input_named_id() -> None: @@ -1894,3 +1829,15 @@ def test_input_named_id() -> None: ] ) assert exit_code == 0, stderr + + +def test_make_template() -> None: + """End-to-end test of --make-template, especially for mypyc mode.""" + exit_code, stdout, stderr = get_main_output( + [ + "--make-template", + "--debug", + get_data("tests/CometAdapter.cwl"), + ] + ) + assert exit_code == 0, stderr diff --git a/tests/test_fetch.py b/tests/test_fetch.py index e55491d90..55c4695fe 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Any, List, Optional +from typing import Any from urllib.parse import urljoin, urlsplit import pytest @@ -21,11 +21,11 @@ class CWLTestFetcher(Fetcher): def __init__( self, cache: CacheType, - session: Optional[requests.sessions.Session], + session: requests.sessions.Session | None, ) -> None: """Create a Fetcher that provides a fixed result for testing purposes.""" - def fetch_text(self, url: str, content_types: Optional[List[str]] = None) -> str: + def fetch_text(self, url: str, content_types: list[str] | None = None) -> str: if url == "baz:bar/foo.cwl": return """ cwlVersion: v1.0 @@ -36,7 +36,7 @@ def fetch_text(self, url: str, content_types: Optional[List[str]] = None) -> str """ raise RuntimeError("Not foo.cwl, was %s" % url) - def check_exists(self, url): # type: (str) -> bool + def check_exists(self, url: str) -> bool: return url == "baz:bar/foo.cwl" def urljoin(self, base: str, url: str) -> str: diff --git a/tests/test_http_input.py b/tests/test_http_input.py index 6b4d9b479..e80260ff9 100644 --- a/tests/test_http_input.py +++ b/tests/test_http_input.py @@ -1,10 +1,7 @@ import os -import sys from datetime import datetime from pathlib import Path -from typing import List -import pytest from pytest_httpserver import HTTPServer from cwltool.pathmapper import PathMapper @@ -15,7 +12,7 @@ def test_http_path_mapping(tmp_path: Path) -> None: input_file_path = ( "https://raw.githubusercontent.com/common-workflow-language/cwltool/main/tests/2.fasta" ) - base_file: List[CWLObjectType] = [ + base_file: list[CWLObjectType] = [ { "class": "File", "location": "https://raw.githubusercontent.com/common-workflow-language/" @@ -34,7 +31,6 @@ def test_http_path_mapping(tmp_path: Path) -> None: assert ">Sequence 561 BP; 135 A; 106 C; 98 G; 222 T; 0 other;" in contents -@pytest.mark.skipif(sys.version_info < (3, 7), reason="timesout on CI") def test_modification_date(tmp_path: Path) -> None: """Local copies of remote files should preserve last modification date.""" # Initialize the server @@ -58,7 +54,7 @@ def test_modification_date(tmp_path: Path) -> None: ) location = httpserver.url_for(f"/{remote_file_name}") - base_file: List[CWLObjectType] = [ + base_file: list[CWLObjectType] = [ { "class": "File", "location": location, diff --git a/tests/test_iwdr.py b/tests/test_iwdr.py index 8d4138cb7..52da1fb5b 100644 --- a/tests/test_iwdr.py +++ b/tests/test_iwdr.py @@ -376,3 +376,16 @@ def test_iwdr_permutations_singularity_inplace( ) == 0 ) + + +def test_iwdr_writable_secondaryfiles(tmp_path: Path) -> None: + """Confirm that a writable primary file and its secondary files are still co-located.""" + err_code, _, _ = get_main_output( + [ + "--outdir", + str(tmp_path), + get_data("tests/iwdr_writable_secfile.cwl"), + get_data("tests/iwdr_writable_secfile-inputs.json"), + ] + ) + assert err_code == 0 diff --git a/tests/test_js_sandbox.py b/tests/test_js_sandbox.py index f4839e8a0..2c5df6339 100644 --- a/tests/test_js_sandbox.py +++ b/tests/test_js_sandbox.py @@ -5,7 +5,7 @@ import shutil import threading from pathlib import Path -from typing import Any, List +from typing import Any import pytest from cwl_utils import sandboxjs @@ -48,8 +48,8 @@ def test_value_from_two_concatenated_expressions() -> None: def hide_nodejs(temp_dir: Path) -> str: """Generate a new PATH that hides node{js,}.""" - paths: List[str] = os.environ.get("PATH", "").split(":") - names: List[str] = [] + paths: list[str] = os.environ.get("PATH", "").split(":") + names: list[str] = [] for name in ("nodejs", "node"): path = shutil.which(name) if path: diff --git a/tests/test_loop.py b/tests/test_loop.py index bf908196d..e8a043611 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -1,8 +1,8 @@ """Test the 1.3 loop feature.""" import json +from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import MutableMapping, MutableSequence from cwltool.main import main diff --git a/tests/test_loop_ext.py b/tests/test_loop_ext.py index 499dd17b4..1769d64ad 100644 --- a/tests/test_loop_ext.py +++ b/tests/test_loop_ext.py @@ -1,8 +1,8 @@ """Test the prototype cwltool:Loop extension.""" import json +from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import MutableMapping, MutableSequence from cwltool.main import main diff --git a/tests/test_misc_cli.py b/tests/test_misc_cli.py index 307153e16..be314cad4 100644 --- a/tests/test_misc_cli.py +++ b/tests/test_misc_cli.py @@ -1,5 +1,7 @@ """Tests for various command line options.""" +import pytest + from cwltool.utils import versionstring from .util import get_data, get_main_output, needs_docker @@ -26,9 +28,13 @@ def test_empty_cmdling() -> None: assert "CWL document required, no input file was provided" in stderr -def test_tool_help() -> None: +def test_tool_help(monkeypatch: pytest.MonkeyPatch) -> None: """Test --tool-help.""" - return_code, stdout, stderr = get_main_output(["--tool-help", get_data("tests/echo.cwl")]) + return_code, stdout, stderr = get_main_output( + ["--tool-help", get_data("tests/echo.cwl")], + extra_env={"NO_COLOR": "1"}, + monkeypatch=monkeypatch, + ) assert return_code == 0 assert "job_order Job input json file" in stdout diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 643907a39..9683ffa22 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -3,23 +3,25 @@ import json import os.path import sys +from collections.abc import Generator, MutableMapping +from importlib.resources import files from io import StringIO from pathlib import Path -from typing import Any, Generator, List, MutableMapping, Optional, Tuple +from typing import Any, cast import pytest from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import Names +from schema_salad.ref_resolver import file_uri from schema_salad.utils import yaml_no_ts import cwltool.load_tool import cwltool.singularity import cwltool.udocker from cwltool.command_line_tool import CommandLineTool -from cwltool.context import LoadingContext, RuntimeContext +from cwltool.context import RuntimeContext from cwltool.main import main from cwltool.mpi import MpiConfig, MPIRequirementName -from cwltool.utils import files from .util import get_data, working_directory @@ -75,12 +77,12 @@ def __init__(self): else: self.indata = sys.stdin.read().encode(sys.stdin.encoding) - def run_once(self, args: List[str]): + def run_once(self, args: list[str]): subprocess.run( args, input=self.indata, stdout=sys.stdout, stderr=sys.stderr ).check_returncode() - def run_many(self, n: int, args: List[str]): + def run_many(self, n: int, args: list[str]): for i in range(n): self.run_once(args) @@ -122,7 +124,7 @@ def make_processes_input(np: int, tmp_path: Path) -> Path: return input_file -def cwltool_args(fake_mpi_conf: str) -> List[str]: +def cwltool_args(fake_mpi_conf: str) -> list[str]: return ["--enable-ext", "--enable-dev", "--mpi-config-file", fake_mpi_conf] @@ -291,15 +293,22 @@ def schema_ext11() -> Generator[Names, None, None]: mpiReq = CommentedMap({"class": MPIRequirementName, "processes": 1}) containerReq = CommentedMap({"class": "DockerRequirement"}) -basetool = CommentedMap({"cwlVersion": "v1.1", "inputs": CommentedSeq(), "outputs": CommentedSeq()}) +basetool = CommentedMap( + { + "cwlVersion": "v1.1", + "class": "CommandLineTool", + "inputs": CommentedSeq(), + "outputs": CommentedSeq(), + } +) def mk_tool( schema: Names, - opts: List[str], - reqs: Optional[List[CommentedMap]] = None, - hints: Optional[List[CommentedMap]] = None, -) -> Tuple[LoadingContext, RuntimeContext, CommentedMap]: + opts: list[str], + reqs: list[CommentedMap] | None = None, + hints: list[CommentedMap] | None = None, +) -> tuple[RuntimeContext, CommandLineTool]: tool = basetool.copy() if reqs is not None: @@ -309,53 +318,57 @@ def mk_tool( args = cwltool.argparser.arg_parser().parse_args(opts) args.enable_ext = True + args.basedir = os.path.dirname(os.path.abspath(".")) rc = RuntimeContext(vars(args)) lc = cwltool.main.setup_loadingContext(None, rc, args) lc.avsc_names = schema - return lc, rc, tool + tool["id"] = file_uri(os.path.abspath("./mktool.cwl")) + assert lc.loader is not None + lc.loader.idx[tool["id"]] = tool + return rc, cast(CommandLineTool, cwltool.load_tool.load_tool(tool, lc)) def test_singularity(schema_ext11: Names) -> None: - lc, rc, tool = mk_tool(schema_ext11, ["--singularity"], reqs=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, ["--singularity"], reqs=[mpiReq, containerReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.singularity.SingularityCommandLineJob def test_udocker(schema_ext11: Names) -> None: - lc, rc, tool = mk_tool(schema_ext11, ["--udocker"], reqs=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, ["--udocker"], reqs=[mpiReq, containerReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.udocker.UDockerCommandLineJob def test_docker_hint(schema_ext11: Names) -> None: # Docker hint, MPI required - lc, rc, tool = mk_tool(schema_ext11, [], hints=[containerReq], reqs=[mpiReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], hints=[containerReq], reqs=[mpiReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.job.CommandLineJob def test_docker_required(schema_ext11: Names) -> None: # Docker required, MPI hinted - lc, rc, tool = mk_tool(schema_ext11, [], reqs=[containerReq], hints=[mpiReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], reqs=[containerReq], hints=[mpiReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.docker.DockerCommandLineJob def test_docker_mpi_both_required(schema_ext11: Names) -> None: # Both required - error - lc, rc, tool = mk_tool(schema_ext11, [], reqs=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], reqs=[mpiReq, containerReq]) with pytest.raises(cwltool.errors.UnsupportedRequirement): - clt.make_job_runner(rc) + clt._init_job({}, rc) + clt.make_job_runner(rc) def test_docker_mpi_both_hinted(schema_ext11: Names) -> None: # Both hinted - error - lc, rc, tool = mk_tool(schema_ext11, [], hints=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], hints=[mpiReq, containerReq]) with pytest.raises(cwltool.errors.UnsupportedRequirement): - clt.make_job_runner(rc) + clt._init_job({}, rc) + clt.make_job_runner(rc) diff --git a/tests/test_override.py b/tests/test_override.py index 980c853bb..93c836c84 100644 --- a/tests/test_override.py +++ b/tests/test_override.py @@ -1,6 +1,5 @@ import json from io import StringIO -from typing import Dict, List import pytest @@ -76,7 +75,7 @@ @needs_docker @pytest.mark.parametrize("parameters,result", override_parameters) -def test_overrides(parameters: List[str], result: Dict[str, str]) -> None: +def test_overrides(parameters: list[str], result: dict[str, str]) -> None: sio = StringIO() assert main(parameters, stdout=sio) == 0 @@ -119,7 +118,7 @@ def test_overrides(parameters: List[str], result: Dict[str, str]) -> None: @needs_docker @pytest.mark.parametrize("parameters,expected_error", failing_override_parameters) -def test_overrides_fails(parameters: List[str], expected_error: str) -> None: +def test_overrides_fails(parameters: list[str], expected_error: str) -> None: sio = StringIO() assert main(parameters, stderr=sio) == 1 diff --git a/tests/test_pack.py b/tests/test_pack.py index 1d38e35e8..a65996f8f 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -5,7 +5,6 @@ from functools import partial from io import StringIO from pathlib import Path -from typing import Dict import pytest from schema_salad.utils import yaml_no_ts @@ -95,7 +94,7 @@ def test_pack_fragment() -> None: def test_pack_rewrites() -> None: - rewrites: Dict[str, str] = {} + rewrites: dict[str, str] = {} loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/default-wf5.cwl")) loadingContext.do_update = False diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 01ab7fe17..096de9942 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -1,7 +1,7 @@ import urllib.parse from io import BytesIO from pathlib import Path -from typing import IO, Any, List, cast +from typing import IO, Any, cast import pytest from ruamel.yaml.comments import CommentedMap @@ -112,7 +112,7 @@ def test_unicode_in_output_files(tmp_path: Path, filename: str) -> None: class StubFsAccess(StdFsAccess): """Stub fs access object that doesn't rely on the filesystem.""" - def glob(self, pattern: str) -> List[str]: + def glob(self, pattern: str) -> list[str]: """glob.""" return [pattern] diff --git a/tests/test_pathmapper.py b/tests/test_pathmapper.py index b7cf2f6a1..4ffac24bd 100644 --- a/tests/test_pathmapper.py +++ b/tests/test_pathmapper.py @@ -1,5 +1,3 @@ -from typing import List, Tuple - import pytest from cwltool.pathmapper import PathMapper @@ -10,7 +8,7 @@ def test_subclass() -> None: class SubPathMapper(PathMapper): def __init__( self, - referenced_files: List[CWLObjectType], + referenced_files: list[CWLObjectType], basedir: str, stagedir: str, new: str, @@ -81,7 +79,7 @@ def test_normalizeFilesDirs(name: str, file_dir: CWLObjectType, expected: CWLObj @pytest.mark.parametrize("filename,expected", basename_generation_parameters) -def test_basename_field_generation(filename: str, expected: Tuple[str, str]) -> None: +def test_basename_field_generation(filename: str, expected: tuple[str, str]) -> None: nameroot, nameext = expected expected2 = { "class": "File", diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 83eb61c22..d7a2a698b 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -3,8 +3,9 @@ import pickle import sys import urllib +from collections.abc import Generator from pathlib import Path -from typing import IO, Any, Generator, cast +from typing import IO, Any, cast import arcp import bagit @@ -31,12 +32,23 @@ SCHEMA = Namespace("http://schema.org/") CWLPROV = Namespace("https://w3id.org/cwl/prov#") OA = Namespace("http://www.w3.org/ns/oa#") +FOAF = Namespace("http://xmlns.com/foaf/0.1/") -def cwltool(tmp_path: Path, *args: Any) -> Path: +TEST_ORCID = "https://orcid.org/0000-0003-4862-3349" + + +def cwltool(tmp_path: Path, *args: Any, with_orcid: bool = False) -> Path: prov_folder = tmp_path / "provenance" prov_folder.mkdir() - new_args = ["--provenance", str(prov_folder)] + new_args = [ + "--enable-user-provenance", + "--enable-host-provenance", + "--provenance", + str(prov_folder), + ] + if with_orcid: + new_args.extend(["--orcid", TEST_ORCID]) new_args.extend(args) # Run within a temporary directory to not pollute git checkout tmp_dir = tmp_path / "cwltool-run" @@ -48,61 +60,81 @@ def cwltool(tmp_path: Path, *args: Any) -> Path: @needs_docker -def test_hello_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_hello_workflow(tmp_path: Path, with_orcid: bool) -> None: check_provenance( cwltool( tmp_path, get_data("tests/wf/hello-workflow.cwl"), "--usermessage", "Hello workflow", - ) + with_orcid=with_orcid, + ), + with_orcid=with_orcid, ) @needs_docker -def test_hello_single_tool(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_hello_single_tool(tmp_path: Path, with_orcid: bool) -> None: check_provenance( cwltool( tmp_path, get_data("tests/wf/hello_single_tool.cwl"), "--message", "Hello tool", + with_orcid=with_orcid, ), single_tool=True, + with_orcid=with_orcid, ) @needs_docker -def test_revsort_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_revsort_workflow(tmp_path: Path, with_orcid: bool) -> None: folder = cwltool( tmp_path, get_data("tests/wf/revsort.cwl"), get_data("tests/wf/revsort-job.json"), + with_orcid=with_orcid, ) check_output_object(folder) - check_provenance(folder) + check_provenance(folder, with_orcid=with_orcid) @needs_docker -def test_revsort_workflow_shortcut(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_revsort_workflow_shortcut(tmp_path: Path, with_orcid: bool) -> None: """Confirm that using 'cwl:tool' shortcut still snapshots the CWL files.""" folder = cwltool( tmp_path, get_data("tests/wf/revsort-job-shortcut.json"), + with_orcid=with_orcid, ) check_output_object(folder) - check_provenance(folder) + check_provenance(folder, with_orcid=with_orcid) assert not (folder / "snapshot" / "revsort-job-shortcut.json").exists() assert len(list((folder / "snapshot").iterdir())) == 4 @needs_docker -def test_nested_workflow(tmp_path: Path) -> None: - check_provenance(cwltool(tmp_path, get_data("tests/wf/nested.cwl")), nested=True) +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_nested_workflow(tmp_path: Path, with_orcid: bool) -> None: + check_provenance( + cwltool( + tmp_path, + get_data("tests/wf/nested.cwl"), + with_orcid=with_orcid, + ), + nested=True, + with_orcid=with_orcid, + ) @needs_docker -def test_secondary_files_implicit(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_implicit(tmp_path: Path, with_orcid: bool) -> None: file1 = tmp_path / "foo1.txt" file1idx = tmp_path / "foo1.txt.idx" @@ -112,13 +144,20 @@ def test_secondary_files_implicit(tmp_path: Path) -> None: f.write("bar") # secondary will be picked up by .idx - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf.cwl"), "--file1", str(file1)) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf.cwl"), + "--file1", + str(file1), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) check_secondary_files(folder) @needs_docker -def test_secondary_files_explicit(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_explicit(tmp_path: Path, with_orcid: bool) -> None: # Deliberately do NOT have common basename or extension file1dir = tmp_path / "foo" file1dir.mkdir() @@ -153,22 +192,33 @@ def test_secondary_files_explicit(tmp_path: Path) -> None: j = json.dumps(job, ensure_ascii=True) fp.write(j.encode("ascii")) - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf.cwl"), str(jobJson)) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf.cwl"), + str(jobJson), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) check_secondary_files(folder) @needs_docker -def test_secondary_files_output(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_output(tmp_path: Path, with_orcid: bool) -> None: # secondary will be picked up by .idx - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf-out.cwl")) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf-out.cwl"), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) # Skipped, not the same secondary files as above # self.check_secondary_files() @needs_docker -def test_directory_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_directory_workflow(tmp_path: Path, with_orcid: bool) -> None: dir2 = tmp_path / "dir2" dir2.mkdir() sha1 = { @@ -184,8 +234,14 @@ def test_directory_workflow(tmp_path: Path) -> None: with open(dir2 / x, "w", encoding="ascii") as f: f.write(x) - folder = cwltool(tmp_path, get_data("tests/wf/directory.cwl"), "--dir", str(dir2)) - check_provenance(folder, directory=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/directory.cwl"), + "--dir", + str(dir2), + with_orcid=with_orcid, + ) + check_provenance(folder, directory=True, with_orcid=with_orcid) # Output should include ls stdout of filenames a b c on each line file_list = ( @@ -208,10 +264,12 @@ def test_directory_workflow(tmp_path: Path) -> None: @needs_docker -def test_no_data_files(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_no_data_files(tmp_path: Path, with_orcid: bool) -> None: folder = cwltool( tmp_path, get_data("tests/wf/conditional_step_no_inputs.cwl"), + with_orcid=with_orcid, ) check_bagit(folder) @@ -262,6 +320,7 @@ def check_provenance( single_tool: bool = False, directory: bool = False, secondary_files: bool = False, + with_orcid: bool = False, ) -> None: check_folders(base_path) check_bagit(base_path) @@ -272,6 +331,7 @@ def check_provenance( single_tool=single_tool, directory=directory, secondary_files=secondary_files, + with_orcid=with_orcid, ) @@ -462,6 +522,7 @@ def check_prov( single_tool: bool = False, directory: bool = False, secondary_files: bool = False, + with_orcid: bool = False, ) -> None: prov_file = base_path / "metadata" / "provenance" / "primary.cwlprov.nt" assert prov_file.is_file(), f"Can't find {prov_file}" @@ -484,7 +545,6 @@ def check_prov( # the has_provenance annotations in manifest.json instead # run should have been started by a wf engine - engines = set(g.subjects(RDF.type, WFPROV.WorkflowEngine)) assert engines, "Could not find WorkflowEngine" assert len(engines) == 1, "Found too many WorkflowEngines: %s" % engines @@ -501,6 +561,39 @@ def check_prov( PROV.SoftwareAgent, ) in g, "Engine not declared as SoftwareAgent" + # run should be associated to the user + accounts = set(g.subjects(RDF.type, FOAF.OnlineAccount)) + assert len(accounts) == 1 + account = accounts.pop() + people = set(g.subjects(RDF.type, SCHEMA.Person)) + assert len(people) == 1, "Can't find associated person in workflow run" + person = people.pop() + if with_orcid: + assert person == URIRef(TEST_ORCID) + else: + account_names = set(g.objects(account, FOAF.accountName)) + assert len(account_names) == 1 + account_name = cast(Literal, account_names.pop()) + machine_user = provenance._whoami()[0] + assert account_name.value == machine_user + + # find the random UUID assigned to cwltool + tool_agents = set(g.subjects(RDF.type, PROV.SoftwareAgent)) + n_all_agents = 2 + len(tool_agents) + agents = set(g.subjects(RDF.type, PROV.Agent)) + assert ( + len(agents) == n_all_agents + ), "There should be 1 agent per tool (engine), 1 user agent, and 1 cwltool agent" + agents.remove(person) + agents.remove(engine) # the main tool + remain_agents = agents - tool_agents + assert len(remain_agents) == 1 + assert ( + account, + PROV.actedOnBehalfOf, + person, + ) in g, "Association of cwltool agent acting for user is missing" + if single_tool: activities = set(g.subjects(RDF.type, PROV.Activity)) assert len(activities) == 1, "Too many activities: %s" % activities diff --git a/tests/test_relocate.py b/tests/test_relocate.py index 81877c776..692e995fa 100644 --- a/tests/test_relocate.py +++ b/tests/test_relocate.py @@ -1,18 +1,13 @@ import json import os import shutil -import sys +from io import StringIO from pathlib import Path from cwltool.main import main from .util import get_data, needs_docker -if sys.version_info[0] < 3: - from StringIO import StringIO -else: - from io import StringIO - @needs_docker def test_for_910(tmp_path: Path) -> None: diff --git a/tests/test_secrets.py b/tests/test_secrets.py index bd90bee78..206b001bd 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -1,7 +1,7 @@ import shutil import tempfile +from collections.abc import Callable from io import StringIO -from typing import Callable, Dict, List, Tuple, Union import pytest @@ -13,7 +13,7 @@ @pytest.fixture -def secrets() -> Tuple[SecretStore, CWLObjectType]: +def secrets() -> tuple[SecretStore, CWLObjectType]: """Fixture to return a secret store.""" sec_store = SecretStore() job: CWLObjectType = {"foo": "bar", "baz": "quux"} @@ -22,7 +22,7 @@ def secrets() -> Tuple[SecretStore, CWLObjectType]: return sec_store, job -def test_obscuring(secrets: Tuple[SecretStore, CWLObjectType]) -> None: +def test_obscuring(secrets: tuple[SecretStore, CWLObjectType]) -> None: """Basic test of secret store.""" storage, obscured = secrets assert obscured["foo"] != "bar" @@ -41,8 +41,8 @@ def test_obscuring(secrets: Tuple[SecretStore, CWLObjectType]) -> None: @pytest.mark.parametrize("factory,expected", obscured_factories_expected) def test_secrets( factory: Callable[[str], CWLObjectType], - expected: Union[str, List[str], Dict[str, str]], - secrets: Tuple[SecretStore, CWLObjectType], + expected: str | list[str] | dict[str, str], + secrets: tuple[SecretStore, CWLObjectType], ) -> None: storage, obscured = secrets obs = obscured["foo"] diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 0512f2e28..1139dfbc7 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -2,7 +2,6 @@ import shutil from pathlib import Path -from typing import Any import pytest @@ -19,7 +18,7 @@ @needs_singularity_2_6 -def test_singularity_pullfolder(tmp_path: Path, monkeypatch: Any) -> None: +def test_singularity_pullfolder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test singularity respects SINGULARITY_PULLFOLDER.""" workdir = tmp_path / "working_dir_new" workdir.mkdir() diff --git a/tests/test_subclass_mypyc.py b/tests/test_subclass_mypyc.py index aa4964b34..14cdc9dc6 100644 --- a/tests/test_subclass_mypyc.py +++ b/tests/test_subclass_mypyc.py @@ -13,11 +13,13 @@ from cwltool.builder import Builder from cwltool.command_line_tool import CommandLineTool, ExpressionTool from cwltool.context import LoadingContext, RuntimeContext +from cwltool.procgenerator import ProcessGenerator from cwltool.stdfsaccess import StdFsAccess from cwltool.update import INTERNAL_VERSION from cwltool.workflow import Workflow from .test_anon_types import snippet +from .util import get_data @pytest.mark.parametrize("snippet", snippet) @@ -52,6 +54,35 @@ def test_pickle_unpickle_workflow(snippet: CommentedMap) -> None: assert pickle.loads(stream) +def test_pickle_processgenerator() -> None: + """We can pickle ProcessGenerator.""" + + pytoolgen = get_data("tests/wf/generator/pytoolgen.cwl") + a = ProcessGenerator( + CommentedMap( + { + "cwlVersion": "v1.0", + "id": f"file://{pytoolgen}", + "$namespaces": {"cwltool": "http://commonwl.org/cwltool#"}, + "class": "cwltool:ProcessGenerator", + "inputs": {}, + "outputs": {}, + "run": { + "id": f"file://{pytoolgen}#f9f617af-3088-4ade-bbf3-acd1d6f51355", + "cwlVersion": "v1.0", + "class": "CommandLineTool", + "inputs": {}, + "outputs": {}, + "baseCommand": "echo", + }, + }, + ), + LoadingContext(), + ) + + assert pickle.dumps(a) + + def test_serialize_builder() -> None: """We can pickle Builder.""" runtime_context = RuntimeContext() diff --git a/tests/test_subgraph.py b/tests/test_subgraph.py index eb99124ea..fd5d22af0 100644 --- a/tests/test_subgraph.py +++ b/tests/test_subgraph.py @@ -225,6 +225,7 @@ def test_single_process_subwf_subwf_inline_step(tmp_path: Path) -> None: [ "--outdir", str(tmp_path), + "--debug", "--single-process", "step1/stepX/stepY", get_data("tests/subgraph/count-lines17-wf.cwl.json"), diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index 73fe240d0..18a588cf8 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -6,7 +6,7 @@ import subprocess import sys from pathlib import Path -from typing import List, cast +from typing import cast import pytest from ruamel.yaml.comments import CommentedMap @@ -318,7 +318,7 @@ def test_docker_tmpdir_prefix(tmp_path: Path) -> None: "docker", ) job = DockerCommandLineJob(builder, {}, CommandLineTool.make_path_mapper, [], [], "") - runtime: List[str] = [] + runtime: list[str] = [] volume_writable_file = MapperEnt( resolved=get_data("tests/2.fastq"), target="foo", type=None, staged=None diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py index 11ce5e3db..2bf7a809a 100644 --- a/tests/test_toolargparse.py +++ b/tests/test_toolargparse.py @@ -1,7 +1,7 @@ import argparse +from collections.abc import Callable from io import StringIO from pathlib import Path -from typing import Callable, List import pytest @@ -296,7 +296,7 @@ def test_argparser_without_doc() -> None: ), ], ) -def test_argparse_append_with_default(job_order: List[str], expected_values: List[str]) -> None: +def test_argparse_append_with_default(job_order: list[str], expected_values: list[str]) -> None: """ Confirm that the appended arguments must not include the default. diff --git a/tests/test_trs.py b/tests/test_trs.py index 5f94ce58a..c51b6e12b 100644 --- a/tests/test_trs.py +++ b/tests/test_trs.py @@ -1,4 +1,4 @@ -from typing import Any, Optional +from typing import Any from unittest import mock from unittest.mock import MagicMock @@ -10,7 +10,7 @@ class MockResponse1: def __init__( - self, json_data: Any, status_code: int, raise_for_status: Optional[bool] = None + self, json_data: Any, status_code: int, raise_for_status: bool | None = None ) -> None: """Create a fake return object for requests.Session.head.""" self.json_data = json_data @@ -28,7 +28,7 @@ def mocked_requests_head(*args: Any, **kwargs: Any) -> MockResponse1: class MockResponse2: def __init__( - self, json_data: Any, status_code: int, raise_for_status: Optional[bool] = None + self, json_data: Any, status_code: int, raise_for_status: bool | None = None ) -> None: """Create a fake return object for requests.Session.get.""" self.json_data = json_data diff --git a/tests/test_validate.py b/tests/test_validate.py index 171a6b6c1..8e664d9aa 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -1,5 +1,7 @@ """Tests --validation.""" +import io +import logging import re from .util import get_data, get_main_output @@ -43,13 +45,83 @@ def test_validate_with_invalid_input_object() -> None: ] ) assert exit_code == 1 - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Invalid job input record" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert "Invalid job input record" in stdout assert ( "tests/wf/1st-workflow_bad_inputs.yml:2:1: * the 'ex' field is not " - "valid because the value is not string" in stderr + "valid because the value is not string" in stdout ) assert ( "tests/wf/1st-workflow_bad_inputs.yml:1:1: * the 'inp' field is not " - "valid because is not a dict. Expected a File object." in stderr + "valid because is not a dict. Expected a File object." in stdout + ) + + +def test_validate_quiet() -> None: + """Ensure that --validate --quiet prints the correct amount of information.""" + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + "--quiet", + get_data("tests/CometAdapter.cwl"), + ] + ) + assert exit_code == 0 + stdout = re.sub(r"\s\s+", " ", stdout) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "tests/CometAdapter.cwl:10:3: object id" in stdout + assert "tests/CometAdapter.cwl#out' previously defined" in stdout + + +def test_validate_no_warnings() -> None: + """Ensure that --validate --no-warnings doesn't print any warnings.""" + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + "--no-warnings", + get_data("tests/CometAdapter.cwl"), + ] ) + assert exit_code == 0 + stdout = re.sub(r"\s\s+", " ", stdout) + stderr = re.sub(r"\s\s+", " ", stderr) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "WARNING" not in stdout + assert "WARNING" not in stderr + assert "tests/CometAdapter.cwl:9:3: object id" not in stdout + assert "tests/CometAdapter.cwl:9:3: object id" not in stderr + assert "tests/CometAdapter.cwl#out' previously defined" not in stdout + assert "tests/CometAdapter.cwl#out' previously defined" not in stderr + + +def test_validate_custom_logger() -> None: + """Custom log handling test.""" + custom_log = io.StringIO() + handler = logging.StreamHandler(custom_log) + handler.setLevel(logging.DEBUG) + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + get_data("tests/CometAdapter.cwl"), + ], + logger_handler=handler, + ) + custom_log_text = custom_log.getvalue() + assert exit_code == 0 + custom_log_text = re.sub(r"\s\s+", " ", custom_log_text) + stdout = re.sub(r"\s\s+", " ", stdout) + stderr = re.sub(r"\s\s+", " ", stderr) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "INFO" in custom_log_text + assert "WARNING" not in stdout + assert "WARNING" not in stderr + assert "WARNING" in custom_log_text + assert "tests/CometAdapter.cwl:10:3: object id" not in stdout + assert "tests/CometAdapter.cwl:10:3: object id" not in stderr + assert "tests/CometAdapter.cwl:10:3: object id" in custom_log_text + assert "tests/CometAdapter.cwl#out' previously defined" not in stdout + assert "tests/CometAdapter.cwl#out' previously defined" not in stderr + assert "tests/CometAdapter.cwl#out' previously defined" in custom_log_text diff --git a/tests/trs/Dockstore.cwl b/tests/trs/Dockstore.cwl old mode 100644 new mode 100755 diff --git a/tests/trs/md5sum-tool.cwl b/tests/trs/md5sum-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/trs/md5sum-workflow.cwl b/tests/trs/md5sum-workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/utf_doc_example.cwl b/tests/utf_doc_example.cwl old mode 100644 new mode 100755 diff --git a/tests/util.py b/tests/util.py index 0547cfa9a..235c3e6c3 100644 --- a/tests/util.py +++ b/tests/util.py @@ -8,16 +8,17 @@ import shutil import subprocess import sys +from collections.abc import Generator, Mapping from contextlib import ExitStack +from importlib.resources import as_file, files from pathlib import Path -from typing import Dict, Generator, List, Mapping, Optional, Tuple, Union +from typing import Any import pytest from cwltool.env_to_stdout import deserialize_env from cwltool.main import main from cwltool.singularity import is_version_2_6, is_version_3_or_newer -from cwltool.utils import as_file, files def force_default_container(default_container_id: str, _: str) -> str: @@ -65,7 +66,7 @@ def get_data(filename: str) -> str: reason="Requires the podman executable on the system path.", ) -_env_accepts_null: Optional[bool] = None +_env_accepts_null: bool | None = None def env_accepts_null() -> bool: @@ -83,11 +84,12 @@ def env_accepts_null() -> bool: def get_main_output( - args: List[str], - replacement_env: Optional[Mapping[str, str]] = None, - extra_env: Optional[Mapping[str, str]] = None, - monkeypatch: Optional[pytest.MonkeyPatch] = None, -) -> Tuple[Optional[int], str, str]: + args: list[str], + replacement_env: Mapping[str, str] | None = None, + extra_env: Mapping[str, str] | None = None, + monkeypatch: pytest.MonkeyPatch | None = None, + **extra_kwargs: Any, +) -> tuple[int | None, str, str]: """Run cwltool main. args: the command line args to call it with @@ -112,7 +114,7 @@ def get_main_output( monkeypatch.setenv(k, v) try: - rc = main(argsl=args, stdout=stdout, stderr=stderr) + rc = main(argsl=args, stdout=stdout, stderr=stderr, **extra_kwargs) except SystemExit as e: if isinstance(e.code, int): rc = e.code @@ -127,13 +129,13 @@ def get_main_output( def get_tool_env( tmp_path: Path, - flag_args: List[str], - inputs_file: Optional[str] = None, - replacement_env: Optional[Mapping[str, str]] = None, - extra_env: Optional[Mapping[str, str]] = None, - monkeypatch: Optional[pytest.MonkeyPatch] = None, - runtime_env_accepts_null: Optional[bool] = None, -) -> Dict[str, str]: + flag_args: list[str], + inputs_file: str | None = None, + replacement_env: Mapping[str, str] | None = None, + extra_env: Mapping[str, str] | None = None, + monkeypatch: pytest.MonkeyPatch | None = None, + runtime_env_accepts_null: bool | None = None, +) -> dict[str, str]: """Get the env vars for a tool's invocation.""" # GNU env accepts the -0 option to end each variable's # printing with "\0". No such luck on BSD-ish. @@ -167,7 +169,7 @@ def get_tool_env( @contextlib.contextmanager -def working_directory(path: Union[str, Path]) -> Generator[None, None, None]: +def working_directory(path: str | Path) -> Generator[None, None, None]: """Change working directory and returns to previous on exit.""" prev_cwd = Path.cwd() os.chdir(path) diff --git a/tests/wc-tool-bad-hints.cwl b/tests/wc-tool-bad-hints.cwl old mode 100644 new mode 100755 diff --git a/tests/wc-tool-bad-reqs.cwl b/tests/wc-tool-bad-reqs.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/1496.cwl b/tests/wf/1496.cwl old mode 100644 new mode 100755 index 74f6ef49e..d5da025dd --- a/tests/wf/1496.cwl +++ b/tests/wf/1496.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool diff --git a/tests/wf/1590.cwl b/tests/wf/1590.cwl old mode 100644 new mode 100755 index 19c8ad331..de602a500 --- a/tests/wf/1590.cwl +++ b/tests/wf/1590.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "baseCommand": [ "cat" diff --git a/tests/wf/1st-workflow.cwl b/tests/wf/1st-workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/811-12.cwl b/tests/wf/811-12.cwl old mode 100644 new mode 100755 index f4403f45b..ab0b2bd92 --- a/tests/wf/811-12.cwl +++ b/tests/wf/811-12.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: Workflow diff --git a/tests/wf/811.cwl b/tests/wf/811.cwl old mode 100644 new mode 100755 index 16a4c828d..8ef826fdc --- a/tests/wf/811.cwl +++ b/tests/wf/811.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: Workflow diff --git a/tests/wf/816_tool.cwl b/tests/wf/816_tool.cwl old mode 100644 new mode 100755 index 00395abe8..c48d3e373 --- a/tests/wf/816_tool.cwl +++ b/tests/wf/816_tool.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool requirements: diff --git a/tests/wf/816_wf.cwl b/tests/wf/816_wf.cwl old mode 100644 new mode 100755 index a64b130b6..9db143916 --- a/tests/wf/816_wf.cwl +++ b/tests/wf/816_wf.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Workflow cwlVersion: v1.0 inputs: diff --git a/tests/wf/910.cwl b/tests/wf/910.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/arguments.cwl b/tests/wf/arguments.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad-stderr-expr.cwl b/tests/wf/bad-stderr-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad-stdin-expr.cwl b/tests/wf/bad-stdin-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad-stdout-expr.cwl b/tests/wf/bad-stdout-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad_networkaccess.cwl b/tests/wf/bad_networkaccess.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad_timelimit.cwl b/tests/wf/bad_timelimit.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/cache_test_workflow.cwl b/tests/wf/cache_test_workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/cat-tool.cwl b/tests/wf/cat-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/conditional_step_no_inputs.cwl b/tests/wf/conditional_step_no_inputs.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/conflict.cwl b/tests/wf/conflict.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/cores_float.cwl b/tests/wf/cores_float.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/default-dir5.cwl b/tests/wf/default-dir5.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/default-wf5.cwl b/tests/wf/default-wf5.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/double-nested.cwl b/tests/wf/double-nested.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/expect_trick_packed.cwl b/tests/wf/expect_trick_packed.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/extract_region_specs.cwl b/tests/wf/extract_region_specs.cwl old mode 100644 new mode 100755 index 279fa4400..437222a13 --- a/tests/wf/extract_region_specs.cwl +++ b/tests/wf/extract_region_specs.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "cwlVersion": "v1.0", "class": "CommandLineTool", diff --git a/tests/wf/floats_small_and_large.cwl b/tests/wf/floats_small_and_large.cwl old mode 100644 new mode 100755 index 434327361..1637bd9af --- a/tests/wf/floats_small_and_large.cwl +++ b/tests/wf/floats_small_and_large.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool baseCommand: echo diff --git a/tests/wf/generator/pytoolgen.cwl b/tests/wf/generator/pytoolgen.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/inp-filelist.txt b/tests/wf/inp-filelist.txt new file mode 100644 index 000000000..232ddf670 --- /dev/null +++ b/tests/wf/inp-filelist.txt @@ -0,0 +1,9999 @@ +example_input_file1.txt +example_input_file2.txt +example_input_file3.txt +example_input_file4.txt +example_input_file5.txt +example_input_file6.txt +example_input_file7.txt +example_input_file8.txt +example_input_file9.txt +example_input_file10.txt +example_input_file11.txt +example_input_file12.txt +example_input_file13.txt +example_input_file14.txt +example_input_file15.txt +example_input_file16.txt +example_input_file17.txt +example_input_file18.txt +example_input_file19.txt +example_input_file20.txt +example_input_file21.txt +example_input_file22.txt +example_input_file23.txt +example_input_file24.txt +example_input_file25.txt +example_input_file26.txt +example_input_file27.txt +example_input_file28.txt +example_input_file29.txt +example_input_file30.txt +example_input_file31.txt +example_input_file32.txt +example_input_file33.txt +example_input_file34.txt +example_input_file35.txt +example_input_file36.txt +example_input_file37.txt +example_input_file38.txt +example_input_file39.txt +example_input_file40.txt +example_input_file41.txt +example_input_file42.txt +example_input_file43.txt +example_input_file44.txt +example_input_file45.txt +example_input_file46.txt +example_input_file47.txt +example_input_file48.txt +example_input_file49.txt +example_input_file50.txt +example_input_file51.txt +example_input_file52.txt +example_input_file53.txt +example_input_file54.txt +example_input_file55.txt +example_input_file56.txt +example_input_file57.txt +example_input_file58.txt +example_input_file59.txt +example_input_file60.txt +example_input_file61.txt +example_input_file62.txt +example_input_file63.txt +example_input_file64.txt +example_input_file65.txt +example_input_file66.txt +example_input_file67.txt +example_input_file68.txt +example_input_file69.txt +example_input_file70.txt +example_input_file71.txt +example_input_file72.txt +example_input_file73.txt +example_input_file74.txt +example_input_file75.txt +example_input_file76.txt +example_input_file77.txt +example_input_file78.txt +example_input_file79.txt +example_input_file80.txt +example_input_file81.txt +example_input_file82.txt +example_input_file83.txt +example_input_file84.txt +example_input_file85.txt +example_input_file86.txt +example_input_file87.txt +example_input_file88.txt +example_input_file89.txt +example_input_file90.txt +example_input_file91.txt +example_input_file92.txt +example_input_file93.txt +example_input_file94.txt +example_input_file95.txt +example_input_file96.txt +example_input_file97.txt +example_input_file98.txt +example_input_file99.txt +example_input_file100.txt +example_input_file101.txt +example_input_file102.txt +example_input_file103.txt +example_input_file104.txt +example_input_file105.txt +example_input_file106.txt +example_input_file107.txt +example_input_file108.txt +example_input_file109.txt +example_input_file110.txt +example_input_file111.txt +example_input_file112.txt +example_input_file113.txt +example_input_file114.txt +example_input_file115.txt +example_input_file116.txt +example_input_file117.txt +example_input_file118.txt +example_input_file119.txt +example_input_file120.txt +example_input_file121.txt +example_input_file122.txt +example_input_file123.txt +example_input_file124.txt +example_input_file125.txt +example_input_file126.txt +example_input_file127.txt +example_input_file128.txt +example_input_file129.txt +example_input_file130.txt +example_input_file131.txt +example_input_file132.txt +example_input_file133.txt +example_input_file134.txt +example_input_file135.txt +example_input_file136.txt +example_input_file137.txt +example_input_file138.txt +example_input_file139.txt +example_input_file140.txt +example_input_file141.txt +example_input_file142.txt +example_input_file143.txt +example_input_file144.txt +example_input_file145.txt +example_input_file146.txt +example_input_file147.txt +example_input_file148.txt +example_input_file149.txt +example_input_file150.txt +example_input_file151.txt +example_input_file152.txt +example_input_file153.txt +example_input_file154.txt +example_input_file155.txt +example_input_file156.txt +example_input_file157.txt +example_input_file158.txt +example_input_file159.txt +example_input_file160.txt +example_input_file161.txt +example_input_file162.txt +example_input_file163.txt +example_input_file164.txt +example_input_file165.txt +example_input_file166.txt +example_input_file167.txt +example_input_file168.txt +example_input_file169.txt +example_input_file170.txt +example_input_file171.txt +example_input_file172.txt +example_input_file173.txt +example_input_file174.txt +example_input_file175.txt +example_input_file176.txt +example_input_file177.txt +example_input_file178.txt +example_input_file179.txt +example_input_file180.txt +example_input_file181.txt +example_input_file182.txt +example_input_file183.txt +example_input_file184.txt +example_input_file185.txt +example_input_file186.txt +example_input_file187.txt +example_input_file188.txt +example_input_file189.txt +example_input_file190.txt +example_input_file191.txt +example_input_file192.txt +example_input_file193.txt +example_input_file194.txt +example_input_file195.txt +example_input_file196.txt +example_input_file197.txt +example_input_file198.txt +example_input_file199.txt +example_input_file200.txt +example_input_file201.txt +example_input_file202.txt +example_input_file203.txt +example_input_file204.txt +example_input_file205.txt +example_input_file206.txt +example_input_file207.txt +example_input_file208.txt +example_input_file209.txt +example_input_file210.txt +example_input_file211.txt +example_input_file212.txt +example_input_file213.txt +example_input_file214.txt +example_input_file215.txt +example_input_file216.txt +example_input_file217.txt +example_input_file218.txt +example_input_file219.txt +example_input_file220.txt +example_input_file221.txt +example_input_file222.txt +example_input_file223.txt +example_input_file224.txt +example_input_file225.txt +example_input_file226.txt +example_input_file227.txt +example_input_file228.txt +example_input_file229.txt +example_input_file230.txt +example_input_file231.txt +example_input_file232.txt +example_input_file233.txt +example_input_file234.txt +example_input_file235.txt +example_input_file236.txt +example_input_file237.txt +example_input_file238.txt +example_input_file239.txt +example_input_file240.txt +example_input_file241.txt +example_input_file242.txt +example_input_file243.txt +example_input_file244.txt +example_input_file245.txt +example_input_file246.txt +example_input_file247.txt +example_input_file248.txt +example_input_file249.txt +example_input_file250.txt +example_input_file251.txt +example_input_file252.txt +example_input_file253.txt +example_input_file254.txt +example_input_file255.txt +example_input_file256.txt +example_input_file257.txt +example_input_file258.txt +example_input_file259.txt +example_input_file260.txt +example_input_file261.txt +example_input_file262.txt +example_input_file263.txt +example_input_file264.txt +example_input_file265.txt +example_input_file266.txt +example_input_file267.txt +example_input_file268.txt +example_input_file269.txt +example_input_file270.txt +example_input_file271.txt +example_input_file272.txt +example_input_file273.txt +example_input_file274.txt +example_input_file275.txt +example_input_file276.txt +example_input_file277.txt +example_input_file278.txt +example_input_file279.txt +example_input_file280.txt +example_input_file281.txt +example_input_file282.txt +example_input_file283.txt +example_input_file284.txt +example_input_file285.txt +example_input_file286.txt +example_input_file287.txt +example_input_file288.txt +example_input_file289.txt +example_input_file290.txt +example_input_file291.txt +example_input_file292.txt +example_input_file293.txt +example_input_file294.txt +example_input_file295.txt +example_input_file296.txt +example_input_file297.txt +example_input_file298.txt +example_input_file299.txt +example_input_file300.txt +example_input_file301.txt +example_input_file302.txt +example_input_file303.txt +example_input_file304.txt +example_input_file305.txt +example_input_file306.txt +example_input_file307.txt +example_input_file308.txt +example_input_file309.txt +example_input_file310.txt +example_input_file311.txt +example_input_file312.txt +example_input_file313.txt +example_input_file314.txt +example_input_file315.txt +example_input_file316.txt +example_input_file317.txt +example_input_file318.txt +example_input_file319.txt +example_input_file320.txt +example_input_file321.txt +example_input_file322.txt +example_input_file323.txt +example_input_file324.txt +example_input_file325.txt +example_input_file326.txt +example_input_file327.txt +example_input_file328.txt +example_input_file329.txt +example_input_file330.txt +example_input_file331.txt +example_input_file332.txt +example_input_file333.txt +example_input_file334.txt +example_input_file335.txt +example_input_file336.txt +example_input_file337.txt +example_input_file338.txt +example_input_file339.txt +example_input_file340.txt +example_input_file341.txt +example_input_file342.txt +example_input_file343.txt +example_input_file344.txt +example_input_file345.txt +example_input_file346.txt +example_input_file347.txt +example_input_file348.txt +example_input_file349.txt +example_input_file350.txt +example_input_file351.txt +example_input_file352.txt +example_input_file353.txt +example_input_file354.txt +example_input_file355.txt +example_input_file356.txt +example_input_file357.txt +example_input_file358.txt +example_input_file359.txt +example_input_file360.txt +example_input_file361.txt +example_input_file362.txt +example_input_file363.txt +example_input_file364.txt +example_input_file365.txt +example_input_file366.txt +example_input_file367.txt +example_input_file368.txt +example_input_file369.txt +example_input_file370.txt +example_input_file371.txt +example_input_file372.txt +example_input_file373.txt +example_input_file374.txt +example_input_file375.txt +example_input_file376.txt +example_input_file377.txt +example_input_file378.txt +example_input_file379.txt +example_input_file380.txt +example_input_file381.txt +example_input_file382.txt +example_input_file383.txt +example_input_file384.txt +example_input_file385.txt +example_input_file386.txt +example_input_file387.txt +example_input_file388.txt +example_input_file389.txt +example_input_file390.txt +example_input_file391.txt +example_input_file392.txt +example_input_file393.txt +example_input_file394.txt +example_input_file395.txt +example_input_file396.txt +example_input_file397.txt +example_input_file398.txt +example_input_file399.txt +example_input_file400.txt +example_input_file401.txt +example_input_file402.txt +example_input_file403.txt +example_input_file404.txt +example_input_file405.txt +example_input_file406.txt +example_input_file407.txt +example_input_file408.txt +example_input_file409.txt +example_input_file410.txt +example_input_file411.txt +example_input_file412.txt +example_input_file413.txt +example_input_file414.txt +example_input_file415.txt +example_input_file416.txt +example_input_file417.txt +example_input_file418.txt +example_input_file419.txt +example_input_file420.txt +example_input_file421.txt +example_input_file422.txt +example_input_file423.txt +example_input_file424.txt +example_input_file425.txt +example_input_file426.txt +example_input_file427.txt +example_input_file428.txt +example_input_file429.txt +example_input_file430.txt +example_input_file431.txt +example_input_file432.txt +example_input_file433.txt +example_input_file434.txt +example_input_file435.txt +example_input_file436.txt +example_input_file437.txt +example_input_file438.txt +example_input_file439.txt +example_input_file440.txt +example_input_file441.txt +example_input_file442.txt +example_input_file443.txt +example_input_file444.txt +example_input_file445.txt +example_input_file446.txt +example_input_file447.txt +example_input_file448.txt +example_input_file449.txt +example_input_file450.txt +example_input_file451.txt +example_input_file452.txt +example_input_file453.txt +example_input_file454.txt +example_input_file455.txt +example_input_file456.txt +example_input_file457.txt +example_input_file458.txt +example_input_file459.txt +example_input_file460.txt +example_input_file461.txt +example_input_file462.txt +example_input_file463.txt +example_input_file464.txt +example_input_file465.txt +example_input_file466.txt +example_input_file467.txt +example_input_file468.txt +example_input_file469.txt +example_input_file470.txt +example_input_file471.txt +example_input_file472.txt +example_input_file473.txt +example_input_file474.txt +example_input_file475.txt +example_input_file476.txt +example_input_file477.txt +example_input_file478.txt +example_input_file479.txt +example_input_file480.txt +example_input_file481.txt +example_input_file482.txt +example_input_file483.txt +example_input_file484.txt +example_input_file485.txt +example_input_file486.txt +example_input_file487.txt +example_input_file488.txt +example_input_file489.txt +example_input_file490.txt +example_input_file491.txt +example_input_file492.txt +example_input_file493.txt +example_input_file494.txt +example_input_file495.txt +example_input_file496.txt +example_input_file497.txt +example_input_file498.txt +example_input_file499.txt +example_input_file500.txt +example_input_file501.txt +example_input_file502.txt +example_input_file503.txt +example_input_file504.txt +example_input_file505.txt +example_input_file506.txt +example_input_file507.txt +example_input_file508.txt +example_input_file509.txt +example_input_file510.txt +example_input_file511.txt +example_input_file512.txt +example_input_file513.txt +example_input_file514.txt +example_input_file515.txt +example_input_file516.txt +example_input_file517.txt +example_input_file518.txt +example_input_file519.txt +example_input_file520.txt +example_input_file521.txt +example_input_file522.txt +example_input_file523.txt +example_input_file524.txt +example_input_file525.txt +example_input_file526.txt +example_input_file527.txt +example_input_file528.txt +example_input_file529.txt +example_input_file530.txt +example_input_file531.txt +example_input_file532.txt +example_input_file533.txt +example_input_file534.txt +example_input_file535.txt +example_input_file536.txt +example_input_file537.txt +example_input_file538.txt +example_input_file539.txt +example_input_file540.txt +example_input_file541.txt +example_input_file542.txt +example_input_file543.txt +example_input_file544.txt +example_input_file545.txt +example_input_file546.txt +example_input_file547.txt +example_input_file548.txt +example_input_file549.txt +example_input_file550.txt +example_input_file551.txt +example_input_file552.txt +example_input_file553.txt +example_input_file554.txt +example_input_file555.txt +example_input_file556.txt +example_input_file557.txt +example_input_file558.txt +example_input_file559.txt +example_input_file560.txt +example_input_file561.txt +example_input_file562.txt +example_input_file563.txt +example_input_file564.txt +example_input_file565.txt +example_input_file566.txt +example_input_file567.txt +example_input_file568.txt +example_input_file569.txt +example_input_file570.txt +example_input_file571.txt +example_input_file572.txt +example_input_file573.txt +example_input_file574.txt +example_input_file575.txt +example_input_file576.txt +example_input_file577.txt +example_input_file578.txt +example_input_file579.txt +example_input_file580.txt +example_input_file581.txt +example_input_file582.txt +example_input_file583.txt +example_input_file584.txt +example_input_file585.txt +example_input_file586.txt +example_input_file587.txt +example_input_file588.txt +example_input_file589.txt +example_input_file590.txt +example_input_file591.txt +example_input_file592.txt +example_input_file593.txt +example_input_file594.txt +example_input_file595.txt +example_input_file596.txt +example_input_file597.txt +example_input_file598.txt +example_input_file599.txt +example_input_file600.txt +example_input_file601.txt +example_input_file602.txt +example_input_file603.txt +example_input_file604.txt +example_input_file605.txt +example_input_file606.txt +example_input_file607.txt +example_input_file608.txt +example_input_file609.txt +example_input_file610.txt +example_input_file611.txt +example_input_file612.txt +example_input_file613.txt +example_input_file614.txt +example_input_file615.txt +example_input_file616.txt +example_input_file617.txt +example_input_file618.txt +example_input_file619.txt +example_input_file620.txt +example_input_file621.txt +example_input_file622.txt +example_input_file623.txt +example_input_file624.txt +example_input_file625.txt +example_input_file626.txt +example_input_file627.txt +example_input_file628.txt +example_input_file629.txt +example_input_file630.txt +example_input_file631.txt +example_input_file632.txt +example_input_file633.txt +example_input_file634.txt +example_input_file635.txt +example_input_file636.txt +example_input_file637.txt +example_input_file638.txt +example_input_file639.txt +example_input_file640.txt +example_input_file641.txt +example_input_file642.txt +example_input_file643.txt +example_input_file644.txt +example_input_file645.txt +example_input_file646.txt +example_input_file647.txt +example_input_file648.txt +example_input_file649.txt +example_input_file650.txt +example_input_file651.txt +example_input_file652.txt +example_input_file653.txt +example_input_file654.txt +example_input_file655.txt +example_input_file656.txt +example_input_file657.txt +example_input_file658.txt +example_input_file659.txt +example_input_file660.txt +example_input_file661.txt +example_input_file662.txt +example_input_file663.txt +example_input_file664.txt +example_input_file665.txt +example_input_file666.txt +example_input_file667.txt +example_input_file668.txt +example_input_file669.txt +example_input_file670.txt +example_input_file671.txt +example_input_file672.txt +example_input_file673.txt +example_input_file674.txt +example_input_file675.txt +example_input_file676.txt +example_input_file677.txt +example_input_file678.txt +example_input_file679.txt +example_input_file680.txt +example_input_file681.txt +example_input_file682.txt +example_input_file683.txt +example_input_file684.txt +example_input_file685.txt +example_input_file686.txt +example_input_file687.txt +example_input_file688.txt +example_input_file689.txt +example_input_file690.txt +example_input_file691.txt +example_input_file692.txt +example_input_file693.txt +example_input_file694.txt +example_input_file695.txt +example_input_file696.txt +example_input_file697.txt +example_input_file698.txt +example_input_file699.txt +example_input_file700.txt +example_input_file701.txt +example_input_file702.txt +example_input_file703.txt +example_input_file704.txt +example_input_file705.txt +example_input_file706.txt +example_input_file707.txt +example_input_file708.txt +example_input_file709.txt +example_input_file710.txt +example_input_file711.txt +example_input_file712.txt +example_input_file713.txt +example_input_file714.txt +example_input_file715.txt +example_input_file716.txt +example_input_file717.txt +example_input_file718.txt +example_input_file719.txt +example_input_file720.txt +example_input_file721.txt +example_input_file722.txt +example_input_file723.txt +example_input_file724.txt +example_input_file725.txt +example_input_file726.txt +example_input_file727.txt +example_input_file728.txt +example_input_file729.txt +example_input_file730.txt +example_input_file731.txt +example_input_file732.txt +example_input_file733.txt +example_input_file734.txt +example_input_file735.txt +example_input_file736.txt +example_input_file737.txt +example_input_file738.txt +example_input_file739.txt +example_input_file740.txt +example_input_file741.txt +example_input_file742.txt +example_input_file743.txt +example_input_file744.txt +example_input_file745.txt +example_input_file746.txt +example_input_file747.txt +example_input_file748.txt +example_input_file749.txt +example_input_file750.txt +example_input_file751.txt +example_input_file752.txt +example_input_file753.txt +example_input_file754.txt +example_input_file755.txt +example_input_file756.txt +example_input_file757.txt +example_input_file758.txt +example_input_file759.txt +example_input_file760.txt +example_input_file761.txt +example_input_file762.txt +example_input_file763.txt +example_input_file764.txt +example_input_file765.txt +example_input_file766.txt +example_input_file767.txt +example_input_file768.txt +example_input_file769.txt +example_input_file770.txt +example_input_file771.txt +example_input_file772.txt +example_input_file773.txt +example_input_file774.txt +example_input_file775.txt +example_input_file776.txt +example_input_file777.txt +example_input_file778.txt +example_input_file779.txt +example_input_file780.txt +example_input_file781.txt +example_input_file782.txt +example_input_file783.txt +example_input_file784.txt +example_input_file785.txt +example_input_file786.txt +example_input_file787.txt +example_input_file788.txt +example_input_file789.txt +example_input_file790.txt +example_input_file791.txt +example_input_file792.txt +example_input_file793.txt +example_input_file794.txt +example_input_file795.txt +example_input_file796.txt +example_input_file797.txt +example_input_file798.txt +example_input_file799.txt +example_input_file800.txt +example_input_file801.txt +example_input_file802.txt +example_input_file803.txt +example_input_file804.txt +example_input_file805.txt +example_input_file806.txt +example_input_file807.txt +example_input_file808.txt +example_input_file809.txt +example_input_file810.txt +example_input_file811.txt +example_input_file812.txt +example_input_file813.txt +example_input_file814.txt +example_input_file815.txt +example_input_file816.txt +example_input_file817.txt +example_input_file818.txt +example_input_file819.txt +example_input_file820.txt +example_input_file821.txt +example_input_file822.txt +example_input_file823.txt +example_input_file824.txt +example_input_file825.txt +example_input_file826.txt +example_input_file827.txt +example_input_file828.txt +example_input_file829.txt +example_input_file830.txt +example_input_file831.txt +example_input_file832.txt +example_input_file833.txt +example_input_file834.txt +example_input_file835.txt +example_input_file836.txt +example_input_file837.txt +example_input_file838.txt +example_input_file839.txt +example_input_file840.txt +example_input_file841.txt +example_input_file842.txt +example_input_file843.txt +example_input_file844.txt +example_input_file845.txt +example_input_file846.txt +example_input_file847.txt +example_input_file848.txt +example_input_file849.txt +example_input_file850.txt +example_input_file851.txt +example_input_file852.txt +example_input_file853.txt +example_input_file854.txt +example_input_file855.txt +example_input_file856.txt +example_input_file857.txt +example_input_file858.txt +example_input_file859.txt +example_input_file860.txt +example_input_file861.txt +example_input_file862.txt +example_input_file863.txt +example_input_file864.txt +example_input_file865.txt +example_input_file866.txt +example_input_file867.txt +example_input_file868.txt +example_input_file869.txt +example_input_file870.txt +example_input_file871.txt +example_input_file872.txt +example_input_file873.txt +example_input_file874.txt +example_input_file875.txt +example_input_file876.txt +example_input_file877.txt +example_input_file878.txt +example_input_file879.txt +example_input_file880.txt +example_input_file881.txt +example_input_file882.txt +example_input_file883.txt +example_input_file884.txt +example_input_file885.txt +example_input_file886.txt +example_input_file887.txt +example_input_file888.txt +example_input_file889.txt +example_input_file890.txt +example_input_file891.txt +example_input_file892.txt +example_input_file893.txt +example_input_file894.txt +example_input_file895.txt +example_input_file896.txt +example_input_file897.txt +example_input_file898.txt +example_input_file899.txt +example_input_file900.txt +example_input_file901.txt +example_input_file902.txt +example_input_file903.txt +example_input_file904.txt +example_input_file905.txt +example_input_file906.txt +example_input_file907.txt +example_input_file908.txt +example_input_file909.txt +example_input_file910.txt +example_input_file911.txt +example_input_file912.txt +example_input_file913.txt +example_input_file914.txt +example_input_file915.txt +example_input_file916.txt +example_input_file917.txt +example_input_file918.txt +example_input_file919.txt +example_input_file920.txt +example_input_file921.txt +example_input_file922.txt +example_input_file923.txt +example_input_file924.txt +example_input_file925.txt +example_input_file926.txt +example_input_file927.txt +example_input_file928.txt +example_input_file929.txt +example_input_file930.txt +example_input_file931.txt +example_input_file932.txt +example_input_file933.txt +example_input_file934.txt +example_input_file935.txt +example_input_file936.txt +example_input_file937.txt +example_input_file938.txt +example_input_file939.txt +example_input_file940.txt +example_input_file941.txt +example_input_file942.txt +example_input_file943.txt +example_input_file944.txt +example_input_file945.txt +example_input_file946.txt +example_input_file947.txt +example_input_file948.txt +example_input_file949.txt +example_input_file950.txt +example_input_file951.txt +example_input_file952.txt +example_input_file953.txt +example_input_file954.txt +example_input_file955.txt +example_input_file956.txt +example_input_file957.txt +example_input_file958.txt +example_input_file959.txt +example_input_file960.txt +example_input_file961.txt +example_input_file962.txt +example_input_file963.txt +example_input_file964.txt +example_input_file965.txt +example_input_file966.txt +example_input_file967.txt +example_input_file968.txt +example_input_file969.txt +example_input_file970.txt +example_input_file971.txt +example_input_file972.txt +example_input_file973.txt +example_input_file974.txt +example_input_file975.txt +example_input_file976.txt +example_input_file977.txt +example_input_file978.txt +example_input_file979.txt +example_input_file980.txt +example_input_file981.txt +example_input_file982.txt +example_input_file983.txt +example_input_file984.txt +example_input_file985.txt +example_input_file986.txt +example_input_file987.txt +example_input_file988.txt +example_input_file989.txt +example_input_file990.txt +example_input_file991.txt +example_input_file992.txt +example_input_file993.txt +example_input_file994.txt +example_input_file995.txt +example_input_file996.txt +example_input_file997.txt +example_input_file998.txt +example_input_file999.txt +example_input_file1000.txt +example_input_file1001.txt +example_input_file1002.txt +example_input_file1003.txt +example_input_file1004.txt +example_input_file1005.txt +example_input_file1006.txt +example_input_file1007.txt +example_input_file1008.txt +example_input_file1009.txt +example_input_file1010.txt +example_input_file1011.txt +example_input_file1012.txt +example_input_file1013.txt +example_input_file1014.txt +example_input_file1015.txt +example_input_file1016.txt +example_input_file1017.txt +example_input_file1018.txt +example_input_file1019.txt +example_input_file1020.txt +example_input_file1021.txt +example_input_file1022.txt +example_input_file1023.txt +example_input_file1024.txt +example_input_file1025.txt +example_input_file1026.txt +example_input_file1027.txt +example_input_file1028.txt +example_input_file1029.txt +example_input_file1030.txt +example_input_file1031.txt +example_input_file1032.txt +example_input_file1033.txt +example_input_file1034.txt +example_input_file1035.txt +example_input_file1036.txt +example_input_file1037.txt +example_input_file1038.txt +example_input_file1039.txt +example_input_file1040.txt +example_input_file1041.txt +example_input_file1042.txt +example_input_file1043.txt +example_input_file1044.txt +example_input_file1045.txt +example_input_file1046.txt +example_input_file1047.txt +example_input_file1048.txt +example_input_file1049.txt +example_input_file1050.txt +example_input_file1051.txt +example_input_file1052.txt +example_input_file1053.txt +example_input_file1054.txt +example_input_file1055.txt +example_input_file1056.txt +example_input_file1057.txt +example_input_file1058.txt +example_input_file1059.txt +example_input_file1060.txt +example_input_file1061.txt +example_input_file1062.txt +example_input_file1063.txt +example_input_file1064.txt +example_input_file1065.txt +example_input_file1066.txt +example_input_file1067.txt +example_input_file1068.txt +example_input_file1069.txt +example_input_file1070.txt +example_input_file1071.txt +example_input_file1072.txt +example_input_file1073.txt +example_input_file1074.txt +example_input_file1075.txt +example_input_file1076.txt +example_input_file1077.txt +example_input_file1078.txt +example_input_file1079.txt +example_input_file1080.txt +example_input_file1081.txt +example_input_file1082.txt +example_input_file1083.txt +example_input_file1084.txt +example_input_file1085.txt +example_input_file1086.txt +example_input_file1087.txt +example_input_file1088.txt +example_input_file1089.txt +example_input_file1090.txt +example_input_file1091.txt +example_input_file1092.txt +example_input_file1093.txt +example_input_file1094.txt +example_input_file1095.txt +example_input_file1096.txt +example_input_file1097.txt +example_input_file1098.txt +example_input_file1099.txt +example_input_file1100.txt +example_input_file1101.txt +example_input_file1102.txt +example_input_file1103.txt +example_input_file1104.txt +example_input_file1105.txt +example_input_file1106.txt +example_input_file1107.txt +example_input_file1108.txt +example_input_file1109.txt +example_input_file1110.txt +example_input_file1111.txt +example_input_file1112.txt +example_input_file1113.txt +example_input_file1114.txt +example_input_file1115.txt +example_input_file1116.txt +example_input_file1117.txt +example_input_file1118.txt +example_input_file1119.txt +example_input_file1120.txt +example_input_file1121.txt +example_input_file1122.txt +example_input_file1123.txt +example_input_file1124.txt +example_input_file1125.txt +example_input_file1126.txt +example_input_file1127.txt +example_input_file1128.txt +example_input_file1129.txt +example_input_file1130.txt +example_input_file1131.txt +example_input_file1132.txt +example_input_file1133.txt +example_input_file1134.txt +example_input_file1135.txt +example_input_file1136.txt +example_input_file1137.txt +example_input_file1138.txt +example_input_file1139.txt +example_input_file1140.txt +example_input_file1141.txt +example_input_file1142.txt +example_input_file1143.txt +example_input_file1144.txt +example_input_file1145.txt +example_input_file1146.txt +example_input_file1147.txt +example_input_file1148.txt +example_input_file1149.txt +example_input_file1150.txt +example_input_file1151.txt +example_input_file1152.txt +example_input_file1153.txt +example_input_file1154.txt +example_input_file1155.txt +example_input_file1156.txt +example_input_file1157.txt +example_input_file1158.txt +example_input_file1159.txt +example_input_file1160.txt +example_input_file1161.txt +example_input_file1162.txt +example_input_file1163.txt +example_input_file1164.txt +example_input_file1165.txt +example_input_file1166.txt +example_input_file1167.txt +example_input_file1168.txt +example_input_file1169.txt +example_input_file1170.txt +example_input_file1171.txt +example_input_file1172.txt +example_input_file1173.txt +example_input_file1174.txt +example_input_file1175.txt +example_input_file1176.txt +example_input_file1177.txt +example_input_file1178.txt +example_input_file1179.txt +example_input_file1180.txt +example_input_file1181.txt +example_input_file1182.txt +example_input_file1183.txt +example_input_file1184.txt +example_input_file1185.txt +example_input_file1186.txt +example_input_file1187.txt +example_input_file1188.txt +example_input_file1189.txt +example_input_file1190.txt +example_input_file1191.txt +example_input_file1192.txt +example_input_file1193.txt +example_input_file1194.txt +example_input_file1195.txt +example_input_file1196.txt +example_input_file1197.txt +example_input_file1198.txt +example_input_file1199.txt +example_input_file1200.txt +example_input_file1201.txt +example_input_file1202.txt +example_input_file1203.txt +example_input_file1204.txt +example_input_file1205.txt +example_input_file1206.txt +example_input_file1207.txt +example_input_file1208.txt +example_input_file1209.txt +example_input_file1210.txt +example_input_file1211.txt +example_input_file1212.txt +example_input_file1213.txt +example_input_file1214.txt +example_input_file1215.txt +example_input_file1216.txt +example_input_file1217.txt +example_input_file1218.txt +example_input_file1219.txt +example_input_file1220.txt +example_input_file1221.txt +example_input_file1222.txt +example_input_file1223.txt +example_input_file1224.txt +example_input_file1225.txt +example_input_file1226.txt +example_input_file1227.txt +example_input_file1228.txt +example_input_file1229.txt +example_input_file1230.txt +example_input_file1231.txt +example_input_file1232.txt +example_input_file1233.txt +example_input_file1234.txt +example_input_file1235.txt +example_input_file1236.txt +example_input_file1237.txt +example_input_file1238.txt +example_input_file1239.txt +example_input_file1240.txt +example_input_file1241.txt +example_input_file1242.txt +example_input_file1243.txt +example_input_file1244.txt +example_input_file1245.txt +example_input_file1246.txt +example_input_file1247.txt +example_input_file1248.txt +example_input_file1249.txt +example_input_file1250.txt +example_input_file1251.txt +example_input_file1252.txt +example_input_file1253.txt +example_input_file1254.txt +example_input_file1255.txt +example_input_file1256.txt +example_input_file1257.txt +example_input_file1258.txt +example_input_file1259.txt +example_input_file1260.txt +example_input_file1261.txt +example_input_file1262.txt +example_input_file1263.txt +example_input_file1264.txt +example_input_file1265.txt +example_input_file1266.txt +example_input_file1267.txt +example_input_file1268.txt +example_input_file1269.txt +example_input_file1270.txt +example_input_file1271.txt +example_input_file1272.txt +example_input_file1273.txt +example_input_file1274.txt +example_input_file1275.txt +example_input_file1276.txt +example_input_file1277.txt +example_input_file1278.txt +example_input_file1279.txt +example_input_file1280.txt +example_input_file1281.txt +example_input_file1282.txt +example_input_file1283.txt +example_input_file1284.txt +example_input_file1285.txt +example_input_file1286.txt +example_input_file1287.txt +example_input_file1288.txt +example_input_file1289.txt +example_input_file1290.txt +example_input_file1291.txt +example_input_file1292.txt +example_input_file1293.txt +example_input_file1294.txt +example_input_file1295.txt +example_input_file1296.txt +example_input_file1297.txt +example_input_file1298.txt +example_input_file1299.txt +example_input_file1300.txt +example_input_file1301.txt +example_input_file1302.txt +example_input_file1303.txt +example_input_file1304.txt +example_input_file1305.txt +example_input_file1306.txt +example_input_file1307.txt +example_input_file1308.txt +example_input_file1309.txt +example_input_file1310.txt +example_input_file1311.txt +example_input_file1312.txt +example_input_file1313.txt +example_input_file1314.txt +example_input_file1315.txt +example_input_file1316.txt +example_input_file1317.txt +example_input_file1318.txt +example_input_file1319.txt +example_input_file1320.txt +example_input_file1321.txt +example_input_file1322.txt +example_input_file1323.txt +example_input_file1324.txt +example_input_file1325.txt +example_input_file1326.txt +example_input_file1327.txt +example_input_file1328.txt +example_input_file1329.txt +example_input_file1330.txt +example_input_file1331.txt +example_input_file1332.txt +example_input_file1333.txt +example_input_file1334.txt +example_input_file1335.txt +example_input_file1336.txt +example_input_file1337.txt +example_input_file1338.txt +example_input_file1339.txt +example_input_file1340.txt +example_input_file1341.txt +example_input_file1342.txt +example_input_file1343.txt +example_input_file1344.txt +example_input_file1345.txt +example_input_file1346.txt +example_input_file1347.txt +example_input_file1348.txt +example_input_file1349.txt +example_input_file1350.txt +example_input_file1351.txt +example_input_file1352.txt +example_input_file1353.txt +example_input_file1354.txt +example_input_file1355.txt +example_input_file1356.txt +example_input_file1357.txt +example_input_file1358.txt +example_input_file1359.txt +example_input_file1360.txt +example_input_file1361.txt +example_input_file1362.txt +example_input_file1363.txt +example_input_file1364.txt +example_input_file1365.txt +example_input_file1366.txt +example_input_file1367.txt +example_input_file1368.txt +example_input_file1369.txt +example_input_file1370.txt +example_input_file1371.txt +example_input_file1372.txt +example_input_file1373.txt +example_input_file1374.txt +example_input_file1375.txt +example_input_file1376.txt +example_input_file1377.txt +example_input_file1378.txt +example_input_file1379.txt +example_input_file1380.txt +example_input_file1381.txt +example_input_file1382.txt +example_input_file1383.txt +example_input_file1384.txt +example_input_file1385.txt +example_input_file1386.txt +example_input_file1387.txt +example_input_file1388.txt +example_input_file1389.txt +example_input_file1390.txt +example_input_file1391.txt +example_input_file1392.txt +example_input_file1393.txt +example_input_file1394.txt +example_input_file1395.txt +example_input_file1396.txt +example_input_file1397.txt +example_input_file1398.txt +example_input_file1399.txt +example_input_file1400.txt +example_input_file1401.txt +example_input_file1402.txt +example_input_file1403.txt +example_input_file1404.txt +example_input_file1405.txt +example_input_file1406.txt +example_input_file1407.txt +example_input_file1408.txt +example_input_file1409.txt +example_input_file1410.txt +example_input_file1411.txt +example_input_file1412.txt +example_input_file1413.txt +example_input_file1414.txt +example_input_file1415.txt +example_input_file1416.txt +example_input_file1417.txt +example_input_file1418.txt +example_input_file1419.txt +example_input_file1420.txt +example_input_file1421.txt +example_input_file1422.txt +example_input_file1423.txt +example_input_file1424.txt +example_input_file1425.txt +example_input_file1426.txt +example_input_file1427.txt +example_input_file1428.txt +example_input_file1429.txt +example_input_file1430.txt +example_input_file1431.txt +example_input_file1432.txt +example_input_file1433.txt +example_input_file1434.txt +example_input_file1435.txt +example_input_file1436.txt +example_input_file1437.txt +example_input_file1438.txt +example_input_file1439.txt +example_input_file1440.txt +example_input_file1441.txt +example_input_file1442.txt +example_input_file1443.txt +example_input_file1444.txt +example_input_file1445.txt +example_input_file1446.txt +example_input_file1447.txt +example_input_file1448.txt +example_input_file1449.txt +example_input_file1450.txt +example_input_file1451.txt +example_input_file1452.txt +example_input_file1453.txt +example_input_file1454.txt +example_input_file1455.txt +example_input_file1456.txt +example_input_file1457.txt +example_input_file1458.txt +example_input_file1459.txt +example_input_file1460.txt +example_input_file1461.txt +example_input_file1462.txt +example_input_file1463.txt +example_input_file1464.txt +example_input_file1465.txt +example_input_file1466.txt +example_input_file1467.txt +example_input_file1468.txt +example_input_file1469.txt +example_input_file1470.txt +example_input_file1471.txt +example_input_file1472.txt +example_input_file1473.txt +example_input_file1474.txt +example_input_file1475.txt +example_input_file1476.txt +example_input_file1477.txt +example_input_file1478.txt +example_input_file1479.txt +example_input_file1480.txt +example_input_file1481.txt +example_input_file1482.txt +example_input_file1483.txt +example_input_file1484.txt +example_input_file1485.txt +example_input_file1486.txt +example_input_file1487.txt +example_input_file1488.txt +example_input_file1489.txt +example_input_file1490.txt +example_input_file1491.txt +example_input_file1492.txt +example_input_file1493.txt +example_input_file1494.txt +example_input_file1495.txt +example_input_file1496.txt +example_input_file1497.txt +example_input_file1498.txt +example_input_file1499.txt +example_input_file1500.txt +example_input_file1501.txt +example_input_file1502.txt +example_input_file1503.txt +example_input_file1504.txt +example_input_file1505.txt +example_input_file1506.txt +example_input_file1507.txt +example_input_file1508.txt +example_input_file1509.txt +example_input_file1510.txt +example_input_file1511.txt +example_input_file1512.txt +example_input_file1513.txt +example_input_file1514.txt +example_input_file1515.txt +example_input_file1516.txt +example_input_file1517.txt +example_input_file1518.txt +example_input_file1519.txt +example_input_file1520.txt +example_input_file1521.txt +example_input_file1522.txt +example_input_file1523.txt +example_input_file1524.txt +example_input_file1525.txt +example_input_file1526.txt +example_input_file1527.txt +example_input_file1528.txt +example_input_file1529.txt +example_input_file1530.txt +example_input_file1531.txt +example_input_file1532.txt +example_input_file1533.txt +example_input_file1534.txt +example_input_file1535.txt +example_input_file1536.txt +example_input_file1537.txt +example_input_file1538.txt +example_input_file1539.txt +example_input_file1540.txt +example_input_file1541.txt +example_input_file1542.txt +example_input_file1543.txt +example_input_file1544.txt +example_input_file1545.txt +example_input_file1546.txt +example_input_file1547.txt +example_input_file1548.txt +example_input_file1549.txt +example_input_file1550.txt +example_input_file1551.txt +example_input_file1552.txt +example_input_file1553.txt +example_input_file1554.txt +example_input_file1555.txt +example_input_file1556.txt +example_input_file1557.txt +example_input_file1558.txt +example_input_file1559.txt +example_input_file1560.txt +example_input_file1561.txt +example_input_file1562.txt +example_input_file1563.txt +example_input_file1564.txt +example_input_file1565.txt +example_input_file1566.txt +example_input_file1567.txt +example_input_file1568.txt +example_input_file1569.txt +example_input_file1570.txt +example_input_file1571.txt +example_input_file1572.txt +example_input_file1573.txt +example_input_file1574.txt +example_input_file1575.txt +example_input_file1576.txt +example_input_file1577.txt +example_input_file1578.txt +example_input_file1579.txt +example_input_file1580.txt +example_input_file1581.txt +example_input_file1582.txt +example_input_file1583.txt +example_input_file1584.txt +example_input_file1585.txt +example_input_file1586.txt +example_input_file1587.txt +example_input_file1588.txt +example_input_file1589.txt +example_input_file1590.txt +example_input_file1591.txt +example_input_file1592.txt +example_input_file1593.txt +example_input_file1594.txt +example_input_file1595.txt +example_input_file1596.txt +example_input_file1597.txt +example_input_file1598.txt +example_input_file1599.txt +example_input_file1600.txt +example_input_file1601.txt +example_input_file1602.txt +example_input_file1603.txt +example_input_file1604.txt +example_input_file1605.txt +example_input_file1606.txt +example_input_file1607.txt +example_input_file1608.txt +example_input_file1609.txt +example_input_file1610.txt +example_input_file1611.txt +example_input_file1612.txt +example_input_file1613.txt +example_input_file1614.txt +example_input_file1615.txt +example_input_file1616.txt +example_input_file1617.txt +example_input_file1618.txt +example_input_file1619.txt +example_input_file1620.txt +example_input_file1621.txt +example_input_file1622.txt +example_input_file1623.txt +example_input_file1624.txt +example_input_file1625.txt +example_input_file1626.txt +example_input_file1627.txt +example_input_file1628.txt +example_input_file1629.txt +example_input_file1630.txt +example_input_file1631.txt +example_input_file1632.txt +example_input_file1633.txt +example_input_file1634.txt +example_input_file1635.txt +example_input_file1636.txt +example_input_file1637.txt +example_input_file1638.txt +example_input_file1639.txt +example_input_file1640.txt +example_input_file1641.txt +example_input_file1642.txt +example_input_file1643.txt +example_input_file1644.txt +example_input_file1645.txt +example_input_file1646.txt +example_input_file1647.txt +example_input_file1648.txt +example_input_file1649.txt +example_input_file1650.txt +example_input_file1651.txt +example_input_file1652.txt +example_input_file1653.txt +example_input_file1654.txt +example_input_file1655.txt +example_input_file1656.txt +example_input_file1657.txt +example_input_file1658.txt +example_input_file1659.txt +example_input_file1660.txt +example_input_file1661.txt +example_input_file1662.txt +example_input_file1663.txt +example_input_file1664.txt +example_input_file1665.txt +example_input_file1666.txt +example_input_file1667.txt +example_input_file1668.txt +example_input_file1669.txt +example_input_file1670.txt +example_input_file1671.txt +example_input_file1672.txt +example_input_file1673.txt +example_input_file1674.txt +example_input_file1675.txt +example_input_file1676.txt +example_input_file1677.txt +example_input_file1678.txt +example_input_file1679.txt +example_input_file1680.txt +example_input_file1681.txt +example_input_file1682.txt +example_input_file1683.txt +example_input_file1684.txt +example_input_file1685.txt +example_input_file1686.txt +example_input_file1687.txt +example_input_file1688.txt +example_input_file1689.txt +example_input_file1690.txt +example_input_file1691.txt +example_input_file1692.txt +example_input_file1693.txt +example_input_file1694.txt +example_input_file1695.txt +example_input_file1696.txt +example_input_file1697.txt +example_input_file1698.txt +example_input_file1699.txt +example_input_file1700.txt +example_input_file1701.txt +example_input_file1702.txt +example_input_file1703.txt +example_input_file1704.txt +example_input_file1705.txt +example_input_file1706.txt +example_input_file1707.txt +example_input_file1708.txt +example_input_file1709.txt +example_input_file1710.txt +example_input_file1711.txt +example_input_file1712.txt +example_input_file1713.txt +example_input_file1714.txt +example_input_file1715.txt +example_input_file1716.txt +example_input_file1717.txt +example_input_file1718.txt +example_input_file1719.txt +example_input_file1720.txt +example_input_file1721.txt +example_input_file1722.txt +example_input_file1723.txt +example_input_file1724.txt +example_input_file1725.txt +example_input_file1726.txt +example_input_file1727.txt +example_input_file1728.txt +example_input_file1729.txt +example_input_file1730.txt +example_input_file1731.txt +example_input_file1732.txt +example_input_file1733.txt +example_input_file1734.txt +example_input_file1735.txt +example_input_file1736.txt +example_input_file1737.txt +example_input_file1738.txt +example_input_file1739.txt +example_input_file1740.txt +example_input_file1741.txt +example_input_file1742.txt +example_input_file1743.txt +example_input_file1744.txt +example_input_file1745.txt +example_input_file1746.txt +example_input_file1747.txt +example_input_file1748.txt +example_input_file1749.txt +example_input_file1750.txt +example_input_file1751.txt +example_input_file1752.txt +example_input_file1753.txt +example_input_file1754.txt +example_input_file1755.txt +example_input_file1756.txt +example_input_file1757.txt +example_input_file1758.txt +example_input_file1759.txt +example_input_file1760.txt +example_input_file1761.txt +example_input_file1762.txt +example_input_file1763.txt +example_input_file1764.txt +example_input_file1765.txt +example_input_file1766.txt +example_input_file1767.txt +example_input_file1768.txt +example_input_file1769.txt +example_input_file1770.txt +example_input_file1771.txt +example_input_file1772.txt +example_input_file1773.txt +example_input_file1774.txt +example_input_file1775.txt +example_input_file1776.txt +example_input_file1777.txt +example_input_file1778.txt +example_input_file1779.txt +example_input_file1780.txt +example_input_file1781.txt +example_input_file1782.txt +example_input_file1783.txt +example_input_file1784.txt +example_input_file1785.txt +example_input_file1786.txt +example_input_file1787.txt +example_input_file1788.txt +example_input_file1789.txt +example_input_file1790.txt +example_input_file1791.txt +example_input_file1792.txt +example_input_file1793.txt +example_input_file1794.txt +example_input_file1795.txt +example_input_file1796.txt +example_input_file1797.txt +example_input_file1798.txt +example_input_file1799.txt +example_input_file1800.txt +example_input_file1801.txt +example_input_file1802.txt +example_input_file1803.txt +example_input_file1804.txt +example_input_file1805.txt +example_input_file1806.txt +example_input_file1807.txt +example_input_file1808.txt +example_input_file1809.txt +example_input_file1810.txt +example_input_file1811.txt +example_input_file1812.txt +example_input_file1813.txt +example_input_file1814.txt +example_input_file1815.txt +example_input_file1816.txt +example_input_file1817.txt +example_input_file1818.txt +example_input_file1819.txt +example_input_file1820.txt +example_input_file1821.txt +example_input_file1822.txt +example_input_file1823.txt +example_input_file1824.txt +example_input_file1825.txt +example_input_file1826.txt +example_input_file1827.txt +example_input_file1828.txt +example_input_file1829.txt +example_input_file1830.txt +example_input_file1831.txt +example_input_file1832.txt +example_input_file1833.txt +example_input_file1834.txt +example_input_file1835.txt +example_input_file1836.txt +example_input_file1837.txt +example_input_file1838.txt +example_input_file1839.txt +example_input_file1840.txt +example_input_file1841.txt +example_input_file1842.txt +example_input_file1843.txt +example_input_file1844.txt +example_input_file1845.txt +example_input_file1846.txt +example_input_file1847.txt +example_input_file1848.txt +example_input_file1849.txt +example_input_file1850.txt +example_input_file1851.txt +example_input_file1852.txt +example_input_file1853.txt +example_input_file1854.txt +example_input_file1855.txt +example_input_file1856.txt +example_input_file1857.txt +example_input_file1858.txt +example_input_file1859.txt +example_input_file1860.txt +example_input_file1861.txt +example_input_file1862.txt +example_input_file1863.txt +example_input_file1864.txt +example_input_file1865.txt +example_input_file1866.txt +example_input_file1867.txt +example_input_file1868.txt +example_input_file1869.txt +example_input_file1870.txt +example_input_file1871.txt +example_input_file1872.txt +example_input_file1873.txt +example_input_file1874.txt +example_input_file1875.txt +example_input_file1876.txt +example_input_file1877.txt +example_input_file1878.txt +example_input_file1879.txt +example_input_file1880.txt +example_input_file1881.txt +example_input_file1882.txt +example_input_file1883.txt +example_input_file1884.txt +example_input_file1885.txt +example_input_file1886.txt +example_input_file1887.txt +example_input_file1888.txt +example_input_file1889.txt +example_input_file1890.txt +example_input_file1891.txt +example_input_file1892.txt +example_input_file1893.txt +example_input_file1894.txt +example_input_file1895.txt +example_input_file1896.txt +example_input_file1897.txt +example_input_file1898.txt +example_input_file1899.txt +example_input_file1900.txt +example_input_file1901.txt +example_input_file1902.txt +example_input_file1903.txt +example_input_file1904.txt +example_input_file1905.txt +example_input_file1906.txt +example_input_file1907.txt +example_input_file1908.txt +example_input_file1909.txt +example_input_file1910.txt +example_input_file1911.txt +example_input_file1912.txt +example_input_file1913.txt +example_input_file1914.txt +example_input_file1915.txt +example_input_file1916.txt +example_input_file1917.txt +example_input_file1918.txt +example_input_file1919.txt +example_input_file1920.txt +example_input_file1921.txt +example_input_file1922.txt +example_input_file1923.txt +example_input_file1924.txt +example_input_file1925.txt +example_input_file1926.txt +example_input_file1927.txt +example_input_file1928.txt +example_input_file1929.txt +example_input_file1930.txt +example_input_file1931.txt +example_input_file1932.txt +example_input_file1933.txt +example_input_file1934.txt +example_input_file1935.txt +example_input_file1936.txt +example_input_file1937.txt +example_input_file1938.txt +example_input_file1939.txt +example_input_file1940.txt +example_input_file1941.txt +example_input_file1942.txt +example_input_file1943.txt +example_input_file1944.txt +example_input_file1945.txt +example_input_file1946.txt +example_input_file1947.txt +example_input_file1948.txt +example_input_file1949.txt +example_input_file1950.txt +example_input_file1951.txt +example_input_file1952.txt +example_input_file1953.txt +example_input_file1954.txt +example_input_file1955.txt +example_input_file1956.txt +example_input_file1957.txt +example_input_file1958.txt +example_input_file1959.txt +example_input_file1960.txt +example_input_file1961.txt +example_input_file1962.txt +example_input_file1963.txt +example_input_file1964.txt +example_input_file1965.txt +example_input_file1966.txt +example_input_file1967.txt +example_input_file1968.txt +example_input_file1969.txt +example_input_file1970.txt +example_input_file1971.txt +example_input_file1972.txt +example_input_file1973.txt +example_input_file1974.txt +example_input_file1975.txt +example_input_file1976.txt +example_input_file1977.txt +example_input_file1978.txt +example_input_file1979.txt +example_input_file1980.txt +example_input_file1981.txt +example_input_file1982.txt +example_input_file1983.txt +example_input_file1984.txt +example_input_file1985.txt +example_input_file1986.txt +example_input_file1987.txt +example_input_file1988.txt +example_input_file1989.txt +example_input_file1990.txt +example_input_file1991.txt +example_input_file1992.txt +example_input_file1993.txt +example_input_file1994.txt +example_input_file1995.txt +example_input_file1996.txt +example_input_file1997.txt +example_input_file1998.txt +example_input_file1999.txt +example_input_file2000.txt +example_input_file2001.txt +example_input_file2002.txt +example_input_file2003.txt +example_input_file2004.txt +example_input_file2005.txt +example_input_file2006.txt +example_input_file2007.txt +example_input_file2008.txt +example_input_file2009.txt +example_input_file2010.txt +example_input_file2011.txt +example_input_file2012.txt +example_input_file2013.txt +example_input_file2014.txt +example_input_file2015.txt +example_input_file2016.txt +example_input_file2017.txt +example_input_file2018.txt +example_input_file2019.txt +example_input_file2020.txt +example_input_file2021.txt +example_input_file2022.txt +example_input_file2023.txt +example_input_file2024.txt +example_input_file2025.txt +example_input_file2026.txt +example_input_file2027.txt +example_input_file2028.txt +example_input_file2029.txt +example_input_file2030.txt +example_input_file2031.txt +example_input_file2032.txt +example_input_file2033.txt +example_input_file2034.txt +example_input_file2035.txt +example_input_file2036.txt +example_input_file2037.txt +example_input_file2038.txt +example_input_file2039.txt +example_input_file2040.txt +example_input_file2041.txt +example_input_file2042.txt +example_input_file2043.txt +example_input_file2044.txt +example_input_file2045.txt +example_input_file2046.txt +example_input_file2047.txt +example_input_file2048.txt +example_input_file2049.txt +example_input_file2050.txt +example_input_file2051.txt +example_input_file2052.txt +example_input_file2053.txt +example_input_file2054.txt +example_input_file2055.txt +example_input_file2056.txt +example_input_file2057.txt +example_input_file2058.txt +example_input_file2059.txt +example_input_file2060.txt +example_input_file2061.txt +example_input_file2062.txt +example_input_file2063.txt +example_input_file2064.txt +example_input_file2065.txt +example_input_file2066.txt +example_input_file2067.txt +example_input_file2068.txt +example_input_file2069.txt +example_input_file2070.txt +example_input_file2071.txt +example_input_file2072.txt +example_input_file2073.txt +example_input_file2074.txt +example_input_file2075.txt +example_input_file2076.txt +example_input_file2077.txt +example_input_file2078.txt +example_input_file2079.txt +example_input_file2080.txt +example_input_file2081.txt +example_input_file2082.txt +example_input_file2083.txt +example_input_file2084.txt +example_input_file2085.txt +example_input_file2086.txt +example_input_file2087.txt +example_input_file2088.txt +example_input_file2089.txt +example_input_file2090.txt +example_input_file2091.txt +example_input_file2092.txt +example_input_file2093.txt +example_input_file2094.txt +example_input_file2095.txt +example_input_file2096.txt +example_input_file2097.txt +example_input_file2098.txt +example_input_file2099.txt +example_input_file2100.txt +example_input_file2101.txt +example_input_file2102.txt +example_input_file2103.txt +example_input_file2104.txt +example_input_file2105.txt +example_input_file2106.txt +example_input_file2107.txt +example_input_file2108.txt +example_input_file2109.txt +example_input_file2110.txt +example_input_file2111.txt +example_input_file2112.txt +example_input_file2113.txt +example_input_file2114.txt +example_input_file2115.txt +example_input_file2116.txt +example_input_file2117.txt +example_input_file2118.txt +example_input_file2119.txt +example_input_file2120.txt +example_input_file2121.txt +example_input_file2122.txt +example_input_file2123.txt +example_input_file2124.txt +example_input_file2125.txt +example_input_file2126.txt +example_input_file2127.txt +example_input_file2128.txt +example_input_file2129.txt +example_input_file2130.txt +example_input_file2131.txt +example_input_file2132.txt +example_input_file2133.txt +example_input_file2134.txt +example_input_file2135.txt +example_input_file2136.txt +example_input_file2137.txt +example_input_file2138.txt +example_input_file2139.txt +example_input_file2140.txt +example_input_file2141.txt +example_input_file2142.txt +example_input_file2143.txt +example_input_file2144.txt +example_input_file2145.txt +example_input_file2146.txt +example_input_file2147.txt +example_input_file2148.txt +example_input_file2149.txt +example_input_file2150.txt +example_input_file2151.txt +example_input_file2152.txt +example_input_file2153.txt +example_input_file2154.txt +example_input_file2155.txt +example_input_file2156.txt +example_input_file2157.txt +example_input_file2158.txt +example_input_file2159.txt +example_input_file2160.txt +example_input_file2161.txt +example_input_file2162.txt +example_input_file2163.txt +example_input_file2164.txt +example_input_file2165.txt +example_input_file2166.txt +example_input_file2167.txt +example_input_file2168.txt +example_input_file2169.txt +example_input_file2170.txt +example_input_file2171.txt +example_input_file2172.txt +example_input_file2173.txt +example_input_file2174.txt +example_input_file2175.txt +example_input_file2176.txt +example_input_file2177.txt +example_input_file2178.txt +example_input_file2179.txt +example_input_file2180.txt +example_input_file2181.txt +example_input_file2182.txt +example_input_file2183.txt +example_input_file2184.txt +example_input_file2185.txt +example_input_file2186.txt +example_input_file2187.txt +example_input_file2188.txt +example_input_file2189.txt +example_input_file2190.txt +example_input_file2191.txt +example_input_file2192.txt +example_input_file2193.txt +example_input_file2194.txt +example_input_file2195.txt +example_input_file2196.txt +example_input_file2197.txt +example_input_file2198.txt +example_input_file2199.txt +example_input_file2200.txt +example_input_file2201.txt +example_input_file2202.txt +example_input_file2203.txt +example_input_file2204.txt +example_input_file2205.txt +example_input_file2206.txt +example_input_file2207.txt +example_input_file2208.txt +example_input_file2209.txt +example_input_file2210.txt +example_input_file2211.txt +example_input_file2212.txt +example_input_file2213.txt +example_input_file2214.txt +example_input_file2215.txt +example_input_file2216.txt +example_input_file2217.txt +example_input_file2218.txt +example_input_file2219.txt +example_input_file2220.txt +example_input_file2221.txt +example_input_file2222.txt +example_input_file2223.txt +example_input_file2224.txt +example_input_file2225.txt +example_input_file2226.txt +example_input_file2227.txt +example_input_file2228.txt +example_input_file2229.txt +example_input_file2230.txt +example_input_file2231.txt +example_input_file2232.txt +example_input_file2233.txt +example_input_file2234.txt +example_input_file2235.txt +example_input_file2236.txt +example_input_file2237.txt +example_input_file2238.txt +example_input_file2239.txt +example_input_file2240.txt +example_input_file2241.txt +example_input_file2242.txt +example_input_file2243.txt +example_input_file2244.txt +example_input_file2245.txt +example_input_file2246.txt +example_input_file2247.txt +example_input_file2248.txt +example_input_file2249.txt +example_input_file2250.txt +example_input_file2251.txt +example_input_file2252.txt +example_input_file2253.txt +example_input_file2254.txt +example_input_file2255.txt +example_input_file2256.txt +example_input_file2257.txt +example_input_file2258.txt +example_input_file2259.txt +example_input_file2260.txt +example_input_file2261.txt +example_input_file2262.txt +example_input_file2263.txt +example_input_file2264.txt +example_input_file2265.txt +example_input_file2266.txt +example_input_file2267.txt +example_input_file2268.txt +example_input_file2269.txt +example_input_file2270.txt +example_input_file2271.txt +example_input_file2272.txt +example_input_file2273.txt +example_input_file2274.txt +example_input_file2275.txt +example_input_file2276.txt +example_input_file2277.txt +example_input_file2278.txt +example_input_file2279.txt +example_input_file2280.txt +example_input_file2281.txt +example_input_file2282.txt +example_input_file2283.txt +example_input_file2284.txt +example_input_file2285.txt +example_input_file2286.txt +example_input_file2287.txt +example_input_file2288.txt +example_input_file2289.txt +example_input_file2290.txt +example_input_file2291.txt +example_input_file2292.txt +example_input_file2293.txt +example_input_file2294.txt +example_input_file2295.txt +example_input_file2296.txt +example_input_file2297.txt +example_input_file2298.txt +example_input_file2299.txt +example_input_file2300.txt +example_input_file2301.txt +example_input_file2302.txt +example_input_file2303.txt +example_input_file2304.txt +example_input_file2305.txt +example_input_file2306.txt +example_input_file2307.txt +example_input_file2308.txt +example_input_file2309.txt +example_input_file2310.txt +example_input_file2311.txt +example_input_file2312.txt +example_input_file2313.txt +example_input_file2314.txt +example_input_file2315.txt +example_input_file2316.txt +example_input_file2317.txt +example_input_file2318.txt +example_input_file2319.txt +example_input_file2320.txt +example_input_file2321.txt +example_input_file2322.txt +example_input_file2323.txt +example_input_file2324.txt +example_input_file2325.txt +example_input_file2326.txt +example_input_file2327.txt +example_input_file2328.txt +example_input_file2329.txt +example_input_file2330.txt +example_input_file2331.txt +example_input_file2332.txt +example_input_file2333.txt +example_input_file2334.txt +example_input_file2335.txt +example_input_file2336.txt +example_input_file2337.txt +example_input_file2338.txt +example_input_file2339.txt +example_input_file2340.txt +example_input_file2341.txt +example_input_file2342.txt +example_input_file2343.txt +example_input_file2344.txt +example_input_file2345.txt +example_input_file2346.txt +example_input_file2347.txt +example_input_file2348.txt +example_input_file2349.txt +example_input_file2350.txt +example_input_file2351.txt +example_input_file2352.txt +example_input_file2353.txt +example_input_file2354.txt +example_input_file2355.txt +example_input_file2356.txt +example_input_file2357.txt +example_input_file2358.txt +example_input_file2359.txt +example_input_file2360.txt +example_input_file2361.txt +example_input_file2362.txt +example_input_file2363.txt +example_input_file2364.txt +example_input_file2365.txt +example_input_file2366.txt +example_input_file2367.txt +example_input_file2368.txt +example_input_file2369.txt +example_input_file2370.txt +example_input_file2371.txt +example_input_file2372.txt +example_input_file2373.txt +example_input_file2374.txt +example_input_file2375.txt +example_input_file2376.txt +example_input_file2377.txt +example_input_file2378.txt +example_input_file2379.txt +example_input_file2380.txt +example_input_file2381.txt +example_input_file2382.txt +example_input_file2383.txt +example_input_file2384.txt +example_input_file2385.txt +example_input_file2386.txt +example_input_file2387.txt +example_input_file2388.txt +example_input_file2389.txt +example_input_file2390.txt +example_input_file2391.txt +example_input_file2392.txt +example_input_file2393.txt +example_input_file2394.txt +example_input_file2395.txt +example_input_file2396.txt +example_input_file2397.txt +example_input_file2398.txt +example_input_file2399.txt +example_input_file2400.txt +example_input_file2401.txt +example_input_file2402.txt +example_input_file2403.txt +example_input_file2404.txt +example_input_file2405.txt +example_input_file2406.txt +example_input_file2407.txt +example_input_file2408.txt +example_input_file2409.txt +example_input_file2410.txt +example_input_file2411.txt +example_input_file2412.txt +example_input_file2413.txt +example_input_file2414.txt +example_input_file2415.txt +example_input_file2416.txt +example_input_file2417.txt +example_input_file2418.txt +example_input_file2419.txt +example_input_file2420.txt +example_input_file2421.txt +example_input_file2422.txt +example_input_file2423.txt +example_input_file2424.txt +example_input_file2425.txt +example_input_file2426.txt +example_input_file2427.txt +example_input_file2428.txt +example_input_file2429.txt +example_input_file2430.txt +example_input_file2431.txt +example_input_file2432.txt +example_input_file2433.txt +example_input_file2434.txt +example_input_file2435.txt +example_input_file2436.txt +example_input_file2437.txt +example_input_file2438.txt +example_input_file2439.txt +example_input_file2440.txt +example_input_file2441.txt +example_input_file2442.txt +example_input_file2443.txt +example_input_file2444.txt +example_input_file2445.txt +example_input_file2446.txt +example_input_file2447.txt +example_input_file2448.txt +example_input_file2449.txt +example_input_file2450.txt +example_input_file2451.txt +example_input_file2452.txt +example_input_file2453.txt +example_input_file2454.txt +example_input_file2455.txt +example_input_file2456.txt +example_input_file2457.txt +example_input_file2458.txt +example_input_file2459.txt +example_input_file2460.txt +example_input_file2461.txt +example_input_file2462.txt +example_input_file2463.txt +example_input_file2464.txt +example_input_file2465.txt +example_input_file2466.txt +example_input_file2467.txt +example_input_file2468.txt +example_input_file2469.txt +example_input_file2470.txt +example_input_file2471.txt +example_input_file2472.txt +example_input_file2473.txt +example_input_file2474.txt +example_input_file2475.txt +example_input_file2476.txt +example_input_file2477.txt +example_input_file2478.txt +example_input_file2479.txt +example_input_file2480.txt +example_input_file2481.txt +example_input_file2482.txt +example_input_file2483.txt +example_input_file2484.txt +example_input_file2485.txt +example_input_file2486.txt +example_input_file2487.txt +example_input_file2488.txt +example_input_file2489.txt +example_input_file2490.txt +example_input_file2491.txt +example_input_file2492.txt +example_input_file2493.txt +example_input_file2494.txt +example_input_file2495.txt +example_input_file2496.txt +example_input_file2497.txt +example_input_file2498.txt +example_input_file2499.txt +example_input_file2500.txt +example_input_file2501.txt +example_input_file2502.txt +example_input_file2503.txt +example_input_file2504.txt +example_input_file2505.txt +example_input_file2506.txt +example_input_file2507.txt +example_input_file2508.txt +example_input_file2509.txt +example_input_file2510.txt +example_input_file2511.txt +example_input_file2512.txt +example_input_file2513.txt +example_input_file2514.txt +example_input_file2515.txt +example_input_file2516.txt +example_input_file2517.txt +example_input_file2518.txt +example_input_file2519.txt +example_input_file2520.txt +example_input_file2521.txt +example_input_file2522.txt +example_input_file2523.txt +example_input_file2524.txt +example_input_file2525.txt +example_input_file2526.txt +example_input_file2527.txt +example_input_file2528.txt +example_input_file2529.txt +example_input_file2530.txt +example_input_file2531.txt +example_input_file2532.txt +example_input_file2533.txt +example_input_file2534.txt +example_input_file2535.txt +example_input_file2536.txt +example_input_file2537.txt +example_input_file2538.txt +example_input_file2539.txt +example_input_file2540.txt +example_input_file2541.txt +example_input_file2542.txt +example_input_file2543.txt +example_input_file2544.txt +example_input_file2545.txt +example_input_file2546.txt +example_input_file2547.txt +example_input_file2548.txt +example_input_file2549.txt +example_input_file2550.txt +example_input_file2551.txt +example_input_file2552.txt +example_input_file2553.txt +example_input_file2554.txt +example_input_file2555.txt +example_input_file2556.txt +example_input_file2557.txt +example_input_file2558.txt +example_input_file2559.txt +example_input_file2560.txt +example_input_file2561.txt +example_input_file2562.txt +example_input_file2563.txt +example_input_file2564.txt +example_input_file2565.txt +example_input_file2566.txt +example_input_file2567.txt +example_input_file2568.txt +example_input_file2569.txt +example_input_file2570.txt +example_input_file2571.txt +example_input_file2572.txt +example_input_file2573.txt +example_input_file2574.txt +example_input_file2575.txt +example_input_file2576.txt +example_input_file2577.txt +example_input_file2578.txt +example_input_file2579.txt +example_input_file2580.txt +example_input_file2581.txt +example_input_file2582.txt +example_input_file2583.txt +example_input_file2584.txt +example_input_file2585.txt +example_input_file2586.txt +example_input_file2587.txt +example_input_file2588.txt +example_input_file2589.txt +example_input_file2590.txt +example_input_file2591.txt +example_input_file2592.txt +example_input_file2593.txt +example_input_file2594.txt +example_input_file2595.txt +example_input_file2596.txt +example_input_file2597.txt +example_input_file2598.txt +example_input_file2599.txt +example_input_file2600.txt +example_input_file2601.txt +example_input_file2602.txt +example_input_file2603.txt +example_input_file2604.txt +example_input_file2605.txt +example_input_file2606.txt +example_input_file2607.txt +example_input_file2608.txt +example_input_file2609.txt +example_input_file2610.txt +example_input_file2611.txt +example_input_file2612.txt +example_input_file2613.txt +example_input_file2614.txt +example_input_file2615.txt +example_input_file2616.txt +example_input_file2617.txt +example_input_file2618.txt +example_input_file2619.txt +example_input_file2620.txt +example_input_file2621.txt +example_input_file2622.txt +example_input_file2623.txt +example_input_file2624.txt +example_input_file2625.txt +example_input_file2626.txt +example_input_file2627.txt +example_input_file2628.txt +example_input_file2629.txt +example_input_file2630.txt +example_input_file2631.txt +example_input_file2632.txt +example_input_file2633.txt +example_input_file2634.txt +example_input_file2635.txt +example_input_file2636.txt +example_input_file2637.txt +example_input_file2638.txt +example_input_file2639.txt +example_input_file2640.txt +example_input_file2641.txt +example_input_file2642.txt +example_input_file2643.txt +example_input_file2644.txt +example_input_file2645.txt +example_input_file2646.txt +example_input_file2647.txt +example_input_file2648.txt +example_input_file2649.txt +example_input_file2650.txt +example_input_file2651.txt +example_input_file2652.txt +example_input_file2653.txt +example_input_file2654.txt +example_input_file2655.txt +example_input_file2656.txt +example_input_file2657.txt +example_input_file2658.txt +example_input_file2659.txt +example_input_file2660.txt +example_input_file2661.txt +example_input_file2662.txt +example_input_file2663.txt +example_input_file2664.txt +example_input_file2665.txt +example_input_file2666.txt +example_input_file2667.txt +example_input_file2668.txt +example_input_file2669.txt +example_input_file2670.txt +example_input_file2671.txt +example_input_file2672.txt +example_input_file2673.txt +example_input_file2674.txt +example_input_file2675.txt +example_input_file2676.txt +example_input_file2677.txt +example_input_file2678.txt +example_input_file2679.txt +example_input_file2680.txt +example_input_file2681.txt +example_input_file2682.txt +example_input_file2683.txt +example_input_file2684.txt +example_input_file2685.txt +example_input_file2686.txt +example_input_file2687.txt +example_input_file2688.txt +example_input_file2689.txt +example_input_file2690.txt +example_input_file2691.txt +example_input_file2692.txt +example_input_file2693.txt +example_input_file2694.txt +example_input_file2695.txt +example_input_file2696.txt +example_input_file2697.txt +example_input_file2698.txt +example_input_file2699.txt +example_input_file2700.txt +example_input_file2701.txt +example_input_file2702.txt +example_input_file2703.txt +example_input_file2704.txt +example_input_file2705.txt +example_input_file2706.txt +example_input_file2707.txt +example_input_file2708.txt +example_input_file2709.txt +example_input_file2710.txt +example_input_file2711.txt +example_input_file2712.txt +example_input_file2713.txt +example_input_file2714.txt +example_input_file2715.txt +example_input_file2716.txt +example_input_file2717.txt +example_input_file2718.txt +example_input_file2719.txt +example_input_file2720.txt +example_input_file2721.txt +example_input_file2722.txt +example_input_file2723.txt +example_input_file2724.txt +example_input_file2725.txt +example_input_file2726.txt +example_input_file2727.txt +example_input_file2728.txt +example_input_file2729.txt +example_input_file2730.txt +example_input_file2731.txt +example_input_file2732.txt +example_input_file2733.txt +example_input_file2734.txt +example_input_file2735.txt +example_input_file2736.txt +example_input_file2737.txt +example_input_file2738.txt +example_input_file2739.txt +example_input_file2740.txt +example_input_file2741.txt +example_input_file2742.txt +example_input_file2743.txt +example_input_file2744.txt +example_input_file2745.txt +example_input_file2746.txt +example_input_file2747.txt +example_input_file2748.txt +example_input_file2749.txt +example_input_file2750.txt +example_input_file2751.txt +example_input_file2752.txt +example_input_file2753.txt +example_input_file2754.txt +example_input_file2755.txt +example_input_file2756.txt +example_input_file2757.txt +example_input_file2758.txt +example_input_file2759.txt +example_input_file2760.txt +example_input_file2761.txt +example_input_file2762.txt +example_input_file2763.txt +example_input_file2764.txt +example_input_file2765.txt +example_input_file2766.txt +example_input_file2767.txt +example_input_file2768.txt +example_input_file2769.txt +example_input_file2770.txt +example_input_file2771.txt +example_input_file2772.txt +example_input_file2773.txt +example_input_file2774.txt +example_input_file2775.txt +example_input_file2776.txt +example_input_file2777.txt +example_input_file2778.txt +example_input_file2779.txt +example_input_file2780.txt +example_input_file2781.txt +example_input_file2782.txt +example_input_file2783.txt +example_input_file2784.txt +example_input_file2785.txt +example_input_file2786.txt +example_input_file2787.txt +example_input_file2788.txt +example_input_file2789.txt +example_input_file2790.txt +example_input_file2791.txt +example_input_file2792.txt +example_input_file2793.txt +example_input_file2794.txt +example_input_file2795.txt +example_input_file2796.txt +example_input_file2797.txt +example_input_file2798.txt +example_input_file2799.txt +example_input_file2800.txt +example_input_file2801.txt +example_input_file2802.txt +example_input_file2803.txt +example_input_file2804.txt +example_input_file2805.txt +example_input_file2806.txt +example_input_file2807.txt +example_input_file2808.txt +example_input_file2809.txt +example_input_file2810.txt +example_input_file2811.txt +example_input_file2812.txt +example_input_file2813.txt +example_input_file2814.txt +example_input_file2815.txt +example_input_file2816.txt +example_input_file2817.txt +example_input_file2818.txt +example_input_file2819.txt +example_input_file2820.txt +example_input_file2821.txt +example_input_file2822.txt +example_input_file2823.txt +example_input_file2824.txt +example_input_file2825.txt +example_input_file2826.txt +example_input_file2827.txt +example_input_file2828.txt +example_input_file2829.txt +example_input_file2830.txt +example_input_file2831.txt +example_input_file2832.txt +example_input_file2833.txt +example_input_file2834.txt +example_input_file2835.txt +example_input_file2836.txt +example_input_file2837.txt +example_input_file2838.txt +example_input_file2839.txt +example_input_file2840.txt +example_input_file2841.txt +example_input_file2842.txt +example_input_file2843.txt +example_input_file2844.txt +example_input_file2845.txt +example_input_file2846.txt +example_input_file2847.txt +example_input_file2848.txt +example_input_file2849.txt +example_input_file2850.txt +example_input_file2851.txt +example_input_file2852.txt +example_input_file2853.txt +example_input_file2854.txt +example_input_file2855.txt +example_input_file2856.txt +example_input_file2857.txt +example_input_file2858.txt +example_input_file2859.txt +example_input_file2860.txt +example_input_file2861.txt +example_input_file2862.txt +example_input_file2863.txt +example_input_file2864.txt +example_input_file2865.txt +example_input_file2866.txt +example_input_file2867.txt +example_input_file2868.txt +example_input_file2869.txt +example_input_file2870.txt +example_input_file2871.txt +example_input_file2872.txt +example_input_file2873.txt +example_input_file2874.txt +example_input_file2875.txt +example_input_file2876.txt +example_input_file2877.txt +example_input_file2878.txt +example_input_file2879.txt +example_input_file2880.txt +example_input_file2881.txt +example_input_file2882.txt +example_input_file2883.txt +example_input_file2884.txt +example_input_file2885.txt +example_input_file2886.txt +example_input_file2887.txt +example_input_file2888.txt +example_input_file2889.txt +example_input_file2890.txt +example_input_file2891.txt +example_input_file2892.txt +example_input_file2893.txt +example_input_file2894.txt +example_input_file2895.txt +example_input_file2896.txt +example_input_file2897.txt +example_input_file2898.txt +example_input_file2899.txt +example_input_file2900.txt +example_input_file2901.txt +example_input_file2902.txt +example_input_file2903.txt +example_input_file2904.txt +example_input_file2905.txt +example_input_file2906.txt +example_input_file2907.txt +example_input_file2908.txt +example_input_file2909.txt +example_input_file2910.txt +example_input_file2911.txt +example_input_file2912.txt +example_input_file2913.txt +example_input_file2914.txt +example_input_file2915.txt +example_input_file2916.txt +example_input_file2917.txt +example_input_file2918.txt +example_input_file2919.txt +example_input_file2920.txt +example_input_file2921.txt +example_input_file2922.txt +example_input_file2923.txt +example_input_file2924.txt +example_input_file2925.txt +example_input_file2926.txt +example_input_file2927.txt +example_input_file2928.txt +example_input_file2929.txt +example_input_file2930.txt +example_input_file2931.txt +example_input_file2932.txt +example_input_file2933.txt +example_input_file2934.txt +example_input_file2935.txt +example_input_file2936.txt +example_input_file2937.txt +example_input_file2938.txt +example_input_file2939.txt +example_input_file2940.txt +example_input_file2941.txt +example_input_file2942.txt +example_input_file2943.txt +example_input_file2944.txt +example_input_file2945.txt +example_input_file2946.txt +example_input_file2947.txt +example_input_file2948.txt +example_input_file2949.txt +example_input_file2950.txt +example_input_file2951.txt +example_input_file2952.txt +example_input_file2953.txt +example_input_file2954.txt +example_input_file2955.txt +example_input_file2956.txt +example_input_file2957.txt +example_input_file2958.txt +example_input_file2959.txt +example_input_file2960.txt +example_input_file2961.txt +example_input_file2962.txt +example_input_file2963.txt +example_input_file2964.txt +example_input_file2965.txt +example_input_file2966.txt +example_input_file2967.txt +example_input_file2968.txt +example_input_file2969.txt +example_input_file2970.txt +example_input_file2971.txt +example_input_file2972.txt +example_input_file2973.txt +example_input_file2974.txt +example_input_file2975.txt +example_input_file2976.txt +example_input_file2977.txt +example_input_file2978.txt +example_input_file2979.txt +example_input_file2980.txt +example_input_file2981.txt +example_input_file2982.txt +example_input_file2983.txt +example_input_file2984.txt +example_input_file2985.txt +example_input_file2986.txt +example_input_file2987.txt +example_input_file2988.txt +example_input_file2989.txt +example_input_file2990.txt +example_input_file2991.txt +example_input_file2992.txt +example_input_file2993.txt +example_input_file2994.txt +example_input_file2995.txt +example_input_file2996.txt +example_input_file2997.txt +example_input_file2998.txt +example_input_file2999.txt +example_input_file3000.txt +example_input_file3001.txt +example_input_file3002.txt +example_input_file3003.txt +example_input_file3004.txt +example_input_file3005.txt +example_input_file3006.txt +example_input_file3007.txt +example_input_file3008.txt +example_input_file3009.txt +example_input_file3010.txt +example_input_file3011.txt +example_input_file3012.txt +example_input_file3013.txt +example_input_file3014.txt +example_input_file3015.txt +example_input_file3016.txt +example_input_file3017.txt +example_input_file3018.txt +example_input_file3019.txt +example_input_file3020.txt +example_input_file3021.txt +example_input_file3022.txt +example_input_file3023.txt +example_input_file3024.txt +example_input_file3025.txt +example_input_file3026.txt +example_input_file3027.txt +example_input_file3028.txt +example_input_file3029.txt +example_input_file3030.txt +example_input_file3031.txt +example_input_file3032.txt +example_input_file3033.txt +example_input_file3034.txt +example_input_file3035.txt +example_input_file3036.txt +example_input_file3037.txt +example_input_file3038.txt +example_input_file3039.txt +example_input_file3040.txt +example_input_file3041.txt +example_input_file3042.txt +example_input_file3043.txt +example_input_file3044.txt +example_input_file3045.txt +example_input_file3046.txt +example_input_file3047.txt +example_input_file3048.txt +example_input_file3049.txt +example_input_file3050.txt +example_input_file3051.txt +example_input_file3052.txt +example_input_file3053.txt +example_input_file3054.txt +example_input_file3055.txt +example_input_file3056.txt +example_input_file3057.txt +example_input_file3058.txt +example_input_file3059.txt +example_input_file3060.txt +example_input_file3061.txt +example_input_file3062.txt +example_input_file3063.txt +example_input_file3064.txt +example_input_file3065.txt +example_input_file3066.txt +example_input_file3067.txt +example_input_file3068.txt +example_input_file3069.txt +example_input_file3070.txt +example_input_file3071.txt +example_input_file3072.txt +example_input_file3073.txt +example_input_file3074.txt +example_input_file3075.txt +example_input_file3076.txt +example_input_file3077.txt +example_input_file3078.txt +example_input_file3079.txt +example_input_file3080.txt +example_input_file3081.txt +example_input_file3082.txt +example_input_file3083.txt +example_input_file3084.txt +example_input_file3085.txt +example_input_file3086.txt +example_input_file3087.txt +example_input_file3088.txt +example_input_file3089.txt +example_input_file3090.txt +example_input_file3091.txt +example_input_file3092.txt +example_input_file3093.txt +example_input_file3094.txt +example_input_file3095.txt +example_input_file3096.txt +example_input_file3097.txt +example_input_file3098.txt +example_input_file3099.txt +example_input_file3100.txt +example_input_file3101.txt +example_input_file3102.txt +example_input_file3103.txt +example_input_file3104.txt +example_input_file3105.txt +example_input_file3106.txt +example_input_file3107.txt +example_input_file3108.txt +example_input_file3109.txt +example_input_file3110.txt +example_input_file3111.txt +example_input_file3112.txt +example_input_file3113.txt +example_input_file3114.txt +example_input_file3115.txt +example_input_file3116.txt +example_input_file3117.txt +example_input_file3118.txt +example_input_file3119.txt +example_input_file3120.txt +example_input_file3121.txt +example_input_file3122.txt +example_input_file3123.txt +example_input_file3124.txt +example_input_file3125.txt +example_input_file3126.txt +example_input_file3127.txt +example_input_file3128.txt +example_input_file3129.txt +example_input_file3130.txt +example_input_file3131.txt +example_input_file3132.txt +example_input_file3133.txt +example_input_file3134.txt +example_input_file3135.txt +example_input_file3136.txt +example_input_file3137.txt +example_input_file3138.txt +example_input_file3139.txt +example_input_file3140.txt +example_input_file3141.txt +example_input_file3142.txt +example_input_file3143.txt +example_input_file3144.txt +example_input_file3145.txt +example_input_file3146.txt +example_input_file3147.txt +example_input_file3148.txt +example_input_file3149.txt +example_input_file3150.txt +example_input_file3151.txt +example_input_file3152.txt +example_input_file3153.txt +example_input_file3154.txt +example_input_file3155.txt +example_input_file3156.txt +example_input_file3157.txt +example_input_file3158.txt +example_input_file3159.txt +example_input_file3160.txt +example_input_file3161.txt +example_input_file3162.txt +example_input_file3163.txt +example_input_file3164.txt +example_input_file3165.txt +example_input_file3166.txt +example_input_file3167.txt +example_input_file3168.txt +example_input_file3169.txt +example_input_file3170.txt +example_input_file3171.txt +example_input_file3172.txt +example_input_file3173.txt +example_input_file3174.txt +example_input_file3175.txt +example_input_file3176.txt +example_input_file3177.txt +example_input_file3178.txt +example_input_file3179.txt +example_input_file3180.txt +example_input_file3181.txt +example_input_file3182.txt +example_input_file3183.txt +example_input_file3184.txt +example_input_file3185.txt +example_input_file3186.txt +example_input_file3187.txt +example_input_file3188.txt +example_input_file3189.txt +example_input_file3190.txt +example_input_file3191.txt +example_input_file3192.txt +example_input_file3193.txt +example_input_file3194.txt +example_input_file3195.txt +example_input_file3196.txt +example_input_file3197.txt +example_input_file3198.txt +example_input_file3199.txt +example_input_file3200.txt +example_input_file3201.txt +example_input_file3202.txt +example_input_file3203.txt +example_input_file3204.txt +example_input_file3205.txt +example_input_file3206.txt +example_input_file3207.txt +example_input_file3208.txt +example_input_file3209.txt +example_input_file3210.txt +example_input_file3211.txt +example_input_file3212.txt +example_input_file3213.txt +example_input_file3214.txt +example_input_file3215.txt +example_input_file3216.txt +example_input_file3217.txt +example_input_file3218.txt +example_input_file3219.txt +example_input_file3220.txt +example_input_file3221.txt +example_input_file3222.txt +example_input_file3223.txt +example_input_file3224.txt +example_input_file3225.txt +example_input_file3226.txt +example_input_file3227.txt +example_input_file3228.txt +example_input_file3229.txt +example_input_file3230.txt +example_input_file3231.txt +example_input_file3232.txt +example_input_file3233.txt +example_input_file3234.txt +example_input_file3235.txt +example_input_file3236.txt +example_input_file3237.txt +example_input_file3238.txt +example_input_file3239.txt +example_input_file3240.txt +example_input_file3241.txt +example_input_file3242.txt +example_input_file3243.txt +example_input_file3244.txt +example_input_file3245.txt +example_input_file3246.txt +example_input_file3247.txt +example_input_file3248.txt +example_input_file3249.txt +example_input_file3250.txt +example_input_file3251.txt +example_input_file3252.txt +example_input_file3253.txt +example_input_file3254.txt +example_input_file3255.txt +example_input_file3256.txt +example_input_file3257.txt +example_input_file3258.txt +example_input_file3259.txt +example_input_file3260.txt +example_input_file3261.txt +example_input_file3262.txt +example_input_file3263.txt +example_input_file3264.txt +example_input_file3265.txt +example_input_file3266.txt +example_input_file3267.txt +example_input_file3268.txt +example_input_file3269.txt +example_input_file3270.txt +example_input_file3271.txt +example_input_file3272.txt +example_input_file3273.txt +example_input_file3274.txt +example_input_file3275.txt +example_input_file3276.txt +example_input_file3277.txt +example_input_file3278.txt +example_input_file3279.txt +example_input_file3280.txt +example_input_file3281.txt +example_input_file3282.txt +example_input_file3283.txt +example_input_file3284.txt +example_input_file3285.txt +example_input_file3286.txt +example_input_file3287.txt +example_input_file3288.txt +example_input_file3289.txt +example_input_file3290.txt +example_input_file3291.txt +example_input_file3292.txt +example_input_file3293.txt +example_input_file3294.txt +example_input_file3295.txt +example_input_file3296.txt +example_input_file3297.txt +example_input_file3298.txt +example_input_file3299.txt +example_input_file3300.txt +example_input_file3301.txt +example_input_file3302.txt +example_input_file3303.txt +example_input_file3304.txt +example_input_file3305.txt +example_input_file3306.txt +example_input_file3307.txt +example_input_file3308.txt +example_input_file3309.txt +example_input_file3310.txt +example_input_file3311.txt +example_input_file3312.txt +example_input_file3313.txt +example_input_file3314.txt +example_input_file3315.txt +example_input_file3316.txt +example_input_file3317.txt +example_input_file3318.txt +example_input_file3319.txt +example_input_file3320.txt +example_input_file3321.txt +example_input_file3322.txt +example_input_file3323.txt +example_input_file3324.txt +example_input_file3325.txt +example_input_file3326.txt +example_input_file3327.txt +example_input_file3328.txt +example_input_file3329.txt +example_input_file3330.txt +example_input_file3331.txt +example_input_file3332.txt +example_input_file3333.txt +example_input_file3334.txt +example_input_file3335.txt +example_input_file3336.txt +example_input_file3337.txt +example_input_file3338.txt +example_input_file3339.txt +example_input_file3340.txt +example_input_file3341.txt +example_input_file3342.txt +example_input_file3343.txt +example_input_file3344.txt +example_input_file3345.txt +example_input_file3346.txt +example_input_file3347.txt +example_input_file3348.txt +example_input_file3349.txt +example_input_file3350.txt +example_input_file3351.txt +example_input_file3352.txt +example_input_file3353.txt +example_input_file3354.txt +example_input_file3355.txt +example_input_file3356.txt +example_input_file3357.txt +example_input_file3358.txt +example_input_file3359.txt +example_input_file3360.txt +example_input_file3361.txt +example_input_file3362.txt +example_input_file3363.txt +example_input_file3364.txt +example_input_file3365.txt +example_input_file3366.txt +example_input_file3367.txt +example_input_file3368.txt +example_input_file3369.txt +example_input_file3370.txt +example_input_file3371.txt +example_input_file3372.txt +example_input_file3373.txt +example_input_file3374.txt +example_input_file3375.txt +example_input_file3376.txt +example_input_file3377.txt +example_input_file3378.txt +example_input_file3379.txt +example_input_file3380.txt +example_input_file3381.txt +example_input_file3382.txt +example_input_file3383.txt +example_input_file3384.txt +example_input_file3385.txt +example_input_file3386.txt +example_input_file3387.txt +example_input_file3388.txt +example_input_file3389.txt +example_input_file3390.txt +example_input_file3391.txt +example_input_file3392.txt +example_input_file3393.txt +example_input_file3394.txt +example_input_file3395.txt +example_input_file3396.txt +example_input_file3397.txt +example_input_file3398.txt +example_input_file3399.txt +example_input_file3400.txt +example_input_file3401.txt +example_input_file3402.txt +example_input_file3403.txt +example_input_file3404.txt +example_input_file3405.txt +example_input_file3406.txt +example_input_file3407.txt +example_input_file3408.txt +example_input_file3409.txt +example_input_file3410.txt +example_input_file3411.txt +example_input_file3412.txt +example_input_file3413.txt +example_input_file3414.txt +example_input_file3415.txt +example_input_file3416.txt +example_input_file3417.txt +example_input_file3418.txt +example_input_file3419.txt +example_input_file3420.txt +example_input_file3421.txt +example_input_file3422.txt +example_input_file3423.txt +example_input_file3424.txt +example_input_file3425.txt +example_input_file3426.txt +example_input_file3427.txt +example_input_file3428.txt +example_input_file3429.txt +example_input_file3430.txt +example_input_file3431.txt +example_input_file3432.txt +example_input_file3433.txt +example_input_file3434.txt +example_input_file3435.txt +example_input_file3436.txt +example_input_file3437.txt +example_input_file3438.txt +example_input_file3439.txt +example_input_file3440.txt +example_input_file3441.txt +example_input_file3442.txt +example_input_file3443.txt +example_input_file3444.txt +example_input_file3445.txt +example_input_file3446.txt +example_input_file3447.txt +example_input_file3448.txt +example_input_file3449.txt +example_input_file3450.txt +example_input_file3451.txt +example_input_file3452.txt +example_input_file3453.txt +example_input_file3454.txt +example_input_file3455.txt +example_input_file3456.txt +example_input_file3457.txt +example_input_file3458.txt +example_input_file3459.txt +example_input_file3460.txt +example_input_file3461.txt +example_input_file3462.txt +example_input_file3463.txt +example_input_file3464.txt +example_input_file3465.txt +example_input_file3466.txt +example_input_file3467.txt +example_input_file3468.txt +example_input_file3469.txt +example_input_file3470.txt +example_input_file3471.txt +example_input_file3472.txt +example_input_file3473.txt +example_input_file3474.txt +example_input_file3475.txt +example_input_file3476.txt +example_input_file3477.txt +example_input_file3478.txt +example_input_file3479.txt +example_input_file3480.txt +example_input_file3481.txt +example_input_file3482.txt +example_input_file3483.txt +example_input_file3484.txt +example_input_file3485.txt +example_input_file3486.txt +example_input_file3487.txt +example_input_file3488.txt +example_input_file3489.txt +example_input_file3490.txt +example_input_file3491.txt +example_input_file3492.txt +example_input_file3493.txt +example_input_file3494.txt +example_input_file3495.txt +example_input_file3496.txt +example_input_file3497.txt +example_input_file3498.txt +example_input_file3499.txt +example_input_file3500.txt +example_input_file3501.txt +example_input_file3502.txt +example_input_file3503.txt +example_input_file3504.txt +example_input_file3505.txt +example_input_file3506.txt +example_input_file3507.txt +example_input_file3508.txt +example_input_file3509.txt +example_input_file3510.txt +example_input_file3511.txt +example_input_file3512.txt +example_input_file3513.txt +example_input_file3514.txt +example_input_file3515.txt +example_input_file3516.txt +example_input_file3517.txt +example_input_file3518.txt +example_input_file3519.txt +example_input_file3520.txt +example_input_file3521.txt +example_input_file3522.txt +example_input_file3523.txt +example_input_file3524.txt +example_input_file3525.txt +example_input_file3526.txt +example_input_file3527.txt +example_input_file3528.txt +example_input_file3529.txt +example_input_file3530.txt +example_input_file3531.txt +example_input_file3532.txt +example_input_file3533.txt +example_input_file3534.txt +example_input_file3535.txt +example_input_file3536.txt +example_input_file3537.txt +example_input_file3538.txt +example_input_file3539.txt +example_input_file3540.txt +example_input_file3541.txt +example_input_file3542.txt +example_input_file3543.txt +example_input_file3544.txt +example_input_file3545.txt +example_input_file3546.txt +example_input_file3547.txt +example_input_file3548.txt +example_input_file3549.txt +example_input_file3550.txt +example_input_file3551.txt +example_input_file3552.txt +example_input_file3553.txt +example_input_file3554.txt +example_input_file3555.txt +example_input_file3556.txt +example_input_file3557.txt +example_input_file3558.txt +example_input_file3559.txt +example_input_file3560.txt +example_input_file3561.txt +example_input_file3562.txt +example_input_file3563.txt +example_input_file3564.txt +example_input_file3565.txt +example_input_file3566.txt +example_input_file3567.txt +example_input_file3568.txt +example_input_file3569.txt +example_input_file3570.txt +example_input_file3571.txt +example_input_file3572.txt +example_input_file3573.txt +example_input_file3574.txt +example_input_file3575.txt +example_input_file3576.txt +example_input_file3577.txt +example_input_file3578.txt +example_input_file3579.txt +example_input_file3580.txt +example_input_file3581.txt +example_input_file3582.txt +example_input_file3583.txt +example_input_file3584.txt +example_input_file3585.txt +example_input_file3586.txt +example_input_file3587.txt +example_input_file3588.txt +example_input_file3589.txt +example_input_file3590.txt +example_input_file3591.txt +example_input_file3592.txt +example_input_file3593.txt +example_input_file3594.txt +example_input_file3595.txt +example_input_file3596.txt +example_input_file3597.txt +example_input_file3598.txt +example_input_file3599.txt +example_input_file3600.txt +example_input_file3601.txt +example_input_file3602.txt +example_input_file3603.txt +example_input_file3604.txt +example_input_file3605.txt +example_input_file3606.txt +example_input_file3607.txt +example_input_file3608.txt +example_input_file3609.txt +example_input_file3610.txt +example_input_file3611.txt +example_input_file3612.txt +example_input_file3613.txt +example_input_file3614.txt +example_input_file3615.txt +example_input_file3616.txt +example_input_file3617.txt +example_input_file3618.txt +example_input_file3619.txt +example_input_file3620.txt +example_input_file3621.txt +example_input_file3622.txt +example_input_file3623.txt +example_input_file3624.txt +example_input_file3625.txt +example_input_file3626.txt +example_input_file3627.txt +example_input_file3628.txt +example_input_file3629.txt +example_input_file3630.txt +example_input_file3631.txt +example_input_file3632.txt +example_input_file3633.txt +example_input_file3634.txt +example_input_file3635.txt +example_input_file3636.txt +example_input_file3637.txt +example_input_file3638.txt +example_input_file3639.txt +example_input_file3640.txt +example_input_file3641.txt +example_input_file3642.txt +example_input_file3643.txt +example_input_file3644.txt +example_input_file3645.txt +example_input_file3646.txt +example_input_file3647.txt +example_input_file3648.txt +example_input_file3649.txt +example_input_file3650.txt +example_input_file3651.txt +example_input_file3652.txt +example_input_file3653.txt +example_input_file3654.txt +example_input_file3655.txt +example_input_file3656.txt +example_input_file3657.txt +example_input_file3658.txt +example_input_file3659.txt +example_input_file3660.txt +example_input_file3661.txt +example_input_file3662.txt +example_input_file3663.txt +example_input_file3664.txt +example_input_file3665.txt +example_input_file3666.txt +example_input_file3667.txt +example_input_file3668.txt +example_input_file3669.txt +example_input_file3670.txt +example_input_file3671.txt +example_input_file3672.txt +example_input_file3673.txt +example_input_file3674.txt +example_input_file3675.txt +example_input_file3676.txt +example_input_file3677.txt +example_input_file3678.txt +example_input_file3679.txt +example_input_file3680.txt +example_input_file3681.txt +example_input_file3682.txt +example_input_file3683.txt +example_input_file3684.txt +example_input_file3685.txt +example_input_file3686.txt +example_input_file3687.txt +example_input_file3688.txt +example_input_file3689.txt +example_input_file3690.txt +example_input_file3691.txt +example_input_file3692.txt +example_input_file3693.txt +example_input_file3694.txt +example_input_file3695.txt +example_input_file3696.txt +example_input_file3697.txt +example_input_file3698.txt +example_input_file3699.txt +example_input_file3700.txt +example_input_file3701.txt +example_input_file3702.txt +example_input_file3703.txt +example_input_file3704.txt +example_input_file3705.txt +example_input_file3706.txt +example_input_file3707.txt +example_input_file3708.txt +example_input_file3709.txt +example_input_file3710.txt +example_input_file3711.txt +example_input_file3712.txt +example_input_file3713.txt +example_input_file3714.txt +example_input_file3715.txt +example_input_file3716.txt +example_input_file3717.txt +example_input_file3718.txt +example_input_file3719.txt +example_input_file3720.txt +example_input_file3721.txt +example_input_file3722.txt +example_input_file3723.txt +example_input_file3724.txt +example_input_file3725.txt +example_input_file3726.txt +example_input_file3727.txt +example_input_file3728.txt +example_input_file3729.txt +example_input_file3730.txt +example_input_file3731.txt +example_input_file3732.txt +example_input_file3733.txt +example_input_file3734.txt +example_input_file3735.txt +example_input_file3736.txt +example_input_file3737.txt +example_input_file3738.txt +example_input_file3739.txt +example_input_file3740.txt +example_input_file3741.txt +example_input_file3742.txt +example_input_file3743.txt +example_input_file3744.txt +example_input_file3745.txt +example_input_file3746.txt +example_input_file3747.txt +example_input_file3748.txt +example_input_file3749.txt +example_input_file3750.txt +example_input_file3751.txt +example_input_file3752.txt +example_input_file3753.txt +example_input_file3754.txt +example_input_file3755.txt +example_input_file3756.txt +example_input_file3757.txt +example_input_file3758.txt +example_input_file3759.txt +example_input_file3760.txt +example_input_file3761.txt +example_input_file3762.txt +example_input_file3763.txt +example_input_file3764.txt +example_input_file3765.txt +example_input_file3766.txt +example_input_file3767.txt +example_input_file3768.txt +example_input_file3769.txt +example_input_file3770.txt +example_input_file3771.txt +example_input_file3772.txt +example_input_file3773.txt +example_input_file3774.txt +example_input_file3775.txt +example_input_file3776.txt +example_input_file3777.txt +example_input_file3778.txt +example_input_file3779.txt +example_input_file3780.txt +example_input_file3781.txt +example_input_file3782.txt +example_input_file3783.txt +example_input_file3784.txt +example_input_file3785.txt +example_input_file3786.txt +example_input_file3787.txt +example_input_file3788.txt +example_input_file3789.txt +example_input_file3790.txt +example_input_file3791.txt +example_input_file3792.txt +example_input_file3793.txt +example_input_file3794.txt +example_input_file3795.txt +example_input_file3796.txt +example_input_file3797.txt +example_input_file3798.txt +example_input_file3799.txt +example_input_file3800.txt +example_input_file3801.txt +example_input_file3802.txt +example_input_file3803.txt +example_input_file3804.txt +example_input_file3805.txt +example_input_file3806.txt +example_input_file3807.txt +example_input_file3808.txt +example_input_file3809.txt +example_input_file3810.txt +example_input_file3811.txt +example_input_file3812.txt +example_input_file3813.txt +example_input_file3814.txt +example_input_file3815.txt +example_input_file3816.txt +example_input_file3817.txt +example_input_file3818.txt +example_input_file3819.txt +example_input_file3820.txt +example_input_file3821.txt +example_input_file3822.txt +example_input_file3823.txt +example_input_file3824.txt +example_input_file3825.txt +example_input_file3826.txt +example_input_file3827.txt +example_input_file3828.txt +example_input_file3829.txt +example_input_file3830.txt +example_input_file3831.txt +example_input_file3832.txt +example_input_file3833.txt +example_input_file3834.txt +example_input_file3835.txt +example_input_file3836.txt +example_input_file3837.txt +example_input_file3838.txt +example_input_file3839.txt +example_input_file3840.txt +example_input_file3841.txt +example_input_file3842.txt +example_input_file3843.txt +example_input_file3844.txt +example_input_file3845.txt +example_input_file3846.txt +example_input_file3847.txt +example_input_file3848.txt +example_input_file3849.txt +example_input_file3850.txt +example_input_file3851.txt +example_input_file3852.txt +example_input_file3853.txt +example_input_file3854.txt +example_input_file3855.txt +example_input_file3856.txt +example_input_file3857.txt +example_input_file3858.txt +example_input_file3859.txt +example_input_file3860.txt +example_input_file3861.txt +example_input_file3862.txt +example_input_file3863.txt +example_input_file3864.txt +example_input_file3865.txt +example_input_file3866.txt +example_input_file3867.txt +example_input_file3868.txt +example_input_file3869.txt +example_input_file3870.txt +example_input_file3871.txt +example_input_file3872.txt +example_input_file3873.txt +example_input_file3874.txt +example_input_file3875.txt +example_input_file3876.txt +example_input_file3877.txt +example_input_file3878.txt +example_input_file3879.txt +example_input_file3880.txt +example_input_file3881.txt +example_input_file3882.txt +example_input_file3883.txt +example_input_file3884.txt +example_input_file3885.txt +example_input_file3886.txt +example_input_file3887.txt +example_input_file3888.txt +example_input_file3889.txt +example_input_file3890.txt +example_input_file3891.txt +example_input_file3892.txt +example_input_file3893.txt +example_input_file3894.txt +example_input_file3895.txt +example_input_file3896.txt +example_input_file3897.txt +example_input_file3898.txt +example_input_file3899.txt +example_input_file3900.txt +example_input_file3901.txt +example_input_file3902.txt +example_input_file3903.txt +example_input_file3904.txt +example_input_file3905.txt +example_input_file3906.txt +example_input_file3907.txt +example_input_file3908.txt +example_input_file3909.txt +example_input_file3910.txt +example_input_file3911.txt +example_input_file3912.txt +example_input_file3913.txt +example_input_file3914.txt +example_input_file3915.txt +example_input_file3916.txt +example_input_file3917.txt +example_input_file3918.txt +example_input_file3919.txt +example_input_file3920.txt +example_input_file3921.txt +example_input_file3922.txt +example_input_file3923.txt +example_input_file3924.txt +example_input_file3925.txt +example_input_file3926.txt +example_input_file3927.txt +example_input_file3928.txt +example_input_file3929.txt +example_input_file3930.txt +example_input_file3931.txt +example_input_file3932.txt +example_input_file3933.txt +example_input_file3934.txt +example_input_file3935.txt +example_input_file3936.txt +example_input_file3937.txt +example_input_file3938.txt +example_input_file3939.txt +example_input_file3940.txt +example_input_file3941.txt +example_input_file3942.txt +example_input_file3943.txt +example_input_file3944.txt +example_input_file3945.txt +example_input_file3946.txt +example_input_file3947.txt +example_input_file3948.txt +example_input_file3949.txt +example_input_file3950.txt +example_input_file3951.txt +example_input_file3952.txt +example_input_file3953.txt +example_input_file3954.txt +example_input_file3955.txt +example_input_file3956.txt +example_input_file3957.txt +example_input_file3958.txt +example_input_file3959.txt +example_input_file3960.txt +example_input_file3961.txt +example_input_file3962.txt +example_input_file3963.txt +example_input_file3964.txt +example_input_file3965.txt +example_input_file3966.txt +example_input_file3967.txt +example_input_file3968.txt +example_input_file3969.txt +example_input_file3970.txt +example_input_file3971.txt +example_input_file3972.txt +example_input_file3973.txt +example_input_file3974.txt +example_input_file3975.txt +example_input_file3976.txt +example_input_file3977.txt +example_input_file3978.txt +example_input_file3979.txt +example_input_file3980.txt +example_input_file3981.txt +example_input_file3982.txt +example_input_file3983.txt +example_input_file3984.txt +example_input_file3985.txt +example_input_file3986.txt +example_input_file3987.txt +example_input_file3988.txt +example_input_file3989.txt +example_input_file3990.txt +example_input_file3991.txt +example_input_file3992.txt +example_input_file3993.txt +example_input_file3994.txt +example_input_file3995.txt +example_input_file3996.txt +example_input_file3997.txt +example_input_file3998.txt +example_input_file3999.txt +example_input_file4000.txt +example_input_file4001.txt +example_input_file4002.txt +example_input_file4003.txt +example_input_file4004.txt +example_input_file4005.txt +example_input_file4006.txt +example_input_file4007.txt +example_input_file4008.txt +example_input_file4009.txt +example_input_file4010.txt +example_input_file4011.txt +example_input_file4012.txt +example_input_file4013.txt +example_input_file4014.txt +example_input_file4015.txt +example_input_file4016.txt +example_input_file4017.txt +example_input_file4018.txt +example_input_file4019.txt +example_input_file4020.txt +example_input_file4021.txt +example_input_file4022.txt +example_input_file4023.txt +example_input_file4024.txt +example_input_file4025.txt +example_input_file4026.txt +example_input_file4027.txt +example_input_file4028.txt +example_input_file4029.txt +example_input_file4030.txt +example_input_file4031.txt +example_input_file4032.txt +example_input_file4033.txt +example_input_file4034.txt +example_input_file4035.txt +example_input_file4036.txt +example_input_file4037.txt +example_input_file4038.txt +example_input_file4039.txt +example_input_file4040.txt +example_input_file4041.txt +example_input_file4042.txt +example_input_file4043.txt +example_input_file4044.txt +example_input_file4045.txt +example_input_file4046.txt +example_input_file4047.txt +example_input_file4048.txt +example_input_file4049.txt +example_input_file4050.txt +example_input_file4051.txt +example_input_file4052.txt +example_input_file4053.txt +example_input_file4054.txt +example_input_file4055.txt +example_input_file4056.txt +example_input_file4057.txt +example_input_file4058.txt +example_input_file4059.txt +example_input_file4060.txt +example_input_file4061.txt +example_input_file4062.txt +example_input_file4063.txt +example_input_file4064.txt +example_input_file4065.txt +example_input_file4066.txt +example_input_file4067.txt +example_input_file4068.txt +example_input_file4069.txt +example_input_file4070.txt +example_input_file4071.txt +example_input_file4072.txt +example_input_file4073.txt +example_input_file4074.txt +example_input_file4075.txt +example_input_file4076.txt +example_input_file4077.txt +example_input_file4078.txt +example_input_file4079.txt +example_input_file4080.txt +example_input_file4081.txt +example_input_file4082.txt +example_input_file4083.txt +example_input_file4084.txt +example_input_file4085.txt +example_input_file4086.txt +example_input_file4087.txt +example_input_file4088.txt +example_input_file4089.txt +example_input_file4090.txt +example_input_file4091.txt +example_input_file4092.txt +example_input_file4093.txt +example_input_file4094.txt +example_input_file4095.txt +example_input_file4096.txt +example_input_file4097.txt +example_input_file4098.txt +example_input_file4099.txt +example_input_file4100.txt +example_input_file4101.txt +example_input_file4102.txt +example_input_file4103.txt +example_input_file4104.txt +example_input_file4105.txt +example_input_file4106.txt +example_input_file4107.txt +example_input_file4108.txt +example_input_file4109.txt +example_input_file4110.txt +example_input_file4111.txt +example_input_file4112.txt +example_input_file4113.txt +example_input_file4114.txt +example_input_file4115.txt +example_input_file4116.txt +example_input_file4117.txt +example_input_file4118.txt +example_input_file4119.txt +example_input_file4120.txt +example_input_file4121.txt +example_input_file4122.txt +example_input_file4123.txt +example_input_file4124.txt +example_input_file4125.txt +example_input_file4126.txt +example_input_file4127.txt +example_input_file4128.txt +example_input_file4129.txt +example_input_file4130.txt +example_input_file4131.txt +example_input_file4132.txt +example_input_file4133.txt +example_input_file4134.txt +example_input_file4135.txt +example_input_file4136.txt +example_input_file4137.txt +example_input_file4138.txt +example_input_file4139.txt +example_input_file4140.txt +example_input_file4141.txt +example_input_file4142.txt +example_input_file4143.txt +example_input_file4144.txt +example_input_file4145.txt +example_input_file4146.txt +example_input_file4147.txt +example_input_file4148.txt +example_input_file4149.txt +example_input_file4150.txt +example_input_file4151.txt +example_input_file4152.txt +example_input_file4153.txt +example_input_file4154.txt +example_input_file4155.txt +example_input_file4156.txt +example_input_file4157.txt +example_input_file4158.txt +example_input_file4159.txt +example_input_file4160.txt +example_input_file4161.txt +example_input_file4162.txt +example_input_file4163.txt +example_input_file4164.txt +example_input_file4165.txt +example_input_file4166.txt +example_input_file4167.txt +example_input_file4168.txt +example_input_file4169.txt +example_input_file4170.txt +example_input_file4171.txt +example_input_file4172.txt +example_input_file4173.txt +example_input_file4174.txt +example_input_file4175.txt +example_input_file4176.txt +example_input_file4177.txt +example_input_file4178.txt +example_input_file4179.txt +example_input_file4180.txt +example_input_file4181.txt +example_input_file4182.txt +example_input_file4183.txt +example_input_file4184.txt +example_input_file4185.txt +example_input_file4186.txt +example_input_file4187.txt +example_input_file4188.txt +example_input_file4189.txt +example_input_file4190.txt +example_input_file4191.txt +example_input_file4192.txt +example_input_file4193.txt +example_input_file4194.txt +example_input_file4195.txt +example_input_file4196.txt +example_input_file4197.txt +example_input_file4198.txt +example_input_file4199.txt +example_input_file4200.txt +example_input_file4201.txt +example_input_file4202.txt +example_input_file4203.txt +example_input_file4204.txt +example_input_file4205.txt +example_input_file4206.txt +example_input_file4207.txt +example_input_file4208.txt +example_input_file4209.txt +example_input_file4210.txt +example_input_file4211.txt +example_input_file4212.txt +example_input_file4213.txt +example_input_file4214.txt +example_input_file4215.txt +example_input_file4216.txt +example_input_file4217.txt +example_input_file4218.txt +example_input_file4219.txt +example_input_file4220.txt +example_input_file4221.txt +example_input_file4222.txt +example_input_file4223.txt +example_input_file4224.txt +example_input_file4225.txt +example_input_file4226.txt +example_input_file4227.txt +example_input_file4228.txt +example_input_file4229.txt +example_input_file4230.txt +example_input_file4231.txt +example_input_file4232.txt +example_input_file4233.txt +example_input_file4234.txt +example_input_file4235.txt +example_input_file4236.txt +example_input_file4237.txt +example_input_file4238.txt +example_input_file4239.txt +example_input_file4240.txt +example_input_file4241.txt +example_input_file4242.txt +example_input_file4243.txt +example_input_file4244.txt +example_input_file4245.txt +example_input_file4246.txt +example_input_file4247.txt +example_input_file4248.txt +example_input_file4249.txt +example_input_file4250.txt +example_input_file4251.txt +example_input_file4252.txt +example_input_file4253.txt +example_input_file4254.txt +example_input_file4255.txt +example_input_file4256.txt +example_input_file4257.txt +example_input_file4258.txt +example_input_file4259.txt +example_input_file4260.txt +example_input_file4261.txt +example_input_file4262.txt +example_input_file4263.txt +example_input_file4264.txt +example_input_file4265.txt +example_input_file4266.txt +example_input_file4267.txt +example_input_file4268.txt +example_input_file4269.txt +example_input_file4270.txt +example_input_file4271.txt +example_input_file4272.txt +example_input_file4273.txt +example_input_file4274.txt +example_input_file4275.txt +example_input_file4276.txt +example_input_file4277.txt +example_input_file4278.txt +example_input_file4279.txt +example_input_file4280.txt +example_input_file4281.txt +example_input_file4282.txt +example_input_file4283.txt +example_input_file4284.txt +example_input_file4285.txt +example_input_file4286.txt +example_input_file4287.txt +example_input_file4288.txt +example_input_file4289.txt +example_input_file4290.txt +example_input_file4291.txt +example_input_file4292.txt +example_input_file4293.txt +example_input_file4294.txt +example_input_file4295.txt +example_input_file4296.txt +example_input_file4297.txt +example_input_file4298.txt +example_input_file4299.txt +example_input_file4300.txt +example_input_file4301.txt +example_input_file4302.txt +example_input_file4303.txt +example_input_file4304.txt +example_input_file4305.txt +example_input_file4306.txt +example_input_file4307.txt +example_input_file4308.txt +example_input_file4309.txt +example_input_file4310.txt +example_input_file4311.txt +example_input_file4312.txt +example_input_file4313.txt +example_input_file4314.txt +example_input_file4315.txt +example_input_file4316.txt +example_input_file4317.txt +example_input_file4318.txt +example_input_file4319.txt +example_input_file4320.txt +example_input_file4321.txt +example_input_file4322.txt +example_input_file4323.txt +example_input_file4324.txt +example_input_file4325.txt +example_input_file4326.txt +example_input_file4327.txt +example_input_file4328.txt +example_input_file4329.txt +example_input_file4330.txt +example_input_file4331.txt +example_input_file4332.txt +example_input_file4333.txt +example_input_file4334.txt +example_input_file4335.txt +example_input_file4336.txt +example_input_file4337.txt +example_input_file4338.txt +example_input_file4339.txt +example_input_file4340.txt +example_input_file4341.txt +example_input_file4342.txt +example_input_file4343.txt +example_input_file4344.txt +example_input_file4345.txt +example_input_file4346.txt +example_input_file4347.txt +example_input_file4348.txt +example_input_file4349.txt +example_input_file4350.txt +example_input_file4351.txt +example_input_file4352.txt +example_input_file4353.txt +example_input_file4354.txt +example_input_file4355.txt +example_input_file4356.txt +example_input_file4357.txt +example_input_file4358.txt +example_input_file4359.txt +example_input_file4360.txt +example_input_file4361.txt +example_input_file4362.txt +example_input_file4363.txt +example_input_file4364.txt +example_input_file4365.txt +example_input_file4366.txt +example_input_file4367.txt +example_input_file4368.txt +example_input_file4369.txt +example_input_file4370.txt +example_input_file4371.txt +example_input_file4372.txt +example_input_file4373.txt +example_input_file4374.txt +example_input_file4375.txt +example_input_file4376.txt +example_input_file4377.txt +example_input_file4378.txt +example_input_file4379.txt +example_input_file4380.txt +example_input_file4381.txt +example_input_file4382.txt +example_input_file4383.txt +example_input_file4384.txt +example_input_file4385.txt +example_input_file4386.txt +example_input_file4387.txt +example_input_file4388.txt +example_input_file4389.txt +example_input_file4390.txt +example_input_file4391.txt +example_input_file4392.txt +example_input_file4393.txt +example_input_file4394.txt +example_input_file4395.txt +example_input_file4396.txt +example_input_file4397.txt +example_input_file4398.txt +example_input_file4399.txt +example_input_file4400.txt +example_input_file4401.txt +example_input_file4402.txt +example_input_file4403.txt +example_input_file4404.txt +example_input_file4405.txt +example_input_file4406.txt +example_input_file4407.txt +example_input_file4408.txt +example_input_file4409.txt +example_input_file4410.txt +example_input_file4411.txt +example_input_file4412.txt +example_input_file4413.txt +example_input_file4414.txt +example_input_file4415.txt +example_input_file4416.txt +example_input_file4417.txt +example_input_file4418.txt +example_input_file4419.txt +example_input_file4420.txt +example_input_file4421.txt +example_input_file4422.txt +example_input_file4423.txt +example_input_file4424.txt +example_input_file4425.txt +example_input_file4426.txt +example_input_file4427.txt +example_input_file4428.txt +example_input_file4429.txt +example_input_file4430.txt +example_input_file4431.txt +example_input_file4432.txt +example_input_file4433.txt +example_input_file4434.txt +example_input_file4435.txt +example_input_file4436.txt +example_input_file4437.txt +example_input_file4438.txt +example_input_file4439.txt +example_input_file4440.txt +example_input_file4441.txt +example_input_file4442.txt +example_input_file4443.txt +example_input_file4444.txt +example_input_file4445.txt +example_input_file4446.txt +example_input_file4447.txt +example_input_file4448.txt +example_input_file4449.txt +example_input_file4450.txt +example_input_file4451.txt +example_input_file4452.txt +example_input_file4453.txt +example_input_file4454.txt +example_input_file4455.txt +example_input_file4456.txt +example_input_file4457.txt +example_input_file4458.txt +example_input_file4459.txt +example_input_file4460.txt +example_input_file4461.txt +example_input_file4462.txt +example_input_file4463.txt +example_input_file4464.txt +example_input_file4465.txt +example_input_file4466.txt +example_input_file4467.txt +example_input_file4468.txt +example_input_file4469.txt +example_input_file4470.txt +example_input_file4471.txt +example_input_file4472.txt +example_input_file4473.txt +example_input_file4474.txt +example_input_file4475.txt +example_input_file4476.txt +example_input_file4477.txt +example_input_file4478.txt +example_input_file4479.txt +example_input_file4480.txt +example_input_file4481.txt +example_input_file4482.txt +example_input_file4483.txt +example_input_file4484.txt +example_input_file4485.txt +example_input_file4486.txt +example_input_file4487.txt +example_input_file4488.txt +example_input_file4489.txt +example_input_file4490.txt +example_input_file4491.txt +example_input_file4492.txt +example_input_file4493.txt +example_input_file4494.txt +example_input_file4495.txt +example_input_file4496.txt +example_input_file4497.txt +example_input_file4498.txt +example_input_file4499.txt +example_input_file4500.txt +example_input_file4501.txt +example_input_file4502.txt +example_input_file4503.txt +example_input_file4504.txt +example_input_file4505.txt +example_input_file4506.txt +example_input_file4507.txt +example_input_file4508.txt +example_input_file4509.txt +example_input_file4510.txt +example_input_file4511.txt +example_input_file4512.txt +example_input_file4513.txt +example_input_file4514.txt +example_input_file4515.txt +example_input_file4516.txt +example_input_file4517.txt +example_input_file4518.txt +example_input_file4519.txt +example_input_file4520.txt +example_input_file4521.txt +example_input_file4522.txt +example_input_file4523.txt +example_input_file4524.txt +example_input_file4525.txt +example_input_file4526.txt +example_input_file4527.txt +example_input_file4528.txt +example_input_file4529.txt +example_input_file4530.txt +example_input_file4531.txt +example_input_file4532.txt +example_input_file4533.txt +example_input_file4534.txt +example_input_file4535.txt +example_input_file4536.txt +example_input_file4537.txt +example_input_file4538.txt +example_input_file4539.txt +example_input_file4540.txt +example_input_file4541.txt +example_input_file4542.txt +example_input_file4543.txt +example_input_file4544.txt +example_input_file4545.txt +example_input_file4546.txt +example_input_file4547.txt +example_input_file4548.txt +example_input_file4549.txt +example_input_file4550.txt +example_input_file4551.txt +example_input_file4552.txt +example_input_file4553.txt +example_input_file4554.txt +example_input_file4555.txt +example_input_file4556.txt +example_input_file4557.txt +example_input_file4558.txt +example_input_file4559.txt +example_input_file4560.txt +example_input_file4561.txt +example_input_file4562.txt +example_input_file4563.txt +example_input_file4564.txt +example_input_file4565.txt +example_input_file4566.txt +example_input_file4567.txt +example_input_file4568.txt +example_input_file4569.txt +example_input_file4570.txt +example_input_file4571.txt +example_input_file4572.txt +example_input_file4573.txt +example_input_file4574.txt +example_input_file4575.txt +example_input_file4576.txt +example_input_file4577.txt +example_input_file4578.txt +example_input_file4579.txt +example_input_file4580.txt +example_input_file4581.txt +example_input_file4582.txt +example_input_file4583.txt +example_input_file4584.txt +example_input_file4585.txt +example_input_file4586.txt +example_input_file4587.txt +example_input_file4588.txt +example_input_file4589.txt +example_input_file4590.txt +example_input_file4591.txt +example_input_file4592.txt +example_input_file4593.txt +example_input_file4594.txt +example_input_file4595.txt +example_input_file4596.txt +example_input_file4597.txt +example_input_file4598.txt +example_input_file4599.txt +example_input_file4600.txt +example_input_file4601.txt +example_input_file4602.txt +example_input_file4603.txt +example_input_file4604.txt +example_input_file4605.txt +example_input_file4606.txt +example_input_file4607.txt +example_input_file4608.txt +example_input_file4609.txt +example_input_file4610.txt +example_input_file4611.txt +example_input_file4612.txt +example_input_file4613.txt +example_input_file4614.txt +example_input_file4615.txt +example_input_file4616.txt +example_input_file4617.txt +example_input_file4618.txt +example_input_file4619.txt +example_input_file4620.txt +example_input_file4621.txt +example_input_file4622.txt +example_input_file4623.txt +example_input_file4624.txt +example_input_file4625.txt +example_input_file4626.txt +example_input_file4627.txt +example_input_file4628.txt +example_input_file4629.txt +example_input_file4630.txt +example_input_file4631.txt +example_input_file4632.txt +example_input_file4633.txt +example_input_file4634.txt +example_input_file4635.txt +example_input_file4636.txt +example_input_file4637.txt +example_input_file4638.txt +example_input_file4639.txt +example_input_file4640.txt +example_input_file4641.txt +example_input_file4642.txt +example_input_file4643.txt +example_input_file4644.txt +example_input_file4645.txt +example_input_file4646.txt +example_input_file4647.txt +example_input_file4648.txt +example_input_file4649.txt +example_input_file4650.txt +example_input_file4651.txt +example_input_file4652.txt +example_input_file4653.txt +example_input_file4654.txt +example_input_file4655.txt +example_input_file4656.txt +example_input_file4657.txt +example_input_file4658.txt +example_input_file4659.txt +example_input_file4660.txt +example_input_file4661.txt +example_input_file4662.txt +example_input_file4663.txt +example_input_file4664.txt +example_input_file4665.txt +example_input_file4666.txt +example_input_file4667.txt +example_input_file4668.txt +example_input_file4669.txt +example_input_file4670.txt +example_input_file4671.txt +example_input_file4672.txt +example_input_file4673.txt +example_input_file4674.txt +example_input_file4675.txt +example_input_file4676.txt +example_input_file4677.txt +example_input_file4678.txt +example_input_file4679.txt +example_input_file4680.txt +example_input_file4681.txt +example_input_file4682.txt +example_input_file4683.txt +example_input_file4684.txt +example_input_file4685.txt +example_input_file4686.txt +example_input_file4687.txt +example_input_file4688.txt +example_input_file4689.txt +example_input_file4690.txt +example_input_file4691.txt +example_input_file4692.txt +example_input_file4693.txt +example_input_file4694.txt +example_input_file4695.txt +example_input_file4696.txt +example_input_file4697.txt +example_input_file4698.txt +example_input_file4699.txt +example_input_file4700.txt +example_input_file4701.txt +example_input_file4702.txt +example_input_file4703.txt +example_input_file4704.txt +example_input_file4705.txt +example_input_file4706.txt +example_input_file4707.txt +example_input_file4708.txt +example_input_file4709.txt +example_input_file4710.txt +example_input_file4711.txt +example_input_file4712.txt +example_input_file4713.txt +example_input_file4714.txt +example_input_file4715.txt +example_input_file4716.txt +example_input_file4717.txt +example_input_file4718.txt +example_input_file4719.txt +example_input_file4720.txt +example_input_file4721.txt +example_input_file4722.txt +example_input_file4723.txt +example_input_file4724.txt +example_input_file4725.txt +example_input_file4726.txt +example_input_file4727.txt +example_input_file4728.txt +example_input_file4729.txt +example_input_file4730.txt +example_input_file4731.txt +example_input_file4732.txt +example_input_file4733.txt +example_input_file4734.txt +example_input_file4735.txt +example_input_file4736.txt +example_input_file4737.txt +example_input_file4738.txt +example_input_file4739.txt +example_input_file4740.txt +example_input_file4741.txt +example_input_file4742.txt +example_input_file4743.txt +example_input_file4744.txt +example_input_file4745.txt +example_input_file4746.txt +example_input_file4747.txt +example_input_file4748.txt +example_input_file4749.txt +example_input_file4750.txt +example_input_file4751.txt +example_input_file4752.txt +example_input_file4753.txt +example_input_file4754.txt +example_input_file4755.txt +example_input_file4756.txt +example_input_file4757.txt +example_input_file4758.txt +example_input_file4759.txt +example_input_file4760.txt +example_input_file4761.txt +example_input_file4762.txt +example_input_file4763.txt +example_input_file4764.txt +example_input_file4765.txt +example_input_file4766.txt +example_input_file4767.txt +example_input_file4768.txt +example_input_file4769.txt +example_input_file4770.txt +example_input_file4771.txt +example_input_file4772.txt +example_input_file4773.txt +example_input_file4774.txt +example_input_file4775.txt +example_input_file4776.txt +example_input_file4777.txt +example_input_file4778.txt +example_input_file4779.txt +example_input_file4780.txt +example_input_file4781.txt +example_input_file4782.txt +example_input_file4783.txt +example_input_file4784.txt +example_input_file4785.txt +example_input_file4786.txt +example_input_file4787.txt +example_input_file4788.txt +example_input_file4789.txt +example_input_file4790.txt +example_input_file4791.txt +example_input_file4792.txt +example_input_file4793.txt +example_input_file4794.txt +example_input_file4795.txt +example_input_file4796.txt +example_input_file4797.txt +example_input_file4798.txt +example_input_file4799.txt +example_input_file4800.txt +example_input_file4801.txt +example_input_file4802.txt +example_input_file4803.txt +example_input_file4804.txt +example_input_file4805.txt +example_input_file4806.txt +example_input_file4807.txt +example_input_file4808.txt +example_input_file4809.txt +example_input_file4810.txt +example_input_file4811.txt +example_input_file4812.txt +example_input_file4813.txt +example_input_file4814.txt +example_input_file4815.txt +example_input_file4816.txt +example_input_file4817.txt +example_input_file4818.txt +example_input_file4819.txt +example_input_file4820.txt +example_input_file4821.txt +example_input_file4822.txt +example_input_file4823.txt +example_input_file4824.txt +example_input_file4825.txt +example_input_file4826.txt +example_input_file4827.txt +example_input_file4828.txt +example_input_file4829.txt +example_input_file4830.txt +example_input_file4831.txt +example_input_file4832.txt +example_input_file4833.txt +example_input_file4834.txt +example_input_file4835.txt +example_input_file4836.txt +example_input_file4837.txt +example_input_file4838.txt +example_input_file4839.txt +example_input_file4840.txt +example_input_file4841.txt +example_input_file4842.txt +example_input_file4843.txt +example_input_file4844.txt +example_input_file4845.txt +example_input_file4846.txt +example_input_file4847.txt +example_input_file4848.txt +example_input_file4849.txt +example_input_file4850.txt +example_input_file4851.txt +example_input_file4852.txt +example_input_file4853.txt +example_input_file4854.txt +example_input_file4855.txt +example_input_file4856.txt +example_input_file4857.txt +example_input_file4858.txt +example_input_file4859.txt +example_input_file4860.txt +example_input_file4861.txt +example_input_file4862.txt +example_input_file4863.txt +example_input_file4864.txt +example_input_file4865.txt +example_input_file4866.txt +example_input_file4867.txt +example_input_file4868.txt +example_input_file4869.txt +example_input_file4870.txt +example_input_file4871.txt +example_input_file4872.txt +example_input_file4873.txt +example_input_file4874.txt +example_input_file4875.txt +example_input_file4876.txt +example_input_file4877.txt +example_input_file4878.txt +example_input_file4879.txt +example_input_file4880.txt +example_input_file4881.txt +example_input_file4882.txt +example_input_file4883.txt +example_input_file4884.txt +example_input_file4885.txt +example_input_file4886.txt +example_input_file4887.txt +example_input_file4888.txt +example_input_file4889.txt +example_input_file4890.txt +example_input_file4891.txt +example_input_file4892.txt +example_input_file4893.txt +example_input_file4894.txt +example_input_file4895.txt +example_input_file4896.txt +example_input_file4897.txt +example_input_file4898.txt +example_input_file4899.txt +example_input_file4900.txt +example_input_file4901.txt +example_input_file4902.txt +example_input_file4903.txt +example_input_file4904.txt +example_input_file4905.txt +example_input_file4906.txt +example_input_file4907.txt +example_input_file4908.txt +example_input_file4909.txt +example_input_file4910.txt +example_input_file4911.txt +example_input_file4912.txt +example_input_file4913.txt +example_input_file4914.txt +example_input_file4915.txt +example_input_file4916.txt +example_input_file4917.txt +example_input_file4918.txt +example_input_file4919.txt +example_input_file4920.txt +example_input_file4921.txt +example_input_file4922.txt +example_input_file4923.txt +example_input_file4924.txt +example_input_file4925.txt +example_input_file4926.txt +example_input_file4927.txt +example_input_file4928.txt +example_input_file4929.txt +example_input_file4930.txt +example_input_file4931.txt +example_input_file4932.txt +example_input_file4933.txt +example_input_file4934.txt +example_input_file4935.txt +example_input_file4936.txt +example_input_file4937.txt +example_input_file4938.txt +example_input_file4939.txt +example_input_file4940.txt +example_input_file4941.txt +example_input_file4942.txt +example_input_file4943.txt +example_input_file4944.txt +example_input_file4945.txt +example_input_file4946.txt +example_input_file4947.txt +example_input_file4948.txt +example_input_file4949.txt +example_input_file4950.txt +example_input_file4951.txt +example_input_file4952.txt +example_input_file4953.txt +example_input_file4954.txt +example_input_file4955.txt +example_input_file4956.txt +example_input_file4957.txt +example_input_file4958.txt +example_input_file4959.txt +example_input_file4960.txt +example_input_file4961.txt +example_input_file4962.txt +example_input_file4963.txt +example_input_file4964.txt +example_input_file4965.txt +example_input_file4966.txt +example_input_file4967.txt +example_input_file4968.txt +example_input_file4969.txt +example_input_file4970.txt +example_input_file4971.txt +example_input_file4972.txt +example_input_file4973.txt +example_input_file4974.txt +example_input_file4975.txt +example_input_file4976.txt +example_input_file4977.txt +example_input_file4978.txt +example_input_file4979.txt +example_input_file4980.txt +example_input_file4981.txt +example_input_file4982.txt +example_input_file4983.txt +example_input_file4984.txt +example_input_file4985.txt +example_input_file4986.txt +example_input_file4987.txt +example_input_file4988.txt +example_input_file4989.txt +example_input_file4990.txt +example_input_file4991.txt +example_input_file4992.txt +example_input_file4993.txt +example_input_file4994.txt +example_input_file4995.txt +example_input_file4996.txt +example_input_file4997.txt +example_input_file4998.txt +example_input_file4999.txt +example_input_file5000.txt +example_input_file5001.txt +example_input_file5002.txt +example_input_file5003.txt +example_input_file5004.txt +example_input_file5005.txt +example_input_file5006.txt +example_input_file5007.txt +example_input_file5008.txt +example_input_file5009.txt +example_input_file5010.txt +example_input_file5011.txt +example_input_file5012.txt +example_input_file5013.txt +example_input_file5014.txt +example_input_file5015.txt +example_input_file5016.txt +example_input_file5017.txt +example_input_file5018.txt +example_input_file5019.txt +example_input_file5020.txt +example_input_file5021.txt +example_input_file5022.txt +example_input_file5023.txt +example_input_file5024.txt +example_input_file5025.txt +example_input_file5026.txt +example_input_file5027.txt +example_input_file5028.txt +example_input_file5029.txt +example_input_file5030.txt +example_input_file5031.txt +example_input_file5032.txt +example_input_file5033.txt +example_input_file5034.txt +example_input_file5035.txt +example_input_file5036.txt +example_input_file5037.txt +example_input_file5038.txt +example_input_file5039.txt +example_input_file5040.txt +example_input_file5041.txt +example_input_file5042.txt +example_input_file5043.txt +example_input_file5044.txt +example_input_file5045.txt +example_input_file5046.txt +example_input_file5047.txt +example_input_file5048.txt +example_input_file5049.txt +example_input_file5050.txt +example_input_file5051.txt +example_input_file5052.txt +example_input_file5053.txt +example_input_file5054.txt +example_input_file5055.txt +example_input_file5056.txt +example_input_file5057.txt +example_input_file5058.txt +example_input_file5059.txt +example_input_file5060.txt +example_input_file5061.txt +example_input_file5062.txt +example_input_file5063.txt +example_input_file5064.txt +example_input_file5065.txt +example_input_file5066.txt +example_input_file5067.txt +example_input_file5068.txt +example_input_file5069.txt +example_input_file5070.txt +example_input_file5071.txt +example_input_file5072.txt +example_input_file5073.txt +example_input_file5074.txt +example_input_file5075.txt +example_input_file5076.txt +example_input_file5077.txt +example_input_file5078.txt +example_input_file5079.txt +example_input_file5080.txt +example_input_file5081.txt +example_input_file5082.txt +example_input_file5083.txt +example_input_file5084.txt +example_input_file5085.txt +example_input_file5086.txt +example_input_file5087.txt +example_input_file5088.txt +example_input_file5089.txt +example_input_file5090.txt +example_input_file5091.txt +example_input_file5092.txt +example_input_file5093.txt +example_input_file5094.txt +example_input_file5095.txt +example_input_file5096.txt +example_input_file5097.txt +example_input_file5098.txt +example_input_file5099.txt +example_input_file5100.txt +example_input_file5101.txt +example_input_file5102.txt +example_input_file5103.txt +example_input_file5104.txt +example_input_file5105.txt +example_input_file5106.txt +example_input_file5107.txt +example_input_file5108.txt +example_input_file5109.txt +example_input_file5110.txt +example_input_file5111.txt +example_input_file5112.txt +example_input_file5113.txt +example_input_file5114.txt +example_input_file5115.txt +example_input_file5116.txt +example_input_file5117.txt +example_input_file5118.txt +example_input_file5119.txt +example_input_file5120.txt +example_input_file5121.txt +example_input_file5122.txt +example_input_file5123.txt +example_input_file5124.txt +example_input_file5125.txt +example_input_file5126.txt +example_input_file5127.txt +example_input_file5128.txt +example_input_file5129.txt +example_input_file5130.txt +example_input_file5131.txt +example_input_file5132.txt +example_input_file5133.txt +example_input_file5134.txt +example_input_file5135.txt +example_input_file5136.txt +example_input_file5137.txt +example_input_file5138.txt +example_input_file5139.txt +example_input_file5140.txt +example_input_file5141.txt +example_input_file5142.txt +example_input_file5143.txt +example_input_file5144.txt +example_input_file5145.txt +example_input_file5146.txt +example_input_file5147.txt +example_input_file5148.txt +example_input_file5149.txt +example_input_file5150.txt +example_input_file5151.txt +example_input_file5152.txt +example_input_file5153.txt +example_input_file5154.txt +example_input_file5155.txt +example_input_file5156.txt +example_input_file5157.txt +example_input_file5158.txt +example_input_file5159.txt +example_input_file5160.txt +example_input_file5161.txt +example_input_file5162.txt +example_input_file5163.txt +example_input_file5164.txt +example_input_file5165.txt +example_input_file5166.txt +example_input_file5167.txt +example_input_file5168.txt +example_input_file5169.txt +example_input_file5170.txt +example_input_file5171.txt +example_input_file5172.txt +example_input_file5173.txt +example_input_file5174.txt +example_input_file5175.txt +example_input_file5176.txt +example_input_file5177.txt +example_input_file5178.txt +example_input_file5179.txt +example_input_file5180.txt +example_input_file5181.txt +example_input_file5182.txt +example_input_file5183.txt +example_input_file5184.txt +example_input_file5185.txt +example_input_file5186.txt +example_input_file5187.txt +example_input_file5188.txt +example_input_file5189.txt +example_input_file5190.txt +example_input_file5191.txt +example_input_file5192.txt +example_input_file5193.txt +example_input_file5194.txt +example_input_file5195.txt +example_input_file5196.txt +example_input_file5197.txt +example_input_file5198.txt +example_input_file5199.txt +example_input_file5200.txt +example_input_file5201.txt +example_input_file5202.txt +example_input_file5203.txt +example_input_file5204.txt +example_input_file5205.txt +example_input_file5206.txt +example_input_file5207.txt +example_input_file5208.txt +example_input_file5209.txt +example_input_file5210.txt +example_input_file5211.txt +example_input_file5212.txt +example_input_file5213.txt +example_input_file5214.txt +example_input_file5215.txt +example_input_file5216.txt +example_input_file5217.txt +example_input_file5218.txt +example_input_file5219.txt +example_input_file5220.txt +example_input_file5221.txt +example_input_file5222.txt +example_input_file5223.txt +example_input_file5224.txt +example_input_file5225.txt +example_input_file5226.txt +example_input_file5227.txt +example_input_file5228.txt +example_input_file5229.txt +example_input_file5230.txt +example_input_file5231.txt +example_input_file5232.txt +example_input_file5233.txt +example_input_file5234.txt +example_input_file5235.txt +example_input_file5236.txt +example_input_file5237.txt +example_input_file5238.txt +example_input_file5239.txt +example_input_file5240.txt +example_input_file5241.txt +example_input_file5242.txt +example_input_file5243.txt +example_input_file5244.txt +example_input_file5245.txt +example_input_file5246.txt +example_input_file5247.txt +example_input_file5248.txt +example_input_file5249.txt +example_input_file5250.txt +example_input_file5251.txt +example_input_file5252.txt +example_input_file5253.txt +example_input_file5254.txt +example_input_file5255.txt +example_input_file5256.txt +example_input_file5257.txt +example_input_file5258.txt +example_input_file5259.txt +example_input_file5260.txt +example_input_file5261.txt +example_input_file5262.txt +example_input_file5263.txt +example_input_file5264.txt +example_input_file5265.txt +example_input_file5266.txt +example_input_file5267.txt +example_input_file5268.txt +example_input_file5269.txt +example_input_file5270.txt +example_input_file5271.txt +example_input_file5272.txt +example_input_file5273.txt +example_input_file5274.txt +example_input_file5275.txt +example_input_file5276.txt +example_input_file5277.txt +example_input_file5278.txt +example_input_file5279.txt +example_input_file5280.txt +example_input_file5281.txt +example_input_file5282.txt +example_input_file5283.txt +example_input_file5284.txt +example_input_file5285.txt +example_input_file5286.txt +example_input_file5287.txt +example_input_file5288.txt +example_input_file5289.txt +example_input_file5290.txt +example_input_file5291.txt +example_input_file5292.txt +example_input_file5293.txt +example_input_file5294.txt +example_input_file5295.txt +example_input_file5296.txt +example_input_file5297.txt +example_input_file5298.txt +example_input_file5299.txt +example_input_file5300.txt +example_input_file5301.txt +example_input_file5302.txt +example_input_file5303.txt +example_input_file5304.txt +example_input_file5305.txt +example_input_file5306.txt +example_input_file5307.txt +example_input_file5308.txt +example_input_file5309.txt +example_input_file5310.txt +example_input_file5311.txt +example_input_file5312.txt +example_input_file5313.txt +example_input_file5314.txt +example_input_file5315.txt +example_input_file5316.txt +example_input_file5317.txt +example_input_file5318.txt +example_input_file5319.txt +example_input_file5320.txt +example_input_file5321.txt +example_input_file5322.txt +example_input_file5323.txt +example_input_file5324.txt +example_input_file5325.txt +example_input_file5326.txt +example_input_file5327.txt +example_input_file5328.txt +example_input_file5329.txt +example_input_file5330.txt +example_input_file5331.txt +example_input_file5332.txt +example_input_file5333.txt +example_input_file5334.txt +example_input_file5335.txt +example_input_file5336.txt +example_input_file5337.txt +example_input_file5338.txt +example_input_file5339.txt +example_input_file5340.txt +example_input_file5341.txt +example_input_file5342.txt +example_input_file5343.txt +example_input_file5344.txt +example_input_file5345.txt +example_input_file5346.txt +example_input_file5347.txt +example_input_file5348.txt +example_input_file5349.txt +example_input_file5350.txt +example_input_file5351.txt +example_input_file5352.txt +example_input_file5353.txt +example_input_file5354.txt +example_input_file5355.txt +example_input_file5356.txt +example_input_file5357.txt +example_input_file5358.txt +example_input_file5359.txt +example_input_file5360.txt +example_input_file5361.txt +example_input_file5362.txt +example_input_file5363.txt +example_input_file5364.txt +example_input_file5365.txt +example_input_file5366.txt +example_input_file5367.txt +example_input_file5368.txt +example_input_file5369.txt +example_input_file5370.txt +example_input_file5371.txt +example_input_file5372.txt +example_input_file5373.txt +example_input_file5374.txt +example_input_file5375.txt +example_input_file5376.txt +example_input_file5377.txt +example_input_file5378.txt +example_input_file5379.txt +example_input_file5380.txt +example_input_file5381.txt +example_input_file5382.txt +example_input_file5383.txt +example_input_file5384.txt +example_input_file5385.txt +example_input_file5386.txt +example_input_file5387.txt +example_input_file5388.txt +example_input_file5389.txt +example_input_file5390.txt +example_input_file5391.txt +example_input_file5392.txt +example_input_file5393.txt +example_input_file5394.txt +example_input_file5395.txt +example_input_file5396.txt +example_input_file5397.txt +example_input_file5398.txt +example_input_file5399.txt +example_input_file5400.txt +example_input_file5401.txt +example_input_file5402.txt +example_input_file5403.txt +example_input_file5404.txt +example_input_file5405.txt +example_input_file5406.txt +example_input_file5407.txt +example_input_file5408.txt +example_input_file5409.txt +example_input_file5410.txt +example_input_file5411.txt +example_input_file5412.txt +example_input_file5413.txt +example_input_file5414.txt +example_input_file5415.txt +example_input_file5416.txt +example_input_file5417.txt +example_input_file5418.txt +example_input_file5419.txt +example_input_file5420.txt +example_input_file5421.txt +example_input_file5422.txt +example_input_file5423.txt +example_input_file5424.txt +example_input_file5425.txt +example_input_file5426.txt +example_input_file5427.txt +example_input_file5428.txt +example_input_file5429.txt +example_input_file5430.txt +example_input_file5431.txt +example_input_file5432.txt +example_input_file5433.txt +example_input_file5434.txt +example_input_file5435.txt +example_input_file5436.txt +example_input_file5437.txt +example_input_file5438.txt +example_input_file5439.txt +example_input_file5440.txt +example_input_file5441.txt +example_input_file5442.txt +example_input_file5443.txt +example_input_file5444.txt +example_input_file5445.txt +example_input_file5446.txt +example_input_file5447.txt +example_input_file5448.txt +example_input_file5449.txt +example_input_file5450.txt +example_input_file5451.txt +example_input_file5452.txt +example_input_file5453.txt +example_input_file5454.txt +example_input_file5455.txt +example_input_file5456.txt +example_input_file5457.txt +example_input_file5458.txt +example_input_file5459.txt +example_input_file5460.txt +example_input_file5461.txt +example_input_file5462.txt +example_input_file5463.txt +example_input_file5464.txt +example_input_file5465.txt +example_input_file5466.txt +example_input_file5467.txt +example_input_file5468.txt +example_input_file5469.txt +example_input_file5470.txt +example_input_file5471.txt +example_input_file5472.txt +example_input_file5473.txt +example_input_file5474.txt +example_input_file5475.txt +example_input_file5476.txt +example_input_file5477.txt +example_input_file5478.txt +example_input_file5479.txt +example_input_file5480.txt +example_input_file5481.txt +example_input_file5482.txt +example_input_file5483.txt +example_input_file5484.txt +example_input_file5485.txt +example_input_file5486.txt +example_input_file5487.txt +example_input_file5488.txt +example_input_file5489.txt +example_input_file5490.txt +example_input_file5491.txt +example_input_file5492.txt +example_input_file5493.txt +example_input_file5494.txt +example_input_file5495.txt +example_input_file5496.txt +example_input_file5497.txt +example_input_file5498.txt +example_input_file5499.txt +example_input_file5500.txt +example_input_file5501.txt +example_input_file5502.txt +example_input_file5503.txt +example_input_file5504.txt +example_input_file5505.txt +example_input_file5506.txt +example_input_file5507.txt +example_input_file5508.txt +example_input_file5509.txt +example_input_file5510.txt +example_input_file5511.txt +example_input_file5512.txt +example_input_file5513.txt +example_input_file5514.txt +example_input_file5515.txt +example_input_file5516.txt +example_input_file5517.txt +example_input_file5518.txt +example_input_file5519.txt +example_input_file5520.txt +example_input_file5521.txt +example_input_file5522.txt +example_input_file5523.txt +example_input_file5524.txt +example_input_file5525.txt +example_input_file5526.txt +example_input_file5527.txt +example_input_file5528.txt +example_input_file5529.txt +example_input_file5530.txt +example_input_file5531.txt +example_input_file5532.txt +example_input_file5533.txt +example_input_file5534.txt +example_input_file5535.txt +example_input_file5536.txt +example_input_file5537.txt +example_input_file5538.txt +example_input_file5539.txt +example_input_file5540.txt +example_input_file5541.txt +example_input_file5542.txt +example_input_file5543.txt +example_input_file5544.txt +example_input_file5545.txt +example_input_file5546.txt +example_input_file5547.txt +example_input_file5548.txt +example_input_file5549.txt +example_input_file5550.txt +example_input_file5551.txt +example_input_file5552.txt +example_input_file5553.txt +example_input_file5554.txt +example_input_file5555.txt +example_input_file5556.txt +example_input_file5557.txt +example_input_file5558.txt +example_input_file5559.txt +example_input_file5560.txt +example_input_file5561.txt +example_input_file5562.txt +example_input_file5563.txt +example_input_file5564.txt +example_input_file5565.txt +example_input_file5566.txt +example_input_file5567.txt +example_input_file5568.txt +example_input_file5569.txt +example_input_file5570.txt +example_input_file5571.txt +example_input_file5572.txt +example_input_file5573.txt +example_input_file5574.txt +example_input_file5575.txt +example_input_file5576.txt +example_input_file5577.txt +example_input_file5578.txt +example_input_file5579.txt +example_input_file5580.txt +example_input_file5581.txt +example_input_file5582.txt +example_input_file5583.txt +example_input_file5584.txt +example_input_file5585.txt +example_input_file5586.txt +example_input_file5587.txt +example_input_file5588.txt +example_input_file5589.txt +example_input_file5590.txt +example_input_file5591.txt +example_input_file5592.txt +example_input_file5593.txt +example_input_file5594.txt +example_input_file5595.txt +example_input_file5596.txt +example_input_file5597.txt +example_input_file5598.txt +example_input_file5599.txt +example_input_file5600.txt +example_input_file5601.txt +example_input_file5602.txt +example_input_file5603.txt +example_input_file5604.txt +example_input_file5605.txt +example_input_file5606.txt +example_input_file5607.txt +example_input_file5608.txt +example_input_file5609.txt +example_input_file5610.txt +example_input_file5611.txt +example_input_file5612.txt +example_input_file5613.txt +example_input_file5614.txt +example_input_file5615.txt +example_input_file5616.txt +example_input_file5617.txt +example_input_file5618.txt +example_input_file5619.txt +example_input_file5620.txt +example_input_file5621.txt +example_input_file5622.txt +example_input_file5623.txt +example_input_file5624.txt +example_input_file5625.txt +example_input_file5626.txt +example_input_file5627.txt +example_input_file5628.txt +example_input_file5629.txt +example_input_file5630.txt +example_input_file5631.txt +example_input_file5632.txt +example_input_file5633.txt +example_input_file5634.txt +example_input_file5635.txt +example_input_file5636.txt +example_input_file5637.txt +example_input_file5638.txt +example_input_file5639.txt +example_input_file5640.txt +example_input_file5641.txt +example_input_file5642.txt +example_input_file5643.txt +example_input_file5644.txt +example_input_file5645.txt +example_input_file5646.txt +example_input_file5647.txt +example_input_file5648.txt +example_input_file5649.txt +example_input_file5650.txt +example_input_file5651.txt +example_input_file5652.txt +example_input_file5653.txt +example_input_file5654.txt +example_input_file5655.txt +example_input_file5656.txt +example_input_file5657.txt +example_input_file5658.txt +example_input_file5659.txt +example_input_file5660.txt +example_input_file5661.txt +example_input_file5662.txt +example_input_file5663.txt +example_input_file5664.txt +example_input_file5665.txt +example_input_file5666.txt +example_input_file5667.txt +example_input_file5668.txt +example_input_file5669.txt +example_input_file5670.txt +example_input_file5671.txt +example_input_file5672.txt +example_input_file5673.txt +example_input_file5674.txt +example_input_file5675.txt +example_input_file5676.txt +example_input_file5677.txt +example_input_file5678.txt +example_input_file5679.txt +example_input_file5680.txt +example_input_file5681.txt +example_input_file5682.txt +example_input_file5683.txt +example_input_file5684.txt +example_input_file5685.txt +example_input_file5686.txt +example_input_file5687.txt +example_input_file5688.txt +example_input_file5689.txt +example_input_file5690.txt +example_input_file5691.txt +example_input_file5692.txt +example_input_file5693.txt +example_input_file5694.txt +example_input_file5695.txt +example_input_file5696.txt +example_input_file5697.txt +example_input_file5698.txt +example_input_file5699.txt +example_input_file5700.txt +example_input_file5701.txt +example_input_file5702.txt +example_input_file5703.txt +example_input_file5704.txt +example_input_file5705.txt +example_input_file5706.txt +example_input_file5707.txt +example_input_file5708.txt +example_input_file5709.txt +example_input_file5710.txt +example_input_file5711.txt +example_input_file5712.txt +example_input_file5713.txt +example_input_file5714.txt +example_input_file5715.txt +example_input_file5716.txt +example_input_file5717.txt +example_input_file5718.txt +example_input_file5719.txt +example_input_file5720.txt +example_input_file5721.txt +example_input_file5722.txt +example_input_file5723.txt +example_input_file5724.txt +example_input_file5725.txt +example_input_file5726.txt +example_input_file5727.txt +example_input_file5728.txt +example_input_file5729.txt +example_input_file5730.txt +example_input_file5731.txt +example_input_file5732.txt +example_input_file5733.txt +example_input_file5734.txt +example_input_file5735.txt +example_input_file5736.txt +example_input_file5737.txt +example_input_file5738.txt +example_input_file5739.txt +example_input_file5740.txt +example_input_file5741.txt +example_input_file5742.txt +example_input_file5743.txt +example_input_file5744.txt +example_input_file5745.txt +example_input_file5746.txt +example_input_file5747.txt +example_input_file5748.txt +example_input_file5749.txt +example_input_file5750.txt +example_input_file5751.txt +example_input_file5752.txt +example_input_file5753.txt +example_input_file5754.txt +example_input_file5755.txt +example_input_file5756.txt +example_input_file5757.txt +example_input_file5758.txt +example_input_file5759.txt +example_input_file5760.txt +example_input_file5761.txt +example_input_file5762.txt +example_input_file5763.txt +example_input_file5764.txt +example_input_file5765.txt +example_input_file5766.txt +example_input_file5767.txt +example_input_file5768.txt +example_input_file5769.txt +example_input_file5770.txt +example_input_file5771.txt +example_input_file5772.txt +example_input_file5773.txt +example_input_file5774.txt +example_input_file5775.txt +example_input_file5776.txt +example_input_file5777.txt +example_input_file5778.txt +example_input_file5779.txt +example_input_file5780.txt +example_input_file5781.txt +example_input_file5782.txt +example_input_file5783.txt +example_input_file5784.txt +example_input_file5785.txt +example_input_file5786.txt +example_input_file5787.txt +example_input_file5788.txt +example_input_file5789.txt +example_input_file5790.txt +example_input_file5791.txt +example_input_file5792.txt +example_input_file5793.txt +example_input_file5794.txt +example_input_file5795.txt +example_input_file5796.txt +example_input_file5797.txt +example_input_file5798.txt +example_input_file5799.txt +example_input_file5800.txt +example_input_file5801.txt +example_input_file5802.txt +example_input_file5803.txt +example_input_file5804.txt +example_input_file5805.txt +example_input_file5806.txt +example_input_file5807.txt +example_input_file5808.txt +example_input_file5809.txt +example_input_file5810.txt +example_input_file5811.txt +example_input_file5812.txt +example_input_file5813.txt +example_input_file5814.txt +example_input_file5815.txt +example_input_file5816.txt +example_input_file5817.txt +example_input_file5818.txt +example_input_file5819.txt +example_input_file5820.txt +example_input_file5821.txt +example_input_file5822.txt +example_input_file5823.txt +example_input_file5824.txt +example_input_file5825.txt +example_input_file5826.txt +example_input_file5827.txt +example_input_file5828.txt +example_input_file5829.txt +example_input_file5830.txt +example_input_file5831.txt +example_input_file5832.txt +example_input_file5833.txt +example_input_file5834.txt +example_input_file5835.txt +example_input_file5836.txt +example_input_file5837.txt +example_input_file5838.txt +example_input_file5839.txt +example_input_file5840.txt +example_input_file5841.txt +example_input_file5842.txt +example_input_file5843.txt +example_input_file5844.txt +example_input_file5845.txt +example_input_file5846.txt +example_input_file5847.txt +example_input_file5848.txt +example_input_file5849.txt +example_input_file5850.txt +example_input_file5851.txt +example_input_file5852.txt +example_input_file5853.txt +example_input_file5854.txt +example_input_file5855.txt +example_input_file5856.txt +example_input_file5857.txt +example_input_file5858.txt +example_input_file5859.txt +example_input_file5860.txt +example_input_file5861.txt +example_input_file5862.txt +example_input_file5863.txt +example_input_file5864.txt +example_input_file5865.txt +example_input_file5866.txt +example_input_file5867.txt +example_input_file5868.txt +example_input_file5869.txt +example_input_file5870.txt +example_input_file5871.txt +example_input_file5872.txt +example_input_file5873.txt +example_input_file5874.txt +example_input_file5875.txt +example_input_file5876.txt +example_input_file5877.txt +example_input_file5878.txt +example_input_file5879.txt +example_input_file5880.txt +example_input_file5881.txt +example_input_file5882.txt +example_input_file5883.txt +example_input_file5884.txt +example_input_file5885.txt +example_input_file5886.txt +example_input_file5887.txt +example_input_file5888.txt +example_input_file5889.txt +example_input_file5890.txt +example_input_file5891.txt +example_input_file5892.txt +example_input_file5893.txt +example_input_file5894.txt +example_input_file5895.txt +example_input_file5896.txt +example_input_file5897.txt +example_input_file5898.txt +example_input_file5899.txt +example_input_file5900.txt +example_input_file5901.txt +example_input_file5902.txt +example_input_file5903.txt +example_input_file5904.txt +example_input_file5905.txt +example_input_file5906.txt +example_input_file5907.txt +example_input_file5908.txt +example_input_file5909.txt +example_input_file5910.txt +example_input_file5911.txt +example_input_file5912.txt +example_input_file5913.txt +example_input_file5914.txt +example_input_file5915.txt +example_input_file5916.txt +example_input_file5917.txt +example_input_file5918.txt +example_input_file5919.txt +example_input_file5920.txt +example_input_file5921.txt +example_input_file5922.txt +example_input_file5923.txt +example_input_file5924.txt +example_input_file5925.txt +example_input_file5926.txt +example_input_file5927.txt +example_input_file5928.txt +example_input_file5929.txt +example_input_file5930.txt +example_input_file5931.txt +example_input_file5932.txt +example_input_file5933.txt +example_input_file5934.txt +example_input_file5935.txt +example_input_file5936.txt +example_input_file5937.txt +example_input_file5938.txt +example_input_file5939.txt +example_input_file5940.txt +example_input_file5941.txt +example_input_file5942.txt +example_input_file5943.txt +example_input_file5944.txt +example_input_file5945.txt +example_input_file5946.txt +example_input_file5947.txt +example_input_file5948.txt +example_input_file5949.txt +example_input_file5950.txt +example_input_file5951.txt +example_input_file5952.txt +example_input_file5953.txt +example_input_file5954.txt +example_input_file5955.txt +example_input_file5956.txt +example_input_file5957.txt +example_input_file5958.txt +example_input_file5959.txt +example_input_file5960.txt +example_input_file5961.txt +example_input_file5962.txt +example_input_file5963.txt +example_input_file5964.txt +example_input_file5965.txt +example_input_file5966.txt +example_input_file5967.txt +example_input_file5968.txt +example_input_file5969.txt +example_input_file5970.txt +example_input_file5971.txt +example_input_file5972.txt +example_input_file5973.txt +example_input_file5974.txt +example_input_file5975.txt +example_input_file5976.txt +example_input_file5977.txt +example_input_file5978.txt +example_input_file5979.txt +example_input_file5980.txt +example_input_file5981.txt +example_input_file5982.txt +example_input_file5983.txt +example_input_file5984.txt +example_input_file5985.txt +example_input_file5986.txt +example_input_file5987.txt +example_input_file5988.txt +example_input_file5989.txt +example_input_file5990.txt +example_input_file5991.txt +example_input_file5992.txt +example_input_file5993.txt +example_input_file5994.txt +example_input_file5995.txt +example_input_file5996.txt +example_input_file5997.txt +example_input_file5998.txt +example_input_file5999.txt +example_input_file6000.txt +example_input_file6001.txt +example_input_file6002.txt +example_input_file6003.txt +example_input_file6004.txt +example_input_file6005.txt +example_input_file6006.txt +example_input_file6007.txt +example_input_file6008.txt +example_input_file6009.txt +example_input_file6010.txt +example_input_file6011.txt +example_input_file6012.txt +example_input_file6013.txt +example_input_file6014.txt +example_input_file6015.txt +example_input_file6016.txt +example_input_file6017.txt +example_input_file6018.txt +example_input_file6019.txt +example_input_file6020.txt +example_input_file6021.txt +example_input_file6022.txt +example_input_file6023.txt +example_input_file6024.txt +example_input_file6025.txt +example_input_file6026.txt +example_input_file6027.txt +example_input_file6028.txt +example_input_file6029.txt +example_input_file6030.txt +example_input_file6031.txt +example_input_file6032.txt +example_input_file6033.txt +example_input_file6034.txt +example_input_file6035.txt +example_input_file6036.txt +example_input_file6037.txt +example_input_file6038.txt +example_input_file6039.txt +example_input_file6040.txt +example_input_file6041.txt +example_input_file6042.txt +example_input_file6043.txt +example_input_file6044.txt +example_input_file6045.txt +example_input_file6046.txt +example_input_file6047.txt +example_input_file6048.txt +example_input_file6049.txt +example_input_file6050.txt +example_input_file6051.txt +example_input_file6052.txt +example_input_file6053.txt +example_input_file6054.txt +example_input_file6055.txt +example_input_file6056.txt +example_input_file6057.txt +example_input_file6058.txt +example_input_file6059.txt +example_input_file6060.txt +example_input_file6061.txt +example_input_file6062.txt +example_input_file6063.txt +example_input_file6064.txt +example_input_file6065.txt +example_input_file6066.txt +example_input_file6067.txt +example_input_file6068.txt +example_input_file6069.txt +example_input_file6070.txt +example_input_file6071.txt +example_input_file6072.txt +example_input_file6073.txt +example_input_file6074.txt +example_input_file6075.txt +example_input_file6076.txt +example_input_file6077.txt +example_input_file6078.txt +example_input_file6079.txt +example_input_file6080.txt +example_input_file6081.txt +example_input_file6082.txt +example_input_file6083.txt +example_input_file6084.txt +example_input_file6085.txt +example_input_file6086.txt +example_input_file6087.txt +example_input_file6088.txt +example_input_file6089.txt +example_input_file6090.txt +example_input_file6091.txt +example_input_file6092.txt +example_input_file6093.txt +example_input_file6094.txt +example_input_file6095.txt +example_input_file6096.txt +example_input_file6097.txt +example_input_file6098.txt +example_input_file6099.txt +example_input_file6100.txt +example_input_file6101.txt +example_input_file6102.txt +example_input_file6103.txt +example_input_file6104.txt +example_input_file6105.txt +example_input_file6106.txt +example_input_file6107.txt +example_input_file6108.txt +example_input_file6109.txt +example_input_file6110.txt +example_input_file6111.txt +example_input_file6112.txt +example_input_file6113.txt +example_input_file6114.txt +example_input_file6115.txt +example_input_file6116.txt +example_input_file6117.txt +example_input_file6118.txt +example_input_file6119.txt +example_input_file6120.txt +example_input_file6121.txt +example_input_file6122.txt +example_input_file6123.txt +example_input_file6124.txt +example_input_file6125.txt +example_input_file6126.txt +example_input_file6127.txt +example_input_file6128.txt +example_input_file6129.txt +example_input_file6130.txt +example_input_file6131.txt +example_input_file6132.txt +example_input_file6133.txt +example_input_file6134.txt +example_input_file6135.txt +example_input_file6136.txt +example_input_file6137.txt +example_input_file6138.txt +example_input_file6139.txt +example_input_file6140.txt +example_input_file6141.txt +example_input_file6142.txt +example_input_file6143.txt +example_input_file6144.txt +example_input_file6145.txt +example_input_file6146.txt +example_input_file6147.txt +example_input_file6148.txt +example_input_file6149.txt +example_input_file6150.txt +example_input_file6151.txt +example_input_file6152.txt +example_input_file6153.txt +example_input_file6154.txt +example_input_file6155.txt +example_input_file6156.txt +example_input_file6157.txt +example_input_file6158.txt +example_input_file6159.txt +example_input_file6160.txt +example_input_file6161.txt +example_input_file6162.txt +example_input_file6163.txt +example_input_file6164.txt +example_input_file6165.txt +example_input_file6166.txt +example_input_file6167.txt +example_input_file6168.txt +example_input_file6169.txt +example_input_file6170.txt +example_input_file6171.txt +example_input_file6172.txt +example_input_file6173.txt +example_input_file6174.txt +example_input_file6175.txt +example_input_file6176.txt +example_input_file6177.txt +example_input_file6178.txt +example_input_file6179.txt +example_input_file6180.txt +example_input_file6181.txt +example_input_file6182.txt +example_input_file6183.txt +example_input_file6184.txt +example_input_file6185.txt +example_input_file6186.txt +example_input_file6187.txt +example_input_file6188.txt +example_input_file6189.txt +example_input_file6190.txt +example_input_file6191.txt +example_input_file6192.txt +example_input_file6193.txt +example_input_file6194.txt +example_input_file6195.txt +example_input_file6196.txt +example_input_file6197.txt +example_input_file6198.txt +example_input_file6199.txt +example_input_file6200.txt +example_input_file6201.txt +example_input_file6202.txt +example_input_file6203.txt +example_input_file6204.txt +example_input_file6205.txt +example_input_file6206.txt +example_input_file6207.txt +example_input_file6208.txt +example_input_file6209.txt +example_input_file6210.txt +example_input_file6211.txt +example_input_file6212.txt +example_input_file6213.txt +example_input_file6214.txt +example_input_file6215.txt +example_input_file6216.txt +example_input_file6217.txt +example_input_file6218.txt +example_input_file6219.txt +example_input_file6220.txt +example_input_file6221.txt +example_input_file6222.txt +example_input_file6223.txt +example_input_file6224.txt +example_input_file6225.txt +example_input_file6226.txt +example_input_file6227.txt +example_input_file6228.txt +example_input_file6229.txt +example_input_file6230.txt +example_input_file6231.txt +example_input_file6232.txt +example_input_file6233.txt +example_input_file6234.txt +example_input_file6235.txt +example_input_file6236.txt +example_input_file6237.txt +example_input_file6238.txt +example_input_file6239.txt +example_input_file6240.txt +example_input_file6241.txt +example_input_file6242.txt +example_input_file6243.txt +example_input_file6244.txt +example_input_file6245.txt +example_input_file6246.txt +example_input_file6247.txt +example_input_file6248.txt +example_input_file6249.txt +example_input_file6250.txt +example_input_file6251.txt +example_input_file6252.txt +example_input_file6253.txt +example_input_file6254.txt +example_input_file6255.txt +example_input_file6256.txt +example_input_file6257.txt +example_input_file6258.txt +example_input_file6259.txt +example_input_file6260.txt +example_input_file6261.txt +example_input_file6262.txt +example_input_file6263.txt +example_input_file6264.txt +example_input_file6265.txt +example_input_file6266.txt +example_input_file6267.txt +example_input_file6268.txt +example_input_file6269.txt +example_input_file6270.txt +example_input_file6271.txt +example_input_file6272.txt +example_input_file6273.txt +example_input_file6274.txt +example_input_file6275.txt +example_input_file6276.txt +example_input_file6277.txt +example_input_file6278.txt +example_input_file6279.txt +example_input_file6280.txt +example_input_file6281.txt +example_input_file6282.txt +example_input_file6283.txt +example_input_file6284.txt +example_input_file6285.txt +example_input_file6286.txt +example_input_file6287.txt +example_input_file6288.txt +example_input_file6289.txt +example_input_file6290.txt +example_input_file6291.txt +example_input_file6292.txt +example_input_file6293.txt +example_input_file6294.txt +example_input_file6295.txt +example_input_file6296.txt +example_input_file6297.txt +example_input_file6298.txt +example_input_file6299.txt +example_input_file6300.txt +example_input_file6301.txt +example_input_file6302.txt +example_input_file6303.txt +example_input_file6304.txt +example_input_file6305.txt +example_input_file6306.txt +example_input_file6307.txt +example_input_file6308.txt +example_input_file6309.txt +example_input_file6310.txt +example_input_file6311.txt +example_input_file6312.txt +example_input_file6313.txt +example_input_file6314.txt +example_input_file6315.txt +example_input_file6316.txt +example_input_file6317.txt +example_input_file6318.txt +example_input_file6319.txt +example_input_file6320.txt +example_input_file6321.txt +example_input_file6322.txt +example_input_file6323.txt +example_input_file6324.txt +example_input_file6325.txt +example_input_file6326.txt +example_input_file6327.txt +example_input_file6328.txt +example_input_file6329.txt +example_input_file6330.txt +example_input_file6331.txt +example_input_file6332.txt +example_input_file6333.txt +example_input_file6334.txt +example_input_file6335.txt +example_input_file6336.txt +example_input_file6337.txt +example_input_file6338.txt +example_input_file6339.txt +example_input_file6340.txt +example_input_file6341.txt +example_input_file6342.txt +example_input_file6343.txt +example_input_file6344.txt +example_input_file6345.txt +example_input_file6346.txt +example_input_file6347.txt +example_input_file6348.txt +example_input_file6349.txt +example_input_file6350.txt +example_input_file6351.txt +example_input_file6352.txt +example_input_file6353.txt +example_input_file6354.txt +example_input_file6355.txt +example_input_file6356.txt +example_input_file6357.txt +example_input_file6358.txt +example_input_file6359.txt +example_input_file6360.txt +example_input_file6361.txt +example_input_file6362.txt +example_input_file6363.txt +example_input_file6364.txt +example_input_file6365.txt +example_input_file6366.txt +example_input_file6367.txt +example_input_file6368.txt +example_input_file6369.txt +example_input_file6370.txt +example_input_file6371.txt +example_input_file6372.txt +example_input_file6373.txt +example_input_file6374.txt +example_input_file6375.txt +example_input_file6376.txt +example_input_file6377.txt +example_input_file6378.txt +example_input_file6379.txt +example_input_file6380.txt +example_input_file6381.txt +example_input_file6382.txt +example_input_file6383.txt +example_input_file6384.txt +example_input_file6385.txt +example_input_file6386.txt +example_input_file6387.txt +example_input_file6388.txt +example_input_file6389.txt +example_input_file6390.txt +example_input_file6391.txt +example_input_file6392.txt +example_input_file6393.txt +example_input_file6394.txt +example_input_file6395.txt +example_input_file6396.txt +example_input_file6397.txt +example_input_file6398.txt +example_input_file6399.txt +example_input_file6400.txt +example_input_file6401.txt +example_input_file6402.txt +example_input_file6403.txt +example_input_file6404.txt +example_input_file6405.txt +example_input_file6406.txt +example_input_file6407.txt +example_input_file6408.txt +example_input_file6409.txt +example_input_file6410.txt +example_input_file6411.txt +example_input_file6412.txt +example_input_file6413.txt +example_input_file6414.txt +example_input_file6415.txt +example_input_file6416.txt +example_input_file6417.txt +example_input_file6418.txt +example_input_file6419.txt +example_input_file6420.txt +example_input_file6421.txt +example_input_file6422.txt +example_input_file6423.txt +example_input_file6424.txt +example_input_file6425.txt +example_input_file6426.txt +example_input_file6427.txt +example_input_file6428.txt +example_input_file6429.txt +example_input_file6430.txt +example_input_file6431.txt +example_input_file6432.txt +example_input_file6433.txt +example_input_file6434.txt +example_input_file6435.txt +example_input_file6436.txt +example_input_file6437.txt +example_input_file6438.txt +example_input_file6439.txt +example_input_file6440.txt +example_input_file6441.txt +example_input_file6442.txt +example_input_file6443.txt +example_input_file6444.txt +example_input_file6445.txt +example_input_file6446.txt +example_input_file6447.txt +example_input_file6448.txt +example_input_file6449.txt +example_input_file6450.txt +example_input_file6451.txt +example_input_file6452.txt +example_input_file6453.txt +example_input_file6454.txt +example_input_file6455.txt +example_input_file6456.txt +example_input_file6457.txt +example_input_file6458.txt +example_input_file6459.txt +example_input_file6460.txt +example_input_file6461.txt +example_input_file6462.txt +example_input_file6463.txt +example_input_file6464.txt +example_input_file6465.txt +example_input_file6466.txt +example_input_file6467.txt +example_input_file6468.txt +example_input_file6469.txt +example_input_file6470.txt +example_input_file6471.txt +example_input_file6472.txt +example_input_file6473.txt +example_input_file6474.txt +example_input_file6475.txt +example_input_file6476.txt +example_input_file6477.txt +example_input_file6478.txt +example_input_file6479.txt +example_input_file6480.txt +example_input_file6481.txt +example_input_file6482.txt +example_input_file6483.txt +example_input_file6484.txt +example_input_file6485.txt +example_input_file6486.txt +example_input_file6487.txt +example_input_file6488.txt +example_input_file6489.txt +example_input_file6490.txt +example_input_file6491.txt +example_input_file6492.txt +example_input_file6493.txt +example_input_file6494.txt +example_input_file6495.txt +example_input_file6496.txt +example_input_file6497.txt +example_input_file6498.txt +example_input_file6499.txt +example_input_file6500.txt +example_input_file6501.txt +example_input_file6502.txt +example_input_file6503.txt +example_input_file6504.txt +example_input_file6505.txt +example_input_file6506.txt +example_input_file6507.txt +example_input_file6508.txt +example_input_file6509.txt +example_input_file6510.txt +example_input_file6511.txt +example_input_file6512.txt +example_input_file6513.txt +example_input_file6514.txt +example_input_file6515.txt +example_input_file6516.txt +example_input_file6517.txt +example_input_file6518.txt +example_input_file6519.txt +example_input_file6520.txt +example_input_file6521.txt +example_input_file6522.txt +example_input_file6523.txt +example_input_file6524.txt +example_input_file6525.txt +example_input_file6526.txt +example_input_file6527.txt +example_input_file6528.txt +example_input_file6529.txt +example_input_file6530.txt +example_input_file6531.txt +example_input_file6532.txt +example_input_file6533.txt +example_input_file6534.txt +example_input_file6535.txt +example_input_file6536.txt +example_input_file6537.txt +example_input_file6538.txt +example_input_file6539.txt +example_input_file6540.txt +example_input_file6541.txt +example_input_file6542.txt +example_input_file6543.txt +example_input_file6544.txt +example_input_file6545.txt +example_input_file6546.txt +example_input_file6547.txt +example_input_file6548.txt +example_input_file6549.txt +example_input_file6550.txt +example_input_file6551.txt +example_input_file6552.txt +example_input_file6553.txt +example_input_file6554.txt +example_input_file6555.txt +example_input_file6556.txt +example_input_file6557.txt +example_input_file6558.txt +example_input_file6559.txt +example_input_file6560.txt +example_input_file6561.txt +example_input_file6562.txt +example_input_file6563.txt +example_input_file6564.txt +example_input_file6565.txt +example_input_file6566.txt +example_input_file6567.txt +example_input_file6568.txt +example_input_file6569.txt +example_input_file6570.txt +example_input_file6571.txt +example_input_file6572.txt +example_input_file6573.txt +example_input_file6574.txt +example_input_file6575.txt +example_input_file6576.txt +example_input_file6577.txt +example_input_file6578.txt +example_input_file6579.txt +example_input_file6580.txt +example_input_file6581.txt +example_input_file6582.txt +example_input_file6583.txt +example_input_file6584.txt +example_input_file6585.txt +example_input_file6586.txt +example_input_file6587.txt +example_input_file6588.txt +example_input_file6589.txt +example_input_file6590.txt +example_input_file6591.txt +example_input_file6592.txt +example_input_file6593.txt +example_input_file6594.txt +example_input_file6595.txt +example_input_file6596.txt +example_input_file6597.txt +example_input_file6598.txt +example_input_file6599.txt +example_input_file6600.txt +example_input_file6601.txt +example_input_file6602.txt +example_input_file6603.txt +example_input_file6604.txt +example_input_file6605.txt +example_input_file6606.txt +example_input_file6607.txt +example_input_file6608.txt +example_input_file6609.txt +example_input_file6610.txt +example_input_file6611.txt +example_input_file6612.txt +example_input_file6613.txt +example_input_file6614.txt +example_input_file6615.txt +example_input_file6616.txt +example_input_file6617.txt +example_input_file6618.txt +example_input_file6619.txt +example_input_file6620.txt +example_input_file6621.txt +example_input_file6622.txt +example_input_file6623.txt +example_input_file6624.txt +example_input_file6625.txt +example_input_file6626.txt +example_input_file6627.txt +example_input_file6628.txt +example_input_file6629.txt +example_input_file6630.txt +example_input_file6631.txt +example_input_file6632.txt +example_input_file6633.txt +example_input_file6634.txt +example_input_file6635.txt +example_input_file6636.txt +example_input_file6637.txt +example_input_file6638.txt +example_input_file6639.txt +example_input_file6640.txt +example_input_file6641.txt +example_input_file6642.txt +example_input_file6643.txt +example_input_file6644.txt +example_input_file6645.txt +example_input_file6646.txt +example_input_file6647.txt +example_input_file6648.txt +example_input_file6649.txt +example_input_file6650.txt +example_input_file6651.txt +example_input_file6652.txt +example_input_file6653.txt +example_input_file6654.txt +example_input_file6655.txt +example_input_file6656.txt +example_input_file6657.txt +example_input_file6658.txt +example_input_file6659.txt +example_input_file6660.txt +example_input_file6661.txt +example_input_file6662.txt +example_input_file6663.txt +example_input_file6664.txt +example_input_file6665.txt +example_input_file6666.txt +example_input_file6667.txt +example_input_file6668.txt +example_input_file6669.txt +example_input_file6670.txt +example_input_file6671.txt +example_input_file6672.txt +example_input_file6673.txt +example_input_file6674.txt +example_input_file6675.txt +example_input_file6676.txt +example_input_file6677.txt +example_input_file6678.txt +example_input_file6679.txt +example_input_file6680.txt +example_input_file6681.txt +example_input_file6682.txt +example_input_file6683.txt +example_input_file6684.txt +example_input_file6685.txt +example_input_file6686.txt +example_input_file6687.txt +example_input_file6688.txt +example_input_file6689.txt +example_input_file6690.txt +example_input_file6691.txt +example_input_file6692.txt +example_input_file6693.txt +example_input_file6694.txt +example_input_file6695.txt +example_input_file6696.txt +example_input_file6697.txt +example_input_file6698.txt +example_input_file6699.txt +example_input_file6700.txt +example_input_file6701.txt +example_input_file6702.txt +example_input_file6703.txt +example_input_file6704.txt +example_input_file6705.txt +example_input_file6706.txt +example_input_file6707.txt +example_input_file6708.txt +example_input_file6709.txt +example_input_file6710.txt +example_input_file6711.txt +example_input_file6712.txt +example_input_file6713.txt +example_input_file6714.txt +example_input_file6715.txt +example_input_file6716.txt +example_input_file6717.txt +example_input_file6718.txt +example_input_file6719.txt +example_input_file6720.txt +example_input_file6721.txt +example_input_file6722.txt +example_input_file6723.txt +example_input_file6724.txt +example_input_file6725.txt +example_input_file6726.txt +example_input_file6727.txt +example_input_file6728.txt +example_input_file6729.txt +example_input_file6730.txt +example_input_file6731.txt +example_input_file6732.txt +example_input_file6733.txt +example_input_file6734.txt +example_input_file6735.txt +example_input_file6736.txt +example_input_file6737.txt +example_input_file6738.txt +example_input_file6739.txt +example_input_file6740.txt +example_input_file6741.txt +example_input_file6742.txt +example_input_file6743.txt +example_input_file6744.txt +example_input_file6745.txt +example_input_file6746.txt +example_input_file6747.txt +example_input_file6748.txt +example_input_file6749.txt +example_input_file6750.txt +example_input_file6751.txt +example_input_file6752.txt +example_input_file6753.txt +example_input_file6754.txt +example_input_file6755.txt +example_input_file6756.txt +example_input_file6757.txt +example_input_file6758.txt +example_input_file6759.txt +example_input_file6760.txt +example_input_file6761.txt +example_input_file6762.txt +example_input_file6763.txt +example_input_file6764.txt +example_input_file6765.txt +example_input_file6766.txt +example_input_file6767.txt +example_input_file6768.txt +example_input_file6769.txt +example_input_file6770.txt +example_input_file6771.txt +example_input_file6772.txt +example_input_file6773.txt +example_input_file6774.txt +example_input_file6775.txt +example_input_file6776.txt +example_input_file6777.txt +example_input_file6778.txt +example_input_file6779.txt +example_input_file6780.txt +example_input_file6781.txt +example_input_file6782.txt +example_input_file6783.txt +example_input_file6784.txt +example_input_file6785.txt +example_input_file6786.txt +example_input_file6787.txt +example_input_file6788.txt +example_input_file6789.txt +example_input_file6790.txt +example_input_file6791.txt +example_input_file6792.txt +example_input_file6793.txt +example_input_file6794.txt +example_input_file6795.txt +example_input_file6796.txt +example_input_file6797.txt +example_input_file6798.txt +example_input_file6799.txt +example_input_file6800.txt +example_input_file6801.txt +example_input_file6802.txt +example_input_file6803.txt +example_input_file6804.txt +example_input_file6805.txt +example_input_file6806.txt +example_input_file6807.txt +example_input_file6808.txt +example_input_file6809.txt +example_input_file6810.txt +example_input_file6811.txt +example_input_file6812.txt +example_input_file6813.txt +example_input_file6814.txt +example_input_file6815.txt +example_input_file6816.txt +example_input_file6817.txt +example_input_file6818.txt +example_input_file6819.txt +example_input_file6820.txt +example_input_file6821.txt +example_input_file6822.txt +example_input_file6823.txt +example_input_file6824.txt +example_input_file6825.txt +example_input_file6826.txt +example_input_file6827.txt +example_input_file6828.txt +example_input_file6829.txt +example_input_file6830.txt +example_input_file6831.txt +example_input_file6832.txt +example_input_file6833.txt +example_input_file6834.txt +example_input_file6835.txt +example_input_file6836.txt +example_input_file6837.txt +example_input_file6838.txt +example_input_file6839.txt +example_input_file6840.txt +example_input_file6841.txt +example_input_file6842.txt +example_input_file6843.txt +example_input_file6844.txt +example_input_file6845.txt +example_input_file6846.txt +example_input_file6847.txt +example_input_file6848.txt +example_input_file6849.txt +example_input_file6850.txt +example_input_file6851.txt +example_input_file6852.txt +example_input_file6853.txt +example_input_file6854.txt +example_input_file6855.txt +example_input_file6856.txt +example_input_file6857.txt +example_input_file6858.txt +example_input_file6859.txt +example_input_file6860.txt +example_input_file6861.txt +example_input_file6862.txt +example_input_file6863.txt +example_input_file6864.txt +example_input_file6865.txt +example_input_file6866.txt +example_input_file6867.txt +example_input_file6868.txt +example_input_file6869.txt +example_input_file6870.txt +example_input_file6871.txt +example_input_file6872.txt +example_input_file6873.txt +example_input_file6874.txt +example_input_file6875.txt +example_input_file6876.txt +example_input_file6877.txt +example_input_file6878.txt +example_input_file6879.txt +example_input_file6880.txt +example_input_file6881.txt +example_input_file6882.txt +example_input_file6883.txt +example_input_file6884.txt +example_input_file6885.txt +example_input_file6886.txt +example_input_file6887.txt +example_input_file6888.txt +example_input_file6889.txt +example_input_file6890.txt +example_input_file6891.txt +example_input_file6892.txt +example_input_file6893.txt +example_input_file6894.txt +example_input_file6895.txt +example_input_file6896.txt +example_input_file6897.txt +example_input_file6898.txt +example_input_file6899.txt +example_input_file6900.txt +example_input_file6901.txt +example_input_file6902.txt +example_input_file6903.txt +example_input_file6904.txt +example_input_file6905.txt +example_input_file6906.txt +example_input_file6907.txt +example_input_file6908.txt +example_input_file6909.txt +example_input_file6910.txt +example_input_file6911.txt +example_input_file6912.txt +example_input_file6913.txt +example_input_file6914.txt +example_input_file6915.txt +example_input_file6916.txt +example_input_file6917.txt +example_input_file6918.txt +example_input_file6919.txt +example_input_file6920.txt +example_input_file6921.txt +example_input_file6922.txt +example_input_file6923.txt +example_input_file6924.txt +example_input_file6925.txt +example_input_file6926.txt +example_input_file6927.txt +example_input_file6928.txt +example_input_file6929.txt +example_input_file6930.txt +example_input_file6931.txt +example_input_file6932.txt +example_input_file6933.txt +example_input_file6934.txt +example_input_file6935.txt +example_input_file6936.txt +example_input_file6937.txt +example_input_file6938.txt +example_input_file6939.txt +example_input_file6940.txt +example_input_file6941.txt +example_input_file6942.txt +example_input_file6943.txt +example_input_file6944.txt +example_input_file6945.txt +example_input_file6946.txt +example_input_file6947.txt +example_input_file6948.txt +example_input_file6949.txt +example_input_file6950.txt +example_input_file6951.txt +example_input_file6952.txt +example_input_file6953.txt +example_input_file6954.txt +example_input_file6955.txt +example_input_file6956.txt +example_input_file6957.txt +example_input_file6958.txt +example_input_file6959.txt +example_input_file6960.txt +example_input_file6961.txt +example_input_file6962.txt +example_input_file6963.txt +example_input_file6964.txt +example_input_file6965.txt +example_input_file6966.txt +example_input_file6967.txt +example_input_file6968.txt +example_input_file6969.txt +example_input_file6970.txt +example_input_file6971.txt +example_input_file6972.txt +example_input_file6973.txt +example_input_file6974.txt +example_input_file6975.txt +example_input_file6976.txt +example_input_file6977.txt +example_input_file6978.txt +example_input_file6979.txt +example_input_file6980.txt +example_input_file6981.txt +example_input_file6982.txt +example_input_file6983.txt +example_input_file6984.txt +example_input_file6985.txt +example_input_file6986.txt +example_input_file6987.txt +example_input_file6988.txt +example_input_file6989.txt +example_input_file6990.txt +example_input_file6991.txt +example_input_file6992.txt +example_input_file6993.txt +example_input_file6994.txt +example_input_file6995.txt +example_input_file6996.txt +example_input_file6997.txt +example_input_file6998.txt +example_input_file6999.txt +example_input_file7000.txt +example_input_file7001.txt +example_input_file7002.txt +example_input_file7003.txt +example_input_file7004.txt +example_input_file7005.txt +example_input_file7006.txt +example_input_file7007.txt +example_input_file7008.txt +example_input_file7009.txt +example_input_file7010.txt +example_input_file7011.txt +example_input_file7012.txt +example_input_file7013.txt +example_input_file7014.txt +example_input_file7015.txt +example_input_file7016.txt +example_input_file7017.txt +example_input_file7018.txt +example_input_file7019.txt +example_input_file7020.txt +example_input_file7021.txt +example_input_file7022.txt +example_input_file7023.txt +example_input_file7024.txt +example_input_file7025.txt +example_input_file7026.txt +example_input_file7027.txt +example_input_file7028.txt +example_input_file7029.txt +example_input_file7030.txt +example_input_file7031.txt +example_input_file7032.txt +example_input_file7033.txt +example_input_file7034.txt +example_input_file7035.txt +example_input_file7036.txt +example_input_file7037.txt +example_input_file7038.txt +example_input_file7039.txt +example_input_file7040.txt +example_input_file7041.txt +example_input_file7042.txt +example_input_file7043.txt +example_input_file7044.txt +example_input_file7045.txt +example_input_file7046.txt +example_input_file7047.txt +example_input_file7048.txt +example_input_file7049.txt +example_input_file7050.txt +example_input_file7051.txt +example_input_file7052.txt +example_input_file7053.txt +example_input_file7054.txt +example_input_file7055.txt +example_input_file7056.txt +example_input_file7057.txt +example_input_file7058.txt +example_input_file7059.txt +example_input_file7060.txt +example_input_file7061.txt +example_input_file7062.txt +example_input_file7063.txt +example_input_file7064.txt +example_input_file7065.txt +example_input_file7066.txt +example_input_file7067.txt +example_input_file7068.txt +example_input_file7069.txt +example_input_file7070.txt +example_input_file7071.txt +example_input_file7072.txt +example_input_file7073.txt +example_input_file7074.txt +example_input_file7075.txt +example_input_file7076.txt +example_input_file7077.txt +example_input_file7078.txt +example_input_file7079.txt +example_input_file7080.txt +example_input_file7081.txt +example_input_file7082.txt +example_input_file7083.txt +example_input_file7084.txt +example_input_file7085.txt +example_input_file7086.txt +example_input_file7087.txt +example_input_file7088.txt +example_input_file7089.txt +example_input_file7090.txt +example_input_file7091.txt +example_input_file7092.txt +example_input_file7093.txt +example_input_file7094.txt +example_input_file7095.txt +example_input_file7096.txt +example_input_file7097.txt +example_input_file7098.txt +example_input_file7099.txt +example_input_file7100.txt +example_input_file7101.txt +example_input_file7102.txt +example_input_file7103.txt +example_input_file7104.txt +example_input_file7105.txt +example_input_file7106.txt +example_input_file7107.txt +example_input_file7108.txt +example_input_file7109.txt +example_input_file7110.txt +example_input_file7111.txt +example_input_file7112.txt +example_input_file7113.txt +example_input_file7114.txt +example_input_file7115.txt +example_input_file7116.txt +example_input_file7117.txt +example_input_file7118.txt +example_input_file7119.txt +example_input_file7120.txt +example_input_file7121.txt +example_input_file7122.txt +example_input_file7123.txt +example_input_file7124.txt +example_input_file7125.txt +example_input_file7126.txt +example_input_file7127.txt +example_input_file7128.txt +example_input_file7129.txt +example_input_file7130.txt +example_input_file7131.txt +example_input_file7132.txt +example_input_file7133.txt +example_input_file7134.txt +example_input_file7135.txt +example_input_file7136.txt +example_input_file7137.txt +example_input_file7138.txt +example_input_file7139.txt +example_input_file7140.txt +example_input_file7141.txt +example_input_file7142.txt +example_input_file7143.txt +example_input_file7144.txt +example_input_file7145.txt +example_input_file7146.txt +example_input_file7147.txt +example_input_file7148.txt +example_input_file7149.txt +example_input_file7150.txt +example_input_file7151.txt +example_input_file7152.txt +example_input_file7153.txt +example_input_file7154.txt +example_input_file7155.txt +example_input_file7156.txt +example_input_file7157.txt +example_input_file7158.txt +example_input_file7159.txt +example_input_file7160.txt +example_input_file7161.txt +example_input_file7162.txt +example_input_file7163.txt +example_input_file7164.txt +example_input_file7165.txt +example_input_file7166.txt +example_input_file7167.txt +example_input_file7168.txt +example_input_file7169.txt +example_input_file7170.txt +example_input_file7171.txt +example_input_file7172.txt +example_input_file7173.txt +example_input_file7174.txt +example_input_file7175.txt +example_input_file7176.txt +example_input_file7177.txt +example_input_file7178.txt +example_input_file7179.txt +example_input_file7180.txt +example_input_file7181.txt +example_input_file7182.txt +example_input_file7183.txt +example_input_file7184.txt +example_input_file7185.txt +example_input_file7186.txt +example_input_file7187.txt +example_input_file7188.txt +example_input_file7189.txt +example_input_file7190.txt +example_input_file7191.txt +example_input_file7192.txt +example_input_file7193.txt +example_input_file7194.txt +example_input_file7195.txt +example_input_file7196.txt +example_input_file7197.txt +example_input_file7198.txt +example_input_file7199.txt +example_input_file7200.txt +example_input_file7201.txt +example_input_file7202.txt +example_input_file7203.txt +example_input_file7204.txt +example_input_file7205.txt +example_input_file7206.txt +example_input_file7207.txt +example_input_file7208.txt +example_input_file7209.txt +example_input_file7210.txt +example_input_file7211.txt +example_input_file7212.txt +example_input_file7213.txt +example_input_file7214.txt +example_input_file7215.txt +example_input_file7216.txt +example_input_file7217.txt +example_input_file7218.txt +example_input_file7219.txt +example_input_file7220.txt +example_input_file7221.txt +example_input_file7222.txt +example_input_file7223.txt +example_input_file7224.txt +example_input_file7225.txt +example_input_file7226.txt +example_input_file7227.txt +example_input_file7228.txt +example_input_file7229.txt +example_input_file7230.txt +example_input_file7231.txt +example_input_file7232.txt +example_input_file7233.txt +example_input_file7234.txt +example_input_file7235.txt +example_input_file7236.txt +example_input_file7237.txt +example_input_file7238.txt +example_input_file7239.txt +example_input_file7240.txt +example_input_file7241.txt +example_input_file7242.txt +example_input_file7243.txt +example_input_file7244.txt +example_input_file7245.txt +example_input_file7246.txt +example_input_file7247.txt +example_input_file7248.txt +example_input_file7249.txt +example_input_file7250.txt +example_input_file7251.txt +example_input_file7252.txt +example_input_file7253.txt +example_input_file7254.txt +example_input_file7255.txt +example_input_file7256.txt +example_input_file7257.txt +example_input_file7258.txt +example_input_file7259.txt +example_input_file7260.txt +example_input_file7261.txt +example_input_file7262.txt +example_input_file7263.txt +example_input_file7264.txt +example_input_file7265.txt +example_input_file7266.txt +example_input_file7267.txt +example_input_file7268.txt +example_input_file7269.txt +example_input_file7270.txt +example_input_file7271.txt +example_input_file7272.txt +example_input_file7273.txt +example_input_file7274.txt +example_input_file7275.txt +example_input_file7276.txt +example_input_file7277.txt +example_input_file7278.txt +example_input_file7279.txt +example_input_file7280.txt +example_input_file7281.txt +example_input_file7282.txt +example_input_file7283.txt +example_input_file7284.txt +example_input_file7285.txt +example_input_file7286.txt +example_input_file7287.txt +example_input_file7288.txt +example_input_file7289.txt +example_input_file7290.txt +example_input_file7291.txt +example_input_file7292.txt +example_input_file7293.txt +example_input_file7294.txt +example_input_file7295.txt +example_input_file7296.txt +example_input_file7297.txt +example_input_file7298.txt +example_input_file7299.txt +example_input_file7300.txt +example_input_file7301.txt +example_input_file7302.txt +example_input_file7303.txt +example_input_file7304.txt +example_input_file7305.txt +example_input_file7306.txt +example_input_file7307.txt +example_input_file7308.txt +example_input_file7309.txt +example_input_file7310.txt +example_input_file7311.txt +example_input_file7312.txt +example_input_file7313.txt +example_input_file7314.txt +example_input_file7315.txt +example_input_file7316.txt +example_input_file7317.txt +example_input_file7318.txt +example_input_file7319.txt +example_input_file7320.txt +example_input_file7321.txt +example_input_file7322.txt +example_input_file7323.txt +example_input_file7324.txt +example_input_file7325.txt +example_input_file7326.txt +example_input_file7327.txt +example_input_file7328.txt +example_input_file7329.txt +example_input_file7330.txt +example_input_file7331.txt +example_input_file7332.txt +example_input_file7333.txt +example_input_file7334.txt +example_input_file7335.txt +example_input_file7336.txt +example_input_file7337.txt +example_input_file7338.txt +example_input_file7339.txt +example_input_file7340.txt +example_input_file7341.txt +example_input_file7342.txt +example_input_file7343.txt +example_input_file7344.txt +example_input_file7345.txt +example_input_file7346.txt +example_input_file7347.txt +example_input_file7348.txt +example_input_file7349.txt +example_input_file7350.txt +example_input_file7351.txt +example_input_file7352.txt +example_input_file7353.txt +example_input_file7354.txt +example_input_file7355.txt +example_input_file7356.txt +example_input_file7357.txt +example_input_file7358.txt +example_input_file7359.txt +example_input_file7360.txt +example_input_file7361.txt +example_input_file7362.txt +example_input_file7363.txt +example_input_file7364.txt +example_input_file7365.txt +example_input_file7366.txt +example_input_file7367.txt +example_input_file7368.txt +example_input_file7369.txt +example_input_file7370.txt +example_input_file7371.txt +example_input_file7372.txt +example_input_file7373.txt +example_input_file7374.txt +example_input_file7375.txt +example_input_file7376.txt +example_input_file7377.txt +example_input_file7378.txt +example_input_file7379.txt +example_input_file7380.txt +example_input_file7381.txt +example_input_file7382.txt +example_input_file7383.txt +example_input_file7384.txt +example_input_file7385.txt +example_input_file7386.txt +example_input_file7387.txt +example_input_file7388.txt +example_input_file7389.txt +example_input_file7390.txt +example_input_file7391.txt +example_input_file7392.txt +example_input_file7393.txt +example_input_file7394.txt +example_input_file7395.txt +example_input_file7396.txt +example_input_file7397.txt +example_input_file7398.txt +example_input_file7399.txt +example_input_file7400.txt +example_input_file7401.txt +example_input_file7402.txt +example_input_file7403.txt +example_input_file7404.txt +example_input_file7405.txt +example_input_file7406.txt +example_input_file7407.txt +example_input_file7408.txt +example_input_file7409.txt +example_input_file7410.txt +example_input_file7411.txt +example_input_file7412.txt +example_input_file7413.txt +example_input_file7414.txt +example_input_file7415.txt +example_input_file7416.txt +example_input_file7417.txt +example_input_file7418.txt +example_input_file7419.txt +example_input_file7420.txt +example_input_file7421.txt +example_input_file7422.txt +example_input_file7423.txt +example_input_file7424.txt +example_input_file7425.txt +example_input_file7426.txt +example_input_file7427.txt +example_input_file7428.txt +example_input_file7429.txt +example_input_file7430.txt +example_input_file7431.txt +example_input_file7432.txt +example_input_file7433.txt +example_input_file7434.txt +example_input_file7435.txt +example_input_file7436.txt +example_input_file7437.txt +example_input_file7438.txt +example_input_file7439.txt +example_input_file7440.txt +example_input_file7441.txt +example_input_file7442.txt +example_input_file7443.txt +example_input_file7444.txt +example_input_file7445.txt +example_input_file7446.txt +example_input_file7447.txt +example_input_file7448.txt +example_input_file7449.txt +example_input_file7450.txt +example_input_file7451.txt +example_input_file7452.txt +example_input_file7453.txt +example_input_file7454.txt +example_input_file7455.txt +example_input_file7456.txt +example_input_file7457.txt +example_input_file7458.txt +example_input_file7459.txt +example_input_file7460.txt +example_input_file7461.txt +example_input_file7462.txt +example_input_file7463.txt +example_input_file7464.txt +example_input_file7465.txt +example_input_file7466.txt +example_input_file7467.txt +example_input_file7468.txt +example_input_file7469.txt +example_input_file7470.txt +example_input_file7471.txt +example_input_file7472.txt +example_input_file7473.txt +example_input_file7474.txt +example_input_file7475.txt +example_input_file7476.txt +example_input_file7477.txt +example_input_file7478.txt +example_input_file7479.txt +example_input_file7480.txt +example_input_file7481.txt +example_input_file7482.txt +example_input_file7483.txt +example_input_file7484.txt +example_input_file7485.txt +example_input_file7486.txt +example_input_file7487.txt +example_input_file7488.txt +example_input_file7489.txt +example_input_file7490.txt +example_input_file7491.txt +example_input_file7492.txt +example_input_file7493.txt +example_input_file7494.txt +example_input_file7495.txt +example_input_file7496.txt +example_input_file7497.txt +example_input_file7498.txt +example_input_file7499.txt +example_input_file7500.txt +example_input_file7501.txt +example_input_file7502.txt +example_input_file7503.txt +example_input_file7504.txt +example_input_file7505.txt +example_input_file7506.txt +example_input_file7507.txt +example_input_file7508.txt +example_input_file7509.txt +example_input_file7510.txt +example_input_file7511.txt +example_input_file7512.txt +example_input_file7513.txt +example_input_file7514.txt +example_input_file7515.txt +example_input_file7516.txt +example_input_file7517.txt +example_input_file7518.txt +example_input_file7519.txt +example_input_file7520.txt +example_input_file7521.txt +example_input_file7522.txt +example_input_file7523.txt +example_input_file7524.txt +example_input_file7525.txt +example_input_file7526.txt +example_input_file7527.txt +example_input_file7528.txt +example_input_file7529.txt +example_input_file7530.txt +example_input_file7531.txt +example_input_file7532.txt +example_input_file7533.txt +example_input_file7534.txt +example_input_file7535.txt +example_input_file7536.txt +example_input_file7537.txt +example_input_file7538.txt +example_input_file7539.txt +example_input_file7540.txt +example_input_file7541.txt +example_input_file7542.txt +example_input_file7543.txt +example_input_file7544.txt +example_input_file7545.txt +example_input_file7546.txt +example_input_file7547.txt +example_input_file7548.txt +example_input_file7549.txt +example_input_file7550.txt +example_input_file7551.txt +example_input_file7552.txt +example_input_file7553.txt +example_input_file7554.txt +example_input_file7555.txt +example_input_file7556.txt +example_input_file7557.txt +example_input_file7558.txt +example_input_file7559.txt +example_input_file7560.txt +example_input_file7561.txt +example_input_file7562.txt +example_input_file7563.txt +example_input_file7564.txt +example_input_file7565.txt +example_input_file7566.txt +example_input_file7567.txt +example_input_file7568.txt +example_input_file7569.txt +example_input_file7570.txt +example_input_file7571.txt +example_input_file7572.txt +example_input_file7573.txt +example_input_file7574.txt +example_input_file7575.txt +example_input_file7576.txt +example_input_file7577.txt +example_input_file7578.txt +example_input_file7579.txt +example_input_file7580.txt +example_input_file7581.txt +example_input_file7582.txt +example_input_file7583.txt +example_input_file7584.txt +example_input_file7585.txt +example_input_file7586.txt +example_input_file7587.txt +example_input_file7588.txt +example_input_file7589.txt +example_input_file7590.txt +example_input_file7591.txt +example_input_file7592.txt +example_input_file7593.txt +example_input_file7594.txt +example_input_file7595.txt +example_input_file7596.txt +example_input_file7597.txt +example_input_file7598.txt +example_input_file7599.txt +example_input_file7600.txt +example_input_file7601.txt +example_input_file7602.txt +example_input_file7603.txt +example_input_file7604.txt +example_input_file7605.txt +example_input_file7606.txt +example_input_file7607.txt +example_input_file7608.txt +example_input_file7609.txt +example_input_file7610.txt +example_input_file7611.txt +example_input_file7612.txt +example_input_file7613.txt +example_input_file7614.txt +example_input_file7615.txt +example_input_file7616.txt +example_input_file7617.txt +example_input_file7618.txt +example_input_file7619.txt +example_input_file7620.txt +example_input_file7621.txt +example_input_file7622.txt +example_input_file7623.txt +example_input_file7624.txt +example_input_file7625.txt +example_input_file7626.txt +example_input_file7627.txt +example_input_file7628.txt +example_input_file7629.txt +example_input_file7630.txt +example_input_file7631.txt +example_input_file7632.txt +example_input_file7633.txt +example_input_file7634.txt +example_input_file7635.txt +example_input_file7636.txt +example_input_file7637.txt +example_input_file7638.txt +example_input_file7639.txt +example_input_file7640.txt +example_input_file7641.txt +example_input_file7642.txt +example_input_file7643.txt +example_input_file7644.txt +example_input_file7645.txt +example_input_file7646.txt +example_input_file7647.txt +example_input_file7648.txt +example_input_file7649.txt +example_input_file7650.txt +example_input_file7651.txt +example_input_file7652.txt +example_input_file7653.txt +example_input_file7654.txt +example_input_file7655.txt +example_input_file7656.txt +example_input_file7657.txt +example_input_file7658.txt +example_input_file7659.txt +example_input_file7660.txt +example_input_file7661.txt +example_input_file7662.txt +example_input_file7663.txt +example_input_file7664.txt +example_input_file7665.txt +example_input_file7666.txt +example_input_file7667.txt +example_input_file7668.txt +example_input_file7669.txt +example_input_file7670.txt +example_input_file7671.txt +example_input_file7672.txt +example_input_file7673.txt +example_input_file7674.txt +example_input_file7675.txt +example_input_file7676.txt +example_input_file7677.txt +example_input_file7678.txt +example_input_file7679.txt +example_input_file7680.txt +example_input_file7681.txt +example_input_file7682.txt +example_input_file7683.txt +example_input_file7684.txt +example_input_file7685.txt +example_input_file7686.txt +example_input_file7687.txt +example_input_file7688.txt +example_input_file7689.txt +example_input_file7690.txt +example_input_file7691.txt +example_input_file7692.txt +example_input_file7693.txt +example_input_file7694.txt +example_input_file7695.txt +example_input_file7696.txt +example_input_file7697.txt +example_input_file7698.txt +example_input_file7699.txt +example_input_file7700.txt +example_input_file7701.txt +example_input_file7702.txt +example_input_file7703.txt +example_input_file7704.txt +example_input_file7705.txt +example_input_file7706.txt +example_input_file7707.txt +example_input_file7708.txt +example_input_file7709.txt +example_input_file7710.txt +example_input_file7711.txt +example_input_file7712.txt +example_input_file7713.txt +example_input_file7714.txt +example_input_file7715.txt +example_input_file7716.txt +example_input_file7717.txt +example_input_file7718.txt +example_input_file7719.txt +example_input_file7720.txt +example_input_file7721.txt +example_input_file7722.txt +example_input_file7723.txt +example_input_file7724.txt +example_input_file7725.txt +example_input_file7726.txt +example_input_file7727.txt +example_input_file7728.txt +example_input_file7729.txt +example_input_file7730.txt +example_input_file7731.txt +example_input_file7732.txt +example_input_file7733.txt +example_input_file7734.txt +example_input_file7735.txt +example_input_file7736.txt +example_input_file7737.txt +example_input_file7738.txt +example_input_file7739.txt +example_input_file7740.txt +example_input_file7741.txt +example_input_file7742.txt +example_input_file7743.txt +example_input_file7744.txt +example_input_file7745.txt +example_input_file7746.txt +example_input_file7747.txt +example_input_file7748.txt +example_input_file7749.txt +example_input_file7750.txt +example_input_file7751.txt +example_input_file7752.txt +example_input_file7753.txt +example_input_file7754.txt +example_input_file7755.txt +example_input_file7756.txt +example_input_file7757.txt +example_input_file7758.txt +example_input_file7759.txt +example_input_file7760.txt +example_input_file7761.txt +example_input_file7762.txt +example_input_file7763.txt +example_input_file7764.txt +example_input_file7765.txt +example_input_file7766.txt +example_input_file7767.txt +example_input_file7768.txt +example_input_file7769.txt +example_input_file7770.txt +example_input_file7771.txt +example_input_file7772.txt +example_input_file7773.txt +example_input_file7774.txt +example_input_file7775.txt +example_input_file7776.txt +example_input_file7777.txt +example_input_file7778.txt +example_input_file7779.txt +example_input_file7780.txt +example_input_file7781.txt +example_input_file7782.txt +example_input_file7783.txt +example_input_file7784.txt +example_input_file7785.txt +example_input_file7786.txt +example_input_file7787.txt +example_input_file7788.txt +example_input_file7789.txt +example_input_file7790.txt +example_input_file7791.txt +example_input_file7792.txt +example_input_file7793.txt +example_input_file7794.txt +example_input_file7795.txt +example_input_file7796.txt +example_input_file7797.txt +example_input_file7798.txt +example_input_file7799.txt +example_input_file7800.txt +example_input_file7801.txt +example_input_file7802.txt +example_input_file7803.txt +example_input_file7804.txt +example_input_file7805.txt +example_input_file7806.txt +example_input_file7807.txt +example_input_file7808.txt +example_input_file7809.txt +example_input_file7810.txt +example_input_file7811.txt +example_input_file7812.txt +example_input_file7813.txt +example_input_file7814.txt +example_input_file7815.txt +example_input_file7816.txt +example_input_file7817.txt +example_input_file7818.txt +example_input_file7819.txt +example_input_file7820.txt +example_input_file7821.txt +example_input_file7822.txt +example_input_file7823.txt +example_input_file7824.txt +example_input_file7825.txt +example_input_file7826.txt +example_input_file7827.txt +example_input_file7828.txt +example_input_file7829.txt +example_input_file7830.txt +example_input_file7831.txt +example_input_file7832.txt +example_input_file7833.txt +example_input_file7834.txt +example_input_file7835.txt +example_input_file7836.txt +example_input_file7837.txt +example_input_file7838.txt +example_input_file7839.txt +example_input_file7840.txt +example_input_file7841.txt +example_input_file7842.txt +example_input_file7843.txt +example_input_file7844.txt +example_input_file7845.txt +example_input_file7846.txt +example_input_file7847.txt +example_input_file7848.txt +example_input_file7849.txt +example_input_file7850.txt +example_input_file7851.txt +example_input_file7852.txt +example_input_file7853.txt +example_input_file7854.txt +example_input_file7855.txt +example_input_file7856.txt +example_input_file7857.txt +example_input_file7858.txt +example_input_file7859.txt +example_input_file7860.txt +example_input_file7861.txt +example_input_file7862.txt +example_input_file7863.txt +example_input_file7864.txt +example_input_file7865.txt +example_input_file7866.txt +example_input_file7867.txt +example_input_file7868.txt +example_input_file7869.txt +example_input_file7870.txt +example_input_file7871.txt +example_input_file7872.txt +example_input_file7873.txt +example_input_file7874.txt +example_input_file7875.txt +example_input_file7876.txt +example_input_file7877.txt +example_input_file7878.txt +example_input_file7879.txt +example_input_file7880.txt +example_input_file7881.txt +example_input_file7882.txt +example_input_file7883.txt +example_input_file7884.txt +example_input_file7885.txt +example_input_file7886.txt +example_input_file7887.txt +example_input_file7888.txt +example_input_file7889.txt +example_input_file7890.txt +example_input_file7891.txt +example_input_file7892.txt +example_input_file7893.txt +example_input_file7894.txt +example_input_file7895.txt +example_input_file7896.txt +example_input_file7897.txt +example_input_file7898.txt +example_input_file7899.txt +example_input_file7900.txt +example_input_file7901.txt +example_input_file7902.txt +example_input_file7903.txt +example_input_file7904.txt +example_input_file7905.txt +example_input_file7906.txt +example_input_file7907.txt +example_input_file7908.txt +example_input_file7909.txt +example_input_file7910.txt +example_input_file7911.txt +example_input_file7912.txt +example_input_file7913.txt +example_input_file7914.txt +example_input_file7915.txt +example_input_file7916.txt +example_input_file7917.txt +example_input_file7918.txt +example_input_file7919.txt +example_input_file7920.txt +example_input_file7921.txt +example_input_file7922.txt +example_input_file7923.txt +example_input_file7924.txt +example_input_file7925.txt +example_input_file7926.txt +example_input_file7927.txt +example_input_file7928.txt +example_input_file7929.txt +example_input_file7930.txt +example_input_file7931.txt +example_input_file7932.txt +example_input_file7933.txt +example_input_file7934.txt +example_input_file7935.txt +example_input_file7936.txt +example_input_file7937.txt +example_input_file7938.txt +example_input_file7939.txt +example_input_file7940.txt +example_input_file7941.txt +example_input_file7942.txt +example_input_file7943.txt +example_input_file7944.txt +example_input_file7945.txt +example_input_file7946.txt +example_input_file7947.txt +example_input_file7948.txt +example_input_file7949.txt +example_input_file7950.txt +example_input_file7951.txt +example_input_file7952.txt +example_input_file7953.txt +example_input_file7954.txt +example_input_file7955.txt +example_input_file7956.txt +example_input_file7957.txt +example_input_file7958.txt +example_input_file7959.txt +example_input_file7960.txt +example_input_file7961.txt +example_input_file7962.txt +example_input_file7963.txt +example_input_file7964.txt +example_input_file7965.txt +example_input_file7966.txt +example_input_file7967.txt +example_input_file7968.txt +example_input_file7969.txt +example_input_file7970.txt +example_input_file7971.txt +example_input_file7972.txt +example_input_file7973.txt +example_input_file7974.txt +example_input_file7975.txt +example_input_file7976.txt +example_input_file7977.txt +example_input_file7978.txt +example_input_file7979.txt +example_input_file7980.txt +example_input_file7981.txt +example_input_file7982.txt +example_input_file7983.txt +example_input_file7984.txt +example_input_file7985.txt +example_input_file7986.txt +example_input_file7987.txt +example_input_file7988.txt +example_input_file7989.txt +example_input_file7990.txt +example_input_file7991.txt +example_input_file7992.txt +example_input_file7993.txt +example_input_file7994.txt +example_input_file7995.txt +example_input_file7996.txt +example_input_file7997.txt +example_input_file7998.txt +example_input_file7999.txt +example_input_file8000.txt +example_input_file8001.txt +example_input_file8002.txt +example_input_file8003.txt +example_input_file8004.txt +example_input_file8005.txt +example_input_file8006.txt +example_input_file8007.txt +example_input_file8008.txt +example_input_file8009.txt +example_input_file8010.txt +example_input_file8011.txt +example_input_file8012.txt +example_input_file8013.txt +example_input_file8014.txt +example_input_file8015.txt +example_input_file8016.txt +example_input_file8017.txt +example_input_file8018.txt +example_input_file8019.txt +example_input_file8020.txt +example_input_file8021.txt +example_input_file8022.txt +example_input_file8023.txt +example_input_file8024.txt +example_input_file8025.txt +example_input_file8026.txt +example_input_file8027.txt +example_input_file8028.txt +example_input_file8029.txt +example_input_file8030.txt +example_input_file8031.txt +example_input_file8032.txt +example_input_file8033.txt +example_input_file8034.txt +example_input_file8035.txt +example_input_file8036.txt +example_input_file8037.txt +example_input_file8038.txt +example_input_file8039.txt +example_input_file8040.txt +example_input_file8041.txt +example_input_file8042.txt +example_input_file8043.txt +example_input_file8044.txt +example_input_file8045.txt +example_input_file8046.txt +example_input_file8047.txt +example_input_file8048.txt +example_input_file8049.txt +example_input_file8050.txt +example_input_file8051.txt +example_input_file8052.txt +example_input_file8053.txt +example_input_file8054.txt +example_input_file8055.txt +example_input_file8056.txt +example_input_file8057.txt +example_input_file8058.txt +example_input_file8059.txt +example_input_file8060.txt +example_input_file8061.txt +example_input_file8062.txt +example_input_file8063.txt +example_input_file8064.txt +example_input_file8065.txt +example_input_file8066.txt +example_input_file8067.txt +example_input_file8068.txt +example_input_file8069.txt +example_input_file8070.txt +example_input_file8071.txt +example_input_file8072.txt +example_input_file8073.txt +example_input_file8074.txt +example_input_file8075.txt +example_input_file8076.txt +example_input_file8077.txt +example_input_file8078.txt +example_input_file8079.txt +example_input_file8080.txt +example_input_file8081.txt +example_input_file8082.txt +example_input_file8083.txt +example_input_file8084.txt +example_input_file8085.txt +example_input_file8086.txt +example_input_file8087.txt +example_input_file8088.txt +example_input_file8089.txt +example_input_file8090.txt +example_input_file8091.txt +example_input_file8092.txt +example_input_file8093.txt +example_input_file8094.txt +example_input_file8095.txt +example_input_file8096.txt +example_input_file8097.txt +example_input_file8098.txt +example_input_file8099.txt +example_input_file8100.txt +example_input_file8101.txt +example_input_file8102.txt +example_input_file8103.txt +example_input_file8104.txt +example_input_file8105.txt +example_input_file8106.txt +example_input_file8107.txt +example_input_file8108.txt +example_input_file8109.txt +example_input_file8110.txt +example_input_file8111.txt +example_input_file8112.txt +example_input_file8113.txt +example_input_file8114.txt +example_input_file8115.txt +example_input_file8116.txt +example_input_file8117.txt +example_input_file8118.txt +example_input_file8119.txt +example_input_file8120.txt +example_input_file8121.txt +example_input_file8122.txt +example_input_file8123.txt +example_input_file8124.txt +example_input_file8125.txt +example_input_file8126.txt +example_input_file8127.txt +example_input_file8128.txt +example_input_file8129.txt +example_input_file8130.txt +example_input_file8131.txt +example_input_file8132.txt +example_input_file8133.txt +example_input_file8134.txt +example_input_file8135.txt +example_input_file8136.txt +example_input_file8137.txt +example_input_file8138.txt +example_input_file8139.txt +example_input_file8140.txt +example_input_file8141.txt +example_input_file8142.txt +example_input_file8143.txt +example_input_file8144.txt +example_input_file8145.txt +example_input_file8146.txt +example_input_file8147.txt +example_input_file8148.txt +example_input_file8149.txt +example_input_file8150.txt +example_input_file8151.txt +example_input_file8152.txt +example_input_file8153.txt +example_input_file8154.txt +example_input_file8155.txt +example_input_file8156.txt +example_input_file8157.txt +example_input_file8158.txt +example_input_file8159.txt +example_input_file8160.txt +example_input_file8161.txt +example_input_file8162.txt +example_input_file8163.txt +example_input_file8164.txt +example_input_file8165.txt +example_input_file8166.txt +example_input_file8167.txt +example_input_file8168.txt +example_input_file8169.txt +example_input_file8170.txt +example_input_file8171.txt +example_input_file8172.txt +example_input_file8173.txt +example_input_file8174.txt +example_input_file8175.txt +example_input_file8176.txt +example_input_file8177.txt +example_input_file8178.txt +example_input_file8179.txt +example_input_file8180.txt +example_input_file8181.txt +example_input_file8182.txt +example_input_file8183.txt +example_input_file8184.txt +example_input_file8185.txt +example_input_file8186.txt +example_input_file8187.txt +example_input_file8188.txt +example_input_file8189.txt +example_input_file8190.txt +example_input_file8191.txt +example_input_file8192.txt +example_input_file8193.txt +example_input_file8194.txt +example_input_file8195.txt +example_input_file8196.txt +example_input_file8197.txt +example_input_file8198.txt +example_input_file8199.txt +example_input_file8200.txt +example_input_file8201.txt +example_input_file8202.txt +example_input_file8203.txt +example_input_file8204.txt +example_input_file8205.txt +example_input_file8206.txt +example_input_file8207.txt +example_input_file8208.txt +example_input_file8209.txt +example_input_file8210.txt +example_input_file8211.txt +example_input_file8212.txt +example_input_file8213.txt +example_input_file8214.txt +example_input_file8215.txt +example_input_file8216.txt +example_input_file8217.txt +example_input_file8218.txt +example_input_file8219.txt +example_input_file8220.txt +example_input_file8221.txt +example_input_file8222.txt +example_input_file8223.txt +example_input_file8224.txt +example_input_file8225.txt +example_input_file8226.txt +example_input_file8227.txt +example_input_file8228.txt +example_input_file8229.txt +example_input_file8230.txt +example_input_file8231.txt +example_input_file8232.txt +example_input_file8233.txt +example_input_file8234.txt +example_input_file8235.txt +example_input_file8236.txt +example_input_file8237.txt +example_input_file8238.txt +example_input_file8239.txt +example_input_file8240.txt +example_input_file8241.txt +example_input_file8242.txt +example_input_file8243.txt +example_input_file8244.txt +example_input_file8245.txt +example_input_file8246.txt +example_input_file8247.txt +example_input_file8248.txt +example_input_file8249.txt +example_input_file8250.txt +example_input_file8251.txt +example_input_file8252.txt +example_input_file8253.txt +example_input_file8254.txt +example_input_file8255.txt +example_input_file8256.txt +example_input_file8257.txt +example_input_file8258.txt +example_input_file8259.txt +example_input_file8260.txt +example_input_file8261.txt +example_input_file8262.txt +example_input_file8263.txt +example_input_file8264.txt +example_input_file8265.txt +example_input_file8266.txt +example_input_file8267.txt +example_input_file8268.txt +example_input_file8269.txt +example_input_file8270.txt +example_input_file8271.txt +example_input_file8272.txt +example_input_file8273.txt +example_input_file8274.txt +example_input_file8275.txt +example_input_file8276.txt +example_input_file8277.txt +example_input_file8278.txt +example_input_file8279.txt +example_input_file8280.txt +example_input_file8281.txt +example_input_file8282.txt +example_input_file8283.txt +example_input_file8284.txt +example_input_file8285.txt +example_input_file8286.txt +example_input_file8287.txt +example_input_file8288.txt +example_input_file8289.txt +example_input_file8290.txt +example_input_file8291.txt +example_input_file8292.txt +example_input_file8293.txt +example_input_file8294.txt +example_input_file8295.txt +example_input_file8296.txt +example_input_file8297.txt +example_input_file8298.txt +example_input_file8299.txt +example_input_file8300.txt +example_input_file8301.txt +example_input_file8302.txt +example_input_file8303.txt +example_input_file8304.txt +example_input_file8305.txt +example_input_file8306.txt +example_input_file8307.txt +example_input_file8308.txt +example_input_file8309.txt +example_input_file8310.txt +example_input_file8311.txt +example_input_file8312.txt +example_input_file8313.txt +example_input_file8314.txt +example_input_file8315.txt +example_input_file8316.txt +example_input_file8317.txt +example_input_file8318.txt +example_input_file8319.txt +example_input_file8320.txt +example_input_file8321.txt +example_input_file8322.txt +example_input_file8323.txt +example_input_file8324.txt +example_input_file8325.txt +example_input_file8326.txt +example_input_file8327.txt +example_input_file8328.txt +example_input_file8329.txt +example_input_file8330.txt +example_input_file8331.txt +example_input_file8332.txt +example_input_file8333.txt +example_input_file8334.txt +example_input_file8335.txt +example_input_file8336.txt +example_input_file8337.txt +example_input_file8338.txt +example_input_file8339.txt +example_input_file8340.txt +example_input_file8341.txt +example_input_file8342.txt +example_input_file8343.txt +example_input_file8344.txt +example_input_file8345.txt +example_input_file8346.txt +example_input_file8347.txt +example_input_file8348.txt +example_input_file8349.txt +example_input_file8350.txt +example_input_file8351.txt +example_input_file8352.txt +example_input_file8353.txt +example_input_file8354.txt +example_input_file8355.txt +example_input_file8356.txt +example_input_file8357.txt +example_input_file8358.txt +example_input_file8359.txt +example_input_file8360.txt +example_input_file8361.txt +example_input_file8362.txt +example_input_file8363.txt +example_input_file8364.txt +example_input_file8365.txt +example_input_file8366.txt +example_input_file8367.txt +example_input_file8368.txt +example_input_file8369.txt +example_input_file8370.txt +example_input_file8371.txt +example_input_file8372.txt +example_input_file8373.txt +example_input_file8374.txt +example_input_file8375.txt +example_input_file8376.txt +example_input_file8377.txt +example_input_file8378.txt +example_input_file8379.txt +example_input_file8380.txt +example_input_file8381.txt +example_input_file8382.txt +example_input_file8383.txt +example_input_file8384.txt +example_input_file8385.txt +example_input_file8386.txt +example_input_file8387.txt +example_input_file8388.txt +example_input_file8389.txt +example_input_file8390.txt +example_input_file8391.txt +example_input_file8392.txt +example_input_file8393.txt +example_input_file8394.txt +example_input_file8395.txt +example_input_file8396.txt +example_input_file8397.txt +example_input_file8398.txt +example_input_file8399.txt +example_input_file8400.txt +example_input_file8401.txt +example_input_file8402.txt +example_input_file8403.txt +example_input_file8404.txt +example_input_file8405.txt +example_input_file8406.txt +example_input_file8407.txt +example_input_file8408.txt +example_input_file8409.txt +example_input_file8410.txt +example_input_file8411.txt +example_input_file8412.txt +example_input_file8413.txt +example_input_file8414.txt +example_input_file8415.txt +example_input_file8416.txt +example_input_file8417.txt +example_input_file8418.txt +example_input_file8419.txt +example_input_file8420.txt +example_input_file8421.txt +example_input_file8422.txt +example_input_file8423.txt +example_input_file8424.txt +example_input_file8425.txt +example_input_file8426.txt +example_input_file8427.txt +example_input_file8428.txt +example_input_file8429.txt +example_input_file8430.txt +example_input_file8431.txt +example_input_file8432.txt +example_input_file8433.txt +example_input_file8434.txt +example_input_file8435.txt +example_input_file8436.txt +example_input_file8437.txt +example_input_file8438.txt +example_input_file8439.txt +example_input_file8440.txt +example_input_file8441.txt +example_input_file8442.txt +example_input_file8443.txt +example_input_file8444.txt +example_input_file8445.txt +example_input_file8446.txt +example_input_file8447.txt +example_input_file8448.txt +example_input_file8449.txt +example_input_file8450.txt +example_input_file8451.txt +example_input_file8452.txt +example_input_file8453.txt +example_input_file8454.txt +example_input_file8455.txt +example_input_file8456.txt +example_input_file8457.txt +example_input_file8458.txt +example_input_file8459.txt +example_input_file8460.txt +example_input_file8461.txt +example_input_file8462.txt +example_input_file8463.txt +example_input_file8464.txt +example_input_file8465.txt +example_input_file8466.txt +example_input_file8467.txt +example_input_file8468.txt +example_input_file8469.txt +example_input_file8470.txt +example_input_file8471.txt +example_input_file8472.txt +example_input_file8473.txt +example_input_file8474.txt +example_input_file8475.txt +example_input_file8476.txt +example_input_file8477.txt +example_input_file8478.txt +example_input_file8479.txt +example_input_file8480.txt +example_input_file8481.txt +example_input_file8482.txt +example_input_file8483.txt +example_input_file8484.txt +example_input_file8485.txt +example_input_file8486.txt +example_input_file8487.txt +example_input_file8488.txt +example_input_file8489.txt +example_input_file8490.txt +example_input_file8491.txt +example_input_file8492.txt +example_input_file8493.txt +example_input_file8494.txt +example_input_file8495.txt +example_input_file8496.txt +example_input_file8497.txt +example_input_file8498.txt +example_input_file8499.txt +example_input_file8500.txt +example_input_file8501.txt +example_input_file8502.txt +example_input_file8503.txt +example_input_file8504.txt +example_input_file8505.txt +example_input_file8506.txt +example_input_file8507.txt +example_input_file8508.txt +example_input_file8509.txt +example_input_file8510.txt +example_input_file8511.txt +example_input_file8512.txt +example_input_file8513.txt +example_input_file8514.txt +example_input_file8515.txt +example_input_file8516.txt +example_input_file8517.txt +example_input_file8518.txt +example_input_file8519.txt +example_input_file8520.txt +example_input_file8521.txt +example_input_file8522.txt +example_input_file8523.txt +example_input_file8524.txt +example_input_file8525.txt +example_input_file8526.txt +example_input_file8527.txt +example_input_file8528.txt +example_input_file8529.txt +example_input_file8530.txt +example_input_file8531.txt +example_input_file8532.txt +example_input_file8533.txt +example_input_file8534.txt +example_input_file8535.txt +example_input_file8536.txt +example_input_file8537.txt +example_input_file8538.txt +example_input_file8539.txt +example_input_file8540.txt +example_input_file8541.txt +example_input_file8542.txt +example_input_file8543.txt +example_input_file8544.txt +example_input_file8545.txt +example_input_file8546.txt +example_input_file8547.txt +example_input_file8548.txt +example_input_file8549.txt +example_input_file8550.txt +example_input_file8551.txt +example_input_file8552.txt +example_input_file8553.txt +example_input_file8554.txt +example_input_file8555.txt +example_input_file8556.txt +example_input_file8557.txt +example_input_file8558.txt +example_input_file8559.txt +example_input_file8560.txt +example_input_file8561.txt +example_input_file8562.txt +example_input_file8563.txt +example_input_file8564.txt +example_input_file8565.txt +example_input_file8566.txt +example_input_file8567.txt +example_input_file8568.txt +example_input_file8569.txt +example_input_file8570.txt +example_input_file8571.txt +example_input_file8572.txt +example_input_file8573.txt +example_input_file8574.txt +example_input_file8575.txt +example_input_file8576.txt +example_input_file8577.txt +example_input_file8578.txt +example_input_file8579.txt +example_input_file8580.txt +example_input_file8581.txt +example_input_file8582.txt +example_input_file8583.txt +example_input_file8584.txt +example_input_file8585.txt +example_input_file8586.txt +example_input_file8587.txt +example_input_file8588.txt +example_input_file8589.txt +example_input_file8590.txt +example_input_file8591.txt +example_input_file8592.txt +example_input_file8593.txt +example_input_file8594.txt +example_input_file8595.txt +example_input_file8596.txt +example_input_file8597.txt +example_input_file8598.txt +example_input_file8599.txt +example_input_file8600.txt +example_input_file8601.txt +example_input_file8602.txt +example_input_file8603.txt +example_input_file8604.txt +example_input_file8605.txt +example_input_file8606.txt +example_input_file8607.txt +example_input_file8608.txt +example_input_file8609.txt +example_input_file8610.txt +example_input_file8611.txt +example_input_file8612.txt +example_input_file8613.txt +example_input_file8614.txt +example_input_file8615.txt +example_input_file8616.txt +example_input_file8617.txt +example_input_file8618.txt +example_input_file8619.txt +example_input_file8620.txt +example_input_file8621.txt +example_input_file8622.txt +example_input_file8623.txt +example_input_file8624.txt +example_input_file8625.txt +example_input_file8626.txt +example_input_file8627.txt +example_input_file8628.txt +example_input_file8629.txt +example_input_file8630.txt +example_input_file8631.txt +example_input_file8632.txt +example_input_file8633.txt +example_input_file8634.txt +example_input_file8635.txt +example_input_file8636.txt +example_input_file8637.txt +example_input_file8638.txt +example_input_file8639.txt +example_input_file8640.txt +example_input_file8641.txt +example_input_file8642.txt +example_input_file8643.txt +example_input_file8644.txt +example_input_file8645.txt +example_input_file8646.txt +example_input_file8647.txt +example_input_file8648.txt +example_input_file8649.txt +example_input_file8650.txt +example_input_file8651.txt +example_input_file8652.txt +example_input_file8653.txt +example_input_file8654.txt +example_input_file8655.txt +example_input_file8656.txt +example_input_file8657.txt +example_input_file8658.txt +example_input_file8659.txt +example_input_file8660.txt +example_input_file8661.txt +example_input_file8662.txt +example_input_file8663.txt +example_input_file8664.txt +example_input_file8665.txt +example_input_file8666.txt +example_input_file8667.txt +example_input_file8668.txt +example_input_file8669.txt +example_input_file8670.txt +example_input_file8671.txt +example_input_file8672.txt +example_input_file8673.txt +example_input_file8674.txt +example_input_file8675.txt +example_input_file8676.txt +example_input_file8677.txt +example_input_file8678.txt +example_input_file8679.txt +example_input_file8680.txt +example_input_file8681.txt +example_input_file8682.txt +example_input_file8683.txt +example_input_file8684.txt +example_input_file8685.txt +example_input_file8686.txt +example_input_file8687.txt +example_input_file8688.txt +example_input_file8689.txt +example_input_file8690.txt +example_input_file8691.txt +example_input_file8692.txt +example_input_file8693.txt +example_input_file8694.txt +example_input_file8695.txt +example_input_file8696.txt +example_input_file8697.txt +example_input_file8698.txt +example_input_file8699.txt +example_input_file8700.txt +example_input_file8701.txt +example_input_file8702.txt +example_input_file8703.txt +example_input_file8704.txt +example_input_file8705.txt +example_input_file8706.txt +example_input_file8707.txt +example_input_file8708.txt +example_input_file8709.txt +example_input_file8710.txt +example_input_file8711.txt +example_input_file8712.txt +example_input_file8713.txt +example_input_file8714.txt +example_input_file8715.txt +example_input_file8716.txt +example_input_file8717.txt +example_input_file8718.txt +example_input_file8719.txt +example_input_file8720.txt +example_input_file8721.txt +example_input_file8722.txt +example_input_file8723.txt +example_input_file8724.txt +example_input_file8725.txt +example_input_file8726.txt +example_input_file8727.txt +example_input_file8728.txt +example_input_file8729.txt +example_input_file8730.txt +example_input_file8731.txt +example_input_file8732.txt +example_input_file8733.txt +example_input_file8734.txt +example_input_file8735.txt +example_input_file8736.txt +example_input_file8737.txt +example_input_file8738.txt +example_input_file8739.txt +example_input_file8740.txt +example_input_file8741.txt +example_input_file8742.txt +example_input_file8743.txt +example_input_file8744.txt +example_input_file8745.txt +example_input_file8746.txt +example_input_file8747.txt +example_input_file8748.txt +example_input_file8749.txt +example_input_file8750.txt +example_input_file8751.txt +example_input_file8752.txt +example_input_file8753.txt +example_input_file8754.txt +example_input_file8755.txt +example_input_file8756.txt +example_input_file8757.txt +example_input_file8758.txt +example_input_file8759.txt +example_input_file8760.txt +example_input_file8761.txt +example_input_file8762.txt +example_input_file8763.txt +example_input_file8764.txt +example_input_file8765.txt +example_input_file8766.txt +example_input_file8767.txt +example_input_file8768.txt +example_input_file8769.txt +example_input_file8770.txt +example_input_file8771.txt +example_input_file8772.txt +example_input_file8773.txt +example_input_file8774.txt +example_input_file8775.txt +example_input_file8776.txt +example_input_file8777.txt +example_input_file8778.txt +example_input_file8779.txt +example_input_file8780.txt +example_input_file8781.txt +example_input_file8782.txt +example_input_file8783.txt +example_input_file8784.txt +example_input_file8785.txt +example_input_file8786.txt +example_input_file8787.txt +example_input_file8788.txt +example_input_file8789.txt +example_input_file8790.txt +example_input_file8791.txt +example_input_file8792.txt +example_input_file8793.txt +example_input_file8794.txt +example_input_file8795.txt +example_input_file8796.txt +example_input_file8797.txt +example_input_file8798.txt +example_input_file8799.txt +example_input_file8800.txt +example_input_file8801.txt +example_input_file8802.txt +example_input_file8803.txt +example_input_file8804.txt +example_input_file8805.txt +example_input_file8806.txt +example_input_file8807.txt +example_input_file8808.txt +example_input_file8809.txt +example_input_file8810.txt +example_input_file8811.txt +example_input_file8812.txt +example_input_file8813.txt +example_input_file8814.txt +example_input_file8815.txt +example_input_file8816.txt +example_input_file8817.txt +example_input_file8818.txt +example_input_file8819.txt +example_input_file8820.txt +example_input_file8821.txt +example_input_file8822.txt +example_input_file8823.txt +example_input_file8824.txt +example_input_file8825.txt +example_input_file8826.txt +example_input_file8827.txt +example_input_file8828.txt +example_input_file8829.txt +example_input_file8830.txt +example_input_file8831.txt +example_input_file8832.txt +example_input_file8833.txt +example_input_file8834.txt +example_input_file8835.txt +example_input_file8836.txt +example_input_file8837.txt +example_input_file8838.txt +example_input_file8839.txt +example_input_file8840.txt +example_input_file8841.txt +example_input_file8842.txt +example_input_file8843.txt +example_input_file8844.txt +example_input_file8845.txt +example_input_file8846.txt +example_input_file8847.txt +example_input_file8848.txt +example_input_file8849.txt +example_input_file8850.txt +example_input_file8851.txt +example_input_file8852.txt +example_input_file8853.txt +example_input_file8854.txt +example_input_file8855.txt +example_input_file8856.txt +example_input_file8857.txt +example_input_file8858.txt +example_input_file8859.txt +example_input_file8860.txt +example_input_file8861.txt +example_input_file8862.txt +example_input_file8863.txt +example_input_file8864.txt +example_input_file8865.txt +example_input_file8866.txt +example_input_file8867.txt +example_input_file8868.txt +example_input_file8869.txt +example_input_file8870.txt +example_input_file8871.txt +example_input_file8872.txt +example_input_file8873.txt +example_input_file8874.txt +example_input_file8875.txt +example_input_file8876.txt +example_input_file8877.txt +example_input_file8878.txt +example_input_file8879.txt +example_input_file8880.txt +example_input_file8881.txt +example_input_file8882.txt +example_input_file8883.txt +example_input_file8884.txt +example_input_file8885.txt +example_input_file8886.txt +example_input_file8887.txt +example_input_file8888.txt +example_input_file8889.txt +example_input_file8890.txt +example_input_file8891.txt +example_input_file8892.txt +example_input_file8893.txt +example_input_file8894.txt +example_input_file8895.txt +example_input_file8896.txt +example_input_file8897.txt +example_input_file8898.txt +example_input_file8899.txt +example_input_file8900.txt +example_input_file8901.txt +example_input_file8902.txt +example_input_file8903.txt +example_input_file8904.txt +example_input_file8905.txt +example_input_file8906.txt +example_input_file8907.txt +example_input_file8908.txt +example_input_file8909.txt +example_input_file8910.txt +example_input_file8911.txt +example_input_file8912.txt +example_input_file8913.txt +example_input_file8914.txt +example_input_file8915.txt +example_input_file8916.txt +example_input_file8917.txt +example_input_file8918.txt +example_input_file8919.txt +example_input_file8920.txt +example_input_file8921.txt +example_input_file8922.txt +example_input_file8923.txt +example_input_file8924.txt +example_input_file8925.txt +example_input_file8926.txt +example_input_file8927.txt +example_input_file8928.txt +example_input_file8929.txt +example_input_file8930.txt +example_input_file8931.txt +example_input_file8932.txt +example_input_file8933.txt +example_input_file8934.txt +example_input_file8935.txt +example_input_file8936.txt +example_input_file8937.txt +example_input_file8938.txt +example_input_file8939.txt +example_input_file8940.txt +example_input_file8941.txt +example_input_file8942.txt +example_input_file8943.txt +example_input_file8944.txt +example_input_file8945.txt +example_input_file8946.txt +example_input_file8947.txt +example_input_file8948.txt +example_input_file8949.txt +example_input_file8950.txt +example_input_file8951.txt +example_input_file8952.txt +example_input_file8953.txt +example_input_file8954.txt +example_input_file8955.txt +example_input_file8956.txt +example_input_file8957.txt +example_input_file8958.txt +example_input_file8959.txt +example_input_file8960.txt +example_input_file8961.txt +example_input_file8962.txt +example_input_file8963.txt +example_input_file8964.txt +example_input_file8965.txt +example_input_file8966.txt +example_input_file8967.txt +example_input_file8968.txt +example_input_file8969.txt +example_input_file8970.txt +example_input_file8971.txt +example_input_file8972.txt +example_input_file8973.txt +example_input_file8974.txt +example_input_file8975.txt +example_input_file8976.txt +example_input_file8977.txt +example_input_file8978.txt +example_input_file8979.txt +example_input_file8980.txt +example_input_file8981.txt +example_input_file8982.txt +example_input_file8983.txt +example_input_file8984.txt +example_input_file8985.txt +example_input_file8986.txt +example_input_file8987.txt +example_input_file8988.txt +example_input_file8989.txt +example_input_file8990.txt +example_input_file8991.txt +example_input_file8992.txt +example_input_file8993.txt +example_input_file8994.txt +example_input_file8995.txt +example_input_file8996.txt +example_input_file8997.txt +example_input_file8998.txt +example_input_file8999.txt +example_input_file9000.txt +example_input_file9001.txt +example_input_file9002.txt +example_input_file9003.txt +example_input_file9004.txt +example_input_file9005.txt +example_input_file9006.txt +example_input_file9007.txt +example_input_file9008.txt +example_input_file9009.txt +example_input_file9010.txt +example_input_file9011.txt +example_input_file9012.txt +example_input_file9013.txt +example_input_file9014.txt +example_input_file9015.txt +example_input_file9016.txt +example_input_file9017.txt +example_input_file9018.txt +example_input_file9019.txt +example_input_file9020.txt +example_input_file9021.txt +example_input_file9022.txt +example_input_file9023.txt +example_input_file9024.txt +example_input_file9025.txt +example_input_file9026.txt +example_input_file9027.txt +example_input_file9028.txt +example_input_file9029.txt +example_input_file9030.txt +example_input_file9031.txt +example_input_file9032.txt +example_input_file9033.txt +example_input_file9034.txt +example_input_file9035.txt +example_input_file9036.txt +example_input_file9037.txt +example_input_file9038.txt +example_input_file9039.txt +example_input_file9040.txt +example_input_file9041.txt +example_input_file9042.txt +example_input_file9043.txt +example_input_file9044.txt +example_input_file9045.txt +example_input_file9046.txt +example_input_file9047.txt +example_input_file9048.txt +example_input_file9049.txt +example_input_file9050.txt +example_input_file9051.txt +example_input_file9052.txt +example_input_file9053.txt +example_input_file9054.txt +example_input_file9055.txt +example_input_file9056.txt +example_input_file9057.txt +example_input_file9058.txt +example_input_file9059.txt +example_input_file9060.txt +example_input_file9061.txt +example_input_file9062.txt +example_input_file9063.txt +example_input_file9064.txt +example_input_file9065.txt +example_input_file9066.txt +example_input_file9067.txt +example_input_file9068.txt +example_input_file9069.txt +example_input_file9070.txt +example_input_file9071.txt +example_input_file9072.txt +example_input_file9073.txt +example_input_file9074.txt +example_input_file9075.txt +example_input_file9076.txt +example_input_file9077.txt +example_input_file9078.txt +example_input_file9079.txt +example_input_file9080.txt +example_input_file9081.txt +example_input_file9082.txt +example_input_file9083.txt +example_input_file9084.txt +example_input_file9085.txt +example_input_file9086.txt +example_input_file9087.txt +example_input_file9088.txt +example_input_file9089.txt +example_input_file9090.txt +example_input_file9091.txt +example_input_file9092.txt +example_input_file9093.txt +example_input_file9094.txt +example_input_file9095.txt +example_input_file9096.txt +example_input_file9097.txt +example_input_file9098.txt +example_input_file9099.txt +example_input_file9100.txt +example_input_file9101.txt +example_input_file9102.txt +example_input_file9103.txt +example_input_file9104.txt +example_input_file9105.txt +example_input_file9106.txt +example_input_file9107.txt +example_input_file9108.txt +example_input_file9109.txt +example_input_file9110.txt +example_input_file9111.txt +example_input_file9112.txt +example_input_file9113.txt +example_input_file9114.txt +example_input_file9115.txt +example_input_file9116.txt +example_input_file9117.txt +example_input_file9118.txt +example_input_file9119.txt +example_input_file9120.txt +example_input_file9121.txt +example_input_file9122.txt +example_input_file9123.txt +example_input_file9124.txt +example_input_file9125.txt +example_input_file9126.txt +example_input_file9127.txt +example_input_file9128.txt +example_input_file9129.txt +example_input_file9130.txt +example_input_file9131.txt +example_input_file9132.txt +example_input_file9133.txt +example_input_file9134.txt +example_input_file9135.txt +example_input_file9136.txt +example_input_file9137.txt +example_input_file9138.txt +example_input_file9139.txt +example_input_file9140.txt +example_input_file9141.txt +example_input_file9142.txt +example_input_file9143.txt +example_input_file9144.txt +example_input_file9145.txt +example_input_file9146.txt +example_input_file9147.txt +example_input_file9148.txt +example_input_file9149.txt +example_input_file9150.txt +example_input_file9151.txt +example_input_file9152.txt +example_input_file9153.txt +example_input_file9154.txt +example_input_file9155.txt +example_input_file9156.txt +example_input_file9157.txt +example_input_file9158.txt +example_input_file9159.txt +example_input_file9160.txt +example_input_file9161.txt +example_input_file9162.txt +example_input_file9163.txt +example_input_file9164.txt +example_input_file9165.txt +example_input_file9166.txt +example_input_file9167.txt +example_input_file9168.txt +example_input_file9169.txt +example_input_file9170.txt +example_input_file9171.txt +example_input_file9172.txt +example_input_file9173.txt +example_input_file9174.txt +example_input_file9175.txt +example_input_file9176.txt +example_input_file9177.txt +example_input_file9178.txt +example_input_file9179.txt +example_input_file9180.txt +example_input_file9181.txt +example_input_file9182.txt +example_input_file9183.txt +example_input_file9184.txt +example_input_file9185.txt +example_input_file9186.txt +example_input_file9187.txt +example_input_file9188.txt +example_input_file9189.txt +example_input_file9190.txt +example_input_file9191.txt +example_input_file9192.txt +example_input_file9193.txt +example_input_file9194.txt +example_input_file9195.txt +example_input_file9196.txt +example_input_file9197.txt +example_input_file9198.txt +example_input_file9199.txt +example_input_file9200.txt +example_input_file9201.txt +example_input_file9202.txt +example_input_file9203.txt +example_input_file9204.txt +example_input_file9205.txt +example_input_file9206.txt +example_input_file9207.txt +example_input_file9208.txt +example_input_file9209.txt +example_input_file9210.txt +example_input_file9211.txt +example_input_file9212.txt +example_input_file9213.txt +example_input_file9214.txt +example_input_file9215.txt +example_input_file9216.txt +example_input_file9217.txt +example_input_file9218.txt +example_input_file9219.txt +example_input_file9220.txt +example_input_file9221.txt +example_input_file9222.txt +example_input_file9223.txt +example_input_file9224.txt +example_input_file9225.txt +example_input_file9226.txt +example_input_file9227.txt +example_input_file9228.txt +example_input_file9229.txt +example_input_file9230.txt +example_input_file9231.txt +example_input_file9232.txt +example_input_file9233.txt +example_input_file9234.txt +example_input_file9235.txt +example_input_file9236.txt +example_input_file9237.txt +example_input_file9238.txt +example_input_file9239.txt +example_input_file9240.txt +example_input_file9241.txt +example_input_file9242.txt +example_input_file9243.txt +example_input_file9244.txt +example_input_file9245.txt +example_input_file9246.txt +example_input_file9247.txt +example_input_file9248.txt +example_input_file9249.txt +example_input_file9250.txt +example_input_file9251.txt +example_input_file9252.txt +example_input_file9253.txt +example_input_file9254.txt +example_input_file9255.txt +example_input_file9256.txt +example_input_file9257.txt +example_input_file9258.txt +example_input_file9259.txt +example_input_file9260.txt +example_input_file9261.txt +example_input_file9262.txt +example_input_file9263.txt +example_input_file9264.txt +example_input_file9265.txt +example_input_file9266.txt +example_input_file9267.txt +example_input_file9268.txt +example_input_file9269.txt +example_input_file9270.txt +example_input_file9271.txt +example_input_file9272.txt +example_input_file9273.txt +example_input_file9274.txt +example_input_file9275.txt +example_input_file9276.txt +example_input_file9277.txt +example_input_file9278.txt +example_input_file9279.txt +example_input_file9280.txt +example_input_file9281.txt +example_input_file9282.txt +example_input_file9283.txt +example_input_file9284.txt +example_input_file9285.txt +example_input_file9286.txt +example_input_file9287.txt +example_input_file9288.txt +example_input_file9289.txt +example_input_file9290.txt +example_input_file9291.txt +example_input_file9292.txt +example_input_file9293.txt +example_input_file9294.txt +example_input_file9295.txt +example_input_file9296.txt +example_input_file9297.txt +example_input_file9298.txt +example_input_file9299.txt +example_input_file9300.txt +example_input_file9301.txt +example_input_file9302.txt +example_input_file9303.txt +example_input_file9304.txt +example_input_file9305.txt +example_input_file9306.txt +example_input_file9307.txt +example_input_file9308.txt +example_input_file9309.txt +example_input_file9310.txt +example_input_file9311.txt +example_input_file9312.txt +example_input_file9313.txt +example_input_file9314.txt +example_input_file9315.txt +example_input_file9316.txt +example_input_file9317.txt +example_input_file9318.txt +example_input_file9319.txt +example_input_file9320.txt +example_input_file9321.txt +example_input_file9322.txt +example_input_file9323.txt +example_input_file9324.txt +example_input_file9325.txt +example_input_file9326.txt +example_input_file9327.txt +example_input_file9328.txt +example_input_file9329.txt +example_input_file9330.txt +example_input_file9331.txt +example_input_file9332.txt +example_input_file9333.txt +example_input_file9334.txt +example_input_file9335.txt +example_input_file9336.txt +example_input_file9337.txt +example_input_file9338.txt +example_input_file9339.txt +example_input_file9340.txt +example_input_file9341.txt +example_input_file9342.txt +example_input_file9343.txt +example_input_file9344.txt +example_input_file9345.txt +example_input_file9346.txt +example_input_file9347.txt +example_input_file9348.txt +example_input_file9349.txt +example_input_file9350.txt +example_input_file9351.txt +example_input_file9352.txt +example_input_file9353.txt +example_input_file9354.txt +example_input_file9355.txt +example_input_file9356.txt +example_input_file9357.txt +example_input_file9358.txt +example_input_file9359.txt +example_input_file9360.txt +example_input_file9361.txt +example_input_file9362.txt +example_input_file9363.txt +example_input_file9364.txt +example_input_file9365.txt +example_input_file9366.txt +example_input_file9367.txt +example_input_file9368.txt +example_input_file9369.txt +example_input_file9370.txt +example_input_file9371.txt +example_input_file9372.txt +example_input_file9373.txt +example_input_file9374.txt +example_input_file9375.txt +example_input_file9376.txt +example_input_file9377.txt +example_input_file9378.txt +example_input_file9379.txt +example_input_file9380.txt +example_input_file9381.txt +example_input_file9382.txt +example_input_file9383.txt +example_input_file9384.txt +example_input_file9385.txt +example_input_file9386.txt +example_input_file9387.txt +example_input_file9388.txt +example_input_file9389.txt +example_input_file9390.txt +example_input_file9391.txt +example_input_file9392.txt +example_input_file9393.txt +example_input_file9394.txt +example_input_file9395.txt +example_input_file9396.txt +example_input_file9397.txt +example_input_file9398.txt +example_input_file9399.txt +example_input_file9400.txt +example_input_file9401.txt +example_input_file9402.txt +example_input_file9403.txt +example_input_file9404.txt +example_input_file9405.txt +example_input_file9406.txt +example_input_file9407.txt +example_input_file9408.txt +example_input_file9409.txt +example_input_file9410.txt +example_input_file9411.txt +example_input_file9412.txt +example_input_file9413.txt +example_input_file9414.txt +example_input_file9415.txt +example_input_file9416.txt +example_input_file9417.txt +example_input_file9418.txt +example_input_file9419.txt +example_input_file9420.txt +example_input_file9421.txt +example_input_file9422.txt +example_input_file9423.txt +example_input_file9424.txt +example_input_file9425.txt +example_input_file9426.txt +example_input_file9427.txt +example_input_file9428.txt +example_input_file9429.txt +example_input_file9430.txt +example_input_file9431.txt +example_input_file9432.txt +example_input_file9433.txt +example_input_file9434.txt +example_input_file9435.txt +example_input_file9436.txt +example_input_file9437.txt +example_input_file9438.txt +example_input_file9439.txt +example_input_file9440.txt +example_input_file9441.txt +example_input_file9442.txt +example_input_file9443.txt +example_input_file9444.txt +example_input_file9445.txt +example_input_file9446.txt +example_input_file9447.txt +example_input_file9448.txt +example_input_file9449.txt +example_input_file9450.txt +example_input_file9451.txt +example_input_file9452.txt +example_input_file9453.txt +example_input_file9454.txt +example_input_file9455.txt +example_input_file9456.txt +example_input_file9457.txt +example_input_file9458.txt +example_input_file9459.txt +example_input_file9460.txt +example_input_file9461.txt +example_input_file9462.txt +example_input_file9463.txt +example_input_file9464.txt +example_input_file9465.txt +example_input_file9466.txt +example_input_file9467.txt +example_input_file9468.txt +example_input_file9469.txt +example_input_file9470.txt +example_input_file9471.txt +example_input_file9472.txt +example_input_file9473.txt +example_input_file9474.txt +example_input_file9475.txt +example_input_file9476.txt +example_input_file9477.txt +example_input_file9478.txt +example_input_file9479.txt +example_input_file9480.txt +example_input_file9481.txt +example_input_file9482.txt +example_input_file9483.txt +example_input_file9484.txt +example_input_file9485.txt +example_input_file9486.txt +example_input_file9487.txt +example_input_file9488.txt +example_input_file9489.txt +example_input_file9490.txt +example_input_file9491.txt +example_input_file9492.txt +example_input_file9493.txt +example_input_file9494.txt +example_input_file9495.txt +example_input_file9496.txt +example_input_file9497.txt +example_input_file9498.txt +example_input_file9499.txt +example_input_file9500.txt +example_input_file9501.txt +example_input_file9502.txt +example_input_file9503.txt +example_input_file9504.txt +example_input_file9505.txt +example_input_file9506.txt +example_input_file9507.txt +example_input_file9508.txt +example_input_file9509.txt +example_input_file9510.txt +example_input_file9511.txt +example_input_file9512.txt +example_input_file9513.txt +example_input_file9514.txt +example_input_file9515.txt +example_input_file9516.txt +example_input_file9517.txt +example_input_file9518.txt +example_input_file9519.txt +example_input_file9520.txt +example_input_file9521.txt +example_input_file9522.txt +example_input_file9523.txt +example_input_file9524.txt +example_input_file9525.txt +example_input_file9526.txt +example_input_file9527.txt +example_input_file9528.txt +example_input_file9529.txt +example_input_file9530.txt +example_input_file9531.txt +example_input_file9532.txt +example_input_file9533.txt +example_input_file9534.txt +example_input_file9535.txt +example_input_file9536.txt +example_input_file9537.txt +example_input_file9538.txt +example_input_file9539.txt +example_input_file9540.txt +example_input_file9541.txt +example_input_file9542.txt +example_input_file9543.txt +example_input_file9544.txt +example_input_file9545.txt +example_input_file9546.txt +example_input_file9547.txt +example_input_file9548.txt +example_input_file9549.txt +example_input_file9550.txt +example_input_file9551.txt +example_input_file9552.txt +example_input_file9553.txt +example_input_file9554.txt +example_input_file9555.txt +example_input_file9556.txt +example_input_file9557.txt +example_input_file9558.txt +example_input_file9559.txt +example_input_file9560.txt +example_input_file9561.txt +example_input_file9562.txt +example_input_file9563.txt +example_input_file9564.txt +example_input_file9565.txt +example_input_file9566.txt +example_input_file9567.txt +example_input_file9568.txt +example_input_file9569.txt +example_input_file9570.txt +example_input_file9571.txt +example_input_file9572.txt +example_input_file9573.txt +example_input_file9574.txt +example_input_file9575.txt +example_input_file9576.txt +example_input_file9577.txt +example_input_file9578.txt +example_input_file9579.txt +example_input_file9580.txt +example_input_file9581.txt +example_input_file9582.txt +example_input_file9583.txt +example_input_file9584.txt +example_input_file9585.txt +example_input_file9586.txt +example_input_file9587.txt +example_input_file9588.txt +example_input_file9589.txt +example_input_file9590.txt +example_input_file9591.txt +example_input_file9592.txt +example_input_file9593.txt +example_input_file9594.txt +example_input_file9595.txt +example_input_file9596.txt +example_input_file9597.txt +example_input_file9598.txt +example_input_file9599.txt +example_input_file9600.txt +example_input_file9601.txt +example_input_file9602.txt +example_input_file9603.txt +example_input_file9604.txt +example_input_file9605.txt +example_input_file9606.txt +example_input_file9607.txt +example_input_file9608.txt +example_input_file9609.txt +example_input_file9610.txt +example_input_file9611.txt +example_input_file9612.txt +example_input_file9613.txt +example_input_file9614.txt +example_input_file9615.txt +example_input_file9616.txt +example_input_file9617.txt +example_input_file9618.txt +example_input_file9619.txt +example_input_file9620.txt +example_input_file9621.txt +example_input_file9622.txt +example_input_file9623.txt +example_input_file9624.txt +example_input_file9625.txt +example_input_file9626.txt +example_input_file9627.txt +example_input_file9628.txt +example_input_file9629.txt +example_input_file9630.txt +example_input_file9631.txt +example_input_file9632.txt +example_input_file9633.txt +example_input_file9634.txt +example_input_file9635.txt +example_input_file9636.txt +example_input_file9637.txt +example_input_file9638.txt +example_input_file9639.txt +example_input_file9640.txt +example_input_file9641.txt +example_input_file9642.txt +example_input_file9643.txt +example_input_file9644.txt +example_input_file9645.txt +example_input_file9646.txt +example_input_file9647.txt +example_input_file9648.txt +example_input_file9649.txt +example_input_file9650.txt +example_input_file9651.txt +example_input_file9652.txt +example_input_file9653.txt +example_input_file9654.txt +example_input_file9655.txt +example_input_file9656.txt +example_input_file9657.txt +example_input_file9658.txt +example_input_file9659.txt +example_input_file9660.txt +example_input_file9661.txt +example_input_file9662.txt +example_input_file9663.txt +example_input_file9664.txt +example_input_file9665.txt +example_input_file9666.txt +example_input_file9667.txt +example_input_file9668.txt +example_input_file9669.txt +example_input_file9670.txt +example_input_file9671.txt +example_input_file9672.txt +example_input_file9673.txt +example_input_file9674.txt +example_input_file9675.txt +example_input_file9676.txt +example_input_file9677.txt +example_input_file9678.txt +example_input_file9679.txt +example_input_file9680.txt +example_input_file9681.txt +example_input_file9682.txt +example_input_file9683.txt +example_input_file9684.txt +example_input_file9685.txt +example_input_file9686.txt +example_input_file9687.txt +example_input_file9688.txt +example_input_file9689.txt +example_input_file9690.txt +example_input_file9691.txt +example_input_file9692.txt +example_input_file9693.txt +example_input_file9694.txt +example_input_file9695.txt +example_input_file9696.txt +example_input_file9697.txt +example_input_file9698.txt +example_input_file9699.txt +example_input_file9700.txt +example_input_file9701.txt +example_input_file9702.txt +example_input_file9703.txt +example_input_file9704.txt +example_input_file9705.txt +example_input_file9706.txt +example_input_file9707.txt +example_input_file9708.txt +example_input_file9709.txt +example_input_file9710.txt +example_input_file9711.txt +example_input_file9712.txt +example_input_file9713.txt +example_input_file9714.txt +example_input_file9715.txt +example_input_file9716.txt +example_input_file9717.txt +example_input_file9718.txt +example_input_file9719.txt +example_input_file9720.txt +example_input_file9721.txt +example_input_file9722.txt +example_input_file9723.txt +example_input_file9724.txt +example_input_file9725.txt +example_input_file9726.txt +example_input_file9727.txt +example_input_file9728.txt +example_input_file9729.txt +example_input_file9730.txt +example_input_file9731.txt +example_input_file9732.txt +example_input_file9733.txt +example_input_file9734.txt +example_input_file9735.txt +example_input_file9736.txt +example_input_file9737.txt +example_input_file9738.txt +example_input_file9739.txt +example_input_file9740.txt +example_input_file9741.txt +example_input_file9742.txt +example_input_file9743.txt +example_input_file9744.txt +example_input_file9745.txt +example_input_file9746.txt +example_input_file9747.txt +example_input_file9748.txt +example_input_file9749.txt +example_input_file9750.txt +example_input_file9751.txt +example_input_file9752.txt +example_input_file9753.txt +example_input_file9754.txt +example_input_file9755.txt +example_input_file9756.txt +example_input_file9757.txt +example_input_file9758.txt +example_input_file9759.txt +example_input_file9760.txt +example_input_file9761.txt +example_input_file9762.txt +example_input_file9763.txt +example_input_file9764.txt +example_input_file9765.txt +example_input_file9766.txt +example_input_file9767.txt +example_input_file9768.txt +example_input_file9769.txt +example_input_file9770.txt +example_input_file9771.txt +example_input_file9772.txt +example_input_file9773.txt +example_input_file9774.txt +example_input_file9775.txt +example_input_file9776.txt +example_input_file9777.txt +example_input_file9778.txt +example_input_file9779.txt +example_input_file9780.txt +example_input_file9781.txt +example_input_file9782.txt +example_input_file9783.txt +example_input_file9784.txt +example_input_file9785.txt +example_input_file9786.txt +example_input_file9787.txt +example_input_file9788.txt +example_input_file9789.txt +example_input_file9790.txt +example_input_file9791.txt +example_input_file9792.txt +example_input_file9793.txt +example_input_file9794.txt +example_input_file9795.txt +example_input_file9796.txt +example_input_file9797.txt +example_input_file9798.txt +example_input_file9799.txt +example_input_file9800.txt +example_input_file9801.txt +example_input_file9802.txt +example_input_file9803.txt +example_input_file9804.txt +example_input_file9805.txt +example_input_file9806.txt +example_input_file9807.txt +example_input_file9808.txt +example_input_file9809.txt +example_input_file9810.txt +example_input_file9811.txt +example_input_file9812.txt +example_input_file9813.txt +example_input_file9814.txt +example_input_file9815.txt +example_input_file9816.txt +example_input_file9817.txt +example_input_file9818.txt +example_input_file9819.txt +example_input_file9820.txt +example_input_file9821.txt +example_input_file9822.txt +example_input_file9823.txt +example_input_file9824.txt +example_input_file9825.txt +example_input_file9826.txt +example_input_file9827.txt +example_input_file9828.txt +example_input_file9829.txt +example_input_file9830.txt +example_input_file9831.txt +example_input_file9832.txt +example_input_file9833.txt +example_input_file9834.txt +example_input_file9835.txt +example_input_file9836.txt +example_input_file9837.txt +example_input_file9838.txt +example_input_file9839.txt +example_input_file9840.txt +example_input_file9841.txt +example_input_file9842.txt +example_input_file9843.txt +example_input_file9844.txt +example_input_file9845.txt +example_input_file9846.txt +example_input_file9847.txt +example_input_file9848.txt +example_input_file9849.txt +example_input_file9850.txt +example_input_file9851.txt +example_input_file9852.txt +example_input_file9853.txt +example_input_file9854.txt +example_input_file9855.txt +example_input_file9856.txt +example_input_file9857.txt +example_input_file9858.txt +example_input_file9859.txt +example_input_file9860.txt +example_input_file9861.txt +example_input_file9862.txt +example_input_file9863.txt +example_input_file9864.txt +example_input_file9865.txt +example_input_file9866.txt +example_input_file9867.txt +example_input_file9868.txt +example_input_file9869.txt +example_input_file9870.txt +example_input_file9871.txt +example_input_file9872.txt +example_input_file9873.txt +example_input_file9874.txt +example_input_file9875.txt +example_input_file9876.txt +example_input_file9877.txt +example_input_file9878.txt +example_input_file9879.txt +example_input_file9880.txt +example_input_file9881.txt +example_input_file9882.txt +example_input_file9883.txt +example_input_file9884.txt +example_input_file9885.txt +example_input_file9886.txt +example_input_file9887.txt +example_input_file9888.txt +example_input_file9889.txt +example_input_file9890.txt +example_input_file9891.txt +example_input_file9892.txt +example_input_file9893.txt +example_input_file9894.txt +example_input_file9895.txt +example_input_file9896.txt +example_input_file9897.txt +example_input_file9898.txt +example_input_file9899.txt +example_input_file9900.txt +example_input_file9901.txt +example_input_file9902.txt +example_input_file9903.txt +example_input_file9904.txt +example_input_file9905.txt +example_input_file9906.txt +example_input_file9907.txt +example_input_file9908.txt +example_input_file9909.txt +example_input_file9910.txt +example_input_file9911.txt +example_input_file9912.txt +example_input_file9913.txt +example_input_file9914.txt +example_input_file9915.txt +example_input_file9916.txt +example_input_file9917.txt +example_input_file9918.txt +example_input_file9919.txt +example_input_file9920.txt +example_input_file9921.txt +example_input_file9922.txt +example_input_file9923.txt +example_input_file9924.txt +example_input_file9925.txt +example_input_file9926.txt +example_input_file9927.txt +example_input_file9928.txt +example_input_file9929.txt +example_input_file9930.txt +example_input_file9931.txt +example_input_file9932.txt +example_input_file9933.txt +example_input_file9934.txt +example_input_file9935.txt +example_input_file9936.txt +example_input_file9937.txt +example_input_file9938.txt +example_input_file9939.txt +example_input_file9940.txt +example_input_file9941.txt +example_input_file9942.txt +example_input_file9943.txt +example_input_file9944.txt +example_input_file9945.txt +example_input_file9946.txt +example_input_file9947.txt +example_input_file9948.txt +example_input_file9949.txt +example_input_file9950.txt +example_input_file9951.txt +example_input_file9952.txt +example_input_file9953.txt +example_input_file9954.txt +example_input_file9955.txt +example_input_file9956.txt +example_input_file9957.txt +example_input_file9958.txt +example_input_file9959.txt +example_input_file9960.txt +example_input_file9961.txt +example_input_file9962.txt +example_input_file9963.txt +example_input_file9964.txt +example_input_file9965.txt +example_input_file9966.txt +example_input_file9967.txt +example_input_file9968.txt +example_input_file9969.txt +example_input_file9970.txt +example_input_file9971.txt +example_input_file9972.txt +example_input_file9973.txt +example_input_file9974.txt +example_input_file9975.txt +example_input_file9976.txt +example_input_file9977.txt +example_input_file9978.txt +example_input_file9979.txt +example_input_file9980.txt +example_input_file9981.txt +example_input_file9982.txt +example_input_file9983.txt +example_input_file9984.txt +example_input_file9985.txt +example_input_file9986.txt +example_input_file9987.txt +example_input_file9988.txt +example_input_file9989.txt +example_input_file9990.txt +example_input_file9991.txt +example_input_file9992.txt +example_input_file9993.txt +example_input_file9994.txt +example_input_file9995.txt +example_input_file9996.txt +example_input_file9997.txt +example_input_file9998.txt +example_input_file9999.txt diff --git a/tests/wf/input_named_id.cwl b/tests/wf/input_named_id.cwl old mode 100644 new mode 100755 index e559f967b..82a0353aa --- a/tests/wf/input_named_id.cwl +++ b/tests/wf/input_named_id.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner label: FeatureFinderIdentification doc: "" inputs: diff --git a/tests/wf/iwd-container-entryname1.cwl b/tests/wf/iwd-container-entryname1.cwl new file mode 100644 index 000000000..3fe16c9e8 --- /dev/null +++ b/tests/wf/iwd-container-entryname1.cwl @@ -0,0 +1,23 @@ +cwlVersion: v1.2 +class: CommandLineTool +doc: | + When executing in a container, entryname can have an absolute path + to a mount location inside the container. +inputs: + filelist: File +outputs: + head: + type: File + outputBinding: + glob: head.txt +requirements: + DockerRequirement: + dockerPull: docker.io/debian:stable-slim + dockerOutputDirectory: /output + InitialWorkDirRequirement: + listing: + - entryname: /tmp2j3y7rpb/input/stuff.txt # Give it a weird prefix to minimize chance of conflict with a real file + entry: $(inputs.filelist) + ShellCommandRequirement: {} +arguments: + - {shellQuote: false, valueFrom: "head -n10 /tmp2j3y7rpb/input/stuff.txt > /output/head.txt"} diff --git a/tests/wf/iwd-container-entryname3.cwl b/tests/wf/iwd-container-entryname3.cwl new file mode 100644 index 000000000..cfa14e631 --- /dev/null +++ b/tests/wf/iwd-container-entryname3.cwl @@ -0,0 +1,24 @@ +cwlVersion: v1.2 +class: CommandLineTool +doc: | + Must fail if entryname is an absolute path and DockerRequirement is + not in the 'requirements' section. +inputs: + filelist: File +outputs: + head: + type: File + outputBinding: + glob: head.txt +hints: + DockerRequirement: + dockerPull: docker.io/debian:stable-slim + dockerOutputDirectory: /output +requirements: + InitialWorkDirRequirement: + listing: + - entryname: /tmp2j3y7rpb/input/stuff.txt # Give it a weird prefix to minimize chance of conflict with a real file + entry: $(inputs.filelist) + ShellCommandRequirement: {} +arguments: + - {shellQuote: false, valueFrom: "head -n10 /tmp2j3y7rpb/input/stuff.txt > /output/head.txt"} diff --git a/tests/wf/iwd-passthrough1.cwl b/tests/wf/iwd-passthrough1.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/iwdr-empty.cwl b/tests/wf/iwdr-empty.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/iwdr-entry.cwl b/tests/wf/iwdr-entry.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/iwdr-passthrough-successive.cwl b/tests/wf/iwdr-passthrough-successive.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/literalfile.cwl b/tests/wf/literalfile.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/loadContents-input.yml b/tests/wf/loadContents-input.yml new file mode 100644 index 000000000..a44fd27ca --- /dev/null +++ b/tests/wf/loadContents-input.yml @@ -0,0 +1,3 @@ +filelist: + class: File + location: inp-filelist.txt diff --git a/tests/wf/malformed_outputs.cwl b/tests/wf/malformed_outputs.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/missing-tool.cwl b/tests/wf/missing-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_env.cwl b/tests/wf/mpi_env.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_expr.cwl b/tests/wf/mpi_expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_line_count.cwl b/tests/wf/mpi_line_count.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_simple.cwl b/tests/wf/mpi_simple.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_simple_wf.cwl b/tests/wf/mpi_simple_wf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/nested.cwl b/tests/wf/nested.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/networkaccess-fail.cwl b/tests/wf/networkaccess-fail.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/networkaccess.cwl b/tests/wf/networkaccess.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/no-parameters-echo.cwl b/tests/wf/no-parameters-echo.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/nvidia-smi-cc.cwl b/tests/wf/nvidia-smi-cc.cwl old mode 100644 new mode 100755 index a4f315b0e..92e781d26 --- a/tests/wf/nvidia-smi-cc.cwl +++ b/tests/wf/nvidia-smi-cc.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi-container.cwl b/tests/wf/nvidia-smi-container.cwl old mode 100644 new mode 100755 index 84fd72d83..22092fae4 --- a/tests/wf/nvidia-smi-container.cwl +++ b/tests/wf/nvidia-smi-container.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi-max.cwl b/tests/wf/nvidia-smi-max.cwl old mode 100644 new mode 100755 index d3d4d5e9c..1f8687d4d --- a/tests/wf/nvidia-smi-max.cwl +++ b/tests/wf/nvidia-smi-max.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi-range.cwl b/tests/wf/nvidia-smi-range.cwl old mode 100644 new mode 100755 index 19d3ea43c..2e131a373 --- a/tests/wf/nvidia-smi-range.cwl +++ b/tests/wf/nvidia-smi-range.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi.cwl b/tests/wf/nvidia-smi.cwl old mode 100644 new mode 100755 index 8e227d0c5..8a17f8e17 --- a/tests/wf/nvidia-smi.cwl +++ b/tests/wf/nvidia-smi.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/operation/abstract-cosifer.cwl b/tests/wf/operation/abstract-cosifer.cwl old mode 100644 new mode 100755 index 057620844..2a50cded7 --- a/tests/wf/operation/abstract-cosifer.cwl +++ b/tests/wf/operation/abstract-cosifer.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Operation cwlVersion: v1.2 diff --git a/tests/wf/operation/operation-single.cwl b/tests/wf/operation/operation-single.cwl old mode 100644 new mode 100755 index 2119bf02e..63b6a6bf0 --- a/tests/wf/operation/operation-single.cwl +++ b/tests/wf/operation/operation-single.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Workflow cwlVersion: v1.2 id: abstract_cosifer_workflow diff --git a/tests/wf/optional-numerical-output-0.cwl b/tests/wf/optional-numerical-output-0.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/optional_src_mandatory_sink.cwl b/tests/wf/optional_src_mandatory_sink.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/packed-with-loadlisting.cwl b/tests/wf/packed-with-loadlisting.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/packed_no_main.cwl b/tests/wf/packed_no_main.cwl old mode 100644 new mode 100755 index 8307a083e..aafd0de58 --- a/tests/wf/packed_no_main.cwl +++ b/tests/wf/packed_no_main.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 $graph: - id: echo diff --git a/tests/wf/paramref_arguments_roundtrip.cwl b/tests/wf/paramref_arguments_roundtrip.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/paramref_arguments_self.cwl b/tests/wf/paramref_arguments_self.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/record_outputeval.cwl b/tests/wf/record_outputeval.cwl old mode 100644 new mode 100755 index 45daf9b4d..e7b59cb6c --- a/tests/wf/record_outputeval.cwl +++ b/tests/wf/record_outputeval.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool requirements: diff --git a/tests/wf/resreq_expr_float_v1_0.cwl b/tests/wf/resreq_expr_float_v1_0.cwl old mode 100644 new mode 100755 index d5ea08cde..a36877c0f --- a/tests/wf/resreq_expr_float_v1_0.cwl +++ b/tests/wf/resreq_expr_float_v1_0.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool requirements: diff --git a/tests/wf/resreq_expr_float_v1_2.cwl b/tests/wf/resreq_expr_float_v1_2.cwl old mode 100644 new mode 100755 index cfb1b2a13..ca72364b7 --- a/tests/wf/resreq_expr_float_v1_2.cwl +++ b/tests/wf/resreq_expr_float_v1_2.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool requirements: diff --git a/tests/wf/scatter2.cwl b/tests/wf/scatter2.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/scatter2_subwf.cwl b/tests/wf/scatter2_subwf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/schemadef-bug-1473.cwl b/tests/wf/schemadef-bug-1473.cwl old mode 100644 new mode 100755 index ad87ae08e..beab3211d --- a/tests/wf/schemadef-bug-1473.cwl +++ b/tests/wf/schemadef-bug-1473.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "$graph": [ { diff --git a/tests/wf/schemadef-tool-12.cwl b/tests/wf/schemadef-tool-12.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/schemadef-tool.cwl b/tests/wf/schemadef-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/sec-tool.cwl b/tests/wf/sec-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/sec-wf-out.cwl b/tests/wf/sec-wf-out.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/sec-wf.cwl b/tests/wf/sec-wf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/secret_job.cwl b/tests/wf/secret_job.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/secret_wf.cwl b/tests/wf/secret_wf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/separate_without_prefix.cwl b/tests/wf/separate_without_prefix.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/shm_size.cwl b/tests/wf/shm_size.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/storage_float.cwl b/tests/wf/storage_float.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/tar-param.cwl b/tests/wf/tar-param.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/three_step_color.cwl b/tests/wf/three_step_color.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/timelimit-fail.cwl b/tests/wf/timelimit-fail.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/timelimit.cwl b/tests/wf/timelimit.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/touch_tool.cwl b/tests/wf/touch_tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/trick_defaults.cwl b/tests/wf/trick_defaults.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/trick_defaults2.cwl b/tests/wf/trick_defaults2.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/vf-concat.cwl b/tests/wf/vf-concat.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/workreuse-fail.cwl b/tests/wf/workreuse-fail.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/workreuse.cwl b/tests/wf/workreuse.cwl old mode 100644 new mode 100755 diff --git a/tests/with_doc.cwl b/tests/with_doc.cwl old mode 100644 new mode 100755 diff --git a/tests/without_doc.cwl b/tests/without_doc.cwl old mode 100644 new mode 100755 diff --git a/tox.ini b/tox.ini index c75d0cc47..5350201cd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = - py3{8,9,10,11,12,13}-lint - py3{8,9,10,11,12,13}-unit - py3{8,9,10,11,12,13}-bandit - py3{8,9,10,11,12,13}-mypy + py3{10,11,12,13,14}-lint + py3{10,11,12,13,14}-unit + py3{10,11,12,13,14}-bandit + py3{10,11,12,13,14}-mypy py312-lintreadme py312-shellcheck py312-pydocstyle @@ -11,27 +11,26 @@ envlist = skip_missing_interpreters = True [pytest] -addopts=--ignore cwltool/schemas -n logical --dist worksteal +addopts=--ignore cwltool/schemas -n logical --dist worksteal -rsfE testpaths = tests [gh-actions] python = - 3.8: py38 - 3.9: py39 3.10: py310 3.11: py311 3.12: py312 3.13: py313 + 3.14: py314 [testenv] skipsdist = - py3{8,9,10,11,12,13}-!{unit,mypy,lintreadme} = True + py3{10,11,12,13,14}-!{unit,mypy,lintreadme} = True description = - py3{8,9,10,11,12,13}-unit: Run the unit tests - py3{8,9,10,11,12,13}-lint: Lint the Python code - py3{8,9,10,11,12,13}-bandit: Search for common security issues - py3{8,9,10,11,12,13}-mypy: Check for type safety + py3{10,11,12,13,14}-unit: Run the unit tests + py3{10,11,12,13,14}-lint: Lint the Python code + py3{10,11,12,13,14}-bandit: Search for common security issues + py3{10,11,12,13,14}-mypy: Check for type safety py312-pydocstyle: docstring style checker py312-shellcheck: syntax check for shell scripts py312-lintreadme: Lint the README.rst→.md conversion @@ -44,14 +43,14 @@ passenv = SINGULARITY_FAKEROOT extras = - py3{8,9,10,11,12,13}-unit: deps + py3{10,11,12,13,14}-unit: deps deps = - py3{8,9,10,11,12,13}-{unit,lint,bandit,mypy}: -rrequirements.txt - py3{8,9,10,11,12,13}-{unit,mypy}: -rtest-requirements.txt - py3{8,9,10,11,12,13}-lint: -rlint-requirements.txt - py3{8,9,10,11,12,13}-bandit: bandit - py3{8,9,10,11,12,13}-mypy: -rmypy-requirements.txt + py3{10,11,12,13,14}-{unit,lint,bandit,mypy}: -rrequirements.txt + py3{10,11,12,13,14}-{unit,mypy}: -rtest-requirements.txt + py3{10,11,12,13,14}-lint: -rlint-requirements.txt + py3{10,11,12,13,14}-bandit: bandit + py3{10,11,12,13,14}-mypy: -rmypy-requirements.txt py312-pydocstyle: pydocstyle py312-pydocstyle: diff-cover py312-lintreadme: twine @@ -63,20 +62,19 @@ setenv = HOME = {envtmpdir} commands_pre = - py3{8,9,10,11,12,13}-unit: python -m pip install -U pip setuptools wheel + py3{10,11,12,13}-unit: python -m pip install -U pip setuptools wheel py312-lintreadme: python -m build --outdir {distdir} commands = - py3{8,9,10,11,12,13}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} - py3{8,9,10,11,12,13}-bandit: bandit -r cwltool - py3{8,9,10,11,12,13}-lint: make flake8 format-check codespell-check - py3{8,9,10,11,12,13}-mypy: make mypy PYTEST_EXTRA={posargs} - py3{8,9,10,11,12}-mypy: make mypyc PYTEST_EXTRA={posargs} + py3{10,11,12,13,14}-unit: make coverage-report coverage.xml PYTEST_EXTRA="{posargs}" + py3{10,11,12,13,14}-bandit: bandit -r cwltool + py3{10,11,12,13,14}-lint: make flake8 format-check codespell-check + py3{10,11,12,13,14}-mypy: make mypy mypyc PYTEST_EXTRA="{posargs}" py312-shellcheck: make shellcheck py312-pydocstyle: make diff_pydocstyle_report py312-lintreadme: twine check {distdir}/* skip_install = - py3{8,9,10,11,12,13}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true + py3{10,11,12,13,14}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true allowlist_externals = make