diff --git a/.github/workflows/scripts-unit-tests.yml b/.github/workflows/scripts-unit-tests.yml new file mode 100644 index 0000000..d1578ab --- /dev/null +++ b/.github/workflows/scripts-unit-tests.yml @@ -0,0 +1,30 @@ +name: scripts-unit-tests +env: + PYTHON_VERSION: 3.11 +on: + pull_request: + paths: + - 'scripts/validate_metadata/**' +jobs: + execute-scripts-unit-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install test dependencies + run: | + pip install -r scripts/validate_metadata/requirements.txt + + - name: Install Test dependencies + run: | + pip install -r scripts/validate_metadata/requirements.txt + + - name: Run tests + run: pytest scripts/validate_metadata diff --git a/.github/workflows/validate-metadata-schema.yml b/.github/workflows/validate-metadata-schema.yml new file mode 100644 index 0000000..82e35c6 --- /dev/null +++ b/.github/workflows/validate-metadata-schema.yml @@ -0,0 +1,49 @@ +name: validate-metadata-schema +env: + PYTHON_VERSION: 3.11 + RETRIEVAL_SCRIPT_PATH: $GITHUB_WORKSPACE/.github/resources/scripts/get_items_for_validation.sh + VALIDATION_SCRIPT_PATH: $GITHUB_WORKSPACE/scripts/validate_metadata/validate_metadata.py +on: + pull_request: + paths: + - 'components/**' + - 'pipelines/**' + - 'scripts/**' + - 'third_party/components/**' + - 'third_party/pipelines/**' + +jobs: + validate-metadata-schema: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Test dependencies + run: | + pip install -r scripts/validate_metadata/requirements.txt + + - name: Retrieve new components and/or pipelines + id: get-new-items + run: ${{ env.RETRIEVAL_SCRIPT_PATH }} "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" validate_metadata + + #ToDo: Utilize .github/scripts/find-changed-components-and-pipelines.sh script once merged in from https://github.com/kubeflow/pipelines-components/pull/7 + - name: Validate new core/third-party components and/or pipelines + if: ${{ steps.get-new-items.outputs.new_items_list != '' }} + run: | + NEW_ITEMS_ARRAY="${{ steps.get-new-items.outputs.new_items_list }}" + + # Set IFS to a comma, so that the shell will split the string by commas. + IFS=',' + + for item in $NEW_ITEMS_ARRAY; do + FILE_PATH="$GITHUB_WORKSPACE/$item" + echo "Processing item: $item" + python "${{ env.VALIDATION_SCRIPT_PATH }}" --item $FILE_PATH + done diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/validate_metadata/__init__.py b/scripts/validate_metadata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/validate_metadata/requirements.txt b/scripts/validate_metadata/requirements.txt new file mode 100644 index 0000000..fc9d9ca --- /dev/null +++ b/scripts/validate_metadata/requirements.txt @@ -0,0 +1,3 @@ +PyYAML==6.0.3 +semver==3.0.4 +pytest==7.1.2 diff --git a/scripts/validate_metadata/test_data/directories_metadata/missing_metadata_file/OWNERS b/scripts/validate_metadata/test_data/directories_metadata/missing_metadata_file/OWNERS new file mode 100644 index 0000000..fa71bd1 --- /dev/null +++ b/scripts/validate_metadata/test_data/directories_metadata/missing_metadata_file/OWNERS @@ -0,0 +1,2 @@ +approvers: +- sample-approver diff --git a/scripts/validate_metadata/test_data/directories_metadata/missing_owners_file/metadata.yaml b/scripts/validate_metadata/test_data/directories_metadata/missing_owners_file/metadata.yaml new file mode 100644 index 0000000..7f0ac4d --- /dev/null +++ b/scripts/validate_metadata/test_data/directories_metadata/missing_owners_file/metadata.yaml @@ -0,0 +1,22 @@ +tier: third_party +name: happy-path-component +stability: alpha | beta | stable +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5' + - name: Trainer + version: '>=2.0' + external_services: + - name: Argo Workflows + version: "3.6" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false + pytest: optional +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/directories_metadata/not_a_dir.txt b/scripts/validate_metadata/test_data/directories_metadata/not_a_dir.txt new file mode 100644 index 0000000..e69de29 diff --git a/scripts/validate_metadata/test_data/directories_metadata/valid/OWNERS b/scripts/validate_metadata/test_data/directories_metadata/valid/OWNERS new file mode 100644 index 0000000..fa71bd1 --- /dev/null +++ b/scripts/validate_metadata/test_data/directories_metadata/valid/OWNERS @@ -0,0 +1,2 @@ +approvers: +- sample-approver diff --git a/scripts/validate_metadata/test_data/directories_metadata/valid/metadata.yaml b/scripts/validate_metadata/test_data/directories_metadata/valid/metadata.yaml new file mode 100644 index 0000000..ac1065a --- /dev/null +++ b/scripts/validate_metadata/test_data/directories_metadata/valid/metadata.yaml @@ -0,0 +1,21 @@ +name: component_valid +tier: third_party +stability: alpha +dependencies: + kubeflow: + - name: Pipelines + version: '2.5.0' + - name: Trainer + version: '2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_ci.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_ci.yaml new file mode 100644 index 0000000..3e5258a --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_ci.yaml @@ -0,0 +1,20 @@ +name: invalid-ci +tier: core +stability: alpha +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: invalid-ci-value +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_ci_category.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_ci_category.yaml new file mode 100644 index 0000000..ca2fb85 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_ci_category.yaml @@ -0,0 +1,22 @@ +name: invalid-ci-category +tier: core +stability: stable +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false + invalid_ci_category: invalid_value +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_ci_dependency_probe.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_ci_dependency_probe.yaml new file mode 100644 index 0000000..017be5b --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_ci_dependency_probe.yaml @@ -0,0 +1,21 @@ +name: invalid-ci-dependency-probe +tier: third_party +stability: beta +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: invalid-probe-value +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_dependencies_category.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_dependencies_category.yaml new file mode 100644 index 0000000..0c2d0f8 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_dependencies_category.yaml @@ -0,0 +1,24 @@ +name: invalid-dependencies-category +tier: core +stability: beta +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" + invalid_dependency_category: + - name: Invalid Dependency + version: "1.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_dependencies_type.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_dependencies_type.yaml new file mode 100644 index 0000000..0dd3447 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_dependencies_type.yaml @@ -0,0 +1,13 @@ +name: invalid-dependencies-type +tier: third_party +stability: alpha +dependencies: "invalid-dependencies-type" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_dependency_semantic_versioning.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_dependency_semantic_versioning.yaml new file mode 100644 index 0000000..510e6aa --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_dependency_semantic_versioning.yaml @@ -0,0 +1,21 @@ +name: invalid-dependency-semantic-versioning +tier: third_party +stability: stable +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_links.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_links.yaml new file mode 100644 index 0000000..bdf0168 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_links.yaml @@ -0,0 +1,19 @@ +name: invalid-links +tier: core +stability: alpha +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: https://invalid-link diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_name.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_name.yaml new file mode 100644 index 0000000..b5c661e --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_name.yaml @@ -0,0 +1,21 @@ +name: 2 +tier: core +stability: beta +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_stability.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_stability.yaml new file mode 100644 index 0000000..846bbd3 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_stability.yaml @@ -0,0 +1,21 @@ +name: invalid-stability +tier: third_party +stability: invalid-stability +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_tag_array_type.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_tag_array_type.yaml new file mode 100644 index 0000000..43f3fb2 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_tag_array_type.yaml @@ -0,0 +1,21 @@ +name: invalid-tag-array-type +tier: core +stability: stable +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - 1 + - 2 +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_tag_type.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_tag_type.yaml new file mode 100644 index 0000000..ec98443 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_tag_type.yaml @@ -0,0 +1,19 @@ +name: invalid-tag-type +tier: core +stability: stable +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: "tags" +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_tier.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_tier.yaml new file mode 100644 index 0000000..e20be68 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_tier.yaml @@ -0,0 +1,21 @@ +name: invalid-tier +tier: invalid_tier +stability: alpha +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/invalid_verified_date.yaml b/scripts/validate_metadata/test_data/metadata/invalid/invalid_verified_date.yaml new file mode 100644 index 0000000..7f15846 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/invalid_verified_date.yaml @@ -0,0 +1,21 @@ +name: invalid-verified-date +tier: core +stability: beta +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2024-11-20T0 +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/missing_dependencies.yaml b/scripts/validate_metadata/test_data/metadata/invalid/missing_dependencies.yaml new file mode 100644 index 0000000..71deeae --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/missing_dependencies.yaml @@ -0,0 +1,12 @@ +name: missing-dependencies +tier: third_party +stability: stable +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/missing_kfp_dependency.yaml b/scripts/validate_metadata/test_data/metadata/invalid/missing_kfp_dependency.yaml new file mode 100644 index 0000000..b21cdf0 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/missing_kfp_dependency.yaml @@ -0,0 +1,19 @@ +name: missing-kfp-dependency +tier: core +stability: alpha +dependencies: + kubeflow: + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/missing_name.yaml b/scripts/validate_metadata/test_data/metadata/invalid/missing_name.yaml new file mode 100644 index 0000000..5f26fc1 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/missing_name.yaml @@ -0,0 +1,20 @@ +tier: core +stability: stable +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/missing_stability.yaml b/scripts/validate_metadata/test_data/metadata/invalid/missing_stability.yaml new file mode 100644 index 0000000..2599ad0 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/missing_stability.yaml @@ -0,0 +1,20 @@ +name: missing-stability +tier: third_party +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/missing_tier.yaml b/scripts/validate_metadata/test_data/metadata/invalid/missing_tier.yaml new file mode 100644 index 0000000..598baec --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/missing_tier.yaml @@ -0,0 +1,20 @@ +name: missing-tier +stability: beta +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/missing_verified_date.yaml b/scripts/validate_metadata/test_data/metadata/invalid/missing_verified_date.yaml new file mode 100644 index 0000000..f0ceb1e --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/missing_verified_date.yaml @@ -0,0 +1,20 @@ +name: missing-verified-date +tier: core +stability: alpha +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/invalid/passed_verified_date.yaml b/scripts/validate_metadata/test_data/metadata/invalid/passed_verified_date.yaml new file mode 100644 index 0000000..bb9a320 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/invalid/passed_verified_date.yaml @@ -0,0 +1,21 @@ +name: passed-verified-date +tier: third_party +stability: stable +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2024-11-10T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/valid/custom_links_category.yaml b/scripts/validate_metadata/test_data/metadata/valid/custom_links_category.yaml new file mode 100644 index 0000000..cf353da --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/valid/custom_links_category.yaml @@ -0,0 +1,22 @@ +name: links-subfield +tier: third_party +stability: alpha +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + custom_link_category: "" + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/valid/excluding_ci.yaml b/scripts/validate_metadata/test_data/metadata/valid/excluding_ci.yaml new file mode 100644 index 0000000..a8cbdd5 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/valid/excluding_ci.yaml @@ -0,0 +1,16 @@ +name: test_component +tier: third_party +stability: beta +dependencies: + kubeflow: + - name: Pipelines + version: '2.5.0' + - name: Trainer + version: '2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +lastVerified: 2025-03-15T00:00:00Z +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/valid/excluding_ci_dependency_probe.yaml b/scripts/validate_metadata/test_data/metadata/valid/excluding_ci_dependency_probe.yaml new file mode 100644 index 0000000..0b463d9 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/valid/excluding_ci_dependency_probe.yaml @@ -0,0 +1,16 @@ +name: excluding-ci-dependency-probe +tier: third_party +stability: alpha +dependencies: + kubeflow: + - name: Pipelines + version: '2.5.0' + - name: Trainer + version: '2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +lastVerified: 2025-03-15T00:00:00Z +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/valid/excluding_ext_dependencies.yaml b/scripts/validate_metadata/test_data/metadata/valid/excluding_ext_dependencies.yaml new file mode 100644 index 0000000..3f59747 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/valid/excluding_ext_dependencies.yaml @@ -0,0 +1,15 @@ +name: test_component +tier: core +stability: beta +dependencies: + kubeflow: + - name: Pipelines + version: '2.5.0' + - name: Trainer + version: '2.0.0' +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/valid/excluding_links.yaml b/scripts/validate_metadata/test_data/metadata/valid/excluding_links.yaml new file mode 100644 index 0000000..60a4483 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/valid/excluding_links.yaml @@ -0,0 +1,15 @@ +name: test_component +tier: third_party +stability: alpha +dependencies: + kubeflow: + - name: Pipelines + version: '2.5.0' + - name: Trainer + version: '2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false diff --git a/scripts/validate_metadata/test_data/metadata/valid/excluding_tags.yaml b/scripts/validate_metadata/test_data/metadata/valid/excluding_tags.yaml new file mode 100644 index 0000000..248d727 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/valid/excluding_tags.yaml @@ -0,0 +1,21 @@ +name: test_component +tier: core +stability: stable +dependencies: + kubeflow: + - name: Pipelines + version: '2.5.0' + - name: Trainer + version: '2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/valid/missing_ci_dependency_probe.yaml b/scripts/validate_metadata/test_data/metadata/valid/missing_ci_dependency_probe.yaml new file mode 100644 index 0000000..7f3e22f --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/valid/missing_ci_dependency_probe.yaml @@ -0,0 +1,19 @@ +tier: core +name: missing-ci-dependency-probe +stability: alpha +dependencies: + kubeflow: + - name: Pipelines + version: '>=2.5.0' + - name: Trainer + version: '>=2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +tags: + - training + - evaluation +lastVerified: 2025-03-15T00:00:00Z +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/metadata/valid/valid_metadata.yaml b/scripts/validate_metadata/test_data/metadata/valid/valid_metadata.yaml new file mode 100644 index 0000000..aad3b09 --- /dev/null +++ b/scripts/validate_metadata/test_data/metadata/valid/valid_metadata.yaml @@ -0,0 +1,18 @@ +name: test_component +tier: third_party +stability: alpha +dependencies: + kubeflow: + - name: Pipelines + version: '2.5.0' + - name: Trainer + version: '2.0.0' + external_services: + - name: Argo Workflows + version: "3.6.0" +lastVerified: 2025-03-15T00:00:00Z +ci: + skip_dependency_probe: false +links: + documentation: https://kubeflow.org/components/happy-path-component + issue_tracker: https://github.com/kubeflow/kfp-components/issues diff --git a/scripts/validate_metadata/test_data/owners/invalid/owners_empty.txt b/scripts/validate_metadata/test_data/owners/invalid/owners_empty.txt new file mode 100644 index 0000000..e69de29 diff --git a/scripts/validate_metadata/test_data/owners/invalid/owners_missing_approvers.txt b/scripts/validate_metadata/test_data/owners/invalid/owners_missing_approvers.txt new file mode 100644 index 0000000..4a6318f --- /dev/null +++ b/scripts/validate_metadata/test_data/owners/invalid/owners_missing_approvers.txt @@ -0,0 +1,2 @@ +reviewers: +- sample-reviewer diff --git a/scripts/validate_metadata/test_data/owners/invalid/owners_typo_approvers.txt b/scripts/validate_metadata/test_data/owners/invalid/owners_typo_approvers.txt new file mode 100644 index 0000000..576effc --- /dev/null +++ b/scripts/validate_metadata/test_data/owners/invalid/owners_typo_approvers.txt @@ -0,0 +1,2 @@ +approver: +- sample-approver diff --git a/scripts/validate_metadata/test_data/owners/valid/owners_approvers_and_reviewer.txt b/scripts/validate_metadata/test_data/owners/valid/owners_approvers_and_reviewer.txt new file mode 100644 index 0000000..50efde0 --- /dev/null +++ b/scripts/validate_metadata/test_data/owners/valid/owners_approvers_and_reviewer.txt @@ -0,0 +1,5 @@ +approvers: +- sample-approver + +reviewers: +- sample-reviewer diff --git a/scripts/validate_metadata/test_data/owners/valid/owners_multiple_approvers.txt b/scripts/validate_metadata/test_data/owners/valid/owners_multiple_approvers.txt new file mode 100644 index 0000000..09de636 --- /dev/null +++ b/scripts/validate_metadata/test_data/owners/valid/owners_multiple_approvers.txt @@ -0,0 +1,3 @@ +approvers: +- sample-approver +- sample-approver-2 diff --git a/scripts/validate_metadata/test_data/owners/valid/owners_single_approver.txt b/scripts/validate_metadata/test_data/owners/valid/owners_single_approver.txt new file mode 100644 index 0000000..0037369 --- /dev/null +++ b/scripts/validate_metadata/test_data/owners/valid/owners_single_approver.txt @@ -0,0 +1,2 @@ +approvers: +- sample-approver \ No newline at end of file diff --git a/scripts/validate_metadata/validate_metadata.py b/scripts/validate_metadata/validate_metadata.py new file mode 100644 index 0000000..316a390 --- /dev/null +++ b/scripts/validate_metadata/validate_metadata.py @@ -0,0 +1,317 @@ +import argparse +import os +import sys +from itertools import pairwise + +import yaml +from datetime import datetime, timezone +from pathlib import Path +import logging + +from semver import Version + +# The following ordered fields are required in a metadata.yaml file. +REQUIRED_FIELDS = ["name", "tier", "stability", "dependencies", "lastVerified"] +# The following fields are optional in a metadata.yaml file.' +OPTIONAL_FIELDS = ["tags", "ci", "links"] +# 'Tier' must be 'core' or 'third-party'. +TIER_OPTIONS = ["core", "third_party"] +# 'Stability' must be 'alpha', 'beta', or 'stable'. +STABILITY_OPTIONS = ["alpha", "beta", "stable"] +# 'Dependencies' must contain 'kubeflow' and can contain 'external_services'. +DEPENDENCIES_FIELDS = ["kubeflow", "external_services"] +# A given dependency must contain 'name' and 'version' fields. +DEPENDENCY=["name", "version"] +# Comparison operators for dependency versions. +COMPARISON = {">=", "<=", "=="} + +OWNERS="OWNERS" +METADATA="metadata.yaml" + +class ValidationError(Exception): + """Custom exception for validation errors that should be displayed + without traceback and can take a custom message. + """ + def __init__(self, message: str = "A validation error occurred."): + # Call the base class constructor with the message + super().__init__(message) + + # Store the message in an attribute (optional, but good practice) + self.message = message + +def parse_args() -> argparse.Namespace: + """Parse and validate command-line arguments. + + Returns: + argParse.Namespace: Parsed and validated arguments. + """ + parser = argparse.ArgumentParser( + description="Validate metadata schema for Kubeflow Pipelines pipelines/components", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" + # For example, from project root: + python -m scripts.validate_metadata --dir components/data_processing/sample_component + """ + ) + + parser.add_argument( + '--dir', + type=validate_dir, + required=True, + help='Path to the component or pipeline directory (must contain OWNERS and metadata.yaml files)' + ) + + return parser.parse_args() + +def validate_dir(path: str) -> Path: + """Validate that the input path is a valid directory and contains required files. + + Args: + path: String representation of the path to the component or pipeline directory. + + Returns: + Path: Validated Path object to the component or pipeline directory. + + Raises: + argparse.ArgumentTypeError: If validation fails. + """ + path = Path(path) + print(os.getcwd()) + if not path.exists(): + raise argparse.ArgumentTypeError("Directory '{}' does not exist".format(path)) + + if not path.is_dir(): + raise argparse.ArgumentTypeError("'{}' is not a directory".format(path)) + + file_path = path / OWNERS + if not file_path.exists(): + raise argparse.ArgumentTypeError("{} does not contain an {} file".format(path, OWNERS)) + + metadata_file = path / METADATA + if not metadata_file.exists(): + raise argparse.ArgumentTypeError("'{}' does not contain a {} file".format(path, METADATA)) + + return path + +def validate_owners_file(filepath: Path): + """Validate that the OWNERS file contains at least one approver under the 'approvers' heading. + + Args: + filepath: Path object representing the filepath to the OWNERS file. + + Raises: + ValidationError: If filepath input is not a file, heading 'approvers:' is missing, or no approvers are listed. + """ + if not filepath.is_file(): + raise ValidationError("{} is not a valid filepath.".format(filepath)) + + with open(filepath) as f: + for line, next_line in pairwise(f): + if line.startswith("approvers:") and next_line.startswith("-") and len(next_line) > 2: + logging.info("OWNERS file at {} contains at least one approver under heading 'approvers:'.".format(filepath)) + return + + # If this line is reached, no approvers were found. + raise ValidationError("OWNERS file at {} requires 1+ approver under heading 'approvers:'.".format(filepath)) + +def validate_metadata_yaml(filepath: Path) -> bool: + """Validate that the input filepath represents a metadata.yaml file with a valid schema. + + Args: + filepath: Path object representing the filepath to the metadata.yaml file. + + Raise: + ValidationError: If 'lastVerified' empty, or validate_date_verified() or validate_required_fields() fails. + """ + if not filepath.is_file(): + raise ValidationError("{} is not a valid filepath.".format(filepath)) + with open(filepath) as f: + metadata = yaml.safe_load(f) + + # Validate metadata.yaml has been verified within one year of the current date. + if "lastVerified" not in metadata: + raise ValidationError("Metadata at {} has corresponding metadata.yaml with no 'lastVerified' value.".format(filepath)) + + last_verified = metadata.get("lastVerified") + if not validate_date_verified(last_verified): + raise ValidationError("Metadata at {} has corresponding metadata.yaml with invalid 'lastVerified' value: {}.".format(filepath, last_verified)) + + # Validate required fields and their corresponding values. + validate_required_fields(metadata) + +def validate_date_verified(last_verified: datetime) -> bool: + """Validate that the input date is RFC-3339-formatted and within 1 year of the current date. + + Args: + last_verified: Input datetime date to be validated. + + Returns: + bool: True if the input date is valid, False otherwise. + + Examples: + '2025-03-15T00:00:00Z' -> True [As of November 2025] + '2025-03-15' -> False + '2024-03-15T00:00:00Z' -> False + """ + # Validate input date formatting. + if not isinstance(last_verified, datetime): + logging.error("'lastVerified' should be format YYYY-MM-DDT00:00:00Z, but instead is: {}.".format(last_verified)) + return False + # Validate input date to be within 1 year of the current date. + today = datetime.now(timezone.utc) + delta = abs((today - last_verified).days) + if delta >= 365: + logging.error("'lastVerified' should be within 1 year of current date, but is {} days over.".format(delta)) + return False + return True + +def validate_required_fields(metadata: dict): + """Validates that all required fields are present in the input dictionary and have valid values, + and that no invalid fields are present. + + Args: + metadata: dictionary object containing nested metadata fields. + + Raises: + ValidationError: If validation fails. + """ + # Convert metadata keys to a list for comparison purposes. + input_metadata_fields = list(metadata.keys()) + # Optional fields should not be validated against required fields. Remove optional fields for this check. + for field in OPTIONAL_FIELDS: + if field in input_metadata_fields: + input_metadata_fields.remove(field) + + # Retrieve name name. + name = metadata.get("name") + if name is None: + raise ValidationError("Missing required field 'name' in {}.".format(METADATA)) + if not isinstance(name, str): + raise ValidationError("{} value identified in field 'name' in {}: '{}'. Value for 'name' must be string.".format(type(name).__name__, METADATA, name)) + + # Convert metadata keys to a set and compare against REQUIRED_FIELDS set. + input_fields_set = set(input_metadata_fields) + required_fields_set = set(REQUIRED_FIELDS) + if required_fields_set != input_fields_set: + missing_fields = required_fields_set - input_fields_set + if len(missing_fields) > 0: + raise ValidationError("Missing required field(s) in {} for '{}': {}.".format(METADATA, name, missing_fields)) + extra_fields = input_fields_set - required_fields_set + if len(extra_fields) > 0: + raise ValidationError("Unexpected field(s) in {} for '{}': {}.".format(METADATA, name, extra_fields)) + # Compare input fields against REQUIRED FIELDS as lists to verify elements are ordered correctly. + if list(input_metadata_fields) != REQUIRED_FIELDS: + raise ValidationError("Field(s) located incorrectly in {} for '{}'. Expected order is {}.".format(METADATA, name, REQUIRED_FIELDS)) + + # Validate field values. + for field in metadata: + value_type = type(metadata.get(field)).__name__ + + if field == "tier": + tier_val = metadata.get("tier") + if tier_val not in TIER_OPTIONS: + raise ValidationError("Invalid 'tier' value in {} for '{}': '{}'. Expected a scalar string from the following options: {}.".format(METADATA, name, tier_val, TIER_OPTIONS)) + + elif field == "stability": + stability_val = metadata.get("stability") + if stability_val not in STABILITY_OPTIONS: + raise ValidationError("Invalid 'stability' value in {} for '{}': '{}'. Expected one of: {}.".format(METADATA, name, stability_val, STABILITY_OPTIONS)) + + elif field == "dependencies": + # Dependencies should be a dictionary. + dependency_val = metadata.get("dependencies") + if not isinstance(dependency_val, dict): + raise ValidationError("{} value identified for field 'dependencies' in {} for '{}'. Value must be array.".format(value_type, METADATA, name)) + dependency_types = set(dependency_val.keys()) + + # Dependencies should contain 'kubeflow' and can contain 'external_services'. + if not (dependency_types == {"kubeflow"} or dependency_types == {"kubeflow", "external_services"}): + raise ValidationError("The following field(s) were found in dependencies: {}. Expected {}.".format(list(dependency_val.keys()), DEPENDENCIES_FIELDS)) + + # Kubeflow Pipelines is a required dependency. + kf_dependencies = dependency_val.get("kubeflow") + ext_dependencies = dependency_val.get("external_services") + if not isinstance(kf_dependencies, list) or (ext_dependencies is not None and not isinstance(ext_dependencies, list)): + raise ValidationError("Dependency sub-types for '{}' should contain lists but instead are {} and {}.".format(name, type(kf_dependencies), type(ext_dependencies))) + kfp_present = any(d.get('name') == 'Pipelines' for d in kf_dependencies) + if not kfp_present: + raise ValidationError("{} for '{}' is missing Kubeflow Pipelines dependency.".format(METADATA, name)) + + # Dependency versions must be correctly formatted by semantic versioning. + invalid_dependencies = get_invalid_versions(kf_dependencies) + get_invalid_versions(ext_dependencies) + if len(invalid_dependencies) > 0: + raise ValidationError("{} for '{}' contains one or more dependencies with invalid semantic versioning: {}.".format(METADATA, name, invalid_dependencies)) + + elif field == "tags": + tags_val = metadata.get("tags") + if not (isinstance(tags_val, list)): + raise ValidationError("{} value identified in field 'tags' in {} for '{}'. Value must be string array.".format(value_type, METADATA, name)) + if not all(isinstance(item, str) for item in tags_val): + raise ValidationError("The following tags in {} for '{}': {}. Expected an array of scalar strings.".format(METADATA, name, tags_val)) + elif field == "ci": + ci_val = metadata.get("ci") + if not isinstance(ci_val, dict): + raise ValidationError("{} value identified for field 'ci' in {} for '{}'. Value must be dictionary.".format(value_type, METADATA, name)) + keys = set(ci_val.keys()) + if not (keys == {"skip_dependency_probe"}): + raise ValidationError("The following field(s) were found in field 'ci' in {} for '{}': {}. Only field 'skip_dependency_probe' is valid.".format(METADATA, name, list(ci_val.keys()))) + probe = ci_val.get("skip_dependency_probe") + if probe is not None and not isinstance(probe, bool): + raise ValidationError("{} expects a boolean value for skip_dependency_probe but {} value provided: '{}'.".format(METADATA, type(probe).__name__, probe)) + + elif field == "links": + links_value = metadata.get("links") + if not isinstance(links_value, dict): + raise ValidationError("{} value identified in field 'links' in {} for '{}'. Value must be dictionary.".format(value_type, METADATA, name)) + +def get_invalid_versions(dependencies: list[dict]) -> list[dict]: + """Return a list of the input dependencies that contain invalid semantic versioning. + + Args: + dependencies: list[dict] of dependencies to be validated + + Return: + dependencies: list[dict] of invalid dependencies + """ + if dependencies is None: + return [] + invalid : list[dict] = [] + for dependency in dependencies: + version = dependency.get("version") + # If the dependency version is null or non-string, it is invalid. + if version is None or not isinstance(version, str): + invalid.append(dependency) + # Strip leading '==', '>=' or '<=' from dependency version, if applicable. + if len(version) > 1 and version[:2] in COMPARISON: + version = version[2:] + if not Version.is_valid(version): + invalid.append(dependency) + return invalid + +def main(): + """Main entry point for the CLI.""" + args = parse_args() + input_dir = args.component + + # Validate OWNERS + try: + owners_file_path = input_dir / OWNERS + validate_owners_file(owners_file_path) + except ValidationError as e: + logging.error("Validation Error: %s", e) + sys.exit(1) + + # Validate metadata.yaml + try: + metadata_file_path = input_dir / METADATA + validate_metadata_yaml(metadata_file_path) + except ValidationError as e: + logging.error("Validation Error: %s", e) + sys.exit(1) + + # Validation successful. + logging.info("Validation successful for {}.".format(input_dir)) + print("Validation successful for {}.".format(input_dir)) + +if __name__ == "__main__": + main() diff --git a/scripts/validate_metadata/validate_metadata_test.py b/scripts/validate_metadata/validate_metadata_test.py new file mode 100644 index 0000000..2bcac6a --- /dev/null +++ b/scripts/validate_metadata/validate_metadata_test.py @@ -0,0 +1,273 @@ +import argparse +import builtins +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import pytest + +from scripts.validate_metadata import validate_metadata +from scripts.validate_metadata.validate_metadata import ValidationError + +INVALID_METADATA_DIR = "scripts/validate_metadata/test_data/metadata/invalid/" +VALID_METADATA_DIR = "scripts/validate_metadata/test_data/metadata/valid/" +INVALID_OWNERS_DIR = "scripts/validate_metadata/test_data/owners/invalid/" +VALID_OWNERS_DIR = "scripts/validate_metadata/test_data/owners/valid/" +TEST_DIRS= "scripts/validate_metadata/test_data/directories_metadata/" + +@dataclass +class ValidateMetadataTestFile: + file_name: str + expected_exception: Optional[builtins.type[Exception]] + expected_exception_msg: Optional[str] + +@dataclass +class ValidateMetadataTestDir: + dir_name: str + expected_exception: Optional[builtins.type[Exception]] + expected_exception_msg: Optional[str] + +@pytest.mark.parametrize( + 'test_data', [ + ValidateMetadataTestFile( + file_name='valid_metadata.yaml', + expected_exception=None, + expected_exception_msg="" + ), + ValidateMetadataTestFile( + file_name='excluding_tags.yaml', + expected_exception=None, + expected_exception_msg="" + ), + ValidateMetadataTestFile( + file_name='excluding_ext_dependencies.yaml', + expected_exception=None, + expected_exception_msg="" + ), + ValidateMetadataTestFile( + file_name='excluding_ci.yaml', + expected_exception=None, + expected_exception_msg="" + ), + ValidateMetadataTestFile( + file_name='excluding_ci_dependency_probe.yaml', + expected_exception=None, + expected_exception_msg="" + ), + ValidateMetadataTestFile( + file_name='excluding_links.yaml', + expected_exception=None, + expected_exception_msg="" + ), + ValidateMetadataTestFile( + file_name='custom_links_category.yaml', + expected_exception=None, + expected_exception_msg="" + ), + ] +) +def test_validate_metadata_yaml_success(test_data): + valid_yaml = validate_metadata.validate_metadata_yaml(filepath=Path(VALID_METADATA_DIR + test_data.file_name)) + # Asserts that no exceptions have been raised. + assert True + +@pytest.mark.parametrize( + 'test_data', [ + ValidateMetadataTestFile( + file_name='this_file_does_not_exist.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("scripts/validate_metadata/test_data/metadata/invalid/this_file_does_not_exist.yaml is not a valid filepath.") + ), + ValidateMetadataTestFile( + file_name='missing_verified_date.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("Metadata at scripts/validate_metadata/test_data/metadata/invalid/missing_verified_date.yaml has corresponding metadata.yaml with no 'lastVerified' value.") + ), + ValidateMetadataTestFile( + file_name='invalid_verified_date.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("Metadata at scripts/validate_metadata/test_data/metadata/invalid/invalid_verified_date.yaml has corresponding metadata.yaml with invalid 'lastVerified' value: 2024-11-20T0.") + ), + ValidateMetadataTestFile( + file_name='passed_verified_date.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("Metadata at scripts/validate_metadata/test_data/metadata/invalid/passed_verified_date.yaml has corresponding metadata.yaml with invalid 'lastVerified' value: 2024-11-10 00:00:00+00:00.") + ), + ValidateMetadataTestFile( + file_name='missing_name.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("Missing required field 'name' in metadata.yaml.") + ), + ValidateMetadataTestFile( + file_name='invalid_name.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("int value identified in field 'name' in metadata.yaml: '2'. Value for 'name' must be string.") + ), + ValidateMetadataTestFile( + file_name='missing_tier.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("Missing required field(s) in metadata.yaml for 'missing-tier': {'tier'}.") + ), + ValidateMetadataTestFile( + file_name='invalid_tier.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("Invalid 'tier' value in metadata.yaml for 'invalid-tier': 'invalid_tier'. Expected a scalar string from the following options: ['core', 'third_party'].") + ), + ValidateMetadataTestFile( + file_name='missing_stability.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("Missing required field(s) in metadata.yaml for 'missing-stability': {'stability'}.") + ), + ValidateMetadataTestFile( + file_name='invalid_stability.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("Invalid 'stability' value in metadata.yaml for 'invalid-stability': 'invalid-stability'. Expected one of: ['alpha', 'beta', 'stable'].")), + ValidateMetadataTestFile( + file_name='missing_dependencies.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("Missing required field(s) in metadata.yaml for 'missing-dependencies': {'dependencies'}.") + ), + ValidateMetadataTestFile( + file_name='invalid_dependencies_type.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("str value identified for field 'dependencies' in metadata.yaml for 'invalid-dependencies-type'. Value must be array.") + ), + ValidateMetadataTestFile( + file_name='invalid_dependencies_category.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("The following field(s) were found in dependencies: ['kubeflow', 'external_services', 'invalid_dependency_category']. Expected ['kubeflow', 'external_services'].") + ), + ValidateMetadataTestFile( + file_name='missing_kfp_dependency.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("metadata.yaml for 'missing-kfp-dependency' is missing Kubeflow Pipelines dependency.") + ), + ValidateMetadataTestFile( + file_name='invalid_dependency_semantic_versioning.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("metadata.yaml for 'invalid-dependency-semantic-versioning' contains one or more dependencies with invalid semantic versioning: [{'name': 'Argo Workflows', 'version': '3.6'}].") + ), + ValidateMetadataTestFile( + file_name='invalid_tag_type.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("str value identified in field 'tags' in metadata.yaml for 'invalid-tag-type'. Value must be string array.") + ), + ValidateMetadataTestFile( + file_name='invalid_tag_array_type.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("The following tags in metadata.yaml for 'invalid-tag-array-type': [1, 2]. Expected an array of scalar strings.") + ), + ValidateMetadataTestFile( + file_name='invalid_ci.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("str value identified for field 'ci' in metadata.yaml for 'invalid-ci'. Value must be dictionary.") + ), + ValidateMetadataTestFile( + file_name='invalid_ci_category.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("The following field(s) were found in field 'ci' in metadata.yaml for 'invalid-ci-category': ['skip_dependency_probe', 'invalid_ci_category']. Only field 'skip_dependency_probe' is valid") + ), + ValidateMetadataTestFile( + file_name='invalid_ci_dependency_probe.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("metadata.yaml expects a boolean value for skip_dependency_probe but str value provided: 'invalid-probe-value'.") + ), + ValidateMetadataTestFile( + file_name='invalid_links.yaml', + expected_exception=ValidationError, + expected_exception_msg=re.escape("str value identified in field 'links' in metadata.yaml for 'invalid-links'. Value must be dictionary.") + ), + ]) +def test_validate_metadata_yaml_failure(test_data): + with pytest.raises(test_data.expected_exception, + match=test_data.expected_exception_msg): + validate_metadata.validate_metadata_yaml(filepath=Path(INVALID_METADATA_DIR + test_data.file_name)) + +@pytest.mark.parametrize( + 'test_data', [ + ValidateMetadataTestFile( + file_name='owners_single_approver.txt', + expected_exception=None, + expected_exception_msg=None + ), + ValidateMetadataTestFile( + file_name='owners_multiple_approvers.txt', + expected_exception=None, + expected_exception_msg=None + ), + ValidateMetadataTestFile( + file_name='owners_approvers_and_reviewer.txt', + expected_exception=None, + expected_exception_msg=None + ) + ] +) +def test_validate_owners_yaml_success(test_data): + validate_metadata.validate_owners_file(filepath=Path(VALID_OWNERS_DIR + test_data.file_name)) + # Asserts that no exceptions have been raised. + assert True + +@pytest.mark.parametrize( + 'test_data', [ + ValidateMetadataTestFile( + file_name='owners_empty.txt', + expected_exception=ValidationError, + expected_exception_msg=re.escape("OWNERS file at scripts/validate_metadata/test_data/owners/invalid/owners_empty.txt requires 1+ approver under heading 'approvers:'.") + ), + ValidateMetadataTestFile( + file_name='owners_missing_approvers.txt', + expected_exception=ValidationError, + expected_exception_msg=re.escape("OWNERS file at scripts/validate_metadata/test_data/owners/invalid/owners_missing_approvers.txt requires 1+ approver under heading 'approvers:'.") + ), + ValidateMetadataTestFile( + file_name='owners_typo_approvers.txt', + expected_exception=ValidationError, + expected_exception_msg=re.escape("OWNERS file at scripts/validate_metadata/test_data/owners/invalid/owners_typo_approvers.txt requires 1+ approver under heading 'approvers:'.") + ) + ] +) +def test_validate_owners_yaml_failure(test_data): + with pytest.raises(ValidationError, + match=test_data.expected_exception_msg): + validate_metadata.validate_owners_file(filepath=Path(INVALID_OWNERS_DIR + test_data.file_name)) + +@pytest.mark.parametrize( + 'test_data', [ + ValidateMetadataTestDir( + dir_name='missing', + expected_exception=argparse.ArgumentTypeError, + expected_exception_msg="" + ), + ValidateMetadataTestDir( + dir_name='dir_is_not_dir.txt', + expected_exception=argparse.ArgumentTypeError, + expected_exception_msg="" + ), + ValidateMetadataTestDir( + dir_name='missing_owners_file', + expected_exception=argparse.ArgumentTypeError, + expected_exception_msg="" + ), + ValidateMetadataTestDir( + dir_name='missing_metadata_file', + expected_exception=argparse.ArgumentTypeError, + expected_exception_msg="" + ) + ]) +def test_validate_metadata_files_in_dir_failure(test_data): + with pytest.raises(test_data.expected_exception, + match=test_data.expected_exception_msg): + validate_metadata.validate_dir(path=TEST_DIRS + test_data.dir_name) + +@pytest.mark.parametrize( + 'test_data', [ + ValidateMetadataTestDir( + dir_name='valid', + expected_exception=None, + expected_exception_msg=None + ) + ]) +def test_validate_metadata_files_in_dir_success(test_data): + files_present = validate_metadata.validate_dir(path=TEST_DIRS + test_data.dir_name) + assert files_present == Path('scripts/validate_metadata/test_data/directories_metadata/valid')