Merge commit '7f49d7e651a02eac57af3ef387aec53c76845e2e'

Author: qiuosier

.coveragerc

@@ -14,7 +14,6 @@ omit =
     ads/common/analyzer.py
     ads/common/artifact/*
     ads/common/function/*
-    ads/dataflow/*
     ads/dataset/dask_series.py
     ads/dataset/factory.py
     ads/environment/*
@@ -48,7 +47,6 @@ omit =
     ads/common/analyzer.py
     ads/common/artifact/*
     ads/common/function/*
-    ads/dataflow/*
     ads/dataset/dask_series.py
     ads/dataset/factory.py
     ads/environment/*

.github/workflows/add-3plicense-warning.yml

@@ -0,0 +1,43 @@
+name: "Add 3P License Warning to PR"
+
+on:
+  pull_request:
+    paths:
+      - setup.py
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  warning-message-3plicense:
+    name: Add ⚠️ Warning about 3p license
+    runs-on: ubuntu-latest
+    if: ${{ success() }} && ${{ github.event.issue.pull_request }}
+    steps:
+      - run: |
+          BODY_MSG=$(cat << EOF
+          ⚠️ This PR changed **setup.py** file. ⚠️ 
+            - PR Creator must update 📃 THIRD_PARTY_LICENSES.txt, if any 📚 library added/removed in **setup.py**.
+            - PR Approver must confirm 📃 THIRD_PARTY_LICENSES.txt updated, if any 📚 library added/removed in **setup.py**.
+          EOF
+          )
+          echo "BODY_MSG<<EOF" >> $GITHUB_ENV
+          echo "$BODY_MSG" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ github.token }}
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `${{ env.BODY_MSG }}`
+            })

.github/workflows/publish-to-readthedocs.yml

@@ -1,18 +1,7 @@
 name: "Publish Docs"
 
-on:
-  # Auto-trigger this workflow on tag creation
-  push:
-    tags:
-      - 'v*.*.*'
-  # Auto-trigger this workflow on merge to main changes in docs/** folder
-  pull_request_target:
-    types:
-      - closed
-    branches:
-      - main
-    paths:
-      - 'docs/**'
+# To run this workflow manually from the Actions tab
+on: workflow_dispatch
 
 env:
   RTDS_ADS_PROJECT: https://readthedocs.org/api/v3/projects/accelerated-data-science
@@ -24,17 +13,16 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: When PR ✅ merged - Trigger Readthedocs build
-        if: github.event_name == 'pull_request_target' && github.event.pull_request.merged == true
+      - name: When tag 🏷️ pushed - Trigger Readthedocs build
+        if: github.event_name == 'push' && startsWith(github.ref_name, 'v')
         run: |
+          # trigger build/publish of latest version
           curl \
             -X POST \
             -H "Authorization: Token $RTDS_ADS_TOKEN" $RTDS_ADS_PROJECT/versions/latest/builds/
-      - name: When tag 🏷️ pushed - Trigger Readthedocs build
-        if: github.event_name == 'push' && startsWith(github.ref_name, 'v')
-        run: |
-          # add 10 minutes wait time for readthedocs see freshly created tag
-          sleep 10m
+          # add 15 minutes wait time for readthedocs see freshly created tag
+          sleep 15m
+          # trigger build/publish of v*.*.* version
           curl \
             -X POST \
             -H "Authorization: Token $RTDS_ADS_TOKEN" $RTDS_ADS_PROJECT/versions/${{ github.ref_name }}/builds/

.github/workflows/run-unittests.yml

@@ -135,7 +135,7 @@ jobs:
     needs: test
     if: ${{ success() }} && ${{ github.event.issue.pull_request }}
     env:
-      COMPARE_BRANCH: develop
+      COMPARE_BRANCH: main
 
     steps:
       - name: "Checkout current branch"
@@ -152,9 +152,14 @@ jobs:
           COV_BODY_INTRO="📌 Overall coverage:\n\n"
           echo COV_BODY="$COV_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
 
+          # Prepare file paths to .coverage files
+          # Filenames taken from job.test last step with name - "Save coverage files"
+          FILE_UNITARY="cov-reports-unitary/.coverage"; [[ ! -f $FILE_UNITARY ]] && FILE_UNITARY="" 
+          FILE_MODEL="cov-reports-model/.coverage"; [[ ! -f $FILE_MODEL ]] && FILE_MODEL=""
+          
           # Combine coverage files
           pip install coverage
-          coverage combine cov-reports-unitary/.coverage cov-reports-model/.coverage
+          coverage combine $FILE_UNITARY $FILE_MODEL
 
           # Make html report
           coverage html

.gitleaks.toml

@@ -0,0 +1,40 @@
+title = "Gitleaks Config"
+
+# Gitleaks feature, extending the existing base config from:
+# https://github.com/zricethezav/gitleaks/blob/master/config/gitleaks.toml
+[extend]
+useDefault = true
+
+# Allowlist's 'stopwords' and 'regexes' excludes any secrets or mathching patterns from the current repository.
+# Paths listed in allowlist will not be scanned.
+[allowlist]
+    description = "Global allow list"
+    stopwords = ["test_password", "sample_key"]
+    regexes = [
+        '''example-password''',
+        '''this-is-not-the-secret''',
+        '''<redacted>'''
+    ]
+    paths = [
+        '''tests/integration/tests_configs.yaml'''
+    ]
+
+# Describe rule to search real ocids
+[[rules]]
+    description = "Real ocids"
+    id = "ocid"
+    regex = '''ocid[123]\.[a-z1-9A-Z]*\.oc\d\.[a-z1-9A-Z]*\.[a-z1-9A-Z]+'''
+    keywords = [
+        "ocid"
+    ]
+
+# Describe rule to search generic secrets
+[[rules]]
+    description = "Generic secret"
+    id = "generic-secret"
+    regex = '''(?i)((key|api|token|secret|passwd|password|psw|pass|pswd)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z!@#$%^&*<>\\\-_.=]{3,100})['\"]'''
+    entropy = 0
+    secretGroup = 4
+    keywords = [
+        "key","api","token","secret","passwd","password", "psw", "pass", "pswd"
+    ]

.pre-commit-config.yaml

@@ -1,10 +1,12 @@
 repos:
+# Standard hooks
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
     -   id: check-ast
+        exclude: ^docs/
     -   id: check-docstring-first
-        exclude: ^tests/
+        exclude: ^(docs/|tests/)
     -   id: check-json
     -   id: check-merge-conflict
     -   id: check-yaml
@@ -14,14 +16,34 @@ repos:
     -   id: pretty-format-json
         args: ['--autofix']
     -   id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+        exclude: ^docs/
+# Black, the code formatter, natively supports pre-commit
 -   repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.3.0
     hooks:
     -   id: black
+        exclude: ^docs/
+# Regex based rst files common mistakes detector
 -   repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.9.0
+    rev: v1.10.0
     hooks:
     -   id: rst-backticks
+        files: ^docs/
     -   id: rst-inline-touching-normal
-
-exclude: ^(docs/)
+        files: ^docs/
+# Hardcoded secrets and ocids detector
+-   repo: https://github.com/gitleaks/gitleaks
+    rev: v8.17.0
+    hooks:
+    -   id: gitleaks
+# Oracle copyright checker
+-   repo: https://github.com/oracle-samples/oci-data-science-ai-samples/
+    rev: cbe0136
+    hooks:
+    -   id: check-copyright
+        name: check-copyright
+        entry: .pre-commit-scripts/check-copyright.py
+        language: script
+        types_or: ['python', 'shell', 'bash']
+        exclude: ^docs/

.readthedocs.yaml

@@ -0,0 +1,19 @@
+# Read the Docs configuration file
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.9"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: docs/source/conf.py
+
+# Optionally declare the Python requirements required to build your docs
+python:
+   install:
+   - requirements: docs/requirements.txt

CONTRIBUTING.md

@@ -31,6 +31,15 @@ git commit --signoff
 Only pull requests from committers that can be verified as having signed the OCA
 are accepted.
 
+## Branching strategy
+
+Use GitHub flow branching strategy. With the GitHub flow, there are only 2 branches: main and feature.
+
+1. Every change that is worked on is branched directly off of main into a feature branch.
+2. Once a feature is ready it is tested on the feature branch and the code is reviewed before being merged to main.
+
+Hotfixes, bugfixes are treated the same as feature branches.
+
 ## Pull request process
 
 1. Ensure there is an issue created to track and discuss the fix or enhancement