CI: add internal testing (#140)

* list commands
* update requirements
* adding tests
* install default
* fixing
* Apply suggestions from code review

Co-authored-by: Aki Nitta <nitta@akihironitta.com>

Author: Borda

.actions/assistant.py

@@ -328,7 +328,7 @@ def bash_render(folder: str) -> str:
         cmd.append(f"python .actions/assistant.py update-env-details {folder}")
         # copy and add to version the enriched meta config
         cmd += [f"cp {meta_file} {pub_meta}", f"git add {pub_meta}"]
-        # if thumb image is linked to to the notebook, copy and version it too
+        # if thumb image is linked to the notebook, copy and version it too
         if thumb_file:
             cmd += [f"cp {thumb_file} {pub_thumb}", f"git add {pub_thumb}"]
         # add the generated notebook to version
@@ -361,7 +361,7 @@ def bash_test(folder: str) -> str:
 
         cmd.append(f"# available: {AssistantCLI.DEVICE_ACCELERATOR}")
         if AssistantCLI._valid_accelerator(folder):
-            cmd.append(f"python -m pytest {ipynb_file} -v --nbval")
+            cmd.append(f"python -m pytest {ipynb_file} -v --nbval --nbval-cell-timeout=300")
         else:
             warn("Invalid notebook's accelerator for this device. So no tests will be run!!!", RuntimeWarning)
         # deactivate and clean local environment
@@ -443,7 +443,7 @@ def group_folders(
         fpath_gitdiff: str,
         fpath_change_folders: str = "changed-folders.txt",
         fpath_drop_folders: str = "dropped-folders.txt",
-        fpaths_actual_dirs: Sequence[str] = tuple(),
+        fpath_actual_dirs: Sequence[str] = tuple(),
         strict: bool = True,
         root_path: str = "",
     ) -> None:
@@ -458,23 +458,23 @@ def group_folders(
 
             fpath_change_folders: output file with changed folders
             fpath_drop_folders: output file with deleted folders
-            fpaths_actual_dirs: files with listed all folder in particular stat
+            fpath_actual_dirs: files with listed all folder in particular stat
             strict: raise error if some folder outside skipped does not have valid meta file
             root_path: path to the root tobe added for all local folder paths in files
 
         Example:
-            >> python assistant.py group-folders ../target-diff.txt --fpaths_actual_dirs "['../dirs-main.txt', '../dirs-publication.txt']"
+            >> python assistant.py group-folders ../target-diff.txt --fpath_actual_dirs "['../dirs-main.txt', '../dirs-publication.txt']"
         """
         with open(fpath_gitdiff) as fp:
             changed = [ln.strip() for ln in fp.readlines()]
         dirs = [os.path.dirname(ln) for ln in changed]
         # not empty paths
         dirs = [ln for ln in dirs if ln]
 
-        if fpaths_actual_dirs:
-            assert isinstance(fpaths_actual_dirs, list)
-            assert all(os.path.isfile(p) for p in fpaths_actual_dirs)
-            dir_sets = [{ln.strip() for ln in open(fp).readlines()} for fp in fpaths_actual_dirs]
+        if fpath_actual_dirs:
+            assert isinstance(fpath_actual_dirs, list)
+            assert all(os.path.isfile(p) for p in fpath_actual_dirs)
+            dir_sets = [{ln.strip() for ln in open(fp).readlines()} for fp in fpath_actual_dirs]
             # get only different
             dirs += list(set.union(*dir_sets) - set.intersection(*dir_sets))
 
@@ -564,7 +564,7 @@ def copy_notebooks(
         path_docs_ipynb: str = "notebooks",
         path_docs_images: str = "_static/images",
         patterns: Sequence[str] = (".", "**"),
-    ):
+    ) -> None:
         """Copy all notebooks from a folder to doc folder.
 
         Args:
@@ -609,13 +609,13 @@ def copy_notebooks(
             ipynb_content.append(os.path.join("notebooks", sub_ipynb))
 
     @staticmethod
-    def update_env_details(dir_path: str):
+    def update_env_details(folder: str) -> None:
         """Export the actual packages used in runtime.
 
         Args:
-             dir_path: path to the folder
+             folder: path to the folder
         """
-        meta = AssistantCLI._load_meta(dir_path)
+        meta = AssistantCLI._load_meta(folder)
         # default is COU runtime
         with open(PATH_REQ_DEFAULT) as fp:
             req = fp.readlines()
@@ -636,9 +636,16 @@ def _parse_package_name(pkg: str, keys: str = " <=>[]@", egg_name: str = "#egg="
         meta["environment"] = [env[r] for r in require]
         meta["published"] = datetime.now().isoformat()
 
-        fmeta = os.path.join(DIR_NOTEBOOKS, dir_path) + ".yaml"
+        fmeta = os.path.join(DIR_NOTEBOOKS, folder) + ".yaml"
         yaml.safe_dump(meta, stream=open(fmeta, "w"), sort_keys=False)
 
+    @staticmethod
+    def list_dirs(folder: str = ".notebooks") -> str:
+        """List all sub-folders in a given tree including any ipynb."""
+        ipynbs = glob.glob(os.path.join(folder, "*.ipynb")) + glob.glob(os.path.join(folder, "**", "*.ipynb"))
+        ipynbs = sorted(os.path.splitext(os.path.sep.join(p.split(os.path.sep)[1:]))[0] for p in ipynbs)
+        return os.linesep.join(ipynbs)
+
 
 if __name__ == "__main__":
     fire.Fire(AssistantCLI)

.actions/git-diff-sync.sh

@@ -18,19 +18,13 @@ git checkout $2
 b2="${2//'/'/'_'}"
 printf "Branch alias: $b2\n"
 # list all dirs in target branch
-python -c "
-import os
-from glob import glob
-from os.path import sep, splitext
-ipynbs = glob('.notebooks/*.ipynb') + glob('.notebooks/**/*.ipynb')
-ipynbs = sorted([splitext(sep.join(p.split(sep)[1:]))[0] for p in ipynbs])
-print(os.linesep.join(ipynbs))" > "dirs-$b2.txt"
+python .actions/assistant.py list_dirs > "dirs-$b2.txt"
 cat "dirs-$b2.txt"
 
 printf "\n\n"
 git merge --ff -s resolve origin/$1
 
-python .actions/assistant.py group-folders target-diff.txt --fpaths_actual_dirs "['dirs-$b1.txt', 'dirs-$b2.txt']"
+python .actions/assistant.py group-folders target-diff.txt --fpath_actual_dirs "['dirs-$b1.txt', 'dirs-$b2.txt']"
 printf "\n\nChanged folders:\n"
 cat changed-folders.txt
 printf "\n\nDropped folders:\n"

.actions/requirements.txt

@@ -2,4 +2,5 @@ Fire
 tqdm
 PyYAML
 wcmatch
+requests
 pip

.actions/test_cli.py

@@ -0,0 +1,25 @@
+import os
+
+import pytest
+from assistant import AssistantCLI
+
+_PATH_ROOT = os.path.dirname(os.path.dirname(__file__))
+_PATH_TEMPLATES = os.path.join(_PATH_ROOT, "templates")
+_PATH_DIR_SIMPLE = os.path.join(_PATH_TEMPLATES, "simple")
+_PATH_DIR_TITANIC = os.path.join(_PATH_TEMPLATES, "titanic")
+
+
+@pytest.mark.parametrize(
+    "cmd,args",
+    [
+        ("list_dirs", []),
+        ("bash_render", [_PATH_DIR_SIMPLE]),
+        ("bash_test", [_PATH_DIR_SIMPLE]),
+        # ("group_folders", [_PATH_SIMPLE]),
+        ("augment_script", [_PATH_DIR_SIMPLE]),
+        ("copy_notebooks", [_PATH_ROOT]),
+        ("update_env_details", [_PATH_DIR_SIMPLE]),
+    ],
+)
+def test_assistant_commands(cmd: str, args: list):
+    AssistantCLI().__getattribute__(cmd)(*args)

.github/workflows/ci_test-acts.yml

@@ -0,0 +1,69 @@
+name: CI internal
+
+# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
+on:  # Trigger the workflow on push or pull request, but only for the main branch
+  push: {}
+  pull_request:
+    branches: [main]
+
+jobs:
+  pytest-internal:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04 ]
+        python-version: ["3.9"]
+    # Timeout: https://stackoverflow.com/a/59076067/4521646
+    timeout-minutes: 10
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install requirements
+      run: |
+        pip --version
+        pip install -q -r .actions/requirements.txt
+        pip install -q "pytest==6.*" coverage jupytext
+        # this is needed to be able to run package version parsing test
+        pip install -q matplotlib -r requirements/default.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+
+    - name: Convert templates
+      run: |
+        jupytext --set-formats ipynb,py:percent templates/simple/template.py
+        jupytext --set-formats ipynb,py:percent templates/titanic/tutorial.py
+        # mv templates/simple/template.ipynb templates/simple.ipynb
+
+    - name: Testing
+      run: |
+        coverage run -m pytest .actions -v --junitxml="junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml"
+
+    - name: Upload pytest test results
+      uses: actions/upload-artifact@v2
+      with:
+        name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}
+        path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml
+      if: failure()
+
+    - name: Statistics
+      if: success()
+      run: |
+        coverage report
+        coverage xml
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      if: always()
+      # see: https://github.com/actions/toolkit/issues/399
+      continue-on-error: true
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: coverage.xml
+        flags: pytest,${{ runner.os }}
+        name: CLI-coverage
+        fail_ci_if_error: false

README.md

@@ -1,7 +1,8 @@
 # PytorchLightning Tutorials
 
+[![CI internal](https://github.com/PyTorchLightning/lightning-tutorials/actions/workflows/ci_test-acts.yml/badge.svg?event=push)](https://github.com/PyTorchLightning/lightning-tutorials/actions/workflows/ci_test-acts.yml)
 [![Build Status](https://dev.azure.com/PytorchLightning/Tutorials/_apis/build/status/PyTorchLightning.Tutorials-publishing?branchName=main)](https://dev.azure.com/PytorchLightning/Tutorials/_build/latest?definitionId=11&branchName=main)
-[![Code formatting](https://github.com/PyTorchLightning/lightning-tutorials/actions/workflows/ci_code-format.yml/badge.svg?event=push)](https://github.com/PyTorchLightning/lightning-tutorials/actions/workflows/ci_code-format.yml)
+[![codecov](https://codecov.io/gh/PyTorchLightning/lightning-tutorials/branch/main/graph/badge.svg?token=C6T3XOOR56)](https://codecov.io/gh/PyTorchLightning/lightning-tutorials)
 [![Deploy Docs](https://github.com/PyTorchLightning/lightning-tutorials/actions/workflows/docs-deploy.yml/badge.svg)](https://github.com/PyTorchLightning/lightning-tutorials/actions/workflows/docs-deploy.yml)
 [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/PyTorchLightning/lightning-tutorials/main.svg)](https://results.pre-commit.ci/latest/github/PyTorchLightning/lightning-tutorials/main)
 

setup.cfg

@@ -8,7 +8,6 @@ addopts =
     --strict
     --doctest-modules
     --color=yes
-    --nbval-cell-timeout=300
 
 [coverage:report]
 exclude_lines =