CI/CD: store runtime requirements (#151)

* store runtime requirements
* cat meta
* ci docs
* convert-ipynb

Author: Borda

.actions/assistant.py

@@ -308,7 +308,6 @@ def bash_render(folder: str) -> str:
         ipynb_file, meta_file, thumb_file = AssistantCLI._valid_folder(folder, ext=".ipynb")
         pub_ipynb = os.path.join(DIR_NOTEBOOKS, f"{folder}.ipynb")
         pub_dir = os.path.dirname(pub_ipynb)
-        pub_meta = os.path.join(DIR_NOTEBOOKS, f"{folder}.yaml")
         thumb_ext = os.path.splitext(thumb_file)[-1] if thumb_file else "."
         pub_thumb = os.path.join(DIR_NOTEBOOKS, f"{folder}{thumb_ext}") if thumb_file else ""
         cmd.append(f"mkdir -p {pub_dir}")
@@ -325,9 +324,9 @@ def bash_render(folder: str) -> str:
                 warn("Invalid notebook's accelerator for this device. So no outputs will be generated.", RuntimeWarning)
                 cmd.append(f"cp {ipynb_file} {pub_ipynb}")
         # Export the actual packages used in runtime
-        cmd.append(f"python .actions/assistant.py update-env-details {folder}")
+        cmd.append(f"meta_file=$(python .actions/assistant.py update-env-details {folder})")
         # copy and add to version the enriched meta config
-        cmd += [f"cp {meta_file} {pub_meta}", f"git add {pub_meta}"]
+        cmd += ["echo $meta_file", "cat $meta_file", "git add $meta_file"]
         # if thumb image is linked to the notebook, copy and version it too
         if thumb_file:
             cmd += [f"cp {thumb_file} {pub_thumb}", f"git add {pub_thumb}"]
@@ -358,6 +357,10 @@ def bash_test(folder: str) -> str:
         # and install specific packages
         pip_req, pip_args = AssistantCLI._parse_requirements(folder)
         cmd += [f"pip install {pip_req} {pip_args}", "pip list"]
+        # Export the actual packages used in runtime
+        cmd.append(f"meta_file=$(python .actions/assistant.py update-env-details {folder} --base_path .)")
+        # show created meta config
+        cmd += ["echo $meta_file", "cat $meta_file"]
 
         cmd.append(f"# available: {AssistantCLI.DEVICE_ACCELERATOR}")
         if AssistantCLI._valid_accelerator(folder):
@@ -369,7 +372,7 @@ def bash_test(folder: str) -> str:
         return os.linesep.join(cmd)
 
     @staticmethod
-    def augment_script(folder: str) -> None:
+    def convert_ipynb(folder: str) -> None:
         """Add template header and footer to the python base script.
 
         Args:
@@ -609,7 +612,7 @@ def copy_notebooks(
             ipynb_content.append(os.path.join("notebooks", sub_ipynb))
 
     @staticmethod
-    def update_env_details(folder: str) -> None:
+    def update_env_details(folder: str, base_path: str = DIR_NOTEBOOKS) -> str:
         """Export the actual packages used in runtime.
 
         Args:
@@ -636,8 +639,9 @@ def _parse_package_name(pkg: str, keys: str = " <=>[]@", egg_name: str = "#egg="
         meta["environment"] = [env[r] for r in require]
         meta["published"] = datetime.now().isoformat()
 
-        fmeta = os.path.join(DIR_NOTEBOOKS, folder) + ".yaml"
+        fmeta = os.path.join(base_path, folder) + ".yaml"
         yaml.safe_dump(meta, stream=open(fmeta, "w"), sort_keys=False)
+        return fmeta
 
     @staticmethod
     def list_dirs(folder: str = "", include_file_ext: str = "") -> str:

.actions/test_cli.py

@@ -21,7 +21,7 @@ def _path_in_dir(fname: str, folder: str = _PATH_ROOT) -> str:
         ("bash_render", [_PATH_DIR_SIMPLE]),
         ("bash_test", [_PATH_DIR_SIMPLE]),
         ("group_folders", [_path_in_dir("master-diff.txt"), _path_in_dir("dirs-b1.txt"), _path_in_dir("dirs-b2.txt")]),
-        ("augment_script", [_PATH_DIR_SIMPLE]),
+        ("convert_ipynb", [_PATH_DIR_SIMPLE]),
         ("copy_notebooks", [_PATH_ROOT]),
         ("update_env_details", [_PATH_DIR_SIMPLE]),
     ],

.azure/ipynb-publish.yml

@@ -87,7 +87,7 @@ jobs:
 
     - bash: |
         while IFS= read -r line; do
-            python .actions/assistant.py augment-script $line
+            python .actions/assistant.py convert-ipynb $line
         done <<< $(cat changed-folders.txt)
       condition: gt(variables['changed.folders'], 0)
       displayName: 'Generate notebook'

.azure/ipynb-tests.yml

@@ -63,7 +63,7 @@ jobs:
 
     - bash: |
         while IFS= read -r line; do
-            python .actions/assistant.py augment-script $line
+            python .actions/assistant.py convert-ipynb $line
         done <<< $(cat changed-folders.txt)
       condition: gt(variables['changed.folders'], 0)
       displayName: 'Generate notebook'

.github/workflows/ci_docs.yml

@@ -1,6 +1,8 @@
 name: validate Docs
 
 on:  # Trigger the workflow on push or pull request
+  push:
+    branches: [main]
   pull_request: {}
   schedule:
     # At the end of every day
@@ -70,7 +72,7 @@ jobs:
         if: steps.changed.outputs.dirs != 0
         run: |
           while IFS= read -r line; do
-              python .actions/assistant.py augment-script $line
+              python .actions/assistant.py convert-ipynb $line
               python .actions/assistant.py bash-render $line > .actions/_ipynb-render.sh
               cat .actions/_ipynb-render.sh
               bash .actions/_ipynb-render.sh

.github/workflows/ci_testing.yml

@@ -64,7 +64,7 @@ jobs:
         if: steps.changed.outputs.dirs != 0
         run: |
           while IFS= read -r line; do
-              python .actions/assistant.py augment-script $line
+              python .actions/assistant.py convert-ipynb $line
           done <<< $(cat changed-folders.txt)
         shell: bash
 

templates/simple/.meta.yml

@@ -3,7 +3,7 @@ author: PL team
 created: 2021-06-15
 updated: 2021-06-17
 license: CC
-build: 6
+build: 7
 description: |
   This is a template to show how to contribute a tutorial.
 requirements: