ci: use script outputs (#157)

* set output file
* ci: update

Author: Borda

.actions/assistant.py

@@ -18,6 +18,8 @@
 _PATH_HERE = os.path.dirname(__file__)
 _PATH_ROOT = os.path.dirname(_PATH_HERE)
 PATH_REQ_DEFAULT = os.path.join(_PATH_ROOT, "requirements", "default.txt")
+PATH_SCRIPT_RENDER = os.path.join(_PATH_HERE, "_ipynb-render.sh")
+PATH_SCRIPT_TEST = os.path.join(_PATH_HERE, "_ipynb-test.sh")
 REPO_NAME = "lightning-tutorials"
 COLAB_REPO_LINK = "https://colab.research.google.com/github/PytorchLightning"
 BRANCH_DEFAULT = "main"
@@ -293,11 +295,12 @@ def _bash_download_data(folder: str) -> List[str]:
         return cmd
 
     @staticmethod
-    def bash_render(folder: str) -> str:
+    def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[str]:
         """Prepare bash script for running rendering of a particular notebook.
 
         Args:
             folder: name/path to a folder with notebook files
+            output_file: if defined, stream the commands to the file
 
         Returns:
             string with nash script content
@@ -332,14 +335,18 @@ def bash_render(folder: str) -> str:
             cmd += [f"cp {thumb_file} {pub_thumb}", f"git add {pub_thumb}"]
         # add the generated notebook to version
         cmd.append(f"git add {pub_ipynb}")
-        return os.linesep.join(cmd)
+        if not output_file:
+            return os.linesep.join(cmd)
+        with open(output_file, "w") as fp:
+            fp.write(os.linesep.join(cmd))
 
     @staticmethod
-    def bash_test(folder: str) -> str:
+    def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str]:
         """Prepare bash script for running tests of a particular notebook.
 
         Args:
             folder: name/path to a folder with notebook files
+            output_file: if defined, stream the commands to the file
 
         Returns:
             string with nash script content
@@ -369,7 +376,10 @@ def bash_test(folder: str) -> str:
             warn("Invalid notebook's accelerator for this device. So no tests will be run!!!", RuntimeWarning)
         # deactivate and clean local environment
         cmd += ["deactivate", f"rm -rf {os.path.join(folder, 'venv')}"]
-        return os.linesep.join(cmd)
+        if not output_file:
+            return os.linesep.join(cmd)
+        with open(output_file, "w") as fp:
+            fp.write(os.linesep.join(cmd))
 
     @staticmethod
     def convert_ipynb(folder: str) -> None:

.azure/ipynb-publish.yml

@@ -105,7 +105,7 @@ jobs:
         mkdir $(PATH_DATASETS)
         # render the actual notebooks
         while IFS= read -r line; do
-            python .actions/assistant.py bash-render $line > .actions/_ipynb-render.sh
+            python .actions/assistant.py bash-render $line
             cat .actions/_ipynb-render.sh
             bash .actions/_ipynb-render.sh
         done <<< $(cat changed-folders.txt)

.azure/ipynb-tests.yml

@@ -79,7 +79,7 @@ jobs:
     - bash: |
         set -e
         while IFS= read -r line; do
-            python .actions/assistant.py bash-test $line > .actions/_ipynb-test.sh
+            python .actions/assistant.py bash-test $line
             cat .actions/_ipynb-test.sh
             bash .actions/_ipynb-test.sh
         done <<< $(cat changed-folders.txt)

.github/workflows/ci_docs.yml

@@ -73,7 +73,7 @@ jobs:
         run: |
           while IFS= read -r line; do
               python .actions/assistant.py convert-ipynb $line
-              python .actions/assistant.py bash-render $line > .actions/_ipynb-render.sh
+              python .actions/assistant.py bash-render $line
               cat .actions/_ipynb-render.sh
               bash .actions/_ipynb-render.sh
           done <<< $(cat changed-folders.txt)

.github/workflows/ci_testing.yml

@@ -74,8 +74,9 @@ jobs:
           KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
           KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
         run: |
+          set -e
           while IFS= read -r line; do
-              python .actions/assistant.py bash-test $line > .actions/_ipynb-test.sh
+              python .actions/assistant.py bash-test $line
               cat .actions/_ipynb-test.sh
               bash .actions/_ipynb-test.sh
           done <<< $(cat changed-folders.txt)

Makefile

@@ -16,7 +16,7 @@ ipynb: init ${IPYNB}
 %.ipynb: %/.meta.y*ml
 	@echo $<
 	python .actions/assistant.py augment-script $(shell dirname $<)
-	python .actions/assistant.py bash-render $(shell dirname $<) > .actions/_ipynb-render.sh
+	python .actions/assistant.py bash-render $(shell dirname $<)
 	bash .actions/_ipynb-render.sh
 
 docs: clean

templates/simple/.meta.yml

@@ -3,7 +3,7 @@ author: PL team
 created: 2021-06-15
 updated: 2021-06-17
 license: CC
-build: 7
+build: 8
 description: |
   This is a template to show how to contribute a tutorial.
 requirements: