disable github actions and experiment with datapaths

Author: TinasheMTapera

.github/workflows/deploy.yaml

@@ -1,14 +1,14 @@
-name: Deploy to GitHub Pages
+# name: Deploy to GitHub Pages
 
-permissions:
-  contents: write
-  pages: write
+# permissions:
+#   contents: write
+#   pages: write
 
-on:
-  push:
-    branches: [ "main", "master" ]
-  workflow_dispatch:
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps: [uses: fastai/workflows/quarto-ghp@master]
+# on:
+#   push:
+#     branches: [ "main", "master" ]
+#   workflow_dispatch:
+# jobs:
+#   deploy:
+#     runs-on: ubuntu-latest
+#     steps: [uses: fastai/workflows/quarto-ghp@master]

.github/workflows/test.yaml

@@ -1,7 +1,7 @@
-name: CI
-on:  [workflow_dispatch, pull_request, push]
+# name: CI
+# on:  [workflow_dispatch, pull_request, push]
 
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps: [uses: fastai/workflows/nbdev-ci@master]
+# jobs:
+#   test:
+#     runs-on: ubuntu-latest
+#     steps: [uses: fastai/workflows/nbdev-ci@master]

conf/config.yaml

@@ -1,3 +1,7 @@
+defaults:
+  - _self_
+  - datapaths: datapaths
+
 query: 
   product_type: reanalysis
   variable: ["2m_dewpoint_temperature", "2m_temperature", "skin_temperature"]
@@ -9,6 +13,7 @@ query:
   data_format: netcdf
   download_format: unarchived
 
+# logging config
 hydra:
   run:
     dir: logs/${now:%Y-%m-%d}/${now:%H-%M-%S}

conf/datapaths.yaml

@@ -0,0 +1,3 @@
+# if the files are stored in local, then paths are null
+input: null
+output: null

conf/datapaths/datapaths.yaml

@@ -7,6 +7,7 @@
                 'lib_path': 'era5_sandbox'},
   'syms': { 'era5_sandbox.core': { 'era5_sandbox.core._expand_path': ('core.html#_expand_path', 'era5_sandbox/core.py'),
                                    'era5_sandbox.core.describe': ('core.html#describe', 'era5_sandbox/core.py'),
+                                   'era5_sandbox.core.main': ('core.html#main', 'era5_sandbox/core.py'),
                                    'era5_sandbox.core.testAPI': ('core.html#testapi', 'era5_sandbox/core.py')},
             'era5_sandbox.download': { 'era5_sandbox.download._validate_query': ( 'download_raw_data.html#_validate_query',
                                                                                   'era5_sandbox/download.py'),

era5_sandbox/_modidx.py

@@ -3,7 +3,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../notes/00_core.ipynb.
 
 # %% auto 0
-__all__ = ['describe', 'testAPI']
+__all__ = ['describe', 'testAPI', 'main']
 
 # %% ../notes/00_core.ipynb 3
 import os
@@ -28,7 +28,6 @@ def _expand_path(
     return path
 
 # %% ../notes/00_core.ipynb 5
-@hydra.main(version_base=None, config_path="../conf", config_name="config")
 def describe(
     cfg: DictConfig=None,  # Configuration file
     )-> None:
@@ -41,7 +40,6 @@ def describe(
     print(OmegaConf.to_yaml(cfg))
 
 # %% ../notes/00_core.ipynb 6
-@hydra.main(version_base=None, config_path="../conf", config_name="config")
 def testAPI(
     cfg: DictConfig=None,
     output_path:str=None,
@@ -92,3 +90,9 @@ def testAPI(
         print("Did you set up your API key with CDS? If not, please visit https://cds.climate.copernicus.eu/how-to-api#install-the-cds-api-client")
         print("Error: {}".format(e))
         return False
+
+# %% ../notes/00_core.ipynb 10
+@hydra.main(version_base=None, config_path="../conf", config_name="config")
+def main(cfg: DictConfig) -> None:
+    describe(cfg=cfg)
+    testAPI(cfg=cfg)

era5_sandbox/core.py

@@ -72,7 +72,6 @@
    "outputs": [],
    "source": [
     "#| export\n",
-    "@hydra.main(version_base=None, config_path=\"../conf\", config_name=\"config\")\n",
     "def describe(\n",
     "    cfg: DictConfig=None,  # Configuration file\n",
     "    )-> None:\n",
@@ -87,12 +86,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "#| export\n",
-    "@hydra.main(version_base=None, config_path=\"../conf\", config_name=\"config\")\n",
     "def testAPI(\n",
     "    cfg: DictConfig=None,\n",
     "    output_path:str=None,\n",
@@ -192,7 +190,7 @@
    "source": [
     "### Importing the Main Function\n",
     "\n",
-    "Important: using main in nbdev is a little bit tricky. We need to define the main function in the module and then when we export the notebook to script, we need to add the `nbdev.imports.IN_NOTEBOOK` variable. This way, the main function will only be executed when we run the notebook and not when we import the module.\n",
+    "Important: using `__main__` in nbdev and Hydra is a little bit tricky. We need to define the main function in the module ONLY ONCE and then when we export the notebook to script, we need to add the `nbdev.imports.IN_NOTEBOOK` variable. This way, the main function will only be executed when we run the notebook and not when we import the module.\n",
     "\n",
     "```python\n",
     "from nbdev.imports import IN_NOTEBOOK\n",
@@ -201,6 +199,19 @@
     "You'll see this listed throughout the notebooks."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "@hydra.main(version_base=None, config_path=\"../conf\", config_name=\"config\")\n",
+    "def main(cfg: DictConfig) -> None:\n",
+    "    describe(cfg=cfg)\n",
+    "    testAPI(cfg=cfg)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -212,8 +223,7 @@
     "except: IN_NOTEBOOK=False\n",
     "\n",
     "if __name__ == \"__main__\" and not IN_NOTEBOOK:\n",
-    "    # for testing\n",
-    "    describe()"
+    "    main()"
    ]
   },
   {