hydra works with nbdev

Author: TinasheMTapera

conf/config.yaml

@@ -0,0 +1,8 @@
+db: 
+  driver: mysql
+  user: omry
+  password: secret
+
+hydra:
+  run:
+    dir: logs/${now:%Y-%m-%d}/${now:%H-%M-%S}

era5_sandbox/__pycache__/__init__.cpython-313.pyc

@@ -28,19 +28,28 @@ def _expand_path(
     return path
 
 # %% ../notes/00_core.ipynb 5
-def describe():
-    print("This package fetches ERA5 data.")
+@hydra.main(version_base=None, config_path="../conf", config_name="config")
+def describe(
+    cfg: DictConfig=None,  # Configuration file
+    )-> None:
+    "Describe the configuration file used by Hydra for the pipeline"
+    
+    if cfg is None:
+        cfg = OmegaConf.create()
+        
+    print("This package fetches ERA5 data. The following is the config file used by Hydra for the pipeline:\n")
+    print(OmegaConf.to_yaml(cfg))
 
 # %% ../notes/00_core.ipynb 6
-#@hydra.main(version_base=None, config_path=here() / "conf", config_name="config")
+@hydra.main(version_base=None, config_path="../conf", config_name="config")
 def testAPI(
+    cfg: DictConfig=None,
     output_path:str=None,
     dataset:str="reanalysis-era5-pressure-levels",
-    remove:bool=False,
-    cfg: DictConfig = None
+    remove:bool=True
     )-> bool:    
 
-    #print(OmegaConf.to_yaml(cfg))
+    print(OmegaConf.to_yaml(cfg))
 
     try:
         client = cdsapi.Client()
@@ -80,6 +89,6 @@ def testAPI(
 
     except Exception as e:
         print("API connection test failed.")
-        print("Did you set up your API key with CDS? If not, please visit https://cds.climate.copernicus.eu/api-how-to")
+        print("Did you set up your API key with CDS? If not, please visit https://cds.climate.copernicus.eu/how-to-api#install-the-cds-api-client")
         print("Error: {}".format(e))
         return False

era5_sandbox/__pycache__/core.cpython-313.pyc

@@ -31,7 +31,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -46,7 +46,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -72,26 +72,35 @@
    "outputs": [],
    "source": [
     "#| export\n",
-    "def describe():\n",
-    "    print(\"This package fetches ERA5 data.\")"
+    "@hydra.main(version_base=None, config_path=\"../conf\", config_name=\"config\")\n",
+    "def describe(\n",
+    "    cfg: DictConfig=None,  # Configuration file\n",
+    "    )-> None:\n",
+    "    \"Describe the configuration file used by Hydra for the pipeline\"\n",
+    "    \n",
+    "    if cfg is None:\n",
+    "        cfg = OmegaConf.create()\n",
+    "        \n",
+    "    print(\"This package fetches ERA5 data. The following is the config file used by Hydra for the pipeline:\\n\")\n",
+    "    print(OmegaConf.to_yaml(cfg))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [],
    "source": [
     "#| export\n",
-    "#@hydra.main(version_base=None, config_path=here() / \"conf\", config_name=\"config\")\n",
+    "@hydra.main(version_base=None, config_path=\"../conf\", config_name=\"config\")\n",
     "def testAPI(\n",
+    "    cfg: DictConfig=None,\n",
     "    output_path:str=None,\n",
     "    dataset:str=\"reanalysis-era5-pressure-levels\",\n",
-    "    remove:bool=False,\n",
-    "    cfg: DictConfig = None\n",
+    "    remove:bool=True\n",
     "    )-> bool:    \n",
     "    \n",
-    "    #print(OmegaConf.to_yaml(cfg))\n",
+    "    print(OmegaConf.to_yaml(cfg))\n",
     "\n",
     "    try:\n",
     "        client = cdsapi.Client()\n",
@@ -131,11 +140,52 @@
     "\n",
     "    except Exception as e:\n",
     "        print(\"API connection test failed.\")\n",
-    "        print(\"Did you set up your API key with CDS? If not, please visit https://cds.climate.copernicus.eu/api-how-to\")\n",
+    "        print(\"Did you set up your API key with CDS? If not, please visit https://cds.climate.copernicus.eu/how-to-api#install-the-cds-api-client\")\n",
     "        print(\"Error: {}\".format(e))\n",
     "        return False"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can see that this API tester tool works with Hydra configuration:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "This package fetches ERA5 data. The following is the config file used by Hydra for the pipeline:\n",
+      "\n",
+      "db:\n",
+      "  driver: mysql\n",
+      "  user: omry\n",
+      "  password: secret\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from hydra import initialize, compose\n",
+    "from omegaconf import OmegaConf\n",
+    "\n",
+    "# unfortunately, we have to use the initialize function to load the config file\n",
+    "# this is because the @hydra decorator does not work with Notebooks very well\n",
+    "# this is a known issue with Hydra: https://gist.github.com/bdsaglam/586704a98336a0cf0a65a6e7c247d248\n",
+    "# \n",
+    "# just use the relative path from the notebook to the config dir\n",
+    "with initialize(version_base=None, config_path=\"../conf\"):\n",
+    "    cfg = compose(config_name='config.yaml')\n",
+    "\n",
+    "describe(cfg)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -144,10 +194,8 @@
    "source": [
     "#export\n",
     "if __name__ == \"__main__\":\n",
-    "    describe()\n",
-    "    testAPI()\n",
-    "else:\n",
-    "    pass"
+    "    # for testing\n",
+    "    describe()"
    ]
   },
   {
@@ -168,7 +216,15 @@
    "name": "python3"
   },
   "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
    "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
    "version": "3.13.2"
   }
  },