fixed the implementation of a main function

Author: TinasheMTapera

.gitignore

@@ -1,2 +1,4 @@
 .micromamba
-data
+logs/*
+.snakemake/*
+.DS_Store

conf/config.yaml

@@ -1,7 +1,13 @@
-db: 
-  driver: mysql
-  user: omry
-  password: secret
+query: 
+  product_type: reanalysis
+  variable: ["2m_dewpoint_temperature", "2m_temperature", "skin_temperature"]
+  year: [2010, 2011]
+  month: [1, 2, 3]
+  day: [1, 2, 3, 4, 5]
+  time: [0, 6, 12, 18]
+  area: [0, 360, -90, 90]
+  data_format: netcdf
+  download_format: unarchived
 
 hydra:
   run:

conf/datapaths.yaml

@@ -0,0 +1,4 @@
+*
+!*/
+!.gitignore
+!README.md

data/.gitignore

@@ -7,4 +7,10 @@
                 'lib_path': 'era5_sandbox'},
   'syms': { 'era5_sandbox.core': { 'era5_sandbox.core._expand_path': ('core.html#_expand_path', 'era5_sandbox/core.py'),
                                    'era5_sandbox.core.describe': ('core.html#describe', 'era5_sandbox/core.py'),
-                                   'era5_sandbox.core.testAPI': ('core.html#testapi', 'era5_sandbox/core.py')}}}
+                                   'era5_sandbox.core.testAPI': ('core.html#testapi', 'era5_sandbox/core.py')},
+            'era5_sandbox.download': { 'era5_sandbox.download._validate_query': ( 'download_raw_data.html#_validate_query',
+                                                                                  'era5_sandbox/download.py'),
+                                       'era5_sandbox.download.download': ('download_raw_data.html#download', 'era5_sandbox/download.py'),
+                                       'era5_sandbox.download.fetch_MDG_GADM': ( 'download_raw_data.html#fetch_mdg_gadm',
+                                                                                 'era5_sandbox/download.py'),
+                                       'era5_sandbox.download.main': ('download_raw_data.html#main', 'era5_sandbox/download.py')}}}

era5_sandbox/__pycache__/__init__.cpython-313.pyc

@@ -0,0 +1,88 @@
+"""This module downloads the raw data from CDS and saves it in the local directory"""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../notes/01_download_raw_data.ipynb.
+
+# %% auto 0
+__all__ = ['fetch_MDG_GADM', 'download', 'main']
+
+# %% ../notes/01_download_raw_data.ipynb 4
+import os
+import hydra
+import cdsapi
+from fastcore.script import call_parse # for the console script
+
+from omegaconf import DictConfig, OmegaConf
+
+# %% ../notes/01_download_raw_data.ipynb 5
+def _validate_query(
+        query_body: DictConfig
+    )->bool:
+    '''
+    Check that the query is valid
+    '''
+
+    required_keys = ['product_type', 'variable', 'year', 'month', 'day', 'time', 'area', 'data_format', 'download_format']
+    if not all([key in query_body.keys() for key in required_keys]):
+        print(f"Missing required key in query. Required keys are {required_keys}")
+        print("Query validation failed")
+        raise ValueError("Invalid query")
+        
+    else:
+        return query_body
+
+# %% ../notes/01_download_raw_data.ipynb 6
+def fetch_MDG_GADM(
+        output_file: str="gadm41_MDG.gpkg" # file path to save the GADM data
+    )-> str:
+    '''
+    Fetch the GADM data for Madagascar
+    https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/gadm41_MDG.gpkg
+    '''
+
+    output_file_path = os.path.join(os.getcwd(), output_file)
+    if os.path.exists(output_file_path):
+        print("GADM data already exists")
+        return output_file_path
+    
+    print("Fetching GADM bounding box data for Madagascar")
+    os.system("curl --output {} https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/gadm41_MDG.gpkg".format(output_file))
+    print("GADM data fetched")
+    
+    return output_file_path
+
+# %% ../notes/01_download_raw_data.ipynb 7
+def download(
+        cfg: DictConfig,  # hydra configuration file
+        output_dir: str = "data/input/", # output directory
+        dataset: str = "reanalysis-era5-pressure-levels", # dataset to download
+        testing: bool = True  # testing mode
+    )->None:
+    '''
+    Send the query to the API and download the data
+    '''
+    
+    client = cdsapi.Client()
+    
+    query = _validate_query(cfg.query)
+    
+    # Send the query to the client
+    if not testing:
+        client.retrieve(dataset, query).download(os.path.join(output_dir, "{}_{}.nc".format(query.year, query.month)))
+    else:
+        print(f"Testing mode. Not downloading data. Query is {query}")
+
+    print("Done")
+
+# %% ../notes/01_download_raw_data.ipynb 8
+@hydra.main(config_path="../conf", config_name="config", version_base=None)
+def main(cfg: DictConfig) -> None:
+    download(cfg=cfg)
+
+# %% ../notes/01_download_raw_data.ipynb 9
+try: from nbdev.imports import IN_NOTEBOOK
+except: IN_NOTEBOOK=False
+
+if __name__ == "__main__" and not IN_NOTEBOOK:
+    print('Running from __main__ ...')
+
+    main()

era5_sandbox/__pycache__/core.cpython-313.pyc

@@ -186,14 +186,32 @@
     "describe(cfg)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Importing the Main Function\n",
+    "\n",
+    "Important: using main in nbdev is a little bit tricky. We need to define the main function in the module and then when we export the notebook to script, we need to add the `nbdev.imports.IN_NOTEBOOK` variable. This way, the main function will only be executed when we run the notebook and not when we import the module.\n",
+    "\n",
+    "```python\n",
+    "from nbdev.imports import IN_NOTEBOOK\n",
+    "```\n",
+    "\n",
+    "You'll see this listed throughout the notebooks."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "#export\n",
-    "if __name__ == \"__main__\":\n",
+    "try: from nbdev.imports import IN_NOTEBOOK\n",
+    "except: IN_NOTEBOOK=False\n",
+    "\n",
+    "if __name__ == \"__main__\" and not IN_NOTEBOOK:\n",
     "    # for testing\n",
     "    describe()"
    ]