download script works with hydra config and snakemake sbatch

Author: TinasheMTapera

.github/workflows/deploy.yaml renamed to _.github/workflows/deploy.yaml

@@ -7,13 +7,14 @@ development_mode: false
 CDS_API_KEY: 
   path: "$HOME/.cdsapirc"
 
+gadm_file: "https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/gadm41_MDG.gpkg"
+
 query: 
   product_type: reanalysis
-  gadm_file: "https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/gadm41_MDG.gpkg"
   # check precipitation
   # variable: ["2m_dewpoint_temperature", "2m_temperature", "skin_temperature", "total_precipitation"]
   variable: ["2m_dewpoint_temperature", "2m_temperature"]
-  year: [2010, 2011]
+  year: [2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024]
   month: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
   day: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
   time: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]

.github/workflows/test.yaml renamed to _.github/workflows/test.yaml

@@ -47,9 +47,11 @@
     "import hydra\n",
     "import cdsapi\n",
     "import geopandas as gpd\n",
-    "from core import _expand_path\n",
     "from pyprojroot import here\n",
-    "from omegaconf import DictConfig, ListConfig, OmegaConf"
+    "from omegaconf import DictConfig, ListConfig, OmegaConf\n",
+    "\n",
+    "try: from era5_sandbox.core import _expand_path\n",
+    "except: from core import _expand_path"
    ]
   },
   {
@@ -144,7 +146,8 @@
     "    bbox[2] = round(bbox[2], round_to) + buffer\n",
     "    bbox[3] = round(bbox[3], round_to) + buffer\n",
     "    \n",
-    "    bbox = [bbox[0], bbox[2], bbox[1], bbox[3]]\n",
+    "    # The bounding box from total_bounds ([min_x, min_y, max_x, max_y]) differs from the CDS API area format ([North, West, South, East]). The CDS API area format is used to specify the area of interest for the data download. The bounding box from total_bounds is in the format ([min_x, min_y, max_x, max_y]). To convert the bounding box to the CDS API area format, we need to rearrange the values as follows:\n",
+    "    bbox = [bbox[3], bbox[0], bbox[1], bbox[2]]\n",
     "\n",
     "    return bbox\n"
    ]
@@ -168,16 +171,17 @@
     "    testing = cfg.development_mode  # for testing\n",
     "    output_dir = here(\"data/input\") # output directory\n",
     "    \n",
+    "    target =os.path.join(_expand_path(output_dir), \"{}_{}.nc\".format(cfg.query['year'], cfg.query['month']))\n",
+    "    \n",
     "    client = cdsapi.Client()\n",
     "    \n",
     "    query = _validate_query(cfg.query)\n",
     "    \n",
     "    # Send the query to the client\n",
     "    if not testing:\n",
-    "        bounds = create_bounding_box(cfg.query['gadm_file'])\n",
+    "        bounds = create_bounding_box(cfg['gadm_file'])\n",
     "        query['area'] = bounds\n",
-    "        del query['gadm_file']\n",
-    "        client.retrieve(dataset, query).download(os.path.join(_expand_path(output_dir), \"{}_{}.nc\".format(query['year'], query['month'])))\n",
+    "        client.retrieve(dataset, query).download(target)\n",
     "    else:\n",
     "        print(f\"Testing mode. Not downloading data. Query is {query}\")\n",
     "\n",
@@ -237,7 +241,7 @@
     "\n",
     "if __name__ == \"__main__\" and not IN_NOTEBOOK:\n",
     "    print('Running from __main__ ...')\n",
-    "\n",
+    "    \n",
     "    main()"
    ]
   },

conf/config.yaml

@@ -61,5 +61,3 @@
         'nbdev': [f'{cfg.get("lib_name")}={cfg.get("lib_name")}._modidx:d']
     },
     **setup_cfg)
-
-

notes/01_download_raw_data.ipynb

@@ -3,6 +3,6 @@
 #SBATCH -p test # partition (queue)
 #SBATCH -c 12 # number of cores 
 #SBATCH --mem 90GB # memory 
-#SBATCH -t 0-02:00 # time (D-HH:MM)
+#SBATCH -t 0-10:00 # time (D-HH:MM)
 
 snakemake --cores 6

setup.py

@@ -1,4 +1,4 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: era5_sandbox
 Version: 0.0.1
 Summary: Sandbox environment for era5 development
@@ -27,6 +27,11 @@ Requires-Dist: nbdev
 Requires-Dist: cdsapi
 Requires-Dist: hydra-core
 Requires-Dist: ipykernel
+Requires-Dist: snakemake
+Requires-Dist: geopandas
+Requires-Dist: xarray
+Requires-Dist: matplotlib
+Requires-Dist: cartopy
 Provides-Extra: dev
 Dynamic: author
 Dynamic: author-email
@@ -36,6 +41,7 @@ Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: keywords
 Dynamic: license
+Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python

snakemake.sbatch

@@ -5,5 +5,10 @@ nbdev
 cdsapi
 hydra-core
 ipykernel
+snakemake
+geopandas
+xarray
+matplotlib
+cartopy
 
 [dev]