Add section on training a neural network with PyTorch and xbatcher to user guide (#258)

Author: andersy005

ci/requirements/doc.yml

@@ -17,12 +17,13 @@ dependencies:
   - sphinx-design
   - xarray
   # For examples
-  - adlfs
+  - s3fs
   - ipykernel
   - nbsphinx
   - netcdf4
   - pooch
   - zarr
+  - pytorch
   # Editable xbatcher installation
   - pip
 

ci/requirements/environment.yml

@@ -9,7 +9,7 @@ dependencies:
   - numpy
   - xarray
   # Dev dependencies
-  - adlfs
+  - s3fs
   - asv
   - pre-commit
   - pytest

doc/conf.py

@@ -109,7 +109,11 @@
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build', '**.ipynb_checkpoints']
+exclude_patterns = [
+    '_build',
+    '**.ipynb_checkpoints',
+    'user-guide/create-fashion-mnist-dataset.ipynb',
+]
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.

doc/demo.ipynb

@@ -47,18 +47,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "store = 'az://carbonplan-share/example_cmip6_data.zarr'\n",
+    "store = 's3://carbonplan-share/xbatcher/example_cmip6_data.zarr'\n",
     "ds = xr.open_dataset(\n",
-    "    store,\n",
-    "    engine='zarr',\n",
-    "    chunks={},\n",
-    "    backend_kwargs={'storage_options': {'account_name': 'carbonplan'}},\n",
+    "    store, engine='zarr', chunks={}, backend_kwargs={'storage_options': {'anon': True}}\n",
     ")\n",
     "\n",
-    "# the attributes contain a lot of useful information, but clutter the print out when we inspect the outputs\n",
-    "# throughout this demo, clearing it to avoid confusion\n",
-    "ds.attrs = {}\n",
-    "\n",
     "# inspect the dataset\n",
     "ds"
    ]
@@ -393,7 +386,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.9.13 ('xbatcher-tests')",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -407,7 +400,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.13 | packaged by conda-forge | (default, Mar 25 2022, 06:04:14) \n[Clang 12.0.1 ]"
+   "version": "3.11.9"
   },
   "vscode": {
    "interpreter": {

doc/user-guide/caching.ipynb

@@ -218,7 +218,7 @@
     "\n",
     "# Set up S3 filesystem (you'll need appropriate credentials)\n",
     "s3 = s3fs.S3FileSystem(anon=False)\n",
-    "store = s3.get_mapper('s3://my-bucket/my-cache.zarr')\n",
+    "cache = s3.get_mapper('s3://my-bucket/my-cache.zarr')\n",
     "\n",
     "# Use this cache with BatchGenerator\n",
     "gen_s3 = xbatcher.BatchGenerator(ds, input_dims={'lat': 10, 'lon': 10}, cache=cache)\n",

doc/user-guide/create-fashion-mnist-dataset.ipynb

@@ -6,3 +6,4 @@ User Guide
    :caption: Contents:
 
    caching
+   training-a-neural-network-with-Pytorch-and-xbatcher