add broadcast view example notebook, docker-compose file

Author: minrk

docs/source/examples/broadcast/Broadcast view.ipynb

@@ -0,0 +1,3 @@
+FROM jupyter/scipy-notebook:4a6f5b7e5db1
+RUN mamba install -yq openmpi mpi4py
+RUN pip install --upgrade https://github.com/ipython/ipyparallel/archive/HEAD.tar.gz

docs/source/examples/broadcast/Dockerfile

@@ -19,12 +19,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "import socket\n",
@@ -40,37 +35,48 @@
    "metadata": {},
    "source": [
     "For this demo, I will connect to a cluster with engines started with MPI.\n",
-    "If you have MPI and mpi4py on your machine, you can start a local cluster with MPI with:\n",
     "\n",
-    "    ipcluster start -n 8 --engines=MPI --profile mpi"
+    "One way to do so would be:\n",
+    "\n",
+    "    ipcluster start -n 64 --engines=MPI --profile mpi\n",
+    "    \n",
+    "In this directory is a docker-compose file to simulate multiple engine sets in "
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "mpi_profile = 'mpi'\n",
-    "rc = ipp.Client(profile=mpi_profile)\n",
-    "eall = rc[:]\n",
-    "root = rc[-1]"
+    "rc = ipp.Client(profile=\"mpi\")\n",
+    "rc.wait_for_engines(32)\n",
+    "eall = rc.broadcast_view(coalescing=True)\n",
+    "root = rc[0]"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rc.ids"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "root['a'] = 5"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
    "outputs": [],
    "source": [
     "%px from mpi4py.MPI import COMM_WORLD as MPI"
@@ -79,12 +85,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "mpi_ranks = eall.apply_async(lambda : MPI.Get_rank()).get_dict()\n",
@@ -95,44 +96,29 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "sz = 1024\n",
+    "sz = 512\n",
     "data = np.random.random((sz, sz))\n",
-    "data = data.dot(data.T)"
+    "data.nbytes // (1024 * 1024)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "%%time \n",
     "ar = eall.push({'data': data}, block=False)\n",
-    "ar.wait_interactive()"
+    "ar.wait_interactive()\n"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "@ipp.interactive\n",
@@ -156,12 +142,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "%%time\n",
@@ -172,23 +153,13 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "eall.apply_sync(np.linalg.norm, parallel.Reference('data'), 2)"
+    "%%px\n",
+    "import numpy as np\n",
+    "np.linalg.norm(data, 2)\n"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

docs/source/examples/MPI Broadcast.ipynb renamed to docs/source/examples/broadcast/MPI Broadcast.ipynb

@@ -0,0 +1,58 @@
+services:
+  # need /ipython to be writable by the container user (1000)
+  # can't seem to start volumes with the right permissions
+  init-permissions:
+    image: alpine
+    volumes:
+      - ipython:/ipython
+    command: chown -Rv 1000:1000 /ipython
+
+  controller:
+    depends_on:
+      - init-permissions
+      # condition: service_completed_successfully
+
+    build: .
+    volumes:
+      - ipython:/ipython
+    environment:
+      IPYTHONDIR: /ipython
+    command: |
+      ipcontroller --location controller --ip 0.0.0.0 --nodb --HubFactory.broadcast_scheduler_depth=2
+    deploy:
+      restart_policy:
+        condition: on-failure
+  engines:
+    build: .
+    depends_on:
+      - controller
+    volumes_from:
+      - controller
+    environment:
+      IPYTHONDIR: /ipython
+    links:
+      - controller
+    command: |
+      ipcluster engines -n 16
+    deploy:
+      mode: replicated
+      replicas: 4
+      restart_policy:
+        condition: on-failure
+
+  client:
+    build: .
+    ports:
+      - "9999:8888"
+    links:
+      - controller
+    volumes_from:
+      - controller
+    volumes:
+      - .:/home/jovyan/example
+    environment:
+      IPYTHONDIR: /ipython
+      JUPYTER_CONFIG_DIR: /ipython/jupyter
+
+volumes:
+  ipython: {}