Migrate to docker containers

Author: smartspot2

.dockerignore

@@ -0,0 +1 @@
+node_modules/

.pre-commit-config.yaml

@@ -9,11 +9,11 @@ repos:
       - id: trailing-whitespace
   # python
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.3.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/isort
-    rev: 5.11.4
+    rev: 5.12.0
     hooks:
       - id: isort
   - repo: local

Dockerfile.django

@@ -0,0 +1,29 @@
+# set up environment variables
+FROM python:3.9.16
+
+ENV POETRY_VIRTUALENVS_CREATE false
+ENV POETRY_VERSION 1.3.1
+ENV POETRY_HOME /opt/poetry
+ENV POETRY_NO_INTERACTION 1
+ENV VIRTUAL_ENV /venv
+ENV PYTHONUNBUFFERED 1
+
+ENV PATH $POETRY_HOME/bin:$PATH
+
+# set up dependencies, create virtual environment
+RUN mkdir -p /opt/csm_web
+WORKDIR /opt/csm_web
+
+# install poetry
+RUN curl -sSL https://install.python-poetry.org | python3 -
+# create and activate virtual environment for poetry
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH $VIRTUAL_ENV/bin:$PATH
+
+# install python dependencies
+COPY poetry.lock pyproject.toml ./
+RUN poetry install --no-root --with=dev
+
+# start database
+COPY ./docker-django-entrypoint.sh ./
+ENTRYPOINT ["./docker-django-entrypoint.sh"]

Dockerfile.node

@@ -0,0 +1,29 @@
+FROM node:18
+
+# install inotify-tools for webpack reloading
+# RUN apt install inotify-tools
+
+RUN mkdir -p /opt/csm_web \
+    # change ownership of /opt/csm_web to install dependencies
+    && chown node:node /opt/csm_web
+
+WORKDIR /opt/csm_web
+
+USER node
+
+# install npm dependencies
+COPY --chown=node:node package.json package-lock.json ./
+RUN npm install && npm cache clean --force
+ENV PATH /opt/csm_web/node_modules/.bin:$PATH
+
+WORKDIR /opt/csm_web/app
+
+# change to root to allow permission changes in entrypoint
+USER root
+
+# specify entrypoint to execute pre-command tasks
+COPY ./docker-node-entrypoint.sh ./
+ENTRYPOINT ["./docker-node-entrypoint.sh"]
+
+# start continuous compilation
+CMD ["npm", "run", "watch"]

README.md

@@ -11,20 +11,28 @@ If you're unfamiliar with CSM and/or its web applications, check out [this repos
 We don't know what specific minimum version you would need for any of the following software, but the most recent version of any of the below should work.
 
 - Python 3.9.13
-  - It is recommended that you use [`pyenv`](https://github.com/pyenv/pyenv) to manage python versions, so that you can use a consistent python version for `csm_web`, and another python version for your other projects.
+  - It is recommended that you use a python version manager like [`pyenv`](https://github.com/pyenv/pyenv) or [`asdf`](https://asdf-vm.com) (with [`asdf-python`](https://github.com/asdf-community/asdf-python)), so that you can use a consistent python version for `csm_web`, and another python version for your other projects.
 - [`poetry`](https://python-poetry.org/docs/#installation)
+  - We use poetry to manage python dependencies; this should be installed _outside_ of a virtual environment.
+  - Although everything will be run through Docker containers, you should use Poetry to get the dependencies locally for editing.
 - `npm`
-  - It is recommended that you use [`nvm`](https://github.com/nvm-sh/nvm) to manage node/npm versions, so that you can use a consistent node/npm version for `csm_web`, and another verison for your other projects.
-- [PostgreSQL](https://www.postgresql.org/download/)
-- [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli#download-and-install)
+  - It is recommended that you use a node version manager like [`nvm`](https://github.com/nvm-sh/nvm), [`n`](https://github.com/tj/n), or [`asdf`](https://asdf-vm.com) (with [`asdf-nodejs`](https://github.com/asdf-vm/asdf-nodejs)), so that you can use a consistent node/npm version for `csm_web`, and another verison for your other projects.
+  - Although everything will be run through Docker containers, you should get the dependencies locally for editing.
+- [Docker](https://www.docker.com)
+  - Your development environment will be hosted through docker containers, so that you do not need to do much local setup.
+- [PostgreSQL](https://www.postgresql.org/download/) (optional)
+  - This should not be necessary now that we have migrated to Docker, but install it if any issues arise when editing.
+- [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli#download-and-install) (optional)
   - Create an account on [Heroku](https://id.heroku.com/login) and [login](https://devcenter.heroku.com/articles/heroku-cli#getting-started)
-- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html)
+  - This is not completely necessary for the application to work locally; it is only used for interactions with the production/staging environment.
+- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) (optional)
   - We use an S3 bucket to store course resources. See [here](https://aws.amazon.com/s3/) to get started.
-- Log in to AWS CLI (`aws configure`) This will prompt an interactive session to enter login credentials.
-  - AWS Access Key ID: (ask tech chair)
-  - AWS Secret Access Key: (ask tech chair)
-  - Default region name: `us-east-1`
-  - Default output format: `json`
+  - Log in to AWS CLI (`aws configure`) This will prompt an interactive session to enter login credentials.
+    - AWS Access Key ID: (ask tech chair)
+    - AWS Secret Access Key: (ask tech chair)
+    - Default region name: `us-east-1`
+    - Default output format: `json`
+  - This is not completely necessary for the application to work locally; it is only used for interactions with the resources page in production/staging.
 
 Other miscellaneous requirements will be installed by the commands below.
 
@@ -34,23 +42,23 @@ To ensure package version consistency and avoid polluting your global package in
 
 Firstly, make sure you have the right python version (see `runtime.txt` for the expected python version to install). If you're using `pyenv` to manage python versions (this is recommended), you can install the specified python version with `pyenv install <version>`.
 
-From a terminal in the top level of the project directory, run `python3 -m venv venv`; if your system python version is different from the version required here, and you're using `pyenv`, run `PYENV_VERSION=<version> python3 -m venv venv` instead (for example, `PYENV_VERSION=3.9.13 python3 -m venv venv`). This will initialize a new virtual environment in the `venv/` folder, with the correct base python version.
+Next, make sure that your current python version is correct (i.e. as specified in the previous section); if it is different, then change to the correct python version. That is, with `pyenv`, run `pyenv local <version>`; with `asdf`, run `asdf local python <version>`.
 
-To activate the environment, run `source venv/bin/activate`. You will need to run this command every time you open a new terminal.
+Finally, run `./setup.sh`. This will install additional requirements needed by the server, and set up some necessary environment variables. In particular, the setup script installs all dependencies locally and builds the Docker images.
 
-Finally, run `./setup.sh`. This will install additional requirements needed by the server, and set up some necessary environment variables. You should _not_ be running this script after it has succeeded and set up the environment for the first time.
+Note that generally, you should not need to run `setup.sh` after first setting up the repository.
 
 ## Running
 
-To start the Django server, run `python3 csm_web/manage.py runserver` and visit `localhost:8000` in your browser.
+To start the Django server and other services, make sure Docker is up and run `docker compose up -d`. This will start Django, automatically compile and watch frontend files, and start a development database. (The `-d` puts the process in the background.)
 
-Run `python3 csm_web/manage.py createtestdata` to generate some test data. If you ran `./setup.sh`,
-this was done for you.
+To generate test data, run `docker compose exec django python3 csm_web/manage.py createtestdata`. In general, if you'd like to run any commands in the Django docker container, run `docker compose exec django <command>`. (You can make an alias in your shell if you'd like to avoid typing all of this each time.)
 
-_If you are working on the frontend_:
+If all of the above has worked, visit [http://localhost:8000](http://localhost:8000) in your browser and you should see a log in screen; don't actually use this to actually log in locally. Visit [http://localhost:8000/admin/](http://localhost:8000/admin/) to log in instead.
 
-Run `npm run watch`, which will automatically rebuild the JS bundle if any changes to the frontend JS are detected.
-Alternatively you can run `npm run dev` manually each time you make changes to the frontend.
+Any changes will automatically reload the server in the docker containers, but you will usually need to force refresh (`ctrl + shift + R` or `cmd + shift + R` on most browsers) for frontend changes to be reflected (this clears the browser cache for the page).
+
+During development, you should use the virtual environment as much as possible---while Docker makes this less necessary, your choice of editor may require the dependencies in the virtual environment. To activate the virtual environment, you can use `poetry shell` (this will start a new nested shell instance), or you can use `source .venv/bin/activate` (more generally, `source $(poetry env info --path)/bin/activate`).
 
 ## Troubleshooting
 
@@ -122,5 +130,4 @@ could not connect to server: Connection refused
         TCP/IP connections on port 5432?
 ```
 
-Your postgres server is likely not running. On a mac (which is the only platform we've done local
-testing on), run `brew services start postgres` before invoking `runserver` again.
+Your PostgreSQL server is likely not running. On MacOS, run `brew services start postgres` before invoking `runserver` again; on Unix, run `sudo service postgresql restart` before invoking `runserver` again.

csm_web/csm_web/settings.py

@@ -115,10 +115,10 @@
     DATABASES = {
         "default": {
             "ENGINE": "psqlextra.backend",
-            "NAME": "csm_web_dev",
-            "USER": "postgres",
+            "NAME": os.environ.get("POSTGRES_DB", "csm_web_dev"),
+            "USER": os.environ.get("POSTGRES_USER", "postgres"),
             "PASSWORD": os.environ.get("POSTGRES_PASSWORD", ""),
-            "HOST": "localhost",
+            "HOST": os.environ.get("POSTGRES_HOST", "localhost"),
             "PORT": "5432",
         }
     }

cypress/support/commands.ts

@@ -38,11 +38,14 @@ Cypress.Commands.add(
   "setupDB",
   (script_path: string, func_name: string, options: SetupDBOptions = { force: false, mutate: false }) => {
     // validate arguments
-    expect(script_path.match(/[a-zA-Z0-9_\/\.\-]/)).to.not.be.null;
+    expect(script_path.match(/[a-zA-Z0-9_/.-]/)).to.not.be.null;
     expect(func_name.match(/[a-zA-Z0-9_]/)).to.not.be.null;
 
+    // insert prefix in case of docker container; default to empty
+    const prefix = Cypress.env("DOCKER_PREFIX") ?? "";
+
     // run setup script
-    let command = `python3 cypress/db/_setup.py "${script_path}" "${func_name}"`;
+    let command = `${prefix} python3 cypress/db/_setup.py "${script_path}" "${func_name}"`;
     if (options.force) {
       command += " --force";
     }
@@ -57,7 +60,9 @@ Cypress.Commands.add(
  * Initialize the Django database and cache
  */
 Cypress.Commands.add("initDB", () => {
-  cy._exec("python3 cypress/db/_setup.py --init");
+  // insert prefix in case of docker container; default to empty
+  const prefix = Cypress.env("DOCKER_PREFIX") ?? "";
+  cy._exec(`${prefix} python3 cypress/db/_setup.py --init`);
 });
 
 /**

docker-compose.yml

@@ -0,0 +1,65 @@
+services:
+  postgres:
+    image: postgres:14
+    healthcheck:
+      test: /usr/bin/pg_isready
+      interval: 5s
+      timeout: 10s
+      retries: 120
+    environment:
+      POSTGRES_DB: csm_web_dev
+      PGUSER: postgres
+      POSTGRES_HOST_AUTH_METHOD: trust
+  node:
+    build:
+      context: .
+      dockerfile: Dockerfile.node
+    volumes:
+      # mount source files
+      - type: bind
+        source: ./
+        target: /opt/csm_web/app
+        read_only: true
+      # output from webpack
+      - type: bind
+        source: ./csm_web/frontend/static/frontend/
+        target: /opt/csm_web/app/csm_web/frontend/static/frontend/
+      # mount package files
+      - type: bind
+        source: ./package.json
+        target: /opt/csm_web/package.json
+        read_only: true
+      - type: bind
+        source: ./package-lock.json
+        target: /opt/csm_web/package-lock.json
+        read_only: true
+      # prevent node modules from being overwritten on host
+      - notused:/opt/csm_web/app/node_modules
+  django:
+    tty: true
+    build:
+      context: .
+      dockerfile: Dockerfile.django
+    env_file: .env
+    environment:
+      POSTGRES_DB: csm_web_dev
+      POSTGRES_USER: postgres
+      POSTGRES_HOST: postgres
+    ports:
+      - 8000:8000
+    volumes:
+      - type: bind
+        source: ./
+        target: /opt/csm_web
+        read_only: true
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+networks:
+  default:
+    name: csm_web_default
+
+# volume to exclude files from being mounted
+volumes:
+  notused:

docker-django-entrypoint.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+python3 csm_web/manage.py migrate
+exec python3 csm_web/manage.py runserver 0.0.0.0:8000

docker-node-entrypoint.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+chmod -R a+w /opt/csm_web/app/csm_web/frontend/static/frontend/
+exec runuser -u node "$@"