Merge pull request #1225 from yambottle/contribusion.md

datajoint-python developer guide

Author: dimitri-yatsenko

.gitignore

@@ -144,7 +144,7 @@ venv.bak/
 .ropeproject
 
 # mkdocs documentation
-/site
+/docs/site
 
 # mypy
 .mypy_cache/

CONTRIBUTION.md

@@ -139,6 +139,6 @@ DataJoint (https://datajoint.com).
 - [DataJoint Elements](https://datajoint.com/docs/elements/) - Catalog of example pipelines for neuroscience experiments
 
 - Contribute
-  - [Development Environment](https://datajoint.com/docs/core/datajoint-python/latest/develop/)
+  - [Contribution Guidelines](https://datajoint.com/docs/about/contribute/)
 
-  - [Guidelines](https://datajoint.com/docs/about/contribute/)
+  - [Developer Guide](https://datajoint.com/docs/core/datajoint-python/latest/develop/)

README.md

@@ -1,12 +1,17 @@
 # https://github.com/DavidAnson/markdownlint
 # https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md
+MD007: false # Unordered list indentation
 MD009: false # permit trailing spaces
 MD013:
-    line_length: "88" # Line length limits
+    # previously we defined line_length to 88 which is better for python
+    # but not for markdown
+    line_length:
+        - strict: false
     tables: false # disable for tables
     headings: false # disable for headings
-    code_blocks: false # disable for code blocks
+MD029: false # Ordered list item prefix
 MD030: false # Number of spaces after a list
+MD032: false # Lists should be surrounded by blank lines
 MD033: # HTML elements allowed
     allowed_elements:
         - "div"

docs/.docker/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3
+
+WORKDIR /main
+COPY ./docs/pip_requirements.txt /main/docs/pip_requirements.txt
+COPY ./datajoint /main/datajoint/
+COPY ./pyproject.toml /main/pyproject.toml
+
+RUN \
+    # Install docs dependencies
+    pip install --no-cache-dir -r /main/docs/pip_requirements.txt && \
+    # Install datajoint
+    pip install --no-cache-dir -e /main/
+
+# Install dependencies first and use docker cache
+# modify docs content won't cause image rebuild
+COPY ./docs/ /main/docs/

docs/.docker/apk_requirements.txt

@@ -1,16 +1,97 @@
-# Docs Contributions
+# Contribute to DataJoint Documentation
 
-Docs contributors should be aware of the following extensions, with the corresponding
-settings files, that were used in developing these docs:
+This is the home for DataJoint software documentation as hosted at https://datajoint.com/docs/core/datajoint-python/latest/
 
-- [MarkdownLinter](https://github.com/DavidAnson/markdownlint):
+## VSCode Linter Extensions and Settings
+
+The following extensions were used in developing these docs, with the corresponding
+settings files:
+
+- Recommended extensions are already specified in `.vscode/extensions.json`, it will ask you to install them when you open the project if you haven't installed them.
+- settings in `.vscode/settings.json`
+- [MarkdownLinter](https://marketplace.visualstudio.com/items?itemName=DavidAnson.vscode-markdownlint):
   - `.markdownlint.yaml` establishes settings for various
   [linter rules](https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md)
   - `.vscode/settings.json` formatting on save to fix linting
 
-- [CSpell](https://github.com/streetsidesoftware/vscode-spell-checker): `cspell.json`
+- [CSpell](https://marketplace.visualstudio.com/items?itemName=streetsidesoftware.code-spell-checker): `cspell.json`
 has various ignored words.
 
-- [ReWrap](https://github.com/stkb/Rewrap/): `.vscode/settings.json` allows toggling
+- [ReWrap](https://marketplace.visualstudio.com/items?itemName=stkb.rewrap): `.vscode/settings.json` allows toggling
 automated hard wrapping for files at 88 characters. This can also be keymapped to be
 performed on individual paragraphs, see documentation.
+
+## With Virtual Environment
+
+conda
+```bash
+conda create -n djdocs -y
+conda activate djdocs
+```
+venv
+```bash
+python -m venv .venv
+source .venv/bin/activate
+```
+
+Then install the required packages:
+```bash
+# go to the repo's root directory to generate API docs
+# cd ~/datajoint-python/
+
+# install mkdocs related requirements
+pip install -r ./docs/pip_requirements.txt
+# install datajoint, since API docs are generated from the package
+pip install -e .[dev]
+```
+
+Run mkdocs at: http://127.0.0.1:8000/
+```bash
+# go to the repo's root directory to generate API docs
+# cd ~/datajoint-python/
+
+# It will automatically reload the docs when changes are made
+mkdocs serve --config-file ./docs/mkdocs.yaml
+```
+
+## With Docker
+
+> We mostly use Docker to simplify docs deployment
+
+Ensure you have `Docker` and `Docker Compose` installed.
+
+Then run the following:
+```bash
+# It will automatically reload the docs when changes are made
+MODE="LIVE" docker compose up --build
+```
+
+Navigate to http://127.0.0.1:8000/ to preview the changes.
+
+This setup supports live-reloading so all that is needed is to save the markdown files
+and/or `mkdocs.yaml` file to trigger a reload.
+
+## Mkdocs Warning Explanation
+
+> TL;DR: We need to do it this way for hosting, please keep it as is.
+
+```log
+INFO    -  Doc file 'index.md' contains an unrecognized relative link './develop', it was left as is. Did you mean
+           'develop.md'?
+```
+
+- We use reverse proxy to proxy our docs sites, here is the proxy flow:
+  - You hit `datajoint.com/*` on your browser
+  - It'll bring you to the reverse proxy server first, that you wouldn't notice
+  - when your URL ends with:
+    - `/` is the company page
+    - `/docs/` is the landing/navigation page hosted by datajoint/datajoint-docs's github pages
+    - `/docs/core/datajoint-python/` is the actual docs site hosted by datajoint/datajoint-python's github pages
+    - `/docs/elements/element-*/` is the actual docs site hosted by each element's github pages
+
+
+```log
+WARNING -  Doc file 'query/operators.md' contains a link '../../../images/concepts-operators-restriction.png', but
+           the target '../../images/concepts-operators-restriction.png' is not found among documentation files.
+```
+- We use Github Pages to host our docs, the image references needs to follow the mkdocs's build directory structure, under `site/` directory once you run mkdocs.

docs/.markdownlint.yaml

@@ -1,36 +1,34 @@
-# MODE="LIVE|QA|BUILD" PACKAGE=datajoint UPSTREAM_REPO=https://github.com/datajoint/datajoint-python.git HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build
-version: "2.4"
+# MODE="LIVE|QA|BUILD" PACKAGE=datajoint UPSTREAM_REPO=https://github.com/datajoint/datajoint-python.git docker compose up --build
 services:
   docs:
     build:
-      dockerfile: docs/.docker/Dockerfile
-      context: ../
-      args:
-        - PACKAGE
-    image: ${PACKAGE}_python-docs
+      # some docs need to be dynamically generated from the datajoint PACKAGE
+      context: ..
+      dockerfile: docs/Dockerfile
+    image: datajoint-python-docs
     environment:
-      - PACKAGE
-      - UPSTREAM_REPO
-      - MODE
+      MODE: ${MODE:-LIVE} # specify mode: LIVE, QA, BUILD
+      # specify package to generate API docs
+      PACKAGE: ${PACKAGE:-datajoint}
+      UPSTREAM_REPO: ${UPSTREAM_REPO:-https://github.com/datajoint/datajoint-python.git}
     volumes:
       - ..:/main
-    user: ${HOST_UID}:anaconda
     ports:
-      - 80:80
+      - 8000:8000
     command:
-      - sh
+      - bash
       - -c
       - |
         set -e
         if echo "$${MODE}" | grep -i live &>/dev/null; then
-            mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80
+            mkdocs serve --config-file /main/docs/mkdocs.yaml -a 0.0.0.0:8000
         elif echo "$${MODE}" | grep -iE "qa|build" &>/dev/null; then
             git branch -D gh-pages || true
             git fetch $${UPSTREAM_REPO} gh-pages:gh-pages || true
-            mike deploy --config-file ./docs/mkdocs.yaml -u $$(grep -oE '\d+\.\d+' /main/$${PACKAGE}/version.py) latest
-            mike set-default --config-file ./docs/mkdocs.yaml latest
+            mike deploy --ignore-remote-status --config-file /main/docs/mkdocs.yaml -u $$(grep -oP '\d+\.\d+' /main/$${PACKAGE}/version.py) latest
+            # mike set-default --config-file /main/docs/mkdocs.yaml latest
             if echo "$${MODE}" | grep -i qa &>/dev/null; then
-                mike serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80
+                mike serve --config-file /main/docs/mkdocs.yaml -a 0.0.0.0:8000
             fi
         else
             echo "Unexpected mode..."

docs/Dockerfile

@@ -139,8 +139,8 @@ markdown_extensions:
   - toc:
       permalink: true
   - pymdownx.emoji:
-      emoji_index: !!python/name:materialx.emoji.twemoji
-      emoji_generator: !!python/name:materialx.emoji.to_svg
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
       options:
         custom_icons:
           - .overrides/.icons