Merge pull request #1187 from datajoint/migrate-pyproject

Replace setup.py with pyproject.toml

Author: ethho

.devcontainer/devcontainer.json

@@ -5,7 +5,7 @@
 	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
 	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
 	"dockerComposeFile": [
-		"../LNX-docker-compose.yml",
+		"../docker-compose.yaml",
 		"docker-compose.yml"
 	],
 	// The 'service' property is the name of the service for the container that VS Code should

.github/workflows/development.yaml

@@ -29,10 +29,13 @@ jobs:
       COMPOSE_HTTP_TIMEOUT: "120"
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{matrix.py_ver}}
       - name: Validate version and release notes
         run: |
           DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
-          RELEASE_BODY=$(python -c \
+          RELEASE_BODY=$(python3 -c \
             'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \
           )
           echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
@@ -41,8 +44,8 @@ jobs:
           echo "EOF" >> $GITHUB_ENV
       - name: Build pip artifacts
         run: |
-          export HOST_UID=$(id -u)
-          docker compose -f docker-compose-build.yaml up --exit-code-from app --build
+          python3 -m pip install build
+          python3 -m build .
           echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
       - if: matrix.py_ver == '3.9' && matrix.distro == 'debian'
         name: Add pip artifacts
@@ -70,12 +73,6 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{matrix.py_ver}}
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 black
-      - name: Run syntax tests
-        run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
       - name: Run primary tests
         env:
           PY_VER: ${{matrix.py_ver}}
@@ -87,7 +84,7 @@ jobs:
           COMPOSE_HTTP_TIMEOUT: "120"
         run: |
           export HOST_UID=$(id -u)
-          docker compose -f LNX-docker-compose.yml up --build --exit-code-from app
+          docker compose --profile test up --quiet-pull --build --exit-code-from djtest djtest
   lint:
     runs-on: ubuntu-latest
     strategy:
@@ -101,8 +98,8 @@ jobs:
           python-version: ${{matrix.py_ver}}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install flake8 black==24.2.0
+          python3 -m pip install --upgrade pip
+          python3 -m pip install ".[test]"
       - name: Run syntax tests
         run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
       - name: Run style tests
@@ -138,7 +135,7 @@ jobs:
           export PACKAGE=datajoint
           export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git
           export HOST_UID=$(id -u)
-          docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build
+          docker compose -f docs/docker-compose.yaml up --quiet-pull --exit-code-from docs --build
           git push origin gh-pages
   publish-release:
     if: |
@@ -219,7 +216,7 @@ jobs:
       - name: Publish pip release
         run: |
           export HOST_UID=$(id -u)
-          docker compose -f docker-compose-build.yaml run \
+          docker compose run --build --quiet-pull \
             -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \
             sh -c "pip install twine && python -m twine upload dist/*"
       - name: Login to DockerHub

Dockerfile

@@ -1,11 +1,24 @@
-ARG IMAGE=jupyter/docker-stacks-foundation
-ARG PY_VER=3.9
-ARG DISTRO=debian
+ARG IMAGE=mambaorg/micromamba:1.5-bookworm-slim
 FROM ${IMAGE}
-RUN conda install -y -n base -c conda-forge python=${PY_VER} && \
-    conda clean -afy
-COPY --chown=anaconda:anaconda ./setup.py ./datajoint.pub ./requirements.txt /main/
-COPY --chown=anaconda:anaconda ./datajoint /main/datajoint
+
+ARG CONDA_BIN=micromamba
+ARG PY_VER=3.9
+ARG HOST_UID=1000
+
+RUN ${CONDA_BIN} install --no-pin -qq -y -n base -c conda-forge \
+        python=${PY_VER} pip setuptools git graphviz pydot && \
+    ${CONDA_BIN} clean -qq -afy
+ENV PATH="$PATH:/home/mambauser/.local/bin"
+
+COPY --chown=${HOST_UID:-1000}:mambauser ./pyproject.toml ./README.md ./LICENSE.txt /main/
+COPY --chown=${HOST_UID:-1000}:mambauser ./datajoint /main/datajoint
+
+VOLUME /src
+WORKDIR /src
+USER root
 RUN \
-    pip install --no-cache-dir /main && \
+    chown -R ${HOST_UID:-1000}:mambauser /main && \
+    chown -R ${HOST_UID:-1000}:mambauser /src && \
+    ${CONDA_BIN} run -n base pip install -q --no-cache-dir /main && \
     rm -r /main/*
+USER ${MAMBA_USER}

MANIFEST.in

@@ -136,7 +136,7 @@ def __call__(
         format=None,
         as_dict=None,
         squeeze=False,
-        download_path="."
+        download_path=".",
     ):
         """
         Fetches the expression results from the database into an np.array or list of dictionaries and

datajoint/fetch.py

@@ -27,7 +27,7 @@ def __init__(
         *,
         secure=False,
         proxy_server=None,
-        **_
+        **_,
     ):
         # from https://docs.min.io/docs/python-client-api-reference
         self.client = minio.Minio(

datajoint/s3.py

@@ -1,12 +1,6 @@
-# PY_VER=3.8 MYSQL_VER=5.7 DISTRO=alpine MINIO_VER=RELEASE.2022-08-11T04-37-28Z HOST_UID=$(id -u) docker compose -f LNX-docker-compose.yml up --exit-code-from app --build
-version: "2.4"
-x-net:
-  &net
-  networks:
-    - main
+# DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) docker compose --profile test up --build --exit-code-from djtest djtest
 services:
   db:
-    <<: *net
     image: datajoint/mysql:${MYSQL_VER:-8.0}
     environment:
       - MYSQL_ROOT_PASSWORD=${DJ_PASS:-password}
@@ -21,7 +15,6 @@ services:
       retries: 5
       interval: 15s
   minio:
-    <<: *net
     image: minio/minio:${MINIO_VER:-RELEASE.2022-08-11T04-37-28Z}
     environment:
       - MINIO_ACCESS_KEY=datajoint
@@ -34,74 +27,53 @@ services:
     command: server --address ":9000" /data
     healthcheck:
       test:
-        [
-          "CMD",
-          "curl",
-          "--fail",
-          "http://minio:9000/minio/health/live"
-        ]
+        - "CMD"
+        - "curl"
+        - "--fail"
+        - "http://minio:9000/minio/health/live"
       timeout: 30s
       retries: 5
       interval: 15s
-  fakeservices.datajoint.io:
-    <<: *net
-    image: datajoint/nginx:latest
-    environment:
-      - ADD_db_TYPE=DATABASE
-      - ADD_db_ENDPOINT=db:3306
-      - ADD_minio_TYPE=MINIO
-      - ADD_minio_ENDPOINT=minio:9000
-      - ADD_minio_PORT=80 # allow unencrypted connections
-      - ADD_minio_PREFIX=/datajoint
-    # ports:
-    #   - "80:80"
-    #   - "443:443"
-    #   - "3306:3306"
   app:
-    <<: *net
-    image: datajoint/djtest:py${PY_VER:-3.8}-${DISTRO:-alpine}
+    image: datajoint/datajoint:${DJ_VERSION:-latest}
     build:
       context: .
       dockerfile: Dockerfile
       args:
         PY_VER: ${PY_VER:-3.8}
-        DISTRO: ${DISTRO:-alpine}
+        HOST_UID: ${HOST_UID:-1000}
     depends_on:
       db:
         condition: service_healthy
       minio:
         condition: service_healthy
-      fakeservices.datajoint.io:
-        condition: service_healthy
     environment:
-      - DJ_HOST=fakeservices.datajoint.io
+      - DJ_HOST=db
       - DJ_USER=root
       - DJ_PASS=password
-      - DJ_TEST_HOST=fakeservices.datajoint.io
+      - DJ_TEST_HOST=db
       - DJ_TEST_USER=datajoint
       - DJ_TEST_PASSWORD=datajoint
-      - S3_ENDPOINT=fakeservices.datajoint.io
+      - S3_ENDPOINT=minio:9000
       - S3_ACCESS_KEY=datajoint
       - S3_SECRET_KEY=datajoint
       - S3_BUCKET=datajoint.test
       - PYTHON_USER=dja
       - JUPYTER_PASSWORD=datajoint
-      - DISPLAY
     working_dir: /src
+    user: ${HOST_UID:-1000}:mambauser
+    volumes:
+      - .:/src
+  djtest:
+    extends:
+      service: app
+    profiles: ["test"]
     command:
       - sh
       - -c
       - |
         set -e
-        pip install -e .
-        pip list --format=freeze | grep datajoint
-        pytest -sv --cov-report term-missing --cov=datajoint tests
-    # ports:
-    #   - "8888:8888"
-    user: ${HOST_UID:-1000}:anaconda
-    volumes:
-      - .:/src
-      - /tmp/.X11-unix:/tmp/.X11-unix:rw
-      # - ./notebooks:/home/dja/notebooks
-networks:
-  main:
+        pip install -q -e ".[test]"
+        pip freeze | grep datajoint
+        pytest --cov-report term-missing --cov=datajoint tests
+

LNX-docker-compose.yml renamed to docker-compose.yaml

@@ -0,0 +1,71 @@
+[project]
+name = "datajoint"
+version = "0.14.2"
+dependencies = [
+  "numpy",
+  "pymysql>=0.7.2",
+  "pyparsing",
+  "ipython",
+  "pandas",
+  "tqdm",
+  "networkx",
+  "pydot",
+  "minio>=7.0.0",
+  "matplotlib",
+  "otumat",
+  "faker",
+  "cryptography",
+  "urllib3"
+]
+requires-python = ">=3.8,<4.0"
+authors = [
+  {name = "Dimitri Yatsenko", email = "dimitri@datajoint.com"},
+  {name = "Raphael Guzman"},
+  {name = "Edgar Walker"},
+  {name = "DataJoint Contributors", email = "support@datajoint.com"},
+]
+maintainers = [
+  {name = "Dimitri Yatsenko", email = "dimitri@datajoint.com"},
+  {name = "DataJoint Contributors", email = "support@datajoint.com"},
+]
+description = "A relational data pipeline framework."
+readme = "README.md"
+license = {file = "LICENSE.txt"}
+keywords = [
+    "database",
+    "data pipelines",
+    "scientific computing",
+    "automated research workflows",
+]
+classifiers = [
+  "Programming Language :: Python"
+]
+
+[project.optional-dependencies]
+test = [
+  "pytest",
+  "pytest-cov",
+  "black==24.2.0",
+  "flake8",
+]
+
+[project.urls]
+Homepage = "https://datajoint.com/docs"
+Documentation = "https://datajoint.com/docs"
+Repository = "https://github.com/datajoint/datajoint-python"
+"Bug Tracker" = "https://github.com/datajoint/datajoint-python/issues"
+Changelog = "https://github.com/datajoint/datajoint-python/blob/master/CHANGELOG.md"
+
+[project.entry-points."console_scripts"]
+dj = "datajoint.cli:cli"
+datajoint = "datajoint.cli:cli"
+
+[tool.setuptools]
+packages = ["datajoint"]
+
+[build-system]
+requires = [
+    "setuptools>=60",
+    "setuptools-scm>=8.0"
+]
+build-backend = "setuptools.build_meta"