refactor: update test suite to run locally (#195)

This change aims to simplify testing of PySlurm. The test suite, using
pytest, leverages a Slurm container for running tests locally.

Instructions are added to the README to use a test container, or an
existing Slurm cluster that you have access to.

Author: giovtorres

Pipfile

@@ -8,6 +8,10 @@ cython = "*"
 
 [dev-packages]
 black = "*"
+pytest = "*"
+pytest-sugar = "*"
+pytest-testinfra = "*"
+isort = "*"
 
 [pipenv]
 allow_prereleases = true

Pipfile.lock

@@ -75,6 +75,62 @@ the pyslurm module once it is built:
     make html
 
 
+Testing
+*******
+
+PySlurm requires an installation of Slurm.
+
+Using a Test Container
+----------------------
+
+To run tests locally without an existing Slurm cluster, `docker` and
+`docker-compose` is required.
+
+Clone the project::
+
+    git clone https://github.com/PySlurm/pyslurm.git
+    cd pyslurm
+
+Start the Slurm container in the background::
+
+    docker-compose up -d
+
+The cluster takes a few seconds to start all the required Slurm services. Tail the logs::
+
+    docker logs -f slurmctl
+
+When the cluster is ready, you will see the following log message::
+
+    Cluster is now available
+
+Press CTRL+C to stop tailing the logs. Slurm is now running in a container in detached mode. `docker-compose` also bind mounds the git directory
+inside the container at `/pyslurm` so that the container has access to the test cases.
+
+Install test dependencies::
+
+    pipenv sync --dev
+
+Execute the tests inside the container::
+
+    pipenv run pytest -sv scripts/run_tests_in_container.py
+
+When testing is complete, stop the running Slurm container::
+
+    docker-compose down
+
+Testing on an Existing Slurm Cluster
+------------------------------------
+
+You may also choose to clone the project and run tests on a node where Slurm is already compiled and installed::
+
+    git clone https://github.com/PySlurm/pyslurm.git
+    cd pyslurm
+    python3.9 setup.py build
+    python3.9 setup.py install
+    ./scripts/configure.sh
+    pipenv sync --dev
+    pipenv run pytest -sv
+
 Authors
 *******
 

README.rst

@@ -0,0 +1,14 @@
+version: "3.8"
+
+services:
+  slurm:
+    image: giovtorres/docker-centos7-slurm:20.11.8
+    hostname: slurmctl
+    container_name: slurmctl
+    stdin_open: true
+    tty: true
+    working_dir: /pyslurm
+    environment:
+      PYTHON: "3.9"
+    volumes:
+      - ${PWD}:/pyslurm

docker-compose.yml

@@ -16,5 +16,6 @@
 from .pyslurm import *
 from .__version__ import __version__
 
+
 def version():
     return __version__

pyslurm/__init__.py

@@ -52,7 +52,7 @@ then
 fi
 
 # Add the cluster to the slurm database
-sacctmgr --immediate add cluster name=linux
+#sacctmgr --immediate add cluster name=linux
 
 # Restart Slurm components to apply configuration changes
 supervisorctl restart slurmctld
@@ -66,14 +66,7 @@ supervisorctl restart slurmd
 # Wait for nodes to become IDLE
 wait_for_cmd "sinfo | grep -q normal.*idle" "Waiting for nodes to transition to IDLE"
 
-# Print the PySlurm version
-echo "---> PySlurm version"
-python$PYTHON -c "import pyslurm; print(pyslurm.version())"
-
-# Print the Slurm API version
-echo "---> Slurm API version"
-python$PYTHON -c "import pyslurm; print(pyslurm.slurm_api_version())"
-
+# TODO: convert to setup method/fixture
 # Submit test job with jobstep via srun for testing
 echo "---> Submitting sleep job"
 sbatch --wrap="srun sleep 1000"

scripts/configure.sh

@@ -0,0 +1,9 @@
+import testinfra
+
+
+def test_run():
+    host = testinfra.get_host(f"docker://slurmctl")
+    print(host.check_output("python3.9 setup.py build"))
+    print(host.check_output("python3.9 setup.py install"))
+    print(host.check_output("./scripts/configure.sh"))
+    print(host.check_output("pytest -v"))

scripts/run_tests_in_container.py

@@ -48,7 +48,6 @@ def get_user():
         ["squeue", "-O", "username", "-h"], stdout=subprocess.PIPE, stderr=None
     ).communicate()
     for username in users[0].splitlines():
-        print(username.decode())
         uid = pwd.getpwnam("{}".format(username.strip().decode()))
         yield username.strip().decode(), uid.pw_uid