fix: make proWES run locally

.gitignore

@@ -221,3 +221,4 @@ tags
 !.gitkeep
 celeryd.pid
 *.modified.yaml
+data

Makefile

@@ -0,0 +1,54 @@
+default: help
+
+# Color variables
+BOLD = \033[1m
+PURPLE = \033[35m
+GRAY = \033[37m
+CYAN = \033[36m
+NC = \033[0m
+
+.PHONY: help
+help:
+	@echo "\nUsage: make [target] ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
+	@echo "Available targets:\n"
+	@echo "Development Environment ---------------------------------------------------"
+	@echo "  ${BOLD}${PURPLE}clean-venv${NC} ${GRAY}(cv)${NC}: ${CYAN}Remove virtual environment.${NC}"
+	@echo "  ${BOLD}${PURPLE}install${NC} ${GRAY}(i)${NC}: ${CYAN}Install package in development mode.${NC}"
+	@echo "  ${BOLD}${PURPLE}venv${NC} ${GRAY}(v)${NC}: ${CYAN}Create virtual environment.${NC}\n"
+	@echo "Development Services -----------------------------------------------------"
+	@echo "  ${BOLD}${PURPLE}dev-app${NC} ${GRAY}(da)${NC}: ${CYAN}Run only the prowes Flask app locally.${NC}"
+	@echo "  ${BOLD}${PURPLE}dev-celery${NC} ${GRAY}(dc)${NC}: ${CYAN}Run only the Celery worker locally.${NC}"
+	@echo "  ${BOLD}${PURPLE}dev-docker${NC} ${GRAY}(dd)${NC}: ${CYAN}Run all docker dev services.${NC}"
+
+# Development Environment
+.PHONY: clean-venv cv
+clean-venv cv:
+	@echo "Removing virtual environment..."
+	rm -rf .venv
+
+.PHONY: install i
+install i:
+	@echo "Installing package in development mode..."
+	pip install -e .
+
+.PHONY: venv v
+venv v:
+	@echo "Creating virtual environment..."
+	python -m venv .venv
+	@echo "Run 'source .venv/bin/activate' to activate the virtual environment"
+
+# Development Services
+.PHONY: dev-docker dd
+dev-docker dd:
+	@echo "Starting Flask app..."
+	docker compose -f docker-compose.dev.yaml up -d
+
+.PHONY: dev-app da
+dev-app da:
+	@echo "Starting FOCA app..."
+	python pro_wes/app.py
+
+.PHONY: dev-celery dc
+dev-celery dc:
+	@echo "Starting Celery worker..."
+	cd pro_wes && celery -A celery_worker worker -E --loglevel=info

docker-compose.dev.yaml

@@ -0,0 +1,23 @@
+services:
+  rabbitmq:
+    image: rabbitmq:3-management
+    restart: unless-stopped
+    links:
+      - mongodb
+    ports:
+      - "5672:5672"
+
+  mongodb:
+    image: mongo:3.2
+    restart: unless-stopped
+    volumes:
+      - ${PROWES_DATA_DIR:-../data/pro_wes}/db:${PWD}/data/db
+    ports:
+      - "27017:27017"
+
+  flower:
+    image: mher/flower:0.9.7
+    restart: unless-stopped
+    command: flower --broker=amqp://guest:guest@rabbitmq:5672// --port=5555
+    ports:
+      - "5555:5555"

docker-compose.yaml

@@ -1,4 +1,3 @@
-version: '3.6'
 services:
 
   prowes-worker:
@@ -22,7 +21,7 @@ services:
     restart: unless-stopped
     links:
       - mongodb
-    command: bash -c "cd /app/pro_wes; gunicorn -c gunicorn.py wsgi:app"
+    command: bash -c "cd /app; gunicorn -c pro_wes/gunicorn.py pro_wes.wsgi:app"
     ports:
       - "8090:8080"
     volumes:

pro_wes/app.py

@@ -1,5 +1,6 @@
 """proWES application entry point."""
 
+import os
 from pathlib import Path
 
 from connexion import App
@@ -16,8 +17,13 @@ def init_app() -> App:
     Returns:
         FOCA application.
     """
+    _parent_dir = Path(__file__).resolve().parent
+    if os.environ.get("ENVIRONMENT") == "DEV":
+        config_file = _parent_dir / "config.dev.yaml"
+    else:
+        config_file = _parent_dir / "config.yaml"
     foca = Foca(
-        config_file=Path(__file__).resolve().parent / "config.yaml",
+        config_file=config_file,
         custom_config_model="pro_wes.config_models.CustomConfig",
     )
     app = foca.create_app()
@@ -47,5 +53,6 @@ def run_app(app: App) -> None:
 
 
 if __name__ == "__main__":
+    os.environ.setdefault("ENVIRONMENT", "DEV")
     foca_app = init_app()
     run_app(app=foca_app)

pro_wes/config.dev.yaml

@@ -0,0 +1,154 @@
+# FOCA configuration
+# Available in app context as attributes of `current_app.config.foca`
+# Automatically validated via FOCA
+# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html
+
+# Server configuration
+# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.ServerConfig
+server:
+  host: "0.0.0.0"
+  port: 8080
+  debug: True
+  environment: development
+  testing: False
+  use_reloader: False
+
+# Security configuration
+# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.SecurityConfig
+security:
+  auth:
+    add_key_to_claims: True
+    algorithms:
+      - RS256
+    allow_expired: False
+    audience: null
+    validation_methods:
+      - userinfo
+      - public_key
+    validation_checks: any
+
+# Database configuration
+# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.DBConfig
+db:
+  host: localhost
+  port: 27017
+  dbs:
+    runStore:
+      collections:
+        runs:
+          indexes:
+            - keys:
+                run_id: 1
+                task_id: 1
+              options:
+                "unique": True
+                "sparse": True
+        service_info:
+          indexes:
+            - keys:
+                id: 1
+
+# API configuration
+# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.APIConfig
+api:
+  specs:
+    - path:
+        - pro_wes/api/20201124.tag_1_0_1.workflow_execution_service.openapi.yaml
+        - pro_wes/api/additions.openapi.yaml
+      add_operation_fields:
+        x-openapi-router-controller: pro_wes.ga4gh.wes.controllers
+      add_security_fields:
+        x-bearerInfoFunc: app.validate_token
+      disable_auth: True
+      connexion:
+        strict_validation: True
+        # workaround until cwl-WES responses are fixed
+        validate_responses: False
+        base_path: /ga4gh/wes/v1
+        options:
+          swagger_ui: True
+          serve_spec: True
+
+# Logging configuration
+# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.LogConfig
+log:
+  version: 1
+  disable_existing_loggers: False
+  formatters:
+    standard:
+      class: logging.Formatter
+      style: "{"
+      format: "[{asctime}: {levelname:<8}] {message} [{name}]"
+  handlers:
+    console:
+      class: logging.StreamHandler
+      level: 20
+      formatter: standard
+      stream: ext://sys.stderr
+  root:
+    level: 10
+    handlers: [console]
+
+# Background job configuration
+# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.JobsConfig
+jobs:
+  host: rabbitmq
+  port: 5672
+  backend: "rpc://"
+  include:
+    - pro_wes.tasks.track_run_progress
+
+# Exception configuration
+# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.ExceptionConfig
+exceptions:
+  required_members: [["message"], ["code"]]
+  status_member: ["code"]
+  exceptions: pro_wes.exceptions.exceptions
+
+# Custom configuration
+# Available in app context as attributes of `current_app.config.foca`
+custom:
+  defaults:
+    timeout: 2
+  post_runs:
+    db_insert_attempts: 10
+    id_charset: string.ascii_uppercase + string.digits
+    id_length: 6
+    polling_attempts: 100
+    polling_wait: 3
+    storage_path: "./data"
+    timeout_job: null
+    timeout_post: null
+  list_runs:
+    default_page_size: 5
+  service_info:
+    auth_instructions_url: "https://example.org/auth_instructions"
+    default_workflow_engine_parameters: []
+    id: v1.wes.ga4gh.org.example
+    name: "proWES example deployment"
+    supported_filesystem_protocols:
+      - https
+    supported_wes_versions:
+      - 1.0.1
+    tags:
+      key: "value"
+    type:
+      group: "org.ga4gh"
+      artifact: "wes"
+      version: "1.0.1"
+    workflow_engine_versions:
+      cwl-engine: "1.2.3"
+    workflow_type_versions:
+      CWL:
+        workflow_type_version:
+          - v1.0
+    description: "WES gateway service"
+    organization:
+      name: "Example organization"
+      url: "https://example.org"
+    contactUrl: "support@example.org"
+    documentationUrl: "https://example.org/docs"
+    createdAt: "2020-01-01T00:00:00Z"
+    updatedAt: "2022-12-31T00:00:00Z"
+    environment: "test"
+    version: "0.18.0"

pro_wes/config.yaml

@@ -53,8 +53,8 @@ db:
 api:
   specs:
     - path:
-        - api/20201124.tag_1_0_1.workflow_execution_service.openapi.yaml
-        - api/additions.openapi.yaml
+        - pro_wes/api/20201124.tag_1_0_1.workflow_execution_service.openapi.yaml
+        - pro_wes/api/additions.openapi.yaml
       add_operation_fields:
         x-openapi-router-controller: pro_wes.ga4gh.wes.controllers
       add_security_fields:

requirements.txt

@@ -2,3 +2,4 @@ connexion<3
 email-validator>=2.1.0,<3
 foca>=0.12.1
 gunicorn>=20.1.0,<21
+pymongo==4.7  # Cf. https://github.com/elixir-cloud-aai/foca/issues/246