retrieve config prefix individually (#33)

* retrieve config prefix individually

* retrieve results path from config and name

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix bug

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: lkstrp

.github/workflows/codeql.yml

@@ -89,4 +89,4 @@ jobs:
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v3
       with:
-        category: "/language:${{matrix.language}}"
+        category: "/language:${{matrix.language}}"

.github/workflows/publish-reporter-image.yml

@@ -20,7 +20,7 @@ jobs:
     - name: 'Build Inventory Image'
       run: |
         docker build . --tag ghcr.io/${{ github.repository_owner }}/pypsa-reporter:latest --file docker/reporter/Dockerfile
-        
+
     - name: 'Push Inventory Image'
-      run: |
-        docker push ghcr.io/${{ github.repository_owner }}/pypsa-reporter:latest
+      run: |-
+        docker push ghcr.io/${{ github.repository_owner }}/pypsa-reporter:latest

docker/reporter/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-slim
+FROM python:3.13-slim
 
 LABEL org.opencontainers.image.source https://github.com/lkstrp/pypsa-validator
 

pyproject.toml

@@ -10,15 +10,16 @@ authors = ["Lukas Trippe <lkstrp@pm.me>"]
 
 
 [tool.poetry.dependencies]
-python = "^3.7"
+python = "^3.12"
 # Add your specific dependencies here:
 requests = "^2.32.3"
-numpy = "^2.2.0"
+numpy = "^2.2.1"
 openpyxl = "^3.1.5"
-matplotlib = "^3.9.3"
+matplotlib = "^3.10.0"
 seaborn = "^0.13.2"
-GitPython = "^3.1.43"
-python-gitlab = "^5.1.0"
+GitPython = "^3.1.44"
+python-gitlab = "^5.3.1"
+pyyaml = "^6.0.2"
 # Add more as needed...
 
 

reporter/__main__.py

@@ -12,7 +12,6 @@ def main():
     report = "".join(comps)
 
     print(report)
-    import os
 
     publish(report)
 

reporter/components/general.py

@@ -3,7 +3,8 @@
 import pandas as pd
 
 from reporter.components.utils import create_details_block
-from reporter.meta.config import PLOTS, PREFIX
+from reporter.meta.config import PLOTS
+from reporter.meta.repo import PATH_FEAT_RESULTS, PATH_MAIN_RESULTS
 from reporter.utils import URL_IMAGE_PLACEHOLDER
 
 
@@ -15,8 +16,8 @@ def plots_table() -> str:
     for plot in plots_list:
         if not plot:
             continue
-        url_a = URL_IMAGE_PLACEHOLDER.format(f"main/results/{PREFIX}/{plot}")
-        url_b = URL_IMAGE_PLACEHOLDER.format(f"feat/results/{PREFIX}/{plot}")
+        url_a = URL_IMAGE_PLACEHOLDER.format(f"{PATH_MAIN_RESULTS}/{plot}")
+        url_b = URL_IMAGE_PLACEHOLDER.format(f"{PATH_FEAT_RESULTS}/{plot}")
         rows.append(
             [
                 f'<img src="{url_a}" alt="Image not available">',

reporter/meta/config.py

@@ -1,6 +1,11 @@
-from reporter.meta.paths import PATH_ARTIFACTS, PATH_REPO
+from pathlib import Path
+
 from reporter.utils import read_env_var
 
+# Mounted Paths
+PATH_ARTIFACTS = Path("/artifacts")
+PATH_REPO = Path("/repo")
+
 # Project
 PLATFORM = read_env_var("PLATFORM", "gitlab")
 DOMAIN = read_env_var("DOMAIN", "https://gitlab.com")
@@ -15,13 +20,26 @@
 # Configuration
 PLOTS = read_env_var("PLOTS")
 
+PATH_CONFIG = read_env_var("PATH_CONFIG")
 
-prefixes = [
-    prefix
-    for prefix in (PATH_ARTIFACTS / "main" / "results").iterdir()
-    if prefix.is_dir()
+# Do some checks on mounted paths
+if not PATH_ARTIFACTS.exists():
+    msg = f"Path '{PATH_ARTIFACTS}' must be mounted."
+    raise OSError(msg)
+needed_subdir = [
+    "feat/benchmarks",
+    "feat/results",
+    "feat/logs",
+    "feat/.snakemake/log",
+    "main/benchmarks",
+    "main/results",
+    "main/logs",
+    "main/.snakemake/log",
 ]
+if not all((PATH_ARTIFACTS / subdir).exists() for subdir in needed_subdir):
+    msg = f"Path '{PATH_ARTIFACTS}' must contain subdirectories: {needed_subdir}"
+    raise OSError(msg)
 
-PREFIX = prefixes[0].name # TODO: Handle different prefixes
-
-PATH_CONFIG = PATH_REPO / read_env_var("PATH_CONFIG")
+if not PATH_REPO.exists():
+    msg = f"Path '{PATH_REPO}' must be mounted."
+    raise OSError(msg)

reporter/meta/paths.py

@@ -10,10 +10,12 @@
     - BEHIND_COUNT: Number of commits behind the main branch.
 """
 
+from pathlib import Path
+
 import git
+import yaml
 
-from reporter.meta.config import HASH_FEAT, HASH_MAIN, PATH_CONFIG
-from reporter.meta.paths import PATH_REPO
+from reporter.meta.config import HASH_FEAT, HASH_MAIN, PATH_CONFIG, PATH_REPO
 
 repo = git.Repo(PATH_REPO)
 
@@ -54,3 +56,23 @@
         )
     )
 )
+
+# Read in hash specific config
+file_blob = repo.commit(HASH_MAIN).tree / str(PATH_CONFIG)
+CONFIG_MAIN = yaml.safe_load(file_blob.data_stream.read().decode("utf-8"))
+file_blob = repo.commit(HASH_FEAT).tree / str(PATH_CONFIG)
+CONFIG_FEAT = yaml.safe_load(file_blob.data_stream.read().decode("utf-8"))
+
+
+PATH_MAIN_RESULTS = (
+    Path("main")
+    / "results"
+    / CONFIG_MAIN.get("run", {}).get("prefix", "")
+    / CONFIG_MAIN.get("run", {}).get("name", "")
+)
+PATH_FEAT_RESULTS = (
+    Path("feat")
+    / "results"
+    / CONFIG_FEAT.get("run", {}).get("prefix", "")
+    / CONFIG_FEAT.get("run", {}).get("name", "")
+)

reporter/meta/repo.py

@@ -1,8 +1,9 @@
-import gitlab
 import re
+
+import gitlab
+
+from reporter.meta.config import DOMAIN, PATH_ARTIFACTS, PLATFORM
 from reporter.utils import REPORT_IDENTIFIER, read_env_var
-from reporter.meta.paths import PATH_ARTIFACTS
-from reporter.meta.config import PREFIX, PLATFORM, DOMAIN
 
 
 def publish(text: str) -> None:
@@ -23,7 +24,9 @@ def publish_on_gitlab(text: str) -> None:
 
     def _upload_image_to_github(name):
         path = PATH_ARTIFACTS / name
-        assert path.exists(), f"File {path} does not exist"
+        if not path.exists():
+            msg = f"File {path} does not exist. Are you sure you specified the correct configuration?"
+            raise FileNotFoundError(msg)
         uploaded_file = project.upload(name, filedata=open(path, "rb"))
         return uploaded_file["url"]