Add exclude option

Author: egeakman

Makefile

@@ -1,20 +1,28 @@
 # Variables for the project
 # =========================
-CONFERENCE ?= ep2024
+CONFERENCE ?= ep2025
 DATA_DIR ?= ./data/public/$(CONFERENCE)/
 
 # Variables for remote host
 # =========================
 VPS_USER  ?= static_content_user
 VPS_HOST  ?= static.europython.eu
-VPS_PATH  ?= /home/$(VPS_USER)/content/programapi/$(CONFERENCE)/releases
+VPS_PATH  ?= /home/$(VPS_USER)/content/static/programme/$(CONFERENCE)/releases
 REMOTE_CMD=ssh $(VPS_USER)@$(VPS_HOST)
 
 # Variables for deploy
-# ==========================
+# ====================
 TIMESTAMP ?= $(shell date +%Y%m%d%H%M%S)
 FORCE_DEPLOY ?= false
 
+# Optional arguments
+# ==================
+EXCLUDE ?=
+WARN_DUPES ?= false
+
+# Convert EXCLUDE space-separated list to repeated --exclude flags
+EXCLUDE_FLAGS = $(foreach item,$(EXCLUDE),--exclude $(item))
+
 dev:
 	uv sync --dev
 
@@ -27,13 +35,13 @@ deps/install:
 install: deps/install
 
 download:
-	python -m src.download
+	python -m src.download $(EXCLUDE_FLAGS)
 
 transform:
 ifeq ($(WARN_DUPES), true)
-	python -m src.transform --warn-dupes
+	python -m src.transform $(EXCLUDE_FLAGS) --warn-dupes
 else
-	python -m src.transform
+	python -m src.transform $(EXCLUDE_FLAGS)
 endif
 
 all: download transform

src/config.py

@@ -5,8 +5,8 @@
 
 
 class Config:
-    event = "europython-2024"
-    event_dir_name = "ep2024"
+    event = "europython-2025"
+    event_dir_name = "ep2025"
     project_root = Path(__file__).resolve().parents[1]
     raw_path = Path(f"{project_root}/data/raw/{event_dir_name}")
     public_path = Path(f"{project_root}/data/public/{event_dir_name}")

src/download.py

@@ -1,11 +1,23 @@
 import json
+from argparse import ArgumentParser
 from typing import Any
 
 import requests
 from tqdm import tqdm
 
 from src.config import Config
 
+parser = ArgumentParser(description="Download Pretalx data for EuroPython processing.")
+parser.add_argument(
+    "-e",
+    "--exclude",
+    choices=["schedule", "youtube"],
+    action="append",
+    help="Exclude certain resources from download.",
+)
+args = parser.parse_args()
+exclude = set(args.exclude or [])
+
 headers = {
     "Accept": "application/json, text/javascript",
     "Authorization": f"Token {Config.token()}",
@@ -14,24 +26,27 @@
 base_url = f"https://pretalx.com/api/events/{Config.event}/"
 schedule_url = base_url + "schedules/latest/"
 
+# Build resource list dynamically based on exclusions
 resources = [
-    # Questions need to be passed to include answers in the same endpoint,
-    # saving us later time with joining the answers.
     "submissions?questions=all&state=confirmed",
     "speakers?questions=all",
-    "p/youtube",
 ]
 
+if "youtube" not in exclude:
+    resources.append("p/youtube")
+
 Config.raw_path.mkdir(parents=True, exist_ok=True)
 
 for resource in resources:
-    url = base_url + f"{resource}"
+    # To get the resource name without extra parameters
+    resource_name = resource.split("?")[0].split("/")[-1]
+    url = base_url + resource
 
     res0: list[dict[str, Any]] = []
     data: dict[str, Any] = {"next": url}
     n = 0
 
-    pbar = tqdm(desc=f"Downloading {resource}", unit=" page", dynamic_ncols=True)
+    pbar = tqdm(desc=f"Downloading {resource_name}", unit=" page", dynamic_ncols=True)
 
     while url := data["next"]:
         n += 1
@@ -46,24 +61,25 @@
 
     pbar.close()
 
-    # To get the resource name without extra parameters
-    filename = resource.split("?")[0].split("/")[-1]
-    filename = f"{filename}_latest.json"
+    # Save the data to a file 
+    filename = f"{resource_name}_latest.json"
     filepath = Config.raw_path / filename
 
     with open(filepath, "w") as fd:
         json.dump(res0, fd)
 
+# Download schedule unless excluded
+if "schedule" not in exclude:
+    print("Downloading schedule...", end="")
+    response = requests.get(schedule_url, headers=headers)
 
-# Download schedule
-response = requests.get(schedule_url, headers=headers)
+    if response.status_code != 200:
+        raise Exception(f"Error {response.status_code}: {response.text}")
 
-if response.status_code != 200:
-    raise Exception(f"Error {response.status_code}: {response.text}")
-
-data = response.json()
-filename = "schedule_latest.json"
-filepath = Config.raw_path / filename
+    data = response.json()
+    filename = "schedule_latest.json"
+    filepath = Config.raw_path / filename
 
-with open(filepath, "w") as fd:
-    json.dump(data, fd)
+    with open(filepath, "w") as fd:
+        json.dump(data, fd)
+    print(" done.")

src/transform.py

@@ -1,4 +1,4 @@
-import sys
+from argparse import ArgumentParser
 
 from src.config import Config
 from src.utils.parse import Parse
@@ -7,43 +7,88 @@
 from src.utils.utils import Utils
 
 if __name__ == "__main__":
-    print(f"Parsing the data from {Config.raw_path}...")
+    parser = ArgumentParser(description="Transform data from Pretalx to EuroPython format and save it.")
+    parser.add_argument(
+        "-w",
+        "--warn-dupes",
+        action="store_true",
+        help="Warn about duplicates in the data.",
+    )
+    parser.add_argument(
+        "-e",
+        "--exclude",
+        choices=["schedule", "youtube"],
+        action="append",
+        help="Exclude certain data from transformation.",
+    )
+    args = parser.parse_args()
+    exclude = set(args.exclude or [])
+
+
+    print(f"Parsing submissions from {Config.raw_path}/submissions_latest.json...", end="")
     pretalx_submissions = Parse.publishable_submissions(
         Config.raw_path / "submissions_latest.json"
     )
+    print(" done.")
+
+    print(f"\nParsing speakers from {Config.raw_path}/speakers_latest.json...", end="")
     pretalx_speakers = Parse.publishable_speakers(
         Config.raw_path / "speakers_latest.json", pretalx_submissions.keys()
     )
-    pretalx_schedule = Parse.schedule(Config.raw_path / "schedule_latest.json")
+    print(" done.")
 
-    # Parse the YouTube data
-    youtube_data = Parse.youtube(Config.raw_path / "youtube_latest.json")
+    if "youtube" not in exclude:
+        print(f"Parsing YouTube data from {Config.raw_path}/youtube_latest.json...", end="")
+        youtube_data = Parse.youtube(Config.raw_path / "youtube_latest.json")
+        print(" done.")
+    else:
+        youtube_data = {}
 
-    print("Computing timing relationships...")
+    print("\nComputing timing relationships...", end="")
     TimingRelationships.compute(pretalx_submissions.values())
+    print(" done.")
 
-    print("Transforming the data...")
+    print("\nTransforming submissions...", end="")
     ep_sessions = Transform.pretalx_submissions_to_europython_sessions(
         pretalx_submissions,
         youtube_data,
     )
+    print(" done.")
+
+    print("\nTransforming speakers...", end="")
     ep_speakers = Transform.pretalx_speakers_to_europython_speakers(pretalx_speakers)
-    ep_schedule = Transform.pretalx_schedule_to_europython_schedule(
-        pretalx_schedule.breaks, ep_sessions, ep_speakers
-    )
+    print(" done.")
 
     # Warn about duplicates if the flag is set
-    if len(sys.argv) > 1 and sys.argv[1] == "--warn-dupes":
+    if args.warn_dupes:
         Utils.warn_duplicates(
             session_attributes_to_check=["title"],
             speaker_attributes_to_check=["name"],
             sessions_to_check=ep_sessions,
             speakers_to_check=ep_speakers,
         )
 
-    print(f"Writing the data to {Config.public_path}...")
+    print(f"\nWriting sessions to {Config.public_path}/sessions.json...", end="")
     Utils.write_to_file(Config.public_path / "sessions.json", ep_sessions)
+    print(" done.")
+
+    print(f"\nWriting speakers to {Config.public_path}/speakers.json...", end="")
     Utils.write_to_file(Config.public_path / "speakers.json", ep_speakers)
-    Utils.write_to_file(
-        Config.public_path / "schedule.json", ep_schedule, direct_dump=True
-    )
+    print(" done.")
+
+    if "schedule" not in exclude:
+        print("\nParsing schedule from {Config.raw_path}/schedule_latest.json...", end="")
+        pretalx_schedule = Parse.schedule(Config.raw_path / "schedule_latest.json")
+        print(" done.")
+
+        print(f"\nTransforming the schedule...", end="")
+        ep_schedule = Transform.pretalx_schedule_to_europython_schedule(
+            pretalx_schedule.breaks, ep_sessions, ep_speakers
+        )
+        print(" done.")
+
+        print(f"\nWriting schedule to {Config.public_path}/schedule.json...", end="")
+        Utils.write_to_file(
+            Config.public_path / "schedule.json", ep_schedule, direct_dump=True
+        )
+        print(" done.")

src/utils/utils.py

@@ -75,7 +75,8 @@ def warn_duplicates(
         Warns about duplicate attributes in the given objects
         """
         print(
-            f"Checking for duplicate {'s, '.join(session_attributes_to_check)}s in sessions..."
+            f"\nChecking for duplicate {'s, '.join(session_attributes_to_check)}s in sessions...",
+            end="",
         )
         duplicate_sessions = Utils.find_duplicate_attributes(
             sessions_to_check, session_attributes_to_check
@@ -84,9 +85,11 @@ def warn_duplicates(
         for attribute, codes in duplicate_sessions.items():
             if len(codes) > 1:
                 print(f"Duplicate ``{attribute}`` in sessions: {codes}")
+        print(" done.")
 
         print(
-            f"Checking for duplicate {'s, '.join(speaker_attributes_to_check)}s in speakers..."
+            f"Checking for duplicate {'s, '.join(speaker_attributes_to_check)}s in speakers...",
+            end="",
         )
         duplicate_speakers = Utils.find_duplicate_attributes(
             speakers_to_check, speaker_attributes_to_check
@@ -95,6 +98,7 @@ def warn_duplicates(
         for attribute, codes in duplicate_speakers.items():
             if len(codes) > 1:
                 print(f"Duplicate ``{attribute}`` in speakers: {codes}")
+        print(" done.")
 
     @staticmethod
     def compute_unique_slugs_by_attribute(