Update gts_run_all_repos to include paths and referrer files

 - Add db.query_referring function
 - Adjust db.migrate_csv to handle Referring data

Author: astrochun

github_stats_pages/db.py

@@ -47,6 +47,10 @@ def migrate_csv(
     log.info(f"[yellow]Loading: {filename}")
     df = pd.read_csv(filename, header=None, skiprows=skip_rows, names=names)
     log.info(f"Size of dataframe: {len(df)}")
+    if model.__name__ == "Referring":  # Add date since this isn't included
+        file_date = filename.name[:10]
+        df.insert(loc=0, column="date", value=file_date)
+
     if model.__name__ == "Paths":
         repository_names = [a.split("/")[2] for a in df["path"].values]
         df.insert(1, "repository_name", repository_names)
@@ -56,7 +60,12 @@ def migrate_csv(
         query_results = list(
             map(func, df["repository_name"], df["date"], df["path"])
         )
-    else:
+    elif model.__name__ == "Referring":
+        func = partial(query_referring, engine=engine, model=model)
+        query_results = list(
+            map(func, df["repository_name"], df["date"], df["site"])
+        )
+    else:  # For Clone and Traffic
         func = partial(query, engine=engine, model=model)
         query_results = list(map(func, df["repository_name"], df["date"]))
 
@@ -125,3 +134,25 @@ def query_path(
             return result.one()
         except NoResultFound:
             return
+
+
+def query_referring(
+    repository_name: str,
+    date: str,
+    site: str,
+    engine: Engine,
+    model: Union[Type[SQLModel], Referring],
+) -> Union[SQLModel, Referring, None]:
+
+    with Session(engine) as session:
+        result = session.exec(
+            select(model).where(
+                model.repository_name == repository_name,
+                model.date == date,
+                model.site == site,
+            )
+        )
+        try:
+            return result.one()
+        except NoResultFound:
+            return

scripts/gts_run_all_repos

@@ -59,7 +59,9 @@ if __name__ == "__main__":
     p_cwd = Path.cwd()
     p_data = p_cwd / "data"
 
-    for datatype, model in zip(["clone", "traffic"], [db.Clone, db.Traffic]):
+    model_names = ["clone", "traffic", "paths", "referrer"]
+    models = [db.Clone, db.Traffic, db.Paths, db.Referring]
+    for datatype, model in zip(model_names, models):
         for f in p_cwd.glob(f"????-??-??-???-???-*{datatype}-stats.csv"):
             db.migrate_csv(f, model, engine, skip_rows=1)