Merge pull request #42 from mcantu/main

Support multiple topics in repository search

Author: zkoppert

README.md

@@ -40,6 +40,8 @@ jobs:
       env:
         GH_TOKEN: ${{ secrets.GH_TOKEN }}
         ORGANIZATION: ${{ secrets.ORGANIZATION }}
+        # for multiple topics, add them after a comma eg:
+        # TOPIC: inner-source,actions,security,python
         TOPIC: inner-source
 ```
 

crawler.py

@@ -10,7 +10,6 @@
 from dotenv import load_dotenv
 
 if __name__ == "__main__":
-
     # Load env variables from file
     dotenv_path = join(dirname(__file__), ".env")
     load_dotenv(dotenv_path)
@@ -24,56 +23,69 @@
 
     # Set the topic
     topic = os.getenv("TOPIC")
+    if not topic:
+        raise ValueError("TOPIC environment variable not set")
+
+    # If multiple topics, split topics by comma
+    topics = [t.strip() for t in topic.split(",")]
     organization = os.getenv("ORGANIZATION")
 
-    # Get all repos from organization
-    search_string = "org:{} topic:{}".format(organization, topic)
-    all_repos = gh.search_repositories(search_string)
+    # Create empty list for repos
     repo_list = []
-
-    for repo in all_repos:
-        if repo is not None:
-            print("{0}".format(repo.repository))
-            full_repository = repo.repository.refresh()
-
-            innersource_repo = repo.as_dict()
-            innersource_repo["_InnerSourceMetadata"] = {}
-
-            # fetch innersource.json
-            try:
-                content = repo.repository.file_contents("/innersource.json").content
-                metadata = json.loads(b64decode(content))
-
-                innersource_repo["_InnerSourceMetadata"] = metadata
-            except github3.exceptions.NotFoundError:
-                # innersource.json not found in repository, but it's not required
-                pass
-
-            # fetch repository participation
-            participation = repo.repository.weekly_commit_count()
-            innersource_repo["_InnerSourceMetadata"]["participation"] = participation[
-                "all"
-            ]
-
-            # fetch contributing guidelines
-            try:
-                # if CONTRIBUTING.md exists in the repository, link to that instead of repo root
-                content = repo.repository.file_contents("/CONTRIBUTING.md").content
+    # Set for repos that have already been added to the list
+    repo_set = set()
+
+    # Iterate over topics, search for matching repositories, and process unique ones
+    for topic in topics:
+        search_string = "org:{} topic:{}".format(organization, topic)
+        all_repos = gh.search_repositories(search_string)
+
+        # For each repo in the search results, check if it's unique and add it to repo_set
+        for repo in all_repos:
+            if repo is not None and repo.repository.full_name not in repo_set:
+                repo_set.add(repo.repository.full_name)
+                print("{0}".format(repo.repository))
+
+                innersource_repo = repo.as_dict()
+                innersource_repo["_InnerSourceMetadata"] = {}
+
+                # fetch innersource.json
+                try:
+                    content = repo.repository.file_contents("/innersource.json").content
+                    metadata = json.loads(b64decode(content))
+
+                    innersource_repo["_InnerSourceMetadata"] = metadata
+                except github3.exceptions.NotFoundError:
+                    # innersource.json not found in repository, but it's not required
+                    pass
+
+                # fetch repository participation
+                participation = repo.repository.weekly_commit_count()
                 innersource_repo["_InnerSourceMetadata"][
-                    "guidelines"
-                ] = "CONTRIBUTING.md"
-            except github3.exceptions.NotFoundError:
-                # CONTRIBUTING.md not found in repository, but it's not required
-                pass
-
-            # fetch repository topics
-            topics = repo.repository.topics()
-            innersource_repo["_InnerSourceMetadata"]["topics"] = topics.names
-
-            # calculate score
-            innersource_repo["score"] = repo_activity.score.calculate(innersource_repo)
-
-            repo_list.append(innersource_repo)
+                    "participation"
+                ] = participation["all"]
+
+                # fetch contributing guidelines
+                try:
+                    # if CONTRIBUTING.md exists in the repository, link to that instead of repo root
+                    content = repo.repository.file_contents("/CONTRIBUTING.md").content
+                    innersource_repo["_InnerSourceMetadata"][
+                        "guidelines"
+                    ] = "CONTRIBUTING.md"
+                except github3.exceptions.NotFoundError:
+                    # CONTRIBUTING.md not found in repository, but it's not required
+                    pass
+
+                # fetch repository topics
+                repo_topics = repo.repository.topics()
+                innersource_repo["_InnerSourceMetadata"]["topics"] = repo_topics.names
+
+                # calculate score
+                innersource_repo["score"] = repo_activity.score.calculate(
+                    innersource_repo
+                )
+
+                repo_list.append(innersource_repo)
 
     # Write each repository to a repos.json file
     with open("repos.json", "w") as f: