Merge pull request #13 from zkoppert/only-fetch-matching-repos

Only retrieve repos with matching topic in org

Author: zkoppert

.env-example

@@ -1,2 +1,3 @@
 GH_TOKEN=' '
 TOPIC='inner-source'
+ORGANIZATION=' '

README.md

@@ -10,8 +10,9 @@ This project creates a `repos.json` that can be utilized by the [SAP InnerSource
 ## Usage
 
 1. Copy `.env-example` to `.env`
-1. Fill out the `.env` file with a token from a machine user that only has access to the org to scan
+1. Fill out the `.env` file with a _token_ from a user that has access to the organization to scan (listed below)
 1. Fill out the `.env` file with the exact _topic_ name you are searching for
+1. Fill out the `.env` file with the exact _organization_ that you want to search in
 1. Run `python3 ./crawler.py`, which will create a `repos.json` file containing the relevant metadata for the GitHub repos for the given _topic_
 1. Copy `repos.json` to your instance of the [SAP-InnerSource-Portal][SAP-InnerSource-Portal] and launch the portal as outlined in their installation instructions
 

crawler.py

@@ -6,7 +6,6 @@
 
 import github3
 from dotenv import load_dotenv
-from ratelimiter import RateLimiter
 
 if __name__ == "__main__":
 
@@ -17,27 +16,22 @@
     # Auth to GitHub.com
     gh = github3.login(token=os.getenv("GH_TOKEN"))
 
-    # Get all repos from organization
-    all_repos = gh.repositories()
-    rate_limiter = RateLimiter(max_calls=10, period=1)
-    repo_list = []
     # Set the topic
     topic = os.getenv("TOPIC")
+    organization = os.getenv("ORGANIZATION")
+
+    # Get all repos from organization
+    search_string = "org:" + organization + " topic:" + topic
+    all_repos = gh.search_repositories(search_string)
+    repo_list = []
 
-    with rate_limiter:
-        for repo in all_repos:
-            if repo is not None:
-                try:
-                    repo_topic = repo.topics()
-                except Exception:
-                    print("skipping 404")
-                else:
-                    if topic in repo_topic.names:
-                        print("{0}".format(repo))
-                        full_repository = repo.refresh()
-                        # TODO: #7 For each resulting project add a key _InnerSourceMetadata
-                        # Add stuff here about innersource.json data before appending to list
-                        repo_list.append(full_repository.as_dict())
+    for repo in all_repos:
+        if repo is not None:
+            # TODO: #7 For each resulting project add a key _InnerSourceMetadata
+            print("{0}".format(repo.repository))
+            full_repository = repo.repository.refresh()
+            # Add stuff here about innersource.json data before appending to list
+            repo_list.append(repo.as_dict())
 
     # Write each repository to a repos.json file
     with open("repos.json", "w") as f:

requirements.txt

@@ -1,3 +1,2 @@
 github3.py==1.3.0
 python-dotenv==0.15.0
-ratelimiter==1.2.0.post0