Custom Projects List Feature

.gitignore

@@ -376,4 +376,6 @@ $RECYCLE.BIN/
 # Windows shortcuts
 *.lnk
 
+cache/*
+
 # End of https://www.toptal.com/developers/gitignore/api/java,pycharm+all,intellij+all,python,macos,windows,linux

README.md

@@ -63,12 +63,52 @@ python3 move_org_projects_under_project_list_then_unfollow.py <LGTM_PROJECT_LIST
 python3 follow_repos_by_search_term_via_code_instances.py <LANGUAGE> <SEARCH_TERM>
 
 # Finds repositories given a search term. Under the hood, the script searches Github for repositories that match the provided search term.
-python3 follow_repos_by_search_term.py <LANGUAGE> <SEARCH_TERM>
+python3 follow_repos_by_search_term.py <LANGUAGE> <SEARCH_TERM> <CUSTOM_LIST_NAME>(optional)
 
 # Finds top repositories that have a minimum 500 stars and use the provided programming language.
-python3 follow_top_repos_by_star_count.py <LANGUAGE>  
+python3 follow_top_repos_by_star_count.py <LANGUAGE> <CUSTOM_LIST_NAME>(optional)
+
+# Unfollows all projects you're currently following that are not in a custom list.
+python3 unfollow_all_followed_projects.py
 ```
 
+## The Custom Projects Lists Feature
+In developing these collection of scripts, we realized that when a user follows thousands of repos in their LGTM account, there is a chance that the LGTM account will break. You won't be able to use the query console and some API
+calls will be broken.
+
+To resolve this, we decided to create a feature users can opt-in. The "Custom Projects Lists" feature does the following:
+
+- Follows all repos (aka project) in your LGTM account.
+- Stores every project you follow in a txt file.
+- At a later date (we suggest 24 hours), the user may run a follow-up command that will take the repos followed, add them to a LGTM custom list, and finally unfollow the projects in the user's LGTM account.
+
+Although these steps are tedious, this is the best work-around we've found. We avoid bricking the LGTM account when projects are placed in custom lists. Also, we typically wait 24 hours since if the project is new to LGTM it will want to first process the project and projects being processed can't be added to custom lists.
+
+Finally, by having custom lists we hope that the security researcher will have an easier time picking which repos they want to test.
+
+### How To Run The Custom Projects Lists Feature
+In some of the commands above, you will see the <CUSTOM_LIST_NAME> option. This is optional for all
+commands. This CUSTOM_LIST_NAME represents the name of a LGTM project list that will be created and used to add projects to. Any projects found from that command will then be added to the LGTM custom list. Let's show an example below to get a better idea of how this works:
+
+1. Run a command passing in the name of the custom list name. The command below will follow Javascript repos and generate a cache file of every repo you follow for the project list called "cool_javascript_projects".
+
+    `python3 follow_top_repos_by_star_count.py javascript big_ole_js_projects`
+
+2. Wait 1 - 24 hours.
+
+3. Run the command below. This will take a cached file you created earlier, create a LGTM custom project list, add the projects to that project list, and finally unfollow the repositories in your LGTM account.
+
+    `python3 move_repos_to_lgtm_lists.py`
+
+Note: When naming a project custom list name, please use alphanumeric, dashes, and underscore characters only.
+
+### Build Processes By LGTM
+LGTM can't move projects that are being processed into custom lists. To resolve this, we've added a check that confirms whether or not all projects you plan on moving to a custom list are processed. If a project isn't processed, we will not move any projects into the custom list and you'll receive the following error:
+
+> The <CACHED_FILE_NAME> can't be processed at this time because a project build is still in progress.
+
+If you receive this error, wait a few hours and run the script again.
+
 ## Legal
 
 The author of this script assumes no liability for your use of this project, including,

auto_sort_projects.py

@@ -71,7 +71,7 @@
     project_list_name = gh_org_to_project_list_name[org]
     project_list_id = site.get_or_create_project_list(project_list_name)
     for project in org_to_projects[org]:
-        if project.is_protoproject:
+        if project.is_protoproject():
             print('Unable to add project to project list since it is a protoproject. %s' % project)
             continue
         site.load_into_project_list(project_list_id, [project.key])

follow_repos_by_search_term.py

@@ -1,29 +1,35 @@
 from typing import List
-from lgtm import LGTMSite
+from lgtm import LGTMSite, LGTMDataFilters
+
+import utils.cacher
 import utils.github_dates
 import utils.github_api
 
 import sys
 import time
 
-def save_project_to_lgtm(site: 'LGTMSite', repo_name: str):
+def save_project_to_lgtm(site: 'LGTMSite', repo_name: str) -> dict:
     print("About to save: " + repo_name)
     # Another throttle. Considering we are sending a request to Github
     # owned properties twice in a small time-frame, I would prefer for
     # this to be here.
     time.sleep(1)
 
     repo_url: str = 'https://github.com/' + repo_name
-    site.follow_repository(repo_url)
+    project = site.follow_repository(repo_url)
     print("Saved the project: " + repo_name)
+    return project
 
-def find_and_save_projects_to_lgtm(language: str, search_term: str):
+def find_and_save_projects_to_lgtm(language: str, search_term: str) -> List[str]:
     github = utils.github_api.create()
     site = LGTMSite.create_from_file()
+    saved_project_data: List[str] = []
 
     for date_range in utils.github_dates.generate_dates():
-        repos = github.search_repositories(query=f'language:{language} created:{date_range} {search_term}')
+        repos = github.search_repositories(query=f'stars:>5 language:{language} fork:false created:{date_range} {search_term}')
 
+        # TODO: This occasionally returns requests.exceptions.ConnectionError which is annoying as hell.
+        # It would be nice if we built in exception handling.
         for repo in repos:
             # Github has rate limiting in place hence why we add a sleep here. More info can be found here:
             # https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting
@@ -32,7 +38,15 @@ def find_and_save_projects_to_lgtm(language: str, search_term: str):
             if repo.archived or repo.fork:
                 continue
 
-            save_project_to_lgtm(site, repo.full_name)
+            saved_project = save_project_to_lgtm(site, repo.full_name)
+
+            simple_project = LGTMDataFilters.build_simple_project(saved_project)
+
+            if simple_project.is_valid_project:
+                saved_data = f'{simple_project.display_name},{simple_project.key},{simple_project.project_type}'
+                saved_project_data.append(saved_data)
+
+    return saved_project_data
 
 if len(sys.argv) < 3:
     print("Please make sure you provided a language and search term")
@@ -42,4 +56,9 @@ def find_and_save_projects_to_lgtm(language: str, search_term: str):
 search_term = sys.argv[2]
 
 print(f'Following repos for the {language} language that contain the \'{search_term}\' search term.')
-find_and_save_projects_to_lgtm(language, search_term)
+saved_project_data = find_and_save_projects_to_lgtm(language, search_term)
+
+# If the user provided a second arg then they want to create a custom list.
+if len(sys.argv) <= 4:
+    custom_list_name = sys.argv[3]
+    utils.cacher.write_project_data_to_file(saved_project_data, custom_list_name)

follow_top_repos_by_star_count.py

@@ -1,28 +1,32 @@
 from typing import List
-from lgtm import LGTMSite
+from lgtm import LGTMSite, LGTMDataFilters
 
 import utils.github_dates
 import utils.github_api
+import utils.cacher
 import sys
 import time
 
-def save_project_to_lgtm(site: 'LGTMSite', repo_name: str):
+def save_project_to_lgtm(site: 'LGTMSite', repo_name: str) -> dict:
     print("Adding: " + repo_name)
     # Another throttle. Considering we are sending a request to Github
     # owned properties twice in a small time-frame, I would prefer for
     # this to be here.
     time.sleep(1)
 
     repo_url: str = 'https://github.com/' + repo_name
-    site.follow_repository(repo_url)
+    project = site.follow_repository(repo_url)
+
     print("Saved the project: " + repo_name)
+    return project
 
-def find_and_save_projects_to_lgtm(language: str):
+def find_and_save_projects_to_lgtm(language: str) -> List[str]:
     github = utils.github_api.create()
     site = LGTMSite.create_from_file()
+    saved_project_data: List[str] = []
 
     for date_range in utils.github_dates.generate_dates():
-        repos = github.search_repositories(query=f'stars:>500 created:{date_range} sort:stars language:{language}')
+        repos = github.search_repositories(query=f'stars:>500 created:{date_range} fork:false sort:stars language:{language}')
 
         for repo in repos:
             # Github has rate limiting in place hence why we add a sleep here. More info can be found here:
@@ -32,7 +36,14 @@ def find_and_save_projects_to_lgtm(language: str):
             if repo.archived or repo.fork:
                 continue
 
-            save_project_to_lgtm(site, repo.full_name)
+            saved_project = save_project_to_lgtm(site, repo.full_name)
+            simple_project = LGTMDataFilters.build_simple_project(saved_project)
+
+            if simple_project.is_valid_project:
+                saved_data = f'{simple_project.display_name},{simple_project.key},{simple_project.project_type}'
+                saved_project_data.append(saved_data)
+
+    return saved_project_data
 
 if len(sys.argv) < 2:
     print("Please provide a language you want to search")
@@ -41,4 +52,9 @@ def find_and_save_projects_to_lgtm(language: str):
 language = sys.argv[1].capitalize()
 
 print('Following the top repos for %s' % language)
-find_and_save_projects_to_lgtm(language)
+saved_project_data = find_and_save_projects_to_lgtm(language)
+
+# If the user provided a second arg then they want to create a custom list.
+if len(sys.argv) <= 3:
+    custom_list_name = sys.argv[2]
+    utils.cacher.write_project_data_to_file(saved_project_data, custom_list_name)