update ER and resolve bug

Author: Maximilian Karl

docs/ER_diagram/ER_diagram.drawio

@@ -230,9 +230,9 @@ class Files():
         Methods
         -------
         to_list()
-            Returns a list of all filenames.
+            Returns a list of all pandas filenames.
         to_dict()
-            Returns a dict with the folder as key and the list of all filenames as value.
+            Returns a dict with the folder as key and the list of all pandas filenames as value.
         
         """
         DATA_DIR = ""
@@ -242,34 +242,40 @@ def to_list(cls) -> list:
             """
             to_list(cls)
 
-            Returns a list of all filenames.
+            Returns a list of all pandas filenames.
             
             Returns
             -------
             list
-                List of all filenames.
+                List of all pandas filenames.
 
             """
             filenames = []
             for var, value in vars(cls).items():
-                if isinstance(value,str) and not "DIR" in var and not var.startswith("__"):
-                    filenames.append(value)
+                if isinstance(value,str):
+                    if value.endswith(".p") and not var.startswith("__"):
+                        filenames.append(value)
+                else:
+                    if hasattr(value, "to_list"):
+                        for item in value.to_list():
+                            filenames.append(item)
             return filenames
 
         @classmethod
         def to_dict(cls) -> dict:
             """
             to_dict(cls)
             
-            Returns a dict with the folder as key and the list of all filenames as value.
+            Returns a dict with the folder as key and the list of all pandas filenames as value.
             
             Returns
             -------
             dict
-                Dictionary with the folder as key and the list of all filenames as value.
+                Dictionary with the folder as key and the list of all pandas filenames as value.
 
             """
             return {cls.DATA_DIR: cls.to_list()}
+
 
     class UserFiles(Files):
         """

docs/ER_diagram/ER_diagram.png

@@ -113,14 +113,12 @@ def __init__(self, git_releases: bool = True, issues_params: Issues.Params = Iss
             self.version = version
             self.workflows_params = workflows_params
 
-    class Files():
+    class Files(Core.Files):
         """
         A file class that holds all file names and the folder name.
 
         Attributes
         ----------
-        DATA_DIR : str
-            Folder name for this module.
         REPOS : str
             Filename of the repos json file.
         GIT_RELEASES : GitReleases.Files
@@ -135,8 +133,8 @@ class Files():
             Folder and files for version.
         WORKFLOWS : Workflows.Files
             Folder and files for workflows.
-        CORE : Core.Files
-            Folder and files for core.
+        USER : Core.UserFiles
+            User pandas files.
 
         Methods
         -------
@@ -153,48 +151,27 @@ class Files():
         REPOSITORY = Repository.Files
         VERSION = Version.Files
         WORKFLOWS = Workflows.Files
-        CORE = Core.Files
+        USER = Core.UserFiles
 
-        @staticmethod
-        def to_list() -> list:
+        @classmethod
+        def to_dict(cls) -> dict:
             """
-            to_list()
-
-            Returns a list of all filenames.
+            to_dict(cls)
             
-            Returns
-            -------
-            list
-                List of all filenames.
-
-            """
-            return [
-                GitHub2Pandas.Files.GIT_RELEASES,
-                GitHub2Pandas.Files.ISSUES,
-                GitHub2Pandas.Files.PULL_REQUESTS,
-                GitHub2Pandas.Files.REPOSITORY,
-                GitHub2Pandas.Files.VERSION,
-                GitHub2Pandas.Files.WORKFLOWS,
-                GitHub2Pandas.Files.CORE
-            ]
-
-        @staticmethod
-        def to_dict() -> dict:
-            """
-            to_dict()
-            
-            Returns a dict with the folder as key and the list of all filenames as value.
+            Returns a dict with the folder as key and the list of all pandas filenames as value.
             
             Returns
             -------
             dict
-                Dictionary with the folder as key and the list of all filenames as value.
+                Dictionary with the folder as key and the list of all pandas filenames as value.
 
             """
-            d = {}
-            for files in GitHub2Pandas.Files.to_list():
-                d.update(files.to_dict())
-            return d
+            files = {}
+            for var, value in vars(cls).items():
+                if not isinstance(value,str):
+                    if hasattr(value, "DATA_DIR"):
+                        files[value.DATA_DIR] = value.to_list()
+            return files
 
     def __init__(self, github_token: str, data_root_dir: Path, request_maximum: int = 40000, log_level: int = logging.INFO) -> None:
         """
@@ -537,7 +514,7 @@ def save_tables_to_excel(repo_data_dir: Path, filename: str = "GitHub2Pandas") -
         writer = pd.ExcelWriter(Path(repo_data_dir,f'{filename}.xlsx'), engine='xlsxwriter')
         for folder, files in GitHub2Pandas.Files.to_dict().items():
             for file in files:
-                if not isinstance(file,dict):
+                if file.endswith(".p"):
                     df = GitHub2Pandas.get_pandas_data_frame(repo_data_dir,folder,file)
                     df.to_excel(writer, sheet_name=file[:-2])
         writer.save()

github2pandas/core.py

@@ -115,19 +115,6 @@ class Files(Core.Files):
         PULL_REQUESTS_REACTIONS = "PullRequestsReactions.p"
         REVIEWS = "Reviews.p"
 
-        @staticmethod
-        def to_list() -> list:
-            return [
-                PullRequests.Files.PULL_REQUESTS,
-                PullRequests.Files.REVIEWS_COMMENTS,
-                PullRequests.Files.PULL_REQUESTS_REACTIONS,
-                PullRequests.Files.REVIEWS
-            ]
-
-        @staticmethod
-        def to_dict() -> dict:
-            return {PullRequests.Files.DATA_DIR: PullRequests.Files.to_list()}
-
     def __init__(self, github_connection: Github, repo: GitHubRepository, data_root_dir: Path, request_maximum: int = 40000, log_level: int = logging.INFO) -> None:
         """
         __init__(self, github_connection, repo, data_root_dir, request_maximum, log_level)
@@ -292,7 +279,7 @@ def generate_pandas_tables(self, check_for_updates: bool = False, params: Params
                     number = int(issues_df.loc[count,"number"])
                     pull_request = self.save_api_call(self.repo.get_pull, number)
                     self.extract_pull_request(pull_request, params)
-        if params.review_comment:
+        if params.review_comments:
             # extract comments
             self.extract_with_updated_and_since(
                 self.repo.get_pulls_comments,
@@ -302,7 +289,7 @@ def generate_pandas_tables(self, check_for_updates: bool = False, params: Params
         if extract_pull_requests:
             pull_request_df = DataFrame(self.__pull_request_list)
             self.save_pandas_data_frame(PullRequests.Files.PULL_REQUESTS, pull_request_df)
-        if params.review_comment:
+        if params.review_comments:
             review_comment_df = DataFrame(self.__review_comment_list)
             self.save_pandas_data_frame(PullRequests.Files.REVIEWS_COMMENTS, review_comment_df)
         if params.reviews:
@@ -357,7 +344,7 @@ def extract_pull_request(self, pull_request: GitHubPullRequest, params: Params)
             for i in range(self.request_maximum):
                 try:
                     commit = self.get_save_api_data(commits, i)
-                    pull_request_data["commits"].append(commit.sha)
+                    pull_request_data["commit_shas"].append(commit.sha)
                 except IndexError:
                     break
         self.__pull_request_list.append(pull_request_data)

github2pandas/github2pandas.py

@@ -7,7 +7,7 @@
 from github2pandas.core import Core
 from github2pandas.github2pandas import GitHub2Pandas
 
-class TestPullRequests(unittest.TestCase):
+class TestGitHub2Pandas(unittest.TestCase):
     """
     Test case for GitHub2Pandas class.
     """
@@ -21,13 +21,11 @@ class TestPullRequests(unittest.TestCase):
     data_root_dir = Path("test_data")
     log_level = logging.DEBUG
 
-    def __init__(self, methodName: str = ...) -> None:
+    def __init__(self, methodName: str = "...") -> None:
         super().__init__(methodName)
         if self.data_root_dir.exists() and self.data_root_dir.is_dir():
             shutil.rmtree(self.data_root_dir, onerror=Core.file_error_handling)
         self.data_root_dir.mkdir(parents=True, exist_ok=True)
-        print(Core.UserFiles.to_list())
-        print(Core.UserFiles.to_dict())
         pass
 
     def test_get_repository_information(self):
@@ -55,8 +53,15 @@ def test_get_all_data_frames(self):
         repo_data_dir = Path(self.data_root_dir,self.git_repo_owner,self.git_repo_name)
         for files in GitHub2Pandas.Files.to_list():
             for file in files.to_list():
-                if isinstance(file, str):
+                if file.endswith(".p"):
                     df = GitHub2Pandas.get_pandas_data_frame(repo_data_dir, files.DATA_DIR, file)
 
 if __name__ == "__main__":
-    unittest.main()
+    #unittest.main()
+    files = GitHub2Pandas.Files.to_list()
+    files = GitHub2Pandas.Files.to_dict()
+    data_root_dir = Path("test_data")
+    git_repo_name = "github2pandas"
+    git_repo_owner = "TUBAF-IFI-DiPiT"
+    GitHub2Pandas.save_tables_to_excel(Path(data_root_dir,git_repo_owner,git_repo_name))
+    pass