Calculate min confidence of two files changing together (normalized co-change count)

Author: JohT

cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_code_files.cypher

@@ -1,7 +1,8 @@
 // Take the already existing "CHANGED_TOGETHER_WITH" relationship between git files and apply it to resolved file nodes. Requires "Add_CHANGED_TOGETHER_WITH_relationships_to_git_files".
 
 MATCH (firstGitFile:Git&File&!Repository)-[gitChange:CHANGED_TOGETHER_WITH]-(secondGitFile:Git&File&!Repository)
-WHERE elementId(firstGitFile) < elementId(secondGitFile)
+//De-duplicating the pairs of files isn't necessary, because the dependency relation is directed.
+//WHERE elementId(firstGitFile) < elementId(secondGitFile)
 MATCH (firstGitFile)-[:RESOLVES_TO]->(firstCodeFile:File&!Git&!Repository)
 MATCH (secondGitFile)-[:RESOLVES_TO]->(secondCodeFile:File&!Git&!Repository)
  CALL (firstCodeFile, secondCodeFile, gitChange) {

cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_git_files.cypher

@@ -1,45 +1,84 @@
 // Connect git files that where changed together frequently with "CHANGED_TOGETHER_WITH"
 
-MATCH (global_git_commit:Git:Commit)
- WITH count(global_git_commit) AS globalCommitCount
+// Determine global file count, global file count threshold (filter out refactoring commits) and global update commits
+MATCH (git_commit_global:Git:Commit)-[:CONTAINS_CHANGE]->(:Git:Change)-[:UPDATES]->(git_file_global:Git:File)
+WHERE git_file_global.deletedAt IS NULL
+ WITH git_commit_global, count(DISTINCT git_file_global) AS commitFileCount
+ WITH percentileDisc(commitFileCount, 0.95) AS globalFileCountThreshold
+     ,count(git_commit_global)              AS globalUpdateCommitCount
+// Main section
 MATCH (git_commit:Git:Commit)-[:CONTAINS_CHANGE]->(git_change:Git:Change)-[:UPDATES]->(git_file:Git:File)
 MATCH (git_repository:Git&Repository)-[:HAS_FILE]->(git_file)
 WHERE git_file.deletedAt IS NULL
 // Order files to assure, that pairs of distinct files are grouped together (fileA, fileB) without (fileB, fileA)
 ORDER BY git_commit.sha, git_file.relativePath
- WITH globalCommitCount
+ WITH globalFileCountThreshold
+     ,globalUpdateCommitCount
      ,git_commit.sha             AS commitHash
      ,collect(DISTINCT git_file) AS filesInCommit
 // Limit the file count to min. 2 (changed together) and
 // max. 50 (reduce permutations, improve performance, filter out large refactorings that usually affect many files)
 WHERE size(filesInCommit) >= 2
-  AND size(filesInCommit) <= 50
+  AND size(filesInCommit) <= globalFileCountThreshold
 // Collect distinct pairwise (..., 2, 2) combinations of all files in the list
- WITH globalCommitCount
+ WITH globalFileCountThreshold
+     ,globalUpdateCommitCount
      ,commitHash
      ,apoc.coll.combinations(filesInCommit, 2, 2) AS fileCombinations
 UNWIND fileCombinations AS fileCombination
-  WITH globalCommitCount
+  WITH globalFileCountThreshold
+      ,globalUpdateCommitCount
       ,fileCombination
-      ,count(DISTINCT commitHash)      AS commitCount
-      ,collect(DISTINCT commitHash)    AS commitHashes
+      ,count(DISTINCT commitHash)      AS updateCommitCount
+      ,collect(DISTINCT commitHash)    AS updateCommitHashes
+// Deactivated: 
 // Filter out file pairs that where changed not very often together 
 // In detail: More than 0.1 per mille compared to overall commit count
-WHERE commitCount > globalCommitCount * 0.001 
- WITH fileCombination[0] AS firstFile
+// WHERE updateCommitCount > globalUpdateCommitCount * 0.001 
+ WITH *
+     ,fileCombination[0] AS firstFile
      ,fileCombination[1] AS secondFile
-     ,commitCount
-     ,commitHashes
-// Create the new relationship "CHANGED_TOGETHER_WITH" and set the property "commitCount" on it
- CALL (firstFile, secondFile, commitCount, commitHashes) {
+ WITH *
+     // Get the lowest number of git update commits of both files (file pair) 
+     ,CASE WHEN firstFile.updateCommitCount < secondFile.updateCommitCount
+           THEN firstFile.updateCommitCount
+           ELSE secondFile.updateCommitCount
+      END AS minUpdateCommitCount
+      // Calculate update commit support by dividing the update commit count by the overall commit count for both files
+     ,toFloat(firstFile.updateCommitCount)  / globalUpdateCommitCount AS firstFileUpdateSupport
+     ,toFloat(secondFile.updateCommitCount) / globalUpdateCommitCount AS secondFileUpdateSupport
+ WITH *
+      // Expected likelihood that the first and the second file change together given complete randomness
+     ,firstFileUpdateSupport * secondFileUpdateSupport AS expectedCoUpdateSupport
+ WITH firstFile
+     ,secondFile
+     ,updateCommitHashes
+     ,updateCommitCount
+     // Out of all the times the less frequently changed file was touched, how often did it co-occur with the other file?
+     ,toFloat(updateCommitCount) / minUpdateCommitCount    AS updateCommitMinConfidence
+     // Compared to all commits in general, how high is the percentage of the commits where both files changed together?
+     ,toFloat(updateCommitCount) / globalUpdateCommitCount AS updateCommitSupport
+     ,expectedCoUpdateSupport
+// Create the new relationship "CHANGED_TOGETHER_WITH" and set the property "updateCommitCount" on it
+ CALL (firstFile, secondFile, updateCommitCount, updateCommitHashes, updateCommitMinConfidence, updateCommitSupport, expectedCoUpdateSupport) {
        MERGE (firstFile)-[pairwiseChange:CHANGED_TOGETHER_WITH]-(secondFile)
-         SET pairwiseChange.commitCount  = commitCount
-            ,pairwiseChange.commitHashes = commitHashes
-       } IN TRANSACTIONS
+         SET pairwiseChange.updateCommitCount         = updateCommitCount
+            ,pairwiseChange.updateCommitHashes        = updateCommitHashes
+            ,pairwiseChange.updateCommitMinConfidence = updateCommitMinConfidence
+            ,pairwiseChange.updateCommitSupport       = updateCommitSupport
+            ,pairwiseChange.updateCommitLift          = updateCommitSupport / expectedCoUpdateSupport
+       } IN TRANSACTIONS OF 500 ROWS
 // Return one row with some statistics about the found pairs and their commit counts
-RETURN max(commitCount)                  AS maxCommitCount
-      ,avg(commitCount)                  AS avgCommitCount
-      ,percentileDisc(commitCount, 0.5)  AS percentile50CommitCount
-      ,percentileDisc(commitCount, 0.9)  AS percentile90CommitCount
-      ,percentileDisc(commitCount, 0.95) AS percentile95CommitCount
-      ,count(*)                          AS pairCount
+RETURN min(updateCommitCount)                    AS minCommitCount
+      ,max(updateCommitCount)                    AS maxCommitCount
+      ,avg(updateCommitCount)                    AS avgCommitCount
+      ,percentileDisc(updateCommitCount, 0.5)    AS percentile50CommitCount
+      ,percentileDisc(updateCommitCount, 0.9)    AS percentile90CommitCount
+      ,percentileDisc(updateCommitCount, 0.95)   AS percentile95CommitCount
+      ,min(updateCommitMinConfidence)                  AS minMinConfidence
+      ,max(updateCommitMinConfidence)                  AS maxMinConfidence
+      ,avg(updateCommitMinConfidence)                  AS avgMinConfidence
+      ,percentileDisc(updateCommitMinConfidence, 0.5)  AS percentile50MinConfidence
+      ,percentileDisc(updateCommitMinConfidence, 0.9)  AS percentile90MinConfidence
+      ,percentileDisc(updateCommitMinConfidence, 0.95) AS percentile95MinConfidence
+      ,count(*)                                        AS pairCount

cypher/GitLog/List_git_files_that_were_changed_together_with_another_file.cypher

@@ -3,6 +3,11 @@
 MATCH (firstGitFile:Git&File&!Repository)-[gitChange:CHANGED_TOGETHER_WITH]-(secondGitFile:Git&File&!Repository)
 MATCH (gitRepository:Git&Repository)-[:HAS_FILE]->(firstGitFile)
 UNWIND gitChange.commitHashes AS commitHash
-RETURN gitRepository.name + '/' + firstGitFile.relativePath AS filePath
+  WITH gitRepository.name + '/' + firstGitFile.relativePath AS filePath
       ,count(DISTINCT commitHash)                           AS commitCount
+      ,sum(firstGitFile.updateCommitCount)           AS fileUpdateCount
+  WITH *
+      // Out of all the times the file was touched, how often did it co-occur with other files?
+      ,CASE WHEN fileUpdateCount > 0 THEN toFloat(commitCount) / fileUpdateCount ELSE 0.0 END AS coChangeRate
+RETURN filePath, commitCount, coChangeRate
 ORDER BY commitCount DESC

cypher/GitLog/List_pairwise_changed_files_with_dependencies.cypher

@@ -4,36 +4,16 @@ MATCH (firstCodeFile:File)-[dependency:DEPENDS_ON]->(secondCodeFile:File)
 MATCH (firstCodeFile)-[pairwiseChange:CHANGED_TOGETHER_WITH]-(secondCodeFile)
 //De-duplicating the pairs of files isn't necessary, because the dependency relation is directed.
 //WHERE elementId(firstCodeFile) < elementId(secondCodeFile)
- WITH  firstCodeFile.fileName      AS firstFileName
-      ,secondCodeFile.fileName     AS secondFileName
+ WITH  firstCodeFile.fileName       AS firstFileName
+      ,secondCodeFile.fileName      AS secondFileName
       ,coalesce(dependency.weight, dependency.cardinality)    AS dependencyWeight
-      ,pairwiseChange.commitCount  AS commitCount
+      ,pairwiseChange.commitCount   AS commitCount
+      ,pairwiseChange.updateCommitMinConfidence AS updateCommitMinConfidence
       ,dependency.fileDistanceAsFewestChangeDirectoryCommands AS fileDistanceAsFewestChangeDirectoryCommands
 RETURN dependencyWeight
       ,commitCount
+      ,updateCommitMinConfidence
       ,fileDistanceAsFewestChangeDirectoryCommands
       // ,count(*)                    AS occurrences
       // ,collect(firstFileName + ' -> ' + secondFileName)[0..3] AS examples
-ORDER BY dependencyWeight, commitCount
-
-// MATCH (firstCodeFile:File)-[dependency:DEPENDS_ON]->(secondCodeFile:File)
-// MATCH (firstCodeFile)-[pairwiseChange:CHANGED_TOGETHER_WITH]-(secondCodeFile)
-// WHERE elementId(firstCodeFile) < elementId(secondCodeFile)
-// RETURN firstCodeFile.fileName  AS firstFileName
-//       ,secondCodeFile.fileName AS secondFileName
-//       ,dependency.weight           AS dependencyWeight
-//       ,pairwiseChange.commitCount  AS commitCount
-// ORDER BY dependencyWeight, commitCount
-
-//  MATCH (g1:!Git&File)-[relation:CHANGED_TOGETHER_WITH|DEPENDS_ON]-(g2:!Git&File) 
-//   WITH count(DISTINCT relation)   AS relatedFilesCount
-//       ,collect(DISTINCT relation) AS relations
-// UNWIND relations AS relation
-//   WITH relatedFilesCount
-//       ,coalesce(relation.commitCount, 0)                                 AS commitCount
-//       ,coalesce(relation.weight, 0)                                      AS dependencyWeight
-//       ,coalesce(relation.fileDistanceAsFewestChangeDirectoryCommands, 0) AS fileDistanceAsFewestChangeDirectoryCommands
-// RETURN dependencyWeight
-//       ,commitCount
-//       ,fileDistanceAsFewestChangeDirectoryCommands
-// ORDER BY dependencyWeight, commitCount
+ORDER BY dependencyWeight, commitCount

cypher/GitLog/Set_number_of_git_plugin_update_commits.cypher

@@ -0,0 +1,11 @@
+// Set updateCommitCount property on Git File nodes when git commits with Update modifier (detected by the plugin) are present
+
+MATCH (git_file:File&Git)<-[:UPDATES]-(:Git&Change)<-[:CONTAINS_CHANGE]-(git_commit:Git&Commit)
+WHERE git_file.deletedAt IS NULL
+ WITH git_file, count(DISTINCT git_commit.sha) AS updateCommitCount
+  SET git_file.updateCommitCount = updateCommitCount
+ WITH git_file, updateCommitCount
+MATCH (code_file:File&!Git)<-[:RESOLVES_TO]-(git_file)
+  SET code_file.updateCommitCount = updateCommitCount
+RETURN count(DISTINCT code_file)                      AS codeFileUpdates
+      ,collect(DISTINCT code_file.name)[0..4]         AS codeFileExample

jupyter/GitHistoryGeneral.ipynb

@@ -1198,6 +1198,7 @@
     "pairwise_changed_git_files = pairwise_changed_git_files.groupby(['directoryPath']).aggregate(\n",
     "    pairwiseChangeCommitCount=pd.NamedAgg(column=\"commitCount\", aggfunc=\"sum\"),\n",
     "    pairwiseChangeFileCount=pd.NamedAgg(column=\"filePath\", aggfunc=\"count\"),\n",
+    "    pairwiseChangeAverageRate=pd.NamedAgg(column=\"coChangeRate\", aggfunc=\"mean\"),\n",
     ")\n",
     "pairwise_changed_git_files.reset_index(inplace=True)\n",
     "\n",
@@ -1220,6 +1221,7 @@
     "\n",
     "pairwise_changed_git_files['pairwiseChangeCommitCount'] = pairwise_changed_git_files['pairwiseChangeCommitCount'].fillna(0).astype(int)\n",
     "pairwise_changed_git_files['pairwiseChangeFileCount'] = pairwise_changed_git_files['pairwiseChangeFileCount'].fillna(0).astype(int)\n",
+    "pairwise_changed_git_files['pairwiseChangeAverageRate'] = pairwise_changed_git_files['pairwiseChangeAverageRate'].fillna(0).astype(float)\n",
     "pairwise_changed_git_files.reset_index(inplace=True)\n",
     "\n",
     "# Debug\n",
@@ -1399,7 +1401,13 @@
     "    display(pearsonr(pairwise_changed_git_files_with_dependencies['commitCount'], pairwise_changed_git_files_with_dependencies['dependencyWeight']))\n",
     "\n",
     "    display(\"Spearman Correlation with p-value for commitCount and dependencyWeight\")\n",
-    "    display(spearmanr(pairwise_changed_git_files_with_dependencies['commitCount'], pairwise_changed_git_files_with_dependencies['dependencyWeight']))"
+    "    display(spearmanr(pairwise_changed_git_files_with_dependencies['commitCount'], pairwise_changed_git_files_with_dependencies['dependencyWeight']))\n",
+    "\n",
+    "    display(\"Pearson Correlation with p-value for updateCommitMinConfidence and dependencyWeight\")\n",
+    "    display(pearsonr(pairwise_changed_git_files_with_dependencies['updateCommitMinConfidence'], pairwise_changed_git_files_with_dependencies['dependencyWeight']))\n",
+    "\n",
+    "    display(\"Spearman Correlation with p-value for updateCommitMinConfidence and dependencyWeight\")\n",
+    "    display(spearmanr(pairwise_changed_git_files_with_dependencies['updateCommitMinConfidence'], pairwise_changed_git_files_with_dependencies['dependencyWeight']))"
    ]
   },
   {
@@ -1431,6 +1439,34 @@
     "        figure.write_image(**get_plotly_figure_write_image_settings(\"PairwiseChangedFilesVsDependencyWeight\"))"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "75264b82",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Scatter plot of all pairs of files with their min confidence (normalized update commit count) on the x axis and dependency weight on the y axis\n",
+    "\n",
+    "if pairwise_changed_git_files_with_dependencies.empty:\n",
+    "    print(\"No data to plot\")\n",
+    "else:\n",
+    "    figure = plotly_graph_objects.Figure(plotly_graph_objects.Scatter(\n",
+    "        x=pairwise_changed_git_files_with_dependencies['updateCommitMinConfidence'], \n",
+    "        y=pairwise_changed_git_files_with_dependencies['dependencyWeight'],\n",
+    "        mode='markers',\n",
+    "    ))\n",
+    "    figure.update_layout(\n",
+    "        **plotly_bar_layout_base_settings,\n",
+    "        title='Pairwise changed files: Min confidence co-change rate vs. dependency weight',\n",
+    "        xaxis_title='co-change rate (min confidence, normalized update commit count)',\n",
+    "        yaxis_title='dependency weight',\n",
+    "    )\n",
+    "    figure.show(**plotly_treemap_figure_show_settings)\n",
+    "    if is_command_line_execution():\n",
+    "        figure.write_image(**get_plotly_figure_write_image_settings(\"PairwiseChangedFilesVsDependencyWeight\"))"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "14e87aff",

scripts/importGit.sh

@@ -160,6 +160,8 @@ postGitPluginImport() {
 
   echo "importGit: Add numberOfGitCommits property to nodes with matching file names..."
   execute_cypher "${GIT_LOG_CYPHER_DIR}/Set_number_of_git_plugin_commits.cypher"
+  echo "importGit: Add updateCommitCount property to file nodes and code nodes with matching file names..."
+  execute_cypher "${GIT_LOG_CYPHER_DIR}/Set_number_of_git_plugin_update_commits.cypher"
 }
 
 postAggregatedGitLogImport() {