Calculate min confidence of two files changing together (normalized co-change count)

JohT · JohT · commit 1020280d3ff2 · 2025-08-09T21:45:14.000+02:00
diff --git a/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_code_files.cypher b/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_code_files.cypher
@@ -1,7 +1,8 @@
 // Take the already existing "CHANGED_TOGETHER_WITH" relationship between git files and apply it to resolved file nodes. Requires "Add_CHANGED_TOGETHER_WITH_relationships_to_git_files".
 
 MATCH (firstGitFile:Git&File&!Repository)-[gitChange:CHANGED_TOGETHER_WITH]-(secondGitFile:Git&File&!Repository)
-WHERE elementId(firstGitFile) < elementId(secondGitFile)
+//De-duplicating the pairs of files isn't necessary, because the dependency relation is directed.
+//WHERE elementId(firstGitFile) < elementId(secondGitFile)
 MATCH (firstGitFile)-[:RESOLVES_TO]->(firstCodeFile:File&!Git&!Repository)
 MATCH (secondGitFile)-[:RESOLVES_TO]->(secondCodeFile:File&!Git&!Repository)
  CALL (firstCodeFile, secondCodeFile, gitChange) {
diff --git a/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_git_files.cypher b/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_git_files.cypher
@@ -1,45 +1,68 @@
 // Connect git files that where changed together frequently with "CHANGED_TOGETHER_WITH"
 
-MATCH (global_git_commit:Git:Commit)
- WITH count(global_git_commit) AS globalCommitCount
+MATCH (git_commit_global:Git:Commit)-[:CONTAINS_CHANGE]->(:Git:Change)-[:UPDATES]->(git_file_global:Git:File)
+WHERE git_file_global.deletedAt IS NULL
+ WITH git_commit_global, count(DISTINCT git_file_global) AS commitfileCount
+ WITH percentileDisc(commitfileCount, 0.95) AS globalFileCountThreshold
+     ,count(git_commit_global)              AS globalUpdateCommitCount
+
 MATCH (git_commit:Git:Commit)-[:CONTAINS_CHANGE]->(git_change:Git:Change)-[:UPDATES]->(git_file:Git:File)
 MATCH (git_repository:Git&Repository)-[:HAS_FILE]->(git_file)
 WHERE git_file.deletedAt IS NULL
 // Order files to assure, that pairs of distinct files are grouped together (fileA, fileB) without (fileB, fileA)
 ORDER BY git_commit.sha, git_file.relativePath
- WITH globalCommitCount
+ WITH globalFileCountThreshold
+     ,globalUpdateCommitCount
      ,git_commit.sha             AS commitHash
      ,collect(DISTINCT git_file) AS filesInCommit
 // Limit the file count to min. 2 (changed together) and
 // max. 50 (reduce permutations, improve performance, filter out large refactorings that usually affect many files)
 WHERE size(filesInCommit) >= 2
-  AND size(filesInCommit) <= 50
+  AND size(filesInCommit) <= globalFileCountThreshold
 // Collect distinct pairwise (..., 2, 2) combinations of all files in the list
- WITH globalCommitCount
+ WITH globalFileCountThreshold
+     ,globalUpdateCommitCount
      ,commitHash
      ,apoc.coll.combinations(filesInCommit, 2, 2) AS fileCombinations
 UNWIND fileCombinations AS fileCombination
-  WITH globalCommitCount
+  WITH globalFileCountThreshold
+      ,globalUpdateCommitCount
       ,fileCombination
       ,count(DISTINCT commitHash)      AS commitCount
       ,collect(DISTINCT commitHash)    AS commitHashes
 // Filter out file pairs that where changed not very often together 
 // In detail: More than 0.1 per mille compared to overall commit count
-WHERE commitCount > globalCommitCount * 0.001 
+// WHERE commitCount > globalUpdateCommitCount * 0.001 
+ WITH *
+     // Get the lowest number of git update commits of both files (file pair) 
+     ,CASE WHEN fileCombination[0].numberOfGitUpdateCommits < fileCombination[1].numberOfGitUpdateCommits
+           THEN fileCombination[0].numberOfGitUpdateCommits
+           ELSE fileCombination[1].numberOfGitUpdateCommits
+      END AS minNumberOfGitUpdateCommits
  WITH fileCombination[0] AS firstFile
      ,fileCombination[1] AS secondFile
      ,commitCount
+     // Out of all the times the less frequently changed file was touched, how often did it co-occur with the other file?
+     ,toFloat(commitCount) / minNumberOfGitUpdateCommits AS minConfidence
      ,commitHashes
 // Create the new relationship "CHANGED_TOGETHER_WITH" and set the property "commitCount" on it
- CALL (firstFile, secondFile, commitCount, commitHashes) {
+ CALL (firstFile, secondFile, commitCount, minConfidence, commitHashes) {
        MERGE (firstFile)-[pairwiseChange:CHANGED_TOGETHER_WITH]-(secondFile)
-         SET pairwiseChange.commitCount  = commitCount
-            ,pairwiseChange.commitHashes = commitHashes
-       } IN TRANSACTIONS
+         SET pairwiseChange.commitCount   = commitCount
+            ,pairwiseChange.minConfidence = minConfidence
+            ,pairwiseChange.commitHashes  = commitHashes
+       } IN TRANSACTIONS OF 500 ROWS
 // Return one row with some statistics about the found pairs and their commit counts
-RETURN max(commitCount)                  AS maxCommitCount
-      ,avg(commitCount)                  AS avgCommitCount
-      ,percentileDisc(commitCount, 0.5)  AS percentile50CommitCount
-      ,percentileDisc(commitCount, 0.9)  AS percentile90CommitCount
-      ,percentileDisc(commitCount, 0.95) AS percentile95CommitCount
-      ,count(*)                          AS pairCount
+RETURN min(commitCount)                    AS minCommitCount
+      ,max(commitCount)                    AS maxCommitCount
+      ,avg(commitCount)                    AS avgCommitCount
+      ,percentileDisc(commitCount, 0.5)    AS percentile50CommitCount
+      ,percentileDisc(commitCount, 0.9)    AS percentile90CommitCount
+      ,percentileDisc(commitCount, 0.95)   AS percentile95CommitCount
+      ,min(minConfidence)                  AS minMinConfidence
+      ,max(minConfidence)                  AS maxMinConfidence
+      ,avg(minConfidence)                  AS avgMinConfidence
+      ,percentileDisc(minConfidence, 0.5)  AS percentile50MinConfidence
+      ,percentileDisc(minConfidence, 0.9)  AS percentile90MinConfidence
+      ,percentileDisc(minConfidence, 0.95) AS percentile95MinConfidence
+      ,count(*)                            AS pairCount
diff --git a/cypher/GitLog/List_git_files_that_were_changed_together_with_another_file.cypher b/cypher/GitLog/List_git_files_that_were_changed_together_with_another_file.cypher
@@ -3,6 +3,11 @@
 MATCH (firstGitFile:Git&File&!Repository)-[gitChange:CHANGED_TOGETHER_WITH]-(secondGitFile:Git&File&!Repository)
 MATCH (gitRepository:Git&Repository)-[:HAS_FILE]->(firstGitFile)
 UNWIND gitChange.commitHashes AS commitHash
-RETURN gitRepository.name + '/' + firstGitFile.relativePath AS filePath
+  WITH gitRepository.name + '/' + firstGitFile.relativePath AS filePath
       ,count(DISTINCT commitHash)                           AS commitCount
+      ,sum(firstGitFile.numberOfGitUpdateCommits)           AS fileUpdateCount
+  WITH *
+      // Out of all the times the file was touched, how often did it co-occur with other files?
+      ,CASE WHEN fileUpdateCount > 0 THEN toFloat(commitCount) / fileUpdateCount ELSE 0.0 END AS coChangeRate
+RETURN filePath, commitCount, coChangeRate
 ORDER BY commitCount DESC
diff --git a/cypher/GitLog/List_pairwise_changed_files_with_dependencies.cypher b/cypher/GitLog/List_pairwise_changed_files_with_dependencies.cypher
@@ -4,36 +4,16 @@ MATCH (firstCodeFile:File)-[dependency:DEPENDS_ON]->(secondCodeFile:File)
 MATCH (firstCodeFile)-[pairwiseChange:CHANGED_TOGETHER_WITH]-(secondCodeFile)
 //De-duplicating the pairs of files isn't necessary, because the dependency relation is directed.
 //WHERE elementId(firstCodeFile) < elementId(secondCodeFile)
- WITH  firstCodeFile.fileName      AS firstFileName
-      ,secondCodeFile.fileName     AS secondFileName
+ WITH  firstCodeFile.fileName       AS firstFileName
+      ,secondCodeFile.fileName      AS secondFileName
       ,coalesce(dependency.weight, dependency.cardinality)    AS dependencyWeight
-      ,pairwiseChange.commitCount  AS commitCount
+      ,pairwiseChange.commitCount   AS commitCount
+      ,pairwiseChange.minConfidence AS minConfidence
       ,dependency.fileDistanceAsFewestChangeDirectoryCommands AS fileDistanceAsFewestChangeDirectoryCommands
 RETURN dependencyWeight
       ,commitCount
+      ,minConfidence
       ,fileDistanceAsFewestChangeDirectoryCommands
       // ,count(*)                    AS occurrences
       // ,collect(firstFileName + ' -> ' + secondFileName)[0..3] AS examples
-ORDER BY dependencyWeight, commitCount
-
-// MATCH (firstCodeFile:File)-[dependency:DEPENDS_ON]->(secondCodeFile:File)
-// MATCH (firstCodeFile)-[pairwiseChange:CHANGED_TOGETHER_WITH]-(secondCodeFile)
-// WHERE elementId(firstCodeFile) < elementId(secondCodeFile)
-// RETURN firstCodeFile.fileName  AS firstFileName
-//       ,secondCodeFile.fileName AS secondFileName
-//       ,dependency.weight           AS dependencyWeight
-//       ,pairwiseChange.commitCount  AS commitCount
-// ORDER BY dependencyWeight, commitCount
-
-//  MATCH (g1:!Git&File)-[relation:CHANGED_TOGETHER_WITH|DEPENDS_ON]-(g2:!Git&File) 
-//   WITH count(DISTINCT relation)   AS relatedFilesCount
-//       ,collect(DISTINCT relation) AS relations
-// UNWIND relations AS relation
-//   WITH relatedFilesCount
-//       ,coalesce(relation.commitCount, 0)                                 AS commitCount
-//       ,coalesce(relation.weight, 0)                                      AS dependencyWeight
-//       ,coalesce(relation.fileDistanceAsFewestChangeDirectoryCommands, 0) AS fileDistanceAsFewestChangeDirectoryCommands
-// RETURN dependencyWeight
-//       ,commitCount
-//       ,fileDistanceAsFewestChangeDirectoryCommands
-// ORDER BY dependencyWeight, commitCount
+ORDER BY dependencyWeight, commitCount
diff --git a/cypher/GitLog/Set_number_of_git_plugin_update_commits.cypher b/cypher/GitLog/Set_number_of_git_plugin_update_commits.cypher
@@ -0,0 +1,10 @@
+// Set numberOfGitUpdateCommits property on Git File nodes when git commits with Update modifier (detected by the plugin) are present
+
+MATCH (git_file:File&Git)<-[:UPDATES]-(:Git&Change)<-[:CONTAINS_CHANGE]-(git_commit:Git&Commit)
+ WITH git_file, count(DISTINCT git_commit.sha) AS numberOfGitUpdateCommits
+  SET git_file.numberOfGitUpdateCommits = numberOfGitUpdateCommits
+ WITH git_file, numberOfGitUpdateCommits
+MATCH (code_file:File&!Git)<-[:RESOLVES_TO]-(git_file)
+  SET code_file.numberOfGitUpdateCommits = numberOfGitUpdateCommits
+RETURN count(DISTINCT code_file)                      AS codeFileUpdates
+      ,collect(DISTINCT code_file.name)[0..4]         AS codeFileExample
diff --git a/jupyter/GitHistoryGeneral.ipynb b/jupyter/GitHistoryGeneral.ipynb
@@ -1198,6 +1198,7 @@
     "pairwise_changed_git_files = pairwise_changed_git_files.groupby(['directoryPath']).aggregate(\n",
     "    pairwiseChangeCommitCount=pd.NamedAgg(column=\"commitCount\", aggfunc=\"sum\"),\n",
     "    pairwiseChangeFileCount=pd.NamedAgg(column=\"filePath\", aggfunc=\"count\"),\n",
+    "    pairwiseChangeAverageRate=pd.NamedAgg(column=\"coChangeRate\", aggfunc=\"mean\"),\n",
     ")\n",
     "pairwise_changed_git_files.reset_index(inplace=True)\n",
     "\n",
@@ -1220,6 +1221,7 @@
     "\n",
     "pairwise_changed_git_files['pairwiseChangeCommitCount'] = pairwise_changed_git_files['pairwiseChangeCommitCount'].fillna(0).astype(int)\n",
     "pairwise_changed_git_files['pairwiseChangeFileCount'] = pairwise_changed_git_files['pairwiseChangeFileCount'].fillna(0).astype(int)\n",
+    "pairwise_changed_git_files['pairwiseChangeAverageRate'] = pairwise_changed_git_files['pairwiseChangeAverageRate'].fillna(0).astype(float)\n",
     "pairwise_changed_git_files.reset_index(inplace=True)\n",
     "\n",
     "# Debug\n",
@@ -1399,7 +1401,13 @@
     "    display(pearsonr(pairwise_changed_git_files_with_dependencies['commitCount'], pairwise_changed_git_files_with_dependencies['dependencyWeight']))\n",
     "\n",
     "    display(\"Spearman Correlation with p-value for commitCount and dependencyWeight\")\n",
-    "    display(spearmanr(pairwise_changed_git_files_with_dependencies['commitCount'], pairwise_changed_git_files_with_dependencies['dependencyWeight']))"
+    "    display(spearmanr(pairwise_changed_git_files_with_dependencies['commitCount'], pairwise_changed_git_files_with_dependencies['dependencyWeight']))\n",
+    "\n",
+    "    display(\"Pearson Correlation with p-value for minConfidence and dependencyWeight\")\n",
+    "    display(pearsonr(pairwise_changed_git_files_with_dependencies['minConfidence'], pairwise_changed_git_files_with_dependencies['dependencyWeight']))\n",
+    "\n",
+    "    display(\"Spearman Correlation with p-value for minConfidence and dependencyWeight\")\n",
+    "    display(spearmanr(pairwise_changed_git_files_with_dependencies['minConfidence'], pairwise_changed_git_files_with_dependencies['dependencyWeight']))"
    ]
   },
   {
@@ -1431,6 +1439,34 @@
     "        figure.write_image(**get_plotly_figure_write_image_settings(\"PairwiseChangedFilesVsDependencyWeight\"))"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "75264b82",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Scatter plot of all pairs of files with their min confidence (normalized update commit count) on the x axis and dependency weight on the y axis\n",
+    "\n",
+    "if pairwise_changed_git_files_with_dependencies.empty:\n",
+    "    print(\"No data to plot\")\n",
+    "else:\n",
+    "    figure = plotly_graph_objects.Figure(plotly_graph_objects.Scatter(\n",
+    "        x=pairwise_changed_git_files_with_dependencies['minConfidence'], \n",
+    "        y=pairwise_changed_git_files_with_dependencies['dependencyWeight'],\n",
+    "        mode='markers',\n",
+    "    ))\n",
+    "    figure.update_layout(\n",
+    "        **plotly_bar_layout_base_settings,\n",
+    "        title='Pairwise changed files: Min confidence co-change rate vs. dependency weight',\n",
+    "        xaxis_title='co-change rate (min confidence, normalized update commit count)',\n",
+    "        yaxis_title='dependency weight',\n",
+    "    )\n",
+    "    figure.show(**plotly_treemap_figure_show_settings)\n",
+    "    if is_command_line_execution():\n",
+    "        figure.write_image(**get_plotly_figure_write_image_settings(\"PairwiseChangedFilesVsDependencyWeight\"))"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "14e87aff",
diff --git a/scripts/importGit.sh b/scripts/importGit.sh
@@ -160,6 +160,8 @@ postGitPluginImport() {
 
   echo "importGit: Add numberOfGitCommits property to nodes with matching file names..."
   execute_cypher "${GIT_LOG_CYPHER_DIR}/Set_number_of_git_plugin_commits.cypher"
+  echo "importGit: Add numberOfGitUpdateCommits property to file nodes and code nodes with matching file names..."
+  execute_cypher "${GIT_LOG_CYPHER_DIR}/Set_number_of_git_plugin_update_commits.cypher"
 }
 
 postAggregatedGitLogImport() {

Original file line number	Diff line number	Diff line change
`@@ -160,6 +160,8 @@ postGitPluginImport() {`
`160`	`160`
`161`	`161`	`echo "importGit: Add numberOfGitCommits property to nodes with matching file names..."`
`162`	`162`	`execute_cypher "${GIT_LOG_CYPHER_DIR}/Set_number_of_git_plugin_commits.cypher"`
	`163`	`+ echo "importGit: Add numberOfGitUpdateCommits property to file nodes and code nodes with matching file names..."`
	`164`	`+ execute_cypher "${GIT_LOG_CYPHER_DIR}/Set_number_of_git_plugin_update_commits.cypher"`
`163`	`165`	`}`
`164`	`166`
`165`	`167`	`postAggregatedGitLogImport() {`