Merge pull request #398 from JohT/feature/anomaly-detection

Anomaly Detection

Author: JohT

.gitignore

@@ -93,5 +93,11 @@ coverage/
 .ipynb_checkpoints
 *.nbconvert*
 
+# Python
+__pycache__/
+
 # Python environments
-.conda
+.conda
+
+# Optuna (and other) Database data
+*.db

COMMANDS.md

@@ -8,6 +8,7 @@
     - [Examples](#examples)
         - [Start an analysis with CSV reports only](#start-an-analysis-with-csv-reports-only)
         - [Start an analysis with Jupyter reports only](#start-an-analysis-with-jupyter-reports-only)
+        - [Start an analysis with Python reports only](#start-an-analysis-with-python-reports-only)
         - [Start an analysis with PDF generation](#start-an-analysis-with-pdf-generation)
         - [Start an analysis without importing git log data](#start-an-analysis-without-importing-git-log-data)
         - [Only run setup and explore the Graph manually](#only-run-setup-and-explore-the-graph-manually)
@@ -102,6 +103,14 @@ If only the Jupyter reports are needed e.g. when the CSV reports had already bee
 ./../../scripts/analysis/analyze.sh --report Jupyter
 ```
 
+#### Start an analysis with Python reports only
+
+If you only need Python reports, e.g. to skip Chromium Browser dependency, the this can be done with:
+
+```shell
+./../../scripts/analysis/analyze.sh --report Python
+```
+
 #### Start an analysis with PDF generation
 
 Note: Generating a PDF from a Jupyter notebook using [nbconvert](https://nbconvert.readthedocs.io) takes some time and might even fail due to a timeout error.

GETTING_STARTED.md

@@ -84,16 +84,22 @@ Use these optional command line options as needed:
   ./../../scripts/analysis/analyze.sh --report Csv
   ```
 
-- Jupyter notebook reports when Python and Conda are installed:
+- Jupyter notebook reports when Python and Conda are installed (and Chromium Browser for PDF generation):
 
   ```shell
   ./../../scripts/analysis/analyze.sh --report Jupyter
   ```
 
+- Python reports when Python and Conda are installed (without Chromium Browser for PDF generation):
+
+  ```shell
+  ./../../scripts/analysis/analyze.sh --report Python
+  ```
+
 - Graph visualizations when Node.js and npm are installed:
 
   ```shell
-  ./../../scripts/analysis/analyze.sh --report Jupyter
+  ./../../scripts/analysis/analyze.sh --report Visualization
   ```
 
 - All reports with Python, Conda, Node.js and npm installed:

README.md

@@ -17,6 +17,7 @@ Contained within this repository is a comprehensive and automated code graph ana
 - Easily integrable into your [continuous integration pipeline](./INTEGRATION.md)
 - More than 130 CSV reports for dependencies, metrics, cycles, annotations, algorithms and many more
 - Jupyter notebook reports for dependencies, metrics, visibility and many more
+- Anomaly detection powered by unsupervised machine learning and explainable AI
 - Graph structure visualization
 - Automated reference document generation
 - Runtime and library independent automation using [shell scripts](./scripts/SCRIPTS.md)
@@ -27,6 +28,7 @@ Contained within this repository is a comprehensive and automated code graph ana
 
 ### :newspaper: News
 
+- August 2025: Anomaly detection powered by unsupervised machine learning and explainable AI
 - May 2025: Migrated to [Neo4j 2025.x](https://neo4j.com/docs/upgrade-migration-guide/current/version-2025/upgrade) and Java 21.
 
 ### :notebook: Jupyter Notebook Reports
@@ -148,6 +150,10 @@ The [Code Structure Analysis Pipeline](./.github/workflows/internal-java-code-an
   - [Neo4j Python Driver](https://neo4j.com/docs/api/python-driver)
   - [openTSNE](https://github.com/pavlin-policar/openTSNE)
   - [wordcloud](https://github.com/amueller/word_cloud)
+  - [umap](https://umap-learn.readthedocs.io)
+  - [scikit-learn](https://scikit-learn.org)
+  - [optuna](https://optuna.org)
+  - [SHAP](https://github.com/shap/shap)
 - [Graph Visualization](./graph-visualization/README.md) uses [node.js](https://nodejs.org/de) and the dependencies listed in [package.json](./graph-visualization/package.json).
 - [HPCC-Systems (High Performance Computing Cluster) Web-Assembly (JavaScript)](https://github.com/hpcc-systems/hpcc-js-wasm) containing a wrapper for GraphViz to visualize graph structures.
 - [GraphViz](https://gitlab.com/graphviz/graphviz) for CLI Graph Visualization

cypher/Community_Detection/Community_Detection_2b_Leiden_Tuneable_Statistics.cypher

@@ -0,0 +1,31 @@
+//Community Detection Leiden Statistics
+
+CALL gds.leiden.stats(
+ $dependencies_projection + '-cleaned', {
+  gamma: toFloat($dependencies_leiden_gamma),
+  theta: toFloat($dependencies_leiden_theta),
+  maxLevels: toInteger($dependencies_leiden_max_levels),
+  tolerance: 0.0000001,
+  consecutiveIds: true,
+  relationshipWeightProperty: $dependencies_projection_weight_property
+})
+YIELD nodeCount
+     ,communityCount
+     ,ranLevels
+     ,modularity
+     ,modularities
+     ,communityDistribution
+RETURN nodeCount
+      ,communityCount
+      ,ranLevels
+      ,modularity
+      ,modularities
+      ,communityDistribution.min
+      ,communityDistribution.mean
+      ,communityDistribution.max
+      ,communityDistribution.p50
+      ,communityDistribution.p75
+      ,communityDistribution.p90
+      ,communityDistribution.p95
+      ,communityDistribution.p99
+      ,communityDistribution.p999

cypher/Community_Detection/Community_Detection_2d_Leiden_Tuneable_Write.cypher

@@ -0,0 +1,40 @@
+//Community Detection Leiden Write property communityLeidenId
+
+CALL gds.leiden.write(
+ $dependencies_projection + '-cleaned', {
+  gamma: toFloat($dependencies_leiden_gamma),
+  theta: toFloat($dependencies_leiden_theta),
+  maxLevels: toInteger($dependencies_leiden_max_levels),
+  tolerance: 0.0000001,
+  consecutiveIds: true,
+  relationshipWeightProperty: $dependencies_projection_weight_property,
+  writeProperty: $dependencies_projection_write_property
+})
+YIELD preProcessingMillis
+     ,computeMillis
+     ,writeMillis
+     ,postProcessingMillis
+     ,nodePropertiesWritten
+     ,communityCount
+     ,ranLevels
+     ,modularity
+     ,modularities
+     ,communityDistribution
+RETURN preProcessingMillis
+      ,computeMillis
+      ,writeMillis
+      ,postProcessingMillis
+      ,nodePropertiesWritten
+      ,communityCount
+      ,ranLevels
+      ,modularity
+      ,communityDistribution.min
+      ,communityDistribution.mean
+      ,communityDistribution.max
+      ,communityDistribution.p50
+      ,communityDistribution.p75
+      ,communityDistribution.p90
+      ,communityDistribution.p95
+      ,communityDistribution.p99
+      ,communityDistribution.p999
+      ,modularities

cypher/Dependencies_Projection/Dependencies_13_Sample_Projected_Graph.cypher

@@ -0,0 +1,11 @@
+// Creates a smaller projection by sampling the original graph using "Common Neighbour Aware Random Walk"
+
+CALL gds.graph.sample.cnarw(
+  $dependencies_projection + '-sampled-cleaned',
+  $dependencies_projection,
+  {
+    samplingRatio: toFloat($dependencies_projection_sampling_ratio)
+  }
+)
+YIELD graphName, fromGraphName, nodeCount, relationshipCount, startNodeCount, projectMillis
+RETURN graphName, fromGraphName, nodeCount, relationshipCount, startNodeCount, projectMillis

cypher/Dependencies_Projection/Dependencies_14_Write_Batch_Data.cypher

@@ -0,0 +1,7 @@
+// Writes batch data back into the database for code units when working with a dependencies projection. Variables: dependencies_projection_rows, dependencies_projection_node
+
+UNWIND $dependencies_projection_rows AS row
+MATCH (codeUnit)
+WHERE elementId(codeUnit) = row.nodeId
+  AND $dependencies_projection_node IN labels(codeUnit) 
+  SET codeUnit += row.properties

cypher/Exploration/Explore_node_properties_non_null.cypher

@@ -0,0 +1,16 @@
+// Explore non null node property counts for the selected node label. Variables: projection_node_label
+
+  MATCH (selectedNode)
+  WHERE $projection_node_label IN labels(selectedNode)
+   WITH *, keys(selectedNode) AS nodeProperties
+ UNWIND nodeProperties    AS nodeProperty
+   WITH *
+ ORDER BY nodeProperty
+  WHERE selectedNode[nodeProperty] IS NOT NULL
+   WITH  nodeProperty, count(*) AS nonNullCount
+// RETURN nodeProperty, nonNullCount 
+// ORDER BY nodeProperty, nonNullCount
+ RETURN nonNullCount
+       ,count(DISTINCT nodeProperty) AS propertyCount
+       ,collect(nodeProperty)        AS properties
+ORDER BY nonNullCount DESC

cypher/Node_Embeddings/Node_Embeddings_1d_Fast_Random_Projection_Stream.cypher

@@ -16,8 +16,9 @@ OPTIONAL MATCH (projectRoot:Directory)<-[:HAS_ROOT]-(proj:TS:Project)-[:CONTAINS
    WITH *, last(split(projectRoot.absoluteFileName, '/')) AS projectName   
  RETURN DISTINCT 
         coalesce(codeUnit.fqn, codeUnit.globalFqn, codeUnit.fileName, codeUnit.signature, codeUnit.name) AS codeUnitName
-       ,codeUnit.name AS shortCodeUnitName
-       ,coalesce(artifactName, projectName) AS projectName
-       ,coalesce(codeUnit.communityLeidenId, 0) AS communityId
+       ,codeUnit.name                               AS shortCodeUnitName
+       ,elementId(codeUnit)                         AS nodeElementId
+       ,coalesce(artifactName, projectName)         AS projectName
+       ,coalesce(codeUnit.communityLeidenId, 0)     AS communityId
        ,coalesce(codeUnit.centralityPageRank, 0.01) AS centrality
        ,embedding