Introduce anomaly detection pipeline with tuned clustering

Author: JohT

domains/anomaly-detection/anomalyDetectionPipeline.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+# Pipeline that coordinates anomaly detection using the Graph Data Science Library of Neo4j.
+# It requires an already running Neo4j graph database with already scanned and analyzed artifacts.
+# The results will be written into the sub directory reports/anomaly-detection.
+
+# Note that "scripts/prepareAnalysis.sh" is required to run prior to this script.
+
+# Requires executeQueryFunctions.sh, projectionFunctions.sh, cleanupAfterReportGeneration.sh
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+# Overrideable Constants (defaults also defined in sub scripts)
+REPORTS_DIRECTORY=${REPORTS_DIRECTORY:-"reports"}
+
+## Get this "scripts/reports" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution.
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+ANOMALY_DETECTION_SCRIPT_DIR=${ANOMALY_DETECTION_SCRIPT_DIR:-$(CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)}
+echo "anomalyDetectionPipeline: ANOMALY_DETECTION_SCRIPT_DIR=${ANOMALY_DETECTION_SCRIPT_DIR}"
+# Get the "scripts" directory by taking the path of this script and going one directory up.
+SCRIPTS_DIR=${SCRIPTS_DIR:-"${ANOMALY_DETECTION_SCRIPT_DIR}/../../scripts"} # Repository directory containing the shell scripts
+# Get the "cypher" directory by taking the path of this script and going two directory up and then to "cypher".
+CYPHER_DIR=${CYPHER_DIR:-"${ANOMALY_DETECTION_SCRIPT_DIR}/../../cypher"}
+
+# Function to display script usage
+usage() {
+  echo -e "${COLOR_ERROR}" >&2
+  echo "Usage: $0 [--verbose]" >&2
+  echo -e "${COLOR_DEFAULT}" >&2
+  exit 1
+}
+
+# Default values
+verboseMode="" # either "" or "--verbose"
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+  key="$1"
+  value="${2}"
+
+  case ${key} in
+    --verbose)
+      verboseMode="--verbose"
+      ;;
+    *)
+      echo -e "${COLOR_ERROR}anomalyDetectionPipeline: Error: Unknown option: ${key}${COLOR_DEFAULT}" >&2
+      usage
+      ;;
+  esac
+  shift || true # ignore error when there are no more arguments
+done
+
+# Define functions to execute a cypher query from within the given file (first and only argument)
+source "${SCRIPTS_DIR}/executeQueryFunctions.sh"
+
+# Define functions to create and delete Graph Projections like "createUndirectedDependencyProjection"
+source "${SCRIPTS_DIR}/projectionFunctions.sh"
+
+# Create report directory
+REPORT_NAME="anomaly-detection"
+FULL_REPORT_DIRECTORY="${REPORTS_DIRECTORY}/${REPORT_NAME}"
+mkdir -p "${FULL_REPORT_DIRECTORY}"
+
+# Query Parameter Keys
+CREATE_PROJECTION="dependencies_projection"
+READ_PROJECTION="projection_name"
+
+PROJECTION_NODE="dependencies_projection_node"
+ALGORITHM_NODE="projection_node_label"
+
+PROJECTION_WEIGHT="dependencies_projection_weight_property"
+ALGORITHM_WEIGHT="projection_weight_property"
+
+# Code independent parameters
+LEIDEN_COMMUNITY_WRITE_PROPERTY="projection_write_property=communityLeidenIdTuned"
+
+# -- Java Artifact Node Embeddings -------------------------------
+
+if createUndirectedDependencyProjection "${CREATE_PROJECTION}=artifact-embeddings" "${PROJECTION_NODE}=Artifact" "${PROJECTION_WEIGHT}=weight"; then
+    time "${ANOMALY_DETECTION_SCRIPT_DIR}/tunedLeidenCommunityDetection.py" "${READ_PROJECTION}=artifact-embeddings" "${ALGORITHM_NODE}=Artifact" "${ALGORITHM_WEIGHT}=weight" "${LEIDEN_COMMUNITY_WRITE_PROPERTY}" ${verboseMode}
+    time "${ANOMALY_DETECTION_SCRIPT_DIR}/tunedNodeEmbeddingClustering.py" "${READ_PROJECTION}=artifact-embeddings" "${ALGORITHM_NODE}=Artifact" "${ALGORITHM_WEIGHT}=weight" ${verboseMode}
+fi
+
+# -- Java Package Node Embeddings --------------------------------
+
+if createUndirectedDependencyProjection "${CREATE_PROJECTION}=package-embeddings" "${PROJECTION_NODE}=Package" "${PROJECTION_WEIGHT}=weight25PercentInterfaces"; then
+    time "${ANOMALY_DETECTION_SCRIPT_DIR}/tunedLeidenCommunityDetection.py" "${READ_PROJECTION}=package-embeddings" "${ALGORITHM_NODE}=Package" "${ALGORITHM_WEIGHT}=weight25PercentInterfaces" "${LEIDEN_COMMUNITY_WRITE_PROPERTY}" ${verboseMode}
+    time "${ANOMALY_DETECTION_SCRIPT_DIR}/tunedNodeEmbeddingClustering.py" "${READ_PROJECTION}=package-embeddings" "${ALGORITHM_NODE}=Package" "${ALGORITHM_WEIGHT}=weight25PercentInterfaces" ${verboseMode}
+fi
+
+# -- Java Type Node Embeddings -----------------------------------
+
+if createUndirectedJavaTypeDependencyProjection "${CREATE_PROJECTION}=type-embeddings"; then
+    time "${ANOMALY_DETECTION_SCRIPT_DIR}/tunedLeidenCommunityDetection.py" "${READ_PROJECTION}=type-embeddings" "${ALGORITHM_NODE}=Type" "${ALGORITHM_WEIGHT}=weight" "${LEIDEN_COMMUNITY_WRITE_PROPERTY}" ${verboseMode}
+    time "${ANOMALY_DETECTION_SCRIPT_DIR}/tunedNodeEmbeddingClustering.py" "${READ_PROJECTION}=type-embeddings" "${ALGORITHM_NODE}=Type" "${ALGORITHM_WEIGHT}=weight" ${verboseMode}
+fi
+
+# -- Typescript Module Node Embeddings ---------------------------
+
+if createUndirectedDependencyProjection "${CREATE_PROJECTION}=typescript-module-embedding" "${PROJECTION_NODE}=Module" "${PROJECTION_WEIGHT}=lowCouplingElement25PercentWeight"; then
+    time "${ANOMALY_DETECTION_SCRIPT_DIR}/tunedLeidenCommunityDetection.py" "${READ_PROJECTION}=typescript-module-embedding" "${ALGORITHM_NODE}=Module" "${ALGORITHM_WEIGHT}=lowCouplingElement25PercentWeight" "${LEIDEN_COMMUNITY_WRITE_PROPERTY}" ${verboseMode}
+    time "${ANOMALY_DETECTION_SCRIPT_DIR}/tunedNodeEmbeddingClustering.py" "${READ_PROJECTION}=typescript-module-embedding" "${ALGORITHM_NODE}=Module" "${ALGORITHM_WEIGHT}=lowCouplingElement25PercentWeight" ${verboseMode}
+fi
+
+# ---------------------------------------------------------------
+
+# Clean-up after report generation. Empty reports will be deleted.
+source "${SCRIPTS_DIR}/cleanupAfterReportGeneration.sh" "${FULL_REPORT_DIRECTORY}"
+
+echo "anomalyDetectionPipeline: $(date +'%Y-%m-%dT%H:%M:%S%z') Successfully finished."