Generate analysis results references documentation

JohT · JohT · commit 2751ae09af8f · 2025-01-26T11:43:55.000+01:00
diff --git a/.github/workflows/internal-report-reference-documentation.yml b/.github/workflows/internal-report-reference-documentation.yml
@@ -0,0 +1,77 @@
+name: Generate report reference documentation
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "analysis-results/**" # Only run on changed analysis results
+      - "!analysis-results/*.md" # Ignore report reference documentation changes (endless loop prevention)
+      - ".github/workflows/internal-report-reference-documentation.yml" # Also run when this file was changed
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "analysis-results/**" # Only run on changed analysis results
+      - "!analysis-results/*.md" # Ignore report reference documentation changes (endless loop prevention)
+      - ".github/workflows/internal-report-reference-documentation.yml" # Also run when this file was changed
+
+jobs:
+  generate-report-reference-documentation:
+    runs-on: ubuntu-latest
+    outputs:
+      generated_documents_changed: ${{ steps.set-generated_documents_changed.outputs.generated_documents_changed }}
+      documentation-upload-name: ${{ steps.set-documentation-upload-name.outputs.documentation-upload-name }}
+      
+    steps:
+      - name: Checkout git repository
+        uses: actions/checkout@v4
+
+      - name: Generate report reference document
+        working-directory: analysis-results
+        run: |
+          ./../documentation/analysis-reports-reference/generateReportReferences.sh
+
+      - name: Use git to detect changes in the regenerated documentation and set generated_documents_changed
+        working-directory: analysis-results
+        id: set-generated_documents_changed
+        run: |
+          git add -N *.md # Add all new files to the index so that git diff also works for newly generated files
+          git diff --quiet || echo "generated_documents_changed=true" >> "$GITHUB_OUTPUT"
+          git --no-pager diff --name-only
+
+      - name: Display generated_documents_changed
+        run: echo "generated_documents_changed=${{ steps.set-generated_documents_changed.outputs.generated_documents_changed }}"
+
+      - name: Generate ARTIFACT_UPLOAD_ID
+        if: steps.set-generated_documents_changed.outputs.generated_documents_changed == 'true'
+        run: echo "ARTIFACT_UPLOAD_ID=$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 10)" >> $GITHUB_ENV
+ 
+      - name: Set documentation-upload-name
+        if: steps.set-generated_documents_changed.outputs.generated_documents_changed == 'true'
+        id: set-documentation-upload-name
+        run: echo "documentation-upload-name=report-reference-documentation-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
+
+      - name: Archive generated report references documents
+        if: steps.set-generated_documents_changed.outputs.generated_documents_changed == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.set-documentation-upload-name.outputs.documentation-upload-name }}
+          path: ./analysis-results/*.md
+          if-no-files-found: error
+          retention-days: 5
+
+
+  commit-analysis-results:
+    name: Commit Analysis Results
+    needs: [generate-report-reference-documentation]
+    uses: ./.github/workflows/internal-commit-results.yml
+    if: needs.generate-report-reference-documentation.outputs.generated_documents_changed == 'true'
+    with:
+      commit-author-name: "${{ github.event.repository.name }} Continuous Integration"
+      commit-author-email: "7671054+JohT@users.noreply.github.com"
+      commit-message: "Automated code structure analysis results (CI)"
+      commit-directory: "./analysis-results/*.md"
+      uploaded-artifact-name: ${{ needs.generate-report-reference-documentation.outputs.documentation-upload-name }}
+    secrets:
+      repository-commit-token: ${{ secrets.WORKFLOW_GIT_ACCESS_TOKEN }}
diff --git a/.github/workflows/java-code-analysis.yml b/.github/workflows/java-code-analysis.yml
@@ -7,6 +7,7 @@ on:
     # Ignore changes in documentation, general configuration and analysis-results for push events
     paths-ignore: 
       - 'analysis-results/**'
+      - 'documentation/**'
       - '**/*.md'
       - '**/*.txt'
       - '**/*.css'
@@ -24,6 +25,7 @@ on:
     # Ignore changes in documentation, general configuration and analysis-results for pull request events
     paths-ignore: 
       - 'analysis-results/**'
+      - 'documentation/**'
       - '**/*.md'
       - '**/*.txt'
       - '**/*.css'
diff --git a/.github/workflows/typescript-code-analysis.yml b/.github/workflows/typescript-code-analysis.yml
@@ -7,6 +7,7 @@ on:
     # Ignore changes in documentation, general configuration and analysis-results for push events
     paths-ignore: 
       - 'analysis-results/**'
+      - 'documentation/**'
       - '**/*.md'
       - '**/*.txt'
       - '**/*.css'
@@ -24,6 +25,7 @@ on:
     # Ignore changes in documentation, general configuration and analysis-results for pull request events
     paths-ignore: 
       - 'analysis-results/**'
+      - 'documentation/**'
       - '**/*.md'
       - '**/*.txt'
       - '**/*.css'
diff --git a/COMMANDS.md b/COMMANDS.md
@@ -0,0 +1,42 @@
+# Code Graph Analysis Pipeline - Commands
+
+<!-- TOC -->
+
+- [Generate Markdown References](#generate-markdown-references)
+    - [Generate CSV Cypher Query Results Report Reference](#generate-csv-cypher-query-results-report-reference)
+    - [Generate Jupyter Notebook Report Reference](#generate-jupyter-notebook-report-reference)
+    - [Generate Image Reference](#generate-image-reference)
+
+<!-- /TOC -->
+
+## Generate Markdown References
+
+### Generate CSV Cypher Query Results Report Reference
+
+Change into the [analysis-results](./analysis-results/) directory e.g. with `cd analysis-results` and then execute the script [generateCsvReportReference.sh](./documentation/analysis-reports-reference/generateCsvReportReference.sh) with the following command:
+
+👉**Note:** This script is automatically triggered in the pipeline [internal-report-reference-documentation.yml](.github/workflows/internal-report-reference-documentation.yml) and doesn't need to be executed manually normally.
+
+```script
+./../documentation/analysis-reports-reference/generateCsvReportReference.sh
+```
+
+### Generate Jupyter Notebook Report Reference
+
+Change into the [analysis-results](./analysis-results/) directory e.g. with `cd analysis-results` and then execute the script [generateJupyterReportReference.sh](./documentation/analysis-reports-reference/generateJupyterReportReference.sh) with the following command:
+
+👉**Note:** This script is automatically triggered in the pipeline [internal-report-reference-documentation.yml](.github/workflows/internal-report-reference-documentation.yml) and doesn't need to be executed manually normally.
+
+```script
+./../documentation/analysis-reports-reference/generateJupyterReportReference.sh
+```
+
+### Generate Image Reference
+
+Change into the [analysis-results](./analysis-results/) directory e.g. with `cd analysis-results` and then execute the script [generateImageReference.sh](./documentation/analysis-reports-reference/generateImageReference.sh) with the following command:
+
+👉**Note:** This script is automatically triggered in the pipeline [internal-report-reference-documentation.yml](.github/workflows/internal-report-reference-documentation.yml) and doesn't need to be executed manually normally.
+
+```script
+./../documentation/analysis-reports-reference/generateImageReference.sh
+```
diff --git a/README.md b/README.md
@@ -83,6 +83,18 @@ Here are ten examples from over a hundred reports generated by the analysis. The
 
 <img src="./analysis-results/AxonFramework/AxonFramework-4.10.3/wordcloud/Wordcloud_files/Wordcloud_17_0.png" width="600" alt="Word cloud of git authors">
 
+## :page_with_curl: CSV Cypher Query Report Reference
+
+[CSV_REPORTS.md](./analysis-results/CSV_REPORTS.md) lists all CSV Cypher query result reports inside the [results](./results) directory. It can be generated as described in [Generate CSV Report Reference](./COMMANDS.md#generate-csv-cypher-query-report-reference).
+
+## :page_with_curl: Jupyter Notebook Report Reference
+
+[JUPYTER_REPORTS.md](./analysis-results/JUPYTER_REPORTS.md) lists all Jupyter Notebook reports inside the [results](./results) directory. It can be generated as described in [Generate Jupyter Notebook Report Reference](./COMMANDS.md#generate-jupyter-notebook-report-reference).
+
+## :camera: Image Reference
+
+[IMAGES.md](./analysis-results/IMAGES.md) lists all PNG images inside the [results](./results) directory. It can be generated as described in [Generate Image Reference](./COMMANDS.md#generate-image-reference).
+
 ## Keeping the Analysis Workflow Updated with Renovate
 
 This repository uses [Renovate](https://docs.renovatebot.com) to automatically update the analysis workflow to the latest version. To enable this, add the following extension to your Renovate configuration:
diff --git a/documentation/analysis-reports-reference/generateCsvReportReference.sh b/documentation/analysis-reports-reference/generateCsvReportReference.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+# Generates "CSV_REPORTS.md" containing a reference to all CSV cypher query reports in this directory and its subdirectories.
+
+# Note: This script was generated by Chat-GPT after some messages back and forth:
+# https://chat.openai.com/share/0bd3cde7-32d0-460d-830c-79b7d00a2492
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+# Output Markdown file name
+markdown_file="CSV_REPORTS.md"
+
+# Function to count rows in a CSV file
+count_rows() {
+  wc -l < "$1"
+}
+
+# Function to extract the source query name from CSV header
+get_source_query() {
+  header=$(head -n 1 "$1")
+  source_query=$(echo "$header" | sed -n 's/.*Source Cypher File: \(.*\)"/\1/p')
+  echo "$source_query"
+}
+
+echo "generateCsvReportReference: Generating ${markdown_file}..."
+
+# Create the Markdown header
+{
+  echo "# CSV Cypher Query Result Reports Reference"
+  echo ""
+  echo "This document lists all CSV Cypher query result reports found in the current directory and its subdirectories."
+  echo "Each entry in the table below includes the file name, the analysis type, the number of rows in the CSV, and the source Cypher query."
+  echo "This file was automatically generated using the [generateCsvReportReference](./../documentation/analysis-reports-reference/generateCsvReportReference.sh) script."
+  echo ""
+  # Create the Markdown table header
+  echo "| CSV File | Analysis | Number of Rows | Source Query |"
+  echo "| -------- | -------- | -------------- | ------------ |"
+} > "$markdown_file"
+
+# Find and process CSV files
+find . -type f -name "*.csv" | sort | while IFS= read -r csv_file; do
+  num_rows=$(count_rows "$csv_file")
+  source_query=$(get_source_query "$csv_file")
+  
+  # Get the main directory name
+  main_dir=$(dirname "$csv_file" | cut -d '/' -f 3)
+  
+  # Get the base name of the file
+  base_name=$(basename "$csv_file")
+  
+  # Escape special characters for Markdown
+  csv_link=$(echo "$csv_file" | sed 's/\[/\\[/g; s/\]/\\]/g')
+  
+  # Create a link to the source query
+  source_query_link="./../cypher/$source_query"
+  
+  # Append a new row to the Markdown table
+  echo "| [$base_name]($csv_link) | $main_dir | $num_rows | [$source_query]($source_query_link) |" >> "$markdown_file"
+done
+
+echo "generateCsvReportReference: Successfully generated ${markdown_file}."
diff --git a/documentation/analysis-reports-reference/generateImageReference.sh b/documentation/analysis-reports-reference/generateImageReference.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Generates "IMAGES.md" containing a reference to all images (PNG) in this directory and its subdirectories.
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+# Markdown file name
+markdown_file="IMAGES.md"
+
+echo "generateImageReference: Generating ${markdown_file}..."
+
+{ 
+  echo "# Image Reference" 
+  echo ""
+  echo "This document serves as a reference for all images (PNG) in the current directory and its subdirectories."
+  echo "It provides a table listing each file and the analysis it belongs to."
+  echo "This file was generated with the script [generateImageReference.sh](./../documentation/analysis-reports-reference/generateImageReference.sh)."
+  echo ""
+  echo "Image  | Analysis |"
+  echo "-------|----------|" 
+} > ${markdown_file}
+
+# Loop through all Markdown files in the current directory
+find . -type f -name "*.png" | sort | while read -r image_file; do
+    # Trim leading and trailing whitespace
+    description=$(echo "${description}" | awk '{$1=$1;print}')
+
+    # Extract the script file name without the path
+    filename=$(basename "$image_file")
+    
+    if [ "$filename" = "${markdown_file}" ]; then
+      continue
+    fi
+    
+    # Extract the script file path without the name
+    pathname=$(dirname "$image_file")
+    
+    # Extract the second part of the path after ./ that contains the analysis directory name
+    analysisname=$(echo "${pathname}" | cut -d/ -f 3)
+
+    # Create a link to the script file in the table
+    link="[${filename}](${image_file})"
+    
+    # Add the script file and its description to the Markdown table
+    echo "| ${link} | ${analysisname%%.} |" >> ${markdown_file}
+done
+
+echo "generateImageReference: Successfully generated ${markdown_file}."
diff --git a/documentation/analysis-reports-reference/generateJupyterReportReference.sh b/documentation/analysis-reports-reference/generateJupyterReportReference.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+# Generates "JUPYTER_REPORTS.md" containing a reference to all Jupyter Notebook Markdown reports in this directory and its subdirectories.
+# This script was generated by Chat-GPT after some messages back and forth and then tuned manually.
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+# Markdown file name
+markdown_file="JUPYTER_REPORTS.md"
+
+echo "generateJupyterReportReference: Generating ${markdown_file}..."
+
+{ 
+  echo "# Jupyter Notebook Reports Reference" 
+  echo ""
+  echo "This document serves as a reference for all Jupyter Notebook reports in the current directory and its subdirectories."
+  echo "It provides a table listing each file and its corresponding description found in the first header line."
+  echo "This file was generated with the script [generateJupyterReportReference.sh](./../documentation/analysis-reports-reference/generateJupyterReportReference.sh)."
+  echo ""
+  echo "Report | Analysis | Description"
+  echo "-------|----------|------------" 
+} > ${markdown_file}
+
+# Loop through all Markdown files in the current directory
+find . -type f -name "*.md" | sort | while read -r report_file; do
+    # Extract the first non-empty line that starts with '#'
+    report_file_header_line=$(grep -m 1 -e '^#\+.*' "${report_file}")
+
+    # Remove leading '#' characters and trim leading/trailing spaces
+    description=$(echo "${report_file_header_line}" | sed -E 's/^#+\s*//')
+    
+    # Trim leading and trailing whitespace
+    description=$(echo "${description}" | awk '{$1=$1;print}')
+
+    # Extract the script file name without the path
+    filename=$(basename "$report_file")
+    
+    if [ "$filename" = "${markdown_file}" ]; then
+      continue
+    fi
+    
+    # Extract the script file path without the name
+    pathname=$(dirname "$report_file")
+    
+    # Extract the second part of the path after ./ that contains the analysis directory name
+    analysisname=$(echo "${pathname}" | cut -d/ -f 3)
+
+    # Create a link to the script file in the table
+    link="[${filename}](${report_file})"
+    
+    # Add the script file and its description to the Markdown table
+    echo "| ${link} | ${analysisname%%.} | ${description} |" >> ${markdown_file}
+done
+
+echo "generateJupyterReportReference: Successfully generated ${markdown_file}."
diff --git a/documentation/analysis-reports-reference/generateReportReferences.sh b/documentation/analysis-reports-reference/generateReportReferences.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Triggers the regeneration of all reference documentations for the reports inside the "results" directory.
+# The generation to the reference documentations is delegated to the dedicated scripts.
+
+# Notice that this scripts needs to be executed within the "results" directory.
+
+# Requires generateJupyterReportReference.sh, generateCsvReportReference.sh, generateImageReference.sh
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+## Get this "scripts" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+DOCUMENTATION_SCRIPT_DIR=${DOCUMENTATION_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
+echo "generateReportReferences: SCRIPTS_DIR=${DOCUMENTATION_SCRIPT_DIR}"
+
+# Generate JUPYTER_REPORTS.md containing a reference to all Jupyter Notebook Markdown reports in the "results" directory and its subdirectories.
+source "${DOCUMENTATION_SCRIPT_DIR}/generateJupyterReportReference.sh"
+
+# Generate CSV_REPORTS.md containing a reference to all CSV cypher query reports in the "results" directory and its subdirectories.
+source "${DOCUMENTATION_SCRIPT_DIR}/generateCsvReportReference.sh"
+
+# Generate IMAGES.md containing a reference to all PNG images in the "results" directory and its subdirectories.
+source "${DOCUMENTATION_SCRIPT_DIR}/generateImageReference.sh"
