Generate analysis results references documentation

Author: JohT

.github/documentation/analysis-reports-reference/generateCsvReportReference.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Generates "CSV_REPORTS.md" containing a reference to all CSV cypher query reports in this directory and its subdirectories.
+
+# Note: This script was generated by Chat-GPT after some messages back and forth:
+# https://chat.openai.com/share/0bd3cde7-32d0-460d-830c-79b7d00a2492
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+# Output Markdown file name
+markdown_file="CSV_REPORTS.md"
+
+# Function to count rows in a CSV file
+count_rows() {
+  wc -l < "$1"
+}
+
+# Function to extract the source query name from CSV header
+get_source_query() {
+  header=$(head -n 1 "$1")
+  source_query=$(echo "$header" | sed -n 's/.*Source Cypher File: \(.*\)"/\1/p')
+  echo "$source_query"
+}
+
+echo "generateCsvReportReference: Generating ${markdown_file}..."
+
+# Create the Markdown header
+{
+  echo "# CSV Cypher Query Reports Reference"
+  echo ""
+  echo "This document serves as a reference for all CSV Cypher query reports in the current directory and its subdirectories. It provides a table listing each file, its corresponding analysis, the row count and the source query. This file was generated with the script [generateCsvReportsReference](./../scripts/documentation/generateCsvReportsReference.sh)."
+  echo ""
+  
+  # Create the Markdown table header
+  echo "| CSV File | Analysis | Number of Rows | Source Query |"
+  echo "| -------- | -------- | -------------- | ------------ |"
+} > "$markdown_file"
+
+# Find and process CSV files
+find . -type f -name "*.csv" | sort | while IFS= read -r csv_file; do
+  num_rows=$(count_rows "$csv_file")
+  source_query=$(get_source_query "$csv_file")
+  
+  # Get the main directory name
+  main_dir=$(dirname "$csv_file" | cut -d '/' -f 2)
+  
+  # Get the base name of the file
+  base_name=$(basename "$csv_file")
+  
+  # Escape special characters for Markdown
+  csv_link=$(echo "$csv_file" | sed 's/\[/\\[/g; s/\]/\\]/g')
+  
+  # Create a link to the source query
+  source_query_link="./../cypher/$source_query"
+  
+  # Append a new row to the Markdown table
+  echo "| [$base_name]($csv_link) | $main_dir | $num_rows | [$source_query]($source_query_link) |" >> "$markdown_file"
+done
+
+echo "generateCsvReportReference: Successfully generated ${markdown_file}."

.github/documentation/analysis-reports-reference/generateImageReference.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Generates "IMAGES.md" containing a reference to all images (PNG) in this directory and its subdirectories.
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+# Markdown file name
+markdown_file="IMAGES.md"
+
+echo "generateImageReference: Generating ${markdown_file}..."
+
+{ 
+  echo "# Image Reference" 
+  echo ""
+  echo "This document serves as a reference for all images (PNG) in the current directory and its subdirectories."
+  echo "It provides a table listing each file and the analysis it belongs to."
+  echo "This file was generated with the script [generateImageReference.sh](./../scripts/documentation/generateImageReference.sh)."
+  echo ""
+  echo "Image  | Analysis |"
+  echo "-------|----------|" 
+} > ${markdown_file}
+
+# Loop through all Markdown files in the current directory
+find . -type f -name "*.png" | sort | while read -r image_file; do
+    # Trim leading and trailing whitespace
+    description=$(echo "${description}" | awk '{$1=$1;print}')
+
+    # Extract the script file name without the path
+    filename=$(basename "$image_file")
+    
+    if [ "$filename" = "${markdown_file}" ]; then
+      continue
+    fi
+    
+    # Extract the script file path without the name
+    pathname=$(dirname "$image_file")
+    
+    # Extract the second part of the path after ./ that contains the analysis directory name
+    analysisname=$(echo "${pathname}" | cut -d/ -f2)
+
+    # Create a link to the script file in the table
+    link="[${filename}](${image_file})"
+    
+    # Add the script file and its description to the Markdown table
+    echo "| ${link} | ${analysisname%%.} |" >> ${markdown_file}
+done
+
+echo "generateImageReference: Successfully generated ${markdown_file}."

.github/documentation/analysis-reports-reference/generateJupyterReportReference.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+# Generates "JUPYTER_REPORTS.md" containing a reference to all Jupyter Notebook Markdown reports in this directory and its subdirectories.
+# This script was generated by Chat-GPT after some messages back and forth and then tuned manually.
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+# Markdown file name
+markdown_file="JUPYTER_REPORTS.md"
+
+echo "generateJupyterReportReference: Generating ${markdown_file}..."
+
+{ 
+  echo "# Jupyter Notebook Reports Reference" 
+  echo ""
+  echo "This document serves as a reference for all Jupyter Notebook reports in the current directory and its subdirectories."
+  echo "It provides a table listing each file and its corresponding description found in the first header line."
+  echo "This file was generated with the script [generateMarkdownReference.sh](./../scripts/documentation/generateMarkdownReference.sh)."
+  echo ""
+  echo "Report | Analysis | Description"
+  echo "-------|----------|------------" 
+} > ${markdown_file}
+
+# Loop through all Markdown files in the current directory
+find . -type f -name "*.md" | sort | while read -r report_file; do
+    # Extract the first non-empty line that starts with '#'
+    report_file_header_line=$(grep -m 1 -e '^#\+.*' "${report_file}")
+
+    # Remove leading '#' characters and trim leading/trailing spaces
+    description=$(echo "${report_file_header_line}" | sed -E 's/^#+\s*//')
+    
+    # Trim leading and trailing whitespace
+    description=$(echo "${description}" | awk '{$1=$1;print}')
+
+    # Extract the script file name without the path
+    filename=$(basename "$report_file")
+    
+    if [ "$filename" = "${markdown_file}" ]; then
+      continue
+    fi
+    
+    # Extract the script file path without the name
+    pathname=$(dirname "$report_file")
+    
+    # Extract the second part of the path after ./ that contains the analysis directory name
+    analysisname=$(echo "${pathname}" | cut -d/ -f2)
+
+    # Create a link to the script file in the table
+    link="[${filename}](${report_file})"
+    
+    # Add the script file and its description to the Markdown table
+    echo "| ${link} | ${analysisname%%.} | ${description} |" >> ${markdown_file}
+done
+
+echo "generateJupyterReportReference: Successfully generated ${markdown_file}."

.github/documentation/analysis-reports-reference/generateReportReferences.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Triggers the regeneration of all reference documentations for the reports inside the "results" directory.
+# The generation to the reference documentations is delegated to the dedicated scripts.
+
+# Notice that this scripts needs to be executed within the "results" directory.
+
+# Requires generateJupyterReportReference.sh, generateCsvReportReference.sh, generateImageReference.sh
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+## Get this "scripts" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+DOCUMENTATION_SCRIPT_DIR=${DOCUMENTATION_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
+echo "generateReportReferences: SCRIPTS_DIR=${DOCUMENTATION_SCRIPT_DIR}"
+
+# Generate JUPYTER_REPORTS.md containing a reference to all Jupyter Notebook Markdown reports in the "results" directory and its subdirectories.
+source "${DOCUMENTATION_SCRIPT_DIR}/generateJupyterReportReference.sh"
+
+# Generate CSV_REPORTS.md containing a reference to all CSV cypher query reports in the "results" directory and its subdirectories.
+source "${DOCUMENTATION_SCRIPT_DIR}/generateCsvReportReference.sh"
+
+# Generate IMAGES.md containing a reference to all PNG images in the "results" directory and its subdirectories.
+source "${DOCUMENTATION_SCRIPT_DIR}/generateImageReference.sh"

.github/workflows/internal-report-reference-documentation.yml

@@ -0,0 +1,71 @@
+name: Generate report reference documentation
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "analysis-results/**" # Only run on changed analysis results
+      - "!analysis-results/*.md" # Ignore report reference documentation changes (endless loop prevention)
+      - ".github/workflows/internal-report-reference-documentation.yml" # Also run when this file was changed
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "analysis-results/**" # Only run on changed analysis results
+      - "!analysis-results/*.md" # Ignore report reference documentation changes (endless loop prevention)
+      - ".github/workflows/internal-report-reference-documentation.yml" # Also run when this file was changed
+
+jobs:
+  generate-report-reference-documentation:
+    runs-on: ubuntu-latest
+    outputs:
+      generated_documents_changed: ${{ steps.set-generated_documents_changed.outputs.generated_documents_changed }}
+      documentation-upload-name: ${{ steps.set-documentation-upload-name.outputs.documentation-upload-name }}
+      
+    steps:
+      - name: Checkout git repository
+        uses: actions/checkout@v4
+
+      - name: Generate report reference document
+        working-directory: analysis-results
+        run: |
+          ./../documentation/analysis-reports-reference/generateReportReferences.sh
+
+      - name: Use git to detect changes in the regenerated documentation and set generated_documents_changed
+        id: set-generated_documents_changed
+        run: git diff --quiet || echo "generated_documents_changed=true" >> "$GITHUB_OUTPUT"
+
+      - name: Display generated_documents_changed
+        run: echo "generated_documents_changed=${{ steps.set-generated_documents_changed.outputs.generated_documents_changed }}"
+
+      - name: Generate ARTIFACT_UPLOAD_ID
+        run: echo "ARTIFACT_UPLOAD_ID=$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 10)" >> $GITHUB_ENV
+ 
+      - name: Set documentation-upload-name
+        id: set-documentation-upload-name
+        run: echo "documentation-upload-name=report-reference-documentation-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
+
+      - name: Archive generated report references documents
+        if: steps.set-generated_documents_changed.outputs.generated_documents_changed == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-reference-documentation
+          path: ./analysis-results/*.md
+          if-no-files-found: error
+          retention-days: 5
+
+
+  commit-analysis-results:
+    name: Commit Analysis Results
+    needs: [generate-report-reference-documentation]
+    uses: ./.github/workflows/internal-commit-results.yml
+    if: needs.generate-report-reference-documentation.outputs.generated_documents_changed == 'true'
+    with:
+      commit-author-name: "${{ github.event.repository.name }} Continuous Integration"
+      commit-author-email: "7671054+JohT@users.noreply.github.com"
+      commit-message: "Automated code structure analysis results (CI)"
+      commit-directory: "./analysis-results/*.md"
+      uploaded-artifact-name: ${{ needs.generate-report-reference-documentation.outputs.documentation-upload-name }}
+    secrets:
+      repository-commit-token: ${{ secrets.WORKFLOW_GIT_ACCESS_TOKEN }}