diff --git a/scripts/executeQuery.sh b/scripts/executeQuery.sh index 9c662e6a1..e5a1468dd 100755 --- a/scripts/executeQuery.sh +++ b/scripts/executeQuery.sh @@ -14,9 +14,18 @@ # -> "--no_source_reference" to not append the cypher query file name as last CSV column # -> any following key=value arguments are used as query parameters +# Requires markdown/formatQueryResultAsMarkdownTable.sh + # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) set -o errexit -o pipefail +## Get this "scripts" directory if not already set +# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. +# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. +# This way non-standard tools like readlink aren't needed. +SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts +#echo "executeQuery: SCRIPTS_DIR=$SCRIPTS_DIR" >&2 + # Overrideable Defaults NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} # Neo4j HTTP API port for executing queries NEO4J_HTTP_TRANSACTION_ENDPOINT=${NEO4J_HTTP_TRANSACTION_ENDPOINT:-"db/neo4j/tx/commit"} # Since Neo4j v5: "db//tx/commit", Neo4j v4: "db/data/transaction/commit" @@ -35,29 +44,34 @@ fi cypher_query_file_name="" no_source_reference=false omit_query_error_highlighting=false +output_markdown_table=false query_parameters="" # Input Arguments: Function to print usage information print_usage() { - echo "executeQuery Usage: $0 [--no-source-reference-column] [--omit-query-error-highlighting]" >&2 + echo "executeQuery Usage: $0 [--no-source-reference-column] [--omit-query-error-highlighting] [--output-markdown-table]" >&2 echo "Options:" >&2 echo " --no-source-reference-column: Exclude the source reference column" >&2 echo " --omit-query-error-highlighting: Log query errors in same color as infos" >&2 + echo " --output-markdown-table: Output the result as markdown table instead of CSV" >&2 } # Input Arguments: Parse the command-line arguments while [[ $# -gt 0 ]]; do arg="$1" - case $arg in --no-source-reference-column) no_source_reference=true shift ;; - --omit_query_error_highlighting) + --omit-query-error-highlighting) omit_query_error_highlighting=true shift ;; + --output-markdown-table) + output_markdown_table=true + shift + ;; *) if [[ -z "${cypher_query_file_name}" ]]; then # Input Arguments: Read the first unnamed input argument containing the name of the cypher file @@ -134,11 +148,16 @@ if [[ -n "${error_message}" ]]; then exit 1 fi -# Output results in CSV format -if [ "${no_source_reference}" = true ] ; then - echo -n "${cypher_query_result}" | jq -r '(.results[0])? | .columns,(.data[].row)? | map(if type == "array" then join(",") else . end) | flatten | @csv' +if [ "${output_markdown_table}" = "true" ] ; then + echo "executeQuery: Will output in Markdown Table Format" >&2 + echo -n "${cypher_query_result}" | "${SCRIPTS_DIR}/markdown/formatQueryResultAsMarkdownTable.sh" else - cypher_query_file_relative_name=${cypher_query_file_name#/**/cypher/} - sourceFileReferenceInfo="Source Cypher File: ${cypher_query_file_relative_name}" - echo -n "${cypher_query_result}" | jq -r --arg sourceReference "${sourceFileReferenceInfo}" '(.results[0])? | .columns + [$sourceReference], (.data[].row)? + [""] | map(if type == "array" then join(",") else . end) | flatten | @csv' + # Output results in CSV format + if [ "${no_source_reference}" = true ] ; then + echo -n "${cypher_query_result}" | jq -r '(.results[0])? | .columns,(.data[].row)? | map(if type == "array" then join(",") else . end) | flatten | @csv' + else + cypher_query_file_relative_name=${cypher_query_file_name#/**/cypher/} + sourceFileReferenceInfo="Source Cypher File: ${cypher_query_file_relative_name}" + echo -n "${cypher_query_result}" | jq -r --arg sourceReference "${sourceFileReferenceInfo}" '(.results[0])? | .columns + [$sourceReference], (.data[].row)? + [""] | map(if type == "array" then join(",") else . end) | flatten | @csv' + fi fi \ No newline at end of file diff --git a/scripts/markdown/embedMarkdownIncludes.sh b/scripts/markdown/embedMarkdownIncludes.sh new file mode 100755 index 000000000..e54a58315 --- /dev/null +++ b/scripts/markdown/embedMarkdownIncludes.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# Processes a template_markdown_file markdown file, replacing placeholders like "" with the contents of the specified markdown files. The files to include needs to be in the "includes" subdirectory. + +# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) +set -o errexit -o pipefail + +## Get this "scripts" directory if not already set +# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. +# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. +# This way non-standard tools like readlink aren't needed. +MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts +#echo "embedMarkdownIncludes: MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR}" >&2 + +template_markdown_file="$1" +include_directory="includes" + +awk -v include_directory="${include_directory}" ' + # Check if the filename is safe + function is_safe(path) { + if (substr(path, 1, 1) == "/") return 0 + if (path ~ /\.\./) return 0 + return 1 + } + + function include_file(path, fullpath, line) { + fullpath = include_directory "/" path + + if (!is_safe(path)) { + print "ERROR: illegal include path: " path > "/dev/stderr" + exit 1 + } + + if ((getline test < fullpath) < 0) { + print "ERROR: missing file " fullpath > "/dev/stderr" + exit 1 + } + close(fullpath) + + while ((getline line < fullpath) > 0) { + print line + } + close(fullpath) + } + + { + # Look for the include marker using index+substr (portable) + if ($0 ~ /") + fname = substr($0, start, end - start) + gsub(/^[ \t]+|[ \t]+$/, "", fname) # trim spaces + + include_file(fname) + } else { + print + } + } +' "${template_markdown_file}" + +#echo "embedMarkdownIncludes: $(date +'%Y-%m-%dT%H:%M:%S%z') Successfully finished." >&2 \ No newline at end of file diff --git a/scripts/markdown/formatQueryResultAsMarkdownTable.sh b/scripts/markdown/formatQueryResultAsMarkdownTable.sh new file mode 100755 index 000000000..f418e4bc6 --- /dev/null +++ b/scripts/markdown/formatQueryResultAsMarkdownTable.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Takes the input stream (Cypher query result in JSON format) and formats it as a Markdown table. + +# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) +set -o errexit -o pipefail + +## Get this "scripts" directory if not already set +# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. +# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. +# This way non-standard tools like readlink aren't needed. +MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts +#echo "formatQueryResultAsMarkdownTable: MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR}" >&2 + +echo "formatQueryResultAsMarkdownTable: Will output in Markdown Table Format" >&2 + +# Read all input (including multiline) into cypher_query_result +cypher_query_result=$(cat) + +echo -n "${cypher_query_result}" | jq -r ' + # Take the first query result + .results[0] as $result + + # Extract the column names + | $result.columns as $columns + + # Build the Markdown header row + | ( "| " + ( $columns | join(" | ") ) + " |" ) + + # Build the Markdown separator row + , ( "| " + ( $columns | map("---") | join(" | ") ) + " |" ) + + # Build one row for each data entry + , ( $result.data[].row + | map(tostring) + | "| " + ( join(" | ") ) + " |" + ) +' + +#echo "formatQueryResultAsMarkdownTable: $(date +'%Y-%m-%dT%H:%M:%S%z') Successfully finished." >&2 \ No newline at end of file diff --git a/scripts/markdown/testEmbedMarkdownIncludes.sh b/scripts/markdown/testEmbedMarkdownIncludes.sh new file mode 100755 index 000000000..17661554b --- /dev/null +++ b/scripts/markdown/testEmbedMarkdownIncludes.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +# Tests template processing for markdown by embedding includes. + +# Requires embedMarkdownIncludes.sh + +# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) +set -o errexit -o pipefail + +## Get this "scripts" directory if not already set +# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. +# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. +# This way non-standard tools like readlink aren't needed. +MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts +echo "testEmbedMarkdownIncludes: MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR}" >&2 + +tearDown() { + # echo "testEmbedMarkdownIncludes: Tear down tests...." + rm -rf "${temporaryTestDirectory}" +} + +successful() { + local COLOR_SUCCESSFUL="\033[0;32m" # green + local COLOR_DEFAULT='\033[0m' + + echo -e "testEmbedMarkdownIncludes: ${COLOR_SUCCESSFUL}Tests finished successfully.${COLOR_DEFAULT}" + + tearDown +} + +fail() { + local COLOR_ERROR='\033[0;31m' # red + local COLOR_DEFAULT='\033[0m' + + local errorMessage="${1}" + + echo -e "testEmbedMarkdownIncludes: ${COLOR_ERROR}${errorMessage}${COLOR_DEFAULT}" + tearDown + return 1 +} + +echo "testEmbedMarkdownIncludes: Starting tests...." + +# Create testing resources +temporaryTestDirectory=$(mktemp -d 2>/dev/null || mktemp -d -t 'temporaryTestDirectory') + +testMarkdownTemplate="${temporaryTestDirectory}/testMarkdownTemplate.md" +echo "" > "${testMarkdownTemplate}" + +# Setup test files +mkdir -p "${temporaryTestDirectory}/includes" + +# ------------------------------------------------------------ +# Test case -- +# ------------------------------------------------------------ +echo "testEmbedMarkdownIncludes: 1.) An existing include file is correctly embedded." + +# - Setup +testIncludeFile="includes/testInclude.md" +expected_test_include_content="This is the included content for the test." +echo "${expected_test_include_content}" > "${temporaryTestDirectory}/${testIncludeFile}" + +# - Execute script under test +embeddedContent=$(cd "${temporaryTestDirectory}"; "${MARKDOWN_SCRIPTS_DIR}/embedMarkdownIncludes.sh" "${testMarkdownTemplate}" ) + +# - Verify results +if [ "${embeddedContent}" != "${expected_test_include_content}" ]; then + fail "1.) Test failed: Expected embedded content to be '${expected_test_include_content}', but got '${embeddedContent}'." +fi + +# ------------------------------------------------------------ +# Test case -- +# ------------------------------------------------------------ +echo "testEmbedMarkdownIncludes: 2.) A missing include file results in an error." + +# - Setup +testMarkdownTemplateMissingInclude="testMarkdownTemplateMissingInclude.md" +echo "" > "${temporaryTestDirectory}/${testMarkdownTemplateMissingInclude}" + +# - Execute script under test +set +o errexit +errorOutput=$(cd "${temporaryTestDirectory}"; { "${MARKDOWN_SCRIPTS_DIR}/embedMarkdownIncludes.sh" "${testMarkdownTemplateMissingInclude}" 2>&1 1>/dev/null; } ) +exitCode=$? +set -o errexit + +# - Verify results +if [ ${exitCode} -eq 0 ]; then + fail "2.) Test failed: Expected an error due to missing include file, but the script succeeded." +fi +if [[ "${errorOutput}" != *"ERROR: missing file"* ]]; then + fail "2.) Test failed: Expected error message to contain 'ERROR: missing file', but got '${errorOutput}'." +fi + +successful +return 0 \ No newline at end of file diff --git a/scripts/markdown/testFormatQueryResultAsMarkdownTable.sh b/scripts/markdown/testFormatQueryResultAsMarkdownTable.sh new file mode 100755 index 000000000..2d20983b3 --- /dev/null +++ b/scripts/markdown/testFormatQueryResultAsMarkdownTable.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +# Tests formatting of Cypher query results as Markdown table. + +# Requires formatQueryResultAsMarkdownTable.sh + +# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) +set -o errexit -o pipefail + +## Get this "scripts" directory if not already set +# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. +# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. +# This way non-standard tools like readlink aren't needed. +MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts +#echo "testFormatQueryResultAsMarkdownTable: MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR}" >&2 + +tearDown() { + # echo "testFormatQueryResultAsMarkdownTable: Tear down tests...." + rm -rf "${temporaryTestDirectory}" +} + +successful() { + local COLOR_SUCCESSFUL="\033[0;32m" # green + local COLOR_DEFAULT='\033[0m' + + echo -e "testFormatQueryResultAsMarkdownTable: ${COLOR_SUCCESSFUL}Tests finished successfully.${COLOR_DEFAULT}" + + tearDown +} + +fail() { + local COLOR_ERROR='\033[0;31m' # red + local COLOR_DEFAULT='\033[0m' + + local errorMessage="${1}" + + echo -e "testFormatQueryResultAsMarkdownTable: ${COLOR_ERROR}${errorMessage}${COLOR_DEFAULT}" + tearDown + return 1 +} + +echo "testFormatQueryResultAsMarkdownTable: Starting tests...." + +# Create testing resources +temporaryTestDirectory=$(mktemp -d 2>/dev/null || mktemp -d -t 'temporaryTestDirectory') + +# ------------------------------------------------------------ +# Test case -- +# ------------------------------------------------------------ +echo "testFormatQueryResultAsMarkdownTable: 1.) Convert a simple query result to a Markdown table." + +# Read expected result from test_data_cypher_query_result_simple_expected +expected_result=$(<"${MARKDOWN_SCRIPTS_DIR}/test_data_cypher_query_result_simple_expected.md") + +# - Execute script under test +embeddedContent=$(cd "${temporaryTestDirectory}"; cat "${MARKDOWN_SCRIPTS_DIR}/test_data_cypher_query_result_simple.json" | "${MARKDOWN_SCRIPTS_DIR}/formatQueryResultAsMarkdownTable.sh") + +# - Verify results +if [ "${embeddedContent}" != "${expected_result}" ]; then + fail "1.) Test failed: Expected Markdown table to be \n${expected_result}, but got:\n${embeddedContent}" +fi + + +echo "testFormatQueryResultAsMarkdownTable: 2.) Convert an array query result to a Markdown table." + +# Read expected result from test_data_cypher_query_result_simple_expected +expected_result=$(<"${MARKDOWN_SCRIPTS_DIR}/test_data_cypher_query_result_with_array_expected.md") + +# - Execute script under test +embeddedContent=$(cd "${temporaryTestDirectory}"; cat "${MARKDOWN_SCRIPTS_DIR}/test_data_cypher_query_result_with_array.json" | "${MARKDOWN_SCRIPTS_DIR}/formatQueryResultAsMarkdownTable.sh") + +# - Verify results +if [ "${embeddedContent}" != "${expected_result}" ]; then + fail "2.) Test failed: Expected Markdown table to be \n${expected_result}, but got:\n${embeddedContent}" +fi + +successful +return 0 \ No newline at end of file diff --git a/scripts/markdown/test_data_cypher_query_result_simple.json b/scripts/markdown/test_data_cypher_query_result_simple.json new file mode 100644 index 000000000..17ce42756 --- /dev/null +++ b/scripts/markdown/test_data_cypher_query_result_simple.json @@ -0,0 +1,35 @@ +{ + "results": [ + { + "columns": [ + "nodeLabel", + "nodesWithThatLabel", + "nodesWithThatLabelPercent" + ], + "data": [ + { + "row": [ + "Git", + 234151, + 77.73524646765112 + ] + }, + { + "row": [ + "Change", + 213807, + 70.98128917454584 + ] + }, + { + "row": [ + "Update", + 138781, + 46.073581748645495 + ] + } + ] + } + ], + "errors": [] +} \ No newline at end of file diff --git a/scripts/markdown/test_data_cypher_query_result_simple_expected.md b/scripts/markdown/test_data_cypher_query_result_simple_expected.md new file mode 100644 index 000000000..e756f1076 --- /dev/null +++ b/scripts/markdown/test_data_cypher_query_result_simple_expected.md @@ -0,0 +1,5 @@ +| nodeLabel | nodesWithThatLabel | nodesWithThatLabelPercent | +| --- | --- | --- | +| Git | 234151 | 77.73524646765112 | +| Change | 213807 | 70.98128917454584 | +| Update | 138781 | 46.073581748645495 | \ No newline at end of file diff --git a/scripts/markdown/test_data_cypher_query_result_with_array.json b/scripts/markdown/test_data_cypher_query_result_with_array.json new file mode 100644 index 000000000..a4e1f0a85 --- /dev/null +++ b/scripts/markdown/test_data_cypher_query_result_with_array.json @@ -0,0 +1,47 @@ +{ + "results": [ + { + "columns": [ + "nodeLabel", + "nodesWithThatLabel", + "nodesWithThatLabelPercent", + "keys" + ], + "data": [ + { + "row": [ + "Git", + 234151, + 77.73524646765112, + [ + "name", + "fileName", + "modificationKind" + ] + ] + }, + { + "row": [ + "Change", + 213807, + 70.98128917454584, + [ + "modificationKind" + ] + ] + }, + { + "row": [ + "Update", + 138781, + 46.073581748645495, + [ + "modificationKind" + ] + ] + } + ] + } + ], + "errors": [] +} \ No newline at end of file diff --git a/scripts/markdown/test_data_cypher_query_result_with_array_expected.md b/scripts/markdown/test_data_cypher_query_result_with_array_expected.md new file mode 100644 index 000000000..e278799f4 --- /dev/null +++ b/scripts/markdown/test_data_cypher_query_result_with_array_expected.md @@ -0,0 +1,5 @@ +| nodeLabel | nodesWithThatLabel | nodesWithThatLabelPercent | keys | +| --- | --- | --- | --- | +| Git | 234151 | 77.73524646765112 | ["name","fileName","modificationKind"] | +| Change | 213807 | 70.98128917454584 | ["modificationKind"] | +| Update | 138781 | 46.073581748645495 | ["modificationKind"] | \ No newline at end of file diff --git a/scripts/runTests.sh b/scripts/runTests.sh index 7b2147cd3..e662e6fd1 100755 --- a/scripts/runTests.sh +++ b/scripts/runTests.sh @@ -15,18 +15,18 @@ LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defa # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. # This way non-standard tools like readlink aren't needed. SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts -echo "runTests: SCRIPTS_DIR=${SCRIPTS_DIR}" >&2 +# echo "runTests: SCRIPTS_DIR=${SCRIPTS_DIR}" >&2 # Run all report scripts -for test_script_file in "${SCRIPTS_DIR}"/test*.sh; do - test_script_filename=$(basename -- "${test_script_file}"); +find "${SCRIPTS_DIR}" -type f -name 'test*.sh' | while read -r test_script_file; do + test_script_filename=$(basename -- "${test_script_file}") test_script_filename="${test_script_filename%.*}" # Remove file extension - echo "${LOG_GROUP_START}Run ${test_script_filename}"; - echo "runTests: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${test_script_filename}..."; + echo "${LOG_GROUP_START}Run ${test_script_filename}" + echo "runTests: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${test_script_filename}..." source "${test_script_file}" - echo "runTests: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${test_script_filename}"; - echo "${LOG_GROUP_END}"; + echo "runTests: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${test_script_filename}" + echo "${LOG_GROUP_END}" done \ No newline at end of file diff --git a/scripts/testDetectChangedFiles.sh b/scripts/testDetectChangedFiles.sh index 693f8ac62..3dfea4b03 100755 --- a/scripts/testDetectChangedFiles.sh +++ b/scripts/testDetectChangedFiles.sh @@ -24,7 +24,6 @@ successful() { echo -e "testDetectChangedFiles: ${COLOR_SUCCESSFUL}Tests finished successfully.${COLOR_DEFAULT}" tearDown - exit 0 } fail() { @@ -35,7 +34,7 @@ fail() { echo -e "testDetectChangedFiles: ${COLOR_ERROR}${errorMessage}${COLOR_DEFAULT}" tearDown - exit 1 + return 1 } echo "testDetectChangedFiles: Starting tests...." @@ -129,4 +128,5 @@ if [ "${changeDetectionReturnCode}" = "0" ]; then fail "13.) Tests failed: Expected return code 0 if nothing changed with a protocol prefixed file, but got ${changeDetectionReturnCode}." fi -successful \ No newline at end of file +successful +return 0 \ No newline at end of file diff --git a/scripts/waitForNeo4jHttpFunctions.sh b/scripts/waitForNeo4jHttpFunctions.sh index ec01192c8..f07c50745 100644 --- a/scripts/waitForNeo4jHttpFunctions.sh +++ b/scripts/waitForNeo4jHttpFunctions.sh @@ -25,7 +25,7 @@ echo "waitForNeo4jHttp: CYPHER_DIR=${CYPHER_DIR}" source "${SCRIPTS_DIR}/executeQueryFunctions.sh" testDatabase() { - execute_cypher "${CYPHER_DIR}/Count_nodes_and_relationships.cypher" "--no-source-reference-column" "--omit_query_error_highlighting" + execute_cypher "${CYPHER_DIR}/Count_nodes_and_relationships.cypher" "--no-source-reference-column" "--omit-query-error-highlighting" } isDatabaseQueryable() {