feat(ci): Add gdl checker in workflow

Author: lonrun

.github/workflows/check_gdl_workflow.yml

@@ -0,0 +1,55 @@
+name: GDL script file checker
+on:
+  push:
+    branches-ignore:
+      - 'none'  
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  checking-job:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository to the runner
+        uses: actions/checkout@v4
+
+      - name: Install locales
+        run: |
+          sudo apt-get update && sudo apt-get install -y locales
+          sudo locale-gen zh_CN.UTF-8
+        env:
+          LANG: zh_CN.UTF-8
+          LANGUAGE: zh_CN:zh:en_US:en
+
+      - name: Download and extract the latest sparrow-cli release
+        run: |
+          ASSET_NAME="sparrow-cli.*.linux.tar.gz" # This pattern should match only the tar.gz file
+          mkdir -p $HOME/sparrow-cli
+      
+          # Use GitHub API to get the latest release information
+          RELEASE_INFO=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/codefuse-ai/CodeFuse-Query/releases/latest")
+      
+          # Extract the asset download URL for the asset name specified
+          # The test function is used to ensure we match only the tar.gz file, not the checksum file
+          ASSET_URL=$(echo "$RELEASE_INFO" | jq --arg asset_name "$ASSET_NAME" -r '.assets[] | select(.name | test($asset_name)) | select(.content_type == "application/x-gzip").browser_download_url')
+      
+          # Check if the asset URL is empty or not
+          if [ -z "$ASSET_URL" ]; then
+            echo "Error: Asset URL is empty."
+            exit 1
+          fi
+      
+          # Download and extract the asset
+          echo "Downloading $ASSET_URL to $HOME/sparrow-cli/sparrow-cli.tar.gz"
+          curl -fL --retry 5 "$ASSET_URL" | tar -xz -C $HOME/sparrow-cli
+        env:
+          # The GitHub token is needed for API requests to avoid rate limits
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set execute permissions for script
+        run: chmod +x ./tool/aci/check_gdl.sh
+        
+      - name: Run GDL script checking
+        run: ./tool/aci/check_gdl.sh .
+        env:
+          LC_ALL: zh_CN.UTF-8

tool/aci/check_gdl.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+
+: '
+This script performs validation and compilation of Godel script files (.gs and .gdl).
+
+Usage:
+  ./check_gdl.sh <directory>
+
+Arguments:
+  <directory>    The directory to scan for Godel script files. The script will
+                 search for .gs and .gdl files to compile and validate.
+
+Description:
+  The script does the following:
+  - Validates that a directory is provided as an argument.
+  - Changes to the specified directory.
+  - Finds all .gs and .gdl files within the specified directory (excluding specific paths).
+  - For each located library directory, it concatenates the library files and compiles them.
+  - For each script file, it runs a separate compilation process and checks for errors.
+  - Reports any compilation errors and terminates execution if an error occurs.
+  - If no errors occur, it reports successful compilation for each file.
+
+Requires:
+  - The "sparrow-cli" tool must be installed and available under the user"s home directory.
+  - Command "find" available on the system (commonly available on Unix-like systems).
+  - Command "mktemp" available on the system for creating temporary files.
+  - Command "date" available on the system for time measurements.
+
+Author: AntGroup
+Date: 2024-01-16
+Version: 1.0
+
+'
+
+set +x
+
+# Check if the parameter is empty
+if [ -z "$1" ]; then
+  echo "Please provide a directory as an argument"
+  exit 1
+fi
+
+# Change to the directory
+cd "$1" || exit 1
+
+sparrow_godel_script="$HOME/sparrow-cli/sparrow-cli/godel-script/usr/bin/godel"
+sparrow_lib_1_0="$HOME/sparrow-cli/sparrow-cli/lib-1.0"
+
+# Define get_files function
+get_files() {
+  find "$1" -type f \( -name "*$2" \) -print
+}
+
+# Define rebuild_lib function
+rebuild_lib() {
+  local lib_path="$1"
+  local lib="$2"
+  local gdl_list=()
+  local output_file
+  local tmp_out
+  local start_time
+  local end_time
+  local elapsed_time
+
+  gdl_list+=($(get_files "$lib_path" ".gs"))
+  gdl_list+=($(get_files "$lib_path" ".gdl"))
+
+  output_file=$(mktemp "tempfile.XXXXXX.gdl")
+  trap 'rm -f "$output_file"' EXIT
+
+  echo "// script" > "$output_file"
+  for file_name in "${gdl_list[@]}"; do
+    cat "$file_name" >> "$output_file"
+  done
+
+  tmp_out=$(mktemp "tempfile.XXXXXX.gdl")
+  trap 'rm -f "$tmp_out"' EXIT
+
+  start_time=$(date +%s%3N)
+  if ! "$sparrow_godel_script" "$output_file" -o "$tmp_out"; then
+    echo "$lib_path lib compile error, please check it yourself" >&2
+    exit 1
+  fi
+
+  mv "$tmp_out" "$sparrow_lib_1_0/coref.$lib.gdl"
+
+  end_time=$(date +%s%3N)
+  elapsed_time=$((end_time - start_time))
+  echo "$lib_path lib compile success time: ${elapsed_time} milliseconds" >&2
+}
+
+# Define get_language function
+get_language() {
+  local dir="$1"
+  local dirname
+  local language
+
+  dirname=$(dirname "$dir")
+  language=$(basename "$dirname")
+  echo "$language"
+}
+
+# Get libs directories
+directories=($(find "$PWD" -type d \( -path "$PWD/language/*/lib" -o -path "$PWD/language/*/libs" \) -print))
+
+# Get libs
+for dir in "${directories[@]}"; do
+  lang=$(get_language "$dir")
+  echo "Building lib for $lang ..."
+  rebuild_lib "$dir" "$lang"
+done
+
+# Define get_target_files function
+get_target_files() {
+  find "$1" -type f \( -name "*.gs" -o -name "*.gdl" \) -not -name "tempfile.*.gdl" -not -path "$1/language/*/lib/*"
+}
+
+files=$(get_target_files "$PWD")
+
+# Iterate over the files
+for file in $files; do
+  output=$(("$sparrow_godel_script" "$file" -p "$sparrow_lib_1_0" -o "${file%.*}_tmp.gdl") 2>&1)
+
+  # Check if the output is not empty
+  if [ -n "$output" ]; then
+    echo "The file $file produced the following output:"
+    echo "$output"
+    echo "Please check if this file is a godel script (.gs) or a godel 1.0 script (.gdl)"
+    exit 1
+  else
+    echo "$file build successful"
+  fi
+
+  # Remove temporary file
+  rm -f "${file%.*}_tmp.gdl"
+done
+
+exit 0