Merge pull request #3168 from Blargian/separate-out-new-build

Improve build-scripts

Author: Blargian

.gitignore

@@ -37,9 +37,7 @@ active_links.txt
 active_links.json
 
 # Files used by scripts to autogenerate settings and do spellcheck
-FormatFactorySettingsDeclaration.h
-FormatFactorySettings.h
-Settings.cpp
+scripts/settings/tmp
 
 # Files whose content gets autogenerated
 docs/en/cloud/manage/api/invitations-api-reference.md

README.md

@@ -69,28 +69,22 @@ You can run a copy of this website locally within a few steps. Some folks find t
     # ✨  Done in 6.44s.
     ```
 
-1. Use Yarn to grab the latest documentation changes from the `clickhouse/clickhouse` repository:
+1. Use Yarn to grab the latest documentation changes from the `ClickHouse/ClickHouse` repository:
 
     ```shell
-    yarn prep-from-master
+    yarn copy-clickhouse-repo-docs
 
-    # yarn run v1.22.19
-    # Cloning into '/tmp/ch_temp_14714'...
+    # Cloning into 'ClickHouse'...
     # ...
-    # Prep completed
-    # ✨  Done in 16.44s
+    # Copying docs from ClickHouse ...
+    # Successfully executed copy from master
+    # ✨  Done in 18.56s.
     ```
 
-    Alternatively, you can use a local copy of the `clickhouse/clickhouse` if you already have that repository cloned locally.
+    Alternatively, you can use a local copy of `ClickHouse/ClickHouse` if you already have that repository cloned locally with `-l`.
 
     ```shell
-    yarn prep-from-local "/Users/johnny/clickhouse/"
-
-    # yarn run v1.22.19
-    # $ bash -c 'array_root=($npm ...
-    # ...
-    # Prep completed
-    ✨  Done in 0.47s.
+    yarn copy-clickhouse-repo-docs -l "/Users/johnny/clickhouse/"
     ```
 
 1. Start the local web-server:

copyClickhouseRepoDocs.sh

@@ -318,7 +318,7 @@ const config = {
 				async postBuild({ siteConfig = {}, routesPaths = [], outDir }) {
 					// Print out to console all the rendered routes.
 					routesPaths.map((route) => {
-						console.log(route)
+						//console.log(route)
 					})
 				},
 			}

docusaurus.config.js

@@ -4,22 +4,22 @@
   "private": true,
   "config": {
     "prep_array_en": "docs/en/development docs/en/engines docs/en/getting-started docs/en/interfaces docs/en/operations docs/en/sql-reference",
-    "prep_array_root": "docs/ru docs/zh"
+    "prep_array_root": "docs/ru docs/zh",
+    "autogen_needed_files": "src/Core/FormatFactorySettings.h src/Core/Settings.cpp CHANGELOG.md"
   },
   "scripts": {
-    "build": "docusaurus build",
+    "build": "yarn copy-clickhouse-repo-docs && yarn generate-changelog && yarn autogenerate-settings && yarn build-api-doc && yarn build-swagger && docusaurus build",
     "clear": "docusaurus clear && bash ./placeholderReset.sh",
     "deploy": "docusaurus deploy",
     "docusaurus": "docusaurus",
-    "prep-from-local": "bash scripts/prep-from-local.sh",
-    "prep-from-master": "bash scripts/prep-from-master.sh",
-    "copy-clickhouse-repo-docs": "bash ./copyClickhouseRepoDocs.sh",
+    "copy-clickhouse-repo-docs": "bash scripts/copy-clickhouse-repo-docs.sh",
     "serve": "docusaurus serve",
     "build-api-doc": "node clickhouseapi.js",
-    "build-swagger": "npx @redocly/cli build-docs  https://api.clickhouse.cloud/v1 --output build/en/cloud/manage/api/swagger.html",
-    "auto-generate-settings": "bash ./scripts/settings/autogenerate-settings.sh",
-    "auto-generate-table-of-contents": "bash ./scripts/autogenerate-table-of-contents.sh",
-    "new-build": "yarn copy-clickhouse-repo-docs && yarn auto-generate-settings && yarn auto-generate-table-of-contents && yarn build-api-doc && yarn build && yarn build-swagger",
+    "build-swagger": "npx @redocly/cli build-docs https://api.clickhouse.cloud/v1 --output build/en/cloud/manage/api/swagger.html",
+    "prep-from-local": "bash ./scripts/copy-clickhouse-repo-docs.sh -l $1",
+    "autogenerate-settings": "bash ./scripts/settings/autogenerate-settings.sh",
+    "generate-changelog": "bash ./scripts/generate-changelog.sh",
+    "new-build": "yarn copy-clickhouse-repo-docs && yarn generate-changelog && yarn autogenerate-settings && yarn build-api-doc && yarn build-swagger && docusaurus build",
     "start": "docusaurus start",
     "swizzle": "docusaurus swizzle",
     "write-heading-ids": "docusaurus write-heading-ids",
@@ -32,7 +32,6 @@
     "@docusaurus/theme-mermaid": "3.7.0",
     "@docusaurus/theme-search-algolia": "^3.7.0",
     "@mdx-js/react": "^3.1.0",
-    "@yaireo/tagify": "^4.33.2",
     "@radix-ui/react-navigation-menu": "^1.2.4",
     "axios": "^1.7.9",
     "clsx": "^2.1.0",
@@ -62,6 +61,7 @@
     "@playwright/test": "^1.49.1",
     "@types/react": "^19.0.4",
     "cheerio": "^1.0.0",
+    "rsync": "^0.6.1",
     "typescript": "^5.7.3"
   },
   "browserslist": {

package.json

@@ -0,0 +1,143 @@
+#!/bin/bash
+
+function parse_args() {
+
+  while getopts "hl:" opt; do
+    case "$opt" in
+      h)
+        # Display the usage information and exit when -h is provided
+        echo "Usage: $0 [-l path_to_local_clickhouse_repo]"
+        echo ""
+        echo "Options:"
+        echo "  -l   Path to a local copy of the ClickHouse repository."
+        echo "  -h   Display this help message."
+        exit 0
+        ;;
+      l)
+        local_path="$OPTARG"
+        ;;
+      \?)
+        echo "Invalid option: -$OPTARG" >&2
+        exit 1
+        ;;
+    esac
+  done
+
+  echo "$local_path"
+}
+
+# Validate local path exists and ends in ClickHouse
+function validate_local() {
+
+  if [[ -d "$local_path" && "$local_path" == *"ClickHouse" ]]; then
+    return 0
+  else
+    echo "Please provide a valid path to your local ClickHouse repository."
+    exit 1
+  fi
+}
+
+# Copy files/folders using rsync (or fallback to cp)
+copy_item() {
+  local source="$1"
+  local destination="$2"
+
+  if $has_rsync; then
+    rsync -a "$source" "$destination"
+  else
+    cp -r "$source" "$destination"
+    if [ $? -ne 0 ]; then
+      echo "Error copying $source to $destination"
+      error=1
+    fi
+  fi
+}
+
+# Define function to copy docs locally
+function copy_docs_locally() {
+  local local_path=$1
+
+  # Validate local path only if it's provided
+  if [[ -n "$local_path" ]]; then
+    validate_local "$local_path"
+  fi
+
+  # Read package.json to get list of docs folders and files
+  package_json=$(cat "$(pwd)/package.json")
+
+  # Extract docs_folders_en
+  docs_folders_en=$(echo "$package_json" | awk -F'"' '/"prep_array_en":/{print $4}')
+
+  # Extract docs_folders_other
+  docs_folders_other=$(echo "$package_json" | awk -F'"' '/"prep_array_root":/{print $4}')
+
+  # Extract files_for_autogen_settings
+  files_for_autogen_settings=$(echo "$package_json" | awk -F'"' '/"autogen_needed_files":/{print $4}')
+
+  if [ "$docs_folders_en" = "" ] || [ "$docs_folders_other" = "" ] || [ "$files_for_autogen_settings" = "" ]
+  then
+    echo "An error occurred trying to extract directory and file names from package.json"
+    exit 1
+  fi
+
+  error=0
+  # Copy docs folders
+  for folder in $docs_folders_en; do
+    copy_item "$local_path/$folder" "docs/en"
+  done
+
+  for folder in $docs_folders_other; do
+    copy_item "$local_path/$folder" "docs/"
+  done
+
+  # check if tmp directory exists, if not make it
+  if [[ -d "scripts/tmp" ]]; then
+    echo "scripts/tmp already exists"
+  else
+    mkdir scripts/tmp
+  fi
+  # Copy files for autogen settings
+  for source_file in $files_for_autogen_settings; do
+    copy_item "$local_path/$source_file" "scripts/tmp"
+  done
+
+  if [ "$error" -eq 1 ]; then
+    echo "an error occurred copying the files"
+    exit 1
+  fi
+}
+
+main() {
+  parse_args "$@"
+
+  # Check if rsync is available
+  has_rsync=false
+  if command -v rsync &> /dev/null; then
+    echo "rsync found"
+    has_rsync=true
+  else
+    echo "rsync not found - falling back to use cp command. For faster dev builds we recommend installing rsync."
+  fi
+
+  # If no local path is provided, clone the ClickHouse repo
+
+  if [[ -z "$local_path" ]]; then
+
+    git clone --depth 1 --branch master https://github.com/ClickHouse/ClickHouse
+
+    # Copy docs from cloned repository
+    copy_docs_locally "$(pwd)/ClickHouse"
+
+    # Remove cloned repository
+    rm -rf ClickHouse
+
+    echo "Successfully executed copy from master"
+  else
+    # Copy docs from the provided local path
+    validate_local "$local_path"
+    copy_docs_locally "$local_path"
+    echo "Successfully executed local copy"
+  fi
+}
+
+main "$@"

scripts/copy-clickhouse-repo-docs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+script_dir=$(dirname "$0")  # Get the directory of the script
+parent_dir=$(realpath "$script_dir") # Get the parent directory
+
+echo "Running Changelog Generation Script"
+cp "$parent_dir/../docs/en/_placeholders/changelog/_index.md" "$parent_dir/../docs/en/whats-new/changelog/index.md"
+if grep -q "^# $(date +%Y) Changelog" "$parent_dir/tmp/CHANGELOG.md"; then
+  echo "Generating $(date +%Y) Changelog..."
+    sed "/^# $(date +%Y) Changelog/d" "$parent_dir/tmp/CHANGELOG.md" > temp.txt
+    echo "Changelog copied to temp.txt"
+    cat temp.txt >> "$parent_dir/../docs/en/whats-new/changelog/index.md"
+    echo "Changelog written to docs/en/whats-new/changelog/index.md"
+    rm -f temp.txt
+    echo "$(date +%Y) Changelog was updated."
+else
+    current_year="$(date +%Y)"
+    previous_year="$(($current_year - 1))"
+    echo "No Changelog found for $current_year."
+    echo -e ":::note\nThere have been no new releases yet for $current_year.  \n View changelog for the year [$previous_year](/docs/en/whats-new/changelog/$previous_year).\n:::" >> "$parent_dir/../docs/en/whats-new/changelog/index.md"
+fi