merge main into branch and solve conflicts

Author: Blargian

.github/CODEOWNERS

@@ -1,3 +1,3 @@
 *                      @ClickHouse/docs
-/docs/integrations/data-ingestion/clickpipes/ @ClickHouse/clickpipes @ClickHouse/docs
 /docs/integrations/ @ClickHouse/integrations-ecosystem @ClickHouse/docs
+/docs/integrations/data-ingestion/clickpipes/ @ClickHouse/clickpipes @ClickHouse/docs

.github/workflows/check-build.yml

@@ -16,15 +16,15 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        check_type: [spellcheck, kbcheck, md-lint]
+        check_type: [spellcheck, kbcheck, md-lint, glossary-check]
     steps:
       # Add setup steps per check here
       - uses: actions/checkout@v4
       - name: Install Aspell
         if: matrix.check_type == 'spellcheck'
         run: sudo apt-get update && sudo apt-get install -y aspell aspell-en
       - name: Set up Python
-        if: matrix.check_type == 'kbcheck'
+        if: matrix.check_type == 'kbcheck' || matrix.check_type == 'glossary-check'
         run: |
           curl -Ls https://astral.sh/uv/install.sh | sh
           uv clean
@@ -39,31 +39,25 @@ jobs:
         run: yarn add -D markdownlint-cli2
 
       # Run the checks here
-      - name: Run checks
-        id: check_step
-        run: |
-          if [[ "${{ matrix.check_type }}" == "spellcheck" ]]; then
-            yarn check-spelling
-          exit_code=$?
-          elif [[ "${{ matrix.check_type }}" == "kbcheck" ]]; then
-            yarn check-kb
-          exit_code=$?
-          elif [[ "${{ matrix.check_type }}" == "md-lint" ]]; then
-            yarn check-markdown
-          exit_code=$?
-          fi
-          
-          if [[ $exit_code -ne 0 ]]; then
-            echo "::error::${{ matrix.check_type }} check failed. See logs for details."
-            exit 1
-          fi
+      - name: Run spellcheck
+        if: matrix.check_type == 'spellcheck'
+        run: yarn check-spelling
 
-      - name: Set check status
-        if: steps.check_step.outcome != 'success'
-        uses: actions/github-script@v6
-        with:
-          script: |
-            core.setFailed('${{ matrix.check_type }} check failed.');
+      - name: Run KB check  
+        if: matrix.check_type == 'kbcheck'
+        run: yarn check-kb
+
+      - name: Run markdown lint
+        if: matrix.check_type == 'md-lint' 
+        run: yarn check-markdown
+
+      - name: Run glossary check
+        if: matrix.check_type == 'glossary-check'
+        run: |
+          echo "Extracting glossary from markdown..."
+          python3 scripts/glossary/extract-glossary-terms.py
+          echo "Checking glossary coverage..."
+          python3 scripts/glossary/wrap-glossary-terms.py --check || echo "::warning::Glossary check found unwrapped terms (non-blocking)"
           
   check_overall_status:
     needs: stylecheck
@@ -74,5 +68,4 @@ jobs:
         if: needs.stylecheck.result != 'success'
         run: |
           echo "::error::One or more checks of the style check failed."
-          exit 1
-          
+          exit 1

.github/workflows/trademark-cla-approval.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Generate Token
         id: generate-token
         continue-on-error: true
-        uses: actions/create-github-app-token@v1
+        uses: actions/create-github-app-token@v2
         with:
           app-id: "${{ secrets.WORKFLOW_AUTH_PUBLIC_APP_ID }}"
           private-key: "${{ secrets.WORKFLOW_AUTH_PUBLIC_PRIVATE_KEY }}"

.github/workflows/trademark-cla-notice.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Generate Token
         id: generate-token
         continue-on-error: true
-        uses: actions/create-github-app-token@v1
+        uses: actions/create-github-app-token@v2
         with:
           app-id: "${{ secrets.WORKFLOW_AUTH_PUBLIC_APP_ID }}"
           private-key: "${{ secrets.WORKFLOW_AUTH_PUBLIC_PRIVATE_KEY }}"

README.md

@@ -80,6 +80,10 @@ You can run a copy of this website locally within a few steps. Some folks find t
     # [INFO] Use `npm run serve` command to test your build locally.
     # ✨  Done in 105.96s.
     ```
+   
+> [!TIP]
+> If the build command is failing due to broken anchors, 
+> it is possible to ignore these temporarily by running `ON_BROKEN_ANCHORS=ignore yarn build` instead.
 
 1. Start the local web-server:
 
@@ -134,9 +138,9 @@ Please assign any pull request (PR) against an issue; this helps the docs team t
 
 Check out the GitHub docs for a refresher on [how to create a pull request](https://docs.github.com/en/desktop/working-with-your-remote-repository-on-github-or-github-enterprise/creating-an-issue-or-pull-request-from-github-desktop).
 
-### Style guidelines
+### Style and contribution guidelines
 
-For documentation style guidelines, see ["Style guide"](/contribute/style-guide.md). 
+For documentation style guidelines, see ["Style guide"](/contribute/style-guide.md).
 
 To check spelling and markdown is correct locally run:
 

code_snippets/ClickStack/config-unstructured-logs-with-processor.yaml

@@ -0,0 +1,43 @@
+receivers:
+  filelog:
+    include:
+      - /opt/data/logs/access-unstructured.log
+    start_at: beginning
+    operators:
+      - type: regex_parser
+        regex: '^(?P<ip>[\d.]+)\s+-\s+-\s+\[(?P<timestamp>[^\]]+)\]\s+"(?P<method>[A-Z]+)\s+(?P<url>[^\s]+)\s+HTTP/[^\s]+"\s+(?P<status>\d+)\s+(?P<size>\d+)\s+"(?P<referrer>[^"]*)"\s+"(?P<user_agent>[^"]*)"'
+        timestamp:
+          parse_from: attributes.timestamp
+          layout: '%d/%b/%Y:%H:%M:%S %z'
+          #22/Jan/2019:03:56:14 +0330
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 100
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 2048
+    spike_limit_mib: 256
+exporters:
+  # HTTP setup
+  otlphttp/hdx:
+    endpoint: 'http://localhost:4318'
+    headers:
+      authorization: <YOUR_INGESTION_API_KEY>
+    compression: gzip
+
+  # gRPC setup (alternative)
+  otlp/hdx:
+    endpoint: 'localhost:4317'
+    headers:
+      authorization: <YOUR_API_INGESTION_KEY>
+    compression: gzip
+service:
+  telemetry:
+    metrics:
+      address: 0.0.0.0:9888 # Modified as 2 collectors running on same host
+  pipelines:
+    logs:
+      receivers: [filelog]
+      processors: [batch]
+      exporters: [otlphttp/hdx]

contribute/autogenerated-documentation-from-source.md

@@ -61,18 +61,16 @@ TO DO
 ## Functions
 
 Documentation for functions is autogenerated from the documentation in system tables (`system.functions`) which is 
-registered along with the function registration in the C++ code.
+registered along with the function registration in the C++ code. ([Example](https://github.com/ClickHouse/ClickHouse/blob/a0216c8120f00479f50237e41c897c2bf244e1ef/src/Functions/FunctionChar.cpp#L116-L158))
 
-To add a new page:
-- add SQL to generate the markdown from system tables
-- update `autogenerate-settings.sh` in the section which specifies which markdown file contents to
-  copy to which file in between the `<!--AUTOGENERATED_START-->` and `<!--AUTOGENERATED_END-->`
+This [script](https://github.com/ClickHouse/clickhouse-docs/blob/main/scripts/settings/autogenerate-settings.sh) is used to generate
+the markdown docs for functions.
 
-The following pages are autogenerated from the source code for functions:
-- [Arithmetic](/sql-reference/functions/arithmetic-functions) ([SQL](../scripts/settings/arithmetic-functions.sql))
-- [Arrays](/sql-reference/functions/array-functions) ([SQL](../scripts/settings/array-functions.sql))
-- [Bit](/sql-reference/functions/bit-functions) ([SQL](../scripts/settings/bit-functions.sql))
-- [Bitmap](/sql-reference/functions/bitmap-functions) ([SQL](../scripts/settings/bitmap-functions.sql))
-- [Comparison](/sql-reference/functions/comparison-functions) ([SQL](../scripts/settings/comparison-functions.sql))
-- [Conditional](/sql-reference/functions/conditional-functions) ([SQL](../scripts/settings/conditional-functions.sql))
+Adding a new page can be done by:
 
+1. Adding the function [category](https://github.com/ClickHouse/ClickHouse/blob/85700c135ccad89d3651a7a92ca63bb989743ba6/src/Common/FunctionDocumentation.cpp#L222-L273) to the [list of function categories to generate](https://github.com/ClickHouse/clickhouse-docs/blob/c31d64bc5d71c6c3afc5b520dbb73386e303447b/scripts/settings/autogenerate-settings.sh#L253-L278).
+2. The docs are automatically generated for the categories defined above by [this SQL code](https://github.com/ClickHouse/clickhouse-docs/blob/main/scripts/settings/generate-functions.sql). It produces a temporary markdown file with convention: `category-functions.md`. If the category name contains spaces, e.g. "String Splitting" then the temporary file will be `string_splitting-functions.md`.
+3. Register the temporary file name [here](https://github.com/ClickHouse/clickhouse-docs/blob/c31d64bc5d71c6c3afc5b520dbb73386e303447b/scripts/settings/autogenerate-settings.sh#L372)
+4. Register the destination file you want to insert the markdown into [here](https://github.com/ClickHouse/clickhouse-docs/blob/c31d64bc5d71c6c3afc5b520dbb73386e303447b/scripts/settings/autogenerate-settings.sh#L399)
+
+The script takes the content of the temporary markdown file and inserts it into the destination file between tags `<!--AUTOGENERATED_START-->` and `<!--AUTOGENERATED_END-->`. If those tags do not exist you will get a warning.

contribute/style-guide.md

@@ -16,6 +16,7 @@ sidebar_label: 'Using clickhouse-local database'
 slug: /chdb/guides/clickhouse-local
 description: 'Learn how to use a clickhouse-local database with chDB'
 keywords: ['chdb', 'clickhouse-local']
+doc_type: 'reference' (or 'guide' or 'changelog')
 ---
 ```
 
@@ -26,7 +27,7 @@ keywords: ['chdb', 'clickhouse-local']
 There is a custom Docusaurus plugin which runs on build that makes the following
 checks on front-matter:
 
-- title, description and slug are specified.
+- title, description, slug and doc_type (either `reference`, `guide`, `landingpage` or `changelog`) are specified.
 - keywords use flow style arrays with single quoted items e.g. 
   `keywords: ['integrations']`
 - single quotes are used for title, description, slug, sidebar_label
@@ -112,12 +113,75 @@ SELECT * FROM system.contributors;
 \```
 ```
 
+Note: in the snippet above `\` is used only for formatting purposes in this guide.
+You should not include it when you write markdown.
+
 Code blocks:
 - Should always have a language defined immediately next to the opening 3
   backticks, without any space.
 - Have a title (optional) such as 'Query' or 'Response'
 - Use language `response` if it is for the result of a query.
 
+#### Importing code from files or URLs
+
+There are a few additional parameters you can include on a code block if you want
+to import code.
+
+To import from a file use `file=`:
+
+```text
+\```python file=code_snippets/integrations/example.py
+Code will be inserted here
+\```
+```
+
+When `yarn build` is run, the code from the file will be inserted as text into
+the code block.
+
+To import from a url use `url=`:
+
+```text
+\```python url=https://raw.githubusercontent.com/ClickHouse/clickhouse-connect/refs/heads/main/examples/pandas_examples.py
+Code will be inserted here
+\```
+```
+
+**File paths are relative to the root of the docs repository**
+
+You should commit the code inserted to the snippet as we want people (or LLMs) 
+reading the markdown to be able to see the code. The advantage of importing code
+to snippets this way is that you can test your snippets externally or store them
+wherever you want.
+
+If you want to only import a section from a file, surround the section with `docs-start`
+and `docs-end` comments, for example:
+
+```python
+a = 200
+b = 33
+#docs-start
+if b > a:
+  print("b is greater than a")
+elif a == b:
+  print("a and b are equal")
+else:
+  print("a is greater than b")
+#docs-end
+```
+
+Only the code between those comments will be pulled.
+
+If you want to make multiple code snippets from one file then you can use the `snippet` parameter:
+
+```markdown
+
+\```python url=https://raw.githubusercontent.com/ClickHouse/clickhouse-connect/refs/heads/main/examples/pandas_examples.py snippet=1
+Code will be inserted here
+\```
+```
+
+You will then use `docs-start-1`, `docs-end-1` comments for the first snippet, `docs-start-2`, `docs-end-2` for the second snippet and so on.
+
 ### Highlighting
 
 You can highlight lines in a code block using the following keywords:
@@ -412,3 +476,48 @@ vale --filter='.Name == "ClickHouse.Headings"' docs/integrations
 This will run only the rule named `Headings` on
 the `docs/integrations` directory. Specifying a specific markdown
 file is also possible.
+
+## Vertical numbered stepper
+
+It is possible to render numbered steppers, as seen [here](https://clickhouse.com/docs/getting-started/quick-start/cloud)
+for example, using the following syntax:
+
+`<VerticalStepper headerLevel="hN"></VerticalStepper>`
+
+For example:
+
+```markdown
+<VerticalStepper headerLevel="h2">
+## Header 1 {#explicit-anchor-1}
+
+Some content...
+
+## Header 2 {#explicit-anchor-2}
+
+Some more content...
+
+</VerticalStepper>
+```
+
+You should specify `N` as the header level you want the vertical stepper to render
+for. In the example above, it is `h2` as we are using `##`. Use `h3` for `###`,
+`h4` for `####` etc.
+
+The component also works with numbered lists using `headerLevel="list"`. For example:
+
+```markdown
+<VerticalStepper headerLevel="h2">
+
+1. First list item
+
+Some content...
+
+2. Second list item
+
+Some more content...
+
+</VerticalStepper>
+```
+
+In this case, the first paragraph will be taken to be the label (the text next
+to the numbered circles of the vertical stepper) of the stepper.