Merge branch 'main' of https://github.com/ClickHouse/clickhouse-docs into otel-collector-advanced-config

dhtclk · dhtclk · commit ad92b7b2446d · 2025-11-05T11:44:46.000-06:00
diff --git a/.github/workflows/badges-notify.yml b/.github/workflows/badges-notify.yml
@@ -13,7 +13,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Make script executable
         run: chmod +x scripts/badger.sh
diff --git a/.github/workflows/build-search.yml b/.github/workflows/build-search.yml
@@ -20,10 +20,10 @@ jobs:
 
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Set up Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: '20' 
 
diff --git a/.github/workflows/check-build.yml b/.github/workflows/check-build.yml
@@ -19,7 +19,7 @@ jobs:
         check_type: [spellcheck, kbcheck, md-lint, glossary-check]
     steps:
       # Add setup steps per check here
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Install Aspell
         if: matrix.check_type == 'spellcheck'
         run: sudo apt-get update && sudo apt-get install -y aspell aspell-en
@@ -31,7 +31,7 @@ jobs:
           uv python install 3.12 --verbose
       - name: Setup md-lint environment
         if: matrix.check_type == 'md-lint'
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v6
         with:
           node-version: 20
       - name: Install markdownlint-cli2
diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
@@ -21,10 +21,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Set up Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v6
         with:
           node-version: '20'  # Adjust Node.js version as needed
           cache: 'yarn'       # Cache yarn dependencies
diff --git a/.github/workflows/trademark-cla-approval.yml b/.github/workflows/trademark-cla-approval.yml
@@ -170,7 +170,7 @@ jobs:
 
       - name: Check out repository
         if: success() && steps.process-comment.outputs.pr_number != ''
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           # For forked PRs, we need to fetch from the fork and checkout the SHA
diff --git a/.github/workflows/trademark-cla-notice.yml b/.github/workflows/trademark-cla-notice.yml
@@ -25,7 +25,7 @@ jobs:
           private-key: "${{ secrets.WORKFLOW_AUTH_PUBLIC_PRIVATE_KEY }}"
 
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           # Use the GitHub App token if available, otherwise fallback to GITHUB_TOKEN
diff --git a/.github/workflows/vale-linter.yml b/.github/workflows/vale-linter.yml
@@ -20,7 +20,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           path: .
diff --git a/docs/cloud/reference/03_billing/01_billing_overview.md b/docs/cloud/reference/03_billing/01_billing_overview.md
@@ -181,14 +181,14 @@ Best for: large scale, mission critical deployments that have stringent security
 
 A ClickHouse Credit is a unit of credit toward Customer's usage of ClickHouse Cloud equal to one (1) US dollar, to be applied based on ClickHouse's then-current published price list.
 
-### Where can I find legacy pricing? {#find-legacy-pricing}
-
-Legacy pricing information can be found [here](https://clickhouse.com/pricing?legacy=true).
-
 :::note 
 If you are being billed through Stripe then you will see that 1 CHC is equal to \$0.01 USD on your Stripe invoice. This is to allow accurate billing on Stripe due to their limitation on not being able to bill fractional quantities of our standard SKU of 1 CHC = \$1 USD.
 :::
 
+### Where can I find legacy pricing? {#find-legacy-pricing}
+
+Legacy pricing information can be found [here](https://clickhouse.com/pricing?legacy=true).
+
 ### How is compute metered? {#how-is-compute-metered}
 
 ClickHouse Cloud meters compute on a per-minute basis, in 8G RAM increments. 
diff --git a/docs/cloud/reference/03_billing/index.md b/docs/cloud/reference/03_billing/index.md
@@ -10,7 +10,9 @@ This section of the documentation covers topics related to billing, and contains
 
 | Page                                  | Description                                                          |
 |---------------------------------------|----------------------------------------------------------------------|
-| [Overview](/cloud/marketplace/marketplace-billing)                       | Overview and FAQ pages for marketplace billing.                      | 
+| [Overview](/cloud/manage/billing/overview)                       | Pricing examples and FAQs for billing.                      | 
 | [Payment Thresholds](/cloud/billing/payment-thresholds)             | Learn more about how payment thresholds work and how to adjust them. |
 | [Troubleshooting Billing Issues](/manage/clickhouse-cloud-billing-compliance) | Troubleshoot common billing issues.                                  |
 | [Marketplace](/cloud/manage/marketplace/)                    | Landing page for further marketplace related topics.                 |
+
+https://clickhouse.com/docs/cloud/
diff --git a/docs/integrations/data-ingestion/clickpipes/object-storage/02_reference.md b/docs/integrations/data-ingestion/clickpipes/object-storage/02_reference.md
@@ -75,12 +75,9 @@ https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/{documents-01,doc
 ## Continuous Ingest {#continuous-ingest}
 ClickPipes supports continuous ingestion from S3, GCS, Azure Blob Storage, and DigitalOcean Spaces. When enabled, ClickPipes continuously ingests data from the specified path, and polls for new files at a rate of once every 30 seconds. However, new files must be lexically greater than the last ingested file. This means that they must be named in a way that defines the ingestion order. For instance, files named `file1`, `file2`, `file3`, etc., will be ingested sequentially. If a new file is added with a name like `file0`, ClickPipes will not ingest it because it is not lexically greater than the last ingested file.
 
-## Archive table {#archive-table}
-ClickPipes will create a table next to your destination table with the postfix `s3_clickpipe_<clickpipe_id>_archive`. This table will contain a list of all the files that have been ingested by the ClickPipe. This table is used to track files during ingestion and can be used to verify files have been ingested. The archive table has a [TTL](/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-ttl) of 7 days.
+## Tracking ingested files {#tracking-ingested-files}
 
-:::note
-These tables will not be visible using ClickHouse Cloud SQL Console, you will need to connect via an external client either using HTTPS or Native connection to read them.
-:::
+To track which files have been ingested include the `_file` [virtual column](/sql-reference/table-functions/s3#virtual-columns) in the field mappings. The `_file` virtual column contains the filename of the source object, making it easy to query and identify which files have been processed.
 
 ## Authentication {#authentication}
 
diff --git a/docs/integrations/data-visualization/index.md b/docs/integrations/data-visualization/index.md
@@ -35,7 +35,7 @@ Now that your data is in ClickHouse, it's time to analyze it, which often involv
 - [Explo](./community_integrations/explo-and-clickhouse.md)
 - [Fabi.ai](./community_integrations/fabi-and-clickhouse.md)
 - [Grafana](./grafana/index.md)
-- [Lightdash] (./lightdash-and-clickhouse.md)
+- [Lightdash](./lightdash-and-clickhouse.md)
 - [Looker](./looker-and-clickhouse.md)
 - [Luzmo](./community_integrations/luzmo-and-clickhouse.md)
 - [Metabase](./metabase-and-clickhouse.md)
@@ -75,4 +75,4 @@ Now that your data is in ClickHouse, it's time to analyze it, which often involv
 | [Rocket BI](./community_integrations/rocketbi-and-clickhouse.md)            | Native connector              | ✅      | ❌          |                                                                                                                                         |
 | [Tableau Desktop](./tableau/tableau-and-clickhouse.md)       | ClickHouse official connector | ✅      | ✅          |                                                                                                               |
 | [Tableau Online](./tableau/tableau-online-and-clickhouse.md) | MySQL interface               | ✅      | ✅          | Works with some limitations, see [the documentation](./tableau/tableau-online-and-clickhouse.md) for more details            |
-| [Zing Data](./community_integrations/zingdata-and-clickhouse.md)            | Native connector              | ✅      | ✅          |                                                                                                                                         |
+| [Zing Data](./community_integrations/zingdata-and-clickhouse.md)            | Native connector              | ✅      | ✅          |                                                                                                                                         |
diff --git a/docs/integrations/language-clients/java/client/_snippets/_v0_8.mdx b/docs/integrations/language-clients/java/client/_snippets/_v0_8.mdx
@@ -382,6 +382,128 @@ try (InsertResponse response = client.insert(TABLE_NAME, events).get()) {
 }
 ```
 
+### insert(String tableName, DataStreamWriter writer, ClickHouseFormat format, InsertSettings settings)
+**Beta** 
+
+This API method allows to pass a writer object that will encode data directly into an output stream. Data will be compressed by the client.
+There is a configuration option in `InsertSettings` called `appCompressedData` that allows to turn off client compression and let application to send compressed stream. 
+Examples shows major usecases this API was designed for. 
+
+`com.clickhouse.client.api.DataStreamWriter` is a functional interface with a method `onOutput` that is called by the client when output stream is ready for data to be written. This interface has
+another method `onRetry` with default implementation. This method is called when retry logic is triggered and mainly used to reset data source if applicable. 
+
+
+**Signatures**
+```java
+CompletableFuture<InsertResponse> insert(String tableName,              // name of destination table
+                                         DataStreamWriter writer,       // data writer instance 
+                                         ClickHouseFormat format,       // data format in which the writer encodes data 
+                                         InsertSettings settings)       // operation settings
+```
+
+**Parameters**
+
+`tableName` - name of the target table. 
+
+`writer` - data writer instance.
+
+`format` - data format in which the writer encodes data. 
+
+`settings` - request settings.
+
+**Return value**
+
+Future of `InsertResponse` type - the result of the operation and additional information like server side metrics.
+
+**Examples**
+
+Writing a collection of JSON objects encoded as string values using `JSONEachRow` format: 
+```java showLineNumbers
+
+final int EXECUTE_CMD_TIMEOUT = 10; // seconds
+final String tableName = "events";
+final String tableCreate = "CREATE TABLE \"" + tableName + "\" " +
+        " (name String, " +
+        "  v1 Float32, " +
+        "  v2 Float32, " +
+        "  attrs Nullable(String), " +
+        "  corrected_time DateTime('UTC') DEFAULT now()," +
+        "  special_attr Nullable(Int8) DEFAULT -1)" +
+        "  Engine = MergeTree ORDER by ()";
+
+client.execute("DROP TABLE IF EXISTS " + tableName).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS);
+client.execute(createTableSQL).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS);
+
+String correctedTime = Instant.now().atZone(ZoneId.of("UTC")).format(DataTypeUtils.DATETIME_FORMATTER);
+String[] rows = new String[] {
+        "{ \"name\": \"foo1\", \"v1\": 0.3, \"v2\": 0.6, \"attrs\": \"a=1,b=2,c=5\", \"corrected_time\": \"" + correctedTime + "\", \"special_attr\": 10}",
+        "{ \"name\": \"foo1\", \"v1\": 0.3, \"v2\": 0.6, \"attrs\": \"a=1,b=2,c=5\", \"corrected_time\": \"" + correctedTime + "\"}",
+        "{ \"name\": \"foo1\", \"v1\": 0.3, \"v2\": 0.6, \"attrs\": \"a=1,b=2,c=5\" }",
+        "{ \"name\": \"foo1\", \"v1\": 0.3, \"v2\": 0.6 }",
+};
+
+
+try (InsertResponse response = client.insert(tableName, out -> {
+    // writing raw bytes 
+    for (String row : rows) {
+        out.write(row.getBytes());
+    }
+
+}, ClickHouseFormat.JSONEachRow, new InsertSettings()).get()) {
+
+    System.out.println("Rows written: " + response.getWrittenRows());
+}
+
+```
+
+Writing already compressed data:
+```java showLineNumbers
+String tableName = "very_long_table_name_with_uuid_" + UUID.randomUUID().toString().replace('-', '_');
+String tableCreate = "CREATE TABLE \"" + tableName + "\" " +
+        " (name String, " +
+        "  v1 Float32, " +
+        "  v2 Float32, " +
+        "  attrs Nullable(String), " +
+        "  corrected_time DateTime('UTC') DEFAULT now()," +
+        "  special_attr Nullable(Int8) DEFAULT -1)" +
+        "  Engine = MergeTree ORDER by ()";
+
+client.execute("DROP TABLE IF EXISTS " + tableName).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS);
+client.execute(createTableSQL).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS);
+
+String correctedTime = Instant.now().atZone(ZoneId.of("UTC")).format(DataTypeUtils.DATETIME_FORMATTER);
+String[] data = new String[] {
+        "{ \"name\": \"foo1\", \"v1\": 0.3, \"v2\": 0.6, \"attrs\": \"a=1,b=2,c=5\", \"corrected_time\": \"" + correctedTime + "\", \"special_attr\": 10}",
+        "{ \"name\": \"foo1\", \"v1\": 0.3, \"v2\": 0.6, \"attrs\": \"a=1,b=2,c=5\", \"corrected_time\": \"" + correctedTime + "\"}",
+        "{ \"name\": \"foo1\", \"v1\": 0.3, \"v2\": 0.6, \"attrs\": \"a=1,b=2,c=5\" }",
+        "{ \"name\": \"foo1\", \"v1\": 0.3, \"v2\": 0.6 }",
+};
+
+
+// This step is only for showcase. Real application would have already compressed data. 
+byte[][] compressedData = new byte[data.length][];
+for (int i = 0 ; i < data.length; i++) {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    GZIPOutputStream gz = new GZIPOutputStream(baos);
+    gz.write(data[i].getBytes(StandardCharsets.UTF_8));
+    gz.finish();
+    compressedData[i] = baos.toByteArray();
+}
+
+InsertSettings insertSettings = new InsertSettings()
+        .appCompressedData(true, "gzip"); // defining compression algorithm (sent via HTTP headers)
+
+try (InsertResponse response = client.insert(tableName, out -> {
+    // Writing data 
+    for (byte[] row : compressedData) {
+        out.write(row);
+    }
+}, ClickHouseFormat.JSONEachRow, insertSettings).get()) {
+    System.out.println("Rows written: " + response.getWrittenRows());
+}    
+
+```
+
 ### InsertSettings {#insertsettings}
 
 Configuration options for insert operations.
diff --git a/docs/use-cases/observability/clickstack/deployment/hyperdx-clickhouse-cloud.md b/docs/use-cases/observability/clickstack/deployment/hyperdx-clickhouse-cloud.md
@@ -18,6 +18,7 @@ import hyperdx_cloud_landing from '@site/static/images/use-cases/observability/h
 import hyperdx_cloud_datasource from '@site/static/images/use-cases/observability/hyperdx_cloud_datasource.png';
 import hyperdx_create_new_source from '@site/static/images/use-cases/observability/hyperdx_create_new_source.png';
 import hyperdx_create_trace_datasource from '@site/static/images/use-cases/observability/hyperdx_create_trace_datasource.png';
+import read_only from '@site/static/images/clickstack/read-only-access.png';
 import { TrackedLink } from '@site/src/components/GalaxyTrackedLink/GalaxyTrackedLink';
 
 <PrivatePreviewBadge/>
@@ -286,6 +287,25 @@ For users looking to explore the HyperDX interface only, we recommend our [sampl
 
 <Image img={hyperdx_cloud_landing} alt="ClickHouse Cloud HyperDX Landing" size="lg"/>
 
+### User permissions {#user-permissions}
+
+Users accessing HyperDX are automatically authenticated using their ClickHouse Cloud console credentials. Access is controlled through SQL console permissions configured in the service settings.
+
+#### To configure user access {#configure-access}
+
+1. Navigate to your service in the ClickHouse Cloud console
+2. Go to **Settings** → **SQL Console Access**
+3. Set the appropriate permission level for each user:
+   - **Service Admin → Full Access** - Required for enabling alerts
+   - **Service Read Only → Read Only** - Can view observability data and create dashboards
+   - **No access** - Cannot access HyperDX
+
+<Image img={read_only} alt="ClickHouse Cloud Read Only"/>
+
+:::important Alerts require admin access
+To enable alerts, at least one user with **Service Admin** permissions (mapped to **Full Access** in the SQL Console Access dropdown) must log into HyperDX at least once. This provisions a dedicated user in the database that runs alert queries.
+:::
+
 ### Create a data source {#create-a-datasource}
 
 HyperDX is Open Telemetry native but not Open Telemetry exclusive - users can use their own table schemas if desired.
diff --git a/docs/use-cases/observability/clickstack/integration-examples/nginx-logs.md b/docs/use-cases/observability/clickstack/integration-examples/nginx-logs.md
@@ -1,10 +1,10 @@
 ---
 slug: /use-cases/observability/clickstack/integrations/nginx
-title: 'Monitoring Nginx Logs with ClickStack'
-sidebar_label: 'Nginx Logs'
+title: 'Monitoring nginx Logs with ClickStack'
+sidebar_label: 'nginx Logs'
 pagination_prev: null
 pagination_next: null
-description: 'Monitoring Nginx with ClickStack'
+description: 'Monitoring nginx with ClickStack'
 doc_type: 'guide'
 ---
 
@@ -16,7 +16,7 @@ import example_dashboard from '@site/static/images/clickstack/example-logs-dashb
 import log_view from '@site/static/images/clickstack/log-view.png';
 import search_view from '@site/static/images/clickstack/nginx-logs-search-view.png';
 
-# Monitoring Nginx Logs with ClickStack {#nginx-clickstack}
+# Monitoring nginx Logs with ClickStack {#nginx-clickstack}
 
 :::note[TL;DR]
 This guide shows you how to monitor nginx with ClickStack by configuring the OpenTelemetry collector to ingest nginx access logs. You'll learn how to:
diff --git a/docs/use-cases/observability/clickstack/integration-examples/nginx-traces.md b/docs/use-cases/observability/clickstack/integration-examples/nginx-traces.md
@@ -1,10 +1,10 @@
 ---
 slug: /use-cases/observability/clickstack/integrations/nginx-traces
-title: 'Monitoring Nginx Traces with ClickStack'
-sidebar_label: 'Nginx Traces'
+title: 'Monitoring nginx Traces with ClickStack'
+sidebar_label: 'nginx Traces'
 pagination_prev: null
 pagination_next: null
-description: 'Monitoring Nginx Traces with ClickStack'
+description: 'Monitoring nginx Traces with ClickStack'
 doc_type: 'guide'
 keywords: ['ClickStack', 'nginx', 'traces', 'otel']
 ---
@@ -16,7 +16,7 @@ import finish_import from '@site/static/images/clickstack/finish-trace-dashboard
 import example_dashboard from '@site/static/images/clickstack/example-trace-dashboard.png';
 import view_traces from '@site/static/images/clickstack/nginx-traces-search-view.png';
 
-# Monitoring Nginx Traces with ClickStack {#nginx-traces-clickstack}
+# Monitoring nginx Traces with ClickStack {#nginx-traces-clickstack}
 
 :::note[TL;DR]
 This guide shows you how to capture distributed traces from your existing nginx installation and visualize them in ClickStack. You'll learn how to:
diff --git a/plugins/frontmatter-validation/customParseFrontMatter.js b/plugins/frontmatter-validation/customParseFrontMatter.js
@@ -5,6 +5,14 @@ const fs = require('fs');
 // List to track files with issues
 let filesWithIssues = [];
 
+// Valid doc_type values enum
+const VALID_DOC_TYPES = [
+    'guide',
+    'reference',
+    'changelog',
+    'landing-page',
+];
+
 // Exceptions list configuration
 const EXCEPTIONS_FILE_PATH = path.join(process.cwd(), 'plugins/frontmatter-validation/frontmatter-exceptions.txt');
 let exceptionList = [];
@@ -140,6 +148,7 @@ async function customParseFrontMatter(params) {
     try {
         // Use Docusaurus's default parser to get the frontmatter data
         const parsedData = await defaultParseFrontMatter(params);
+        
         // Check for required fields
         const requiredFields = ['title', 'slug', 'description', 'doc_type'];
         for (const field of requiredFields) {
@@ -148,6 +157,13 @@ async function customParseFrontMatter(params) {
             }
         }
 
+        // Validate doc_type against enum
+        if (parsedData.frontMatter.doc_type) {
+            if (!VALID_DOC_TYPES.includes(parsedData.frontMatter.doc_type)) {
+                issues.push(`invalid doc_type '${parsedData.frontMatter.doc_type}'. Must be one of: ${VALID_DOC_TYPES.join(', ')}`);
+            }
+        }
+
         // Check optional fields format
         if (parsedData.frontMatter["keywords"] && !Array.isArray(parsedData.frontMatter["keywords"])) {
             issues.push('keywords must be a list');
@@ -376,5 +392,7 @@ module.exports = {
     },
     // Export exception list management functions
     reloadExceptions,
-    getExceptions: () => [...exceptionList]
+    getExceptions: () => [...exceptionList],
+    // Export valid doc types for reference
+    getValidDocTypes: () => [...VALID_DOC_TYPES]
 };
diff --git a/static/images/clickstack/read-only-access.png b/static/images/clickstack/read-only-access.png
