merge main into branch and fix merge conflicts

Author: Blargian

.gitignore

@@ -45,6 +45,10 @@ docs/cloud/manage/api/keys-api-reference.md
 docs/cloud/manage/api/members-api-reference.md
 docs/cloud/manage/api/organizations-api-reference.md
 docs/cloud/manage/api/services-api-reference.md
+docs/cloud/manage/api/privateEndpointConfig-api-reference.md
+docs/cloud/manage/api/prometheus-api-reference.md
+docs/cloud/manage/api/usageCost-api-reference.md
+docs/whats-new/changelog/index.md
 .vscode
 .aspell.en.prepl
 *.md.bak

README.md

@@ -69,22 +69,16 @@ You can run a copy of this website locally within a few steps. Some folks find t
     # ✨  Done in 6.44s.
     ```
 
-1. Use Yarn to grab the latest documentation changes from the `ClickHouse/ClickHouse` repository:
+1. Use Yarn to grab the latest documentation changes from the `ClickHouse/ClickHouse` repository and build the documentation using `yarn-build`:
 
     ```shell
-    yarn copy-clickhouse-repo-docs
+    yarn build
 
     # Cloning into 'ClickHouse'...
     # ...
-    # Copying docs from ClickHouse ...
-    # Successfully executed copy from master
-    # ✨  Done in 18.56s.
-    ```
-
-    Alternatively, you can use a local copy of `ClickHouse/ClickHouse` if you already have that repository cloned locally with `-l`.
-
-    ```shell
-    yarn copy-clickhouse-repo-docs -l "/Users/johnny/clickhouse/"
+    # [SUCCESS] Generated static files in "build".
+    # [INFO] Use `npm run serve` command to test your build locally.
+    # ✨  Done in 105.96s.
     ```
 
 1. Start the local web-server:
@@ -104,6 +98,18 @@ You can run a copy of this website locally within a few steps. Some folks find t
 
 If you want to build a static copy of this repository that doesn't require a constant server running to view, you can use `yarn build` instead of `yarn start`. The `yarn build` will output a static copy of the website in the `/build` directory. This process takes around 10 minutes to complete on an M1 Macbook with 8GB RAM.
 
+1. Should you wish to make changes to documents which are located in the ClickHouse/ClickHouse repo and want to visualize what the changes made look like locally you can use `yarn copy-clickhouse-repo-docs -l` and provide a path to the ClickHouse repository on your local machine, for example:
+
+   ```
+   # yarn copy-clickhouse-repo-docs -l /Users/user/Desktop/ClickHouse
+   # Successfully executed local copy
+   # ✨  Done in 1.15s.
+   ```
+
+We recommend to install rsync in order to only copy what is needed, however the script will fallback to using `cp` if rsync is not available.
+
+Running `yarn copy-clickhouse-repo-docs` without any arguments will pull in the latest docs changes from github.
+
 ### Notes {#notes}
 
 Here are some things to keep in mind when building a local copy of the ClickHouse docs site.
@@ -126,6 +132,10 @@ Please assign any pull request (PR) against an issue; this helps the docs team t
 
 Check out the GitHub docs for a refresher on [how to create a pull request](https://docs.github.com/en/desktop/working-with-your-remote-repository-on-github-or-github-enterprise/creating-an-issue-or-pull-request-from-github-desktop).
 
+### Style guidelines
+
+For style guidelines, see ["Style guide"](/contribute/style-guide.md).
+
 ### Tests and CI/CD {#tests-and-cicd}
 
 There are five workflows that run against PRs in this repo:

clickhouseapi.js

@@ -87,7 +87,7 @@ function generateDocusaurusMarkdown(spec, groupedEndpoints, prefix) {
       if (operation.responses && operation.responses['200'].content["application/json"]) {
         const rawSchema = operation.responses['200'].content["application/json"].schema
         const result = rawSchema.properties.result
-
+        
         if (result) {
           markdownContent += `\n### Response\n\n`;
 
@@ -96,54 +96,77 @@ function generateDocusaurusMarkdown(spec, groupedEndpoints, prefix) {
           const schema = rawSchema.properties.result.type === 'array' ?
             result.items['$ref'].split('/').pop() : result['$ref'].split('/').pop()
 
-          const bodyParamAttrs = spec.components.schemas[schema].properties
-          const bodyParams = Object.keys(bodyParamAttrs)
-          const sampleResponseObj = {}
-
+          const extractedFields = extractFields(result, spec.components.schemas, undefined);
           markdownContent += `| Name | Type | Description |\n`
           markdownContent += `| :--- | :--- | :---------- |\n`
+          markdownContent += extractedFields.markdown
+          markdownContent += '\n'
+          markdownContent += `\n#### Sample response\n\n`;
+          markdownContent += '```\n'
+          markdownContent += `${JSON.stringify(extractedFields.json, 0, 2)}`
+          markdownContent += '\n```\n'
+      }
+    }
+    }
+  }
+
+  return markdownContent;
+}
+
+function extractFields(result, schemas, fieldPrefix) {
+  const schemaRef = result.type === 'array' ? result.items['$ref'].split('/').pop() : result['$ref'].split('/').pop();
+  const bodyParamAttrs = schemas[schemaRef].properties;
+  const bodyParams = Object.keys(bodyParamAttrs);
+  const resObj = {
+    markdown: '',
+    json: {}
+  }
+  for (const parameter of bodyParams) {
+      const newPrefix = fieldPrefix ? `${fieldPrefix}.${parameter}` : parameter;
+    if (bodyParamAttrs[parameter]['$ref']) {
+  
+      const nestedObj = extractFields(bodyParamAttrs[parameter], schemas, newPrefix)
+      resObj.markdown += nestedObj.markdown
+      resObj.json[parameter] = nestedObj.json
+    }
+    else {
+      const paramType = bodyParamAttrs[parameter].format || bodyParamAttrs[parameter].type;
+      resObj.markdown +=  `| ${newPrefix} | ${paramType || ''} | ${bodyParamAttrs[parameter].description || ''} | \n`;
+      resObj.json[parameter] = returnParamTypeSample(bodyParamAttrs[parameter].format || bodyParamAttrs[parameter].type);
+    }
+  }
+  return resObj;
+}
 
-          for (const parameter of bodyParams) {
-            const paramType = bodyParamAttrs[parameter].format || bodyParamAttrs[parameter].type
-            markdownContent += `| ${parameter} | ${paramType || ''} | ${bodyParamAttrs[parameter].description || ''} | \n`
-            
-            switch (paramType) {
-              case 'uuid':
-                sampleResponseObj[parameter] = 'uuid';
+function returnParamTypeSample(paramType) {
+  let result;
+  switch(paramType) {
+    case 'uuid':
+                result = 'uuid';
                 break;
               case 'string':
-                sampleResponseObj[parameter] = 'string';
+                result = 'string';
                 break;
               case 'number':
-                sampleResponseObj[parameter] = 0;
+                result = 0;
                 break;
               case 'array':
-                sampleResponseObj[parameter] = 'Array';
+                result = 'Array';
                 break;
               case 'boolean':
-                sampleResponseObj[parameter] = 'boolean';
+                result = 'boolean';
                 break;
               case 'date-time':
-                sampleResponseObj[parameter] = 'date-time';
+                result = 'date-time';
+                break;
+              case 'date':
+                result = 'date';
                 break;
               case 'email':
-                sampleResponseObj[parameter] = 'email';
+                result = 'email';
                 break;
-            }
-          }
-
-          markdownContent += `\n#### Sample response\n\n`;
-          markdownContent += '```\n'
-          markdownContent += `${JSON.stringify(sampleResponseObj, 0, 2)}`
-          markdownContent += '\n```\n'
-        }
-
-        
-      }
-    }
   }
-
-  return markdownContent;
+  return result;
 }
 
 async function main() {

contribute/style-guide.md

@@ -0,0 +1,125 @@
+# ClickHouse docs style guide
+
+In this document, you will find a number of style guidelines for writing documentation.
+The rules in this guide are intended to assist in helping us to create a high
+quality documentation offering on par with the quality of ClickHouse itself.
+
+## YAML front matter
+
+Begin every new markdown document with YAML front-matter:
+
+```markdown
+---
+title: Using a clickhouse-local database
+sidebar_label: Using clickhouse-local database
+slug: /chdb/guides/clickhouse-local
+description: Learn how to use a clickhouse-local database with chDB
+keywords: [chdb, clickhouse-local]
+---
+```
+
+## Explicit header tags
+
+Due to the way our translation system works, it is necessary to add an explicit
+anchor tag next to every header, such as `{#explicit-anchor-tag}`.
+
+For example:
+
+```markdown
+## My header {#my-header}
+```
+
+### Associated markdown rule or CI check
+- [`custom-anchor-headings`](/scripts/.markdownlint-cli2.yaml)
+
+## Images
+
+In all cases images get added to the `static/images` directory of the repository.
+Under the images directory you should place the images according to the folder
+structure of the docs.
+
+For example, if you wanted to add images to `clickhouse-local.md` which is 
+located at `/docs/chdb/guides/clickhouse-local.md`. You should add the 
+images at `/static/images/chdb/guides/`.
+
+Try to use descriptive names in lower case. For example:
+- `clickhouse-local-1.png`
+- `clickhouse-local-2.png`
+- `clickhouse-local-3.png` etc.
+
+In the markdown document import the images under the YAML frontmatter:
+
+```markdown
+---
+title: Using a clickhouse-local database
+sidebar_label: Using clickhouse-local database
+slug: /chdb/guides/clickhouse-local
+description: Learn how to use a clickhouse-local database with chDB
+keywords: [chdb, clickhouse-local]
+---
+
+import clickhouse-local-1 from '@site/static/images/chdb/guides/clickhouse-local-1.png'
+import clickhouse-local-2 from '@site/static/images/chdb/guides/clickhouse-local-2.png'
+import clickhouse-local-3 from '@site/static/images/chdb/guides/clickhouse-local-3.png'
+```
+
+Use the `<img/>` tag to place your image in the appropriate place:
+
+```markdown
+Here is some example text which refers to the image below:
+
+<img src={clickhouse-local-1}
+    alt='DESCRIPTION OF THE IMAGE'
+    style={{width: '800px'}} // optional
+/>
+
+Here is another paragraph...
+```
+
+## Codeblocks
+
+Codeblocks are defined using backticks. For example:
+
+```text
+\```sql title='Query'
+SELECT * FROM system.contributors;
+\```
+```
+
+Code blocks:
+- Should always have a language defined immediately next to the opening 3
+  backticks, without any space.
+- Have a title (optional) such as 'Query' or 'Response'
+- Use language `response` if it is for the result of a query.
+
+### Associated markdown rule or CI check
+
+- [`MD040` enforces that codeblocks have a language specified](/scripts/.markdownlint-cli2.yaml)
+
+## Broken links
+
+Making a change to a page slug or moving a file can result in broken links in
+numerous places across the docs. To mitigate this we use the built in link checker
+provided by Docusaurus. If broken links are found then docs check will fail with
+an error message something like this:
+
+```text
+Exhaustive list of all broken links found:
+
+- Broken link on source page path = /docs/observability/integrating-opentelemetry:
+   -> linking to /docs/observability/schema-design#using-maps
+- Broken link on source page path = /docs/operations/server-configuration-parameters/settings:
+   -> linking to ../../../engines/table-engines/mergetree-family/mergetree#mergetree-table-ttl (resolved as: /engines/table-engines/mergetree-family/mergetree#mergetree-table-ttl)
+- Broken link on source page path = /docs/partitions:
+   -> linking to /docs/optimize/sparse-primary-indexes#data-is-organized-into-granules-for-parallel-data-processing
+```
+
+To fix the links, find the source page, given after `source page path =` and search within it for the
+link given after `linking to`. As slugs are resolved relative to `/docs` you can exclude this from your
+search. E.g don't search `/docs/observability/schema-design#using-maps` but `/observability/schema-design#using-maps`
+
+**note** if you can't find the link on the page but you're sure that you are looking in the file 
+reported by the failing broken link checker, the link is most likely found in a snippet which is
+imported by that page. Find the snippet location from the import statement at the top of the page.
+
+

docs/_snippets/_GCS_authentication_and_bucket.md

@@ -1,43 +1,52 @@
 
+import GCS_bucket_1 from '@site/static/images/integrations/data-ingestion/s3/GCS-bucket-1.png';
+import GCS_bucket_2 from '@site/static/images/integrations/data-ingestion/s3/GCS-bucket-2.png';
+import GCS_create_service_account_key from '@site/static/images/integrations/data-ingestion/s3/GCS-create-a-service-account-key.png';
+import GCS_create_service_account_0 from '@site/static/images/integrations/data-ingestion/s3/GCS-create-service-account-0.png';
+import GCS_create_service_account_a from '@site/static/images/integrations/data-ingestion/s3/GCS-create-service-account-a.png';
+import GCS_create_service_account_2 from '@site/static/images/integrations/data-ingestion/s3/GCS-create-service-account-2.png';
+import GCS_create_service_account_3 from '@site/static/images/integrations/data-ingestion/s3/GCS-create-service-account-3.png';
+import GCS_guide_key from '@site/static/images/integrations/data-ingestion/s3/GCS-guide-key.png';
+
 <details>
     <summary>Create GCS buckets and an HMAC key</summary>
 
 ### ch_bucket_us_east1 {#ch_bucket_us_east1}
 
-![Add a bucket](@site/docs/integrations/data-ingestion/s3/images/GCS-bucket-1.png)
+<img src={GCS_bucket_1} alt="Creating a GCS bucket in US East 1" />
 
 ### ch_bucket_us_east4 {#ch_bucket_us_east4}
 
-![Add a bucket](@site/docs/integrations/data-ingestion/s3/images/GCS-bucket-2.png)
+<img src={GCS_bucket_2} alt="Creating a GCS bucket in US East 4" />
 
 ### Generate an Access key {#generate-an-access-key}
 
 ### Create a service account HMAC key and secret {#create-a-service-account-hmac-key-and-secret}
 
 Open **Cloud Storage > Settings > Interoperability** and either choose an existing **Access key**, or **CREATE A KEY FOR A SERVICE ACCOUNT**.  This guide covers the path for creating a new key for a new service account.
 
-![Add a bucket](@site/docs/integrations/data-ingestion/s3/images/GCS-create-a-service-account-key.png)
+<img src={GCS_create_service_account_key} alt="Generating a service account HMAC key in GCS" />
 
 ### Add a new service account {#add-a-new-service-account}
 
 If this is a project with no existing service account, **CREATE NEW ACCOUNT**.
 
-![Add a bucket](@site/docs/integrations/data-ingestion/s3/images/GCS-create-service-account-0.png)
+<img src={GCS_create_service_account_0} alt="Adding a new service account in GCS" />
 
 There are three steps to creating the service account, in the first step give the account a meaningful name, ID, and description.
 
-![Add a bucket](@site/docs/integrations/data-ingestion/s3/images/GCS-create-service-account-a.png)
+<img src={GCS_create_service_account_a} alt="Defining a new service account name and ID in GCS" />
 
 In the Interoperability settings dialog the IAM role **Storage Object Admin** role is recommended; select that role in step two.
 
-![Add a bucket](@site/docs/integrations/data-ingestion/s3/images/GCS-create-service-account-2.png)
+<img src={GCS_create_service_account_2} alt="Selecting IAM role Storage Object Admin in GCS" />
 
 Step three is optional and not used in this guide.  You may allow users to have these privileges based on your policies.
 
-![Add a bucket](@site/docs/integrations/data-ingestion/s3/images/GCS-create-service-account-3.png)
+<img src={GCS_create_service_account_3} alt="Configuring additional settings for the new service account in GCS" />
 
 The service account HMAC key will be displayed.  Save this information, as it will be used in the ClickHouse configuration.
 
-![Add a bucket](@site/docs/integrations/data-ingestion/s3/images/GCS-guide-key.png)
+<img src={GCS_guide_key} alt="Retrieving the generated HMAC key for GCS" />
 
 </details>