Merge pull request #3530 from Blargian/autogenerate_global_server_settings

Autogenerate global server settings

Author: Blargian

README.md

@@ -134,7 +134,12 @@ Check out the GitHub docs for a refresher on [how to create a pull request](http
 
 ### Style guidelines
 
-For style guidelines, see ["Style guide"](/contribute/style-guide.md).
+For documentation style guidelines, see ["Style guide"](/contribute/style-guide.md).
+
+### Generating documentation from source code
+
+For an overview of how reference documentation such as settings, system tables
+and functions are generated from the source code, see ["Generating documentation from source code"](/contribute/autogenerated-documentation-from-source.md)
 
 ### Tests and CI/CD {#tests-and-cicd}
 

contribute/autogenerated-documentation-from-source.md

@@ -0,0 +1,48 @@
+# Generating documentation from source code
+
+A [script in our docs repo](https://github.com/ClickHouse/clickhouse-docs/blob/main/scripts/settings/autogenerate-settings.sh) 
+extracts setting names, descriptions, default values, etc. from ClickHouse's source code.
+
+## Session settings
+
+Documentation for session settings are extracted from [`src/Core/Settings.cpp`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.cpp).
+
+## Format settings
+
+Documentation for session settings are extracted from [`src/Core/FormatFactorySettings.h`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/FormatFactorySettings.h).
+
+## Server settings
+
+Documentation for session settings are extracted from [`src/Core/ServerSettings.cpp`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/ServerSettings.cpp).
+
+Note that this file contains only a fraction of all server settings. 
+The reason is that server settings can be nested and `system.server_settings` 
+(which is built from `src/Core/ServerSettings.cpp`) and cannot represent nested
+settings.
+
+Example:
+
+```yaml
+<query_cache>
+    <max_size_in_bytes>1073741824</max_size_in_bytes>
+    <max_entries>1024</max_entries>
+    <max_entry_size_in_bytes>1048576</max_entry_size_in_bytes>
+    <max_entry_size_in_rows>30000000</max_entry_size_in_rows>
+</query_cache>
+```
+As a result of the above, you will find the server settings which are not found 
+in `system.server_settings` documented in file `_server_settings_outside_source.md`
+
+The auto-generation script reads these in, combines them with the ones from 
+`system.settings` and appends the formatted settings to `settings.md`.
+
+**As such, if you need to make a change to the settings you see on the 
+[server settings](clickhouse.com/docs/operations/server-configuration-parameters/) 
+page, you will need to check if the setting is in `system.server_settings`. 
+If it is, then please edit the setting description in the source code documentation
+in [`ServerSettings.cpp`](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/ServerSettings.cpp) 
+or else edit `_server_settings_outside_source.md`.**
+
+## System tables
+
+## Functions

package.json

@@ -4,7 +4,7 @@
   "private": true,
   "config": {
     "clickhouse_repo_folders": "docs/en/development docs/en/engines docs/en/getting-started docs/en/interfaces docs/en/operations docs/en/sql-reference",
-    "autogen_needed_files": "src/Core/FormatFactorySettings.h src/Core/Settings.cpp CHANGELOG.md"
+    "autogen_needed_files": "src/Core/FormatFactorySettings.h src/Core/Settings.cpp CHANGELOG.md docs/en/operations/server-configuration-parameters/_server_settings_outside_source.md"
   },
   "scripts": {
     "build": "yarn copy-clickhouse-repo-docs && yarn run-markdown-linter && yarn generate-changelog && yarn autogenerate-settings && yarn autogenerate-table-of-contents && yarn build-api-doc && yarn build-swagger && yarn build-docs",

scripts/settings/autogenerate-settings.sh

@@ -119,8 +119,85 @@ SELECT prefix || (SELECT groupConcat(*) FROM main_content)
 INTO OUTFILE 'settings.md' TRUNCATE FORMAT LineAsString
 " > /dev/null || { echo "Failed to Autogenerate Core settings"; exit 1; }
 
+# Auto generate global server settings
+./clickhouse -q "
+WITH
+    server_settings_outside_source AS
+    (
+        SELECT
+            arrayJoin(extractAllGroups(raw_blob, '## (\\w+)(?:\\s[^\n]+)?\n\\s+((?:[^#]|#[^#]|##[^ ])+)')) AS g,
+            g[1] AS name,
+            replaceRegexpAll(replaceRegexpAll(g[2], '\n(Type|Default( value)?): [^\n]+\n', ''), '^\n+|\n+$', '') AS doc
+        FROM file('_server_settings_outside_source.md', RawBLOB)
+    ),
+    server_settings_in_source AS
+    (
+        SELECT
+	        name,
+	        replaceRegexpAll(description, '(?m)^[ \t]+', '') AS description
+        FROM system.server_settings
+    ),
+    combined_server_settings AS
+    (
+        SELECT
+            name,
+            description
+        FROM server_settings_in_source
+        UNION ALL
+        SELECT
+            name,
+            doc AS description
+        FROM server_settings_outside_source
+    ),
+    formatted_settings AS
+    (
+        SELECT
+            format('## {} {}\n\n{}\n\n', name, lcase('{#'||name||'}'), description) AS formatted_text
+        FROM combined_server_settings
+        ORDER BY name ASC
+    ),
+    prefix_text AS
+    (
+        SELECT
+            '---
+description: ''This section contains descriptions of server settings i.e settings
+  which cannot be changed at the session or query level.''
+keywords: [''global server settings'']
+sidebar_label: ''Server Settings''
+sidebar_position: 57
+slug: /operations/server-configuration-parameters/settings
+title: ''Server Settings''
+---
+
+import Tabs from ''@theme/Tabs'';
+import TabItem from ''@theme/TabItem'';
+import SystemLogParameters from ''@site/docs/operations/server-configuration-parameters/_snippets/_system-log-parameters.md''
+
+# Server Settings
+
+This section contains descriptions of server settings. These are settings which
+cannot be changed at the session or query level.
+
+For more information on configuration files in ClickHouse see [""Configuration Files""](/operations/configuration-files).
+
+Other settings are described in the ""[Settings](/operations/settings/overview)"" section.
+Before studying the settings, we recommend reading the [Configuration files](/operations/configuration-files)
+section and note the use of substitutions (the `incl` and `optional` attributes).
+
+' AS prefix_content
+    )
+SELECT
+    arrayStringConcat([
+        (SELECT prefix_content FROM prefix_text),
+        arrayStringConcat(groupArray(formatted_text), '')
+    ], '')
+FROM formatted_settings
+INTO OUTFILE 'server_settings.md'
+TRUNCATE FORMAT LineAsString" > /dev/null || { echo "Failed to Autogenerate Format settings"; exit 1; }
+
 mv settings-formats.md "$root/docs/operations/settings" || { echo "Failed to move generated settings-format.md"; exit 1; }
 mv settings.md "$root/docs/operations/settings" || { echo "Failed to move generated settings.md"; exit 1; }
+mv server_settings.md "$root/docs/operations/server-configuration-parameters/settings.md" || { echo "Failed to move generated server_settings.md"; exit 1; }
 
 echo "[$SCRIPT_NAME] Auto-generation of settings markdown pages completed successfully"