Skip to content

Commit 484399b

Browse files
BlargianBlargian
authored
Merge pull request #3572 from Blargian/beta_experimental_global_server_settings
Autogenerate settings refactor
2 parents abce100 + a40b7a7 commit 484399b

File tree

4 files changed

+164
-164
lines changed

4 files changed

+164
-164
lines changed

‎scripts/settings/autogenerate-settings.sh‎

Lines changed: 8 additions & 164 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ else
1616
exit 1
1717
fi
1818

19-
2019
target_dir=$(dirname "$(dirname "$(realpath "$0")")")
2120
SCRIPT_NAME=$(basename "$0")
21+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
2222
tmp_dir="$target_dir/tmp"
2323

2424
mkdir -p "$tmp_dir" || exit 1
@@ -41,173 +41,17 @@ fi
4141

4242
echo "Downloaded to: $script_path"
4343
echo "[$SCRIPT_NAME] Auto-generating settings"
44-
45-
# Autogenerate Format settings
4644
chmod +x "$script_path" || { echo "Error: Failed to set execute permission"; exit 1; }
47-
4845
root=$(dirname "$(dirname "$(realpath "$tmp_dir")")")
4946

50-
./clickhouse -q "
51-
WITH
52-
'FormatFactorySettings.h' AS cpp_file,
53-
settings_from_cpp AS
54-
(
55-
SELECT extract(line, 'DECLARE\\(\\w+, (\\w+),') AS name
56-
FROM file(cpp_file, LineAsString)
57-
WHERE match(line, '^\\s*DECLARE\\(')
58-
),
59-
main_content AS
60-
(
61-
SELECT format('## {} {}\\n{}\\n\\nType: \`{}\`\\n\\nDefault: \`{}\`\\n\\n{}\\n\\n',
62-
name, '{#'||name||'}', multiIf(tier == 'Experimental', '<ExperimentalBadge/>', tier == 'Beta', '<BetaBadge/>', ''), type, default, trim(BOTH '\\n' FROM description))
63-
FROM system.settings WHERE name IN settings_from_cpp
64-
ORDER BY name
65-
),
66-
'---
67-
title: ''Format Settings''
68-
sidebar_label: ''Format Settings''
69-
slug: /operations/settings/formats
70-
toc_max_heading_level: 2
71-
description: ''Settings which control input and output formats.''
72-
---
73-
74-
import ExperimentalBadge from \'@theme/badges/ExperimentalBadge\';
75-
import BetaBadge from \'@theme/badges/BetaBadge\';
76-
77-
<!-- Autogenerated -->
78-
These settings are autogenerated from [source](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/FormatFactorySettings.h).
79-
80-
' AS prefix
81-
SELECT prefix || (SELECT groupConcat(*) FROM main_content)
82-
INTO OUTFILE 'settings-formats.md' TRUNCATE FORMAT LineAsString
83-
" > /dev/null || { echo "Failed to Autogenerate Format settings"; exit 1; }
84-
85-
# Autogenerate settings
86-
./clickhouse -q "
87-
WITH
88-
'Settings.cpp' AS cpp_file,
89-
settings_from_cpp AS
90-
(
91-
SELECT extract(line, 'DECLARE\\(\\w+, (\\w+),') AS name
92-
FROM file(cpp_file, LineAsString)
93-
WHERE match(line, '^\\s*DECLARE\\(')
94-
),
95-
main_content AS
96-
(
97-
SELECT format('## {} {}\\n{}\\n{}\\n\\nType: {}\\n\\nDefault value: {}\\n\\n{}\\n\\n',
98-
name, '{#'||name||'}', multiIf(tier == 'Experimental', '<ExperimentalBadge/>', tier == 'Beta', '<BetaBadge/>', ''), if(description LIKE '%Only has an effect in ClickHouse Cloud%', '\\n<CloudAvailableBadge/>', ''), type, default, replaceOne(trim(BOTH '\\n' FROM description), ' and [MaterializedMySQL](../../engines/database-engines/materialized-mysql.md)',''))
99-
FROM system.settings WHERE name IN settings_from_cpp
100-
ORDER BY name
101-
),
102-
'---
103-
title: ''Session Settings''
104-
sidebar_label: ''Session Settings''
105-
slug: /operations/settings/settings
106-
toc_max_heading_level: 2
107-
description: ''Settings which are found in the ``system.settings`` table.''
108-
---
109-
110-
import ExperimentalBadge from \'@theme/badges/ExperimentalBadge\';
111-
import BetaBadge from \'@theme/badges/BetaBadge\';
112-
import CloudAvailableBadge from \'@theme/badges/CloudAvailableBadge\';
113-
114-
<!-- Autogenerated -->
115-
All below settings are also available in table [system.settings](/docs/operations/system-tables/settings). These settings are autogenerated from [source](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.cpp).
116-
117-
' AS prefix
118-
SELECT prefix || (SELECT groupConcat(*) FROM main_content)
119-
INTO OUTFILE 'settings.md' TRUNCATE FORMAT LineAsString
120-
" > /dev/null || { echo "Failed to Autogenerate Core settings"; exit 1; }
121-
122-
# Auto generate global server settings
123-
./clickhouse -q "
124-
WITH
125-
server_settings_outside_source AS
126-
(
127-
SELECT
128-
arrayJoin(extractAllGroups(raw_blob, '## (\\w+)(?:\\s[^\n]+)?\n\\s+((?:[^#]|#[^#]|##[^ ])+)')) AS g,
129-
g[1] AS name,
130-
replaceRegexpAll(replaceRegexpAll(g[2], '\n(Type|Default( value)?): [^\n]+\n', ''), '^\n+|\n+$', '') AS doc
131-
FROM file('_server_settings_outside_source.md', RawBLOB)
132-
),
133-
server_settings_in_source AS
134-
(
135-
SELECT
136-
name,
137-
replaceRegexpAll(description, '(?m)^[ \t]+', '') AS description,
138-
type,
139-
default
140-
FROM system.server_settings
141-
),
142-
combined_server_settings AS
143-
(
144-
SELECT
145-
name,
146-
description,
147-
type,
148-
default
149-
FROM server_settings_in_source
150-
UNION ALL
151-
SELECT
152-
name,
153-
doc AS description,
154-
'' AS type,
155-
'' AS default
156-
FROM server_settings_outside_source
157-
),
158-
formatted_settings AS
159-
(
160-
SELECT
161-
format(
162-
'## {} {}\n\n{}{}{}\n\n',
163-
name,
164-
lcase('{#'||name||'}'),
165-
if(type != '', concat('Type: \`', type, '\`\n\n'), ''),
166-
if(default != '', concat('Default: \`', default, '\`\n\n'), ''),
167-
description
168-
) AS formatted_text
169-
FROM combined_server_settings
170-
ORDER BY name ASC
171-
),
172-
prefix_text AS
173-
(
174-
SELECT
175-
'---
176-
description: ''This section contains descriptions of server settings i.e settings
177-
which cannot be changed at the session or query level.''
178-
keywords: [''global server settings'']
179-
sidebar_label: ''Server Settings''
180-
sidebar_position: 57
181-
slug: /operations/server-configuration-parameters/settings
182-
title: ''Server Settings''
183-
---
184-
185-
import Tabs from ''@theme/Tabs'';
186-
import TabItem from ''@theme/TabItem'';
187-
import SystemLogParameters from ''@site/docs/operations/server-configuration-parameters/_snippets/_system-log-parameters.md''
188-
189-
# Server Settings
190-
191-
This section contains descriptions of server settings. These are settings which
192-
cannot be changed at the session or query level.
193-
194-
For more information on configuration files in ClickHouse see [""Configuration Files""](/operations/configuration-files).
195-
196-
Other settings are described in the ""[Settings](/operations/settings/overview)"" section.
197-
Before studying the settings, we recommend reading the [Configuration files](/operations/configuration-files)
198-
section and note the use of substitutions (the `incl` and `optional` attributes).
199-
200-
' AS prefix_content
201-
)
202-
SELECT
203-
arrayStringConcat([
204-
(SELECT prefix_content FROM prefix_text),
205-
arrayStringConcat(groupArray(formatted_text), '')
206-
], '')
207-
FROM formatted_settings
208-
INTO OUTFILE 'server_settings.md'
209-
TRUNCATE FORMAT LineAsString" > /dev/null || { echo "Failed to Autogenerate Format settings"; exit 1; }
47+
# Autogenerate settings for all .sql files in directory
48+
for SQL_FILE in "$SCRIPT_DIR"/*.sql; do
49+
if [ -f "$SQL_FILE" ]; then
50+
./clickhouse --queries-file "$SQL_FILE" > /dev/null || { echo "Failed to generate some settings"; exit 1; }
51+
fi
52+
done
21053

54+
# move across files to where they need to be
21155
mv settings-formats.md "$root/docs/operations/settings" || { echo "Failed to move generated settings-format.md"; exit 1; }
21256
mv settings.md "$root/docs/operations/settings" || { echo "Failed to move generated settings.md"; exit 1; }
21357
mv server_settings.md "$root/docs/operations/server-configuration-parameters/settings.md" || { echo "Failed to move generated server_settings.md"; exit 1; }

‎scripts/settings/format-settings.sql‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
WITH
2+
'FormatFactorySettings.h' AS cpp_file,
3+
settings_from_cpp AS
4+
(
5+
SELECT extract(line, 'DECLARE\\(\\w+, (\\w+),') AS name
6+
FROM file(cpp_file, LineAsString)
7+
WHERE match(line, '^\\s*DECLARE\\(')
8+
),
9+
main_content AS
10+
(
11+
SELECT format('## {} {} {} \n\n|Type|Default|\n|---|---|\n|`{}`|`{}`|\n\n{}\n\n',
12+
name, '{#'||name||'}', multiIf(tier == 'Experimental', '<ExperimentalBadge/>', tier == 'Beta', '<BetaBadge/>', ''), type, default, trim(BOTH '\\n' FROM description))
13+
FROM system.settings WHERE name IN settings_from_cpp
14+
ORDER BY name
15+
),
16+
'' ||
17+
'---
18+
title: ''Format Settings''
19+
sidebar_label: ''Format Settings''
20+
slug: /operations/settings/formats
21+
toc_max_heading_level: 2
22+
description: ''Settings which control input and output formats.''
23+
---
24+
25+
import ExperimentalBadge from \'@theme/badges/ExperimentalBadge\';
26+
import BetaBadge from \'@theme/badges/BetaBadge\';
27+
28+
<!-- Autogenerated -->
29+
These settings are autogenerated from [source](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/FormatFactorySettings.h).
30+
31+
' AS prefix
32+
SELECT prefix || (SELECT groupConcat(*) FROM main_content)
33+
INTO OUTFILE 'settings-formats.md' TRUNCATE FORMAT LineAsString

‎scripts/settings/global-server-settings.sql‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
WITH
2+
server_settings_outside_source AS
3+
(
4+
SELECT
5+
arrayJoin(extractAllGroups(raw_blob, '## (\\w+)(?:\\s[^\n]+)?\n\\s+((?:[^#]|#[^#]|##[^ ])+)')) AS g,
6+
g[1] AS name,
7+
replaceRegexpAll(
8+
replaceRegexpAll(g[2], '\n(Type|Default( value)?): [^\n]+\n', ''), '^\n+|\n+$','') AS doc
9+
FROM file('_server_settings_outside_source.md', RawBLOB)),
10+
server_settings_in_source AS
11+
(
12+
SELECT
13+
name,
14+
replaceRegexpAll(description, '(?m)^[ \t]+', '') AS description,
15+
type,
16+
default
17+
FROM system.server_settings
18+
),
19+
combined_server_settings AS
20+
(
21+
SELECT
22+
name,
23+
description,
24+
type,
25+
default
26+
FROM server_settings_in_source
27+
UNION ALL
28+
SELECT
29+
name,
30+
doc AS description,
31+
'' AS type,
32+
'' AS default
33+
FROM server_settings_outside_source
34+
),
35+
formatted_settings AS
36+
(
37+
SELECT
38+
format(
39+
'## {}{}{}{}\n\n',
40+
name,
41+
lcase(' {#'|| name ||'} \n\n'),
42+
if(type != '' AND default != '', format('|Type|Default|\n|---|---|\n|`{}`|`{}`|\n\n',type, default), ''),
43+
description
44+
) AS formatted_text
45+
FROM combined_server_settings
46+
ORDER BY name ASC
47+
),
48+
'---
49+
description: ''This section contains descriptions of server settings i.e settings
50+
which cannot be changed at the session or query level.''
51+
keywords: [''global server settings'']
52+
sidebar_label: ''Server Settings''
53+
sidebar_position: 57
54+
slug: /operations/server-configuration-parameters/settings
55+
title: ''Server Settings''
56+
---
57+
58+
import Tabs from ''@theme/Tabs'';
59+
import TabItem from ''@theme/TabItem'';
60+
import SystemLogParameters from ''@site/docs/operations/server-configuration-parameters/_snippets/_system-log-parameters.md''
61+
62+
# Server Settings
63+
64+
This section contains descriptions of server settings. These are settings which
65+
cannot be changed at the session or query level.
66+
67+
For more information on configuration files in ClickHouse see [""Configuration Files""](/operations/configuration-files).
68+
69+
Other settings are described in the ""[Settings](/operations/settings/overview)"" section.
70+
Before studying the settings, we recommend reading the [Configuration files](/operations/configuration-files)
71+
section and note the use of substitutions (the `incl` and `optional` attributes).
72+
73+
' AS prefix_content
74+
SELECT
75+
arrayStringConcat(
76+
[
77+
prefix_content,
78+
arrayStringConcat(groupArray(formatted_text),'')
79+
],
80+
''
81+
)
82+
FROM formatted_settings
83+
INTO OUTFILE 'server_settings.md'
84+
TRUNCATE FORMAT LineAsString

‎scripts/settings/session-settings.sql‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
WITH
2+
'Settings.cpp' AS cpp_file,
3+
settings_from_cpp AS
4+
(
5+
SELECT extract(line, 'DECLARE\\(\\w+, (\\w+),') AS name
6+
FROM file(cpp_file, LineAsString)
7+
WHERE match(line, '^\\s*DECLARE\\(')
8+
),
9+
main_content AS
10+
(
11+
SELECT
12+
format('## {}{}{}{}{}{}\n\n',
13+
name,
14+
' {#'||name||'} \n\n',
15+
multiIf(tier == 'Experimental', '<ExperimentalBadge/>\n\n', tier == 'Beta', '<BetaBadge/>\n\n', ''),
16+
if(description LIKE '%Only has an effect in ClickHouse Cloud%', '<CloudAvailableBadge/>\n\n', ''),
17+
if(type != '' AND default != '', format('|Type|Default|\n|---|---|\n|`{}`|`{}`|\n\n',type, default), ''),
18+
replaceOne(trim(BOTH '\\n' FROM description), ' and [MaterializedMySQL](../../engines/database-engines/materialized-mysql.md)',''))
19+
FROM system.settings WHERE name IN settings_from_cpp
20+
ORDER BY name
21+
),
22+
'---
23+
title: ''Session Settings''
24+
sidebar_label: ''Session Settings''
25+
slug: /operations/settings/settings
26+
toc_max_heading_level: 2
27+
description: ''Settings which are found in the ``system.settings`` table.''
28+
---
29+
30+
import ExperimentalBadge from \'@theme/badges/ExperimentalBadge\';
31+
import BetaBadge from \'@theme/badges/BetaBadge\';
32+
import CloudAvailableBadge from \'@theme/badges/CloudAvailableBadge\';
33+
34+
<!-- Autogenerated -->
35+
All below settings are also available in table [system.settings](/docs/operations/system-tables/settings). These settings are autogenerated from [source](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.cpp).
36+
37+
' AS prefix
38+
SELECT prefix || (SELECT groupConcat(*) FROM main_content)
39+
INTO OUTFILE 'settings.md' TRUNCATE FORMAT LineAsString

0 commit comments

Comments
 (0)