Merge pull request #3205 from ClickHouse/click_analytics

click analytics for search

Author: gingerwizard

.github/workflows/build-search.yml

@@ -15,8 +15,10 @@ env:
 
 jobs:
   update-search:
-    if: github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'update search') && github.event.pull_request.base.ref == 'main'
-    #if: contains(github.event.pull_request.labels.*.name, 'update search') # Updated to trigger directly on PRs with the label
+    if: |
+      github.event_name == 'workflow_dispatch' || 
+      github.event_name == 'schedule' || 
+      (github.event_name == 'pull_request' && github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'update search') && github.event.pull_request.base.ref == 'main')    #if: contains(github.event.pull_request.labels.*.name, 'update search') # Updated to trigger directly on PRs with the label
     runs-on: ubuntu-latest
 
     steps:

docs/en/engines/table-engines/integrations/s3queue.md

@@ -4,6 +4,8 @@ sidebar_position: 181
 sidebar_label: S3Queue
 ---
 
+import ScalePlanFeatureBadge from '@theme/badges/ScalePlanFeatureBadge'
+
 # S3Queue Table Engine
 
 This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ecosystem and allows streaming import. This engine is similar to the [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) engines, but provides S3-specific features.
@@ -194,6 +196,8 @@ Engine supports all s3 related settings. For more information about S3 settings
 
 ## S3 role-based access
 
+<ScalePlanFeatureBadge feature="S3 Role-Based Access" />
+
 The s3Queue table engine supports role-based access.
 Refer to the documentation [here](https://clickhouse.com/docs/en/cloud/security/secure-s3) for steps to configure a role to access your bucket.
 

package.json

@@ -52,7 +52,8 @@
     "remark-docusaurus-tabs": "^0.2.0",
     "remark-link-rewrite": "^1.0.7",
     "remark-math": "^6.0.0",
-    "sass": "^1.83.1"
+    "sass": "^1.83.1",
+    "search-insights": "^2.17.3"
   },
   "devDependencies": {
     "@argos-ci/cli": "^2.5.5",

scripts/search/README.md

@@ -61,3 +61,7 @@ Algolia configs to try:
 - minProximity - 1
 - minWordSizefor2Typos - 7
 - minWordSizefor1Typo- 3
+
+Implement:
+- per page ranking as metadata
+- omit page from index

scripts/search/compute_ndcg.py

@@ -1,6 +1,10 @@
 import csv
 import math
 import argparse
+from urllib.parse import urlparse
+from bs4 import BeautifulSoup
+
+import requests
 from algoliasearch.search.client import SearchClientSync
 
 ALGOLIA_INDEX_NAME = "clickhouse"
@@ -25,6 +29,33 @@ def compute_dcg(relevance_scores):
     return sum(rel / math.log2(idx + 2) for idx, rel in enumerate(relevance_scores))
 
 
+def verify_link(link):
+    if not link:
+        return True
+    """Verify that a given link is valid and exists, including checking anchor existence."""
+    parsed_url = urlparse(link)
+    base_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
+    anchor = parsed_url.fragment  # Extract anchor if present
+
+    try:
+        response = requests.get(base_url, timeout=10, allow_redirects=True)
+        response.raise_for_status()  # Raise an error for HTTP failures (4xx, 5xx)
+    except requests.RequestException as e:
+        print(f"Error: Unable to reach {base_url} - {e}")
+        exit(1)
+
+    # If no anchor, the base page is sufficient
+    if not anchor:
+        return True
+
+    # Parse the page and check for the anchor
+    soup = BeautifulSoup(response.text, 'html.parser')
+    element = soup.find(id=anchor) or soup.find(attrs={"name": anchor})  # Check for both `id` and `name`
+
+    if not element:
+        print(f"Error: Anchor #{anchor} not found in {base_url}")
+        exit(1)
+
 def compute_ndcg(expected_links, retrieved_links, k):
     """Compute normalized DCG."""
     relevance_scores = [1 if link in expected_links else 0 for link in retrieved_links[:k]]
@@ -36,14 +67,26 @@ def compute_ndcg(expected_links, retrieved_links, k):
     return dcg / idcg if idcg > 0 else 0
 
 
-def main(input_csv, detailed, k=3):
+def main(input_csv, detailed, validate, k=3):
     """Main function to compute nDCG for search terms in a CSV."""
     with open(input_csv, mode='r', newline='', encoding='utf-8') as file:
         reader = csv.reader(file)
         rows = list(reader)
     results = []
     total_ndcg = 0
 
+    if validate:
+        print("Validating links...")
+        for row in rows:
+            for link in row[1:4]:
+                if link:
+                    print(f"Checking link {link}...", end="")
+                    verify_link(link)
+                    print("OK")
+        print("All links validated.")
+    else:
+        print("Skipping link validation.")
+
     for row in rows:
         term = row[0]
         # Remove duplicates in expected links - can happen as some docs return same url
@@ -97,6 +140,12 @@ def main(input_csv, detailed, k=3):
         action="store_true",
         help="Print detailed results for each search term."
     )
+    parser.add_argument(
+        "-v",
+        "--validate",
+        action="store_true",
+        help="Validate links."
+    )
     args = parser.parse_args()
 
-    main(args.input_csv, args.detailed)
+    main(args.input_csv, args.detailed, args.validate)

scripts/search/index_pages.py

@@ -186,6 +186,14 @@ def extract_links_from_content(content):
     return re.findall(link_pattern, content)
 
 
+def remove_markdown_links(text):
+    # Remove inline Markdown links: [text](url)
+    text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', text)
+    # Remove autolinks: <http://example.com>
+    text = re.sub(r'<https?://[^>]+>', '', text)
+    return text
+
+
 # best effort at creating links between docs - handling both md and urls. Challenge here some files import others
 # e.g. /opt/clickhouse-docs/docs/en/sql-reference/formats.mdx - we don't recursively resolve here
 def update_page_links(directory, base_directory, page_path, url, content):
@@ -249,7 +257,8 @@ def parse_markdown_content(metadata, content):
             current_subdoc['type'] = 'lvl1'
             current_subdoc['object_id'] = custom_slugify(heading_slug)
             current_subdoc['hierarchy']['lvl1'] = current_h1
-            current_subdoc['hierarchy']['lvl0'] = current_h1 if metadata.get('title', '') == '' else metadata.get('title', '')
+            current_subdoc['hierarchy']['lvl0'] = current_h1 if metadata.get('title', '') == '' else metadata.get(
+                'title', '')
         elif line.startswith('## '):
             if current_subdoc:
                 yield from split_large_document(current_subdoc)
@@ -356,6 +365,7 @@ def process_markdown_directory(directory, base_directory):
                         sub_doc['anchor'] = anchor
                         update_page_links(directory, base_directory, metadata.get('file_path', ''), sub_doc['url'],
                                           sub_doc['content'])
+                        sub_doc['content'] = remove_markdown_links(sub_doc['content'])
                         yield sub_doc
 
 

scripts/search/results.csv

@@ -83,7 +83,7 @@ keeper,https://clickhouse.com/docs/en/guides/sre/keeper/clickhouse-keeper,https:
 type,https://clickhouse.com/docs/en/sql-reference/data-types,https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions,
 nullable,https://clickhouse.com/docs/en/sql-reference/data-types/nullable,https://clickhouse.com/docs/en/cloud/bestpractices/avoid-nullable-columns,https://clickhouse.com/docs/en/sql-reference/functions/functions-for-nulls
 projection,https://clickhouse.com/docs/en/sql-reference/statements/alter/projection,https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#projections,https://clickhouse.com/docs/knowledgebase/projection_example
-jdbc,https://clickhouse.com/docs/en/engines/table-engines/integrations/jdbc,https://clickhouse.com/docs/en/interfaces/jdbc,https://clickhouse.com/docs/en/integrations/java/jdbc-driver
+jdbc,https://clickhouse.com/docs/en/interfaces/jdbc,https://clickhouse.com/docs/en/integrations/java/jdbc-v2, https://clickhouse.com/docs/en/engines/table-engines/integrations/jdbc
 ifnull,https://clickhouse.com/docs/en/sql-reference/functions/functions-for-nulls#ifnull,https://clickhouse.com/docs/en/sql-reference/functions/conditional-functions,
 any,https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/any,https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/first_value,
 optimize,https://clickhouse.com/docs/en/sql-reference/statements/optimize,https://clickhouse.com/docs/en/optimize,https://clickhouse.com/docs/en/cloud/bestpractices/avoid-optimize-final
@@ -138,7 +138,7 @@ nested,https://clickhouse.com/docs/en/sql-reference/data-types/nested-data-struc
 sample,https://clickhouse.com/docs/en/sql-reference/statements/select/sample,,
 distinct,https://clickhouse.com/docs/en/sql-reference/statements/select/distinct,,
 hash,https://clickhouse.com/docs/en/sql-reference/functions/hash-functions,,
-codec,https://clickhouse.com/docs/en/sql-reference/statements/create/table#column_compression_codec,https://clickhouse.com/docs/en/data-compression/compression-in-clickhouse,https://clickhouse.com/docs/observability/use-cases/observability/schema-design#using-codecs
+codec,https://clickhouse.com/docs/en/sql-reference/statements/create/table#column_compression_codec,https://clickhouse.com/docs/en/data-compression/compression-in-clickhouse,https://clickhouse.com/docs/en/use-cases/observability/schema-design#using-codecs
 timestamp,https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions#timestamp,,
 drop,https://clickhouse.com/docs/en/sql-reference/statements/drop,,
 parquet,https://clickhouse.com/docs/en/integrations/data-formats/parquet,https://clickhouse.com/docs/knowledgebase/ingest-parquet-files-in-s3,
@@ -151,7 +151,7 @@ materi,https://clickhouse.com/docs/en/materialized-view,,
 max_threads,https://clickhouse.com/docs/en/operations/settings/settings#max_threads,,
 limit,https://clickhouse.com/docs/en/sql-reference/statements/select/limit,,
 toint,https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions,,
-shard,https://clickhouse.com/docs/concepts/concepts/glossary#shard,https://clickhouse.com/docs/en/concepts/glossary,
+shard,https://clickhouse.com/docs/en/concepts/glossary#shard,https://clickhouse.com/docs/en/concepts/glossary,
 timeout,https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings,https://clickhouse.com/docs/en/operations/settings/settings#timeout_overflow_mode,
 date_diff,https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions#date_diff,,
 default,https://clickhouse.com/docs/en/operations/settings/settings-users,https://clickhouse.com/docs/knowledgebase/remove-default-user,
@@ -181,7 +181,7 @@ config.xml,https://clickhouse.com/docs/en/operations/server-configuration-parame
 where,https://clickhouse.com/docs/en/sql-reference/statements/select/where,,
 replica,https://clickhouse.com/docs/en/architecture/horizontal-scaling,,
 last,https://clickhouse.com/docs/en/sql-reference/window-functions/last_value,,
-version,https://clickhouse.com/docs/en/sql-reference/functions/other-functions#version,https://clickhouse.com/docswhich-clickhouse-version-to-use-in-production,
+version,https://clickhouse.com/docs/en/sql-reference/functions/other-functions#version,https://clickhouse.com/docs/en/faq/operations/production,
 greatest,https://clickhouse.com/docs/en/sql-reference/functions/conditional-functions#greatest,https://clickhouse.com/docs/en/sql-reference/functions/other-functions#greatest,
 nan,https://clickhouse.com/docs/en/sql-reference/data-types/float#nan-and-inf,,
 log,https://clickhouse.com/docs/en/sql-reference/functions/math-functions#log,,
@@ -206,7 +206,8 @@ min_insert_block_size_rows,https://clickhouse.com/docs/en/operations/settings/se
 allow_experimental_parallel_reading_from_replicas,https://clickhouse.com/docs/en/operations/settings/settings#allow_experimental_parallel_reading_from_replicas,,
 join_algorithm,https://clickhouse.com/docs/en/operations/settings/settings#join_algorithm,,
 max_memory_usage,https://clickhouse.com/docs/en/operations/settings/settings#max_memory_usage,,
-max_bytes_before_external_group_by,https://clickhouse.com/docs/en/operations/settings/settings#max_bytes_before_external_group_by,,
+max_bytes_before_external_group_by,https://clickhouse.com/docs/en/operations/settings/settings#max_bytes_before_external_group_by,https://clickhouse.com/docs/en/operations/settings/query-complexity#settings-max_bytes_before_external_group_by,https://clickhouse.com/docs/en/sql-reference/statements/select/group-by#group-by-in-external-memory
 max_bytes_before_external_sort,https://clickhouse.com/docs/en/operations/settings/settings#max_bytes_before_external_sort,,
 result_overflow_mode,https://clickhouse.com/docs/en/operations/settings/settings#result_overflow_mode,,
 use_query_cache,https://clickhouse.com/docs/en/operations/settings/settings#use_query_cache,,
+date time best effort,https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#parsedatetimebesteffortusorzero,https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#parsedatetimebesteffort

scripts/search/settings.json

@@ -22,7 +22,9 @@
       "url_without_anchor",
       "type"
     ],
-    "camelCaseAttributes": [],
+    "camelCaseAttributes": [
+      "h2"
+    ],
     "attributeCriteriaComputedByMinProximity": false,
     "distinct": true,
     "unretrievableAttributes": null,

src/lib/google/google.js

@@ -0,0 +1,41 @@
+/**
+ * Extracts the user ID from the Google Analytics device ID.
+ * @example `GA1.1.xxxxxxxxxx.xxxxxxxxxx => xxxxxxxxxx-xxxxxxxxxx`
+ * @link https://support.google.com/analytics/answer/11397207
+ */
+const extractGoogleAnalyticsUserIdFromCookie = (gaCookie) => {
+    if (gaCookie) {
+        // Remove the Google Analytics tracker from the device ID.
+        const userIdParts = gaCookie.split('.').slice(-2);
+        if (userIdParts.length === 2) {
+        return userIdParts.join('-');
+        }
+    }
+    return undefined;
+};
+  
+const getBrowserCookie = (cookieName) => {
+    // In React Native environments, `document.cookie` doesn't exist.
+    if (typeof document !== 'object' || typeof document.cookie !== 'string') {
+      return undefined;
+    }
+    const name = cookieName + '=';
+    const decodedCookie = decodeURIComponent(document.cookie);
+    const ca = decodedCookie.split(';');
+    for (let i = 0; i < ca.length; i++) {
+      let c = ca[i].trim();
+      if (c.startsWith(name)) {
+        return c.substring(name.length);
+      }
+    }
+    return undefined;
+};
+  
+/**
+ * Returns the Google Analytics User ID from a browser cookie name.
+ * @example `getGoogleAnalyticsUserIdFromBrowserCookie('_ga')`
+ */
+export const getGoogleAnalyticsUserIdFromBrowserCookie = (cookieName) => {
+    const browserCookie = getBrowserCookie(cookieName);
+    return browserCookie ? extractGoogleAnalyticsUserIdFromCookie(browserCookie) : undefined;
+};