diff --git a/config/_default/config.toml b/config/_default/config.toml index 8ecc3ecb..ad7164a3 100644 --- a/config/_default/config.toml +++ b/config/_default/config.toml @@ -72,7 +72,7 @@ description = "Documentation for AxoSyslog, the scalable security data processor # The version number for the version of the docs represented in this doc set. # Used in the "version-banner" partial to display a version number for the # current doc set. - version = "4.19.0" + version = "4.20.0" version_menu_canonicallinks = true # A link to latest version of the docs. Used in the "version-banner" partial to @@ -172,11 +172,11 @@ description = "Documentation for AxoSyslog, the scalable security data processor [params.product] name = "AxoSyslog" abbrev = "AxoSyslog" -version = "4.19" +version = "4.20" # techversion includes patch version number, needed for install/image commands # configversion is needed in the config file examples -techversion = "4.19.1" -configversion = "4.19" +techversion = "4.20.0" +configversion = "4.20" syslog-ng = "syslog-ng" selinux = "SELinux" apparmor = "AppArmor" diff --git a/content/chapter-destinations/clickhouse/_index.md b/content/chapter-destinations/clickhouse/_index.md index 336587c0..cb8d75cb 100644 --- a/content/chapter-destinations/clickhouse/_index.md +++ b/content/chapter-destinations/clickhouse/_index.md @@ -79,6 +79,51 @@ This destination has the following options: {{< include-headless "chunk/option-destination-diskbuffer.md" >}} +## format() + +| | | +| -------- | -------------------------- | +| Type: | `JSONEachRow`, `JSONCompactEachRow`, or `Protobuf` | +| Default: | see description | + +Available in {{< product >}} 4.20 and later. + +*Description:* Specifies the data format to use when sending data to Clickhouse. + +By default, `format()` is set to: + +- `Protobuf` if the [`proto-var()`](#proto-var) option is set, or +- `JSONEachRow` if the [`json-var()`](#json-var) option is set. + +Starting with {{< product >}} 4.20, you can also use `format(JSONCompactEachRow)` (when `json-var()` is also set) to use a more compact, array-based JSON representation. For example: + +```shell +destination { + clickhouse ( + # ... + json-var(json("$my_filterx_json_variable")) + format("JSONCompactEachRow") + # ... + ); +}; +``` + +In the `JSONEachRow` format each line is a JSON object, making it more readable. For example: + +```json +{"id":1,"name":"foo","value":42} +{"id":2,"name":"bar","value":17} +``` + +In the `JSONCompactEachRow` format each row is a compact array-based row: + +```json +[1,"foo",42] +[2,"bar",17] +``` + +Note that if the data's actual format doesn't match the selected format, ClickHouse returns a `CANNOT_PARSE_INPUT_ASSERTION_FAILED` error message. + {{< include-headless "chunk/option-destination-frac-digits.md" >}} {{< include-headless "chunk/option-grpc-headers.md" >}} @@ -264,6 +309,8 @@ By default, sending data to ClickHouse doesn't propagate the type information of *Description:* The username used for authentication. +{{< include-headless "chunk/option-destination-worker-partition-buckets.md" >}} + {{< include-headless "chunk/option-destination-http-worker-partition-key.md" >}} diff --git a/content/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/_index.md b/content/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/_index.md index ef6cad4a..f410c4f7 100644 --- a/content/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/_index.md +++ b/content/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/_index.md @@ -422,6 +422,8 @@ In case the server on the specified URL returns a redirect request, {{% param "p {{% include-headless "chunk/option-destination-http-use-system-cert-store.md" %}} +{{< include-headless "chunk/option-destination-worker-partition-buckets.md" >}} + {{< include-headless "chunk/option-destination-http-worker-partition-key.md" >}} diff --git a/content/chapter-destinations/destination-loki/_index.md b/content/chapter-destinations/destination-loki/_index.md index 30fa9699..489242e5 100644 --- a/content/chapter-destinations/destination-loki/_index.md +++ b/content/chapter-destinations/destination-loki/_index.md @@ -167,6 +167,8 @@ loki( *Description:* The URL of the Loki endpoint, including the gRPC listen port of your Loki deployment. +{{< include-headless "chunk/option-destination-worker-partition-buckets.md" >}} + {{< include-headless "chunk/option-destination-http-worker-partition-key.md" >}} diff --git a/content/chapter-destinations/destination-s3/_index.md b/content/chapter-destinations/destination-s3/_index.md index 09b1fd2e..01ea8703 100644 --- a/content/chapter-destinations/destination-s3/_index.md +++ b/content/chapter-destinations/destination-s3/_index.md @@ -60,6 +60,17 @@ s3( All of these strategies can be used individually, or together. +The name of the object can be further modified by the following options: + +- [`object-key-suffix()`](#object-key-suffix): A custom suffix that comes after the timestamp/index added by the object creation strategies. +- [`compression()`](#compression): For compressed objects, `.gz` is appended to the very end of the object name. + +To summarize, the different options (if set) modify the name of the object in the following order: + +``` +object-key()object-key-timestamp()max-object-size()object-key-suffix().gz(if compression is enabled) +``` + ## Upload options {{% param "product.abbrev" %}} uploads objects using the multipart upload API. {{% param "product.abbrev" %}} composes chunks locally. When a chunk reaches the size set in `chunk-size()` (by default 5 MiB), the chunk is uploaded. When an object is finished, the multipart upload is completed and S3 merges the chunks. @@ -119,7 +130,9 @@ If you configure an invalid value, the default is used. | Type: | boolean | | Default: | `no` | -*Description:* Setting `compression(yes)` enables gzip compression, and implicitly adds a `.gz` suffix to the created object's key. You can set the level of the compression using the `compresslevel()` option (0-9). +*Description:* Setting `compression(yes)` enables gzip compression, and implicitly adds a `.gz` suffix to the very end of the created object's key. You can set the level of the compression using the `compresslevel()` option (0-9). + +{{< include-headless "chunk/destination-s3-object-name.md" >}} ## compresslevel() @@ -174,6 +187,8 @@ Available in {{< product >}} 4.8 and later. *Description:* The maximal size of the S3 object. If an object reaches this size, {{% param "product_name" %}} appends an index ("-1", "-2", ...) to the end of the object key and starts a new object after rotation. +{{< include-headless "chunk/destination-s3-object-name.md" >}} + ## max-pending-uploads() | | | @@ -193,7 +208,20 @@ Available in {{< product >}} 4.8 and later. | Type: | template | | Default: | N/A | -*Description:* The [object key](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html) (or key name), which uniquely identifies the object in an Amazon S3 bucket. Note that a suffix may be appended to this object key depending on the [naming strategies](#creating-objects) used. Example: `my-logs/${HOSTNAME}/`. +*Description:* The [object key](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html) (or key name), which uniquely identifies the object in an Amazon S3 bucket. Note that a suffix may be appended to this object key depending on the [naming strategies](#creating-objects) and other options used. Example: `my-logs/${HOSTNAME}/`. + +## object-key-suffix() + +| | | +| -------- | --------- | +| Type: | template | +| Default: | empty string | + +Available in {{< product >}} 4.20 and later. + +*Description:* A suffix added to the object key. + +{{< include-headless "chunk/destination-s3-object-name.md" >}} ## object-key-timestamp() @@ -204,6 +232,8 @@ Available in {{< product >}} 4.8 and later. *Description:* The `object-key-timestamp()` option can be used to set a datetime-related template, which is appended to the end of the object key, for example: `"${R_MONTH_ABBREV}${R_DAY}"`. When a log message arrives with a newer timestamp template resolution, the previous timestamped object gets finished and a new one is started with the new timestamp. If an older message arrives, it doesn`t reopen the old object, but starts a new object with the key having an index appended to the old object. +{{< include-headless "chunk/destination-s3-object-name.md" >}} + {{< include-headless "chunk/option-persist-name.md" >}} ## region() @@ -280,6 +310,17 @@ If you configure an invalid value, the default is used. *Description:* The number of {{% param "product_name" %}} worker threads that are used to upload data to S3 from this destination. +## use-checksum() + +| | | +| -------- | --------- | +| Type: | `when_supported` or `when_required` | +| Default: | `when_supported` | + +Available in {{< product >}} 4.20 and later. + +*Description:* Change the default checksum settings for S3 compatible solutions that don't support checksums. + ## template() | | | diff --git a/content/chapter-destinations/destination-syslog-ng-otlp/_index.md b/content/chapter-destinations/destination-syslog-ng-otlp/_index.md index de38efeb..fe7487ab 100644 --- a/content/chapter-destinations/destination-syslog-ng-otlp/_index.md +++ b/content/chapter-destinations/destination-syslog-ng-otlp/_index.md @@ -1,12 +1,12 @@ --- -title: "syslog-ng-otlp(): Forward logs to another node using OpenTelemetry" +title: "axosyslog-otlp(): Forward logs to another node using OpenTelemetry" weight: 6750 -driver: "syslog-ng-otlp()" +driver: "axosyslog-otlp()" short_description: "Forward logs to another node using OpenTelemetry" --- -Available in {{% param "product.abbrev" %}} version 4.4 and later. +Available in {{% param "product.abbrev" %}} version 4.12 and later. (From version 4.4 to 4.11, this driver was called `syslog-ng-otlp()`.) {{< include-headless "chunk/syslog-ng-otlp-intro.md" >}} @@ -72,6 +72,8 @@ The `syslog-ng-otlp()` destination has the following options. *Description:* The URL of the {{% param "product.abbrev" %}} receiver. +{{< include-headless "chunk/option-destination-worker-partition-buckets.md" >}} + {{< include-headless "chunk/option-destination-http-worker-partition-key.md" >}} diff --git a/content/chapter-destinations/google-bigquery/_index.md b/content/chapter-destinations/google-bigquery/_index.md index c7563b0b..77515c7d 100644 --- a/content/chapter-destinations/google-bigquery/_index.md +++ b/content/chapter-destinations/google-bigquery/_index.md @@ -193,6 +193,8 @@ Alternatively, you can set the schema with the [`protobuf-schema()`](#protobuf-s *Description:* The URL of the Google BigQuery where the logs are sent. +{{< include-headless "chunk/option-destination-worker-partition-buckets.md" >}} + {{< include-headless "chunk/option-destination-http-worker-partition-key.md" >}} {{< include-headless "chunk/option-destination-threaded-workers.md" >}} diff --git a/content/chapter-destinations/google-pubsub-grpc/_index.md b/content/chapter-destinations/google-pubsub-grpc/_index.md index 76b5b8e5..1ba38853 100644 --- a/content/chapter-destinations/google-pubsub-grpc/_index.md +++ b/content/chapter-destinations/google-pubsub-grpc/_index.md @@ -176,6 +176,8 @@ auth( *Description:* An alias for [`service_endpoint`](#service-endpoint). +{{< include-headless "chunk/option-destination-worker-partition-buckets.md" >}} + {{< include-headless "chunk/option-destination-http-worker-partition-key.md" >}} diff --git a/content/chapter-destinations/opentelemetry/_index.md b/content/chapter-destinations/opentelemetry/_index.md index 123ce7f5..5c6a7573 100644 --- a/content/chapter-destinations/opentelemetry/_index.md +++ b/content/chapter-destinations/opentelemetry/_index.md @@ -111,6 +111,8 @@ log non_otel_to_otel_tls { *Description:* The URL of the OpenTelemetry receiver. +{{< include-headless "chunk/option-destination-worker-partition-buckets.md" >}} + {{< include-headless "chunk/option-destination-http-worker-partition-key.md" >}} diff --git a/content/chapter-nonsequential-processing/_index.md b/content/chapter-nonsequential-processing/_index.md index 71d31bad..7473eac7 100644 --- a/content/chapter-nonsequential-processing/_index.md +++ b/content/chapter-nonsequential-processing/_index.md @@ -57,3 +57,7 @@ log { ``` Staring with {{< product >}} version 4.17, you can use the `batch-size()` option to specify how many consecutive messages should be processed by a single `parallelize()` worker. This ensures that this many messages preserve their order on the destination side, and also improves `parallelize()` performance. A value around 100 is recommended for `batch-size()`. Default value: `0` (batching is disabled). + + diff --git a/content/chapter-sources/opentelemetry/_index.md b/content/chapter-sources/opentelemetry/_index.md index 017355c5..20d467c6 100644 --- a/content/chapter-sources/opentelemetry/_index.md +++ b/content/chapter-sources/opentelemetry/_index.md @@ -37,6 +37,8 @@ log otel_forward_mode_alts { {{< include-headless "chunk/option-source-concurrent-requests.md" >}} +{{< include-headless "chunk/option-source-otlp-keep-alive.md" >}} + ## keep-hostname() The `opentelemetry()` source ignores this option and uses the address of the OTLP peer as the HOST. diff --git a/content/chapter-sources/source-syslog-ng-otlp/_index.md b/content/chapter-sources/source-syslog-ng-otlp/_index.md index 86c32ef7..ea46d520 100644 --- a/content/chapter-sources/source-syslog-ng-otlp/_index.md +++ b/content/chapter-sources/source-syslog-ng-otlp/_index.md @@ -1,12 +1,12 @@ --- -title: "syslog-ng-otlp(): Receive logs from another node using OpenTelemetry" +title: "axosyslog-otlp(): Receive logs from another node using OpenTelemetry" weight: 3950 -driver: "syslog-ng-otlp()" +driver: "axosyslog-otlp()" short_description: "Receive logs from another node using OpenTelemetry" --- -Available in {{% param "product.abbrev" %}} version 4.4 and later. +Available in {{% param "product.abbrev" %}} version 4.12 and later. (From version 4.4 to 4.11, this driver was called `syslog-ng-otlp()`.) {{< include-headless "chunk/syslog-ng-otlp-intro.md" >}} @@ -65,6 +65,8 @@ The `syslog-ng-otlp()` source has the following options. {{< include-headless "chunk/option-source-host-override.md" >}} +{{< include-headless "chunk/option-source-otlp-keep-alive.md" >}} + ## keep-hostname() The `syslog-ng-otlp()` source ignores this option and uses the hostname from the message as the `${HOST}`. diff --git a/content/headless/chunk/destination-s3-object-name.md b/content/headless/chunk/destination-s3-object-name.md new file mode 100644 index 00000000..78929a3f --- /dev/null +++ b/content/headless/chunk/destination-s3-object-name.md @@ -0,0 +1,4 @@ +--- +--- + +For a summary of how the different options affect the name of the object, see [Creating objects](#creating-objects). diff --git a/content/headless/chunk/option-destination-worker-partition-buckets.md b/content/headless/chunk/option-destination-worker-partition-buckets.md new file mode 100644 index 00000000..dd8deb94 --- /dev/null +++ b/content/headless/chunk/option-destination-worker-partition-buckets.md @@ -0,0 +1,11 @@ +--- +--- + +## worker-partition-buckets() + +| | | +| -------- | ------ | +| Type: | template | +| Default: | | + +*Description:* The `worker-partition-buckets()` option determines the number of worker threads used for the `worker-partition-key()`. Note that the number set by `worker-partition-buckets()` should be lower than the number of `workers()`. diff --git a/content/headless/chunk/option-source-otlp-keep-alive.md b/content/headless/chunk/option-source-otlp-keep-alive.md new file mode 100644 index 00000000..5e57c274 --- /dev/null +++ b/content/headless/chunk/option-source-otlp-keep-alive.md @@ -0,0 +1,14 @@ +--- +--- + + +## keep-alive() + +| | | +| -------- | --------- | +| Type: | `yes` or `no` | +| Default: | `yes` | + +Available in {{< product >}} 4.20 and later. + +*Description:* Client connections can be kept alive during reload, avoiding unnecessary retry backoffs and other error messages on the client side. diff --git a/content/whats-new/_index.md b/content/whats-new/_index.md index a28df3d1..a8a203d8 100644 --- a/content/whats-new/_index.md +++ b/content/whats-new/_index.md @@ -6,6 +6,17 @@ weight: 10 {{< include-headless "banner-new-to-axosyslog.md" >}} +## Version 4.20 (2025-11-20) + +- The `clickhouse()` destination now has a `format()` option, allowing you to send data in a [compact `JSONCompactEachRow`]({{< relref "/chapter-destinations/clickhouse/_index.md#format" >}}) format. +- The [`opentelemetry()`]({{< relref "/chapter-sources/opentelemetry/_index.md#keep-alive" >}}) and [`axosyslog-otlp()`]({{< relref "/chapter-sources/source-syslog-ng-otlp/_index.md#keep-alive" >}}) (formerly called `syslog-ng-otlp()`) sources now support the `keep-alive()` option, and is enabled by default. +- The [`s3()` destination]({{< relref "/chapter-destinations/destination-s3/_index.md" >}}) has two new options: + + - You can now set a suffix for your S3 objects using the [`object-key-suffix()` option]({{< relref "/chapter-destinations/destination-s3/_index.md#object-key-suffix" >}}). + - You can change the default checksum settings for S3 compatible solutions that don't support checksums using the [`use-checksum()` option]({{< relref "/chapter-destinations/destination-s3/_index.md#use-checksum" >}}). + +- The `http()` and other threaded destinations now have a [`worker-partition-buckets()` option]({{< relref "/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/_index.md#worker-partition-buckets" >}}) that determines the number of worker threads used for the `worker-partition-key()`. + ## Version 4.19 (2025-10-15) - The [`dict_to_pairs`]({{< relref "/filterx/function-reference.md#dict-to-pairs" >}}) FilterX function can convert a dictionary to a list of pairs. diff --git a/themes/docsy-axoflow b/themes/docsy-axoflow index 11db25e3..41948e98 160000 --- a/themes/docsy-axoflow +++ b/themes/docsy-axoflow @@ -1 +1 @@ -Subproject commit 11db25e31a98aa5d2d4737a80cb7789d8fa7b2a8 +Subproject commit 41948e98c3dddbda8b9d8eec0a1cc8b6652090bf