diff --git a/.github/workflows/publish-version-5.4.yaml b/.github/workflows/publish-version-5.4.yaml new file mode 100644 index 000000000..3202e3502 --- /dev/null +++ b/.github/workflows/publish-version-5.4.yaml @@ -0,0 +1,86 @@ +name: Publish version 5.4 + +env: + doc_versionnumber: "5.4" + +on: + push: + branches: + - release-5.4 + workflow_dispatch: + +jobs: + build: + name: Build + runs-on: ubuntu-latest + + permissions: + contents: write + pages: write + id-token: write + + concurrency: + group: "pages" + cancel-in-progress: false + + environment: + name: github-pages-test + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: release-5.4 + submodules: 'recursive' + + - name: Set up Pages + id: pages + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 + + - name: Set up Hugo + uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0 + with: + hugo-version: '0.110.0' + extended: true + + - name: Set up Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 18 + + - name: Install dependencies + run: | + cd themes/docsy + npm install + + - name: Set up PostCSS + run: npm install --save-dev autoprefixer postcss-cli postcss + + - name: Build + run: hugo --environment production --baseURL ${{ steps.pages.outputs.base_url }}/${{ env.doc_versionnumber }}/ + + # - name: Upload artifact + # uses: actions/upload-pages-artifact@64bcae551a7b18bcb9a09042ddf1960979799187 # v1.0.8 + # with: + # path: ./public/ + + - name: Checkout code to update + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: 'gh-pages-test' + path: 'tmp/gh-pages' + # - name: Display file structure + # run: ls -R + - name: Copy built site to GH pages + run: | + rm -rf tmp/gh-pages/${{ env.doc_versionnumber }} + mkdir -p tmp/gh-pages/${{ env.doc_versionnumber }} + mv public/* tmp/gh-pages/${{ env.doc_versionnumber }} + - name: Commit & Push changes + uses: actions-js/push@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + message: 'Publish updated docs for ${{ env.doc_versionnumber }}, ${{ github.event.repository.pushed_at}}' + branch: 'gh-pages-test' + directory: 'tmp/gh-pages' diff --git a/config/_default/config.toml b/config/_default/config.toml index ccb1c1bef..1312b0e56 100644 --- a/config/_default/config.toml +++ b/config/_default/config.toml @@ -65,7 +65,7 @@ weight = 1 # Used in the "version-banner" partial to display a version number for the # current doc set. - version = "5.3" + version = "5.4" version_menu = "Releases" version_menu_canonicallinks = true version_menu_pagelinks = true @@ -169,9 +169,13 @@ twitter = "AxoflowIO" ####################### # Add your release versions here [[params.versions]] - version = "latest (5.3)" + version = "latest (5.4)" githubbranch = "master" url = "" +[[params.versions]] + version = "5.3" + githubbranch = "release-5.3" + url = "/5.3/" [[params.versions]] version = "5.2" githubbranch = "release-5.2" diff --git a/content/docs/configuration/crds/v1beta1/_index.md b/content/docs/configuration/crds/v1beta1/_index.md index 29970c748..57dfbcfb7 100644 --- a/content/docs/configuration/crds/v1beta1/_index.md +++ b/content/docs/configuration/crds/v1beta1/_index.md @@ -8,6 +8,7 @@ For more information please click on the name | Name | Description | Version | |---|---|---| +| **[AxoSyslog](axosyslog_types/)** | AxoSyslog is the schema for the AxoSyslogs API. | v1beta1 | | **[EventTailer](../extensions/eventtailer_types/)** | Eventtailer's main goal is to listen kubernetes events and transmit their changes to stdout. This way the logging-operator is able to process them. | extensions | | **[HostTailer](../extensions/hosttailer_types/)** | HostTailer's main goal is to tail custom files and transmit their changes to stdout. This way the logging-operator is able to process them. | extensions | | **[ClusterFlow](clusterflow_types/)** | ClusterFlow is the Schema for the clusterflows API | v1beta1 | diff --git a/content/docs/configuration/crds/v1beta1/axosyslog_types.md b/content/docs/configuration/crds/v1beta1/axosyslog_types.md new file mode 100644 index 000000000..75be3f487 --- /dev/null +++ b/content/docs/configuration/crds/v1beta1/axosyslog_types.md @@ -0,0 +1,122 @@ +--- +title: AxoSyslog +weight: 200 +generated_file: true +--- + +AxoSyslog is the schema for the AxoSyslogs API. + +Available in Logging operator version 5.4 and later. + +### (metav1.TypeMeta, required) {#axosyslog-} + + +### metadata (metav1.ObjectMeta, optional) {#axosyslog-metadata} + + +### spec (AxoSyslogSpec, optional) {#axosyslog-spec} + + +### status (AxoSyslogStatus, optional) {#axosyslog-status} + + + +## AxoSyslogSpec + +AxoSyslogSpec defines the desired state of AxoSyslog. + +### destinations ([]Destination, optional) {#axosyslogspec-destinations} + +Destinations is a list of destinations to be rendered in the AxoSyslog configuration. + + +### logPaths ([]LogPath, optional) {#axosyslogspec-logpaths} + +LogPaths is a list of log paths to be rendered in the AxoSyslog configuration. + + + +## LogPath + +LogPath defines a single log path that will be rendered in the AxoSyslog configuration. + +### destination (string, optional) {#logpath-destination} + +Name of a destination to be used in the log path. + + +### filterx (string, optional) {#logpath-filterx} + +FilterX block to be rendered within the log path. + + + +## Destination + +Destination defines a single destination that will be rendered in the AxoSyslog configuration. + +### config (string, optional) {#destination-config} + +Config is the configuration for the destination. + + +### name (string, optional) {#destination-name} + +Name of the destination. + + + +## AxoSyslogStatus + +AxoSyslogStatus defines the observed state of AxoSyslog. + +### problems ([]string, optional) {#axosyslogstatus-problems} + +Problems with the AxoSyslog resource. + + +### problemsCount (int, optional) {#axosyslogstatus-problemscount} + +Number of problems with the AxoSyslog resource. + + +### sources ([]Source, optional) {#axosyslogstatus-sources} + +Sources configured for AxoSyslog. + + + +## Source + +Source represents the log sources for AxoSyslog. + +### otlp (*OTLPSource, optional) {#source-otlp} + +OTLP specific configuration. + + + +## OTLPSource + +OTLPSource contains configuration for OpenTelemetry Protocol sources. + +### endpoint (string, optional) {#otlpsource-endpoint} + +Endpoint for the OTLP source. + + + +## AxoSyslogList + +AxoSyslogList contains a list of AxoSyslog + +### (metav1.TypeMeta, required) {#axosysloglist-} + + +### metadata (metav1.ListMeta, optional) {#axosysloglist-metadata} + + +### items ([]AxoSyslog, required) {#axosysloglist-items} + + + diff --git a/content/docs/configuration/crds/v1beta1/fluentbit_types.md b/content/docs/configuration/crds/v1beta1/fluentbit_types.md index 1d3c927b4..0cc63bd86 100644 --- a/content/docs/configuration/crds/v1beta1/fluentbit_types.md +++ b/content/docs/configuration/crds/v1beta1/fluentbit_types.md @@ -127,9 +127,16 @@ Disable Kubernetes metadata filter ### disableVarLibDockerContainers (*bool, optional) {#fluentbitspec-disablevarlibdockercontainers} +Available in Logging operator version 5.4 and later. + DisableVarLibDockerContainers controls whether the /var/lib/docker/containers volume is mounted. If true, the volume is NOT mounted. If false (default), the volume is mounted. +### disableVarLog (*bool, optional) {#fluentbitspec-disablevarlog} + +DisableVarLog controls whether the /var/log volume is mounted. If true, the volume is NOT mounted. If false (default), the volume is mounted. + + ### enableUpstream (bool, optional) {#fluentbitspec-enableupstream} @@ -675,6 +682,30 @@ Allow Kubernetes Pods to suggest a pre-defined Parser (read more about it in Kub Default: Off +Important: When enabling this setting, you must also configure +`inputTail.multiline.parser` with appropriate parsers (typically including `cri` for +Container Runtime Interface logs). Without this configuration, the annotation-based +parsing will not work correctly. + +#### Example Configuration + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: FluentbitAgent +metadata: + name: example-fluentbit +spec: + inputTail: + multiline.parser: [cri] # Required when K8S-Logging.Parser is enabled + filterKubernetes: + K8S-Logging.Parser: "On" + Merge_Log_Key: "parsed" # Optional but recommended to prevent key conflicts + + # Once configured, you can use annotations on your pods: + annotations: + fluentbit.io/parser: "my-custom-parser" +``` + ### Keep_Log (string, optional) {#filterkubernetes-keep_log} When Keep_Log is disabled, the log field is removed from the incoming message once it has been successfully merged (Merge_Log must be enabled as well). diff --git a/content/docs/configuration/crds/v1beta1/output_types.md b/content/docs/configuration/crds/v1beta1/output_types.md index 0d454c153..6ac992464 100644 --- a/content/docs/configuration/crds/v1beta1/output_types.md +++ b/content/docs/configuration/crds/v1beta1/output_types.md @@ -47,6 +47,9 @@ OutputSpec defines the desired state of Output ### kinesisStream (*output.KinesisStreamOutputConfig, optional) {#outputspec-kinesisstream} +### lmLogs (*output.LMLogsOutputConfig, optional) {#outputspec-lmlogs} + + ### logdna (*output.LogDNAOutput, optional) {#outputspec-logdna} diff --git a/content/docs/configuration/plugins/outputs/elasticsearch.md b/content/docs/configuration/plugins/outputs/elasticsearch.md index 1e08a7be2..09c3b7d0f 100644 --- a/content/docs/configuration/plugins/outputs/elasticsearch.md +++ b/content/docs/configuration/plugins/outputs/elasticsearch.md @@ -109,6 +109,12 @@ Specify an existing index template for the data stream. If not present, a new te Default: data_stream_name +### data_stream_template_use_index_patterns_wildcard (*bool, optional) {#elasticsearch-data_stream_template_use_index_patterns_wildcard} + +https://github.com/uken/fluent-plugin-elasticsearch?tab=readme-ov-file#data_stream_template_use_index_patterns_wildcard + +Default: true + ### default_elasticsearch_version (string, optional) {#elasticsearch-default_elasticsearch_version} This parameter changes that ES plugin assumes default Elasticsearch version. diff --git a/content/docs/configuration/plugins/outputs/forward.md b/content/docs/configuration/plugins/outputs/forward.md index fc8b56eab..4fa1a45a9 100644 --- a/content/docs/configuration/plugins/outputs/forward.md +++ b/content/docs/configuration/plugins/outputs/forward.md @@ -17,6 +17,11 @@ Default: 190 [Buffer](../buffer/) +### compress (string, optional) {#forwardoutput-compress} + +Fluentd supports transparent data compression, the only value accepted is "gzip" + + ### connect_timeout (int, optional) {#forwardoutput-connect_timeout} The timeout time for socket connect. When the connection timed out during establishment, Errno::ETIMEDOUT is raised. diff --git a/content/docs/configuration/plugins/outputs/lm_logs.md b/content/docs/configuration/plugins/outputs/lm_logs.md new file mode 100644 index 000000000..c114e50d1 --- /dev/null +++ b/content/docs/configuration/plugins/outputs/lm_logs.md @@ -0,0 +1,109 @@ +--- +title: LogicMonitor Logs +weight: 200 +generated_file: true +--- + +## Overview + +The LogicMonitor Logs output plugin for Fluentd sends log records to LogicMonitor Logs via the LM API. For details, see [https://github.com/logicmonitor/lm-logs-fluentd](https://github.com/logicmonitor/lm-logs-fluentd). + +## Example output configurations + +```yaml +spec: + lmLogs: + company_name: mycompany + access_id: + valueFrom: + secretKeyRef: + name: lm-credentials + key: access_id + access_key: + valueFrom: + secretKeyRef: + name: lm-credentials + key: access_key + resource_mapping: '{"kubernetes.host": "system.hostname"}' + flush_interval: 60s + debug: false +``` + + +## Configuration + +### access_id (*secret.Secret, optional) {#logicmonitorlogs-access_id} + +LM API Token access ID [Secret](../secret/) + + +### access_key (*secret.Secret, optional) {#logicmonitorlogs-access_key} + +LM API Token access key [Secret](../secret/) + + +### bearer_token (*secret.Secret, optional) {#logicmonitorlogs-bearer_token} + +LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all three are specified, `LMv1 token(access_id and access_key)` will be used for authentication with LogicMonitor [Secret](../secret/) + + +### buffer (*Buffer, optional) {#logicmonitorlogs-buffer} + +[Buffer](../buffer/) + + +### company_domain (string, optional) {#logicmonitorlogs-company_domain} + +LogicMonitor account domain. For example, for the `test.logicmonitor.com` URL, `company_domain` is `logicmonitor.com` + +Default: `logicmonitor.com` + +### company_name (string, required) {#logicmonitorlogs-company_name} + +LogicMonitor account name + + +### debug (*bool, optional) {#logicmonitorlogs-debug} + +When true, the plugin logs more information to the Fluentd console + + +### device_less_logs (*bool, optional) {#logicmonitorlogs-device_less_logs} + +When true, do not map logs with any resource. Record must have service when true + +Default: false + +### flush_interval (string, optional) {#logicmonitorlogs-flush_interval} + +Defines the time in seconds to wait before sending batches of logs to LogicMonitor (default: 60s) + +Default: 60s + +### force_encoding (string, optional) {#logicmonitorlogs-force_encoding} + +Specify charset when logs contains invalid utf-8 characters + + +### format (*Format, optional) {#logicmonitorlogs-format} + +[Format](../format/) + + +### http_proxy (string, optional) {#logicmonitorlogs-http_proxy} + +http proxy string eg. http://user:pass@proxy.server:port + + +### include_metadata (*bool, optional) {#logicmonitorlogs-include_metadata} + +When true, appends additional metadata to the log + +Default: false + +### resource_mapping (string, required) {#logicmonitorlogs-resource_mapping} + +The mapping that defines the source of the log event to the LM resource. In this case, the `` in the incoming event is mapped to the value of` ` + + + diff --git a/content/docs/configuration/plugins/outputs/vmware_loginsight.md b/content/docs/configuration/plugins/outputs/vmware_loginsight.md index b7abfe92a..ce170c81f 100644 --- a/content/docs/configuration/plugins/outputs/vmware_loginsight.md +++ b/content/docs/configuration/plugins/outputs/vmware_loginsight.md @@ -161,7 +161,7 @@ Default: json ### shorten_keys (map[string]string, optional) {#vmwareloginsight-shorten_keys} -Keys from log event to rewrite for instance from 'kubernetes_namespace' to 'k8s_namespace' tags will be rewritten with substring substitution and applied in the order present in the hash. Hashes enumerate their values in the order that the corresponding keys were inserted, see: https://ruby-doc.org/core-2.2.2/Hash.html +Keys from log event to rewrite for instance from 'kubernetes_namespace' to 'k8s_namespace' tags will be rewritten with substring substitution and applied in the order present in the hash. Hashes enumerate their values in the order that the corresponding keys were inserted, see: https://ruby-doc.org/core-2.2.2/Hash.html Default: { 'kubernetes_':'k8s_', 'namespace':'ns', 'labels_':'', '_name':'', '_hash':'', 'container_':'' } diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/syslog.md b/content/docs/configuration/plugins/syslog-ng-outputs/syslog.md index 9d1ec3c9f..95fde32de 100644 --- a/content/docs/configuration/plugins/syslog-ng-outputs/syslog.md +++ b/content/docs/configuration/plugins/syslog-ng-outputs/syslog.md @@ -103,6 +103,12 @@ Specifies how many lines are flushed to a destination at a time. For details, se Address of the destination host +### ip-protocol (*int, optional) {#syslogoutput-ip-protocol} + +IP protocol + +Default: 4 + ### persist_name (string, optional) {#syslogoutput-persist_name} Unique name for the syslog-ng driver. If you receive the following error message during syslog-ng startup, set the `persist-name()` option of the duplicate drivers: `Error checking the uniqueness of the persist names, please override it with persist-name option. Shutting down.` See the [documentation of the AxoSyslog syslog-ng distribution](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/#persist-name) for more information. diff --git a/content/docs/image-versions.md b/content/docs/image-versions.md index 33a4d7168..8116c7da9 100644 --- a/content/docs/image-versions.md +++ b/content/docs/image-versions.md @@ -5,6 +5,22 @@ weight: 750 Logging operator uses the following image versions. +## Logging operator version 5.4 + +| Image repository | GitHub repository | Version | +| -------- | --- | -- | +| ghcr.io/kube-logging/logging-operator/node-exporter | https://github.com/kube-logging/logging-operator/tree/master/images/node-exporter | 5.4.0 | +| ghcr.io/kube-logging/logging-operator/config-reloader | https://github.com/kube-logging/logging-operator/tree/master/images/config-reloader | 5.4.0 | +| ghcr.io/kube-logging/logging-operator/fluentd-drain-watch | https://github.com/kube-logging/logging-operator/tree/master/images/fluentd-drain-watch | 5.4.0 | +| registry.k8s.io/pause | https://github.com/kubernetes/registry.k8s.io | 3.9 | +| docker.io/library/busybox | https://github.com/docker-library/busybox | latest | +| ghcr.io/axoflow/axosyslog | https://github.com/axoflow/axosyslog/ | 4.11.0 | +| ghcr.io/fluent/fluent-bit | https://github.com/fluent/fluent-bit | 3.2.5 | +| ghcr.io/kube-logging/logging-operator/fluentd | https://github.com/kube-logging/logging-operator/tree/master/images/fluentd | 5.4.0-full | +| ghcr.io/axoflow/axosyslog-metrics-exporter | https://github.com/axoflow/axosyslog-metrics-exporter | 0.0.9 | +| ghcr.io/kube-logging/logging-operator/syslog-ng-reloader | https://github.com/kube-logging/logging-operator/tree/master/images/syslog-ng-reloader | 5.4.0 | +| ghcr.io/kube-logging/eventrouter | https://github.com/kube-logging/eventrouter | 0.4.0 | + ## Logging operator version 5.3 | Image repository | GitHub repository | Version | diff --git a/content/docs/whats-new/_index.md b/content/docs/whats-new/_index.md index 4f0054683..15f9a8f40 100644 --- a/content/docs/whats-new/_index.md +++ b/content/docs/whats-new/_index.md @@ -3,6 +3,122 @@ title: What's new weight: 50 --- +## Version 5.4 + +The following are the highlights and main changes of Logging operator 5.4. For a complete list of changes and bugfixes, see the [Logging operator 5.4 releases page](https://github.com/kube-logging/logging-operator/releases/tag/5.4.0). + +- The new [LogicMonitor Logs]({{< relref "/docs/configuration/plugins/outputs/lm_logs.md" >}}) Fluentd output plugin allows you to send logs to [LM Logs of LogicMonitor](https://www.logicmonitor.com/). +- The Elasticsearch Fluent plugin has been updated to 6.0, making [wildcards available in index patterns by default](/docs/configuration/plugins/outputs/elasticsearch.md#elasticsearch-data_stream_template_use_index_patterns_wildcard). +- You can now use the `compress` option in the [forward output]({{< relref "/docs/configuration/plugins/outputs/forward.md#forwardoutput-compress" >}}) to enable gzip compression. +- You can [disable mounting the `/var/log` volume]({{< relref "/docs/configuration/crds/v1beta1/fluentbit_types.md#fluentbitspec-disablevarlog" >}}) in Fluent Bit. This is useful when you're not permitted to mount host volumes, and collect host logs some other way. +- Initial basic implementation of the upcoming [AxoSyslog custom resource]({{< relref "/docs/configuration/crds/v1beta1/axosyslog_types.md" >}}). + +### Upcoming deprecations and breaking changes + +We are planning on deprecating the following components in the next major release. These breaking changes will be part of version 6.0.0 (scheduled for July 14), while 5.4.0 remains officially supported until October 6. (If needed, patch releases from community contributions for version 5.4.0 will be supported even after 6th of October.) + +#### NodeAgent CRD and inline NodeAgents in the Logging resource + +[NodeAgents]({{< relref "/docs/configuration/crds/v1beta1/node_agent_types.md" >}}) were an attempt to generalize log agents configuration, but it never got out PoC status, with the main feature of running Fluent Bit on Windows hosts. The code hasn't been updated recently, and the original FluentbitSpec (in the Logging resource and in the separate FluentbitAgent resource) and the features behind it have significantly diverged. + +Last year we've introduced [Telemetry Controller](https://github.com/kube-logging/telemetry-controller) as a replacement for the NodeAgent and FluentbitAgent resources, with additional multi-tenant capabilities and more flexible agent-side log selection. + +- If you're using NodeAgent on Windows, get in touch with us (the Logging operator maintainers) over the [community channels]({{< relref "/docs/community.md" >}}), so we can help you find a viable path forward using the Telemetry Controller. +- Non-windows users should either migrate to [FluentbitAgent]({{< relref "/docs/configuration/crds/v1beta1/fluentbit_types.md" >}}), or to the [Telemetry Controller](https://github.com/kube-logging/telemetry-controller). In case you need help with either case, [feel free to contact us]({{< relref "/docs/community.md" >}}). + +#### hostTailer in the Helm chart + +Configuring a hostTailer in the [Logging operator Helm chart](https://github.com/kube-logging/logging-operator/tree/master/charts/logging-operator) is deprecated in favor of using hostTailers. Migrate your hostTailer configuration like this: + +- Old configuration: + + ```yaml + hostTailer: + # -- HostTailer + enabled: false + # -- name of HostTailer + name: hosttailer + image: + # -- repository of eventTailer image + repository: + # -- tag of eventTailer image + tag: + # -- pullPolicy of eventTailer image + pullPolicy: + # -- imagePullSecrets of eventTailer image + imagePullSecrets: [] + # -- workloadMetaOverrides of HostTailer + workloadMetaOverrides: + # -- workloadOverrides of HostTailer + workloadOverrides: + # -- configure fileTailers of HostTailer + # example: + # - name: sample-file + # path: /var/log/sample-file + # disabled: false + # buffer_max_size: + # buffer_chunk_size: + # skip_long_lines: + # read_from_head: false + # containerOverrides: + # image: + fileTailers: [] + # -- configure systemdTailers of HostTailer + # example: + # - name: system-sample + # disabled: false + # systemdFilter: kubelet.service + # maxEntries: 20 + # containerOverrides: + # image: + systemdTailers: **[]** + ``` + +- The new configuration will be similar to this: + + ```yaml + hostTailers: + # -- Enable all hostTailers + enabled: false + # -- List of hostTailers configurations + instances: [] + # - name: hosttailer + # -- Enable hostTailer + # enabled: true + # image: + # -- repository of eventTailer image + # repository: + # -- tag of eventTailer image + # tag: + # -- pullPolicy of eventTailer image + # pullPolicy: + # -- imagePullSecrets of eventTailer image + # imagePullSecrets: [] + # -- workloadMetaOverrides of HostTailer + # workloadMetaOverrides: {} + # -- workloadOverrides of HostTailer + # workloadOverrides: {} + # -- configure fileTailers of HostTailer + # fileTailers: + # - name: sample-file + # path: /var/log/sample-file + # disabled: false + # buffer_max_size: + # buffer_chunk_size: + # skip_long_lines: + # read_from_head: false + # containerOverrides: + # image: + # -- configure systemdTailers of HostTailer + # systemdTailers: + # - name: system-sample + # disabled: false + # systemdFilter: kubelet.service + # maxEntries: 20 + # containerOverrides: + # image: + ``` + ## Version 5.3 The following are the highlights and main changes of Logging operator 5.3. For a complete list of changes and bugfixes, see the [Logging operator 5.3 releases page](https://github.com/kube-logging/logging-operator/releases/tag/5.3.0).