From bf7c56ef69d5aca207e286b7497e4dee7f0a198b Mon Sep 17 00:00:00 2001 From: May Lee Date: Tue, 11 Nov 2025 14:06:03 -0500 Subject: [PATCH] add env vars --- .../destinations/google_pubsub.md | 7 +------ .../destinations/kafka.md | 17 +---------------- .../guide/environment_variables.md | 9 +++++++++ .../sources/opentelemetry.md | 10 +--------- .../observability_pipelines/sources/socket.md | 4 ++++ .../destination_env_vars/google_pubsub.en.md | 7 +++++++ .../destination_env_vars/kafka.en.md | 16 ++++++++++++++++ .../source_env_vars/opentelemetry.en.md | 11 +++++++++++ .../source_env_vars/socket.md | 5 ++--- 9 files changed, 52 insertions(+), 34 deletions(-) create mode 100644 layouts/shortcodes/observability_pipelines/configure_existing_pipelines/destination_env_vars/google_pubsub.en.md create mode 100644 layouts/shortcodes/observability_pipelines/configure_existing_pipelines/destination_env_vars/kafka.en.md create mode 100644 layouts/shortcodes/observability_pipelines/configure_existing_pipelines/source_env_vars/opentelemetry.en.md diff --git a/content/en/observability_pipelines/destinations/google_pubsub.md b/content/en/observability_pipelines/destinations/google_pubsub.md index f4d2cad1c0f81..c0357b01253be 100644 --- a/content/en/observability_pipelines/destinations/google_pubsub.md +++ b/content/en/observability_pipelines/destinations/google_pubsub.md @@ -114,11 +114,7 @@ Set up the Google Pub/Sub destination and its environment variables when you [se {{< img src="observability_pipelines/destinations/google_pubsub_env_var.png" alt="The install page showing the Google Pub/Sub environment variable field" style="width:70%;" >}} -By default the Worker sends data to the global endpoint: `https://pubsub.googleapis.com`. - -If your Pub/Sub topic is region-specific, configure the Google Pub/Sub alternative endpoint URL with the regional endpoint. See [About Pub/Sub endpoints][5] for more information. - -Stored as the environment variable: `DD_OP_DESTINATION_GCP_PUBSUB_ENDPOINT_URL`. +{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/google_pubsub %}} ## Troubleshooting @@ -163,7 +159,6 @@ A batch of events is flushed when one of these parameters is met. See [event bat [2]: https://cloud.google.com/docs/authentication#auth-flowchart [3]: https://cloud.google.com/pubsub/docs/access-control#roles [4]: https://console.cloud.google.com/iam-admin/serviceaccounts -[5]: https://cloud.google.com/pubsub/docs/reference/service_apis_overview#pubsub_endpoints [6]: /observability_pipelines/destinations/#event-batching [7]:https://cloud.google.com/kubernetes-engine/docs/concepts/workload-identity [8]: /observability_pipelines/monitoring_and_maintenance/pipeline_usage_metrics/ diff --git a/content/en/observability_pipelines/destinations/kafka.md b/content/en/observability_pipelines/destinations/kafka.md index 3109c7f3fe949..25d1392b07a3d 100644 --- a/content/en/observability_pipelines/destinations/kafka.md +++ b/content/en/observability_pipelines/destinations/kafka.md @@ -78,22 +78,7 @@ Click **Advanced** if you want to set any of the following fields: ### Set environment variables -#### Kafka bootstrap servers -- The host and port of the Kafka bootstrap servers. -- This is the bootstrap server that the client uses to connect to the Kafka cluster and discover all the other hosts in the cluster. The host and port must be entered in the format of `host:port`, such as `10.14.22.123:9092`. If there is more than one server, use commas to separate them. -- Stored as the environment variable: `DD_OP_DESTINATION_KAFKA_BOOTSTRAP_SERVERS`. - -#### TLS (when enabled) - -- If TLS is enabled, the Kafka TLS passphrase is needed. -- Stored as the environment variable: `DD_OP_DESTINATION_KAFKA_KEY_PASS`. - -#### SASL (when enabled) - -- Kafka SASL username - - Stored as the environment variable: `DD_OP_DESTINATION_KAFKA_SASL_USERNAME`. -- Kafka SASL password - - Stored as the environment variable: `DD_OP_DESTINATION_KAFKA_SASL_PASSWORD`. +{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/kafka %}} ## librdkafka options diff --git a/content/en/observability_pipelines/guide/environment_variables.md b/content/en/observability_pipelines/guide/environment_variables.md index f81c466ce7552..01e87af347f35 100644 --- a/content/en/observability_pipelines/guide/environment_variables.md +++ b/content/en/observability_pipelines/guide/environment_variables.md @@ -39,6 +39,9 @@ Some Observability Pipelines components require setting up environment variables ### Logstash {{% observability_pipelines/configure_existing_pipelines/source_env_vars/logstash %}} +### OpenTelemetry +{{% observability_pipelines/configure_existing_pipelines/source_env_vars/opentelemetry %}} + ### Socket {{% observability_pipelines/configure_existing_pipelines/source_env_vars/socket %}} @@ -91,6 +94,12 @@ Some Observability Pipelines components require setting up environment variables ### Elasticsearch {{% observability_pipelines/configure_existing_pipelines/destination_env_vars/elasticsearch %}} +### Kafka +{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/kafka %}} + +### Google Pub/Sub +{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/google_pubsub %}} + ### HTTP Client {{% observability_pipelines/configure_existing_pipelines/destination_env_vars/http_client %}} diff --git a/content/en/observability_pipelines/sources/opentelemetry.md b/content/en/observability_pipelines/sources/opentelemetry.md index ea4a37ba542b3..f99de2e227d88 100644 --- a/content/en/observability_pipelines/sources/opentelemetry.md +++ b/content/en/observability_pipelines/sources/opentelemetry.md @@ -32,15 +32,7 @@ Optionally, toggle the switch to enable TLS. If you enable TLS, the following ce {{< img src="observability_pipelines/sources/otel_env_vars.png" alt="The install page showing the OpenTelemetry environment variable field" style="width:75%;" >}} -You must provide both HTTP and gRPC endpoints. Configure your OTLP exporters to point to one of these endpoints. See [Send logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker) for more information. - -- HTTP listener address - - The Observability Pipelines Worker listens to this socket address to receive logs from the OTel collector. - - Stored as the environment variable `DD_OP_SOURCE_OTEL_HTTP_ADDRESS`. - -- gRPC listener address - - The Observability Pipelines Worker listens to this socket address to receive logs from the OTel collector. - - Stored as the environment variable `DD_OP_SOURCE_OTEL_GRPC_ADDRESS`. +{{% observability_pipelines/configure_existing_pipelines/source_env_vars/splunk_hec %}} ## Send logs to the Observability Pipelines Worker diff --git a/content/en/observability_pipelines/sources/socket.md b/content/en/observability_pipelines/sources/socket.md index 6aebc4a4ebb6a..413028dabaf95 100644 --- a/content/en/observability_pipelines/sources/socket.md +++ b/content/en/observability_pipelines/sources/socket.md @@ -15,4 +15,8 @@ Select and set up this source when you [set up a pipeline][1]. The information b {{% observability_pipelines/source_settings/socket %}} +## Set the environment variables + +{{% observability_pipelines/configure_existing_pipelines/source_env_vars/socket %}} + [1]: /observability_pipelines/configuration/set_up_pipelines/ \ No newline at end of file diff --git a/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/destination_env_vars/google_pubsub.en.md b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/destination_env_vars/google_pubsub.en.md new file mode 100644 index 0000000000000..32846608c0f3f --- /dev/null +++ b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/destination_env_vars/google_pubsub.en.md @@ -0,0 +1,7 @@ +By default the Worker sends data to the global endpoint: `https://pubsub.googleapis.com`. + +If your Pub/Sub topic is region-specific, configure the Google Pub/Sub alternative endpoint URL with the regional endpoint. See [About Pub/Sub endpoints][10240] for more information. + +Stored as the environment variable: `DD_OP_DESTINATION_GCP_PUBSUB_ENDPOINT_URL`. + +[10240]: https://cloud.google.com/pubsub/docs/reference/service_apis_overview#pubsub_endpoints \ No newline at end of file diff --git a/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/destination_env_vars/kafka.en.md b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/destination_env_vars/kafka.en.md new file mode 100644 index 0000000000000..87a230544299b --- /dev/null +++ b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/destination_env_vars/kafka.en.md @@ -0,0 +1,16 @@ +#### Kafka bootstrap servers +- The host and port of the Kafka bootstrap servers. +- This is the bootstrap server that the client uses to connect to the Kafka cluster and discover all the other hosts in the cluster. The host and port must be entered in the format of `host:port`, such as `10.14.22.123:9092`. If there is more than one server, use commas to separate them. +- Stored as the environment variable: `DD_OP_DESTINATION_KAFKA_BOOTSTRAP_SERVERS`. + +#### TLS (when enabled) + +- If TLS is enabled, the Kafka TLS passphrase is needed. +- Stored as the environment variable: `DD_OP_DESTINATION_KAFKA_KEY_PASS`. + +#### SASL (when enabled) + +- Kafka SASL username + - Stored as the environment variable: `DD_OP_DESTINATION_KAFKA_SASL_USERNAME`. +- Kafka SASL password + - Stored as the environment variable: `DD_OP_DESTINATION_KAFKA_SASL_PASSWORD`. \ No newline at end of file diff --git a/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/source_env_vars/opentelemetry.en.md b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/source_env_vars/opentelemetry.en.md new file mode 100644 index 0000000000000..0b96350e53e4b --- /dev/null +++ b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/source_env_vars/opentelemetry.en.md @@ -0,0 +1,11 @@ +You must provide both HTTP and gRPC endpoints. Configure your OTLP exporters to point to one of these endpoints. See [Send logs to the Observability Pipelines Worker][10230] for more information. + +- HTTP listener address + - The Observability Pipelines Worker listens to this socket address to receive logs from the OTel collector. + - Stored as the environment variable `DD_OP_SOURCE_OTEL_HTTP_ADDRESS`. + +- gRPC listener address + - The Observability Pipelines Worker listens to this socket address to receive logs from the OTel collector. + - Stored as the environment variable `DD_OP_SOURCE_OTEL_GRPC_ADDRESS`. + + [10230]: /observability_pipelines/sources/opentelemetry/#send-logs-to-the-observability-pipelines-worker \ No newline at end of file diff --git a/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/source_env_vars/socket.md b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/source_env_vars/socket.md index 5b6869c1d01d7..e4f5137c1e63a 100644 --- a/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/source_env_vars/socket.md +++ b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/source_env_vars/socket.md @@ -1,7 +1,6 @@ - Socket address: - The address and port where the Observability Pipelines Worker listens for incoming logs. - - Stored as the environment variable `DD_OP_SOURCE_SOCKET_ADDRESS` + - Stored as the environment variable `DD_OP_SOURCE_SOCKET_ADDRESS`. -- TLS passphrase: - - Used if TLS is enabled. +- TLS passphrase (when enabled) - Stored as the environment variable `DD_OP_SOURCE_SOCKET_KEY_PASS`.