Merge pull request #23 from kube-logging/demo-openobserve

feat(demo): add demo using OpenObserve's OTLP GRPC ingestion

Author: kristofgyuracz

api/telemetry/v1alpha1/oteloutput_types.go

@@ -32,10 +32,10 @@ type OtelOutputSpec struct {
 
 // OTLP grpc exporter config ref: https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/otlpexporter/config.go
 type OTLPgrpc struct {
-	QueueConfig        QueueSettings `json:"sending_queue,omitempty"`
-	RetryConfig        BackOffConfig `json:"retry_on_failure,omitempty"`
-	TimeoutSettings    `json:",inline"`
-	GRPCClientSettings `json:",inline"`
+	QueueConfig        QueueSettings `json:"sending_queue,omitempty" yaml:"sending_queue,omitempty"`
+	RetryConfig        BackOffConfig `json:"retry_on_failure,omitempty" yaml:"retry_on_failure,omitempty"`
+	TimeoutSettings    `json:",inline" yaml:",inline"`
+	GRPCClientSettings `json:",inline" yaml:",inline"`
 }
 
 // OtelOutputStatus defines the observed state of OtelOutput
@@ -52,15 +52,15 @@ type OtelOutput struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
-	Spec   OtelOutputSpec   `json:"spec,omitempty"`
-	Status OtelOutputStatus `json:"status,omitempty"`
+	Spec   OtelOutputSpec   `json:"spec,omitempty" yaml:"spec,omitempty"`
+	Status OtelOutputStatus `json:"status,omitempty" `
 }
 
 //+kubebuilder:object:root=true
 
 // OtelOutputList contains a list of OtelOutput
 type OtelOutputList struct {
-	metav1.TypeMeta `json:",inline"`
+	metav1.TypeMeta `json:",inline" yaml:",inline"`
 	metav1.ListMeta `json:"metadata,omitempty"`
 	Items           []OtelOutput `json:"items"`
 }

api/telemetry/v1alpha1/otlp_config.go

@@ -23,101 +23,101 @@ import (
 type TimeoutSettings struct {
 	// Timeout is the timeout for every attempt to send data to the backend.
 	// A zero timeout means no timeout.
-	Timeout time.Duration `json:"timeout,omitempty"`
+	Timeout time.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"`
 }
 
 // QueueSettings defines configuration for queueing batches before sending to the consumerSender.
 type QueueSettings struct {
 	// Enabled indicates whether to not enqueue batches before sending to the consumerSender.
-	Enabled bool `json:"enabled,omitempty"`
+	Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"`
 	// NumConsumers is the number of consumers from the queue.
-	NumConsumers int `json:"num_consumers,omitempty"`
+	NumConsumers int `json:"num_consumers,omitempty" yaml:"num_consumers,omitempty"`
 	// QueueSize is the maximum number of batches allowed in queue at a given time.
-	QueueSize int `json:"queue_size,omitempty"`
+	QueueSize int `json:"queue_size,omitempty" yaml:"queue_size,omitempty"`
 	// StorageID if not empty, enables the persistent storage and uses the component specified
 	// as a storage extension for the persistent queue
-	StorageID string `json:"storage,omitempty"` //TODO this is *component.ID at Otel
+	StorageID string `json:"storage,omitempty" yaml:"storage,omitempty"` //TODO this is *component.ID at Otel
 }
 
 // BackOffConfig defines configuration for retrying batches in case of export failure.
 // The current supported strategy is exponential backoff.
 type BackOffConfig struct {
 	// Enabled indicates whether to not retry sending batches in case of export failure.
-	Enabled bool `json:"enabled,omitempty"`
+	Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"`
 	// InitialInterval the time to wait after the first failure before retrying.
-	InitialInterval time.Duration `json:"initial_interval,omitempty"`
+	InitialInterval time.Duration `json:"initial_interval,omitempty" yaml:"initial_interval,omitempty" `
 	// RandomizationFactor is a random factor used to calculate next backoffs
 	// Randomized interval = RetryInterval * (1 ± RandomizationFactor)
-	RandomizationFactor string `json:"randomization_factor,omitempty"`
+	RandomizationFactor string `json:"randomization_factor,omitempty" yaml:"randomization_factor,omitempty"`
 	// Multiplier is the value multiplied by the backoff interval bounds
-	Multiplier string `json:"multiplier,omitempty"`
+	Multiplier string `json:"multiplier,omitempty" yaml:"multiplier,omitempty"`
 	// MaxInterval is the upper bound on backoff interval. Once this value is reached the delay between
 	// consecutive retries will always be `MaxInterval`.
-	MaxInterval time.Duration `json:"max_interval,omitempty"`
+	MaxInterval time.Duration `json:"max_interval,omitempty" yaml:"max_interval,omitempty"`
 	// MaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch.
 	// Once this value is reached, the data is discarded. If set to 0, the retries are never stopped.
-	MaxElapsedTime time.Duration `json:"max_elapsed_time,omitempty"`
+	MaxElapsedTime time.Duration `json:"max_elapsed_time,omitempty" yaml:"max_elapsed_time,omitempty"`
 }
 
 // KeepaliveClientConfig exposes the keepalive.ClientParameters to be used by the exporter.
 // Refer to the original data-structure for the meaning of each parameter:
 // https://godoc.org/google.golang.org/grpc/keepalive#ClientParameters
 type KeepaliveClientConfig struct {
-	Time                time.Duration `json:"time,omitempty"`
-	Timeout             time.Duration `json:"timeout,omitempty"`
-	PermitWithoutStream bool          `json:"permit_without_stream,omitempty"`
+	Time                time.Duration `json:"time,omitempty" yaml:"time,omitempty"`
+	Timeout             time.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"`
+	PermitWithoutStream bool          `json:"permit_without_stream,omitempty" yaml:"permit_without_stream,omitempty"`
 }
 
 // GRPCClientSettings defines common settings for a gRPC client configuration.
 type GRPCClientSettings struct {
 	// The target to which the exporter is going to send traces or metrics,
 	// using the gRPC protocol. The valid syntax is described at
 	// https://github.com/grpc/grpc/blob/master/doc/naming.md.
-	Endpoint string `json:"endpoint"`
+	Endpoint string `json:"endpoint" yaml:"endpoint"`
 
 	// The compression key for supported compression types within collector.
-	Compression configcompression.CompressionType `json:"compression,omitempty"`
+	Compression configcompression.CompressionType `json:"compression,omitempty" yaml:"compression,omitempty"`
 
 	// TLSSetting struct exposes TLS client configuration.
-	TLSSetting TLSClientSetting `json:"tls,omitempty"`
+	TLSSetting TLSClientSetting `json:"tls,omitempty" yaml:"tls,omitempty"`
 
 	// The keepalive parameters for gRPC client. See grpc.WithKeepaliveParams.
 	// (https://godoc.org/google.golang.org/grpc#WithKeepaliveParams).
-	Keepalive *KeepaliveClientConfig `json:"keepalive,omitempty"`
+	Keepalive *KeepaliveClientConfig `json:"keepalive,omitempty" yaml:"keepalive,omitempty"`
 
 	// ReadBufferSize for gRPC client. See grpc.WithReadBufferSize.
 	// (https://godoc.org/google.golang.org/grpc#WithReadBufferSize).
-	ReadBufferSize int `json:"read_buffer_size,omitempty"`
+	ReadBufferSize int `json:"read_buffer_size,omitempty" yaml:"read_buffer_size,omitempty"`
 
 	// WriteBufferSize for gRPC gRPC. See grpc.WithWriteBufferSize.
 	// (https://godoc.org/google.golang.org/grpc#WithWriteBufferSize).
-	WriteBufferSize int `json:"write_buffer_size,omitempty"`
+	WriteBufferSize int `json:"write_buffer_size,omitempty" yaml:"write_buffer_size,omitempty"`
 
 	// WaitForReady parameter configures client to wait for ready state before sending data.
 	// (https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md)
-	WaitForReady bool `json:"wait_for_ready,omitempty"`
+	WaitForReady bool `json:"wait_for_ready,omitempty" yaml:"wait_for_ready,omitempty"`
 
 	// The headers associated with gRPC requests.
-	Headers map[string]string `json:"headers,omitempty"`
+	Headers map[string]string `json:"headers,omitempty" yaml:"headers,omitempty"`
 
 	// Sets the balancer in grpclb_policy to discover the servers. Default is pick_first.
 	// https://github.com/grpc/grpc-go/blob/master/examples/features/load_balancing/README.md
-	BalancerName string `json:"balancer_name,omitempty"`
+	BalancerName string `json:"balancer_name,omitempty" yaml:"balancer_name,omitempty"`
 
 	// WithAuthority parameter configures client to rewrite ":authority" header
 	// (godoc.org/google.golang.org/grpc#WithAuthority)
-	Authority string `json:"authority,omitempty"`
+	Authority string `json:"authority,omitempty" yaml:"authority,omitempty"`
 
 	// Auth configuration for outgoing RPCs.
-	Auth string `json:"auth,omitempty"` //TODO this is a reference *configauth.Authentication
+	Auth string `json:"auth,omitempty" yaml:"auth,omitempty"` //TODO this is a reference *configauth.Authentication
 }
 
 // TLSClientSetting contains TLS configurations that are specific to client
 // connections in addition to the common configurations. This should be used by
 // components configuring TLS client connections.
 type TLSClientSetting struct {
 	// squash ensures fields are correctly decoded in embedded struct.
-	//TLSSetting `json:",inline"`
+	TLSSetting `json:",inline" yaml:",inline"`
 
 	// These are config options specific to client connections.
 
@@ -127,13 +127,13 @@ type TLSClientSetting struct {
 	// (InsecureSkipVerify in the tls Config). Please refer to
 	// https://godoc.org/crypto/tls#Config for more information.
 	// (optional, default false)
-	Insecure bool `json:"insecure,omitempty"`
+	Insecure bool `json:"insecure,omitempty" yaml:"insecure,omitempty"`
 	// InsecureSkipVerify will enable TLS but not verify the certificate.
-	InsecureSkipVerify bool `json:"insecure_skip_verify,omitempty"`
+	InsecureSkipVerify bool `json:"insecure_skip_verify,omitempty" yaml:"insecure_skip_verify,omitempty"`
 	// ServerName requested by client for virtual hosting.
 	// This sets the ServerName in the TLSConfig. Please refer to
 	// https://godoc.org/crypto/tls#Config for more information. (optional)
-	ServerName string `json:"server_name_override,omitempty"`
+	ServerName string `json:"server_name_override,omitempty" yaml:"server_name_override,omitempty"`
 }
 
 // TLSSetting exposes the common client and server TLS configurations.
@@ -143,32 +143,32 @@ type TLSSetting struct {
 	// Path to the CA cert. For a client this verifies the server certificate.
 	// For a server this verifies client certificates. If empty uses system root CA.
 	// (optional)
-	CAFile string `json:"ca_file,omitempty"`
+	CAFile string `json:"ca_file,omitempty" yaml:"ca_file,omitempty"`
 
 	// In memory PEM encoded cert. (optional)
-	CAPem string `json:"ca_pem,omitempty"`
+	CAPem string `json:"ca_pem,omitempty" yaml:"ca_pem,omitempty"`
 
 	// Path to the TLS cert to use for TLS required connections. (optional)
-	CertFile string `json:"cert_file,omitempty"`
+	CertFile string `json:"cert_file,omitempty" yaml:"cert_file,omitempty"`
 
 	// In memory PEM encoded TLS cert to use for TLS required connections. (optional)
-	CertPem string `json:"cert_pem,omitempty"`
+	CertPem string `json:"cert_pem,omitempty" yaml:"cert_pem,omitempty"`
 
 	// Path to the TLS key to use for TLS required connections. (optional)
-	KeyFile string `json:"key_file,omitempty"`
+	KeyFile string `json:"key_file,omitempty" yaml:"key_file,omitempty"`
 
 	// In memory PEM encoded TLS key to use for TLS required connections. (optional)
-	KeyPem string `json:"key_pem,omitempty"`
+	KeyPem string `json:"key_pem,omitempty" yaml:"key_pem,omitempty"`
 
 	// MinVersion sets the minimum TLS version that is acceptable.
 	// If not set, TLS 1.2 will be used. (optional)
-	MinVersion string `json:"min_version,omitempty"`
+	MinVersion string `json:"min_version,omitempty" yaml:"min_version,omitempty"`
 
 	// MaxVersion sets the maximum TLS version that is acceptable.
 	// If not set, refer to crypto/tls for defaults. (optional)
-	MaxVersion string `json:"max_version,omitempty"`
+	MaxVersion string `json:"max_version,omitempty" yaml:"max_version,omitempty"`
 
 	// ReloadInterval specifies the duration after which the certificate will be reloaded
 	// If not set, it will never be reloaded (optional)
-	ReloadInterval time.Duration `json:"reload_interval,omitempty"`
+	ReloadInterval time.Duration `json:"reload_interval,omitempty" yaml:"reload_interval,omitempty"`
 }

api/telemetry/v1alpha1/zz_generated.deepcopy.go

@@ -165,6 +165,23 @@ spec:
                   tls:
                     description: TLSSetting struct exposes TLS client configuration.
                     properties:
+                      ca_file:
+                        description: |-
+                          Path to the CA cert. For a client this verifies the server certificate.
+                          For a server this verifies client certificates. If empty uses system root CA.
+                          (optional)
+                        type: string
+                      ca_pem:
+                        description: In memory PEM encoded cert. (optional)
+                        type: string
+                      cert_file:
+                        description: Path to the TLS cert to use for TLS required
+                          connections. (optional)
+                        type: string
+                      cert_pem:
+                        description: In memory PEM encoded TLS cert to use for TLS
+                          required connections. (optional)
+                        type: string
                       insecure:
                         description: |-
                           In gRPC when set to true, this is used to disable the client transport security.
@@ -178,6 +195,30 @@ spec:
                         description: InsecureSkipVerify will enable TLS but not verify
                           the certificate.
                         type: boolean
+                      key_file:
+                        description: Path to the TLS key to use for TLS required connections.
+                          (optional)
+                        type: string
+                      key_pem:
+                        description: In memory PEM encoded TLS key to use for TLS
+                          required connections. (optional)
+                        type: string
+                      max_version:
+                        description: |-
+                          MaxVersion sets the maximum TLS version that is acceptable.
+                          If not set, refer to crypto/tls for defaults. (optional)
+                        type: string
+                      min_version:
+                        description: |-
+                          MinVersion sets the minimum TLS version that is acceptable.
+                          If not set, TLS 1.2 will be used. (optional)
+                        type: string
+                      reload_interval:
+                        description: |-
+                          ReloadInterval specifies the duration after which the certificate will be reloaded
+                          If not set, it will never be reloaded (optional)
+                        format: int64
+                        type: integer
                       server_name_override:
                         description: |-
                           ServerName requested by client for virtual hosting.

config/crd/bases/telemetry.kube-logging.dev_oteloutputs.yaml

@@ -0,0 +1,73 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: collector
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    nsSelector: example-tenant
+  name: example-tenant-ns
+---
+apiVersion: telemetry.kube-logging.dev/v1alpha1
+kind: Collector
+metadata:
+  name: example-collector
+spec:
+  controlNamespace: collector
+  tenantSelector:
+    matchLabels:
+      collectorLabel: example-collector
+---
+apiVersion: telemetry.kube-logging.dev/v1alpha1
+kind: Tenant
+metadata:
+  labels:
+    collectorLabel: example-collector
+  name: example-tenant
+spec:
+  subscriptionNamespaceSelectors:
+    - matchLabels:
+        nsSelector: example-tenant
+  logSourceNamespaceSelectors:
+    - matchLabels:
+        nsSelector: example-tenant
+---
+apiVersion: telemetry.kube-logging.dev/v1alpha1
+kind: Subscription
+metadata:
+  name: subscription-sample-1
+  namespace: example-tenant-ns
+spec:
+  ottl: 'route()'
+  outputs:
+    - name: otlp-openobserve
+      namespace: collector
+---
+apiVersion: telemetry.kube-logging.dev/v1alpha1
+kind: Subscription
+metadata:
+  name: subscription-sample-2
+  namespace: example-tenant-ns
+spec:
+  ottl: 'route()'
+  outputs:
+    - name: otlp-openobserve
+      namespace: collector
+---
+apiVersion: telemetry.kube-logging.dev/v1alpha1
+kind: OtelOutput
+metadata:
+  name: otlp-openobserve
+  namespace: collector
+spec:
+  otlp:
+    endpoint: openobserve-otlp-grpc.openobserve.svc.cluster.local:5081
+    headers:
+      # echo -n username:org_pwd | base64 
+      Authorization: "Basic cm9vdEBleGFtcGxlLmNvbTpkREN6Z213eVVkMTlmVzZs"
+      organization: default
+      stream-name: default
+    tls:
+      insecure: true