diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 33705383..e3ab71e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,5 +17,6 @@ The following guides are a work-in-progress with a focus on helping code contrib - [Getting Started](contributing/getting-started.md) - [Getting Started - Step by Step](contributing/step-by-step.md) - [Test](contributing/test.md) +- [Configuration](contributing/config.md) - [Destinations](contributing/destinations.md) - [MQ](contributing/mq.md) diff --git a/contributing/config.md b/contributing/config.md index a7584484..99d443a2 100644 --- a/contributing/config.md +++ b/contributing/config.md @@ -1,6 +1,134 @@ # Config -TBD. This document should go into more details about various Outpost configuration. +This document provides guidelines for working with Outpost configuration. + +## Adding New Configuration Fields + +When adding new configuration fields to Outpost, follow these steps to ensure consistency and proper logging: + +### 1. Define the Configuration Field + +Add your new field to the appropriate config struct in `internal/config/`: + +```go +type Config struct { + // ... existing fields ... + MyNewField string `yaml:"my_new_field" env:"MY_NEW_FIELD" desc:"Description of the field" required:"N"` +} +``` + +### 2. Add Default Values (if applicable) + +Update `InitDefaults()` in `internal/config/config.go`: + +```go +func (c *Config) InitDefaults() { + // ... existing defaults ... + c.MyNewField = "default_value" +} +``` + +### 3. Update Configuration Logging ⚠️ IMPORTANT + +**To maintain visibility into startup configuration, you MUST update the configuration logging helper** in `internal/config/logging.go`: + +#### For General Configuration Fields + +Add your field to `LogConfigurationSummary()`: + +```go +func (c *Config) LogConfigurationSummary() []zap.Field { + fields := []zap.Field{ + // ... existing fields ... + + // For non-sensitive fields: + zap.String("my_new_field", c.MyNewField), + + // For sensitive fields (passwords, secrets, keys): + zap.Bool("my_secret_field_configured", c.MySecretField != ""), + + // ... rest of fields ... + } + return fields +} +``` + +#### For Message Queue Configuration + +If adding MQ-specific fields, update `getMQSpecificFields()`: + +```go +func (c *Config) getMQSpecificFields(mqType string) []zap.Field { + switch mqType { + case "rabbitmq": + return []zap.Field{ + // ... existing fields ... + zap.String("rabbitmq_my_field", c.MQs.RabbitMQ.MyField), + } + // ... other cases ... + } +} +``` + +### 4. Guidelines for Sensitive Data + +**Always mask sensitive data in logs:** + +- ✅ **DO**: Use `zap.Bool("field_configured", value != "")` for secrets +- ✅ **DO**: Use helper functions like `maskURL()` for URLs with credentials +- ❌ **DON'T**: Log actual passwords, API keys, tokens, or secrets +- ❌ **DON'T**: Log full connection strings with credentials + +**Examples:** + +```go +// Good - shows if configured without exposing value +zap.Bool("api_key_configured", c.APIKey != "") + +// Good - masks credentials in URL +zap.String("database_url", maskPostgresURLHost(c.PostgresURL)) + +// Bad - exposes sensitive data +zap.String("api_key", c.APIKey) // ❌ NEVER DO THIS +``` + +### 5. Update Validation (if needed) + +If your field requires validation, update `Validate()` in `internal/config/validation.go`. + +### 6. Update Documentation + +Don't forget to regenerate the configuration documentation: + +```bash +go generate ./internal/config/... +``` + +This will update `docs/pages/references/configuration.mdx` with your new field's description. + +## Configuration Logging Checklist + +When adding or modifying configuration fields, use this checklist: + +- [ ] Field added to appropriate struct with `yaml`, `env`, `desc`, and `required` tags +- [ ] Default value added to `InitDefaults()` (if applicable) +- [ ] **Field added to `LogConfigurationSummary()` in `internal/config/logging.go`** +- [ ] **Sensitive fields are masked (showing only if configured, not actual value)** +- [ ] MQ-specific fields added to `getMQSpecificFields()` (if applicable) +- [ ] Validation added (if required) +- [ ] Documentation regenerated with `go generate` +- [ ] Changes tested with `LOG_LEVEL=info` to verify logs appear correctly + +## Why Configuration Logging Matters + +Configuration logging serves several critical purposes: + +1. **Troubleshooting**: When users report issues, configuration logs help identify misconfiguration quickly +2. **Security Auditing**: Shows what's configured without exposing sensitive values +3. **Deployment Verification**: Confirms the application started with expected configuration +4. **Documentation**: Provides a real-world example of what configuration is being used + +Keeping configuration logging up-to-date prevents "configuration drift" where the code and logs don't match, making troubleshooting harder. ## MQs diff --git a/internal/app/app.go b/internal/app/app.go index 56e7e90d..7d0d1f34 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -53,19 +53,14 @@ func (a *App) PreRun(ctx context.Context) error { if err := a.setupLogger(); err != nil { return err } + defer func() { if r := recover(); r != nil { a.logger.Error("panic during PreRun", zap.Any("panic", r)) } }() - a.logger.Info("starting outpost", - zap.String("config_path", a.config.ConfigFilePath()), - zap.String("service", a.config.MustGetService().String())) - - if a.config.DeploymentID != "" { - a.logger.Info("deployment configured", zap.String("deployment_id", a.config.DeploymentID)) - } + a.logger.Info("starting outpost", a.config.LogConfigurationSummary()...) if err := a.configureIDGenerators(); err != nil { return err diff --git a/internal/config/logging.go b/internal/config/logging.go new file mode 100644 index 00000000..09710186 --- /dev/null +++ b/internal/config/logging.go @@ -0,0 +1,185 @@ +package config + +import ( + "strings" + + "go.uber.org/zap" +) + +// LogConfigurationSummary returns zap fields with configuration summary, masking sensitive data +// +// ⚠️ IMPORTANT: When adding new configuration fields, you MUST update this function +// to include them in the startup logs. This helps with troubleshooting and ensures +// configuration visibility. +// +// Guidelines: +// - For non-sensitive fields: use zap.String(), zap.Int(), zap.Bool(), etc. +// - For sensitive fields (secrets, passwords, keys): use zap.Bool("field_configured", value != "") +// - For URLs with credentials: use helper functions like maskURL() or maskPostgresURLHost() +// +// See contributing/config.md for detailed guidelines on configuration logging. +func (c *Config) LogConfigurationSummary() []zap.Field { + fields := []zap.Field{ + // General + zap.String("service", c.Service), + zap.String("config_file_path", func() string { + if c.configPath != "" { + return c.configPath + } + return "none (using defaults and environment variables)" + }()), + zap.String("log_level", c.LogLevel), + zap.Bool("audit_log", c.AuditLog), + zap.String("deployment_id", c.DeploymentID), + zap.Strings("topics", c.Topics), + zap.String("organization_name", c.OrganizationName), + zap.String("http_user_agent", c.HTTPUserAgent), + + // API + zap.Int("api_port", c.APIPort), + zap.Bool("api_key_configured", c.APIKey != ""), + zap.Bool("api_jwt_secret_configured", c.APIJWTSecret != ""), + zap.String("gin_mode", c.GinMode), + + // Application + zap.Bool("aes_encryption_secret_configured", c.AESEncryptionSecret != ""), + + // Redis + zap.String("redis_host", c.Redis.Host), + zap.Int("redis_port", c.Redis.Port), + zap.Bool("redis_password_configured", c.Redis.Password != ""), + zap.Int("redis_database", c.Redis.Database), + zap.Bool("redis_tls_enabled", c.Redis.TLSEnabled), + zap.Bool("redis_cluster_enabled", c.Redis.ClusterEnabled), + + // PostgreSQL + zap.Bool("postgres_configured", c.PostgresURL != ""), + zap.String("postgres_host", maskPostgresURLHost(c.PostgresURL)), + + // Message Queue + zap.String("mq_type", c.MQs.GetInfraType()), + + // Consumers + zap.Int("publish_max_concurrency", c.PublishMaxConcurrency), + zap.Int("delivery_max_concurrency", c.DeliveryMaxConcurrency), + zap.Int("log_max_concurrency", c.LogMaxConcurrency), + + // Delivery Retry + zap.Ints("retry_schedule", c.RetrySchedule), + zap.Int("retry_interval_seconds", c.RetryIntervalSeconds), + zap.Int("retry_max_limit", c.RetryMaxLimit), + + // Event Delivery + zap.Int("max_destinations_per_tenant", c.MaxDestinationsPerTenant), + zap.Int("delivery_timeout_seconds", c.DeliveryTimeoutSeconds), + + // Idempotency + zap.Int("publish_idempotency_key_ttl", c.PublishIdempotencyKeyTTL), + zap.Int("delivery_idempotency_key_ttl", c.DeliveryIdempotencyKeyTTL), + + // Log batcher + zap.Int("log_batch_threshold_seconds", c.LogBatchThresholdSeconds), + zap.Int("log_batch_size", c.LogBatchSize), + + // Telemetry + zap.Bool("telemetry_disabled", c.Telemetry.Disabled || c.DisableTelemetry), + + // Alert + zap.String("alert_callback_url", maskURL(c.Alert.CallbackURL)), + zap.Int("alert_consecutive_failure_count", c.Alert.ConsecutiveFailureCount), + zap.Bool("alert_auto_disable_destination", c.Alert.AutoDisableDestination), + + // ID Generation + zap.String("idgen_type", c.IDGen.Type), + zap.String("idgen_event_prefix", c.IDGen.EventPrefix), + } + + // Add MQ-specific fields based on type + mqType := c.MQs.GetInfraType() + fields = append(fields, c.getMQSpecificFields(mqType)...) + + return fields +} + +// getMQSpecificFields returns MQ-specific configuration fields +// +// ⚠️ IMPORTANT: When adding new MQ configuration fields, update the appropriate case +// in this function to include them in startup logs. +func (c *Config) getMQSpecificFields(mqType string) []zap.Field { + switch mqType { + case "rabbitmq": + return []zap.Field{ + zap.String("rabbitmq_url", maskURL(c.MQs.RabbitMQ.ServerURL)), + zap.String("rabbitmq_exchange", c.MQs.RabbitMQ.Exchange), + zap.String("rabbitmq_delivery_queue", c.MQs.RabbitMQ.DeliveryQueue), + zap.String("rabbitmq_log_queue", c.MQs.RabbitMQ.LogQueue), + } + case "awssqs": + return []zap.Field{ + zap.Bool("aws_access_key_configured", c.MQs.AWSSQS.AccessKeyID != ""), + zap.Bool("aws_secret_key_configured", c.MQs.AWSSQS.SecretAccessKey != ""), + zap.String("aws_region", c.MQs.AWSSQS.Region), + zap.String("aws_delivery_queue", c.MQs.AWSSQS.DeliveryQueue), + zap.String("aws_log_queue", c.MQs.AWSSQS.LogQueue), + } + case "gcppubsub": + return []zap.Field{ + zap.Bool("gcp_credentials_configured", c.MQs.GCPPubSub.ServiceAccountCredentials != ""), + zap.String("gcp_project_id", c.MQs.GCPPubSub.Project), + zap.String("gcp_delivery_topic", c.MQs.GCPPubSub.DeliveryTopic), + zap.String("gcp_delivery_subscription", c.MQs.GCPPubSub.DeliverySubscription), + zap.String("gcp_log_topic", c.MQs.GCPPubSub.LogTopic), + zap.String("gcp_log_subscription", c.MQs.GCPPubSub.LogSubscription), + } + case "azureservicebus": + return []zap.Field{ + zap.Bool("azure_connection_string_configured", c.MQs.AzureServiceBus.ConnectionString != ""), + zap.String("azure_delivery_topic", c.MQs.AzureServiceBus.DeliveryTopic), + zap.String("azure_delivery_subscription", c.MQs.AzureServiceBus.DeliverySubscription), + zap.String("azure_log_topic", c.MQs.AzureServiceBus.LogTopic), + zap.String("azure_log_subscription", c.MQs.AzureServiceBus.LogSubscription), + } + default: + return []zap.Field{} + } +} + +// maskURL masks credentials in a URL +func maskURL(url string) string { + if url == "" { + return "" + } + // Basic masking for URLs with credentials + // Format: protocol://user:password@host:port + if idx := strings.Index(url, "://"); idx != -1 { + protocol := url[:idx+3] + rest := url[idx+3:] + if atIdx := strings.Index(rest, "@"); atIdx != -1 { + host := rest[atIdx:] + return protocol + "***:***" + host + } + } + return url +} + +// maskPostgresURLHost extracts and returns just the host from a postgres URL +func maskPostgresURLHost(url string) string { + if url == "" { + return "" + } + + // postgres://user:password@host:port/database?params + if idx := strings.Index(url, "@"); idx != -1 { + rest := url[idx+1:] + // Get host:port before the database name + if slashIdx := strings.Index(rest, "/"); slashIdx != -1 { + return rest[:slashIdx] + } + // No database name, get host:port before params + if qIdx := strings.Index(rest, "?"); qIdx != -1 { + return rest[:qIdx] + } + return rest + } + return "not configured" +}