basic integratation of open telemetry

.dockerignore

@@ -0,0 +1,18 @@
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.Python
+.venv
+venv/
+ENV/
+env/
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+.coverage
+htmlcov/
+.DS_Store
+*.log
+

README.md

@@ -90,6 +90,23 @@ Watch as Codeflash:
 7. Shows impressive speedups (up to 90x in some cases!)
 
 
+## 📊 OpenTelemetry Integration
+
+This project uses OpenTelemetry auto-instrumentation (the standard pattern used by large open-source projects) for observability.
+
+**Quick Start:**
+```bash
+# Run with auto-instrumentation (recommended)
+opentelemetry-instrument python examples/run_all_traces.py
+
+# Or use Docker for visualization
+cd src/telemetry && docker-compose up -d  # Start Jaeger
+OTEL_EXPORTER_TYPE=otlp python examples/run_all_traces.py
+# View traces at: http://localhost:16686
+```
+
+For detailed setup instructions, see [src/telemetry/README.md](src/telemetry/README.md).
+
 ## 🤝 Need Help?
 
 Join our [Discord community](https://www.codeflash.ai/discord) for support and to connect with other developers who love fast code.

env.example

@@ -0,0 +1,69 @@
+# OpenTelemetry Configuration
+# Copy this file to .env and update values as needed
+
+# Enable/Disable OpenTelemetry SDK
+# Set to "true" to disable telemetry, "false" to enable (default: false)
+OTEL_SDK_DISABLED=false
+
+# Service Information
+# Service name (default: "optimize-me")
+OTEL_SERVICE_NAME=optimize-me
+
+# Service version (default: "0.1.0")
+OTEL_SERVICE_VERSION=0.1.0
+
+# Exporter Configuration
+# Type of exporter: "console" (development), "otlp" (production), "datadog", or "jaeger"
+# (default: "console", or "datadog" if DD_API_KEY is set)
+OTEL_EXPORTER_TYPE=console
+
+# OTLP Exporter Endpoint
+# Endpoint for OTLP exporter (used when OTEL_EXPORTER_TYPE is "otlp" or "jaeger")
+# Format: http://host:port (default: "http://localhost:4318")
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+
+# OTLP Traces Exporter (for opentelemetry-instrument)
+# Used by opentelemetry-instrument command: "console" or "otlp"
+# (default: not set, uses OTEL_EXPORTER_TYPE)
+OTEL_TRACES_EXPORTER=console
+
+# Sampling Rate
+# Sampling rate for traces (0.0 to 1.0)
+# 1.0 = 100% sampling (all traces), 0.1 = 10% sampling (default: 1.0)
+OTEL_TRACES_SAMPLER_ARG=1.0
+
+# Log Level
+# Logging level for OpenTelemetry (DEBUG, INFO, WARNING, ERROR)
+# (default: "INFO")
+OTEL_LOG_LEVEL=INFO
+
+# Datadog Configuration (optional)
+# These are used when OTEL_EXPORTER_TYPE is "datadog" or when DD_API_KEY is set
+# Required for Datadog exporter and Docker Compose
+# 
+# How to get your API key:
+# 1. Sign in to Datadog: https://app.datadoghq.com (or create a free account)
+# 2. Go to: Organization Settings → API Keys
+#    Direct link: https://app.datadoghq.com/organization-settings/api-keys
+# 3. Click "New Key" or "Create Key"
+# 4. Give it a name and copy the key (you'll only see it once!)
+# 5. Set it here or export as: export DD_API_KEY=your-key
+DD_API_KEY=
+
+# Datadog Site (default: "datadoghq.com")
+# Options: datadoghq.com, datadoghq.eu, us3.datadoghq.com, etc.
+DD_SITE=datadoghq.com
+
+# Datadog Environment (default: "development")
+DD_ENV=development
+
+# Datadog Service Name (optional, defaults to OTEL_SERVICE_NAME)
+DD_SERVICE=optimize-me
+
+# Datadog Service Version (optional, defaults to OTEL_SERVICE_VERSION)
+DD_VERSION=0.1.0
+
+# Datadog Agent URL (optional, for native Datadog protocol)
+# When using Docker Compose, this is automatically http://localhost:8126
+# When using OTLP (recommended), use http://localhost:4317
+DD_AGENT_URL=http://localhost:8126

examples/run_all_traces.py

@@ -0,0 +1,257 @@
+"""
+Run all instrumented functions and display their trace outputs.
+
+This script demonstrates OpenTelemetry auto-instrumentation.
+Functions are automatically instrumented via OpenTelemetry auto-instrumentation
+(no decorators needed for libraries like NumPy and Pandas).
+
+Usage:
+    # With console exporter (default)
+    python examples/run_all_traces.py
+
+    # With OTLP exporter (requires docker-compose up)
+    # First: cd src/telemetry && docker-compose up -d
+    OTEL_TRACES_EXPORTER=otlp OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 python examples/run_all_traces.py
+
+    # Or use opentelemetry-instrument command (recommended)
+    # Note: Set OTEL_TRACES_EXPORTER=console to see traces in console
+    OTEL_TRACES_EXPORTER=console opentelemetry-instrument python examples/run_all_traces.py
+"""
+import os
+import sys
+from pathlib import Path
+
+# Add project root to path
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+import numpy as np
+import pandas as pd
+
+from src.telemetry import setup_telemetry
+from src.telemetry.auto_instrumentation import auto_instrument_package
+
+# Initialize OpenTelemetry with auto-instrumentation FIRST
+# Uses environment variables if set, otherwise defaults to console exporter
+print("=" * 80)
+print("Initializing OpenTelemetry with auto-instrumentation...")
+print("=" * 80)
+
+# Enable debug logging if needed
+import logging
+if os.getenv("OTEL_LOG_LEVEL", "").upper() == "DEBUG":
+    logging.basicConfig(level=logging.DEBUG)
+
+# Check if running via opentelemetry-instrument (which sets up OTel automatically)
+# opentelemetry-instrument uses OTEL_TRACES_EXPORTER, but we also support OTEL_EXPORTER_TYPE for compatibility
+exporter_type = os.getenv("OTEL_TRACES_EXPORTER") or os.getenv("OTEL_EXPORTER_TYPE", "console")
+exporter_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317")
+
+# Detect if running under opentelemetry-instrument
+# opentelemetry-instrument sets up its own TracerProvider, so we shouldn't replace it
+from opentelemetry import trace
+existing_provider = trace.get_tracer_provider()
+is_opentelemetry_instrument = (
+    existing_provider is not None 
+    and not isinstance(existing_provider, trace.NoOpTracerProvider)
+    and type(existing_provider).__name__ in ("ProxyTracerProvider", "TracerProvider")
+)
+
+if is_opentelemetry_instrument:
+    print("Detected opentelemetry-instrument - using existing OpenTelemetry setup")
+    print(f"Exporter: {exporter_type}, Endpoint: {exporter_endpoint}")
+    # Still call setup_telemetry to ensure console exporter is added if needed
+    # setup_telemetry handles ProxyTracerProvider by creating a new TracerProvider
+    # This ensures we have a console exporter configured for immediate output
+    setup_telemetry(
+        service_name="optimize-me",
+        service_version="0.1.0",
+        exporter_type=exporter_type,  # Use console when OTEL_TRACES_EXPORTER=console
+        exporter_endpoint=exporter_endpoint,
+        use_auto_instrumentation=True,
+    )
+    if exporter_type == "console":
+        print("✅ Console exporter configured - traces will appear below")
+else:
+    # Call setup_telemetry to create TracerProvider and configure exporter
+    setup_telemetry(
+        service_name="optimize-me",
+        service_version="0.1.0",
+        exporter_type=exporter_type,
+        exporter_endpoint=exporter_endpoint,
+        use_auto_instrumentation=True,  # Use auto-instrumentation (standard pattern)
+    )
+
+# Auto-instrument all custom functions in src package
+# This automatically traces all functions without requiring decorators
+# IMPORTANT: This must happen BEFORE importing the modules
+print("\nSetting up auto-instrumentation for custom functions...")
+auto_instrument_package(
+    'src',
+    include_private=False,  # Don't instrument private functions (starting with _)
+    exclude_modules=['src.tests', 'src.telemetry']  # Exclude test and telemetry modules
+)
+print("✅ Auto-instrumentation enabled - all functions will be traced automatically")
+
+# NOW import modules - functions will be automatically wrapped
+from src.numerical.optimization import gradient_descent
+from src.algorithms.graph import graph_traversal, find_node_clusters, PathFinder, calculate_node_betweenness
+from src.algorithms.dynamic_programming import fibonacci, matrix_sum, matrix_chain_order, coin_change, knapsack
+from src.data_processing.dataframe import dataframe_filter, groupby_mean, dataframe_merge
+from src.statistics.descriptive import describe, correlation
+
+print("\n" + "=" * 80)
+print("RUNNING ALL INSTRUMENTED FUNCTIONS")
+print("=" * 80)
+print("\nTraces will appear as JSON objects below each function call.\n")
+
+# ============================================================================
+# Numerical Optimization
+# ============================================================================
+print("\n--- Numerical Optimization ---")
+print("Running gradient_descent...")
+X = np.array([[1, 2], [3, 4], [5, 6]])
+y = np.array([1, 2, 3])
+weights = gradient_descent(X, y, learning_rate=0.01, iterations=100)
+print(f"Result: {weights}\n")
+
+# ============================================================================
+# Graph Algorithms
+# ============================================================================
+print("\n--- Graph Algorithms ---")
+
+print("Running graph_traversal...")
+graph = {1: {2, 3}, 2: {4}, 3: {4}, 4: {}}
+visited = graph_traversal(graph, 1)
+print(f"Result: {visited}\n")
+
+print("Running find_node_clusters...")
+nodes = [{"id": 1}, {"id": 2}, {"id": 3}, {"id": 4}]
+edges = [{"source": 1, "target": 2}, {"source": 3, "target": 4}]
+clusters = find_node_clusters(nodes, edges)
+print(f"Result: {clusters}\n")
+
+print("Running PathFinder.find_shortest_path...")
+path_finder = PathFinder({"A": ["B", "C"], "B": ["D"], "C": ["D"], "D": []})
+path = path_finder.find_shortest_path("A", "D")
+print(f"Result: {path}\n")
+
+print("Running calculate_node_betweenness...")
+nodes_list = ["A", "B", "C", "D"]
+edges_list = [{"source": "A", "target": "B"}, {"source": "B", "target": "C"}, {"source": "C", "target": "D"}]
+betweenness = calculate_node_betweenness(nodes_list, edges_list)
+print(f"Result: {betweenness}\n")
+
+# ============================================================================
+# Dynamic Programming
+# ============================================================================
+print("\n--- Dynamic Programming ---")
+
+print("Running fibonacci...")
+fib_result = fibonacci(10)
+print(f"Result: {fib_result}\n")
+
+print("Running matrix_sum...")
+matrix = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
+matrix_result = matrix_sum(matrix)
+print(f"Result: {matrix_result}\n")
+
+print("Running matrix_chain_order...")
+matrices = [(10, 20), (20, 30), (30, 40)]
+chain_result = matrix_chain_order(matrices)
+print(f"Result: {chain_result}\n")
+
+print("Running coin_change...")
+coins = [1, 2, 5]
+amount = 5
+coin_result = coin_change(coins, amount, 0)
+print(f"Result: {coin_result}\n")
+
+print("Running knapsack...")
+weights = [10, 20, 30]
+values = [60, 100, 120]
+capacity = 50
+knapsack_result = knapsack(weights, values, capacity, len(weights))
+print(f"Result: {knapsack_result}\n")
+
+# ============================================================================
+# Data Processing
+# ============================================================================
+print("\n--- Data Processing ---")
+
+print("Running dataframe_filter...")
+df = pd.DataFrame({"A": [1, 2, 3, 4, 5], "B": [10, 20, 30, 40, 50]})
+filtered = dataframe_filter(df, "A", 3)
+print(f"Result:\n{filtered}\n")
+
+print("Running groupby_mean...")
+df_group = pd.DataFrame({
+    "group": ["A", "A", "B", "B", "C"],
+    "value": [10, 20, 30, 40, 50]
+})
+grouped = groupby_mean(df_group, "group", "value")
+print(f"Result: {grouped}\n")
+
+print("Running dataframe_merge...")
+df_left = pd.DataFrame({"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"]})
+df_right = pd.DataFrame({"id": [2, 3, 4], "age": [25, 30, 35]})
+merged = dataframe_merge(df_left, df_right, "id", "id")
+print(f"Result:\n{merged}\n")
+
+# ============================================================================
+# Statistics
+# ============================================================================
+print("\n--- Statistics ---")
+
+print("Running describe...")
+series = pd.Series([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+stats = describe(series)
+print(f"Result: {stats}\n")
+
+print("Running correlation...")
+df_corr = pd.DataFrame({
+    "x": [1, 2, 3, 4, 5],
+    "y": [2, 4, 6, 8, 10],
+    "z": [1, 3, 5, 7, 9]
+})
+corr_result = correlation(df_corr)
+print(f"Result: {corr_result}\n")
+
+print("=" * 80)
+print("ALL FUNCTIONS EXECUTED")
+print("=" * 80)
+
+if exporter_type == "console":
+    print("\n✅ Traces printed to console above (JSON format)")
+    print("\nTo view traces in Jaeger UI:")
+    print("  1. Start services: cd src/telemetry && docker-compose up -d")
+    print("  2. Set environment: export OTEL_TRACES_EXPORTER=otlp")
+    print("  3. Run: python examples/run_all_traces.py")
+    print("  4. Open: http://localhost:16686")
+elif exporter_type == "otlp":
+    print(f"\n✅ Traces sent to OTLP endpoint: {exporter_endpoint}")
+    print("\nView traces in Jaeger UI: http://localhost:16686")
+    print("(Make sure docker-compose is running: cd src/telemetry && docker-compose up -d)")
+
+print("\nOpenTelemetry auto-instrumentation automatically captured:")
+print("  - NumPy operations (array operations)")
+print("  - Pandas operations (DataFrame operations)")
+print("  - Function execution times")
+print("  - Service information (service.name, service.version)")
+print("\nFor more details, see: src/telemetry/README.md")
+
+# Force flush spans to ensure they're exported (especially for console exporter)
+try:
+    from opentelemetry import trace
+    from opentelemetry.sdk.trace import TracerProvider
+    provider = trace.get_tracer_provider()
+    if isinstance(provider, TracerProvider):
+        if hasattr(provider, "force_flush"):
+            provider.force_flush(timeout_millis=5000)  # Wait up to 5 seconds
+            print("\n✅ Spans flushed to exporter")
+except Exception as e:
+    if os.getenv("OTEL_LOG_LEVEL", "").upper() == "DEBUG":
+        print(f"\n⚠️ Could not flush spans: {e}")
+
+print()
+