[.NET] Custom Metrics E2E Test

.github/workflows/dotnet-ec2-default-test.yml

@@ -229,9 +229,26 @@ jobs:
           --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
           --rollup'
 
+      - name: Validate custom metrics
+        id: cwagent-metric-validation
+        if: (success() || steps.log-validation.outcome == 'failure' || steps.log-validation.outcome == 'failure') && !cancelled()
+        run: ./gradlew validator:run --args='-c dotnet/ec2/default/custom-metric-validation.yml
+          --testing-id ${{ env.TESTING_ID }}
+          --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
+          --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8081
+          --region ${{ env.E2E_TEST_AWS_REGION }}
+          --metric-namespace CWAgent
+          --log-group ${{ env.LOG_GROUP_NAME }}
+          --service-name dotnet-sample-application-${{ env.TESTING_ID }}
+          --remote-service-name dotnet-sample-remote-application-${{ env.TESTING_ID }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}
+          --instance-ami ${{ env.EC2_INSTANCE_AMI }}
+          --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
+          --rollup'
+
       - name: Validate generated traces
         id: trace-validation
-        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
+        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure' || steps.custom-metric-validation.outcome == 'failure') && !cancelled()
         run: ./gradlew validator:run --args='-c dotnet/ec2/default/trace-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
@@ -258,7 +275,7 @@ jobs:
         if: always()
         id: validation-result
         run: |
-          if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
+          if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.custom-metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
             echo "validation-result=success" >> $GITHUB_OUTPUT
           else
             echo "validation-result=failure" >> $GITHUB_OUTPUT

.github/workflows/test.yml

@@ -0,0 +1,23 @@
+## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+## SPDX-License-Identifier: Apache-2.0
+
+# This is a reusable workflow for running the Enablement test for App Signals.
+# It is meant to be called from another workflow.
+# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview
+name: Test
+on:
+  push:
+    branches:
+      - DOTNET_Custom_metrics
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  dotnet-ec2-default:
+    uses: ./.github/workflows/dotnet-ec2-default-test.yml
+    secrets: inherit
+    with:
+      caller-workflow-name: 'test'
+      aws-region: 'us-east-1'

sample-apps/dotnet/asp_frontend_service/Controllers/AppController.cs

@@ -9,6 +9,13 @@
 using Amazon.S3;
 using Microsoft.AspNetCore.Mvc;
 using Amazon.S3.Model;
+using System.Diagnostics.Metrics;
+using System.Collections.Generic;
+using OpenTelemetry;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Exporter;
+
 
 namespace asp_frontend_service.Controllers;
 
@@ -22,6 +29,53 @@ public class AppController : ControllerBase
     private static bool threadStarted = false;
     private readonly AmazonS3Client s3Client = new AmazonS3Client();
     private readonly HttpClient httpClient = new HttpClient();
+    private static readonly Meter meter = new Meter("myMeter");
+    private static readonly Counter<int> agentBasedCounter = meter.CreateCounter<int>("agent_based_counter");
+    private static readonly Histogram<double> agentBasedHistogram = meter.CreateHistogram<double>("agent_based_histogram");
+    private static readonly UpDownCounter<int> agentBasedGauge = meter.CreateUpDownCounter<int>("agent_based_gauge");
+
+    // Custom pipeline metrics - only create if specific env vars exist
+    private static readonly Meter? pipelineMeter;
+    private static readonly Counter<int>? customPipelineCounter;
+    private static readonly Histogram<double>? customPipelineHistogram;
+    private static readonly UpDownCounter<int>? customPipelineGauge;
+    private static readonly MeterProvider? pipelineMeterProvider;
+
+    static AppController()
+    {
+        var serviceName = Environment.GetEnvironmentVariable("SERVICE_NAME");
+        var deploymentEnv = Environment.GetEnvironmentVariable("DEPLOYMENT_ENVIRONMENT_NAME");
+
+        if (!string.IsNullOrEmpty(serviceName) && !string.IsNullOrEmpty(deploymentEnv))
+        {
+            var pipelineResource = ResourceBuilder.CreateDefault()
+                .AddAttributes(new Dictionary<string, object>
+                {
+                    ["service.name"] = serviceName,
+                    ["deployment.environment.name"] = deploymentEnv
+                })
+                .Build();
+
+            pipelineMeterProvider = Sdk.CreateMeterProviderBuilder()
+                .SetResourceBuilder(ResourceBuilder.CreateDefault().AddAttributes(new Dictionary<string, object>
+                {
+                    ["service.name"] = serviceName,
+                    ["deployment.environment.name"] = deploymentEnv
+                }))
+                .AddOtlpExporter(options =>
+                {
+                    options.Endpoint = new Uri("http://localhost:4318/v1/metrics");
+                    options.Protocol = OtlpExportProtocol.HttpProtobuf;
+                })
+                .AddMeter("myMeter")
+                .Build();
+
+            pipelineMeter = new Meter("myMeter");
+            customPipelineCounter = pipelineMeter.CreateCounter<int>("custom_pipeline_counter", "1", "pipeline export counter");
+            customPipelineHistogram = pipelineMeter.CreateHistogram<double>("custom_pipeline_histogram", "ms", "pipeline export histogram");
+            customPipelineGauge = pipelineMeter.CreateUpDownCounter<int>("custom_pipeline_gauge", "1", "pipeline export gauge");
+        }
+    }
 
     private static readonly Thread thread = new Thread(() =>
             {
@@ -50,7 +104,6 @@ public AppController()
     {
         if (!threadStarted)
         {
-            Console.WriteLine("Starting thread");
             threadStarted = true;
             thread.Start();
         }
@@ -69,7 +122,32 @@ public string OutgoingHttp()
     [Route("/aws-sdk-call")]
     public string AWSSDKCall([FromQuery] string testingId)
     {
-       var request = new GetBucketLocationRequest()
+        var random = new Random();
+
+        // Agent-based metrics
+        var histogramValue = random.NextDouble() * 100;
+        var gaugeValue = random.Next(-10, 11);
+
+        agentBasedCounter.Add(1, new KeyValuePair<string, object?>("Operation", "counter"));
+        agentBasedHistogram.Record(histogramValue, new KeyValuePair<string, object?>("Operation", "histogram"));
+        agentBasedGauge.Add(gaugeValue, new KeyValuePair<string, object?>("Operation", "gauge"));
+
+        // Custom pipeline metrics - only record if pipeline exists
+        if (customPipelineCounter != null)
+        {
+            customPipelineCounter.Add(1, new KeyValuePair<string, object?>("Operation", "pipeline_counter"));
+            customPipelineHistogram?.Record(random.Next(100, 1001), new KeyValuePair<string, object?>("Operation", "pipeline_histogram"));
+            customPipelineGauge?.Add(random.Next(-10, 11), new KeyValuePair<string, object?>("Operation", "pipeline_gauge"));
+        }
+
+
+        var bucketName = "e2e-test-bucket-name";
+        if (!string.IsNullOrEmpty(testingId))
+        {
+            bucketName += "-" + testingId;
+        }
+
+        var request = new GetBucketLocationRequest()
             {
                BucketName = testingId
             };

sample-apps/dotnet/asp_frontend_service/Startup.cs

@@ -7,6 +7,9 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
+using System.Collections.Generic;
 
 namespace asp_frontend_service;
 
@@ -25,6 +28,22 @@ public void ConfigureServices(IServiceCollection services)
         services.AddControllers();
 
         AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
+
+        // Configure OpenTelemetry for custom pipeline metrics
+        services.AddOpenTelemetry()
+            .ConfigureResource(resource => resource
+                .AddService(Environment.GetEnvironmentVariable("SERVICE_NAME") ?? "dotnet-sample-application")
+                .AddAttributes(new Dictionary<string, object> { { "Telemetry.Source", "UserMetric" } }))
+            .WithMetrics(metrics => metrics
+                .AddAspNetCoreInstrumentation()
+                .AddMeter("myMeter")
+                .AddOtlpExporter(options => {
+                    options.Endpoint = new Uri(Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT") ?? "http://localhost:4318/v1/metrics");
+                })
+                .AddConsoleExporter((exporterOptions, metricReaderOptions) =>
+                {
+                    metricReaderOptions.PeriodicExportingMetricReaderOptions.ExportIntervalMilliseconds = 1000;
+                }));
     }
 
     // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.

sample-apps/dotnet/asp_frontend_service/asp_frontend_service.csproj

@@ -8,6 +8,11 @@
   <ItemGroup>
     <PackageReference Include="AWSSDK.S3" Version="3.7.310.7" />
     <PackageReference Include="AWSSDK.SecurityToken" Version="3.7.300.120" />
+    <PackageReference Include="OpenTelemetry" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>

terraform/dotnet/ec2/default/amazon-cloudwatch-agent.json

@@ -10,7 +10,11 @@
   },
   "logs": {
     "metrics_collected": {
-      "application_signals": {}
+      "application_signals": {},
+      "otlp": {
+        "grpc_endpoint": "0.0.0.0:4317",
+        "http_endpoint": "0.0.0.0:4318"
+      }
     }
   }
 }

terraform/dotnet/ec2/default/main.tf

@@ -132,8 +132,8 @@ resource "null_resource" "main_service_setup" {
       ${var.get_adot_distro_command}
 
       # Get and run the sample application with configuration
-      aws s3 cp ${var.sample_app_zip} ./dotnet-sample-app.zip
-      unzip -o dotnet-sample-app.zip
+      aws s3 cp ${var.sample_app_zip} ./dotnet-sample-app-delete-me.zip
+      unzip -o dotnet-sample-app-delete-me.zip
 
       # Get Absolute Path
       current_dir=$(pwd)
@@ -153,8 +153,13 @@ resource "null_resource" "main_service_setup" {
       export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
       export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4316
       export OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT=http://127.0.0.1:4316/v1/metrics
-      export OTEL_METRICS_EXPORTER=none
-      export OTEL_RESOURCE_ATTRIBUTES=service.name=dotnet-sample-application-${var.test_id}
+      export OTEL_METRICS_EXPORTER=otlp
+      export OTEL_DOTNET_AUTO_METRICS_ADDITIONAL_SOURCES=myMeter
+      export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:4318/v1/metrics
+      export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=http/protobuf
+      export SERVICE_NAME='dotnet-sample-application-${var.test_id}'
+      export DEPLOYMENT_ENVIRONMENT_NAME='ec2:default'
+      export OTEL_RESOURCE_ATTRIBUTES="service.name=$${SERVICE_NAME},deployment.environment.name=$${DEPLOYMENT_ENVIRONMENT_NAME}"
       export OTEL_AWS_APPLICATION_SIGNALS_ENABLED=true
       export OTEL_AWS_APPLICATION_SIGNALS_RUNTIME_ENABLED=false
       export OTEL_TRACES_SAMPLER=always_on
@@ -240,8 +245,8 @@ resource "null_resource" "remote_service_setup" {
       ${var.get_adot_distro_command}
 
       # Get and run the sample application with configuration
-      aws s3 cp ${var.sample_app_zip} ./dotnet-sample-app.zip
-      unzip -o dotnet-sample-app.zip
+      aws s3 cp ${var.sample_app_zip} ./dotnet-sample-app-delete-me.zip
+      unzip -o dotnet-sample-app-delete-me.zip
 
       # Get Absolute Path
       current_dir=$(pwd)

validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java

@@ -315,6 +315,9 @@ public enum PredefinedExpectedTemplate implements FileConfig {
   DOTNET_EC2_WINDOWS_DEFAULT_AWS_SDK_CALL_METRIC("/expected-data-template/dotnet/ec2/windows/aws-sdk-call-metric.mustache"),
   DOTNET_EC2_WINDOWS_DEFAULT_AWS_SDK_CALL_TRACE("/expected-data-template/dotnet/ec2/windows/aws-sdk-call-trace.mustache"),
 
+    /** DOTNET EC2 Default Custom Metrics Test Case Validations */
+  DOTNET_EC2_DEFAULT_AWS_OTEL_CUSTOM_METRIC("/expected-data-template/dotnet/ec2/default/aws-otel-custom-metrics.mustache"),
+
   DOTNET_EC2_WINDOWS_DEFAULT_REMOTE_SERVICE_LOG("/expected-data-template/dotnet/ec2/windows/remote-service-log.mustache"),
   DOTNET_EC2_WINDOWS_DEFAULT_REMOTE_SERVICE_METRIC("/expected-data-template/dotnet/ec2/windows/remote-service-metric.mustache"),
 //  Because of a time sync issue, block the remote service trace check for now