[Node] Custom Metrics E2E Test

.github/workflows/node-ec2-default-test.yml

@@ -45,7 +45,7 @@ env:
   NODE_VERSION: ${{ inputs.node-version }}
   CPU_ARCHITECTURE: ${{ inputs.cpu-architecture }}
   ADOT_INSTRUMENTATION_NAME: ${{ inputs.staging-instrumentation-name }}
-  SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}/node-sample-app.zip
+  SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}/node-sample-app-delete-me.zip
   E2E_TEST_ACCOUNT_ID: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}
   E2E_TEST_ROLE_NAME: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }}
   METRIC_NAMESPACE: ApplicationSignals
@@ -244,9 +244,27 @@ jobs:
           --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
           --rollup'
 
+      - name: Validate custom metrics
+        id: cwagent-metric-validation
+        if: (success() || steps.log-validation.outcome == 'failure') && !cancelled()
+        run: ./gradlew validator:run --args='-c node/ec2/default/custom-metric-validation.yml
+          --testing-id ${{ env.TESTING_ID }}
+          --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
+          --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001
+          --region ${{ inputs.aws-region }}
+          --account-id ${{ env.ACCOUNT_ID }}
+          --metric-namespace CWAgent
+          --log-group ${{ env.LOG_GROUP_NAME }}
+          --service-name node-sample-application-${{ env.TESTING_ID }}
+          --remote-service-name node-sample-remote-application-${{ env.TESTING_ID }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
+          --instance-ami ${{ env.EC2_INSTANCE_AMI }}
+          --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
+          --rollup'
+
       - name: Validate generated traces
         id: trace-validation
-        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
+        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure' || steps.cwagent-metric-validation.outcome == 'failure') && !cancelled()
         run: ./gradlew validator:run --args='-c node/ec2/default/trace-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
@@ -273,7 +291,7 @@ jobs:
         if: always()
         id: validation-result
         run: |
-          if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
+          if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.cwagent-metric-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
             echo "validation-result=success" >> $GITHUB_OUTPUT
           else
             echo "validation-result=failure" >> $GITHUB_OUTPUT

.github/workflows/test.yml

@@ -0,0 +1,23 @@
+## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+## SPDX-License-Identifier: Apache-2.0
+
+# This is a reusable workflow for running the Enablement test for App Signals.
+# It is meant to be called from another workflow.
+# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview
+name: Test
+on:
+  push:
+    branches:
+      - Node_Custom_metrics
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  node-ec2-default:
+    uses: ./.github/workflows/node-ec2-default-test.yml
+    secrets: inherit
+    with:
+      caller-workflow-name: 'test'
+      aws-region: 'us-east-1'

sample-apps/node/frontend-service/index.js

@@ -5,6 +5,9 @@ const express = require('express');
 const mysql = require('mysql2');
 const bunyan = require('bunyan');
 const { S3Client, GetBucketLocationCommand } = require('@aws-sdk/client-s3');
+const opentelemetry = require('@opentelemetry/sdk-node');
+const { metrics } = require('@opentelemetry/api');
+const { randomInt } = require('crypto');
 
 const PORT = parseInt(process.env.SAMPLE_APP_PORT || '8000', 10);
 
@@ -13,6 +16,61 @@ const app = express();
 // Create bunyan logger
 const logger = bunyan.createLogger({name: 'express-app', level: 'info'});
 
+let pipelineMeter = null;
+
+if (process.env.SERVICE_NAME && process.env.DEPLOYMENT_ENVIRONMENT_NAME) {
+    const { Resource } = require('@opentelemetry/resources');
+    const { MeterProvider, PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics');
+    const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-proto');
+
+    const serviceName = process.env.SERVICE_NAME;
+    const deploymentEnv = process.env.DEPLOYMENT_ENVIRONMENT_NAME;
+
+    const pipelineResource = new Resource({
+        // SEMRESATTRS_DEPLOYMENT_ENVIRONMENT_NAME maps to dimension 'deployment.name' so "deployment.environment.name" used 
+        // to assign value correctly.
+        'service.name': serviceName,
+        'deployment.environment.name': deploymentEnv
+    });
+
+    const pipelineMetricExporter = new OTLPMetricExporter({
+        url: 'http://localhost:4317'
+    });
+
+    const pipelineMetricReader = new PeriodicExportingMetricReader({
+        exporter: pipelineMetricExporter,
+        exportIntervalMillis: 1000
+    });
+
+    const pipelineMeterProvider = new MeterProvider({
+        resource: pipelineResource,
+        readers: [pipelineMetricReader]
+    });
+
+    pipelineMeter = pipelineMeterProvider.getMeter('myMeter');
+}
+
+
+const meter = metrics.getMeter('myMeter');
+const agent_based_counter = meter.createCounter('agent_based_counter', {description: 'agent export counter'});
+const agent_based_histogram = meter.createHistogram('agent_based_histogram', {description: 'agent export histogram'});
+const agent_based_gauge = meter.createUpDownCounter('agent_based_gauge', {description: 'agent export gauge'});
+
+let custom_pipeline_counter = null;
+let custom_pipeline_histogram = null;
+let custom_pipeline_gauge = null;
+
+if (pipelineMeter) {
+  custom_pipeline_counter = pipelineMeter.createCounter('custom_pipeline_counter', {unit: '1', description: 'pipeline export counter'});
+  custom_pipeline_histogram = pipelineMeter.createHistogram('custom_pipeline_histogram', {description: 'pipeline export histogram'});
+  custom_pipeline_gauge = pipelineMeter.createUpDownCounter('custom_pipeline_gauge', {unit: '1', description: 'pipeline export gauge'});
+}
+
+console.log('=== Metrics Setup Complete ===');
+app.get('/', (req, res) => {
+  res.send('Node.js Application Started! Available endpoints: /healthcheck, /aws-sdk-call, /outgoing-http-call, /remote-service, /client-call, /mysql');
+});
+
 app.get('/healthcheck', (req, res) => {
   logger.info('/healthcheck called successfully');
   res.send('healthcheck');
@@ -44,9 +102,17 @@ app.get('/aws-sdk-call', async (req, res) => {
   const s3Client = new S3Client({ region: 'us-east-1' });
   const bucketName = 'e2e-test-bucket-name-' + (req.query.testingId || 'MISSING_ID');
 
-  // Add custom warning log for validation testing
-  const warningMsg = "This is a custom log for validation testing";
-  logger.warn(warningMsg);
+  // Increment counter/histogram/gauge for agent export
+  agent_based_counter.add(1, { Operation : 'counter' });
+  agent_based_histogram.record(randomInt(100,1001), { Operation : 'histogram' });
+  agent_based_gauge.add(randomInt(-10, 11), { Operation : 'gauge' });
+
+  // Increment counter/histogram/gauge for pipeline export
+  if (custom_pipeline_counter) {
+    custom_pipeline_counter.add(1, { Operation : 'pipeline_counter' });
+    custom_pipeline_histogram.record(randomInt(100,1001), { Operation : 'pipeline_histogram' });
+    custom_pipeline_gauge.add(randomInt(-10, 11), { Operation : 'pipeline_gauge' });
+  }
 
   try {
     await s3Client.send(
@@ -120,6 +186,8 @@ app.get('/client-call', (req, res) => {
   makeAsyncCall = true;
 });
 
+
+
 app.get('/mysql', (req, res) => {
   // Create a connection to the MySQL database
   const connection = mysql.createConnection({

sample-apps/node/frontend-service/package.json

@@ -10,11 +10,18 @@
   "author": "",
   "license": "Apache-2.0",
   "dependencies": {
-    "@aws-sdk/client-s3": "3.621.0",
+    "@aws-sdk/client-s3": "^3.621.0",
+    "@aws/aws-distro-opentelemetry-node-autoinstrumentation": "^0.8.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-metrics-otlp-proto": "^0.57.1",
+    "@opentelemetry/resources": "^1.30.1",
+    "@opentelemetry/sdk-metrics": "^1.30.1",
+    "@opentelemetry/semantic-conventions": "^1.37.0",
+    "@opentelemetry/sdk-node": "^0.57.0",
     "@types/express": "^4.17.21",
     "@types/node": "^20.14.6",
+    "bunyan": "^1.8.15",
     "express": "^4.21.2",
-    "mysql2": "^3.11.0",
-    "bunyan": "^1.8.15"
+    "mysql2": "^3.15.3"
   }
 }

terraform/node/ec2/default/amazon-cloudwatch-agent.json

@@ -10,7 +10,11 @@
   },
   "logs": {
     "metrics_collected": {
-      "application_signals": {}
+      "application_signals": {},
+      "otlp": {
+        "grpc_endpoint": "0.0.0.0:4317",
+        "http_endpoint": "0.0.0.0:4318"
+      }
     }
   }
 }

terraform/node/ec2/default/main.tf

@@ -141,8 +141,8 @@ resource "null_resource" "main_service_setup" {
       sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:./amazon-cloudwatch-agent.json
 
       # Get and run the sample application with configuration
-      aws s3 cp ${var.sample_app_zip} ./node-sample-app.zip
-      unzip -o node-sample-app.zip
+      aws s3 cp ${var.sample_app_zip} ./node-sample-app-delete-me.zip
+      unzip -o node-sample-app-delete-me.zip
 
       # Enter appropriate service folder
       cd frontend-service
@@ -159,14 +159,19 @@ resource "null_resource" "main_service_setup" {
       # Export environment variables for instrumentation
       # Note: We use OTEL_NODE_DISABLED_INSTRUMENTATIONS=fs,dns,express to avoid
       # having to validate around the telemetry generated for middleware
-      tmux send-keys -t frontend 'export OTEL_METRICS_EXPORTER=none' C-m
+      tmux send-keys -t frontend 'export OTEL_METRICS_EXPORTER=otlp' C-m
       tmux send-keys -t frontend 'export OTEL_TRACES_EXPORTER=otlp' C-m
       tmux send-keys -t frontend 'export OTEL_AWS_APPLICATION_SIGNALS_ENABLED=true' C-m
       tmux send-keys -t frontend 'export OTEL_AWS_APPLICATION_SIGNALS_RUNTIME_ENABLED=false' C-m
       tmux send-keys -t frontend 'export OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT=http://localhost:4316/v1/metrics' C-m
       tmux send-keys -t frontend 'export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4316/v1/traces' C-m
       tmux send-keys -t frontend 'export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf' C-m
       tmux send-keys -t frontend 'export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=http/protobuf' C-m
+      tmux send-keys -t frontend 'export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:4318/v1/metrics' C-m
+      tmux send-keys -t frontend 'export OTEL_EXPORTER_OTLP_METRICS_INSECURE=true' C-m
+      tmux send-keys -t frontend 'export OTEL_RESOURCE_ATTRIBUTES="service.name=node-sample-application-${var.test_id},deployment.environment.name=ec2:default"' C-m
+      tmux send-keys -t frontend 'export TESTING_ID=${var.test_id}' C-m
+      tmux send-keys -t frontend 'export AWS_REGION=${var.aws_region}' C-m
       tmux send-keys -t frontend 'export OTEL_NODE_DISABLED_INSTRUMENTATIONS=fs,dns,express' C-m
       tmux send-keys -t frontend 'export OTEL_SERVICE_NAME=node-sample-application-${var.test_id}' C-m
       tmux send-keys -t frontend 'export OTEL_TRACES_SAMPLER=always_on' C-m
@@ -258,8 +263,8 @@ resource "null_resource" "remote_service_setup" {
       sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:./amazon-cloudwatch-agent.json
 
       # Get and run the sample application with configuration
-      aws s3 cp ${var.sample_app_zip} ./node-sample-app.zip
-      unzip -o node-sample-app.zip
+      aws s3 cp ${var.sample_app_zip} ./node-sample-app-delete-me.zip
+      unzip -o node-sample-app-delete-me.zip
 
       # Enter appropriate service folder
       cd remote-service

validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java

@@ -422,6 +422,9 @@ public enum PredefinedExpectedTemplate implements FileConfig {
   NODE_EC2_DEFAULT_AWS_SDK_CALL_METRIC("/expected-data-template/node/ec2/default/aws-sdk-call-metric.mustache"),
   NODE_EC2_DEFAULT_AWS_SDK_CALL_TRACE("/expected-data-template/node/ec2/default/aws-sdk-call-trace.mustache"),
 
+  /** Node EC2 Default Custom Metrics Test Case Validations */
+  NODE_EC2_DEFAULT_AWS_OTEL_CUSTOM_METRIC("/expected-data-template/node/ec2/default/aws-otel-custom-metrics.mustache"),
+
   NODE_EC2_DEFAULT_REMOTE_SERVICE_LOG("/expected-data-template/node/ec2/default/remote-service-log.mustache"),
   NODE_EC2_DEFAULT_REMOTE_SERVICE_METRIC("/expected-data-template/node/ec2/default/remote-service-metric.mustache"),
   NODE_EC2_DEFAULT_REMOTE_SERVICE_TRACE("/expected-data-template/node/ec2/default/remote-service-trace.mustache"),