Small fixes for local monitoring setup,

Fix job-executor with disabled otel-monitoring

Author: MSpryszynski

handler.js

@@ -19,7 +19,7 @@ const {
     cpuMetric,
     memoryMetric,
     cTimeMetric,
-} = require('./metrics-definition');
+} = process.env.HF_VAR_ENABLE_OTEL === "1" ? require('./metrics-definition') : {};
 
 const {
     procfs,

metrics-definition.js

@@ -1,4 +1,4 @@
-const meter = process.env.HF_VAR_ENABLE_TRACING === "1" ? require("./metrics.js")("hyperflow-job-executor"): undefined;
+const meter = process.env.HF_VAR_ENABLE_OTEL === "1" ? require("./metrics.js")("hyperflow-job-executor"): undefined;
 
 
 const cpuMetric = meter.createObservableGauge('cpu-usage', {

monitoring-local-setup/docker-compose.yml

@@ -20,12 +20,13 @@ services:
     container_name: opensearch
     environment:
       - cluster.name=opensearch-cluster
-      - node.name=opensearc
+      - node.name=opensearch
       - discovery.seed_hosts=opensearch
       - cluster.initial_cluster_manager_nodes=opensearch
       - bootstrap.memory_lock=true
       - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
       - OPENSEARCH_INITIAL_ADMIN_PASSWORD=Hyperflow1!
+      - plugins.security.disabled=true
     ulimits:
       memlock:
         soft: -1
@@ -48,7 +49,8 @@ services:
     expose:
       - "5601"
     environment:
-      OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
+      OPENSEARCH_HOSTS: '["http://opensearch:9200"]'
+      DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
     networks:
       - backend
   data-prepper:

monitoring-local-setup/otel-collector-config.yaml

@@ -25,10 +25,6 @@ exporters:
       insecure: true
       insecure_skip_verify: true
 
-  logging:
-    verbosity: detailed
-
-
 service:
   pipelines:
     traces:
@@ -41,5 +37,5 @@ service:
       exporters: [otlp/metrics]
     logs:
       receivers: [otlp]
-      exporters: [logging, otlp/logs]
+      exporters: [otlp/logs]
 

monitoring-local-setup/pipelines.yaml

@@ -16,12 +16,13 @@ raw-pipeline:
     - otel_trace_raw:
   sink:
     - opensearch:
-        hosts: ["https://opensearch:9200"]
+        hosts: ["http://opensearch:9200"]
         insecure: true
         username: admin
         password: "Hyperflow1!"
         index_type: custom
         index: hyperflow_traces
+        ssl: false
 service-map-pipeline:
   delay: "100"
   source:
@@ -31,11 +32,12 @@ service-map-pipeline:
     - service_map_stateful:
   sink:
     - opensearch:
-        hosts: ["https://opensearch:9200"]
+        hosts: ["http://opensearch:9200"]
         insecure: true
         username: admin
         password: "Hyperflow1!"
         index_type: trace-analytics-service-map
+        ssl: false
 
 
 metrics-pipeline:
@@ -44,22 +46,24 @@ metrics-pipeline:
       ssl: false
   sink:
     - opensearch:
-        hosts: ["https://opensearch:9200"]
+        hosts: ["http://opensearch:9200"]
         insecure: true
         username: admin
         password: "Hyperflow1!"
         index_type: custom
         index: hyperflow_metrics
+        ssl: false
 
 logs-pipeline:
   source:
     otel_logs_source:
       ssl: false
   sink:
     - opensearch:
-        hosts: ["https://opensearch:9200"]
+        hosts: ["http://opensearch:9200"]
         insecure: true
         username: admin
         password: "Hyperflow1!"
         index: hyperflow_logs
+        ssl: false
 

package.json

@@ -15,7 +15,11 @@
     "connector.js",
     "handler.js",
     "jobexec.js",
-    "hflow-job-listener.js"
+    "hflow-job-listener.js",
+    "logs.js",
+    "metrics.js",
+    "metrics-definition.js",
+    "tracing.js"
   ],
   "dependencies": {
     "@opentelemetry/api": "^1.9.0",