engine v1 improved dequeue selection algorithm (#1632)

* engine v1 fairness improvements (env based queue selection)

* Add ability to bias env selection by concurrency limit and capacity

* Add some randomization to the list of queues within each evn

* Configure biases using env vars

* Typo

* revert v3 catalog project ref change

* Fix v2 marqs

Author: ericallam

.vscode/launch.json

@@ -22,6 +22,15 @@
       "cwd": "${workspaceFolder}/apps/webapp",
       "sourceMaps": true
     },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug fairDequeuingStrategy.test.ts",
+      "command": "pnpm run test -t FairDequeuingStrategy",
+      "envFile": "${workspaceFolder}/.env",
+      "cwd": "${workspaceFolder}/apps/webapp",
+      "sourceMaps": true
+    },
     {
       "type": "chrome",
       "request": "launch",

apps/webapp/app/env.server.ts

@@ -199,6 +199,8 @@ const EnvironmentSchema = z.object({
   INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED: z.string().default("0"),
   INTERNAL_OTEL_TRACE_DISABLED: z.string().default("0"),
 
+  INTERNAL_OTEL_LOG_EXPORTER_URL: z.string().optional(),
+
   ORG_SLACK_INTEGRATION_CLIENT_ID: z.string().optional(),
   ORG_SLACK_INTEGRATION_CLIENT_SECRET: z.string().optional(),
 
@@ -221,9 +223,12 @@ const EnvironmentSchema = z.object({
     .number()
     .int()
     .default(60 * 1000 * 15),
-  MARQS_SHARED_QUEUE_SELECTION_COUNT: z.coerce.number().int().default(36),
-  MARQS_DEV_QUEUE_SELECTION_COUNT: z.coerce.number().int().default(12),
+  MARQS_SHARED_QUEUE_LIMIT: z.coerce.number().int().default(1000),
+  MARQS_DEV_QUEUE_LIMIT: z.coerce.number().int().default(1000),
   MARQS_MAXIMUM_NACK_COUNT: z.coerce.number().int().default(64),
+  MARQS_CONCURRENCY_LIMIT_BIAS: z.coerce.number().default(0.75),
+  MARQS_AVAILABLE_CAPACITY_BIAS: z.coerce.number().default(0.3),
+  MARQS_QUEUE_AGE_RANDOMIZATION_BIAS: z.coerce.number().default(0.25),
 
   PROD_TASK_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(),