App OOM fixtures

.github/workflows/deploy.yml

@@ -64,6 +64,7 @@ jobs:
     runs-on: grid-bot-infra
 
     needs: parse-input-components
+    environment: ${{ github.event.inputs.nomad_environment }}
 
     if: ${{ needs.parse-input-components.outputs.nomad-files != '{}' && needs.parse-input-components.outputs.nomad-files != '' }}
 
@@ -76,16 +77,7 @@ jobs:
         uses: nferch/setup-nomad@v4.0.0
         env:
           NOMAD_TLS_SKIP_VERIFY: 1
-
-      - name: Set Initial GitHub Deployment Status
-        uses: chrnorm/deployment-action@v2
-        id: deployment
-        continue-on-error: true
-        with:
-          token: "${{ secrets.DEPLOYER_TOKEN }}"
-          environment: "${{ github.event.inputs.nomad_environment }}"
-          description: "Components: ${{ github.event.inputs.components }}"
-
+
       - name: Deploy Nomad Jobs
         uses: actions/github-script@v7
         id: deploy-nomad-jobs
@@ -121,15 +113,3 @@ jobs:
               core.setFailed(`Failed to deploy the following jobs: ${failedJobs.join(', ')}`);
             }
 
-      - name: Set GitHub Deployment Status
-        uses: chrnorm/deployment-status@v2
-        continue-on-error: true
-        with:
-          token: "${{ secrets.DEPLOYER_TOKEN }}"
-          deployment-id: ${{ steps.deployment.outputs.deployment_id }}
-          description: "Components: ${{ github.event.inputs.components }}"
-
-          # If the "deploy" step fails, the deployment status will be set to "failure"
-          # If the "deploy" step succeeds, the deployment status will be set to "success"
-          state: "${{ steps.deploy-nomad-jobs.outcome }}"
-

services/grid-bot/.component.yaml

@@ -1,9 +1,5 @@
 component: grid-bot
 
-# This is only used by the build worklow,
-# it determines how the component is built
-# Docker only relevant when the argument
-# 
 build:
   project_file: src/Grid.Bot.csproj
   component_directory: ./.deploy
@@ -40,32 +36,27 @@ deployment:
           grpc:
             static: 5000
           http:
-            static: 8888
+            static: 8882
       services:
         - name: ${{ env.NOMAD_ENVIRONMENT }}-grid-bot
           port: metrics
           tags:
             - ${{ env.NOMAD_ENVIRONMENT }}
+            - prometheus # For prometheus collector
           checks:
-            - type: http
-              path: /metrics
+            - type: tcp
 
-        - name: ${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web
-          port: http
-          tags:
-            - ${{ env.NOMAD_ENVIRONMENT }}
-            - "traefik.enable=true"
-            - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-http.rule=(Host(`clientsettingscdn.sitetest4.robloxlabs.com`) || Host(`versioncompatibility.api.sitetest4.robloxlabs.com`))"
-            - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-http.entrypoints=http"
-
-        - name: ${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-https
+        - name: ${{ env.NOMAD_ENVIRONMENT }}-grid-bot
           port: http
           tags:
             - ${{ env.NOMAD_ENVIRONMENT }}
             - "traefik.enable=true"
-            - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-https.rule=(Host(`clientsettingscdn.sitetest4.robloxlabs.com`) || Host(`versioncompatibility.api.sitetest4.robloxlabs.com`))"
-            - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-https.entrypoints=https"
-            - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-https.tls=true"
+            - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web.rule=(Host(`clientsettingscdn.sitetest4.robloxlabs.com`) || Host(`versioncompatibility.api.sitetest4.robloxlabs.com`) || Host(`avatar.sitetest4.robloxlabs.com`))"
+            - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web.entrypoints=http"
+
+            - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-secure.rule=(Host(`clientsettingscdn.sitetest4.robloxlabs.com`) || Host(`versioncompatibility.api.sitetest4.robloxlabs.com`) || Host(`avatar.sitetest4.robloxlabs.com`))"
+            - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-secure.entrypoints=https"
+            - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-secure.tls=true"
 
       volumes:
         - '/var/run/docker.sock:/var/run/docker.sock'

services/grid-bot/Dockerfile

@@ -1,9 +1,12 @@
-# Base Image: net8.0
 FROM mcr.microsoft.com/dotnet/aspnet:8.0.1-jammy
 
 WORKDIR /opt/grid
 COPY . /opt/grid/
 
+RUN apt-get -y update && \
+    apt-get install -y dotnet-sdk-8.0
+RUN dotnet tool install --global dotnet-dump && dotnet tool install --global dotnet-counters
+
 COPY ./ssl/global-root-ca.crt /usr/local/share/ca-certificates/global-root-ca.crt
 RUN chmod 644 /usr/local/share/ca-certificates/global-root-ca.crt && update-ca-certificates
 

services/grid-bot/lib/Directory.Build.props

@@ -7,8 +7,6 @@
     <TargetFramework>net8.0</TargetFramework>
     <LangVersion>preview</LangVersion>
     <RootNamespace>Grid.Bot</RootNamespace>
-
-    <Configurations>debug;release</Configurations>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -41,4 +39,4 @@
   <PropertyGroup Label="RootDirectory">
     <RootDirectory>$(MSBuildThisFileDirectory)..\..\..\</RootDirectory>
   </PropertyGroup>
-</Project>
+</Project>

services/grid-bot/lib/commands/Shared.Commands.csproj

@@ -14,12 +14,12 @@
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' != 'true' ">
-    <PackageReference Include="mfdlabs.FileSystem" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.FloodCheckers.Core" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.FloodCheckers.Redis" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Text" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Networking" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Configuration" Version="1.0.9" /> 
+    <PackageReference Include="mfdlabs.FileSystem" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.FloodCheckers.Core" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.FloodCheckers.Redis" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Text" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Networking" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Configuration" Version="1.1.1" /> 
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' == 'true' ">

services/grid-bot/lib/events/Events/OnLogMessage.cs

@@ -103,21 +103,21 @@ public Task Invoke(LogMessage message)
         if (message.Exception != null)
         {
 #if DEBUG || DEBUG_LOGGING_IN_PROD
-#if !DEBUG // Don't log these exceptions outside of debug mode.
-            if (message.Exception is GatewayReconnectException)
-                return Task.CompletedTask;
+            if (!_settings.DebugAllowGatewayWebsocketExceptions) {
+                if (message.Exception is GatewayReconnectException)
+                    return Task.CompletedTask;
 
-            // Closed web socket exceptions are expected when the bot is shutting down.
-            if (message.Exception.InnerException is WebSocketException)
-                return Task.CompletedTask;
+                // Closed web socket exceptions are expected when the bot is shutting down.
+                if (message.Exception.InnerException is WebSocketException)
+                    return Task.CompletedTask;
 
-            if (message.Exception is WebSocketClosedException || message.Exception.InnerException is WebSocketClosedException)
-                return Task.CompletedTask;
+                if (message.Exception is WebSocketClosedException || message.Exception.InnerException is WebSocketClosedException)
+                    return Task.CompletedTask;
+            }
 
             if (message.Exception is TaskCanceledException &&
                 !_settings.DebugAllowTaskCanceledExceptions)
                 return Task.CompletedTask;
-#endif
 
             // Temporary fix for discord-net/Discord.Net#3128
             // Just keep it out of Backtrace and increment a counter.

services/grid-bot/lib/events/Events/OnReady.cs

@@ -15,7 +15,6 @@
 using Threading;
 using Text.Extensions;
 
-
 /// <summary>
 /// Event handler to be invoked when a shard is ready,
 /// </summary>
@@ -62,7 +61,7 @@ OnCommandExecuted onCommandExecutedEvent
 {
     private static readonly Assembly _commandsAssembly = Assembly.Load("Shared.Commands");
 
-    private Atomic<int> _shardCount = 0; // needs to be atomic due to the race situation here.
+    private OnceFlag _initializeClientOnceFlag = new();
 
     private readonly DiscordSettings _discordSettings = discordSettings ?? throw new ArgumentNullException(nameof(discordSettings));
     private readonly MaintenanceSettings _maintenanceSettings = maintenanceSettings ?? throw new ArgumentNullException(nameof(maintenanceSettings));
@@ -85,21 +84,17 @@ private static string GetStatusText(string updateText)
     /// Invoe the event handler.
     /// </summary>
     /// <param name="shard">The client for the shard.</param>
-    public async Task Invoke(DiscordSocketClient shard)
+    public Task Invoke(DiscordSocketClient shard)
     {
-        _shardCount++;
-
         _logger.Debug(
             "Shard '{0}' ready as '{0}#{1}'",
             shard.ShardId,
             _client.CurrentUser.Username,
             _client.CurrentUser.Discriminator
         );
 
-        if (_shardCount == _client.Shards.Count)
+        Call.Once(ref _initializeClientOnceFlag, async () =>
         {
-            _logger.Debug("Final shard ready!");
-
             await _interactionService.AddModulesAsync(_commandsAssembly, _services);
             await _commandService.AddModulesAsync(_commandsAssembly, _services);
 
@@ -136,6 +131,8 @@ public async Task Invoke(DiscordSocketClient shard)
                 _client.SetGameAsync(
                     _discordSettings.BotStatusMessage
                 );
-        }
+        });
+
+        return Task.CompletedTask;
     }
 }

services/grid-bot/lib/events/Shared.Events.csproj

@@ -18,7 +18,7 @@
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' != 'true' ">
-    <PackageReference Include="mfdlabs.Logging" Version="1.0.9" />
+    <PackageReference Include="mfdlabs.Logging" Version="1.1.1" />
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' == 'true' ">

services/grid-bot/lib/settings/Providers/DiscordSettings.cs

@@ -28,12 +28,20 @@ public class DiscordSettings : BaseSettingsProvider
 
 #if DEBUG || DEBUG_LOGGING_IN_PROD
     /// <summary>
-    /// Can task cancelled exceptions be loggeD?
+    /// Can task cancelled exceptions be logged?
     /// </summary>
     public bool DebugAllowTaskCanceledExceptions => GetOrDefault(
         nameof(DebugAllowTaskCanceledExceptions),
         false
     );
+
+    /// <summary>
+    /// Can websocket or gateway exceptions be logged?
+    /// </summary>
+    public bool DebugAllowGatewayWebsocketExceptions => GetOrDefault(
+        nameof(DebugAllowGatewayWebsocketExceptions),
+        false
+    );
 #endif
 
 #if DEBUG

services/grid-bot/lib/settings/Shared.Settings.csproj

@@ -16,16 +16,16 @@
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' != 'true' ">
-    <PackageReference Include="mfdlabs.Configuration" Version="1.0.9" /> 
-    <PackageReference Include="mfdlabs.Redis" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.ServiceDiscovery" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.FloodCheckers.Redis" Version="1.0.9" />
-
-    <PackageReference Include="mfdlabs.Grid.ProcessManagement.Docker" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Grid.ProcessManagement.Core" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Grid.ProcessManagement" Version="1.0.9" />
+    <PackageReference Include="mfdlabs.Configuration" Version="1.1.1" /> 
+    <PackageReference Include="mfdlabs.Redis" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.ServiceDiscovery" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.FloodCheckers.Redis" Version="1.1.1" />
+
+    <PackageReference Include="mfdlabs.Grid.ProcessManagement.Docker" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Grid.ProcessManagement.Core" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Grid.ProcessManagement" Version="1.1.1" />
 
-    <PackageReference Include="mfdlabs.Thumbnails.Client" Version="1.0.9" />
+    <PackageReference Include="mfdlabs.Thumbnails.Client" Version="1.1.1" />
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' == 'true' ">

services/grid-bot/lib/utility/Shared.Utility.csproj

@@ -13,20 +13,20 @@
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' != 'true' ">
-    <PackageReference Include="mfdlabs.FileSystem" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.FloodCheckers.Core" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.FloodCheckers.Redis" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Grid.Commands" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Grid.Client" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Logging" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Redis" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.ServiceDiscovery" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Threading" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Users.Client" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Grid.ProcessManagement.Docker" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Grid.ProcessManagement.Core" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Networking" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Thumbnails.Client" Version="1.0.9" />
+    <PackageReference Include="mfdlabs.FileSystem" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.FloodCheckers.Core" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.FloodCheckers.Redis" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Grid.Commands" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Grid.Client" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Logging" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Redis" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.ServiceDiscovery" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Threading" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Users.Client" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Grid.ProcessManagement.Docker" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Grid.ProcessManagement.Core" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Networking" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Thumbnails.Client" Version="1.1.1" />
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' == 'true' ">
@@ -58,4 +58,4 @@
     <EmbeddedResource Include="Lua/LuaVMTemplate.lua" />
     <None Remove="Lua/LuaVMTemplate.lua" />
   </ItemGroup>
-</Project>
+</Project>

services/grid-bot/lib/web/Extensions/IServiceProviderExtensions.cs

@@ -156,4 +156,4 @@ public static void UseWebServer(this IServiceProvider services, IEnumerable<stri
         // Start the web server
         Task.Factory.StartNew(() => app.Run(webSettings.WebServerBindAddress), TaskCreationOptions.LongRunning);
     }
-}
+}

services/grid-bot/lib/web/Grid.Bot.Web.csproj

@@ -13,11 +13,11 @@
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' != 'true' ">
-    <PackageReference Include="mfdlabs.FileSystem" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Logging" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Redis" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.ServiceDiscovery" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Threading" Version="1.0.9" />
+    <PackageReference Include="mfdlabs.FileSystem" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Logging" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Redis" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.ServiceDiscovery" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Threading" Version="1.1.1" />
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' == 'true' ">

services/grid-bot/src/Grid.Bot.csproj

@@ -4,7 +4,6 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <Configurations>debug;release</Configurations>
     <OutputType>Exe</OutputType>
 
     <AutoGenerateBindingRedirects>True</AutoGenerateBindingRedirects>
@@ -25,9 +24,9 @@
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' != 'true' ">
-    <PackageReference Include="mfdlabs.Random" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Redis" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Configuration.Core" Version="1.0.9" />
+    <PackageReference Include="mfdlabs.Random" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Redis" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Configuration.Core" Version="1.1.1" />
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' == 'true' ">

services/recovery/src/Grid.Bot.Recovery.csproj

@@ -4,7 +4,6 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <Configurations>debug;release</Configurations>
     <OutputType>Exe</OutputType>
 
     <AutoGenerateBindingRedirects>True</AutoGenerateBindingRedirects>
@@ -15,9 +14,9 @@
   </PropertyGroup> 
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' != 'true' ">
-    <PackageReference Include="mfdlabs.Configuration" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Logging" Version="1.0.9" />
-    <PackageReference Include="mfdlabs.Networking" Version="1.0.9" />
+    <PackageReference Include="mfdlabs.Configuration" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Logging" Version="1.1.1" />
+    <PackageReference Include="mfdlabs.Networking" Version="1.1.1" />
   </ItemGroup>
 
   <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' == 'true' ">