Re-enable classroom limits and update alerts and metrics (#371)

* update threshold and alarm

* add logs

* update log

* update log

* remove debug help and update config

* update name

Author: molly-moen

cicd/3-app/javabuilder/config/production.config.json

@@ -2,11 +2,12 @@
   "Parameters": {
     "BaseDomainName": "code.org",
     "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU",
-    "ProvisionedConcurrentExecutions": "150",
-    "ReservedConcurrentExecutions": "1650",
+    "ProvisionedConcurrentExecutions": "50",
+    "ReservedConcurrentExecutions": "500",
     "LimitPerHour": "1000",
     "LimitPerDay": "5000",
-    "SilenceAlerts": "false"
+    "SilenceAlerts": "false",
+    "TeacherLimitPerHour": "25000"
   },
   "Tags" : {
     "EnvType" : "production"

cicd/3-app/javabuilder/template.yml.erb

@@ -1012,12 +1012,11 @@ Resources:
         ComparisonOperator: GreaterThanThreshold
         TreatMissingData: notBreaching
 
-  HighClassroomsBlockedAlarm:
+  ClassroomBlockedAlarm:
     Type: AWS::CloudWatch::Alarm
     Properties:
-        AlarmName: !Sub "${SubDomainName}_high_classrooms_blocked"
-        AlarmDescription: Unusually high number of classrooms being newly blocked by our throttling
-            thresholds.
+        AlarmName: !Sub "${SubDomainName}_classroom_blocked"
+        AlarmDescription: A classroom was newly blocked by our threshold. Investigate if this is classroom should be blocked.
         ActionsEnabled: true
         OKActions: []
         AlarmActions:
@@ -1032,8 +1031,8 @@ Resources:
         Period: 60
         EvaluationPeriods: 1
         DatapointsToAlarm: 1
-        Threshold: 10
-        ComparisonOperator: GreaterThanThreshold
+        Threshold: 1
+        ComparisonOperator: GreaterThanOrEqualToThreshold
         TreatMissingData: notBreaching
 
   HighUnknownTokensAlarm:

javabuilder-authorizer/metrics_reporter.rb

@@ -31,6 +31,23 @@ def log(metric_name)
     @client.put_metric_data({namespace: "Javabuilder", metric_data: [metric_data]})
   end
 
+  def log_count_with_value(metric_name, value)
+    metric_data = {
+      metric_name: metric_name,
+      dimensions: [
+        {
+          name: "functionName",
+          value: function_name
+        }
+      ],
+      unit: "Count",
+      value: value
+    }
+
+    @client.put_metric_data({namespace: "Javabuilder", metric_data: [metric_data]})
+  end
+
+
   private
 
   # ARN is of the format arn:aws:lambda:{region}:{account_id}:function:{lambda_name}

javabuilder-authorizer/token_status.rb

@@ -43,4 +43,5 @@ module TokenStatus
 
   NEW_USER_BLOCKED = 'NewUserBlocked'.freeze
   NEW_CLASSROOM_BLOCKED = 'NewClassroomBlocked'.freeze
+  CLASSROOM_HOURLY_REQUEST_COUNT = 'ClassroomHourlyRequestCount'.freeze
 end

javabuilder-authorizer/token_validator.rb

@@ -26,16 +26,11 @@ def validate
     # TODO: uncomment other throttling thresholds once we have fine-tuned those limits
     return error(TOKEN_USED) unless log_token
     return error(USER_BLOCKED) if user_blocked?
-    # return error(CLASSROOM_BLOCKED) if teachers_blocked?
+    return error(CLASSROOM_BLOCKED) if teachers_blocked?
     hourly_usage_response = user_usage(ONE_HOUR_SECONDS)
     return error(USER_BLOCKED) if user_over_hourly_limit?(hourly_usage_response)
     # return error(USER_BLOCKED) if user_over_daily_limit?
-    # return error(CLASSROOM_BLOCKED) if teachers_over_hourly_limit?
-    
-    # for now, check and log if the teacher is over the hourly limit, but still 
-    # allow those users through until we fine tune the limit
-    teachers_over_hourly_limit?
-
+    return error(CLASSROOM_BLOCKED) if teachers_over_hourly_limit?
 
     near_limit_detail = get_user_near_hourly_limit_detail(hourly_usage_response.count)
     log_requests
@@ -130,6 +125,10 @@ def teachers_over_hourly_limit?
         puts "teacher_associated_requests query has paginated responses. user_id #{@user_id} teacher_id #{teacher_id}"
       end
 
+      # Log the count of requests per hour to a metric. We are logging this for now so we can determine if we
+      # have set a good threshold for blocking classrooms.
+      @metrics_reporter.log_count_with_value(CLASSROOM_HOURLY_REQUEST_COUNT, response.count)
+
       if response.count > ENV['teacher_limit_per_hour'].to_i
         begin
           @dynamodb_client.put_item(