diff --git a/app/jobs/runtime/events_cleanup.rb b/app/jobs/runtime/events_cleanup.rb index a1841576395..873c6a7f319 100644 --- a/app/jobs/runtime/events_cleanup.rb +++ b/app/jobs/runtime/events_cleanup.rb @@ -9,7 +9,7 @@ def initialize(cutoff_age_in_days) end def perform - Database::OldRecordCleanup.new(Event, cutoff_age_in_days).delete + Database::OldRecordCleanup.new(Event, cutoff_age_in_days:).delete end def job_name_in_configuration diff --git a/app/repositories/app_usage_event_repository.rb b/app/repositories/app_usage_event_repository.rb index 51d751a8cd1..8955c326b19 100644 --- a/app/repositories/app_usage_event_repository.rb +++ b/app/repositories/app_usage_event_repository.rb @@ -152,7 +152,7 @@ def purge_and_reseed_started_apps! end def delete_events_older_than(cutoff_age_in_days) - Database::OldRecordCleanup.new(AppUsageEvent, cutoff_age_in_days, keep_at_least_one_record: true).delete + Database::OldRecordCleanup.new(AppUsageEvent, cutoff_age_in_days: cutoff_age_in_days, keep_at_least_one_record: true, keep_running_records: true).delete end private diff --git a/app/repositories/service_usage_event_repository.rb b/app/repositories/service_usage_event_repository.rb index 35fbef5a1f0..1f55434c621 100644 --- a/app/repositories/service_usage_event_repository.rb +++ b/app/repositories/service_usage_event_repository.rb @@ -92,7 +92,7 @@ def purge_and_reseed_service_instances! end def delete_events_older_than(cutoff_age_in_days) - Database::OldRecordCleanup.new(ServiceUsageEvent, cutoff_age_in_days, keep_at_least_one_record: true).delete + Database::OldRecordCleanup.new(ServiceUsageEvent, cutoff_age_in_days: cutoff_age_in_days, keep_at_least_one_record: true, keep_running_records: true).delete end end end diff --git a/db/migrations/20251103141149_add_lifecycle_index_to_usage_events.rb b/db/migrations/20251103141149_add_lifecycle_index_to_usage_events.rb new file mode 100644 index 00000000000..728394f11c0 --- /dev/null +++ b/db/migrations/20251103141149_add_lifecycle_index_to_usage_events.rb @@ -0,0 +1,68 @@ +Sequel.migration do + no_transaction # to use the 'concurrently' option + + up do + if database_type == :postgres + VCAP::Migration.with_concurrent_timeout(self) do + add_index :app_usage_events, %i[state app_guid id], + name: :app_usage_events_lifecycle_index, + if_not_exists: true, + concurrently: true + + add_index :service_usage_events, %i[state service_instance_guid id], + name: :service_usage_events_lifecycle_index, + if_not_exists: true, + concurrently: true + end + + elsif database_type == :mysql + alter_table :app_usage_events do + # rubocop:disable Sequel/ConcurrentIndex + add_index %i[state app_guid id], name: :app_usage_events_lifecycle_index unless @db.indexes(:app_usage_events).include?(:app_usage_events_lifecycle_index) + # rubocop:enable Sequel/ConcurrentIndex + end + + alter_table :service_usage_events do + # rubocop:disable Sequel/ConcurrentIndex + unless @db.indexes(:service_usage_events).include?(:service_usage_events_lifecycle_index) + add_index %i[state service_instance_guid id], + name: :service_usage_events_lifecycle_index + end + # rubocop:enable Sequel/ConcurrentIndex + end + end + end + + down do + if database_type == :postgres + VCAP::Migration.with_concurrent_timeout(self) do + drop_index :app_usage_events, %i[state app_guid id], + name: :app_usage_events_lifecycle_index, + if_exists: true, + concurrently: true + + drop_index :service_usage_events, %i[state service_instance_guid id], + name: :service_usage_events_lifecycle_index, + if_exists: true, + concurrently: true + end + end + + if database_type == :mysql + alter_table :app_usage_events do + # rubocop:disable Sequel/ConcurrentIndex + drop_index %i[state app_guid id], name: :app_usage_events_lifecycle_index if @db.indexes(:app_usage_events).include?(:app_usage_events_lifecycle_index) + # rubocop:enable Sequel/ConcurrentIndex + end + + alter_table :service_usage_events do + # rubocop:disable Sequel/ConcurrentIndex + if @db.indexes(:service_usage_events).include?(:service_usage_events_lifecycle_index) + drop_index %i[state service_instance_guid id], + name: :service_usage_events_lifecycle_index + end + # rubocop:enable Sequel/ConcurrentIndex + end + end + end +end diff --git a/lib/database/old_record_cleanup.rb b/lib/database/old_record_cleanup.rb index 44227d84507..455d6a629e2 100644 --- a/lib/database/old_record_cleanup.rb +++ b/lib/database/old_record_cleanup.rb @@ -3,16 +3,17 @@ module Database class OldRecordCleanup class NoCurrentTimestampError < StandardError; end - attr_reader :model, :days_ago, :keep_at_least_one_record + attr_reader :model, :cutoff_age_in_days, :keep_at_least_one_record, :keep_running_records - def initialize(model, days_ago, keep_at_least_one_record: false) + def initialize(model, cutoff_age_in_days:, keep_at_least_one_record: false, keep_running_records: false) @model = model - @days_ago = days_ago + @cutoff_age_in_days = cutoff_age_in_days @keep_at_least_one_record = keep_at_least_one_record + @keep_running_records = keep_running_records end def delete - cutoff_date = current_timestamp_from_database - days_ago.to_i.days + cutoff_date = current_timestamp_from_database - cutoff_age_in_days.to_i.days old_records = model.dataset.where(Sequel.lit('created_at < ?', cutoff_date)) if keep_at_least_one_record @@ -21,6 +22,8 @@ def delete end logger.info("Cleaning up #{old_records.count} #{model.table_name} table rows") + old_records = exclude_running_records(old_records) if keep_running_records + Database::BatchDelete.new(old_records, 1000).delete end @@ -35,5 +38,66 @@ def current_timestamp_from_database def logger @logger ||= Steno.logger('cc.old_record_cleanup') end + + def exclude_running_records(old_records) + return old_records unless has_duration?(model) + + beginning_string = beginning_string(model) + ending_string = ending_string(model) + guid_symbol = guid_symbol(model) + + raise "Invalid duration model: #{model}" if beginning_string.nil? || ending_string.nil? || guid_symbol.nil? + + # Create subqueries for START and STOP records within the old records set + # Using from_self creates a subquery, allowing us to reference these in complex joins + initial_records = old_records.where(state: beginning_string).from_self(alias: :initial_records) + final_records = old_records.where(state: ending_string).from_self(alias: :final_records) + + # For each START record, check if there exists a STOP record that: + # 1. Has the same resource GUID (app_guid or service_instance_guid) + # 2. Was created AFTER the START record (higher ID implies later creation) + exists_condition = final_records.where(Sequel[:final_records][guid_symbol] => Sequel[:initial_records][guid_symbol]).where do + Sequel[:final_records][:id] > Sequel[:initial_records][:id] + end.select(1).exists + + prunable_initial_records = initial_records.where(exists_condition) + + # Include records with states other than START/STOP + other_records = old_records.exclude(state: [beginning_string, ending_string]) + + # Return the UNION of: + # 1. START records that have a matching STOP (safe to delete) + # 2. All STOP records (always safe to delete) + # 3. Other state records (always safe to delete) + prunable_initial_records.union(final_records, all: true).union(other_records, all: true) + end + + def has_duration?(model) + return true if model == VCAP::CloudController::AppUsageEvent + return true if model == VCAP::CloudController::ServiceUsageEvent + + false + end + + def beginning_string(model) + return VCAP::CloudController::ProcessModel::STARTED if model == VCAP::CloudController::AppUsageEvent + return VCAP::CloudController::Repositories::ServiceUsageEventRepository::CREATED_EVENT_STATE if model == VCAP::CloudController::ServiceUsageEvent + + nil + end + + def ending_string(model) + return VCAP::CloudController::ProcessModel::STOPPED if model == VCAP::CloudController::AppUsageEvent + return VCAP::CloudController::Repositories::ServiceUsageEventRepository::DELETED_EVENT_STATE if model == VCAP::CloudController::ServiceUsageEvent + + nil + end + + def guid_symbol(model) + return :app_guid if model == VCAP::CloudController::AppUsageEvent + return :service_instance_guid if model == VCAP::CloudController::ServiceUsageEvent + + nil + end end end diff --git a/spec/unit/jobs/runtime/app_usage_events_cleanup_spec.rb b/spec/unit/jobs/runtime/app_usage_events_cleanup_spec.rb index 8018ee0ee69..ce4f1df9956 100644 --- a/spec/unit/jobs/runtime/app_usage_events_cleanup_spec.rb +++ b/spec/unit/jobs/runtime/app_usage_events_cleanup_spec.rb @@ -5,7 +5,7 @@ module Jobs::Runtime RSpec.describe AppUsageEventsCleanup, job_context: :worker do let(:cutoff_age_in_days) { 30 } let(:logger) { double(Steno::Logger, info: nil) } - let!(:event_before_threshold) { AppUsageEvent.make(created_at: (cutoff_age_in_days + 1).days.ago) } + let!(:event_before_threshold) { AppUsageEvent.make(created_at: (cutoff_age_in_days + 1).days.ago, state: 'STOPPED') } let!(:event_after_threshold) { AppUsageEvent.make(created_at: (cutoff_age_in_days - 1).days.ago) } subject(:job) do diff --git a/spec/unit/jobs/services/service_usage_events_cleanup_spec.rb b/spec/unit/jobs/services/service_usage_events_cleanup_spec.rb index 181ad89e18d..da11beb39b5 100644 --- a/spec/unit/jobs/services/service_usage_events_cleanup_spec.rb +++ b/spec/unit/jobs/services/service_usage_events_cleanup_spec.rb @@ -5,7 +5,7 @@ module Jobs::Services RSpec.describe ServiceUsageEventsCleanup, job_context: :worker do let(:cutoff_age_in_days) { 30 } let(:logger) { double(Steno::Logger, info: nil) } - let!(:event_before_threshold) { ServiceUsageEvent.make(created_at: (cutoff_age_in_days + 1).days.ago) } + let!(:event_before_threshold) { ServiceUsageEvent.make(created_at: (cutoff_age_in_days + 1).days.ago, state: 'DELETED') } let!(:event_after_threshold) { ServiceUsageEvent.make(created_at: (cutoff_age_in_days - 1).days.ago) } subject(:job) do diff --git a/spec/unit/lib/database/old_record_cleanup_spec.rb b/spec/unit/lib/database/old_record_cleanup_spec.rb index 7b2258bbaec..867bf013f90 100644 --- a/spec/unit/lib/database/old_record_cleanup_spec.rb +++ b/spec/unit/lib/database/old_record_cleanup_spec.rb @@ -8,8 +8,10 @@ let!(:fresh_event) { VCAP::CloudController::Event.make(created_at: 1.day.ago + 1.minute) } + # ==================== CORE FUNCTIONALITY ==================== + it 'deletes records older than specified days' do - record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::Event, 1) + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::Event, cutoff_age_in_days: 1) expect do record_cleanup.delete @@ -20,23 +22,14 @@ expect { stale_event2.reload }.to raise_error(Sequel::NoExistingObject) end - context "when there are no records at all but you're trying to keep at least one" do - it "doesn't keep one because there aren't any to keep" do - record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::ServiceUsageEvent, 1, keep_at_least_one_record: true) - - expect { record_cleanup.delete }.not_to raise_error - expect(VCAP::CloudController::ServiceUsageEvent.count).to eq(0) - end - end - it 'only retrieves the current timestamp from the database once' do expect(VCAP::CloudController::Event.db).to receive(:fetch).with('SELECT CURRENT_TIMESTAMP as now').once.and_call_original - record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::Event, 1) + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::Event, cutoff_age_in_days: 1) record_cleanup.delete end it 'keeps the last row when :keep_at_least_one_record is true even if it is older than the cutoff date' do - record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::Event, 0, keep_at_least_one_record: true) + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::Event, cutoff_age_in_days: 0, keep_at_least_one_record: true) expect do record_cleanup.delete @@ -46,5 +39,320 @@ expect { stale_event1.reload }.to raise_error(Sequel::NoExistingObject) expect { stale_event2.reload }.to raise_error(Sequel::NoExistingObject) end + + context "when there are no records at all but you're trying to keep at least one" do + it "doesn't keep one because there aren't any to keep" do + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::ServiceUsageEvent, cutoff_age_in_days: 1, keep_at_least_one_record: true) + + expect { record_cleanup.delete }.not_to raise_error + expect(VCAP::CloudController::ServiceUsageEvent.count).to eq(0) + end + end + + # ==================== KEEP_RUNNING_RECORDS: AppUsageEvent ==================== + + it 'keeps AppUsageEvent start record when there is no corresponding stop record' do + stale_app_usage_event_start = VCAP::CloudController::AppUsageEvent.make(created_at: 2.days.ago, state: 'STARTED', app_guid: 'guid1') + + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::AppUsageEvent, cutoff_age_in_days: 1, keep_at_least_one_record: false, keep_running_records: true) + record_cleanup.delete + expect(stale_app_usage_event_start.reload).to be_present + expect(VCAP::CloudController::AppUsageEvent.count).to eq(1) + end + + it 'keeps AppUsageEvent start record when stop record is fresh' do + stale_app_usage_event_start = VCAP::CloudController::AppUsageEvent.make(created_at: 2.days.ago, state: 'STARTED', app_guid: 'guid1') + fresh_app_usage_event_stop = VCAP::CloudController::AppUsageEvent.make(created_at: 1.day.ago + 1.minute, state: 'STOPPED', app_guid: 'guid1') + + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::AppUsageEvent, cutoff_age_in_days: 1, keep_at_least_one_record: false, keep_running_records: true) + record_cleanup.delete + expect(stale_app_usage_event_start.reload).to be_present + expect(fresh_app_usage_event_stop.reload).to be_present + end + + it 'keeps AppUsageEvent start record when stop record was inserted first' do + stale_app_usage_event_stop = VCAP::CloudController::AppUsageEvent.make(created_at: 3.days.ago, state: 'STOPPED', app_guid: 'guid1') + stale_app_usage_event_start = VCAP::CloudController::AppUsageEvent.make(created_at: 2.days.ago, state: 'STARTED', app_guid: 'guid1') + + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::AppUsageEvent, cutoff_age_in_days: 1, keep_at_least_one_record: false, keep_running_records: true) + record_cleanup.delete + expect(stale_app_usage_event_start.reload).to be_present + expect { stale_app_usage_event_stop.reload }.to raise_error(Sequel::NoExistingObject) + end + + it 'deletes old AppUsageEvent records when they have a corresponding stop record' do + app_guid = 'app-with-multiple-cycles' + + cycle1_start = VCAP::CloudController::AppUsageEvent.make(created_at: 10.days.ago, state: 'STARTED', app_guid: app_guid) + cycle1_stop = VCAP::CloudController::AppUsageEvent.make(created_at: 9.days.ago, state: 'STOPPED', app_guid: app_guid) + + cycle2_start = VCAP::CloudController::AppUsageEvent.make(created_at: 8.days.ago, state: 'STARTED', app_guid: app_guid) + cycle2_stop = VCAP::CloudController::AppUsageEvent.make(created_at: 7.days.ago, state: 'STOPPED', app_guid: app_guid) + + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::AppUsageEvent, cutoff_age_in_days: 1, keep_at_least_one_record: false, keep_running_records: true) + record_cleanup.delete + + expect { cycle1_start.reload }.to raise_error(Sequel::NoExistingObject) + expect { cycle1_stop.reload }.to raise_error(Sequel::NoExistingObject) + expect { cycle2_start.reload }.to raise_error(Sequel::NoExistingObject) + expect { cycle2_stop.reload }.to raise_error(Sequel::NoExistingObject) + end + + it 'deletes a single old AppUsageEvent stop record with no start record' do + single_stop = VCAP::CloudController::AppUsageEvent.make(created_at: 10.days.ago, state: 'STOPPED', app_guid: 'stopped-app') + + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::AppUsageEvent, cutoff_age_in_days: 1, keep_at_least_one_record: false, keep_running_records: true) + record_cleanup.delete + + expect { single_stop.reload }.to raise_error(Sequel::NoExistingObject) + expect(VCAP::CloudController::AppUsageEvent.count).to eq(0) + end + + # ==================== KEEP_RUNNING_RECORDS: ServiceUsageEvent ==================== + + it 'keeps ServiceUsageEvent create record when there is no corresponding delete record' do + stale_service_usage_event_create = VCAP::CloudController::ServiceUsageEvent.make(created_at: 2.days.ago, state: 'CREATED', service_instance_guid: 'guid1') + + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::ServiceUsageEvent, cutoff_age_in_days: 1, keep_at_least_one_record: false, keep_running_records: true) + record_cleanup.delete + expect(stale_service_usage_event_create.reload).to be_present + expect(VCAP::CloudController::ServiceUsageEvent.count).to eq(1) + end + + it 'keeps ServiceUsageEvent create record when delete record is fresh' do + stale_service_usage_event_create = VCAP::CloudController::ServiceUsageEvent.make(created_at: 2.days.ago, state: 'CREATED', service_instance_guid: 'guid1') + fresh_service_usage_event_delete = VCAP::CloudController::ServiceUsageEvent.make(created_at: 1.day.ago + 1.minute, state: 'DELETED', service_instance_guid: 'guid1') + + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::ServiceUsageEvent, cutoff_age_in_days: 1, keep_at_least_one_record: false, keep_running_records: true) + record_cleanup.delete + expect(stale_service_usage_event_create.reload).to be_present + expect(fresh_service_usage_event_delete.reload).to be_present + end + + it 'keeps ServiceUsageEvent create record when delete record was inserted first' do + stale_service_usage_event_delete = VCAP::CloudController::ServiceUsageEvent.make(created_at: 3.days.ago, state: 'DELETED', service_instance_guid: 'guid1') + stale_service_usage_event_create = VCAP::CloudController::ServiceUsageEvent.make(created_at: 2.days.ago, state: 'CREATED', service_instance_guid: 'guid1') + + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::ServiceUsageEvent, cutoff_age_in_days: 1, keep_at_least_one_record: false, keep_running_records: true) + record_cleanup.delete + expect(stale_service_usage_event_create.reload).to be_present + expect { stale_service_usage_event_delete.reload }.to raise_error(Sequel::NoExistingObject) + end + + it 'keeps all ServiceUsageEvent created records when there is no corresponding deleted record' do + service_guid = 'multi-create-service' + + create1 = VCAP::CloudController::ServiceUsageEvent.make(created_at: 10.days.ago, state: 'CREATED', service_instance_guid: service_guid) + create2 = VCAP::CloudController::ServiceUsageEvent.make(created_at: 8.days.ago, state: 'CREATED', service_instance_guid: service_guid) + create3 = VCAP::CloudController::ServiceUsageEvent.make(created_at: 6.days.ago, state: 'CREATED', service_instance_guid: service_guid) + + record_cleanup = Database::OldRecordCleanup.new( + VCAP::CloudController::ServiceUsageEvent, + cutoff_age_in_days: 1, + keep_running_records: true + ) + record_cleanup.delete + + expect(create1.reload).to be_present + expect(create2.reload).to be_present + expect(create3.reload).to be_present + end + + it 'deletes ServiceUsageEvent deleted records without a create' do + service_guid = 'orphan-delete-service' + + orphan_delete = VCAP::CloudController::ServiceUsageEvent.make( + created_at: 10.days.ago, + state: 'DELETED', + service_instance_guid: service_guid + ) + + record_cleanup = Database::OldRecordCleanup.new( + VCAP::CloudController::ServiceUsageEvent, + cutoff_age_in_days: 1, + keep_running_records: true + ) + record_cleanup.delete + + expect { orphan_delete.reload }.to raise_error(Sequel::NoExistingObject) + end + + # ==================== EDGE CASES & DATA INTEGRITY ==================== + + it 'deletes records with non-lifecycle states when keep_running_records is true' do + # Create records with various states, all old + buildpack_event1 = VCAP::CloudController::AppUsageEvent.make(created_at: 3.days.ago, state: 'BUILDPACK_SET', app_guid: 'app1') + buildpack_event2 = VCAP::CloudController::AppUsageEvent.make(created_at: 2.days.ago, state: 'BUILDPACK_SET', app_guid: 'app2') + task_event = VCAP::CloudController::AppUsageEvent.make(created_at: 2.days.ago, state: 'TASK_STOPPED', app_guid: 'app3') + + record_cleanup = Database::OldRecordCleanup.new(VCAP::CloudController::AppUsageEvent, cutoff_age_in_days: 1, keep_at_least_one_record: false, keep_running_records: true) + record_cleanup.delete + + expect { buildpack_event1.reload }.to raise_error(Sequel::NoExistingObject) + expect { buildpack_event2.reload }.to raise_error(Sequel::NoExistingObject) + expect { task_event.reload }.to raise_error(Sequel::NoExistingObject) + end + + it 'deletes AppUsageEvent records when stop record created before start record' do + app_guid = 'time-travel-app' + + start_event = VCAP::CloudController::AppUsageEvent.make( + created_at: 2.days.ago, + state: 'STARTED', + app_guid: app_guid + ) + + stop_event = VCAP::CloudController::AppUsageEvent.make( + created_at: 3.days.ago, # Earlier timestamp but higher ID + state: 'STOPPED', + app_guid: app_guid + ) + + record_cleanup = Database::OldRecordCleanup.new( + VCAP::CloudController::AppUsageEvent, + cutoff_age_in_days: 1, + keep_running_records: true + ) + record_cleanup.delete + + expect { start_event.reload }.to raise_error(Sequel::NoExistingObject) + expect { stop_event.reload }.to raise_error(Sequel::NoExistingObject) + end + + it 'keeps multiple AppUsageEvent start records when there is no stop record' do + app_guid = 'multi-start-app' + + # Multiple START events for same app + start1 = VCAP::CloudController::AppUsageEvent.make(created_at: 5.days.ago, state: 'STARTED', app_guid: app_guid) + start2 = VCAP::CloudController::AppUsageEvent.make(created_at: 4.days.ago, state: 'STARTED', app_guid: app_guid) + start3 = VCAP::CloudController::AppUsageEvent.make(created_at: 3.days.ago, state: 'STARTED', app_guid: app_guid) + + record_cleanup = Database::OldRecordCleanup.new( + VCAP::CloudController::AppUsageEvent, + cutoff_age_in_days: 1, + keep_running_records: true + ) + record_cleanup.delete + + expect(start1.reload).to be_present + expect(start2.reload).to be_present + expect(start3.reload).to be_present + end + + it 'deletes multiple AppUsageEvent stop records for the same app when there is only a single start' do + app_guid = 'multi-stop-app' + + start_event = VCAP::CloudController::AppUsageEvent.make(created_at: 5.days.ago, state: 'STARTED', app_guid: app_guid) + stop1 = VCAP::CloudController::AppUsageEvent.make(created_at: 4.days.ago, state: 'STOPPED', app_guid: app_guid) + stop2 = VCAP::CloudController::AppUsageEvent.make(created_at: 3.days.ago, state: 'STOPPED', app_guid: app_guid) + stop3 = VCAP::CloudController::AppUsageEvent.make(created_at: 2.days.ago, state: 'STOPPED', app_guid: app_guid) + + record_cleanup = Database::OldRecordCleanup.new( + VCAP::CloudController::AppUsageEvent, + cutoff_age_in_days: 1, + keep_running_records: true + ) + record_cleanup.delete + + # START has a STOP after it, so it should be deleted + expect { start_event.reload }.to raise_error(Sequel::NoExistingObject) + + # All STOPs should be deleted + expect { stop1.reload }.to raise_error(Sequel::NoExistingObject) + expect { stop2.reload }.to raise_error(Sequel::NoExistingObject) + expect { stop3.reload }.to raise_error(Sequel::NoExistingObject) + end + + it 'deletes old AppUsageEvent records with corresponding stop record even if app_guid is an empty string' do + empty_guid_start = VCAP::CloudController::AppUsageEvent.make(created_at: 5.days.ago, state: 'STARTED', app_guid: '') + different_empty_start = VCAP::CloudController::AppUsageEvent.make(created_at: 4.days.ago, state: 'STARTED', app_guid: '') + empty_guid_stop = VCAP::CloudController::AppUsageEvent.make(created_at: 3.days.ago, state: 'STOPPED', app_guid: '') + + record_cleanup = Database::OldRecordCleanup.new( + VCAP::CloudController::AppUsageEvent, + cutoff_age_in_days: 1, + keep_running_records: true + ) + record_cleanup.delete + + # Both STARTs with empty string have a STOP with empty string after them + expect { empty_guid_start.reload }.to raise_error(Sequel::NoExistingObject) + expect { different_empty_start.reload }.to raise_error(Sequel::NoExistingObject) + expect { empty_guid_stop.reload }.to raise_error(Sequel::NoExistingObject) + end + + it 'works when cutoff_age_in_days in 0' do + old_start = VCAP::CloudController::AppUsageEvent.make( + created_at: 1.second.ago, + state: 'STARTED', + app_guid: 'running-app' + ) + + record_cleanup = Database::OldRecordCleanup.new( + VCAP::CloudController::AppUsageEvent, + cutoff_age_in_days: 0, + keep_running_records: true + ) + record_cleanup.delete + + expect(old_start.reload).to be_present + end + + it 'does not error if database is empty' do + VCAP::CloudController::AppUsageEvent.dataset.delete + + record_cleanup = Database::OldRecordCleanup.new( + VCAP::CloudController::AppUsageEvent, + cutoff_age_in_days: 1, + keep_running_records: true + ) + + expect { record_cleanup.delete }.not_to raise_error + end + + # ==================== FEATURE FLAG COMBINATIONS ==================== + + it 'deletes all old AppUsageEvent records when keep_running_records is false' do + app_guid = 'no-keep-running-app' + + old_start = VCAP::CloudController::AppUsageEvent.make(created_at: 5.days.ago, state: 'STARTED', app_guid: app_guid) + old_stop = VCAP::CloudController::AppUsageEvent.make(created_at: 4.days.ago, state: 'STOPPED', app_guid: app_guid) + old_running_start = VCAP::CloudController::AppUsageEvent.make(created_at: 3.days.ago, state: 'STARTED', app_guid: 'running-app') + + record_cleanup = Database::OldRecordCleanup.new( + VCAP::CloudController::AppUsageEvent, + cutoff_age_in_days: 1, + keep_running_records: false # Feature disabled + ) + record_cleanup.delete + + # All old records deleted, including running app START + expect { old_start.reload }.to raise_error(Sequel::NoExistingObject) + expect { old_stop.reload }.to raise_error(Sequel::NoExistingObject) + expect { old_running_start.reload }.to raise_error(Sequel::NoExistingObject) + end + + it 'keep_at_least_one_record preserves last record even if it has a stop' do + app_guid = 'last-record-stopped-app' + + old_start = VCAP::CloudController::AppUsageEvent.make(created_at: 10.days.ago, state: 'STARTED', app_guid: app_guid) + last_stop = VCAP::CloudController::AppUsageEvent.make(created_at: 9.days.ago, state: 'STOPPED', app_guid: app_guid) + + record_cleanup = Database::OldRecordCleanup.new( + VCAP::CloudController::AppUsageEvent, + cutoff_age_in_days: 1, + keep_at_least_one_record: true, + keep_running_records: true + ) + record_cleanup.delete + + # keep_at_least_one_record is applied BEFORE keep_running_records + # So the last record (STOP) is excluded from deletion + # Then keep_running_records logic runs on the remaining old_records + # The START has a STOP with higher ID, but that STOP was excluded from old_records + # So the START doesn't find a matching STOP in the old_records dataset and is kept + expect(old_start.reload).to be_present # Kept - no STOP in old_records to match + expect(last_stop.reload).to be_present # Kept - last record + end end end