Skip to content

Commit 7887112

Browse files
niemyjskiniemyjski
committed
Updates for Issue #14
@ejsmith Can you please look over these api changes and let me know your thoughts.. It makes some things like deletes have a bit more logic but only in one spot..
1 parent ea56b97 commit 7887112

File tree

6 files changed

+101
-98
lines changed

6 files changed

+101
-98
lines changed

‎src/configuration/SettingsManager.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,12 @@ export class SettingsManager {
3535
}
3636

3737
private static getSavedServerSettings(config:Configuration):Object {
38-
return config.storage.get(this._configPath, 1)[0] || {};
38+
return config.storage.get(this._configPath) || {};
3939
}
4040

4141
public static checkVersion(version:number, config:Configuration):void {
4242
if (version) {
43-
var savedConfigVersion = parseInt(<string>config.storage.get(`${this._configPath}-version`, 1)[0]);
43+
var savedConfigVersion = parseInt(<string>config.storage.get(`${this._configPath}-version`));
4444
if (isNaN(savedConfigVersion) || version > savedConfigVersion) {
4545
config.log.info(`Updating settings from v${(!isNaN(savedConfigVersion) ? savedConfigVersion : 0)} to v${version}`);
4646
this.updateSettings(config);

‎src/queue/DefaultEventQueue-spec.ts‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,43 +4,53 @@ import { IEvent } from '../models/IEvent';
44
describe('DefaultEventQueue', () => {
55
it('should enqueue event', () => {
66
var config = new Configuration({ apiKey:'LhhP1C9gijpSKCslHHCvwdSIz298twx271n1l6xw', serverUrl:'http://localhost:50000'});
7-
expect(config.storage.count()).toBe(0);
7+
expect(config.storage.getList().length).toBe(0);
88
var event:IEvent = { type: 'log', reference_id: '123454321' };
99
config.queue.enqueue(event);
10-
expect(config.storage.count()).toBe(1);
10+
expect(config.storage.getList().length).toBe(1);
1111
});
1212

1313
it('should process queue', () => {
1414
var config = new Configuration({ apiKey:'LhhP1C9gijpSKCslHHCvwdSIz298twx271n1l6xw', serverUrl:'http://localhost:50000'});
15-
expect(config.storage.count()).toBe(0);
15+
expect(config.storage.getList().length).toBe(0);
1616
var event:IEvent = { type: 'log', reference_id: '123454321' };
1717
config.queue.enqueue(event);
18-
expect(config.storage.count()).toBe(1);
18+
expect(config.storage.getList().length).toBe(1);
1919
config.queue.process();
20-
expect(config.storage.count()).toBe(0);
20+
21+
if (!(<any>config.queue)._suspendProcessingUntil) {
22+
expect(config.storage.getList().length).toBe(0);
23+
} else {
24+
expect(config.storage.getList().length).toBe(1);
25+
}
2126
});
2227

2328
it('should discard event submission', () => {
2429
var config = new Configuration({ apiKey:'LhhP1C9gijpSKCslHHCvwdSIz298twx271n1l6xw', serverUrl:'http://localhost:50000'});
25-
expect(config.storage.count()).toBe(0);
30+
expect(config.storage.getList().length).toBe(0);
2631
config.queue.suspendProcessing(1, true);
2732

2833
var event:IEvent = { type: 'log', reference_id: '123454321' };
2934
config.queue.enqueue(event);
30-
expect(config.storage.count()).toBe(0);
35+
expect(config.storage.getList().length).toBe(0);
3136
});
3237

3338
it('should suspend processing', (done) => {
3439
var config = new Configuration({ apiKey:'LhhP1C9gijpSKCslHHCvwdSIz298twx271n1l6xw', serverUrl:'http://localhost:50000'});
35-
expect(config.storage.count()).toBe(0);
40+
expect(config.storage.getList().length).toBe(0);
3641
config.queue.suspendProcessing(.0001);
3742

3843
var event:IEvent = { type: 'log', reference_id: '123454321' };
3944
config.queue.enqueue(event);
40-
expect(config.storage.count()).toBe(1);
45+
expect(config.storage.getList().length).toBe(1);
4146

4247
setTimeout(() => {
43-
expect(config.storage.count()).toBe(0);
48+
if (!(<any>config.queue)._suspendProcessingUntil) {
49+
expect(config.storage.getList().length).toBe(0);
50+
} else {
51+
expect(config.storage.getList().length).toBe(1);
52+
}
53+
4454
done();
4555
}, 10000);
4656
}, 21000);

‎src/queue/DefaultEventQueue.ts‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -46,23 +46,32 @@ export class DefaultEventQueue implements IEventQueue {
4646
}
4747

4848
public enqueue(event:IEvent): void {
49-
var config:Configuration = this._config; // Optmization for minifier.
49+
var config:Configuration = this._config; // Optimization for minifier.
5050
this.ensureQueueTimer();
5151

5252
if (this.areQueuedItemsDiscarded()) {
5353
config.log.info('Queue items are currently being discarded. The event will not be queued.');
5454
return;
5555
}
5656

57-
var key = `${this.queuePath()}-${new Date().toJSON()}-${Utils.randomNumber()}`;
57+
var key = `ex-q-${new Date().toJSON()}-${Utils.randomNumber()}`;
5858
config.log.info(`Enqueuing event: ${key} type=${event.type} ${!!event.reference_id ? 'refid=' + event.reference_id : ''}`);
5959
config.storage.save(key, event);
6060
}
6161

6262
public process(): void {
63+
function getEvents(events:{ path:string, value:IEvent }[]):IEvent[] {
64+
var items:IEvent[] = [];
65+
for (var index = 0; index < events.length; index++) {
66+
items.push(events[index].value);
67+
}
68+
69+
return items;
70+
}
71+
6372
const queueNotProcessed:string = 'The queue will not be processed.'; // optimization for minifier.
64-
var config:Configuration = this._config; // Optmization for minifier.
65-
var log:ILog = config.log; // Optmization for minifier.
73+
var config:Configuration = this._config; // Optimization for minifier.
74+
var log:ILog = config.log; // Optimization for minifier.
6675

6776
this.ensureQueueTimer();
6877

@@ -84,14 +93,14 @@ export class DefaultEventQueue implements IEventQueue {
8493
this._processingQueue = true;
8594

8695
try {
87-
var events = config.storage.get(this.queuePath(), config.submissionBatchSize);
96+
var events = config.storage.getList('ex-q', config.submissionBatchSize);
8897
if (!events || events.length == 0) {
8998
this._processingQueue = false;
9099
return;
91100
}
92101

93102
log.info(`Sending ${events.length} events to ${config.serverUrl}.`);
94-
config.submissionClient.postEvents(events, config, (response:SubmissionResponse) => {
103+
config.submissionClient.postEvents(getEvents(events), config, (response:SubmissionResponse) => {
95104
this.processSubmissionResponse(response, events);
96105
log.info('Finished processing queue.');
97106
this._processingQueue = false;
@@ -103,21 +112,21 @@ export class DefaultEventQueue implements IEventQueue {
103112
}
104113
}
105114

106-
private processSubmissionResponse(response:SubmissionResponse, events:IEvent[]): void {
107-
const noSubmission:string = 'The event will not be submitted.'; // Optmization for minifier.
108-
var config:Configuration = this._config; // Optmization for minifier.
109-
var log:ILog = config.log; // Optmization for minifier.
115+
private processSubmissionResponse(response:SubmissionResponse, events:{ path:string, value:IEvent }[]): void {
116+
const noSubmission:string = 'The event will not be submitted.'; // Optimization for minifier.
117+
var config:Configuration = this._config; // Optimization for minifier.
118+
var log:ILog = config.log; // Optimization for minifier.
110119

111120
if (response.success) {
112121
log.info(`Sent ${events.length} events.`);
122+
this.removeEvents(events);
113123
return;
114124
}
115125

116126
if (response.serviceUnavailable) {
117127
// You are currently over your rate limit or the servers are under stress.
118128
log.error('Server returned service unavailable.');
119129
this.suspendProcessing();
120-
this.requeueEvents(events);
121130
return;
122131
}
123132

@@ -132,13 +141,15 @@ export class DefaultEventQueue implements IEventQueue {
132141
// The api key was suspended or could not be authorized.
133142
log.info(`Unable to authenticate, please check your configuration. ${noSubmission}`);
134143
this.suspendProcessing(15);
144+
this.removeEvents(events);
135145
return;
136146
}
137147

138148
if (response.notFound || response.badRequest) {
139149
// The service end point could not be found.
140150
log.error(`Error while trying to submit data: ${response.message}`);
141151
this.suspendProcessing(60 * 4);
152+
this.removeEvents(events);
142153
return;
143154
}
144155

@@ -147,9 +158,9 @@ export class DefaultEventQueue implements IEventQueue {
147158
if (config.submissionBatchSize > 1) {
148159
log.error(`${message} Retrying with smaller batch size.`);
149160
config.submissionBatchSize = Math.max(1, Math.round(config.submissionBatchSize / 1.5));
150-
this.requeueEvents(events);
151161
} else {
152162
log.error(`${message} ${noSubmission}`);
163+
this.removeEvents(events);
153164
}
154165

155166
return;
@@ -158,7 +169,6 @@ export class DefaultEventQueue implements IEventQueue {
158169
if (!response.success) {
159170
log.error(`Error submitting events: ${ response.message}`);
160171
this.suspendProcessing();
161-
this.requeueEvents(events);
162172
}
163173
}
164174

@@ -175,7 +185,7 @@ export class DefaultEventQueue implements IEventQueue {
175185
}
176186

177187
public suspendProcessing(durationInMinutes?:number, discardFutureQueuedItems?:boolean, clearQueue?:boolean): void {
178-
var config:Configuration = this._config; // Optmization for minifier.
188+
var config:Configuration = this._config; // Optimization for minifier.
179189

180190
if (!durationInMinutes || durationInMinutes <= 0) {
181191
durationInMinutes = 5;
@@ -188,21 +198,15 @@ export class DefaultEventQueue implements IEventQueue {
188198
this._discardQueuedItemsUntil = new Date(new Date().getTime() + (durationInMinutes * 60000));
189199
}
190200

191-
if (!clearQueue) {
192-
return;
201+
if (clearQueue) {
202+
// Account is over the limit and we want to ensure that the sample size being sent in will contain newer errors.
203+
this.removeEvents(config.storage.getList('ex-q'));
193204
}
194-
195-
// Account is over the limit and we want to ensure that the sample size being sent in will contain newer errors.
196-
try {
197-
config.storage.clear(this.queuePath());
198-
} catch (Exception) { }
199205
}
200206

201-
private requeueEvents(events:IEvent[]): void {
202-
this._config.log.info(`Requeuing ${events.length} events.`);
203-
204-
for (var index = 0; index < events.length; index++) {
205-
this.enqueue(events[index]);
207+
private removeEvents(events:{ path:string, value:IEvent }[]) {
208+
for (var index = 0; index < (events || []).length; index++) {
209+
this._config.storage.remove(events[index].path);
206210
}
207211
}
208212

@@ -213,8 +217,4 @@ export class DefaultEventQueue implements IEventQueue {
213217
private areQueuedItemsDiscarded(): boolean {
214218
return this._discardQueuedItemsUntil && this._discardQueuedItemsUntil > new Date();
215219
}
216-
217-
private queuePath(): string {
218-
return 'ex-q';
219-
}
220220
}

‎src/storage/IStorage.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
export interface IStorage<T>{
22
save<T>(path:string, value:T):boolean;
3-
get(searchPattern?:string, limit?:number):T[];
4-
clear(searchPattern?:string):void;
5-
count(searchPattern?:string):number;
3+
get(path:string):T;
4+
getList(searchPattern?:string, limit?:number):{ path:string, value:T }[];
5+
remove(path:string):void;
66
}

‎src/storage/InMemoryStorage-spec.ts‎

Lines changed: 36 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,17 @@ import { InMemoryStorage } from './InMemoryStorage';
44
describe('InMemoryStorage', () => {
55
it('should save events', () => {
66
var storage = new InMemoryStorage<IEvent>();
7-
var key = 'ex-LhhP1C9gi-q-';
7+
var key = 'ex-q-';
88
var event1:IEvent = { type: 'log', reference_id: key + '123454321' };
99
var event2:IEvent = { type: 'log', reference_id: key + '098765432' };
10-
expect(storage.count()).toBe(0);
10+
expect(storage.getList().length).toBe(0);
1111
storage.save(event1.reference_id, event1);
12-
expect(storage.count()).toBe(1);
13-
expect(storage.count(key)).toBe(1);
12+
expect(storage.getList().length).toBe(1);
13+
expect(storage.getList(key).length).toBe(1);
1414
storage.save(event2.reference_id, event2);
15-
expect(storage.count()).toBe(2);
16-
expect(storage.count(key)).toBe(2);
17-
expect(storage.count(key + '1')).toBe(1);
15+
expect(storage.getList().length).toBe(2);
16+
expect(storage.getList(key).length).toBe(2);
17+
expect(storage.getList(key + '1').length).toBe(1);
1818
});
1919

2020
it('should get saved events', () => {
@@ -26,58 +26,68 @@ describe('InMemoryStorage', () => {
2626
var event4:IEvent = { type: 'log', reference_id: key + '14' };
2727
var event5:IEvent = { type: 'log', reference_id: key + '15' };
2828
var event6:IEvent = { type: 'log', reference_id: key + '16' };
29-
expect(storage.count()).toBe(0);
29+
expect(storage.getList().length).toBe(0);
3030

3131
storage.save(event1.reference_id, event1);
3232
storage.save(event2.reference_id, event2);
3333
storage.save(event3.reference_id, event3);
3434
storage.save(event4.reference_id, event4);
3535
storage.save(event5.reference_id, event5);
3636
storage.save(event6.reference_id, event6);
37-
expect(storage.count()).toBe(6);
37+
expect(storage.getList().length).toBe(6);
3838

39-
var ev = storage.get(event1.reference_id, 1)[0];
39+
var ev = storage.get(event1.reference_id);
4040
expect(ev).toEqual(event1);
41-
expect(storage.count()).toBe(5);
41+
expect(ev).toEqual(storage.getList(event1.reference_id, 1)[0].value);
42+
expect(storage.getList().length).toBe(6);
43+
storage.remove(event1.reference_id);
44+
expect(storage.get(event1.reference_id)).toBe(null);
45+
expect(storage.getList().length).toBe(5);
4246

43-
ev = storage.get(event2.reference_id, 1)[0];
47+
ev = storage.getList(event2.reference_id, 1)[0].value;
4448
expect(ev).toEqual(event2);
45-
expect(storage.count()).toBe(4);
49+
storage.remove(event2.reference_id);
50+
expect(storage.getList().length).toBe(4);
4651

47-
var events = storage.get(key, 2);
52+
var events = storage.getList(key, 2);
4853
expect(events.length).toBe(2);
49-
expect(events[0]).not.toEqual(events[1]);
50-
expect(storage.count()).toBe(2);
54+
expect(events[0].value).not.toEqual(events[1].value);
55+
storage.remove(events[0].path);
56+
storage.remove(events[1].path);
57+
expect(storage.getList().length).toBe(2);
5158

52-
events = storage.get(key);
59+
events = storage.getList(key);
5360
expect(events.length).toBe(2);
54-
expect(events[0]).not.toEqual(events[1]);
55-
expect(storage.count()).toBe(0);
61+
expect(events[0].value).not.toEqual(events[1].value);
5662
});
5763

5864
it('should clear all events', () => {
5965
var storage = new InMemoryStorage<IEvent>();
60-
var key = 'ex-LhhP1C9gi-q-';
66+
var key = 'ex-q-';
6167
var event1:IEvent = { type: 'log', reference_id: key + '11' };
6268
var event2:IEvent = { type: 'log', reference_id: key + '12' };
6369
var event3:IEvent = { type: 'log', reference_id: key + '13' };
6470
var event4:IEvent = { type: 'log', reference_id: key + '14' };
6571
var event5:IEvent = { type: 'log', reference_id: key + '15' };
6672
var event6:IEvent = { type: 'log', reference_id: key + '16' };
67-
expect(storage.count()).toBe(0);
73+
expect(storage.getList().length).toBe(0);
6874

6975
storage.save(event1.reference_id, event1);
7076
storage.save(event2.reference_id, event2);
7177
storage.save(event3.reference_id, event3);
7278
storage.save(event4.reference_id, event4);
7379
storage.save(event5.reference_id, event5);
7480
storage.save(event6.reference_id, event6);
75-
expect(storage.count()).toBe(6);
81+
expect(storage.getList().length).toBe(6);
82+
83+
storage.remove(event1.reference_id);
84+
expect(storage.getList().length).toBe(5);
7685

77-
storage.clear(event1.reference_id);
78-
expect(storage.count()).toBe(5);
86+
var events = storage.getList();
87+
for (var index = 0; index < events.length; index++) {
88+
storage.remove(events[index].path);
89+
}
7990

80-
storage.clear();
81-
expect(storage.count()).toBe(0);
91+
expect(storage.getList().length).toBe(0);
8292
});
8393
});

0 commit comments

Comments
 (0)