Skip to content

Commit 65f1739

Browse files
eli0shineli0shin
authored
Merge pull request #11 from eli0shin/ignore-old-events
add ignoreOldEvents and maxAge
2 parents c0d1e2a + 9d3bea9 commit 65f1739

File tree

4 files changed

+96
-62
lines changed

4 files changed

+96
-62
lines changed

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "firebase-framework",
3-
"version": "1.0.0",
3+
"version": "1.1.0",
44
"description": "A framework designed to increase productivity in serverless applications using firebase",
55
"main": "src/index.js",
66
"scripts": {

‎readme.md‎

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,9 @@
22

33
## Getting Started
44

5-
To begin you will need `nodejs` and `firebase-tools`
5+
This documentation describes features of the `firebase-framework` but generally does not cover firebase options in general. For example, to find out how you can specify a specific version of nodejs see the firebase docs here: https://firebase.google.com/docs/functions/manage-functions#set_nodejs_version
66

7-
nodejs v8.15.x is recommended because cloud functions by default uses node 8. \
8-
While it is not the most recent LTS build it can be downloaded here:
9-
<https://nodejs.org/download/release/v8.15.1/>
7+
To begin you will need `nodejs` and `firebase-tools`
108

119
To install `firebase-tools` run
1210

@@ -86,7 +84,7 @@ module.exports = {
8684
routes: [
8785
{
8886
path: '/',
89-
function: req => [200, { message: 'hello-world' }],
87+
function: (req) => [200, { message: 'hello-world' }],
9088
},
9189
],
9290
};
@@ -106,20 +104,21 @@ module.exports = [hello];
106104
- Service configuration is exported from the `index.js` file in the service's directory
107105
- the config object's structure is as follows:
108106

109-
| key | required | type | description |
110-
| -------------- | -------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
111-
| basePath | true | string | defines the services base-path |
112-
| resourcePath | false | string | defines which documents in the db should be published when changed ex: `'posts/{id}'` (uses the cloud functions firestore triggers syntax) |
113-
| schema | false | object | should contain a reference to the `require`d schema file |
114-
| postSchema | false | object | optional alternative used for services that require special fields during creation |
115-
| publishChanges | false | boolean | whether the service should publish changes to it's data as messages on cloud pub sub |
116-
| withModifiers | false | boolean | declares that the schema can contain `writeModifier` keys that define a function that will modify values before they are processes/saved |
117-
| middleware | false | Array | ExpressJs middleware that will apply to all routes in the service |
118-
| routes | false | Array | these are the functions triggered within the service by http requests (see routes below) |
119-
| events | false | Array | pub sub events that the service will listed to (see events below) |
120-
| schedule | false | Array | cloud schedules that will trigger functions within this service (see schedule below) |
121-
| keepAlive | false | boolean | whether a scheduled function should be set up that will trigger the http function (routes) every 5 minutes to prevent cold starts\* |
122-
| runtimeOptions | false | object | An object containing 2 optional properties. `memory`: amount of memory to allocate to the function, possible values are: '128MB', '256MB', '512MB', '1GB', and '2GB'. `timeoutSeconds`: timeout for the function in seconds, possible values are 0 to 540. |
107+
| key | required | type | description |
108+
| -------------- | --------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
109+
| basePath | true | string | defines the services base-path |
110+
| resourcePath | false | string | defines which documents in the db should be published when changed ex: `'posts/{id}'` (uses the cloud functions firestore triggers syntax) |
111+
| schema | false | object | should contain a reference to the `require`d schema file |
112+
| postSchema | false | object | optional alternative used for services that require special fields during creation |
113+
| publishChanges | false | boolean | whether the service should publish changes to it's data as messages on cloud pub sub |
114+
| withModifiers | false | boolean | declares that the schema can contain `writeModifier` keys that define a function that will modify values before they are processes/saved |
115+
| middleware | false | Array | ExpressJs middleware that will apply to all routes in the service |
116+
| routes | false | Array | these are the functions triggered within the service by http requests (see routes below) |
117+
| events | false | Array | pub sub events that the service will listed to (see events below) |
118+
| schedule | false | Array | cloud schedules that will trigger functions within this service (see schedule below) |
119+
| keepAlive | false | boolean | whether a scheduled function should be set up that will trigger the http function (routes) every 5 minutes to prevent cold starts\* |
120+
| runtimeOptions | false | object | Firebase runtime options usually passed to `runWith` as documented here: https://firebase.google.com/docs/reference/functions/function_configuration.runtimeoptions|
121+
| maxAge | undefined | number | The maxAge in ms before the event listener will not attempt to process an event. Can be used to prevent infinite retries of a retry-able event |
123122

124123
\* Though billing is required, you can expect the overall cost to be manageable, as each Cloud Scheduler job costs \$0.10 (USD) per month, and there is an allowance of three free jobs per Google account (as of the time of writing). \* The keepAlive feature adds a route to the service at '/heartbeat'. This will not conflict with wildcard routes in the service but would conflict with a route named the same.
125124

@@ -142,7 +141,7 @@ module.exports = [hello];
142141
| type | false | string | the event type to listen to. This is passed to the subscriber but will not affect which message in the topic trigger the subscriber, it can function as a not about which types of events from the topic the function cares about |
143142
| function | false | function | to be executed when the described event is triggered |
144143
| ensureIdempotent | false | boolean | whether the framework should check messages against a store (requires a firestore database in the project) to ensure that messages are never processed more than once |
145-
| runtimeOptions | false | object | An object containing 2 optional properties. `memory`: amount of memory to allocate to the function, possible values are: '128MB', '256MB', '512MB', '1GB', and '2GB'. `timeoutSeconds`: timeout for the function in seconds, possible values are 0 to 540. |
144+
| runtimeOptions | false | object | Firebase runtime options usually passed to `runWith` as documented here: https://firebase.google.com/docs/reference/functions/function_configuration.runtimeoptions |
146145

147146
#### Schedule
148147

@@ -151,7 +150,7 @@ module.exports = [hello];
151150
| name | true | string | The name of the schedule (it can be anything) |
152151
| time | true | string | Both Unix Crontab and AppEngine syntax are supported by Google Cloud Scheduler. |
153152
| function | false | function | to be executed when the cronjob is run |
154-
| runtimeOptions | false | object | An object containing 2 optional properties. `memory`: amount of memory to allocate to the function, possible values are: '128MB', '256MB', '512MB', '1GB', and '2GB'. `timeoutSeconds`: timeout for the function in seconds, possible values are 0 to 540. |
153+
| runtimeOptions | false | object | Firebase runtime options usually passed to `runWith` as documented here: https://firebase.google.com/docs/reference/functions/function_configuration.runtimeoptions |
155154

156155
## Service Schemas
157156

‎src/createFunctions.js‎

Lines changed: 51 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const parseMessage = require('./pubSub/parseMessage');
55
const withIdempotency = require('./ensureIdempotent');
66
const keepFunctionAlive = require('./keepFunctionAlive');
77
const configStore = require('./configStore');
8+
const ignoreOldEvents = require('./ignoreOldEvents');
89

910
const fromEntries = [(acc, [key, value]) => ({ ...acc, [key]: value }), {}];
1011

@@ -57,21 +58,24 @@ const setupKeepAlive = (config, service) =>
5758
}
5859
: {};
5960

60-
const setupSchedule = (config, service) => ({
61-
time,
62-
name,
63-
function: toExecute,
64-
timeZone = 'America/New_York',
65-
runtimeOptions,
66-
}) => [
67-
[`${service.basePath}_${name}`],
68-
functions
69-
.region(config.region)
70-
.runWith(runtimeOptions || service.runtimeOptions || {})
71-
.pubsub.schedule(time)
72-
.timeZone(timeZone)
73-
.onRun(toExecute),
74-
];
61+
const setupSchedule =
62+
(config, service) =>
63+
({
64+
time,
65+
name,
66+
function: toExecute,
67+
timeZone = 'America/New_York',
68+
runtimeOptions,
69+
}) =>
70+
[
71+
[`${service.basePath}_${name}`],
72+
functions
73+
.region(config.region)
74+
.runWith(runtimeOptions || service.runtimeOptions || {})
75+
.pubsub.schedule(time)
76+
.timeZone(timeZone)
77+
.onRun(toExecute),
78+
];
7579

7680
const setupSchedules = (config, service) =>
7781
Array.isArray(service.schedule)
@@ -80,30 +84,37 @@ const setupSchedules = (config, service) =>
8084
.reduce(...fromEntries)
8185
: {};
8286

83-
const setupEvent = (config, service) => ({
84-
topic,
85-
type = '',
86-
function: toExecute,
87-
ensureIdempotent = false,
88-
runtimeOptions,
89-
}) => {
90-
const functionName = `${service.basePath}_${topic}${type ? `_${type}` : ''}`;
91-
92-
return [
93-
functionName,
94-
functions
95-
.region(config.region)
96-
.runWith(runtimeOptions || service.runtimeOptions || {})
97-
.pubsub.topic(topic)
98-
.onPublish(
99-
parseMessage(
100-
ensureIdempotent
101-
? withIdempotency(functionName, toExecute)
102-
: toExecute
103-
)
104-
),
105-
];
106-
};
87+
const setupEvent =
88+
(config, service) =>
89+
({
90+
topic,
91+
type = '',
92+
function: toExecute,
93+
ensureIdempotent = false,
94+
maxAge,
95+
runtimeOptions,
96+
}) => {
97+
const functionName = `${service.basePath}_${topic}${
98+
type ? `_${type}` : ''
99+
}`;
100+
101+
const handlerWithIdempotency = ensureIdempotent
102+
? withIdempotency(functionName, toExecute)
103+
: toExecute;
104+
105+
const handler = Boolean(maxAge)
106+
? ignoreOldEvents(maxAge, handlerWithIdempotency)
107+
: handlerWithIdempotency;
108+
109+
return [
110+
functionName,
111+
functions
112+
.region(config.region)
113+
.runWith(runtimeOptions || service.runtimeOptions || {})
114+
.pubsub.topic(topic)
115+
.onPublish(parseMessage(handler)),
116+
];
117+
};
107118

108119
const setupEvents = (config, service) =>
109120
Array.isArray(service.events)
@@ -122,7 +133,7 @@ const createFunctions = (config = {}, services) => {
122133
configStore.config = { ...configStore.config, ...config };
123134

124135
return services
125-
.map(service => parseConfig(configStore.config, service))
136+
.map((service) => parseConfig(configStore.config, service))
126137
.reduce(...flattenObjects);
127138
};
128139

‎src/ignoreOldEvents.js‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/**
2+
* If an event continues to retry for a long period of time
3+
* there may be a reason to just drop it and stop retrying
4+
* This util, when applied, will bail out of a handler if
5+
* it is greater than the maxAge set.
6+
*/
7+
8+
function setupIgnoreOldEvents(maxAge, handler) {
9+
return function ignoreOldEvents(...args) {
10+
const [_message, context] = args;
11+
const eventAgeMs = Date.now() - Date.parse(context.timestamp);
12+
13+
if (eventAgeMs > maxAge) {
14+
console.log(
15+
`Dropping event ${context.eventId} with age[ms]: ${eventAgeMs}`
16+
);
17+
return true;
18+
}
19+
20+
return handler(...args);
21+
};
22+
}
23+
24+
module.exports = setupIgnoreOldEvents;

0 commit comments

Comments
 (0)