feat(winston): separate winston ecsFormat to composable ecsFields and ecsStringify (take 2) (#161)

This adds `ecsFields` and `ecsStringify` formats. `ecsFormat` is now just a
composition of the two. Named import is now preferred, but the old default
export remains for now for backwards compat.

The single downside here is that the order of serialized fields no longer
follows the spec (https://github.com/elastic/ecs-logging/blob/main/spec/spec.json)
suggestion of having `@timestamp` then `log.level` then `message` then
the rest ordering. This was a trade-off with avoiding creating an additional
object for each log record.

Fixes: #57
Obsoletes: #65

Author: trentm

docs/winston.asciidoc

@@ -26,18 +26,18 @@ $ npm install @elastic/ecs-winston-format
 
 [source,js]
 ----
-const winston = require('winston')
-const ecsFormat = require('@elastic/ecs-winston-format')
+const winston = require('winston');
+const { ecsFormat } = require('@elastic/ecs-winston-format');
 
 const logger = winston.createLogger({
   format: ecsFormat(/* options */), <1>
   transports: [
     new winston.transports.Console()
   ]
-})
+});
 
-logger.info('hi')
-logger.error('oops there is a problem', { err: new Error('boom') })
+logger.info('hi');
+logger.error('oops there is a problem', { err: new Error('boom') });
 ----
 <1> Pass the ECS formatter to winston here.
 
@@ -58,19 +58,19 @@ NOTE: You might like to try out our tutorial using Node.js ECS logging with wins
 
 [source,js]
 ----
-const winston = require('winston')
-const ecsFormat = require('@elastic/ecs-winston-format')
+const winston = require('winston');
+const { ecsFormat } = require('@elastic/ecs-winston-format');
 
 const logger = winston.createLogger({
   level: 'info',
   format: ecsFormat(/* options */), <1>
   transports: [
     new winston.transports.Console()
   ]
-})
+});
 
-logger.info('hi')
-logger.error('oops there is a problem', { foo: 'bar' })
+logger.info('hi');
+logger.error('oops there is a problem', { foo: 'bar' });
 ----
 <1> See available options <<winston-ref,below>>.
 
@@ -99,17 +99,17 @@ For https://github.com/elastic/ecs-logging-nodejs/blob/main/packages/ecs-winston
 
 [source,js]
 ----
-const winston = require('winston')
-const ecsFormat = require('@elastic/ecs-winston-format')
+const winston = require('winston');
+const { ecsFormat } = require('@elastic/ecs-winston-format');
 const logger = winston.createLogger({
   format: ecsFormat(), <1>
   transports: [
     new winston.transports.Console()
   ]
-})
+});
 
-const myErr = new Error('boom')
-logger.info('oops', { err: myErr }) <2>
+const myErr = new Error('boom');
+logger.info('oops', { err: myErr }); <2>
 ----
 
 will yield (pretty-printed for readability):
@@ -153,27 +153,27 @@ objects when passed as the `req` and `res` meta fields, respectively.
 
 [source,js]
 ----
-const http = require('http')
-const winston = require('winston')
-const ecsFormat = require('@elastic/ecs-winston-format')
+const http = require('http');
+const winston = require('winston');
+const { ecsFormat } = require('@elastic/ecs-winston-format');
 
 const logger = winston.createLogger({
   level: 'info',
   format: ecsFormat({ convertReqRes: true }), <1>
   transports: [
     new winston.transports.Console()
   ]
-})
+});
 
-const server = http.createServer(handler)
+const server = http.createServer(handler);
 server.listen(3000, () => {
   logger.info('listening at http://localhost:3000')
-})
+});
 
 function handler (req, res) {
-  res.setHeader('Foo', 'Bar')
-  res.end('ok')
-  logger.info('handled request', { req, res }) <2>
+  res.setHeader('Foo', 'Bar');
+  res.end('ok');
+  logger.info('handled request', { req, res }); <2>
 }
 ----
 <1> use `convertReqRes` option
@@ -271,6 +271,18 @@ const logger = winston.createLogger({
 })
 ----
 
+[float]
+[[winston-limitations]]
+=== Limitations and Considerations
+
+The https://github.com/elastic/ecs-logging/tree/main/spec[ecs-logging spec]
+suggests that the first three fields in log records should be `@timestamp`,
+`log.level`, and `message`. As of version 1.5.0, this formatter does *not*
+follow this suggestion. It would be possible but would require creating a new
+Object in `ecsFields` for each log record. Given that ordering of ecs-logging
+fields is for *human readability* and does not affect interoperability, the
+decision was made to prefer performance.
+
 [float]
 [[winston-ref]]
 === Reference
@@ -289,4 +301,49 @@ const logger = winston.createLogger({
 ** `serviceNodeName` +{type-string}+ A "service.node.name" value. If specified this overrides any value from an active APM agent.
 ** `eventDataset` +{type-string}+ A "event.dataset" value. If specified this overrides the default of using `${serviceVersion}`.
 
-Create a formatter for winston that emits in ECS Logging format.
+Create a formatter for winston that emits in ECS Logging format. This is a
+single format that handles both <<winston-ref-ecsFields>> and <<winston-ref-ecsStringify>>.
+The following two are equivalent:
+
+[source,js]
+----
+const { ecsFormat, ecsFields, ecsStringify } = require('@elastic/ecs-winston-format');
+const winston = require('winston');
+
+const logger = winston.createLogger({
+  format: ecsFormat(/* options */),
+  // ...
+});
+
+const logger = winston.createLogger({
+  format: winston.format.combine(
+    ecsFields(/* options */),
+    ecsStringify()
+  ),
+  // ...
+});
+----
+
+[float]
+[[winston-ref-ecsFields]]
+==== `ecsFields([options])`
+
+* `options` +{type-object}+ The following options are supported:
+** `convertErr` +{type-boolean}+ Whether to convert a logged `err` field to ECS error fields. *Default:* `true`.
+** `convertReqRes` +{type-boolean}+ Whether to logged `req` and `res` HTTP request and response fields to ECS HTTP, User agent, and URL fields. *Default:* `false`.
+** `apmIntegration` +{type-boolean}+ Whether to enable APM agent integration. *Default:* `true`.
+** `serviceName` +{type-string}+ A "service.name" value. If specified this overrides any value from an active APM agent.
+** `serviceVersion` +{type-string}+ A "service.version" value. If specified this overrides any value from an active APM agent.
+** `serviceEnvironment` +{type-string}+ A "service.environment" value. If specified this overrides any value from an active APM agent.
+** `serviceNodeName` +{type-string}+ A "service.node.name" value. If specified this overrides any value from an active APM agent.
+** `eventDataset` +{type-string}+ A "event.dataset" value. If specified this overrides the default of using `${serviceVersion}`.
+
+Create a formatter for winston that converts fields on the log record info
+objecct to ECS Logging format.
+
+[float]
+[[winston-ref-ecsStringify]]
+==== `ecsStringify([options])`
+
+Create a formatter for winston that stringifies/serializes the log record to
+JSON. (This is very similar to `logform.json()`.)

packages/ecs-winston-format/CHANGELOG.md

@@ -2,6 +2,50 @@
 
 ## Unreleased
 
+- Add `ecsFields` and `ecsStringify` exports that are winston formatters
+  that separate the gathering of ECS fields (`ecsFields`) and the
+  stringification of a log record to an ecs-logging JSON object
+  (`ecsStringify`). This allows for better composability using
+  [`winston.format.combine`](https://github.com/winstonjs/logform#combine).
+
+  The preferred way to import now changes to:
+
+  ```js
+  const { ecsFormat } = require('@elastic/ecs-winston-format'); // NEW
+  ```
+
+  The old way will be deprecated and removed in the future:
+
+  ```js
+  const ecsFormat = require('@elastic/ecs-winston-format'); // OLD
+  ```
+
+  Common usage will still use `ecsFormat` in the same way:
+
+  ```js
+  const { ecsFormat } = require('@elastic/ecs-winston-format');
+  const log = winston.createLogger({
+      format: ecsFormat(<options>),
+      // ...
+  ```
+
+  However, one can use the separated formatters as follows:
+
+  ```js
+  const { ecsFields, ecsStringify } = require('@elastic/ecs-winston-format');
+  const log = winston.createLogger({
+      format: winston.format.combine(
+          ecsFields(<options>),
+          // Add a custom formatter to redact fields here.
+          ecsStringify()
+      ),
+      // ...
+  ```
+
+  One good use case is for redaction of sensitive fields in the log record
+  as in https://github.com/elastic/ecs-logging-nodejs/issues/57. See a
+  complete example at [examples/redact-fields.js](./examples/redact-fields.js).
+
 - Fix/improve serialization of error details to `error.*` fields for the
   various ways a Winston logger handles `Error` instances.
 

packages/ecs-winston-format/README.md

@@ -27,19 +27,19 @@ npm install @elastic/ecs-winston-format
 ## Usage
 
 ```js
-const winston = require('winston')
-const ecsFormat = require('@elastic/ecs-winston-format')
+const winston = require('winston');
+const { ecsFormat } = require('@elastic/ecs-winston-format');
 
 const logger = winston.createLogger({
   level: 'info',
   format: ecsFormat(/* options */),
   transports: [
     new winston.transports.Console()
   ]
-})
+});
 
-logger.info('hi')
-logger.error('oops there is a problem', { foo: 'bar' })
+logger.info('hi');
+logger.error('oops there is a problem', { foo: 'bar' });
 ```
 
 Running this script will produce log output similar to the following:

packages/ecs-winston-format/examples/basic-without-ecs-format.js

@@ -28,6 +28,7 @@ const logger = winston.createLogger({
   // winston format:
   format: winston.format.combine(
     winston.format.timestamp(),
+    winston.format.errors({ stack: true, cause: true }),
     winston.format.json()
   ),
   transports: [
@@ -36,4 +37,8 @@ const logger = winston.createLogger({
 })
 
 logger.info('hi')
-logger.error('oops there is a problem', { foo: 'bar' })
+logger.warn('look out', { foo: 'bar' })
+
+const err = new Error('boom', { cause: new Error('the cause') })
+err.code = 42
+logger.error('here is an exception', err)

packages/ecs-winston-format/examples/basic.js

@@ -18,13 +18,14 @@
 'use strict'
 
 const winston = require('winston')
-const ecsFormat = require('../') // @elastic/ecs-winston-format
+const { ecsFormat } = require('../') // @elastic/ecs-winston-format
 
 const logger = winston.createLogger({
   level: 'info',
   format: ecsFormat(),
-  // Compare to:
+  // Compare to the following (see "basic-without-ecs-format.js"):
   // format: winston.format.combine(
+  //   winston.format.timestamp(),
   //   winston.format.errors({stack: true, cause: true}),
   //   winston.format.json()
   // ),

packages/ecs-winston-format/examples/error.js

@@ -18,7 +18,7 @@
 'use strict'
 
 const winston = require('winston')
-const ecsFormat = require('../') // @elastic/ecs-winston-format
+const { ecsFormat } = require('../') // @elastic/ecs-winston-format
 
 const logger = winston.createLogger({
   level: 'info',

packages/ecs-winston-format/examples/express.js

@@ -20,7 +20,7 @@
 // This shows how one could use @elastic/ecs-winston-format with Express.
 // This implements simple Express middleware to do so.
 
-const ecsFormat = require('../') // @elastic/ecs-winston-format
+const { ecsFormat } = require('../') // @elastic/ecs-winston-format
 const express = require('express')
 const winston = require('winston')
 

packages/ecs-winston-format/examples/http-with-elastic-apm.js

@@ -38,7 +38,7 @@ const apm = require('elastic-apm-node').start({
 
 const http = require('http')
 const winston = require('winston')
-const ecsFormat = require('../') // @elastic/ecs-winston-format
+const { ecsFormat } = require('../') // @elastic/ecs-winston-format
 
 const logger = winston.createLogger({
   level: 'info',

packages/ecs-winston-format/examples/http.js

@@ -19,7 +19,7 @@
 
 const http = require('http')
 const winston = require('winston')
-const ecsFormat = require('../') // @elastic/ecs-winston-format
+const { ecsFormat } = require('../') // @elastic/ecs-winston-format
 
 const logger = winston.createLogger({
   level: 'info',