From f364e8fbcffe10904e16f33c99864674cba3cdca Mon Sep 17 00:00:00 2001 From: Yasser Fadl Date: Mon, 21 Oct 2019 19:21:56 +0100 Subject: [PATCH 1/5] feat: Adding pino logger --- package-lock.json | 72 +++++++++++++++ package.json | 2 + src/config/config-initialiser.ts | 16 +++- src/services/logger/pino/pino-logger.ts | 90 +++++++++++++++++++ .../logger/{ => std}/std-out-logger.spec.ts | 2 +- .../logger/{ => std}/std-out-logger.ts | 2 +- 6 files changed, 178 insertions(+), 6 deletions(-) create mode 100644 src/services/logger/pino/pino-logger.ts rename src/services/logger/{ => std}/std-out-logger.spec.ts (95%) rename src/services/logger/{ => std}/std-out-logger.ts (98%) diff --git a/package-lock.json b/package-lock.json index 20e70d23d..264fcc8b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -416,6 +416,23 @@ "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==", "dev": true }, + "@types/pino": { + "version": "5.8.12", + "resolved": "https://registry.npmjs.org/@types/pino/-/pino-5.8.12.tgz", + "integrity": "sha512-hIqsIogqsiPhL/0VJhvWNo859rpFrQ+K0iVdtJBzP0alRYUJot9bSl7Nhypnpi+uvYMrzhfrRuNMzyvFyvKKNA==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/pino-std-serializers": "*", + "@types/sonic-boom": "*" + } + }, + "@types/pino-std-serializers": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@types/pino-std-serializers/-/pino-std-serializers-2.4.0.tgz", + "integrity": "sha512-eAdu+NW1IkCdmp85SnhyKha+OOREQMT9lXaoICQxa7bhSauRiLzu3WSNt9Mf2piuJvWeXF/G0hGWHr63xNpIRA==", + "dev": true + }, "@types/sinon": { "version": "7.0.13", "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-7.0.13.tgz", @@ -432,6 +449,15 @@ "@types/sinon": "*" } }, + "@types/sonic-boom": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@types/sonic-boom/-/sonic-boom-0.7.0.tgz", + "integrity": "sha512-AfqR0fZMoUXUNwusgXKxcE9DPlHNDHQp6nKYUd4PSRpLobF5CCevSpyTEBcVZreqaWKCnGBr9KI1fHMTttoB7A==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/ws": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-6.0.3.tgz", @@ -2199,6 +2225,16 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "fast-redact": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-2.0.0.tgz", + "integrity": "sha512-zxpkULI9W9MNTK2sJ3BpPQrTEXFNESd2X6O1tXMFpK/XM0G5c5Rll2EVYZH2TqI3xRGK/VaJ+eEOt7pnENJpeA==" + }, + "fast-safe-stringify": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", + "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" + }, "fd-slicer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", @@ -2292,6 +2328,11 @@ "is-buffer": "~2.0.3" } }, + "flatstr": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz", + "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==" + }, "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -5559,6 +5600,24 @@ "pinkie": "^2.0.0" } }, + "pino": { + "version": "5.13.5", + "resolved": "https://registry.npmjs.org/pino/-/pino-5.13.5.tgz", + "integrity": "sha512-NSArDZnjIXgzTLsYA5EhYwLiMe2OmGJ73760Wt5Vj44kUcuPJk4ub29BKtWXGAMwVmW1cQ7Q8jQaLjY/5Gxqcw==", + "requires": { + "fast-redact": "^2.0.0", + "fast-safe-stringify": "^2.0.7", + "flatstr": "^1.0.9", + "pino-std-serializers": "^2.3.0", + "quick-format-unescaped": "^3.0.3", + "sonic-boom": "^0.7.5" + } + }, + "pino-std-serializers": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-2.4.2.tgz", + "integrity": "sha512-WaL504dO8eGs+vrK+j4BuQQq6GLKeCCcHaMB2ItygzVURcL1CycwNEUHTD/lHFHs/NL5qAz2UKrjYWXKSf4aMQ==" + }, "pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -5737,6 +5796,11 @@ "strict-uri-encode": "^1.0.0" } }, + "quick-format-unescaped": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-3.0.3.tgz", + "integrity": "sha512-dy1yjycmn9blucmJLXOfZDx1ikZJUi6E8bBZLnhPG5gBrVhHXx2xVyqqgKBubVNEXmx51dBACMHpoMQK/N/AXQ==" + }, "raw-body": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", @@ -6263,6 +6327,14 @@ } } }, + "sonic-boom": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-0.7.6.tgz", + "integrity": "sha512-k9E2QQ4zxuVRLDW+ZW6ISzJs3wlEorVdmM7ApDgor7wsGKSDG5YGHsGmgLY4XYh4DMlr/2ap2BWAE7yTFJtWnQ==", + "requires": { + "flatstr": "^1.0.12" + } + }, "sort-keys": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", diff --git a/package.json b/package.json index a2ca76f8b..95610bca3 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "mkdirp": "^0.5.1", "mqtt-connection": "^4.0.0", "needle": "^2.4.0", + "pino": "^5.13.5", "source-map-support": "^0.5.13", "uWebSockets.js": "github:uNetworking/uWebSockets.js#v15.11.0", "ws": "^7.1.2" @@ -66,6 +67,7 @@ "@types/mocha": "^5.2.7", "@types/needle": "^2.0.4", "@types/node": "^12.7.8", + "@types/pino": "^5.8.12", "@types/sinon": "^7.0.13", "@types/sinon-chai": "^3.2.3", "@types/ws": "^6.0.3", diff --git a/src/config/config-initialiser.ts b/src/config/config-initialiser.ts index f275db6c9..368987892 100644 --- a/src/config/config-initialiser.ts +++ b/src/config/config-initialiser.ts @@ -17,7 +17,8 @@ import { FileBasedAuthentication } from '../services/authentication/file/file-ba import { HttpAuthentication } from '../services/authentication/http/http-authentication' import { NoopStorage } from '../services/storage/noop-storage' import { LocalCache } from '../services/cache/local-cache' -import { StdOutLogger } from '../services/logger/std-out-logger' +import { StdOutLogger } from '../services/logger/std/std-out-logger' +import { PinoLogger } from '../services/logger/pino/pino-logger' import { LocalMonitoring } from '../services/monitoring/noop-monitoring' import { DistributedLockRegistry } from '../services/lock/distributed-lock-registry' import { DistributedStateRegistryFactory } from '../services/cluster-state/distributed-state-registry-factory' @@ -116,10 +117,17 @@ function handleLogger (config: DeepstreamConfig, services: DeepstreamServices): configOptions.colors = commandLineArguments.colors } let LoggerClass - if (config.logger.type === 'default') { - LoggerClass = StdOutLogger - } else { + if (config.logger.name === 'pino') { + LoggerClass = PinoLogger + } else if (config.logger.name || config.logger.path) { LoggerClass = resolvePluginClass(config.logger, 'logger', config.logLevel) + if (!LoggerClass) { + throw new Error(`unable to resolve plugin ${config.logger.name || config.logger.path}`) + } + } + + if (!LoggerClass) { + LoggerClass = StdOutLogger } if (configOptions instanceof Array) { diff --git a/src/services/logger/pino/pino-logger.ts b/src/services/logger/pino/pino-logger.ts new file mode 100644 index 000000000..dcccd3dc2 --- /dev/null +++ b/src/services/logger/pino/pino-logger.ts @@ -0,0 +1,90 @@ +import * as pino from 'pino' +import { LOG_LEVEL, DeepstreamPlugin, DeepstreamLogger, DeepstreamServices, NamespacedLogger, EVENT } from '../../../../ds-types/src' + +const DSToPino: { [index: number]: string } = { + [LOG_LEVEL.DEBUG]: 'debug', + [LOG_LEVEL.FATAL]: 'fatal', + [LOG_LEVEL.ERROR]: 'error', + [LOG_LEVEL.WARN]: 'warn', + [LOG_LEVEL.INFO]: 'info', +} + +export class PinoLogger extends DeepstreamPlugin implements DeepstreamLogger { + public description = 'Pino Logger' + private logger: pino.Logger = pino() + + constructor (pluginOptions: {}, private services: DeepstreamServices) { + super() + } + + /** + * Return true if logging is enabled. This is used in deepstream to stop generating useless complex strings + * that we know will never be logged. + */ + public shouldLog (logLevel: LOG_LEVEL): boolean { + return this.logger.isLevelEnabled(DSToPino[logLevel]) + } + + /** + * Set the log level desired by deepstream. Since deepstream uses LOG_LEVEL this needs to be mapped + * to whatever your libary uses (this is usually just conversion stored in a static map) + */ + public setLogLevel (logLevel: LOG_LEVEL): void { + this.logger.level = DSToPino[logLevel] + } + + /** + * Log as info + */ + public info (event: string, message?: string, metaData?: any): void { + this.logger.info({ message, event }) + } + + /** + * Log as debug + */ + public debug (event: string, message?: string, metaData?: any): void { + this.logger.debug({ message, event }) + } + + /** + * Log as warn + */ + public warn (event: string, message?: string, metaData?: any): void { + this.logger.warn({ message, event }) + } + + /** + * Log as error + */ + public error (event: string, message?: string, metaData?: any): void { + this.logger.error({ message, event }) + } + + /** + * Log as error + */ + public fatal (event: string, message?: string, metaData?: any): void { + this.logger.fatal({ message, event }) + this.services.notifyFatalException() + } + + /** + * Create a namespaced logger, used by plugins. This could either be a new instance of a logger + * or just a thin wrapper to add the namespace at the beginning of the log method. + */ + public getNameSpace (namespace: string): NamespacedLogger { + return { + shouldLog: this.shouldLog.bind(this), + fatal: this.log.bind(this, LOG_LEVEL.FATAL, namespace), + error: this.log.bind(this, LOG_LEVEL.ERROR, namespace), + warn: this.log.bind(this, LOG_LEVEL.WARN, namespace), + info: this.log.bind(this, LOG_LEVEL.INFO, namespace), + debug: this.log.bind(this, LOG_LEVEL.DEBUG, namespace), + } + } + + private log (logLevel: LOG_LEVEL, namespace: string, event: EVENT, message: string) { + this.logger[logLevel]({ namespace, event, message }) + } +} diff --git a/src/services/logger/std-out-logger.spec.ts b/src/services/logger/std/std-out-logger.spec.ts similarity index 95% rename from src/services/logger/std-out-logger.spec.ts rename to src/services/logger/std/std-out-logger.spec.ts index 1603d696f..ff3c754f6 100644 --- a/src/services/logger/std-out-logger.spec.ts +++ b/src/services/logger/std/std-out-logger.spec.ts @@ -3,7 +3,7 @@ import { expect } from 'chai' import {spy} from 'sinon' import { StdOutLogger } from './std-out-logger' -import { LOG_LEVEL, EVENT } from '../../../ds-types/src/index'; +import { LOG_LEVEL, EVENT } from '../../../../ds-types/src/index'; describe('logs to stdout and stderr', () => { const logger = new StdOutLogger({ color: false }) diff --git a/src/services/logger/std-out-logger.ts b/src/services/logger/std/std-out-logger.ts similarity index 98% rename from src/services/logger/std-out-logger.ts rename to src/services/logger/std/std-out-logger.ts index ac726a9d1..0fa3cb2a5 100644 --- a/src/services/logger/std-out-logger.ts +++ b/src/services/logger/std/std-out-logger.ts @@ -1,5 +1,5 @@ import chalk from 'chalk' -import { DeepstreamPlugin, DeepstreamLogger, DeepstreamServices, DeepstreamConfig, LOG_LEVEL, NamespacedLogger, EVENT, MetaData } from '../../../ds-types/src/index' +import { DeepstreamPlugin, DeepstreamLogger, DeepstreamServices, DeepstreamConfig, LOG_LEVEL, NamespacedLogger, EVENT, MetaData } from '../../../../ds-types/src/index' const EOL = require('os').EOL From 9e921ccf783d71176c68f3465881fa2ca9e14e86 Mon Sep 17 00:00:00 2001 From: Yasser Fadl Date: Mon, 21 Oct 2019 19:23:21 +0100 Subject: [PATCH 2/5] fix: small guard to ignore binary frames until V5 --- src/connection-endpoint/text/connection-endpoint.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/connection-endpoint/text/connection-endpoint.ts b/src/connection-endpoint/text/connection-endpoint.ts index ee51fdff5..3171fb4c9 100644 --- a/src/connection-endpoint/text/connection-endpoint.ts +++ b/src/connection-endpoint/text/connection-endpoint.ts @@ -2,7 +2,7 @@ import ConnectionEndpoint, {WebSocketServerConfig} from '../websocket/connection import * as textMessageParser from './protocol/message-parser' import * as textMessageBuilder from './protocol/message-builder' import {createWSSocketWrapper} from './socket-wrapper-factory' -import { DeepstreamServices, SocketWrapper, DeepstreamConfig, UnauthenticatedSocketWrapper } from '../../../ds-types/src/index' +import { DeepstreamServices, SocketWrapper, DeepstreamConfig, UnauthenticatedSocketWrapper, EVENT } from '../../../ds-types/src/index' import { Dictionary } from 'ts-essentials' import * as WebSocket from 'ws' import { IncomingMessage, Server } from 'http' @@ -68,9 +68,11 @@ export class WSTextConnectionEndpoint extends ConnectionEndpoint { }) websocket.on('message', (msg: string) => { - const messages = textMessageParser.parse(msg) - if (messages.length > 0) { - this.connections.get(websocket)!.onMessage(messages) + if (typeof msg !== 'string') { + const messages = textMessageParser.parse(msg) + if (messages.length > 0) { + this.connections.get(websocket)!.onMessage(messages) + } } }) From 639f448fce20ef3587390e6577aa0091dc3278aa Mon Sep 17 00:00:00 2001 From: Yasser Fadl Date: Mon, 21 Oct 2019 19:23:39 +0100 Subject: [PATCH 3/5] fix: small guard to ignore binary frames until V5 --- src/connection-endpoint/text/connection-endpoint.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connection-endpoint/text/connection-endpoint.ts b/src/connection-endpoint/text/connection-endpoint.ts index 3171fb4c9..29c5cb770 100644 --- a/src/connection-endpoint/text/connection-endpoint.ts +++ b/src/connection-endpoint/text/connection-endpoint.ts @@ -2,7 +2,7 @@ import ConnectionEndpoint, {WebSocketServerConfig} from '../websocket/connection import * as textMessageParser from './protocol/message-parser' import * as textMessageBuilder from './protocol/message-builder' import {createWSSocketWrapper} from './socket-wrapper-factory' -import { DeepstreamServices, SocketWrapper, DeepstreamConfig, UnauthenticatedSocketWrapper, EVENT } from '../../../ds-types/src/index' +import { DeepstreamServices, SocketWrapper, DeepstreamConfig, UnauthenticatedSocketWrapper } from '../../../ds-types/src/index' import { Dictionary } from 'ts-essentials' import * as WebSocket from 'ws' import { IncomingMessage, Server } from 'http' @@ -68,7 +68,7 @@ export class WSTextConnectionEndpoint extends ConnectionEndpoint { }) websocket.on('message', (msg: string) => { - if (typeof msg !== 'string') { + if (typeof msg === 'string') { const messages = textMessageParser.parse(msg) if (messages.length > 0) { this.connections.get(websocket)!.onMessage(messages) From ef447574bc3aff4195387e8e9d55afacce250472 Mon Sep 17 00:00:00 2001 From: Yasser Fadl Date: Mon, 21 Oct 2019 19:25:12 +0100 Subject: [PATCH 4/5] 4.3.0 --- CHANGELOG.md | 11 +++++++++++ package-lock.json | 2 +- package.json | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2eef4a102..f9f5e98f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## [4.3.0] - 2019.10.21 + +### Feat + +Adding a PINO logger for JSON logging (name: pino) + +### Fix + +Don't crash when sending a binary message frame to a V3 text endpoint. This has a more +robust solution in V5 with logging. + ## [4.2.5] - 2019.10.01 ### Fix diff --git a/package-lock.json b/package-lock.json index 264fcc8b3..9e66e0f83 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "@deepstream/server", - "version": "4.2.5", + "version": "4.3.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 95610bca3..abe38e2fd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@deepstream/server", - "version": "4.2.5", + "version": "4.3.0", "description": "a scalable server for realtime webapps", "main": "./dist/src/deepstream.io.js", "bin": { From 12bf8964a80db4212426925b893604a369f13abe Mon Sep 17 00:00:00 2001 From: Yasser Fadl Date: Mon, 21 Oct 2019 19:34:15 +0100 Subject: [PATCH 5/5] fix: Removing gitmodule --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index a8ed98ca4..8d3f78b26 100644 --- a/.gitmodules +++ b/.gitmodules @@ -28,9 +28,6 @@ [submodule "ds-types"] path = ds-types url = git@github.com:deepstreamIO/deepstream.io-types.git -[submodule "connectors/monitoring/http"] - path = connectors/monitoring/http - url = git@github.com:deepstreamIO/deepstream.io-monitoring-http.git [submodule "connectors/clusterNode/redis"] path = connectors/clusterNode/redis url = git@github.com:deepstreamIO/deepstream.io-clusternode-redis.git