From f8221571af5af4164adbde91d9941b4a416ed731 Mon Sep 17 00:00:00 2001 From: Sean Grove Date: Tue, 27 Jul 2021 12:09:12 -0700 Subject: [PATCH 1/5] Make secrets lookup just-in-time and dynamic --- src/lib/consts.js | 2 + src/lib/secrets.js | 145 +++++++++++++++++++++++++++++++++++------- src/lib/services.json | 18 +----- 3 files changed, 128 insertions(+), 37 deletions(-) diff --git a/src/lib/consts.js b/src/lib/consts.js index ae2176cb..25b96981 100644 --- a/src/lib/consts.js +++ b/src/lib/consts.js @@ -2,10 +2,12 @@ const BUILDER_FUNCTIONS_FLAG = true const HTTP_STATUS_METHOD_NOT_ALLOWED = 405 const HTTP_STATUS_OK = 200 const METADATA_VERSION = 1 +const ONEGRAPH_AUTHLIFY_APP_ID = '4d3de9a5-722f-4d27-9c96-2ac43c93c004' module.exports = { BUILDER_FUNCTIONS_FLAG, HTTP_STATUS_METHOD_NOT_ALLOWED, HTTP_STATUS_OK, METADATA_VERSION, + ONEGRAPH_AUTHLIFY_APP_ID } diff --git a/src/lib/secrets.js b/src/lib/secrets.js index 71886b11..64987206 100644 --- a/src/lib/secrets.js +++ b/src/lib/secrets.js @@ -1,33 +1,134 @@ -const process = require('process') +const https = require("https"); +const process = require("process"); +const { ONEGRAPH_AUTHLIFY_APP_ID } = require("./consts"); -const services = require('./services.json') +function camelize(text) { + text = text.replace(/[-_\s.]+(.)?/g, (_, c) => (c ? c.toUpperCase() : "")); + return text.substr(0, 1).toLowerCase() + text.substr(1); +} + +// The services will be camelized versions of the OneGraph service enums +// unless overridden by the serviceNormalizeOverrides object +const serviceNormalizeOverrides = { + // Keys are the OneGraph service enums, values are the desired `secret.` names + GITHUB: "gitHub", +}; + +function oneGraphRequest(secretToken, body) { + return new Promise((resolve, reject) => { + const options = { + host: "serve.onegraph.com", + path: "/graphql?app_id=" + ONEGRAPH_AUTHLIFY_APP_ID, + port: 443, + method: "POST", + headers: { + Authorization: "Bearer " + secretToken, + "Content-Type": "application/json", + Accept: "application/json", + "Content-Length": body ? Buffer.byteLength(body) : 0, + }, + }; -const getSecrets = () => - Object.entries(services).reduce((secrets, [serviceName, service]) => { - const serviceSecrets = [] - // This is so if there are no secrets we don't add an empty object - Object.entries(service.tokens).forEach(([tokenName, token]) => { - if (token in process.env) { - serviceSecrets.push([tokenName, process.env[token]]) + const req = https.request(options, (res) => { + if (res.statusCode !== 200) { + return reject(new Error(res.statusCode)); } - }) - if (serviceSecrets.length !== 0) { - // No Object.fromEntries in node < 12 - return { - ...secrets, - [serviceName]: serviceSecrets.reduce((acc, [tokenName, token]) => ({ ...acc, [tokenName]: token }), {}), + + var body = []; + + res.on("data", function (chunk) { + body.push(chunk); + }); + + res.on("end", function () { + const data = Buffer.concat(body).toString(); + try { + body = JSON.parse(data); + } catch (e) { + reject(e); + } + resolve(body); + }); + }); + + req.on("error", (e) => { + reject(e.message); + }); + + req.write(body); + + req.end(); + }); +} + +let secrets = {}; + +// Note: We may want to have configurable "sets" of secrets, +// e.g. "dev" and "prod" +const getSecrets = async () => { + const secretToken = process.env.ONEGRAPH_AUTHLIFY_TOKEN; + + if (!secretToken) { + return {}; + } + + // Cache in memory for the life of the serverless process + if (secrets[secretToken]) { + return secrets[secretToken]; + } + + const doc = `query FindLoggedInServicesQuery { + me { + serviceMetadata { + loggedInServices { + friendlyServiceName + service + isLoggedIn + bearerToken + } } } - return secrets - }, {}) + }`; + + const body = JSON.stringify({ query: doc }); + + const result = await oneGraphRequest( + secretToken, + new TextEncoder().encode(body) + ); + + const services = + result.data && + result.data.me && + result.data.me.serviceMetadata && + result.data.me.serviceMetadata.loggedInServices; + + if (services) { + const newSecrets = services.reduce((acc, service) => { + const normalized = + serviceNormalizeOverrides[service.service] || + camelize(service.friendlyServiceName); + acc[normalized] = service; + return acc; + }, {}); + + secrets[secretToken] = newSecrets; + return newSecrets; + } else { + return {}; + } +}; // eslint-disable-next-line promise/prefer-await-to-callbacks -const withSecrets = (handler) => (event, context, callback) => { - const secrets = getSecrets() - return handler(event, { ...context, secrets }, callback) -} +const withSecrets = (handler) => async (event, context, callback) => { + const secrets = await getSecrets(); + + return handler(event, { ...context, secrets }, callback); +}; module.exports = { + // Fine-grained control during the preview, less necessary with a more proactive OneGraph solution getSecrets, + // The common usage of this module withSecrets, -} +}; diff --git a/src/lib/services.json b/src/lib/services.json index 18e164e1..e859f19b 100644 --- a/src/lib/services.json +++ b/src/lib/services.json @@ -1,17 +1,5 @@ { - "github": { - "tokens": { - "token": "ONEGRAPH_GITHUB_TOKEN" - } - }, - "spotify": { - "tokens": { - "token": "ONEGRAPH_SPOTIFY_TOKEN" - } - }, - "salesforce": { - "tokens": { - "token": "ONEGRAPH_SALESFORCE_TOKEN" - } - } + "gitHub": null, + "spotify": null, + "salesforce": null } From d0dd0bfc91033207acf026169be4c57285cfb5ad Mon Sep 17 00:00:00 2001 From: Sean Grove Date: Tue, 27 Jul 2021 12:43:41 -0700 Subject: [PATCH 2/5] fix: fix ci issues Fix eslint and prettier --- src/lib/consts.js | 2 +- src/lib/secrets.js | 148 +++++++++++++++++++++++---------------------- 2 files changed, 77 insertions(+), 73 deletions(-) diff --git a/src/lib/consts.js b/src/lib/consts.js index 25b96981..f71ab0b4 100644 --- a/src/lib/consts.js +++ b/src/lib/consts.js @@ -9,5 +9,5 @@ module.exports = { HTTP_STATUS_METHOD_NOT_ALLOWED, HTTP_STATUS_OK, METADATA_VERSION, - ONEGRAPH_AUTHLIFY_APP_ID + ONEGRAPH_AUTHLIFY_APP_ID, } diff --git a/src/lib/secrets.js b/src/lib/secrets.js index 64987206..f585f3a3 100644 --- a/src/lib/secrets.js +++ b/src/lib/secrets.js @@ -1,80 +1,102 @@ -const https = require("https"); -const process = require("process"); -const { ONEGRAPH_AUTHLIFY_APP_ID } = require("./consts"); +const Buffer = require('buffer') +const https = require('https') +const process = require('process') -function camelize(text) { - text = text.replace(/[-_\s.]+(.)?/g, (_, c) => (c ? c.toUpperCase() : "")); - return text.substr(0, 1).toLowerCase() + text.substr(1); +const { ONEGRAPH_AUTHLIFY_APP_ID } = require('./consts') + +const camelize = function (text) { + const safe = text.replace(/[-_\s.]+(.)?/g, (_, sub) => (sub ? sub.toUpperCase() : '')) + return safe.slice(0, 1).toLowerCase() + safe.slice(1) } // The services will be camelized versions of the OneGraph service enums // unless overridden by the serviceNormalizeOverrides object const serviceNormalizeOverrides = { // Keys are the OneGraph service enums, values are the desired `secret.` names - GITHUB: "gitHub", -}; + GITHUB: 'gitHub', +} -function oneGraphRequest(secretToken, body) { +const oneGraphRequest = function (secretToken, requestBody) { return new Promise((resolve, reject) => { + const port = 443 + const options = { - host: "serve.onegraph.com", - path: "/graphql?app_id=" + ONEGRAPH_AUTHLIFY_APP_ID, - port: 443, - method: "POST", + host: 'serve.onegraph.com', + path: `/graphql?app_id=${ONEGRAPH_AUTHLIFY_APP_ID}`, + port, + method: 'POST', headers: { - Authorization: "Bearer " + secretToken, - "Content-Type": "application/json", - Accept: "application/json", - "Content-Length": body ? Buffer.byteLength(body) : 0, + Authorization: `Bearer ${secretToken}`, + 'Content-Type': 'application/json', + Accept: 'application/json', + 'Content-Length': requestBody ? Buffer.byteLength(requestBody) : 0, }, - }; + } const req = https.request(options, (res) => { if (res.statusCode !== 200) { - return reject(new Error(res.statusCode)); + return reject(new Error(res.statusCode)) } - var body = []; + let body = [] - res.on("data", function (chunk) { - body.push(chunk); - }); + res.on('data', (chunk) => { + body.push(chunk) + }) - res.on("end", function () { - const data = Buffer.concat(body).toString(); + res.on('end', () => { + const data = Buffer.concat(body).toString() try { - body = JSON.parse(data); - } catch (e) { - reject(e); + body = JSON.parse(data) + } catch (error) { + reject(error) } - resolve(body); - }); - }); + resolve(body) + }) + }) + + req.on('error', (error) => { + reject(error.message) + }) + + req.write(requestBody) + + req.end() + }) +} + +const formatSecrets = (result) => { + const services = + result.data && result.data.me && result.data.me.serviceMetadata && result.data.me.serviceMetadata.loggedInServices - req.on("error", (e) => { - reject(e.message); - }); + if (services) { + const newSecrets = services.reduce((acc, service) => { + const normalized = serviceNormalizeOverrides[service.service] || camelize(service.friendlyServiceName) + // eslint-disable-next-line no-param-reassign + acc[normalized] = service + return acc + }, {}) - req.write(body); + return newSecrets + } - req.end(); - }); + return {} } -let secrets = {}; +const secretsCache = {} // Note: We may want to have configurable "sets" of secrets, // e.g. "dev" and "prod" const getSecrets = async () => { - const secretToken = process.env.ONEGRAPH_AUTHLIFY_TOKEN; + const secretToken = process.env.ONEGRAPH_AUTHLIFY_TOKEN if (!secretToken) { - return {}; + return {} } // Cache in memory for the life of the serverless process - if (secrets[secretToken]) { - return secrets[secretToken]; + if (secretsCache[secretToken]) { + return secretsCache[secretToken] } const doc = `query FindLoggedInServicesQuery { @@ -88,47 +110,29 @@ const getSecrets = async () => { } } } - }`; + }` - const body = JSON.stringify({ query: doc }); + const body = JSON.stringify({ query: doc }) - const result = await oneGraphRequest( - secretToken, - new TextEncoder().encode(body) - ); + // eslint-disable-next-line node/no-unsupported-features/node-builtins + const result = await oneGraphRequest(secretToken, new TextEncoder().encode(body)) - const services = - result.data && - result.data.me && - result.data.me.serviceMetadata && - result.data.me.serviceMetadata.loggedInServices; + const newSecrets = formatSecrets(result) + secretsCache[secretToken] = newSecrets - if (services) { - const newSecrets = services.reduce((acc, service) => { - const normalized = - serviceNormalizeOverrides[service.service] || - camelize(service.friendlyServiceName); - acc[normalized] = service; - return acc; - }, {}); - - secrets[secretToken] = newSecrets; - return newSecrets; - } else { - return {}; - } -}; + return newSecrets +} // eslint-disable-next-line promise/prefer-await-to-callbacks const withSecrets = (handler) => async (event, context, callback) => { - const secrets = await getSecrets(); + const secrets = await getSecrets() - return handler(event, { ...context, secrets }, callback); -}; + return handler(event, { ...context, secrets }, callback) +} module.exports = { // Fine-grained control during the preview, less necessary with a more proactive OneGraph solution getSecrets, // The common usage of this module withSecrets, -}; +} From 393a711d0dd35bd4577c2db1f56c735e01b85187 Mon Sep 17 00:00:00 2001 From: Sean Grove Date: Sun, 1 Aug 2021 19:33:45 -0700 Subject: [PATCH 3/5] feat: update TypeScript definitions and auth metadata Expands on the GraphQL query for the service auths --- package-lock.json | 6 ++++-- package.json | 4 ++-- src/lib/secrets.d.ts | 23 +++++++++++++++++++++-- src/lib/secrets.js | 26 +++++++++++++++++--------- src/lib/services.json | 3 ++- 5 files changed, 46 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index e25d1af2..93be6b3d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "@netlify/functions", + "name": "@sgrove/netlify-functions", "version": "0.7.3-handle-secrets.1", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "@netlify/functions", + "name": "@sgrove/netlify-functions", "version": "0.7.3-handle-secrets.1", "license": "MIT", "dependencies": { @@ -2409,6 +2409,7 @@ "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", + "fsevents": "~2.3.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", @@ -6636,6 +6637,7 @@ "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", "dev": true, "dependencies": { + "graceful-fs": "^4.1.6", "universalify": "^2.0.0" }, "optionalDependencies": { diff --git a/package.json b/package.json index 95557658..2206e842 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { - "name": "@netlify/functions", + "name": "@sgrove/netlify-functions", "main": "./src/main.js", "types": "./src/main.d.ts", - "version": "0.7.3-handle-secrets.1", + "version": "0.7.3-handle-secrets.3", "description": "JavaScript utilities for Netlify Functions", "files": [ "src/**/*.js", diff --git a/src/lib/secrets.d.ts b/src/lib/secrets.d.ts index 2c202276..2bacf278 100644 --- a/src/lib/secrets.d.ts +++ b/src/lib/secrets.d.ts @@ -2,14 +2,33 @@ import { Context } from '../function/context' import { Handler } from '../function/handler' import * as services from './services.json' +export type Service = { + friendlyServiceName: string + service: string + isLoggedIn: boolean + bearerToken: string | null + grantedScopes: Array<{ + scope: string + scopeInfo: { + category: string | null + scope: string + display: string + isDefault: boolean + isRequired: boolean + description: string | null + title: string | null + } + }> | null +} + export type Services = typeof services export type ServiceKey = keyof Services -export type ServiceTokens = Services[T]['tokens'] +export type ServiceTokens = Service export type NetlifySecrets = { - [K in ServiceKey]?: ServiceTokens + [K in ServiceKey]?: Service } export interface ContextWithSecrets extends Context { diff --git a/src/lib/secrets.js b/src/lib/secrets.js index f585f3a3..8c5f3524 100644 --- a/src/lib/secrets.js +++ b/src/lib/secrets.js @@ -1,4 +1,4 @@ -const Buffer = require('buffer') +const { Buffer } = require('buffer') const https = require('https') const process = require('process') @@ -83,8 +83,6 @@ const formatSecrets = (result) => { return {} } -const secretsCache = {} - // Note: We may want to have configurable "sets" of secrets, // e.g. "dev" and "prod" const getSecrets = async () => { @@ -94,11 +92,10 @@ const getSecrets = async () => { return {} } - // Cache in memory for the life of the serverless process - if (secretsCache[secretToken]) { - return secretsCache[secretToken] - } - + // We select for more than we typeically need here + // in order to allow for some metaprogramming for + // consumers downstream. Also, the data is typically + // static and shouldn't add any measurable overhead. const doc = `query FindLoggedInServicesQuery { me { serviceMetadata { @@ -107,6 +104,18 @@ const getSecrets = async () => { service isLoggedIn bearerToken + grantedScopes { + scope + scopeInfo { + category + scope + display + isDefault + isRequired + description + title + } + } } } } @@ -118,7 +127,6 @@ const getSecrets = async () => { const result = await oneGraphRequest(secretToken, new TextEncoder().encode(body)) const newSecrets = formatSecrets(result) - secretsCache[secretToken] = newSecrets return newSecrets } diff --git a/src/lib/services.json b/src/lib/services.json index e859f19b..2eed9bf5 100644 --- a/src/lib/services.json +++ b/src/lib/services.json @@ -1,5 +1,6 @@ { "gitHub": null, "spotify": null, - "salesforce": null + "salesforce": null, + "stripe": null } From e5f8842c7055edcbe13d52867e8d2f218da27674 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Mon, 2 Aug 2021 10:30:06 -0700 Subject: [PATCH 4/5] feat: add warning if Autlify has not been set up --- package-lock.json | 6 ++---- package.json | 4 ++-- src/lib/secrets.js | 5 ++++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 93be6b3d..e25d1af2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "@sgrove/netlify-functions", + "name": "@netlify/functions", "version": "0.7.3-handle-secrets.1", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "@sgrove/netlify-functions", + "name": "@netlify/functions", "version": "0.7.3-handle-secrets.1", "license": "MIT", "dependencies": { @@ -2409,7 +2409,6 @@ "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", - "fsevents": "~2.3.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", @@ -6637,7 +6636,6 @@ "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", "dev": true, "dependencies": { - "graceful-fs": "^4.1.6", "universalify": "^2.0.0" }, "optionalDependencies": { diff --git a/package.json b/package.json index 2206e842..95557658 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { - "name": "@sgrove/netlify-functions", + "name": "@netlify/functions", "main": "./src/main.js", "types": "./src/main.d.ts", - "version": "0.7.3-handle-secrets.3", + "version": "0.7.3-handle-secrets.1", "description": "JavaScript utilities for Netlify Functions", "files": [ "src/**/*.js", diff --git a/src/lib/secrets.js b/src/lib/secrets.js index 8c5f3524..1b82dba4 100644 --- a/src/lib/secrets.js +++ b/src/lib/secrets.js @@ -89,10 +89,13 @@ const getSecrets = async () => { const secretToken = process.env.ONEGRAPH_AUTHLIFY_TOKEN if (!secretToken) { + console.warn( + 'withSecrets is not set up. Visit Netlify Labs to enable it or trigger a new deploy if it has been enabled.', + ) return {} } - // We select for more than we typeically need here + // We select for more than we typically need here // in order to allow for some metaprogramming for // consumers downstream. Also, the data is typically // static and shouldn't add any measurable overhead. From 6c0d8e348116a4485e73a04adde70cfaab1ffd36 Mon Sep 17 00:00:00 2001 From: Daniel Woelfel Date: Tue, 3 Aug 2021 08:47:00 -0700 Subject: [PATCH 5/5] feat: remove withContext and implement review feedback --- src/lib/secrets.d.ts | 48 +++++++++++++++++-------------------------- src/lib/secrets.js | 21 ++++++------------- src/lib/services.json | 6 ------ 3 files changed, 25 insertions(+), 50 deletions(-) delete mode 100644 src/lib/services.json diff --git a/src/lib/secrets.d.ts b/src/lib/secrets.d.ts index 2bacf278..6821695f 100644 --- a/src/lib/secrets.d.ts +++ b/src/lib/secrets.d.ts @@ -1,42 +1,32 @@ -import { Context } from '../function/context' -import { Handler } from '../function/handler' -import * as services from './services.json' +export type ScopeInfo = { + category: string | null + scope: string + display: string + isDefault: boolean + isRequired: boolean + description: string | null + title: string | null +} + +export type Scope = { + scope: string + scopeInfo: ScopeInfo | null +} export type Service = { friendlyServiceName: string service: string isLoggedIn: boolean bearerToken: string | null - grantedScopes: Array<{ - scope: string - scopeInfo: { - category: string | null - scope: string - display: string - isDefault: boolean - isRequired: boolean - description: string | null - title: string | null - } - }> | null + grantedScopes: Array | null } -export type Services = typeof services - -export type ServiceKey = keyof Services - -export type ServiceTokens = Service - export type NetlifySecrets = { - [K in ServiceKey]?: Service -} - -export interface ContextWithSecrets extends Context { - secrets: NetlifySecrets + gitHub?: Service | null + spotify?: Service | null + salesforce?: Service | null + stripe?: Service | null } -export type HandlerWithSecrets = Handler - export declare const getSecrets: () => NetlifySecrets -export declare const withSecrets: (handler: HandlerWithSecrets) => Handler diff --git a/src/lib/secrets.js b/src/lib/secrets.js index 1b82dba4..072f06dc 100644 --- a/src/lib/secrets.js +++ b/src/lib/secrets.js @@ -17,6 +17,8 @@ const serviceNormalizeOverrides = { } const oneGraphRequest = function (secretToken, requestBody) { + // eslint-disable-next-line node/no-unsupported-features/node-builtins + const requestBodyBuffer = Buffer.from(new TextEncoder().encode(requestBody)) return new Promise((resolve, reject) => { const port = 443 @@ -29,7 +31,7 @@ const oneGraphRequest = function (secretToken, requestBody) { Authorization: `Bearer ${secretToken}`, 'Content-Type': 'application/json', Accept: 'application/json', - 'Content-Length': requestBody ? Buffer.byteLength(requestBody) : 0, + 'Content-Length': Buffer.byteLength(requestBodyBuffer), }, } @@ -59,7 +61,7 @@ const oneGraphRequest = function (secretToken, requestBody) { reject(error.message) }) - req.write(requestBody) + req.write(requestBodyBuffer) req.end() }) @@ -90,7 +92,7 @@ const getSecrets = async () => { if (!secretToken) { console.warn( - 'withSecrets is not set up. Visit Netlify Labs to enable it or trigger a new deploy if it has been enabled.', + 'getSecrets is not set up. Visit Netlify Labs to enable it or trigger a new deploy if it has been enabled.', ) return {} } @@ -126,24 +128,13 @@ const getSecrets = async () => { const body = JSON.stringify({ query: doc }) - // eslint-disable-next-line node/no-unsupported-features/node-builtins - const result = await oneGraphRequest(secretToken, new TextEncoder().encode(body)) + const result = await oneGraphRequest(secretToken, body) const newSecrets = formatSecrets(result) return newSecrets } -// eslint-disable-next-line promise/prefer-await-to-callbacks -const withSecrets = (handler) => async (event, context, callback) => { - const secrets = await getSecrets() - - return handler(event, { ...context, secrets }, callback) -} - module.exports = { - // Fine-grained control during the preview, less necessary with a more proactive OneGraph solution getSecrets, - // The common usage of this module - withSecrets, } diff --git a/src/lib/services.json b/src/lib/services.json deleted file mode 100644 index 2eed9bf5..00000000 --- a/src/lib/services.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "gitHub": null, - "spotify": null, - "salesforce": null, - "stripe": null -}