From 42b513ac66585f82f0ea99ec868cb0e7f3f9ab91 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 27 Apr 2020 18:18:08 -0500 Subject: [PATCH 01/12] feat/update status endpoint --- src/server/es/index.js | 26 ++++++++++++++++++++++++++ src/server/server.js | 10 ++++++++-- src/server/status.js | 15 +++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 src/server/status.js diff --git a/src/server/es/index.js b/src/server/es/index.js index 6585be36..826b269e 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -308,6 +308,32 @@ class ES { ); } + /** + * Get all es indices and their alias + */ + getAllESIndices() { + const indicesArray = this.config.indices.map((e) => e.index); + if (this.config.configIndex) { + indicesArray.push(this.config.configIndex); + } + return this.client.indices.getAlias({ + index: indicesArray, + }).then((resp) => { + try { + return { + statusCode: resp.statusCode, + indices: { + ...resp.body, + }, + }; + } catch (err) { + throw new Error(err); + } + }, (err) => { + throw new Error(err); + }); + } + /** * Check if the field is array */ diff --git a/src/server/server.js b/src/server/server.js index 996883c3..ca238e55 100644 --- a/src/server/server.js +++ b/src/server/server.js @@ -14,6 +14,7 @@ import headerParser from './utils/headerParser'; import getAuthHelperInstance from './auth/authHelper'; import downloadRouter from './download'; import CodedError from './utils/error'; +import statusRouter from './status'; const app = express(); app.use(cors()); @@ -47,8 +48,13 @@ const startServer = () => { }); // simple health check endpoint - app.get('/_status', (req, res) => { - res.send('hello guppy'); + // eslint-disable-next-line no-unused-vars + app.get('/_status', statusRouter, (req, res, err, next) => { + if (err instanceof CodedError) { + res.status(err.code).send(err.msg); + } else { + res.status(500).send(err); + } }); // download endpoint for fetching data directly from es diff --git a/src/server/status.js b/src/server/status.js new file mode 100644 index 00000000..f5807c64 --- /dev/null +++ b/src/server/status.js @@ -0,0 +1,15 @@ + +import esInstance from './es/index'; + + +const statusRouter = async (req, res, next) => { + try { + const data = await esInstance.getAllESIndices(); + res.send(data); + } catch (err) { + next(err); + } + return 0; +}; + +export default statusRouter; From 1a48f5cf02e742200340b2950b61655dc863af13 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Tue, 5 May 2020 13:15:28 -0500 Subject: [PATCH 02/12] chore: status ep response --- src/server/es/index.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index 326b44f6..289f3058 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -327,16 +327,17 @@ class ES { try { return { statusCode: resp.statusCode, + warnings: resp.warnings, indices: { ...resp.body, }, }; } catch (err) { - throw new Error(err); + return { + ...err, + }; } - }, (err) => { - throw new Error(err); - }); + }, (err) => err); } /** From 4e5b7a5531c8d2e4f0194ce41208b029843d29b2 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Tue, 5 May 2020 13:21:47 -0500 Subject: [PATCH 03/12] fix: logic for status --- src/server/es/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index 289f3058..921749b3 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -333,11 +333,11 @@ class ES { }, }; } catch (err) { - return { - ...err, - }; + throw new Error(err); } - }, (err) => err); + }, (err) => { + throw new Error(err); + }); } /** From e71f0a989749867579a9a1bd2525f3a0bf471d92 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Tue, 5 May 2020 17:15:03 -0500 Subject: [PATCH 04/12] feat: rearrange endpoints --- src/server/endpoints.js | 21 +++++++++++++++++++++ src/server/server.js | 5 ++++- src/server/status.js | 15 --------------- 3 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 src/server/endpoints.js delete mode 100644 src/server/status.js diff --git a/src/server/endpoints.js b/src/server/endpoints.js new file mode 100644 index 00000000..ff1c273d --- /dev/null +++ b/src/server/endpoints.js @@ -0,0 +1,21 @@ + +import esInstance from './es/index'; +import { gitVersion, gitCommit } from './version'; + +export const statusRouter = async (req, res, next) => { + try { + const data = await esInstance.getAllESIndices(); + res.send(data); + } catch (err) { + next(err); + } + return 0; +}; + +export const versionRouter = async (req, res) => { + res.send({ + version: gitVersion, + commit: gitCommit, + }); + return 0; +}; diff --git a/src/server/server.js b/src/server/server.js index ca238e55..0a708548 100644 --- a/src/server/server.js +++ b/src/server/server.js @@ -14,7 +14,7 @@ import headerParser from './utils/headerParser'; import getAuthHelperInstance from './auth/authHelper'; import downloadRouter from './download'; import CodedError from './utils/error'; -import statusRouter from './status'; +import { statusRouter, versionRouter } from './endpoints'; const app = express(); app.use(cors()); @@ -57,6 +57,9 @@ const startServer = () => { } }); + // eslint-disable-next-line no-unused-vars + app.get('/_version', versionRouter); + // download endpoint for fetching data directly from es app.post('/download', downloadRouter, (err, req, res, next) => { // eslint-disable-line no-unused-vars diff --git a/src/server/status.js b/src/server/status.js deleted file mode 100644 index f5807c64..00000000 --- a/src/server/status.js +++ /dev/null @@ -1,15 +0,0 @@ - -import esInstance from './es/index'; - - -const statusRouter = async (req, res, next) => { - try { - const data = await esInstance.getAllESIndices(); - res.send(data); - } catch (err) { - next(err); - } - return 0; -}; - -export default statusRouter; From a0e335a466213bc85a28d1014eec8e74a2dd7ffd Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Tue, 5 May 2020 17:15:11 -0500 Subject: [PATCH 05/12] feat: version ep --- .travis.yml | 12 ++++++++---- Dockerfile | 4 ++-- src/server/version.js | 2 ++ 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 src/server/version.js diff --git a/.travis.yml b/.travis.yml index 8d144165..b0081b1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,15 @@ node_js: - "10" services: - - docker + - docker + +before_install: + - COMMIT=`git rev-parse HEAD` && echo "export const gitCommit = \"${COMMIT}\";" >versions.js + - VERSION=`git describe --always --tags` && echo "export const gitVersion =\"${VERSION}\";" >>versions.js install: - - npm install + - npm install script: - - npm run eslint - - npm run test \ No newline at end of file + - npm run eslint + - npm run test \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d4bc0789..e97ce2a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,8 @@ RUN apt-get update \ COPY . /guppy/ WORKDIR /guppy -RUN COMMIT=`git rev-parse HEAD` && echo "export const guppyCommit = \"${COMMIT}\";" >versions.js -RUN VERSION=`git describe --always --tags` && echo "export const guppyVersion =\"${VERSION}\";" >>versions.js +RUN COMMIT=`git rev-parse HEAD` && echo "export const gitCommit = \"${COMMIT}\";" >versions.js +RUN VERSION=`git describe --always --tags` && echo "export const gitVersion =\"${VERSION}\";" >>versions.js RUN /bin/rm -rf .git RUN /bin/rm -rf node_modules diff --git a/src/server/version.js b/src/server/version.js new file mode 100644 index 00000000..1505bcb6 --- /dev/null +++ b/src/server/version.js @@ -0,0 +1,2 @@ +export const gitCommit = 'N/A'; +export const gitVersion = 'N/A'; From c70a6ec80cfec525002a6746932c02a486e33188 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Wed, 6 May 2020 10:01:20 -0500 Subject: [PATCH 06/12] fix: version script --- .travis.yml | 4 ++-- Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index b0081b1c..5d33f3a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,8 @@ services: - docker before_install: - - COMMIT=`git rev-parse HEAD` && echo "export const gitCommit = \"${COMMIT}\";" >versions.js - - VERSION=`git describe --always --tags` && echo "export const gitVersion =\"${VERSION}\";" >>versions.js + - COMMIT=`git rev-parse HEAD` && echo "export const gitCommit = \"${COMMIT}\";" >version.js + - VERSION=`git describe --always --tags` && echo "export const gitVersion =\"${VERSION}\";" >>version.js install: - npm install diff --git a/Dockerfile b/Dockerfile index e97ce2a1..0347d6b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,8 @@ RUN apt-get update \ COPY . /guppy/ WORKDIR /guppy -RUN COMMIT=`git rev-parse HEAD` && echo "export const gitCommit = \"${COMMIT}\";" >versions.js -RUN VERSION=`git describe --always --tags` && echo "export const gitVersion =\"${VERSION}\";" >>versions.js +RUN COMMIT=`git rev-parse HEAD` && echo "export const gitCommit = \"${COMMIT}\";" >version.js +RUN VERSION=`git describe --always --tags` && echo "export const gitVersion =\"${VERSION}\";" >>version.js RUN /bin/rm -rf .git RUN /bin/rm -rf node_modules From f9ed396311a6809fceb4b0e8fd87f07134cfaa10 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Wed, 6 May 2020 10:12:19 -0500 Subject: [PATCH 07/12] fix: script path --- .travis.yml | 4 ++-- Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d33f3a1..0d62b72b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,8 @@ services: - docker before_install: - - COMMIT=`git rev-parse HEAD` && echo "export const gitCommit = \"${COMMIT}\";" >version.js - - VERSION=`git describe --always --tags` && echo "export const gitVersion =\"${VERSION}\";" >>version.js + - COMMIT=`git rev-parse HEAD` && echo "export const gitCommit = \"${COMMIT}\";" >src/server/version.js + - VERSION=`git describe --always --tags` && echo "export const gitVersion =\"${VERSION}\";" >>src/server/version.js install: - npm install diff --git a/Dockerfile b/Dockerfile index 0347d6b0..dd45e1c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,8 @@ RUN apt-get update \ COPY . /guppy/ WORKDIR /guppy -RUN COMMIT=`git rev-parse HEAD` && echo "export const gitCommit = \"${COMMIT}\";" >version.js -RUN VERSION=`git describe --always --tags` && echo "export const gitVersion =\"${VERSION}\";" >>version.js +RUN COMMIT=`git rev-parse HEAD` && echo "export const gitCommit = \"${COMMIT}\";" >src/server/version.js +RUN VERSION=`git describe --always --tags` && echo "export const gitVersion =\"${VERSION}\";" >>src/server/version.js RUN /bin/rm -rf .git RUN /bin/rm -rf node_modules From d6f0fc4a21c7b8ab937ce9a4433617b08061f2af Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 16 Nov 2020 13:54:05 -0600 Subject: [PATCH 08/12] Feat/new ep zakir mod (#102) * feat: add array fields to /_status endpoint --- src/server/endpoints.js | 1 - src/server/es/index.js | 11 ++++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/server/endpoints.js b/src/server/endpoints.js index ff1c273d..994f01fa 100644 --- a/src/server/endpoints.js +++ b/src/server/endpoints.js @@ -1,4 +1,3 @@ - import esInstance from './es/index'; import { gitVersion, gitCommit } from './version'; diff --git a/src/server/es/index.js b/src/server/es/index.js index d3a0ab5e..a10019d7 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -330,11 +330,20 @@ class ES { index: indicesArray, }).then((resp) => { try { + const indicesMetadata = resp.body; + const indicesWithArrayFields = Object.keys(this.arrayFields); + for (let i = 0; i < indicesWithArrayFields.length; i += 1) { + const indexName = indicesWithArrayFields[i]; + if (!indicesMetadata[indexName]) { + indicesMetadata[indexName] = {}; + } + indicesMetadata[indexName].arrayFields = this.arrayFields[indexName]; + } return { statusCode: resp.statusCode, warnings: resp.warnings, indices: { - ...resp.body, + ...indicesMetadata, }, }; } catch (err) { From e576ed6ddec59a93ca5e036e8c58b6e325b8dcd1 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 16 Nov 2020 14:39:32 -0600 Subject: [PATCH 09/12] rebuild quay --- src/server/es/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server/es/index.js b/src/server/es/index.js index a10019d7..439bb047 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -339,6 +339,7 @@ class ES { } indicesMetadata[indexName].arrayFields = this.arrayFields[indexName]; } + return { statusCode: resp.statusCode, warnings: resp.warnings, From a81eca9352fb7ad7d26c450b51a4c896983df88e Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 16 Nov 2020 14:39:53 -0600 Subject: [PATCH 10/12] rebuild quay --- src/server/es/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index 439bb047..a10019d7 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -339,7 +339,6 @@ class ES { } indicesMetadata[indexName].arrayFields = this.arrayFields[indexName]; } - return { statusCode: resp.statusCode, warnings: resp.warnings, From 801d51ef04e016b162e3922042430f760ab6e878 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 16 Nov 2020 14:55:17 -0600 Subject: [PATCH 11/12] fix quay issue --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index dd45e1c8..214d00b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM quay.io/cdis/ubuntu:16.04 ENV DEBIAN_FRONTEND=noninteractive From 0959ea08407fcc7eb2b138850c7e89350a9859ab Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 16 Nov 2020 16:50:31 -0600 Subject: [PATCH 12/12] feat: update README --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 177dce62..1d630bb4 100644 --- a/README.md +++ b/README.md @@ -107,5 +107,15 @@ npm start > > (E.g., `"tier_access_sensitive_record_exclusion_field": "sensitive"` in the Guppy config tells Guppy to look for a field in the ES index called `sensitive`, and to exclude records in the ES index which have `sensitive: "true"`) -### Download Endpoint: -Guppy has another special endpoint `/download` for just fetching raw data from elasticsearch. please see [here](https://github.com/uc-cdis/guppy/blob/master/doc/download.md) for more details. +### Additional Guppy Endpoints: +Guppy has a special endpoint `/download` for just fetching raw data from elasticsearch. This endpoint can be used to overcome Elastic Search's 10k record limit. Please see [here](https://github.com/uc-cdis/guppy/blob/master/doc/download.md) for details. + +Guppy's `/_status` endpoint yields health check and array field information. This endpoint is publicly accessible and returns output of the form +``` +{"statusCode":200,"warnings":null,"indices":{"":{"aliases":{"alias-name":{}},"arrayFields":[""]}}} +``` + +The `/_version` endpoint yields version and commit information. This endpoint is publicly accessible and returns output of the form +``` +{"version":"","commit":""} +``` \ No newline at end of file