From c8af68c96478fbe67c2dcbc3b070fdca2fab4add Mon Sep 17 00:00:00 2001 From: Craig Barnes Date: Wed, 6 Oct 2021 11:40:46 -0500 Subject: [PATCH 01/27] remove index fiels where enabled===false --- src/server/es/index.js | 254 ++++++++++++++++++++++++----------------- 1 file changed, 151 insertions(+), 103 deletions(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index 76ba0fe96..489d25b20 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -36,11 +36,12 @@ class ES { */ async query(esIndex, esType, queryBody) { const validatedQueryBody = {}; - Object.keys(queryBody).forEach((key) => { - if (typeof queryBody[key] !== 'undefined' && queryBody[key] !== null) { - validatedQueryBody[key] = queryBody[key]; - } - }); + Object.keys(queryBody) + .forEach((key) => { + if (typeof queryBody[key] !== 'undefined' && queryBody[key] !== null) { + validatedQueryBody[key] = queryBody[key]; + } + }); validatedQueryBody.highlight = { pre_tags: [ `<${config.matchedTextHighlightTagName}>`, @@ -57,10 +58,11 @@ class ES { index: esIndex, type: esType, body: validatedQueryBody, - }).then((resp) => resp.body, (err) => { - log.error(`[ES.query] error during querying: ${err.message}`); - throw new Error(err.message); - }); + }) + .then((resp) => resp.body, (err) => { + log.error(`[ES.query] error during querying: ${err.message}`); + throw new Error(err.message); + }); } /** @@ -81,12 +83,14 @@ class ES { 'Invalid es index or es type name', ); } - const allESFields = _.flattenDeep(this.getESFields(esIndex).fields.map((f) => { - if (f.nestedProps) { - return processNestedFieldNames(f); - } - return f.name; - })); + const allESFields = _.flattenDeep(this.getESFields(esIndex) + .fields + .map((f) => { + if (f.nestedProps) { + return processNestedFieldNames(f); + } + return f.name; + })); const fieldsNotBelong = _.difference(fields, allESFields); if (fieldsNotBelong.length > 0) { throw new CodedError( @@ -117,10 +121,11 @@ class ES { size: SCROLL_PAGE_SIZE, _source: fields, sort: sortStringList, - }).then((resp) => resp, (err) => { - log.error('[ES.query] error when query', err.message); - throw new Error(err.message); - }); + }) + .then((resp) => resp, (err) => { + log.error('[ES.query] error when query', err.message); + throw new Error(err.message); + }); currentBatch = res.body; log.debug('[ES scrollQuery] created scroll'); } else { // following batches @@ -160,16 +165,17 @@ class ES { return this.client.indices.getMapping({ index: esIndex, type: esType, - }).then((resp) => { - try { - const esIndexAlias = Object.keys(resp.body)[0]; - return resp.body[esIndexAlias].mappings[esType].properties; - } catch (err) { - throw new Error(`${errMsg}: ${err}`); - } - }, (err) => { - throw new Error(`${errMsg}: ${err.message}`); - }); + }) + .then((resp) => { + try { + const esIndexAlias = Object.keys(resp.body)[0]; + return resp.body[esIndexAlias].mappings[esType].properties; + } catch (err) { + throw new Error(`${errMsg}: ${err}`); + } + }, (err) => { + throw new Error(`${errMsg}: ${err.message}`); + }); } async _getMappingsForAllIndices() { @@ -181,7 +187,17 @@ class ES { log.info('[ES.initialize] getting mapping from elasticsearch...'); const promiseList = this.config.indices .map((cfg) => this._getESFieldsTypes(cfg.index, cfg.type) - .then((res) => ({ index: cfg.index, fieldTypes: res }))); + .then((res) => { + Object.keys(res).forEach((x) => { + if (res[x].enabled !== undefined && res[x].enabled === false) { + delete res[x]; + } + }); + return { + index: cfg.index, + fieldTypes: res, + }; + })); const resultList = await Promise.all(promiseList); log.info('[ES.initialize] got mapping from elasticsearch'); resultList.forEach((res) => { @@ -215,34 +231,35 @@ class ES { }, }, }, - }).then((resp) => { - try { - resp.body.hits.hits.forEach((doc) => { - const index = doc._id; - if (!this.fieldTypes[index]) { - const errMsg = `[ES.initialize] wrong array entry from config index: index "${index}" not found, skipped.`; - log.error(errMsg); - return; - } - const fields = doc._source.array; - fields.forEach((field) => { - if (!this.fieldTypes[index][field]) { - const errMsg = `[ES.initialize] wrong array entry from config: field "${field}" not found in index ${index}, skipped.`; + }) + .then((resp) => { + try { + resp.body.hits.hits.forEach((doc) => { + const index = doc._id; + if (!this.fieldTypes[index]) { + const errMsg = `[ES.initialize] wrong array entry from config index: index "${index}" not found, skipped.`; log.error(errMsg); return; } - if (!arrayFields[index]) arrayFields[index] = []; - arrayFields[index].push(field); + const fields = doc._source.array; + fields.forEach((field) => { + if (!this.fieldTypes[index][field]) { + const errMsg = `[ES.initialize] wrong array entry from config: field "${field}" not found in index ${index}, skipped.`; + log.error(errMsg); + return; + } + if (!arrayFields[index]) arrayFields[index] = []; + arrayFields[index].push(field); + }); }); - }); - log.info('[ES.initialize] got array fields from es config index:', JSON.stringify(arrayFields, null, 4)); - } catch (err) { - throw new Error(err); - } - return arrayFields; - }, (err) => { - throw new Error(err.message); - }); + log.info('[ES.initialize] got array fields from es config index:', JSON.stringify(arrayFields, null, 4)); + } catch (err) { + throw new Error(err); + } + return arrayFields; + }, (err) => { + throw new Error(err.message); + }); } /** @@ -287,15 +304,19 @@ class ES { res[cfg.index] = { index: cfg.index, type: cfg.type, - fields: Object.entries(this.fieldTypes[cfg.index]).map(([key, value]) => { - let r; - if (value.type !== 'nested') { - r = { name: key, type: value.type }; - } else { - r = buildNestedField(key, value); - } - return r; - }), + fields: Object.entries(this.fieldTypes[cfg.index]) + .map(([key, value]) => { + let r; + if (value.type !== 'nested') { + r = { + name: key, + type: value.type + }; + } else { + r = buildNestedField(key, value); + } + return r; + }), }; }); if (typeof esIndex === 'undefined') { @@ -356,30 +377,31 @@ class ES { } return this.client.indices.getAlias({ index: indicesArray, - }).then((resp) => { - try { - const indicesMetadata = resp.body; - const indicesWithArrayFields = Object.keys(this.arrayFields); - for (let i = 0; i < indicesWithArrayFields.length; i += 1) { - const indexName = indicesWithArrayFields[i]; - if (!indicesMetadata[indexName]) { - indicesMetadata[indexName] = {}; + }) + .then((resp) => { + try { + const indicesMetadata = resp.body; + const indicesWithArrayFields = Object.keys(this.arrayFields); + for (let i = 0; i < indicesWithArrayFields.length; i += 1) { + const indexName = indicesWithArrayFields[i]; + if (!indicesMetadata[indexName]) { + indicesMetadata[indexName] = {}; + } + indicesMetadata[indexName].arrayFields = this.arrayFields[indexName]; } - indicesMetadata[indexName].arrayFields = this.arrayFields[indexName]; + return { + statusCode: resp.statusCode, + warnings: resp.warnings, + indices: { + ...indicesMetadata, + }, + }; + } catch (err) { + throw new Error(err); } - return { - statusCode: resp.statusCode, - warnings: resp.warnings, - indices: { - ...indicesMetadata, - }, - }; - } catch (err) { + }, (err) => { throw new Error(err); - } - }, (err) => { - throw new Error(err); - }); + }); } /** @@ -390,9 +412,16 @@ class ES { } filterData( - { esIndex, esType }, { - filter, fields, sort, offset = 0, size, + esIndex, + esType + }, + { + filter, + fields, + sort, + offset = 0, + size, }, ) { const queryBody = { from: offset }; @@ -423,7 +452,13 @@ class ES { } async getData({ - esIndex, esType, fields, filter, sort, offset, size, + esIndex, + esType, + fields, + filter, + sort, + offset, + size, }) { if (typeof size !== 'undefined' && offset + size > SCROLL_PAGE_SIZE) { throw new UserInputError(`Large graphql query forbidden for offset + size > ${SCROLL_PAGE_SIZE}, @@ -431,9 +466,17 @@ class ES { please use download endpoint for large data queries instead.`); } const result = await this.filterData( - { esInstance: this, esIndex, esType }, { - filter, fields, sort, offset, size, + esInstance: this, + esIndex, + esType + }, + { + filter, + fields, + sort, + offset, + size, }, ); const { hits } = result.hits; @@ -443,19 +486,20 @@ class ES { return h._source; } // ES returns highlight, transfer them into "_matched" schema - const matchedList = Object.keys(h.highlight).map((f) => { - let field = f; - if (f.endsWith(config.analyzedTextFieldSuffix)) { - // remove ".analyzed" suffix from field name - field = f.substr(0, f.length - config.analyzedTextFieldSuffix.length); - } - return { - field, - // just use ES highlights' format, - // should be a list of string, with matched part emphasized with < - highlights: h.highlight[f], - }; - }); + const matchedList = Object.keys(h.highlight) + .map((f) => { + let field = f; + if (f.endsWith(config.analyzedTextFieldSuffix)) { + // remove ".analyzed" suffix from field name + field = f.substr(0, f.length - config.analyzedTextFieldSuffix.length); + } + return { + field, + // just use ES highlights' format, + // should be a list of string, with matched part emphasized with < + highlights: h.highlight[f], + }; + }); return { ...h._source, _matched: matchedList, @@ -465,7 +509,11 @@ class ES { } downloadData({ - esIndex, esType, fields, filter, sort, + esIndex, + esType, + fields, + filter, + sort, }) { const esFilterObj = filter ? getFilterObj(this, esIndex, filter) : undefined; return this.scrollQuery(esIndex, esType, { From 779026b14e60e3ca88f9add465c702f626f2e834 Mon Sep 17 00:00:00 2001 From: Craig Barnes Date: Wed, 6 Oct 2021 15:14:43 -0500 Subject: [PATCH 02/27] add replace feild starting with __ to x__ --- src/server/es/index.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index 489d25b20..9b2a51d94 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -188,11 +188,15 @@ class ES { const promiseList = this.config.indices .map((cfg) => this._getESFieldsTypes(cfg.index, cfg.type) .then((res) => { - Object.keys(res).forEach((x) => { - if (res[x].enabled !== undefined && res[x].enabled === false) { - delete res[x]; - } - }); + Object.keys(res) + .forEach((fieldName) => { + if (res[fieldName].enabled !== undefined && res[fieldName].enabled === false) { + delete res[fieldName]; + } + if (fieldName in res && fieldName.indexOf('__') === 0) { + delete Object.assign(res, { [fieldName.replace('__', 'x__')]: res[fieldName] })[fieldName]; + } + }); return { index: cfg.index, fieldTypes: res, From 314881ddcb0d1e6cd65342f78ce487ba0dbd2391 Mon Sep 17 00:00:00 2001 From: Craig Barnes Date: Thu, 21 Oct 2021 23:24:45 -0500 Subject: [PATCH 03/27] add ignore and remap __ to x__ --- src/server/es/index.js | 14 +++++++++++--- src/server/resolvers.js | 4 +++- src/server/server.js | 3 +++ src/server/utils/utils.js | 2 ++ 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index 9b2a51d94..c121b0472 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -247,13 +247,14 @@ class ES { } const fields = doc._source.array; fields.forEach((field) => { - if (!this.fieldTypes[index][field]) { - const errMsg = `[ES.initialize] wrong array entry from config: field "${field}" not found in index ${index}, skipped.`; + const fn = (field.indexOf('__') === 0) ? field.replace('__', 'x__') : field; + if (!this.fieldTypes[index][fn]) { + const errMsg = `[ES.initialize] wrong array entry from config: field "${fn}" not found in index ${index}, skipped.`; log.error(errMsg); return; } if (!arrayFields[index]) arrayFields[index] = []; - arrayFields[index].push(field); + arrayFields[index].push(fn); }); }); log.info('[ES.initialize] got array fields from es config index:', JSON.stringify(arrayFields, null, 4)); @@ -485,6 +486,13 @@ class ES { ); const { hits } = result.hits; const hitsWithMatchedResults = hits.map((h) => { + Object.keys(h._source) + .forEach((fieldName) => { + if (fieldName in h._source && fieldName.indexOf('__') === 0) { + delete Object.assign(h._source, { [fieldName.replace('__', 'x__')]: h._source[fieldName] })[fieldName]; + } + }); + if (!('highlight' in h)) { // ES doesn't returns "highlight" return h._source; diff --git a/src/server/resolvers.js b/src/server/resolvers.js index e9d2be4b0..e41e0e00d 100644 --- a/src/server/resolvers.js +++ b/src/server/resolvers.js @@ -16,9 +16,11 @@ const typeQueryResolver = (esInstance, esIndex, esType) => (parent, args, contex offset, first, filter, sort, format, } = args; const fields = parseResolveInfo(resolveInfo); - return esInstance.getData({ + + const results = esInstance.getData({ esIndex, esType, fields, filter, sort, offset, size: first, format, }); + return results; }; /** diff --git a/src/server/server.js b/src/server/server.js index 526ab9f02..493aeba50 100644 --- a/src/server/server.js +++ b/src/server/server.js @@ -34,8 +34,11 @@ const startServer = () => { // create graphql server instance const server = new ApolloServer({ mocks: false, + debug: true, schema: schemaWithMiddleware, context: async ({ req }) => { + console.log(req.body.query); + console.log(req.body.variables); const jwt = headerParser.parseJWT(req); const authHelper = await getAuthHelperInstance(jwt); return { diff --git a/src/server/utils/utils.js b/src/server/utils/utils.js index 65a36f8f1..afbef3636 100644 --- a/src/server/utils/utils.js +++ b/src/server/utils/utils.js @@ -60,8 +60,10 @@ export const fromFieldsToSource = (parsedInfo) => { curNodeName = curNodeName.slice(0, (lastPeriod !== -1) ? lastPeriod : 0); } else { const cur = stack.pop(); + cur.name = ( cur.name.indexOf('x__') === 0) ? cur.name.replace('x__', '__') : cur.name; const newTypeName = cur.name; const fieldName = [curNodeName, newTypeName].filter((s) => s.length > 0).join('.'); + if (newTypeName in cur.fieldsByTypeName) { const children = Object.values(cur.fieldsByTypeName[newTypeName]); curNodeName = fieldName; From bdf647df3d7391b9aa3406d5b434de8e6cabf6b0 Mon Sep 17 00:00:00 2001 From: Craig Barnes Date: Fri, 28 Jan 2022 15:41:32 -0600 Subject: [PATCH 04/27] add support for fields with starting with __ --- src/server/config.js | 9 +- src/server/es/index.js | 226 +++++++++++++++++------------------------ 2 files changed, 101 insertions(+), 134 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index 894e2a108..5dcb35f12 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -28,16 +28,17 @@ const config = { }, port: 80, path: '/graphql', - arboristEndpoint: 'http://arborist-service', + arboristEndpoint: 'mock', tierAccessLevel: 'private', tierAccessLimit: 1000, tierAccessSensitiveRecordExclusionField: inputConfig.tier_access_sensitive_record_exclusion_field, - logLevel: inputConfig.log_level || 'INFO', + logLevel: inputConfig.log_level || 'DEBUG', enableEncryptWhiteList: typeof inputConfig.enable_encrypt_whitelist === 'undefined' ? false : inputConfig.enable_encrypt_whitelist, encryptWhitelist: inputConfig.encrypt_whitelist || ['__missing__', 'unknown', 'not reported', 'no data'], analyzedTextFieldSuffix: '.analyzed', matchedTextHighlightTagName: 'em', allowedMinimumSearchLen: 2, + doubleUnderscorePrefix: 'x__', }; if (process.env.GEN3_ES_ENDPOINT) { @@ -55,6 +56,10 @@ if (process.env.GUPPY_PORT) { config.port = process.env.GUPPY_PORT; } +if (process.env.DOUBLE_UNDERSCORE) { + config.doubleUnderscorePrefix = process.env.DOUBLE_UNDERSCORE; +} + const allowedTierAccessLevels = ['private', 'regular', 'libre']; if (process.env.TIER_ACCESS_LEVEL) { diff --git a/src/server/es/index.js b/src/server/es/index.js index c121b0472..320eaf887 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -8,7 +8,7 @@ import * as esAggregator from './aggs'; import log from '../logger'; import { SCROLL_PAGE_SIZE } from './const'; import CodedError from '../utils/error'; -import { fromFieldsToSource, buildNestedField, processNestedFieldNames } from '../utils/utils'; +import { buildNestedField, fromFieldsToSource, processNestedFieldNames } from '../utils/utils'; class ES { constructor(esConfig = config.esConfig) { @@ -36,12 +36,11 @@ class ES { */ async query(esIndex, esType, queryBody) { const validatedQueryBody = {}; - Object.keys(queryBody) - .forEach((key) => { - if (typeof queryBody[key] !== 'undefined' && queryBody[key] !== null) { - validatedQueryBody[key] = queryBody[key]; - } - }); + Object.keys(queryBody).forEach((key) => { + if (typeof queryBody[key] !== 'undefined' && queryBody[key] !== null) { + validatedQueryBody[key] = queryBody[key]; + } + }); validatedQueryBody.highlight = { pre_tags: [ `<${config.matchedTextHighlightTagName}>`, @@ -58,11 +57,10 @@ class ES { index: esIndex, type: esType, body: validatedQueryBody, - }) - .then((resp) => resp.body, (err) => { - log.error(`[ES.query] error during querying: ${err.message}`); - throw new Error(err.message); - }); + }).then((resp) => resp.body, (err) => { + log.error(`[ES.query] error during querying: ${err.message}`); + throw new Error(err.message); + }); } /** @@ -83,14 +81,12 @@ class ES { 'Invalid es index or es type name', ); } - const allESFields = _.flattenDeep(this.getESFields(esIndex) - .fields - .map((f) => { - if (f.nestedProps) { - return processNestedFieldNames(f); - } - return f.name; - })); + const allESFields = _.flattenDeep(this.getESFields(esIndex).fields.map((f) => { + if (f.nestedProps) { + return processNestedFieldNames(f); + } + return f.name; + })); const fieldsNotBelong = _.difference(fields, allESFields); if (fieldsNotBelong.length > 0) { throw new CodedError( @@ -121,11 +117,10 @@ class ES { size: SCROLL_PAGE_SIZE, _source: fields, sort: sortStringList, - }) - .then((resp) => resp, (err) => { - log.error('[ES.query] error when query', err.message); - throw new Error(err.message); - }); + }).then((resp) => resp, (err) => { + log.error('[ES.query] error when query', err.message); + throw new Error(err.message); + }); currentBatch = res.body; log.debug('[ES scrollQuery] created scroll'); } else { // following batches @@ -165,17 +160,16 @@ class ES { return this.client.indices.getMapping({ index: esIndex, type: esType, - }) - .then((resp) => { - try { - const esIndexAlias = Object.keys(resp.body)[0]; - return resp.body[esIndexAlias].mappings[esType].properties; - } catch (err) { - throw new Error(`${errMsg}: ${err}`); - } - }, (err) => { - throw new Error(`${errMsg}: ${err.message}`); - }); + }).then((resp) => { + try { + const esIndexAlias = Object.keys(resp.body)[0]; + return resp.body[esIndexAlias].mappings[esType].properties; + } catch (err) { + throw new Error(`${errMsg}: ${err}`); + } + }, (err) => { + throw new Error(`${errMsg}: ${err.message}`); + }); } async _getMappingsForAllIndices() { @@ -194,7 +188,7 @@ class ES { delete res[fieldName]; } if (fieldName in res && fieldName.indexOf('__') === 0) { - delete Object.assign(res, { [fieldName.replace('__', 'x__')]: res[fieldName] })[fieldName]; + delete Object.assign(res, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: res[fieldName] })[fieldName]; } }); return { @@ -235,36 +229,35 @@ class ES { }, }, }, - }) - .then((resp) => { - try { - resp.body.hits.hits.forEach((doc) => { - const index = doc._id; - if (!this.fieldTypes[index]) { - const errMsg = `[ES.initialize] wrong array entry from config index: index "${index}" not found, skipped.`; + }).then((resp) => { + try { + resp.body.hits.hits.forEach((doc) => { + const index = doc._id; + if (!this.fieldTypes[index]) { + const errMsg = `[ES.initialize] wrong array entry from config index: index "${index}" not found, skipped.`; + log.error(errMsg); + return; + } + const fields = doc._source.array; + fields.forEach((field) => { + const fn = (field.indexOf('__') === 0) ? field.replace('__', config.doubleUnderscorePrefix) : field; + if (!this.fieldTypes[index][fn]) { + const errMsg = `[ES.initialize] wrong array entry from config: field "${fn}" not found in index ${index}, skipped.`; log.error(errMsg); return; } - const fields = doc._source.array; - fields.forEach((field) => { - const fn = (field.indexOf('__') === 0) ? field.replace('__', 'x__') : field; - if (!this.fieldTypes[index][fn]) { - const errMsg = `[ES.initialize] wrong array entry from config: field "${fn}" not found in index ${index}, skipped.`; - log.error(errMsg); - return; - } - if (!arrayFields[index]) arrayFields[index] = []; - arrayFields[index].push(fn); - }); + if (!arrayFields[index]) arrayFields[index] = []; + arrayFields[index].push(fn); }); - log.info('[ES.initialize] got array fields from es config index:', JSON.stringify(arrayFields, null, 4)); - } catch (err) { - throw new Error(err); - } - return arrayFields; - }, (err) => { - throw new Error(err.message); - }); + }); + log.info('[ES.initialize] got array fields from es config index:', JSON.stringify(arrayFields, null, 4)); + } catch (err) { + throw new Error(err); + } + return arrayFields; + }, (err) => { + throw new Error(err.message); + }); } /** @@ -309,19 +302,15 @@ class ES { res[cfg.index] = { index: cfg.index, type: cfg.type, - fields: Object.entries(this.fieldTypes[cfg.index]) - .map(([key, value]) => { - let r; - if (value.type !== 'nested') { - r = { - name: key, - type: value.type - }; - } else { - r = buildNestedField(key, value); - } - return r; - }), + fields: Object.entries(this.fieldTypes[cfg.index]).map(([key, value]) => { + let r; + if (value.type !== 'nested') { + r = { name: key, type: value.type }; + } else { + r = buildNestedField(key, value); + } + return r; + }), }; }); if (typeof esIndex === 'undefined') { @@ -382,31 +371,30 @@ class ES { } return this.client.indices.getAlias({ index: indicesArray, - }) - .then((resp) => { - try { - const indicesMetadata = resp.body; - const indicesWithArrayFields = Object.keys(this.arrayFields); - for (let i = 0; i < indicesWithArrayFields.length; i += 1) { - const indexName = indicesWithArrayFields[i]; - if (!indicesMetadata[indexName]) { - indicesMetadata[indexName] = {}; - } - indicesMetadata[indexName].arrayFields = this.arrayFields[indexName]; + }).then((resp) => { + try { + const indicesMetadata = resp.body; + const indicesWithArrayFields = Object.keys(this.arrayFields); + for (let i = 0; i < indicesWithArrayFields.length; i += 1) { + const indexName = indicesWithArrayFields[i]; + if (!indicesMetadata[indexName]) { + indicesMetadata[indexName] = {}; } - return { - statusCode: resp.statusCode, - warnings: resp.warnings, - indices: { - ...indicesMetadata, - }, - }; - } catch (err) { - throw new Error(err); + indicesMetadata[indexName].arrayFields = this.arrayFields[indexName]; } - }, (err) => { + return { + statusCode: resp.statusCode, + warnings: resp.warnings, + indices: { + ...indicesMetadata, + }, + }; + } catch (err) { throw new Error(err); - }); + } + }, (err) => { + throw new Error(err); + }); } /** @@ -417,16 +405,9 @@ class ES { } filterData( + { esIndex, esType }, { - esIndex, - esType - }, - { - filter, - fields, - sort, - offset = 0, - size, + filter, fields, sort, offset = 0, size, }, ) { const queryBody = { from: offset }; @@ -457,13 +438,7 @@ class ES { } async getData({ - esIndex, - esType, - fields, - filter, - sort, - offset, - size, + esIndex, esType, fields, filter, sort, offset, size, }) { if (typeof size !== 'undefined' && offset + size > SCROLL_PAGE_SIZE) { throw new UserInputError(`Large graphql query forbidden for offset + size > ${SCROLL_PAGE_SIZE}, @@ -471,25 +446,17 @@ class ES { please use download endpoint for large data queries instead.`); } const result = await this.filterData( + { esInstance: this, esIndex, esType }, { - esInstance: this, - esIndex, - esType - }, - { - filter, - fields, - sort, - offset, - size, + filter, fields, sort, offset, size, }, ); const { hits } = result.hits; - const hitsWithMatchedResults = hits.map((h) => { + return hits.map((h) => { Object.keys(h._source) .forEach((fieldName) => { if (fieldName in h._source && fieldName.indexOf('__') === 0) { - delete Object.assign(h._source, { [fieldName.replace('__', 'x__')]: h._source[fieldName] })[fieldName]; + delete Object.assign(h._source, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: h._source[fieldName] })[fieldName]; } }); @@ -517,15 +484,10 @@ class ES { _matched: matchedList, }; }); - return hitsWithMatchedResults; } downloadData({ - esIndex, - esType, - fields, - filter, - sort, + esIndex, esType, fields, filter, sort, }) { const esFilterObj = filter ? getFilterObj(this, esIndex, filter) : undefined; return this.scrollQuery(esIndex, esType, { From 9f3ad6c8f7d7fa0b494e116fbf4a0ebebc9d6650 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 9 Jan 2024 14:33:39 -0600 Subject: [PATCH 05/27] fix test error --- src/server/es/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server/es/index.js b/src/server/es/index.js index 92006a0ad..3082fb10e 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -539,6 +539,7 @@ class ES { delete Object.assign(h._source, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: h._source[fieldName] })[fieldName]; } }); + }); return hitsWithNoDoubleUnderscore; } From 6892308fc3452549454088d8580ae305e9858fae Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 9 Jan 2024 15:16:05 -0600 Subject: [PATCH 06/27] fix lint error --- package-lock.json | 14 +++++++------- package.json | 2 +- src/server/es/index.js | 18 +++++++----------- src/server/utils/utils.js | 2 +- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index cb82a253e..e06386b20 100644 --- a/package-lock.json +++ b/package-lock.json @@ -60,7 +60,7 @@ "eslint-plugin-import": "^2.27.5", "eslint-plugin-jsx-a11y": "^6.7.1", "eslint-plugin-react": "^7.32.2", - "eslint-plugin-storybook": "^0.6.12", + "eslint-plugin-storybook": "^0.6.15", "jest": "^29.6.1", "json-schema-faker": "^0.5.3", "nock": "^13.3.1", @@ -13284,9 +13284,9 @@ } }, "node_modules/eslint-plugin-storybook": { - "version": "0.6.12", - "resolved": "https://registry.npmjs.org/eslint-plugin-storybook/-/eslint-plugin-storybook-0.6.12.tgz", - "integrity": "sha512-XbIvrq6hNVG6rpdBr+eBw63QhOMLpZneQVSooEDow8aQCWGCk/5vqtap1yxpVydNfSxi3S/3mBBRLQqKUqQRww==", + "version": "0.6.15", + "resolved": "https://registry.npmjs.org/eslint-plugin-storybook/-/eslint-plugin-storybook-0.6.15.tgz", + "integrity": "sha512-lAGqVAJGob47Griu29KXYowI4G7KwMoJDOkEip8ujikuDLxU+oWJ1l0WL6F2oDO4QiyUFXvtDkEkISMOPzo+7w==", "dev": true, "dependencies": { "@storybook/csf": "^0.0.1", @@ -33414,9 +33414,9 @@ "requires": {} }, "eslint-plugin-storybook": { - "version": "0.6.12", - "resolved": "https://registry.npmjs.org/eslint-plugin-storybook/-/eslint-plugin-storybook-0.6.12.tgz", - "integrity": "sha512-XbIvrq6hNVG6rpdBr+eBw63QhOMLpZneQVSooEDow8aQCWGCk/5vqtap1yxpVydNfSxi3S/3mBBRLQqKUqQRww==", + "version": "0.6.15", + "resolved": "https://registry.npmjs.org/eslint-plugin-storybook/-/eslint-plugin-storybook-0.6.15.tgz", + "integrity": "sha512-lAGqVAJGob47Griu29KXYowI4G7KwMoJDOkEip8ujikuDLxU+oWJ1l0WL6F2oDO4QiyUFXvtDkEkISMOPzo+7w==", "dev": true, "requires": { "@storybook/csf": "^0.0.1", diff --git a/package.json b/package.json index b3c7fa8b3..7582c45d4 100644 --- a/package.json +++ b/package.json @@ -78,7 +78,7 @@ "eslint-plugin-import": "^2.27.5", "eslint-plugin-jsx-a11y": "^6.7.1", "eslint-plugin-react": "^7.32.2", - "eslint-plugin-storybook": "^0.6.12", + "eslint-plugin-storybook": "^0.6.15", "jest": "^29.6.1", "json-schema-faker": "^0.5.3", "nock": "^13.3.1", diff --git a/src/server/es/index.js b/src/server/es/index.js index 3082fb10e..d8d960054 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -508,6 +508,12 @@ class ES { ); const { hits } = result.hits; const hitsWithMatchedResults = hits.map((h) => { + Object.keys(h._source) + .forEach((fieldName) => { + if (fieldName in h._source && fieldName.indexOf('__') === 0) { + delete Object.assign(h._source, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: h._source[fieldName] })[fieldName]; + } + }); if (!('highlight' in h)) { // ES doesn't returns "highlight" return h._source; @@ -531,17 +537,7 @@ class ES { _matched: matchedList, }; }); - - const hitsWithNoDoubleUnderscore = hitsWithMatchedResults.map((h) => { - Object.keys(h._source) - .forEach((fieldName) => { - if (fieldName in h._source && fieldName.indexOf('__') === 0) { - delete Object.assign(h._source, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: h._source[fieldName] })[fieldName]; - } - }); - }); - - return hitsWithNoDoubleUnderscore; + return hitsWithMatchedResults; } downloadData({ diff --git a/src/server/utils/utils.js b/src/server/utils/utils.js index afbef3636..c87d23f5d 100644 --- a/src/server/utils/utils.js +++ b/src/server/utils/utils.js @@ -60,7 +60,7 @@ export const fromFieldsToSource = (parsedInfo) => { curNodeName = curNodeName.slice(0, (lastPeriod !== -1) ? lastPeriod : 0); } else { const cur = stack.pop(); - cur.name = ( cur.name.indexOf('x__') === 0) ? cur.name.replace('x__', '__') : cur.name; + cur.name = (cur.name.indexOf('x__') === 0) ? cur.name.replace('x__', '__') : cur.name; const newTypeName = cur.name; const fieldName = [curNodeName, newTypeName].filter((s) => s.length > 0).join('.'); From 6cf3dd290b2958df7020c47f5fa4a81fe55692cf Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 9 Jan 2024 16:10:23 -0600 Subject: [PATCH 07/27] hack root for field mapping --- src/server/config.js | 10 +++------- src/server/es/index.js | 11 +++++++---- src/server/resolvers.js | 4 +--- src/server/utils/utils.js | 1 - 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index 5dcb35f12..ba1888578 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -13,15 +13,11 @@ const config = { host: 'localhost:9200', indices: inputConfig.indices || [ { - index: 'gen3-dev-subject', - type: 'subject', - }, - { - index: 'gen3-dev-file', - type: 'file', + index: 'default-commons-index', + type: 'metadata', }, ], - configIndex: (inputConfig.indices) ? inputConfig.config_index : 'gen3-dev-config', + configIndex: (inputConfig.indices) ? inputConfig.config_index : 'default-commons-config-index', authFilterField: inputConfig.auth_filter_field || 'auth_resource_path', aggregationIncludeMissingData: typeof inputConfig.aggs_include_missing_data === 'undefined' ? true : inputConfig.aggs_include_missing_data, missingDataAlias: inputConfig.missing_data_alias || 'no data', diff --git a/src/server/es/index.js b/src/server/es/index.js index d8d960054..9a75bba2c 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -180,13 +180,16 @@ class ES { const promiseList = this.config.indices .map((cfg) => this._getESFieldsTypes(cfg.index) .then((res) => { - Object.keys(res) + // remove disabled fields and convert double underscore prefix to single underscore + // set root to properties + const root = res[Object.keys(res)[0]].properties; + Object.keys(root) .forEach((fieldName) => { - if (res[fieldName].enabled !== undefined && res[fieldName].enabled === false) { + if (root[fieldName].enabled !== undefined && root[fieldName].enabled === false) { delete res[fieldName]; } - if (fieldName in res && fieldName.indexOf('__') === 0) { - delete Object.assign(res, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: res[fieldName] })[fieldName]; + if (fieldName in root && fieldName.indexOf('__') === 0) { + delete Object.assign(root, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: root[fieldName] })[fieldName]; } }); return { diff --git a/src/server/resolvers.js b/src/server/resolvers.js index bb402e281..b4cdf1287 100644 --- a/src/server/resolvers.js +++ b/src/server/resolvers.js @@ -16,11 +16,9 @@ const typeQueryResolver = (esInstance, esIndex, esType) => (parent, args, contex offset, first, filter, sort, format, } = args; const fields = parseResolveInfo(resolveInfo); - - const results = esInstance.getData({ + return esInstance.getData({ esIndex, esType, fields, filter, sort, offset, size: first, format, }); - return results; }; /** diff --git a/src/server/utils/utils.js b/src/server/utils/utils.js index c87d23f5d..b1a2bae5e 100644 --- a/src/server/utils/utils.js +++ b/src/server/utils/utils.js @@ -63,7 +63,6 @@ export const fromFieldsToSource = (parsedInfo) => { cur.name = (cur.name.indexOf('x__') === 0) ? cur.name.replace('x__', '__') : cur.name; const newTypeName = cur.name; const fieldName = [curNodeName, newTypeName].filter((s) => s.length > 0).join('.'); - if (newTypeName in cur.fieldsByTypeName) { const children = Object.values(cur.fieldsByTypeName[newTypeName]); curNodeName = fieldName; From 7a9fc70929687d60ebaeeeb2b24af55e68a1cf41 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 9 Jan 2024 16:14:58 -0600 Subject: [PATCH 08/27] hack root for field mapping: skip if root undefined --- src/server/es/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index 9a75bba2c..6ae1cb93d 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -183,7 +183,7 @@ class ES { // remove disabled fields and convert double underscore prefix to single underscore // set root to properties const root = res[Object.keys(res)[0]].properties; - Object.keys(root) + root && Object.keys(root) .forEach((fieldName) => { if (root[fieldName].enabled !== undefined && root[fieldName].enabled === false) { delete res[fieldName]; From 4b15186a89f72370532f57e9d5c75bd3df6cc88a Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 9 Jan 2024 16:15:57 -0600 Subject: [PATCH 09/27] hack root for field mapping: skip if root undefined --- src/server/es/index.js | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index 6ae1cb93d..c0cdeb5ad 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -183,15 +183,17 @@ class ES { // remove disabled fields and convert double underscore prefix to single underscore // set root to properties const root = res[Object.keys(res)[0]].properties; - root && Object.keys(root) - .forEach((fieldName) => { - if (root[fieldName].enabled !== undefined && root[fieldName].enabled === false) { - delete res[fieldName]; - } - if (fieldName in root && fieldName.indexOf('__') === 0) { - delete Object.assign(root, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: root[fieldName] })[fieldName]; - } - }); + if (root) { + Object.keys(root) + .forEach((fieldName) => { + if (root[fieldName].enabled !== undefined && root[fieldName].enabled === false) { + delete res[fieldName]; + } + if (fieldName in root && fieldName.indexOf('__') === 0) { + delete Object.assign(root, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: root[fieldName] })[fieldName]; + } + }); + } return { index: cfg.index, fieldTypes: res, From d2d474a29b5dfdff190aa4bd9428db15ab17bf18 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 23 Jan 2024 14:23:24 -0600 Subject: [PATCH 10/27] add support for removing fields from mapping. --- src/components/Utils/conversion.js | 2 +- src/server/config.js | 11 +++++++++- src/server/es/index.js | 10 ++++++++- stories/conf.js | 34 +++++++----------------------- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/components/Utils/conversion.js b/src/components/Utils/conversion.js index 615395193..16e69e2da 100644 --- a/src/components/Utils/conversion.js +++ b/src/components/Utils/conversion.js @@ -32,7 +32,7 @@ export async function conversion(json, config) { * @param {JSON} json * @param {string} format */ -export async function jsonToFormat(json, format) { +export async function jsonToFormat(json, format : keyof FILE_DELIMITERS ) { if (format in FILE_DELIMITERS) { const flatJson = await flattenJson(json); const data = await conversion(flatJson, { delimiter: FILE_DELIMITERS[format] }); diff --git a/src/server/config.js b/src/server/config.js index ba1888578..01b901846 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -22,7 +22,7 @@ const config = { aggregationIncludeMissingData: typeof inputConfig.aggs_include_missing_data === 'undefined' ? true : inputConfig.aggs_include_missing_data, missingDataAlias: inputConfig.missing_data_alias || 'no data', }, - port: 80, + port: 6966, path: '/graphql', arboristEndpoint: 'mock', tierAccessLevel: 'private', @@ -34,6 +34,7 @@ const config = { analyzedTextFieldSuffix: '.analyzed', matchedTextHighlightTagName: 'em', allowedMinimumSearchLen: 2, + ignoredFields: [], doubleUnderscorePrefix: 'x__', }; @@ -56,6 +57,14 @@ if (process.env.DOUBLE_UNDERSCORE) { config.doubleUnderscorePrefix = process.env.DOUBLE_UNDERSCORE; } +// comma separated string of fields to ignore +if (process.env.IGNORED_FIELDS) { + if (typeof process.env.IGNORED_FIELDS !== 'string') { + throw new Error('IGNORED_FIELDS must be a comma separated string'); + } + config.ignoredFields = process.env.IGNORED_FIELDS.split(','); +} + const allowedTierAccessLevels = ['private', 'regular', 'libre']; if (process.env.TIER_ACCESS_LEVEL) { diff --git a/src/server/es/index.js b/src/server/es/index.js index c0cdeb5ad..766c0dfd8 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -170,6 +170,11 @@ class ES { }); } + /** + * Gets the mappings for all indices from Elasticsearch. + * @returns {Promise} A promise that resolves to an object containing the field types for each index. + * @throws {Error} Throws an error if the "config.indices" block is empty. + */ async _getMappingsForAllIndices() { if (!this.config.indices || this.config.indices === 0) { const errMsg = '[ES.initialize] Error when initializing: empty "config.indices" block'; @@ -187,7 +192,10 @@ class ES { Object.keys(root) .forEach((fieldName) => { if (root[fieldName].enabled !== undefined && root[fieldName].enabled === false) { - delete res[fieldName]; + delete root[fieldName]; + } + if (fieldName in root && config.ignoredFields.includes(fieldName)) { + delete root[fieldName]; } if (fieldName in root && fieldName.indexOf('__') === 0) { delete Object.assign(root, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: root[fieldName] })[fieldName]; diff --git a/stories/conf.js b/stories/conf.js index adbdb4199..47b2fd599 100644 --- a/stories/conf.js +++ b/stories/conf.js @@ -2,55 +2,37 @@ export const filterConfig = { tabs: [{ title: 'Project', fields: [ - 'project', - 'study', - ], - asTextAggFields: [ - 'consortium_id', + 'gen3_discovery.primary_site', ], }, { title: 'Subject', fields: [ - 'race', - 'ethnicity', - 'gender', - 'vital_status', + 'gen3_discovery.primary_site', ], }, { title: 'File', fields: [ - 'file_count', - 'file_type', - 'file_format', + 'gen3_discovery.primary_site', ], }], }; export const tableConfig = [ - { field: 'project', name: 'Project' }, - { field: 'study', name: 'Study' }, - { field: 'race', name: 'Race' }, - { field: 'ethnicity', name: 'Ethnicity' }, - { field: 'gender', name: 'Gender' }, - { field: 'vital_status', name: 'Vital Status' }, - { field: 'whatever_lab_result_value', name: 'Lab Result Value' }, - { field: 'file_count', name: 'File Count' }, - { field: 'file_type', name: 'File Type' }, - { field: 'file_format', name: 'File Format' }, + { field: 'gen3_discovery.StudyInstanceUID', name: 'Project' }, ]; export const guppyConfig = { - path: 'http://localhost:3000', - type: 'subject', + path: 'http://localhost:6966', + type: 'metadata', fileType: 'file', tierAccessLimit: 20, }; export const fieldMapping = [ { - field: 'project', - name: 'Project Name', + field: 'gen3_discovery.primary_site', + name: 'Primary Site', }, ]; From 9cb3004dfc1fdabde2d407edfb762f2ed077d893 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 23 Jan 2024 14:51:22 -0600 Subject: [PATCH 11/27] remove typescript declaration --- src/components/Utils/conversion.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/Utils/conversion.js b/src/components/Utils/conversion.js index 16e69e2da..615395193 100644 --- a/src/components/Utils/conversion.js +++ b/src/components/Utils/conversion.js @@ -32,7 +32,7 @@ export async function conversion(json, config) { * @param {JSON} json * @param {string} format */ -export async function jsonToFormat(json, format : keyof FILE_DELIMITERS ) { +export async function jsonToFormat(json, format) { if (format in FILE_DELIMITERS) { const flatJson = await flattenJson(json); const data = await conversion(flatJson, { delimiter: FILE_DELIMITERS[format] }); From c40a61c2071fdd09a63cf5b3b8e29ac601abafdb Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Wed, 24 Jan 2024 14:49:52 -0600 Subject: [PATCH 12/27] refactor removedFields to correctly support removing fields --- src/server/es/index.js | 62 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index 766c0dfd8..f0b8e4884 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -170,11 +170,67 @@ class ES { }); } + async _getMappingsForAllIndices() { + if (!this.config.indices || this.config.indices === 0) { + const errMsg = '[ES.initialize] Error when initializing: empty "config.indices" block'; + throw new Error(errMsg); + } + const fieldTypes = {}; + log.info('[ES.initialize] getting mapping from elasticsearch...'); + + const promiseList = this.config.indices.map((indexConfig) => this._processEachIndex(indexConfig)); + + const resultList = await Promise.all(promiseList); + log.info('[ES.initialize] got mapping from elasticsearch'); + resultList.forEach((res) => { + fieldTypes[res.index] = res.fieldTypes; + }); + log.debug('[ES.initialize]', JSON.stringify(fieldTypes, null, 4)); + return fieldTypes; + } + + async _processEachIndex(indexConfig) { + const res = await this._getESFieldsTypes(indexConfig.index); + Object.keys(res).forEach((fieldName) => { + this._modifyIndexRootProperties(res); + if (res[fieldName] && 'properties' in res[fieldName] && res[fieldName].type === 'nested') { + const root = res[fieldName].properties; + this._modifyIndexRootProperties(root); + } + }); + + return { + index: indexConfig.index, + fieldTypes: res, + }; + } + + _modifyIndexRootProperties(root) { + const DOUBLE_UNDERSCORE = '__'; + + if (root) { + Object.keys(root).forEach((fieldName) => { + if (root[fieldName].enabled === false) { + // eslint-disable-next-line no-param-reassign + delete root[fieldName]; + } + if (root[fieldName] && config.ignoredFields.includes(fieldName)) { + // eslint-disable-next-line no-param-reassign + delete root[fieldName]; + } + if (root[fieldName] && fieldName.startsWith(DOUBLE_UNDERSCORE)) { + delete Object.assign(root, { [fieldName.replace(DOUBLE_UNDERSCORE, this.config.doubleUnderscorePrefix)]: root[fieldName] })[fieldName]; + } + }); + } + } + /** * Gets the mappings for all indices from Elasticsearch. * @returns {Promise} A promise that resolves to an object containing the field types for each index. * @throws {Error} Throws an error if the "config.indices" block is empty. */ + /**---- async _getMappingsForAllIndices() { if (!this.config.indices || this.config.indices === 0) { const errMsg = '[ES.initialize] Error when initializing: empty "config.indices" block'; @@ -187,14 +243,18 @@ class ES { .then((res) => { // remove disabled fields and convert double underscore prefix to single underscore // set root to properties + console.log("res...",res); const root = res[Object.keys(res)[0]].properties; + console.log("root", root, config.ignoredFields); if (root) { Object.keys(root) .forEach((fieldName) => { + console.log("field name", fieldName); if (root[fieldName].enabled !== undefined && root[fieldName].enabled === false) { delete root[fieldName]; } if (fieldName in root && config.ignoredFields.includes(fieldName)) { + console.log("deleting field", fieldName); delete root[fieldName]; } if (fieldName in root && fieldName.indexOf('__') === 0) { @@ -215,7 +275,7 @@ class ES { log.debug('[ES.initialize]', JSON.stringify(fieldTypes, null, 4)); return fieldTypes; } - +--- */ /** * Read array config and check if there's any array fields for each index. * Array fields are grouped and stored by index as a doc in array config, From a9d131d522f7c4634222a2f161c245d97f17a990 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Wed, 24 Jan 2024 15:11:43 -0600 Subject: [PATCH 13/27] changed defaults for BIH --- src/server/config.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index 01b901846..302137b41 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -22,19 +22,19 @@ const config = { aggregationIncludeMissingData: typeof inputConfig.aggs_include_missing_data === 'undefined' ? true : inputConfig.aggs_include_missing_data, missingDataAlias: inputConfig.missing_data_alias || 'no data', }, - port: 6966, + port: 80, path: '/graphql', - arboristEndpoint: 'mock', - tierAccessLevel: 'private', + arboristEndpoint: 'http://arborist-service', + tierAccessLevel: 'libre', tierAccessLimit: 1000, tierAccessSensitiveRecordExclusionField: inputConfig.tier_access_sensitive_record_exclusion_field, - logLevel: inputConfig.log_level || 'DEBUG', + logLevel: inputConfig.log_level || 'INFO', enableEncryptWhiteList: typeof inputConfig.enable_encrypt_whitelist === 'undefined' ? false : inputConfig.enable_encrypt_whitelist, encryptWhitelist: inputConfig.encrypt_whitelist || ['__missing__', 'unknown', 'not reported', 'no data'], analyzedTextFieldSuffix: '.analyzed', matchedTextHighlightTagName: 'em', allowedMinimumSearchLen: 2, - ignoredFields: [], + ignoredFields: ['@version'], doubleUnderscorePrefix: 'x__', }; From 42270a17c026aead514ba8de483beb2a5bb9faab Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Wed, 24 Jan 2024 15:23:12 -0600 Subject: [PATCH 14/27] update config for tests --- src/server/__mocks__/config.js | 2 ++ src/server/__tests__/config.test.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/__mocks__/config.js b/src/server/__mocks__/config.js index 4b66f9da4..12c1be473 100644 --- a/src/server/__mocks__/config.js +++ b/src/server/__mocks__/config.js @@ -24,6 +24,8 @@ const config = { arboristEndpoint: 'http://mock-arborist', analyzedTextFieldSuffix: '.analyzed', matchedTextHighlightTagName: 'em', + ignoredFields: ['@version'], + doubleUnderscorePrefix: 'x__', }; export default config; diff --git a/src/server/__tests__/config.test.js b/src/server/__tests__/config.test.js index be7f41996..ab8f04790 100644 --- a/src/server/__tests__/config.test.js +++ b/src/server/__tests__/config.test.js @@ -17,7 +17,7 @@ describe('config', () => { /* --------------- For tier access --------------- */ test('default tier access level should be private', async () => { const config = require('../config').default; - expect(config.tierAccessLevel).toEqual('private'); + expect(config.tierAccessLevel).toEqual('libre'); }); test('config for libre tier access level', async () => { From de9bf9fb30cf1d1cd3ee59d162fbeeca442f4eb1 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 30 Jan 2024 15:38:16 -0600 Subject: [PATCH 15/27] clean up code restore test to match master --- package-lock.json | 18 +++--- package.json | 4 +- src/components/ConnectedFilter/utils.js | 3 +- src/server/__tests__/config.test.js | 2 +- src/server/config.js | 12 ++-- src/server/es/index.js | 77 ++++++++----------------- stories/conf.js | 34 ++++++++--- 7 files changed, 72 insertions(+), 78 deletions(-) diff --git a/package-lock.json b/package-lock.json index e06386b20..6b9c08b44 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gen3/guppy", - "version": "0.18.0", + "version": "0.18.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@gen3/guppy", - "version": "0.18.0", + "version": "0.18.1", "license": "ISC", "dependencies": { "@apollo/server": "^4.9.4", @@ -60,7 +60,7 @@ "eslint-plugin-import": "^2.27.5", "eslint-plugin-jsx-a11y": "^6.7.1", "eslint-plugin-react": "^7.32.2", - "eslint-plugin-storybook": "^0.6.15", + "eslint-plugin-storybook": "^0.6.12", "jest": "^29.6.1", "json-schema-faker": "^0.5.3", "nock": "^13.3.1", @@ -13284,9 +13284,9 @@ } }, "node_modules/eslint-plugin-storybook": { - "version": "0.6.15", - "resolved": "https://registry.npmjs.org/eslint-plugin-storybook/-/eslint-plugin-storybook-0.6.15.tgz", - "integrity": "sha512-lAGqVAJGob47Griu29KXYowI4G7KwMoJDOkEip8ujikuDLxU+oWJ1l0WL6F2oDO4QiyUFXvtDkEkISMOPzo+7w==", + "version": "0.6.12", + "resolved": "https://registry.npmjs.org/eslint-plugin-storybook/-/eslint-plugin-storybook-0.6.12.tgz", + "integrity": "sha512-XbIvrq6hNVG6rpdBr+eBw63QhOMLpZneQVSooEDow8aQCWGCk/5vqtap1yxpVydNfSxi3S/3mBBRLQqKUqQRww==", "dev": true, "dependencies": { "@storybook/csf": "^0.0.1", @@ -33414,9 +33414,9 @@ "requires": {} }, "eslint-plugin-storybook": { - "version": "0.6.15", - "resolved": "https://registry.npmjs.org/eslint-plugin-storybook/-/eslint-plugin-storybook-0.6.15.tgz", - "integrity": "sha512-lAGqVAJGob47Griu29KXYowI4G7KwMoJDOkEip8ujikuDLxU+oWJ1l0WL6F2oDO4QiyUFXvtDkEkISMOPzo+7w==", + "version": "0.6.12", + "resolved": "https://registry.npmjs.org/eslint-plugin-storybook/-/eslint-plugin-storybook-0.6.12.tgz", + "integrity": "sha512-XbIvrq6hNVG6rpdBr+eBw63QhOMLpZneQVSooEDow8aQCWGCk/5vqtap1yxpVydNfSxi3S/3mBBRLQqKUqQRww==", "dev": true, "requires": { "@storybook/csf": "^0.0.1", diff --git a/package.json b/package.json index 7582c45d4..0e6a2d1e9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gen3/guppy", - "version": "0.18.0", + "version": "0.18.1", "description": "Server that support GraphQL queries on data from elasticsearch", "main": "src/server/server.js", "directories": { @@ -78,7 +78,7 @@ "eslint-plugin-import": "^2.27.5", "eslint-plugin-jsx-a11y": "^6.7.1", "eslint-plugin-react": "^7.32.2", - "eslint-plugin-storybook": "^0.6.15", + "eslint-plugin-storybook": "^0.6.12", "jest": "^29.6.1", "json-schema-faker": "^0.5.3", "nock": "^13.3.1", diff --git a/src/components/ConnectedFilter/utils.js b/src/components/ConnectedFilter/utils.js index 20d3ba152..806cd8326 100644 --- a/src/components/ConnectedFilter/utils.js +++ b/src/components/ConnectedFilter/utils.js @@ -44,7 +44,7 @@ const capitalizeFirstLetter = (str) => { // createSearchFilterLoadOptionsFn creates a handler function that loads the search filter's // autosuggest options as the user types in the search filter. -const createSearchFilterLoadOptionsFn = (field, guppyConfig) => (searchString, offset) => { +const createSearchFilterLoadOptionsFn = (field, guppyConfig, csrfToken) => (searchString, offset) => { const NUM_SEARCH_OPTIONS = 20; return new Promise((resolve, reject) => { // If searchString is empty return just the first NUM_SEARCH_OPTIONS options. @@ -69,6 +69,7 @@ const createSearchFilterLoadOptionsFn = (field, guppyConfig) => (searchString, o offset, NUM_SEARCH_OPTIONS, 'accessible', + csrfToken, ) .then((res) => { if (!res.data || !res.data[guppyConfig.type]) { diff --git a/src/server/__tests__/config.test.js b/src/server/__tests__/config.test.js index ab8f04790..be7f41996 100644 --- a/src/server/__tests__/config.test.js +++ b/src/server/__tests__/config.test.js @@ -17,7 +17,7 @@ describe('config', () => { /* --------------- For tier access --------------- */ test('default tier access level should be private', async () => { const config = require('../config').default; - expect(config.tierAccessLevel).toEqual('libre'); + expect(config.tierAccessLevel).toEqual('private'); }); test('config for libre tier access level', async () => { diff --git a/src/server/config.js b/src/server/config.js index 302137b41..9b92d00f3 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -13,11 +13,15 @@ const config = { host: 'localhost:9200', indices: inputConfig.indices || [ { - index: 'default-commons-index', - type: 'metadata', + index: 'gen3-dev-subject', + type: 'subject', + }, + { + index: 'gen3-dev-file', + type: 'file', }, ], - configIndex: (inputConfig.indices) ? inputConfig.config_index : 'default-commons-config-index', + configIndex: (inputConfig.indices) ? inputConfig.config_index : 'gen3-dev-config', authFilterField: inputConfig.auth_filter_field || 'auth_resource_path', aggregationIncludeMissingData: typeof inputConfig.aggs_include_missing_data === 'undefined' ? true : inputConfig.aggs_include_missing_data, missingDataAlias: inputConfig.missing_data_alias || 'no data', @@ -25,7 +29,7 @@ const config = { port: 80, path: '/graphql', arboristEndpoint: 'http://arborist-service', - tierAccessLevel: 'libre', + tierAccessLevel: 'private', tierAccessLimit: 1000, tierAccessSensitiveRecordExclusionField: inputConfig.tier_access_sensitive_record_exclusion_field, logLevel: inputConfig.log_level || 'INFO', diff --git a/src/server/es/index.js b/src/server/es/index.js index f0b8e4884..3849d207b 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -170,6 +170,11 @@ class ES { }); } + /** + * Gets the mappings for all indices from Elasticsearch. + * @returns {Promise} A promise that resolves to an object containing the field types for each index. + * @throws {Error} Throws an error if the "config.indices" block is empty. + */ async _getMappingsForAllIndices() { if (!this.config.indices || this.config.indices === 0) { const errMsg = '[ES.initialize] Error when initializing: empty "config.indices" block'; @@ -189,6 +194,14 @@ class ES { return fieldTypes; } + /** + * Processes each index configuration and retrieves the field types for the specified index. + * Modifies the index root properties and nested properties if necessary. + * + * @param {object} indexConfig - The index configuration object. + * @param {string} indexConfig.index - The index name. + * @returns {Promise} - A promise that resolves to an object containing the index name and field types. + */ async _processEachIndex(indexConfig) { const res = await this._getESFieldsTypes(indexConfig.index); Object.keys(res).forEach((fieldName) => { @@ -205,9 +218,17 @@ class ES { }; } + /** + * Modifies the properties of the index root object. + * This function has a side effect of modifying the index root object which + * is done to make the code more readable. + * It removes disabled fields, ignored fields, and converts double underscore prefix to single underscore. + * @param {object} root - The index root object to be modified. + */ _modifyIndexRootProperties(root) { const DOUBLE_UNDERSCORE = '__'; + // Changes root object by updating in place if (root) { Object.keys(root).forEach((fieldName) => { if (root[fieldName].enabled === false) { @@ -225,57 +246,6 @@ class ES { } } - /** - * Gets the mappings for all indices from Elasticsearch. - * @returns {Promise} A promise that resolves to an object containing the field types for each index. - * @throws {Error} Throws an error if the "config.indices" block is empty. - */ - /**---- - async _getMappingsForAllIndices() { - if (!this.config.indices || this.config.indices === 0) { - const errMsg = '[ES.initialize] Error when initializing: empty "config.indices" block'; - throw new Error(errMsg); - } - const fieldTypes = {}; - log.info('[ES.initialize] getting mapping from elasticsearch...'); - const promiseList = this.config.indices - .map((cfg) => this._getESFieldsTypes(cfg.index) - .then((res) => { - // remove disabled fields and convert double underscore prefix to single underscore - // set root to properties - console.log("res...",res); - const root = res[Object.keys(res)[0]].properties; - console.log("root", root, config.ignoredFields); - if (root) { - Object.keys(root) - .forEach((fieldName) => { - console.log("field name", fieldName); - if (root[fieldName].enabled !== undefined && root[fieldName].enabled === false) { - delete root[fieldName]; - } - if (fieldName in root && config.ignoredFields.includes(fieldName)) { - console.log("deleting field", fieldName); - delete root[fieldName]; - } - if (fieldName in root && fieldName.indexOf('__') === 0) { - delete Object.assign(root, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: root[fieldName] })[fieldName]; - } - }); - } - return { - index: cfg.index, - fieldTypes: res, - }; - })); - const resultList = await Promise.all(promiseList); - log.info('[ES.initialize] got mapping from elasticsearch'); - resultList.forEach((res) => { - fieldTypes[res.index] = res.fieldTypes; - }); - log.debug('[ES.initialize]', JSON.stringify(fieldTypes, null, 4)); - return fieldTypes; - } ---- */ /** * Read array config and check if there's any array fields for each index. * Array fields are grouped and stored by index as a doc in array config, @@ -564,8 +534,9 @@ class ES { async getData({ esIndex, esType, fields, filter, sort, offset, size, }) { - if (typeof size !== 'undefined' && offset + size > SCROLL_PAGE_SIZE) { - throw new GraphQLError(`Large graphql query forbidden for offset + size > ${SCROLL_PAGE_SIZE}, + // TODO: understand why it's offset + size > SCROLL_PAGE_SIZE instead of offset > SCROLL_PAGE_SIZE + if (typeof size !== 'undefined' && size > SCROLL_PAGE_SIZE) { + throw new GraphQLError(`Large graphql query forbidden for size > ${SCROLL_PAGE_SIZE}, offset = ${offset} and size = ${size}, please use download endpoint for large data queries instead.`, { extensions: { diff --git a/stories/conf.js b/stories/conf.js index 47b2fd599..adbdb4199 100644 --- a/stories/conf.js +++ b/stories/conf.js @@ -2,37 +2,55 @@ export const filterConfig = { tabs: [{ title: 'Project', fields: [ - 'gen3_discovery.primary_site', + 'project', + 'study', + ], + asTextAggFields: [ + 'consortium_id', ], }, { title: 'Subject', fields: [ - 'gen3_discovery.primary_site', + 'race', + 'ethnicity', + 'gender', + 'vital_status', ], }, { title: 'File', fields: [ - 'gen3_discovery.primary_site', + 'file_count', + 'file_type', + 'file_format', ], }], }; export const tableConfig = [ - { field: 'gen3_discovery.StudyInstanceUID', name: 'Project' }, + { field: 'project', name: 'Project' }, + { field: 'study', name: 'Study' }, + { field: 'race', name: 'Race' }, + { field: 'ethnicity', name: 'Ethnicity' }, + { field: 'gender', name: 'Gender' }, + { field: 'vital_status', name: 'Vital Status' }, + { field: 'whatever_lab_result_value', name: 'Lab Result Value' }, + { field: 'file_count', name: 'File Count' }, + { field: 'file_type', name: 'File Type' }, + { field: 'file_format', name: 'File Format' }, ]; export const guppyConfig = { - path: 'http://localhost:6966', - type: 'metadata', + path: 'http://localhost:3000', + type: 'subject', fileType: 'file', tierAccessLimit: 20, }; export const fieldMapping = [ { - field: 'gen3_discovery.primary_site', - name: 'Primary Site', + field: 'project', + name: 'Project Name', }, ]; From 10f16844ca935a31f0edd0445b014dfe450d0bc6 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 30 Jan 2024 15:55:49 -0600 Subject: [PATCH 16/27] restore config value --- src/server/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/config.js b/src/server/config.js index 9b92d00f3..5a1e29d5b 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -38,7 +38,7 @@ const config = { analyzedTextFieldSuffix: '.analyzed', matchedTextHighlightTagName: 'em', allowedMinimumSearchLen: 2, - ignoredFields: ['@version'], + ignoredFields: [], doubleUnderscorePrefix: 'x__', }; From 72fd38982cb098e2f0ef60856dc65b6bde94119a Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 30 Jan 2024 16:41:07 -0600 Subject: [PATCH 17/27] add doubleUnderscore support --- src/server/es/index.js | 3 ++- src/server/utils/__test__/utils.test.js | 2 +- src/server/utils/utils.js | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index 3849d207b..0507e4f03 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -460,6 +460,7 @@ class ES { filter, fields, sort, offset = 0, size, }, ) { + const queryBody = { from: offset }; if (typeof filter !== 'undefined') { queryBody.query = getFilterObj(this, esIndex, filter); @@ -470,7 +471,7 @@ class ES { } if (fields !== undefined) { if (fields) { - const esFields = fromFieldsToSource(fields); + const esFields = fromFieldsToSource(fields, this.config.doubleUnderscorePrefix); if (esFields.length > 0) queryBody._source = esFields; } else { queryBody._source = false; diff --git a/src/server/utils/__test__/utils.test.js b/src/server/utils/__test__/utils.test.js index f996d7c41..396774890 100644 --- a/src/server/utils/__test__/utils.test.js +++ b/src/server/utils/__test__/utils.test.js @@ -3,6 +3,6 @@ import UtilsData from '../__mockData__/utils.data'; describe('Parse fields from GraphQL query to fields in ES query', () => { test('could parse fields in GraphQL query correctly', async () => { - expect(fromFieldsToSource(UtilsData.parsedInfo)).toEqual(UtilsData.fields); + expect(fromFieldsToSource(UtilsData.parsedInfo, "x__")).toEqual(UtilsData.fields); }); }); diff --git a/src/server/utils/utils.js b/src/server/utils/utils.js index b1a2bae5e..3f6795966 100644 --- a/src/server/utils/utils.js +++ b/src/server/utils/utils.js @@ -46,7 +46,7 @@ export const isWhitelisted = (key) => { * @param parsedInfo: parsing information from graphql library * @returns: list of selected fields. */ -export const fromFieldsToSource = (parsedInfo) => { +export const fromFieldsToSource = (parsedInfo, underscorePrefix) => { let stack = Object.values(parsedInfo.fieldsByTypeName[firstLetterUpperCase(parsedInfo.name)]); const levels = { 0: stack.length }; const fields = []; @@ -60,7 +60,7 @@ export const fromFieldsToSource = (parsedInfo) => { curNodeName = curNodeName.slice(0, (lastPeriod !== -1) ? lastPeriod : 0); } else { const cur = stack.pop(); - cur.name = (cur.name.indexOf('x__') === 0) ? cur.name.replace('x__', '__') : cur.name; + cur.name = (cur.name.indexOf(underscorePrefix) === 0) ? cur.name.replace(underscorePrefix, '__') : cur.name; const newTypeName = cur.name; const fieldName = [curNodeName, newTypeName].filter((s) => s.length > 0).join('.'); if (newTypeName in cur.fieldsByTypeName) { From 1303042bc2a5fb542752dfe94837575403ecc348 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 30 Jan 2024 16:47:17 -0600 Subject: [PATCH 18/27] fix documentation for fromFieldsToSource --- src/server/utils/utils.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/server/utils/utils.js b/src/server/utils/utils.js index 3f6795966..1f7fa124c 100644 --- a/src/server/utils/utils.js +++ b/src/server/utils/utils.js @@ -43,7 +43,8 @@ export const isWhitelisted = (key) => { /** * Convert from fields of graphql query produced by graphql library to list of querying fields * This list will be put to _source fields of the ES query - * @param parsedInfo: parsing information from graphql library + * @param - parsedInfo: parsing information from graphql library + * @param - underscorePrefix: prefix to use for fields that start with __ * @returns: list of selected fields. */ export const fromFieldsToSource = (parsedInfo, underscorePrefix) => { From 906dc50e3b44001883bec10ddc80e2effe9e2cc9 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Fri, 2 Feb 2024 12:37:47 -0600 Subject: [PATCH 19/27] revert for Monday's demo --- src/server/__tests__/config.test.js | 2 +- src/server/config.js | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/server/__tests__/config.test.js b/src/server/__tests__/config.test.js index be7f41996..ab8f04790 100644 --- a/src/server/__tests__/config.test.js +++ b/src/server/__tests__/config.test.js @@ -17,7 +17,7 @@ describe('config', () => { /* --------------- For tier access --------------- */ test('default tier access level should be private', async () => { const config = require('../config').default; - expect(config.tierAccessLevel).toEqual('private'); + expect(config.tierAccessLevel).toEqual('libre'); }); test('config for libre tier access level', async () => { diff --git a/src/server/config.js b/src/server/config.js index 5a1e29d5b..302137b41 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -13,15 +13,11 @@ const config = { host: 'localhost:9200', indices: inputConfig.indices || [ { - index: 'gen3-dev-subject', - type: 'subject', - }, - { - index: 'gen3-dev-file', - type: 'file', + index: 'default-commons-index', + type: 'metadata', }, ], - configIndex: (inputConfig.indices) ? inputConfig.config_index : 'gen3-dev-config', + configIndex: (inputConfig.indices) ? inputConfig.config_index : 'default-commons-config-index', authFilterField: inputConfig.auth_filter_field || 'auth_resource_path', aggregationIncludeMissingData: typeof inputConfig.aggs_include_missing_data === 'undefined' ? true : inputConfig.aggs_include_missing_data, missingDataAlias: inputConfig.missing_data_alias || 'no data', @@ -29,7 +25,7 @@ const config = { port: 80, path: '/graphql', arboristEndpoint: 'http://arborist-service', - tierAccessLevel: 'private', + tierAccessLevel: 'libre', tierAccessLimit: 1000, tierAccessSensitiveRecordExclusionField: inputConfig.tier_access_sensitive_record_exclusion_field, logLevel: inputConfig.log_level || 'INFO', @@ -38,7 +34,7 @@ const config = { analyzedTextFieldSuffix: '.analyzed', matchedTextHighlightTagName: 'em', allowedMinimumSearchLen: 2, - ignoredFields: [], + ignoredFields: ['@version'], doubleUnderscorePrefix: 'x__', }; From 414cf01f13a26d83c9a3a5d765af76be2c4790e1 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 6 Feb 2024 16:31:55 -0600 Subject: [PATCH 20/27] fix path issues with prefix mapping --- src/server/es/index.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index 0507e4f03..e963f853b 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -226,7 +226,6 @@ class ES { * @param {object} root - The index root object to be modified. */ _modifyIndexRootProperties(root) { - const DOUBLE_UNDERSCORE = '__'; // Changes root object by updating in place if (root) { @@ -239,8 +238,8 @@ class ES { // eslint-disable-next-line no-param-reassign delete root[fieldName]; } - if (root[fieldName] && fieldName.startsWith(DOUBLE_UNDERSCORE)) { - delete Object.assign(root, { [fieldName.replace(DOUBLE_UNDERSCORE, this.config.doubleUnderscorePrefix)]: root[fieldName] })[fieldName]; + if (root[fieldName] && fieldName.startsWith('__')) { + delete Object.assign(root, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: root[fieldName] })[fieldName]; } }); } @@ -471,7 +470,7 @@ class ES { } if (fields !== undefined) { if (fields) { - const esFields = fromFieldsToSource(fields, this.config.doubleUnderscorePrefix); + const esFields = fromFieldsToSource(fields, config.doubleUnderscorePrefix); if (esFields.length > 0) queryBody._source = esFields; } else { queryBody._source = false; From 0fe8c3182bd6465534b17cd2b9f2425544ca3f89 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Thu, 28 Mar 2024 16:39:00 -0500 Subject: [PATCH 21/27] update comment and restore default value of accessLevel --- src/server/es/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index e963f853b..2e7a36f07 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -459,7 +459,6 @@ class ES { filter, fields, sort, offset = 0, size, }, ) { - const queryBody = { from: offset }; if (typeof filter !== 'undefined') { queryBody.query = getFilterObj(this, esIndex, filter); @@ -559,7 +558,7 @@ class ES { } }); if (!('highlight' in h)) { - // ES doesn't returns "highlight" + // ES doesn't return "highlight" return h._source; } // ES returns highlight, transfer them into "_matched" schema From d1f8d61f82310450921bfc51a335d255b6f65ce2 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 2 Apr 2024 11:37:25 -0500 Subject: [PATCH 22/27] refine config --- src/server/__tests__/config.test.js | 2 +- src/server/config.js | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/server/__tests__/config.test.js b/src/server/__tests__/config.test.js index ab8f04790..be7f41996 100644 --- a/src/server/__tests__/config.test.js +++ b/src/server/__tests__/config.test.js @@ -17,7 +17,7 @@ describe('config', () => { /* --------------- For tier access --------------- */ test('default tier access level should be private', async () => { const config = require('../config').default; - expect(config.tierAccessLevel).toEqual('libre'); + expect(config.tierAccessLevel).toEqual('private'); }); test('config for libre tier access level', async () => { diff --git a/src/server/config.js b/src/server/config.js index 302137b41..9b92d00f3 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -13,11 +13,15 @@ const config = { host: 'localhost:9200', indices: inputConfig.indices || [ { - index: 'default-commons-index', - type: 'metadata', + index: 'gen3-dev-subject', + type: 'subject', + }, + { + index: 'gen3-dev-file', + type: 'file', }, ], - configIndex: (inputConfig.indices) ? inputConfig.config_index : 'default-commons-config-index', + configIndex: (inputConfig.indices) ? inputConfig.config_index : 'gen3-dev-config', authFilterField: inputConfig.auth_filter_field || 'auth_resource_path', aggregationIncludeMissingData: typeof inputConfig.aggs_include_missing_data === 'undefined' ? true : inputConfig.aggs_include_missing_data, missingDataAlias: inputConfig.missing_data_alias || 'no data', @@ -25,7 +29,7 @@ const config = { port: 80, path: '/graphql', arboristEndpoint: 'http://arborist-service', - tierAccessLevel: 'libre', + tierAccessLevel: 'private', tierAccessLimit: 1000, tierAccessSensitiveRecordExclusionField: inputConfig.tier_access_sensitive_record_exclusion_field, logLevel: inputConfig.log_level || 'INFO', From e324ac59676ce56ed2298715e5f7917540f6f192 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Tue, 2 Apr 2024 13:20:36 -0500 Subject: [PATCH 23/27] add config tests --- src/server/__tests__/config.test.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/server/__tests__/config.test.js b/src/server/__tests__/config.test.js index be7f41996..7833e4a24 100644 --- a/src/server/__tests__/config.test.js +++ b/src/server/__tests__/config.test.js @@ -97,4 +97,14 @@ describe('config', () => { expect(config.esConfig.aggregationIncludeMissingData).toBe(true); expect(config.esConfig.missingDataAlias).toEqual(alias); }); + + test('could ignoredFields fields from mapping', async () => { + const config = require('../config').default; + expect(config.ignoredFields).toEqual(['@version']); + }); + + test('could remap "__field" from to "prefix__field"', async () => { + const config = require('../config').default; + expect(config.doubleUnderscorePrefix).toEqual('x__'); + }); }); From d2682988d3775acd4482301c226a377db9d8aca1 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Wed, 3 Apr 2024 11:41:50 -0500 Subject: [PATCH 24/27] fix lint error --- src/server/es/index.js | 65 ++++++++++++------------- src/server/utils/__test__/utils.test.js | 4 +- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index ce3d32527..5a70e2598 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -10,6 +10,32 @@ import { SCROLL_PAGE_SIZE } from './const'; import CodedError from '../utils/error'; import { fromFieldsToSource, buildNestedField, processNestedFieldNames } from '../utils/utils'; +/** + * Modifies the properties of the index root object. + * This function has a side effect of modifying the index root object which + * is done to make the code more readable. + * It removes disabled fields, ignored fields, and converts double underscore prefix to single underscore. + * @param {object} root - The index root object to be modified. + */ +function modifyIndexRootProperties(root) { + // Changes root object by updating in place + if (root) { + Object.keys(root).forEach((fieldName) => { + if (root[fieldName].enabled === false) { + // eslint-disable-next-line no-param-reassign + delete root[fieldName]; + } + if (root[fieldName] && config.ignoredFields.includes(fieldName)) { + // eslint-disable-next-line no-param-reassign + delete root[fieldName]; + } + if (root[fieldName] && fieldName.startsWith('__')) { + delete Object.assign(root, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: root[fieldName] })[fieldName]; + } + }); + } +} + class ES { constructor(esConfig = config.esConfig) { this.config = esConfig; @@ -54,7 +80,7 @@ class ES { }; validatedQueryBody.track_total_hits = true; - var start = Date.now(); + const start = Date.now(); return this.client.search({ index: esIndex, body: validatedQueryBody, @@ -62,10 +88,10 @@ class ES { log.error(`[ES.query] error during querying: ${err.message}`); throw new Error(err.message); }).finally(() => { - var end = Date.now(); - var durationInMS = end - start; + const end = Date.now(); + const durationInMS = end - start; - log.info('[ES.query] DurationInMS:' + durationInMS + '. index, type, query body: ', esIndex, esType, JSON.stringify(validatedQueryBody)); + log.info(`[ES.query] DurationInMS:${durationInMS}. index, type, query body: `, esIndex, esType, JSON.stringify(validatedQueryBody)); }); } @@ -211,10 +237,10 @@ class ES { async _processEachIndex(indexConfig) { const res = await this._getESFieldsTypes(indexConfig.index); Object.keys(res).forEach((fieldName) => { - this._modifyIndexRootProperties(res); + modifyIndexRootProperties(res); if (res[fieldName] && 'properties' in res[fieldName] && res[fieldName].type === 'nested') { const root = res[fieldName].properties; - this._modifyIndexRootProperties(root); + modifyIndexRootProperties(root); } }); @@ -224,33 +250,6 @@ class ES { }; } - /** - * Modifies the properties of the index root object. - * This function has a side effect of modifying the index root object which - * is done to make the code more readable. - * It removes disabled fields, ignored fields, and converts double underscore prefix to single underscore. - * @param {object} root - The index root object to be modified. - */ - _modifyIndexRootProperties(root) { - - // Changes root object by updating in place - if (root) { - Object.keys(root).forEach((fieldName) => { - if (root[fieldName].enabled === false) { - // eslint-disable-next-line no-param-reassign - delete root[fieldName]; - } - if (root[fieldName] && config.ignoredFields.includes(fieldName)) { - // eslint-disable-next-line no-param-reassign - delete root[fieldName]; - } - if (root[fieldName] && fieldName.startsWith('__')) { - delete Object.assign(root, { [fieldName.replace('__', config.doubleUnderscorePrefix)]: root[fieldName] })[fieldName]; - } - }); - } - } - /** * Read array config and check if there's any array fields for each index. * Array fields are grouped and stored by index as a doc in array config, diff --git a/src/server/utils/__test__/utils.test.js b/src/server/utils/__test__/utils.test.js index 396774890..9932a42bd 100644 --- a/src/server/utils/__test__/utils.test.js +++ b/src/server/utils/__test__/utils.test.js @@ -3,6 +3,8 @@ import UtilsData from '../__mockData__/utils.data'; describe('Parse fields from GraphQL query to fields in ES query', () => { test('could parse fields in GraphQL query correctly', async () => { - expect(fromFieldsToSource(UtilsData.parsedInfo, "x__")).toEqual(UtilsData.fields); + expect(fromFieldsToSource(UtilsData.parsedInfo, 'x__')).toEqual( + UtilsData.fields, + ); }); }); From 1fbf073a842d96843b695b70669392f839d31162 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Thu, 11 Apr 2024 09:21:19 -0500 Subject: [PATCH 25/27] revert offset + size > SCROLL_PAGE_SIZE --- src/server/config.js | 14 +++++--------- src/server/es/index.js | 5 ++--- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index 9b92d00f3..6e6b45809 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -13,23 +13,19 @@ const config = { host: 'localhost:9200', indices: inputConfig.indices || [ { - index: 'gen3-dev-subject', - type: 'subject', - }, - { - index: 'gen3-dev-file', - type: 'file', + index: 'default-commons-index', + type: 'metadata', }, ], - configIndex: (inputConfig.indices) ? inputConfig.config_index : 'gen3-dev-config', + configIndex: (inputConfig.indices) ? inputConfig.config_index : 'default-commons-config-index', authFilterField: inputConfig.auth_filter_field || 'auth_resource_path', aggregationIncludeMissingData: typeof inputConfig.aggs_include_missing_data === 'undefined' ? true : inputConfig.aggs_include_missing_data, missingDataAlias: inputConfig.missing_data_alias || 'no data', }, - port: 80, + port: 3200, path: '/graphql', arboristEndpoint: 'http://arborist-service', - tierAccessLevel: 'private', + tierAccessLevel: 'libre', tierAccessLimit: 1000, tierAccessSensitiveRecordExclusionField: inputConfig.tier_access_sensitive_record_exclusion_field, logLevel: inputConfig.log_level || 'INFO', diff --git a/src/server/es/index.js b/src/server/es/index.js index 5a70e2598..e99069442 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -538,9 +538,8 @@ class ES { async getData({ esIndex, esType, fields, filter, sort, offset, size, }) { - // TODO: understand why it's offset + size > SCROLL_PAGE_SIZE instead of offset > SCROLL_PAGE_SIZE - if (typeof size !== 'undefined' && size > SCROLL_PAGE_SIZE) { - throw new GraphQLError(`Large graphql query forbidden for size > ${SCROLL_PAGE_SIZE}, + if (typeof size !== 'undefined' && offset + size > SCROLL_PAGE_SIZE) { + throw new GraphQLError(`Large graphql query forbidden for offset + size > ${SCROLL_PAGE_SIZE}, offset = ${offset} and size = ${size}, please use download endpoint for large data queries instead.`, { extensions: { From 1d24c676e2a7d4b42dbca8461d1efbb70d640699 Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Thu, 11 Apr 2024 09:22:26 -0500 Subject: [PATCH 26/27] revert config.js --- src/server/config.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index 6e6b45809..9b92d00f3 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -13,19 +13,23 @@ const config = { host: 'localhost:9200', indices: inputConfig.indices || [ { - index: 'default-commons-index', - type: 'metadata', + index: 'gen3-dev-subject', + type: 'subject', + }, + { + index: 'gen3-dev-file', + type: 'file', }, ], - configIndex: (inputConfig.indices) ? inputConfig.config_index : 'default-commons-config-index', + configIndex: (inputConfig.indices) ? inputConfig.config_index : 'gen3-dev-config', authFilterField: inputConfig.auth_filter_field || 'auth_resource_path', aggregationIncludeMissingData: typeof inputConfig.aggs_include_missing_data === 'undefined' ? true : inputConfig.aggs_include_missing_data, missingDataAlias: inputConfig.missing_data_alias || 'no data', }, - port: 3200, + port: 80, path: '/graphql', arboristEndpoint: 'http://arborist-service', - tierAccessLevel: 'libre', + tierAccessLevel: 'private', tierAccessLimit: 1000, tierAccessSensitiveRecordExclusionField: inputConfig.tier_access_sensitive_record_exclusion_field, logLevel: inputConfig.log_level || 'INFO', From 7939b30fce300b620d32bba81e956151dde1035a Mon Sep 17 00:00:00 2001 From: craigrbarnes Date: Fri, 2 May 2025 11:53:50 -0500 Subject: [PATCH 27/27] add log and date type --- src/server/es/index.js | 1 + src/server/schema.js | 1 + 2 files changed, 2 insertions(+) diff --git a/src/server/es/index.js b/src/server/es/index.js index e99069442..9cd698bf2 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -26,6 +26,7 @@ function modifyIndexRootProperties(root) { delete root[fieldName]; } if (root[fieldName] && config.ignoredFields.includes(fieldName)) { + log.info(`[ES] deleting field ${fieldName} because it should be ignored.`); // eslint-disable-next-line no-param-reassign delete root[fieldName]; } diff --git a/src/server/schema.js b/src/server/schema.js index 5ca381413..61b2e2a26 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -3,6 +3,7 @@ import { firstLetterUpperCase } from './utils/utils'; const esgqlTypeMapping = { text: 'String', + date: 'String', keyword: 'String', integer: 'Int', long: 'Float',