From 6eb2c5fbf252d162653d1e89c0709e19ef3ff751 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 30 Nov 2020 14:05:45 -0600 Subject: [PATCH 01/67] feat: per-index tier access middleware --- src/server/config.js | 7 +++++-- src/server/middlewares/index.js | 7 ++++++- .../middlewares/perIndexTierAccessMiddleware/index.js | 0 src/server/middlewares/utils.js | 3 +++ 4 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 src/server/middlewares/perIndexTierAccessMiddleware/index.js create mode 100644 src/server/middlewares/utils.js diff --git a/src/server/config.js b/src/server/config.js index 6f65dd5e..2e657739 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -15,10 +15,12 @@ const config = { { index: 'gen3-dev-subject', type: 'subject', + tier_access_level: 'private', }, { index: 'gen3-dev-file', type: 'file', + tier_access_level: 'private', }, ], configIndex: (inputConfig.indices) ? inputConfig.config_index : 'gen3-dev-config', @@ -30,7 +32,6 @@ const config = { port: 80, path: '/graphql', arboristEndpoint: 'http://arborist-service', - tierAccessLevel: 'private', tierAccessLimit: 1000, tierAccessSensitiveRecordExclusionField: inputConfig.tier_access_sensitive_record_exclusion_field, logLevel: inputConfig.log_level || 'INFO', @@ -72,7 +73,9 @@ if (process.env.ANALYZED_TEXT_FIELD_SUFFIX) { config.analyzedTextFieldSuffix = process.env.ANALYZED_TEXT_FIELD_SUFFIX; } -// only three options for tier access level: 'private' (default), 'regular', and 'libre' +// If the manifest provides a single TIER_ACCESS_LEVEL value (as opposed to index-specific values) +// we ignore the index-specific tiered-access settings. +// This allows for backwards-compatibility and flexibility between commons' use cases. if (process.env.TIER_ACCESS_LEVEL) { if (process.env.TIER_ACCESS_LEVEL !== 'private' && process.env.TIER_ACCESS_LEVEL !== 'regular' diff --git a/src/server/middlewares/index.js b/src/server/middlewares/index.js index eaae74ca..bb018793 100644 --- a/src/server/middlewares/index.js +++ b/src/server/middlewares/index.js @@ -1,8 +1,12 @@ import authMiddleware from './authMiddleware'; import tierAccessMiddleware from './tierAccessMiddleware'; +import perIndexTierAccessMiddleware from './perIndexTierAccessMiddleware'; import config from '../config'; const middlewares = []; + +// If a universal tierAccessLevel has not been applied in the manifest, +// we apply ES-index-specific tiered access settings. switch (config.tierAccessLevel) { case 'libre': break; @@ -13,6 +17,7 @@ switch (config.tierAccessLevel) { middlewares.push(authMiddleware); break; default: - throw new Error(`Invalid tier access level ${config.tierAccessLevel}`); + middlewares.push(perIndexTierAccessMiddleware); + break; } export default middlewares; diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js new file mode 100644 index 00000000..e69de29b diff --git a/src/server/middlewares/utils.js b/src/server/middlewares/utils.js new file mode 100644 index 00000000..678b4657 --- /dev/null +++ b/src/server/middlewares/utils.js @@ -0,0 +1,3 @@ + +// This file contains code shared between the tierAccessMiddleware and the perIndexTierAccessMiddleware. + From 76015344b944b20eebb719dee0a13eed316752be Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 1 Dec 2020 09:20:32 -0600 Subject: [PATCH 02/67] - --- src/server/middlewares/utils.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/middlewares/utils.js b/src/server/middlewares/utils.js index 678b4657..d9ec2216 100644 --- a/src/server/middlewares/utils.js +++ b/src/server/middlewares/utils.js @@ -1,3 +1,5 @@ -// This file contains code shared between the tierAccessMiddleware and the perIndexTierAccessMiddleware. +// This file contains code shared between the tierAccessMiddleware, authMiddleware, and the perIndexTierAccessMiddleware. + + From e98689cf64f279267ed21117d5848b3f73c0590b Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 2 Dec 2020 22:56:03 -0600 Subject: [PATCH 03/67] reorganize code --- .../middlewares/authMiddleware/index.js | 28 +-- .../middlewares/authMiddleware/resolvers.js | 28 +++ src/server/middlewares/index.js | 6 +- .../middlewares/tierAccessMiddleware/index.js | 216 +----------------- .../tierAccessMiddleware/resolvers.js | 216 ++++++++++++++++++ src/server/middlewares/utils.js | 5 - 6 files changed, 250 insertions(+), 249 deletions(-) create mode 100644 src/server/middlewares/authMiddleware/resolvers.js create mode 100644 src/server/middlewares/tierAccessMiddleware/resolvers.js delete mode 100644 src/server/middlewares/utils.js diff --git a/src/server/middlewares/authMiddleware/index.js b/src/server/middlewares/authMiddleware/index.js index a1c5fa70..10f4aeaf 100644 --- a/src/server/middlewares/authMiddleware/index.js +++ b/src/server/middlewares/authMiddleware/index.js @@ -1,31 +1,5 @@ -import assert from 'assert'; -import log from '../../logger'; import config from '../../config'; - -const authMWResolver = async (resolve, root, args, context, info) => { - assert(config.tierAccessLevel === 'private', 'Auth middleware layer only for "private" tier access level'); - const { authHelper } = context; - - // if mock arborist endpoint, just skip auth middleware - if (!config.internalLocalTest) { - if (config.arboristEndpoint === 'mock') { - log.debug('[authMiddleware] using mock arborist endpoint, skip auth middleware'); - return resolve(root, args, context, info); - } - } - - // asking arborist for auth resource list, and add to filter args - const parsedFilter = args.filter; - const appliedFilter = await authHelper.applyAccessibleFilter(parsedFilter); - const newArgs = { - ...args, - filter: appliedFilter, - }; - if (typeof newArgs.filter === 'undefined') { - delete newArgs.filter; - } - return resolve(root, newArgs, context, info); -}; +import authMWResolver from './resolvers'; // apply this middleware to all es types' data/aggregation resolvers const typeMapping = config.esConfig.indices.reduce((acc, item) => { diff --git a/src/server/middlewares/authMiddleware/resolvers.js b/src/server/middlewares/authMiddleware/resolvers.js new file mode 100644 index 00000000..c3bc9819 --- /dev/null +++ b/src/server/middlewares/authMiddleware/resolvers.js @@ -0,0 +1,28 @@ +import log from '../../logger'; +import config from '../../config'; + +const authMWResolver = async (resolve, root, args, context, info) => { + const { authHelper } = context; + + // if mock arborist endpoint, just skip auth middleware + if (!config.internalLocalTest) { + if (config.arboristEndpoint === 'mock') { + log.debug('[authMiddleware] using mock arborist endpoint, skip auth middleware'); + return resolve(root, args, context, info); + } + } + + // asking arborist for auth resource list, and add to filter args + const parsedFilter = args.filter; + const appliedFilter = await authHelper.applyAccessibleFilter(parsedFilter); + const newArgs = { + ...args, + filter: appliedFilter, + }; + if (typeof newArgs.filter === 'undefined') { + delete newArgs.filter; + } + return resolve(root, newArgs, context, info); +}; + +export default authMWResolver; diff --git a/src/server/middlewares/index.js b/src/server/middlewares/index.js index bb018793..39954f64 100644 --- a/src/server/middlewares/index.js +++ b/src/server/middlewares/index.js @@ -1,11 +1,11 @@ import authMiddleware from './authMiddleware'; import tierAccessMiddleware from './tierAccessMiddleware'; -import perIndexTierAccessMiddleware from './perIndexTierAccessMiddleware'; +// import perIndexTierAccessMiddleware from './perIndexTierAccessMiddleware'; import config from '../config'; const middlewares = []; -// If a universal tierAccessLevel has not been applied in the manifest, +// If a universal tierAccessLevel has not been applied in the manifest, // we apply ES-index-specific tiered access settings. switch (config.tierAccessLevel) { case 'libre': @@ -17,7 +17,7 @@ switch (config.tierAccessLevel) { middlewares.push(authMiddleware); break; default: - middlewares.push(perIndexTierAccessMiddleware); + // middlewares.push(perIndexTierAccessMiddleware); break; } export default middlewares; diff --git a/src/server/middlewares/tierAccessMiddleware/index.js b/src/server/middlewares/tierAccessMiddleware/index.js index f13fa066..4234a80f 100644 --- a/src/server/middlewares/tierAccessMiddleware/index.js +++ b/src/server/middlewares/tierAccessMiddleware/index.js @@ -1,218 +1,6 @@ -import _ from 'lodash'; -import assert from 'assert'; -import { ApolloError, UserInputError } from 'apollo-server'; -import log from '../../logger'; import config from '../../config'; -import esInstance from '../../es/index'; -import CodedError from '../../utils/error'; -import { firstLetterUpperCase, isWhitelisted, addTwoFilters } from '../../utils/utils'; - -const ENCRYPT_COUNT = -1; - -const resolverWithAccessibleFilterApplied = ( - resolve, root, args, context, info, authHelper, filter, -) => { - const appliedFilter = authHelper.applyAccessibleFilter(filter); - const newArgs = { - ...args, - filter: appliedFilter, - needEncryptAgg: false, - }; - return resolve(root, newArgs, context, info); -}; - -const resolverWithUnaccessibleFilterApplied = ( - resolve, root, args, context, info, authHelper, filter, -) => { - const appliedFilter = authHelper.applyUnaccessibleFilter(filter); - const newArgs = { - ...args, - filter: appliedFilter, - needEncryptAgg: true, - }; - return resolve(root, newArgs, context, info); -}; - -const tierAccessResolver = ( - { - isRawDataQuery, - esType, - }, -) => async (resolve, root, args, context, info) => { - try { - assert(config.tierAccessLevel === 'regular', 'Tier access middleware layer only for "regular" tier access level'); - const { authHelper } = context; - const esIndex = esInstance.getESIndexByType(esType); - const { filter, filterSelf, accessibility } = args; - - const outOfScopeResourceList = await authHelper.getOutOfScopeResourceList( - esIndex, esType, filter, filterSelf, - ); - // if requesting resources is within allowed resources, return result - if (outOfScopeResourceList.length === 0) { - // unless it's requesting for `unaccessible` data, just resolve this - switch (accessibility) { - case 'accessible': - return resolve(root, { ...args, needEncryptAgg: false }, context, info); - case 'unaccessible': - return resolverWithUnaccessibleFilterApplied( - resolve, root, args, context, info, authHelper, filter, - ); - default: - return resolve(root, { ...args, needEncryptAgg: true }, context, info); - } - } - // else, check if it's raw data query or aggs query - if (isRawDataQuery) { // raw data query for out-of-scope resources are forbidden - if (accessibility === 'accessible') { - return resolverWithAccessibleFilterApplied( - resolve, root, args, context, info, authHelper, filter, - ); - } - log.info('[tierAccessResolver] requesting out-of-scope resources, return 401'); - log.info(`[tierAccessResolver] the following resources are out-of-scope: [${outOfScopeResourceList.join(', ')}]`); - throw new ApolloError('You don\'t have access to all the data you are querying. Try using \'accessibility: accessible\' in your query', 401); - } - - /** - * Here we have a bypass for `regular`-tier-access-leveled commons: - * `accessibility` has 3 options: `all`, `accessible`, and `unaccessible`. - * For `all`, behavior is the same as usual - * For `accessible`, we will apply auth filter on top of filter argument - * For `unaccessible`, we apply unaccessible filters on top of filter argument - */ - const sensitiveRecordExclusionEnabled = !!config.tierAccessSensitiveRecordExclusionField; - if (accessibility === 'all') { - if (sensitiveRecordExclusionEnabled) { - // Sensitive study exclusion is enabled: For all of the projects user does - // not have access to, hide the studies marked 'sensitive' from the aggregation. - // (See doc/queries.md#Tiered_Access_sensitive_record_exclusion) - const projectsUserHasAccessTo = authHelper.getAccessibleResources(); - const sensitiveStudiesFilter = { - OR: [ - { - IN: { - [config.esConfig.authFilterField]: projectsUserHasAccessTo, - }, - }, - { - '!=': { - [config.tierAccessSensitiveRecordExclusionField]: 'true', - }, - }, - ], - }; - return resolve( - root, - { - ...args, - filter: addTwoFilters(filter, sensitiveStudiesFilter), - needEncryptAgg: true, - }, - context, - info, - ); - } - - return resolve( - root, - { - ...args, - filter, - needEncryptAgg: true, - }, - context, - info, - ); - } - if (accessibility === 'accessible') { - // We do not need to apply sensitive studies filter here, because - // user has access to all of these projects. - log.debug('[tierAccessResolver] applying "accessible" to resolver'); - return resolverWithAccessibleFilterApplied( - resolve, root, args, context, info, authHelper, filter, - ); - } - // The below code executes if accessibility === 'unaccessible'. - if (sensitiveRecordExclusionEnabled) { - // Apply sensitive studies filter. Hide the studies marked 'sensitive' from - // the aggregation. - const sensitiveStudiesFilter = { - '!=': { - [config.tierAccessSensitiveRecordExclusionField]: 'true', - }, - }; - return resolverWithUnaccessibleFilterApplied( - resolve, - root, - args, - context, - info, - authHelper, - addTwoFilters(filter, sensitiveStudiesFilter), - ); - } - return resolverWithUnaccessibleFilterApplied( - resolve, root, args, context, info, authHelper, filter, - ); - } catch (err) { - if (err instanceof ApolloError) { - if (err.extensions.code >= 500) { - console.trace(err); // eslint-disable-line no-console - } - } else if (err instanceof CodedError) { - if (err.code >= 500) { - console.trace(err); // eslint-disable-line no-console - } - } else if (!(err instanceof UserInputError)) { - console.trace(err); // eslint-disable-line no-console - } - throw err; - } -}; - -/** - * This resolver middleware is appended after aggregation resolvers, - * it hide number that is less than allowed visible number for regular tier access - * @param {bool} isGettingTotalCount - */ -const hideNumberResolver = (isGettingTotalCount) => async (resolve, root, args, context, info) => { - // for aggregations, hide all counts that are greater than limited number - const { needEncryptAgg } = root; - const result = await resolve(root, args, context, info); - log.debug('[hideNumberResolver] result: ', result); - if (!needEncryptAgg) return result; - - const newRoot = root; - newRoot.accessibility = 'unaccessible'; - const { authHelper } = context; - newRoot.filter = authHelper.applyUnaccessibleFilter(newRoot.filter); - const unaccessibleResult = await resolve(newRoot, args, context, info); - log.debug('[hideNumberResolver] unaccessibleResult: ', unaccessibleResult); - - // if getting total count, only encrypt if unaccessibleResult is between (0, tierAccessLimit) - if (isGettingTotalCount) { - return (unaccessibleResult > 0 - && unaccessibleResult < config.tierAccessLimit) ? ENCRYPT_COUNT : result; - } - - const encryptedResult = result.map((item) => { - // we don't encrypt whitelisted results or if result is not found in unaccessibleResult - if (isWhitelisted(item.key) || !(unaccessibleResult.some((e) => e.key === item.key))) { - return item; - } - // we only encrypt if count from no-access item is small - const unaccessibleResultItem = _.find(unaccessibleResult, (e) => e.key === item.key); - if (unaccessibleResultItem.count < config.tierAccessLimit) { - return { - key: item.key, - count: ENCRYPT_COUNT, - }; - } - return item; - }); - return encryptedResult; -}; +import { firstLetterUpperCase } from '../../utils/utils'; +import { tierAccessResolver, hideNumberResolver } from './resolvers'; // apply this middleware to all es types' data/aggregation resolvers const queryTypeMapping = {}; diff --git a/src/server/middlewares/tierAccessMiddleware/resolvers.js b/src/server/middlewares/tierAccessMiddleware/resolvers.js new file mode 100644 index 00000000..0b411482 --- /dev/null +++ b/src/server/middlewares/tierAccessMiddleware/resolvers.js @@ -0,0 +1,216 @@ +import _ from 'lodash'; +import assert from 'assert'; +import { ApolloError, UserInputError } from 'apollo-server'; +import log from '../../logger'; +import config from '../../config'; +import esInstance from '../../es/index'; +import CodedError from '../../utils/error'; +import { isWhitelisted, addTwoFilters } from '../../utils/utils'; + +const ENCRYPT_COUNT = -1; + +const resolverWithAccessibleFilterApplied = ( + resolve, root, args, context, info, authHelper, filter, +) => { + const appliedFilter = authHelper.applyAccessibleFilter(filter); + const newArgs = { + ...args, + filter: appliedFilter, + needEncryptAgg: false, + }; + return resolve(root, newArgs, context, info); +}; + +const resolverWithUnaccessibleFilterApplied = ( + resolve, root, args, context, info, authHelper, filter, +) => { + const appliedFilter = authHelper.applyUnaccessibleFilter(filter); + const newArgs = { + ...args, + filter: appliedFilter, + needEncryptAgg: true, + }; + return resolve(root, newArgs, context, info); +}; + +export const tierAccessResolver = ( + { + isRawDataQuery, + esType, + }, +) => async (resolve, root, args, context, info) => { + try { + assert(config.tierAccessLevel === 'regular', 'Tier access middleware layer only for "regular" tier access level'); + const { authHelper } = context; + const esIndex = esInstance.getESIndexByType(esType); + const { filter, filterSelf, accessibility } = args; + + const outOfScopeResourceList = await authHelper.getOutOfScopeResourceList( + esIndex, esType, filter, filterSelf, + ); + // if requesting resources is within allowed resources, return result + if (outOfScopeResourceList.length === 0) { + // unless it's requesting for `unaccessible` data, just resolve this + switch (accessibility) { + case 'accessible': + return resolve(root, { ...args, needEncryptAgg: false }, context, info); + case 'unaccessible': + return resolverWithUnaccessibleFilterApplied( + resolve, root, args, context, info, authHelper, filter, + ); + default: + return resolve(root, { ...args, needEncryptAgg: true }, context, info); + } + } + // else, check if it's raw data query or aggs query + if (isRawDataQuery) { // raw data query for out-of-scope resources are forbidden + if (accessibility === 'accessible') { + return resolverWithAccessibleFilterApplied( + resolve, root, args, context, info, authHelper, filter, + ); + } + log.info('[tierAccessResolver] requesting out-of-scope resources, return 401'); + log.info(`[tierAccessResolver] the following resources are out-of-scope: [${outOfScopeResourceList.join(', ')}]`); + throw new ApolloError('You don\'t have access to all the data you are querying. Try using \'accessibility: accessible\' in your query', 401); + } + + /** + * Here we have a bypass for `regular`-tier-access-leveled commons: + * `accessibility` has 3 options: `all`, `accessible`, and `unaccessible`. + * For `all`, behavior is the same as usual + * For `accessible`, we will apply auth filter on top of filter argument + * For `unaccessible`, we apply unaccessible filters on top of filter argument + */ + const sensitiveRecordExclusionEnabled = !!config.tierAccessSensitiveRecordExclusionField; + if (accessibility === 'all') { + if (sensitiveRecordExclusionEnabled) { + // Sensitive study exclusion is enabled: For all of the projects user does + // not have access to, hide the studies marked 'sensitive' from the aggregation. + // (See doc/queries.md#Tiered_Access_sensitive_record_exclusion) + const projectsUserHasAccessTo = authHelper.getAccessibleResources(); + const sensitiveStudiesFilter = { + OR: [ + { + IN: { + [config.esConfig.authFilterField]: projectsUserHasAccessTo, + }, + }, + { + '!=': { + [config.tierAccessSensitiveRecordExclusionField]: 'true', + }, + }, + ], + }; + return resolve( + root, + { + ...args, + filter: addTwoFilters(filter, sensitiveStudiesFilter), + needEncryptAgg: true, + }, + context, + info, + ); + } + + return resolve( + root, + { + ...args, + filter, + needEncryptAgg: true, + }, + context, + info, + ); + } + if (accessibility === 'accessible') { + // We do not need to apply sensitive studies filter here, because + // user has access to all of these projects. + log.debug('[tierAccessResolver] applying "accessible" to resolver'); + return resolverWithAccessibleFilterApplied( + resolve, root, args, context, info, authHelper, filter, + ); + } + // The below code executes if accessibility === 'unaccessible'. + if (sensitiveRecordExclusionEnabled) { + // Apply sensitive studies filter. Hide the studies marked 'sensitive' from + // the aggregation. + const sensitiveStudiesFilter = { + '!=': { + [config.tierAccessSensitiveRecordExclusionField]: 'true', + }, + }; + return resolverWithUnaccessibleFilterApplied( + resolve, + root, + args, + context, + info, + authHelper, + addTwoFilters(filter, sensitiveStudiesFilter), + ); + } + return resolverWithUnaccessibleFilterApplied( + resolve, root, args, context, info, authHelper, filter, + ); + } catch (err) { + if (err instanceof ApolloError) { + if (err.extensions.code >= 500) { + console.trace(err); // eslint-disable-line no-console + } + } else if (err instanceof CodedError) { + if (err.code >= 500) { + console.trace(err); // eslint-disable-line no-console + } + } else if (!(err instanceof UserInputError)) { + console.trace(err); // eslint-disable-line no-console + } + throw err; + } +}; + +/** + * This resolver middleware is appended after aggregation resolvers, + * it hide number that is less than allowed visible number for regular tier access + * @param {bool} isGettingTotalCount + */ +export const hideNumberResolver = (isGettingTotalCount) => async ( + resolve, root, args, context, info) => { + // for aggregations, hide all counts that are greater than limited number + const { needEncryptAgg } = root; + const result = await resolve(root, args, context, info); + log.debug('[hideNumberResolver] result: ', result); + if (!needEncryptAgg) return result; + + const newRoot = root; + newRoot.accessibility = 'unaccessible'; + const { authHelper } = context; + newRoot.filter = authHelper.applyUnaccessibleFilter(newRoot.filter); + const unaccessibleResult = await resolve(newRoot, args, context, info); + log.debug('[hideNumberResolver] unaccessibleResult: ', unaccessibleResult); + + // if getting total count, only encrypt if unaccessibleResult is between (0, tierAccessLimit) + if (isGettingTotalCount) { + return (unaccessibleResult > 0 + && unaccessibleResult < config.tierAccessLimit) ? ENCRYPT_COUNT : result; + } + + const encryptedResult = result.map((item) => { + // we don't encrypt whitelisted results or if result is not found in unaccessibleResult + if (isWhitelisted(item.key) || !(unaccessibleResult.some((e) => e.key === item.key))) { + return item; + } + // we only encrypt if count from no-access item is small + const unaccessibleResultItem = _.find(unaccessibleResult, (e) => e.key === item.key); + if (unaccessibleResultItem.count < config.tierAccessLimit) { + return { + key: item.key, + count: ENCRYPT_COUNT, + }; + } + return item; + }); + return encryptedResult; +}; diff --git a/src/server/middlewares/utils.js b/src/server/middlewares/utils.js deleted file mode 100644 index d9ec2216..00000000 --- a/src/server/middlewares/utils.js +++ /dev/null @@ -1,5 +0,0 @@ - -// This file contains code shared between the tierAccessMiddleware, authMiddleware, and the perIndexTierAccessMiddleware. - - - From 1d802e0cfbcf47cafafa8f60d204f3056e6e5dd7 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 2 Dec 2020 23:04:23 -0600 Subject: [PATCH 04/67] fix unit tests --- src/server/config.js | 5 +++-- src/server/middlewares/index.js | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index 2e657739..a06195e8 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -23,6 +23,7 @@ const config = { tier_access_level: 'private', }, ], + tierAccessLevel: 'private', configIndex: (inputConfig.indices) ? inputConfig.config_index : 'gen3-dev-config', authFilterField: inputConfig.auth_filter_field || 'auth_resource_path', aggregationIncludeMissingData: typeof inputConfig.aggs_include_missing_data === 'undefined' ? true : inputConfig.aggs_include_missing_data, @@ -73,8 +74,8 @@ if (process.env.ANALYZED_TEXT_FIELD_SUFFIX) { config.analyzedTextFieldSuffix = process.env.ANALYZED_TEXT_FIELD_SUFFIX; } -// If the manifest provides a single TIER_ACCESS_LEVEL value (as opposed to index-specific values) -// we ignore the index-specific tiered-access settings. +// In cases where index-scoped tiered-access settings are not provided, +// we fall back on the manifest-provided TIER_ACCESS_LEVEL value. // This allows for backwards-compatibility and flexibility between commons' use cases. if (process.env.TIER_ACCESS_LEVEL) { if (process.env.TIER_ACCESS_LEVEL !== 'private' diff --git a/src/server/middlewares/index.js b/src/server/middlewares/index.js index 39954f64..e3a91599 100644 --- a/src/server/middlewares/index.js +++ b/src/server/middlewares/index.js @@ -17,7 +17,7 @@ switch (config.tierAccessLevel) { middlewares.push(authMiddleware); break; default: - // middlewares.push(perIndexTierAccessMiddleware); + middlewares.push(perIndexTierAccessMiddleware); break; } export default middlewares; From d8620647d744cfc000622d5b89519d36f933ca31 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 2 Dec 2020 23:34:19 -0600 Subject: [PATCH 05/67] fix unit tests --- src/server/config.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index a06195e8..15b8685d 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -23,16 +23,15 @@ const config = { tier_access_level: 'private', }, ], - tierAccessLevel: 'private', configIndex: (inputConfig.indices) ? inputConfig.config_index : 'gen3-dev-config', authFilterField: inputConfig.auth_filter_field || 'auth_resource_path', aggregationIncludeMissingData: typeof inputConfig.aggs_include_missing_data === 'undefined' ? true : inputConfig.aggs_include_missing_data, missingDataAlias: inputConfig.missing_data_alias || 'no data', }, - port: 80, path: '/graphql', arboristEndpoint: 'http://arborist-service', + tierAccessLevel: 'private', tierAccessLimit: 1000, tierAccessSensitiveRecordExclusionField: inputConfig.tier_access_sensitive_record_exclusion_field, logLevel: inputConfig.log_level || 'INFO', From 06f225114a8e2b12d56d093fd06dab36947a2c5f Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 2 Dec 2020 23:35:01 -0600 Subject: [PATCH 06/67] fix unit tests --- src/server/middlewares/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/middlewares/index.js b/src/server/middlewares/index.js index e3a91599..39954f64 100644 --- a/src/server/middlewares/index.js +++ b/src/server/middlewares/index.js @@ -17,7 +17,7 @@ switch (config.tierAccessLevel) { middlewares.push(authMiddleware); break; default: - middlewares.push(perIndexTierAccessMiddleware); + // middlewares.push(perIndexTierAccessMiddleware); break; } export default middlewares; From 77373b1e2abf8a8cb931446e748edce07587903c Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Thu, 3 Dec 2020 13:02:00 -0600 Subject: [PATCH 07/67] draft of new middleware structure --- src/server/middlewares/index.js | 4 +- .../perIndexTierAccessMiddleware/index.js | 53 +++++++++++++++++++ .../perIndexTierAccessMiddleware/resolvers.js | 0 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 src/server/middlewares/perIndexTierAccessMiddleware/resolvers.js diff --git a/src/server/middlewares/index.js b/src/server/middlewares/index.js index 39954f64..7f466a8a 100644 --- a/src/server/middlewares/index.js +++ b/src/server/middlewares/index.js @@ -1,6 +1,6 @@ import authMiddleware from './authMiddleware'; import tierAccessMiddleware from './tierAccessMiddleware'; -// import perIndexTierAccessMiddleware from './perIndexTierAccessMiddleware'; +import perIndexTierAccessMiddleware from './perIndexTierAccessMiddleware'; import config from '../config'; const middlewares = []; @@ -17,7 +17,7 @@ switch (config.tierAccessLevel) { middlewares.push(authMiddleware); break; default: - // middlewares.push(perIndexTierAccessMiddleware); + middlewares.push(perIndexTierAccessMiddleware); break; } export default middlewares; diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index e69de29b..6c33db6c 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -0,0 +1,53 @@ +import config from '../../config'; +import { firstLetterUpperCase } from '../../utils/utils'; +import authMWResolver from '../authMiddleware/resolvers'; +import { tierAccessResolver, hideNumberResolver } from '../tierAccessMiddleware/resolvers'; + +const queryIndexMapping = {}; +const aggsIndexMapping = {}; +const histogramIndexMapping = {}; +const histogramForStringIndexMapping = {}; +const totalCountIndexMapping = {}; + +config.esConfig.indices.reduce((acc, item) => { + if (item.tier_access_level === 'private') { + queryIndexMapping[item.index] = authMWResolver; + aggsIndexMapping[item.index] = authMWResolver; + } else if (item.tier_access_level === 'regular') { + queryIndexMapping[item.index] = tierAccessResolver({ + isRawDataQuery: true, + esType: item.type, + esIndex: item.index, + }); + aggsIndexMapping[item.index] = tierAccessResolver({ esType: item.type, esIndex: item.index }); + const aggregationName = `${firstLetterUpperCase(item.type)}Aggregation`; + totalCountIndexMapping[aggregationName] = { + _totalCount: hideNumberResolver(true), + }; + histogramIndexMapping[item.index] = { histogram: hideNumberResolver(false) }; + histogramForStringIndexMapping[item.index] = { histogram: hideNumberResolver(false) }; + } else if (item.tier_access_level === 'libre') { + // No additional resolvers necessary + } else { + throw new Error(`tier_access_level invalid for index ${item.index}. Please set index-scoped or site-wide tiered-access levels.`); + } + return acc; +}, {}); + +const perIndexTierAccessMiddleware = { + Query: { + ...queryIndexMapping, + }, + Aggregation: { + ...aggsIndexMapping, + }, + ...totalCountIndexMapping, + HistogramForNumber: { + ...histogramIndexMapping, + }, + HistogramForString: { + ...histogramForStringIndexMapping, + }, +}; + +export default perIndexTierAccessMiddleware; diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/resolvers.js b/src/server/middlewares/perIndexTierAccessMiddleware/resolvers.js new file mode 100644 index 00000000..e69de29b From c2c6a595f2a70a77e54a214357d059ce0b6a5735 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Thu, 3 Dec 2020 13:16:31 -0600 Subject: [PATCH 08/67] adjust manifest validation --- src/server/config.js | 12 +++++++++--- .../perIndexTierAccessMiddleware/index.js | 5 ++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index 15b8685d..1f7998ac 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -73,9 +73,15 @@ if (process.env.ANALYZED_TEXT_FIELD_SUFFIX) { config.analyzedTextFieldSuffix = process.env.ANALYZED_TEXT_FIELD_SUFFIX; } -// In cases where index-scoped tiered-access settings are not provided, -// we fall back on the manifest-provided TIER_ACCESS_LEVEL value. -// This allows for backwards-compatibility and flexibility between commons' use cases. +// Either all indices should have explicit index-scoped tiered-access values or +// the manifest should have a site-wide TIER_ACCESS_LEVEL value. +// This approach is backwards-compatible with commons configured for past versions of tiered-access. +config.esConfig.indices.forEach((item) => { + if (!item.tier_access_level && !process.env.TIER_ACCESS_LEVEL) { + throw new Error('Either set all index-scoped tiered-access levels or a site-wide tiered-access level.'); + } +}); + if (process.env.TIER_ACCESS_LEVEL) { if (process.env.TIER_ACCESS_LEVEL !== 'private' && process.env.TIER_ACCESS_LEVEL !== 'regular' diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index 6c33db6c..a0b13b00 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -9,7 +9,7 @@ const histogramIndexMapping = {}; const histogramForStringIndexMapping = {}; const totalCountIndexMapping = {}; -config.esConfig.indices.reduce((acc, item) => { +config.esConfig.indices.forEach((item) => { if (item.tier_access_level === 'private') { queryIndexMapping[item.index] = authMWResolver; aggsIndexMapping[item.index] = authMWResolver; @@ -29,9 +29,8 @@ config.esConfig.indices.reduce((acc, item) => { } else if (item.tier_access_level === 'libre') { // No additional resolvers necessary } else { - throw new Error(`tier_access_level invalid for index ${item.index}. Please set index-scoped or site-wide tiered-access levels.`); + throw new Error(`tier_access_level invalid for index ${item.index}. Either set all index-scoped tiered-access levels or a site-wide tiered-access level.`); } - return acc; }, {}); const perIndexTierAccessMiddleware = { From 3c4c8c9702080308895ca7c52b192c095d639523 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 7 Dec 2020 13:45:07 -0600 Subject: [PATCH 09/67] breaking type-wide middlewares into field-scoped middlewares --- src/server/es/index.js | 14 ++++++++++++++ src/server/middlewares/authMiddleware/index.js | 2 +- .../perIndexTierAccessMiddleware/index.js | 6 +++--- .../middlewares/tierAccessMiddleware/index.js | 13 +++++++------ .../middlewares/tierAccessMiddleware/resolvers.js | 10 ++++++++-- 5 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/server/es/index.js b/src/server/es/index.js index a10019d7..ecf640e0 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -318,6 +318,20 @@ class ES { ); } + /** + * Get es index config by es index name + * Throw 400 error if there's no existing es index of that name + * @param {string} esIndexName + */ + getESIndexConfigByName(esIndexName) { + const indexConfig = this.config.indices.find((i) => i.index === esIndexName); + if (indexConfig) return indexConfig; + throw new CodedError( + 400, + `Invalid es index name: "${esIndexName}"`, + ); + } + /** * Get all es indices and their alias */ diff --git a/src/server/middlewares/authMiddleware/index.js b/src/server/middlewares/authMiddleware/index.js index 10f4aeaf..eb5f2ee8 100644 --- a/src/server/middlewares/authMiddleware/index.js +++ b/src/server/middlewares/authMiddleware/index.js @@ -3,7 +3,7 @@ import authMWResolver from './resolvers'; // apply this middleware to all es types' data/aggregation resolvers const typeMapping = config.esConfig.indices.reduce((acc, item) => { - acc[item.type] = authMWResolver; + acc[item.index] = authMWResolver; return acc; }, {}); const authMiddleware = { diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index a0b13b00..2e65379e 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -7,7 +7,7 @@ const queryIndexMapping = {}; const aggsIndexMapping = {}; const histogramIndexMapping = {}; const histogramForStringIndexMapping = {}; -const totalCountIndexMapping = {}; +const totalCountTypeMapping = {}; config.esConfig.indices.forEach((item) => { if (item.tier_access_level === 'private') { @@ -21,7 +21,7 @@ config.esConfig.indices.forEach((item) => { }); aggsIndexMapping[item.index] = tierAccessResolver({ esType: item.type, esIndex: item.index }); const aggregationName = `${firstLetterUpperCase(item.type)}Aggregation`; - totalCountIndexMapping[aggregationName] = { + totalCountTypeMapping[aggregationName] = { _totalCount: hideNumberResolver(true), }; histogramIndexMapping[item.index] = { histogram: hideNumberResolver(false) }; @@ -40,7 +40,7 @@ const perIndexTierAccessMiddleware = { Aggregation: { ...aggsIndexMapping, }, - ...totalCountIndexMapping, + ...totalCountTypeMapping, HistogramForNumber: { ...histogramIndexMapping, }, diff --git a/src/server/middlewares/tierAccessMiddleware/index.js b/src/server/middlewares/tierAccessMiddleware/index.js index 4234a80f..a4d93f26 100644 --- a/src/server/middlewares/tierAccessMiddleware/index.js +++ b/src/server/middlewares/tierAccessMiddleware/index.js @@ -3,15 +3,16 @@ import { firstLetterUpperCase } from '../../utils/utils'; import { tierAccessResolver, hideNumberResolver } from './resolvers'; // apply this middleware to all es types' data/aggregation resolvers -const queryTypeMapping = {}; -const aggsTypeMapping = {}; +const queryIndexMapping = {}; +const aggsIndexMapping = {}; const totalCountTypeMapping = {}; config.esConfig.indices.forEach((item) => { - queryTypeMapping[item.type] = tierAccessResolver({ + queryIndexMapping[item.index] = tierAccessResolver({ isRawDataQuery: true, esType: item.type, + esIndex: item.index, }); - aggsTypeMapping[item.type] = tierAccessResolver({ esType: item.type }); + aggsIndexMapping[item.index] = tierAccessResolver({ esType: item.type, esIndex: item.index }); const aggregationName = `${firstLetterUpperCase(item.type)}Aggregation`; totalCountTypeMapping[aggregationName] = { _totalCount: hideNumberResolver(true), @@ -19,10 +20,10 @@ config.esConfig.indices.forEach((item) => { }); const tierAccessMiddleware = { Query: { - ...queryTypeMapping, + ...queryIndexMapping, }, Aggregation: { - ...aggsTypeMapping, + ...aggsIndexMapping, }, ...totalCountTypeMapping, HistogramForNumber: { diff --git a/src/server/middlewares/tierAccessMiddleware/resolvers.js b/src/server/middlewares/tierAccessMiddleware/resolvers.js index 0b411482..f8385a36 100644 --- a/src/server/middlewares/tierAccessMiddleware/resolvers.js +++ b/src/server/middlewares/tierAccessMiddleware/resolvers.js @@ -37,12 +37,18 @@ export const tierAccessResolver = ( { isRawDataQuery, esType, + esIndex, }, ) => async (resolve, root, args, context, info) => { try { - assert(config.tierAccessLevel === 'regular', 'Tier access middleware layer only for "regular" tier access level'); + // Assert that either this index is "regular" access or + // that the index has no setting and site-wide config is "regular". + const indexConfig = esInstance.getESIndexConfigByName(esIndex); + const indexIsRegularAccess = indexConfig.tier_access_level == 'regular'; + const tierAccessRegularAndNotIndexScoped = !indexConfig.tier_access_level && config.tierAccessLevel === 'regular'; + assert(indexIsRegularAccess || tierAccessRegularAndNotIndexScoped, 'Tier access middleware layer only for "regular" tier access level'); + const { authHelper } = context; - const esIndex = esInstance.getESIndexByType(esType); const { filter, filterSelf, accessibility } = args; const outOfScopeResourceList = await authHelper.getOutOfScopeResourceList( From a8985db9be3ea56e7b6a32dfcd3e4928171a61c3 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 7 Dec 2020 13:46:01 -0600 Subject: [PATCH 10/67] eslint --- src/server/middlewares/tierAccessMiddleware/resolvers.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/middlewares/tierAccessMiddleware/resolvers.js b/src/server/middlewares/tierAccessMiddleware/resolvers.js index f8385a36..543f81be 100644 --- a/src/server/middlewares/tierAccessMiddleware/resolvers.js +++ b/src/server/middlewares/tierAccessMiddleware/resolvers.js @@ -41,13 +41,13 @@ export const tierAccessResolver = ( }, ) => async (resolve, root, args, context, info) => { try { - // Assert that either this index is "regular" access or + // Assert that either this index is "regular" access or // that the index has no setting and site-wide config is "regular". const indexConfig = esInstance.getESIndexConfigByName(esIndex); - const indexIsRegularAccess = indexConfig.tier_access_level == 'regular'; + const indexIsRegularAccess = indexConfig.tier_access_level === 'regular'; const tierAccessRegularAndNotIndexScoped = !indexConfig.tier_access_level && config.tierAccessLevel === 'regular'; assert(indexIsRegularAccess || tierAccessRegularAndNotIndexScoped, 'Tier access middleware layer only for "regular" tier access level'); - + const { authHelper } = context; const { filter, filterSelf, accessibility } = args; From 12ea6b2786ba90a9f455c7b78eb5014bfea2093f Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 7 Dec 2020 13:57:29 -0600 Subject: [PATCH 11/67] add logs --- src/server/middlewares/index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/server/middlewares/index.js b/src/server/middlewares/index.js index 7f466a8a..68190ba5 100644 --- a/src/server/middlewares/index.js +++ b/src/server/middlewares/index.js @@ -9,14 +9,18 @@ const middlewares = []; // we apply ES-index-specific tiered access settings. switch (config.tierAccessLevel) { case 'libre': + console.log('[Server] applying libre middleware.'); break; case 'regular': + console.log('[Server] applying site-wide regular middleware.'); middlewares.push(tierAccessMiddleware); break; case 'private': + console.log('[Server] applying site-wide private middleware.'); middlewares.push(authMiddleware); break; default: + console.log('[Server] applying index-scoped middleware.'); middlewares.push(perIndexTierAccessMiddleware); break; } From 28a0ebeb8ebdd1d8755c42e70b9e759b64c4e119 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 19 Jan 2021 17:49:12 -0600 Subject: [PATCH 12/67] switched to index-scope querySchema --- src/server/schema.js | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/server/schema.js b/src/server/schema.js index e9dcd161..f4bd5aea 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -55,7 +55,18 @@ const getAggsHistogramName = (gqlType) => { return gqlTypeToAggsHistogramName[gqlType]; }; +const getObjNameFromESIndex = (esIndex) => { + let esTypeObjName = firstLetterUpperCase(esIndex); + // Choosing to replace hyphens with the string "_DASH_". + // It's self-documenting, lends well to readability, and the + // likelihood of a user naming an ES index with the string _DASH_ is low. + esTypeObjName = esTypeObjName.replaceAll('-', '_DASH_'); + return esTypeObjName; +}; + const getQuerySchemaForType = (esType) => { + // This function is now unused in favor of an + // index-scoped query schema. const esTypeObjName = firstLetterUpperCase(esType); return `${esType} ( offset: Int, @@ -66,6 +77,19 @@ const getQuerySchemaForType = (esType) => { ): [${esTypeObjName}]`; }; +const getQuerySchemaForIndex = (esIndex) => { + // This function helps build an index-scoped query schema + // so as to scope middleware by index rather than type. + const esIndexObjName = getObjNameFromESIndex(esIndex); + return `${esIndexObjName} ( + offset: Int, + first: Int, + filter: JSON, + sort: JSON, + accessibility: Accessibility=all, + ): [${esTypeObjName}]`; +}; + const getFieldGQLTypeMapForProperties = (esInstance, esIndex, properties) => { const result = Object.keys(properties).map((field) => { const esFieldType = (properties[field].esType) @@ -162,7 +186,7 @@ const getAggregationSchemaForOneIndex = (esInstance, esIndex, esType) => { export const getQuerySchema = (esConfig) => ` type Query { - ${esConfig.indices.map((cfg) => getQuerySchemaForType(cfg.type)).join('\n')} + ${esConfig.indices.map((cfg) => getQuerySchemaForIndex(cfg.index)).join('\n')} _aggregation: Aggregation _mapping: Mapping } From dfe7856e6237123d9105415fd555299039e6a390 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 19 Jan 2021 18:07:21 -0600 Subject: [PATCH 13/67] eslint --- src/server/middlewares/index.js | 9 +++++---- src/server/schema.js | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/server/middlewares/index.js b/src/server/middlewares/index.js index 68190ba5..4f56d526 100644 --- a/src/server/middlewares/index.js +++ b/src/server/middlewares/index.js @@ -2,6 +2,7 @@ import authMiddleware from './authMiddleware'; import tierAccessMiddleware from './tierAccessMiddleware'; import perIndexTierAccessMiddleware from './perIndexTierAccessMiddleware'; import config from '../config'; +import log from '../logger'; const middlewares = []; @@ -9,18 +10,18 @@ const middlewares = []; // we apply ES-index-specific tiered access settings. switch (config.tierAccessLevel) { case 'libre': - console.log('[Server] applying libre middleware.'); + log.info('[Server] applying libre middleware across indices.'); break; case 'regular': - console.log('[Server] applying site-wide regular middleware.'); + log.info('[Server] applying regular middleware across indices.'); middlewares.push(tierAccessMiddleware); break; case 'private': - console.log('[Server] applying site-wide private middleware.'); + log.info('[Server] applying private middleware across indices.'); middlewares.push(authMiddleware); break; default: - console.log('[Server] applying index-scoped middleware.'); + log.info('[Server] applying index-scoped middleware.'); middlewares.push(perIndexTierAccessMiddleware); break; } diff --git a/src/server/schema.js b/src/server/schema.js index 5b51851a..15271eca 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -58,14 +58,15 @@ const getAggsHistogramName = (gqlType) => { const getObjNameFromESIndex = (esIndex) => { let esTypeObjName = firstLetterUpperCase(esIndex); // Choosing to replace hyphens with the string "_DASH_". - // It's self-documenting, lends well to readability, and the - // likelihood of a user naming an ES index with the string _DASH_ is low. + // It's self-documenting, lends well to readability, and the + // likelihood of a user naming an ES index with the string _DASH_ is low. esTypeObjName = esTypeObjName.replaceAll('-', '_DASH_'); return esTypeObjName; }; +/* eslint-disable no-unused-vars */ const getQuerySchemaForType = (esType) => { - // This function is now unused in favor of an + // This function is now unused in favor of an // index-scoped query schema. const esTypeObjName = firstLetterUpperCase(esType); return `${esType} ( @@ -88,7 +89,7 @@ const getQuerySchemaForIndex = (esIndex) => { filter: JSON, sort: JSON, accessibility: Accessibility=all, - ): [${esTypeObjName}]`; + ): [${esIndexObjName}]`; }; const getFieldGQLTypeMapForProperties = (esInstance, esIndex, properties) => { From e0d6b7309798f870155fc4acf9181cf6750079b3 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 19 Jan 2021 18:14:18 -0600 Subject: [PATCH 14/67] eslint --- src/server/schema.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/schema.js b/src/server/schema.js index 15271eca..2f3102da 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -60,7 +60,7 @@ const getObjNameFromESIndex = (esIndex) => { // Choosing to replace hyphens with the string "_DASH_". // It's self-documenting, lends well to readability, and the // likelihood of a user naming an ES index with the string _DASH_ is low. - esTypeObjName = esTypeObjName.replaceAll('-', '_DASH_'); + esTypeObjName = esTypeObjName.replace(/-/g, '_DASH_'); return esTypeObjName; }; From 6a342d6fd0a1e56154ab457fccc47b0ac9bdf39a Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Thu, 21 Jan 2021 12:16:35 -0600 Subject: [PATCH 15/67] undo schema changes --- src/server/schema.js | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/src/server/schema.js b/src/server/schema.js index 2f3102da..3cdc2ea6 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -55,19 +55,7 @@ const getAggsHistogramName = (gqlType) => { return gqlTypeToAggsHistogramName[gqlType]; }; -const getObjNameFromESIndex = (esIndex) => { - let esTypeObjName = firstLetterUpperCase(esIndex); - // Choosing to replace hyphens with the string "_DASH_". - // It's self-documenting, lends well to readability, and the - // likelihood of a user naming an ES index with the string _DASH_ is low. - esTypeObjName = esTypeObjName.replace(/-/g, '_DASH_'); - return esTypeObjName; -}; - -/* eslint-disable no-unused-vars */ const getQuerySchemaForType = (esType) => { - // This function is now unused in favor of an - // index-scoped query schema. const esTypeObjName = firstLetterUpperCase(esType); return `${esType} ( offset: Int, @@ -79,19 +67,6 @@ const getQuerySchemaForType = (esType) => { ): [${esTypeObjName}]`; }; -const getQuerySchemaForIndex = (esIndex) => { - // This function helps build an index-scoped query schema - // so as to scope middleware by index rather than type. - const esIndexObjName = getObjNameFromESIndex(esIndex); - return `${esIndexObjName} ( - offset: Int, - first: Int, - filter: JSON, - sort: JSON, - accessibility: Accessibility=all, - ): [${esIndexObjName}]`; -}; - const getFieldGQLTypeMapForProperties = (esInstance, esIndex, properties) => { const result = Object.keys(properties).map((field) => { const esFieldType = (properties[field].esType) @@ -188,7 +163,7 @@ const getAggregationSchemaForOneIndex = (esInstance, esIndex, esType) => { export const getQuerySchema = (esConfig) => ` type Query { - ${esConfig.indices.map((cfg) => getQuerySchemaForIndex(cfg.index)).join('\n')} + ${esConfig.indices.map((cfg) => getQuerySchemaForType(cfg.type)).join('\n')} _aggregation: Aggregation _mapping: Mapping } From 6a0c5bbec091da887b1e925bdcd6db5e076eaff7 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Thu, 21 Jan 2021 14:22:04 -0600 Subject: [PATCH 16/67] fix places where i confused ES type with GQL type --- .../middlewares/authMiddleware/index.js | 2 +- .../perIndexTierAccessMiddleware/index.js | 32 +++++++++---------- .../perIndexTierAccessMiddleware/resolvers.js | 0 .../middlewares/tierAccessMiddleware/index.js | 12 +++---- 4 files changed, 23 insertions(+), 23 deletions(-) delete mode 100644 src/server/middlewares/perIndexTierAccessMiddleware/resolvers.js diff --git a/src/server/middlewares/authMiddleware/index.js b/src/server/middlewares/authMiddleware/index.js index eb5f2ee8..10f4aeaf 100644 --- a/src/server/middlewares/authMiddleware/index.js +++ b/src/server/middlewares/authMiddleware/index.js @@ -3,7 +3,7 @@ import authMWResolver from './resolvers'; // apply this middleware to all es types' data/aggregation resolvers const typeMapping = config.esConfig.indices.reduce((acc, item) => { - acc[item.index] = authMWResolver; + acc[item.type] = authMWResolver; return acc; }, {}); const authMiddleware = { diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index 2e65379e..1d9487c9 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -3,49 +3,49 @@ import { firstLetterUpperCase } from '../../utils/utils'; import authMWResolver from '../authMiddleware/resolvers'; import { tierAccessResolver, hideNumberResolver } from '../tierAccessMiddleware/resolvers'; -const queryIndexMapping = {}; -const aggsIndexMapping = {}; -const histogramIndexMapping = {}; -const histogramForStringIndexMapping = {}; +const queryTypeMapping = {}; +const aggsTypeMapping = {}; +const histogramTypeMapping = {}; +const histogramForStringTypeMapping = {}; const totalCountTypeMapping = {}; config.esConfig.indices.forEach((item) => { if (item.tier_access_level === 'private') { - queryIndexMapping[item.index] = authMWResolver; - aggsIndexMapping[item.index] = authMWResolver; + queryTypeMapping[item.type] = authMWResolver; + aggsTypeMapping[item.type] = authMWResolver; } else if (item.tier_access_level === 'regular') { - queryIndexMapping[item.index] = tierAccessResolver({ + queryTypeMapping[item.type] = tierAccessResolver({ isRawDataQuery: true, esType: item.type, - esIndex: item.index, + esIndex: item.type, }); - aggsIndexMapping[item.index] = tierAccessResolver({ esType: item.type, esIndex: item.index }); + aggsTypeMapping[item.type] = tierAccessResolver({ esType: item.type, esIndex: item.type }); const aggregationName = `${firstLetterUpperCase(item.type)}Aggregation`; totalCountTypeMapping[aggregationName] = { _totalCount: hideNumberResolver(true), }; - histogramIndexMapping[item.index] = { histogram: hideNumberResolver(false) }; - histogramForStringIndexMapping[item.index] = { histogram: hideNumberResolver(false) }; + histogramTypeMapping[item.type] = { histogram: hideNumberResolver(false) }; + histogramForStringTypeMapping[item.type] = { histogram: hideNumberResolver(false) }; } else if (item.tier_access_level === 'libre') { // No additional resolvers necessary } else { - throw new Error(`tier_access_level invalid for index ${item.index}. Either set all index-scoped tiered-access levels or a site-wide tiered-access level.`); + throw new Error(`tier_access_level invalid for index ${item.type}. Either set all index-scoped tiered-access levels or a site-wide tiered-access level.`); } }, {}); const perIndexTierAccessMiddleware = { Query: { - ...queryIndexMapping, + ...queryTypeMapping, }, Aggregation: { - ...aggsIndexMapping, + ...aggsTypeMapping, }, ...totalCountTypeMapping, HistogramForNumber: { - ...histogramIndexMapping, + ...histogramTypeMapping, }, HistogramForString: { - ...histogramForStringIndexMapping, + ...histogramForStringTypeMapping, }, }; diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/resolvers.js b/src/server/middlewares/perIndexTierAccessMiddleware/resolvers.js deleted file mode 100644 index e69de29b..00000000 diff --git a/src/server/middlewares/tierAccessMiddleware/index.js b/src/server/middlewares/tierAccessMiddleware/index.js index a4d93f26..86212f30 100644 --- a/src/server/middlewares/tierAccessMiddleware/index.js +++ b/src/server/middlewares/tierAccessMiddleware/index.js @@ -3,16 +3,16 @@ import { firstLetterUpperCase } from '../../utils/utils'; import { tierAccessResolver, hideNumberResolver } from './resolvers'; // apply this middleware to all es types' data/aggregation resolvers -const queryIndexMapping = {}; -const aggsIndexMapping = {}; +const queryTypeMapping = {}; +const aggsTypeMapping = {}; const totalCountTypeMapping = {}; config.esConfig.indices.forEach((item) => { - queryIndexMapping[item.index] = tierAccessResolver({ + queryTypeMapping[item.type] = tierAccessResolver({ isRawDataQuery: true, esType: item.type, esIndex: item.index, }); - aggsIndexMapping[item.index] = tierAccessResolver({ esType: item.type, esIndex: item.index }); + aggsTypeMapping[item.type] = tierAccessResolver({ esType: item.type, esIndex: item.index }); const aggregationName = `${firstLetterUpperCase(item.type)}Aggregation`; totalCountTypeMapping[aggregationName] = { _totalCount: hideNumberResolver(true), @@ -20,10 +20,10 @@ config.esConfig.indices.forEach((item) => { }); const tierAccessMiddleware = { Query: { - ...queryIndexMapping, + ...queryTypeMapping, }, Aggregation: { - ...aggsIndexMapping, + ...aggsTypeMapping, }, ...totalCountTypeMapping, HistogramForNumber: { From b6a8b92242f51da45d5f7be0f2981243f8b1aa33 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Thu, 21 Jan 2021 14:55:55 -0600 Subject: [PATCH 17/67] enhance auth middleware assertion --- src/server/middlewares/authMiddleware/resolvers.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/server/middlewares/authMiddleware/resolvers.js b/src/server/middlewares/authMiddleware/resolvers.js index c3bc9819..53d1db42 100644 --- a/src/server/middlewares/authMiddleware/resolvers.js +++ b/src/server/middlewares/authMiddleware/resolvers.js @@ -1,7 +1,16 @@ import log from '../../logger'; import config from '../../config'; +import esInstance from '../../es/index'; const authMWResolver = async (resolve, root, args, context, info) => { + // Assert that either this index is "private" access or + // that the index has no setting and site-wide config is "private". + const indexConfig = esInstance.getESIndexConfigByName(esIndex); + const indexIsPrivateAccess = indexConfig.tier_access_level === 'private'; + const tierAccessPrivateAndNotIndexScoped = !indexConfig.tier_access_level && config.tierAccessLevel === 'private'; + assert(indexIsPrivateAccess || tierAccessPrivateAndNotIndexScoped, 'Auth middleware layer only for "private" tier access level'); + + const { authHelper } = context; // if mock arborist endpoint, just skip auth middleware From 14bcb67e687d0659a0960e2012bb28d9092181ef Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Thu, 21 Jan 2021 15:23:34 -0600 Subject: [PATCH 18/67] adjusting new resolver logics --- .../middlewares/perIndexTierAccessMiddleware/index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index 1d9487c9..270034a2 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -3,6 +3,12 @@ import { firstLetterUpperCase } from '../../utils/utils'; import authMWResolver from '../authMiddleware/resolvers'; import { tierAccessResolver, hideNumberResolver } from '../tierAccessMiddleware/resolvers'; +const isPrivate = (index) => index.tier_access_level === 'private'; +const isRegular = (index) => index.tier_access_level === 'regular'; +const isLibre = (index) => index.tier_access_level === 'libre'; +const atLeastOneIndexIsPrivate = config.esConfig.indices.some(isPrivate); +const atLeastOneIndexIsRegular = config.esConfig.indices.some(isRegular); + const queryTypeMapping = {}; const aggsTypeMapping = {}; const histogramTypeMapping = {}; From 0609a75af2ccde88f418973d61aaecc7d7342110 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Thu, 21 Jan 2021 15:23:51 -0600 Subject: [PATCH 19/67] adjusting new resolver logics --- .../middlewares/perIndexTierAccessMiddleware/index.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index 270034a2..1d9487c9 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -3,12 +3,6 @@ import { firstLetterUpperCase } from '../../utils/utils'; import authMWResolver from '../authMiddleware/resolvers'; import { tierAccessResolver, hideNumberResolver } from '../tierAccessMiddleware/resolvers'; -const isPrivate = (index) => index.tier_access_level === 'private'; -const isRegular = (index) => index.tier_access_level === 'regular'; -const isLibre = (index) => index.tier_access_level === 'libre'; -const atLeastOneIndexIsPrivate = config.esConfig.indices.some(isPrivate); -const atLeastOneIndexIsRegular = config.esConfig.indices.some(isRegular); - const queryTypeMapping = {}; const aggsTypeMapping = {}; const histogramTypeMapping = {}; From 461d1e534a984a1ae3748bb19bf33a851524078d Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Thu, 21 Jan 2021 15:34:35 -0600 Subject: [PATCH 20/67] eslint --- src/server/middlewares/authMiddleware/resolvers.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/server/middlewares/authMiddleware/resolvers.js b/src/server/middlewares/authMiddleware/resolvers.js index 53d1db42..c3bc9819 100644 --- a/src/server/middlewares/authMiddleware/resolvers.js +++ b/src/server/middlewares/authMiddleware/resolvers.js @@ -1,16 +1,7 @@ import log from '../../logger'; import config from '../../config'; -import esInstance from '../../es/index'; const authMWResolver = async (resolve, root, args, context, info) => { - // Assert that either this index is "private" access or - // that the index has no setting and site-wide config is "private". - const indexConfig = esInstance.getESIndexConfigByName(esIndex); - const indexIsPrivateAccess = indexConfig.tier_access_level === 'private'; - const tierAccessPrivateAndNotIndexScoped = !indexConfig.tier_access_level && config.tierAccessLevel === 'private'; - assert(indexIsPrivateAccess || tierAccessPrivateAndNotIndexScoped, 'Auth middleware layer only for "private" tier access level'); - - const { authHelper } = context; // if mock arborist endpoint, just skip auth middleware From af157a33ed316ee6f3cc7ccef0c225ee3707bf86 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Fri, 22 Jan 2021 01:50:10 -0600 Subject: [PATCH 21/67] fixing bug in manifest var logivs --- src/server/config.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index 1f7998ac..bbaee657 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -15,12 +15,10 @@ const config = { { index: 'gen3-dev-subject', type: 'subject', - tier_access_level: 'private', }, { index: 'gen3-dev-file', type: 'file', - tier_access_level: 'private', }, ], configIndex: (inputConfig.indices) ? inputConfig.config_index : 'gen3-dev-config', @@ -76,12 +74,22 @@ if (process.env.ANALYZED_TEXT_FIELD_SUFFIX) { // Either all indices should have explicit index-scoped tiered-access values or // the manifest should have a site-wide TIER_ACCESS_LEVEL value. // This approach is backwards-compatible with commons configured for past versions of tiered-access. +let allIndicesHaveTierAccessSettings = true; config.esConfig.indices.forEach((item) => { if (!item.tier_access_level && !process.env.TIER_ACCESS_LEVEL) { throw new Error('Either set all index-scoped tiered-access levels or a site-wide tiered-access level.'); } + if (!item.tier_access_level) { + allIndicesHaveTierAccessSettings = false; + } }); +// If there is no site-wide TIER_ACCESS_LEVEL value and the indices all have settings, +// empty out the default TIER_ACCESS_LEVEL from the config. +if (!process.env.TIER_ACCESS_LIMIT && allIndicesHaveTierAccessSettings) { + config.tierAccessLimit = process.env.TIER_ACCESS_LIMIT; +} + if (process.env.TIER_ACCESS_LEVEL) { if (process.env.TIER_ACCESS_LEVEL !== 'private' && process.env.TIER_ACCESS_LEVEL !== 'regular' From 83b897f4730ef458eb0f8d81bdcfdb472f136b39 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Fri, 22 Jan 2021 10:12:06 -0600 Subject: [PATCH 22/67] clarify manifest logic --- src/server/config.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index bbaee657..9c2d46b9 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -55,6 +55,15 @@ if (process.env.GUPPY_PORT) { config.port = process.env.GUPPY_PORT; } +if (process.env.TIER_ACCESS_LEVEL) { + if (process.env.TIER_ACCESS_LEVEL !== 'private' + && process.env.TIER_ACCESS_LEVEL !== 'regular' + && process.env.TIER_ACCESS_LEVEL !== 'libre') { + throw new Error(`Invalid TIER_ACCESS_LEVEL "${process.env.TIER_ACCESS_LEVEL}"`); + } + config.tierAccessLevel = process.env.TIER_ACCESS_LEVEL; +} + if (process.env.TIER_ACCESS_LIMIT) { config.tierAccessLimit = process.env.TIER_ACCESS_LIMIT; } @@ -76,7 +85,7 @@ if (process.env.ANALYZED_TEXT_FIELD_SUFFIX) { // This approach is backwards-compatible with commons configured for past versions of tiered-access. let allIndicesHaveTierAccessSettings = true; config.esConfig.indices.forEach((item) => { - if (!item.tier_access_level && !process.env.TIER_ACCESS_LEVEL) { + if (!item.tier_access_level && !config.tierAccessLevel) { throw new Error('Either set all index-scoped tiered-access levels or a site-wide tiered-access level.'); } if (!item.tier_access_level) { @@ -86,17 +95,8 @@ config.esConfig.indices.forEach((item) => { // If there is no site-wide TIER_ACCESS_LEVEL value and the indices all have settings, // empty out the default TIER_ACCESS_LEVEL from the config. -if (!process.env.TIER_ACCESS_LIMIT && allIndicesHaveTierAccessSettings) { - config.tierAccessLimit = process.env.TIER_ACCESS_LIMIT; -} - -if (process.env.TIER_ACCESS_LEVEL) { - if (process.env.TIER_ACCESS_LEVEL !== 'private' - && process.env.TIER_ACCESS_LEVEL !== 'regular' - && process.env.TIER_ACCESS_LEVEL !== 'libre') { - throw new Error(`Invalid TIER_ACCESS_LEVEL "${process.env.TIER_ACCESS_LEVEL}"`); - } - config.tierAccessLevel = process.env.TIER_ACCESS_LEVEL; +if (allIndicesHaveTierAccessSettings && !process.env.TIER_ACCESS_LIMIT) { + delete config.tierAccessLimit; } // check whitelist is enabled From 137b67372cd9a018861d949458d3512546b03397 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Fri, 22 Jan 2021 11:26:36 -0600 Subject: [PATCH 23/67] clarify manifest logic --- src/server/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/config.js b/src/server/config.js index 9c2d46b9..e1e64435 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -96,7 +96,7 @@ config.esConfig.indices.forEach((item) => { // If there is no site-wide TIER_ACCESS_LEVEL value and the indices all have settings, // empty out the default TIER_ACCESS_LEVEL from the config. if (allIndicesHaveTierAccessSettings && !process.env.TIER_ACCESS_LIMIT) { - delete config.tierAccessLimit; + delete config.tierAccessLevel; } // check whitelist is enabled From 3b85d4efcfd7aea3593379c496014216dd56035e Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 25 Jan 2021 15:12:23 -0600 Subject: [PATCH 24/67] histogram schema --- .../perIndexTierAccessMiddleware/index.js | 12 ++++++------ src/server/schema.js | 12 ++++++++++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index 1d9487c9..ba1557bd 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -5,7 +5,7 @@ import { tierAccessResolver, hideNumberResolver } from '../tierAccessMiddleware/ const queryTypeMapping = {}; const aggsTypeMapping = {}; -const histogramTypeMapping = {}; +const histogramForNumberTypeMapping = {}; const histogramForStringTypeMapping = {}; const totalCountTypeMapping = {}; @@ -24,8 +24,8 @@ config.esConfig.indices.forEach((item) => { totalCountTypeMapping[aggregationName] = { _totalCount: hideNumberResolver(true), }; - histogramTypeMapping[item.type] = { histogram: hideNumberResolver(false) }; - histogramForStringTypeMapping[item.type] = { histogram: hideNumberResolver(false) }; + histogramForNumberTypeMapping[item.type] = hideNumberResolver(false); + histogramForStringTypeMapping[item.type] = hideNumberResolver(false); } else if (item.tier_access_level === 'libre') { // No additional resolvers necessary } else { @@ -41,10 +41,10 @@ const perIndexTierAccessMiddleware = { ...aggsTypeMapping, }, ...totalCountTypeMapping, - HistogramForNumber: { - ...histogramTypeMapping, + RegularAccessHistogramForNumber: { + ...histogramForNumberTypeMapping, }, - HistogramForString: { + RegularAccessHistogramForString: { ...histogramForStringTypeMapping, }, }; diff --git a/src/server/schema.js b/src/server/schema.js index 3cdc2ea6..23109e50 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -147,11 +147,15 @@ const getAggregationType = (entry) => { }; const getAggregationSchemaForOneIndex = (esInstance, esIndex, esType) => { + let histogramTypePrefix = ''; + if (esIndex.tier_access_level === 'regular') { + histogramTypePrefix = 'RegularAccess'; + } const esTypeObjName = firstLetterUpperCase(esType); const fieldGQLTypeMap = getFieldGQLTypeMapForOneIndex(esInstance, esIndex); const fieldAggsTypeMap = fieldGQLTypeMap.filter((f) => f.esType !== 'nested').map((entry) => ({ field: entry.field, - aggType: getAggsHistogramName(entry.type), + aggType: histogramTypePrefix + getAggsHistogramName(entry.type), })); const fieldAggsNestedTypeMap = fieldGQLTypeMap.filter((f) => f.esType === 'nested'); return `type ${esTypeObjName}Aggregation { @@ -191,6 +195,10 @@ export const getAggregationSchema = (esConfig) => ` const getAggregationSchemaForOneNestedIndex = (esInstance, esIndex) => { const fieldGQLTypeMap = getFieldGQLTypeMapForOneIndex(esInstance, esIndex); const fieldAggsNestedTypeMap = fieldGQLTypeMap.filter((f) => f.esType === 'nested'); + let histogramTypePrefix = ''; + if (esIndex.tier_access_level === 'regular') { + histogramTypePrefix = 'RegularAccess'; + } let AggsNestedTypeSchema = ''; while (fieldAggsNestedTypeMap.length > 0) { @@ -207,7 +215,7 @@ const getAggregationSchemaForOneNestedIndex = (esInstance, esIndex) => { ${propsKey}: NestedHistogramFor${firstLetterUpperCase(propsKey)}`; } return ` - ${propsKey}: ${getAggsHistogramName(esgqlTypeMapping[entryType])}`; + ${propsKey}: ${histogramTypePrefix + getAggsHistogramName(esgqlTypeMapping[entryType])}`; })} }`; } From 5154bcc36a03ab65676504b45cd3dbac3db2d1e9 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 25 Jan 2021 16:15:40 -0600 Subject: [PATCH 25/67] histogram schema --- src/server/schema.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/server/schema.js b/src/server/schema.js index 23109e50..5547ebfc 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -146,9 +146,13 @@ const getAggregationType = (entry) => { return ''; }; -const getAggregationSchemaForOneIndex = (esInstance, esIndex, esType) => { +const getAggregationSchemaForOneIndex = (esInstance, esDict) => { + const esIndex = esDict.type; + const esType = esDict.type; let histogramTypePrefix = ''; - if (esIndex.tier_access_level === 'regular') { + // eslint-disable-next-line no-console + console.log('>>>>> inside getAggregationSchemaForOneIndex; esIndex: ', esIndex); + if (Object.prototype.hasOwnProperty.call(esDict, 'tier_access_level') && esDict.tier_access_level === 'regular') { histogramTypePrefix = 'RegularAccess'; } const esTypeObjName = firstLetterUpperCase(esType); @@ -192,11 +196,12 @@ export const getAggregationSchema = (esConfig) => ` * Multi-level nested fields are "flattened" level by level. * For each level of nested field a new type in schema is created. */ -const getAggregationSchemaForOneNestedIndex = (esInstance, esIndex) => { +const getAggregationSchemaForOneNestedIndex = (esInstance, esDict) => { + const esIndex = esDict.type; const fieldGQLTypeMap = getFieldGQLTypeMapForOneIndex(esInstance, esIndex); const fieldAggsNestedTypeMap = fieldGQLTypeMap.filter((f) => f.esType === 'nested'); let histogramTypePrefix = ''; - if (esIndex.tier_access_level === 'regular') { + if (Object.prototype.hasOwnProperty.call(esDict, 'tier_access_level') && esDict.tier_access_level === 'regular') { histogramTypePrefix = 'RegularAccess'; } @@ -224,9 +229,9 @@ const getAggregationSchemaForOneNestedIndex = (esInstance, esIndex) => { return AggsNestedTypeSchema; }; -export const getAggregationSchemaForEachType = (esConfig, esInstance) => esConfig.indices.map((cfg) => getAggregationSchemaForOneIndex(esInstance, cfg.index, cfg.type)).join('\n'); +export const getAggregationSchemaForEachType = (esConfig, esInstance) => esConfig.indices.map((cfg) => getAggregationSchemaForOneIndex(esInstance, cfg)).join('\n'); -export const getAggregationSchemaForEachNestedType = (esConfig, esInstance) => esConfig.indices.map((cfg) => getAggregationSchemaForOneNestedIndex(esInstance, cfg.index)).join('\n'); +export const getAggregationSchemaForEachNestedType = (esConfig, esInstance) => esConfig.indices.map((cfg) => getAggregationSchemaForOneNestedIndex(esInstance, cfg)).join('\n'); export const getMappingSchema = (esConfig) => ` type Mapping { From eed451e17e21fb634535e98359a05d8d36a9b1e3 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 25 Jan 2021 16:25:31 -0600 Subject: [PATCH 26/67] typo fixes --- src/server/schema.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/schema.js b/src/server/schema.js index 5547ebfc..a27950df 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -147,11 +147,11 @@ const getAggregationType = (entry) => { }; const getAggregationSchemaForOneIndex = (esInstance, esDict) => { - const esIndex = esDict.type; + const esIndex = esDict.index; const esType = esDict.type; let histogramTypePrefix = ''; // eslint-disable-next-line no-console - console.log('>>>>> inside getAggregationSchemaForOneIndex; esIndex: ', esIndex); + console.log('>>>>> inside getAggregationSchemaForOneIndex; esIndex: ', esDict); if (Object.prototype.hasOwnProperty.call(esDict, 'tier_access_level') && esDict.tier_access_level === 'regular') { histogramTypePrefix = 'RegularAccess'; } @@ -197,7 +197,7 @@ export const getAggregationSchema = (esConfig) => ` * For each level of nested field a new type in schema is created. */ const getAggregationSchemaForOneNestedIndex = (esInstance, esDict) => { - const esIndex = esDict.type; + const esIndex = esDict.index; const fieldGQLTypeMap = getFieldGQLTypeMapForOneIndex(esInstance, esIndex); const fieldAggsNestedTypeMap = fieldGQLTypeMap.filter((f) => f.esType === 'nested'); let histogramTypePrefix = ''; From 5c8527edf485cc0d9e6dd649078853813836f72a Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 25 Jan 2021 16:49:45 -0600 Subject: [PATCH 27/67] add to schema --- src/server/schema.js | 54 +++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/src/server/schema.js b/src/server/schema.js index a27950df..bf1a7a95 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -233,6 +233,36 @@ export const getAggregationSchemaForEachType = (esConfig, esInstance) => esConfi export const getAggregationSchemaForEachNestedType = (esConfig, esInstance) => esConfig.indices.map((cfg) => getAggregationSchemaForOneNestedIndex(esInstance, cfg)).join('\n'); +const getNumberHistogramSchema = (isRegularAccess) => { + let histogramTypePrefix = ''; + if (isRegularAccess) { + histogramTypePrefix = 'RegularAccess'; + } + return ` + type ${histogramTypePrefix + EnumAggsHistogramName.HISTOGRAM_FOR_NUMBER} { + histogram( + rangeStart: Int, + rangeEnd: Int, + rangeStep: Int, + binCount: Int, + ): [BucketsForNestedNumberAgg], + asTextHistogram: [BucketsForNestedStringAgg] + } + `; +}; + +const getTextHistogramSchema = (isRegularAccess) => { + let histogramTypePrefix = ''; + if (isRegularAccess) { + histogramTypePrefix = 'RegularAccess'; + } + return ` + type ${histogramTypePrefix + EnumAggsHistogramName.HISTOGRAM_FOR_STRING} { + histogram: [BucketsForNestedStringAgg] + } + `; +}; + export const getMappingSchema = (esConfig) => ` type Mapping { ${esConfig.indices.map((cfg) => `${cfg.type} ( @@ -276,11 +306,9 @@ export const buildSchemaString = (esConfig, esInstance) => { const aggregationSchemasForEachNestedType = getAggregationSchemaForEachNestedType(esConfig, esInstance); - const textHistogramSchema = ` - type ${EnumAggsHistogramName.HISTOGRAM_FOR_STRING} { - histogram: [BucketsForNestedStringAgg] - } - `; + const textHistogramSchema = getTextHistogramSchema(false); + + const regularAccessTextHistogramSchema = getTextHistogramSchema(true); const textHistogramBucketSchema = ` type BucketsForNestedStringAgg { @@ -312,17 +340,9 @@ export const buildSchemaString = (esConfig, esInstance) => { } `; - const numberHistogramSchema = ` - type ${EnumAggsHistogramName.HISTOGRAM_FOR_NUMBER} { - histogram( - rangeStart: Int, - rangeEnd: Int, - rangeStep: Int, - binCount: Int, - ): [BucketsForNestedNumberAgg], - asTextHistogram: [BucketsForNestedStringAgg] - } - `; + const numberHistogramSchema = getNumberHistogramSchema(false); + + const regularAccessNumberHistogramSchema = getNumberHistogramSchema(true); const numberHistogramBucketSchema = ` type BucketsForNestedNumberAgg { @@ -351,7 +371,9 @@ export const buildSchemaString = (esConfig, esInstance) => { ${aggregationSchemasForEachType} ${aggregationSchemasForEachNestedType} ${textHistogramSchema} + ${regularAccessTextHistogramSchema} ${numberHistogramSchema} + ${regularAccessNumberHistogramSchema} ${textHistogramBucketSchema} ${nestedMissingFieldsBucketSchema} ${nestedTermsFieldsBucketSchema} From 8e8091c27ba18a484a74575b3f39f2d8379f71c3 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 25 Jan 2021 17:13:35 -0600 Subject: [PATCH 28/67] add to schema --- .../middlewares/perIndexTierAccessMiddleware/index.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index ba1557bd..0899eb9b 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -5,8 +5,6 @@ import { tierAccessResolver, hideNumberResolver } from '../tierAccessMiddleware/ const queryTypeMapping = {}; const aggsTypeMapping = {}; -const histogramForNumberTypeMapping = {}; -const histogramForStringTypeMapping = {}; const totalCountTypeMapping = {}; config.esConfig.indices.forEach((item) => { @@ -24,8 +22,6 @@ config.esConfig.indices.forEach((item) => { totalCountTypeMapping[aggregationName] = { _totalCount: hideNumberResolver(true), }; - histogramForNumberTypeMapping[item.type] = hideNumberResolver(false); - histogramForStringTypeMapping[item.type] = hideNumberResolver(false); } else if (item.tier_access_level === 'libre') { // No additional resolvers necessary } else { @@ -42,10 +38,10 @@ const perIndexTierAccessMiddleware = { }, ...totalCountTypeMapping, RegularAccessHistogramForNumber: { - ...histogramForNumberTypeMapping, + histogram: hideNumberResolver(false), }, RegularAccessHistogramForString: { - ...histogramForStringTypeMapping, + histogram: hideNumberResolver(false), }, }; From a4985b5fd2237a556dff17a4d63fa5a1f3d9fdfd Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 26 Jan 2021 15:35:33 -0600 Subject: [PATCH 29/67] fix client-side error --- src/server/middlewares/perIndexTierAccessMiddleware/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index 0899eb9b..1f2812a2 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -15,9 +15,9 @@ config.esConfig.indices.forEach((item) => { queryTypeMapping[item.type] = tierAccessResolver({ isRawDataQuery: true, esType: item.type, - esIndex: item.type, + esIndex: item.index, }); - aggsTypeMapping[item.type] = tierAccessResolver({ esType: item.type, esIndex: item.type }); + aggsTypeMapping[item.type] = tierAccessResolver({ esType: item.type, esIndex: item.index }); const aggregationName = `${firstLetterUpperCase(item.type)}Aggregation`; totalCountTypeMapping[aggregationName] = { _totalCount: hideNumberResolver(true), From 677fe99135344dbd6f6c5b463205f3a6c25cff5e Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 26 Jan 2021 16:21:44 -0600 Subject: [PATCH 30/67] fix client-side error --- src/server/resolvers.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/server/resolvers.js b/src/server/resolvers.js index 19579582..e9d2be4b 100644 --- a/src/server/resolvers.js +++ b/src/server/resolvers.js @@ -265,6 +265,13 @@ const getResolver = (esConfig, esInstance) => { HistogramForString: { histogram: textHistogramResolver, }, + RegularAccessHistogramForNumber: { + histogram: numericHistogramResolver, + asTextHistogram: textHistogramResolver, + }, + RegularAccessHistogramForString: { + histogram: textHistogramResolver, + }, Mapping: { ...mappingResolvers, }, From 0768a54da8ff3dc0c1e973c733d6eece6f277173 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Fri, 29 Jan 2021 10:44:30 -0600 Subject: [PATCH 31/67] fix logic problem --- src/server/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/config.js b/src/server/config.js index e1e64435..b33897fd 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -95,7 +95,7 @@ config.esConfig.indices.forEach((item) => { // If there is no site-wide TIER_ACCESS_LEVEL value and the indices all have settings, // empty out the default TIER_ACCESS_LEVEL from the config. -if (allIndicesHaveTierAccessSettings && !process.env.TIER_ACCESS_LIMIT) { +if (allIndicesHaveTierAccessSettings) { delete config.tierAccessLevel; } From ff9d749e295169b4faf7131eb08dac242a733b00 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Fri, 29 Jan 2021 10:52:49 -0600 Subject: [PATCH 32/67] adjust comment --- src/server/config.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index b33897fd..e035bab0 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -93,8 +93,7 @@ config.esConfig.indices.forEach((item) => { } }); -// If there is no site-wide TIER_ACCESS_LEVEL value and the indices all have settings, -// empty out the default TIER_ACCESS_LEVEL from the config. +// If the indices all have settings, empty out the default site-wide TIER_ACCESS_LEVEL from the config. if (allIndicesHaveTierAccessSettings) { delete config.tierAccessLevel; } From 73dba726e4218f507d342f6d69a788b3279c684b Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Fri, 29 Jan 2021 11:03:12 -0600 Subject: [PATCH 33/67] eslint --- src/server/config.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/server/config.js b/src/server/config.js index e035bab0..fde1f982 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -93,7 +93,8 @@ config.esConfig.indices.forEach((item) => { } }); -// If the indices all have settings, empty out the default site-wide TIER_ACCESS_LEVEL from the config. +// If the indices all have settings, empty out the default +// site-wide TIER_ACCESS_LEVEL from the config. if (allIndicesHaveTierAccessSettings) { delete config.tierAccessLevel; } From d8fc09a930a53ffa40a0eb8d2165fa7935bad102 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 12:36:22 -0600 Subject: [PATCH 34/67] feat: add doc --- doc/docs:index_scoped_tiered_access.md | 40 ++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 doc/docs:index_scoped_tiered_access.md diff --git a/doc/docs:index_scoped_tiered_access.md b/doc/docs:index_scoped_tiered_access.md new file mode 100644 index 00000000..f7bfb399 --- /dev/null +++ b/doc/docs:index_scoped_tiered_access.md @@ -0,0 +1,40 @@ +# Index-scoped Tiered-Access + +Most commons tend to use a site-wide tiered access configuration that applies across indices. However, some use cases require index-scoped permissioning. One example is the case of an open-access study viewer where studies have a mix of public properties and controlled-access properties. Another example is a Data Explorer that presents data types with different permission requirements meant to serve a variety of audiences. + +For these use cases, tiered-access settings can be specified at the index-level rather than the site-wide level. The expectation is that either a site-wide tiered-access level is set in the global block of the manifest OR all indices in the guppy config block have a tiered-access level set. Guppy will throw an error if the config settings do not meet one of these two expectations. + +You can set index-scoped tiered-access levels using the `tier_access_level` properties index objects in the guppy block of a common's `manifest.json`. Note that the `tier_access_limit` setting is still site-wide and configurable in the manifest's `global` block. +``` +"guppy": { + "indices": [ + { + "index": "subject_regular", + "type": "subject", + "tier_access_level": "regular" + }, + { + "index": "subject_private", + "type": "subject_private", + "tier_access_level": "private" + }, + { + "index": "file_private", + "type": "file", + "tier_access_level": "private" + }, + { + "index": "studies_open", + "type": "studies_open", + "tier_access_level": "libre" + }, + { + "index": "studies_controlled_access", + "type": "studies_controlled_access", + "tier_access_level": "private" + } + ], + "auth_filter_field": "auth_resource_path", + ... + }, +``` \ No newline at end of file From 09c4164042f2877fb39b21b92913fb71f47103fb Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 12:39:26 -0600 Subject: [PATCH 35/67] feat: add doc --- doc/docs:index_scoped_tiered_access.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/docs:index_scoped_tiered_access.md b/doc/docs:index_scoped_tiered_access.md index f7bfb399..ac9293be 100644 --- a/doc/docs:index_scoped_tiered_access.md +++ b/doc/docs:index_scoped_tiered_access.md @@ -1,10 +1,10 @@ # Index-scoped Tiered-Access -Most commons tend to use a site-wide tiered access configuration that applies across indices. However, some use cases require index-scoped permissioning. One example is the case of an open-access study viewer where studies have a mix of public properties and controlled-access properties. Another example is a Data Explorer that presents data types with different permission requirements meant to serve a variety of audiences. +Most commons use a site-wide tiered access configuration that applies across indices. However, some use cases require index-scoped permissioning. One example is the case of an open-access study viewer where studies have a mix of public properties and controlled-access properties. Another example is a Data Explorer that presents data types with different permission requirements meant to serve a variety of audiences. For these use cases, tiered-access settings can be specified at the index-level rather than the site-wide level. -For these use cases, tiered-access settings can be specified at the index-level rather than the site-wide level. The expectation is that either a site-wide tiered-access level is set in the global block of the manifest OR all indices in the guppy config block have a tiered-access level set. Guppy will throw an error if the config settings do not meet one of these two expectations. +Guppy expects that either all indices in the guppy config block will have a tiered-access level set OR that a site-wide tiered-access level is set in the global block of the manifest. Guppy will throw an error if the config settings do not meet one of these two expectations. -You can set index-scoped tiered-access levels using the `tier_access_level` properties index objects in the guppy block of a common's `manifest.json`. Note that the `tier_access_limit` setting is still site-wide and configurable in the manifest's `global` block. +You can set index-scoped tiered-access levels using the `tier_access_level` properties in the guppy block of a common's `manifest.json`. Note that the `tier_access_limit` setting is still site-wide and configurable in the manifest's `global` block. ``` "guppy": { "indices": [ From 9700a00125a2cebf72a2d1b4804e981a7838b0f9 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 12:49:32 -0600 Subject: [PATCH 36/67] feat: add doc --- ...tiered_access.md => index_scoped_tiered_access.md} | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) rename doc/{docs:index_scoped_tiered_access.md => index_scoped_tiered_access.md} (90%) diff --git a/doc/docs:index_scoped_tiered_access.md b/doc/index_scoped_tiered_access.md similarity index 90% rename from doc/docs:index_scoped_tiered_access.md rename to doc/index_scoped_tiered_access.md index ac9293be..7f1eeeed 100644 --- a/doc/docs:index_scoped_tiered_access.md +++ b/doc/index_scoped_tiered_access.md @@ -6,32 +6,33 @@ Guppy expects that either all indices in the guppy config block will have a tier You can set index-scoped tiered-access levels using the `tier_access_level` properties in the guppy block of a common's `manifest.json`. Note that the `tier_access_limit` setting is still site-wide and configurable in the manifest's `global` block. ``` +... "guppy": { "indices": [ { "index": "subject_regular", "type": "subject", - "tier_access_level": "regular" + "tier_access_level": "regular" }, { "index": "subject_private", "type": "subject_private", - "tier_access_level": "private" + "tier_access_level": "private" }, { "index": "file_private", "type": "file", - "tier_access_level": "private" + "tier_access_level": "private" }, { "index": "studies_open", "type": "studies_open", - "tier_access_level": "libre" + "tier_access_level": "libre" }, { "index": "studies_controlled_access", "type": "studies_controlled_access", - "tier_access_level": "private" + "tier_access_level": "private" } ], "auth_filter_field": "auth_resource_path", From c081c6adc7289040d8371f2bae2f3d3ead171837 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 12:57:00 -0600 Subject: [PATCH 37/67] PR feedback: config validation --- src/server/config.js | 9 ++++++--- .../middlewares/perIndexTierAccessMiddleware/index.js | 2 -- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index fde1f982..610348f6 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -55,10 +55,10 @@ if (process.env.GUPPY_PORT) { config.port = process.env.GUPPY_PORT; } +const allowedTierAccessLevels = ['private', 'regular', 'libre']; + if (process.env.TIER_ACCESS_LEVEL) { - if (process.env.TIER_ACCESS_LEVEL !== 'private' - && process.env.TIER_ACCESS_LEVEL !== 'regular' - && process.env.TIER_ACCESS_LEVEL !== 'libre') { + if (!allowedTierAccessLevels.includes(process.env.TIER_ACCESS_LEVEL)) { throw new Error(`Invalid TIER_ACCESS_LEVEL "${process.env.TIER_ACCESS_LEVEL}"`); } config.tierAccessLevel = process.env.TIER_ACCESS_LEVEL; @@ -88,6 +88,9 @@ config.esConfig.indices.forEach((item) => { if (!item.tier_access_level && !config.tierAccessLevel) { throw new Error('Either set all index-scoped tiered-access levels or a site-wide tiered-access level.'); } + if(!allowedTierAccessLevels.includes(item.tier_access_level)) { + throw new Error(`tier_access_level invalid for index ${item.type}.`); + } if (!item.tier_access_level) { allIndicesHaveTierAccessSettings = false; } diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index 1f2812a2..a095b688 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -24,8 +24,6 @@ config.esConfig.indices.forEach((item) => { }; } else if (item.tier_access_level === 'libre') { // No additional resolvers necessary - } else { - throw new Error(`tier_access_level invalid for index ${item.type}. Either set all index-scoped tiered-access levels or a site-wide tiered-access level.`); } }, {}); From 6ade7bb0cec766a939ac89789b5f5f4be8563526 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 13:01:29 -0600 Subject: [PATCH 38/67] PR feedback: add RegularAccessHistograms only on-demand --- src/server/config.js | 2 +- .../perIndexTierAccessMiddleware/index.js | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/server/config.js b/src/server/config.js index 610348f6..6983d076 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -88,7 +88,7 @@ config.esConfig.indices.forEach((item) => { if (!item.tier_access_level && !config.tierAccessLevel) { throw new Error('Either set all index-scoped tiered-access levels or a site-wide tiered-access level.'); } - if(!allowedTierAccessLevels.includes(item.tier_access_level)) { + if (!allowedTierAccessLevels.includes(item.tier_access_level)) { throw new Error(`tier_access_level invalid for index ${item.type}.`); } if (!item.tier_access_level) { diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index a095b688..bfe323a2 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -6,12 +6,14 @@ import { tierAccessResolver, hideNumberResolver } from '../tierAccessMiddleware/ const queryTypeMapping = {}; const aggsTypeMapping = {}; const totalCountTypeMapping = {}; +let atLeastOneIndexIsRegularAccess = false; config.esConfig.indices.forEach((item) => { if (item.tier_access_level === 'private') { queryTypeMapping[item.type] = authMWResolver; aggsTypeMapping[item.type] = authMWResolver; } else if (item.tier_access_level === 'regular') { + atLeastOneIndexIsRegularAccess = true; queryTypeMapping[item.type] = tierAccessResolver({ isRawDataQuery: true, esType: item.type, @@ -35,12 +37,16 @@ const perIndexTierAccessMiddleware = { ...aggsTypeMapping, }, ...totalCountTypeMapping, - RegularAccessHistogramForNumber: { +}; + +if (atLeastOneIndexIsRegularAccess) { + perIndexTierAccessMiddleware.RegularAccessHistogramForNumber = { histogram: hideNumberResolver(false), - }, - RegularAccessHistogramForString: { + }; + + perIndexTierAccessMiddleware.RegularAccessHistogramForString = { histogram: hideNumberResolver(false), - }, -}; + }; +} export default perIndexTierAccessMiddleware; From 859155f44ca5f7d3d55ac0c19a313d83f1a2f744 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 13:12:10 -0600 Subject: [PATCH 39/67] PR feedback: remove redundant check from resolver --- src/server/middlewares/tierAccessMiddleware/resolvers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/middlewares/tierAccessMiddleware/resolvers.js b/src/server/middlewares/tierAccessMiddleware/resolvers.js index 543f81be..5d9bec34 100644 --- a/src/server/middlewares/tierAccessMiddleware/resolvers.js +++ b/src/server/middlewares/tierAccessMiddleware/resolvers.js @@ -45,8 +45,8 @@ export const tierAccessResolver = ( // that the index has no setting and site-wide config is "regular". const indexConfig = esInstance.getESIndexConfigByName(esIndex); const indexIsRegularAccess = indexConfig.tier_access_level === 'regular'; - const tierAccessRegularAndNotIndexScoped = !indexConfig.tier_access_level && config.tierAccessLevel === 'regular'; - assert(indexIsRegularAccess || tierAccessRegularAndNotIndexScoped, 'Tier access middleware layer only for "regular" tier access level'); + const siteIsRegularAccess = config.tierAccessLevel === 'regular'; + assert(indexIsRegularAccess || siteIsRegularAccess, 'Tier access middleware layer only for "regular" tier access level'); const { authHelper } = context; const { filter, filterSelf, accessibility } = args; From 0a6b870b34ea5a3a4b4f879d4857034f674b2e9f Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 13:13:11 -0600 Subject: [PATCH 40/67] PR feedback: remove extra console log --- src/server/schema.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/server/schema.js b/src/server/schema.js index bf1a7a95..e85f3a55 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -150,8 +150,6 @@ const getAggregationSchemaForOneIndex = (esInstance, esDict) => { const esIndex = esDict.index; const esType = esDict.type; let histogramTypePrefix = ''; - // eslint-disable-next-line no-console - console.log('>>>>> inside getAggregationSchemaForOneIndex; esIndex: ', esDict); if (Object.prototype.hasOwnProperty.call(esDict, 'tier_access_level') && esDict.tier_access_level === 'regular') { histogramTypePrefix = 'RegularAccess'; } From d967b56d51bfd71c39969a630d23b15e42c5dd2a Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 13:38:31 -0600 Subject: [PATCH 41/67] PR feedback: histogram RegularAccess ternary operator --- src/server/schema.js | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/src/server/schema.js b/src/server/schema.js index e85f3a55..e886d0b7 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -149,15 +149,13 @@ const getAggregationType = (entry) => { const getAggregationSchemaForOneIndex = (esInstance, esDict) => { const esIndex = esDict.index; const esType = esDict.type; - let histogramTypePrefix = ''; - if (Object.prototype.hasOwnProperty.call(esDict, 'tier_access_level') && esDict.tier_access_level === 'regular') { - histogramTypePrefix = 'RegularAccess'; - } + const histogramTypePrefix = 'RegularAccess'; + const includeHistogramPrefix = Object.prototype.hasOwnProperty.call(esDict, 'tier_access_level') && esDict.tier_access_level === 'regular'; const esTypeObjName = firstLetterUpperCase(esType); const fieldGQLTypeMap = getFieldGQLTypeMapForOneIndex(esInstance, esIndex); const fieldAggsTypeMap = fieldGQLTypeMap.filter((f) => f.esType !== 'nested').map((entry) => ({ field: entry.field, - aggType: histogramTypePrefix + getAggsHistogramName(entry.type), + aggType: (includeHistogramPrefix ? histogramTypePrefix : '') + getAggsHistogramName(entry.type), })); const fieldAggsNestedTypeMap = fieldGQLTypeMap.filter((f) => f.esType === 'nested'); return `type ${esTypeObjName}Aggregation { @@ -198,11 +196,8 @@ const getAggregationSchemaForOneNestedIndex = (esInstance, esDict) => { const esIndex = esDict.index; const fieldGQLTypeMap = getFieldGQLTypeMapForOneIndex(esInstance, esIndex); const fieldAggsNestedTypeMap = fieldGQLTypeMap.filter((f) => f.esType === 'nested'); - let histogramTypePrefix = ''; - if (Object.prototype.hasOwnProperty.call(esDict, 'tier_access_level') && esDict.tier_access_level === 'regular') { - histogramTypePrefix = 'RegularAccess'; - } - + const histogramTypePrefix = 'RegularAccess'; + const includeHistogramPrefix = Object.prototype.hasOwnProperty.call(esDict, 'tier_access_level') && esDict.tier_access_level === 'regular'; let AggsNestedTypeSchema = ''; while (fieldAggsNestedTypeMap.length > 0) { const entry = fieldAggsNestedTypeMap.shift(); @@ -218,7 +213,7 @@ const getAggregationSchemaForOneNestedIndex = (esInstance, esDict) => { ${propsKey}: NestedHistogramFor${firstLetterUpperCase(propsKey)}`; } return ` - ${propsKey}: ${histogramTypePrefix + getAggsHistogramName(esgqlTypeMapping[entryType])}`; + ${propsKey}: ${(includeHistogramPrefix ? histogramTypePrefix : '') + getAggsHistogramName(esgqlTypeMapping[entryType])}`; })} }`; } @@ -232,12 +227,9 @@ export const getAggregationSchemaForEachType = (esConfig, esInstance) => esConfi export const getAggregationSchemaForEachNestedType = (esConfig, esInstance) => esConfig.indices.map((cfg) => getAggregationSchemaForOneNestedIndex(esInstance, cfg)).join('\n'); const getNumberHistogramSchema = (isRegularAccess) => { - let histogramTypePrefix = ''; - if (isRegularAccess) { - histogramTypePrefix = 'RegularAccess'; - } + const histogramTypePrefix = 'RegularAccess'; return ` - type ${histogramTypePrefix + EnumAggsHistogramName.HISTOGRAM_FOR_NUMBER} { + type ${(isRegularAccess ? histogramTypePrefix : '') + EnumAggsHistogramName.HISTOGRAM_FOR_NUMBER} { histogram( rangeStart: Int, rangeEnd: Int, @@ -250,12 +242,9 @@ const getNumberHistogramSchema = (isRegularAccess) => { }; const getTextHistogramSchema = (isRegularAccess) => { - let histogramTypePrefix = ''; - if (isRegularAccess) { - histogramTypePrefix = 'RegularAccess'; - } + const histogramTypePrefix = 'RegularAccess'; return ` - type ${histogramTypePrefix + EnumAggsHistogramName.HISTOGRAM_FOR_STRING} { + type ${(isRegularAccess ? histogramTypePrefix : '') + EnumAggsHistogramName.HISTOGRAM_FOR_STRING} { histogram: [BucketsForNestedStringAgg] } `; From e167e368c8209c013898099a9b5f739640b8dd30 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 13:52:30 -0600 Subject: [PATCH 42/67] PR feedback: add unit test --- src/server/__tests__/schema.test.js | 32 ++++++++++++++++ src/server/schema.js | 57 ++++++++++++++++------------- 2 files changed, 63 insertions(+), 26 deletions(-) diff --git a/src/server/__tests__/schema.test.js b/src/server/__tests__/schema.test.js index 8afecefd..b9e01505 100644 --- a/src/server/__tests__/schema.test.js +++ b/src/server/__tests__/schema.test.js @@ -143,4 +143,36 @@ describe('Schema', () => { expect(removeSpacesNewlinesAndDes(mappingSchema)) .toEqual(removeSpacesAndNewlines(expectedMappingSchema)); }); + + const expectedHistogramSchemas = ` + type HistogramForString { + histogram: [BucketsForNestedStringAgg] + } + type RegularAccessHistogramForString { + histogram: [BucketsForNestedStringAgg] + } + type HistogramForNumber { + histogram( + rangeStart: Int, + rangeEnd: Int, + rangeStep: Int, + binCount: Int, + ): [BucketsForNestedNumberAgg], + asTextHistogram: [BucketsForNestedStringAgg] + } + type RegularAccessHistogramForNumber { + histogram( + rangeStart: Int, + rangeEnd: Int, + rangeStep: Int, + binCount: Int, + ): [BucketsForNestedNumberAgg], + asTextHistogram: [BucketsForNestedStringAgg] + }`; + test('could create histogram schemas for each type', async () => { + await esInstance.initialize(); + const histogramSchemas = getHistogramSchemas(); + expect(removeSpacesNewlinesAndDes(histogramSchemas)) + .toEqual(removeSpacesAndNewlines(expectedHistogramSchemas)); + }); }); diff --git a/src/server/schema.js b/src/server/schema.js index e886d0b7..8d6329cf 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -258,6 +258,20 @@ export const getMappingSchema = (esConfig) => ` } `; +export const getHistogramSchemas = () => { + const textHistogramSchema = getTextHistogramSchema(false); + + const regularAccessTextHistogramSchema = getTextHistogramSchema(true); + + const numberHistogramSchema = getNumberHistogramSchema(false); + + const regularAccessNumberHistogramSchema = getNumberHistogramSchema(true); + + const histogramSchemas = [textHistogramSchema, regularAccessTextHistogramSchema, numberHistogramSchema, regularAccessNumberHistogramSchema].join('\n'); + + return histogramSchemas; +} + export const buildSchemaString = (esConfig, esInstance) => { const querySchema = getQuerySchema(esConfig); @@ -293,10 +307,8 @@ export const buildSchemaString = (esConfig, esInstance) => { const aggregationSchemasForEachNestedType = getAggregationSchemaForEachNestedType(esConfig, esInstance); - const textHistogramSchema = getTextHistogramSchema(false); - - const regularAccessTextHistogramSchema = getTextHistogramSchema(true); - + const histogramSchemas = getHistogramSchemas(); + const textHistogramBucketSchema = ` type BucketsForNestedStringAgg { key: String @@ -306,6 +318,20 @@ export const buildSchemaString = (esConfig, esInstance) => { } `; + const numberHistogramBucketSchema = ` + type BucketsForNestedNumberAgg { + """Lower and higher bounds for this bucket""" + key: [Float] + min: Float + max: Float + avg: Float + sum: Float + count: Int + missingFields: [BucketsForNestedMissingFields] + termsFields: [BucketsForNestedTermsFields] + } + `; + const nestedMissingFieldsBucketSchema = ` type BucketsForNestedMissingFields { field: String @@ -327,24 +353,6 @@ export const buildSchemaString = (esConfig, esInstance) => { } `; - const numberHistogramSchema = getNumberHistogramSchema(false); - - const regularAccessNumberHistogramSchema = getNumberHistogramSchema(true); - - const numberHistogramBucketSchema = ` - type BucketsForNestedNumberAgg { - """Lower and higher bounds for this bucket""" - key: [Float] - min: Float - max: Float - avg: Float - sum: Float - count: Int - missingFields: [BucketsForNestedMissingFields] - termsFields: [BucketsForNestedTermsFields] - } - `; - const mappingSchema = getMappingSchema(esConfig); const schemaStr = ` @@ -357,10 +365,7 @@ export const buildSchemaString = (esConfig, esInstance) => { ${aggregationSchema} ${aggregationSchemasForEachType} ${aggregationSchemasForEachNestedType} - ${textHistogramSchema} - ${regularAccessTextHistogramSchema} - ${numberHistogramSchema} - ${regularAccessNumberHistogramSchema} + ${histogramSchemas} ${textHistogramBucketSchema} ${nestedMissingFieldsBucketSchema} ${nestedTermsFieldsBucketSchema} From 54048c807465466afacd344ade67e83e0af70cb8 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 13:53:01 -0600 Subject: [PATCH 43/67] PR feedback: add unit test --- src/server/schema.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/schema.js b/src/server/schema.js index 8d6329cf..adb775bc 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -262,7 +262,7 @@ export const getHistogramSchemas = () => { const textHistogramSchema = getTextHistogramSchema(false); const regularAccessTextHistogramSchema = getTextHistogramSchema(true); - + const numberHistogramSchema = getNumberHistogramSchema(false); const regularAccessNumberHistogramSchema = getNumberHistogramSchema(true); @@ -270,7 +270,7 @@ export const getHistogramSchemas = () => { const histogramSchemas = [textHistogramSchema, regularAccessTextHistogramSchema, numberHistogramSchema, regularAccessNumberHistogramSchema].join('\n'); return histogramSchemas; -} +}; export const buildSchemaString = (esConfig, esInstance) => { const querySchema = getQuerySchema(esConfig); @@ -308,7 +308,7 @@ export const buildSchemaString = (esConfig, esInstance) => { esInstance); const histogramSchemas = getHistogramSchemas(); - + const textHistogramBucketSchema = ` type BucketsForNestedStringAgg { key: String From 16c9307a50635a8fcec932365b21f3526819f481 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 13:57:02 -0600 Subject: [PATCH 44/67] fix logic error --- src/server/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/config.js b/src/server/config.js index 6983d076..894e2a10 100644 --- a/src/server/config.js +++ b/src/server/config.js @@ -88,7 +88,7 @@ config.esConfig.indices.forEach((item) => { if (!item.tier_access_level && !config.tierAccessLevel) { throw new Error('Either set all index-scoped tiered-access levels or a site-wide tiered-access level.'); } - if (!allowedTierAccessLevels.includes(item.tier_access_level)) { + if (item.tier_access_level && !allowedTierAccessLevels.includes(item.tier_access_level)) { throw new Error(`tier_access_level invalid for index ${item.type}.`); } if (!item.tier_access_level) { From 397048fc4b7687d1bb11c3b6454f3578cef9376d Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 14:00:29 -0600 Subject: [PATCH 45/67] fix logic error --- src/server/__tests__/schema.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server/__tests__/schema.test.js b/src/server/__tests__/schema.test.js index b9e01505..1cecb9f9 100644 --- a/src/server/__tests__/schema.test.js +++ b/src/server/__tests__/schema.test.js @@ -6,6 +6,7 @@ import { getAggregationSchema, getAggregationSchemaForEachType, getMappingSchema, + getHistogramSchemas, } from '../schema'; import esInstance from '../es/index'; import config from '../config'; From 55b19c802e930bd372f58a72b7cab1d984fff26b Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 14:36:22 -0600 Subject: [PATCH 46/67] PR feedback: addd more unit tests --- src/server/__tests__/config.test.js | 18 ++++++++++++ .../test-index-scoped-tier-access.json | 29 +++++++++++++++++++ ...test-invalid-index-scoped-tier-access.json | 14 +++++++++ 3 files changed, 61 insertions(+) create mode 100644 src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json create mode 100644 src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json diff --git a/src/server/__tests__/config.test.js b/src/server/__tests__/config.test.js index 6beb4287..9dc42978 100644 --- a/src/server/__tests__/config.test.js +++ b/src/server/__tests__/config.test.js @@ -37,6 +37,24 @@ describe('config', () => { expect(() => (require('../config'))).toThrow(new Error(`Invalid TIER_ACCESS_LEVEL "${process.env.TIER_ACCESS_LEVEL}"`)); }); + test('should show error if invalid tier access level in guppy block', async () => { + process.env.TIER_ACCESS_LEVEL = null; + const fileName = './testConfigFiles/test-invalid-index-scoped-tier-access.json'; + process.env.GUPPY_CONFIG_FILEPATH = `${__dirname}/${fileName}`; + const invalidItemType = 'subject_private'; + expect(() => (require('../config'))).toThrow(new Error(`tier_access_level invalid for index ${invalidItemType}."`)); + }); + + test('clears out site-wide default tiered-access setting if index-scoped levels set', async () => { + process.env.TIER_ACCESS_LEVEL = null; + const fileName = './testConfigFiles/test-invalid-index-scoped-tier-access.json'; + process.env.GUPPY_CONFIG_FILEPATH = `${__dirname}/${fileName}`; + const config = require('../config').default; + const { indices } = require(fileName); + expect(config.tierAccessLevel).toBe(null); + expect(JSON.stringify(config.esConfig.indices)).toEqual(JSON.stringify(indices)); + }); + /* --------------- For whitelist --------------- */ test('could disable whitelist', async () => { const config = require('../config').default; diff --git a/src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json b/src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json new file mode 100644 index 00000000..645404c9 --- /dev/null +++ b/src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json @@ -0,0 +1,29 @@ +{ + "indices": [ + { + "index": "subject_regular", + "type": "subject", + "tier_access_level": "regular" + }, + { + "index": "subject_private", + "type": "subject_private", + "tier_access_level": "private" + }, + { + "index": "file_private", + "type": "file", + "tier_access_level": "private" + }, + { + "index": "studies_open", + "type": "studies_open", + "tier_access_level": "libre" + }, + { + "index": "studies_controlled_access", + "type": "studies_controlled_access", + "tier_access_level": "private" + } + ] + } \ No newline at end of file diff --git a/src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json b/src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json new file mode 100644 index 00000000..9937f230 --- /dev/null +++ b/src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json @@ -0,0 +1,14 @@ +{ + "indices": [ + { + "index": "subject_regular", + "type": "subject", + "tier_access_level": "regular" + }, + { + "index": "subject_private", + "type": "subject_private", + "tier_access_level": "private____typo" + } + ] + } \ No newline at end of file From 05f1d331955bba89133244d390a8747b767e9157 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 14:44:18 -0600 Subject: [PATCH 47/67] fix travis --- src/server/__tests__/config.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/__tests__/config.test.js b/src/server/__tests__/config.test.js index 9dc42978..d0708363 100644 --- a/src/server/__tests__/config.test.js +++ b/src/server/__tests__/config.test.js @@ -42,12 +42,12 @@ describe('config', () => { const fileName = './testConfigFiles/test-invalid-index-scoped-tier-access.json'; process.env.GUPPY_CONFIG_FILEPATH = `${__dirname}/${fileName}`; const invalidItemType = 'subject_private'; - expect(() => (require('../config'))).toThrow(new Error(`tier_access_level invalid for index ${invalidItemType}."`)); + expect(() => (require('../config'))).toThrow(new Error(`tier_access_level invalid for index ${invalidItemType}.`)); }); test('clears out site-wide default tiered-access setting if index-scoped levels set', async () => { process.env.TIER_ACCESS_LEVEL = null; - const fileName = './testConfigFiles/test-invalid-index-scoped-tier-access.json'; + const fileName = './testConfigFiles/test-index-scoped-tier-access.json'; process.env.GUPPY_CONFIG_FILEPATH = `${__dirname}/${fileName}`; const config = require('../config').default; const { indices } = require(fileName); From 14189cad05d0ab12a59e2fea8bdfe2e04345e450 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Mon, 1 Feb 2021 15:06:02 -0600 Subject: [PATCH 48/67] fix travis --- src/server/__tests__/config.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/__tests__/config.test.js b/src/server/__tests__/config.test.js index d0708363..cac2462b 100644 --- a/src/server/__tests__/config.test.js +++ b/src/server/__tests__/config.test.js @@ -51,7 +51,7 @@ describe('config', () => { process.env.GUPPY_CONFIG_FILEPATH = `${__dirname}/${fileName}`; const config = require('../config').default; const { indices } = require(fileName); - expect(config.tierAccessLevel).toBe(null); + expect(config.tierAccessLevel).toBeUndefined(); expect(JSON.stringify(config.esConfig.indices)).toEqual(JSON.stringify(indices)); }); From fd4ad50e78696f7c0d7e2e895dc21d64837bde59 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 2 Feb 2021 14:19:21 -0600 Subject: [PATCH 49/67] feat: enhance unit test for tierAccessLimit --- src/server/__tests__/config.test.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/server/__tests__/config.test.js b/src/server/__tests__/config.test.js index cac2462b..2ac42bf2 100644 --- a/src/server/__tests__/config.test.js +++ b/src/server/__tests__/config.test.js @@ -47,11 +47,13 @@ describe('config', () => { test('clears out site-wide default tiered-access setting if index-scoped levels set', async () => { process.env.TIER_ACCESS_LEVEL = null; + process.env.TIER_ACCESS_LIMIT = 50; const fileName = './testConfigFiles/test-index-scoped-tier-access.json'; process.env.GUPPY_CONFIG_FILEPATH = `${__dirname}/${fileName}`; const config = require('../config').default; const { indices } = require(fileName); expect(config.tierAccessLevel).toBeUndefined(); + expect(config.tierAccessLevel).toEqual(50); expect(JSON.stringify(config.esConfig.indices)).toEqual(JSON.stringify(indices)); }); From 3ad6a7a6329da3795e44bf21255f461d56947dfe Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 2 Feb 2021 14:19:39 -0600 Subject: [PATCH 50/67] feat: enhance unit test for tierAccessLimit --- src/server/__tests__/config.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/__tests__/config.test.js b/src/server/__tests__/config.test.js index 2ac42bf2..be7f4199 100644 --- a/src/server/__tests__/config.test.js +++ b/src/server/__tests__/config.test.js @@ -53,7 +53,7 @@ describe('config', () => { const config = require('../config').default; const { indices } = require(fileName); expect(config.tierAccessLevel).toBeUndefined(); - expect(config.tierAccessLevel).toEqual(50); + expect(config.tierAccessLimit).toEqual(50); expect(JSON.stringify(config.esConfig.indices)).toEqual(JSON.stringify(indices)); }); From 2b12abc937acd725c742e985ad6045772b1c4c7e Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 2 Feb 2021 14:49:02 -0600 Subject: [PATCH 51/67] PR feedback: fix indentation --- .../testConfigFiles/test-index-scoped-tier-access.json | 10 +++++----- .../test-invalid-index-scoped-tier-access.json | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json b/src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json index 645404c9..e1200f4f 100644 --- a/src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json +++ b/src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json @@ -3,27 +3,27 @@ { "index": "subject_regular", "type": "subject", - "tier_access_level": "regular" + "tier_access_level": "regular" }, { "index": "subject_private", "type": "subject_private", - "tier_access_level": "private" + "tier_access_level": "private" }, { "index": "file_private", "type": "file", - "tier_access_level": "private" + "tier_access_level": "private" }, { "index": "studies_open", "type": "studies_open", - "tier_access_level": "libre" + "tier_access_level": "libre" }, { "index": "studies_controlled_access", "type": "studies_controlled_access", - "tier_access_level": "private" + "tier_access_level": "private" } ] } \ No newline at end of file diff --git a/src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json b/src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json index 9937f230..50ccf4d1 100644 --- a/src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json +++ b/src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json @@ -3,12 +3,12 @@ { "index": "subject_regular", "type": "subject", - "tier_access_level": "regular" + "tier_access_level": "regular" }, { "index": "subject_private", "type": "subject_private", - "tier_access_level": "private____typo" + "tier_access_level": "private____typo" } ] } \ No newline at end of file From 78be005db081a6777f96e46d4a44311742c1ebd4 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 2 Feb 2021 14:50:47 -0600 Subject: [PATCH 52/67] PR feedback: remove unnecessary else-if --- src/server/middlewares/perIndexTierAccessMiddleware/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server/middlewares/perIndexTierAccessMiddleware/index.js b/src/server/middlewares/perIndexTierAccessMiddleware/index.js index bfe323a2..d39d92c8 100644 --- a/src/server/middlewares/perIndexTierAccessMiddleware/index.js +++ b/src/server/middlewares/perIndexTierAccessMiddleware/index.js @@ -24,9 +24,8 @@ config.esConfig.indices.forEach((item) => { totalCountTypeMapping[aggregationName] = { _totalCount: hideNumberResolver(true), }; - } else if (item.tier_access_level === 'libre') { - // No additional resolvers necessary } + // No additional resolvers necessary for tier_access_level == 'libre' }, {}); const perIndexTierAccessMiddleware = { From f6460fef95d8713cc5badb84305bca03ffca40f0 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 2 Feb 2021 14:53:50 -0600 Subject: [PATCH 53/67] PR feedbacks --- src/server/schema.js | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/server/schema.js b/src/server/schema.js index adb775bc..b39e7473 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -17,6 +17,8 @@ const esgqlTypeMapping = { nested: 'Object', }; +const histogramTypePrefix = 'RegularAccess'; + const getGQLType = (esInstance, esIndex, field, esFieldType) => { const gqlType = esgqlTypeMapping[esFieldType]; if (!gqlType) { @@ -146,11 +148,10 @@ const getAggregationType = (entry) => { return ''; }; -const getAggregationSchemaForOneIndex = (esInstance, esDict) => { - const esIndex = esDict.index; - const esType = esDict.type; - const histogramTypePrefix = 'RegularAccess'; - const includeHistogramPrefix = Object.prototype.hasOwnProperty.call(esDict, 'tier_access_level') && esDict.tier_access_level === 'regular'; +const getAggregationSchemaForOneIndex = (esInstance, esConfigElement) => { + const esIndex = esConfigElement.index; + const esType = esConfigElement.type; + const includeHistogramPrefix = Object.prototype.hasOwnProperty.call(esConfigElement, 'tier_access_level') && esConfigElement.tier_access_level === 'regular'; const esTypeObjName = firstLetterUpperCase(esType); const fieldGQLTypeMap = getFieldGQLTypeMapForOneIndex(esInstance, esIndex); const fieldAggsTypeMap = fieldGQLTypeMap.filter((f) => f.esType !== 'nested').map((entry) => ({ @@ -196,7 +197,6 @@ const getAggregationSchemaForOneNestedIndex = (esInstance, esDict) => { const esIndex = esDict.index; const fieldGQLTypeMap = getFieldGQLTypeMapForOneIndex(esInstance, esIndex); const fieldAggsNestedTypeMap = fieldGQLTypeMap.filter((f) => f.esType === 'nested'); - const histogramTypePrefix = 'RegularAccess'; const includeHistogramPrefix = Object.prototype.hasOwnProperty.call(esDict, 'tier_access_level') && esDict.tier_access_level === 'regular'; let AggsNestedTypeSchema = ''; while (fieldAggsNestedTypeMap.length > 0) { @@ -226,9 +226,7 @@ export const getAggregationSchemaForEachType = (esConfig, esInstance) => esConfi export const getAggregationSchemaForEachNestedType = (esConfig, esInstance) => esConfig.indices.map((cfg) => getAggregationSchemaForOneNestedIndex(esInstance, cfg)).join('\n'); -const getNumberHistogramSchema = (isRegularAccess) => { - const histogramTypePrefix = 'RegularAccess'; - return ` +const getNumberHistogramSchema = (isRegularAccess) => ` type ${(isRegularAccess ? histogramTypePrefix : '') + EnumAggsHistogramName.HISTOGRAM_FOR_NUMBER} { histogram( rangeStart: Int, @@ -239,16 +237,12 @@ const getNumberHistogramSchema = (isRegularAccess) => { asTextHistogram: [BucketsForNestedStringAgg] } `; -}; -const getTextHistogramSchema = (isRegularAccess) => { - const histogramTypePrefix = 'RegularAccess'; - return ` +const getTextHistogramSchema = (isRegularAccess) => ` type ${(isRegularAccess ? histogramTypePrefix : '') + EnumAggsHistogramName.HISTOGRAM_FOR_STRING} { histogram: [BucketsForNestedStringAgg] } `; -}; export const getMappingSchema = (esConfig) => ` type Mapping { From 8314686341a2193d7de92f82bba257973e58ed20 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 2 Feb 2021 14:56:21 -0600 Subject: [PATCH 54/67] PR feedback: JSON file newlines --- .../test-index-scoped-tier-access.json | 52 +++++++++---------- ...test-invalid-index-scoped-tier-access.json | 22 ++++---- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json b/src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json index e1200f4f..0dd87e65 100644 --- a/src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json +++ b/src/server/__tests__/testConfigFiles/test-index-scoped-tier-access.json @@ -1,29 +1,29 @@ { "indices": [ - { - "index": "subject_regular", - "type": "subject", - "tier_access_level": "regular" - }, - { - "index": "subject_private", - "type": "subject_private", - "tier_access_level": "private" - }, - { - "index": "file_private", - "type": "file", - "tier_access_level": "private" - }, - { - "index": "studies_open", - "type": "studies_open", - "tier_access_level": "libre" - }, - { - "index": "studies_controlled_access", - "type": "studies_controlled_access", - "tier_access_level": "private" - } + { + "index": "subject_regular", + "type": "subject", + "tier_access_level": "regular" + }, + { + "index": "subject_private", + "type": "subject_private", + "tier_access_level": "private" + }, + { + "index": "file_private", + "type": "file", + "tier_access_level": "private" + }, + { + "index": "studies_open", + "type": "studies_open", + "tier_access_level": "libre" + }, + { + "index": "studies_controlled_access", + "type": "studies_controlled_access", + "tier_access_level": "private" + } ] - } \ No newline at end of file +} diff --git a/src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json b/src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json index 50ccf4d1..a8b5ee94 100644 --- a/src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json +++ b/src/server/__tests__/testConfigFiles/test-invalid-index-scoped-tier-access.json @@ -1,14 +1,14 @@ { "indices": [ - { - "index": "subject_regular", - "type": "subject", - "tier_access_level": "regular" - }, - { - "index": "subject_private", - "type": "subject_private", - "tier_access_level": "private____typo" - } + { + "index": "subject_regular", + "type": "subject", + "tier_access_level": "regular" + }, + { + "index": "subject_private", + "type": "subject_private", + "tier_access_level": "private____typo" + } ] - } \ No newline at end of file +} From 7eff1fc7e722c439f3ae0e7e4e9fc99eed84a6c1 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 2 Feb 2021 15:19:06 -0600 Subject: [PATCH 55/67] PR feedback: update README --- README.md | 19 ++++++++++++++----- doc/index_scoped_tiered_access.md | 12 ++++++------ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 1d630bb4..db00b627 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,13 @@ You could put following as your config files: "indices": [ { "index": "${ES_INDEX_1}", - "type": "${ES_DOC_TYPE_1}" + "type": "${ES_DOC_TYPE_1}", + "tier_access_level": "${ES_TIER_ACCESS_LEVEL_1}" // optional, set this if there is no global tierAccessLevel }, { "index": "${ES_INDEX_2}", - "type": "${ES_DOC_TYPE_2}" + "type": "${ES_DOC_TYPE_2}", + "tier_access_level": "${ES_TIER_ACCESS_LEVEL_2}" // optional, set this if there is no global tierAccessLevel }, ... ], @@ -35,6 +37,8 @@ You could put following as your config files: } ``` +Note: Guppy expects that either all indices in the guppy config block will have a tiered-access level set OR that a site-wide tiered-access level is set in the global block of the manifest. Guppy will throw an error if the config settings do not meet one of these two expectations. See `doc/index_scoped_tiered_access.md` for more information. + Following script will start server using at port 3000, using config file `example_config.json`: ``` @@ -58,7 +62,7 @@ behavior for local test without Arborist, just set `INTERNAL_LOCAL_TEST=true`. P look into `/src/server/auth/utils.js` for more details. ### Tiered Access: -Guppy also support 3 different levels of tier access, by setting `TIER_ACCESS_LEVEL`: +Guppy support 3 different levels of tier access, by setting `TIER_ACCESS_LEVEL`: - `private` by default: only allows access to authorized resources - `regular`: allows all kind of aggregation (with limitation for unauthorized resources), but forbid access to raw data without authorization - `libre`: access to all data @@ -68,7 +72,8 @@ For `regular` level, there's another configuration environment variable `TIER_AC `regular` level commons could also take in a whitelist of values that won't be encrypted. It is set by `config.encrypt_whitelist`. By default the whitelist contains missing values: ['\_\_missing\_\_', 'unknown', 'not reported', 'no data']. Also the whitelist is disabled by default due to security reasons. If you would like to enable whitelist, simply put `enable_encrypt_whitelist: true` in your config. -For example `regular` leveled commons with config looks like this will skip encrypting value `do-not-encrypt-me` even if its count is less than `TIER_ACCESS_LIMIT`: + +For example, a `regular` leveled commons with config looks like this will skip encrypting value `do-not-encrypt-me` even if its count is less than `TIER_ACCESS_LIMIT`: ``` { @@ -89,7 +94,9 @@ For example `regular` leveled commons with config looks like this will skip encr } ``` -For example following script will start a Guppy server with `regular` tier access level, and minimum visible count set to 100: +Tiered-access can be configured in either a site-wide manner or with per-index scoping. To configure tiered-access at the site-wide level, use the tierAccessLevel property in the global block of the manifest.json. The above example is a site-wide tiered-access configuration. + +The following script will start a Guppy server with a site-wide `regular` tier access level, and minimum visible count set to 100: ``` export TIER_ACCESS_LEVEL=regular @@ -97,6 +104,8 @@ export TIER_ACCESS_LIMIT=100 npm start ``` +To learn how to configure Guppy's tiered-access system using a per-index scoping, and which use cases might warrant such a configuration, please see `doc/index_scoped_tiered_access.md`. + > #### Tier Access Sensitive Record Exclusion > It is possible to configure Guppy to hide some records from being returned in `_aggregation` queries when Tiered Access is enabled (tierAccessLevel: "regular"). > The purpose of this is to "hide" information about certain sensitive resources, essentially making this an escape hatch from Tiered Access. diff --git a/doc/index_scoped_tiered_access.md b/doc/index_scoped_tiered_access.md index 7f1eeeed..19cf8450 100644 --- a/doc/index_scoped_tiered_access.md +++ b/doc/index_scoped_tiered_access.md @@ -12,30 +12,30 @@ You can set index-scoped tiered-access levels using the `tier_access_level` prop { "index": "subject_regular", "type": "subject", - "tier_access_level": "regular" + "tier_access_level": "regular" }, { "index": "subject_private", "type": "subject_private", - "tier_access_level": "private" + "tier_access_level": "private" }, { "index": "file_private", "type": "file", - "tier_access_level": "private" + "tier_access_level": "private" }, { "index": "studies_open", "type": "studies_open", - "tier_access_level": "libre" + "tier_access_level": "libre" }, { "index": "studies_controlled_access", "type": "studies_controlled_access", - "tier_access_level": "private" + "tier_access_level": "private" } ], "auth_filter_field": "auth_resource_path", ... }, -``` \ No newline at end of file +``` From 2a31f42549165a37d68ead9b2876484796774b12 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 2 Feb 2021 15:22:22 -0600 Subject: [PATCH 56/67] adjust whitespace --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index db00b627..e69cf8ac 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,6 @@ For `regular` level, there's another configuration environment variable `TIER_AC `regular` level commons could also take in a whitelist of values that won't be encrypted. It is set by `config.encrypt_whitelist`. By default the whitelist contains missing values: ['\_\_missing\_\_', 'unknown', 'not reported', 'no data']. Also the whitelist is disabled by default due to security reasons. If you would like to enable whitelist, simply put `enable_encrypt_whitelist: true` in your config. - For example, a `regular` leveled commons with config looks like this will skip encrypting value `do-not-encrypt-me` even if its count is less than `TIER_ACCESS_LIMIT`: ``` From 3e2da9431d710b8fe97a79ccf002f727491c2953 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 2 Feb 2021 15:29:53 -0600 Subject: [PATCH 57/67] PR feedback: update README --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e69cf8ac..2fe9b041 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ You could put following as your config files: } ``` -Note: Guppy expects that either all indices in the guppy config block will have a tiered-access level set OR that a site-wide tiered-access level is set in the global block of the manifest. Guppy will throw an error if the config settings do not meet one of these two expectations. See `doc/index_scoped_tiered_access.md` for more information. +Note: Guppy expects that either all indices in the guppy config block will have a tier_access_level set OR that a site-wide TIER_ACCESS_LEVEL is set as an environment variable (or in the global block of a commons' manifest). Guppy will throw an error if the config settings do not meet one of these two expectations. See `doc/index_scoped_tiered_access.md` for more information. Following script will start server using at port 3000, using config file `example_config.json`: @@ -67,12 +67,12 @@ Guppy support 3 different levels of tier access, by setting `TIER_ACCESS_LEVEL`: - `regular`: allows all kind of aggregation (with limitation for unauthorized resources), but forbid access to raw data without authorization - `libre`: access to all data -For `regular` level, there's another configuration environment variable `TIER_ACCESS_LIMIT`, which is the minimum visible count for aggregation results. +For the `regular` level, there's another configuration environment variable `TIER_ACCESS_LIMIT`, which is the minimum visible count for aggregation results. -`regular` level commons could also take in a whitelist of values that won't be encrypted. It is set by `config.encrypt_whitelist`. +`regular` level commons can also take in a whitelist of values that won't be encrypted. It is set by `config.encrypt_whitelist`. By default the whitelist contains missing values: ['\_\_missing\_\_', 'unknown', 'not reported', 'no data']. Also the whitelist is disabled by default due to security reasons. If you would like to enable whitelist, simply put `enable_encrypt_whitelist: true` in your config. -For example, a `regular` leveled commons with config looks like this will skip encrypting value `do-not-encrypt-me` even if its count is less than `TIER_ACCESS_LIMIT`: +For example, a `regular` leveled commons with config that looks like this will skip encrypting the value `do-not-encrypt-me` even if its count is less than `TIER_ACCESS_LIMIT`: ``` { From 606b31d3b32db021fdc273def40df1e4a8e6e9b4 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Tue, 2 Feb 2021 15:42:11 -0600 Subject: [PATCH 58/67] fix logic for download endpoint --- src/server/download.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/server/download.js b/src/server/download.js index 981c1ca3..1821d003 100644 --- a/src/server/download.js +++ b/src/server/download.js @@ -18,14 +18,16 @@ const downloadRouter = async (req, res, next) => { try { let appliedFilter; /** - * Tier acces strategy for download endpoint: - * 1. if data commons is secure, add auth filter layer onto filter - * 2. if data commons is regular: + * Tier access strategy for download endpoint: + * 1. if the data commons or the index is secure, add auth filter layer onto filter + * 2. if the data commons or the index is regular: * a. if request contains out-of-access resource, return 401 * b. if request contains only accessible resouces, return response - * 3. if data commons is private, always return reponse without any auth check + * 3. if the data commons or the index is private, always return reponse without any auth check */ - switch (config.tierAccessLevel) { + const tierAccessLevel = config.tierAccessLevel + ? config.tierAccessLevel : esIndex.tier_access_level; + switch (tierAccessLevel) { case 'private': { appliedFilter = authHelper.applyAccessibleFilter(filter); break; @@ -54,8 +56,9 @@ const downloadRouter = async (req, res, next) => { break; } default: - throw new Error(`Invalid TIER_ACCESS_LEVEL "${config.tierAccessLevel}"`); + throw new Error(`Invalid TIER_ACCESS_LEVEL "${tierAccessLevel}"`); } + log.debug('[download] applied filter for tierAccessLevel: ', tierAccessLevel); const data = await esInstance.downloadData({ esIndex, esType: type, filter: appliedFilter, sort, fields, }); From 9eefb462a2df037bf14538afd98bb95df38398ef Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 3 Feb 2021 11:27:54 -0600 Subject: [PATCH 59/67] fix logic for download endpoint --- src/server/download.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/server/download.js b/src/server/download.js index 1821d003..dd0a6f22 100644 --- a/src/server/download.js +++ b/src/server/download.js @@ -10,7 +10,7 @@ const downloadRouter = async (req, res, next) => { type, filter, sort, fields, accessibility, } = req.body; - log.debug('[download] ', JSON.stringify(req.body, null, 4)); + log.info('[download] ', JSON.stringify(req.body, null, 4)); const esIndex = esInstance.getESIndexByType(type); const jwt = headerParser.parseJWT(req); const authHelper = await getAuthHelperInstance(jwt); @@ -33,7 +33,7 @@ const downloadRouter = async (req, res, next) => { break; } case 'regular': { - log.debug('[download] regular commons'); + log.info('[download] regular commons'); if (accessibility === 'accessible') { appliedFilter = authHelper.applyAccessibleFilter(filter); } else { @@ -58,12 +58,13 @@ const downloadRouter = async (req, res, next) => { default: throw new Error(`Invalid TIER_ACCESS_LEVEL "${tierAccessLevel}"`); } - log.debug('[download] applied filter for tierAccessLevel: ', tierAccessLevel); + log.info('[download] applied filter for tierAccessLevel: ', tierAccessLevel); const data = await esInstance.downloadData({ esIndex, esType: type, filter: appliedFilter, sort, fields, }); res.send(data); } catch (err) { + log.error(err) next(err); } return 0; From 44b5ccffaaaf209cb97de2085025a49f271ea995 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 3 Feb 2021 11:55:33 -0600 Subject: [PATCH 60/67] debugging download enddpoint --- src/server/download.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/server/download.js b/src/server/download.js index dd0a6f22..3666d690 100644 --- a/src/server/download.js +++ b/src/server/download.js @@ -12,6 +12,12 @@ const downloadRouter = async (req, res, next) => { log.info('[download] ', JSON.stringify(req.body, null, 4)); const esIndex = esInstance.getESIndexByType(type); + log.info('[download] esIndex: ', esIndex); + log.info('[download] esIndex: ', JSON.stringify(esIndex)); + log.info('[download] config.tierAccessLevel: ', config.tierAccessLevel); + log.info('[download] esIndex.tier_access_level: ', esIndex.tier_access_level); + const tierAccessLevel = (config.tierAccessLevel + ? config.tierAccessLevel : esIndex.tier_access_level); const jwt = headerParser.parseJWT(req); const authHelper = await getAuthHelperInstance(jwt); @@ -25,8 +31,6 @@ const downloadRouter = async (req, res, next) => { * b. if request contains only accessible resouces, return response * 3. if the data commons or the index is private, always return reponse without any auth check */ - const tierAccessLevel = config.tierAccessLevel - ? config.tierAccessLevel : esIndex.tier_access_level; switch (tierAccessLevel) { case 'private': { appliedFilter = authHelper.applyAccessibleFilter(filter); @@ -64,7 +68,7 @@ const downloadRouter = async (req, res, next) => { }); res.send(data); } catch (err) { - log.error(err) + log.error(err); next(err); } return 0; From b3f916bb0ce2e3250e7f3f586e9c46a5e82ba58c Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 3 Feb 2021 12:20:00 -0600 Subject: [PATCH 61/67] debugging download enddpoint --- src/server/download.js | 6 +++--- src/server/es/index.js | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/server/download.js b/src/server/download.js index 3666d690..1e2ed046 100644 --- a/src/server/download.js +++ b/src/server/download.js @@ -11,7 +11,7 @@ const downloadRouter = async (req, res, next) => { } = req.body; log.info('[download] ', JSON.stringify(req.body, null, 4)); - const esIndex = esInstance.getESIndexByType(type); + const esIndex = esInstance.getESIndexConfigByType(type); log.info('[download] esIndex: ', esIndex); log.info('[download] esIndex: ', JSON.stringify(esIndex)); log.info('[download] config.tierAccessLevel: ', config.tierAccessLevel); @@ -42,7 +42,7 @@ const downloadRouter = async (req, res, next) => { appliedFilter = authHelper.applyAccessibleFilter(filter); } else { const outOfScopeResourceList = await authHelper.getOutOfScopeResourceList( - esIndex, type, filter, + esIndex.index, type, filter, ); // if requesting resources > allowed resources, return 401, if (outOfScopeResourceList.length > 0) { @@ -64,7 +64,7 @@ const downloadRouter = async (req, res, next) => { } log.info('[download] applied filter for tierAccessLevel: ', tierAccessLevel); const data = await esInstance.downloadData({ - esIndex, esType: type, filter: appliedFilter, sort, fields, + esIndex.index, esType: type, filter: appliedFilter, sort, fields, }); res.send(data); } catch (err) { diff --git a/src/server/es/index.js b/src/server/es/index.js index ecf640e0..54625998 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -318,6 +318,20 @@ class ES { ); } + /** + * Get es indexConfig by es type + * Throw 400 error if there's no existing es type + * @param {string} esType + */ + getESIndexConfigByType(esType) { + const index = this.config.indices.find((i) => i.type === esType); + if (index) return index; + throw new CodedError( + 400, + `Invalid es type: "${esType}"`, + ); + } + /** * Get es index config by es index name * Throw 400 error if there's no existing es index of that name From 808a7c453a0cef6de7ba9237b98ecaabd2997af0 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 3 Feb 2021 12:29:48 -0600 Subject: [PATCH 62/67] debugging download enddpoint --- src/server/download.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/download.js b/src/server/download.js index 1e2ed046..62b16ee8 100644 --- a/src/server/download.js +++ b/src/server/download.js @@ -64,7 +64,7 @@ const downloadRouter = async (req, res, next) => { } log.info('[download] applied filter for tierAccessLevel: ', tierAccessLevel); const data = await esInstance.downloadData({ - esIndex.index, esType: type, filter: appliedFilter, sort, fields, + esIndex: esIndex.index, esType: type, filter: appliedFilter, sort, fields, }); res.send(data); } catch (err) { From 125998d721fd889e8588e8148fc87faae3ec90e3 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 3 Feb 2021 13:25:14 -0600 Subject: [PATCH 63/67] remove extra prints --- src/server/download.js | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/server/download.js b/src/server/download.js index 62b16ee8..2010eade 100644 --- a/src/server/download.js +++ b/src/server/download.js @@ -10,12 +10,8 @@ const downloadRouter = async (req, res, next) => { type, filter, sort, fields, accessibility, } = req.body; - log.info('[download] ', JSON.stringify(req.body, null, 4)); + log.debug('[download] ', JSON.stringify(req.body, null, 4)); const esIndex = esInstance.getESIndexConfigByType(type); - log.info('[download] esIndex: ', esIndex); - log.info('[download] esIndex: ', JSON.stringify(esIndex)); - log.info('[download] config.tierAccessLevel: ', config.tierAccessLevel); - log.info('[download] esIndex.tier_access_level: ', esIndex.tier_access_level); const tierAccessLevel = (config.tierAccessLevel ? config.tierAccessLevel : esIndex.tier_access_level); const jwt = headerParser.parseJWT(req); @@ -37,7 +33,7 @@ const downloadRouter = async (req, res, next) => { break; } case 'regular': { - log.info('[download] regular commons'); + log.debug('[download] regular commons'); if (accessibility === 'accessible') { appliedFilter = authHelper.applyAccessibleFilter(filter); } else { @@ -46,8 +42,8 @@ const downloadRouter = async (req, res, next) => { ); // if requesting resources > allowed resources, return 401, if (outOfScopeResourceList.length > 0) { - log.info('[download] requesting out-of-scope resources, return 401'); - log.info(`[download] the following resources are out-of-scope: [${outOfScopeResourceList.join(', ')}]`); + log.debug('[download] requesting out-of-scope resources, return 401'); + log.debug(`[download] the following resources are out-of-scope: [${outOfScopeResourceList.join(', ')}]`); throw new CodedError(401, 'You don\'t have access to all the data you are querying. Try using \'accessibility: accessible\' in your query'); } else { // else, go ahead download appliedFilter = filter; @@ -62,7 +58,6 @@ const downloadRouter = async (req, res, next) => { default: throw new Error(`Invalid TIER_ACCESS_LEVEL "${tierAccessLevel}"`); } - log.info('[download] applied filter for tierAccessLevel: ', tierAccessLevel); const data = await esInstance.downloadData({ esIndex: esIndex.index, esType: type, filter: appliedFilter, sort, fields, }); From 600165b21ad7142bbedf6b22634f56d5aaa4b128 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 3 Feb 2021 13:27:24 -0600 Subject: [PATCH 64/67] fix legacy comments --- src/server/download.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/download.js b/src/server/download.js index 2010eade..165bac78 100644 --- a/src/server/download.js +++ b/src/server/download.js @@ -21,11 +21,11 @@ const downloadRouter = async (req, res, next) => { let appliedFilter; /** * Tier access strategy for download endpoint: - * 1. if the data commons or the index is secure, add auth filter layer onto filter + * 1. if the data commons or the index is private, add auth filter layer onto filter * 2. if the data commons or the index is regular: * a. if request contains out-of-access resource, return 401 * b. if request contains only accessible resouces, return response - * 3. if the data commons or the index is private, always return reponse without any auth check + * 3. if the data commons or the index is libre, always return reponse without any auth check */ switch (tierAccessLevel) { case 'private': { From 6e229ca377be229c54a18a984f59d4b2de99b5f2 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 3 Feb 2021 13:31:16 -0600 Subject: [PATCH 65/67] cleaning up --- README.md | 2 +- src/server/download.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2fe9b041..063ac3fc 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ behavior for local test without Arborist, just set `INTERNAL_LOCAL_TEST=true`. P look into `/src/server/auth/utils.js` for more details. ### Tiered Access: -Guppy support 3 different levels of tier access, by setting `TIER_ACCESS_LEVEL`: +The tiered-access setting is configured through either the `TIER_ACCESS_LEVEL` environment variable or the `tier_access_level` properties on individual indices in the esConfig. Guppy supports 3 different levels of tiered access: - `private` by default: only allows access to authorized resources - `regular`: allows all kind of aggregation (with limitation for unauthorized resources), but forbid access to raw data without authorization - `libre`: access to all data diff --git a/src/server/download.js b/src/server/download.js index 165bac78..1c74728a 100644 --- a/src/server/download.js +++ b/src/server/download.js @@ -42,8 +42,8 @@ const downloadRouter = async (req, res, next) => { ); // if requesting resources > allowed resources, return 401, if (outOfScopeResourceList.length > 0) { - log.debug('[download] requesting out-of-scope resources, return 401'); - log.debug(`[download] the following resources are out-of-scope: [${outOfScopeResourceList.join(', ')}]`); + log.info('[download] requesting out-of-scope resources, return 401'); + log.info(`[download] the following resources are out-of-scope: [${outOfScopeResourceList.join(', ')}]`); throw new CodedError(401, 'You don\'t have access to all the data you are querying. Try using \'accessibility: accessible\' in your query'); } else { // else, go ahead download appliedFilter = filter; From 9e25c8b6adccc22c6238879cf6521b5aaad3292f Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 3 Feb 2021 13:53:12 -0600 Subject: [PATCH 66/67] adjust README --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 063ac3fc..1e73e63b 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,6 @@ For example, a `regular` leveled commons with config that looks like this will s } ``` -Tiered-access can be configured in either a site-wide manner or with per-index scoping. To configure tiered-access at the site-wide level, use the tierAccessLevel property in the global block of the manifest.json. The above example is a site-wide tiered-access configuration. - The following script will start a Guppy server with a site-wide `regular` tier access level, and minimum visible count set to 100: ``` From 672b33cb5f3c8dcb41d5973b704d8823a129a731 Mon Sep 17 00:00:00 2001 From: Zakir Gowani Date: Wed, 3 Feb 2021 14:33:58 -0600 Subject: [PATCH 67/67] PR feedback: change variable name --- src/server/download.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/download.js b/src/server/download.js index 1c74728a..b4816ad3 100644 --- a/src/server/download.js +++ b/src/server/download.js @@ -11,9 +11,9 @@ const downloadRouter = async (req, res, next) => { } = req.body; log.debug('[download] ', JSON.stringify(req.body, null, 4)); - const esIndex = esInstance.getESIndexConfigByType(type); + const esIndexConfig = esInstance.getESIndexConfigByType(type); const tierAccessLevel = (config.tierAccessLevel - ? config.tierAccessLevel : esIndex.tier_access_level); + ? config.tierAccessLevel : esIndexConfig.tier_access_level); const jwt = headerParser.parseJWT(req); const authHelper = await getAuthHelperInstance(jwt); @@ -38,7 +38,7 @@ const downloadRouter = async (req, res, next) => { appliedFilter = authHelper.applyAccessibleFilter(filter); } else { const outOfScopeResourceList = await authHelper.getOutOfScopeResourceList( - esIndex.index, type, filter, + esIndexConfig.index, type, filter, ); // if requesting resources > allowed resources, return 401, if (outOfScopeResourceList.length > 0) { @@ -59,7 +59,7 @@ const downloadRouter = async (req, res, next) => { throw new Error(`Invalid TIER_ACCESS_LEVEL "${tierAccessLevel}"`); } const data = await esInstance.downloadData({ - esIndex: esIndex.index, esType: type, filter: appliedFilter, sort, fields, + esIndex: esIndexConfig.index, esType: type, filter: appliedFilter, sort, fields, }); res.send(data); } catch (err) {