From 9c0913f0539bfe207feb860f4a7cefcb1d2ec643 Mon Sep 17 00:00:00 2001 From: ocshawn Date: Tue, 9 May 2023 14:34:21 -0500 Subject: [PATCH 01/10] add cardinalityCount support to guppy connectedFilter and fix bugs --- package-lock.json | 4 ++-- package.json | 2 +- src/components/ConnectedFilter/utils.js | 18 +++++++++++++++--- src/components/Utils/filters.js | 8 +++++++- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3e655eae..68366dff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gen3/guppy", - "version": "0.15.7", + "version": "0.15.8", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@gen3/guppy", - "version": "0.15.7", + "version": "0.15.8", "license": "ISC", "dependencies": { "@elastic/elasticsearch": "~7.13.0", diff --git a/package.json b/package.json index 97af9ae0..4889c28f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gen3/guppy", - "version": "0.15.7", + "version": "0.15.8", "description": "Server that support GraphQL queries on data from elasticsearch", "main": "src/server/server.js", "directories": { diff --git a/src/components/ConnectedFilter/utils.js b/src/components/ConnectedFilter/utils.js index 0eb6f295..052e6212 100644 --- a/src/components/ConnectedFilter/utils.js +++ b/src/components/ConnectedFilter/utils.js @@ -182,7 +182,19 @@ export const excludeSelfFilterFromAggsData = (receivedAggsData, filterResults) = const resultAggsData = {}; const flattenAggsData = flat(receivedAggsData, { safe: true }); Object.keys(flattenAggsData).forEach((field) => { - const actualFieldName = field.replace('.histogram', ''); + const isCardinalityCount = field.includes('._cardinalityCount'); + const actualFieldName = isCardinalityCount + ? field.replace('._cardinalityCount', '') + : field.replace('.histogram', ''); + + if (!resultAggsData[`${actualFieldName}`]) { + resultAggsData[`${actualFieldName}`] = {}; + } + + if (isCardinalityCount) { + resultAggsData[`${actualFieldName}`].cardinalityCount = flattenAggsData[`${field}`]; + return; + } const histogram = flattenAggsData[`${field}`]; if (!histogram) return; if (actualFieldName in filterResults) { @@ -191,9 +203,9 @@ export const excludeSelfFilterFromAggsData = (receivedAggsData, filterResults) = const { selectedValues } = filterResults[`${actualFieldName}`]; resultHistogram = histogram.filter((bucket) => selectedValues.includes(bucket.key)); } - resultAggsData[`${actualFieldName}`] = { histogram: resultHistogram }; + resultAggsData[`${actualFieldName}`].histogram = resultHistogram; } else { - resultAggsData[`${actualFieldName}`] = { histogram: flattenAggsData[`${field}`] }; + resultAggsData[`${actualFieldName}`].histogram = flattenAggsData[`${field}`]; } }); return resultAggsData; diff --git a/src/components/Utils/filters.js b/src/components/Utils/filters.js index 7a14878b..ddfe5f8c 100644 --- a/src/components/Utils/filters.js +++ b/src/components/Utils/filters.js @@ -52,6 +52,11 @@ export const updateCountsInInitialTabsOptions = ( const flattenInitialTabsOptions = flat(initialTabsOptions, { safe: true }); const flattenProcessedTabsOptions = flat(processedTabsOptions, { safe: true }); Object.keys(flattenInitialTabsOptions).forEach((field) => { + // check if _cardinalityCount if so skip + if (field.includes('._cardinalityCount')) { + return; + } + // in flattened tab options, to get actual field name, strip off the last '.histogram' const actualFieldName = field.replace('.histogram', ''); // possible to have '.' in actualFieldName, so use it as a string @@ -65,8 +70,9 @@ export const updateCountsInInitialTabsOptions = ( return; } const histogram = flattenInitialTabsOptions[`${field}`]; - if (!histogram) { + if (!histogram || typeof histogram !== 'object') { console.error(`Guppy did not return histogram data for filter field ${actualFieldName}`); // eslint-disable-line no-console + return; } histogram.forEach((opt) => { const { key } = opt; From faed18aedfa41f8bbe3e6adbeb36a0f7fac2b5b4 Mon Sep 17 00:00:00 2001 From: ocshawn Date: Thu, 11 May 2023 15:10:03 -0500 Subject: [PATCH 02/10] PXP-9147: add _totalCount functionality and convert count chart to use it --- doc/queries.md | 35 ++++++++++++++++++++++++- src/components/ConnectedFilter/utils.js | 8 +++--- src/components/Utils/filters.js | 4 +-- src/components/Utils/queries.js | 12 ++++----- src/server/__tests__/schema.test.js | 4 +++ src/server/es/index.js | 19 ++++++++++++++ src/server/resolvers.js | 11 +++++++- src/server/schema.js | 2 ++ 8 files changed, 81 insertions(+), 14 deletions(-) diff --git a/doc/queries.md b/doc/queries.md index 1781b96a..140f2596 100644 --- a/doc/queries.md +++ b/doc/queries.md @@ -146,7 +146,10 @@ Aggregation query is wrapped within `_aggregation` keyword. Three possible aggre ### 1. Total count aggregation - By using `_totalCount` keyword, return total count of the result. Example: + By using `_totalCount` keyword, return total count of the result. + Can also use `_totalCount` keyword inside text aggregation to get [value count](https://www.elastic.co/guide/en/elasticsearch/reference/8.7/search-aggregations-metrics-valuecount-aggregation.html) of text aggregation + + Example: ``` query ($filter: JSON) { @@ -172,6 +175,36 @@ Example result: } ``` +Text Aggregation Example: + + ``` + query ($filter: JSON) { + _aggregation { + subject(filter: $filter) { + subject_id { + _totalCount + } + } + } +} +``` + +Example result: + +``` +{ + "data": { + "_aggregation": { + "subject": { + "subject_id": { + "_totalCount": 42 + } + } + } + } +} +``` + ### 2. Text aggregation diff --git a/src/components/ConnectedFilter/utils.js b/src/components/ConnectedFilter/utils.js index 052e6212..50ea81a3 100644 --- a/src/components/ConnectedFilter/utils.js +++ b/src/components/ConnectedFilter/utils.js @@ -182,16 +182,16 @@ export const excludeSelfFilterFromAggsData = (receivedAggsData, filterResults) = const resultAggsData = {}; const flattenAggsData = flat(receivedAggsData, { safe: true }); Object.keys(flattenAggsData).forEach((field) => { - const isCardinalityCount = field.includes('._cardinalityCount'); - const actualFieldName = isCardinalityCount - ? field.replace('._cardinalityCount', '') + const isCount = field.includes('._totalCount'); + const actualFieldName = isCount + ? field.replace('._totalCount', '') : field.replace('.histogram', ''); if (!resultAggsData[`${actualFieldName}`]) { resultAggsData[`${actualFieldName}`] = {}; } - if (isCardinalityCount) { + if (isCount) { resultAggsData[`${actualFieldName}`].cardinalityCount = flattenAggsData[`${field}`]; return; } diff --git a/src/components/Utils/filters.js b/src/components/Utils/filters.js index ddfe5f8c..519ce150 100644 --- a/src/components/Utils/filters.js +++ b/src/components/Utils/filters.js @@ -52,8 +52,8 @@ export const updateCountsInInitialTabsOptions = ( const flattenInitialTabsOptions = flat(initialTabsOptions, { safe: true }); const flattenProcessedTabsOptions = flat(processedTabsOptions, { safe: true }); Object.keys(flattenInitialTabsOptions).forEach((field) => { - // check if _cardinalityCount if so skip - if (field.includes('._cardinalityCount')) { + // check if _totalCount if so skip + if (field.includes('._totalCount')) { return; } diff --git a/src/components/Utils/queries.js b/src/components/Utils/queries.js index 19f29bf8..6a8903c3 100644 --- a/src/components/Utils/queries.js +++ b/src/components/Utils/queries.js @@ -23,24 +23,24 @@ const histogramQueryStrForEachField = (field) => { }`); }; -const cardinalityQueryStrForEachField = (field) => { +const countQueryStrForEachField = (field) => { const splittedFieldArray = field.split('.'); const splittedField = splittedFieldArray.shift(); if (splittedFieldArray.length === 0) { return (` ${splittedField} { - _cardinalityCount + _totalCount } `); } return (` ${splittedField} { - ${cardinalityQueryStrForEachField(splittedFieldArray.join('.'))} + ${countQueryStrForEachField(splittedFieldArray.join('.'))} }`); }; -const queryGuppyForAggs = (path, type, fields, cardinalityFields = [], gqlFilter, acc) => { +const queryGuppyForAggs = (path, type, fields, countFields = [], gqlFilter, acc) => { let accessibility = acc; if (accessibility !== 'all' && accessibility !== 'accessible' && accessibility !== 'unaccessible') { accessibility = 'all'; @@ -52,7 +52,7 @@ const queryGuppyForAggs = (path, type, fields, cardinalityFields = [], gqlFilter _aggregation { ${type} (filter: $filter, filterSelf: false, accessibility: ${accessibility}) { ${fields.map((field) => histogramQueryStrForEachField(field))}, - ${cardinalityFields.map((field) => cardinalityQueryStrForEachField(field))} + ${countFields.map((field) => countQueryStrForEachField(field))} } } }`; @@ -63,7 +63,7 @@ const queryGuppyForAggs = (path, type, fields, cardinalityFields = [], gqlFilter _aggregation { ${type} (accessibility: ${accessibility}) { ${fields.map((field) => histogramQueryStrForEachField(field))} - ${cardinalityFields.map((field) => cardinalityQueryStrForEachField(field))} + ${countFields.map((field) => countQueryStrForEachField(field))} } } }`; diff --git a/src/server/__tests__/schema.test.js b/src/server/__tests__/schema.test.js index db80a2ae..57792559 100644 --- a/src/server/__tests__/schema.test.js +++ b/src/server/__tests__/schema.test.js @@ -147,14 +147,17 @@ describe('Schema', () => { const expectedHistogramSchemas = ` type HistogramForString { + _totalCount: Int, _cardinalityCount(precision_threshold:Int=3000): Int, histogram: [BucketsForNestedStringAgg] } type RegularAccessHistogramForString { + _totalCount: Int, _cardinalityCount(precision_threshold:Int=3000): Int, histogram: [BucketsForNestedStringAgg] } type HistogramForNumber { + _totalCount: Int, _cardinalityCount(precision_threshold:Int=3000): Int, histogram( rangeStart: Int, @@ -165,6 +168,7 @@ describe('Schema', () => { asTextHistogram: [BucketsForNestedStringAgg] } type RegularAccessHistogramForNumber { + _totalCount: Int, _cardinalityCount(precision_threshold:Int=3000): Int, histogram( rangeStart: Int, diff --git a/src/server/es/index.js b/src/server/es/index.js index df567057..cb334190 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -429,6 +429,25 @@ class ES { return result.hits.total; } + async getFeildCount(esIndex, esType, filter, field) { + const queryBody = { + size: 0, + aggs: { + [field]: { + value_count: { + field, + }, + }, + }, + }; + if (typeof filter !== 'undefined') { + queryBody.query = getFilterObj(this, esIndex, filter); + } + + const result = await this.query(esIndex, esType, queryBody); + return result.aggregations[field].value; + } + // eslint-disable-next-line camelcase async getCardinalityCount(esIndex, esType, filter, field, precision_threshold) { const queryBody = { diff --git a/src/server/resolvers.js b/src/server/resolvers.js index 0390d47c..cd2e60d3 100644 --- a/src/server/resolvers.js +++ b/src/server/resolvers.js @@ -50,9 +50,13 @@ const typeAggsQueryResolver = (esInstance, esIndex, esType) => (parent, args) => * @param {object} parent */ const aggsTotalQueryResolver = (parent) => { + log.debug('[resolver.aggsTotalQueryResolver] parent', parent); const { - filter, esInstance, esIndex, esType, + filter, esInstance, esIndex, esType, field, } = parent; + if (field) { + return esInstance.getFeildCount(esIndex, esType, filter, field); + } return esInstance.getCount(esIndex, esType, filter); }; @@ -188,6 +192,7 @@ const getFieldAggregationResolverMappings = (esInstance, esIndex) => { * } * } * file_count { + * _totalCount ---> `aggsTotalQueryResolver` * _cardinality ( * precision_threshold: 1000 //optional * ), ---> `cardinalityResolver` @@ -285,20 +290,24 @@ const getResolver = (esConfig, esInstance) => { ...typeAggregationResolvers, ...typeNestedAggregationResolvers, HistogramForNumber: { + _totalCount: aggsTotalQueryResolver, _cardinalityCount: cardinalityResolver, histogram: numericHistogramResolver, asTextHistogram: textHistogramResolver, }, HistogramForString: { + _totalCount: aggsTotalQueryResolver, _cardinalityCount: cardinalityResolver, histogram: textHistogramResolver, }, RegularAccessHistogramForNumber: { + _totalCount: aggsTotalQueryResolver, _cardinalityCount: cardinalityResolver, histogram: numericHistogramResolver, asTextHistogram: textHistogramResolver, }, RegularAccessHistogramForString: { + _totalCount: aggsTotalQueryResolver, _cardinalityCount: cardinalityResolver, histogram: textHistogramResolver, }, diff --git a/src/server/schema.js b/src/server/schema.js index 63b04043..8fb22409 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -228,6 +228,7 @@ export const getAggregationSchemaForEachNestedType = (esConfig, esInstance) => e const getNumberHistogramSchema = (isRegularAccess) => ` type ${(isRegularAccess ? histogramTypePrefix : '') + EnumAggsHistogramName.HISTOGRAM_FOR_NUMBER} { + _totalCount: Int, _cardinalityCount(precision_threshold: Int = 3000): Int, histogram( rangeStart: Int, @@ -241,6 +242,7 @@ const getNumberHistogramSchema = (isRegularAccess) => ` const getTextHistogramSchema = (isRegularAccess) => ` type ${(isRegularAccess ? histogramTypePrefix : '') + EnumAggsHistogramName.HISTOGRAM_FOR_STRING} { + _totalCount: Int, _cardinalityCount(precision_threshold: Int = 3000): Int, histogram: [BucketsForNestedStringAgg] } From b778ededc884ecb8dbe44fedebbebd33643c22dd Mon Sep 17 00:00:00 2001 From: ocshawn Date: Thu, 11 May 2023 15:55:49 -0500 Subject: [PATCH 03/10] PXP-9147: update variable name --- src/components/ConnectedFilter/index.jsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/components/ConnectedFilter/index.jsx b/src/components/ConnectedFilter/index.jsx index a04c073f..d9d76845 100644 --- a/src/components/ConnectedFilter/index.jsx +++ b/src/components/ConnectedFilter/index.jsx @@ -46,7 +46,7 @@ class ConnectedFilter extends React.Component { this.state = { allFields, - countFields: this.props.extraAggsFieldsCardinalityCount, + countFields: this.props.extraAggsFieldsCount, initialAggsData: {}, receivedAggsData: {}, accessibility: ENUM_ACCESSIBILITY.ALL, @@ -320,7 +320,7 @@ ConnectedFilter.propTypes = { })), }).isRequired, extraAggsFields: PropTypes.arrayOf(PropTypes.string), - extraAggsFieldsCardinalityCount: PropTypes.arrayOf(PropTypes.string), + extraAggsFieldsCount: PropTypes.arrayOf(PropTypes.string), guppyConfig: PropTypes.shape({ path: PropTypes.string.isRequired, type: PropTypes.string.isRequired, @@ -348,7 +348,7 @@ ConnectedFilter.propTypes = { ConnectedFilter.defaultProps = { extraAggsFields: [], - extraAggsFieldsCardinalityCount: [], + extraAggsFieldsCount: [], onFilterChange: () => {}, onReceiveNewAggsData: () => {}, className: '', From 46941e3024e75da70c23560503b11cb163ac3f15 Mon Sep 17 00:00:00 2001 From: ocshawn Date: Fri, 12 May 2023 09:19:53 -0500 Subject: [PATCH 04/10] PXP-9147: fix vareable name --- src/components/ConnectedFilter/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/ConnectedFilter/utils.js b/src/components/ConnectedFilter/utils.js index 50ea81a3..e1800ba4 100644 --- a/src/components/ConnectedFilter/utils.js +++ b/src/components/ConnectedFilter/utils.js @@ -192,7 +192,7 @@ export const excludeSelfFilterFromAggsData = (receivedAggsData, filterResults) = } if (isCount) { - resultAggsData[`${actualFieldName}`].cardinalityCount = flattenAggsData[`${field}`]; + resultAggsData[`${actualFieldName}`].totalCount = flattenAggsData[`${field}`]; return; } const histogram = flattenAggsData[`${field}`]; From 17caa88f625e0cc1f8f76381c701503faa2325dd Mon Sep 17 00:00:00 2001 From: ocshawn Date: Mon, 15 May 2023 10:33:37 -0500 Subject: [PATCH 05/10] PXP-9147: improve count chart performance by not sorting into filter values --- src/components/ConnectedFilter/index.jsx | 13 ++++++---- src/components/ConnectedFilter/utils.js | 18 +++----------- src/components/Utils/filters.js | 10 ++++---- src/components/Utils/queries.js | 30 ++++-------------------- src/components/__tests__/filters.test.js | 13 ++++++++-- 5 files changed, 33 insertions(+), 51 deletions(-) diff --git a/src/components/ConnectedFilter/index.jsx b/src/components/ConnectedFilter/index.jsx index d9d76845..d863b34a 100644 --- a/src/components/ConnectedFilter/index.jsx +++ b/src/components/ConnectedFilter/index.jsx @@ -31,7 +31,7 @@ class ConnectedFilter extends React.Component { let allFields = props.accessibleFieldCheckList ? _.union(filterConfigsFields, props.accessibleFieldCheckList) : filterConfigsFields; - allFields = _.union(allFields, this.props.extraAggsFields); + allFields = _.union(allFields, this.props.extraAggsFields, this.props.extraAggsFieldsCount); this.initialTabsOptions = {}; let initialFilter = this.props.adminAppliedPreFilters; @@ -46,7 +46,6 @@ class ConnectedFilter extends React.Component { this.state = { allFields, - countFields: this.props.extraAggsFieldsCount, initialAggsData: {}, receivedAggsData: {}, accessibility: ENUM_ACCESSIBILITY.ALL, @@ -71,7 +70,6 @@ class ConnectedFilter extends React.Component { this.props.guppyConfig.path, this.props.guppyConfig.type, this.state.allFields, - this.state.countFields, this.state.accessibility, this.state.filter, ) @@ -128,7 +126,6 @@ class ConnectedFilter extends React.Component { this.props.guppyConfig.path, this.props.guppyConfig.type, this.state.allFields, - this.state.countFields, mergedFilterResults, this.state.accessibility, ) @@ -172,6 +169,12 @@ class ConnectedFilter extends React.Component { const filtersToDisplay = this.state.filtersApplied; if (this.props.hidden) return null; let processedTabsOptions = this.props.onProcessFilterAggsData(this.state.receivedAggsData); + + // Get filter values + const allFilterValues = this.props.filterConfig.tabs.reduce( + (accumulator, tab) => ([...accumulator, ...tab.fields]), [], + ); + if (Object.keys(this.initialTabsOptions).length === 0) { this.initialTabsOptions = processedTabsOptions; } @@ -182,8 +185,8 @@ class ConnectedFilter extends React.Component { filtersToDisplay, // for tiered access filters this.props.tierAccessLimit ? this.props.accessibleFieldCheckList : [], + allFilterValues, ); - if (Object.keys(filtersToDisplay).length) { // if has applied filters, sort tab options as selected/unselected separately const selectedTabsOptions = {}; diff --git a/src/components/ConnectedFilter/utils.js b/src/components/ConnectedFilter/utils.js index e1800ba4..0eb6f295 100644 --- a/src/components/ConnectedFilter/utils.js +++ b/src/components/ConnectedFilter/utils.js @@ -182,19 +182,7 @@ export const excludeSelfFilterFromAggsData = (receivedAggsData, filterResults) = const resultAggsData = {}; const flattenAggsData = flat(receivedAggsData, { safe: true }); Object.keys(flattenAggsData).forEach((field) => { - const isCount = field.includes('._totalCount'); - const actualFieldName = isCount - ? field.replace('._totalCount', '') - : field.replace('.histogram', ''); - - if (!resultAggsData[`${actualFieldName}`]) { - resultAggsData[`${actualFieldName}`] = {}; - } - - if (isCount) { - resultAggsData[`${actualFieldName}`].totalCount = flattenAggsData[`${field}`]; - return; - } + const actualFieldName = field.replace('.histogram', ''); const histogram = flattenAggsData[`${field}`]; if (!histogram) return; if (actualFieldName in filterResults) { @@ -203,9 +191,9 @@ export const excludeSelfFilterFromAggsData = (receivedAggsData, filterResults) = const { selectedValues } = filterResults[`${actualFieldName}`]; resultHistogram = histogram.filter((bucket) => selectedValues.includes(bucket.key)); } - resultAggsData[`${actualFieldName}`].histogram = resultHistogram; + resultAggsData[`${actualFieldName}`] = { histogram: resultHistogram }; } else { - resultAggsData[`${actualFieldName}`].histogram = flattenAggsData[`${field}`]; + resultAggsData[`${actualFieldName}`] = { histogram: flattenAggsData[`${field}`] }; } }); return resultAggsData; diff --git a/src/components/Utils/filters.js b/src/components/Utils/filters.js index 519ce150..af8e061b 100644 --- a/src/components/Utils/filters.js +++ b/src/components/Utils/filters.js @@ -41,6 +41,7 @@ export const mergeFilters = (userFilter, adminAppliedPreFilter) => { */ export const updateCountsInInitialTabsOptions = ( initialTabsOptions, processedTabsOptions, filtersApplied, accessibleFieldCheckList, + allFilterValues, ) => { const updatedTabsOptions = {}; try { @@ -52,13 +53,14 @@ export const updateCountsInInitialTabsOptions = ( const flattenInitialTabsOptions = flat(initialTabsOptions, { safe: true }); const flattenProcessedTabsOptions = flat(processedTabsOptions, { safe: true }); Object.keys(flattenInitialTabsOptions).forEach((field) => { - // check if _totalCount if so skip - if (field.includes('._totalCount')) { + // in flattened tab options, to get actual field name, strip off the last '.histogram' + const actualFieldName = field.replace('.histogram', ''); + + // check if Filter Value if not skip + if (!allFilterValues.includes(actualFieldName)) { return; } - // in flattened tab options, to get actual field name, strip off the last '.histogram' - const actualFieldName = field.replace('.histogram', ''); // possible to have '.' in actualFieldName, so use it as a string updatedTabsOptions[`${actualFieldName}`] = { histogram: [] }; // if in tiered access mode diff --git a/src/components/Utils/queries.js b/src/components/Utils/queries.js index 6a8903c3..96674da7 100644 --- a/src/components/Utils/queries.js +++ b/src/components/Utils/queries.js @@ -23,24 +23,7 @@ const histogramQueryStrForEachField = (field) => { }`); }; -const countQueryStrForEachField = (field) => { - const splittedFieldArray = field.split('.'); - const splittedField = splittedFieldArray.shift(); - - if (splittedFieldArray.length === 0) { - return (` - ${splittedField} { - _totalCount - } - `); - } - return (` - ${splittedField} { - ${countQueryStrForEachField(splittedFieldArray.join('.'))} - }`); -}; - -const queryGuppyForAggs = (path, type, fields, countFields = [], gqlFilter, acc) => { +const queryGuppyForAggs = (path, type, fields, gqlFilter, acc) => { let accessibility = acc; if (accessibility !== 'all' && accessibility !== 'accessible' && accessibility !== 'unaccessible') { accessibility = 'all'; @@ -51,8 +34,7 @@ const queryGuppyForAggs = (path, type, fields, countFields = [], gqlFilter, acc) const queryWithFilter = `query ($filter: JSON) { _aggregation { ${type} (filter: $filter, filterSelf: false, accessibility: ${accessibility}) { - ${fields.map((field) => histogramQueryStrForEachField(field))}, - ${countFields.map((field) => countQueryStrForEachField(field))} + ${fields.map((field) => histogramQueryStrForEachField(field))} } } }`; @@ -63,7 +45,6 @@ const queryGuppyForAggs = (path, type, fields, countFields = [], gqlFilter, acc) _aggregation { ${type} (accessibility: ${accessibility}) { ${fields.map((field) => histogramQueryStrForEachField(field))} - ${countFields.map((field) => countQueryStrForEachField(field))} } } }`; @@ -286,10 +267,10 @@ export const getGQLFilter = (filterObj) => { }; export const askGuppyAboutAllFieldsAndOptions = ( - path, type, fields, countFields, accessibility, filter, + path, type, fields, accessibility, filter, ) => { const gqlFilter = getGQLFilter(filter); - return queryGuppyForAggs(path, type, fields, countFields, gqlFilter, accessibility); + return queryGuppyForAggs(path, type, fields, gqlFilter, accessibility); }; // eslint-disable-next-line max-len @@ -299,12 +280,11 @@ export const askGuppyForAggregationData = ( path, type, fields, - countFields, filter, accessibility, ) => { const gqlFilter = getGQLFilter(filter); - return queryGuppyForAggs(path, type, fields, countFields, gqlFilter, accessibility); + return queryGuppyForAggs(path, type, fields, gqlFilter, accessibility); }; export const askGuppyForSubAggregationData = ( diff --git a/src/components/__tests__/filters.test.js b/src/components/__tests__/filters.test.js index 60a06c18..fab2a565 100644 --- a/src/components/__tests__/filters.test.js +++ b/src/components/__tests__/filters.test.js @@ -64,6 +64,11 @@ describe('will select user-applied filter for a given key if it is more exclusiv }); describe('can update a small set of tabs with new counts', () => { + const allFilterValues = [ + 'annotated_sex', + 'extra_data', + ]; + const initialTabsOptions = { annotated_sex: { histogram: [ @@ -107,7 +112,7 @@ describe('can update a small set of tabs with new counts', () => { }; const actualUpdatedTabsOptions = updateCountsInInitialTabsOptions( - initialTabsOptions, processedTabsOptions, filtersApplied, + initialTabsOptions, processedTabsOptions, filtersApplied, undefined, allFilterValues, ); test('update tab counts', async () => { @@ -117,6 +122,10 @@ describe('can update a small set of tabs with new counts', () => { }); describe('can update a small set of tabs with new counts, test with ranger slide', () => { + const allFilterValues = [ + 'field1', + 'field2', + ]; const initialTabsOptions = { field1: { histogram: [ @@ -175,7 +184,7 @@ describe('can update a small set of tabs with new counts, test with ranger slide }; const actualUpdatedTabsOptions = updateCountsInInitialTabsOptions( - initialTabsOptions, processedTabsOptions, filtersApplied, + initialTabsOptions, processedTabsOptions, filtersApplied, undefined, allFilterValues, ); test('update tab counts', async () => { From 736aa75469936f59513089432518bd70864d263a Mon Sep 17 00:00:00 2001 From: ocshawn Date: Mon, 15 May 2023 10:54:37 -0500 Subject: [PATCH 06/10] PXP-9147: remove unneded extraAggsFieldsCount --- src/components/ConnectedFilter/index.jsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/components/ConnectedFilter/index.jsx b/src/components/ConnectedFilter/index.jsx index d863b34a..3b7cfc89 100644 --- a/src/components/ConnectedFilter/index.jsx +++ b/src/components/ConnectedFilter/index.jsx @@ -31,7 +31,7 @@ class ConnectedFilter extends React.Component { let allFields = props.accessibleFieldCheckList ? _.union(filterConfigsFields, props.accessibleFieldCheckList) : filterConfigsFields; - allFields = _.union(allFields, this.props.extraAggsFields, this.props.extraAggsFieldsCount); + allFields = _.union(allFields, this.props.extraAggsFields); this.initialTabsOptions = {}; let initialFilter = this.props.adminAppliedPreFilters; @@ -187,6 +187,7 @@ class ConnectedFilter extends React.Component { this.props.tierAccessLimit ? this.props.accessibleFieldCheckList : [], allFilterValues, ); + if (Object.keys(filtersToDisplay).length) { // if has applied filters, sort tab options as selected/unselected separately const selectedTabsOptions = {}; @@ -323,7 +324,6 @@ ConnectedFilter.propTypes = { })), }).isRequired, extraAggsFields: PropTypes.arrayOf(PropTypes.string), - extraAggsFieldsCount: PropTypes.arrayOf(PropTypes.string), guppyConfig: PropTypes.shape({ path: PropTypes.string.isRequired, type: PropTypes.string.isRequired, @@ -351,7 +351,6 @@ ConnectedFilter.propTypes = { ConnectedFilter.defaultProps = { extraAggsFields: [], - extraAggsFieldsCount: [], onFilterChange: () => {}, onReceiveNewAggsData: () => {}, className: '', From 090a4cba7441207e3db6d083545a1a7b6b60f1a9 Mon Sep 17 00:00:00 2001 From: ocshawn Date: Tue, 16 May 2023 16:17:32 -0500 Subject: [PATCH 07/10] PXP-9147: add asTextHistogram to text resolvers, use asTextHistogram for all charts --- src/components/Utils/queries.js | 2 +- src/server/__tests__/schema.test.js | 6 ++++-- src/server/resolvers.js | 2 ++ src/server/schema.js | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/components/Utils/queries.js b/src/components/Utils/queries.js index 96674da7..96efc61f 100644 --- a/src/components/Utils/queries.js +++ b/src/components/Utils/queries.js @@ -11,7 +11,7 @@ const histogramQueryStrForEachField = (field) => { if (splittedFieldArray.length === 0) { return (` ${splittedField} { - histogram { + asTextHistogram { key count } diff --git a/src/server/__tests__/schema.test.js b/src/server/__tests__/schema.test.js index 57792559..431911b2 100644 --- a/src/server/__tests__/schema.test.js +++ b/src/server/__tests__/schema.test.js @@ -149,12 +149,14 @@ describe('Schema', () => { type HistogramForString { _totalCount: Int, _cardinalityCount(precision_threshold:Int=3000): Int, - histogram: [BucketsForNestedStringAgg] + histogram: [BucketsForNestedStringAgg], + asTextHistogram: [BucketsForNestedStringAgg] } type RegularAccessHistogramForString { _totalCount: Int, _cardinalityCount(precision_threshold:Int=3000): Int, - histogram: [BucketsForNestedStringAgg] + histogram: [BucketsForNestedStringAgg], + asTextHistogram: [BucketsForNestedStringAgg] } type HistogramForNumber { _totalCount: Int, diff --git a/src/server/resolvers.js b/src/server/resolvers.js index cd2e60d3..ba878e45 100644 --- a/src/server/resolvers.js +++ b/src/server/resolvers.js @@ -299,6 +299,7 @@ const getResolver = (esConfig, esInstance) => { _totalCount: aggsTotalQueryResolver, _cardinalityCount: cardinalityResolver, histogram: textHistogramResolver, + asTextHistogram: textHistogramResolver, }, RegularAccessHistogramForNumber: { _totalCount: aggsTotalQueryResolver, @@ -310,6 +311,7 @@ const getResolver = (esConfig, esInstance) => { _totalCount: aggsTotalQueryResolver, _cardinalityCount: cardinalityResolver, histogram: textHistogramResolver, + asTextHistogram: textHistogramResolver, }, Mapping: { ...mappingResolvers, diff --git a/src/server/schema.js b/src/server/schema.js index 8fb22409..a7dd919b 100644 --- a/src/server/schema.js +++ b/src/server/schema.js @@ -244,7 +244,8 @@ const getTextHistogramSchema = (isRegularAccess) => ` type ${(isRegularAccess ? histogramTypePrefix : '') + EnumAggsHistogramName.HISTOGRAM_FOR_STRING} { _totalCount: Int, _cardinalityCount(precision_threshold: Int = 3000): Int, - histogram: [BucketsForNestedStringAgg] + histogram: [BucketsForNestedStringAgg], + asTextHistogram: [BucketsForNestedStringAgg] } `; From 2939b3a1f96ae262aefce801d5133d9e871b8bf8 Mon Sep 17 00:00:00 2001 From: ocshawn Date: Wed, 17 May 2023 11:08:41 -0500 Subject: [PATCH 08/10] PXP-9147: fix handaling of asTextHistogram response --- src/components/ConnectedFilter/utils.js | 2 +- src/components/Utils/filters.js | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/components/ConnectedFilter/utils.js b/src/components/ConnectedFilter/utils.js index 0eb6f295..e331fef1 100644 --- a/src/components/ConnectedFilter/utils.js +++ b/src/components/ConnectedFilter/utils.js @@ -182,7 +182,7 @@ export const excludeSelfFilterFromAggsData = (receivedAggsData, filterResults) = const resultAggsData = {}; const flattenAggsData = flat(receivedAggsData, { safe: true }); Object.keys(flattenAggsData).forEach((field) => { - const actualFieldName = field.replace('.histogram', ''); + const actualFieldName = field.replace('.asTextHistogram', ''); const histogram = flattenAggsData[`${field}`]; if (!histogram) return; if (actualFieldName in filterResults) { diff --git a/src/components/Utils/filters.js b/src/components/Utils/filters.js index af8e061b..69203251 100644 --- a/src/components/Utils/filters.js +++ b/src/components/Utils/filters.js @@ -47,14 +47,14 @@ export const updateCountsInInitialTabsOptions = ( try { // flatten the tab options first // { - // project_id.histogram: ... - // visit.visit_label.histogram: ... + // project_id.asTextHistogram: ... + // visit.visit_label.asTextHistogram: ... // } const flattenInitialTabsOptions = flat(initialTabsOptions, { safe: true }); const flattenProcessedTabsOptions = flat(processedTabsOptions, { safe: true }); Object.keys(flattenInitialTabsOptions).forEach((field) => { - // in flattened tab options, to get actual field name, strip off the last '.histogram' - const actualFieldName = field.replace('.histogram', ''); + // in flattened tab options, to get actual field name, strip off the last '.asTextHistogram' + const actualFieldName = field.replace('.asTextHistogram', ''); // check if Filter Value if not skip if (!allFilterValues.includes(actualFieldName)) { From 434819ccf7c6ca72add582406daee59f334614f1 Mon Sep 17 00:00:00 2001 From: ocshawn Date: Wed, 17 May 2023 11:47:12 -0500 Subject: [PATCH 09/10] PXP-9147: update asTextHistogram tests --- src/components/__tests__/filters.test.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/components/__tests__/filters.test.js b/src/components/__tests__/filters.test.js index fab2a565..099fe08c 100644 --- a/src/components/__tests__/filters.test.js +++ b/src/components/__tests__/filters.test.js @@ -71,7 +71,7 @@ describe('can update a small set of tabs with new counts', () => { const initialTabsOptions = { annotated_sex: { - histogram: [ + asTextHistogram: [ { key: 'yellow', count: 137675 }, { key: 'pink', count: 56270 }, { key: 'silver', count: 2020 }, @@ -79,7 +79,7 @@ describe('can update a small set of tabs with new counts', () => { ], }, extra_data: { - histogram: [ + asTextHistogram: [ { key: 'a', count: 2 }, ], }, @@ -87,12 +87,12 @@ describe('can update a small set of tabs with new counts', () => { const processedTabsOptions = { annotated_sex: { - histogram: [ + asTextHistogram: [ { key: 'yellow', count: 1 }, { key: 'orange', count: 107574 }, ], }, - extra_data: { histogram: [] }, + extra_data: { asTextHistogram: [] }, }; const filtersApplied = { annotated_sex: { selectedValues: ['silver'] } }; @@ -128,7 +128,7 @@ describe('can update a small set of tabs with new counts, test with ranger slide ]; const initialTabsOptions = { field1: { - histogram: [ + asTextHistogram: [ { key: 'option1', count: 137675 }, { key: 'option2', count: 56270 }, { key: 'option3', count: 2020 }, @@ -136,7 +136,7 @@ describe('can update a small set of tabs with new counts, test with ranger slide ], }, field2: { - histogram: [ + asTextHistogram: [ { key: [0, 100], count: 100 }, ], }, @@ -144,12 +144,12 @@ describe('can update a small set of tabs with new counts, test with ranger slide const processedTabsOptions = { field1: { - histogram: [ + asTextHistogram: [ { key: 'option3', count: 30 }, ], }, field2: { - histogram: [ + asTextHistogram: [ { key: [4, 39], count: 49, From dee049e8f57cb8ba7a07d4df228d7a4e119e9dca Mon Sep 17 00:00:00 2001 From: ocshawn Date: Fri, 19 May 2023 09:37:18 -0500 Subject: [PATCH 10/10] PXP-9147: fix spelling error and update version number --- package-lock.json | 4 ++-- package.json | 2 +- src/server/es/index.js | 2 +- src/server/resolvers.js | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6ec779b3..a3748fe6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gen3/guppy", - "version": "0.15.8", + "version": "0.16.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@gen3/guppy", - "version": "0.15.8", + "version": "0.16.0", "license": "ISC", "dependencies": { "@elastic/elasticsearch": "~7.13.0", diff --git a/package.json b/package.json index 76ef6535..45f5ad46 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gen3/guppy", - "version": "0.15.8", + "version": "0.16.0", "description": "Server that support GraphQL queries on data from elasticsearch", "main": "src/server/server.js", "directories": { diff --git a/src/server/es/index.js b/src/server/es/index.js index cb334190..0e82b5ba 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -429,7 +429,7 @@ class ES { return result.hits.total; } - async getFeildCount(esIndex, esType, filter, field) { + async getFieldCount(esIndex, esType, filter, field) { const queryBody = { size: 0, aggs: { diff --git a/src/server/resolvers.js b/src/server/resolvers.js index ba878e45..b4cdf128 100644 --- a/src/server/resolvers.js +++ b/src/server/resolvers.js @@ -55,7 +55,7 @@ const aggsTotalQueryResolver = (parent) => { filter, esInstance, esIndex, esType, field, } = parent; if (field) { - return esInstance.getFeildCount(esIndex, esType, filter, field); + return esInstance.getFieldCount(esIndex, esType, filter, field); } return esInstance.getCount(esIndex, esType, filter); };