From 8fae23513397a7c1baa97d021d991fc89c639dcb Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Fri, 8 Sep 2023 12:22:05 -0500 Subject: [PATCH 1/8] fix separate fields --- src/components/ConnectedFilter/index.jsx | 25 +++++++++++-------- src/components/ConnectedFilter/utils.js | 2 +- src/components/GuppyWrapper/index.jsx | 8 +++--- src/components/Utils/filters.js | 10 ++++---- src/components/Utils/queries.js | 31 +++++++++++++----------- src/components/__tests__/filters.test.js | 16 ++++++------ 6 files changed, 51 insertions(+), 41 deletions(-) diff --git a/src/components/ConnectedFilter/index.jsx b/src/components/ConnectedFilter/index.jsx index b384501c..4119bb27 100644 --- a/src/components/ConnectedFilter/index.jsx +++ b/src/components/ConnectedFilter/index.jsx @@ -10,7 +10,6 @@ import { } from './utils'; import { ENUM_ACCESSIBILITY } from '../Utils/const'; import { - askGuppyAboutAllFieldsAndOptions, askGuppyAboutArrayTypes, askGuppyForAggregationData, getAllFieldsFromFilterConfigs, @@ -28,10 +27,13 @@ class ConnectedFilter extends React.Component { super(props); const filterConfigsFields = getAllFieldsFromFilterConfigs(props.filterConfig.tabs); - let allFields = props.accessibleFieldCheckList - ? _.union(filterConfigsFields, props.accessibleFieldCheckList) - : filterConfigsFields; - allFields = _.union(allFields, this.props.extraAggsFields); + const filterConfigsRegularAggFields = filterConfigsFields.fields || []; + const filterConfigsAsTextAggFields = filterConfigsFields.asTextAggFields || []; + const allRegularAggFields = props.accessibleFieldCheckList + ? _.union(filterConfigsRegularAggFields, props.accessibleFieldCheckList) + : filterConfigsRegularAggFields; + // props.extraAggsFields are chart fields, use asTextAgg for all of them + const allAsTextAggFields = _.union(filterConfigsAsTextAggFields, this.props.extraAggsFields); this.initialTabsOptions = {}; let initialFilter = this.props.adminAppliedPreFilters; @@ -47,7 +49,8 @@ class ConnectedFilter extends React.Component { } this.state = { - allFields, + allRegularAggFields, + allAsTextAggFields, initialAggsData: {}, receivedAggsData: {}, accessibility: ENUM_ACCESSIBILITY.ALL, @@ -68,12 +71,13 @@ class ConnectedFilter extends React.Component { if (this.props.onFilterChange) { this.props.onFilterChange(this.state.adminAppliedPreFilters, this.state.accessibility); } - askGuppyAboutAllFieldsAndOptions( + askGuppyForAggregationData( this.props.guppyConfig.path, this.props.guppyConfig.type, - this.state.allFields, - this.state.accessibility, + this.state.allRegularAggFields, + this.state.allAsTextAggFields, this.state.filter, + this.state.accessibility, ) .then((res) => { if (!res.data) { @@ -129,7 +133,8 @@ class ConnectedFilter extends React.Component { askGuppyForAggregationData( this.props.guppyConfig.path, this.props.guppyConfig.type, - this.state.allFields, + this.state.allRegularAggFields, + this.state.allAsTextAggFields, mergedFilterResults, this.state.accessibility, ) diff --git a/src/components/ConnectedFilter/utils.js b/src/components/ConnectedFilter/utils.js index 548e179e..c38582c6 100644 --- a/src/components/ConnectedFilter/utils.js +++ b/src/components/ConnectedFilter/utils.js @@ -189,7 +189,7 @@ export const excludeSelfFilterFromAggsData = (receivedAggsData, filterResults) = const resultAggsData = {}; const flattenAggsData = flat(receivedAggsData, { safe: true }); Object.keys(flattenAggsData).forEach((field) => { - const actualFieldName = field.replace('.asTextHistogram', ''); + const actualFieldName = field.replace('histogram', '').replace('.asTextHistogram', ''); const histogram = flattenAggsData[`${field}`]; if (!histogram) return; if (actualFieldName in filterResults) { diff --git a/src/components/GuppyWrapper/index.jsx b/src/components/GuppyWrapper/index.jsx index d58c0a9a..2303fd07 100644 --- a/src/components/GuppyWrapper/index.jsx +++ b/src/components/GuppyWrapper/index.jsx @@ -64,7 +64,8 @@ class GuppyWrapper extends React.Component { filter: { ...initialFilter }, rawData: [], totalCount: 0, - allFields: [], + allRegularAggFields: [], + allAsTextAggFields: [], rawDataFields: [], accessibleFieldObject: undefined, unaccessibleFieldObject: undefined, @@ -82,7 +83,7 @@ class GuppyWrapper extends React.Component { const rawDataFields = (this.props.rawDataFields && this.props.rawDataFields.length > 0) ? this.props.rawDataFields : fields; this.setState({ - allFields: fields, + allRegularAggFields: fields, rawDataFields, }, () => { this.getDataFromGuppy(this.state.rawDataFields, undefined, true); @@ -326,7 +327,8 @@ class GuppyWrapper extends React.Component { fetchAndUpdateRawData: this.handleFetchAndUpdateRawData.bind(this), downloadRawData: this.handleDownloadRawData.bind(this), downloadRawDataByFields: this.handleDownloadRawDataByFields.bind(this), - allFields: this.state.allFields, + allRegularAggFields: this.state.allRegularAggFields, + allAsTextAggFields: this.state.allAsTextAggFields, accessibleFieldObject: this.state.accessibleFieldObject, unaccessibleFieldObject: this.state.unaccessibleFieldObject, diff --git a/src/components/Utils/filters.js b/src/components/Utils/filters.js index 312a3d5d..7627aafe 100644 --- a/src/components/Utils/filters.js +++ b/src/components/Utils/filters.js @@ -50,14 +50,14 @@ export const updateCountsInInitialTabsOptions = ( try { // flatten the tab options first // { - // project_id.asTextHistogram: ... - // visit.visit_label.asTextHistogram: ... + // project_id.histogram: ... + // visit.visit_label.histogram: ... // } const flattenInitialTabsOptions = flat(initialTabsOptions, { safe: true }); const flattenProcessedTabsOptions = flat(processedTabsOptions, { safe: true }); Object.keys(flattenInitialTabsOptions).forEach((field) => { - // in flattened tab options, to get actual field name, strip off the last '.asTextHistogram' - const actualFieldName = field.replace('.asTextHistogram', ''); + // in flattened tab options, to get actual field name, strip off the last '.histogram' or '.asTextHistogram' + const actualFieldName = field.replace('histogram', '').replace('.asTextHistogram', ''); // check if Filter Value if not skip if (!allFilterValues.includes(actualFieldName)) { @@ -190,7 +190,7 @@ export const buildFilterStatusForURLFilter = (userFilter, tabs) => { const filterStatusArray = tabs.map(() => ([])); for (let tabIndex = 0; tabIndex < tabs.length; tabIndex += 1) { - const allFieldsForThisTab = tabs[tabIndex].fields; + const allFieldsForThisTab = _.union(tabs[tabIndex].fields, tabs[tabIndex].asTextAggFields || []); filterStatusArray[tabIndex] = allFieldsForThisTab.map(() => ({})); for (let i = 0; i < filteringFields.length; i += 1) { const sectionIndex = allFieldsForThisTab.indexOf(filteringFields[i]); diff --git a/src/components/Utils/queries.js b/src/components/Utils/queries.js index 91229706..d713ba10 100644 --- a/src/components/Utils/queries.js +++ b/src/components/Utils/queries.js @@ -5,13 +5,13 @@ const graphqlEndpoint = '/graphql'; const downloadEndpoint = '/download'; const statusEndpoint = '/_status'; -const histogramQueryStrForEachField = (field) => { +const histogramQueryStrForEachField = (field, isAsTextAgg = false) => { const splittedFieldArray = field.split('.'); const splittedField = splittedFieldArray.shift(); if (splittedFieldArray.length === 0) { return (` ${splittedField} { - asTextHistogram { + ${(isAsTextAgg) ? 'asTextHistogram' : 'histogram'} { key count } @@ -23,7 +23,7 @@ const histogramQueryStrForEachField = (field) => { }`); }; -const queryGuppyForAggs = (path, type, fields, gqlFilter, acc) => { +const queryGuppyForAggs = (path, type, regularAggFields, asTextAggFields, gqlFilter, acc) => { let accessibility = acc; if (accessibility !== 'all' && accessibility !== 'accessible' && accessibility !== 'unaccessible') { accessibility = 'all'; @@ -34,7 +34,8 @@ const queryGuppyForAggs = (path, type, fields, gqlFilter, acc) => { const queryWithFilter = `query ($filter: JSON) { _aggregation { ${type} (filter: $filter, filterSelf: false, accessibility: ${accessibility}) { - ${fields.map((field) => histogramQueryStrForEachField(field))} + ${regularAggFields.map((field) => histogramQueryStrForEachField(field, false))} + ${asTextAggFields.map((field) => histogramQueryStrForEachField(field, true))} } } }`; @@ -44,7 +45,8 @@ const queryGuppyForAggs = (path, type, fields, gqlFilter, acc) => { queryBody.query = `query { _aggregation { ${type} (accessibility: ${accessibility}) { - ${fields.map((field) => histogramQueryStrForEachField(field))} + ${regularAggFields.map((field) => histogramQueryStrForEachField(field, false))} + ${asTextAggFields.map((field) => histogramQueryStrForEachField(field, true))} } } }`; @@ -266,23 +268,19 @@ export const getGQLFilter = (filterObj) => { return gqlFilter; }; -export const askGuppyAboutAllFieldsAndOptions = (path, type, fields, accessibility, filter) => { - const gqlFilter = getGQLFilter(filter); - return queryGuppyForAggs(path, type, fields, gqlFilter, accessibility); -}; - // eslint-disable-next-line max-len export const askGuppyAboutArrayTypes = (path) => queryGuppyForStatus(path).then((res) => res.indices); export const askGuppyForAggregationData = ( path, type, - fields, + regularAggFields, + asTextAggFields, filter, accessibility, ) => { const gqlFilter = getGQLFilter(filter); - return queryGuppyForAggs(path, type, fields, gqlFilter, accessibility); + return queryGuppyForAggs(path, type, regularAggFields, asTextAggFields, gqlFilter, accessibility); }; export const askGuppyForSubAggregationData = ( @@ -333,8 +331,13 @@ export const askGuppyForRawData = ( ); }; -export const getAllFieldsFromFilterConfigs = (filterTabConfigs) => filterTabConfigs - .reduce((acc, cur) => acc.concat(cur.fields), []); +export const getAllFieldsFromFilterConfigs = filterTabConfigs => +filterTabConfigs.reduce((acc, cur) => { + Object.keys(cur) + .filter(key => key === 'fields' || key === 'asTextAggFields') + .forEach(key => acc[key] = acc[key].concat(cur[key], [])); + return acc +}, {fields:[], asTextAggFields:[]}); /** * Download all data from guppy using fields, filter, and sort args. diff --git a/src/components/__tests__/filters.test.js b/src/components/__tests__/filters.test.js index b414aef2..15751b9a 100644 --- a/src/components/__tests__/filters.test.js +++ b/src/components/__tests__/filters.test.js @@ -71,7 +71,7 @@ describe('can update a small set of tabs with new counts', () => { const initialTabsOptions = { annotated_sex: { - asTextHistogram: [ + histogram: [ { key: 'yellow', count: 137675 }, { key: 'pink', count: 56270 }, { key: 'silver', count: 2020 }, @@ -79,7 +79,7 @@ describe('can update a small set of tabs with new counts', () => { ], }, extra_data: { - asTextHistogram: [ + histogram: [ { key: 'a', count: 2 }, ], }, @@ -87,12 +87,12 @@ describe('can update a small set of tabs with new counts', () => { const processedTabsOptions = { annotated_sex: { - asTextHistogram: [ + histogram: [ { key: 'yellow', count: 1 }, { key: 'orange', count: 107574 }, ], }, - extra_data: { asTextHistogram: [] }, + extra_data: { histogram: [] }, }; const filtersApplied = { annotated_sex: { selectedValues: ['silver'] } }; @@ -132,7 +132,7 @@ describe('can update a small set of tabs with new counts, test with ranger slide ]; const initialTabsOptions = { field1: { - asTextHistogram: [ + histogram: [ { key: 'option1', count: 137675 }, { key: 'option2', count: 56270 }, { key: 'option3', count: 2020 }, @@ -140,7 +140,7 @@ describe('can update a small set of tabs with new counts, test with ranger slide ], }, field2: { - asTextHistogram: [ + histogram: [ { key: [0, 100], count: 100 }, ], }, @@ -148,12 +148,12 @@ describe('can update a small set of tabs with new counts, test with ranger slide const processedTabsOptions = { field1: { - asTextHistogram: [ + histogram: [ { key: 'option3', count: 30 }, ], }, field2: { - asTextHistogram: [ + histogram: [ { key: [4, 39], count: 49, From 757db305d5563cc4da151e7977fcf74210c27729 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Fri, 8 Sep 2023 12:42:21 -0500 Subject: [PATCH 2/8] fix things --- .eslintrc.js | 14 +++++++++++++- src/components/ConnectedFilter/utils.js | 2 +- src/components/Utils/filters.js | 2 +- src/components/Utils/queries.js | 11 +++++------ 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index b52b9cb1..8a80bb83 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -23,7 +23,19 @@ module.exports = { 'react/no-array-index-key': 'off', "import/no-extraneous-dependencies": ["error", { "devDependencies": true - }] + }], + 'max-len': [ + 'error', + { + code: 150, + ignoreComments: true, + ignoreTrailingComments: true, + ignoreUrls: true, + ignoreStrings: true, + ignoreTemplateLiterals: true, + }, + ], + 'no-return-assign': 'warn' }, overrides: [{ "files": ["src/**/*.test.js"], diff --git a/src/components/ConnectedFilter/utils.js b/src/components/ConnectedFilter/utils.js index c38582c6..0c149061 100644 --- a/src/components/ConnectedFilter/utils.js +++ b/src/components/ConnectedFilter/utils.js @@ -189,7 +189,7 @@ export const excludeSelfFilterFromAggsData = (receivedAggsData, filterResults) = const resultAggsData = {}; const flattenAggsData = flat(receivedAggsData, { safe: true }); Object.keys(flattenAggsData).forEach((field) => { - const actualFieldName = field.replace('histogram', '').replace('.asTextHistogram', ''); + const actualFieldName = field.replace('.histogram', '').replace('.asTextHistogram', ''); const histogram = flattenAggsData[`${field}`]; if (!histogram) return; if (actualFieldName in filterResults) { diff --git a/src/components/Utils/filters.js b/src/components/Utils/filters.js index 7627aafe..b7c4c6d4 100644 --- a/src/components/Utils/filters.js +++ b/src/components/Utils/filters.js @@ -57,7 +57,7 @@ export const updateCountsInInitialTabsOptions = ( const flattenProcessedTabsOptions = flat(processedTabsOptions, { safe: true }); Object.keys(flattenInitialTabsOptions).forEach((field) => { // in flattened tab options, to get actual field name, strip off the last '.histogram' or '.asTextHistogram' - const actualFieldName = field.replace('histogram', '').replace('.asTextHistogram', ''); + const actualFieldName = field.replace('.histogram', '').replace('.asTextHistogram', ''); // check if Filter Value if not skip if (!allFilterValues.includes(actualFieldName)) { diff --git a/src/components/Utils/queries.js b/src/components/Utils/queries.js index d713ba10..275d8ab4 100644 --- a/src/components/Utils/queries.js +++ b/src/components/Utils/queries.js @@ -331,13 +331,12 @@ export const askGuppyForRawData = ( ); }; -export const getAllFieldsFromFilterConfigs = filterTabConfigs => -filterTabConfigs.reduce((acc, cur) => { +export const getAllFieldsFromFilterConfigs = (filterTabConfigs) => filterTabConfigs.reduce((acc, cur) => { Object.keys(cur) - .filter(key => key === 'fields' || key === 'asTextAggFields') - .forEach(key => acc[key] = acc[key].concat(cur[key], [])); - return acc -}, {fields:[], asTextAggFields:[]}); + .filter((key) => key === 'fields' || key === 'asTextAggFields') + .forEach((key) => acc[key] = acc[key].concat(cur[key], [])); + return acc; +}, { fields: [], asTextAggFields: [] }); /** * Download all data from guppy using fields, filter, and sort args. From c6f564a1b8198a97dc7c9cff9324d04bd92a4fea Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Fri, 8 Sep 2023 16:02:36 -0500 Subject: [PATCH 3/8] fix filter --- src/components/Utils/queries.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/components/Utils/queries.js b/src/components/Utils/queries.js index 275d8ab4..48edb520 100644 --- a/src/components/Utils/queries.js +++ b/src/components/Utils/queries.js @@ -246,7 +246,8 @@ export const getGQLFilter = (filterObj) => { } else if (filterValues.__combineMode && !hasSelectedValues && !hasRangeFilter) { // This filter only has a combine setting so far. We can ignore it. return; - } else { + } else if (hasSelectedValues) { + // filter has selected values but we don't know how to process it // eslint-disable-next-line no-console console.error(filterValues); throw new Error('Invalid filter object'); From bc36cb5418cc8141332194448097b6e9619f3ec6 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Fri, 8 Sep 2023 16:14:16 -0500 Subject: [PATCH 4/8] update version --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6f8617f9..37dbf641 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gen3/guppy", - "version": "0.16.1", + "version": "0.17.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@gen3/guppy", - "version": "0.16.1", + "version": "0.17.0", "license": "ISC", "dependencies": { "@apollo/server": "^4.7.5", @@ -30171,7 +30171,7 @@ "object-assign": "^4.1.1", "signal-exit": "^3.0.0", "string-width": "^4.2.3", - "strip-ansi": "6.0.1", + "strip-ansi": "^6.0.1", "wide-align": "^1.1.2" } }, @@ -35918,7 +35918,7 @@ "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "6.0.1" + "strip-ansi": "^6.0.1" }, "dependencies": { "emoji-regex": { diff --git a/package.json b/package.json index 96638a6c..93916454 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gen3/guppy", - "version": "0.16.1", + "version": "0.17.0", "description": "Server that support GraphQL queries on data from elasticsearch", "main": "src/server/server.js", "directories": { From 0a6317cda6e3873f17254b6e1b4a31ec79120614 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 11 Sep 2023 22:45:11 -0500 Subject: [PATCH 5/8] more update --- src/components/ConnectedFilter/index.jsx | 14 +++++++++----- src/components/ConnectedFilter/utils.js | 17 +++++------------ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/components/ConnectedFilter/index.jsx b/src/components/ConnectedFilter/index.jsx index 4119bb27..1519e9d2 100644 --- a/src/components/ConnectedFilter/index.jsx +++ b/src/components/ConnectedFilter/index.jsx @@ -151,11 +151,13 @@ class ConnectedFilter extends React.Component { } getTabsWithSearchFields() { - const newTabs = this.props.filterConfig.tabs.map(({ title, fields, searchFields }) => { + const newTabs = this.props.filterConfig.tabs.map(({ + title, fields, searchFields, asTextAggFields = [], + }) => { if (searchFields) { - return { title, fields: searchFields.concat(fields) }; + return { title, fields: searchFields.concat(fields), asTextAggFields }; } - return { title, fields }; + return { title, fields, asTextAggFields }; }); return newTabs; } @@ -271,9 +273,10 @@ class ConnectedFilter extends React.Component { } if (!processedTabsOptions || Object.keys(processedTabsOptions).length === 0) return null; const { fieldMapping } = this.props; - const tabs = this.props.filterConfig.tabs.map(({ fields, searchFields }, index) => { + const tabs = this.props.filterConfig.tabs.map(({ fields, searchFields, asTextAggFields = [] }, index) => { + const aggFields = _.union(fields, asTextAggFields); const sections = getFilterSections( - fields, + aggFields, searchFields, fieldMapping, processedTabsOptions, @@ -340,6 +343,7 @@ ConnectedFilter.propTypes = { tabs: PropTypes.arrayOf(PropTypes.shape({ title: PropTypes.string, fields: PropTypes.arrayOf(PropTypes.string), + asTextAggFields: PropTypes.arrayOf(PropTypes.string), searchFields: PropTypes.arrayOf(PropTypes.string), })), }).isRequired, diff --git a/src/components/ConnectedFilter/utils.js b/src/components/ConnectedFilter/utils.js index 0c149061..2d7f3f00 100644 --- a/src/components/ConnectedFilter/utils.js +++ b/src/components/ConnectedFilter/utils.js @@ -1,13 +1,6 @@ import flat from 'flat'; import { queryGuppyForRawDataAndTotalCounts } from '../Utils/queries'; -export const getFilterGroupConfig = (filterConfig) => ({ - tabs: filterConfig.tabs.map((t) => ({ - title: t.title, - fields: t.filters.map((f) => f.field), - })), -}); - const getSingleFilterOption = (histogramResult, initHistogramRes, filterValuesToHide) => { if (!histogramResult || !histogramResult.histogram) { throw new Error(`Error parsing field options ${JSON.stringify(histogramResult)}`); @@ -28,14 +21,14 @@ const getSingleFilterOption = (histogramResult, initHistogramRes, filterValuesTo }); return rangeOptions; } - let rawtextOptions = histogramResult.histogram; + let rawTextOptions = histogramResult.histogram; // hide filterValuesToHide from filters // filterValuesToHide added to guppyConfig in data-portal if (filterValuesToHide.length > 0) { - rawtextOptions = histogramResult.histogram + rawTextOptions = histogramResult.histogram .filter((item) => filterValuesToHide.indexOf(item.key) < 0); } - const textOptions = rawtextOptions.map((item) => ({ + const textOptions = rawTextOptions.map((item) => ({ text: item.key, filterType: 'singleSelect', count: item.count, @@ -109,7 +102,7 @@ export const checkIsArrayField = (field, arrayFields) => { }; export const getFilterSections = ( - fields, + aggFields, searchFields, fieldMapping, tabsOptions, @@ -156,7 +149,7 @@ export const getFilterSections = ( }); } - const sections = fields.map((field) => { + const sections = aggFields.map((field) => { const overrideName = fieldMapping.find((entry) => (entry.field === field)); const label = overrideName ? overrideName.name : capitalizeFirstLetter(field); From c038ada715948759ca32b50a5eda2c2e746f71e5 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Tue, 12 Sep 2023 11:21:07 -0500 Subject: [PATCH 6/8] update storybook --- devHelper/scripts/commands.sh | 4 ++-- stories/conf.js | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/devHelper/scripts/commands.sh b/devHelper/scripts/commands.sh index d73932ef..88b3156d 100755 --- a/devHelper/scripts/commands.sh +++ b/devHelper/scripts/commands.sh @@ -47,7 +47,7 @@ curl -iv -X PUT "${ESHOST}/${indexName}" \ } }, "analyzer": { - "ngram_analyzer": { + "ngram_analyzer": { "type": "custom", "tokenizer": "ngram_tokenizer", "filter": [ @@ -91,6 +91,7 @@ curl -iv -X PUT "${ESHOST}/${indexName}" \ "file_type": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } }, "file_format": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } }, "auth_resource_path": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } }, + "consortium_id": { "type": "integer" }, "file_count": { "type": "integer" }, "whatever_lab_result_value": { "type": "float" }, "some_nested_array_field": { @@ -159,4 +160,3 @@ curl -iv -X PUT "${ESHOST}/${configIndexName}" \ function es_indices() { curl -X GET "${ESHOST}/_cat/indices?v" } - diff --git a/stories/conf.js b/stories/conf.js index a8832450..adbdb419 100644 --- a/stories/conf.js +++ b/stories/conf.js @@ -5,6 +5,9 @@ export const filterConfig = { 'project', 'study', ], + asTextAggFields: [ + 'consortium_id', + ], }, { title: 'Subject', From 4cb0772db193b62272153569bd4ab68f77720613 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Wed, 13 Sep 2023 15:11:15 -0500 Subject: [PATCH 7/8] fix filter --- src/components/ConnectedFilter/index.jsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/components/ConnectedFilter/index.jsx b/src/components/ConnectedFilter/index.jsx index 1519e9d2..39ce33a5 100644 --- a/src/components/ConnectedFilter/index.jsx +++ b/src/components/ConnectedFilter/index.jsx @@ -155,9 +155,9 @@ class ConnectedFilter extends React.Component { title, fields, searchFields, asTextAggFields = [], }) => { if (searchFields) { - return { title, fields: searchFields.concat(fields), asTextAggFields }; + return { title, fields: searchFields.concat(fields).concat(asTextAggFields) }; } - return { title, fields, asTextAggFields }; + return { title, fields: fields.concat(asTextAggFields) }; }); return newTabs; } @@ -184,7 +184,7 @@ class ConnectedFilter extends React.Component { // Get filter values const allFilterValues = this.props.filterConfig.tabs.reduce( - (accumulator, tab) => ([...accumulator, ...tab.fields]), + (accumulator, tab) => ([...accumulator, ...tab.fields, ...tab.asTextAggFields || []]), [], ); From 94abf4c1e4793a41b9d99d1755076feddd63200c Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Fri, 15 Sep 2023 09:08:06 -0500 Subject: [PATCH 8/8] es6 docker image for apple silicon --- devHelper/docker/es6.Dockerfile | 51 +++++++++++++++++++++++++++++++++ devHelper/docker/esearch.yml | 4 ++- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 devHelper/docker/es6.Dockerfile diff --git a/devHelper/docker/es6.Dockerfile b/devHelper/docker/es6.Dockerfile new file mode 100644 index 00000000..95fda820 --- /dev/null +++ b/devHelper/docker/es6.Dockerfile @@ -0,0 +1,51 @@ +# From https://gist.github.com/rluvaton/3a8d5953e1ad8236e8953c2e7691e5de + +FROM ubuntu:bionic-20220531 + +# Must be root to install the packages +USER root + +# Install required deps +RUN apt update +RUN apt -y install gnupg wget apt-transport-https coreutils java-common + +# Import Elasticsearch GPG Key +RUN wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - + +# Add Elasticsearch 6.x APT repository +# setting CPU architecture to be amd64 explicity as in case this is being built from ARM (which it should) it would find the elasticsearch package (elasticsearch 6.x doesn't have ARM binary) +RUN echo "deb [arch=amd64] https://artifacts.elastic.co/packages/6.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-6.x.list + +# update after elastic-search repo added +RUN apt-get update + +# Install ARM Amazon JDK +RUN wget https://corretto.aws/downloads/latest/amazon-corretto-8-aarch64-linux-jdk.deb -O amazon-jdk.deb +RUN dpkg --skip-same-version -i amazon-jdk.deb +RUN rm amazon-jdk.deb + +# Install Elasticsearch 6.x +RUN apt-get -y install elasticsearch + +# the user was created when installed the elasticsearch +# Must not be root: +# org.elasticsearch.bootstrap.StartupException: java.lang.RuntimeException: can not run elasticsearch as root +USER elasticsearch + +WORKDIR /usr/share/elasticsearch + +# Append the custom conf + +RUN echo "# ---------------------------------- CUSTOM -----------------------------------" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "# Added because of the following error (TL;DR: X-Pack features are not supported in ARM):" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "# > org.elasticsearch.bootstrap.StartupException:" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "# > ElasticsearchException[X-Pack is not supported and Machine Learning is not available for [linux-aarch64];" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "# > you can use the other X-Pack features (unsupported) by setting xpack.ml.enabled: false in elasticsearch.yml]" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "xpack.ml.enabled: false" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "# Added because we want to listen to requests coming from computers in the network" >> /etc/elasticsearch/elasticsearch.yml +RUN echo "network.host: 0.0.0.0" >> /etc/elasticsearch/elasticsearch.yml + + +ENTRYPOINT [ "./bin/elasticsearch" ] diff --git a/devHelper/docker/esearch.yml b/devHelper/docker/esearch.yml index 4cd7d9ae..5d686729 100644 --- a/devHelper/docker/esearch.yml +++ b/devHelper/docker/esearch.yml @@ -3,7 +3,9 @@ version: "3.3" services: # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-cli-run-prod-mode elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.12 + image: elasticsearch + build: + dockerfile: es6.Dockerfile ports: - "9200:9200" - "9300:9300"