From e8db9dde0d937c91ec0a11a92b8692e74cecb60e Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Fri, 25 Jul 2025 22:59:38 -0500 Subject: [PATCH 1/7] fix nested numeric range query filter --- src/server/es/aggs.js | 53 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/src/server/es/aggs.js b/src/server/es/aggs.js index b02d27df..ff6f64c1 100644 --- a/src/server/es/aggs.js +++ b/src/server/es/aggs.js @@ -7,6 +7,7 @@ import { AGGS_QUERY_NAME, } from './const'; import config from '../config'; +import log from '../logger'; const PAGE_SIZE = 10000; @@ -89,36 +90,66 @@ const processResultsForNestedAgg = (nestedAggFields, item, resultObj) => { * This function appends extra range limitation onto a query body "oldQuery" * export for test * @param {string} field - which field to append range limitation to + * @param {string} nestedPath - the nested field path, if exists * @param {object} oldQuery - the old query body to append * @param {number} rangeStart - the range start * @param {number} rangeEnd - the range end */ -export const appendAdditionalRangeQuery = (field, oldQuery, rangeStart, rangeEnd) => { +export const appendAdditionalRangeQuery = (field, nestedPath, oldQuery, rangeStart, rangeEnd) => { const appendFilter = []; + let updatedFieldName = field; + if (nestedPath) { + updatedFieldName = `${nestedPath}.${field}`; + } if (typeof rangeStart !== 'undefined') { appendFilter.push({ range: { - [field]: { gte: rangeStart }, + [updatedFieldName]: { gte: rangeStart }, }, }); } if (typeof rangeEnd !== 'undefined') { appendFilter.push({ range: { - [field]: { lt: rangeEnd }, + [updatedFieldName]: { lt: rangeEnd }, }, }); } if (appendFilter.length > 0) { - const newQuery = { + let additionalRangeQuery = { bool: { - must: oldQuery ? [ - oldQuery, - [...appendFilter], - ] : [...appendFilter], + must: [...appendFilter], }, }; - return newQuery; + // handle creating range query filter for nested fields + if (nestedPath) { + additionalRangeQuery = { + bool: { + must: [{ + nested: { + path: nestedPath, + query: { ...additionalRangeQuery }, + }, + }], + }, + }; + } + if (!oldQuery) { + return additionalRangeQuery; + } + if ((oldQuery.bool || {}).must) { + if (!Array.isArray(oldQuery.bool.must)) { + log.debug(`Invalid query filter found during processing: ${JSON.stringify(oldQuery, null, 2)}`); + throw new GraphQLError('Invalid query filter found, check debug logging', { + extensions: { + code: 'BAD_USER_INPUT', + }, + }); + } else { + oldQuery.bool.must.push(additionalRangeQuery); + return oldQuery; + } + } } return oldQuery; }; @@ -169,7 +200,7 @@ export const numericGlobalStats = async ( defaultAuthFilter, ); } - queryBody.query = appendAdditionalRangeQuery(field, queryBody.query, rangeStart, rangeEnd); + queryBody.query = appendAdditionalRangeQuery(field, nestedPath, queryBody.query, rangeStart, rangeEnd); let aggsObj = { [AGGS_GLOBAL_STATS_NAME]: { stats: { @@ -261,7 +292,7 @@ export const numericHistogramWithFixedRangeStep = async ( nestedAggFields, ); } - queryBody.query = appendAdditionalRangeQuery(field, queryBody.query, rangeStart, rangeEnd); + queryBody.query = appendAdditionalRangeQuery(field, nestedPath, queryBody.query, rangeStart, rangeEnd); const aggsObj = { [AGGS_GLOBAL_STATS_NAME]: { stats: { From 21caa2e9750b1ee9946f6ebef1a780b60482b9c4 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 28 Jul 2025 11:59:41 -0500 Subject: [PATCH 2/7] fix local testing auth --- src/server/auth/utils.js | 41 ++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/src/server/auth/utils.js b/src/server/auth/utils.js index 2e06980a..c15ca29b 100644 --- a/src/server/auth/utils.js +++ b/src/server/auth/utils.js @@ -26,11 +26,24 @@ export const resourcePathsWithServiceMethodCombination = (userAuthMapping, servi export const getAccessibleResourcesFromArboristasync = async (jwt) => { let data; if (config.internalLocalTest) { + log.info('debug'); data = { - resources: [ // these are just for testing - '/programs/DEV/projects/test', - '/programs/jnkns/projects/jenkins', - ], + // these are just for testing + '/programs/DEV/projects/test': [ + { + service: '*', + method: 'read', + }], + '/programs/jnkns/projects/jenkins': [ + { + service: '*', + method: 'read', + }], + '/guppy_admin': [ + { + service: 'guppy', + method: 'admin_access', + }], }; } else { data = await arboristClient.listAuthMapping(jwt); @@ -54,10 +67,22 @@ export const checkIfUserCanRefreshServer = async (passedData) => { let data = passedData; if (config.internalLocalTest) { data = { - resources: [ // these are just for testing - '/programs/DEV/projects/test', - '/programs/jnkns/projects/jenkins', - ], + // these are just for testing + '/programs/DEV/projects/test': [ + { + service: '*', + method: 'read', + }], + '/programs/jnkns/projects/jenkins': [ + { + service: '*', + method: 'read', + }], + '/guppy_admin': [ + { + service: 'guppy', + method: 'admin_access', + }], }; } From f93d0642398a71f6d84a60e4034f7f512644feb6 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 28 Jul 2025 14:28:14 -0500 Subject: [PATCH 3/7] fix query param --- src/server/es/aggs.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/server/es/aggs.js b/src/server/es/aggs.js index ff6f64c1..6b1f0bc4 100644 --- a/src/server/es/aggs.js +++ b/src/server/es/aggs.js @@ -149,6 +149,9 @@ export const appendAdditionalRangeQuery = (field, nestedPath, oldQuery, rangeSta oldQuery.bool.must.push(additionalRangeQuery); return oldQuery; } + } else { + additionalRangeQuery.bool.must.push(oldQuery); + return additionalRangeQuery; } } return oldQuery; From 293dbe57db12109b8994441f008c43ef99bd84c9 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 28 Jul 2025 15:02:40 -0500 Subject: [PATCH 4/7] add test --- .../mockNumericHistogramFixBinCount.js | 187 +++++++++++++++--- src/server/es/__tests__/aggs.test.js | 76 ++++++- 2 files changed, 225 insertions(+), 38 deletions(-) diff --git a/src/server/__mocks__/mockESData/mockNumericHistogramFixBinCount.js b/src/server/__mocks__/mockESData/mockNumericHistogramFixBinCount.js index 741983d8..fbd57942 100644 --- a/src/server/__mocks__/mockESData/mockNumericHistogramFixBinCount.js +++ b/src/server/__mocks__/mockESData/mockNumericHistogramFixBinCount.js @@ -115,26 +115,24 @@ const mockHistogramFixBinCount = () => { bool: { must: [ { - term: { - gender: 'female', + range: { + file_count: { + gte: 2, + }, }, }, - [ - { - range: { - file_count: { - gte: 2, - }, + { + range: { + file_count: { + lt: 99, }, }, - { - range: { - file_count: { - lt: 99, - }, - }, + }, + { + term: { + gender: 'female', }, - ], + }, ], }, }, @@ -336,26 +334,27 @@ const mockHistogramFixBinCount = () => { bool: { must: [ { - terms: { - gen3_resource_path: ['internal-project-1', 'internal-project-2'], + range: { + file_count: { + gte: 20, + }, }, }, - [ - { - range: { - file_count: { - gte: 20, - }, + { + range: { + file_count: { + lt: 81, }, }, - { - range: { - file_count: { - lt: 81, - }, - }, + }, + { + terms: { + gen3_resource_path: [ + 'internal-project-1', + 'internal-project-2', + ], }, - ], + }, ], }, }, @@ -442,6 +441,134 @@ const mockHistogramFixBinCount = () => { }, }; mockSearchEndpoint(fileCountHistogramFixBinCountQuery3, fileCountHistogramFixBinCountResult3); + + // with nestedPath and binCount applied + const fileCountHistogramFixBinCountQuery4 = { + size: 0, + query: { + bool: { + must: [ + { + nested: { + path: 'visits.follow_ups', + query: { + bool: { + must: [ + { + range: { + 'visits.follow_ups.days_to_follow_up': { + gte: 1, + }, + }, + }, + { + range: { + 'visits.follow_ups.days_to_follow_up': { + lt: 4, + }, + }, + }, + ], + }, + }, + }, + }, + ], + }, + }, + aggs: { + numeric_nested_aggs: { + nested: { + path: 'visits.follow_ups', + }, + aggs: { + numeric_aggs_stats: { + stats: { + field: 'visits.follow_ups.days_to_follow_up', + }, + }, + numeric_aggs: { + histogram: { + field: 'visits.follow_ups.days_to_follow_up', + interval: 0.75, + offset: 0.25, + }, + aggs: { + numeric_item_aggs_stats: { + stats: { + field: 'visits.follow_ups.days_to_follow_up', + }, + }, + }, + }, + }, + }, + }, + track_total_hits: true, + }; + const fileCountHistogramFixBinCountResult4 = { + aggregations: { + numeric_nested_aggs: { + doc_count: 31, + numeric_aggs: { + buckets: [ + { + key: 1.0, + doc_count: 12, + numeric_item_aggs_stats: { + count: 12, + min: 1.0, + max: 1.0, + avg: 1.0, + sum: 12.0, + }, + }, + { + key: 2.0, + doc_count: 19, + numeric_item_aggs_stats: { + count: 19, + min: 2.0, + max: 2.0, + avg: 2.0, + sum: 38.0, + }, + }, + { + key: 3.0, + doc_count: 17, + numeric_item_aggs_stats: { + count: 17, + min: 3.0, + max: 3.0, + avg: 3.0, + sum: 51.0, + }, + }, + { + key: 4.0, + doc_count: 6, + numeric_item_aggs_stats: { + count: 6, + min: 4.0, + max: 4.0, + avg: 4.0, + sum: 24.0, + }, + }, + ], + }, + numeric_aggs_stats: { + count: 54, + min: 1.0, + max: 4.0, + avg: 2.31481481481, + sum: 125.0, + }, + }, + }, + }; + mockSearchEndpoint(fileCountHistogramFixBinCountQuery4, fileCountHistogramFixBinCountResult4); }; export default mockHistogramFixBinCount; diff --git a/src/server/es/__tests__/aggs.test.js b/src/server/es/__tests__/aggs.test.js index f5041841..abe99cbf 100644 --- a/src/server/es/__tests__/aggs.test.js +++ b/src/server/es/__tests__/aggs.test.js @@ -1,5 +1,6 @@ // eslint-disable-next-line import nock from 'nock'; // must import this to enable mock data by nock +import { log, error } from 'console'; import setupMockDataEndpoint from '../../__mocks__/mockDataFromES'; import { appendAdditionalRangeQuery, @@ -37,12 +38,13 @@ describe('could append range limitation onto ES query object', () => { }, }; test('with rangeStart, rangeEnd, and origin query', () => { - const result = appendAdditionalRangeQuery(field, exampleQuery, rangeStart, rangeEnd); + const result = appendAdditionalRangeQuery(field, undefined, exampleQuery, rangeStart, rangeEnd); const expectedResult = { bool: { must: [ + expectedRangeStartPart, + expectedRangeEndPart, exampleQuery, - [expectedRangeStartPart, expectedRangeEndPart], ], }, }; @@ -50,23 +52,23 @@ describe('could append range limitation onto ES query object', () => { }); test('with either rangeStart or rangeEnd', () => { - const result1 = appendAdditionalRangeQuery(field, exampleQuery, rangeStart); + const result1 = appendAdditionalRangeQuery(field, undefined, exampleQuery, rangeStart); const expectedResult1 = { bool: { must: [ + expectedRangeStartPart, exampleQuery, - [expectedRangeStartPart], ], }, }; expect(result1).toEqual(expectedResult1); - const result2 = appendAdditionalRangeQuery(field, exampleQuery, undefined, rangeEnd); + const result2 = appendAdditionalRangeQuery(field, undefined, exampleQuery, undefined, rangeEnd); const expectedResult2 = { bool: { must: [ + expectedRangeEndPart, exampleQuery, - [expectedRangeEndPart], ], }, }; @@ -74,7 +76,7 @@ describe('could append range limitation onto ES query object', () => { }); test('with empty query', () => { - const result = appendAdditionalRangeQuery(field, undefined, rangeStart, rangeEnd); + const result = appendAdditionalRangeQuery(field, undefined, undefined, rangeStart, rangeEnd); const expectedResult = { bool: { must: [ @@ -86,7 +88,7 @@ describe('could append range limitation onto ES query object', () => { expect(result).toEqual(expectedResult); // all empty - const result2 = appendAdditionalRangeQuery(field, undefined); + const result2 = appendAdditionalRangeQuery(field, undefined, undefined); const expectedResult2 = undefined; expect(result2).toEqual(expectedResult2); }); @@ -861,6 +863,64 @@ describe('could aggregate for numeric fields, fixed bin count', () => { ]; expect(result).toEqual(expectedResults); }); + + test('fixed bin count, with nested field in query', async () => { + await esInstance.initialize(); + const result = await numericHistogramWithFixedBinCount( + { esInstance, esIndex, esType }, + { + field: 'days_to_follow_up', binCount: 4, nestedPath: 'visits.follow_ups', + }, + ); + const expectedResults = [ + { + key: [ + 1, + 1.75, + ], + count: 12, + min: 1, + max: 1, + avg: 1, + sum: 12, + }, + { + key: [ + 2, + 2.75, + ], + count: 19, + min: 2, + max: 2, + avg: 2, + sum: 38, + }, + { + key: [ + 3, + 3.75, + ], + count: 17, + min: 3, + max: 3, + avg: 3, + sum: 51, + }, + { + key: [ + 4, + 4.75, + ], + count: 6, + min: 4, + max: 4, + avg: 4, + sum: 24, + }, + ]; + error(JSON.stringify(result, null, 2)); + expect(result).toEqual(expectedResults); + }); }); // see /src/server/__mocks__/mockESData/mockNestedTermsAndMissingAggs.js for mock results From f314a83f714cd729eb730bf303e5c11972679e77 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 28 Jul 2025 15:03:13 -0500 Subject: [PATCH 5/7] cleanup --- src/server/es/__tests__/aggs.test.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/server/es/__tests__/aggs.test.js b/src/server/es/__tests__/aggs.test.js index abe99cbf..56798e8c 100644 --- a/src/server/es/__tests__/aggs.test.js +++ b/src/server/es/__tests__/aggs.test.js @@ -1,6 +1,5 @@ // eslint-disable-next-line import nock from 'nock'; // must import this to enable mock data by nock -import { log, error } from 'console'; import setupMockDataEndpoint from '../../__mocks__/mockDataFromES'; import { appendAdditionalRangeQuery, @@ -918,7 +917,6 @@ describe('could aggregate for numeric fields, fixed bin count', () => { sum: 24, }, ]; - error(JSON.stringify(result, null, 2)); expect(result).toEqual(expectedResults); }); }); From b92310e4be3b331257a9bfb86ed23e8b71775dcd Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Mon, 28 Jul 2025 16:53:46 -0500 Subject: [PATCH 6/7] use native workflow --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2944cda9..8a46d76e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,7 +20,7 @@ jobs: ci: name: Build Image and Push - uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + uses: uc-cdis/.github/.github/workflows/image_build_push_native.yaml@master secrets: ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} From 230e044d30531c6a7e2b1c7fcb0085ab23392401 Mon Sep 17 00:00:00 2001 From: Mingfei Shao Date: Tue, 29 Jul 2025 10:11:42 -0500 Subject: [PATCH 7/7] clean up --- src/server/auth/utils.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/auth/utils.js b/src/server/auth/utils.js index c15ca29b..83606725 100644 --- a/src/server/auth/utils.js +++ b/src/server/auth/utils.js @@ -26,7 +26,6 @@ export const resourcePathsWithServiceMethodCombination = (userAuthMapping, servi export const getAccessibleResourcesFromArboristasync = async (jwt) => { let data; if (config.internalLocalTest) { - log.info('debug'); data = { // these are just for testing '/programs/DEV/projects/test': [