diff --git a/doc/download.md b/doc/download.md index 5f77b8e6..dba5721e 100644 --- a/doc/download.md +++ b/doc/download.md @@ -23,11 +23,13 @@ Example request body: "gender", "race", "file_count", - "subject_id" + "subject_id", + "visit.visit_label" ], "sort": [ { "file_count": "asc" }, - { "gender": "desc" } + { "gender": "desc" }, + { "visit.visit_label": "asc" } ] } ``` @@ -40,25 +42,51 @@ Example result: "subject_id": "78", "file_count": 1, "gender": "female", - "race": "hispanic" + "race": "hispanic", + "visit": [ + { + "visit_label": "label_1" + }, + { + "visit_label": "label_2" + } + ] }, { "subject_id": "45", "file_count": 3, "gender": "female", - "race": "hispanic" + "race": "hispanic", + "visit": [ + { + "visit_label": "label_3" + }, + ... + ] }, { "subject_id": "60", "file_count": 5, "gender": "female", - "race": "asian" + "race": "asian", + "visit": [ + { + "visit_label": "label_X" + }, + ... + ] }, { "subject_id": "58", "file_count": 13, "gender": "male", - "race": "white" + "race": "white", + "visit": [ + { + "visit_label": "label_Y" + }, + ... + ] }, ... ] diff --git a/doc/queries.md b/doc/queries.md index 0b33c251..aea28ba4 100644 --- a/doc/queries.md +++ b/doc/queries.md @@ -30,6 +30,9 @@ Example query: }, { "gender": asc + }, + { + "experiments.experimental_description": asc } ], filter: $filter) { subject_id diff --git a/package-lock.json b/package-lock.json index fb0fb464..7d8f2c65 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "@gen3/guppy", - "version": "0.6.0", + "version": "0.6.1", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -9147,28 +9147,22 @@ "dev": true }, "follow-redirects": { - "version": "1.5.10", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", - "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.11.0.tgz", + "integrity": "sha512-KZm0V+ll8PfBrKwMzdo5D13b1bur9Iq9Zd/RMmAoQQcl2PxxFml8cxXPaaPYVbV0RjNjq1CU7zIzAOqtUPudmA==", "dev": true, "requires": { - "debug": "=3.1.0" + "debug": "^3.0.0" }, "dependencies": { "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", "dev": true, "requires": { - "ms": "2.0.0" + "ms": "^2.1.1" } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", - "dev": true } } }, @@ -10666,9 +10660,9 @@ "dev": true }, "http-proxy": { - "version": "1.18.0", - "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.0.tgz", - "integrity": "sha512-84I2iJM/n1d4Hdgc6y2+qY5mDaz2PUVjlg9znE9byl+q0uC3DeByqBGReQu5tpLK0TAqTIXScRUV+dg7+bUPpQ==", + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", "dev": true, "requires": { "eventemitter3": "^4.0.0", @@ -10677,9 +10671,9 @@ }, "dependencies": { "eventemitter3": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.0.tgz", - "integrity": "sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.4.tgz", + "integrity": "sha512-rlaVLnVxtxvoyLsQQFBx53YmXHDxRIzzTLbdfxqi4yocpSjAxXwkU0cScM5JgSKMqEhrZpnvQ2D9gjylR0AimQ==", "dev": true } } @@ -12861,9 +12855,9 @@ "integrity": "sha512-1RUZVgQlpJSPWYbFSpmudq5nHY1doEIv89gBtF0s4gW1GF2XorxcA/70M5vq7rLv0a6mhOUccRsqkwhwLCIQ2Q==" }, "markdown-to-jsx": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/markdown-to-jsx/-/markdown-to-jsx-6.11.0.tgz", - "integrity": "sha512-RH7LCJQ4RFmPqVeZEesKaO1biRzB/k4utoofmTCp3Eiw6D7qfvK8fzZq/2bjEJAtVkfPrM5SMt5APGf2rnaKMg==", + "version": "6.11.4", + "resolved": "https://registry.npmjs.org/markdown-to-jsx/-/markdown-to-jsx-6.11.4.tgz", + "integrity": "sha512-3lRCD5Sh+tfA52iGgfs/XZiw33f7fFX9Bn55aNnVNUd2GzLDkOWyKYYD8Yju2B1Vn+feiEdgJs8T6Tg0xNokPw==", "dev": true, "requires": { "prop-types": "^15.6.2", @@ -20799,6 +20793,16 @@ "neo-async": "^2.5.0" } }, + "watchpack-chokidar2": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.0.tgz", + "integrity": "sha512-9TyfOyN/zLUbA288wZ8IsMZ+6cbzvsNyEzSBp6e/zkifi6xxbl8SmQ/CxQq32k8NNqrdVEVUVSEf56L4rQ/ZxA==", + "dev": true, + "optional": true, + "requires": { + "chokidar": "^2.1.8" + } + }, "wbuf": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz", @@ -21026,6 +21030,88 @@ "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", "dev": true }, + "anymatch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", + "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==", + "dev": true, + "optional": true, + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, + "binary-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz", + "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==", + "dev": true, + "optional": true + }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "optional": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "optional": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "fsevents": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz", + "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==", + "dev": true, + "optional": true + }, + "glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dev": true, + "optional": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "optional": true, + "requires": { + "binary-extensions": "^2.0.0" + } + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "optional": true + }, + "readdirp": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz", + "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==", + "dev": true, + "optional": true, + "requires": { + "picomatch": "^2.2.1" + } + }, "terser-webpack-plugin": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.3.tgz", @@ -21043,15 +21129,45 @@ "worker-farm": "^1.7.0" } }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "optional": true, + "requires": { + "is-number": "^7.0.0" + } + }, "watchpack": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.6.1.tgz", - "integrity": "sha512-+IF9hfUFOrYOOaKyfaI7h7dquUIOgyEMoQMLA7OP5FxegKA2+XdXThAZ9TU2kucfhDH7rfMHs1oPYziVGWRnZA==", + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.2.tgz", + "integrity": "sha512-ymVbbQP40MFTp+cNMvpyBpBtygHnPzPkHqoIwRRj/0B8KhqQwV8LaKjtbaxF2lK4vl8zN9wCxS46IFCU5K4W0g==", "dev": true, "requires": { - "chokidar": "^2.1.8", + "chokidar": "^3.4.0", "graceful-fs": "^4.1.2", - "neo-async": "^2.5.0" + "neo-async": "^2.5.0", + "watchpack-chokidar2": "^2.0.0" + }, + "dependencies": { + "chokidar": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.0.tgz", + "integrity": "sha512-aXAaho2VJtisB/1fg1+3nlLJqGOuewTzQpd/Tz0yTg2R0e4IGtshYvtjowyEumcBv2z+y4+kc75Mz7j5xJskcQ==", + "dev": true, + "optional": true, + "requires": { + "anymatch": "~3.1.1", + "braces": "~3.0.2", + "fsevents": "~2.1.2", + "glob-parent": "~5.1.0", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.4.0" + } + } } } } diff --git a/package.json b/package.json index ba2955d4..1246bf4c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gen3/guppy", - "version": "0.6.0", + "version": "0.6.1", "description": "Server that support GraphQL queries on data from elasticsearch", "main": "src/server/server.js", "directories": { diff --git a/src/components/Utils/queries.js b/src/components/Utils/queries.js index ecf18f45..52a1e556 100644 --- a/src/components/Utils/queries.js +++ b/src/components/Utils/queries.js @@ -129,6 +129,20 @@ const queryGuppyForSubAgg = ( }); }; +const rawDataQueryStrForEachField = (field) => { + const splittedFieldArray = field.split('.'); + const splittedField = splittedFieldArray.shift(); + if (splittedFieldArray.length === 0) { + return (` + ${splittedField} + `); + } + return (` + ${splittedField} { + ${rawDataQueryStrForEachField(splittedFieldArray.join('.'))} + }`); +}; + const queryGuppyForRawDataAndTotalCounts = ( path, type, @@ -151,9 +165,10 @@ const queryGuppyForRawDataAndTotalCounts = ( if (gqlFilter) { typeAggsLine = `${type} (filter: $filter, accessibility: ${accessibility}) {`; } + const processedFields = fields.map((field) => rawDataQueryStrForEachField(field)); const query = `${queryLine} ${dataTypeLine} - ${fields.join('\n')} + ${processedFields.join('\n')} } _aggregation { ${typeAggsLine} diff --git a/src/server/es/__tests__/sort.test.js b/src/server/es/__tests__/sort.test.js index 0f1d0eb8..5682e61c 100644 --- a/src/server/es/__tests__/sort.test.js +++ b/src/server/es/__tests__/sort.test.js @@ -15,27 +15,109 @@ describe('Transfer GraphQL sort argument to ES sort argument', () => { test('object format sort arg', async () => { await esInstance.initialize(); const graphQLSort1 = { gender: 'asc' }; - const expectedESSort1 = [{ gender: 'asc' }]; + const expectedESSort1 = [ + { + gender: { + order: 'asc', + }, + }, + ]; const resultESSort1 = getESSortBody(graphQLSort1, esInstance, esIndex); expect(resultESSort1).toEqual(expectedESSort1); const graphQLSort2 = { gender: 'asc', file_count: 'desc' }; - const expectedESSort2 = [{ gender: 'asc' }, { file_count: 'desc' }]; + const expectedESSort2 = [ + { + gender: { + order: 'asc', + }, + }, + { + file_count: { + order: 'desc', + }, + }, + ]; const resultESSort2 = getESSortBody(graphQLSort2, esInstance, esIndex); expect(resultESSort2).toEqual(expectedESSort2); + + const graphQLSort3 = { gender: 'asc', file_count: 'desc', 'visits.visit_label': 'asc' }; + const expectedESSort3 = [ + { + gender: { + order: 'asc', + }, + }, + { + file_count: { + order: 'desc', + }, + }, + { + 'visits.visit_label': { + nested: { + path: 'visits', + }, + order: 'asc', + }, + }, + ]; + const resultESSort3 = getESSortBody(graphQLSort3, esInstance, esIndex); + expect(resultESSort3).toEqual(expectedESSort3); }); test('array format sort arg', async () => { await esInstance.initialize(); const graphQLSort1 = [{ gender: 'asc' }]; - const expectedESSort1 = [{ gender: 'asc' }]; + const expectedESSort1 = [ + { + gender: { + order: 'asc', + }, + }, + ]; const resultESSort1 = getESSortBody(graphQLSort1, esInstance, esIndex); expect(resultESSort1).toEqual(expectedESSort1); const graphQLSort2 = [{ gender: 'asc' }, { file_count: 'desc' }]; - const expectedESSort2 = [{ gender: 'asc' }, { file_count: 'desc' }]; + const expectedESSort2 = [ + { + gender: { + order: 'asc', + }, + }, + { + file_count: { + order: 'desc', + }, + }, + ]; const resultESSort2 = getESSortBody(graphQLSort2, esInstance, esIndex); expect(resultESSort2).toEqual(expectedESSort2); + + const graphQLSort3 = [{ gender: 'asc' }, { file_count: 'desc' }, { 'visits.visit_label': 'asc' }]; + const expectedESSort3 = [ + { + gender: { + order: 'asc', + }, + }, + { + file_count: { + order: 'desc', + }, + }, + { + 'visits.visit_label': { + nested: { + path: 'visits', + }, + order: 'asc', + }, + }, + ]; + const resultESSort3 = getESSortBody(graphQLSort3, esInstance, esIndex); + expect(resultESSort3).toEqual(expectedESSort3); }); test('array format sort arg with nonexisting field', async () => { @@ -54,6 +136,11 @@ describe('Transfer GraphQL sort argument to ES sort argument', () => { const graphQLSort = { gender: 'female', invalid_field: 'asc' }; getESSortBody(graphQLSort, esInstance, esIndex); }).toThrow(UserInputError); + + expect(() => { + const graphQLSort = [{ gender: 'female', 'visits.invalid_field': 'asc' }]; + getESSortBody(graphQLSort, esInstance, esIndex); + }).toThrow(UserInputError); }); test('array format sort arg with invalid method', async () => { @@ -72,5 +159,10 @@ describe('Transfer GraphQL sort argument to ES sort argument', () => { const graphQLSort = { gender: 'asc', file_count: 'invalid_method' }; getESSortBody(graphQLSort, esInstance, esIndex); }).toThrow(UserInputError); + + expect(() => { + const graphQLSort = { gender: 'asc', 'visits.visit_label': 'invalid_method' }; + getESSortBody(graphQLSort, esInstance, esIndex); + }).toThrow(UserInputError); }); }); diff --git a/src/server/es/index.js b/src/server/es/index.js index 0dfeddc3..62dff9fa 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -8,7 +8,7 @@ import * as esAggregator from './aggs'; import log from '../logger'; import { SCROLL_PAGE_SIZE } from './const'; import CodedError from '../utils/error'; -import { fromFieldsToSource, buildNestedField } from '../utils/utils'; +import { fromFieldsToSource, buildNestedField, processNestedFieldNames } from '../utils/utils'; class ES { constructor(esConfig = config.esConfig) { @@ -81,8 +81,13 @@ class ES { 'Invalid es index or es type name', ); } - const fieldsNotBelong = _.difference(fields, - this.getESFields(esIndex).fields.map((f) => f.name)); + const allESFields = _.flattenDeep(this.getESFields(esIndex).fields.map((f) => { + if (f.nestedProps) { + return processNestedFieldNames(f); + } + return f.name; + })); + const fieldsNotBelong = _.difference(fields, allESFields); if (fieldsNotBelong.length > 0) { throw new CodedError( 400, diff --git a/src/server/es/sort.js b/src/server/es/sort.js index 71184d87..d7a65148 100644 --- a/src/server/es/sort.js +++ b/src/server/es/sort.js @@ -2,10 +2,29 @@ import { UserInputError } from 'apollo-server'; /** * Transfer graphql sort arg to ES sort object + * e.g.: input graphql sort arg + * [{ gender: 'asc' }, { 'visits.visit_label': 'asc' }] + * output ES sort object + * [ + * { + * gender: { + * order: 'asc', + * }, + * }, + * { + * 'visits.visit_label': { + * nested: { + * path: 'visits', + * }, + * order: 'asc', + * }, + * } + * ] * @param {object} graphqlSort + * @returns a ES sort object */ const getESSortBody = (graphqlSort, esInstance, esIndex) => { - let sortBody; + const sortBody = []; if (typeof graphqlSort !== 'undefined') { let graphqlSortObj = graphqlSort; if (typeof (graphqlSort.length) === 'undefined') { @@ -17,15 +36,46 @@ const getESSortBody = (graphqlSort, esInstance, esIndex) => { throw new UserInputError('Invalid sort argument'); } const field = Object.keys(graphqlSortObj[i])[0]; - if (typeof esInstance.fieldTypes[esIndex][field] === 'undefined') { - throw new UserInputError('Invalid sort argument'); - } const method = graphqlSortObj[i][field]; if (method !== 'asc' && method !== 'desc') { throw new UserInputError('Invalid sort argument'); } + if (!field.includes('.')) { + // non-nested field name, normal check logic + if (typeof esInstance.fieldTypes[esIndex][field] === 'undefined') { + throw new UserInputError('Invalid sort argument'); + } else { + sortBody.push({ + [field]: { + order: method, + }, + }); + } + } else { + // nested field name, check for each parts of name + let nestedFieldNameArray = field.split('.'); + let fieldTypesToCheck = esInstance.fieldTypes[esIndex]; + while (nestedFieldNameArray.length > 0) { + const FieldNameToCheck = nestedFieldNameArray.shift(); + if (fieldTypesToCheck && fieldTypesToCheck[FieldNameToCheck]) { + fieldTypesToCheck = fieldTypesToCheck[FieldNameToCheck].properties; + } else { + throw new UserInputError('Invalid sort argument'); + } + } + // if we got here, everything looks good + nestedFieldNameArray = field.split('.'); + const nestedPath = nestedFieldNameArray.slice(0, nestedFieldNameArray.length - 1).join('.'); + sortBody.push({ + [field]: { + order: method, + nested: { + path: nestedPath, + }, + }, + }); + } } - sortBody = graphqlSortObj; } return sortBody; }; diff --git a/src/server/utils/utils.js b/src/server/utils/utils.js index bb3bd7f6..b0360397 100644 --- a/src/server/utils/utils.js +++ b/src/server/utils/utils.js @@ -98,6 +98,26 @@ export const buildNestedField = (key, value) => { return builtObj; }; +/** + * This function takes a nested field object and parses names of each field + * by concatenating `.` to parent and child field names recursively. + * The returned object is a nested array, which will be deeply flattened later. + * @param field: a nested field object (with `nestedProps`) + */ +export const processNestedFieldNames = (field) => { + const resultArray = []; + field.nestedProps.forEach((prop) => { + if (prop.nestedProps) { + const newField = { ...prop }; + newField.name = `${field.name}.${prop.name}`; + resultArray.push(processNestedFieldNames(newField)); + } else { + resultArray.push(`${field.name}.${prop.name}`); + } + }); + return resultArray; +}; + export const buildNestedFieldMapping = (field, parent) => { if (!field.nestedProps) { return (parent) ? `${parent}.${field.name}` : field.name;