diff --git a/docs_website/docs/changelog/breaking_change.mdx b/docs_website/docs/changelog/breaking_change.mdx index 4ce0f97ae..1c5037d61 100644 --- a/docs_website/docs/changelog/breaking_change.mdx +++ b/docs_website/docs/changelog/breaking_change.mdx @@ -7,6 +7,10 @@ slug: /changelog Here are the list of breaking changes that you should be aware of when updating Querybook: +## v3.32.0 + +Added config `WS_CORS_ALLOWED_ORIGINS` to configure allowed CORS origins for WebSocket connection. This is required for Prod environment. + ## v3.31.0 Upgraded langchain to [0.1.6](https://blog.langchain.dev/langchain-v0-1-0/). diff --git a/docs_website/docs/configurations/infra_config.mdx b/docs_website/docs/configurations/infra_config.mdx index 38528c2f9..490dbb349 100644 --- a/docs_website/docs/configurations/infra_config.mdx +++ b/docs_website/docs/configurations/infra_config.mdx @@ -34,6 +34,10 @@ Otherwise you can also pass the environment variable directly when launching the `FLASK_CACHE_CONFIG` (optional): This can be used to provide caching for API endpoints and internal logic. Follow https://pythonhosted.org/Flask-Cache/ for more details. You should provide a serialized JSON dictionary to be passed into the config. +### WebSocket + +`WS_CORS_ALLOWED_ORIGINS`: (**required for production**): This is the allowed list of origins for CORS. For dev environment, all origins will be allowed. + ### Database `DATABASE_CONN` (**required**): A sqlalchemy connection string to the database. Please check here https://docs.sqlalchemy.org/en/13/core/engines.html for formatting. @@ -100,6 +104,7 @@ You can also add addtional loggers in the event logger plugin. See [Add Event Lo - console: This will print the stats logs to the console. Could be used for debugging purpose. You need to add your own stats logger plugin to use it. See [Add Stats Logger guide](../integrations/add_stats_logger.mdx) for more details. + ## Authentication `AUTH_BACKEND` (optional, defaults to **app.auth.password_auth**): Python path to the authentication file. By default Querybook provides: @@ -119,31 +124,34 @@ the next few configurations are only relevant if you are using OAuth based authe for LDAP authentication: -- `LDAP_CONN`(**required**) -- `LDAP_USE_TLS` (optional, defaults to `False`) -- `LDAP_USE_BIND_USER` (optional, defaults to `False`) - - If `False`: Direct LDAP login - - Additional configuration: - - `LDAP_USER_DN` (**required**) DN with {} for username/etc (ex. `uid={},dc=example,dc=com`) - - Login flow: - - Direct login using formatted `LDAP_USER_DN` + password - - If `True`: Advanced LDAP login using _bind user_ - - Additional configuration: - - `LDAP_BIND_USER` (**required**) Name of a _bind user_ - - `LDAP_BIND_PASSWORD` (**required**) Password of a _bind user_ - - `LDAP_SEARCH` (**required**) LDAP search base (ex. `ou=people,dc=example,dc=com`) - - `LDAP_FILTER` (optional) LDAP filter condition (ex. `(departmentNumber=01000)`) - - `LDAP_UID_FIELD` (optional) Field that matches the username when searching for the account to bind to (defaults to `uid`) - - `LDAP_EMAIL_FIELD`: (optional) Field that matches the user email (default to `mail`) - - `LDAP_LASTNAME_FIELD`: (optional) Field that matches the user surname (default to `sn`) - - `LDAP_FIRSTNAME_FIELD`: (optional) Field that matches the user given name (default to `givenName`) - - `LDAP_FULLNAME_FIELD`: (optional) Field that matches the user full/common name (default to `cn`) - - - Login flow: - 1) Initialized connection for the _bind user_. - 2) Searching the _login user_ using the _bind user_ in LDAP dictionary based on `LDAP_SEARCH` and `LDAP_FILTER`. - 3) The _login user_ credentials are tested in direct login. - 4) If the previous steps were OK, the user is passed on. +- `LDAP_CONN`(**required**) +- `LDAP_USE_TLS` (optional, defaults to `False`) +- `LDAP_USE_BIND_USER` (optional, defaults to `False`) + + - If `False`: Direct LDAP login + - Additional configuration: + - `LDAP_USER_DN` (**required**) DN with {} for username/etc (ex. `uid={},dc=example,dc=com`) + - Login flow: + - Direct login using formatted `LDAP_USER_DN` + password + - If `True`: Advanced LDAP login using _bind user_ + + - Additional configuration: + + - `LDAP_BIND_USER` (**required**) Name of a _bind user_ + - `LDAP_BIND_PASSWORD` (**required**) Password of a _bind user_ + - `LDAP_SEARCH` (**required**) LDAP search base (ex. `ou=people,dc=example,dc=com`) + - `LDAP_FILTER` (optional) LDAP filter condition (ex. `(departmentNumber=01000)`) + - `LDAP_UID_FIELD` (optional) Field that matches the username when searching for the account to bind to (defaults to `uid`) + - `LDAP_EMAIL_FIELD`: (optional) Field that matches the user email (default to `mail`) + - `LDAP_LASTNAME_FIELD`: (optional) Field that matches the user surname (default to `sn`) + - `LDAP_FIRSTNAME_FIELD`: (optional) Field that matches the user given name (default to `givenName`) + - `LDAP_FULLNAME_FIELD`: (optional) Field that matches the user full/common name (default to `cn`) + + - Login flow: + 1. Initialized connection for the _bind user_. + 2. Searching the _login user_ using the _bind user_ in LDAP dictionary based on `LDAP_SEARCH` and `LDAP_FILTER`. + 3. The _login user_ credentials are tested in direct login. + 4. If the previous steps were OK, the user is passed on. If you want to force the user to login again after a certain time, you can the following variable: diff --git a/docs_website/docs/integrations/add_notifier.mdx b/docs_website/docs/integrations/add_notifier.mdx index a51f7dd5d..04c77f0da 100644 --- a/docs_website/docs/integrations/add_notifier.mdx +++ b/docs_website/docs/integrations/add_notifier.mdx @@ -12,11 +12,20 @@ Notifiers provide the option for users to be notified upon completion of their q - Slack - Microsoft Teams +## Provided Notifiers + +The following notifiers are provided by default, and will automatically enable themselves if the necessary configurations are provided: + +- `EmailNotifier`: Sends an email to the user(s) or email address(es) provided. Requires the `EMAILER_CONN` and `QUERYBOOK_EMAIL_ADDRESS` configurations to be set. +- `SlackNotifier`: Sends a message to a Slack channel or user(s). Requires the `QUERYBOOK_SLACK_TOKEN` configuration to be set. + +If no notifiers are enabled and configured, a `NoopNotifier` will be used, which logs the notification message to the server logs along with a suggestion to enable a notifier. + ## Implementation -To keep the notification process standardized, the standard notifiers are included under \/querybook/server/lib/notify/notifiers, -but for custom notifiers create them under \/plugins/notifier_plugin/. -All notifiers must inherit from BaseNotifier that lives in \/querybook/server/lib/notify/base_notifier.py. +To keep the notification process standardized, the standard notifiers are included under `/querybook/server/lib/notify/notifiers`, +but custom notifiers should be created under `/plugins/notifier_plugin/`. +All notifiers must inherit from `BaseNotifier` that lives in `/querybook/server/lib/notify/base_notifier.py`. Here are some fields of notifier that you must configure in the setup process: @@ -29,4 +38,20 @@ Here are some fields of notifier that you must configure in the setup process: If you want to add a notifier that's specific to your own use case, please do so through plugins (See this [Plugin Guide](plugins.mdx) to learn how to setup plugins for Querybook). -Once plugins folder is setup, import the notifier class under `ALL_PLUGIN_NOTIFIERS` in notifier_plugin/**init**.py . +Once plugins folder is setup, import the notifier class under `ALL_PLUGIN_NOTIFIERS` in `notifier_plugin/__init__.py`. + +```python +from lib.notify.notifier.email_notifier import EmailNotifier +from lib.notify.notifier.slack_notifier import SlackNotifier + + +ALL_PLUGIN_NOTIFIERS = [ + EmailNotifier(), + SlackNotifier(), + # Add your notifier here +] +``` + +:::warning +If you configure the `ALL_PLUGIN_NOTIFIERS`, the default notifiers will not enabled automatically. You will need to include them in the notifiers list if you want to use them. +::: diff --git a/package.json b/package.json index b36beb417..ca11426e0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "querybook", - "version": "3.31.2", + "version": "3.32.0", "description": "A Big Data Webapp", "private": true, "scripts": { diff --git a/querybook/config/querybook_default_config.yaml b/querybook/config/querybook_default_config.yaml index 30e5a3f65..42bd611c2 100644 --- a/querybook/config/querybook_default_config.yaml +++ b/querybook/config/querybook_default_config.yaml @@ -54,6 +54,10 @@ LDAP_LASTNAME_FIELD: sn LDAP_FIRSTNAME_FIELD: givenName LDAP_FULLNAME_FIELD: cn +# Websocket CORS allowed origins +WS_CORS_ALLOWED_ORIGINS: + - http://localhost:10001 + # --------------- Result Store --------------- RESULT_STORE_TYPE: db diff --git a/querybook/server/app/flask_app.py b/querybook/server/app/flask_app.py index f675a2282..7041f489a 100644 --- a/querybook/server/app/flask_app.py +++ b/querybook/server/app/flask_app.py @@ -131,7 +131,11 @@ def make_socketio(app): path="-/socket.io", message_queue=QuerybookSettings.REDIS_URL, json=flask_json, - cors_allowed_origins="*", + cors_allowed_origins=( + QuerybookSettings.WS_CORS_ALLOWED_ORIGINS + if QuerybookSettings.PRODUCTION + else "*" + ), ) return socketio diff --git a/querybook/server/env.py b/querybook/server/env.py index 7e59833ab..c27086f23 100644 --- a/querybook/server/env.py +++ b/querybook/server/env.py @@ -55,6 +55,7 @@ class QuerybookSettings(object): PUBLIC_URL = get_env_config("PUBLIC_URL") FLASK_SECRET_KEY = get_env_config("FLASK_SECRET_KEY", optional=False) FLASK_CACHE_CONFIG = get_env_config("FLASK_CACHE_CONFIG") + WS_CORS_ALLOWED_ORIGINS = get_env_config("WS_CORS_ALLOWED_ORIGINS", optional=False) # Celery REDIS_URL = get_env_config("REDIS_URL", optional=False) diff --git a/querybook/server/lib/metastore/utils.py b/querybook/server/lib/metastore/utils.py index d4c3fd518..e20b291bb 100644 --- a/querybook/server/lib/metastore/utils.py +++ b/querybook/server/lib/metastore/utils.py @@ -29,6 +29,8 @@ def _is_table_in_list( for schema_table in self._tables_by_schema[schema]: if schema_table == table or schema_table == "*": return True + elif schema_table.endswith("*") and table.startswith(schema_table[:-1]): + return True return False def is_table_valid( diff --git a/querybook/server/lib/notify/all_notifiers.py b/querybook/server/lib/notify/all_notifiers.py index b52a26bf5..548a8af27 100644 --- a/querybook/server/lib/notify/all_notifiers.py +++ b/querybook/server/lib/notify/all_notifiers.py @@ -1,12 +1,24 @@ +from env import QuerybookSettings + from lib.utils.import_helper import import_module_with_default from .notifier.email_notifier import EmailNotifier +from .notifier.noop_notifier import NoopNotifier +from .notifier.slack_notifier import SlackNotifier + +default_notifiers = [] + +# Auto-load the EmailNotifier / SlackNotifier if configured +if QuerybookSettings.EMAILER_CONN and QuerybookSettings.QUERYBOOK_EMAIL_ADDRESS: + default_notifiers.append(EmailNotifier()) +if QuerybookSettings.QUERYBOOK_SLACK_TOKEN: + default_notifiers.append(SlackNotifier()) + +# If no other notifiers auto-loaded, enable the NoopNotifier +if not default_notifiers: + default_notifiers.append(NoopNotifier()) ALL_PLUGIN_NOTIFIERS = import_module_with_default( - "notifier_plugin", - "ALL_PLUGIN_NOTIFIERS", - default=[ - EmailNotifier(), - ], + "notifier_plugin", "ALL_PLUGIN_NOTIFIERS", default=default_notifiers ) ALL_NOTIFIERS = ALL_PLUGIN_NOTIFIERS diff --git a/querybook/server/lib/notify/notifier/noop_notifier.py b/querybook/server/lib/notify/notifier/noop_notifier.py new file mode 100644 index 000000000..290b74ab2 --- /dev/null +++ b/querybook/server/lib/notify/notifier/noop_notifier.py @@ -0,0 +1,27 @@ +from lib.notify.base_notifier import BaseNotifier +from lib.logger import get_logger + +LOG = get_logger(__file__) + + +class NoopNotifier(BaseNotifier): + @property + def notifier_name(self): + return "noop" + + @property + def notifier_help(self) -> str: + return "Noop notifier does not send any notification. It is used for testing purposes." + + @property + def notifier_format(self): + return "plaintext" + + def notify_recipients(self, recipients, message): + LOG.info(f"📣 Noop notification to {recipients}: {message}") + LOG.info( + "📣 No notifier is configured, please configure a notifier to receive actual notifications!" + ) + + def notify(self, user, message): + self.notify_recipients(recipients=[user.email], message=message) diff --git a/querybook/webapp/components/AppAdmin/AdminMetastore.tsx b/querybook/webapp/components/AppAdmin/AdminMetastore.tsx index 85da9e17c..479634482 100644 --- a/querybook/webapp/components/AppAdmin/AdminMetastore.tsx +++ b/querybook/webapp/components/AppAdmin/AdminMetastore.tsx @@ -11,12 +11,14 @@ import history from 'lib/router-history'; import { generateFormattedDate } from 'lib/utils/datetime'; import { AdminMetastoreResource } from 'resource/admin/metastore'; import { TextButton } from 'ui/Button/Button'; +import { InfoButton } from 'ui/Button/InfoButton'; import { Card } from 'ui/Card/Card'; import { SimpleField } from 'ui/FormikField/SimpleField'; import { GenericCRUD } from 'ui/GenericCRUD/GenericCRUD'; import { Icon } from 'ui/Icon/Icon'; import { Level } from 'ui/Level/Level'; import { Loading } from 'ui/Loading/Loading'; +import { Markdown } from 'ui/Markdown/Markdown'; import { getDefaultFormValue, SmartForm, @@ -302,10 +304,23 @@ export const AdminMetastore: React.FunctionComponent = ({ )}
-
+
ACL Control
+ + {`Access Control Lists (ACL) +are used to limit access to tables in the metastore. If no ACL rules are specified, +all schemas/tables are allowed. Either an allowlist or a denylist can be configured. + +Each value in the list should be in one of the following formats: + +- \`schema.*\`: Allow or deny all tables in a schema +- \`schema.table*\`: Allow or deny all tables in a schema matching a prefix +- \`schema.table\`: Allow or deny a specific table + +This feature affects both the metastore sync and the query engine.`} +
{getMetastoreACLControlDOM( diff --git a/querybook/webapp/components/DataDocLeftSidebar/DataDocLeftSidebar.tsx b/querybook/webapp/components/DataDocLeftSidebar/DataDocLeftSidebar.tsx index 79b64d287..90cbd1c4d 100644 --- a/querybook/webapp/components/DataDocLeftSidebar/DataDocLeftSidebar.tsx +++ b/querybook/webapp/components/DataDocLeftSidebar/DataDocLeftSidebar.tsx @@ -1,6 +1,6 @@ import clsx from 'clsx'; import Resizable from 're-resizable'; -import React, { useCallback, useEffect } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import { DataTableViewMini } from 'components/DataTableViewMini/DataTableViewMini'; @@ -39,7 +39,7 @@ export const DataDocLeftSidebar: React.FunctionComponent = ({ const clearSidebarTableId = () => dispatch(setSidebarTableId(null)); const [contentState, setContentState] = - React.useState('default'); + useState('default'); useEvent( 'keydown', @@ -64,6 +64,7 @@ export const DataDocLeftSidebar: React.FunctionComponent = ({ }, [] ); + useEffect(() => { if (sidebarTableId != null) { setContentState('table'); @@ -76,39 +77,33 @@ export const DataDocLeftSidebar: React.FunctionComponent = ({ const resizeToCollapseSidebar = useResizeToCollapseSidebar( DEFAULT_SIDEBAR_WIDTH, 1 / 3, - React.useCallback(() => setContentState('default'), []) + React.useCallback(() => { + clearSidebarTableId(); + setContentState('default'); + }, []) ); let contentDOM: React.ReactChild; if (contentState === 'contents') { contentDOM = ( - -
- - setContentState('default')} - /> -
- - contents ({TOGGLE_TOC_SHORTCUT}) - - - Click to jump to the corresponding cell. Drag - cells to reorder them. - -
- - -
- +
+ + setContentState('default')} + /> +
+ + contents ({TOGGLE_TOC_SHORTCUT}) + + + Click to jump to the corresponding cell. Drag cells + to reorder them. + +
+ + +
); } else if (contentState === 'table') { contentDOM = ( @@ -143,7 +138,18 @@ export const DataDocLeftSidebar: React.FunctionComponent = ({ hidden: cells.length === 0, })} > - {contentDOM} + {contentState === 'default' ? ( + <> {contentDOM} + ) : ( + + {contentDOM} + + )}
); }; diff --git a/querybook/webapp/lib/query-result/analyzer.ts b/querybook/webapp/lib/query-result/analyzer.ts index edf60e671..fbabb2674 100644 --- a/querybook/webapp/lib/query-result/analyzer.ts +++ b/querybook/webapp/lib/query-result/analyzer.ts @@ -8,6 +8,19 @@ function arrToBigNumber(values: any[]) { } export const columnStatsAnalyzers: IColumnStatsAnalyzer[] = [ + { + key: 'sum', + name: 'Sum', + appliesToType: ['number'], + generator: (values: any[]) => { + const bigNumberArray = arrToBigNumber(values); + const sum = bigNumberArray.reduce( + (s, value) => s.plus(value), + new BigNumber(0) + ); + return sum.toFormat(2); + }, + }, { key: 'average', name: 'Average',