From 78ebe2fce164bbfeef91d0db4370603d2b7ea867 Mon Sep 17 00:00:00 2001 From: danhsiung <35384182+danhsiung@users.noreply.github.com> Date: Mon, 22 Sep 2025 12:33:10 -0700 Subject: [PATCH 1/6] Adding app scaffolding for signl4 --- components/signl4/package.json | 15 +++++++++++++++ components/signl4/signl4.app.mjs | 11 +++++++++++ pnpm-lock.yaml | 3 +++ 3 files changed, 29 insertions(+) create mode 100644 components/signl4/package.json create mode 100644 components/signl4/signl4.app.mjs diff --git a/components/signl4/package.json b/components/signl4/package.json new file mode 100644 index 0000000000000..f51639feaa4ee --- /dev/null +++ b/components/signl4/package.json @@ -0,0 +1,15 @@ +{ + "name": "@pipedream/signl4", + "version": "0.0.1", + "description": "Pipedream SIGNL4 Components", + "main": "signl4.app.mjs", + "keywords": [ + "pipedream", + "signl4" + ], + "homepage": "https://pipedream.com/apps/signl4", + "author": "Pipedream (https://pipedream.com/)", + "publishConfig": { + "access": "public" + } +} \ No newline at end of file diff --git a/components/signl4/signl4.app.mjs b/components/signl4/signl4.app.mjs new file mode 100644 index 0000000000000..989533f6680e9 --- /dev/null +++ b/components/signl4/signl4.app.mjs @@ -0,0 +1,11 @@ +export default { + type: "app", + app: "signl4", + propDefinitions: {}, + methods: { + // this.$auth contains connected account data + authKeys() { + console.log(Object.keys(this.$auth)); + }, + }, +}; \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32d2f100b784d..3d8ddb3b16db9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13147,6 +13147,9 @@ importers: specifier: ^4.0.0 version: 4.0.1 + components/signl4: + specifiers: {} + components/signnow: dependencies: '@pipedream/platform': From 9d2da73592628233f6fc0b287c048bd220183772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Falc=C3=A3o?= <48412907+GTFalcao@users.noreply.github.com> Date: Mon, 22 Sep 2025 17:33:17 -0300 Subject: [PATCH 2/6] Jira - adding pagination to search-issues-with-jql (#18437) * Adding pagination to search-issues-with-jql * Page adjustments * Typo fix --- .../search-issues-with-jql.mjs | 67 ++++++++++++++++--- components/jira/package.json | 2 +- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/components/jira/actions/search-issues-with-jql/search-issues-with-jql.mjs b/components/jira/actions/search-issues-with-jql/search-issues-with-jql.mjs index 1b320bc7033a9..199641e224588 100644 --- a/components/jira/actions/search-issues-with-jql/search-issues-with-jql.mjs +++ b/components/jira/actions/search-issues-with-jql/search-issues-with-jql.mjs @@ -4,7 +4,7 @@ export default { name: "Search Issues with JQL", description: "Search for issues using JQL (Jira Query Language). [See the documentation](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-search/#api-rest-api-3-search-jql-get)", key: "jira-search-issues-with-jql", - version: "0.0.4", + version: "0.1.0", type: "action", props: { jira, @@ -17,16 +17,16 @@ export default { jql: { type: "string", label: "JQL Query", - description: "The [JQL](https://support.atlassian.com/jira-software-cloud/docs/what-is-advanced-search-in-jira-cloud/) query to search for issues", + description: "The JQL query to search for issues. [See the documentation for syntax and examples](https://support.atlassian.com/jira-software-cloud/docs/what-is-advanced-search-in-jira-cloud/)", }, maxResults: { type: "integer", label: "Max Results", - description: "Maximum number of issues to return (default: 50, max: 100)", + description: "Maximum number of issues to return (default: 50)", optional: true, default: 50, min: 1, - max: 100, + max: 5000, }, fields: { type: "string", @@ -90,14 +90,62 @@ export default { optional: true, }, }, + methods: { + getMaxResultsPerPage() { + return Math.min(this.maxResults, 1000); + }, + async paginate({ + params, maxResults, ...otherArgs + }) { + const results = []; + let nextPageToken; + let response; + + do { + response = await this.jira.searchIssues({ + ...otherArgs, + params: { + ...params, + ...(nextPageToken && { + nextPageToken, + }), + }, + }); + + const pageResults = response.issues; + const pageLength = pageResults?.length; + if (!pageLength) { + break; + } + + // If maxResults is specified, only add what we need + if (maxResults && results.length + pageLength > maxResults) { + const remainingSlots = maxResults - results.length; + results.push(...pageResults.slice(0, remainingSlots)); + break; + } else { + results.push(...pageResults); + } + + // Also break if we've reached maxResults exactly + if (maxResults && results.length >= maxResults) { + break; + } + + nextPageToken = response.nextPageToken; + } while (nextPageToken && !response.isLast); + + return results; + }, + }, async run({ $ }) { try { - const response = await this.jira.searchIssues({ + const issues = await this.paginate({ $, cloudId: this.cloudId, params: { jql: this.jql, - maxResults: this.maxResults, + maxResults: this.getMaxResultsPerPage(), fields: this.fields, expand: this.expand ? this.expand.join(",") @@ -106,9 +154,12 @@ export default { fieldsByKeys: this.fieldsByKeys, failFast: this.failFast, }, + maxResults: this.maxResults, }); - $.export("$summary", `Successfully retrieved ${response.issues.length} issues`); - return response; + $.export("$summary", `Successfully retrieved ${issues.length} issues`); + return { + issues, + }; } catch (error) { throw new Error(`Failed to search issues: ${error.message}`); } diff --git a/components/jira/package.json b/components/jira/package.json index cf9fa35512055..68272573ec0ba 100644 --- a/components/jira/package.json +++ b/components/jira/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/jira", - "version": "1.0.4", + "version": "1.1.0", "description": "Pipedream Jira Components", "main": "jira.app.mjs", "keywords": [ From 245b0660918200fa737b2be4856a2cf4e3fca013 Mon Sep 17 00:00:00 2001 From: danhsiung <35384182+danhsiung@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:33:09 -0700 Subject: [PATCH 3/6] Adding app scaffolding for cockpit --- components/cockpit/cockpit.app.mjs | 11 +++++++++++ components/cockpit/package.json | 15 +++++++++++++++ pnpm-lock.yaml | 3 +++ 3 files changed, 29 insertions(+) create mode 100644 components/cockpit/cockpit.app.mjs create mode 100644 components/cockpit/package.json diff --git a/components/cockpit/cockpit.app.mjs b/components/cockpit/cockpit.app.mjs new file mode 100644 index 0000000000000..83c97c68c4e45 --- /dev/null +++ b/components/cockpit/cockpit.app.mjs @@ -0,0 +1,11 @@ +export default { + type: "app", + app: "cockpit", + propDefinitions: {}, + methods: { + // this.$auth contains connected account data + authKeys() { + console.log(Object.keys(this.$auth)); + }, + }, +}; \ No newline at end of file diff --git a/components/cockpit/package.json b/components/cockpit/package.json new file mode 100644 index 0000000000000..d573f3aa9235d --- /dev/null +++ b/components/cockpit/package.json @@ -0,0 +1,15 @@ +{ + "name": "@pipedream/cockpit", + "version": "0.0.1", + "description": "Pipedream Cockpit Components", + "main": "cockpit.app.mjs", + "keywords": [ + "pipedream", + "cockpit" + ], + "homepage": "https://pipedream.com/apps/cockpit", + "author": "Pipedream (https://pipedream.com/)", + "publishConfig": { + "access": "public" + } +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3d8ddb3b16db9..dc63aae233b8c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2934,6 +2934,9 @@ importers: specifier: ^3.0.0 version: 3.0.3 + components/cockpit: + specifiers: {} + components/coda: dependencies: '@pipedream/platform': From f23f304ea74659a479937635def48a567628dcab Mon Sep 17 00:00:00 2001 From: danhsiung <35384182+danhsiung@users.noreply.github.com> Date: Mon, 22 Sep 2025 19:33:08 -0700 Subject: [PATCH 4/6] Adding app scaffolding for kobotoolbox --- components/kobotoolbox/kobotoolbox.app.mjs | 11 +++++++++++ components/kobotoolbox/package.json | 15 +++++++++++++++ pnpm-lock.yaml | 3 +++ 3 files changed, 29 insertions(+) create mode 100644 components/kobotoolbox/kobotoolbox.app.mjs create mode 100644 components/kobotoolbox/package.json diff --git a/components/kobotoolbox/kobotoolbox.app.mjs b/components/kobotoolbox/kobotoolbox.app.mjs new file mode 100644 index 0000000000000..84fb26152ee69 --- /dev/null +++ b/components/kobotoolbox/kobotoolbox.app.mjs @@ -0,0 +1,11 @@ +export default { + type: "app", + app: "kobotoolbox", + propDefinitions: {}, + methods: { + // this.$auth contains connected account data + authKeys() { + console.log(Object.keys(this.$auth)); + }, + }, +}; \ No newline at end of file diff --git a/components/kobotoolbox/package.json b/components/kobotoolbox/package.json new file mode 100644 index 0000000000000..f4d5da6a5cecc --- /dev/null +++ b/components/kobotoolbox/package.json @@ -0,0 +1,15 @@ +{ + "name": "@pipedream/kobotoolbox", + "version": "0.0.1", + "description": "Pipedream KoboToolbox Components", + "main": "kobotoolbox.app.mjs", + "keywords": [ + "pipedream", + "kobotoolbox" + ], + "homepage": "https://pipedream.com/apps/kobotoolbox", + "author": "Pipedream (https://pipedream.com/)", + "publishConfig": { + "access": "public" + } +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dc63aae233b8c..cfc3a254fa106 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7601,6 +7601,9 @@ importers: components/koala_ai: {} + components/kobotoolbox: + specifiers: {} + components/kodagpt: dependencies: '@pipedream/platform': From 0c007dbbb1d61375ca39216f5bbc04005da22092 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 23 Sep 2025 02:41:14 -0400 Subject: [PATCH 5/6] Merging pull request #18434 --- .../actions/list-policies/list-policies.ts | 36 +++++++++++++++++++ components/expensify/package.json | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 components/expensify/actions/list-policies/list-policies.ts diff --git a/components/expensify/actions/list-policies/list-policies.ts b/components/expensify/actions/list-policies/list-policies.ts new file mode 100644 index 0000000000000..ea21ce50e4b36 --- /dev/null +++ b/components/expensify/actions/list-policies/list-policies.ts @@ -0,0 +1,36 @@ +import { defineAction } from "@pipedream/types"; +import expensify from "../../app/expensify.app"; + +export default defineAction({ + key: "expensify-list-policies", + name: "List Policies", + description: "Retrieves a list of policies. [See the documentation](https://integrations.expensify.com/Integration-Server/doc/#policy-list-getter)", + version: "0.0.1", + type: "action", + props: { + expensify, + adminOnly: { + type: "boolean", + label: "Admin Only", + description: "Whether or not to only get policies for which the user is an admin", + optional: true, + }, + userEmail: { + type: "string", + label: "User Email", + description: "Specifies the user to gather the policy list for. You must have been granted third-party access by that user/company domain beforehand.", + optional: true, + }, + }, + async run({ $ }) { + const response = await this.expensify.getPolicyList({ + $, + userEmail: this.userEmail, + adminOnly: this.adminOnly, + }); + + $.export("$summary", `Successfully retrieved ${response?.policyList?.length || 0} policies`); + + return response?.policyList || []; + }, +}) diff --git a/components/expensify/package.json b/components/expensify/package.json index bea532e7eb6eb..182dfef645c5c 100644 --- a/components/expensify/package.json +++ b/components/expensify/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/expensify", - "version": "0.2.0", + "version": "0.3.0", "description": "Pipedream Expensify Components", "main": "dist/app/expensify.app.mjs", "keywords": [ From 946cafdcb1413e6f51092c8043f8e1db069587cf Mon Sep 17 00:00:00 2001 From: Lokesh Chand <91275101+Lokeshchand33@users.noreply.github.com> Date: Tue, 23 Sep 2025 23:31:23 +0530 Subject: [PATCH 6/6] Databricks sql warehouses (#18273) * Added Databricks SQL Warehouses API actions - Added create SQL warehouse action - Added list SQL warehouses action - Added get SQL warehouse action - Added edit SQL warehouse action - Added delete SQL warehouse action - Added start/stop SQL warehouse actions - Added get/set SQL warehouse config actions - Added get/set SQL warehouse permissions actions Implements 11 SQL Warehouses endpoints as discussed in the enhancement issue. * Update Databricks SQL Warehouse docs URLs * fix(databricks): bump component versions and apply lint fixes * fix(databricks): addressed requested changes * addressed coderabbit review feedback * resolved the linting issues * addressed all test failures * addressed coderabbit review feedback * resolved the linting issues * addressed coderabbit review feedback * addressed coderabbit review feedback * resolved the linting issues * updates * Add default value for maxNumClusters * create and edit sql warehouses fixes * create and edit sql warehouse fixes * updates * Added Vector Search Index API actions Summary This PR adds support for Databricks Vector Search Index APIs by implementing 9 new actions. Changes - Added create index action - Added get index action - Added list indexes action - Added delete index action - Added query index action - Added sync index action - Added scan index action - Added upsert index data action - Added delete index data action Resolves #18126 * addressed coderabbit review feedback * version updated * resolved the linting issues * addressed all test failures * addressed coderabbit review feedback * addressed coderabbit review feedback * addressed coderabbit review feedback * updated * updated * updates * fixed failed test cases * updated * updated * updated * fixed failed test cases * fixed failed test cases * resolved conflict * updated * version updated * updated * updated --------- Co-authored-by: Michelle Bergeron Co-authored-by: Leo Vu Co-authored-by: Lokesh chand Co-authored-by: michelle0927 --- .../cancel-all-runs/cancel-all-runs.mjs | 2 +- .../actions/cancel-run/cancel-run.mjs | 2 +- .../create-endpoint/create-endpoint.mjs | 2 +- .../actions/create-job/create-job.mjs | 2 +- .../create-sql-warehouse.mjs | 2 +- .../create-vector-search-index.mjs | 163 ++++++++++++++++++ .../delete-endpoint/delete-endpoint.mjs | 2 +- .../actions/delete-job/delete-job.mjs | 2 +- .../actions/delete-run/delete-run.mjs | 2 +- .../delete-sql-warehouse.mjs | 2 +- .../delete-vector-search-index-data.mjs | 64 +++++++ .../delete-vector-search-index.mjs | 36 ++++ .../edit-sql-warehouse/edit-sql-warehouse.mjs | 2 +- .../actions/export-run/export-run.mjs | 2 +- .../actions/get-endpoint/get-endpoint.mjs | 2 +- .../get-job-permissions.mjs | 2 +- .../databricks/actions/get-job/get-job.mjs | 2 +- .../actions/get-run-output/get-run-output.mjs | 2 +- .../databricks/actions/get-run/get-run.mjs | 2 +- .../get-sql-warehouse-config.mjs | 2 +- .../get-sql-warehouse-permissions.mjs | 2 +- .../get-sql-warehouse/get-sql-warehouse.mjs | 2 +- .../get-vector-search-index.mjs | 40 +++++ .../actions/list-endpoints/list-endpoints.mjs | 2 +- .../actions/list-jobs/list-jobs.mjs | 2 +- .../actions/list-runs/list-runs.mjs | 2 +- .../list-sql-warehouses.mjs | 2 +- .../list-vector-search-indexes.mjs | 36 ++++ .../query-vector-search-index.mjs | 106 ++++++++++++ .../actions/repair-run/repair-run.mjs | 2 +- .../actions/reset-job/reset-job.mjs | 2 +- .../actions/run-job-now/run-job-now.mjs | 2 +- .../scan-vector-search-index.mjs | 64 +++++++ .../set-job-permissions.mjs | 2 +- .../set-sql-warehouse-config.mjs | 2 +- .../set-sql-warehouse-permissions.mjs | 2 +- .../start-sql-warehouse.mjs | 2 +- .../stop-sql-warehouse/stop-sql-warehouse.mjs | 2 +- .../sync-vector-search-index.mjs | 40 +++++ .../actions/update-job/update-job.mjs | 2 +- .../upsert-vector-search-index-data.mjs | 53 ++++++ components/databricks/databricks.app.mjs | 115 +++++++++++- components/databricks/package.json | 2 +- 43 files changed, 749 insertions(+), 34 deletions(-) create mode 100644 components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs create mode 100644 components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs create mode 100644 components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs create mode 100644 components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs create mode 100644 components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs create mode 100644 components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs create mode 100644 components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs create mode 100644 components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs create mode 100644 components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs diff --git a/components/databricks/actions/cancel-all-runs/cancel-all-runs.mjs b/components/databricks/actions/cancel-all-runs/cancel-all-runs.mjs index 592740cc949fd..6cbec9bd8ea1c 100644 --- a/components/databricks/actions/cancel-all-runs/cancel-all-runs.mjs +++ b/components/databricks/actions/cancel-all-runs/cancel-all-runs.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-cancel-all-runs", name: "Cancel All Runs", description: "Cancel all active runs for a job. The runs are canceled asynchronously, so it doesn't prevent new runs from being started. [See the documentation](https://docs.databricks.com/api/workspace/jobs/cancelallruns)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/cancel-run/cancel-run.mjs b/components/databricks/actions/cancel-run/cancel-run.mjs index 9af5f0fd4c76a..bb54ba37e5d37 100644 --- a/components/databricks/actions/cancel-run/cancel-run.mjs +++ b/components/databricks/actions/cancel-run/cancel-run.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-cancel-run", name: "Cancel Run", description: "Cancel a job run. The run is canceled asynchronously, so it may still be running when this request completes. [See the documentation](https://docs.databricks.com/api/workspace/jobs/cancelrun)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/create-endpoint/create-endpoint.mjs b/components/databricks/actions/create-endpoint/create-endpoint.mjs index 33bf7ad4d7a8f..046e54bc31dd1 100644 --- a/components/databricks/actions/create-endpoint/create-endpoint.mjs +++ b/components/databricks/actions/create-endpoint/create-endpoint.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-create-endpoint", name: "Create Endpoint", description: "Create a new vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/createendpoint)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/create-job/create-job.mjs b/components/databricks/actions/create-job/create-job.mjs index 209cf119ac164..708146085ae73 100644 --- a/components/databricks/actions/create-job/create-job.mjs +++ b/components/databricks/actions/create-job/create-job.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-create-job", name: "Create Job", description: "Create a job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/create)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index b953d78090e61..02de4ce9f5c39 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -7,7 +7,7 @@ export default { key: "databricks-create-sql-warehouse", name: "Create SQL Warehouse", description: "Creates a new SQL Warehouse in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/create)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs new file mode 100644 index 0000000000000..492bbca32e0dd --- /dev/null +++ b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs @@ -0,0 +1,163 @@ +import databricks from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; +import { ConfigurationError } from "@pipedream/platform"; + +export default { + key: "databricks-create-vector-search-index", + name: "Create Vector Search Index", + description: + "Creates a new vector search index in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/createindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + name: { + type: "string", + label: "Index Name", + description: + "A unique name for the index (e.g., `main_catalog.docs.en_wiki_index`).", + }, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, + indexType: { + type: "string", + label: "Index Type", + description: "Type of index (`DELTA_SYNC` or `DIRECT_ACCESS`).", + options: [ + "DELTA_SYNC", + "DIRECT_ACCESS", + ], + }, + primaryKey: { + type: "string", + label: "Primary Key", + description: "The primary key column for the index.", + }, + sourceTable: { + type: "string", + label: "Source Table", + description: + "The Delta table backing the index (required for `DELTA_SYNC`).", + optional: true, + }, + columnsToSync: { + type: "string[]", + label: "Columns to Sync", + description: + "List of columns to sync from the source Delta table. Example: `[\"id\", \"text\"]` (required for `DELTA_SYNC`).", + optional: true, + }, + embeddingSourceColumns: { + type: "string[]", + label: "Embedding Source Columns", + description: + "List of embedding source column configs. Each entry is a JSON object string like `{ \"embedding_model_endpoint_name\": \"e5-small-v2\", \"name\": \"text\" }`.Provide when Databricks computes embeddings (DELTA_SYNC).", + optional: true, + }, + schemaJson: { + type: "string", + label: "Schema JSON", + description: + "The schema of the index in JSON format. Example: `{ \"columns\": [{ \"name\": \"id\", \"type\": \"string\" }, { \"name\": \"text_vector\", \"type\": \"array\" }] }`. Required for `DIRECT_ACCESS` indexes.", + optional: true, + }, + pipelineType: { + type: "string", + label: "Pipeline Type", + description: "Pipeline type for syncing (default: TRIGGERED).", + options: [ + "TRIGGERED", + "CONTINUOUS", + ], + optional: true, + default: "TRIGGERED", + }, + }, + + async run({ $ }) { + const payload = { + name: this.name, + endpoint_name: this.endpointName, + index_type: this.indexType, + primary_key: this.primaryKey, + }; + + if (this.indexType === "DELTA_SYNC") { + if (this.schemaJson) { + throw new ConfigurationError( + "`Schema JSON` is not allowed when indexType is DELTA_SYNC.", + ); + } + if (!this.sourceTable) { + throw new ConfigurationError( + "sourceTable is required when indexType is DELTA_SYNC.", + ); + } + + const columnsToSync = Array.isArray(this.columnsToSync) + ? this.columnsToSync + : utils.parseObject(this.columnsToSync); + + const embeddingSourceColumns = utils.parseObject(this.embeddingSourceColumns); + const hasSource = Array.isArray(embeddingSourceColumns) && embeddingSourceColumns.length > 0; + if (!hasSource) { + throw new ConfigurationError( + "embeddingSourceColumns is required when indexType is DELTA_SYNC.", + ); + } + + const deltaSpec = { + source_table: this.sourceTable, + pipeline_type: this.pipelineType || "TRIGGERED", + }; + if (Array.isArray(columnsToSync) && columnsToSync.length > 0) { + deltaSpec.columns_to_sync = columnsToSync; + } + if (hasSource) { + for (const [ + i, + c, + ] of embeddingSourceColumns.entries()) { + if (!c?.name || !c?.embedding_model_endpoint_name) { + throw new ConfigurationError( + `embeddingSourceColumns[${i}] must include "name" and "embedding_model_endpoint_name"`, + ); + } + } + deltaSpec.embedding_source_columns = embeddingSourceColumns; + } + payload.delta_sync_index_spec = deltaSpec; + } + + else if (this.indexType === "DIRECT_ACCESS") { + if (this.sourceTable || this.columnsToSync?.length || this.embeddingSourceColumns?.length) { + throw new ConfigurationError( + "`Source Table`,`Embedding Source Columns` and `Columns to Sync` are not allowed when indexType is DIRECT_ACCESS.", + ); + } + if (!this.schemaJson) { + throw new ConfigurationError( + "schemaJson is required when indexType is DIRECT_ACCESS.", + ); + } + payload.direct_access_index_spec = { + schema_json: this.schemaJson, + }; + } + + const response = await this.databricks.createVectorSearchIndex({ + data: payload, + $, + }); + + $.export( + "$summary", + `Successfully created vector search index: ${response?.name || this.name}`, + ); + return response; + }, +}; diff --git a/components/databricks/actions/delete-endpoint/delete-endpoint.mjs b/components/databricks/actions/delete-endpoint/delete-endpoint.mjs index 4a95bb6f13e2d..6750820463af8 100644 --- a/components/databricks/actions/delete-endpoint/delete-endpoint.mjs +++ b/components/databricks/actions/delete-endpoint/delete-endpoint.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-endpoint", name: "Delete Endpoint", description: "Delete a vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/deleteendpoint)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/delete-job/delete-job.mjs b/components/databricks/actions/delete-job/delete-job.mjs index a5e55b716421b..79b6c528ee93e 100644 --- a/components/databricks/actions/delete-job/delete-job.mjs +++ b/components/databricks/actions/delete-job/delete-job.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-job", name: "Delete Job", description: "Delete a job. Deleted jobs cannot be recovered. [See the documentation](https://docs.databricks.com/api/workspace/jobs/delete)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/delete-run/delete-run.mjs b/components/databricks/actions/delete-run/delete-run.mjs index 2fbda0185965d..b27836b1cabb5 100644 --- a/components/databricks/actions/delete-run/delete-run.mjs +++ b/components/databricks/actions/delete-run/delete-run.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-run", name: "Delete Run", description: "Delete a non-active run. Returns an error if the run is active. [See the documentation](https://docs.databricks.com/api/workspace/jobs/deleterun)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs index 47584608434df..3300848ba3852 100644 --- a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs +++ b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-sql-warehouse", name: "Delete SQL Warehouse", description: "Deletes a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/delete)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs new file mode 100644 index 0000000000000..b605f15edb7c3 --- /dev/null +++ b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs @@ -0,0 +1,64 @@ +import databricks from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; + +export default { + key: "databricks-delete-vector-search-index-data", + name: "Delete Data from Vector Search Index", + description: + "Deletes rows from a Direct Access vector index by primary-key values. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/deletedatavectorindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, + indexName: { + propDefinition: [ + databricks, + "indexName", + ({ endpointName }) => ({ + endpointName, + }), + ], + }, + primaryKeys: { + type: "string[]", + label: "Primary Keys", + description: + "Values of the index’s primary key column to delete (e.g. `1`, `2`). These are the values for the column you set as `primary_key` when the index was created.", + }, + }, + async run({ $ }) { + const parsedKeys = utils.parseObject(this.primaryKeys); + + const keys = (Array.isArray(parsedKeys) + ? parsedKeys + : [ + parsedKeys, + ]) + .map((s) => String(s).trim()) + .filter(Boolean); + + if (!keys.length) { + throw new Error("Please provide at least one primary key to delete."); + } + + const response = await this.databricks.deleteVectorSearchData({ + indexName: this.indexName, + params: { + primary_keys: keys, + }, + $, + }); + + $.export( + "$summary", + `Requested delete of ${keys.length} row(s) from index "${this.indexName}".`, + ); + return response; + }, +}; diff --git a/components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs b/components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs new file mode 100644 index 0000000000000..756f529d003b0 --- /dev/null +++ b/components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs @@ -0,0 +1,36 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-delete-vector-search-index", + name: "Delete Vector Search Index", + description: "Deletes a vector search index in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/deleteindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, + indexName: { + propDefinition: [ + databricks, + "indexName", + ({ endpointName }) => ({ + endpointName, + }), + ], + }, + }, + async run({ $ }) { + const response = await this.databricks.deleteVectorSearchIndex({ + indexName: this.indexName, + $, + }); + + $.export("$summary", `Successfully deleted vector search index: ${this.indexName}`); + return response; + }, +}; diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index 293ee437f120a..b98292e6a5faa 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -7,7 +7,7 @@ export default { key: "databricks-edit-sql-warehouse", name: "Edit SQL Warehouse", description: "Edits the configuration of an existing SQL Warehouse. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/edit)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/export-run/export-run.mjs b/components/databricks/actions/export-run/export-run.mjs index 63fca03ae7303..71e8e28fd7ea9 100644 --- a/components/databricks/actions/export-run/export-run.mjs +++ b/components/databricks/actions/export-run/export-run.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-export-run", name: "Export Run", description: "Export and retrieve the job run task. [See the documentation](https://docs.databricks.com/api/workspace/jobs/exportrun)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/get-endpoint/get-endpoint.mjs b/components/databricks/actions/get-endpoint/get-endpoint.mjs index 1f494f4658be1..0937ff8482688 100644 --- a/components/databricks/actions/get-endpoint/get-endpoint.mjs +++ b/components/databricks/actions/get-endpoint/get-endpoint.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-endpoint", name: "Get Endpoint", description: "Get details of a specific vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/getendpoint)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-job-permissions/get-job-permissions.mjs b/components/databricks/actions/get-job-permissions/get-job-permissions.mjs index 3c9bae4657b9d..c1e77b0564888 100644 --- a/components/databricks/actions/get-job-permissions/get-job-permissions.mjs +++ b/components/databricks/actions/get-job-permissions/get-job-permissions.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-job-permissions", name: "Get Job Permissions", description: "Get permissions of a job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/getpermissions)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/get-job/get-job.mjs b/components/databricks/actions/get-job/get-job.mjs index 6e418684cdd14..fd029c564e3b8 100644 --- a/components/databricks/actions/get-job/get-job.mjs +++ b/components/databricks/actions/get-job/get-job.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-job", name: "Get Job", description: "Retrieves the details for a single job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/get)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/get-run-output/get-run-output.mjs b/components/databricks/actions/get-run-output/get-run-output.mjs index c44b1001a7bfa..6bc6ee495e0cb 100644 --- a/components/databricks/actions/get-run-output/get-run-output.mjs +++ b/components/databricks/actions/get-run-output/get-run-output.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-run-output", name: "Get Run Output", description: "Retrieve the output and metadata of a single task run. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-get-output)", - version: "0.0.4", + version: "0.0.5", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-run/get-run.mjs b/components/databricks/actions/get-run/get-run.mjs index 84d95be7c0d5c..607fee72566b8 100644 --- a/components/databricks/actions/get-run/get-run.mjs +++ b/components/databricks/actions/get-run/get-run.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-run", name: "Get Run", description: "Retrieve the metadata of a run. [See the documentation](https://docs.databricks.com/api/workspace/jobs/getrun)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs index 1fbdda709e71b..3ab986eb06a7a 100644 --- a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs +++ b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse-config", name: "Get SQL Warehouse Config", description: "Retrieves the global configuration for SQL Warehouses. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/getworkspacewarehouseconfig)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs index 4d76f5bb41c74..c7098db9d4156 100644 --- a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse-permissions", name: "Get SQL Warehouse Permissions", description: "Retrieves the permissions for a specific SQL Warehouse. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/getpermissions)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs index 28d131c05c8f1..21fd9701333a9 100644 --- a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs +++ b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse", name: "Get SQL Warehouse", description: "Retrieves details for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/get)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs b/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs new file mode 100644 index 0000000000000..6ead8475541e1 --- /dev/null +++ b/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs @@ -0,0 +1,40 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-get-vector-search-index", + name: "Get Vector Search Index", + description: "Retrieves details about a specific vector search index. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/getindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, + indexName: { + propDefinition: [ + databricks, + "indexName", + ({ endpointName }) => ({ + endpointName, + }), + ], + }, + }, + + async run({ $ }) { + const response = await this.databricks.getVectorSearchIndex({ + indexName: this.indexName, + $, + }); + + $.export( + "$summary", + `Successfully retrieved vector search index: ${this.indexName}`, + ); + return response; + }, +}; diff --git a/components/databricks/actions/list-endpoints/list-endpoints.mjs b/components/databricks/actions/list-endpoints/list-endpoints.mjs index c0fe92e1672e6..9e692d7ecdae7 100644 --- a/components/databricks/actions/list-endpoints/list-endpoints.mjs +++ b/components/databricks/actions/list-endpoints/list-endpoints.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-endpoints", name: "List Endpoints", description: "List all vector search endpoints. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/listendpoints)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-jobs/list-jobs.mjs b/components/databricks/actions/list-jobs/list-jobs.mjs index 9f60a6ea0224b..4781153909c4b 100644 --- a/components/databricks/actions/list-jobs/list-jobs.mjs +++ b/components/databricks/actions/list-jobs/list-jobs.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-list-jobs", name: "List Jobs", description: "List all jobs using automatic pagination. [See the documentation](https://docs.databricks.com/api/workspace/jobs/list)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/list-runs/list-runs.mjs b/components/databricks/actions/list-runs/list-runs.mjs index d2d7fe17cd3f3..e03846d411551 100644 --- a/components/databricks/actions/list-runs/list-runs.mjs +++ b/components/databricks/actions/list-runs/list-runs.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-runs", name: "List Runs", description: "Lists all runs available to the user. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-list)", - version: "0.0.4", + version: "0.0.5", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs index 356622572a9a9..5b63c9c874a82 100644 --- a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs +++ b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-sql-warehouses", name: "List SQL Warehouses", description: "Lists all SQL Warehouses available in the Databricks workspace. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/list)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs new file mode 100644 index 0000000000000..28e71eb08d0a1 --- /dev/null +++ b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs @@ -0,0 +1,36 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-list-vector-search-indexes", + name: "List Vector Search Indexes", + description: "Lists all vector search indexes for a given endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/listindexes)", + version: "0.0.1", + type: "action", + props: { + databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, + }, + + async run({ $ }) { + const { vector_indexes = [] } = await this.databricks.listVectorSearchIndexes({ + params: { + endpoint_name: this.endpointName, + }, + $, + }); + + $.export( + "$summary", + `Successfully retrieved ${vector_indexes.length} index${vector_indexes.length === 1 + ? "" + : "es"}.`, + ); + + return vector_indexes; + }, +}; diff --git a/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs new file mode 100644 index 0000000000000..44f8cfac7685d --- /dev/null +++ b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs @@ -0,0 +1,106 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-query-vector-search-index", + name: "Query Vector Search Index", + description: "Query a specific vector search index in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/queryindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, + indexName: { + propDefinition: [ + databricks, + "indexName", + ({ endpointName }) => ({ + endpointName, + }), + ], + }, + columns: { + type: "string[]", + label: "Columns", + description: "List of column names to include in the response. Example: `[\"id\"]`", + }, + queryText: { + type: "string", + label: "Query Text", + description: "Free-text query for semantic search.", + optional: true, + }, + queryVector: { + type: "string", + label: "Query Vector", + description: "JSON array of floats representing the embedding vector for the query.", + optional: true, + }, + filtersJson: { + type: "string", + label: "Filters JSON", + description: "JSON string representing query filters. Example: `{ \"id <\": 5 }`", + optional: true, + }, + numResults: { + type: "integer", + label: "Number of Results", + description: "Number of results to return. Defaults to 10.", + optional: true, + default: 10, + }, + includeEmbeddings: { + type: "boolean", + label: "Include Embeddings", + description: "Whether to include the embedding vectors in the results.", + optional: true, + }, + }, + + async run({ $ }) { + const payload = { + columns: this.columns, + num_results: this.numResults, + }; + + if (this.queryText) payload.query_text = this.queryText; + + if (this.queryVector) { + try { + payload.query_vector = JSON.parse(this.queryVector); + if ( + !Array.isArray(payload.query_vector) || + payload.query_vector.length === 0 || + !payload.query_vector.every((n) => typeof n === "number" && Number.isFinite(n)) + ) { + throw new Error("`queryVector` must be a non-empty JSON array of finite numbers."); + } + } catch (err) { + throw new Error(`Invalid queryVector JSON: ${err.message}`); + } + } + + if (this.filtersJson) { + payload.filters_json = this.filtersJson; + } + + if (this.includeEmbeddings !== undefined) { + payload.include_embeddings = this.includeEmbeddings; + } + + const response = await this.databricks.queryVectorSearchIndex({ + indexName: this.indexName, + data: payload, + $, + }); + + const count = response?.result?.data_array?.length || 0; + $.export("$summary", `Retrieved ${count} results from index ${this.indexName}`); + + return response; + }, +}; diff --git a/components/databricks/actions/repair-run/repair-run.mjs b/components/databricks/actions/repair-run/repair-run.mjs index e5f9234fdf800..ef4e4015fb5db 100644 --- a/components/databricks/actions/repair-run/repair-run.mjs +++ b/components/databricks/actions/repair-run/repair-run.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-repair-run", name: "Repair Run", description: "Re-run one or more tasks. [See the documentation](https://docs.databricks.com/api/workspace/jobs/repairrun)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/reset-job/reset-job.mjs b/components/databricks/actions/reset-job/reset-job.mjs index cf6acbc2af856..44aa051ea1d3f 100644 --- a/components/databricks/actions/reset-job/reset-job.mjs +++ b/components/databricks/actions/reset-job/reset-job.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-reset-job", name: "Reset Job", description: "Overwrite all settings for the given job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/reset)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/run-job-now/run-job-now.mjs b/components/databricks/actions/run-job-now/run-job-now.mjs index 4318cb98d9e8a..11c6e52ad930c 100644 --- a/components/databricks/actions/run-job-now/run-job-now.mjs +++ b/components/databricks/actions/run-job-now/run-job-now.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-run-job-now", name: "Run Job Now", description: "Run a job now and return the id of the triggered run. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-list)", - version: "0.0.4", + version: "0.0.5", type: "action", props: { databricks, diff --git a/components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs b/components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs new file mode 100644 index 0000000000000..c1036d0d10220 --- /dev/null +++ b/components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs @@ -0,0 +1,64 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-scan-vector-search-index", + name: "Scan Vector Search Index", + description: + "Scans a vector search index and returns entries after the given primary key. [See documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/scanindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, + indexName: { + propDefinition: [ + databricks, + "indexName", + ({ endpointName }) => ({ + endpointName, + }), + ], + }, + lastPrimaryKey: { + type: "string", + label: "Last Primary Key", + description: + "Primary key of the last entry returned in the previous scan. Leave empty to start from the beginning.", + optional: true, + }, + numResults: { + type: "integer", + label: "Number of Results", + description: "Number of results to return (defaults to 10).", + optional: true, + default: 10, + }, + }, + + async run({ $ }) { + const body = {}; + if (this.lastPrimaryKey !== undefined) { + body.last_primary_key = this.lastPrimaryKey; + } + if (this.numResults !== undefined) { + body.num_results = this.numResults; + } + + const response = await this.databricks.scanVectorSearchIndex({ + indexName: this.indexName, + data: body, + $, + }); + + $.export( + "$summary", + `Scanned index "${this.indexName}" and returned ${response?.data?.length ?? 0} entries.`, + ); + return response; + }, +}; diff --git a/components/databricks/actions/set-job-permissions/set-job-permissions.mjs b/components/databricks/actions/set-job-permissions/set-job-permissions.mjs index adf2897ec8faf..5c0cde44682c1 100644 --- a/components/databricks/actions/set-job-permissions/set-job-permissions.mjs +++ b/components/databricks/actions/set-job-permissions/set-job-permissions.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-set-job-permissions", name: "Set Job Permissions", description: "Set permissions on a job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/setpermissions)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index 6c9d4a7b98e6e..c8e1252417614 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -6,7 +6,7 @@ export default { key: "databricks-set-sql-warehouse-config", name: "Set SQL Warehouse Config", description: "Updates the global configuration for SQL Warehouses. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/setworkspacewarehouseconfig)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs index 202d1677dd5a9..9d62bd131465b 100644 --- a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs @@ -6,7 +6,7 @@ export default { key: "databricks-set-sql-warehouse-permissions", name: "Set SQL Warehouse Permissions", description: "Updates the permissions for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/setpermissions)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs index 669a593323040..5cba5641b2bd3 100644 --- a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs +++ b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-start-sql-warehouse", name: "Start SQL Warehouse", description: "Starts a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/start)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs index 0117fca5d801d..8c1b8a932a0a8 100644 --- a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs +++ b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-stop-sql-warehouse", name: "Stop SQL Warehouse", description: "Stops a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/stop)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs b/components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs new file mode 100644 index 0000000000000..ceb744671c506 --- /dev/null +++ b/components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs @@ -0,0 +1,40 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-sync-vector-search-index", + name: "Sync Vector Search Index", + description: "Synchronize a Delta Sync vector search index in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/syncindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, + indexName: { + propDefinition: [ + databricks, + "indexName", + ({ endpointName }) => ({ + endpointName, + }), + ], + }, + }, + async run({ $ }) { + const response = await this.databricks.syncVectorSearchIndex({ + indexName: this.indexName, + $, + }); + + $.export( + "$summary", + `Successfully triggered sync for vector search index: ${this.indexName}`, + ); + + return response; + }, +}; diff --git a/components/databricks/actions/update-job/update-job.mjs b/components/databricks/actions/update-job/update-job.mjs index 6d6afec50bb7f..8cb8f6c81d36d 100644 --- a/components/databricks/actions/update-job/update-job.mjs +++ b/components/databricks/actions/update-job/update-job.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-update-job", name: "Update Job", description: "Update an existing job. Only the fields that are provided will be updated. [See the documentation](https://docs.databricks.com/api/workspace/jobs/update)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs new file mode 100644 index 0000000000000..6e4f5cca592e9 --- /dev/null +++ b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs @@ -0,0 +1,53 @@ +import databricks from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; +import { ConfigurationError } from "@pipedream/platform"; + +export default { + key: "databricks-upsert-vector-search-index-data", + name: "Upsert Vector Search Index Data", + description: "Upserts (inserts/updates) data into an existing vector search index. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/upsertdatavectorindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, + indexName: { + propDefinition: [ + databricks, + "indexName", + ({ endpointName }) => ({ + endpointName, + }), + ], + }, + rows: { + type: "string", + label: "Rows to Upsert", + description: "Array of rows to upsert. Each row should be a JSON object string. Example: `[{ \"id\": \"1\", \"text\": \"hello world\", \"text_vector\": [0.1, 0.2, 0.3] }]`", + }, + }, + + async run({ $ }) { + const parsedRows = utils.parseObject(this.rows); + + if (!Array.isArray(parsedRows) || !parsedRows.length) { + throw new ConfigurationError("rows must be a non-empty JSON array."); + } + + const response = await this.databricks.upsertVectorSearchIndexData({ + indexName: this.indexName, + data: { + inputs_json: JSON.stringify(parsedRows), + }, + $, + }); + + $.export("$summary", `Successfully upserted ${parsedRows.length} row(s) into index ${this.indexName}`); + return response; + }, +}; diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index d4f941e5c922c..607764fdc69e4 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -67,7 +67,7 @@ export default { description: "The name of the vector search endpoint", async options({ prevContext }) { const { - endpoints, next_page_token, + endpoints = [], next_page_token, } = await this.listEndpoints({ params: { page_token: prevContext.page_token, @@ -97,6 +97,26 @@ export default { })) || []; }, }, + indexName: { + type: "string", + label: "Index Name", + description: "The name of the vector search index", + async options({ endpointName }) { + if (!endpointName) { + return []; + } + const { vector_indexes = [] } = await this.listVectorSearchIndexes({ + params: { + endpoint_name: endpointName, + }, + }); + + return vector_indexes.map(({ name }) => ({ + value: name, + label: name, + })); + }, + }, }, methods: { getUrl(path, versionPath = constants.VERSION_PATH.V2_0) { @@ -371,6 +391,99 @@ export default { ...args, }); }, + createVectorSearchIndex(args = {}) { + return this._makeRequest({ + path: "/vector-search/indexes", + method: "POST", + ...args, + }); + }, + + getVectorSearchIndex({ + indexName, ...args + }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}`, + method: "GET", + ...args, + }); + }, + + listVectorSearchIndexes({ + params, ...args + }) { + return this._makeRequest({ + path: "/vector-search/indexes", + method: "GET", + params, + ...args, + }); + }, + + deleteVectorSearchIndex({ + indexName, ...args + }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}`, + method: "DELETE", + ...args, + }); + }, + + queryVectorSearchIndex({ + indexName, ...args + }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}/query`, + method: "POST", + ...args, + }); + }, + + syncVectorSearchIndex({ + indexName, ...args + }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}/sync`, + method: "POST", + ...args, + }); + }, + + deleteVectorSearchData({ + indexName, params, ...args + }) + { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}/delete-data`, + method: "DELETE", + params, + paramsSerializer: { + indexes: null, + }, + ...args, + }); + }, + + upsertVectorSearchIndexData({ + indexName, ...args + }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}/upsert-data`, + method: "POST", + ...args, + }); + }, + + scanVectorSearchIndex({ + indexName, ...args + }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}/scan`, + method: "POST", + ...args, + }); + }, async paginate({ requestor, requestorArgs = {}, maxRequests = 3, resultsKey = "jobs", diff --git a/components/databricks/package.json b/components/databricks/package.json index c82b72576579b..22ed3bf6d6cb6 100644 --- a/components/databricks/package.json +++ b/components/databricks/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/databricks", - "version": "0.4.0", + "version": "0.5.0", "description": "Pipedream Databricks Components", "main": "databricks.app.mjs", "keywords": [