diff --git a/lib/DBSQLClient.ts b/lib/DBSQLClient.ts index a40c740e..e500f3b7 100644 --- a/lib/DBSQLClient.ts +++ b/lib/DBSQLClient.ts @@ -21,6 +21,9 @@ import PlainHttpAuthentication from './connection/auth/PlainHttpAuthentication'; import IDBSQLLogger, { LogLevel } from './contracts/IDBSQLLogger'; import DBSQLLogger from './DBSQLLogger'; +import RestClient from './rest/RestClient'; +import RestDriver from './rest/RestDriver'; + function prependSlash(str: string): string { if (str.length > 0 && str.charAt(0) !== '/') { return `/${str}`; @@ -42,7 +45,8 @@ function getInitialNamespaceOptions(catalogName?: string, schemaName?: string) { } export default class DBSQLClient extends EventEmitter implements IDBSQLClient { - private client: TCLIService.Client | null; + private client: RestClient | null; + private driver: RestDriver | null; private connection: IThriftConnection | null; @@ -63,6 +67,7 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient { this.statusFactory = new StatusFactory(); this.logger = options?.logger || new DBSQLLogger(); this.client = null; + this.driver = null; this.connection = null; this.logger.log(LogLevel.info, 'Created DBSQLClient'); } @@ -99,7 +104,19 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient { this.connection = await this.connectionProvider.connect(this.getConnectionOptions(options), this.authProvider); - this.client = this.thrift.createClient(TCLIService, this.connection.getConnection()); + // this.client = this.thrift.createClient(TCLIService, this.connection.getConnection()); + this.client = new RestClient({ + host: options.host, + warehouseId: + options.path + .split('/') + .filter((s) => s !== '') + .pop() || '', // take last path fragment + headers: { + Authorization: `Basic ${Buffer.from(`token:${options.token}`).toString('base64')}`, + }, + }); + this.driver = new RestDriver(this.client); this.connection.getConnection().on('error', (error: Error) => { // Error.stack already contains error type and message, so log stack if available, @@ -141,11 +158,11 @@ export default class DBSQLClient extends EventEmitter implements IDBSQLClient { * const session = await client.openSession(); */ openSession(request: OpenSessionRequest = {}): Promise { - if (!this.connection?.isConnected()) { + if (!this.connection?.isConnected() || !this.driver) { return Promise.reject(new HiveDriverError('DBSQLClient: connection is lost')); } - const driver = new HiveDriver(this.getClient()); + const driver = this.driver; return driver .openSession({ diff --git a/lib/DBSQLOperation/CompleteOperationHelper.ts b/lib/DBSQLOperation/CompleteOperationHelper.ts index d24b24be..7e1a0875 100644 --- a/lib/DBSQLOperation/CompleteOperationHelper.ts +++ b/lib/DBSQLOperation/CompleteOperationHelper.ts @@ -3,8 +3,10 @@ import HiveDriver from '../hive/HiveDriver'; import StatusFactory from '../factory/StatusFactory'; import Status from '../dto/Status'; +import RestDriver from '../rest/RestDriver'; + export default class CompleteOperationHelper { - private driver: HiveDriver; + private driver: RestDriver; private operationHandle: TOperationHandle; @@ -14,7 +16,7 @@ export default class CompleteOperationHelper { cancelled: boolean = false; - constructor(driver: HiveDriver, operationHandle: TOperationHandle, closeOperation?: TCloseOperationResp) { + constructor(driver: RestDriver, operationHandle: TOperationHandle, closeOperation?: TCloseOperationResp) { this.driver = driver; this.operationHandle = operationHandle; diff --git a/lib/DBSQLOperation/FetchResultsHelper.ts b/lib/DBSQLOperation/FetchResultsHelper.ts index 13dfc6cc..d0a99567 100644 --- a/lib/DBSQLOperation/FetchResultsHelper.ts +++ b/lib/DBSQLOperation/FetchResultsHelper.ts @@ -10,34 +10,14 @@ import { ColumnCode, FetchType, Int64 } from '../hive/Types'; import HiveDriver from '../hive/HiveDriver'; import StatusFactory from '../factory/StatusFactory'; -function checkIfOperationHasMoreRows(response: TFetchResultsResp): boolean { - if (response.hasMoreRows) { - return true; - } - - const columns = response.results?.columns || []; - - if (columns.length === 0) { - return false; - } - - const column: TColumn = columns[0]; +import RestDriver from '../rest/RestDriver'; - const columnValue = - column[ColumnCode.binaryVal] || - column[ColumnCode.boolVal] || - column[ColumnCode.byteVal] || - column[ColumnCode.doubleVal] || - column[ColumnCode.i16Val] || - column[ColumnCode.i32Val] || - column[ColumnCode.i64Val] || - column[ColumnCode.stringVal]; - - return (columnValue?.values?.length || 0) > 0; +function checkIfOperationHasMoreRows(response: TFetchResultsResp): boolean { + return response.hasMoreRows || false; } export default class FetchResultsHelper { - private driver: HiveDriver; + private driver: RestDriver; private operationHandle: TOperationHandle; @@ -50,7 +30,7 @@ export default class FetchResultsHelper { hasMoreRows: boolean = false; constructor( - driver: HiveDriver, + driver: RestDriver, operationHandle: TOperationHandle, prefetchedResults: Array, ) { diff --git a/lib/DBSQLOperation/OperationStatusHelper.ts b/lib/DBSQLOperation/OperationStatusHelper.ts index 45aa74ec..f3ee3dab 100644 --- a/lib/DBSQLOperation/OperationStatusHelper.ts +++ b/lib/DBSQLOperation/OperationStatusHelper.ts @@ -4,6 +4,8 @@ import StatusFactory from '../factory/StatusFactory'; import { WaitUntilReadyOptions } from '../contracts/IOperation'; import OperationStateError from '../errors/OperationStateError'; +import RestDriver from '../rest/RestDriver'; + async function delay(ms?: number): Promise { return new Promise((resolve) => { setTimeout(() => { @@ -13,7 +15,7 @@ async function delay(ms?: number): Promise { } export default class OperationStatusHelper { - private driver: HiveDriver; + private driver: RestDriver; private operationHandle: TOperationHandle; @@ -27,7 +29,7 @@ export default class OperationStatusHelper { hasResultSet: boolean = false; - constructor(driver: HiveDriver, operationHandle: TOperationHandle, operationStatus?: TGetOperationStatusResp) { + constructor(driver: RestDriver, operationHandle: TOperationHandle, operationStatus?: TGetOperationStatusResp) { this.driver = driver; this.operationHandle = operationHandle; this.hasResultSet = operationHandle.hasResultSet; diff --git a/lib/DBSQLOperation/SchemaHelper.ts b/lib/DBSQLOperation/SchemaHelper.ts index 0bc8906a..92b75fd2 100644 --- a/lib/DBSQLOperation/SchemaHelper.ts +++ b/lib/DBSQLOperation/SchemaHelper.ts @@ -7,8 +7,10 @@ import ArrowResult from '../result/ArrowResult'; import HiveDriverError from '../errors/HiveDriverError'; import { definedOrError } from '../utils'; +import RestDriver from '../rest/RestDriver'; + export default class SchemaHelper { - private driver: HiveDriver; + private driver: RestDriver; private operationHandle: TOperationHandle; @@ -16,7 +18,7 @@ export default class SchemaHelper { private metadata?: TGetResultSetMetadataResp; - constructor(driver: HiveDriver, operationHandle: TOperationHandle, metadata?: TGetResultSetMetadataResp) { + constructor(driver: RestDriver, operationHandle: TOperationHandle, metadata?: TGetResultSetMetadataResp) { this.driver = driver; this.operationHandle = operationHandle; this.metadata = metadata; diff --git a/lib/DBSQLOperation/index.ts b/lib/DBSQLOperation/index.ts index 92f286f0..355775a1 100644 --- a/lib/DBSQLOperation/index.ts +++ b/lib/DBSQLOperation/index.ts @@ -15,10 +15,12 @@ import FetchResultsHelper from './FetchResultsHelper'; import CompleteOperationHelper from './CompleteOperationHelper'; import IDBSQLLogger, { LogLevel } from '../contracts/IDBSQLLogger'; +import RestDriver from '../rest/RestDriver'; + const defaultMaxRows = 100000; export default class DBSQLOperation implements IOperation { - private driver: HiveDriver; + private driver: RestDriver; private operationHandle: TOperationHandle; @@ -33,7 +35,7 @@ export default class DBSQLOperation implements IOperation { private _completeOperation: CompleteOperationHelper; constructor( - driver: HiveDriver, + driver: RestDriver, operationHandle: TOperationHandle, logger: IDBSQLLogger, directResults?: TSparkDirectResults, diff --git a/lib/DBSQLSession.ts b/lib/DBSQLSession.ts index 512433b1..191395f6 100644 --- a/lib/DBSQLSession.ts +++ b/lib/DBSQLSession.ts @@ -27,6 +27,7 @@ import StatusFactory from './factory/StatusFactory'; import InfoValue from './dto/InfoValue'; import { definedOrError } from './utils'; import IDBSQLLogger, { LogLevel } from './contracts/IDBSQLLogger'; +import RestDriver from './rest/RestDriver'; import globalConfig from './globalConfig'; const defaultMaxRows = 100000; @@ -74,7 +75,7 @@ function getArrowOptions(): { } export default class DBSQLSession implements IDBSQLSession { - private driver: HiveDriver; + private driver: RestDriver; private sessionHandle: TSessionHandle; @@ -82,7 +83,7 @@ export default class DBSQLSession implements IDBSQLSession { private logger: IDBSQLLogger; - constructor(driver: HiveDriver, sessionHandle: TSessionHandle, logger: IDBSQLLogger) { + constructor(driver: RestDriver, sessionHandle: TSessionHandle, logger: IDBSQLLogger) { this.driver = driver; this.sessionHandle = sessionHandle; this.statusFactory = new StatusFactory(); diff --git a/lib/rest/RestClient.ts b/lib/rest/RestClient.ts new file mode 100644 index 00000000..f03f1bbf --- /dev/null +++ b/lib/rest/RestClient.ts @@ -0,0 +1,95 @@ +import fetch from 'node-fetch'; + +import { + ExecuteStatementRequest, + ExecuteStatementResponse, + CancelExecutionRequest, + GetStatementRequest, + GetStatementResponse, + GetStatementResultChunkNRequest, + ResultData, +} from './Types'; + +export interface RestClientOptions { + host: string; + warehouseId: string; + headers?: Record; +} + +enum HTTPMethod { + GET = 'GET', + POST = 'POST', +} + +export default class RestClient { + private readonly options: RestClientOptions; + + private async doRequest(method: string, path: string, payload: P): Promise { + const { host, headers } = this.options; + const response = await fetch(`https://${host}${path}`, { + method, + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...headers, + }, + body: method !== HTTPMethod.GET ? JSON.stringify(payload) : undefined, + }); + const result = await response.json(); + return result as R; + } + + constructor(options: RestClientOptions) { + this.options = options; + } + + public getWarehouseId() { + return this.options.warehouseId; + } + + public executeStatement(request: ExecuteStatementRequest) { + return this.doRequest( + HTTPMethod.POST, + '/api/2.0/sql/statements/', + request, + ); + } + + public cancelExecution(request: CancelExecutionRequest) { + return this.doRequest( + HTTPMethod.POST, + `/api/2.0/sql/statements/${request.statement_id}/cancel`, + request, + ); + } + + public getStatement(request: GetStatementRequest): Promise { + return this.doRequest( + HTTPMethod.GET, + `/api/2.0/sql/statements/${request.statement_id}`, + request, + ); + } + + public getStatementResultChunkN(request: GetStatementResultChunkNRequest): Promise { + return this.doRequest( + HTTPMethod.GET, + `/api/2.0/sql/statements/${request.statement_id}/result/chunks/${request.chunk_index}`, + request, + ); + } + + public getStatementResultChunk(internalLink: string): Promise { + return this.doRequest(HTTPMethod.GET, internalLink, undefined); + } + + public async fetchExternalLink(url: string): Promise { + const { host, headers } = this.options; + const response = await fetch(url, { + method: HTTPMethod.GET, + headers: {}, + }); + const result = await response.arrayBuffer(); + return Buffer.from(result); + } +} diff --git a/lib/rest/RestDriver.ts b/lib/rest/RestDriver.ts new file mode 100644 index 00000000..832f37c5 --- /dev/null +++ b/lib/rest/RestDriver.ts @@ -0,0 +1,419 @@ +import Int64 from 'node-int64'; + +import { + TCancelDelegationTokenReq, + TCancelDelegationTokenResp, + TCancelOperationReq, + TCancelOperationResp, + TCloseOperationReq, + TCloseOperationResp, + TCloseSessionReq, + TCloseSessionResp, + TColumn, + TColumnDesc, + TExecuteStatementReq, + TExecuteStatementResp, + TFetchResultsReq, + TFetchResultsResp, + TGetCatalogsReq, + TGetCatalogsResp, + TGetColumnsReq, + TGetColumnsResp, + TGetCrossReferenceReq, + TGetCrossReferenceResp, + TGetDelegationTokenReq, + TGetDelegationTokenResp, + TGetFunctionsReq, + TGetFunctionsResp, + TGetInfoReq, + TGetInfoResp, + TGetOperationStatusReq, + TGetOperationStatusResp, + TGetPrimaryKeysReq, + TGetPrimaryKeysResp, + TGetResultSetMetadataReq, + TGetResultSetMetadataResp, + TGetSchemasReq, + TGetSchemasResp, + TGetTablesReq, + TGetTablesResp, + TGetTableTypesReq, + TGetTableTypesResp, + TGetTypeInfoReq, + TGetTypeInfoResp, + THandleIdentifier, + TOpenSessionReq, + TOpenSessionResp, + TOperationHandle, + TOperationState, + TOperationType, + TProtocolVersion, + TRenewDelegationTokenReq, + TRenewDelegationTokenResp, + TRowSet, + TSessionHandle, + TSparkRowSetType, + TStatus, + TStatusCode, + TTypeId, +} from '../../thrift/TCLIService_types'; + +import RestClient from './RestClient'; +import { + ColumnInfoTypeName, + Disposition, + ExecuteStatementResponse, + Format, + ResultData, + ResultSchema, + StatementState, + StatementStatus, + TimeoutAction, +} from './Types'; + +class NotImplementedError extends Error { + constructor() { + super('RestDriver: Method not implemented'); + } +} + +function restOperationStateToThriftOperationState(state: StatementState): TOperationState { + switch (state) { + case StatementState.Canceled: + return TOperationState.CANCELED_STATE; + case StatementState.Closed: + return TOperationState.CLOSED_STATE; + case StatementState.Failed: + return TOperationState.ERROR_STATE; + case StatementState.Pending: + return TOperationState.PENDING_STATE; + case StatementState.Running: + return TOperationState.RUNNING_STATE; + case StatementState.Succeeded: + return TOperationState.FINISHED_STATE; + default: + return TOperationState.UKNOWN_STATE; + } +} + +function restTypeNameToThriftTypeId(typeName: ColumnInfoTypeName): TTypeId { + switch (typeName) { + case ColumnInfoTypeName.Array: + return TTypeId.ARRAY_TYPE; + case ColumnInfoTypeName.Binary: + return TTypeId.BINARY_TYPE; + case ColumnInfoTypeName.Boolean: + return TTypeId.BOOLEAN_TYPE; + case ColumnInfoTypeName.Byte: + return TTypeId.TINYINT_TYPE; // ?? + case ColumnInfoTypeName.Char: + return TTypeId.CHAR_TYPE; + case ColumnInfoTypeName.Date: + return TTypeId.DATE_TYPE; + case ColumnInfoTypeName.Decimal: + return TTypeId.DECIMAL_TYPE; + case ColumnInfoTypeName.Double: + return TTypeId.DOUBLE_TYPE; + case ColumnInfoTypeName.Float: + return TTypeId.FLOAT_TYPE; + case ColumnInfoTypeName.Int: + return TTypeId.INT_TYPE; + case ColumnInfoTypeName.Interval: + return TTypeId.INTERVAL_DAY_TIME_TYPE; // ?? + case ColumnInfoTypeName.Long: + return TTypeId.INT_TYPE; // ?? + case ColumnInfoTypeName.Map: + return TTypeId.MAP_TYPE; + case ColumnInfoTypeName.Null: + return TTypeId.NULL_TYPE; + case ColumnInfoTypeName.Short: + return TTypeId.SMALLINT_TYPE; // ?? + case ColumnInfoTypeName.String: + return TTypeId.STRING_TYPE; + case ColumnInfoTypeName.Struct: + return TTypeId.STRUCT_TYPE; + case ColumnInfoTypeName.Timestamp: + return TTypeId.TIMESTAMP_TYPE; + case ColumnInfoTypeName.UserDefinedType: + return TTypeId.USER_DEFINED_TYPE; + default: + return TTypeId.NULL_TYPE; // ?? + } +} + +export default class RestDriver { + private client: RestClient; + + // REST API doesn't support sessions; we emulate it, but only single session is supported in this PoC + private session?: TOpenSessionReq = undefined; + + // In this PoC we don't support concurrent queries, so just store currently processed query id to simplify things + private currentStatement?: ExecuteStatementResponse = undefined; + + private resultLinks: Array = []; + private nextChunkLinks: Array = []; + + constructor(client: RestClient) { + this.client = client; + } + + private processStatementStatus(status: StatementStatus) { + switch (status.state) { + case StatementState.Canceled: + case StatementState.Closed: + this.currentStatement = undefined; + return; + case StatementState.Failed: + throw new Error(`${status.error.error_code}: ${status.error.message}`); + default: + return; + } + } + + private processResultData(result?: ResultData) { + if (!result) return; + + result.external_links?.forEach((link) => { + if (link.external_link) { + this.resultLinks.push(link.external_link); + } + if (link.next_chunk_internal_link) { + this.nextChunkLinks.push(link.next_chunk_internal_link); + } + }); + } + + private isPendingOrRunning() { + const state = this.currentStatement?.status?.state; + return state === StatementState.Pending || state === StatementState.Running; + } + + private hasResultSet() { + if (this.isPendingOrRunning()) return true; + return this.resultLinks.length > 0 || this.nextChunkLinks.length > 0; + } + + async openSession(request: TOpenSessionReq): Promise { + this.session = request; + return new TOpenSessionResp({ + status: new TStatus({ statusCode: TStatusCode.SUCCESS_STATUS }), + serverProtocolVersion: request.client_protocol || TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V6, + sessionHandle: new TSessionHandle({ + sessionId: new THandleIdentifier({ + guid: Buffer.alloc(16), + secret: Buffer.alloc(0), + }), + }), + }); + } + + async closeSession(request: TCloseSessionReq): Promise { + this.session = undefined; + return new TCloseSessionResp({ + status: new TStatus({ statusCode: TStatusCode.SUCCESS_STATUS }), + }); + } + + async executeStatement(request: TExecuteStatementReq): Promise { + this.currentStatement = await this.client.executeStatement({ + catalog: this.session?.initialNamespace?.catalogName, + schema: this.session?.initialNamespace?.schemaName, + disposition: Disposition.ExternalLinks, + format: Format.ArrowStream, + on_wait_timeout: TimeoutAction.Continue, + wait_timeout: '5s', + warehouse_id: this.client.getWarehouseId(), + statement: request.statement, + }); + + this.processStatementStatus(this.currentStatement.status); + + if (!this.currentStatement) { + return new TExecuteStatementResp({ + status: new TStatus({ statusCode: TStatusCode.INVALID_HANDLE_STATUS }), + }); + } + + this.processResultData(this.currentStatement?.result); + + return new TExecuteStatementResp({ + status: new TStatus({ statusCode: TStatusCode.SUCCESS_STATUS }), + operationHandle: new TOperationHandle({ + operationId: new THandleIdentifier({ + guid: Buffer.alloc(16), + secret: Buffer.alloc(0), + }), + operationType: TOperationType.EXECUTE_STATEMENT, + hasResultSet: this.hasResultSet(), + }), + }); + } + + async getResultSetMetadata(request: TGetResultSetMetadataReq): Promise { + if (!this.currentStatement) { + return new TGetResultSetMetadataResp({ + status: new TStatus({ statusCode: TStatusCode.ERROR_STATUS, errorMessage: 'Invalid handle' }), + }); + } + + const columns: TColumnDesc[] = []; + this.currentStatement.manifest?.schema?.columns?.forEach((column) => { + columns.push({ + columnName: column.name, + position: column.position + 1, // thrift columns are 1-based, rest columns are 0-based + typeDesc: { + types: [ + { + primitiveEntry: { + type: restTypeNameToThriftTypeId(column.type_name), + }, + }, + ], + }, + }); + }); + + return new TGetResultSetMetadataResp({ + status: new TStatus({ statusCode: TStatusCode.SUCCESS_STATUS }), + resultFormat: TSparkRowSetType.ARROW_BASED_SET, + schema: { columns }, + arrowSchema: Buffer.alloc(0), + }); + } + + async fetchResults(request: TFetchResultsReq): Promise { + if (!this.currentStatement) { + return new TFetchResultsResp({ + status: new TStatus({ statusCode: TStatusCode.ERROR_STATUS, errorMessage: 'Invalid handle' }), + }); + } + + if (this.resultLinks.length === 0) { + const nextChunkLink = this.nextChunkLinks.pop(); + if (nextChunkLink) { + const result = await this.client.getStatementResultChunk(nextChunkLink); + this.processResultData(result); + } + } + + const resultLink = this.resultLinks.pop(); + if (resultLink) { + const arrowData = await this.client.fetchExternalLink(resultLink); + return new TFetchResultsResp({ + status: new TStatus({ statusCode: TStatusCode.SUCCESS_STATUS }), + hasMoreRows: this.hasResultSet(), + results: { + startRowOffset: new Int64(0), + rows: [], + arrowBatches: [ + { + batch: arrowData, + rowCount: new Int64(0), + }, + ], + }, + }); + } + + return new TFetchResultsResp({ + status: new TStatus({ statusCode: TStatusCode.ERROR_STATUS, errorMessage: 'No more data' }), + }); + } + + async getInfo(request: TGetInfoReq): Promise { + throw new NotImplementedError(); + } + + async getTypeInfo(request: TGetTypeInfoReq): Promise { + throw new NotImplementedError(); + } + + async getCatalogs(request: TGetCatalogsReq): Promise { + throw new NotImplementedError(); + } + + async getSchemas(request: TGetSchemasReq): Promise { + throw new NotImplementedError(); + } + + async getTables(request: TGetTablesReq): Promise { + throw new NotImplementedError(); + } + + async getTableTypes(request: TGetTableTypesReq): Promise { + throw new NotImplementedError(); + } + + async getColumns(request: TGetColumnsReq): Promise { + throw new NotImplementedError(); + } + + async getFunctions(request: TGetFunctionsReq): Promise { + throw new NotImplementedError(); + } + + async getPrimaryKeys(request: TGetPrimaryKeysReq): Promise { + throw new NotImplementedError(); + } + + async getCrossReference(request: TGetCrossReferenceReq): Promise { + throw new NotImplementedError(); + } + + async getOperationStatus(request: TGetOperationStatusReq): Promise { + if (this.currentStatement && this.isPendingOrRunning()) { + const response = await this.client.getStatement({ statement_id: this.currentStatement.statement_id }); + this.currentStatement.status = response.status; + this.currentStatement.manifest = response.manifest; + this.currentStatement.result = response.result; + + this.processStatementStatus(this.currentStatement.status); + this.processResultData(this.currentStatement?.result); + } + + if (!this.currentStatement) { + return new TGetOperationStatusResp({ + status: new TStatus({ statusCode: TStatusCode.ERROR_STATUS, errorMessage: 'Invalid handle' }), + }); + } + + return new TGetOperationStatusResp({ + status: new TStatus({ statusCode: TStatusCode.SUCCESS_STATUS }), + operationState: restOperationStateToThriftOperationState(this.currentStatement.status.state), + errorCode: 0, + errorMessage: `${this.currentStatement.status.error?.error_code}: ${this.currentStatement.status.error?.message}`, + hasResultSet: this.hasResultSet(), + }); + } + + async cancelOperation(request: TCancelOperationReq): Promise { + if (this.currentStatement) { + await this.client.cancelExecution({ + statement_id: this.currentStatement.statement_id, + }); + this.currentStatement = undefined; + } + return new TCancelOperationResp({ + status: new TStatus({ statusCode: TStatusCode.SUCCESS_STATUS }), + }); + } + + async closeOperation(request: TCloseOperationReq): Promise { + this.currentStatement = undefined; + return new TCloseOperationResp({ + status: new TStatus({ statusCode: TStatusCode.SUCCESS_STATUS }), + }); + } + + async getDelegationToken(request: TGetDelegationTokenReq): Promise { + throw new NotImplementedError(); + } + + async cancelDelegationToken(request: TCancelDelegationTokenReq): Promise { + throw new NotImplementedError(); + } + + async renewDelegationToken(request: TRenewDelegationTokenReq): Promise { + throw new NotImplementedError(); + } +} diff --git a/lib/rest/Types.ts b/lib/rest/Types.ts new file mode 100644 index 00000000..a283b807 --- /dev/null +++ b/lib/rest/Types.ts @@ -0,0 +1,165 @@ +export enum Disposition { + ExternalLinks = 'EXTERNAL_LINKS', + Inline = 'INLINE', +} + +export enum Format { + ArrowStream = 'ARROW_STREAM', + JsonArray = 'JSON_ARRAY', +} + +export enum TimeoutAction { + Cancel = 'CANCEL', + Continue = 'CONTINUE', +} + +export interface ExecuteStatementRequest { + catalog?: string; + disposition: Disposition; + format: Format; + on_wait_timeout: TimeoutAction; + schema?: string; + statement: string; + wait_timeout: string; + warehouse_id: string; +} + +export interface ExecuteStatementResponse { + manifest?: ResultManifest; + result?: ResultData; + statement_id: string; + status: StatementStatus; +} + +export enum StatementState { + Canceled = 'CANCELED', + Closed = 'CLOSED', + Failed = 'FAILED', + Pending = 'PENDING', + Running = 'RUNNING', + Succeeded = 'SUCCEEDED', +} + +export interface StatementStatus { + error: ServiceError; + state: StatementState; +} + +export interface ServiceError { + error_code: ServiceErrorCode; + message: string; +} + +export enum ServiceErrorCode { + Aborted = 'ABORTED', + AlreadyExists = 'ALREADY_EXISTS', + BadRequest = 'BAD_REQUEST', + Cancelled = 'CANCELLED', + DeadlineExceeded = 'DEADLINE_EXCEEDED', + InternalError = 'INTERNAL_ERROR', + IoError = 'IO_ERROR', + NotFound = 'NOT_FOUND', + ResourceExhausted = 'RESOURCE_EXHAUSTED', + ServiceUnderMaintenance = 'SERVICE_UNDER_MAINTENANCE', + TemporarilyUnavailable = 'TEMPORARILY_UNAVAILABLE', + Unauthenticated = 'UNAUTHENTICATED', + Unknown = 'UNKNOWN', + WorkspaceTemporarilyUnavailable = 'WORKSPACE_TEMPORARILY_UNAVAILABLE', +} + +export interface ResultData { + byte_count: number; // int64 + chunk_index: number; + data_array: string[][]; + external_links?: ExternalLink[]; + next_chunk_index?: number; + next_chunk_internal_link?: string; + row_count: number; // int64 + row_offset: number; // int64 +} + +interface ExternalLink { + byte_count: number; // int64 + chunk_index: number; + expiration: string; + external_link: string; + next_chunk_index: number; + next_chunk_internal_link: string; + row_count: number; // int64 + row_offset: number; // int64 +} + +export interface ResultManifest { + chunks: ChunkInfo[]; + format: Format; + schema: ResultSchema; + total_byte_count: number; // int64 + total_chunk_count: number; + total_row_count: number; // int64 +} + +export interface ChunkInfo { + byte_count: number; // int64 + chunk_index: number; + next_chunk_index: number; + next_chunk_internal_link: string; + row_count: number; // int64 + row_offset: number; // int64 +} + +export interface ResultSchema { + column_count: number; + columns: ColumnInfo[]; +} + +export interface ColumnInfo { + name: string; + position: number; + type_interval_type: string; + type_name: ColumnInfoTypeName; + type_precision: number; + type_scale: number; + type_text: string; +} + +export enum ColumnInfoTypeName { + Array = 'ARRAY', + Binary = 'BINARY', + Boolean = 'BOOLEAN', + Byte = 'BYTE', + Char = 'CHAR', + Date = 'DATE', + Decimal = 'DECIMAL', + Double = 'DOUBLE', + Float = 'FLOAT', + Int = 'INT', + Interval = 'INTERVAL', + Long = 'LONG', + Map = 'MAP', + Null = 'NULL', + Short = 'SHORT', + String = 'STRING', + Struct = 'STRUCT', + Timestamp = 'TIMESTAMP', + UserDefinedType = 'USER_DEFINED_TYPE', +} + +export interface CancelExecutionRequest { + statement_id: string; +} + +export interface GetStatementRequest { + statement_id: string; +} + +export interface GetStatementResponse { + manifest?: ResultManifest; + result?: ResultData; + statement_id: string; + status: StatementStatus; +} + +export interface GetStatementResultChunkNRequest { + chunk_index: number; + statement_id: string; +} diff --git a/package-lock.json b/package-lock.json index bcf67e0c..8ddfeb9a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "dependencies": { "apache-arrow": "^10.0.1", "commander": "^9.3.0", + "node-fetch": "^2.6.9", "node-int64": "^0.4.0", "patch-package": "^6.5.0", "thrift": "^0.16.0", @@ -23,6 +24,7 @@ }, "devDependencies": { "@types/node": "^18.11.9", + "@types/node-fetch": "^2.6.2", "@types/node-int64": "^0.4.29", "@types/thrift": "^0.10.11", "@types/uuid": "^8.3.4", @@ -811,6 +813,16 @@ "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==", "dev": true }, + "node_modules/@types/node-fetch": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz", + "integrity": "sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==", + "dev": true, + "dependencies": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, "node_modules/@types/node-int64": { "version": "0.4.29", "resolved": "https://registry.npmjs.org/@types/node-int64/-/node-int64-0.4.29.tgz", @@ -1313,6 +1325,12 @@ "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==", "inBundle": true }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true + }, "node_modules/axe-core": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.4.3.tgz", @@ -1626,6 +1644,18 @@ "text-hex": "1.0.x" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/command-line-args": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", @@ -1879,6 +1909,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/diff": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", @@ -2693,6 +2732,20 @@ "node": ">=8.0.0" } }, + "node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fromentries": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", @@ -3814,6 +3867,27 @@ "node": ">=8.6" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -3960,6 +4034,25 @@ "path-to-regexp": "^1.7.0" } }, + "node_modules/node-fetch": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -5350,6 +5443,11 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "node_modules/triple-beam": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", @@ -5549,6 +5647,20 @@ "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", "dev": true }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -6410,6 +6522,16 @@ "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==", "dev": true }, + "@types/node-fetch": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz", + "integrity": "sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==", + "dev": true, + "requires": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, "@types/node-int64": { "version": "0.4.29", "resolved": "https://registry.npmjs.org/@types/node-int64/-/node-int64-0.4.29.tgz", @@ -6758,6 +6880,12 @@ "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true + }, "axe-core": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.4.3.tgz", @@ -6995,6 +7123,15 @@ "text-hex": "1.0.x" } }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "requires": { + "delayed-stream": "~1.0.0" + } + }, "command-line-args": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", @@ -7186,6 +7323,12 @@ "object-keys": "^1.1.1" } }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true + }, "diff": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", @@ -7816,6 +7959,17 @@ "signal-exit": "^3.0.2" } }, + "form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "fromentries": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", @@ -8621,6 +8775,21 @@ "picomatch": "^2.3.1" } }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "requires": { + "mime-db": "1.52.0" + } + }, "minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -8740,6 +8909,14 @@ "path-to-regexp": "^1.7.0" } }, + "node-fetch": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -9754,6 +9931,11 @@ "is-number": "^7.0.0" } }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "triple-beam": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", @@ -9896,6 +10078,20 @@ "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", "dev": true }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index e434365a..3ba8732c 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "license": "Apache 2.0", "devDependencies": { "@types/node": "^18.11.9", + "@types/node-fetch": "^2.6.2", "@types/node-int64": "^0.4.29", "@types/thrift": "^0.10.11", "@types/uuid": "^8.3.4", @@ -72,6 +73,7 @@ "dependencies": { "apache-arrow": "^10.0.1", "commander": "^9.3.0", + "node-fetch": "^2.6.9", "node-int64": "^0.4.0", "patch-package": "^6.5.0", "thrift": "^0.16.0",