diff --git a/.eslintrc b/.eslintrc index bad5041..5e007f0 100644 --- a/.eslintrc +++ b/.eslintrc @@ -5,6 +5,7 @@ "NodeJS": true }, "rules": { + "jsdoc/no-defaults": 0, "jsdoc/tag-lines": [ "error", "never", diff --git a/doc/api.md b/doc/api.md index 7f8ca22..ab0a65e 100644 --- a/doc/api.md +++ b/doc/api.md @@ -68,7 +68,7 @@ Cloud Files Abstraction * *[.read(filePath, [options])](#Files+read) ⇒ Promise.<Buffer>* * *[.write(filePath, content)](#Files+write) ⇒ Promise.<number>* * *[.getProperties(filePath)](#Files+getProperties) ⇒ [Promise.<RemoteFileProperties>](#RemoteFileProperties)* - * *[.copy(srcPath, destPath, [options])](#Files+copy) ⇒ Promise.<Object.<string, string>>* + * *[.copy(srcPath, destPath, [options])](#Files+copy) ⇒ Promise.<{key: string}>* * *[.generatePresignURL(filePath, options)](#Files+generatePresignURL) ⇒ Promise.<string>* * *[.revokeAllPresignURLs()](#Files+revokeAllPresignURLs) ⇒ void* @@ -185,7 +185,7 @@ Reads properties of a file or directory -### *files.copy(srcPath, destPath, [options]) ⇒ Promise.<Object.<string, string>>* +### *files.copy(srcPath, destPath, [options]) ⇒ Promise.<{key: string}>* ***NodeJS only (streams + fs).*** A utility function to copy files and directories across remote and local Files. This @@ -212,7 +212,7 @@ Rules for copy files are: - not supported **Kind**: instance method of [Files](#Files) -**Returns**: Promise.<Object.<string, string>> - returns a promise resolving to an object +**Returns**: Promise.<{key: string}> - returns a promise resolving to an object containing all copied files from src to dest `{ srcFilePath: destFilePath }` | Param | Type | Default | Description | diff --git a/lib/Files.js b/lib/Files.js index f1d0cc3..ee136c0 100644 --- a/lib/Files.js +++ b/lib/Files.js @@ -640,7 +640,7 @@ class Files { * @param {Function} [options.progressCallback] a function that will be called every * time the operation completes on a single file,the srcPath and destPath to the copied * file are passed as argument to the callback `progressCallback(srcPath, destPath)` - * @returns {Promise>} returns a promise resolving to an object + * @returns {Promise<{key: string}>} returns a promise resolving to an object * containing all copied files from src to dest `{ srcFilePath: destFilePath }` * @memberof Files */ diff --git a/lib/impl/AzureBlobFiles.js b/lib/impl/AzureBlobFiles.js index 676d847..4277299 100644 --- a/lib/impl/AzureBlobFiles.js +++ b/lib/impl/AzureBlobFiles.js @@ -216,64 +216,31 @@ class AzureBlobFiles extends Files { * @memberof AzureBlobFiles * @private */ - async _setAccessPolicy () { + async _setAccessPolicies () { const id = uuidv4() // set access policy with new id and without any permissions await this.containerClientPrivate.setAccessPolicy(undefined, [{ id, accessPolicy: { permission: '' } }]) + await this.containerClientPublic.setAccessPolicy('blob', [{ id, accessPolicy: { permission: '' } }]) } - /** [Internal] get Access policy ID + /** [Internal] get Access policy IDs * @memberof AzureBlobFiles + * @returns {{publicAccessPolicyId, privateAccessPolicyId}} array of access policy IDs * @private */ - async _getAccessPolicy () { - // use API call as this.containerClientPrivate.getAccessPolicy calls fails for policy with empty permissions - const index = this.containerClientPrivate.url.lastIndexOf('/') - const containerName = this.containerClientPrivate.url.substring(index + 1, this.containerClientPrivate.url.length) - - const resource = '/' + this.credentials.storageAccount + '/' + containerName + '\ncomp:acl\nrestype:container' - const date = new Date().toUTCString() - const sign = this._signRequest('GET', resource, date) - - const reqHeaders = { - 'x-ms-date': date, - 'x-ms-version': '2019-02-02', - authorization: 'SharedKey ' + this.credentials.storageAccount + ':' + sign - } - const url = this.containerClientPrivate.url + '?restype=container&comp=acl' - const res = await fetch(url, { method: 'GET', headers: reqHeaders }) - - const acl = await res.text() - const aclObj = xmlJS.xml2js(acl) - let id - if (aclObj.elements) { - const signedIdentifiers = aclObj.elements[0] - if (signedIdentifiers.elements) { - if (signedIdentifiers.elements.length > 1) { - const msg = 'Container has one or more custom policies defined. Either remove all custom policies or use another container.' - logAndThrow(new codes.ERROR_INIT_FAILURE({ messageValues: [msg], sdkDetails: {} })) - } - const signedIdentifier = signedIdentifiers.elements[0].elements - - signedIdentifier.forEach(function (val, index, arr) { - if (val.name === 'Id') { - id = val.elements[0].text - return id - } - }) - } + async _getAccessPolicyIdentifiers () { + return { + privateAccessPolicyId: await _getAccessPolicyID( + this.credentials.storageAccount, + this.credentials.storageAccessKey, + this.containerClientPrivate + ), + publicAccessPolicyId: await _getAccessPolicyID( + this.credentials.storageAccount, + this.credentials.storageAccessKey, + this.containerClientPublic + ) } - return id - } - - /** [Internal] Sign the given request - * @memberof AzureBlobFiles - * @private - */ - _signRequest (method, resource, date) { - const canonicalHeaders = 'x-ms-date:' + date + '\n' + 'x-ms-version:2019-02-02' - const stringToSign = method + '\n\n\n\n\n\n\n\n\n\n\n\n' + canonicalHeaders + '\n' + resource - return crypto.createHmac('sha256', Buffer.from(this.credentials.storageAccessKey, 'base64')).update(stringToSign, 'utf8').digest('base64') } /** @@ -282,19 +249,19 @@ class AzureBlobFiles extends Files { * @private */ async _addAccessPolicyIfNotExists () { - const identifier = await this._getAccessPolicy() - if (identifier) { - logger.debug('found access policy with identifier ' + identifier) + const pols = await this._getAccessPolicyIdentifiers() + if (pols.privateAccessPolicyId && pols.publicAccessPolicyId) { + logger.debug(`found access policies ${JSON.stringify(pols)}`) if (this.hasOwnCredentials) { // check if identifier is custom or not - if (this._isCustomPolicy(identifier)) { - const msg = 'Container has one or more custom policies defined. Either remove all custom policies or use another container.' + if (this._isCustomPolicy(pols.privateAccessPolicyId) || this._isCustomPolicy(pols.publicAccessPolicyId)) { + const msg = 'Custom access policies are defined. Please remove all access policies or use another container.' logAndThrow(new codes.ERROR_INIT_FAILURE({ messageValues: [msg], sdkDetails: {} })) } } } else { logger.debug('adding default access policy') - await this._setAccessPolicy() + await this._setAccessPolicies() // reset both policies } } @@ -583,17 +550,25 @@ class AzureBlobFiles extends Files { } const sharedKeyCredential = new azure.StorageSharedKeyCredential(this.credentials.storageAccount, this.credentials.storageAccessKey) - const containerName = this.credentials.containerName + + let containerName = this.credentials.containerName + let publicAccess = false + if (params.blobName.startsWith('public/')) { + // public blobs are stored on the public container + containerName = containerName + '-public' + publicAccess = true + } + // generate SAS token const expiryTime = new Date(Date.now() + (1000 * params.expiryInSeconds)) - const identifier = await this._getAccessPolicy() + const { privateAccessPolicyId, publicAccessPolicyId } = await this._getAccessPolicyIdentifiers() const permissions = azure.BlobSASPermissions.parse(params.permissions) const commonSasParams = { permissions: permissions.toString(), expiresOn: expiryTime, blobName: params.blobName, - identifier + identifier: publicAccess ? publicAccessPolicyId : privateAccessPolicyId } const sasQueryParamsPrivate = azure.generateBlobSASQueryParameters({ ...commonSasParams, containerName }, sharedKeyCredential) @@ -611,7 +586,7 @@ class AzureBlobFiles extends Files { const msg = 'revokeAllPresignURLs is not supported with Azure Container SAS credentials, please initialize the SDK with Azure storage account credentials instead' logAndThrow(new codes.ERROR_UNSUPPORTED_OPERATION({ messageValues: [msg], sdkDetails: {} })) } - await this._setAccessPolicy() + await this._setAccessPolicies() } /** @@ -638,4 +613,62 @@ class AzureBlobFiles extends Files { } } +/** [Internal] get Access policy ID + * @memberof AzureBlobFiles + * @param {string} storageAccount azure storage account name + * @param {string} storageAccessKey azure storage access key + * @param {azure.ContainerClient} containerClient container client + * @returns {string} access policy ID + * @private + */ +async function _getAccessPolicyID (storageAccount, storageAccessKey, containerClient) { + // use API call as containerClient.getAccessPolicy calls fails for policy with empty permissions + const index = containerClient.url.lastIndexOf('/') + const containerName = containerClient.url.substring(index + 1, containerClient.url.length) + + const resource = '/' + storageAccount + '/' + containerName + '\ncomp:acl\nrestype:container' + const date = new Date().toUTCString() + const sign = _signRequest('GET', resource, date, storageAccessKey) + + const reqHeaders = { + 'x-ms-date': date, + 'x-ms-version': '2019-02-02', + authorization: 'SharedKey ' + storageAccount + ':' + sign + } + const url = containerClient.url + '?restype=container&comp=acl' + const res = await fetch(url, { method: 'GET', headers: reqHeaders }) + + const acl = await res.text() + const aclObj = xmlJS.xml2js(acl) + let id + if (aclObj.elements) { + const signedIdentifiers = aclObj.elements[0] + if (signedIdentifiers.elements) { + if (signedIdentifiers.elements.length > 1) { + const msg = 'Container has more than one access policies defined. Please remove all custom access policies or use another container.' + logAndThrow(new codes.ERROR_INIT_FAILURE({ messageValues: [msg], sdkDetails: {} })) + } + const signedIdentifier = signedIdentifiers.elements[0].elements + + signedIdentifier.forEach(function (val, index, arr) { + if (val.name === 'Id') { + id = val.elements[0].text + return id + } + }) + } + } + return id +} + +/** [Internal] Sign the given request + * @memberof AzureBlobFiles + * @private + */ +function _signRequest (method, resource, date, storageAccessKey) { + const canonicalHeaders = 'x-ms-date:' + date + '\n' + 'x-ms-version:2019-02-02' + const stringToSign = method + '\n\n\n\n\n\n\n\n\n\n\n\n' + canonicalHeaders + '\n' + resource + return crypto.createHmac('sha256', Buffer.from(storageAccessKey, 'base64')).update(stringToSign, 'utf8').digest('base64') +} + module.exports = { AzureBlobFiles } diff --git a/package.json b/package.json index be90a68..e0988e7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@adobe/aio-lib-files", - "version": "4.1.0", + "version": "4.1.1", "description": "An abstraction on top of blob cloud storage exposing a file like API", "main": "lib/init.js", "directories": { @@ -33,23 +33,23 @@ "adobe-io" ], "devDependencies": { - "@adobe/eslint-config-aio-lib-config": "^2.0.1", + "@adobe/eslint-config-aio-lib-config": "^4.0.0", "@types/jest": "^29.5.2", - "eslint": "^8.47.0", + "eslint": "^8.57.1", "eslint-config-standard": "^17.1.0", - "eslint-plugin-import": "^2.28.0", - "eslint-plugin-jest": "^27.2.3", - "eslint-plugin-jsdoc": "^42.0.0", - "eslint-plugin-n": "^15.7", + "eslint-plugin-import": "^2.31.0", + "eslint-plugin-jest": "^27.9.0", + "eslint-plugin-jsdoc": "^48.11.0", + "eslint-plugin-n": "^15.7.0", "eslint-plugin-node": "^11.1.0", - "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-promise": "^6.6.0", "eslint-plugin-standard": "^5.0.0", - "jest": "^28", + "jest": "^29", "jest-junit": "^16.0.0", - "jsdoc-to-markdown": "^5.0.2", - "replace-in-file": "^6.1.0", + "jsdoc-to-markdown": "^9.1.1", + "replace-in-file": "^8.3.0", "tsd-jsdoc": "^2.4.0", - "typescript": "^4.9.5" + "typescript": "^5.7.3" }, "dependencies": { "@adobe/aio-lib-core-errors": "^4", @@ -63,7 +63,7 @@ "lodash.clonedeep": "^4.5.0", "mime-types": "^2.1.24", "node-fetch": "^2.6.0", - "upath": "^1.2.0", + "upath": "^2.0.1", "uuid": "^9.0.0", "xml-js": "^1.6.11" } diff --git a/test/impl/AzureBlobFiles.test.js b/test/impl/AzureBlobFiles.test.js index c3616ef..e11da02 100644 --- a/test/impl/AzureBlobFiles.test.js +++ b/test/impl/AzureBlobFiles.test.js @@ -69,19 +69,28 @@ const testWithProviderError = async (boundFunc, providerMock, errorDetails, file describe('init', () => { const mockContainerCreateIfNotExists = jest.fn() + const mockContainerCreateIfNotExistsPublic = jest.fn() const mockSetAccessPolicy = jest.fn() + const mockSetAccessPolicyPublic = jest.fn() const mockContainerClientInstance = { createIfNotExists: mockContainerCreateIfNotExists, setAccessPolicy: mockSetAccessPolicy, url: fakeSASCredentials.sasURLPrivate } + const mockContainerClientInstancePublic = { + createIfNotExists: mockContainerCreateIfNotExistsPublic, + setAccessPolicy: mockSetAccessPolicyPublic, + url: fakeSASCredentials.sasURLPublic + } beforeEach(async () => { mockContainerCreateIfNotExists.mockReset() + mockContainerCreateIfNotExistsPublic.mockReset() mockSetAccessPolicy.mockReset() + mockSetAccessPolicyPublic.mockReset() // mock needed for byo master keys init azure.BlobServiceClient.mockImplementation(() => { return { - getContainerClient: jest.fn(() => mockContainerClientInstance) + getContainerClient: jest.fn().mockImplementation((containerName) => containerName.includes('-public') ? mockContainerClientInstancePublic : mockContainerClientInstance) } }) // mock for sas creds init @@ -128,16 +137,19 @@ describe('init', () => { fetch.mockResolvedValue({ text: fakeResponse }) + fakeResponse.mockReset() fakeResponse.mockResolvedValue(fakeAccessPolicy) mockContainerCreateIfNotExists.mockResolvedValue('all good') + mockContainerCreateIfNotExistsPublic.mockResolvedValue('all good') }) test('when createIfNotExists containers does not fail', async () => { const files = await AzureBlobFiles.init(fakeUserCredentials) expect(files).toBeInstanceOf(AzureBlobFiles) - expect(mockContainerCreateIfNotExists).toHaveBeenCalledTimes(2) + expect(mockContainerCreateIfNotExists).toHaveBeenCalledTimes(1) + expect(mockContainerCreateIfNotExistsPublic).toHaveBeenCalledTimes(1) expect(mockContainerCreateIfNotExists).toHaveBeenCalledWith() - expect(mockContainerCreateIfNotExists).toHaveBeenCalledWith({ access: 'blob' }) + expect(mockContainerCreateIfNotExistsPublic).toHaveBeenCalledWith({ access: 'blob' }) checkInitDebugLogNoSecrets(fakeUserCredentials.storageAccessKey) }) @@ -157,6 +169,7 @@ describe('init', () => { const files = await AzureBlobFiles.init(fakeUserCredentials) expect(files).toBeInstanceOf(AzureBlobFiles) expect(mockSetAccessPolicy).toHaveBeenCalledWith(undefined, [{ id: 'fake-uuid', accessPolicy: { permission: '' } }]) + expect(mockSetAccessPolicyPublic).toHaveBeenCalledWith('blob', [{ id: 'fake-uuid', accessPolicy: { permission: '' } }]) // for public container }) test('when there is an empty access policy already defined', async () => { fakeResponse.mockResolvedValueOnce(fakeEmptyAccessPolicy) @@ -179,13 +192,13 @@ describe('init', () => { test('when multiple custom access policies are defined', async () => { fakeResponse.mockResolvedValueOnce(fakeMultipleAccessPolicy) - const errorMsg = '[FilesLib:ERROR_INIT_FAILURE] Container has one or more custom policies defined. Either remove all custom policies or use another container.' + const errorMsg = '[FilesLib:ERROR_INIT_FAILURE] Container has more than one access policies defined. Please remove all custom access policies or use another container.' await expect(AzureBlobFiles.init(fakeUserCredentials)).rejects.toThrow(errorMsg) }) test('when custom access policy is defined', async () => { fakeResponse.mockResolvedValueOnce(fakeCustomAccessPolicy) - const errorMsg = '[FilesLib:ERROR_INIT_FAILURE] Container has one or more custom policies defined. Either remove all custom policies or use another container.' + const errorMsg = '[FilesLib:ERROR_INIT_FAILURE] Custom access policies are defined. Please remove all access policies or use another container.' await expect(AzureBlobFiles.init(fakeUserCredentials)).rejects.toThrow(errorMsg) }) }) @@ -843,6 +856,26 @@ describe('_getPresignUrl', () => { const expectedUrl = 'https://fake.blob.core.windows.net/fake/fakesub/afile?fakeSAS' const url = await files._getPresignUrl('fakesub/afile', { expiryInSeconds: 60 }) expect(url).toEqual(expectedUrl) + expect(azure.generateBlobSASQueryParameters).toHaveBeenCalledWith(expect.objectContaining({ + containerName: fakeUserCredentials.containerName, + blobName: 'fakesub/afile', + permissions: 'fakePermissionStr' + }), expect.any(Object)) + }) + + test('_getPresignUrl for public file with correct options default permission own credentials', async () => { + const cleanUrl = 'https://fake.blob.core.windows.net/fake/fakesub/afile' + setMockBlobUrl(cleanUrl) + const files = await AzureBlobFiles.init(fakeUserCredentials) + + const expectedUrl = 'https://fake.blob.core.windows.net/fake/fakesub/afile?fakeSAS' + const url = await files._getPresignUrl('public/afile', { expiryInSeconds: 60 }) + expect(url).toEqual(expectedUrl) + expect(azure.generateBlobSASQueryParameters).toHaveBeenCalledWith(expect.objectContaining({ + containerName: fakeUserCredentials.containerName + '-public', + blobName: 'public/afile', + permissions: 'fakePermissionStr' + }), expect.any(Object)) }) test('_getPresignUrl with correct options explicit permission own credentials', async () => { @@ -865,8 +898,10 @@ describe('_getPresignUrl', () => { }) describe('_revokeAllPresignURLs', () => { + // for byo const mockSetAccessPolicy = jest.fn() - + const mockSetAccessPolicyPublic = jest.fn() + // for tvm const tvm = jest.fn() /** @type {AzureBlobFiles} */ let files @@ -886,10 +921,17 @@ describe('_revokeAllPresignURLs', () => { getBlockBlobClient: () => ({ }) } + const mockContainerClientInstancePublic = { + url: fakeSASCredentials.sasURLPublic, // some fake container url + createIfNotExists: () => Promise.resolve(), + setAccessPolicy: mockSetAccessPolicyPublic, + getBlockBlobClient: () => ({ + }) + } // mock for byo init azure.BlobServiceClient.mockImplementation(() => { return { - getContainerClient: jest.fn(() => mockContainerClientInstance) + getContainerClient: jest.fn(name => name.endsWith('-public') ? mockContainerClientInstancePublic : mockContainerClientInstance) } }) // mock for tvm init @@ -924,6 +966,7 @@ describe('_revokeAllPresignURLs', () => { await files._revokeAllPresignURLs() expect(tvm.revokePresignURLs).not.toHaveBeenCalled() expect(mockSetAccessPolicy).toHaveBeenCalledWith(undefined, [{ accessPolicy: { permission: '' }, id: 'fake-uuid' }]) + expect(mockSetAccessPolicyPublic).toHaveBeenCalledWith('blob', [{ accessPolicy: { permission: '' }, id: 'fake-uuid' }]) }) test('_revokeAllPresignURLs with own sas credentials', async () => {