feat: don't upload the same asset multiple times (#1011)

rix0rrr · web-flow · commit 35937b65ec5f · 2018-11-01T15:04:54.000+01:00
This change implements two asset bandwidth conservation measures: - If a lambda.AssetCode object is reused for multiple Lambdas, the same underyling Asset object will be reused (which leads to the asset data only being uploaded once). - If nonetheless multiple Asset objects are created for the same source data, the data will only be uploaded once and subsequently copied on the server-side to avoid the additional data transfer. Fixes #989.
diff --git a/packages/@aws-cdk/assets/test/integ.multi-assets.expected.json b/packages/@aws-cdk/assets/test/integ.multi-assets.expected.json
@@ -0,0 +1,20 @@
+{
+  "Parameters": {
+    "SampleAsset1S3Bucket469E18FF": {
+      "Type": "String",
+      "Description": "S3 bucket for asset \"aws-cdk-multi-assets/SampleAsset1\""
+    },
+    "SampleAsset1S3VersionKey63A628F0": {
+      "Type": "String",
+      "Description": "S3 key for asset version \"aws-cdk-multi-assets/SampleAsset1\""
+    },
+    "SampleAsset2S3BucketC94C651A": {
+      "Type": "String",
+      "Description": "S3 bucket for asset \"aws-cdk-multi-assets/SampleAsset2\""
+    },
+    "SampleAsset2S3VersionKey3A7E2CC4": {
+      "Type": "String",
+      "Description": "S3 key for asset version \"aws-cdk-multi-assets/SampleAsset2\""
+    }
+  }
+}
diff --git a/packages/@aws-cdk/assets/test/integ.multi-assets.ts b/packages/@aws-cdk/assets/test/integ.multi-assets.ts
@@ -0,0 +1,23 @@
+import cdk = require('@aws-cdk/cdk');
+import path = require('path');
+import assets = require('../lib');
+
+class TestStack extends cdk.Stack {
+  constructor(parent: cdk.App, name: string, props?: cdk.StackProps) {
+    super(parent, name, props);
+
+    // Check that the same asset added multiple times is
+    // uploaded and copied.
+    new assets.FileAsset(this, 'SampleAsset1', {
+      path: path.join(__dirname, 'file-asset.txt')
+    });
+
+    new assets.FileAsset(this, 'SampleAsset2', {
+      path: path.join(__dirname, 'file-asset.txt')
+    });
+  }
+}
+
+const app = new cdk.App();
+new TestStack(app, 'aws-cdk-multi-assets');
+app.run();
diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts
@@ -143,10 +143,13 @@ export class AssetCode extends Code {
   }
 
   public bind(lambda: Func) {
-    this.asset = new assets.Asset(lambda, 'Code', {
-      path: this.path,
-      packaging: this.packaging
-    });
+    // If the same AssetCode is used multiple times, retain only the first instantiation.
+    if (!this.asset) {
+      this.asset = new assets.Asset(lambda, 'Code', {
+        path: this.path,
+        packaging: this.packaging
+      });
+    }
 
     if (!this.asset.isZipArchive) {
       throw new Error(`Asset must be a .zip file or a directory (${this.path})`);
diff --git a/packages/@aws-cdk/aws-lambda/test/test.code.ts b/packages/@aws-cdk/aws-lambda/test/test.code.ts
@@ -36,6 +36,35 @@ export = {
 
       // THEN
       test.throws(() => defineFunction(fileAsset), /Asset must be a \.zip file or a directory/);
+      test.done();
+    },
+
+    'only one Asset object gets created even if multiple functions use the same AssetCode'(test: Test) {
+      // GIVEN
+      const app = new cdk.App();
+      const stack = new cdk.Stack(app, 'MyStack');
+      const directoryAsset = lambda.Code.asset(path.join(__dirname, 'my-lambda-handler'));
+
+      // WHEN
+      new lambda.Function(stack, 'Func1', {
+        handler: 'foom',
+        runtime: lambda.Runtime.NodeJS810,
+        code: directoryAsset
+      });
+
+      new lambda.Function(stack, 'Func2', {
+        handler: 'foom',
+        runtime: lambda.Runtime.NodeJS810,
+        code: directoryAsset
+      });
+
+      // THEN
+      const synthesized = app.synthesizeStack('MyStack');
+
+      // Func1 has an asset, Func2 does not
+      test.deepEqual(synthesized.metadata['/MyStack/Func1/Code'][0].type, 'aws:cdk:asset');
+      test.deepEqual(synthesized.metadata['/MyStack/Func2/Code'], undefined);
+
       test.done();
     }
   }
diff --git a/packages/aws-cdk/lib/api/toolkit-info.ts b/packages/aws-cdk/lib/api/toolkit-info.ts
@@ -21,6 +21,11 @@ export interface Uploaded {
 }
 
 export class ToolkitInfo {
+  /**
+   * A cache of previous uploads done in this session
+   */
+  private readonly previousUploads: {[key: string]: Uploaded} = {};
+
   constructor(private readonly props: {
     sdk: SDK,
     bucketName: string,
@@ -60,17 +65,31 @@ export class ToolkitInfo {
       return { filename, key, changed: false };
     }
 
-    debug(`${url}: uploading`);
-    await s3.putObject({
-      Bucket: bucket,
-      Key: key,
-      Body: data,
-      ContentType: props.contentType
-    }).promise();
-
-    debug(`${url}: upload complete`);
+    const uploaded = { filename, key, changed: true };
+
+    // Upload if it's new or server-side copy if it was already uploaded previously
+    const previous = this.previousUploads[hash];
+    if (previous) {
+      debug(`${url}: copying`);
+      await s3.copyObject({
+        Bucket: bucket,
+        Key: key,
+        CopySource: `${bucket}/${previous.key}`
+      }).promise();
+      debug(`${url}: copy complete`);
+    } else {
+      debug(`${url}: uploading`);
+      await s3.putObject({
+        Bucket: bucket,
+        Key: key,
+        Body: data,
+        ContentType: props.contentType
+      }).promise();
+      debug(`${url}: upload complete`);
+      this.previousUploads[hash] = uploaded;
+    }
 
-    return { filename, key, changed: true };
+    return uploaded;
   }
 
 }
