From a580a1f0c9532f5f8834b9f3abef8a4ee84c7c69 Mon Sep 17 00:00:00 2001 From: Keyong Han Date: Mon, 29 Aug 2022 14:04:02 -0700 Subject: [PATCH 1/6] e2e to build+test --- .../flutter_gallery__transition_perf_e2e.dart | 4 +- dev/devicelab/lib/tasks/gallery.dart | 66 +++++++++++-------- 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart index 8a66049914346..24342fa73f0fb 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/framework/devices.dart'; import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/tasks/gallery.dart'; -Future main() async { +Future main(List args) async { deviceOperatingSystem = DeviceOperatingSystem.android; - await task(createGalleryTransitionE2ETest()); + await task(createGalleryTransitionE2EBuildTest(args)); } diff --git a/dev/devicelab/lib/tasks/gallery.dart b/dev/devicelab/lib/tasks/gallery.dart index 4ce08ab82e354..fd3f5ec772552 100644 --- a/dev/devicelab/lib/tasks/gallery.dart +++ b/dev/devicelab/lib/tasks/gallery.dart @@ -25,14 +25,29 @@ TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) { return GalleryTransitionTest(semanticsEnabled: semanticsEnabled); } +TaskFunction createGalleryTransitionE2EBuildTest( + List args, { + bool semanticsEnabled = false, + bool enableImpeller = false, +}) { + return GalleryTransitionBuildTest( + args, + testFile: semanticsEnabled ? 'transitions_perf_e2e_with_semantics' : 'transitions_perf_e2e', + needFullTimeline: false, + timelineSummaryFile: 'e2e_perf_summary', + transitionDurationFile: null, + timelineTraceFile: null, + driverFile: 'transitions_perf_e2e_test', + enableImpeller: enableImpeller, + ); +} + TaskFunction createGalleryTransitionE2ETest({ bool semanticsEnabled = false, bool enableImpeller = false, }) { return GalleryTransitionTest( - testFile: semanticsEnabled - ? 'transitions_perf_e2e_with_semantics' - : 'transitions_perf_e2e', + testFile: semanticsEnabled ? 'transitions_perf_e2e_with_semantics' : 'transitions_perf_e2e', needFullTimeline: false, timelineSummaryFile: 'e2e_perf_summary', transitionDurationFile: null, @@ -45,14 +60,11 @@ TaskFunction createGalleryTransitionE2ETest({ TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) { return GalleryTransitionTest( semanticsEnabled: semanticsEnabled, - driverFile: semanticsEnabled - ? 'transitions_perf_hybrid_with_semantics_test' - : 'transitions_perf_hybrid_test', + driverFile: semanticsEnabled ? 'transitions_perf_hybrid_with_semantics_test' : 'transitions_perf_hybrid_test', ); } class GalleryTransitionTest { - GalleryTransitionTest({ this.semanticsEnabled = false, this.testFile = 'transitions_perf', @@ -101,23 +113,20 @@ class GalleryTransitionTest { applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk'; } - final String testDriver = driverFile ?? (semanticsEnabled - ? '${testFile}_with_semantics_test' - : '${testFile}_test'); + final String testDriver = + driverFile ?? (semanticsEnabled ? '${testFile}_with_semantics_test' : '${testFile}_test'); section('DRIVE START'); await flutter('drive', options: [ '--no-dds', '--profile', if (enableImpeller) '--enable-impeller', - if (needFullTimeline) - '--trace-startup', + if (needFullTimeline) '--trace-startup', if (applicationBinaryPath != null) '--use-application-binary=$applicationBinaryPath' - else - ...[ - '-t', - 'test_driver/$testFile.dart', - ], + else ...[ + '-t', + 'test_driver/$testFile.dart', + ], '--driver', 'test_driver/$testDriver.dart', '-d', @@ -125,7 +134,8 @@ class GalleryTransitionTest { ]); }); - final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build'; + final String testOutputDirectory = + Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build'; final Map summary = json.decode( file('$testOutputDirectory/$timelineSummaryFile.json').readAsStringSync(), ) as Map; @@ -143,16 +153,14 @@ class GalleryTransitionTest { } final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android; - return TaskResult.success(summary, + return TaskResult.success( + summary, detailFiles: [ - if (transitionDurationFile != null) - '$testOutputDirectory/$transitionDurationFile.json', - if (timelineTraceFile != null) - '$testOutputDirectory/$timelineTraceFile.json', + if (transitionDurationFile != null) '$testOutputDirectory/$transitionDurationFile.json', + if (timelineTraceFile != null) '$testOutputDirectory/$timelineTraceFile.json', ], benchmarkScoreKeys: [ - if (transitionDurationFile != null) - 'missed_transition_count', + if (transitionDurationFile != null) 'missed_transition_count', 'average_frame_build_time_millis', 'worst_frame_build_time_millis', '90th_percentile_frame_build_time_millis', @@ -205,23 +213,26 @@ class GalleryTransitionBuildTest extends BuildTestTask { this.driverFile, this.measureCpuGpu = true, this.measureMemory = true, + this.enableImpeller = false, }) : super(workingDirectory: galleryDirectory); final bool semanticsEnabled; final bool needFullTimeline; final bool measureCpuGpu; final bool measureMemory; + final bool enableImpeller; final String testFile; final String timelineSummaryFile; final String? timelineTraceFile; final String? transitionDurationFile; final String? driverFile; - final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build'; + final String testOutputDirectory = + Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build'; @override void copyArtifacts() { - if(applicationBinaryPath != null) { + if (applicationBinaryPath != null) { copy( file('${galleryDirectory.path}/build/app/outputs/flutter-apk/app-profile.apk'), Directory(applicationBinaryPath!), @@ -247,6 +258,7 @@ class GalleryTransitionBuildTest extends BuildTestTask { final String testDriver = driverFile ?? (semanticsEnabled ? '${testFile}_with_semantics_test' : '${testFile}_test'); return [ '--profile', + if (enableImpeller) '--enable-impeller', if (needFullTimeline) '--trace-startup', '-t', 'test_driver/$testFile.dart', From 2b90afb1d1198e54de7cf7da7c723c01a31b3b1b Mon Sep 17 00:00:00 2001 From: Keyong Han Date: Mon, 29 Aug 2022 15:03:26 -0700 Subject: [PATCH 2/6] no-dds --- dev/devicelab/lib/tasks/gallery.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/devicelab/lib/tasks/gallery.dart b/dev/devicelab/lib/tasks/gallery.dart index fd3f5ec772552..88e466e374908 100644 --- a/dev/devicelab/lib/tasks/gallery.dart +++ b/dev/devicelab/lib/tasks/gallery.dart @@ -257,6 +257,7 @@ class GalleryTransitionBuildTest extends BuildTestTask { List getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) { final String testDriver = driverFile ?? (semanticsEnabled ? '${testFile}_with_semantics_test' : '${testFile}_test'); return [ + '--no-dds', '--profile', if (enableImpeller) '--enable-impeller', if (needFullTimeline) '--trace-startup', From 9a095ff376bbe4aa7f974b40a322f253236bd4c2 Mon Sep 17 00:00:00 2001 From: Keyong Han Date: Mon, 29 Aug 2022 15:16:23 -0700 Subject: [PATCH 3/6] hybrid build+test --- .../flutter_gallery__transition_perf_hybrid.dart | 4 ++-- dev/devicelab/lib/tasks/gallery.dart | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart index ba6afecdec6cb..5108964ac8640 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/framework/devices.dart'; import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/tasks/gallery.dart'; -Future main() async { +Future main(List args) async { deviceOperatingSystem = DeviceOperatingSystem.android; - await task(createGalleryTransitionHybridTest()); + await task(createGalleryTransitionHybridBuildTest(args)); } diff --git a/dev/devicelab/lib/tasks/gallery.dart b/dev/devicelab/lib/tasks/gallery.dart index 88e466e374908..e3a5c2a7c48c2 100644 --- a/dev/devicelab/lib/tasks/gallery.dart +++ b/dev/devicelab/lib/tasks/gallery.dart @@ -57,6 +57,17 @@ TaskFunction createGalleryTransitionE2ETest({ ); } +TaskFunction createGalleryTransitionHybridBuildTest( + List args, { + bool semanticsEnabled = false, +}) { + return GalleryTransitionBuildTest( + args, + semanticsEnabled: semanticsEnabled, + driverFile: semanticsEnabled ? 'transitions_perf_hybrid_with_semantics_test' : 'transitions_perf_hybrid_test', + ); +} + TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) { return GalleryTransitionTest( semanticsEnabled: semanticsEnabled, From 049408ae7f304714f4316715739b9fee871e4b44 Mon Sep 17 00:00:00 2001 From: Keyong Han Date: Tue, 30 Aug 2022 13:10:01 -0700 Subject: [PATCH 4/6] update ci.yaml config --- .ci.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.ci.yaml b/.ci.yaml index ab41655fc1229..b2499531676a7 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -1940,25 +1940,29 @@ targets: tags: > ["devicelab", "android", "linux"] task_name: flutter_gallery__transition_perf - artifact: gallery_app_profile + artifact: gallery__transition_perf - - name: Linux_android flutter_gallery__transition_perf_e2e - recipe: devicelab/devicelab_drone + - name: Linux_build_test flutter_gallery__transition_perf_e2e + recipe: devicelab/devicelab_drone_build_test + bringup: true # New target https://github.com/flutter/flutter/issues/103542 presubmit: false timeout: 60 properties: tags: > ["devicelab", "android", "linux"] task_name: flutter_gallery__transition_perf_e2e + artifact: gallery__transition_perf_e2e - - name: Linux_android flutter_gallery__transition_perf_hybrid - recipe: devicelab/devicelab_drone + - name: Linux_build_test flutter_gallery__transition_perf_hybrid + recipe: devicelab/devicelab_drone_build_test + bringup: true # New target https://github.com/flutter/flutter/issues/103542 presubmit: false timeout: 60 properties: tags: > ["devicelab", "android", "linux"] task_name: flutter_gallery__transition_perf_hybrid + artifact: gallery__transition_perf_hybrid - name: Linux_android flutter_gallery__transition_perf_with_semantics recipe: devicelab/devicelab_drone From 7f0242fe444ca8f5b50c5673700b1a41b931c008 Mon Sep 17 00:00:00 2001 From: Keyong Han Date: Wed, 7 Sep 2022 10:09:28 -0700 Subject: [PATCH 5/6] restore format --- dev/devicelab/lib/tasks/gallery.dart | 45 ++++++++++++++++------------ 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/dev/devicelab/lib/tasks/gallery.dart b/dev/devicelab/lib/tasks/gallery.dart index e3a5c2a7c48c2..4d42944619c5f 100644 --- a/dev/devicelab/lib/tasks/gallery.dart +++ b/dev/devicelab/lib/tasks/gallery.dart @@ -47,7 +47,9 @@ TaskFunction createGalleryTransitionE2ETest({ bool enableImpeller = false, }) { return GalleryTransitionTest( - testFile: semanticsEnabled ? 'transitions_perf_e2e_with_semantics' : 'transitions_perf_e2e', + testFile: semanticsEnabled + ? 'transitions_perf_e2e_with_semantics' + : 'transitions_perf_e2e', needFullTimeline: false, timelineSummaryFile: 'e2e_perf_summary', transitionDurationFile: null, @@ -71,7 +73,9 @@ TaskFunction createGalleryTransitionHybridBuildTest( TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) { return GalleryTransitionTest( semanticsEnabled: semanticsEnabled, - driverFile: semanticsEnabled ? 'transitions_perf_hybrid_with_semantics_test' : 'transitions_perf_hybrid_test', + driverFile: semanticsEnabled + ? 'transitions_perf_hybrid_with_semantics_test' + : 'transitions_perf_hybrid_test', ); } @@ -124,20 +128,23 @@ class GalleryTransitionTest { applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk'; } - final String testDriver = - driverFile ?? (semanticsEnabled ? '${testFile}_with_semantics_test' : '${testFile}_test'); + final String testDriver = driverFile ?? (semanticsEnabled + ? '${testFile}_with_semantics_test' + : '${testFile}_test'); section('DRIVE START'); await flutter('drive', options: [ '--no-dds', '--profile', if (enableImpeller) '--enable-impeller', - if (needFullTimeline) '--trace-startup', + if (needFullTimeline) + '--trace-startup', if (applicationBinaryPath != null) '--use-application-binary=$applicationBinaryPath' - else ...[ - '-t', - 'test_driver/$testFile.dart', - ], + else + ...[ + '-t', + 'test_driver/$testFile.dart', + ], '--driver', 'test_driver/$testDriver.dart', '-d', @@ -145,8 +152,7 @@ class GalleryTransitionTest { ]); }); - final String testOutputDirectory = - Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build'; + final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build'; final Map summary = json.decode( file('$testOutputDirectory/$timelineSummaryFile.json').readAsStringSync(), ) as Map; @@ -164,14 +170,16 @@ class GalleryTransitionTest { } final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android; - return TaskResult.success( - summary, + return TaskResult.success(summary, detailFiles: [ - if (transitionDurationFile != null) '$testOutputDirectory/$transitionDurationFile.json', - if (timelineTraceFile != null) '$testOutputDirectory/$timelineTraceFile.json', + if (transitionDurationFile != null) + '$testOutputDirectory/$transitionDurationFile.json', + if (timelineTraceFile != null) + '$testOutputDirectory/$timelineTraceFile.json', ], benchmarkScoreKeys: [ - if (transitionDurationFile != null) 'missed_transition_count', + if (transitionDurationFile != null) + 'missed_transition_count', 'average_frame_build_time_millis', 'worst_frame_build_time_millis', '90th_percentile_frame_build_time_millis', @@ -238,12 +246,11 @@ class GalleryTransitionBuildTest extends BuildTestTask { final String? transitionDurationFile; final String? driverFile; - final String testOutputDirectory = - Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build'; + final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build'; @override void copyArtifacts() { - if (applicationBinaryPath != null) { + if(applicationBinaryPath != null) { copy( file('${galleryDirectory.path}/build/app/outputs/flutter-apk/app-profile.apk'), Directory(applicationBinaryPath!), From f09e8fed5101ec0c415282f5dccf97332fd24355 Mon Sep 17 00:00:00 2001 From: Keyong Han Date: Wed, 7 Sep 2022 10:12:15 -0700 Subject: [PATCH 6/6] remove format change --- dev/devicelab/lib/tasks/gallery.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/devicelab/lib/tasks/gallery.dart b/dev/devicelab/lib/tasks/gallery.dart index 4d42944619c5f..ae03b16d5f546 100644 --- a/dev/devicelab/lib/tasks/gallery.dart +++ b/dev/devicelab/lib/tasks/gallery.dart @@ -80,6 +80,7 @@ TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) } class GalleryTransitionTest { + GalleryTransitionTest({ this.semanticsEnabled = false, this.testFile = 'transitions_perf', @@ -140,7 +141,7 @@ class GalleryTransitionTest { '--trace-startup', if (applicationBinaryPath != null) '--use-application-binary=$applicationBinaryPath' - else + else ...[ '-t', 'test_driver/$testFile.dart',