Better error messages and fix race condition

There were some missing `await`s, one of which in particular caused the tests to not be properly tested on post-submit, which we only discovered because it made Windows fail with access denied errors.
flutter · auto-submit · Dec 21, 2023 · Dec 13, 2023 · Dec 13, 2023 · Dec 21, 2023
commit 482ff73d638fbef584d3cc2d3516049fd4bec740
diff --git a/packages/flutter_goldens/lib/flutter_goldens.dart b/packages/flutter_goldens/lib/flutter_goldens.dart
@@ -305,7 +305,7 @@ class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator {
     golden = _addPrefix(golden);
     await update(golden, imageBytes);
     final File goldenFile = getGoldenFile(golden);
-    skiaClient.imgtestAdd(golden.path, goldenFile); // throws if the result is false
+    await skiaClient.imgtestAdd(golden.path, goldenFile); // throws if the result is false
     return true;
   }
 

diff --git a/packages/flutter_goldens/lib/skia_client.dart b/packages/flutter_goldens/lib/skia_client.dart
@@ -23,6 +23,14 @@ const String _kWebRendererKey = 'FLUTTER_WEB_RENDERER';
 /// Signature of callbacks used to inject [print] replacements.
 typedef LogCallback = void Function(String);
 
+/// Signature of callbacks used to determine if a Skia Gold command succeeded,
+/// and if not, what the error message should be.
+///
+/// Return null if the given arguments indicate success.
+///
+/// Otherwise, return the error message to show.
+typedef SkiaErrorCallback = String? Function(int exitCode, String stdout, String stderr);
+
 /// Exception thrown when an error is returned from the [SkiaClient].
 class SkiaException implements Exception {
   /// Creates a new `SkiaException` with a required error [message].
@@ -115,8 +123,7 @@ class SkiaGoldClient {
   Future<void> _retry({
     required Future<io.ProcessResult> Function() task,
     required String taskName,
-    bool Function(int exitCode, String stdout, String stderr)? success,
-    String? errorMessage,
+    SkiaErrorCallback? errorMessage,
   }) async {
     Duration delay = const Duration(seconds: 5);
     while (true) {
@@ -144,17 +151,21 @@ class SkiaGoldClient {
         continue; // retry
       }
 
-      if (success == null) {
+      String? message;
+      if (errorMessage != null) {
+        message = errorMessage(result.exitCode, resultStdout, resultStderr);
+        if (message == null) {
+          return; // success
+        }
+      } else {
         if (result.exitCode == 0) {
           return; // success
         }
-      } else if (success(result.exitCode, resultStdout, resultStderr)) {
-        return; // success
       }
 
       final StringBuffer buffer = StringBuffer();
-      if (errorMessage != null) {
-        buffer.writeln(errorMessage);
+      if (message != null) {
+        buffer.writeln(message);
         buffer.writeln();
       }
       buffer.writeln('$taskName failed with exit code ${result.exitCode}.');
@@ -191,11 +202,16 @@ class SkiaGoldClient {
         '--luci',
       ]),
       taskName: 'auth',
-      errorMessage: 'Skia Gold authorization failed.\n'
-                    '\n'
-                    'Luci environments authenticate using the file provided by '
-                    'LUCI_CONTEXT. There may be an error with this file or Gold '
-                    'authentication.',
+      errorMessage: (int exitCode, String resultStdout, String resultStderr) {
+        if (exitCode == 0) {
+          return null;
+        }
+        return 'Skia Gold authorization failed.\n'
+               '\n'
+               'Luci environments authenticate using the file provided by '
+               'LUCI_CONTEXT. There may be an error with this file or Gold '
+               'authentication.';
+      },
     );
   }
 
@@ -252,7 +268,12 @@ class SkiaGoldClient {
     await _retry(
       task: () => process.run(imgtestInitCommand),
       taskName: 'imgtest init',
-      errorMessage: 'An error occurred when initializing golden file test with goldctl.',
+      errorMessage: (int exitCode, String resultStdout, String resultStderr) {
+        if (exitCode == 0) {
+          return null;
+        }
+        return 'An error occurred when initializing golden file test with goldctl.';
+      },
     );
     _initialized = true;
   }
@@ -283,14 +304,23 @@ class SkiaGoldClient {
         ..._getPixelMatchingArguments(),
       ]),
       taskName: 'imgtest add',
-      errorMessage: 'Skia Gold received an unapproved image in post-submit '
-                    'testing. Golden file images in flutter/flutter are triaged '
-                    'in pre-submit during code review for the given PR.\n'
-                    '\n'
-                    'Visit https://flutter-gold.skia.org/ to view and approve '
-                    'the image(s), or revert the associated change. For more '
-                    'information, visit the wiki:\n'
-                    '  https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter'
+      errorMessage: (int exitCode, String resultStdout, String resultStderr) {
+        if (exitCode == 0) {
+          return null;
+        }
+        if (resultStdout.contains('Untriaged') ||
+            resultStdout.contains('negative image')) {
+          return 'Skia Gold received an unapproved image in post-submit '
+                 'testing. Golden file images in flutter/flutter are triaged '
+                 'in pre-submit during code review for the given PR.\n'
+                 '\n'
+                 'Visit https://flutter-gold.skia.org/ to view and approve '
+                 'the image(s), or revert the associated change. For more '
+                 'information, visit the wiki:\n'
+                 '  https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter';
+        }
+        return 'Golden test for "$testName" failed for a reason unrelated to pixel comparison.';
+      },
     );
   }
 
@@ -350,7 +380,12 @@ class SkiaGoldClient {
     await _retry(
       task: () => process.run(imgtestInitCommand),
       taskName: 'imgtest init',
-      errorMessage: 'An error occurred when initializing golden file tryjob with goldctl.'
+      errorMessage: (int exitCode, String resultStdout, String resultStderr) {
+        if (exitCode == 0) {
+          return null;
+        }
+        return 'An error occurred when initializing golden file tryjob with goldctl.';
+      },
     );
     _tryjobInitialized = true;
   }
@@ -375,12 +410,14 @@ class SkiaGoldClient {
         ..._getPixelMatchingArguments(),
       ]),
       taskName: 'imgtest add',
-      success: (int exitCode, String resultStdout, String resultStderr) {
-        return exitCode == 0
-            || resultStdout.contains('Untriaged')
-            || resultStdout.contains('negative image');
+      errorMessage: (int exitCode, String resultStdout, String resultStderr) {
+        if (exitCode == 0 ||
+            resultStdout.contains('Untriaged') ||
+            resultStdout.contains('negative image')) {
+          return null;
+        }
+        return 'Golden test for "$testName" failed for a reason unrelated to pixel comparison.';
       },
-      errorMessage: 'Golden test for "$testName" failed for a reason unrelated to pixel comparison.',
     );
   }
 

diff --git a/packages/flutter_goldens/test/flutter_goldens_test.dart b/packages/flutter_goldens/test/flutter_goldens_test.dart
@@ -554,16 +554,11 @@ void main() {
     process.fallbackProcessResult = io.ProcessResult(123, 1, 'Fallback failure', 'Fallback failure');
 
     expect(
-        skiaClient.imgtestAdd('golden_file_test', goldenFile),
+      skiaClient.imgtestAdd('golden_file_test', goldenFile),
       throwsA(
         isA<SkiaException>().having((SkiaException error) => error.message,
           'message',
-          'Skia Gold received an unapproved image in post-submit testing. '
-          'Golden file images in flutter/flutter are triaged in pre-submit during code review for the given PR.\n'
-          '\n'
-          'Visit https://flutter-gold.skia.org/ to view and approve the image(s), or revert the associated change. '
-          'For more information, visit the wiki:\n'
-          '  https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter\n'
+          'Golden test for "golden_file_test" failed for a reason unrelated to pixel comparison.\n'
           '\n'
           'imgtest add failed with exit code 1.\n'
           '\n'
@@ -770,7 +765,7 @@ void main() {
     expect(log, isEmpty);
   });
 
-  test('FlutterGoldenFileComparator - Post-Submit - calls init during compare', () {
+  test('FlutterGoldenFileComparator - Post-Submit - calls init during compare', () async {
     final Platform platform = FakePlatform(
       environment: <String, String>{'FLUTTER_ROOT': _kFlutterRoot},
       operatingSystem: 'macos'
@@ -786,7 +781,7 @@ void main() {
       log: log.add,
     );
     expect(fakeSkiaClient.initCalls, 0);
-    comparator.compare(
+    await comparator.compare(
       Uint8List.fromList(_kTestPngBytes),
       Uri.parse('flutter.golden_test.1.png'),
     );
@@ -849,7 +844,7 @@ void main() {
     expect(log, isEmpty);
   });
 
-  test('FlutterGoldenFileComparator - Pre-Submit - calls init during compare', () {
+  test('FlutterGoldenFileComparator - Pre-Submit - calls init during compare', () async {
     final FakeSkiaGoldClient fakeSkiaClient = FakeSkiaGoldClient();
     final (FileSystem fs, Directory libDirectory) = createFakeFileSystemWithLibDirectory();
     final List<String> log = <String>[];
@@ -864,7 +859,7 @@ void main() {
       log: log.add,
     );
     expect(fakeSkiaClient.tryInitCalls, 0);
-    comparator.compare(
+    await comparator.compare(
       Uint8List.fromList(_kTestPngBytes),
       Uri.parse('flutter.golden_test.1.png'),
     );
@@ -1062,39 +1057,62 @@ class FakeProcessManager extends Fake implements ProcessManager {
 class FakeSkiaGoldClient extends Fake implements SkiaGoldClient {
   Map<String, String> expectationForTestValues = <String, String>{};
   Exception? getExpectationForTestThrowable;
+
   @override
   Future<String> getExpectationForTest(String testName) async {
+    await null; // force this to be async
     if (getExpectationForTestThrowable != null) {
       throw getExpectationForTestThrowable!;
     }
     return expectationForTestValues[testName] ?? '';
   }
 
   @override
-  Future<void> auth() async {}
+  Future<void> auth() async {
+    await null; // force this to be async
+  }
 
   final List<String> testNames = <String>[];
 
   int initCalls = 0;
+
   @override
-  Future<void> imgtestInit() async => initCalls += 1;
+  Future<void> imgtestInit() async {
+    await null; // force this to be async
+    initCalls += 1;
+  }
+
   @override
   Future<bool> imgtestAdd(String testName, File goldenFile) async {
+    await null; // force this to be async
     testNames.add(testName);
     return true;
   }
 
   int tryInitCalls = 0;
+
   @override
-  Future<void> tryjobInit() async => tryInitCalls += 1;
+  Future<void> tryjobInit() async {
+    await null; // force this to be async
+    tryInitCalls += 1;
+  }
+
   @override
-  Future<bool> tryjobAdd(String testName, File goldenFile) async => true;
+  Future<bool> tryjobAdd(String testName, File goldenFile) async {
+    await null; // force this to be async
+    return true;
+  }
 
   Map<String, List<int>> imageBytesValues = <String, List<int>>{};
+
   @override
-  Future<List<int>> getImageBytes(String imageHash) async => imageBytesValues[imageHash]!;
+  Future<List<int>> getImageBytes(String imageHash) async {
+    await null; // force this to be async
+    return imageBytesValues[imageHash]!;
+  }
 
   Map<String, String> cleanTestNameValues = <String, String>{};
+
   @override
   String cleanTestName(String fileName) => cleanTestNameValues[fileName] ?? '';
 }