fix(deploy): fail a deploy job on unhealthy instance state

evansiroky · evansiroky · commit a8501d7f44b1 · 2020-04-09T14:40:40.000-07:00
diff --git a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java
@@ -181,7 +181,9 @@ public void run () {
             // TODO: should we add separate hooks depending on state of job/sub-tasks (e.g., success, catch, finally)
             // Set status to finished in case duration needed by finishing stage. (e.g., storing the job duration in a
             // database).
-            status.completeSuccessfully("Job complete!");
+            if (cancelMessage.equals("")) {
+                status.completeSuccessfully("Job complete!");
+            }
             jobFinished();
 
             // We retain finished or errored jobs on the server until they are fetched via the API, which implies they
diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java
@@ -512,6 +512,14 @@ private void replaceEC2Servers() {
                 );
                 monitorInitialServerJob.run();
 
+                if (monitorInitialServerJob.status.error) {
+                    // If an error occurred while monitoring the initial server, fail this job and instruct user to inspect
+                    // build logs.
+                    status.fail("Error encountered while building graph. Inspect build logs.");
+                    ServerController.terminateInstances(ec2, graphBuildingInstances);
+                    return;
+                }
+
                 status.update("Graph build is complete!", 40);
                 // If only building graph, job is finished. Note: the graph building EC2 instance should automatically shut
                 // itself down if this flag is turned on (happens in user data). We do not want to proceed with the rest of
@@ -522,13 +530,7 @@ private void replaceEC2Servers() {
                 }
 
                 Persistence.deployments.replace(deployment.id, deployment);
-                if (monitorInitialServerJob.status.error) {
-                    // If an error occurred while monitoring the initial server, fail this job and instruct user to inspect
-                    // build logs.
-                    status.fail("Error encountered while building graph. Inspect build logs.");
-                    ServerController.terminateInstances(ec2, graphBuildingInstances);
-                    return;
-                }
+
                 // Check if a new image of the instance with the completed graph build should be created.
                 if (otpServer.ec2Info.recreateBuildImage) {
                     status.update("Creating build image", 42.5);
diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java
