core: Unfail subgraph only after successful processing

leoyvens · leoyvens · commit 905e76fde78a · 2021-02-16T22:38:17.000-03:00
diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs
@@ -455,6 +455,7 @@ where
     let store_for_err = ctx.inputs.store.cheap_clone();
     let logger = ctx.state.logger.cheap_clone();
     let id_for_err = ctx.inputs.deployment_id.clone();
+    let mut first_run = true;
 
     loop {
         debug!(logger, "Starting or restarting subgraph");
@@ -596,6 +597,18 @@ where
             match res {
                 Ok((c, needs_restart)) => {
                     ctx = c;
+
+                    // Unfail the subgraph if it was previously failed.
+                    // As an optimization we check this only on the first run.
+                    if first_run {
+                        first_run = false;
+
+                        ctx.inputs
+                            .store
+                            .unfail(&ctx.inputs.deployment_id)
+                            .map_err(|_| ())?;
+                    }
+
                     if needs_restart {
                         // Cancel the stream for real
                         ctx.state
diff --git a/graph/src/components/store.rs b/graph/src/components/store.rs
@@ -1080,16 +1080,16 @@ pub trait SubgraphStore: Send + Sync + 'static {
 
     fn unassign_subgraph(&self, id: &SubgraphDeploymentId) -> Result<(), StoreError>;
 
-    /// Start an existing subgraph deployment. This will reset the state of
-    /// the subgraph to a known good state. `ops` needs to contain all the
-    /// operations on the subgraph of subgraphs to reset the metadata of the
-    /// subgraph
+    /// Start an existing subgraph deployment.
     fn start_subgraph_deployment(
         &self,
         logger: &Logger,
         subgraph_id: &SubgraphDeploymentId,
     ) -> Result<(), StoreError>;
 
+    /// Remove the fatal error from a subgraph and check if it is healthy or unhealthy.
+    fn unfail(&self, subgraph_id: &SubgraphDeploymentId) -> Result<(), StoreError>;
+
     /// Load the dynamic data sources for the given deployment
     async fn load_dynamic_data_sources(
         &self,
@@ -1278,6 +1278,10 @@ impl SubgraphStore for MockStore {
         unimplemented!()
     }
 
+    fn unfail(&self, _: &SubgraphDeploymentId) -> Result<(), StoreError> {
+        unimplemented!()
+    }
+
     fn is_deployment_synced(&self, _: &SubgraphDeploymentId) -> Result<bool, Error> {
         unimplemented!()
     }
diff --git a/mock/src/store.rs b/mock/src/store.rs
@@ -185,6 +185,10 @@ impl SubgraphStore for MockStore {
         unimplemented!()
     }
 
+    fn unfail(&self, _: &SubgraphDeploymentId) -> Result<(), StoreError> {
+        unimplemented!()
+    }
+
     fn is_deployment_synced(&self, _: &SubgraphDeploymentId) -> Result<bool, Error> {
         unimplemented!()
     }
diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs
@@ -1071,10 +1071,12 @@ impl DeploymentStore {
         graft_base: Option<(Site, EthereumBlockPointer)>,
     ) -> Result<(), StoreError> {
         let econn = self.get_entity_conn(&site, ReplicaId::Main)?;
-        econn.transaction(|| {
-            deployment::unfail(&econn.conn, &site.deployment)?;
-            econn.start_subgraph(logger, graft_base)
-        })
+        econn.transaction(|| econn.start_subgraph(logger, graft_base))
+    }
+
+    pub(crate) fn unfail(&self, site: Arc<Site>) -> Result<(), StoreError> {
+        let econn = self.get_entity_conn(&site, ReplicaId::Main)?;
+        econn.transaction(|| deployment::unfail(&econn.conn, &site.deployment))
     }
 
     #[cfg(debug_assertions)]
diff --git a/store/postgres/src/store.rs b/store/postgres/src/store.rs
@@ -195,6 +195,10 @@ impl SubgraphStoreTrait for Store {
         self.store.start_subgraph_deployment(logger, subgraph_id)
     }
 
+    fn unfail(&self, id: &SubgraphDeploymentId) -> Result<(), StoreError> {
+        self.store.unfail(id)
+    }
+
     fn is_deployment_synced(&self, id: &SubgraphDeploymentId) -> Result<bool, Error> {
         self.store.is_deployment_synced(id)
     }
diff --git a/store/postgres/src/subgraph_store.rs b/store/postgres/src/subgraph_store.rs
@@ -745,6 +745,11 @@ impl SubgraphStoreTrait for SubgraphStore {
         store.start_subgraph(logger, site, graft_base)
     }
 
+    fn unfail(&self, id: &SubgraphDeploymentId) -> Result<(), StoreError> {
+        let (store, site) = self.store(id)?;
+        store.unfail(site)
+    }
+
     fn is_deployment_synced(&self, id: &SubgraphDeploymentId) -> Result<bool, Error> {
         let (store, _) = self.store(&id)?;
         Ok(store.exists_and_synced(&id)?)
diff --git a/store/postgres/tests/subgraph.rs b/store/postgres/tests/subgraph.rs
@@ -12,7 +12,7 @@ use graph::{
     prelude::SubgraphManifest,
     prelude::SubgraphName,
     prelude::SubgraphVersionSwitchingMode,
-    prelude::{o, slog, CheapClone, Logger, NodeId, SubgraphDeploymentId, SubgraphStore as _},
+    prelude::{CheapClone, NodeId, SubgraphDeploymentId, SubgraphStore as _},
 };
 use graph_store_postgres::layout_for_tests::Connection as Primary;
 use graph_store_postgres::Store;
@@ -554,7 +554,6 @@ fn fail_unfail() {
     }
 
     run_test_sequentially(setup, |store, id| async move {
-        let logger = Logger::root(slog::Discard, o!());
         let query_store = store
             .query_store(id.cheap_clone().into(), false)
             .await
@@ -576,7 +575,7 @@ fn fail_unfail() {
             .unwrap());
 
         // This will unfail the subgraph and delete the fatal error.
-        store.start_subgraph_deployment(&logger, &id).unwrap();
+        store.unfail(&id).unwrap();
 
         // Advance the block ptr to the block of the deleted error.
         transact_entity_operations(&store, id.cheap_clone(), BLOCKS[1], vec![]).unwrap();

Original file line number	Diff line number	Diff line change
`@@ -185,6 +185,10 @@ impl SubgraphStore for MockStore {`
`185`	`185`	`unimplemented!()`
`186`	`186`	`}`
`187`	`187`
	`188`	`+ fn unfail(&self, _: &SubgraphDeploymentId) -> Result<(), StoreError> {`
	`189`	`+ unimplemented!()`
	`190`	`+ }`
	`191`	`+`
`188`	`192`	`fn is_deployment_synced(&self, _: &SubgraphDeploymentId) -> Result<bool, Error> {`
`189`	`193`	`unimplemented!()`
`190`	`194`	`}`
Original file line number	Diff line number	Diff line change
`@@ -195,6 +195,10 @@ impl SubgraphStoreTrait for Store {`
`195`	`195`	`self.store.start_subgraph_deployment(logger, subgraph_id)`
`196`	`196`	`}`
`197`	`197`
	`198`	`+ fn unfail(&self, id: &SubgraphDeploymentId) -> Result<(), StoreError> {`
	`199`	`+ self.store.unfail(id)`
	`200`	`+ }`
	`201`	`+`
`198`	`202`	`fn is_deployment_synced(&self, id: &SubgraphDeploymentId) -> Result<bool, Error> {`
`199`	`203`	`self.store.is_deployment_synced(id)`
`200`	`204`	`}`