Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 624a208

Browse files
bstasyszynbstasyszyn
committed
fix: Exit anchor status monitoring loop on shutdown
Also, updated witness proof monitor to accelerate the next task time if it is known that more records need to be processed. closes #1572 Signed-off-by: Bob Stasyszyn <[email protected]>
1 parent 7017bf0 commit 624a208

12 files changed

Lines changed: 363 additions & 206 deletions

File tree

‎cmd/orb-server/startcmd/params.go‎

Lines changed: 122 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,9 @@ const (
130130
defaultVCTLogMonitoringMaxTreeSize = 50000
131131
defaultVCTLogMonitoringGetEntriesRange = 1000
132132
defaultVCTLogEntriesStoreEnabled = false
133-
defaultAnchorStatusMonitoringInterval = 5 * time.Second
134-
defaultAnchorStatusInProcessGracePeriod = 30 * time.Second
133+
defaultAnchorStatusMonitoringInterval = 10 * time.Second
134+
defaultAnchorStatusMaxRecords = 500
135+
defaultAnchorStatusInProcessGracePeriod = time.Minute
135136
mqDefaultMaxConnectionSubscriptions = 1000
136137
mqDefaultPublisherChannelPoolSize = 25
137138
mqDefaultPublisherConfirmDelivery = true
@@ -226,6 +227,12 @@ const (
226227
"witnessed(completed) as per policy. Defaults to 5s if not set. " +
227228
commonEnvVarUsageText + anchorStatusMonitoringIntervalEnvKey
228229

230+
anchorStatusMaxRecordsFlagName = "anchor-status-max-records"
231+
anchorStatusMaxRecordsEnvKey = "ANCHOR_STATUS_MAX_RECORDS"
232+
anchorStatusMaxRecordsFlagUsage = "The maximum number of anchor status records to process per monitoring interval " +
233+
"Defaults to 500 if not set. " +
234+
commonEnvVarUsageText + anchorStatusMaxRecordsEnvKey
235+
229236
anchorStatusInProcessGracePeriodFlagName = "anchor-status-in-process-grace-period"
230237
anchorStatusInProcessGracePeriodEnvKey = "ANCHOR_STATUS_IN_PROCESS_GRACE_PERIOD"
231238
anchorStatusInProcessGracePeriodFlagUsage = "The period in which witnesses will not be re-selected for 'in-process' anchors." +
@@ -764,47 +771,46 @@ type tlsParameters struct {
764771
}
765772

766773
type orbParameters struct {
767-
http *httpParams
768-
sidetree *sidetreeParams
769-
apServiceParams *apServiceParams
770-
discoveryDomain string
771-
dataURIMediaType datauri.MediaType
772-
batchWriterTimeout time.Duration
773-
cas *casParams
774-
mqParams *mqParams
775-
opQueueParams *opqueue.Config
776-
dbParameters *dbParameters
777-
logLevel string
778-
methodContext []string
779-
baseEnabled bool
780-
allowedOrigins []string
781-
allowedOriginsCacheExpiration time.Duration
782-
anchorCredentialParams *anchorCredentialParams
783-
discovery *discoveryParams
784-
witnessProof *witnessProofParams
785-
syncTimeout uint64
786-
didDiscoveryEnabled bool
787-
unpublishedOperations *unpublishedOperationsStoreParams
788-
resolveFromAnchorOrigin bool
789-
verifyLatestFromAnchorOrigin bool
790-
activityPub *activityPubParams
791-
auth *authParams
792-
enableDevMode bool
793-
enableMaintenanceMode bool
794-
enableVCT bool
795-
nodeInfoRefreshInterval time.Duration
796-
contextProviderURLs []string
797-
dataExpiryCheckInterval time.Duration
798-
taskMgrCheckInterval time.Duration
799-
vct *vctParams
800-
anchorStatusMonitoringInterval time.Duration
801-
anchorStatusInProcessGracePeriod time.Duration
802-
witnessPolicyCacheExpiration time.Duration
803-
kmsParams *kmsParameters
804-
requestTokens map[string]string
805-
allowedDIDWebDomains []*url.URL
806-
observability *observabilityParams
807-
anchorRefPendingRecordLifespan time.Duration
774+
http *httpParams
775+
sidetree *sidetreeParams
776+
apServiceParams *apServiceParams
777+
discoveryDomain string
778+
dataURIMediaType datauri.MediaType
779+
batchWriterTimeout time.Duration
780+
cas *casParams
781+
mqParams *mqParams
782+
opQueueParams *opqueue.Config
783+
dbParameters *dbParameters
784+
logLevel string
785+
methodContext []string
786+
baseEnabled bool
787+
allowedOrigins []string
788+
allowedOriginsCacheExpiration time.Duration
789+
anchorCredentialParams *anchorCredentialParams
790+
discovery *discoveryParams
791+
witnessProof *witnessProofParams
792+
syncTimeout uint64
793+
didDiscoveryEnabled bool
794+
unpublishedOperations *unpublishedOperationsStoreParams
795+
resolveFromAnchorOrigin bool
796+
verifyLatestFromAnchorOrigin bool
797+
activityPub *activityPubParams
798+
auth *authParams
799+
enableDevMode bool
800+
enableMaintenanceMode bool
801+
enableVCT bool
802+
nodeInfoRefreshInterval time.Duration
803+
contextProviderURLs []string
804+
dataExpiryCheckInterval time.Duration
805+
taskMgrCheckInterval time.Duration
806+
vct *vctParams
807+
anchorStatus *anchorStatusParams
808+
witnessPolicyCacheExpiration time.Duration
809+
kmsParams *kmsParameters
810+
requestTokens map[string]string
811+
allowedDIDWebDomains []*url.URL
812+
observability *observabilityParams
813+
anchorRefPendingRecordLifespan time.Duration
808814
}
809815

810816
type observabilityParams struct {
@@ -1127,16 +1133,9 @@ func getOrbParameters(cmd *cobra.Command) (*orbParameters, error) {
11271133
return nil, err
11281134
}
11291135

1130-
anchorStatusMonitoringInterval, err := cmdutil.GetDuration(cmd, anchorStatusMonitoringIntervalFlagName,
1131-
anchorStatusMonitoringIntervalEnvKey, defaultAnchorStatusMonitoringInterval)
1132-
if err != nil {
1133-
return nil, fmt.Errorf("%s: %w", anchorStatusMonitoringIntervalFlagName, err)
1134-
}
1135-
1136-
anchorStatusInProcessGracePeriod, err := cmdutil.GetDuration(cmd, anchorStatusInProcessGracePeriodFlagName,
1137-
anchorStatusInProcessGracePeriodEnvKey, defaultAnchorStatusInProcessGracePeriod)
1136+
anchorStatusParams, err := getAnchorStatusParams(cmd)
11381137
if err != nil {
1139-
return nil, fmt.Errorf("%s: %w", anchorStatusInProcessGracePeriodFlagName, err)
1138+
return nil, err
11401139
}
11411140

11421141
witnessPolicyCacheExpiration, err := cmdutil.GetDuration(cmd, witnessPolicyCacheExpirationFlagName,
@@ -1161,45 +1160,44 @@ func getOrbParameters(cmd *cobra.Command) (*orbParameters, error) {
11611160
}
11621161

11631162
return &orbParameters{
1164-
http: httpParams,
1165-
sidetree: sidetreeParams,
1166-
discoveryDomain: discoveryDomain,
1167-
apServiceParams: apServiceParams,
1168-
allowedOrigins: allowedOrigins,
1169-
allowedOriginsCacheExpiration: allowedOriginsCacheExpiration,
1170-
allowedDIDWebDomains: allowedDIDWebDomains,
1171-
cas: casParams,
1172-
mqParams: mqParams,
1173-
opQueueParams: opQueueParams,
1174-
batchWriterTimeout: batchWriterTimeout,
1175-
anchorCredentialParams: anchorCredentialParams,
1176-
logLevel: loggingLevel,
1177-
dbParameters: dbParams,
1178-
discovery: discoveryParams,
1179-
witnessProof: witnessProofParams,
1180-
syncTimeout: syncTimeout,
1181-
didDiscoveryEnabled: didDiscoveryEnabled,
1182-
unpublishedOperations: unpublishedOperationsParams,
1183-
resolveFromAnchorOrigin: resolveFromAnchorOrigin,
1184-
verifyLatestFromAnchorOrigin: verifyLatestFromAnchorOrigin,
1185-
auth: authParams,
1186-
activityPub: activityPubParams,
1187-
enableDevMode: enableDevMode,
1188-
enableMaintenanceMode: enableMaintenanceMode,
1189-
enableVCT: enableVCT,
1190-
nodeInfoRefreshInterval: nodeInfoRefreshInterval,
1191-
contextProviderURLs: contextProviderURLs,
1192-
dataExpiryCheckInterval: dataExpiryCheckInterval,
1193-
taskMgrCheckInterval: taskMgrCheckInterval,
1194-
vct: vctParams,
1195-
anchorStatusMonitoringInterval: anchorStatusMonitoringInterval,
1196-
anchorStatusInProcessGracePeriod: anchorStatusInProcessGracePeriod,
1197-
witnessPolicyCacheExpiration: witnessPolicyCacheExpiration,
1198-
dataURIMediaType: dataURIMediaType,
1199-
kmsParams: kmsParams,
1200-
requestTokens: requestTokens,
1201-
observability: observabilityParams,
1202-
anchorRefPendingRecordLifespan: anchorRefPendingRecordLifespan,
1163+
http: httpParams,
1164+
sidetree: sidetreeParams,
1165+
discoveryDomain: discoveryDomain,
1166+
apServiceParams: apServiceParams,
1167+
allowedOrigins: allowedOrigins,
1168+
allowedOriginsCacheExpiration: allowedOriginsCacheExpiration,
1169+
allowedDIDWebDomains: allowedDIDWebDomains,
1170+
cas: casParams,
1171+
mqParams: mqParams,
1172+
opQueueParams: opQueueParams,
1173+
batchWriterTimeout: batchWriterTimeout,
1174+
anchorCredentialParams: anchorCredentialParams,
1175+
logLevel: loggingLevel,
1176+
dbParameters: dbParams,
1177+
discovery: discoveryParams,
1178+
witnessProof: witnessProofParams,
1179+
syncTimeout: syncTimeout,
1180+
didDiscoveryEnabled: didDiscoveryEnabled,
1181+
unpublishedOperations: unpublishedOperationsParams,
1182+
resolveFromAnchorOrigin: resolveFromAnchorOrigin,
1183+
verifyLatestFromAnchorOrigin: verifyLatestFromAnchorOrigin,
1184+
auth: authParams,
1185+
activityPub: activityPubParams,
1186+
enableDevMode: enableDevMode,
1187+
enableMaintenanceMode: enableMaintenanceMode,
1188+
enableVCT: enableVCT,
1189+
nodeInfoRefreshInterval: nodeInfoRefreshInterval,
1190+
contextProviderURLs: contextProviderURLs,
1191+
dataExpiryCheckInterval: dataExpiryCheckInterval,
1192+
taskMgrCheckInterval: taskMgrCheckInterval,
1193+
vct: vctParams,
1194+
anchorStatus: anchorStatusParams,
1195+
witnessPolicyCacheExpiration: witnessPolicyCacheExpiration,
1196+
dataURIMediaType: dataURIMediaType,
1197+
kmsParams: kmsParams,
1198+
requestTokens: requestTokens,
1199+
observability: observabilityParams,
1200+
anchorRefPendingRecordLifespan: anchorRefPendingRecordLifespan,
12031201
}, nil
12041202
}
12051203

@@ -1694,6 +1692,38 @@ func getActivityPubIRICacheParameters(cmd *cobra.Command) (int, time.Duration, e
16941692
})
16951693
}
16961694

1695+
type anchorStatusParams struct {
1696+
monitoringInterval time.Duration
1697+
maxRecordsPerInterval int
1698+
inProcessGracePeriod time.Duration
1699+
}
1700+
1701+
func getAnchorStatusParams(cmd *cobra.Command) (*anchorStatusParams, error) {
1702+
monitoringInterval, err := cmdutil.GetDuration(cmd, anchorStatusMonitoringIntervalFlagName,
1703+
anchorStatusMonitoringIntervalEnvKey, defaultAnchorStatusMonitoringInterval)
1704+
if err != nil {
1705+
return nil, fmt.Errorf("%s: %w", anchorStatusMonitoringIntervalFlagName, err)
1706+
}
1707+
1708+
maxRecords, err := cmdutil.GetInt(cmd, anchorStatusMaxRecordsFlagName,
1709+
anchorStatusMaxRecordsEnvKey, defaultAnchorStatusMaxRecords)
1710+
if err != nil {
1711+
return nil, fmt.Errorf("%s: %w", anchorStatusMaxRecordsFlagName, err)
1712+
}
1713+
1714+
inProcessGracePeriod, err := cmdutil.GetDuration(cmd, anchorStatusInProcessGracePeriodFlagName,
1715+
anchorStatusInProcessGracePeriodEnvKey, defaultAnchorStatusInProcessGracePeriod)
1716+
if err != nil {
1717+
return nil, fmt.Errorf("%s: %w", anchorStatusInProcessGracePeriodFlagName, err)
1718+
}
1719+
1720+
return &anchorStatusParams{
1721+
monitoringInterval: monitoringInterval,
1722+
maxRecordsPerInterval: maxRecords,
1723+
inProcessGracePeriod: inProcessGracePeriod,
1724+
}, nil
1725+
}
1726+
16971727
func getAllowedDIDWebDomains(cmd *cobra.Command) ([]*url.URL, error) {
16981728
allowedDIDWebDomainsArray, err := cmdutil.GetUserSetVarFromArrayString(cmd, allowedDIDWebDomainsFlagName,
16991729
allowedDIDWebDomainsEnvKey, true)
@@ -2449,6 +2479,7 @@ func createFlags(startCmd *cobra.Command) {
24492479
startCmd.Flags().StringP(vctLogMonitoringGetEntriesRangeFlagName, "", "", vctLogMonitoringGetEntriesRangeFlagUsage)
24502480
startCmd.Flags().StringP(vctLogEntriesStoreEnabledFlagName, "", "", vctLogEntriesStoreEnabledFlagUsage)
24512481
startCmd.Flags().StringP(anchorStatusMonitoringIntervalFlagName, "", "", anchorStatusMonitoringIntervalFlagUsage)
2482+
startCmd.Flags().StringP(anchorStatusMaxRecordsFlagName, "", "", anchorStatusMaxRecordsFlagUsage)
24522483
startCmd.Flags().StringP(anchorStatusInProcessGracePeriodFlagName, "", "", anchorStatusInProcessGracePeriodFlagUsage)
24532484
startCmd.Flags().StringP(witnessPolicyCacheExpirationFlagName, "", "", witnessPolicyCacheExpirationFlagUsage)
24542485
startCmd.Flags().StringP(activityPubClientCacheSizeFlagName, "", "", activityPubClientCacheSizeFlagUsage)

‎cmd/orb-server/startcmd/params_test.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -839,6 +839,19 @@ func TestStartCmdWithMissingArg(t *testing.T) {
839839
require.Contains(t, err.Error(), "invalid value for anchor-status-monitoring-interval [xxx]")
840840
})
841841

842+
t.Run("anchor status max records", func(t *testing.T) {
843+
restoreEnv := setEnv(t, anchorStatusMaxRecordsEnvKey, "xxx")
844+
defer restoreEnv()
845+
846+
startCmd := GetStartCmd()
847+
848+
startCmd.SetArgs(getTestArgs("localhost:8081", "local", "false", databaseTypeMemOption))
849+
850+
err := startCmd.Execute()
851+
require.Error(t, err)
852+
require.Contains(t, err.Error(), "invalid value for anchor-status-max-records [xxx]")
853+
})
854+
842855
t.Run("anchor status in-process grace period", func(t *testing.T) {
843856
restoreEnv := setEnv(t, anchorStatusInProcessGracePeriodEnvKey, "xxx")
844857
defer restoreEnv()

‎cmd/orb-server/startcmd/start.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -807,15 +807,17 @@ func startOrbServices(parameters *orbParameters) error {
807807
return fmt.Errorf("failed to create witness policy inspector: %s", err.Error())
808808
}
809809

810-
anchorEventStatusStore, err := anchorstatus.New(storeProviders.provider, expiryService,
811-
parameters.witnessProof.maxWitnessDelay, anchorstatus.WithPolicyHandler(policyInspector),
812-
anchorstatus.WithCheckStatusAfterTime(parameters.anchorStatusInProcessGracePeriod))
810+
anchorEventStatusStore, err := anchorstatus.New(storeProviders.provider, taskMgr, expiryService,
811+
parameters.witnessProof.maxWitnessDelay,
812+
anchorstatus.WithPolicyHandler(policyInspector),
813+
anchorstatus.WithMonitoringInterval(parameters.anchorStatus.monitoringInterval),
814+
anchorstatus.WithMaxRecordsPerInterval(parameters.anchorStatus.maxRecordsPerInterval),
815+
anchorstatus.WithCheckStatusAfterTime(parameters.anchorStatus.inProcessGracePeriod),
816+
)
813817
if err != nil {
814818
return fmt.Errorf("failed to create vc status store: %s", err.Error())
815819
}
816820

817-
taskMgr.RegisterTask("anchor-status-monitor", parameters.anchorStatusMonitoringInterval, anchorEventStatusStore.CheckInProcessAnchors)
818-
819821
pubSub := newPubSub(parameters)
820822

821823
proofHandler := proof.New(
@@ -1196,7 +1198,8 @@ func startOrbServices(parameters *orbParameters) error {
11961198
)
11971199

11981200
err = run(httpServer, activityPubService, opQueue, obsrv, batchWriter, taskMgr, apClient,
1199-
nodeInfoService, newMPLifecycleWrapper(mp), tracerProvider, proofMonitoringSvc)
1201+
nodeInfoService, newMPLifecycleWrapper(mp), tracerProvider, proofMonitoringSvc,
1202+
anchorEventStatusStore)
12001203
if err != nil {
12011204
return err
12021205
}

‎internal/pkg/log/fields.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ const (
151151
FieldMaxActivitiesToSync = "maxActivitiesToSync"
152152
FieldNumActivitiesSynced = "numActivitiesSynced"
153153
FieldNextActivitySyncInterval = "nextActivitySyncInterval"
154-
FieldMaxProofMonitorRecords = "maxProofMonitorRecords"
154+
FieldRecordsProcessed = "recordsProcessed"
155155
)
156156

157157
// WithMessageID sets the message-id field.
@@ -905,9 +905,9 @@ func WithNextActivitySyncInterval(value time.Duration) zap.Field {
905905
return zap.Duration(FieldNextActivitySyncInterval, value)
906906
}
907907

908-
// WithMaxProofMonitorRecords sets the maxProofMonitorRecords field.
909-
func WithMaxProofMonitorRecords(value int) zap.Field {
910-
return zap.Int(FieldMaxProofMonitorRecords, value)
908+
// WithRecordsProcessed sets the recordsProcessed field.
909+
func WithRecordsProcessed(value int) zap.Field {
910+
return zap.Int(FieldRecordsProcessed, value)
911911
}
912912

913913
type jsonMarshaller struct {

‎internal/pkg/log/fields_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ func TestStandardFields(t *testing.T) {
6363
WithCreatedTime(now), WithWitnessURI(u1), WithWitnessURIs(u1, u2), WithWitnessPolicy("some policy"),
6464
WithAnchorOrigin(u1.String()), WithOperationType("Create"), WithCoreIndex("1234"),
6565
WithMaxOperationsToRepost(300), WithMaxActivitiesToSync(11), WithNextActivitySyncInterval(3*time.Second),
66-
WithNumActivitiesSynced(123), WithMaxProofMonitorRecords(23),
66+
WithNumActivitiesSynced(123), WithRecordsProcessed(23),
6767
)
6868

6969
t.Logf(stdOut.String())
@@ -123,7 +123,7 @@ func TestStandardFields(t *testing.T) {
123123
require.Equal(t, 11, l.MaxActivitiesToSync)
124124
require.Equal(t, "3s", l.NextActivitySyncInterval)
125125
require.Equal(t, 123, l.NumActivitiesSynced)
126-
require.Equal(t, 23, l.MaxProofMonitorRecords)
126+
require.Equal(t, 23, l.RecordsProcessed)
127127
})
128128

129129
t.Run("json fields 2", func(t *testing.T) {
@@ -453,7 +453,7 @@ type logData struct {
453453
MaxActivitiesToSync int `json:"maxActivitiesToSync"`
454454
NextActivitySyncInterval string `json:"nextActivitySyncInterval"`
455455
NumActivitiesSynced int `json:"numActivitiesSynced"`
456-
MaxProofMonitorRecords int `json:"maxProofMonitorRecords"`
456+
RecordsProcessed int `json:"recordsProcessed"`
457457
}
458458

459459
func unmarshalLogData(t *testing.T, b []byte) *logData {

0 commit comments

Comments
 (0)