Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 38ee45c

Browse files
hufnagelhufnagel
committed
Merge pull request #4007 from samircury/fixing-schema-feeder
fixing #4002
2 parents f3dc500 + 7736869 commit 38ee45c

File tree

5 files changed

+108
-0
lines changed

5 files changed

+108
-0
lines changed

‎src/python/T0/WMBS/Oracle/Create.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,7 @@ def __init__(self, logger = None, dbi = None, params = None):
345345
"""CREATE TABLE workflow_monitoring (
346346
workflow int not null,
347347
tracked int default 0 not null,
348+
closeout int default 0 not null,
348349
primary key (workflow)
349350
) ORGANIZATION INDEX"""
350351

@@ -453,6 +454,8 @@ def __init__(self, logger = None, dbi = None, params = None):
453454
self.indexes[len(self.indexes)] = \
454455
"""CREATE INDEX idx_workflow_monitoring_0 ON workflow_monitoring (checkForZeroState(tracked))"""
455456

457+
self.indexes[len(self.indexes)] = \
458+
"""CREATE INDEX idx_workflow_monitoring_1 ON workflow_monitoring (checkForZeroState(closeout))"""
456459
#
457460
# Constraints
458461
#

‎src/python/T0/WMBS/Oracle/Tier0Feeder/GetNotClosedOutWorkflows.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
"""
2+
_GetNotClosedOutWorkflows_
3+
4+
Oracle implementation of GetNotClosedOutWorkflows
5+
Lists top level filesets not injected to monitoring
6+
"""
7+
from WMCore.Database.DBFormatter import DBFormatter
8+
9+
class GetNotClosedOutWorkflows(DBFormatter):
10+
11+
def execute(self, conn = None, transaction = False):
12+
13+
14+
sql = """SELECT wmbs_subscription.workflow, wmbs_subscription.fileset, wmbs_fileset.open, wmbs_workflow.name
15+
FROM workflow_monitoring
16+
INNER JOIN wmbs_subscription ON
17+
workflow_monitoring.workflow = wmbs_subscription.workflow
18+
INNER JOIN wmbs_workflow ON
19+
wmbs_subscription.workflow = wmbs_workflow.id
20+
INNER JOIN wmbs_fileset ON
21+
wmbs_subscription.fileset = wmbs_fileset.id
22+
WHERE checkForZeroState(workflow_monitoring.closeout) = 0"""
23+
24+
results = self.dbi.processData(sql, [], conn = conn,
25+
transaction = transaction)
26+
27+
return results[0].fetchall()
28+

‎src/python/T0/WMBS/Oracle/Tier0Feeder/MarkCloseoutWorkflowMonitoring.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
"""
2+
_MarkCloseoutWorkflowMonitoring_
3+
4+
Oracle implementation of MarkCloseoutWorkflowMonitoring
5+
Sets a workflow as "closeout=1" in workflow_monitoring table, should be used after successful upload to couchDB
6+
"""
7+
from WMCore.Database.DBFormatter import DBFormatter
8+
9+
class MarkCloseoutWorkflowMonitoring(DBFormatter):
10+
11+
def execute(self, workflowId, conn = None, transaction = False):
12+
13+
sql = """UPDATE workflow_monitoring
14+
SET closeout = 1
15+
WHERE workflow = :WORKFLOW_ID"""
16+
17+
binds = [{'WORKFLOW_ID': workflowId}]
18+
self.dbi.processData(sql, binds, conn = conn,
19+
transaction = transaction)
20+
return

‎src/python/T0Component/Tier0Feeder/Tier0FeederPoller.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,11 @@ def algorithm(self, parameters = None):
213213
#
214214
self.feedCouchMonitoring()
215215

216+
#
217+
# Update Couch when Repack and Express have closed input filesets (analog to old T0 closeout)
218+
#
219+
self.closeOutRealTimeWorkflows()
220+
216221
#
217222
# upload PCL conditions to DropBox
218223
#
@@ -245,6 +250,32 @@ def feedCouchMonitoring(self):
245250
# Here we have to trust the insert, if it doesn't happen will be easy to spot on the logs
246251
markTrackedWorkflowMonitoringDAO.execute(workflowId)
247252

253+
def closeOutRealTimeWorkflows(self):
254+
# This is supposed to keep track (in couch) if Repack and Express have all the files in place (fileset closed).
255+
# found PromptReco workflows should be closed automatically.
256+
workflows = self.getNotClosedOutWorkflowsDAO.execute()
257+
if len(workflows) == 0:
258+
logging.debug("No workflows to publish to couch monitoring, doing nothing")
259+
if workflows:
260+
for workflow in workflows:
261+
(workflowId, filesetId, filesetOpen, workflowName) = workflow
262+
# find returns -1 if the string is not found
263+
if workflowName.find('PromptReco') >= 0:
264+
logging.debug("Closing out instantaneously PromptReco Workflow %s" % workflowName)
265+
self.updateClosedState(workflowName, workflowId)
266+
else :
267+
# Check if fileset (which you already know) is closed or not
268+
# FIXME: No better way to do it? what comes from the DAO is a string, casting bool or int doesn't help much.
269+
# Works like that :
270+
if filesetOpen == '0':
271+
self.updateClosedState(workflowName, workflowId)
272+
273+
def updateClosedState(self, workflowName, workflowId):
274+
response = self.localSummaryCouchDB.updateRequestStatus(workflowName, 'Closed')
275+
if response == "OK" or "EXISTS":
276+
logging.debug("Successfully closed workflow %s" % workflowName)
277+
self.markCloseoutWorkflowMonitoringDAO.execute(workflowId)
278+
248279
def terminate(self, params):
249280
"""
250281
_terminate_

‎test/python/T0Component_t/Tier0Feeder_t/Tier0Feeder_t.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1188,6 +1188,32 @@ def feedCouchMonitoring(self):
11881188
# Here we have to trust the insert, if it doesn't happen will be easy to spot on the logs
11891189
self.markTrackedWorkflowMonitoringDAO.execute(workflowId)
11901190

1191+
def closeOutRealTimeWorkflows(self):
1192+
# This is supposed to keep track (in couch) if Repack and Express have all the files in place (fileset closed).
1193+
# found PromptReco workflows should be closed automatically.
1194+
workflows = self.getNotClosedOutWorkflowsDAO.execute()
1195+
if len(workflows) == 0:
1196+
logging.debug("No workflows to publish to couch monitoring, doing nothing")
1197+
if workflows:
1198+
for workflow in workflows:
1199+
(workflowId, filesetId, filesetOpen, workflowName) = workflow
1200+
# find returns -1 if the string is not found
1201+
if workflowName.find('PromptReco') >= 0:
1202+
logging.debug("Closing out instantaneously PromptReco Workflow %s" % workflowName)
1203+
self.updateClosedState(workflowName, workflowId)
1204+
else :
1205+
# Check if fileset (which you already know) is closed or not
1206+
# FIXME: No better way to do it? what comes from the DAO is a string, casting bool or int doesn't help much.
1207+
# Works like that :
1208+
if filesetOpen == '0':
1209+
self.updateClosedState(workflowName, workflowId)
1210+
1211+
def updateClosedState(self, workflowName, workflowId):
1212+
response = self.localSummaryCouchDB.updateRequestStatus(workflowName, 'Closed')
1213+
if response == "OK" or "EXISTS":
1214+
logging.debug("Successfully closed workflow %s" % workflowName)
1215+
self.markCloseoutWorkflowMonitoringDAO.execute(workflowId)
1216+
11911217
def test00(self):
11921218
"""
11931219
_test00_

0 commit comments

Comments
 (0)