Thanks to visit codestin.com
Credit goes to github.com

Skip to content

make sure repack and express workflows wait for streamer notifications #4154

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hufnagel merged 1 commit into from
Jun 30, 2015
Merged

make sure repack and express workflows wait for streamer notifications #4154

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions src/python/T0/RunConfig/RunConfigAPI.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,12 +221,6 @@ def configureRunStream(tier0Config, run, stream, specDirectory, dqmUploadProxy):
bindsStorageNode = []
bindsPhEDExConfig = []

# mark workflows as injected
wmbsDaoFactory = DAOFactory(package = "WMCore.WMBS",
logger = logging,
dbinterface = myThread.dbi)
markWorkflowsInjectedDAO = wmbsDaoFactory(classname = "Workflow.MarkInjectedWorkflows")

#
# for spec creation, details for all outputs
#
Expand Down Expand Up @@ -621,8 +615,6 @@ def configureRunStream(tier0Config, run, stream, specDirectory, dqmUploadProxy):
'PRIMDS' : primds,
'FILESET' : fileset } )
insertRecoReleaseConfigDAO.execute(bindsRecoReleaseConfig, conn = myThread.transaction.conn, transaction = True)
elif streamConfig.ProcessingStyle == "Express":
markWorkflowsInjectedDAO.execute([workflowName], injected = True, conn = myThread.transaction.conn, transaction = True)
except Exception as ex:
logging.exception(ex)
myThread.transaction.rollback()
Expand Down
2 changes: 1 addition & 1 deletion src/python/T0/WMBS/Oracle/Create.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,7 +450,7 @@ def __init__(self, logger = None, dbi = None, params = None):
"""CREATE INDEX idx_streamer_2 ON streamer (checkForZeroState(used))"""

self.indexes[len(self.indexes)] = \
"""CREATE INDEX idx_streamer_3 ON streamer (checkForZeroOneState(deleted))"""
"""CREATE INDEX idx_streamer_3 ON streamer (checkForZeroState(deleted))"""

self.indexes[len(self.indexes)] = \
"""CREATE INDEX idx_prompt_calib_1 ON prompt_calib (checkForZeroState(finished))"""
Expand Down
42 changes: 0 additions & 42 deletions src/python/T0/WMBS/Oracle/Tier0Feeder/MarkRepackInjected.py
View file
Open in desktop

This file was deleted.

153 changes: 153 additions & 0 deletions src/python/T0/WMBS/Oracle/Tier0Feeder/MarkWorkflowsInjected.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
"""
_MarkWorkflowsInjected_

Oracle implementation of MarkWorkflowsInjected

Implementation depends on whether we assume streamer
notifications to P5 or not.

with P5 streamer notifications:

- Express sets a workflow to injected when the fileset
is closed and all streamers are fed into the Tier0
- Repack has the same requirements and in additions
all datasets for that workflow/stream had their
PromptReco released

without P5 streamer notifications:
- Express is set to injected immediately
- Repack needs all datasets for that workflow/stream
PromptReco released before the workflow is set
to injected

"""

from WMCore.Database.DBFormatter import DBFormatter

class MarkWorkflowsInjected(DBFormatter):

sqlExpressNotify = \
"""UPDATE wmbs_workflow
SET injected = 1
WHERE name in (
SELECT wmbs_workflow.name
FROM run_stream_fileset_assoc
INNER JOIN wmbs_subscription ON
wmbs_subscription.fileset = run_stream_fileset_assoc.fileset
INNER JOIN wmbs_fileset ON
wmbs_fileset.id = wmbs_subscription.fileset AND
wmbs_fileset.open = 0
INNER JOIN wmbs_workflow ON
wmbs_workflow.id = wmbs_subscription.workflow AND
wmbs_workflow.injected = 0
INNER JOIN run_stream_style_assoc ON
run_stream_style_assoc.run_id = run_stream_fileset_assoc.run_id AND
run_stream_style_assoc.stream_id = run_stream_fileset_assoc.stream_id AND
run_stream_style_assoc.style_id = (SELECT id FROM processing_style WHERE name = 'Express')
LEFT OUTER JOIN streamer ON
streamer.run_id = run_stream_fileset_assoc.run_id AND
streamer.stream_id = run_stream_fileset_assoc.stream_id AND
checkForZeroState(streamer.deleted) = 0
WHERE streamer.run_id IS NULL
GROUP BY wmbs_workflow.name
)
"""

sqlRepackNotify = \
"""UPDATE wmbs_workflow
SET injected = 1
WHERE name in (
SELECT wmbs_workflow.name
FROM run_stream_fileset_assoc
INNER JOIN wmbs_subscription ON
wmbs_subscription.fileset = run_stream_fileset_assoc.fileset
INNER JOIN wmbs_fileset ON
wmbs_fileset.id = wmbs_subscription.fileset AND
wmbs_fileset.open = 0
INNER JOIN wmbs_workflow ON
wmbs_workflow.id = wmbs_subscription.workflow AND
wmbs_workflow.injected = 0
INNER JOIN run_stream_style_assoc ON
run_stream_style_assoc.run_id = run_stream_fileset_assoc.run_id AND
run_stream_style_assoc.stream_id = run_stream_fileset_assoc.stream_id AND
run_stream_style_assoc.style_id = (SELECT id FROM processing_style WHERE name = 'Bulk')
LEFT OUTER JOIN streamer ON
streamer.run_id = run_stream_fileset_assoc.run_id AND
streamer.stream_id = run_stream_fileset_assoc.stream_id AND
checkForZeroState(streamer.deleted) = 0
INNER JOIN run_primds_stream_assoc ON
run_primds_stream_assoc.run_id = run_stream_fileset_assoc.run_id AND
run_primds_stream_assoc.stream_id = run_stream_fileset_assoc.stream_id
LEFT OUTER JOIN reco_config ON
reco_config.run_id = run_stream_fileset_assoc.run_id AND
reco_config.primds_id = run_primds_stream_assoc.primds_id
WHERE streamer.run_id IS NULL
GROUP BY wmbs_workflow.name
HAVING COUNT(reco_config.run_id) = COUNT(*)
)
"""

sqlExpressNoNotify = \
"""UPDATE wmbs_workflow
SET injected = 1
WHERE name in (
SELECT wmbs_workflow.name
FROM run_stream_fileset_assoc
INNER JOIN wmbs_subscription ON
wmbs_subscription.fileset = run_stream_fileset_assoc.fileset
INNER JOIN wmbs_workflow ON
wmbs_workflow.id = wmbs_subscription.workflow AND
wmbs_workflow.injected = 0
INNER JOIN run_stream_style_assoc ON
run_stream_style_assoc.run_id = run_stream_fileset_assoc.run_id AND
run_stream_style_assoc.stream_id = run_stream_fileset_assoc.stream_id AND
run_stream_style_assoc.style_id = (SELECT id FROM processing_style WHERE name = 'Express')
GROUP BY wmbs_workflow.name
)
"""

sqlRepackNoNotify = \
"""UPDATE wmbs_workflow
SET injected = 1
WHERE name in (
SELECT wmbs_workflow.name
FROM run_stream_fileset_assoc
INNER JOIN wmbs_subscription ON
wmbs_subscription.fileset = run_stream_fileset_assoc.fileset
INNER JOIN wmbs_workflow ON
wmbs_workflow.id = wmbs_subscription.workflow AND
wmbs_workflow.injected = 0
INNER JOIN run_stream_style_assoc ON
run_stream_style_assoc.run_id = run_stream_fileset_assoc.run_id AND
run_stream_style_assoc.stream_id = run_stream_fileset_assoc.stream_id AND
run_stream_style_assoc.style_id = (SELECT id FROM processing_style WHERE name = 'Bulk')
INNER JOIN run_primds_stream_assoc ON
run_primds_stream_assoc.run_id = run_stream_fileset_assoc.run_id AND
run_primds_stream_assoc.stream_id = run_stream_fileset_assoc.stream_id
LEFT OUTER JOIN reco_config ON
reco_config.run_id = run_stream_fileset_assoc.run_id AND
reco_config.primds_id = run_primds_stream_assoc.primds_id
GROUP BY wmbs_workflow.name
HAVING COUNT(reco_config.run_id) = COUNT(*)
)
"""

def execute(self, streamerNotification, conn = None, transaction = False):

if streamerNotification:

self.dbi.processData(self.sqlExpressNotify, binds = {}, conn = conn,
transaction = transaction)

self.dbi.processData(self.sqlRepackNotify, binds = {}, conn = conn,
transaction = transaction)

else:

self.dbi.processData(self.sqlExpressNoNotify, binds = {}, conn = conn,
transaction = transaction)

self.dbi.processData(self.sqlRepackNoNotify, binds = {}, conn = conn,
transaction = transaction)

return
10 changes: 5 additions & 5 deletions src/python/T0Component/Tier0Feeder/Tier0FeederPoller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def algorithm(self, parameters = None):
findNewExpressRunsDAO = self.daoFactory(classname = "Tier0Feeder.FindNewExpressRuns")
releaseExpressDAO = self.daoFactory(classname = "Tier0Feeder.ReleaseExpress")
feedStreamersDAO = self.daoFactory(classname = "Tier0Feeder.FeedStreamers")
markRepackInjectedDAO = self.daoFactory(classname = "Tier0Feeder.MarkRepackInjected")
markWorkflowsInjectedDAO = self.daoFactory(classname = "Tier0Feeder.MarkWorkflowsInjected")

tier0Config = None
try:
Expand Down Expand Up @@ -203,11 +203,11 @@ def algorithm(self, parameters = None):
self.updateRecoReleaseConfigsT0DataSvc()

#
# check if all datasets for a stream had their PromptReco released
# then mark the repack workflow as injected (if we don't wait, the
# task archiver will cleanup too early)
# mark express and repack workflows as injected if certain conditions are met
# (we don't do it immediately to prevent the TaskArchiver from cleaning up too early)
#
markRepackInjectedDAO.execute(transaction = False)
markWorkflowsInjectedDAO.execute(self.transferSystemBaseDir != None,
transaction = False)

#
# close stream/lumis for run/streams that are active (fileset exists and open)
Expand Down