Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 82186c3

Browse files
hufnagelhufnagel
committed
Merge pull request #4076 from hufnagel/phedex-subscriptions
change the way PhEDex subscriptions are made
2 parents 61303c1 + 47fb2f0 commit 82186c3

File tree

8 files changed

+153
-172
lines changed

8 files changed

+153
-172
lines changed

‎src/python/T0/RunConfig/RunConfigAPI.py‎

Lines changed: 85 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -225,8 +225,7 @@ def configureRunStream(tier0Config, run, stream, specDirectory, dqmUploadProxy):
225225
#
226226
# for PhEDEx subscription settings
227227
#
228-
subscriptions = { 'Express' : [],
229-
'Bulk' : [] }
228+
subscriptions = []
230229

231230
# some hardcoded PhEDEx defaults
232231
expressPhEDExInjectNode = "T2_CH_CERN"
@@ -284,16 +283,15 @@ def configureRunStream(tier0Config, run, stream, specDirectory, dqmUploadProxy):
284283

285284
bindsPhEDExConfig.append( { 'RUN' : run,
286285
'PRIMDS' : specialDataset,
287-
'NODE' : expressPhEDExSubscribeNode,
288-
'CUSTODIAL' : 1,
289-
'REQ_ONLY' : "n",
290-
'PRIO' : "high" } )
286+
'ARCHIVAL_NODE' : None,
287+
'TAPE_NODE' : None,
288+
'DISK_NODE' : expressPhEDExSubscribeNode } )
291289

292-
subscriptions['Express'].append( { 'custodialSites' : [],
293-
'nonCustodialSites' : [expressPhEDExSubscribeNode],
294-
'autoApproveSites' : [expressPhEDExSubscribeNode],
295-
'priority' : "high",
296-
'primaryDataset' : specialDataset } )
290+
subscriptions.append( { 'custodialSites' : [],
291+
'nonCustodialSites' : [ expressPhEDExSubscribeNode ],
292+
'autoApproveSites' : [ expressPhEDExSubscribeNode ],
293+
'priority' : "high",
294+
'primaryDataset' : specialDataset } )
297295

298296
alcaSkim = None
299297
if "ALCARECO" in streamConfig.Express.DataTiers:
@@ -380,49 +378,32 @@ def configureRunStream(tier0Config, run, stream, specDirectory, dqmUploadProxy):
380378
'selectEvents' : selectEvents,
381379
'primaryDataset' : dataset } )
382380

381+
bindsPhEDExConfig.append( { 'RUN' : run,
382+
'PRIMDS' : dataset,
383+
'ARCHIVAL_NODE' : datasetConfig.ArchivalNode,
384+
'TAPE_NODE' : datasetConfig.TapeNode,
385+
'DISK_NODE' : datasetConfig.DiskNode } )
386+
383387
custodialSites = []
384-
nonCustodialSites = []
385388
autoApproveSites = []
386-
387-
if datasetConfig.CustodialNode != None:
388-
389-
custodialSites.append(datasetConfig.CustodialNode)
390-
391-
requestOnly = "y"
392-
if datasetConfig.CustodialAutoApprove:
393-
requestOnly = "n"
394-
autoApproveSites.append(datasetConfig.CustodialNode)
395-
396-
bindsStorageNode.append( { 'NODE' : datasetConfig.CustodialNode } )
397-
398-
bindsPhEDExConfig.append( { 'RUN' : run,
399-
'PRIMDS' : dataset,
400-
'NODE' : datasetConfig.CustodialNode,
401-
'CUSTODIAL' : 1,
402-
'REQ_ONLY' : requestOnly,
403-
'PRIO' : datasetConfig.CustodialPriority } )
404-
405389
if datasetConfig.ArchivalNode != None:
406-
390+
bindsStorageNode.append( { 'NODE' : datasetConfig.ArchivalNode } )
407391
custodialSites.append(datasetConfig.ArchivalNode)
408392
autoApproveSites.append(datasetConfig.ArchivalNode)
409-
410-
bindsStorageNode.append( { 'NODE' : datasetConfig.ArchivalNode } )
411-
412-
bindsPhEDExConfig.append( { 'RUN' : run,
413-
'PRIMDS' : dataset,
414-
'NODE' : datasetConfig.ArchivalNode,
415-
'CUSTODIAL' : 1,
416-
'REQ_ONLY' : "n",
417-
'PRIO' : datasetConfig.CustodialPriority } )
418-
419-
if len(custodialSites) + len(nonCustodialSites) > 0:
420-
subscriptions['Bulk'].append( { 'custodialSites' : custodialSites,
421-
'custodialSubType' : "Replica",
422-
'nonCustodialSites' : nonCustodialSites,
423-
'autoApproveSites' : autoApproveSites,
424-
'priority' : datasetConfig.CustodialPriority,
425-
'primaryDataset' : dataset } )
393+
if datasetConfig.TapeNode != None:
394+
bindsStorageNode.append( { 'NODE' : datasetConfig.TapeNode } )
395+
custodialSites.append(datasetConfig.TapeNode)
396+
if datasetConfig.DiskNode != None:
397+
bindsStorageNode.append( { 'NODE' : datasetConfig.DiskNode } )
398+
399+
if len(custodialSites) > 0:
400+
subscriptions.append( { 'custodialSites' : custodialSites,
401+
'custodialSubType' : "Replica",
402+
'nonCustodialSites' : [],
403+
'autoApproveSites' : autoApproveSites,
404+
'priority' : "high",
405+
'primaryDataset' : dataset,
406+
'dataTier' : "RAW" } )
426407

427408
elif streamConfig.ProcessingStyle == "Express":
428409

@@ -436,16 +417,15 @@ def configureRunStream(tier0Config, run, stream, specDirectory, dqmUploadProxy):
436417

437418
bindsPhEDExConfig.append( { 'RUN' : run,
438419
'PRIMDS' : dataset,
439-
'NODE' : expressPhEDExSubscribeNode,
440-
'CUSTODIAL' : 1,
441-
'REQ_ONLY' : "n",
442-
'PRIO' : "high" } )
443-
444-
subscriptions['Express'].append( { 'custodialSites' : [],
445-
'nonCustodialSites' : [expressPhEDExSubscribeNode],
446-
'autoApproveSites' : [expressPhEDExSubscribeNode],
447-
'priority' : "high",
448-
'primaryDataset' : dataset } )
420+
'ARCHIVAL_NODE' : None,
421+
'TAPE_NODE' : None,
422+
'DISK_NODE' : expressPhEDExSubscribeNode } )
423+
424+
subscriptions.append( { 'custodialSites' : [],
425+
'nonCustodialSites' : [ expressPhEDExSubscribeNode ],
426+
'autoApproveSites' : [ expressPhEDExSubscribeNode ],
427+
'priority' : "high",
428+
'primaryDataset' : dataset } )
449429

450430
#
451431
# finally create WMSpec
@@ -545,13 +525,13 @@ def configureRunStream(tier0Config, run, stream, specDirectory, dqmUploadProxy):
545525
factory = RepackWorkloadFactory()
546526
wmSpec = factory.factoryWorkloadConstruction(workflowName, specArguments)
547527
wmSpec.setPhEDExInjectionOverride(runInfo['bulk_data_loc'])
548-
for subscription in subscriptions['Bulk']:
528+
for subscription in subscriptions:
549529
wmSpec.setSubscriptionInformation(**subscription)
550530
elif streamConfig.ProcessingStyle == "Express":
551531
factory = ExpressWorkloadFactory()
552532
wmSpec = factory.factoryWorkloadConstruction(workflowName, specArguments)
553533
wmSpec.setPhEDExInjectionOverride(expressPhEDExInjectNode)
554-
for subscription in subscriptions['Express']:
534+
for subscription in subscriptions:
555535
wmSpec.setSubscriptionInformation(**subscription)
556536

557537
if streamConfig.ProcessingStyle in [ 'Bulk', 'Express' ]:
@@ -726,28 +706,55 @@ def releasePromptReco(tier0Config, specDirectory, dqmUploadProxy):
726706
'SCRAM_ARCH' : datasetConfig.ScramArch,
727707
'GLOBAL_TAG' : datasetConfig.GlobalTag } )
728708

729-
phedexConfig = phedexConfigs.get(dataset, {})
730-
731-
custodialSites = []
732-
nonCustodialSites = []
733-
autoApproveSites = []
709+
phedexConfig = phedexConfigs[dataset]
734710

735-
for node, config in phedexConfig.items():
711+
if datasetConfig.WriteAOD:
736712

737-
if config['custodial'] == 1:
738-
custodialSites.append(node)
739-
else:
740-
nonCustodialSites.append(node)
713+
custodialSites = []
714+
nonCustodialSites = []
715+
autoApproveSites = []
741716

742-
if config['request_only'] == "n":
743-
autoApproveSites.append(node)
717+
if phedexConfig['tape_node'] != None:
718+
custodialSites.append(phedexConfig['tape_node'])
719+
if phedexConfig['disk_node'] != None:
720+
nonCustodialSites.append(phedexConfig['disk_node'])
721+
autoApproveSites.append(phedexConfig['disk_node'])
744722

745-
if len(custodialSites) + len(nonCustodialSites) > 0:
746723
subscriptions.append( { 'custodialSites' : custodialSites,
747-
'nonCustodialSites' : nonCustodialSites,
748-
'autoApproveSites' : autoApproveSites,
749-
'priority' : config['priority'],
750-
'primaryDataset' : dataset } )
724+
'custodialSubType' : "Replica",
725+
'nonCustodialSites' : nonCustodialSites,
726+
'autoApproveSites' : autoApproveSites,
727+
'priority' : "high",
728+
'primaryDataset' : dataset,
729+
'dataTier' : "AOD" } )
730+
731+
if len(datasetConfig.AlcaSkims) > 0:
732+
if phedexConfig['tape_node'] != None:
733+
subscriptions.append( { 'custodialSites' : [phedexConfig['tape_node']],
734+
'custodialSubType' : "Replica",
735+
'nonCustodialSites' : [],
736+
'autoApproveSites' : [],
737+
'priority' : "high",
738+
'primaryDataset' : dataset,
739+
'dataTier' : "ALCARECO" } )
740+
if datasetConfig.WriteDQM:
741+
if phedexConfig['tape_node'] != None:
742+
subscriptions.append( { 'custodialSites' : [phedexConfig['tape_node']],
743+
'custodialSubType' : "Replica",
744+
'nonCustodialSites' : [],
745+
'autoApproveSites' : [],
746+
'priority' : "high",
747+
'primaryDataset' : dataset,
748+
'dataTier' : "DQM" } )
749+
750+
if datasetConfig.WriteRECO:
751+
if phedexConfig['disk_node'] != None:
752+
subscriptions.append( { 'custodialSites' : [],
753+
'nonCustodialSites' : [phedexConfig['disk_node']],
754+
'autoApproveSites' : [phedexConfig['disk_node']],
755+
'priority' : "high",
756+
'primaryDataset' : dataset,
757+
'dataTier' : "RECO" } )
751758

752759
writeTiers = []
753760
if datasetConfig.WriteRECO:

‎src/python/T0/RunConfig/Tier0Config.py‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -160,14 +160,11 @@
160160
|--> RecoDelayOffset - Time before PromptReco is released for which
161161
| settings are locked in and reco looks released
162162
|
163-
|--> CustodialNode - The custodial PhEDEx storage node for this dataset
164-
|
165163
|--> ArchivalNode - The archival PhEDEx node (should always be CERN T0)
166164
|
167-
|--> CustodialPriority - The priority of the custodial subscription
165+
|--> TapeNode - The tape PhEDEx node (should be T1 _MSS)
168166
|
169-
|--> CustodialAutoApprove - Determine whether or not the custodial
170-
| subscription will be auto approved.
167+
|--> DiskNode - The disk PhEDEx node (should be T1 _Disk)
171168
|
172169
|--> ProcessingVersion - Used for all output from PromptReco
173170
|
@@ -371,14 +368,14 @@ def addDataset(config, datasetName, **settings):
371368
msg = "Tier0Config.addDataset : no write_dqm defined for dataset %s or Default" % datasetName
372369
raise RuntimeError, msg
373370

374-
#
375-
# some optional parameters, Default rules are still used
376-
#
377371
if hasattr(datasetConfig, "ArchivalNode"):
378372
datasetConfig.ArchivalNode = settings.get('archival_node', datasetConfig.ArchivalNode)
379373
else:
380374
datasetConfig.ArchivalNode = settings.get('archival_node', None)
381375

376+
#
377+
# optional parameter, Default rule is still used
378+
#
382379
if hasattr(datasetConfig, "BlockCloseDelay"):
383380
datasetConfig.BlockCloseDelay = settings.get("blockCloseDelay", datasetConfig.BlockCloseDelay)
384381
else:
@@ -390,9 +387,8 @@ def addDataset(config, datasetName, **settings):
390387
datasetConfig.AlcaSkims = settings.get("alca_producers", [])
391388
datasetConfig.DqmSequences = settings.get("dqm_sequences", [])
392389

393-
datasetConfig.CustodialNode = settings.get("custodial_node", None)
394-
datasetConfig.CustodialPriority = settings.get("custodial_priority", "high")
395-
datasetConfig.CustodialAutoApprove = settings.get("custodial_auto_approve", False)
390+
datasetConfig.TapeNode = settings.get("tape_node", None)
391+
datasetConfig.DiskNode = settings.get("disk_node", None)
396392

397393
return
398394

@@ -571,6 +567,9 @@ def addRepackConfig(config, streamName, **options):
571567
else:
572568
streamConfig.Repack.BlockCloseDelay = options.get("blockCloseDelay", 24 * 3600)
573569

570+
streamConfig.Repack.TapeNodes = options.get("tapeNodes", [])
571+
streamConfig.Repack.DiskNodes = options.get("diskNodes", [])
572+
574573
return
575574

576575
def addExpressConfig(config, streamName, **options):

‎src/python/T0/WMBS/Oracle/Create.py‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -326,13 +326,12 @@ def __init__(self, logger = None, dbi = None, params = None):
326326

327327
self.create[len(self.create)] = \
328328
"""CREATE TABLE phedex_config (
329-
run_id int not null,
330-
primds_id int not null,
331-
node_id int not null,
332-
custodial int not null,
333-
request_only char(1) not null,
334-
priority varchar2(10) not null,
335-
primary key (run_id, primds_id, node_id)
329+
run_id int not null,
330+
primds_id int not null,
331+
archival_node_id int,
332+
tape_node_id int,
333+
disk_node_id int,
334+
primary key (run_id, primds_id)
336335
) ORGANIZATION INDEX"""
337336

338337
self.create[len(self.create)] = \
@@ -762,8 +761,20 @@ def __init__(self, logger = None, dbi = None, params = None):
762761

763762
self.constraints[len(self.constraints)] = \
764763
"""ALTER TABLE phedex_config
765-
ADD CONSTRAINT phe_con_nod_id_fk
766-
FOREIGN KEY (node_id)
764+
ADD CONSTRAINT phe_con_arc_nod_id_fk
765+
FOREIGN KEY (archival_node_id)
766+
REFERENCES storage_node(id)"""
767+
768+
self.constraints[len(self.constraints)] = \
769+
"""ALTER TABLE phedex_config
770+
ADD CONSTRAINT phe_con_tap_nod_id_fk
771+
FOREIGN KEY (tape_node_id)
772+
REFERENCES storage_node(id)"""
773+
774+
self.constraints[len(self.constraints)] = \
775+
"""ALTER TABLE phedex_config
776+
ADD CONSTRAINT phe_con_dis_nod_id_fk
777+
FOREIGN KEY (disk_node_id)
767778
REFERENCES storage_node(id)"""
768779

769780
self.constraints[len(self.constraints)] = \

‎src/python/T0/WMBS/Oracle/RunConfig/GetPhEDExConfig.py‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,19 +14,19 @@ class GetPhEDExConfig(DBFormatter):
1414
def execute(self, run, conn = None, transaction = False):
1515

1616
sql = """SELECT primary_dataset.name,
17-
storage_node.name,
18-
phedex_config.custodial,
19-
phedex_config.request_only,
20-
phedex_config.priority
21-
FROM run_primds_stream_assoc
22-
INNER JOIN phedex_config ON
23-
phedex_config.run_id = run_primds_stream_assoc.run_id AND
24-
phedex_config.primds_id = run_primds_stream_assoc.primds_id
17+
archival_node.name,
18+
tape_node.name,
19+
disk_node.name
20+
FROM phedex_config
2521
INNER JOIN primary_dataset ON
2622
primary_dataset.id = phedex_config.primds_id
27-
INNER JOIN storage_node ON
28-
storage_node.id = phedex_config.node_id
29-
WHERE run_primds_stream_assoc.run_id = :RUN
23+
LEFT OUTER JOIN storage_node archival_node ON
24+
archival_node.id = phedex_config.archival_node_id
25+
LEFT OUTER JOIN storage_node tape_node ON
26+
tape_node.id = phedex_config.tape_node_id
27+
LEFT OUTER JOIN storage_node disk_node ON
28+
disk_node.id = phedex_config.disk_node_id
29+
WHERE phedex_config.run_id = :RUN
3030
"""
3131

3232
binds = { 'RUN' : run }
@@ -38,14 +38,12 @@ def execute(self, run, conn = None, transaction = False):
3838
for result in results:
3939

4040
primds = result[0]
41-
node = result[1]
4241

4342
if not resultDict.has_key(primds):
4443
resultDict[primds] = {}
4544

46-
resultDict[primds][node] = {}
47-
resultDict[primds][node]['custodial'] = result[2]
48-
resultDict[primds][node]['request_only'] = result[3]
49-
resultDict[primds][node]['priority'] = result[4]
45+
resultDict[primds]['archival_node'] = result[1]
46+
resultDict[primds]['tape_node'] = result[2]
47+
resultDict[primds]['disk_node'] = result[3]
5048

5149
return resultDict

0 commit comments

Comments
 (0)