Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 397ac2c

Browse files
ameregheamereghe
committed
Merge branch 'master' into fix_validator_flagging_runtime_outliers
2 parents 11ce6b1 + 922314b commit 397ac2c

File tree

9 files changed

+547
-77
lines changed

9 files changed

+547
-77
lines changed

‎boinc_software/cronjobs/acrontab_jobs_sixtadm.txt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# Puppet Name: worksubmit sixtrack 2
55
7,17,27,37,47,57 * * * * boincai11.cern.ch /usr/local/boinc/project/sixtrack/bin/cron.submit-sixtracktest-simo3 -m 3000 > /dev/null 2>&1
66
# Puppet Name: worksubmit sixtrack
7-
1,11,21,31,41,51 * * * * boincai11.cern.ch /usr/local/boinc/project/sixtrack/bin/cron.submit-sixtrack-simo3 -m 3000 > /dev/null 2>&1
7+
1 * * * * boincai11.cern.ch /usr/local/boinc/project/sixtrack/bin/cron.submit-sixtrack-simo3 -m 3000 > /dev/null 2>&1
88
# zip all WUs which were not given back to user
99
# crontab every 3h
1010
30 */3 * * * boincai11.cern.ch cd /share/sixtrack/assimilation ; /usr/local/boinc/project/sixtrack/bin/zip-trashed-WUs.sh > /dev/null 2>&1
@@ -32,6 +32,9 @@
3232
0 4 21 * * lxplus.cern.ch cd /afs/cern.ch/work/b/boinc/boinc ; /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/cronjobs/listDeleteStudies.sh results >> /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/cronjobs/deleteStudies.log 2>&1
3333
0 5 21 * * lxplus.cern.ch cd /afs/cern.ch/work/b/boinc/boinctest ; /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/cronjobs/listDeleteStudies.sh results >> /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/cronjobs/deleteStudies.log 2>&1
3434
#
35+
# tar.gz new job and spool it to EOS
36+
4,14,24,34,44,54 * * * * lxplus.cern.ch /afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/cronjobs/mv2EOS.sh > /dev/null 2>&1
37+
#
3538
#
3639
# new scripts for monitoring BOINC server:
3740
# - server status and plotting;

‎boinc_software/cronjobs/cron.submit-simo3‎

Lines changed: 254 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
# Updated 06.10.2016 A.Mereghetti, added megaZip functionality
88
# Updated 25.03.2017 A.Mereghetti, SixOut.zip + appname (from .desc)
99
# Updated 01.05.2019 A.Mereghetti, appnum + default appNum
10+
# Updated 13.09.2019 A.Mereghetti, EOS spooling
1011
set -e
1112

1213
#Set Variables
@@ -37,6 +38,13 @@ WU_template=templates/${applicationDef}_wu_template.xml
3738
# this template will bring back also the Sixout.zip results
3839
result_template_SixOutZip=templates/${applicationDef}_res_template.xml
3940

41+
export EOS_MGM_URL=root://eosuser.cern.ch
42+
pathInEos=/eos/user/s/sixtadm/spooldirs/uploads/boinc
43+
tmpDirBase=/tmp/sixtadm/`basename $0`/boinc
44+
nMaxRetrial=10
45+
46+
TLpath="/afs/cern.ch/work/s/sixtadm/public/monitor_activity/boinc_software/monitor-boinc-server/general-activity/SixTrack_status_????-??-??.dat"
47+
4048
#Define finctions
4149
abort(){
4250
exitcode=1
@@ -112,7 +120,8 @@ run_spool_megazip(){
112120

113121
megaZip=1
114122

115-
find "$spooldirUpload" -mmin +5 -name "*.zip" | (
123+
# take into account arrival time
124+
find "$spooldirUpload" -mmin +5 -name "*.zip" -printf "%T+\t%p\n" | sort | awk '{print ($2)}' | (
116125
while read workBatch ; do
117126
unzip -t $workBatch >/dev/null 2>&1
118127
if [ $? -ne 0 ] ; then
@@ -123,22 +132,25 @@ run_spool_megazip(){
123132
origPath=`mktemp -d -p /tmp/`
124133
warn "$workBatch being processed in $origPath ..."
125134
cp $workBatch $origPath
126-
# remember that you must run in $boincdir
127-
unzip $workBatch -d $origPath >/dev/null 2>&1
128-
# check for unzipped .desc files
129-
find $origPath -maxdepth 1 -type f -name '*.desc' | (
130-
while read descfile ; do
131-
# process the desc files
132-
if submit_descfile "$descfile" ; then
133-
logstudy "Submitted $WUname"
134-
else
135-
logstudy "Problem submitting $WUname"
136-
fi
137-
done
138-
)
139-
warn " ...going on with next MegaZip."
140-
rm -rf $origPath
141-
rm $workBatch
135+
nForeseen=`unzip -l ${workBatch} | wc -l`
136+
if traffic_light ; then
137+
# remember that you must run in $boincdir
138+
unzip $workBatch -d $origPath >/dev/null 2>&1
139+
# check for unzipped .desc files
140+
find $origPath -maxdepth 1 -type f -name '*.desc' | (
141+
while read descfile ; do
142+
# process the desc files
143+
if submit_descfile "$descfile" ; then
144+
logstudy "Submitted $WUname"
145+
else
146+
logstudy "Problem submitting $WUname"
147+
fi
148+
done
149+
)
150+
warn " ...going on with next MegaZip."
151+
rm -rf $origPath
152+
rm $workBatch
153+
fi
142154
done
143155
)
144156
}
@@ -158,7 +170,8 @@ run_spool(){ # max_jobs_to_submit, max_jobs_perStudy, specific_study
158170

159171
if [ -z "$3" ] ; then
160172
#find the work dirs 2 levels down
161-
local allWorkDirs=`find "$spooldir" -maxdepth 2 -type d -name "work"`
173+
# take into account arrival time
174+
local allWorkDirs=`find "$spooldir" -maxdepth 2 -type d -name "work" -printf "%T+\t%p\n" | sort | awk '{print ($2)}'`
162175
else
163176
#target a specific study
164177
local allWorkDirs=$spooldir/$3/work
@@ -168,39 +181,172 @@ run_spool(){ # max_jobs_to_submit, max_jobs_perStudy, specific_study
168181
for workdir in ${allWorkDirs} ; do
169182
! ${lMaxJobsPerStudy} || local __StudyComplete=0
170183
#check for desc files in the current work dir, and subfolders
184+
# take into account arrival time
171185
if ${lMaxJobsPerStudy} ; then
172-
local allDescs=`find "$workdir" -maxdepth 2 -type f -name '*.desc' | head -n ${max_jobs_perStudy}`
186+
local allDescs=`find "$workdir" -maxdepth 2 -type f -name '*.desc' -printf "%T+\t%p\n" | sort | awk '{print ($2)}' | head -n ${max_jobs_perStudy}`
187+
else
188+
local allDescs=`find "$workdir" -maxdepth 2 -type f -name '*.desc' -printf "%T+\t%p\n" | sort | awk '{print ($2)}'`
189+
fi
190+
nForeseen=`echo "${allDescs}" | wc -l`
191+
let nForeseen=${nForeseen}*2
192+
if traffic_light ; then
193+
for descfile in ${allDescs} ; do
194+
#process the desc files
195+
origPath=`dirname ${descfile}`
196+
! ${lMaxJobsPerStudy} || let __StudyComplete+=1
197+
if submit_descfile "$descfile" ; then
198+
logstudy "Submitted $WUname"
199+
#stop after max_jobs (0=unlimited)
200+
if ${lMaxJobs} ; then
201+
complete=$(( $complete + 1 ))
202+
if [ $complete -ge $max_jobs ] ; then
203+
log "reached ${max_jobs} in total"
204+
break 2
205+
fi
206+
fi
207+
else
208+
logstudy "Problem submitting $WUname"
209+
fi
210+
done
211+
if [ -n "${allDescs}" ] ; then
212+
if ${lMaxJobsPerStudy} ; then
213+
if [ ${__StudyComplete} -eq ${max_jobs_perStudy} ] ; then
214+
log "limit to ${max_jobs_perStudy} reached for ${workdir}"
215+
fi
216+
fi
217+
fi
218+
fi
219+
done
220+
}
221+
222+
run_spool_EOS(){
223+
224+
# get .tar.gz files from ${pathInEos} (path in EOS) and
225+
# ${spooldirUpload} (back-up in AFS in case of problems with EOS)
226+
227+
megaZip=0
228+
origPath=${tmpDirBase}
229+
230+
for __tmpSourceDir in ${pathInEos} ${spooldirUpload} ; do
231+
232+
if [[ "${__tmpSourceDir}" == "/eos"* ]] ; then
233+
gzFiles=`eos find -name "*.tar.gz" ${__tmpSourceDir}`
173234
else
174-
local allDescs=`find "$workdir" -maxdepth 2 -type f -name '*.desc'`
235+
gzFiles=`find ${__tmpSourceDir} -mmin +5 -name "*.tar.gz"`
175236
fi
176-
for descfile in ${allDescs} ; do
177-
#process the desc files
178-
origPath=`dirname ${descfile}`
179-
! ${lMaxJobsPerStudy} || let __StudyComplete+=1
180-
if submit_descfile "$descfile" ; then
181-
logstudy "Submitted $WUname"
182-
#stop after max_jobs (0=unlimited)
183-
if ${lMaxJobs} ; then
184-
complete=$(( $complete + 1 ))
185-
if [ $complete -ge $max_jobs ] ; then
186-
log "reached ${max_jobs} in total"
187-
break 2
188-
fi
237+
238+
log ".tar.gz files in ${__tmpSourceDir}:"
239+
echo "${gzFiles}" | log
240+
241+
for gzFile in ${gzFiles} ; do
242+
243+
gzFileName=`basename ${gzFile}`
244+
245+
if [[ "${__tmpSourceDir}" == "/eos"* ]] ; then
246+
myCommand="xrdcp --cksum adler32 ${EOS_MGM_URL}/${pathInEos}/${gzFileName} ${tmpDirBase}"
247+
loopMe
248+
if [ $? -ne 0 ] ; then
249+
log "unable to download ${gzFileName} from EOS ${EOS_MGM_URL}/${pathInEos}"
250+
continue
189251
fi
190252
else
191-
logstudy "Problem submitting $WUname"
253+
myCommand="cp ${gzFile} ${tmpDirBase}"
254+
loopMe
255+
if [ $? -ne 0 ] ; then
256+
log "unable to download ${gzFileName} from AFS ${spooldirUpload}"
257+
continue
258+
fi
192259
fi
193-
done
194-
if [ -n "${allDescs}" ] ; then
195-
if ${lMaxJobsPerStudy} ; then
196-
if [ ${__StudyComplete} -eq ${max_jobs_perStudy} ] ; then
197-
log "limit to ${max_jobs_perStudy} reached for ${workdir}"
198-
fi
199-
fi
260+
261+
spool_EOS_gunzip || continue
262+
263+
tarFile=${tmpDirBase}/${gzFileName%.gz}
264+
spool_EOS_untar || continue
265+
266+
spool_EOS_descfiles
267+
268+
# clean
269+
if [[ "${__tmpSourceDir}" == "/eos"* ]] ; then
270+
eos rm ${pathInEos}/${gzFileName}
271+
else
272+
rm ${gzFile}
273+
fi
274+
rm ${tarFile}
275+
done
276+
277+
done
278+
}
279+
280+
function spool_EOS_gunzip(){
281+
myCommand="gunzip ${tmpDirBase}/${gzFileName}"
282+
if loopMe ; then
283+
return 0
284+
else
285+
log "unable to gunzip ${gzFileName}"
286+
rm ${tmpDirBase}/${gzFileName}
287+
return 1
288+
fi
289+
}
290+
291+
function spool_EOS_untar(){
292+
nForeseen=`tar -tvf ${tarFile} | wc -l`
293+
# tar -tvf always returns a '.'
294+
nForeseen=$(( $nForeseen -1 ))
295+
if traffic_light ; then
296+
myCommand="tar -xvf ${tarFile} -C ${tmpDirBase}"
297+
loopMe
298+
if [ $? -ne 0 ] ; then
299+
log "unable to untar ${tarFile}"
300+
rm ${tarFile}
301+
return 1
200302
fi
303+
else
304+
rm ${tarFile}
305+
return 2
306+
fi
307+
return 0
308+
}
309+
310+
spool_EOS_descfiles(){
311+
# remember that you must run in $boincdir
312+
for descfile in `find "${tmpDirBase}" -type f -name '*.desc'` ; do
313+
#process the desc files
314+
if submit_descfile "$descfile" ; then
315+
logstudy "Submitted $WUname"
316+
else
317+
logstudy "Problem submitting $WUname"
318+
fi
201319
done
202320
}
203321

322+
function traffic_light(){
323+
# nPresent: how many tasks presently in queue
324+
# nTreated: how many tasks (factor 2 only wrt WUs) will be submitted now
325+
# 0: green
326+
# 1: red
327+
local __nPresentTemp=`tail -1 ${TLpath} 2> /dev/null | awk '{print ($3)}'`
328+
if [ -z "${__nPresentTemp}" ] ; then
329+
if [ -z "${nPresent}" ] ; then
330+
log "unable to get number of queued ${applicationDef} tasks from ${TLpath}"
331+
return 2
332+
else
333+
log "unable to get number of queued ${applicationDef} tasks from ${TLpath} - going on with ${nPresent}"
334+
fi
335+
else
336+
nPresent=${__nPresentTemp}
337+
fi
338+
local __nTemp=$(( ${nTreated} +${nPresent} +${nForeseen} ))
339+
if [ ${__nTemp} -ge ${currentLimit} ] ; then
340+
log "traffic light is red (treated+present+foreseen>limit): ${nTreated}+${nPresent}+${nForeseen}>${currentLimit}"
341+
nResidual=0
342+
return 1
343+
else
344+
log "traffic light is green (treated+present+foreseen>limit): ${nTreated}+${nPresent}+${nForeseen}<${currentLimit}"
345+
let nResidual=${currentLimit}+${__nTemp}
346+
return 0
347+
fi
348+
}
349+
204350
submit_descfile(){
205351
# variables from outer function:
206352
# - megaZip: flag for megaZip;
@@ -309,6 +455,7 @@ create_cmd="$boincdir/bin/create_work \
309455
fi
310456
# in case of megazip, zipfile will be deleted since the
311457
# tmp dir will be deleted
458+
nTreated=$(( $nTreated +2 ))
312459

313460
return 0
314461
}
@@ -406,6 +553,22 @@ clearWUvars(){
406553
unset WUapplication
407554
}
408555

556+
function loopMe(){
557+
# myCommand should be defined before the call
558+
local __reply=1
559+
local __iTrials=0
560+
while [ ${__reply} -ne 0 ] && [ ${__iTrials} -lt ${nMaxRetrial} ] ; do
561+
let __iTrials+=1
562+
log "command: ${myCommand} - run at `date` - trial ${__iTrials}"
563+
${myCommand}
564+
__reply=$?
565+
done
566+
if [ ${__reply} -ne 0 ] ; then
567+
log " ...giving up on command."
568+
fi
569+
return ${__reply}
570+
}
571+
409572
printhelp(){
410573
cat <<EOF
411574
Usage: $(basename $0) [options]
@@ -457,14 +620,58 @@ cd $boincdir
457620
getlock
458621
#Klog
459622

623+
megaZip=0
624+
nResidual=0
625+
460626
# preliminary: remove temp dirs in work older than 1d
461-
log find ${spooldir} -mindepth 3 -maxdepth 3 -mtime +1 -type d -empty -delete -print
462-
find ${spooldir} -mindepth 3 -maxdepth 3 -mtime +1 -type d -empty -delete -print | log
627+
# log find ${spooldir} -mindepth 3 -maxdepth 3 -mtime +1 -type d -empty -delete -print
628+
# find ${spooldir} -mindepth 3 -maxdepth 3 -mtime +1 -type d -empty -delete -print | log
463629

464-
log run_spool $maxjobs $maxjobs_perStudy $studyName
465-
run_spool $maxjobs $maxjobs_perStudy $studyName
630+
# traffic light
631+
currentLimit=`grep -v '#' ${boincdir}/queue_thresholds.txt | awk -v "appName=${applicationDef}" '{if ($1==appName) {print ($2)}}' | tail -1`
632+
if [ -z "${currentLimit}" ] ; then
633+
abort 1 "unable to get threshold of queued ${applicationDef} tasks from ${boincdir}/queue_thresholds.txt"
634+
fi
635+
nTreated=0
636+
nForeseen=0
637+
traffic_light || abort 1 "traffic light: red"
638+
639+
if ! [ -d ${tmpDirBase} ] ; then
640+
mkdir -p ${tmpDirBase}
641+
if [ $? -ne 0 ] ; then
642+
abort 1 "problems in creating ${tmpDirBase} - cannot proceed"
643+
fi
644+
fi
645+
origPath=${tmpDirBase}
646+
log "remaining .tar.gz files in ${tmpDirBase} from previous run"
647+
for gzFile in `find ${tmpDirBase} -name "*.tar.gz"` ; do
648+
gzFileName=`basename ${gzFile}`
649+
spool_EOS_gunzip || continue
650+
tarFile=${tmpDirBase}/${gzFileName%.gz}
651+
spool_EOS_untar || continue
652+
spool_EOS_descfiles
653+
rm ${tarFile}
654+
done
655+
log "remaining .tar files in ${tmpDirBase} from previous run"
656+
for tarFile in `find ${tmpDirBase} -name "*.tar"` ; do
657+
spool_EOS_untar || continue
658+
spool_EOS_descfiles
659+
rm ${tarFile}
660+
done
661+
log "remaining .desc files in ${tmpDirBase} from previous run"
662+
spool_EOS_descfiles
663+
664+
log run_spool_EOS
665+
run_spool_EOS
666+
667+
if [ ${nResidual} -gt 0 ] && [ ${maxjobs} -gt ${nResidual} ] ; then
668+
log "updating maxjobs from ${maxjobs} to ${nResidual} following limit on max number or queued tasks at ${currentLimit}"
669+
maxjobs=${nResidual}
670+
fi
671+
# log run_spool $maxjobs $maxjobs_perStudy $studyName
672+
# run_spool $maxjobs $maxjobs_perStudy $studyName
466673

467-
log run_spool_megazip
468-
run_spool_megazip
674+
# log run_spool_megazip
675+
# run_spool_megazip
469676

470677
log FINISHING

0 commit comments

Comments
 (0)