Support for sending hadd jobs to condor

Tools/python/CRAB3ToolsSh.py

@@ -5,6 +5,7 @@
 import re
 from JMTucker.Tools.CRAB3ToolsBase import *
 from JMTucker.Tools.hadd import HaddBatchResult, hadd
+from JMTucker.Tools.hadd_on_condor import hadd_on_condor
 if crab_global_options.support_automatic_splitting:
     from JMTucker.Tools.Sample import fn_to_sample, norm_from_file
     from JMTucker.Tools import Samples, colors
@@ -106,10 +107,10 @@ def crab_hadd_files(working_dir, lpc_shortcut=False, **kwargs):
 
     return expected, files
 
-def crab_hadd(working_dir, new_name=None, new_dir=None, raise_on_empty=False, chunk_size=900, pattern=None, lpc_shortcut=False, range_filter=None):
+def crab_hadd(working_dir, new_name=None, new_dir=None, raise_on_empty=False, chunk_size=900, pattern=None, lpc_shortcut=False, range_filter=None, submit=False, cmssw_tar_path=''):
     working_dir, new_name, new_dir = crab_hadd_args(working_dir, new_name, new_dir)
     expected, files = crab_hadd_files(working_dir, lpc_shortcut, range_filter=range_filter)
-    result = HaddBatchResult('crab', working_dir, new_name, new_dir, expected, files)
+    result = HaddBatchResult('crab', working_dir, new_name, new_dir, expected, files, submit)
     print '%s: expecting %i files if all jobs succeeded' % (working_dir, expected)
 
     if pattern:
@@ -157,8 +158,11 @@ def crab_hadd(working_dir, new_name=None, new_dir=None, raise_on_empty=False, ch
         if result.success and not new_name.startswith('root://'):
             os.chmod(new_name, 0644)
     else:
-        result.success = hadd(new_name, files)
-
+        if not submit:
+            result.success = hadd(new_name, files)
+        else:
+            result.success = hadd_on_condor(new_name, working_dir, cmssw_tar_path, files)
+
     if automatic_splitting:
         n = norm_from_file(new_name)
         sn, s = fn_to_sample(Samples, new_name)

Tools/python/CondorTools.py

@@ -9,6 +9,7 @@
 from JMTucker.Tools.LumiJSONTools import fjr2ll
 from JMTucker.Tools.general import sub_popen, touch
 from JMTucker.Tools.hadd import HaddBatchResult, hadd
+from JMTucker.Tools.hadd_on_condor import hadd_on_condor
 from JMTucker.Tools import colors
 
 class CSHelpersException(Exception):
@@ -313,10 +314,10 @@ def cs_hadd_files(working_dir, **kwargs):
         files = files[a:b:c]
     return expected, files
 
-def cs_hadd(working_dir, new_name=None, new_dir=None, raise_on_empty=False, chunk_size=900, pattern=None, range_filter=None):
+def cs_hadd(working_dir, new_name=None, new_dir=None, raise_on_empty=False, chunk_size=900, pattern=None, range_filter=None, submit=False, cmssw_tar_path=''):
     working_dir, new_name, new_dir = cs_hadd_args(working_dir, new_name, new_dir)
     expected, files = cs_hadd_files(working_dir, range_filter=range_filter)
-    result = HaddBatchResult('condor', working_dir, new_name, new_dir, expected, files)
+    result = HaddBatchResult('condor', working_dir, new_name, new_dir, expected, files, submit)
     print '%s: expecting %i files if all jobs succeeded' % (working_dir, expected)
 
     if pattern:
@@ -351,8 +352,11 @@ def cs_hadd(working_dir, new_name=None, new_dir=None, raise_on_empty=False, chun
         if result.success and not new_name.startswith('root://'):
             os.chmod(new_name, 0644)
     else:
-        result.success = hadd(new_name, files)
-
+        if not submit:
+            result.success = hadd(new_name, files)
+        else:
+            result.success = hadd_on_condor(new_name, working_dir, cmssw_tar_path, files)
+
     return result
 
 def cs_report(wd, partial=False):

Tools/python/hadd.py

@@ -1,18 +1,20 @@
 #!/usr/bin/env python
 
 import os, subprocess, tempfile, re
+import ROOT 
 from datetime import datetime
 from JMTucker.Tools import colors, eos
 
 class HaddBatchResult(object):
-    def __init__(self, kind, working_dir, new_name, new_dir, expected, files):
+    def __init__(self, kind, working_dir, new_name, new_dir, expected, files, submit):
         self.success = True
         self.kind = kind
         self.working_dir = working_dir
         self.new_name = new_name
         self.new_dir = new_dir
         self.expected = expected
         self.files = files
+        self.submit = submit
 
 class HaddlogParser(object):
     target_re = re.compile(r'hadd Target file: (.*)')
@@ -47,7 +49,6 @@ def hadd(output_fn, input_fns):
     other problems reported by hadd. If so, prints an error to
     stdout. Returns true if success.
     """
-
     l = len(input_fns)
     start = datetime.now()
     print 'hadding %i files to %s at %s' % (l, output_fn, start)
@@ -56,7 +57,7 @@ def hadd(output_fn, input_fns):
     p = subprocess.Popen(args=args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     stdout, stderr = p.communicate()
     assert stderr is None
-
+    
     log_fn = output_fn + '.haddlog'
     is_eos = '/store/' in output_fn # ugh
     while eos.exists(log_fn) if is_eos else os.path.exists(log_fn):

Tools/python/hadd_on_condor.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import subprocess
+import stat
+from JMTucker.Tools.CMSSWTools import cmssw_base
+
+def make_bash_file(cmssw_tar_path, bash_file_path):
+  cmssw_name = cmssw_base().rsplit("/")[-1]
+  bash_meat = '''#!/bin/bash
+workdir=$(pwd)
+echo "Starting job on " `date` #Date/time of start of job
+echo "Running on: `uname -a`" #Condor job is running on this node
+echo "System software: `cat /etc/redhat-release`" #Operating System on that node
+export SCRAM_ARCH=__SCRAM_ARCH__
+source /cvmfs/cms.cern.ch/cmsset_default.sh
+scram project CMSSW __CMSSW_VERSION__ 2>&1 > /dev/null
+scramexit=$?
+if [[ $scramexit -ne 0 ]]; then
+    echo scram exited with code $scramexit
+    exit $scramexit
+fi
+cd __CMSSW_VERSION__
+tar xf ${workdir}/input.tgz
+cd src
+eval `scram ru -sh`
+scram b -j 2 2>&1 > /dev/null
+
+#python JMTucker/Tools/python/hadd.py $1 ${@:2}
+cd ${workdir}
+hadd.py $1 ${@:2}
+''' \
+.replace('__SCRAM_ARCH__',    os.environ['SCRAM_ARCH']) \
+.replace('__CMSSW_VERSION__', os.environ['CMSSW_VERSION'])
+
+  bash_file = open(bash_file_path+'/bash_condor.sh', 'w')
+  bash_file.write(bash_meat)
+  bash_file.close()
+  subprocess.check_call(['chmod', '+x', bash_file_path+'/bash_condor.sh'])
+  return
+
+
+def hadd_on_condor(new_name, job_dir, cmssw_tar_path, input_files):
+  output_file_path = os.path.dirname(os.path.abspath(new_name))
+  new_nice_name = (new_name.rsplit("/")[-1]).replace('.root','')
+  job_dir = os.path.abspath(job_dir)
+  cmssw_tar_path = os.path.abspath(cmssw_tar_path)
+  is_eos = '/store/' in new_name
+  if not is_eos:
+    new_name = new_nice_name+'.root'
+
+  hadd_job_dir = job_dir+'/mhadd/'
+  if not os.path.exists(hadd_job_dir):
+    os.mkdir(hadd_job_dir)
+
+  make_bash_file(cmssw_tar_path, hadd_job_dir)
+
+  #requirements necessary for wisconsin machines -- comment out if not needed
+  #    the CentOS == 7 requires proxy with 2048 bit and will fail with usual 1024
+  #    voms-proxy-init -rfc -valid 144:00 -voms cms -bits 2048
+  condorFile = open(hadd_job_dir+'/submit_'+new_nice_name, 'w')
+  condorFile.write('universe              = vanilla\n')
+  condorFile.write('executable            = '+hadd_job_dir+'/bash_condor.sh\n')
+  condorFile.write('arguments             = '+new_name+' ' + ' '.join(input_files) + '\n')
+  condorFile.write('log                   = '+hadd_job_dir+'/hadd.log\n')
+  condorFile.write('output                = '+hadd_job_dir+'/hadd.out\n')
+  condorFile.write('error                 = '+hadd_job_dir+'/hadd.err\n')
+  condorFile.write('requirements = TARGET.HAS_OSG_WN_CLIENT =?= TRUE\n')
+  condorFile.write('requirements = TARGET.OpSysMajorVer == 7\n')
+  condorFile.write('should_transfer_files = yes\n')
+  condorFile.write('when_to_transfer_output = ON_EXIT\n')
+  condorFile.write('transfer_input_files  = '+cmssw_tar_path+'\n')
+
+  if not is_eos:
+    condorFile.write('Transfer_Output_Files = '+new_nice_name+'.root, '+new_nice_name+'.root.haddlog'+'\n')
+
+  condorFile.write('queue\n')
+  condorFile.close()
+
+  submit_path = os.getcwd()
+  if not is_eos:
+    submit_path = output_file_path
+  args = ['condor_submit '+hadd_job_dir+'/submit_'+new_nice_name]
+  p = subprocess.Popen(args =args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, cwd=submit_path)
+
+  print p.stdout.read()
+
+  return p
+
+_all__ = [
+    'make_bash_file',
+    'hadd_on_condor',
+    ]
+
+if __name__ == '__main__':
+
+  hadd_on_condor(sys.argv[1], sys.argv[2], sys.argv[3:])

Tools/scripts/mhadd

@@ -1,11 +1,12 @@
 #!/usr/bin/env python
 
-import sys, os, glob, argparse
+import sys, os, glob, argparse, subprocess
 from JMTucker.Tools.CRAB3ToolsSh import is_crab_working_dir, crab_hadd_args, crab_hadd_files, crab_hadd
 from JMTucker.Tools.CondorTools import is_cs_dir, cs_hadd_args, cs_hadd_files, cs_hadd
 from JMTucker.Tools.general import touch as _touch
 from JMTucker.Tools.hadd import HaddlogParser
 from JMTucker.Tools import eos, colors
+from JMTucker.Tools.CMSSWTools import cmssw_base, make_tarball
 
 parser = argparse.ArgumentParser(description = 'mhadd: hadd the files from condor/crab directories',
                                  usage = '%(prog)s [options] condor_or_crab_dirs')
@@ -32,6 +33,8 @@ parser.add_argument('-r', '--remove-originals', action='store_true', default=Fal
                     help='Remove the original files.')
 parser.add_argument('-v', '--verbose', action='store_true', default=False,
                     help='Print more info.')
+parser.add_argument('-s', '--submit', action='store_true', default=False,
+                    help='submits hadd jobs to condor')
 
 options = parser.parse_args()
 
@@ -51,6 +54,9 @@ if options.only_notdones:
     options.ignore_done = True
     options.overwrite = True
 
+if options.submit:
+    make_tarball('%s/input.tgz'  % (os.getcwd()), include_python=True, include_interface=True)
+
 ########################################################################
 
 wds = options.positional
@@ -61,7 +67,7 @@ def hadd_(is_crab, *args, **kwargs):
     found_one = True
 
     done = True
-    working_dir = args[0]
+    working_dir = os.path.abspath(args[0])
     if not os.path.isfile(os.path.join(working_dir, 'mmon_done')):
         if options.ignore_done:
             done = False
@@ -72,7 +78,11 @@ def hadd_(is_crab, *args, **kwargs):
     kwargs['chunk_size'] = options.chunk_size
     kwargs['pattern'] = options.pattern
     kwargs['range_filter'] = options.range
+    kwargs['submit'] = options.submit
 
+    if options.submit:
+        kwargs['cmssw_tar_path'] = '%s/input.tgz' % (os.getcwd())
+
     if is_crab:
         kwargs['lpc_shortcut'] = options.lpc_shortcut
         hadd_args, hadd_files, hadd = crab_hadd_args, crab_hadd_files, crab_hadd
@@ -111,13 +121,18 @@ def hadd_(is_crab, *args, **kwargs):
         if options.overwrite:
             print colors.yellow('overwriting existing file %s' % new_name)
             rm(new_name)
+
         else:
             log_fn = new_name + '.haddlog'
+            log_fn_submit = new_name.replace('.root', '') + '.log'
+
             if on_eos:
                 log_fn = eos.fusemount(log_fn)
             njobs = hadd_files(working_dir, **kwargs)[0]
+
             if not exists(log_fn):
                 log_njobs = 1 # assume 1 file copied
+
             else:
                 p = HaddlogParser(log_fn)
                 log_njobs = p.num_sources
@@ -127,6 +142,7 @@ def hadd_(is_crab, *args, **kwargs):
                 if options.verbose:
                     print colors.yellow('skipping existing file %s (which looks complete by njobs and haddlog)' % new_name)
             return
+
 
     result = hadd(*args, **kwargs)
 
@@ -139,6 +155,7 @@ def hadd_(is_crab, *args, **kwargs):
 
 
 for x in wds:
+    x = os.path.abspath(x)
     is_crab = is_crab_working_dir(x)
     if is_crab or is_cs_dir(x):
         hadd_(is_crab, x)