probe_history

#!/usr/bin/env python

import time
import argparse
import subprocess
import sys
import cStringIO
import re
import json
import urllib2

import pymongo


VERSION = '0.5'
GAE_URL = 'http://g-treadmill.appspot.com/testjobs'

int_re = re.compile(r'^\d+$')
float_re = re.compile(r'^\d+\.\d+$')
quoted_re = re.compile(r'^"(.*)"$')

def post_results(job_record):
    """
    Post json record of results to google app engine db
    """
    post_string = json.dumps(job_record)
    request = urllib2.Request(GAE_URL, data = post_string)
    try:
        f = urllib2.urlopen(request)
    except urllib2.URLEroor:
        sys.stderr.write("Got an error posting to {0}\n".format(GAE_URL))
    if f.getcode() != 200:
        sys.stderr.write("Got error posting to {0}: {1}\n".format(GAE_URL, f.getcode()))
    
        
def process_condor_history(db, history_output):
    """
    Process condor history and then insert it into the DB
    """
    history_records = db.history_records
    str_var = cStringIO.StringIO(history_output)
    job_record = {}
    job_count = 0
    line_count = 0
    duplicate_records = 0
    error_records = 0
    for line in str_var.readlines():
        line_count += 1
        if line.strip() != "":
            fields = line.split('=')
            field_val = "".join(fields[1:]).strip()
            # now try to convert the value to the correct type
            try:
                if int_re.match(field_val):
                    field_val = int(field_val)
                elif float_re.match(field_val):
                    field_val = float(field_val)
                elif quoted_re.match(field_val):
                    match = quoted_re.match(field_val)
                    field_val = match.group(1)
            except ValueError:
                pass
            job_record[fields[0].strip()] = field_val
        else:
            job_record['_id'] = "%s.%s" % (job_record['ClusterId'], job_record['ProcId'])
            try:
                record = history_records.find_one({"_id": job_record['_id']})
                if record is not None:
                    duplicate_records += 1
                    continue
                history_records.insert(job_record)
                post_results(job_record)
                job_count += 1
            except pymongo.errors.OperationFailure, e:
                error_records += 1
                sys.stderr.write("Got error inserting record into "
                                 "DB:\n%s\nRecord:%s\n" % (e, job_record))

            job_record = {}
    sys.stdout.write("Read %s lines from condor_history\n" % line_count)
    sys.stdout.write("Inserted %s records into db\n" % job_count)
    sys.stdout.write("Skipped %s duplicate records\n" % duplicate_records)
    sys.stdout.write("Encountered %s records with errors\n" % duplicate_records)


def get_condor_history(start_time=None, end_time=None):
    """
    Get the condor history for the specified time period
    """
    hist_command = ["/usr/bin/condor_history",
                    "-l",
                    "-constraint",
                    "((JobCurrentStartDate > %s) && "
                    "(JobCurrentStartDate < %s))" % (start_time, end_time)]
    proc = subprocess.Popen(hist_command, stdout = subprocess.PIPE)
    history_output  = proc.communicate()[0]
    if proc.returncode != 0:
      sys.stderr.write("Can't call condor_history")
      sys.exit(1)
    return history_output



def run_main():
    """
    Main function, parse arguments and then
    """
    parser = argparse.ArgumentParser(description='Insert condor job history into DB')
    parser.add_argument('--start', default=None,
                        help='String indicating start time in YYYY-MM-DD HH:MM:SS'
                             'format with hours going from 0 to 23 ')
    parser.add_argument('--end', default=None,
                        help='String indicating end time in YYYY-MM-DD HH:MM:SS'
                             'format with hours going from 0 to 23 ')
    parser.add_argument('--process_hours', type=int, help='Start time',
                        default=None)
#    parser.add_argument('config_file', nargs='2', default=None,
#                        help='Path to file with configuration information')
    parser.add_argument('--version', action='version', version=VERSION)
    args = parser.parse_args()
    # need to have either process_hours specified or start and end specified
    #
    if ((args.process_hours is None and
                 args.start is None and
                 args.end is None) or
            (args.process_hours is not None and
                     args.start is not None) or
            (args.process_hours is not None and
                     args.end is not None)):
        sys.stderr.write("Either process_hours or start and end need to be given\n")
        sys.exit(1)

    if args.process_hours:
        end_time = list(time.localtime())
        end_time[4] = 0 # zero out minutes field
        end_time[5] = 0 # same for seconds
        start_time = list(time.localtime())
        start_time[4] = 0 # zero out minutes field
        start_time[5] = 0 # same for seconds
        start_time[3] -= args.process_hours # go back specified number of hours
        end_time = int(round(time.mktime(end_time)))
        start_time = int(round(time.mktime(start_time)))
    else:
        start_time = int(round(time.mktime(time.strptime(args.start,
                                                         "%Y-%m-%D %H:%M:%S"))))
        end_time = int(round(time.mktime(time.strptime(args.end,
                                                       "%Y-%m-%D %H:%M:%S"))))

    db_client = pymongo.MongoClient(host='db.mwt2.org', port=27017)
    db = db_client.condor_history
    history_output = get_condor_history(start_time, end_time)
    process_condor_history(db, history_output)



if __name__ == '__main__':
    run_main()