distsim

#! /usr/bin/env python
 # -*- coding: utf-8 -*-
import argparse
import ConfigParser
import sys
import httplib, urllib
import shutil
import datetime
import os
import shutil

from neurogenesis.features import distsim_simulate, distsim_extract, distsim_plot, distsim_clean, distsim_retry, distsim_dump, distsim_notify
from neurogenesis.util import Logger, PrintColors

from time import sleep

def main():
    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument("command", help="command to be executed. Possible values are: simulate, extract, plot, clean")
    arg_parser.add_argument("--inifile", help="specifies the location of the ini file to be executed.")
    arg_parser.add_argument("--outdir", help="specifies the location where files should be written to.")
    arg_parser.add_argument("--inetdir", help='location of the inet executable on NFS share')
    arg_parser.add_argument("--omnetdir", help="location of the omnet executable on NFS share")
    arg_parser.add_argument("--additionalFiles",  help="additional Files which should be copied to the output dirs")
    arg_parser.add_argument("--nrRanks", help="Nr of ranks the simulation should be distributed to")
    arg_parser.add_argument("--mpiWorker", help="location of the MPI worker script")
    arg_parser.add_argument("--hostfile", help = "location of the host file")
    arg_parser.add_argument("--extractScalarsFile", help= "List of scalars to be extracted")
    arg_parser.add_argument("--extractVectorsFile", help= "List of vectors to be extracted")
    arg_parser.add_argument("--metaFile", help="Meta file where simulation metadata is stored")
    arg_parser.add_argument("--plotScript", help="location of the plot script")
    arg_parser.add_argument("--notificationKeyFile", help="location of the pushover keyfile")
    arg_parser.add_argument("--ignore", help="Token is ignored")
    arg_parser.add_argument("--name", help="name that is used to identify the simulation run later")
    arg_parser.add_argument("--configName", help="name of the config in ini to execute")
    arg_parser.add_argument("--notifyMsg", help="Message content of the notification to be sent to the mobile device")
    arg_parser.add_argument("--versionResults", help="specifies wether metafiles should be versioned separately")
    config = ConfigParser.ConfigParser()

    Logger.printColor(PrintColors.OKGREEN, "This is distsim distributed omnet/inet simulation environment.")

    try:
        config.read("config.ini")
    except ConfigParser.NoSectionError:
        Logger.info("No config.ini found.")

    args = arg_parser.parse_args()
    args = write_default_parameters(args, config)
    args = vars(args)


    command = args['command']
    if command == "simulate":
        import luigi
        from neurogenesis.task import RunSimulationTask
        tasks = []
        tasks.append(RunSimulationTask(args=args))

        while True:
                luigi.build(tasks, workers=1, local_scheduler=False)
                if not tasks[0].complete():
                        Logger.warning("Task could not be completed. Is the cluster busy? Retrying in 15 secs")
                        sleep(15)
                else:
                    break
        sim = tasks[0].result
        if(args["versionResults"]):
                distsim_extract(args, sim)
                try:
                    shutil.copy(args['inifile'], sim.result_dir + args['inifile'])
                except shutil.Error as e:
                    Logger.warning(e)

    elif command == "extract":
        distsim_extract(args)
    elif command == "plot":
        distsim_plot(args)
    elif command == "clean":
        distsim_clean(args)
    elif command == "retry":
        distsim_retry(args)
    elif command == "dump":
        distsim_dump(args)
    elif command == "notify":
        distsim_notify(args)
    else:
        Logger.error("Invlaid command: %s" % str(command))

def write_default_parameters(args, config):

    for key in args.__dict__.keys():
        try:
            args.__dict__[key] = config.get('Base', key)
        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
            pass
    if args.mpiWorker is None:
        args.mpiWorker = "/mnt/distsim/simtools/mpiworker"

    if args.hostfile  is None:
        args.hostfile = "/mnt/distsim/simtools/hostfile"

    #automatically parses hostfile to get the correct number of ranks
    if args.nrRanks is None:
        try:
            with open(args.hostfile, "r") as f:
                h = [[y.split("=") for y in x.strip().split()] for x in f if len(x.strip()) > 1 and x[0] != '#']
            args.nrRanks = reduce(lambda x,y: x+int(y[1][1]), h, 0)
        except IOError as e:
            Logger.warning("No hostfile specified and default location %s not present!" % (args.hostfile))
            args.nrRanks = 0

    if args.outdir is None:
        args.outdir = "/mnt/distsim/simulations/"

    if args.inetdir is None:
        args.inetdir = "/mnt/distsim/inet/src/"

    if args.omnetdir is None:
        args.omnetdir = "/mnt/distsim/omnetpp-5.0/"
    args.omnetdir += "bin/opp_run"

    if args.extractScalarsFile is None:
        args.extractScalarsFile = "scalars.txt"

    if args.extractVectorsFile is None:
        args.extractVectorsFile = "vectors.txt"

    if args.metaFile is None:
        args.metaFile = "meta.pickle"

    if args.plotScript is None:
        args.plotScript = "plot.py"

    if args.notificationKeyFile is None:
        args.notificationKeyFile = "pushover_secret"

    if args.configName is None:
        args.configName = "General"

    if args.notifyMsg is None:
        args.notifyMsg = ""

    if args.versionResults == "true":
        args.versionResults = True
    else:
        args.versionResults = False


    # Advanced plotting configuration
    advanced_plot_parameters = ['x-axis', 'y-axis', ]
    args.advanced_plot_parameters = {}
    for parameter in advanced_plot_parameters:
        try:
            args.advanced_plot_parameters[parameter] = config.get('Plotting', parameter).split('\n')
            print(args.advanced_plot_parameters)
        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
            pass
    return args

if __name__ == '__main__':
    main()