Compiler frontend rework

configure.ac

@@ -23,7 +23,7 @@ AC_PROG_CC([gcc])
 AC_DEFINE_UNQUOTED([GCC_PATH], ["$CC"], [GCC path for dragonegg])
 AC_SUBST(GCC_PATH, $CC)
 
-AX_LLVM([7.0],[16.0],[all])
+AX_LLVM([7.0],[16.0.6],[all])
 AC_SUBST(LLVM_CPPFLAGS, $LLVM_CPPFLAGS)
 
 # Check if Fortran/flang has been enabled (disabled by default)
@@ -37,6 +37,9 @@ fi
 AC_DEFINE_UNQUOTED([FORTRAN_SUPPORT], ["$with_flang"], [enables or disables Fortran/flang support])
 AC_SUBST(FORTRAN_SUPPORT, $with_flang)
 
+# Define Automake var to optionally distribute ceref
+AM_CONDITIONAL([WITH_FLANG],  [test "x$with_flang"  = "xyes"])
+
 # The profiler and gfortran libraries added to LIBS by AC_CHECK_LIB are needed only at
 # cere run time, not when building cere. Therefore we have to reset LIBS.
 LIBS=""
@@ -113,6 +116,7 @@ AC_C_BIGENDIAN(
                [AC_MSG_ERROR([universal endianess not supported])]
                )
 
+
 # Check for Python >= 3
 
 AM_PATH_PYTHON([3])

src/Makefile.am

@@ -1,31 +1,36 @@
 SUBDIRS=ccan rdtsc memory_dump RegionOutliner RegionInstrumentation RegionReplay RegionDump GlobalRename
 
-dist_bin_SCRIPTS=cerec
+dist_bin_SCRIPTS = cerec cerec++
+if WITH_FLANG
+	dist_bin_SCRIPTS += ceref
+endif
+
 pkgpython_PYTHON= \
+	cere/compiler_frontend.py \
 	cere/lel.py \
+	cere/lec.py \
+	cere/vars.py \
+	cere/utils.py \
+	cere/__main__.py \
 	cere/cere_sanity_check.py \
 	cere/ilp_update_graph.py \
 	cere/cere_trace.py \
 	cere/granularity.py \
 	cere/cere_regions.py \
-	cere/__main__.py \
 	cere/cere_select_max_cov.py \
 	cere/cere_selectinv.py \
 	cere/graph_utils.py \
-	cere/vars.py \
 	cere/compress.py \
 	cere/__init__.py \
 	cere/cere_select_ilp.py \
 	cere/cere_profile.py \
-	cere/utils.py \
 	cere/cere_hybrid.py \
 	cere/errors.py \
 	cere/max_cov_update_graph.py \
 	cere/create_graph.py \
 	cere/graph_error.py \
 	cere/cere_check_matching.py \
 	cere/regions_selector.py \
-	cere/lec.py \
 	cere/cere_instrument.py \
 	cere/cere_replay.py \
 	cere/cere_capture.py \

src/cere/compiler_frontend.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python
+# This file is part of CERE.
+#
+# Copyright (c) 2013-2016, Universite de Versailles St-Quentin-en-Yvelines
+#
+# CERE is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License,
+# or (at your option) any later version.
+#
+# CERE is distributed in the hope that it will be useful,  
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with CERE.  If not, see <http://www.gnu.org/licenses/>.  
+
+import argparse
+import os
+import sys
+from cere.vars import *
+import cere.lec as lec
+import cere.lel as lel
+
+# By default, compile for C
+COMPILER = CLANG
+
+option_forbidden = set([("dump", "--instrument"),
+                       ("dump", "--nested-regions"),
+                       ("dump", "--no-nested-regions"),
+                       ("dump", "--instrument-app"),
+                       ("dump", "--hybrid"),
+                       ("dump", "--static"),
+                       ("replay", "--regions-file"),
+                       ("replay", "--nested-regions"),
+                       ("replay", "--no-nested-regions"),
+                       ("replay", "--instrument-app"),
+                       ("replay", "--hybrid")])
+
+#to ignore prefix matching
+class MyParser(argparse.ArgumentParser):
+    def _get_option_tuples(self, option_string):
+        return []
+
+#Allowed option verification
+class OptAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        if (parser.description, option_string) in option_forbidden:
+            exit("Error:can't {option} in {mode} mode".format(
+                 option=option_string, mode=parser.description))
+        else:
+            if (self.const):
+                setattr(namespace, self.dest, self.const)
+            else:
+                setattr(namespace, self.dest, values)
+
+#Initialize Includes parser
+class IncludeAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        namespace.Inc = (namespace.Inc + [option_string + values])
+
+def init_parser_incl(parser, liste):
+    parser.set_defaults(Inc=[])
+    for j in liste:
+        parser.add_argument(j, action=IncludeAction)
+
+# initialize subparsers
+def init_subpars(parser, region_required):
+    parser.add_argument('--region', action=OptAction, required=region_required)
+    parser.add_argument('--regions-file', action=OptAction)
+    parser.add_argument('--invocation', action=OptAction)
+    parser.add_argument('--instrument', action=OptAction, nargs=0,
+                        const=True, default=False)
+    parser.add_argument('--instrument-app', action=OptAction, nargs=0,
+                        const=True, default=False)
+    parser.add_argument('--regions-infos', action=OptAction, default="")
+    parser.add_argument('--hybrid-regions', action=OptAction, default="")
+    parser.add_argument('--extraction-lvl', action=OptAction, default="")
+    parser.add_argument('--cere-objects', action=OptAction, default="")
+    parser.add_argument('--hybrid', action=OptAction, nargs=0,
+                        const=True, default=False)
+    parser.add_argument('--static', action=OptAction, nargs=0,
+                        const=True, default=False)
+    parser.add_argument('--wrapper', action=OptAction, default="")
+    parser.add_argument('-o')
+    parser.add_argument('-c', action=OptAction, nargs=0,
+                        const=True, default=False)
+
+def init_parser_core(parser):
+    '''
+    initialize main parser
+    Create Subparsers for dump replay and original mode and
+    manage version
+    '''
+    parser.add_argument('--version', '-v', action='version',
+                        version='%(prog)s '+VERSION)
+    subparsers = parser.add_subparsers(help='sub-command help')
+    parser_dump = subparsers.add_parser('dump', description='dump')
+    init_subpars(parser_dump, True)
+    parser_dump.set_defaults(func="dump_fun")
+    parser_replay = subparsers.add_parser('replay', description='replay')
+    init_subpars(parser_replay, True)
+    parser_replay.set_defaults(func="replay_fun")
+    parser_instrument = subparsers.add_parser('original',
+                                              description='original')
+    init_subpars(parser_instrument, False)
+    parser_instrument.set_defaults(func="original_fun")
+
+
+def fail_frontend(error_message):
+    exit("Error {prog} : {cmd}".format(prog='compiler_frontend', cmd=error_message))
+
+def safe_system(command):
+    '''
+    Try-catch system call
+    Verify system call and exit with appropriate error message
+    '''
+    ret = os.system(command)
+    if(ret):
+        fail_frontend("safe_system -> " + command)
+
+
+def run(lang, argv):
+    '''
+    Main function
+    '''
+
+    # Select compiler depending on lang
+    if lang == "c":
+        COMPILER = CLANG
+    elif lang == "cpp":
+        COMPILER = CLANGPP
+    elif lang == "fortran":
+        if FORTRAN_SUPPORT:
+            compiler = "flang"
+        else:
+            fail_lec("Fortran support disabled in this build (reconfigure & reinstall using --with-flang).")
+
+    # Get CERE_MODE from environment variable
+    # If CERE_MODE is not defined, we "short-circuit" the normal build process by
+    # performing a simple compiler call by passing the flags directly to the LLVM frontend.
+    # This avoids unnecessary compiltion steps that dump the .ll files and can cause issues,
+    # for instance where build systems such as CMake and autotools build a bunch of test
+    # programs to validate the compiler.
+    try:
+      cere_args = os.environ["CERE_MODE"].split()
+    except KeyError:
+      print("Compiling in normal mode (CERE_MODE undefined)")
+      argv = " ".join(argv[1:])
+      safe_system("{0} {1}".format(COMPILER, argv))
+      exit(0)
+
+
+    # Create parsers
+    parser = MyParser()
+    # This second parser does not remove prefix matching
+    incl_parser = argparse.ArgumentParser()
+    init_parser_core(parser)
+    init_parser_incl(incl_parser, ["-I", "-D"])
+
+    #Parse args
+    args = parser.parse_known_args(cere_args + sys.argv[1:])
+    args2 = incl_parser.parse_known_args(args[1])
+    if args[0].c:
+        lec.compile(args, args2, COMPILER)
+    else:
+        lel.link(args, COMPILER)

src/cere/lec.py

@@ -20,6 +20,9 @@
 
 from cere.vars import *
 
+# By default, compile for C
+COMPILER = CLANG
+
 OMP_FLAGS=""
 if "CERE_OMP" in os.environ:
     OMP_FLAGS="-omp"
@@ -217,24 +220,11 @@ def original_fun(mode_opt, BASE, regions):
 
 def first_compil(INCLUDES, SOURCE, BASE, ext, COMPIL_OPT):
     '''
-    First Compilation
-    Detect source language (fortran or C/C++ for the moment)
-    and compile SOURCE code
+    On first compilation, generate .ll files for later custom passes
     '''
 
-    if ext in FORTRAN_EXTENSIONS:
-        if FORTRAN_SUPPORT:
-            compiler = "flang"
-
-        else:
-            fail_lec("Fortran support disabled in this build (recompile using --with-flang).")
-    elif ext in CXX_EXTENSIONS:
-        compiler = "clang++"
-    else:
-        compiler = "clang"
-
-    safe_system(("{llvm_bindir}/{compiler} {opts} -O0 -g {includes} {source} -S -emit-llvm -o " +
-                "{base}.ll").format(compiler=compiler, llvm_bindir=LLVM_BINDIR, opts=" ".join(COMPIL_OPT), includes=" ".join(INCLUDES),
+    safe_system(("{compiler} {opts} -O0 -g {includes} {source} -S -emit-llvm -o " +
+                "{base}.ll").format(compiler=COMPILER, llvm_bindir=LLVM_BINDIR, opts=" ".join(COMPIL_OPT), includes=" ".join(INCLUDES),
                 source=SOURCE, base=BASE))
 
 def last_compil(INCLUDES, SOURCE, BASE, OBJECT, COMPIL_OPT):
@@ -279,12 +269,14 @@ def last_compil(INCLUDES, SOURCE, BASE, OBJECT, COMPIL_OPT):
                 llvm_bindir=LLVM_BINDIR, opts=" ".join(COMPIL_OPT), includes=" ".join(INCLUDES),
                 backend_flags=BACKEND_FLAGS, source=SOURCE, base=BASE, object=OBJECT))
 
-def compile(args, args2):
+def compile(args, args2, new_compiler):
     function={}
     function["replay_fun"] = replay_fun
     function["dump_fun"] = dump_fun
     function["original_fun"] = original_fun
 
+    COMPILER = new_compiler
+
     SOURCES = []
     if (len(args2[1]) == 0):
         exit("Error:Need source file")

src/cere/lel.py

@@ -23,6 +23,9 @@
 from cere.vars import *
 from cere.compress import compress
 
+# By default, compile for C
+COMPILER = CLANG
+
 def grep(path, regex):
     regex=".*"+regex+".*"
     regObj = re.compile(regex)
@@ -92,7 +95,7 @@ def dump_fun(mode_opt, BINARY, COMPIL_OPT):
     '''
     safe_system(("{link} {opts} -o {binary} {libdir} " +
                  "-Wl,-z,now -lcere_dump -ldl"
-    ).format(link=CLANGPP, binary=BINARY,
+    ).format(link=COMPILER, binary=BINARY,
              opts=COMPIL_OPT, Root=PROJECT_ROOT,libdir=LIBDIR_FLAGS))
 
 def sysout(cmd):
@@ -244,25 +247,26 @@ def original_fun(mode_opt, BINARY, COMPIL_OPT):
 
     if(mode_opt.instrument_app):
         safe_system(("{link} -o {binary} {opts} {libs} {libdir}").format(
-              link=CLANGPP, binary=BINARY, opts=COMPIL_OPT, libs=PROFILE_LIB,
+              link=COMPILER, binary=BINARY, opts=COMPIL_OPT, libs=PROFILE_LIB,
               libdir=LIBDIR_FLAGS))
     elif(mode_opt.instrument):
         safe_system(("{link} -o {binary} {opts} {wrapper} {libdir}").format(
-              link=CLANGPP, binary=BINARY, opts=COMPIL_OPT,
+              link=COMPILER, binary=BINARY, opts=COMPIL_OPT,
               wrapper=mode_opt.wrapper, libdir=LIBDIR_FLAGS))
     else:
-        # NOTE In all linker modes, we now use CLANGPP by default in case we link
-        # some objects containing C++ symbols.
-        safe_system(("{link} -o {binary} {opts}").format(link=CLANGPP,
+        safe_system(("{link} -o {binary} {opts}").format(link=COMPILER,
                 binary=BINARY, opts=COMPIL_OPT))
 
 
 
-def link(args):
+def link(args, new_compiler):
     function={}
     function["replay_fun"] = replay_fun
     function["dump_fun"] = dump_fun
     function["original_fun"] = original_fun
+
+    COMPILER = new_compiler
+
     if (len(args[1]) == 0):
         exit("Error:Need source file")
     objs = ""

src/cere/vars.py.in.in

@@ -21,11 +21,17 @@ VERSION = "@PACKAGE_VERSION@"
 FORTRAN_SUPPORT="@FORTRAN_SUPPORT@"
 CLANG = os.path.join("@LLVM_BINDIR@","clang")
 CLANGPP = os.path.join("@LLVM_BINDIR@","clang++")
+FLANG = os.path.join("@LLVM_BINDIR@","flang")
 PPROF = "@PPROF@"
 LLVM_BINDIR = "@LLVM_BINDIR@"
 GCC="@GCC_PATH@"
 LIBDIR = "%LIBDIR%"
 
+C_EXTENSIONS=[".c", ".C"]
+CXX_EXTENSIONS=[".cpp", ".cc", ".cxx", ".CPP", ".CC", ".CXX"]
+FORTRAN_EXTENSIONS=[".f", ".f90", ".f77", ".F", ".F90", ".F77"]
+SOURCE_EXTENSIONS= C_EXTENSIONS + CXX_EXTENSIONS + FORTRAN_EXTENSIONS
+
 ROOT = os.path.dirname(os.path.realpath(__file__))
 PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__+"/.."))
 PROFILE_LIB = "-lprofiler"
@@ -39,12 +45,8 @@ GLOB_RENAME = LIBDIR + "/libcere_globalrename.so"
 LIBDIR_FLAGS = "-L {libdir} -Wl,--rpath={libdir}".format(libdir=LIBDIR)
 RDTSC_WRAPPER = "-lcere_rdtsc"
 LIKWID = "-llikwid"
-FORTRAN_EXTENSIONS=[".f", ".f90", ".f77", ".F90"]
-CXX_EXTENSIONS=[".cpp", ".cc", ".cxx"]
 DUMPS_DIR = ".cere/dumps/"
 
-SOURCE_EXTENSIONS=[".C", ".c"] + CXX_EXTENSIONS + FORTRAN_EXTENSIONS
-
 INVALID_REGION_FILE = "invalid_regions"
 
 CERE_DIRECTORIES = []