Skip to content

Commit

Permalink
Merge pull request #212 from benchmark-subsetting/compiler-frontend
Browse files Browse the repository at this point in the history
Compiler frontend rework
  • Loading branch information
PurplePachyderm authored Sep 4, 2023
2 parents 6d01712 + 0f9b003 commit e7360db
Show file tree
Hide file tree
Showing 9 changed files with 225 additions and 161 deletions.
6 changes: 5 additions & 1 deletion configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ AC_PROG_CC([gcc])
AC_DEFINE_UNQUOTED([GCC_PATH], ["$CC"], [GCC path for dragonegg])
AC_SUBST(GCC_PATH, $CC)

AX_LLVM([7.0],[16.0],[all])
AX_LLVM([7.0],[16.0.6],[all])
AC_SUBST(LLVM_CPPFLAGS, $LLVM_CPPFLAGS)

# Check if Fortran/flang has been enabled (disabled by default)
Expand All @@ -37,6 +37,9 @@ fi
AC_DEFINE_UNQUOTED([FORTRAN_SUPPORT], ["$with_flang"], [enables or disables Fortran/flang support])
AC_SUBST(FORTRAN_SUPPORT, $with_flang)

# Define Automake var to optionally distribute ceref
AM_CONDITIONAL([WITH_FLANG], [test "x$with_flang" = "xyes"])

# The profiler and gfortran libraries added to LIBS by AC_CHECK_LIB are needed only at
# cere run time, not when building cere. Therefore we have to reset LIBS.
LIBS=""
Expand Down Expand Up @@ -113,6 +116,7 @@ AC_C_BIGENDIAN(
[AC_MSG_ERROR([universal endianess not supported])]
)


# Check for Python >= 3

AM_PATH_PYTHON([3])
Expand Down
15 changes: 10 additions & 5 deletions src/Makefile.am
Original file line number Diff line number Diff line change
@@ -1,31 +1,36 @@
SUBDIRS=ccan rdtsc memory_dump RegionOutliner RegionInstrumentation RegionReplay RegionDump GlobalRename

dist_bin_SCRIPTS=cerec
dist_bin_SCRIPTS = cerec cerec++
if WITH_FLANG
dist_bin_SCRIPTS += ceref
endif

pkgpython_PYTHON= \
cere/compiler_frontend.py \
cere/lel.py \
cere/lec.py \
cere/vars.py \
cere/utils.py \
cere/__main__.py \
cere/cere_sanity_check.py \
cere/ilp_update_graph.py \
cere/cere_trace.py \
cere/granularity.py \
cere/cere_regions.py \
cere/__main__.py \
cere/cere_select_max_cov.py \
cere/cere_selectinv.py \
cere/graph_utils.py \
cere/vars.py \
cere/compress.py \
cere/__init__.py \
cere/cere_select_ilp.py \
cere/cere_profile.py \
cere/utils.py \
cere/cere_hybrid.py \
cere/errors.py \
cere/max_cov_update_graph.py \
cere/create_graph.py \
cere/graph_error.py \
cere/cere_check_matching.py \
cere/regions_selector.py \
cere/lec.py \
cere/cere_instrument.py \
cere/cere_replay.py \
cere/cere_capture.py \
Expand Down
168 changes: 168 additions & 0 deletions src/cere/compiler_frontend.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
#!/usr/bin/env python
# This file is part of CERE.
#
# Copyright (c) 2013-2016, Universite de Versailles St-Quentin-en-Yvelines
#
# CERE is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License,
# or (at your option) any later version.
#
# CERE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with CERE. If not, see <http://www.gnu.org/licenses/>.

import argparse
import os
import sys
from cere.vars import *
import cere.lec as lec
import cere.lel as lel

# By default, compile for C
COMPILER = CLANG

option_forbidden = set([("dump", "--instrument"),
("dump", "--nested-regions"),
("dump", "--no-nested-regions"),
("dump", "--instrument-app"),
("dump", "--hybrid"),
("dump", "--static"),
("replay", "--regions-file"),
("replay", "--nested-regions"),
("replay", "--no-nested-regions"),
("replay", "--instrument-app"),
("replay", "--hybrid")])

#to ignore prefix matching
class MyParser(argparse.ArgumentParser):
def _get_option_tuples(self, option_string):
return []

#Allowed option verification
class OptAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
if (parser.description, option_string) in option_forbidden:
exit("Error:can't {option} in {mode} mode".format(
option=option_string, mode=parser.description))
else:
if (self.const):
setattr(namespace, self.dest, self.const)
else:
setattr(namespace, self.dest, values)

#Initialize Includes parser
class IncludeAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
namespace.Inc = (namespace.Inc + [option_string + values])

def init_parser_incl(parser, liste):
parser.set_defaults(Inc=[])
for j in liste:
parser.add_argument(j, action=IncludeAction)

# initialize subparsers
def init_subpars(parser, region_required):
parser.add_argument('--region', action=OptAction, required=region_required)
parser.add_argument('--regions-file', action=OptAction)
parser.add_argument('--invocation', action=OptAction)
parser.add_argument('--instrument', action=OptAction, nargs=0,
const=True, default=False)
parser.add_argument('--instrument-app', action=OptAction, nargs=0,
const=True, default=False)
parser.add_argument('--regions-infos', action=OptAction, default="")
parser.add_argument('--hybrid-regions', action=OptAction, default="")
parser.add_argument('--extraction-lvl', action=OptAction, default="")
parser.add_argument('--cere-objects', action=OptAction, default="")
parser.add_argument('--hybrid', action=OptAction, nargs=0,
const=True, default=False)
parser.add_argument('--static', action=OptAction, nargs=0,
const=True, default=False)
parser.add_argument('--wrapper', action=OptAction, default="")
parser.add_argument('-o')
parser.add_argument('-c', action=OptAction, nargs=0,
const=True, default=False)

def init_parser_core(parser):
'''
initialize main parser
Create Subparsers for dump replay and original mode and
manage version
'''
parser.add_argument('--version', '-v', action='version',
version='%(prog)s '+VERSION)
subparsers = parser.add_subparsers(help='sub-command help')
parser_dump = subparsers.add_parser('dump', description='dump')
init_subpars(parser_dump, True)
parser_dump.set_defaults(func="dump_fun")
parser_replay = subparsers.add_parser('replay', description='replay')
init_subpars(parser_replay, True)
parser_replay.set_defaults(func="replay_fun")
parser_instrument = subparsers.add_parser('original',
description='original')
init_subpars(parser_instrument, False)
parser_instrument.set_defaults(func="original_fun")


def fail_frontend(error_message):
exit("Error {prog} : {cmd}".format(prog='compiler_frontend', cmd=error_message))

def safe_system(command):
'''
Try-catch system call
Verify system call and exit with appropriate error message
'''
ret = os.system(command)
if(ret):
fail_frontend("safe_system -> " + command)


def run(lang, argv):
'''
Main function
'''

# Select compiler depending on lang
if lang == "c":
COMPILER = CLANG
elif lang == "cpp":
COMPILER = CLANGPP
elif lang == "fortran":
if FORTRAN_SUPPORT:
compiler = "flang"
else:
fail_lec("Fortran support disabled in this build (reconfigure & reinstall using --with-flang).")

# Get CERE_MODE from environment variable
# If CERE_MODE is not defined, we "short-circuit" the normal build process by
# performing a simple compiler call by passing the flags directly to the LLVM frontend.
# This avoids unnecessary compiltion steps that dump the .ll files and can cause issues,
# for instance where build systems such as CMake and autotools build a bunch of test
# programs to validate the compiler.
try:
cere_args = os.environ["CERE_MODE"].split()
except KeyError:
print("Compiling in normal mode (CERE_MODE undefined)")
argv = " ".join(argv[1:])
safe_system("{0} {1}".format(COMPILER, argv))
exit(0)


# Create parsers
parser = MyParser()
# This second parser does not remove prefix matching
incl_parser = argparse.ArgumentParser()
init_parser_core(parser)
init_parser_incl(incl_parser, ["-I", "-D"])

#Parse args
args = parser.parse_known_args(cere_args + sys.argv[1:])
args2 = incl_parser.parse_known_args(args[1])
if args[0].c:
lec.compile(args, args2, COMPILER)
else:
lel.link(args, COMPILER)
26 changes: 9 additions & 17 deletions src/cere/lec.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@

from cere.vars import *

# By default, compile for C
COMPILER = CLANG

OMP_FLAGS=""
if "CERE_OMP" in os.environ:
OMP_FLAGS="-omp"
Expand Down Expand Up @@ -217,24 +220,11 @@ def original_fun(mode_opt, BASE, regions):

def first_compil(INCLUDES, SOURCE, BASE, ext, COMPIL_OPT):
'''
First Compilation
Detect source language (fortran or C/C++ for the moment)
and compile SOURCE code
On first compilation, generate .ll files for later custom passes
'''

if ext in FORTRAN_EXTENSIONS:
if FORTRAN_SUPPORT:
compiler = "flang"

else:
fail_lec("Fortran support disabled in this build (recompile using --with-flang).")
elif ext in CXX_EXTENSIONS:
compiler = "clang++"
else:
compiler = "clang"

safe_system(("{llvm_bindir}/{compiler} {opts} -O0 -g {includes} {source} -S -emit-llvm -o " +
"{base}.ll").format(compiler=compiler, llvm_bindir=LLVM_BINDIR, opts=" ".join(COMPIL_OPT), includes=" ".join(INCLUDES),
safe_system(("{compiler} {opts} -O0 -g {includes} {source} -S -emit-llvm -o " +
"{base}.ll").format(compiler=COMPILER, llvm_bindir=LLVM_BINDIR, opts=" ".join(COMPIL_OPT), includes=" ".join(INCLUDES),
source=SOURCE, base=BASE))

def last_compil(INCLUDES, SOURCE, BASE, OBJECT, COMPIL_OPT):
Expand Down Expand Up @@ -279,12 +269,14 @@ def last_compil(INCLUDES, SOURCE, BASE, OBJECT, COMPIL_OPT):
llvm_bindir=LLVM_BINDIR, opts=" ".join(COMPIL_OPT), includes=" ".join(INCLUDES),
backend_flags=BACKEND_FLAGS, source=SOURCE, base=BASE, object=OBJECT))

def compile(args, args2):
def compile(args, args2, new_compiler):
function={}
function["replay_fun"] = replay_fun
function["dump_fun"] = dump_fun
function["original_fun"] = original_fun

COMPILER = new_compiler

SOURCES = []
if (len(args2[1]) == 0):
exit("Error:Need source file")
Expand Down
18 changes: 11 additions & 7 deletions src/cere/lel.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
from cere.vars import *
from cere.compress import compress

# By default, compile for C
COMPILER = CLANG

def grep(path, regex):
regex=".*"+regex+".*"
regObj = re.compile(regex)
Expand Down Expand Up @@ -92,7 +95,7 @@ def dump_fun(mode_opt, BINARY, COMPIL_OPT):
'''
safe_system(("{link} {opts} -o {binary} {libdir} " +
"-Wl,-z,now -lcere_dump -ldl"
).format(link=CLANGPP, binary=BINARY,
).format(link=COMPILER, binary=BINARY,
opts=COMPIL_OPT, Root=PROJECT_ROOT,libdir=LIBDIR_FLAGS))

def sysout(cmd):
Expand Down Expand Up @@ -244,25 +247,26 @@ def original_fun(mode_opt, BINARY, COMPIL_OPT):

if(mode_opt.instrument_app):
safe_system(("{link} -o {binary} {opts} {libs} {libdir}").format(
link=CLANGPP, binary=BINARY, opts=COMPIL_OPT, libs=PROFILE_LIB,
link=COMPILER, binary=BINARY, opts=COMPIL_OPT, libs=PROFILE_LIB,
libdir=LIBDIR_FLAGS))
elif(mode_opt.instrument):
safe_system(("{link} -o {binary} {opts} {wrapper} {libdir}").format(
link=CLANGPP, binary=BINARY, opts=COMPIL_OPT,
link=COMPILER, binary=BINARY, opts=COMPIL_OPT,
wrapper=mode_opt.wrapper, libdir=LIBDIR_FLAGS))
else:
# NOTE In all linker modes, we now use CLANGPP by default in case we link
# some objects containing C++ symbols.
safe_system(("{link} -o {binary} {opts}").format(link=CLANGPP,
safe_system(("{link} -o {binary} {opts}").format(link=COMPILER,
binary=BINARY, opts=COMPIL_OPT))



def link(args):
def link(args, new_compiler):
function={}
function["replay_fun"] = replay_fun
function["dump_fun"] = dump_fun
function["original_fun"] = original_fun

COMPILER = new_compiler

if (len(args[1]) == 0):
exit("Error:Need source file")
objs = ""
Expand Down
10 changes: 6 additions & 4 deletions src/cere/vars.py.in.in
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,17 @@ VERSION = "@PACKAGE_VERSION@"
FORTRAN_SUPPORT="@FORTRAN_SUPPORT@"
CLANG = os.path.join("@LLVM_BINDIR@","clang")
CLANGPP = os.path.join("@LLVM_BINDIR@","clang++")
FLANG = os.path.join("@LLVM_BINDIR@","flang")
PPROF = "@PPROF@"
LLVM_BINDIR = "@LLVM_BINDIR@"
GCC="@GCC_PATH@"
LIBDIR = "%LIBDIR%"

C_EXTENSIONS=[".c", ".C"]
CXX_EXTENSIONS=[".cpp", ".cc", ".cxx", ".CPP", ".CC", ".CXX"]
FORTRAN_EXTENSIONS=[".f", ".f90", ".f77", ".F", ".F90", ".F77"]
SOURCE_EXTENSIONS= C_EXTENSIONS + CXX_EXTENSIONS + FORTRAN_EXTENSIONS

ROOT = os.path.dirname(os.path.realpath(__file__))
PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__+"/.."))
PROFILE_LIB = "-lprofiler"
Expand All @@ -39,12 +45,8 @@ GLOB_RENAME = LIBDIR + "/libcere_globalrename.so"
LIBDIR_FLAGS = "-L {libdir} -Wl,--rpath={libdir}".format(libdir=LIBDIR)
RDTSC_WRAPPER = "-lcere_rdtsc"
LIKWID = "-llikwid"
FORTRAN_EXTENSIONS=[".f", ".f90", ".f77", ".F90"]
CXX_EXTENSIONS=[".cpp", ".cc", ".cxx"]
DUMPS_DIR = ".cere/dumps/"

SOURCE_EXTENSIONS=[".C", ".c"] + CXX_EXTENSIONS + FORTRAN_EXTENSIONS

INVALID_REGION_FILE = "invalid_regions"

CERE_DIRECTORIES = []
Expand Down
Loading

0 comments on commit e7360db

Please sign in to comment.