From 13b8b1177016a8375531645206ca0833cf1d3241 Mon Sep 17 00:00:00 2001 From: Jason Dogariu Date: Mon, 26 Jul 2021 14:19:09 -0400 Subject: [PATCH] Support LLVM 12 (#503) Works around ORCv1 removal by returning to MCJIT. Includes an update to the Makefile to improve compatibility on a variety of systems and linkers. Co-authored-by: Elliott Slaughter --- .github/workflows/main.yml | 17 ++++++++-- Makefile | 64 +++++++++++++++++++++----------------- src/llvmheaders.h | 2 ++ src/llvmheaders_120.h | 37 ++++++++++++++++++++++ src/tcompiler.cpp | 30 +++++++++++++++--- src/tcwrapper.cpp | 6 ++++ src/unpacklibraries.lua | 25 --------------- travis.sh | 19 +++++++++-- 8 files changed, 137 insertions(+), 63 deletions(-) create mode 100644 src/llvmheaders_120.h delete mode 100644 src/unpacklibraries.lua diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9195c98e..9078ed81 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: ['ubuntu-16.04', 'ubuntu-18.04', 'macos-10.15'] - llvm: ['3.8', '5.0', '6.0', '7', '8', '9', '10', '11'] + llvm: ['3.8', '5.0', '6.0', '7', '8', '9', '10', '11', '12'] cmake: ['0', '1'] cuda: ['0', '1'] static: ['0', '1'] @@ -38,10 +38,12 @@ jobs: llvm: '10' - os: 'ubuntu-16.04' llvm: '11' + - os: 'ubuntu-16.04' + llvm: '12' - os: 'ubuntu-18.04' llvm: '3.8' - # macOS: exclude LLVM 5.0, 8-11 make, cuda/no-static/no-slib + # macOS: exclude LLVM 5.0, 8-12 make, cuda/no-static/no-slib - os: 'macos-10.15' llvm: '5.0' - os: 'macos-10.15' @@ -56,6 +58,9 @@ jobs: - os: 'macos-10.15' llvm: '11' cmake: '0' + - os: 'macos-10.15' + llvm: '12' + cmake: '0' - os: 'macos-10.15' cuda: '1' - os: 'macos-10.15' @@ -103,6 +108,8 @@ jobs: cuda: '1' - llvm: '11' cuda: '1' + - llvm: '12' + cuda: '1' # no-static: only LLVM 8 - llvm: '3.8' @@ -119,6 +126,8 @@ jobs: static: '0' - llvm: '11' static: '0' + - llvm: '12' + static: '0' # no-slib: only LLVM 9 - llvm: '3.8' @@ -135,6 +144,8 @@ jobs: slib: '0' - llvm: '11' slib: '0' + - llvm: '12' + slib: '0' # Moonjit: only LLVM 9 - llvm: '3.8' @@ -151,6 +162,8 @@ jobs: lua: 'moonjit' - llvm: '11' lua: 'moonjit' + - llvm: '12' + lua: 'moonjit' steps: - uses: actions/checkout@v1 - run: ./travis.sh diff --git a/Makefile b/Makefile index 9d84ea62..06090c4d 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ LLVMVERGT4 := $(shell expr $(LLVM_VERSION) \>= 40) FLAGS += -DLLVM_VERSION=$(LLVM_VERSION) -LLVM_NEEDS_CXX14="100 110 111" +LLVM_NEEDS_CXX14="100 110 111 120" ifneq (,$(findstring $(LLVM_VERSION),$(LLVM_NEEDS_CXX14))) CPPFLAGS += -std=c++1y # GCC 5 does not support -std=c++14 flag else @@ -101,37 +101,37 @@ DYNFLAGS = -dynamiclib -single_module -fPIC -install_name "@rpath/terra.dylib" TERRA_STATIC_LIBRARY = -Wl,-force_load,$(LIBRARY) endif -LLVM_LIBRARY_FLAGS += $(LUAJIT_LIB) -LLVM_LIBRARY_FLAGS += $(shell $(LLVM_CONFIG) --ldflags) -L$(CLANG_PREFIX)/lib -LLVM_LIBRARY_FLAGS += -lclangFrontend -lclangDriver \ - -lclangSerialization -lclangCodeGen -lclangParse -lclangSema \ - -lclangAnalysis \ - -lclangEdit -lclangAST -lclangLex -lclangBasic - -CLANG_AST_MATCHERS = "80 90 100 110 111" +CLANG_LIBS += libclangFrontend.a \ + libclangDriver.a \ + libclangSerialization.a \ + libclangCodeGen.a \ + libclangParse.a \ + libclangSema.a \ + libclangAnalysis.a \ + libclangEdit.a \ + libclangAST.a \ + libclangLex.a \ + libclangBasic.a + +CLANG_AST_MATCHERS = "80 90 100 110 111 120" ifneq (,$(findstring $(LLVM_VERSION),$(CLANG_AST_MATCHERS))) -LLVM_LIBRARY_FLAGS += -lclangASTMatchers +CLANG_LIBS += libclangASTMatchers.a endif -# by default, Terra includes only the pieces of the LLVM libraries it needs, -# but this can be a problem if third-party-libraries that also need LLVM are -# used - allow the user to request that some/all of the LLVM components be -# included and re-exported in their entirety +# Get full path to clang libaries +CLANG_LIBFILES := $(patsubst %, $(CLANG_PREFIX)/lib/%, $(CLANG_LIBS)) + ifeq "$(LLVMVERGT4)" "1" LLVM_LIBS += $(shell $(LLVM_CONFIG) --libs --link-static) + LLVM_LIBFILES := $(shell $(LLVM_CONFIG) --libfiles --link-static) else LLVM_LIBS += $(shell $(LLVM_CONFIG) --libs) + LLVM_LIBFILES := $(shell $(LLVM_CONFIG) --libfiles) endif -ifneq ($(REEXPORT_LLVM_COMPONENTS),) - REEXPORT_LIBS := $(shell $(LLVM_CONFIG) --libs $(REEXPORT_LLVM_COMPONENTS)) - ifneq ($(findstring $(UNAME), Linux FreeBSD),) - LLVM_LIBRARY_FLAGS += -Wl,--whole-archive $(REEXPORT_LIBS) -Wl,--no-whole-archive - else - LLVM_LIBRARY_FLAGS += $(REEXPORT_LIBS:%=-Wl,-force_load,%) - endif - LLVM_LIBRARY_FLAGS += $(filter-out $(REEXPORT_LIBS),$(LLVM_LIBS)) -else - LLVM_LIBRARY_FLAGS += $(LLVM_LIBS) + +LLVM_POLLY = "100 110 111 120" +ifneq (,$(findstring $(LLVM_VERSION),$(LLVM_POLLY))) + LLVM_LIBFILES += $(shell $(LLVM_CONFIG) --libdir)/libPolly*.a endif # llvm sometimes requires ncurses and libz, check if they have the symbols, and add them if they do @@ -149,6 +149,8 @@ ifeq ($(UNAME), FreeBSD) SUPPORT_LIBRARY_FLAGS += -lexecinfo -pthread endif +SUPPORT_LIBRARY_FLAGS += -lffi -ledit -lxml2 + PACKAGE_DEPS += $(LUAJIT_LIB) #makes luajit happy on osx 10.6 (otherwise luaL_newstate returns NULL) @@ -229,11 +231,15 @@ release/include/terra/%.h: $(LUAJIT_INCLUDE)/%.h $(LUAJIT_LIB) build/llvm_objects/llvm_list: $(addprefix build/, $(LIBOBJS) $(EXEOBJS)) mkdir -p build/llvm_objects/luajit - $(CXX) -o /dev/null $(addprefix build/, $(LIBOBJS) $(EXEOBJS)) $(LLVM_LIBRARY_FLAGS) $(SUPPORT_LIBRARY_FLAGS) $(LFLAGS) -Wl,-t 2>&1 | egrep "lib(LLVM|clang)" > build/llvm_objects/llvm_list - # extract needed LLVM objects based on a dummy linker invocation - < build/llvm_objects/llvm_list $(LUAJIT) src/unpacklibraries.lua build/llvm_objects - # include all luajit objects, since the entire lua interface is used in terra - + # Extract Luajit + all LLVM & Clang libraries + cd build/llvm_objects; for lib in $(LUAJIT_LIB) $(LLVM_LIBFILES) $(CLANG_LIBFILES); do \ + echo Extracing objects from $$lib; \ + DIR=$$(basename $$lib .a); \ + mkdir -p $$DIR; \ + cd $$DIR; \ + ar x $$lib; \ + cd ..; \ + done build/lua_objects/lj_obj.o: $(LUAJIT_LIB) mkdir -p build/lua_objects diff --git a/src/llvmheaders.h b/src/llvmheaders.h index 3504b66a..26aaf186 100644 --- a/src/llvmheaders.h +++ b/src/llvmheaders.h @@ -62,6 +62,8 @@ #include "llvmheaders_100.h" #elif LLVM_VERSION >= 110 && LLVM_VERSION < 120 #include "llvmheaders_110.h" +#elif LLVM_VERSION == 120 +#include "llvmheaders_120.h" #else #error "unsupported LLVM version" // for OSX code completion diff --git a/src/llvmheaders_120.h b/src/llvmheaders_120.h new file mode 100644 index 00000000..93949901 --- /dev/null +++ b/src/llvmheaders_120.h @@ -0,0 +1,37 @@ +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/Analysis/CallGraph.h" +#include "llvm/IR/DIBuilder.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/Mangler.h" +//#include "llvm/ExecutionEngine/ObjectImage.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Linker/Linker.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/InstVisitor.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" + +#include "llvm/Support/VirtualFileSystem.h" +#include "clang/Rewrite/Core/Rewriter.h" +#include "clang/Rewrite/Frontend/Rewriters.h" +#include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Object/SymbolSize.h" + +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Support/Error.h" + +#define LLVM_PATH_TYPE std::string +#define RAW_FD_OSTREAM_NONE sys::fs::F_None +#define RAW_FD_OSTREAM_BINARY sys::fs::F_None +#define HASFNATTR(attr) \ + getAttributes().hasAttribute(AttributeSet::FunctionIndex, Attribute ::attr) +#define ADDFNATTR(attr) addFnAttr(Attribute ::attr) +#define ATTRIBUTE Attributes diff --git a/src/tcompiler.cpp b/src/tcompiler.cpp index 409ece22..2c6ba99b 100644 --- a/src/tcompiler.cpp +++ b/src/tcompiler.cpp @@ -23,8 +23,10 @@ extern "C" { #if LLVM_VERSION < 50 #include "llvm/ExecutionEngine/MCJIT.h" #else +#if LLVM_VERSION < 120 #include "llvm/ExecutionEngine/OrcMCJITReplacement.h" #endif +#endif #include "llvm/Support/Atomic.h" #include "llvm/Support/FileSystem.h" @@ -390,7 +392,10 @@ static void InitializeJIT(TerraCompilationUnit *CU) { #else .setMCJITMemoryManager(std::make_unique(CU)) #endif - .setUseOrcMCJITReplacement(true); +#if LLVM_VERSION < 120 + .setUseOrcMCJITReplacement(true) +#endif + ; #endif CU->ee = eb.create(); @@ -952,13 +957,21 @@ struct CCallingConv { } template - void addSRetAttr(FnOrCall *r, int idx) { + void addSRetAttr(FnOrCall *r, int idx, Type *ty) { +#if LLVM_VERSION < 120 r->addAttribute(idx, Attribute::StructRet); +#else + r->addAttribute(idx, Attribute::getWithStructRetType(*CU->TT->ctx, ty)); +#endif r->addAttribute(idx, Attribute::NoAlias); } template - void addByValAttr(FnOrCall *r, int idx) { + void addByValAttr(FnOrCall *r, int idx, Type *ty) { +#if LLVM_VERSION < 120 r->addAttribute(idx, Attribute::ByVal); +#else + r->addAttribute(idx, Attribute::getWithByValType(*CU->TT->ctx, ty)); +#endif } template void addExtAttrIfNeeded(TType *t, FnOrCall *r, int idx) { @@ -971,14 +984,14 @@ struct CCallingConv { addExtAttrIfNeeded(info->returntype.type, r, 0); int argidx = 1; if (info->returntype.kind == C_AGGREGATE_MEM) { - addSRetAttr(r, argidx); + addSRetAttr(r, argidx, info->returntype.cctype); argidx++; } for (size_t i = 0; i < info->paramtypes.size(); i++) { Argument *v = &info->paramtypes[i]; if (v->kind == C_AGGREGATE_MEM) { #ifndef _WIN32 - addByValAttr(r, argidx); + addByValAttr(r, argidx, v->cctype); #endif } addExtAttrIfNeeded(v->type, r, argidx); @@ -2488,9 +2501,16 @@ struct FunctionEmitter { DEBUG_ONLY(T) { MDNode *scope = debugScopeForFile(customfilename ? customfilename : obj->string("filename")); +#if LLVM_VERSION < 120 B->SetCurrentDebugLocation(DebugLoc::get( customfilename ? customlinenumber : obj->number("linenumber"), 0, scope)); +#else + B->SetCurrentDebugLocation(DILocation::get( + scope->getContext(), + customfilename ? customlinenumber : obj->number("linenumber"), 0, + scope)); +#endif } } diff --git a/src/tcwrapper.cpp b/src/tcwrapper.cpp index 16e0ab7b..5f3271fc 100644 --- a/src/tcwrapper.cpp +++ b/src/tcwrapper.cpp @@ -328,8 +328,14 @@ class IncludeCVisitor : public RecursiveASTVisitor { } CStyleCastExpr *CreateCast(QualType Ty, CastKind Kind, Expr *E) { TypeSourceInfo *TInfo = Context->getTrivialTypeSourceInfo(Ty, SourceLocation()); +#if LLVM_VERSION < 120 return CStyleCastExpr::Create(*Context, Ty, VK_RValue, Kind, E, 0, TInfo, SourceLocation(), SourceLocation()); +#else + return CStyleCastExpr::Create(*Context, Ty, VK_RValue, Kind, E, 0, + FPOptionsOverride::getFromOpaqueInt(0), TInfo, + SourceLocation(), SourceLocation()); +#endif } IntegerLiteral *LiteralZero() { unsigned IntSize = static_cast(Context->getTypeSize(Context->IntTy)); diff --git a/src/unpacklibraries.lua b/src/unpacklibraries.lua deleted file mode 100644 index ff884d22..00000000 --- a/src/unpacklibraries.lua +++ /dev/null @@ -1,25 +0,0 @@ -local destination = ... - -local function exe(cmd,...) - cmd = string.format(cmd,...) - local res = { os.execute(cmd) } - if type(res[1]) == 'number' and res[1] ~= 0 or not res[1] then - print('Error during '..cmd..':', table.unpack(res)) - error("command failed: "..cmd) - end -end -local function exists(path) - local f = io.open(path) - if not f then return false end - f:close() - return true -end -for line in io.lines() do - line = line:gsub("[()]"," ") - local archivepath,objectfile = line:match("(%S+)%s+(%S+)") - local archivename = archivepath:match("/([^/]*)%.a$") - if not exists( ("%s/%s/%s"):format(destination,archivename,objectfile) ) then - exe("mkdir -p %s/%s",destination,archivename) - exe("cd %s/%s; ar x %s %s",destination,archivename,archivepath,objectfile) - end -end \ No newline at end of file diff --git a/travis.sh b/travis.sh index babfee52..6d038780 100755 --- a/travis.sh +++ b/travis.sh @@ -26,7 +26,16 @@ fi if [[ $(uname) = Linux ]]; then sudo apt-get update -qq - if [[ $LLVM_CONFIG = llvm-config-11 ]]; then + if [[ $LLVM_CONFIG = llvm-config-12 ]]; then + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + sudo add-apt-repository -y "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" + for i in {1..5}; do sudo apt-get update -qq && break || sleep 15; done + sudo apt-get install -y llvm-12-dev clang-12 libclang-12-dev libedit-dev + export CMAKE_PREFIX_PATH=/usr/lib/llvm-12:/usr/share/llvm-12 + if [[ -n $STATIC_LLVM && $STATIC_LLVM -eq 0 ]]; then + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/llvm-12/lib" + fi + elif [[ $LLVM_CONFIG = llvm-config-11 ]]; then wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - sudo add-apt-repository -y "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main" for i in {1..5}; do sudo apt-get update -qq && break || sleep 15; done @@ -81,7 +90,13 @@ if [[ $(uname) = Linux ]]; then fi if [[ $(uname) = Darwin ]]; then - if [[ $LLVM_CONFIG = llvm-config-11 ]]; then + if [[ $LLVM_CONFIG = llvm-config-12 ]]; then + curl -L -O https://github.com/elliottslaughter/llvm-build/releases/download/llvm-12.0.1/clang+llvm-12.0.1-x86_64-apple-darwin.tar.xz + tar xf clang+llvm-12.0.1-x86_64-apple-darwin.tar.xz + ln -s clang+llvm-12.0.1-x86_64-apple-darwin/bin/llvm-config llvm-config-12 + ln -s clang+llvm-12.0.1-x86_64-apple-darwin/bin/clang clang-12 + export CMAKE_PREFIX_PATH=$PWD/clang+llvm-12.0.1-x86_64-apple-darwin + elif [[ $LLVM_CONFIG = llvm-config-11 ]]; then curl -L -O https://github.com/elliottslaughter/llvm-build/releases/download/llvm-11.0.1/clang+llvm-11.0.1-x86_64-apple-darwin.tar.xz tar xf clang+llvm-11.0.1-x86_64-apple-darwin.tar.xz ln -s clang+llvm-11.0.1-x86_64-apple-darwin/bin/llvm-config llvm-config-11