Atomics

clang/lib/Basic/Targets/WebAssembly.h

@@ -212,6 +212,8 @@ class CheerpTargetInfo : public TargetInfo {
     LongDoubleWidth = LongDoubleAlign = 64;
     LongDoubleFormat = &llvm::APFloat::IEEEdouble();
     SizeType = UnsignedInt;
+    // We define these as 32-bit for now, since AsmJS cannot handle 64-bit atomic operations currently.
+    MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 32;
 
     // Use 32-bit integers for two separated bit fields.
     UseBitFieldTypeAlignment = true;

clang/lib/CodeGen/BackendUtil.cpp

@@ -87,7 +87,6 @@
 #include "llvm/Transforms/Scalar/EarlyCSE.h"
 #include "llvm/Transforms/Scalar/GVN.h"
 #include "llvm/Transforms/Scalar/JumpThreading.h"
-#include "llvm/Transforms/Scalar/LowerAtomicPass.h"
 #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
 #include "llvm/Transforms/Utils.h"
 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
@@ -948,8 +947,6 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
             //We need this to track this in custom constructors for DOM types, such as String::String(const char*)
             MPM.addPass(createModuleToFunctionPassAdaptor(cheerp::RequiredPassWrapper<PromotePass>()));
             MPM.addPass(createModuleToFunctionPassAdaptor(cheerp::CheerpNativeRewriterPass()));
-            //Cheerp is single threaded, convert atomic instructions to regular ones
-            MPM.addPass(createModuleToFunctionPassAdaptor(LowerAtomicPass()));
           });
       PB.registerOptimizerLastEPCallback(
           [](ModulePassManager &MPM, OptimizationLevel Level) {

clang/lib/Driver/ToolChains/WebAssembly.cpp

@@ -697,9 +697,14 @@ void cheerp::CheerpOptimizer::ConstructJob(Compilation &C, const JobAction &JA,
   if (Args.hasArg(options::OPT_cheerp_no_icf))
     CmdArgs.push_back("-cheerp-no-icf");
 
+  if (!Args.hasArg(options::OPT_pthread))
+    CmdArgs.push_back("-cheerp-lower-atomics");
+
   addPass("function(CheerpLowerInvoke)");
   if (Args.hasArg(options::OPT_fexceptions))
     CmdArgs.push_back("-cheerp-keep-invokes");
+  // This pass will remove atomics from genericjs functions
+  addPass("CheerpLowerAtomic");
   addPass("function(simplifycfg)");
 
   addPass("CallConstructors");

compiler-rt/lib/sanitizer_common/sanitizer_mutex.h

@@ -163,7 +163,11 @@ class SANITIZER_MUTEX Mutex : CheckedMutex {
   explicit constexpr Mutex(MutexType type = MutexUnchecked)
       : CheckedMutex(type) {}
 
+  // CHEERP: This mutex uses a 64-bit state value, meaning 64-bit atomic operations.
+  // These are not supported in asmjs currently, so we've disabled the Mutex
+  // by returning early from all functions.
   void Lock() SANITIZER_ACQUIRE() {
+    return;
     CheckedMutex::Lock();
     u64 reset_mask = ~0ull;
     u64 state = atomic_load_relaxed(&state_);
@@ -209,6 +213,7 @@ class SANITIZER_MUTEX Mutex : CheckedMutex {
   }
 
   bool TryLock() SANITIZER_TRY_ACQUIRE(true) {
+    return true;
     u64 state = atomic_load_relaxed(&state_);
     for (;;) {
       if (UNLIKELY(state & (kWriterLock | kReaderLockMask)))
@@ -223,6 +228,7 @@ class SANITIZER_MUTEX Mutex : CheckedMutex {
   }
 
   void Unlock() SANITIZER_RELEASE() {
+    return;
     CheckedMutex::Unlock();
     bool wake_writer;
     u64 wake_readers;
@@ -251,6 +257,7 @@ class SANITIZER_MUTEX Mutex : CheckedMutex {
   }
 
   void ReadLock() SANITIZER_ACQUIRE_SHARED() {
+    return;
     CheckedMutex::Lock();
     u64 reset_mask = ~0ull;
     u64 state = atomic_load_relaxed(&state_);
@@ -288,6 +295,7 @@ class SANITIZER_MUTEX Mutex : CheckedMutex {
   }
 
   void ReadUnlock() SANITIZER_RELEASE_SHARED() {
+    return;
     CheckedMutex::Unlock();
     bool wake;
     u64 new_state;
@@ -314,12 +322,14 @@ class SANITIZER_MUTEX Mutex : CheckedMutex {
   // maintaining complex state to work around those situations, the check only
   // checks that the mutex is owned.
   void CheckWriteLocked() const SANITIZER_CHECK_LOCKED() {
+    return;
     CHECK(atomic_load(&state_, memory_order_relaxed) & kWriterLock);
   }
 
   void CheckLocked() const SANITIZER_CHECK_LOCKED() { CheckWriteLocked(); }
 
   void CheckReadLocked() const SANITIZER_CHECK_LOCKED() {
+    return;
     CHECK(atomic_load(&state_, memory_order_relaxed) & kReaderLockMask);
   }
 

llvm/include/llvm/Cheerp/CheerpLowerAtomic.h

@@ -0,0 +1,18 @@
+#ifndef CHEERP_LOWER_ATOMIC_H
+#define CHEERP_LOWER_ATOMIC_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace cheerp
+{
+
+class CheerpLowerAtomicPass : public llvm::PassInfoMixin<CheerpLowerAtomicPass>
+{
+public:
+	llvm::PreservedAnalyses run(llvm::Module& M, llvm::ModuleAnalysisManager& MAM);
+	static bool isRequired() { return true; }
+};
+
+}
+
+#endif

llvm/include/llvm/Cheerp/CommandLine.h

@@ -60,5 +60,6 @@ extern llvm::cl::opt<bool> WasmNoUnalignedMem;
 extern llvm::cl::opt<bool> UseBigInts;
 extern llvm::cl::opt<bool> KeepInvokes;
 extern llvm::cl::opt<bool> PreserveFree;
+extern llvm::cl::opt<bool> LowerAtomics;
 
 #endif //_CHEERP_COMMAND_LINE_H

llvm/include/llvm/Cheerp/GlobalDepsAnalyzer.h

@@ -137,6 +137,8 @@ class GlobalDepsAnalyzer
 	 */
 	bool usesAsmJSMalloc() const { return hasAsmJSMalloc; }
 
+	bool usesAtomics() const { return hasAtomics; }
+
 	bool runOnModule( llvm::Module & );
 
 	void visitType( llvm::Type* t, bool forceTypedArray );
@@ -242,6 +244,9 @@ class GlobalDepsAnalyzer
 	//Extend lifetime of function, visiting them and declaring external
 	void extendLifetime(llvm::Function* F);
 
+	//Determine whether an instruction is atomic.
+	bool isAtomicInstruction(const llvm::Instruction& I);
+
 	std::unordered_set< const llvm::GlobalValue * > reachableGlobals; // Set of all the reachable globals
 
 	FixupMap varsFixups;
@@ -270,6 +275,7 @@ class GlobalDepsAnalyzer
 	bool hasVAArgs;
 	bool hasPointerArrays;
 	bool hasAsmJSCode;
+	bool hasAtomics;
 	bool hasAsmJSMemory;
 	bool hasAsmJSMalloc;
 	bool hasCheerpException;

llvm/include/llvm/Cheerp/NameGenerator.h

@@ -68,6 +68,15 @@ class NameGenerator
 		HANDLE_VAARG,
 		EXCEPTION,
 		FETCHBUFFER,
+		ATOMICLOAD,
+		ATOMICSTORE,
+		ATOMICADD,
+		ATOMICSUB,
+		ATOMICAND,
+		ATOMICOR,
+		ATOMICXOR,
+		ATOMICXCHG,
+		ATOMICCMPXCHG,
 		MEMORY,
 		HEAP8,
 		HEAP16,

llvm/include/llvm/Cheerp/PassRegistry.h

@@ -40,6 +40,7 @@
 #include "llvm/Cheerp/StoreMerging.h"
 #include "llvm/Cheerp/CallConstructors.h"
 #include "llvm/Cheerp/CommandLine.h"
+#include "llvm/Cheerp/CheerpLowerAtomic.h"
 
 namespace cheerp {
 

llvm/include/llvm/Cheerp/WasmOpcodes.h

@@ -143,6 +143,7 @@ enum class WasmOpcode {
 	F64_REINTERPRET_I64 = 0xbf,
 	FC = 0xfc,
 	SIMD = 0xfd,
+	Threads = 0xfe,
 };
 
 enum class WasmS32Opcode {
@@ -373,6 +374,76 @@ enum class WasmSIMDU32U32U32Opcode {
 	V128_STORE64_LANE = 0x5b,
 };
 
+enum class WasmThreadsU32Opcode {
+	ATOMIC_FENCE = 0x3,
+};
+
+enum class WasmThreadsU32U32Opcode {
+	I32_ATOMIC_LOAD = 0x10,
+	I64_ATOMIC_LOAD = 0x11,
+	I32_ATOMIC_LOAD8_U = 0x12,
+	I32_ATOMIC_LOAD16_U = 0x13,
+	I64_ATOMIC_LOAD8_U = 0x14,
+	I64_ATOMIC_LOAD16_U = 0x15,
+	I64_ATOMIC_LOAD32_U = 0x16,
+	I32_ATOMIC_STORE = 0x17,
+	I64_ATOMIC_STORE = 0x18,
+	I32_ATOMIC_STORE8 = 0x19,
+	I32_ATOMIC_STORE16 = 0x1a,
+	I64_ATOMIC_STORE8 = 0x1b,
+	I64_ATOMIC_STORE16 = 0x1c,
+	I64_ATOMIC_STORE32 = 0x1d,
+	I32_ATOMIC_RMW_ADD = 0x1e,
+	I64_ATOMIC_RMW_ADD = 0x1f,
+	I32_ATOMIC_RMW8_ADD_U = 0x20,
+	I32_ATOMIC_RMW16_ADD_U = 0x21,
+	I64_ATOMIC_RMW8_ADD_U = 0x22,
+	I64_ATOMIC_RMW16_ADD_U = 0x23,
+	I64_ATOMIC_RMW32_ADD_U = 0x24,
+	I32_ATOMIC_RMW_SUB = 0x25,
+	I64_ATOMIC_RMW_SUB = 0x26,
+	I32_ATOMIC_RMW8_SUB_U = 0x27,
+	I32_ATOMIC_RMW16_SUB_U = 0x28,
+	I64_ATOMIC_RMW8_SUB_U = 0x29,
+	I64_ATOMIC_RMW16_SUB_U = 0x2a,
+	I64_ATOMIC_RMW32_SUB_U = 0x2b,
+	I32_ATOMIC_RMW_AND = 0x2c,
+	I64_ATOMIC_RMW_AND = 0x2d,
+	I32_ATOMIC_RMW8_AND_U = 0x2e,
+	I32_ATOMIC_RMW16_AND_U = 0x2f,
+	I64_ATOMIC_RMW8_AND_U = 0x30,
+	I64_ATOMIC_RMW16_AND_U = 0x31,
+	I64_ATOMIC_RMW32_AND_U = 0x32,
+	I32_ATOMIC_RMW_OR = 0x33,
+	I64_ATOMIC_RMW_OR = 0x34,
+	I32_ATOMIC_RMW8_OR_U = 0x35,
+	I32_ATOMIC_RMW16_OR_U = 0x36,
+	I64_ATOMIC_RMW8_OR_U = 0x37,
+	I64_ATOMIC_RMW16_OR_U = 0x38,
+	I64_ATOMIC_RMW32_OR_U = 0x39,
+	I32_ATOMIC_RMW_XOR = 0x3a,
+	I64_ATOMIC_RMW_XOR = 0x3b,
+	I32_ATOMIC_RMW8_XOR_U = 0x3c,
+	I32_ATOMIC_RMW16_XOR_U = 0x3d,
+	I64_ATOMIC_RMW8_XOR_U = 0x3e,
+	I64_ATOMIC_RMW16_XOR_U = 0x3f,
+	I64_ATOMIC_RMW32_XOR_U = 0x40,
+	I32_ATOMIC_RMW_XCHG = 0x41,
+	I64_ATOMIC_RMW_XCHG = 0x42,
+	I32_ATOMIC_RMW8_XCHG_U = 0x43,
+	I32_ATOMIC_RMW16_XCHG_U = 0x44,
+	I64_ATOMIC_RMW8_XCHG_U = 0x45,
+	I64_ATOMIC_RMW16_XCHG_U = 0x46,
+	I64_ATOMIC_RMW32_XCHG_U = 0x47,
+	I32_ATOMIC_RMW_CMPXCHG = 0x48,
+	I64_ATOMIC_RMW_CMPXCHG = 0x49,
+	I32_ATOMIC_RMW8_CMPXCHG_U = 0x4a,
+	I32_ATOMIC_RMW16_CMPXCHG_U = 0x4b,
+	I64_ATOMIC_RMW8_CMPXCHG_U = 0x4c,
+	I64_ATOMIC_RMW16_CMPXCHG_U = 0x4d,
+	I64_ATOMIC_RMW32_CMPXCHG_U = 0x4e,
+};
+
 enum class WasmInvalidOpcode {
 	BRANCH_LIKELY = 0x14,
 	BRANCH_UNLIKELY = 0x15,

llvm/include/llvm/Cheerp/WasmWriter.h

@@ -513,6 +513,8 @@ class CheerpWasmWriter final : public CheerpBaseWriter
 	static void encodeInst(WasmSIMDU32U32Opcode opcode, uint32_t i1, uint32_t i2, WasmBuffer& code);
 	static void encodeInst(WasmSIMDU32U32U32Opcode opcode, uint32_t i1, uint32_t i2, uint32_t i3, WasmBuffer& code);
 	static void encodeInst(WasmU32U32Opcode opcode, uint32_t i1, uint32_t i2, WasmBuffer& code);
+	static void encodeInst(WasmThreadsU32Opcode opcode, uint32_t immediate, WasmBuffer& code);
+	static void encodeInst(WasmThreadsU32U32Opcode opcode, uint32_t i1, uint32_t i2, WasmBuffer& code);
 	void encodeInst(WasmInvalidOpcode opcode, WasmBuffer& code);
 	void encodeVectorConstantZero(WasmBuffer& code);
 	void encodeConstantDataVector(WasmBuffer& code, const llvm::ConstantDataVector* cdv);
@@ -528,7 +530,7 @@ class CheerpWasmWriter final : public CheerpBaseWriter
 	void compileICmp(const llvm::ICmpInst& ci, const llvm::CmpInst::Predicate p, WasmBuffer& code);
 	void compileICmp(const llvm::Value* op0, const llvm::Value* op1, const llvm::CmpInst::Predicate p, WasmBuffer& code);
 	void compileFCmp(const llvm::Value* lhs, const llvm::Value* rhs, const llvm::CmpInst::Predicate p, WasmBuffer& code);
-	void encodeLoad(llvm::Type* ty, uint32_t offset, WasmBuffer& code, bool signExtend);
+	void encodeLoad(llvm::Type* ty, uint32_t offset, WasmBuffer& code, bool signExtend, bool atomic);
 	void encodeWasmIntrinsic(WasmBuffer& code, const llvm::Function* F);
 	void encodeBranchTable(WasmBuffer& code, std::vector<uint32_t> table, int32_t defaultBlock);
 	void encodeDataSectionChunk(WasmBuffer& data, uint32_t address, llvm::StringRef buf);

llvm/include/llvm/Cheerp/Writer.h

@@ -453,11 +453,12 @@ class CheerpWriter final : public CheerpBaseWriter
 	COMPILE_INSTRUCTION_FEEDBACK compileNotInlineableInstruction(const llvm::Instruction& I, PARENT_PRIORITY parentPrio);
 	COMPILE_INSTRUCTION_FEEDBACK compileInlineableInstruction(const llvm::Instruction& I, PARENT_PRIORITY parentPrio);
 	COMPILE_INSTRUCTION_FEEDBACK compileCallInstruction(const llvm::CallBase& I, PARENT_PRIORITY parentPrio);
-	void compileLoadElem(const llvm::Value* ptrOp, llvm::Type* Ty, llvm::StructType* STy, POINTER_KIND ptrKind, POINTER_KIND loadKind, bool isOffset, Registerize::REGISTER_KIND regKind, uint32_t structElemIdx, bool asmjs, PARENT_PRIORITY parentPrio);
-	void compileLoadElem(const llvm::Value* ptrOp, llvm::Type* Ty, POINTER_KIND ptrKind, POINTER_KIND loadKind, bool isOffset, Registerize::REGISTER_KIND regKind, bool asmjs, PARENT_PRIORITY parentPrio);
+	void compileLoadElem(const llvm::LoadInst& li, llvm::Type* Ty, llvm::StructType* STy, POINTER_KIND ptrKind, POINTER_KIND loadKind, bool isOffset, Registerize::REGISTER_KIND regKind, uint32_t structElemIdx, bool asmjs, PARENT_PRIORITY parentPrio);
 	void compileLoad(const llvm::LoadInst& li, PARENT_PRIORITY parentPrio);
 	void compileStoreElem(const llvm::StoreInst& si, llvm::Type* Ty, llvm::StructType* STy, POINTER_KIND ptrKind, POINTER_KIND storedKind, bool isOffset, Registerize::REGISTER_KIND regKind, uint32_t structElemIdx, uint32_t elemIdx, bool asmjs);
 	void compileStore(const llvm::StoreInst& si);
+	void compileAtomicRMW(const llvm::AtomicRMWInst& ai, PARENT_PRIORITY parentPrio);
+	void compileAtomicCmpXchg(const llvm::AtomicCmpXchgInst& ai, PARENT_PRIORITY parentPrio);
 
 	void compileSignedInteger(const llvm::Value* v, bool forComparison, PARENT_PRIORITY parentPrio);
 	void compileUnsignedInteger(const llvm::Value* v, bool forAsmJSComparison, PARENT_PRIORITY parentPrio, bool forceTruncation = false);
@@ -692,6 +693,10 @@ class CheerpWriter final : public CheerpBaseWriter
 	 * Compile the function for growing the wasm linear memory
 	 */
 	void compileGrowMem();
+	/**
+	 * Compile the atomic functions
+	 */
+	void compileAtomicFunctions();
 	/**
 	 * Compile an helper function to assign all global heap symbols
 	 */

llvm/lib/CheerpUtils/CMakeLists.txt

@@ -33,6 +33,7 @@ add_llvm_component_library(LLVMCheerpUtils
   BitCastLowering.cpp
   JSStringLiteralLowering.cpp
   MemoryInit.cpp
+  CheerpLowerAtomic.cpp
   )
 
 add_dependencies(LLVMCheerpUtils intrinsics_gen)

llvm/lib/CheerpUtils/CheerpLowerAtomic.cpp

@@ -0,0 +1,27 @@
+#include "llvm/Cheerp/CheerpLowerAtomic.h"
+#include "llvm/Cheerp/CommandLine.h"
+#include "llvm/Transforms/Scalar/LowerAtomicPass.h"
+
+using namespace llvm;
+using namespace cheerp;
+// Module pass that invokes the LLVM LowerAtomicPass on genericjs functions.
+PreservedAnalyses CheerpLowerAtomicPass::run(Module& M, ModuleAnalysisManager& MAM)
+{
+	FunctionAnalysisManager& FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+	FunctionPassManager FPM;
+	FPM.addPass(LowerAtomicPass());
+
+	// Loop over the functions, and only pass genericjs ones to LowerAtomicPass
+	for (Function& F : M)
+	{
+		if (F.isDeclaration())
+			continue;
+
+		if (!LowerAtomics && F.getSection() == "asmjs")
+			continue;
+
+		FPM.run(F, FAM);
+	}
+
+	return PreservedAnalyses::none();
+}

llvm/lib/CheerpUtils/CommandLine.cpp

@@ -97,3 +97,5 @@ llvm::cl::opt<std::string> EnvironName("cheerp-environ-name", llvm::cl::Optional
 
 llvm::cl::opt<std::string> ArgvName("cheerp-argv-name", llvm::cl::Optional,
   llvm::cl::desc("If specified, the identifier name storing the arguments"), llvm::cl::value_desc("name"));
+
+llvm::cl::opt<bool> LowerAtomics("cheerp-lower-atomics", llvm::cl::desc("Lower all atomic operations"));

llvm/lib/CheerpUtils/GlobalDepsAnalyzer.cpp

@@ -144,11 +144,23 @@ void GlobalDepsAnalyzer::replaceFunctionAliasWithAliasee(llvm::Module &module, S
 	}
 }
 
+bool GlobalDepsAnalyzer::isAtomicInstruction(const llvm::Instruction& I)
+{
+	if (isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I) || isa<FenceInst>(I))
+		return true;
+	else if (const llvm::LoadInst* li = dyn_cast<llvm::LoadInst>(&I))
+		return li->isAtomic();
+	else if (const llvm::StoreInst* si = dyn_cast<llvm::StoreInst>(&I))
+		return si->isAtomic();
+	return false;
+}
+
 bool GlobalDepsAnalyzer::runOnModule( llvm::Module & module )
 {
 	DL = &module.getDataLayout();
 	assert(DL);
 	VisitedSet visited;
+	hasAtomics = false;
 
 	replaceFunctionAliasWithAliasee(module, "malloc");
 	replaceFunctionAliasWithAliasee(module, "calloc");
@@ -209,6 +221,9 @@ bool GlobalDepsAnalyzer::runOnModule( llvm::Module & module )
 					break;
 				Instruction& I = *instructionIterator;
 
+				if (isAtomicInstruction(I))
+					hasAtomics = true;
+
 				if (isa<CallInst>(I)) {
 					CallInst* ci = cast<CallInst>(&I);
 					Function* calledFunc = ci->getCalledFunction();

llvm/lib/CheerpUtils/IdenticalCodeFolding.cpp

@@ -526,6 +526,15 @@ bool IdenticalCodeFolding::equivalentInstruction(const llvm::Instruction* A, con
 				equivalentOperand(A->getOperand(0), B->getOperand(0)) &&
 				equivalentOperand(A->getOperand(1), B->getOperand(1)));
 		}
+		case Instruction::AtomicRMW:
+		{
+			const AtomicRMWInst* a = cast<AtomicRMWInst>(A);
+			const AtomicRMWInst* b = cast<AtomicRMWInst>(B);
+			return CacheAndReturn(equivalentType(a->getType(), b->getType()) &&
+				a->getOperation() == b->getOperation() &&
+				equivalentOperand(a->getPointerOperand(), b->getPointerOperand()) &&
+				equivalentOperand(a->getValOperand(), b->getValOperand()));
+		}
 		default:
 		{
 #ifndef NDEBUG