Implement v8 OOM error handling in Go

v8.go

@@ -169,9 +169,12 @@ func NewIsolateWithSnapshot(s *Snapshot) *Isolate {
 }
 
 type ResourceConstraints struct {
+	// MaxOldSpaceSize sets the maximum size of the old object heap in MiB.
 	MaxOldSpaceSize int
 }
 
+// NewIsolateWithConstraints creates a new V8 Isolate applying additional
+// resource constraints to limit the V8 runtime.
 func NewIsolateWithConstraints(constraints ResourceConstraints) *Isolate {
 	v8_init_once.Do(func() { C.v8_init() })
 	var c = C.ResourceConstraints{max_old_space_size: C.int(constraints.MaxOldSpaceSize)}
@@ -217,6 +220,22 @@ func (i *Isolate) convertErrorMsg(error_msg C.Error) error {
 	return err
 }
 
+type OOMErrorCallback func(location string, isHeapOOM bool)
+
+var GlobalOOMErrorHandler OOMErrorCallback
+
+//export OOMErrorHandler
+func OOMErrorHandler(location string, isHeapOOM bool) {
+	if GlobalOOMErrorHandler != nil {
+		GlobalOOMErrorHandler(location, isHeapOOM)
+	}
+}
+
+func (i *Isolate) SetOOMErrorHandler(fn OOMErrorCallback) {
+	GlobalOOMErrorHandler = fn
+	C.v8_Isolate_SetOOMErrorHandler(i.ptr)
+}
+
 // Context is a sandboxed js environment with its own set of built-in objects
 // and functions.  Values and javascript operations within a context are visible
 // only within that context unless the Go code explicitly moves values from one

v8_c_bridge.cc

@@ -1,4 +1,5 @@
 #include "v8_c_bridge.h"
+#include "_cgo_export.h"
 
 #include "libplatform/libplatform.h"
 #include "v8.h"
@@ -182,6 +183,16 @@ StartupData v8_CreateSnapshotDataBlob(const char* js) {
   return StartupData{data.data, data.raw_size};
 }
 
+static void delegateOOMErrorHandler(const char* location, bool is_heap_oom) {
+    GoString str = {location};
+    OOMErrorHandler(str, is_heap_oom);
+}
+
+void v8_Isolate_SetOOMErrorHandler(IsolatePtr isolate_ptr) {
+  v8::Isolate* isolate = static_cast<v8::Isolate*>(isolate_ptr);
+  isolate->SetOOMErrorHandler(delegateOOMErrorHandler);
+}
+
 IsolatePtr v8_Isolate_New(StartupData startup_data, ResourceConstraints* resource_constraints) {
   v8::Isolate::CreateParams create_params;
   create_params.array_buffer_allocator = allocator;

v8_c_bridge.h

@@ -122,6 +122,7 @@ extern IsolatePtr v8_Isolate_New(StartupData data, ResourceConstraints* resource
 extern ContextPtr v8_Isolate_NewContext(IsolatePtr isolate);
 extern void       v8_Isolate_Terminate(IsolatePtr isolate);
 extern void       v8_Isolate_Release(IsolatePtr isolate);
+extern void       v8_Isolate_SetOOMErrorHandler(IsolatePtr isolate);
 
 extern HeapStatistics       v8_Isolate_GetHeapStatistics(IsolatePtr isolate);
 extern void                 v8_Isolate_LowMemoryNotification(IsolatePtr isolate);

v8_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math"
 	"math/big"
+	"os"
 	"reflect"
 	"regexp"
 	"runtime"
@@ -1541,3 +1542,23 @@ func TestPanicHandling(t *testing.T) {
 	_ = NewIsolate()
 	_ = *f
 }
+
+func TestNewIsolateWithConstraints(t *testing.T) {
+	// Creates a v8 runtime where the memory is limited to 10MB and memory is
+	// allocated with a small script until v8 runs out of memory.
+	t.Parallel()
+	isolate := NewIsolateWithConstraints(ResourceConstraints{MaxOldSpaceSize: 10})
+	isolate.SetOOMErrorHandler(func(location string, isHeapOOM bool) {
+		// Success
+		os.Exit(0)
+	})
+	ctx := isolate.NewContext()
+	i := 0
+	for {
+		_, err := ctx.Eval(fmt.Sprintf("var array%d = new Array(100000); array%d.fill(1);", i, i), "test.js")
+		if err != nil {
+			t.Fatalf("Error evaluating javascript, err: %v", err)
+		}
+		i++
+	}
+}