update codegen templates, build nodejs

now uses codegen bin to generate rust and js files
wasm bindgen now imports node classes instead of relying on globals
breaks web / no-modules building, but that's because we're optimizing
for nodejs (commonjs) importing
room for improvement might be having wasm-bindgen export the rust node
structs themselves isntead of templating the JS for those (generated
types would be nice as well)

.gitignore

@@ -3,10 +3,12 @@ node_modules
 yarn.lock
 Cargo.lock
 *.wasm
+codegen/codegen
 
 # Rust codegen
 js/nodes.js
 js/messages.js
+js/index.d.ts
 bindings/src/nodes.rs
 bindings/src/messages.rs
 

Makefile

@@ -1,9 +1,13 @@
 include codegen/build.mk
-include build/build.mk
 include tests/build.mk
 include gh-pages/build.mk
 include benchmark/build.mk
 
+build/nodejs: $(CODEGEN_FILES)
+	wasm-pack build --release --target nodejs --no-typescript --out-dir ../build/nodejs bindings
+	cp js/* build/nodejs
+
+CLEAN += build/nodejs
+
 clean:
 	rm -rf $(CLEAN)
-	touch codegen/build.rs

bindings/Cargo.toml

@@ -12,7 +12,7 @@ default = ["console_error_panic_hook"]
 
 [dependencies]
 js-sys = "0.3.55"
-lib-ruby-parser = {version = "3.0.12"}
+lib-ruby-parser = {version = "4.0"}
 wasm-bindgen = "0.2.63"
 
 # The `console_error_panic_hook` crate provides better debugging of panics by

bindings/src/lib.rs

@@ -46,7 +46,7 @@ where
     }
 }
 
-#[wasm_bindgen]
+#[wasm_bindgen(raw_module = "./types")]
 extern "C" {
     #[wasm_bindgen(js_name = Loc)]
     pub type JsLoc;
@@ -111,29 +111,20 @@ impl IntoJs for u8 {
     }
 }
 
-#[wasm_bindgen]
-extern "C" {
-    #[wasm_bindgen(js_name = Bytes)]
-    pub type JsBytes;
-
-    #[wasm_bindgen(constructor, js_class = Bytes)]
-    fn new(raw: Vec<u8>) -> JsBytes;
-}
-
 impl IntoJs for RustBytes {
-    type Output = JsBytes;
-    fn into_js(self) -> JsBytes {
-        JsBytes::new(self.raw)
+    type Output = Vec<u8>;
+    fn into_js(self) -> Vec<u8> {
+        self.raw
     }
 }
 
-#[wasm_bindgen]
+#[wasm_bindgen(raw_module = "./types")]
 extern "C" {
     #[wasm_bindgen(js_name = Token)]
     pub type JsToken;
 
     #[wasm_bindgen(constructor, js_class = Token)]
-    fn new(token_type: i32, token_value: JsBytes, loc: JsLoc) -> JsToken;
+    fn new(token_type: i32, token_value: Vec<u8>, loc: JsLoc) -> JsToken;
 }
 
 impl IntoJs for RustToken {
@@ -147,7 +138,7 @@ impl IntoJs for RustToken {
     }
 }
 
-#[wasm_bindgen]
+#[wasm_bindgen(raw_module = "./types")]
 extern "C" {
     #[wasm_bindgen(js_name = Diagnostic)]
     pub type JsDiagnostic;
@@ -179,7 +170,7 @@ impl IntoJs for RustErrorLevel {
     }
 }
 
-#[wasm_bindgen]
+#[wasm_bindgen(raw_module = "./types")]
 extern "C" {
     #[wasm_bindgen(js_name = Comment)]
     pub type JsComment;
@@ -207,7 +198,7 @@ impl IntoJs for RustCommentType {
     }
 }
 
-#[wasm_bindgen]
+#[wasm_bindgen(raw_module = "./types")]
 extern "C" {
     #[wasm_bindgen(js_name = MagicComment)]
     pub type JsMagicComment;
@@ -240,7 +231,7 @@ impl IntoJs for RustMagicCommentKind {
     }
 }
 
-#[wasm_bindgen]
+#[wasm_bindgen(raw_module = "./types")]
 extern "C" {
     #[wasm_bindgen(js_name = DecodedInput)]
     pub type JsDecodedInput;
@@ -263,7 +254,7 @@ impl IntoJs for RustDecodedInput {
     }
 }
 
-#[wasm_bindgen]
+#[wasm_bindgen(raw_module = "./types")]
 extern "C" {
     #[wasm_bindgen(js_name = SourceLine)]
     pub type JsSourceLine;
@@ -279,7 +270,7 @@ impl IntoJs for RustSourceLine {
     }
 }
 
-#[wasm_bindgen]
+#[wasm_bindgen(raw_module = "./types")]
 extern "C" {
     #[wasm_bindgen(js_name = ParserResult)]
     pub type JsParserResult;

codegen/build.mk

@@ -1,20 +1,34 @@
-CODEGEN_DEPS = $(wildcard codegen/codegen/*.rs)
-CODEGEN_DEPS += codegen/Cargo.toml
-CODEGEN_DEPS += codegen/build.rs
-DO_CODEGEN = cd codegen && cargo build
+define download_file
+curl "$(1)" --fail-with-body -LSso "$(2).tmp" && mv "$(2).tmp" "$(2)"
+endef
 
-js/nodes.js: $(CODEGEN_DEPS)
-	$(DO_CODEGEN)
+CODEGEN_EXE = codegen/codegen$(EXE)
+$(CODEGEN_EXE):
+	$(call download_file,https://github.com/lib-ruby-parser/lib-ruby-parser/releases/download/codegen-v1.0.1/codegen-x86_64-apple-darwin,$@)
+	chmod +x $(CODEGEN_EXE)
+CLEAN += $(CODEGEN_EXE)
+
+js/index.d.ts: codegen/types.d.ts.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
+CLEAN += js/index.d.ts
+CODEGEN_FILES += js/index.d.ts
+
+js/nodes.js: codegen/nodes.js.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
 CLEAN += js/nodes.js
+CODEGEN_FILES += js/nodes.js
 
-js/messages.js: $(CODEGEN_DEPS)
-	$(DO_CODEGEN)
+js/messages.js: codegen/messages.js.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
 CLEAN += js/messages.js
+CODEGEN_FILES += js/messages.js
 
-bindings/src/nodes.rs: $(CODEGEN_DEPS)
-	$(DO_CODEGEN)
+bindings/src/nodes.rs: codegen/nodes.rs.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
 CLEAN += bindings/src/nodes.rs
+CODEGEN_FILES += bindings/src/nodes.rs
 
-bindings/src/messages.rs: $(CODEGEN_DEPS)
-	$(DO_CODEGEN)
+bindings/src/messages.rs: codegen/messages.rs.liquid $(CODEGEN_EXE)
+	$(CODEGEN_EXE) --template $< --write-to $@
 CLEAN += bindings/src/messages.rs
+CODEGEN_FILES += bindings/src/messages.rs