From c40ca24f82bf891b0b4533b954d3fd03378a7eb0 Mon Sep 17 00:00:00 2001 From: Tooru Fujisawa Date: Sat, 8 Aug 2020 00:23:11 +0900 Subject: [PATCH 1/2] Use SourceSliceIndex for string literal (fixes #624) --- crates/ast/ast.json | 2 +- crates/ast/generate_ast.py | 1 - crates/ast/src/source_atom_set.rs | 1 - crates/emitter/src/ast_emitter.rs | 2 +- crates/emitter/src/emitter.rs | 5 +++++ crates/generated_parser/src/ast_builder.rs | 10 +++++++--- crates/interpreter/src/evaluate.rs | 23 ++++++++++++++++++++-- crates/parser/src/lexer.rs | 2 +- crates/stencil/src/gcthings.rs | 8 ++++++++ gecko-patches.txt | 2 +- 10 files changed, 45 insertions(+), 11 deletions(-) diff --git a/crates/ast/ast.json b/crates/ast/ast.json index e02d91648..5fc0ac8a5 100644 --- a/crates/ast/ast.json +++ b/crates/ast/ast.json @@ -210,7 +210,7 @@ "unicode": "bool" }, "LiteralStringExpression": { - "value": "SourceAtomSetIndex" + "value": "SourceSliceIndex" }, "ArrayExpression": "ArrayExpression", "ArrowExpression": { diff --git a/crates/ast/generate_ast.py b/crates/ast/generate_ast.py index dab3f96aa..ba31aa30c 100755 --- a/crates/ast/generate_ast.py +++ b/crates/ast/generate_ast.py @@ -429,7 +429,6 @@ def emit_variant_none_call(indent, enum_name, variant_name): write(0, "#![allow(dead_code)]") write(0, "") write(0, "use crate::arena;") - write(0, "use crate::source_atom_set::SourceAtomSetIndex;") write(0, "use crate::source_slice_list::SourceSliceIndex;") write(0, "use crate::types::*;") write(0, "") diff --git a/crates/ast/src/source_atom_set.rs b/crates/ast/src/source_atom_set.rs index 25369c977..6240e1ff5 100644 --- a/crates/ast/src/source_atom_set.rs +++ b/crates/ast/src/source_atom_set.rs @@ -111,7 +111,6 @@ macro_rules! for_all_common_atoms { ("while", while_, While), ("with", with, With), ("yield", yield_, Yield), - ("use strict", use_strict, UseStrict), ("__proto__", __proto__, Proto), ); } diff --git a/crates/emitter/src/ast_emitter.rs b/crates/emitter/src/ast_emitter.rs index 1b3480b95..a2280be1e 100644 --- a/crates/emitter/src/ast_emitter.rs +++ b/crates/emitter/src/ast_emitter.rs @@ -453,7 +453,7 @@ impl<'alloc, 'opt> AstEmitter<'alloc, 'opt> { } Expression::LiteralStringExpression { value, .. } => { - let str_index = self.emit.get_atom_gcthing_index(*value); + let str_index = self.emit.get_slice_gcthing_index(*value); self.emit.string(str_index); } diff --git a/crates/emitter/src/emitter.rs b/crates/emitter/src/emitter.rs index 734e4da87..b9a77f1de 100644 --- a/crates/emitter/src/emitter.rs +++ b/crates/emitter/src/emitter.rs @@ -6,6 +6,7 @@ #![allow(dead_code)] use ast::source_atom_set::SourceAtomSetIndex; +use ast::source_slice_list::SourceSliceIndex; use byteorder::{ByteOrder, LittleEndian}; use std::cmp; use std::collections::HashMap; @@ -1384,6 +1385,10 @@ impl InstructionWriter { } } + pub fn get_slice_gcthing_index(&mut self, slice: SourceSliceIndex) -> GCThingIndex { + self.gcthings.push_slice(slice) + } + pub fn get_function_gcthing_index(&mut self, fun_index: ScriptStencilIndex) -> GCThingIndex { self.gcthings.push_function(fun_index) } diff --git a/crates/generated_parser/src/ast_builder.rs b/crates/generated_parser/src/ast_builder.rs index 6de757a96..4149737d9 100644 --- a/crates/generated_parser/src/ast_builder.rs +++ b/crates/generated_parser/src/ast_builder.rs @@ -766,8 +766,9 @@ impl<'alloc> AstBuilder<'alloc> { let loc = token.loc; // Hack: Prevent emission for scripts with "use strict" // directive. - let value = token.value.as_atom(); - if value == CommonSourceAtomSetIndices::use_strict() { + let value = token.value.as_slice(); + let slice = self.slices.borrow().get(value); + if slice == "use strict" { return Err(ParseError::NotImplemented("use strict directive").into()); } @@ -1050,7 +1051,10 @@ impl<'alloc> AstBuilder<'alloc> { &self, token: arena::Box<'alloc, Token>, ) -> Result<'alloc, arena::Box<'alloc, PropertyName<'alloc>>> { - let value = token.value.as_atom(); + let name = self.slices.borrow().get(token.value.as_slice()); + + let value = self.atoms.borrow_mut().insert(name); + if value == CommonSourceAtomSetIndices::__proto__() { return Err(ParseError::NotImplemented("__proto__ as property name").into()); } diff --git a/crates/interpreter/src/evaluate.rs b/crates/interpreter/src/evaluate.rs index 0d70a6415..8e240bbe5 100644 --- a/crates/interpreter/src/evaluate.rs +++ b/crates/interpreter/src/evaluate.rs @@ -39,6 +39,12 @@ trait Helpers<'alloc> { fn read_offset(&self, offset: usize) -> isize; fn read_atom(&self, offset: usize, gcthings: &Vec, atoms: &Vec<&'alloc str>) -> String; + fn read_slice( + &self, + offset: usize, + gcthings: &Vec, + slices: &Vec<&'alloc str>, + ) -> String; } impl<'alloc> Helpers<'alloc> for ImmutableScriptData { @@ -77,6 +83,18 @@ impl<'alloc> Helpers<'alloc> for ImmutableScriptData { _ => panic!("Unexpected GC things for string"), } } + + fn read_slice( + &self, + offset: usize, + gcthings: &Vec, + slices: &Vec<&'alloc str>, + ) -> String { + match gcthings[self.read_i32(offset) as usize] { + GCThing::Slice(index) => slices[usize::from(index)].to_string(), + _ => panic!("Unexpected GC things for string"), + } + } } /// This functions can partially interpreter the bytecode shared with SpiderMonkey. @@ -96,6 +114,7 @@ pub fn evaluate(result: &EmitResult, global: Rc>) -> Result>) -> Result { - stack.push(JSValue::String(immutable_script_data.read_atom( + stack.push(JSValue::String(immutable_script_data.read_slice( pc + 1, gcthings, - atoms, + slices, ))); } diff --git a/crates/parser/src/lexer.rs b/crates/parser/src/lexer.rs index 109558c5c..224be0add 100644 --- a/crates/parser/src/lexer.rs +++ b/crates/parser/src/lexer.rs @@ -1378,7 +1378,7 @@ impl<'alloc> Lexer<'alloc> { Some(c @ '"') | Some(c @ '\'') => { if c == delimiter { - let value = self.string_to_token_value(builder.finish_without_push(&self)); + let value = self.slice_to_token_value(builder.finish_without_push(&self)); return self.set_result( TerminalId::StringLiteral, SourceLocation::new(offset, self.offset()), diff --git a/crates/stencil/src/gcthings.rs b/crates/stencil/src/gcthings.rs index bbe18fd20..e1c40791b 100644 --- a/crates/stencil/src/gcthings.rs +++ b/crates/stencil/src/gcthings.rs @@ -2,6 +2,7 @@ use crate::regexp::RegExpIndex; use crate::scope::ScopeIndex; use crate::script::ScriptStencilIndex; use ast::source_atom_set::SourceAtomSetIndex; +use ast::source_slice_list::SourceSliceIndex; /// Corresponds to js::frontend::GCThingList::ListType /// in m-c/js/src/frontend/BytecodeSection.h. @@ -9,6 +10,7 @@ use ast::source_atom_set::SourceAtomSetIndex; pub enum GCThing { Null, Atom(SourceAtomSetIndex), + Slice(SourceSliceIndex), Function(ScriptStencilIndex), RegExp(RegExpIndex), Scope(ScopeIndex), @@ -51,6 +53,12 @@ impl GCThingList { GCThingIndex::new(index) } + pub fn push_slice(&mut self, slice_index: SourceSliceIndex) -> GCThingIndex { + let index = self.things.len(); + self.things.push(GCThing::Slice(slice_index)); + GCThingIndex::new(index) + } + pub fn push_function(&mut self, fun_index: ScriptStencilIndex) -> GCThingIndex { let index = self.things.len(); self.things.push(GCThing::Function(fun_index)); diff --git a/gecko-patches.txt b/gecko-patches.txt index 17858e08f..f5de7a738 100644 --- a/gecko-patches.txt +++ b/gecko-patches.txt @@ -1 +1 @@ -D85363:1648574 +D86376:1657937 From 8aa6f6e96ff619a15d2b7bc5a49de5bbaadc14ad Mon Sep 17 00:00:00 2001 From: Tooru Fujisawa Date: Sat, 8 Aug 2020 05:40:23 +0900 Subject: [PATCH 2/2] Defer atomization to AstBuilder --- crates/ast/ast.json | 2 +- crates/ast/generate_ast.py | 2 +- crates/emitter/src/object_emitter.rs | 14 +- crates/generated_parser/src/ast_builder.rs | 575 +++++++++--------- .../src/early_error_checker.rs | 28 +- crates/generated_parser/src/early_errors.rs | 78 ++- crates/generated_parser/src/error.rs | 2 +- crates/generated_parser/src/token.rs | 28 +- crates/parser/src/lexer.rs | 296 ++------- crates/parser/src/lib.rs | 6 +- crates/parser/src/parser.rs | 6 +- crates/parser/src/tests.rs | 16 +- 12 files changed, 461 insertions(+), 592 deletions(-) diff --git a/crates/ast/ast.json b/crates/ast/ast.json index 5fc0ac8a5..5f5748834 100644 --- a/crates/ast/ast.json +++ b/crates/ast/ast.json @@ -602,7 +602,7 @@ }, "StaticPropertyName": { "_type": "struct", - "value": "SourceAtomSetIndex" + "value": "SourceSliceIndex" }, "NumericLiteral": { "_type": "struct", diff --git a/crates/ast/generate_ast.py b/crates/ast/generate_ast.py index ba31aa30c..b4152689d 100755 --- a/crates/ast/generate_ast.py +++ b/crates/ast/generate_ast.py @@ -55,7 +55,7 @@ def to_rust_type(self, ast): if self.name in RUST_BUILTIN_TYPES: return self.name if self.name == 'Token': - return "Token" + return "Token<'alloc>" if self.name in ast.type_decls and ast.type_decls[self.name].has_lifetime: return "{}<'alloc>".format(self.name) return self.name diff --git a/crates/emitter/src/object_emitter.rs b/crates/emitter/src/object_emitter.rs index 9f77382aa..1336a720a 100644 --- a/crates/emitter/src/object_emitter.rs +++ b/crates/emitter/src/object_emitter.rs @@ -1,6 +1,6 @@ use crate::ast_emitter::AstEmitter; use crate::emitter::EmitError; -use ast::source_atom_set::SourceAtomSetIndex; +use ast::source_slice_list::SourceSliceIndex; /// Struct for emitting bytecode for a property where its name is string. /// @@ -12,7 +12,7 @@ where F: Fn(&mut AstEmitter) -> Result<(), EmitError>, { pub state: &'a mut ObjectEmitterState, - pub key: SourceAtomSetIndex, + pub key: SourceSliceIndex, pub value: F, } @@ -34,7 +34,7 @@ where // [stack] OBJ } None => { - let name_index = emitter.emit.get_atom_gcthing_index(self.key); + let name_index = emitter.emit.get_slice_gcthing_index(self.key); (self.value)(emitter)?; // [stack] OBJ VALUE @@ -46,12 +46,8 @@ where Ok(()) } - fn to_property_index( - &self, - emitter: &mut AstEmitter, - index: SourceAtomSetIndex, - ) -> Option { - let s = emitter.compilation_info.atoms.get(index); + fn to_property_index(&self, emitter: &mut AstEmitter, index: SourceSliceIndex) -> Option { + let s = emitter.compilation_info.slices.get(index); s.parse::().ok() } } diff --git a/crates/generated_parser/src/ast_builder.rs b/crates/generated_parser/src/ast_builder.rs index 4149737d9..c11a6bd6d 100644 --- a/crates/generated_parser/src/ast_builder.rs +++ b/crates/generated_parser/src/ast_builder.rs @@ -4,6 +4,7 @@ use crate::error::{BoxedParseError, ParseError, Result}; use crate::Token; use ast::{ arena, + source_atom_set::SourceAtomSetIndex, source_atom_set::{CommonSourceAtomSetIndices, SourceAtomSet}, source_location_accessor::SourceLocationAccessor, source_slice_list::SourceSliceList, @@ -87,21 +88,23 @@ impl<'alloc> AstBuilder<'alloc> { // IdentifierReference : Identifier pub fn identifier_reference( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Identifier>> { - self.on_identifier_reference(&token)?; - Ok(self.alloc_with(|| self.identifier(token))) + let token_atom = self.atoms.borrow_mut().insert(token.value.as_str()); + self.on_identifier_reference(&token, token_atom)?; + Ok(self.alloc_with(|| self.identifier_with_atom(token, token_atom))) } // BindingIdentifier : Identifier pub fn binding_identifier( &mut self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, BindingIdentifier>> { - self.on_binding_identifier(&token)?; + let token_atom = self.atoms.borrow_mut().insert(token.value.as_str()); + self.on_binding_identifier(&token, token_atom)?; let loc = token.loc; Ok(self.alloc_with(|| BindingIdentifier { - name: self.identifier(token), + name: self.identifier_with_atom(token, token_atom), loc, })) } @@ -109,13 +112,14 @@ impl<'alloc> AstBuilder<'alloc> { // BindingIdentifier : `yield` pub fn binding_identifier_yield( &mut self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, BindingIdentifier>> { - self.on_binding_identifier(&token)?; + let token_atom = CommonSourceAtomSetIndices::yield_(); + self.on_binding_identifier(&token, token_atom)?; let loc = token.loc; Ok(self.alloc_with(|| BindingIdentifier { name: Identifier { - value: CommonSourceAtomSetIndices::yield_(), + value: token_atom, loc, }, loc, @@ -125,13 +129,14 @@ impl<'alloc> AstBuilder<'alloc> { // BindingIdentifier : `await` pub fn binding_identifier_await( &mut self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, BindingIdentifier>> { - self.on_binding_identifier(&token)?; + let token_atom = CommonSourceAtomSetIndices::await_(); + self.on_binding_identifier(&token, token_atom)?; let loc = token.loc; Ok(self.alloc_with(|| BindingIdentifier { name: Identifier { - value: CommonSourceAtomSetIndices::await_(), + value: token_atom, loc, }, loc, @@ -141,12 +146,13 @@ impl<'alloc> AstBuilder<'alloc> { // LabelIdentifier : Identifier pub fn label_identifier( &mut self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Label>> { - self.on_label_identifier(&token)?; + let token_atom = self.atoms.borrow_mut().insert(token.value.as_str()); + self.on_label_identifier(&token, token_atom)?; let loc = token.loc; Ok(self.alloc_with(|| Label { - value: token.value.as_atom(), + value: token_atom, loc, })) } @@ -154,7 +160,7 @@ impl<'alloc> AstBuilder<'alloc> { // PrimaryExpression : `this` pub fn this_expr( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let loc = token.loc; self.alloc_with(|| Expression::ThisExpression { loc }) @@ -177,9 +183,9 @@ impl<'alloc> AstBuilder<'alloc> { // PrimaryExpression : RegularExpressionLiteral pub fn regexp_literal( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { - let source = self.slices.borrow().get(token.value.as_slice()); + let source = token.value.as_str(); debug_assert!(source.chars().nth(0).unwrap() == '/'); @@ -257,9 +263,9 @@ impl<'alloc> AstBuilder<'alloc> { // CoverParenthesizedExpressionAndArrowParameterList : `(` Expression `)` pub fn cover_parenthesized_expression( &self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, expression: arena::Box<'alloc, Expression<'alloc>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, CoverParenthesized<'alloc>> { self.alloc_with(|| CoverParenthesized::Expression { expression, @@ -688,10 +694,10 @@ impl<'alloc> AstBuilder<'alloc> { // CoverParenthesizedExpressionAndArrowParameterList : `(` Expression `,` `...` BindingPattern `)` pub fn cover_arrow_parameter_list( &self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, parameters: arena::Vec<'alloc, Parameter<'alloc>>, rest: Option>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, CoverParenthesized<'alloc>> { self.alloc_with(|| { CoverParenthesized::Parameters(self.alloc_with(|| FormalParameters { @@ -705,7 +711,7 @@ impl<'alloc> AstBuilder<'alloc> { // Literal : NullLiteral pub fn null_literal( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let loc = token.loc; self.alloc_with(|| Expression::LiteralNullExpression { loc }) @@ -714,28 +720,26 @@ impl<'alloc> AstBuilder<'alloc> { // Literal : BooleanLiteral pub fn boolean_literal( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let loc = token.loc; - let s = token.value.as_atom(); - assert!( - s == CommonSourceAtomSetIndices::true_() || s == CommonSourceAtomSetIndices::false_() - ); + let s = token.value.as_str(); + debug_assert!(s == "true" || s == "false"); self.alloc_with(|| Expression::LiteralBooleanExpression { - value: s == CommonSourceAtomSetIndices::true_(), + value: s == "true", loc, }) } - fn numeric_literal_value(token: arena::Box<'alloc, Token>) -> f64 { + fn numeric_literal_value(token: arena::Box<'alloc, Token<'alloc>>) -> f64 { token.unbox().value.as_number() } // Literal : NumericLiteral pub fn numeric_literal( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { let loc = token.loc; Ok(self.alloc_with(|| { @@ -753,7 +757,7 @@ impl<'alloc> AstBuilder<'alloc> { // * NonDecimalIntegerLiteralBigIntLiteralSuffix pub fn bigint_literal( &self, - _token: arena::Box<'alloc, Token>, + _token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { Err(ParseError::NotImplemented("BigInt").into()) } @@ -761,26 +765,25 @@ impl<'alloc> AstBuilder<'alloc> { // Literal : StringLiteral pub fn string_literal( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { let loc = token.loc; // Hack: Prevent emission for scripts with "use strict" // directive. - let value = token.value.as_slice(); - let slice = self.slices.borrow().get(value); - if slice == "use strict" { + let s = token.value.as_str(); + if s == "use strict" { return Err(ParseError::NotImplemented("use strict directive").into()); } - + let value = self.slices.borrow_mut().push(s); Ok(self.alloc_with(|| Expression::LiteralStringExpression { value, loc })) } // ArrayLiteral : `[` Elision? `]` pub fn array_literal_empty( &self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, elision: Option>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { self.alloc_with(|| { Expression::ArrayExpression(match elision { @@ -799,9 +802,9 @@ impl<'alloc> AstBuilder<'alloc> { // ArrayLiteral : `[` ElementList `]` pub fn array_literal( &self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, mut array: arena::Box<'alloc, ArrayExpression<'alloc>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { array.loc.set_range(open_token.loc, close_token.loc); self.alloc_with(|| Expression::ArrayExpression(array.unbox())) @@ -810,10 +813,10 @@ impl<'alloc> AstBuilder<'alloc> { // ArrayLiteral : `[` ElementList `,` Elision? `]` pub fn array_literal_with_trailing_elision( &self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, mut array: arena::Box<'alloc, ArrayExpression<'alloc>>, elision: Option>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { if let Some(mut more) = elision { self.append(&mut array.elements, &mut more.elements); @@ -901,7 +904,7 @@ impl<'alloc> AstBuilder<'alloc> { // Elision : `,` pub fn elision_single( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, ArrayExpression<'alloc>> { let loc = token.loc; self.alloc_with(|| ArrayExpression { @@ -915,7 +918,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn elision_append( &self, mut array: arena::Box<'alloc, ArrayExpression<'alloc>>, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, ArrayExpression<'alloc>> { let loc = token.loc; self.push(&mut array.elements, ArrayExpressionElement::Elision { loc }); @@ -933,8 +936,8 @@ impl<'alloc> AstBuilder<'alloc> { // ObjectLiteral : `{` `}` pub fn object_literal_empty( &self, - open_token: arena::Box<'alloc, Token>, - close_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { self.alloc_with(|| { Expression::ObjectExpression(ObjectExpression { @@ -948,9 +951,9 @@ impl<'alloc> AstBuilder<'alloc> { // ObjectLiteral : `{` PropertyDefinitionList `,` `}` pub fn object_literal( &self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, mut object: arena::Box<'alloc, ObjectExpression<'alloc>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { object.loc.set_range(open_token.loc, close_token.loc); self.alloc_with(|| Expression::ObjectExpression(object.unbox())) @@ -1035,38 +1038,38 @@ impl<'alloc> AstBuilder<'alloc> { // LiteralPropertyName : IdentifierName pub fn property_name_identifier( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, PropertyName<'alloc>>> { - let value = token.value.as_atom(); - if value == CommonSourceAtomSetIndices::__proto__() { + let s = token.value.as_str(); + if s == "__proto__" { return Err(ParseError::NotImplemented("__proto__ as property name").into()); } let loc = token.loc; + let value = self.slices.borrow_mut().push(s); Ok(self.alloc_with(|| PropertyName::StaticPropertyName(StaticPropertyName { value, loc }))) } // LiteralPropertyName : StringLiteral pub fn property_name_string( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, PropertyName<'alloc>>> { - let name = self.slices.borrow().get(token.value.as_slice()); - - let value = self.atoms.borrow_mut().insert(name); + let s = token.value.as_str(); - if value == CommonSourceAtomSetIndices::__proto__() { + if s == "__proto__" { return Err(ParseError::NotImplemented("__proto__ as property name").into()); } let loc = token.loc; + let value = self.slices.borrow_mut().push(s); Ok(self.alloc_with(|| PropertyName::StaticPropertyName(StaticPropertyName { value, loc }))) } // LiteralPropertyName : NumericLiteral pub fn property_name_numeric( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, PropertyName<'alloc>>> { let loc = token.loc; let value = Self::numeric_literal_value(token); @@ -1081,7 +1084,7 @@ impl<'alloc> AstBuilder<'alloc> { // * NonDecimalIntegerLiteralBigIntLiteralSuffix pub fn property_name_bigint( &self, - _token: arena::Box<'alloc, Token>, + _token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, PropertyName<'alloc>>> { Err(ParseError::NotImplemented("BigInt").into()) } @@ -1089,9 +1092,9 @@ impl<'alloc> AstBuilder<'alloc> { // ComputedPropertyName : `[` AssignmentExpression `]` pub fn computed_property_name( &self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, expression: arena::Box<'alloc, Expression<'alloc>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, PropertyName<'alloc>> { self.alloc_with(|| { PropertyName::ComputedPropertyName(ComputedPropertyName { @@ -1116,16 +1119,14 @@ impl<'alloc> AstBuilder<'alloc> { // TemplateLiteral : NoSubstitutionTemplate pub fn template_literal( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, TemplateExpression<'alloc>> { let loc = token.loc; + let raw_value = self.atoms.borrow_mut().insert(token.value.as_str()); self.alloc_with(|| TemplateExpression { tag: None, elements: self.new_vec_single(TemplateExpressionElement::TemplateElement( - TemplateElement { - raw_value: token.value.as_atom(), - loc, - }, + TemplateElement { raw_value, loc }, )), loc, }) @@ -1134,7 +1135,7 @@ impl<'alloc> AstBuilder<'alloc> { // SubstitutionTemplate : TemplateHead Expression TemplateSpans pub fn substitution_template( &self, - _head: arena::Box<'alloc, Token>, + _head: arena::Box<'alloc, Token<'alloc>>, _expression: arena::Box<'alloc, Expression<'alloc>>, _spans: arena::Box<'alloc, Void>, ) -> Result<'alloc, arena::Box<'alloc, TemplateExpression<'alloc>>> { @@ -1146,7 +1147,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn template_spans( &self, _middle_list: Option>, - _tail: arena::Box<'alloc, Token>, + _tail: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Void>> { Err(ParseError::NotImplemented("template strings").into()) } @@ -1154,7 +1155,7 @@ impl<'alloc> AstBuilder<'alloc> { // TemplateMiddleList : TemplateMiddle Expression pub fn template_middle_list_single( &self, - _middle: arena::Box<'alloc, Token>, + _middle: arena::Box<'alloc, Token<'alloc>>, _expression: arena::Box<'alloc, Expression<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Void>> { Err(ParseError::NotImplemented("template strings").into()) @@ -1164,7 +1165,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn template_middle_list_append( &self, _middle_list: arena::Box<'alloc, Void>, - _middle: arena::Box<'alloc, Token>, + _middle: arena::Box<'alloc, Token<'alloc>>, _expression: arena::Box<'alloc, Expression<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Void>> { Err(ParseError::NotImplemented("template strings").into()) @@ -1176,7 +1177,7 @@ impl<'alloc> AstBuilder<'alloc> { &self, object: arena::Box<'alloc, Expression<'alloc>>, expression: arena::Box<'alloc, Expression<'alloc>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let object_loc = object.get_loc(); self.alloc_with(|| { @@ -1210,9 +1211,9 @@ impl<'alloc> AstBuilder<'alloc> { // OptionalChain : `?.` `[` Expression `]` pub fn optional_computed_member_expr_tail( &self, - start_token: arena::Box<'alloc, Token>, + start_token: arena::Box<'alloc, Token<'alloc>>, expression: arena::Box<'alloc, Expression<'alloc>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { self.alloc_with(|| { Expression::OptionalChain(OptionalChain::ComputedMemberExpressionTail { @@ -1225,8 +1226,8 @@ impl<'alloc> AstBuilder<'alloc> { // OptionalChain : `?.` Expression pub fn optional_static_member_expr_tail( &self, - start_token: arena::Box<'alloc, Token>, - identifier_token: arena::Box<'alloc, Token>, + start_token: arena::Box<'alloc, Token<'alloc>>, + identifier_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let identifier_token_loc = identifier_token.loc; self.alloc_with(|| { @@ -1240,8 +1241,8 @@ impl<'alloc> AstBuilder<'alloc> { // OptionalChain : `?.` PrivateIdentifier pub fn optional_private_field_member_expr_tail( &self, - start_token: arena::Box<'alloc, Token>, - private_identifier: arena::Box<'alloc, Token>, + start_token: arena::Box<'alloc, Token<'alloc>>, + private_identifier: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { let private_identifier_loc = private_identifier.loc; let field = self.private_identifier(private_identifier)?; @@ -1256,7 +1257,7 @@ impl<'alloc> AstBuilder<'alloc> { // OptionalChain : `?.` Arguments pub fn optional_call_expr_tail( &self, - start_token: arena::Box<'alloc, Token>, + start_token: arena::Box<'alloc, Token<'alloc>>, arguments: arena::Box<'alloc, Arguments<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let arguments_loc = arguments.loc; @@ -1280,7 +1281,7 @@ impl<'alloc> AstBuilder<'alloc> { &self, object: arena::Box<'alloc, Expression<'alloc>>, expression: arena::Box<'alloc, Expression<'alloc>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let object_loc = object.get_loc(); self.alloc_with(|| { @@ -1298,7 +1299,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn optional_static_member_expr( &self, object: arena::Box<'alloc, Expression<'alloc>>, - identifier_token: arena::Box<'alloc, Token>, + identifier_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let object_loc = object.get_loc(); let identifier_token_loc = identifier_token.loc; @@ -1317,7 +1318,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn optional_private_field_member_expr( &self, object: arena::Box<'alloc, Expression<'alloc>>, - private_identifier: arena::Box<'alloc, Token>, + private_identifier: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { let object_loc = object.get_loc(); let private_identifier_loc = private_identifier.loc; @@ -1350,31 +1351,37 @@ impl<'alloc> AstBuilder<'alloc> { }) } - fn identifier(&self, token: arena::Box<'alloc, Token>) -> Identifier { + fn identifier_with_atom( + &self, + token: arena::Box<'alloc, Token<'alloc>>, + token_atom: SourceAtomSetIndex, + ) -> Identifier { Identifier { - value: token.value.as_atom(), + value: token_atom, loc: token.loc, } } - fn identifier_name(&self, token: arena::Box<'alloc, Token>) -> IdentifierName { + fn identifier_name(&self, token: arena::Box<'alloc, Token<'alloc>>) -> IdentifierName { + let value = self.atoms.borrow_mut().insert(token.value.as_str()); IdentifierName { - value: token.value.as_atom(), + value, loc: token.loc, } } fn private_identifier( &self, - _token: arena::Box<'alloc, Token>, + _token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, PrivateIdentifier> { Err(ParseError::NotImplemented( "private fields depends on shell switch, that is not supported", ) .into()) /* + let value = self.atoms.borrow_mut().insert(token.value.as_str()); PrivateIdentifier { - value: token.value.as_atom(), + value, loc: token.loc, } */ @@ -1385,7 +1392,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn static_member_expr( &self, object: arena::Box<'alloc, Expression<'alloc>>, - identifier_token: arena::Box<'alloc, Token>, + identifier_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let object_loc = object.get_loc(); let identifier_token_loc = identifier_token.loc; @@ -1414,7 +1421,7 @@ impl<'alloc> AstBuilder<'alloc> { // MemberExpression : `new` MemberExpression Arguments pub fn new_expr_with_arguments( &self, - new_token: arena::Box<'alloc, Token>, + new_token: arena::Box<'alloc, Token<'alloc>>, callee: arena::Box<'alloc, Expression<'alloc>>, arguments: arena::Box<'alloc, Arguments<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { @@ -1430,7 +1437,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn private_field_expr( &self, object: arena::Box<'alloc, Expression<'alloc>>, - private_identifier: arena::Box<'alloc, Token>, + private_identifier: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { let object_loc = object.get_loc(); let field_loc = private_identifier.loc; @@ -1449,9 +1456,9 @@ impl<'alloc> AstBuilder<'alloc> { // SuperProperty : `super` `[` Expression `]` pub fn super_property_computed( &self, - super_token: arena::Box<'alloc, Token>, + super_token: arena::Box<'alloc, Token<'alloc>>, expression: arena::Box<'alloc, Expression<'alloc>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { self.check_super()?; @@ -1470,8 +1477,8 @@ impl<'alloc> AstBuilder<'alloc> { // SuperProperty : `super` `.` IdentifierName pub fn super_property_static( &self, - super_token: arena::Box<'alloc, Token>, - identifier_token: arena::Box<'alloc, Token>, + super_token: arena::Box<'alloc, Token<'alloc>>, + identifier_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { self.check_super()?; @@ -1491,8 +1498,8 @@ impl<'alloc> AstBuilder<'alloc> { // NewTarget : `new` `.` `target` pub fn new_target_expr( &self, - new_token: arena::Box<'alloc, Token>, - target_token: arena::Box<'alloc, Token>, + new_token: arena::Box<'alloc, Token<'alloc>>, + target_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { return self.alloc_with(|| Expression::NewTargetExpression { loc: SourceLocation::from_parts(new_token.loc, target_token.loc), @@ -1502,7 +1509,7 @@ impl<'alloc> AstBuilder<'alloc> { // NewExpression : `new` NewExpression pub fn new_expr_without_arguments( &self, - new_token: arena::Box<'alloc, Token>, + new_token: arena::Box<'alloc, Token<'alloc>>, callee: arena::Box<'alloc, Expression<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let callee_loc = callee.get_loc(); @@ -1538,7 +1545,7 @@ impl<'alloc> AstBuilder<'alloc> { // SuperCall : `super` Arguments pub fn super_call( &self, - super_token: arena::Box<'alloc, Token>, + super_token: arena::Box<'alloc, Token<'alloc>>, arguments: arena::Box<'alloc, Arguments<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { self.check_super()?; @@ -1557,9 +1564,9 @@ impl<'alloc> AstBuilder<'alloc> { // ImportCall : `import` `(` AssignmentExpression `)` pub fn import_call( &self, - import_token: arena::Box<'alloc, Token>, + import_token: arena::Box<'alloc, Token<'alloc>>, argument: arena::Box<'alloc, Expression<'alloc>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { self.alloc_with(|| Expression::ImportCallExpression { argument, @@ -1570,8 +1577,8 @@ impl<'alloc> AstBuilder<'alloc> { // Arguments : `(` `)` pub fn arguments_empty( &self, - open_token: arena::Box<'alloc, Token>, - close_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Arguments<'alloc>> { self.alloc_with(|| Arguments { args: self.new_vec(), @@ -1605,9 +1612,9 @@ impl<'alloc> AstBuilder<'alloc> { pub fn arguments( &self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, mut arguments: arena::Box<'alloc, Arguments<'alloc>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Arguments<'alloc>> { arguments.loc.set_range(open_token.loc, close_token.loc); arguments @@ -1639,7 +1646,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn post_increment_expr( &self, operand: arena::Box<'alloc, Expression<'alloc>>, - operator_token: arena::Box<'alloc, Token>, + operator_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { let operand = self.expression_to_simple_assignment_target(operand)?; let operand_loc = operand.get_loc(); @@ -1657,7 +1664,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn post_decrement_expr( &self, operand: arena::Box<'alloc, Expression<'alloc>>, - operator_token: arena::Box<'alloc, Token>, + operator_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { let operand = self.expression_to_simple_assignment_target(operand)?; let operand_loc = operand.get_loc(); @@ -1674,7 +1681,7 @@ impl<'alloc> AstBuilder<'alloc> { // UpdateExpression : `++` UnaryExpression pub fn pre_increment_expr( &self, - operator_token: arena::Box<'alloc, Token>, + operator_token: arena::Box<'alloc, Token<'alloc>>, operand: arena::Box<'alloc, Expression<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { let operand = self.expression_to_simple_assignment_target(operand)?; @@ -1692,7 +1699,7 @@ impl<'alloc> AstBuilder<'alloc> { // UpdateExpression : `--` UnaryExpression pub fn pre_decrement_expr( &self, - operator_token: arena::Box<'alloc, Token>, + operator_token: arena::Box<'alloc, Token<'alloc>>, operand: arena::Box<'alloc, Expression<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { let operand = self.expression_to_simple_assignment_target(operand)?; @@ -1710,7 +1717,7 @@ impl<'alloc> AstBuilder<'alloc> { // UnaryExpression : `delete` UnaryExpression pub fn delete_expr( &self, - operator_token: arena::Box<'alloc, Token>, + operator_token: arena::Box<'alloc, Token<'alloc>>, operand: arena::Box<'alloc, Expression<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let operand_loc = operand.get_loc(); @@ -1726,7 +1733,7 @@ impl<'alloc> AstBuilder<'alloc> { // UnaryExpression : `void` UnaryExpression pub fn void_expr( &self, - operator_token: arena::Box<'alloc, Token>, + operator_token: arena::Box<'alloc, Token<'alloc>>, operand: arena::Box<'alloc, Expression<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let operand_loc = operand.get_loc(); @@ -1742,7 +1749,7 @@ impl<'alloc> AstBuilder<'alloc> { // UnaryExpression : `typeof` UnaryExpression pub fn typeof_expr( &self, - operator_token: arena::Box<'alloc, Token>, + operator_token: arena::Box<'alloc, Token<'alloc>>, operand: arena::Box<'alloc, Expression<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let operand_loc = operand.get_loc(); @@ -1758,7 +1765,7 @@ impl<'alloc> AstBuilder<'alloc> { // UnaryExpression : `+` UnaryExpression pub fn unary_plus_expr( &self, - operator_token: arena::Box<'alloc, Token>, + operator_token: arena::Box<'alloc, Token<'alloc>>, operand: arena::Box<'alloc, Expression<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let operand_loc = operand.get_loc(); @@ -1774,7 +1781,7 @@ impl<'alloc> AstBuilder<'alloc> { // UnaryExpression : `-` UnaryExpression pub fn unary_minus_expr( &self, - operator_token: arena::Box<'alloc, Token>, + operator_token: arena::Box<'alloc, Token<'alloc>>, operand: arena::Box<'alloc, Expression<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let operand_loc = operand.get_loc(); @@ -1790,7 +1797,7 @@ impl<'alloc> AstBuilder<'alloc> { // UnaryExpression : `~` UnaryExpression pub fn bitwise_not_expr( &self, - operator_token: arena::Box<'alloc, Token>, + operator_token: arena::Box<'alloc, Token<'alloc>>, operand: arena::Box<'alloc, Expression<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let operand_loc = operand.get_loc(); @@ -1806,7 +1813,7 @@ impl<'alloc> AstBuilder<'alloc> { // UnaryExpression : `!` UnaryExpression pub fn logical_not_expr( &self, - operator_token: arena::Box<'alloc, Token>, + operator_token: arena::Box<'alloc, Token<'alloc>>, operand: arena::Box<'alloc, Expression<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let operand_loc = operand.get_loc(); @@ -1819,82 +1826,88 @@ impl<'alloc> AstBuilder<'alloc> { }) } - pub fn equals_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn equals_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::Equals { loc: token.loc } } - pub fn not_equals_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn not_equals_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::NotEquals { loc: token.loc } } - pub fn strict_equals_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn strict_equals_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::StrictEquals { loc: token.loc } } - pub fn strict_not_equals_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn strict_not_equals_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::StrictNotEquals { loc: token.loc } } - pub fn less_than_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn less_than_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::LessThan { loc: token.loc } } - pub fn less_than_or_equal_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn less_than_or_equal_op( + &self, + token: arena::Box<'alloc, Token<'alloc>>, + ) -> BinaryOperator { BinaryOperator::LessThanOrEqual { loc: token.loc } } - pub fn greater_than_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn greater_than_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::GreaterThan { loc: token.loc } } - pub fn greater_than_or_equal_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn greater_than_or_equal_op( + &self, + token: arena::Box<'alloc, Token<'alloc>>, + ) -> BinaryOperator { BinaryOperator::GreaterThanOrEqual { loc: token.loc } } - pub fn in_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn in_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::In { loc: token.loc } } - pub fn instanceof_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn instanceof_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::Instanceof { loc: token.loc } } - pub fn left_shift_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn left_shift_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::LeftShift { loc: token.loc } } - pub fn right_shift_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn right_shift_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::RightShift { loc: token.loc } } - pub fn right_shift_ext_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn right_shift_ext_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::RightShiftExt { loc: token.loc } } - pub fn add_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn add_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::Add { loc: token.loc } } - pub fn sub_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn sub_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::Sub { loc: token.loc } } - pub fn mul_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn mul_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::Mul { loc: token.loc } } - pub fn div_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn div_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::Div { loc: token.loc } } - pub fn mod_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn mod_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::Mod { loc: token.loc } } - pub fn pow_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn pow_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::Pow { loc: token.loc } } - pub fn comma_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn comma_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::Comma { loc: token.loc } } - pub fn coalesce_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn coalesce_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::Coalesce { loc: token.loc } } - pub fn logical_or_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn logical_or_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::LogicalOr { loc: token.loc } } - pub fn logical_and_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn logical_and_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::LogicalAnd { loc: token.loc } } - pub fn bitwise_or_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn bitwise_or_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::BitwiseOr { loc: token.loc } } - pub fn bitwise_xor_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn bitwise_xor_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::BitwiseXor { loc: token.loc } } - pub fn bitwise_and_op(&self, token: arena::Box<'alloc, Token>) -> BinaryOperator { + pub fn bitwise_and_op(&self, token: arena::Box<'alloc, Token<'alloc>>) -> BinaryOperator { BinaryOperator::BitwiseAnd { loc: token.loc } } @@ -2231,76 +2244,94 @@ impl<'alloc> AstBuilder<'alloc> { })) } - pub fn add_assign_op(&self, token: arena::Box<'alloc, Token>) -> CompoundAssignmentOperator { + pub fn add_assign_op( + &self, + token: arena::Box<'alloc, Token<'alloc>>, + ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::Add { loc: token.loc } } - pub fn sub_assign_op(&self, token: arena::Box<'alloc, Token>) -> CompoundAssignmentOperator { + pub fn sub_assign_op( + &self, + token: arena::Box<'alloc, Token<'alloc>>, + ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::Sub { loc: token.loc } } - pub fn mul_assign_op(&self, token: arena::Box<'alloc, Token>) -> CompoundAssignmentOperator { + pub fn mul_assign_op( + &self, + token: arena::Box<'alloc, Token<'alloc>>, + ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::Mul { loc: token.loc } } - pub fn div_assign_op(&self, token: arena::Box<'alloc, Token>) -> CompoundAssignmentOperator { + pub fn div_assign_op( + &self, + token: arena::Box<'alloc, Token<'alloc>>, + ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::Div { loc: token.loc } } - pub fn mod_assign_op(&self, token: arena::Box<'alloc, Token>) -> CompoundAssignmentOperator { + pub fn mod_assign_op( + &self, + token: arena::Box<'alloc, Token<'alloc>>, + ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::Mod { loc: token.loc } } - pub fn pow_assign_op(&self, token: arena::Box<'alloc, Token>) -> CompoundAssignmentOperator { + pub fn pow_assign_op( + &self, + token: arena::Box<'alloc, Token<'alloc>>, + ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::Pow { loc: token.loc } } pub fn left_shift_assign_op( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::LeftShift { loc: token.loc } } pub fn right_shift_assign_op( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::RightShift { loc: token.loc } } pub fn right_shift_ext_assign_op( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::RightShiftExt { loc: token.loc } } pub fn bitwise_or_assign_op( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::Or { loc: token.loc } } pub fn bitwise_xor_assign_op( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::Xor { loc: token.loc } } pub fn bitwise_and_assign_op( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::And { loc: token.loc } } pub fn logical_or_assign_op( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::LogicalOr { loc: token.loc } } pub fn logical_and_assign_op( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::LogicalAnd { loc: token.loc } } pub fn coalesce_assign_op( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> CompoundAssignmentOperator { CompoundAssignmentOperator::Coalesce { loc: token.loc } } @@ -2348,9 +2379,9 @@ impl<'alloc> AstBuilder<'alloc> { // Block : `{` StatementList? `}` pub fn block( &mut self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, statements: Option>>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Block<'alloc>>> { self.check_block_bindings(open_token.loc.start)?; @@ -2368,9 +2399,9 @@ impl<'alloc> AstBuilder<'alloc> { // for Catch pub fn catch_block( &mut self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, statements: Option>>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Block<'alloc>> { // Early Error handling is done in Catch. @@ -2486,7 +2517,7 @@ impl<'alloc> AstBuilder<'alloc> { // LetOrConst : `let` pub fn let_kind( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, VariableDeclarationKind> { self.alloc_with(|| VariableDeclarationKind::Let { loc: token.loc }) } @@ -2494,7 +2525,7 @@ impl<'alloc> AstBuilder<'alloc> { // LetOrConst : `const` pub fn const_kind( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, VariableDeclarationKind> { self.alloc_with(|| VariableDeclarationKind::Const { loc: token.loc }) } @@ -2502,7 +2533,7 @@ impl<'alloc> AstBuilder<'alloc> { // VariableStatement : `var` VariableDeclarationList `;` pub fn variable_statement( &mut self, - var_token: arena::Box<'alloc, Token>, + var_token: arena::Box<'alloc, Token<'alloc>>, declarators: arena::Box<'alloc, arena::Vec<'alloc, VariableDeclarator<'alloc>>>, ) -> arena::Box<'alloc, Statement<'alloc>> { let var_loc = var_token.loc; @@ -2568,10 +2599,10 @@ impl<'alloc> AstBuilder<'alloc> { // ObjectBindingPattern : `{` BindingPropertyList `,` BindingRestProperty? `}` pub fn object_binding_pattern( &self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, properties: arena::Box<'alloc, arena::Vec<'alloc, BindingProperty<'alloc>>>, rest: Option>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Binding<'alloc>> { self.alloc_with(|| { Binding::BindingPattern(BindingPattern::ObjectBinding(ObjectBinding { @@ -2593,11 +2624,11 @@ impl<'alloc> AstBuilder<'alloc> { // ArrayBindingPattern : `[` BindingElementList `,` Elision? BindingRestElement? `]` pub fn array_binding_pattern( &self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, mut elements: arena::Box<'alloc, arena::Vec<'alloc, Option>>>, elision: Option>>, rest: Option>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Binding<'alloc>> { if let Some(elision) = elision { for _ in 0..elision.elements.len() { @@ -2766,7 +2797,7 @@ impl<'alloc> AstBuilder<'alloc> { // EmptyStatement : `;` pub fn empty_statement( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Statement<'alloc>> { self.alloc_with(|| Statement::EmptyStatement { loc: token.loc }) } @@ -2783,7 +2814,7 @@ impl<'alloc> AstBuilder<'alloc> { // IfStatement : `if` `(` Expression `)` Statement pub fn if_statement( &self, - if_token: arena::Box<'alloc, Token>, + if_token: arena::Box<'alloc, Token<'alloc>>, test: arena::Box<'alloc, Expression<'alloc>>, consequent: arena::Box<'alloc, Statement<'alloc>>, alternate: Option>>, @@ -2852,10 +2883,10 @@ impl<'alloc> AstBuilder<'alloc> { // IterationStatement : `do` Statement `while` `(` Expression `)` `;` pub fn do_while_statement( &mut self, - do_token: arena::Box<'alloc, Token>, + do_token: arena::Box<'alloc, Token<'alloc>>, stmt: arena::Box<'alloc, Statement<'alloc>>, test: arena::Box<'alloc, Expression<'alloc>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Statement<'alloc>>> { self.check_single_statement(stmt.get_loc().start)?; @@ -2871,7 +2902,7 @@ impl<'alloc> AstBuilder<'alloc> { // IterationStatement : `while` `(` Expression `)` Statement pub fn while_statement( &mut self, - while_token: arena::Box<'alloc, Token>, + while_token: arena::Box<'alloc, Token<'alloc>>, test: arena::Box<'alloc, Expression<'alloc>>, stmt: arena::Box<'alloc, Statement<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Statement<'alloc>>> { @@ -2891,7 +2922,7 @@ impl<'alloc> AstBuilder<'alloc> { // IterationStatement : `for` `(` `var` VariableDeclarationList `;` Expression? `;` Expression? `)` Statement pub fn for_statement( &mut self, - for_token: arena::Box<'alloc, Token>, + for_token: arena::Box<'alloc, Token<'alloc>>, init: Option>, test: Option>>, update: Option>>, @@ -2904,7 +2935,7 @@ impl<'alloc> AstBuilder<'alloc> { // IterationStatement : `for` `(` ForLexicalDeclaration Expression? `;` Expression? `)` Statement pub fn for_statement_lexical( &mut self, - for_token: arena::Box<'alloc, Token>, + for_token: arena::Box<'alloc, Token<'alloc>>, init: VariableDeclarationOrExpression<'alloc>, test: Option>>, update: Option>>, @@ -2918,7 +2949,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn for_statement_common( &mut self, - for_token: arena::Box<'alloc, Token>, + for_token: arena::Box<'alloc, Token<'alloc>>, init: Option>, test: Option>>, update: Option>>, @@ -2945,7 +2976,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn for_var_declaration( &mut self, - var_token: arena::Box<'alloc, Token>, + var_token: arena::Box<'alloc, Token<'alloc>>, declarators: arena::Box<'alloc, arena::Vec<'alloc, VariableDeclarator<'alloc>>>, ) -> VariableDeclarationOrExpression<'alloc> { let var_loc = var_token.loc; @@ -2983,7 +3014,7 @@ impl<'alloc> AstBuilder<'alloc> { // IterationStatement : `for` `(` `var` BindingIdentifier Initializer `in` Expression `)` Statement pub fn for_in_statement( &mut self, - for_token: arena::Box<'alloc, Token>, + for_token: arena::Box<'alloc, Token<'alloc>>, left: VariableDeclarationOrAssignmentTarget<'alloc>, right: arena::Box<'alloc, Expression<'alloc>>, stmt: arena::Box<'alloc, Statement<'alloc>>, @@ -2995,7 +3026,7 @@ impl<'alloc> AstBuilder<'alloc> { // IterationStatement : `for` `(` ForDeclaration `in` Expression `)` Statement pub fn for_in_statement_lexical( &mut self, - for_token: arena::Box<'alloc, Token>, + for_token: arena::Box<'alloc, Token<'alloc>>, left: VariableDeclarationOrAssignmentTarget<'alloc>, right: arena::Box<'alloc, Expression<'alloc>>, stmt: arena::Box<'alloc, Statement<'alloc>>, @@ -3008,7 +3039,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn for_in_statement_common( &mut self, - for_token: arena::Box<'alloc, Token>, + for_token: arena::Box<'alloc, Token<'alloc>>, left: VariableDeclarationOrAssignmentTarget<'alloc>, right: arena::Box<'alloc, Expression<'alloc>>, stmt: arena::Box<'alloc, Statement<'alloc>>, @@ -3026,7 +3057,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn for_in_or_of_var_declaration( &mut self, - var_token: arena::Box<'alloc, Token>, + var_token: arena::Box<'alloc, Token<'alloc>>, binding: arena::Box<'alloc, Binding<'alloc>>, init: Option>>, ) -> VariableDeclarationOrAssignmentTarget<'alloc> { @@ -3074,7 +3105,7 @@ impl<'alloc> AstBuilder<'alloc> { // IterationStatement : `for` `(` `var` ForBinding `of` AssignmentExpression `)` Statement pub fn for_of_statement( &mut self, - for_token: arena::Box<'alloc, Token>, + for_token: arena::Box<'alloc, Token<'alloc>>, left: VariableDeclarationOrAssignmentTarget<'alloc>, right: arena::Box<'alloc, Expression<'alloc>>, stmt: arena::Box<'alloc, Statement<'alloc>>, @@ -3086,7 +3117,7 @@ impl<'alloc> AstBuilder<'alloc> { // IterationStatement : `for` `(` ForDeclaration `of` AssignmentExpression `)` Statement pub fn for_of_statement_lexical( &mut self, - for_token: arena::Box<'alloc, Token>, + for_token: arena::Box<'alloc, Token<'alloc>>, left: VariableDeclarationOrAssignmentTarget<'alloc>, right: arena::Box<'alloc, Expression<'alloc>>, stmt: arena::Box<'alloc, Statement<'alloc>>, @@ -3099,7 +3130,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn for_of_statement_common( &mut self, - for_token: arena::Box<'alloc, Token>, + for_token: arena::Box<'alloc, Token<'alloc>>, left: VariableDeclarationOrAssignmentTarget<'alloc>, right: arena::Box<'alloc, Expression<'alloc>>, stmt: arena::Box<'alloc, Statement<'alloc>>, @@ -3119,7 +3150,7 @@ impl<'alloc> AstBuilder<'alloc> { // IterationStatement : `for` `await` `(` `var` ForBinding `of` AssignmentExpression `)` Statement pub fn for_await_of_statement( &self, - for_token: arena::Box<'alloc, Token>, + for_token: arena::Box<'alloc, Token<'alloc>>, left: VariableDeclarationOrAssignmentTarget, right: arena::Box<'alloc, Expression<'alloc>>, stmt: arena::Box<'alloc, Statement<'alloc>>, @@ -3131,7 +3162,7 @@ impl<'alloc> AstBuilder<'alloc> { // IterationStatement : `for` `await` `(` ForDeclaration `of` AssignmentExpression `)` Statement pub fn for_await_of_statement_lexical( &mut self, - for_token: arena::Box<'alloc, Token>, + for_token: arena::Box<'alloc, Token<'alloc>>, left: VariableDeclarationOrAssignmentTarget, right: arena::Box<'alloc, Expression<'alloc>>, stmt: arena::Box<'alloc, Statement<'alloc>>, @@ -3144,7 +3175,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn for_await_of_statement_common( &self, - _for_token: arena::Box<'alloc, Token>, + _for_token: arena::Box<'alloc, Token<'alloc>>, _left: VariableDeclarationOrAssignmentTarget, _right: arena::Box<'alloc, Expression<'alloc>>, _stmt: arena::Box<'alloc, Statement<'alloc>>, @@ -3197,7 +3228,7 @@ impl<'alloc> AstBuilder<'alloc> { // ContinueStatement : `continue` LabelIdentifier `;` pub fn continue_statement( &mut self, - continue_token: arena::Box<'alloc, Token>, + continue_token: arena::Box<'alloc, Token<'alloc>>, label: Option>, ) -> Result<'alloc, arena::Box<'alloc, Statement<'alloc>>> { let info = match label { @@ -3235,7 +3266,7 @@ impl<'alloc> AstBuilder<'alloc> { // BreakStatement : `break` LabelIdentifier `;` pub fn break_statement( &mut self, - break_token: arena::Box<'alloc, Token>, + break_token: arena::Box<'alloc, Token<'alloc>>, label: Option>, ) -> Result<'alloc, arena::Box<'alloc, Statement<'alloc>>> { let info = match label { @@ -3270,7 +3301,7 @@ impl<'alloc> AstBuilder<'alloc> { // ReturnStatement : `return` Expression `;` pub fn return_statement( &self, - return_token: arena::Box<'alloc, Token>, + return_token: arena::Box<'alloc, Token<'alloc>>, expression: Option>>, ) -> arena::Box<'alloc, Statement<'alloc>> { let return_loc = return_token.loc; @@ -3284,7 +3315,7 @@ impl<'alloc> AstBuilder<'alloc> { // WithStatement : `with` `(` Expression `)` Statement pub fn with_statement( &self, - with_token: arena::Box<'alloc, Token>, + with_token: arena::Box<'alloc, Token<'alloc>>, object: arena::Box<'alloc, Expression<'alloc>>, body: arena::Box<'alloc, Statement<'alloc>>, ) -> arena::Box<'alloc, Statement<'alloc>> { @@ -3299,7 +3330,7 @@ impl<'alloc> AstBuilder<'alloc> { // SwitchStatement : `switch` `(` Expression `)` CaseBlock pub fn switch_statement( &self, - switch_token: arena::Box<'alloc, Token>, + switch_token: arena::Box<'alloc, Token<'alloc>>, discriminant_expr: arena::Box<'alloc, Expression<'alloc>>, mut cases: arena::Box<'alloc, Statement<'alloc>>, ) -> arena::Box<'alloc, Statement<'alloc>> { @@ -3328,9 +3359,9 @@ impl<'alloc> AstBuilder<'alloc> { // CaseBlock : `{` CaseClauses? `}` pub fn case_block( &mut self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, cases: Option>>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Statement<'alloc>>> { self.check_case_block_binding(open_token.loc.start)?; @@ -3353,11 +3384,11 @@ impl<'alloc> AstBuilder<'alloc> { // CaseBlock : `{` CaseClauses DefaultClause CaseClauses `}` pub fn case_block_with_default( &mut self, - open_token: arena::Box<'alloc, Token>, + open_token: arena::Box<'alloc, Token<'alloc>>, pre_default_cases: Option>>>, default_case: arena::Box<'alloc, SwitchDefault<'alloc>>, post_default_cases: Option>>>, - close_token: arena::Box<'alloc, Token>, + close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Statement<'alloc>>> { self.check_case_block_binding(open_token.loc.start)?; @@ -3403,9 +3434,9 @@ impl<'alloc> AstBuilder<'alloc> { // CaseClause : `case` Expression `:` StatementList pub fn case_clause( &self, - case_token: arena::Box<'alloc, Token>, + case_token: arena::Box<'alloc, Token<'alloc>>, expression: arena::Box<'alloc, Expression<'alloc>>, - colon_token: arena::Box<'alloc, Token>, + colon_token: arena::Box<'alloc, Token<'alloc>>, statements: Option>>>, ) -> arena::Box<'alloc, SwitchCase<'alloc>> { let case_loc = case_token.loc; @@ -3432,8 +3463,8 @@ impl<'alloc> AstBuilder<'alloc> { // DefaultClause : `default` `:` StatementList pub fn default_clause( &self, - default_token: arena::Box<'alloc, Token>, - colon_token: arena::Box<'alloc, Token>, + default_token: arena::Box<'alloc, Token<'alloc>>, + colon_token: arena::Box<'alloc, Token<'alloc>>, statements: Option>>>, ) -> arena::Box<'alloc, SwitchDefault<'alloc>> { let default_loc = default_token.loc; @@ -3475,7 +3506,7 @@ impl<'alloc> AstBuilder<'alloc> { // ThrowStatement : `throw` Expression `;` pub fn throw_statement( &self, - throw_token: arena::Box<'alloc, Token>, + throw_token: arena::Box<'alloc, Token<'alloc>>, expression: arena::Box<'alloc, Expression<'alloc>>, ) -> arena::Box<'alloc, Statement<'alloc>> { let expression_loc = expression.get_loc(); @@ -3490,7 +3521,7 @@ impl<'alloc> AstBuilder<'alloc> { // TryStatement : `try` Block Catch Finally pub fn try_statement( &self, - try_token: arena::Box<'alloc, Token>, + try_token: arena::Box<'alloc, Token<'alloc>>, body: arena::Box<'alloc, Block<'alloc>>, catch_clause: Option>>, finally_block: Option>>, @@ -3525,7 +3556,7 @@ impl<'alloc> AstBuilder<'alloc> { // Catch : `catch` `(` CatchParameter `)` Block pub fn catch( &mut self, - catch_token: arena::Box<'alloc, Token>, + catch_token: arena::Box<'alloc, Token<'alloc>>, binding: arena::Box<'alloc, Binding<'alloc>>, body: arena::Box<'alloc, Block<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, CatchClause<'alloc>>> { @@ -3550,7 +3581,7 @@ impl<'alloc> AstBuilder<'alloc> { // Catch : `catch` `(` CatchParameter `)` Block pub fn catch_no_param( &mut self, - catch_token: arena::Box<'alloc, Token>, + catch_token: arena::Box<'alloc, Token<'alloc>>, body: arena::Box<'alloc, Block<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, CatchClause<'alloc>>> { let catch_loc = catch_token.loc; @@ -3568,7 +3599,7 @@ impl<'alloc> AstBuilder<'alloc> { // DebuggerStatement : `debugger` `;` pub fn debugger_statement( &self, - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, Statement<'alloc>> { self.alloc_with(|| Statement::DebuggerStatement { loc: token.loc }) } @@ -3605,14 +3636,14 @@ impl<'alloc> AstBuilder<'alloc> { // FunctionExpression : `function` BindingIdentifier? `(` FormalParameters `)` `{` FunctionBody `}` pub fn function( &mut self, - function_token: arena::Box<'alloc, Token>, + function_token: arena::Box<'alloc, Token<'alloc>>, name: Option>, - param_open_token: arena::Box<'alloc, Token>, + param_open_token: arena::Box<'alloc, Token<'alloc>>, mut params: arena::Box<'alloc, FormalParameters<'alloc>>, - param_close_token: arena::Box<'alloc, Token>, - body_open_token: arena::Box<'alloc, Token>, + param_close_token: arena::Box<'alloc, Token<'alloc>>, + body_open_token: arena::Box<'alloc, Token<'alloc>>, mut body: arena::Box<'alloc, FunctionBody<'alloc>>, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, Function<'alloc>> { let param_open_loc = param_open_token.loc; let param_close_loc = param_close_token.loc; @@ -3639,14 +3670,14 @@ impl<'alloc> AstBuilder<'alloc> { // AsyncFunctionExpression : `async` `function` `(` FormalParameters `)` `{` AsyncFunctionBody `}` pub fn async_function( &mut self, - async_token: arena::Box<'alloc, Token>, + async_token: arena::Box<'alloc, Token<'alloc>>, name: Option>, - param_open_token: arena::Box<'alloc, Token>, + param_open_token: arena::Box<'alloc, Token<'alloc>>, mut params: arena::Box<'alloc, FormalParameters<'alloc>>, - param_close_token: arena::Box<'alloc, Token>, - body_open_token: arena::Box<'alloc, Token>, + param_close_token: arena::Box<'alloc, Token<'alloc>>, + body_open_token: arena::Box<'alloc, Token<'alloc>>, mut body: arena::Box<'alloc, FunctionBody<'alloc>>, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, Function<'alloc>> { let param_open_loc = param_open_token.loc; let param_close_loc = param_close_token.loc; @@ -3673,14 +3704,14 @@ impl<'alloc> AstBuilder<'alloc> { // GeneratorExpression : `function` `*` BindingIdentifier? `(` FormalParameters `)` `{` GeneratorBody `}` pub fn generator( &mut self, - function_token: arena::Box<'alloc, Token>, + function_token: arena::Box<'alloc, Token<'alloc>>, name: Option>, - param_open_token: arena::Box<'alloc, Token>, + param_open_token: arena::Box<'alloc, Token<'alloc>>, mut params: arena::Box<'alloc, FormalParameters<'alloc>>, - param_close_token: arena::Box<'alloc, Token>, - body_open_token: arena::Box<'alloc, Token>, + param_close_token: arena::Box<'alloc, Token<'alloc>>, + body_open_token: arena::Box<'alloc, Token<'alloc>>, mut body: arena::Box<'alloc, FunctionBody<'alloc>>, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, Function<'alloc>> { let param_open_loc = param_open_token.loc; let param_close_loc = param_close_token.loc; @@ -3707,14 +3738,14 @@ impl<'alloc> AstBuilder<'alloc> { // AsyncGeneratorExpression : `async` `function` `*` BindingIdentifier? `(` FormalParameters `)` `{` AsyncGeneratorBody `}` pub fn async_generator( &mut self, - async_token: arena::Box<'alloc, Token>, + async_token: arena::Box<'alloc, Token<'alloc>>, name: Option>, - param_open_token: arena::Box<'alloc, Token>, + param_open_token: arena::Box<'alloc, Token<'alloc>>, mut params: arena::Box<'alloc, FormalParameters<'alloc>>, - param_close_token: arena::Box<'alloc, Token>, - body_open_token: arena::Box<'alloc, Token>, + param_close_token: arena::Box<'alloc, Token<'alloc>>, + body_open_token: arena::Box<'alloc, Token<'alloc>>, mut body: arena::Box<'alloc, FunctionBody<'alloc>>, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, Function<'alloc>> { let param_open_loc = param_open_token.loc; let param_close_loc = param_close_token.loc; @@ -3873,9 +3904,9 @@ impl<'alloc> AstBuilder<'alloc> { // ConciseBody : `{` FunctionBody `}` pub fn concise_body_block( &self, - body_open_token: arena::Box<'alloc, Token>, + body_open_token: arena::Box<'alloc, Token<'alloc>>, mut body: arena::Box<'alloc, FunctionBody<'alloc>>, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, ArrowExpressionBody<'alloc>> { body.loc .set_range(body_open_token.loc, body_close_token.loc); @@ -3886,12 +3917,12 @@ impl<'alloc> AstBuilder<'alloc> { pub fn method_definition( &mut self, name: arena::Box<'alloc, PropertyName<'alloc>>, - param_open_token: arena::Box<'alloc, Token>, + param_open_token: arena::Box<'alloc, Token<'alloc>>, mut params: arena::Box<'alloc, FormalParameters<'alloc>>, - param_close_token: arena::Box<'alloc, Token>, - body_open_token: arena::Box<'alloc, Token>, + param_close_token: arena::Box<'alloc, Token<'alloc>>, + body_open_token: arena::Box<'alloc, Token<'alloc>>, mut body: arena::Box<'alloc, FunctionBody<'alloc>>, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, MethodDefinition<'alloc>>> { let name_loc = name.get_loc(); let param_open_loc = param_open_token.loc; @@ -3918,11 +3949,11 @@ impl<'alloc> AstBuilder<'alloc> { // MethodDefinition : `get` PropertyName `(` `)` `{` FunctionBody `}` pub fn getter( &self, - get_token: arena::Box<'alloc, Token>, + get_token: arena::Box<'alloc, Token<'alloc>>, name: arena::Box<'alloc, PropertyName<'alloc>>, - body_open_token: arena::Box<'alloc, Token>, + body_open_token: arena::Box<'alloc, Token<'alloc>>, mut body: arena::Box<'alloc, FunctionBody<'alloc>>, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, MethodDefinition<'alloc>> { let body_close_loc = body_close_token.loc; body.loc.set_range(body_open_token.loc, body_close_loc); @@ -3938,14 +3969,14 @@ impl<'alloc> AstBuilder<'alloc> { // MethodDefinition : `set` PropertyName `(` PropertySetParameterList `)` `{` FunctionBody `}` pub fn setter( &mut self, - set_token: arena::Box<'alloc, Token>, + set_token: arena::Box<'alloc, Token<'alloc>>, name: arena::Box<'alloc, PropertyName<'alloc>>, - param_open_token: arena::Box<'alloc, Token>, + param_open_token: arena::Box<'alloc, Token<'alloc>>, mut parameter: arena::Box<'alloc, Parameter<'alloc>>, - param_close_token: arena::Box<'alloc, Token>, - body_open_token: arena::Box<'alloc, Token>, + param_close_token: arena::Box<'alloc, Token<'alloc>>, + body_open_token: arena::Box<'alloc, Token<'alloc>>, mut body: arena::Box<'alloc, FunctionBody<'alloc>>, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, MethodDefinition<'alloc>>> { let param_open_loc = param_open_token.loc; let param_close_loc = param_close_token.loc; @@ -3970,14 +4001,14 @@ impl<'alloc> AstBuilder<'alloc> { // GeneratorMethod : `*` PropertyName `(` UniqueFormalParameters `)` `{` GeneratorBody `}` pub fn generator_method( &mut self, - generator_token: arena::Box<'alloc, Token>, + generator_token: arena::Box<'alloc, Token<'alloc>>, name: arena::Box<'alloc, PropertyName<'alloc>>, - param_open_token: arena::Box<'alloc, Token>, + param_open_token: arena::Box<'alloc, Token<'alloc>>, mut params: arena::Box<'alloc, FormalParameters<'alloc>>, - param_close_token: arena::Box<'alloc, Token>, - body_open_token: arena::Box<'alloc, Token>, + param_close_token: arena::Box<'alloc, Token<'alloc>>, + body_open_token: arena::Box<'alloc, Token<'alloc>>, mut body: arena::Box<'alloc, FunctionBody<'alloc>>, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, MethodDefinition<'alloc>>> { let param_open_loc = param_open_token.loc; let param_close_loc = param_close_token.loc; @@ -4004,7 +4035,7 @@ impl<'alloc> AstBuilder<'alloc> { // YieldExpression : `yield` AssignmentExpression pub fn yield_expr( &self, - yield_token: arena::Box<'alloc, Token>, + yield_token: arena::Box<'alloc, Token<'alloc>>, operand: Option>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let yield_loc = yield_token.loc; @@ -4022,7 +4053,7 @@ impl<'alloc> AstBuilder<'alloc> { // YieldExpression : `yield` `*` AssignmentExpression pub fn yield_star_expr( &self, - yield_token: arena::Box<'alloc, Token>, + yield_token: arena::Box<'alloc, Token<'alloc>>, operand: arena::Box<'alloc, Expression<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let yield_loc = yield_token.loc; @@ -4036,14 +4067,14 @@ impl<'alloc> AstBuilder<'alloc> { // AsyncGeneratorMethod ::= "async" "*" PropertyName "(" UniqueFormalParameters ")" "{" AsyncGeneratorBody "}" pub fn async_generator_method( &mut self, - async_token: arena::Box<'alloc, Token>, + async_token: arena::Box<'alloc, Token<'alloc>>, name: arena::Box<'alloc, PropertyName<'alloc>>, - param_open_token: arena::Box<'alloc, Token>, + param_open_token: arena::Box<'alloc, Token<'alloc>>, mut params: arena::Box<'alloc, FormalParameters<'alloc>>, - param_close_token: arena::Box<'alloc, Token>, - body_open_token: arena::Box<'alloc, Token>, + param_close_token: arena::Box<'alloc, Token<'alloc>>, + body_open_token: arena::Box<'alloc, Token<'alloc>>, mut body: arena::Box<'alloc, FunctionBody<'alloc>>, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, MethodDefinition<'alloc>>> { let param_open_loc = param_open_token.loc; let param_close_loc = param_close_token.loc; @@ -4070,7 +4101,7 @@ impl<'alloc> AstBuilder<'alloc> { // ClassDeclaration : `class` ClassTail pub fn class_declaration( &mut self, - class_token: arena::Box<'alloc, Token>, + class_token: arena::Box<'alloc, Token<'alloc>>, name: Option>, tail: arena::Box<'alloc, ClassExpression<'alloc>>, ) -> arena::Box<'alloc, Statement<'alloc>> { @@ -4106,7 +4137,7 @@ impl<'alloc> AstBuilder<'alloc> { // ClassExpression : `class` BindingIdentifier? ClassTail pub fn class_expression( &mut self, - class_token: arena::Box<'alloc, Token>, + class_token: arena::Box<'alloc, Token<'alloc>>, name: Option>, mut tail: arena::Box<'alloc, ClassExpression<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { @@ -4126,7 +4157,7 @@ impl<'alloc> AstBuilder<'alloc> { body: Option< arena::Box<'alloc, arena::Vec<'alloc, arena::Box<'alloc, ClassElement<'alloc>>>>, >, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> arena::Box<'alloc, ClassExpression<'alloc>> { self.alloc_with(|| ClassExpression { name: None, @@ -4189,7 +4220,7 @@ impl<'alloc> AstBuilder<'alloc> { // ClassElementName : PrivateIdentifier pub fn class_element_name_private( &self, - private_identifier: arena::Box<'alloc, Token>, + private_identifier: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, ClassElementName<'alloc>>> { let name = self.private_identifier(private_identifier)?; Ok(self.alloc_with(|| ClassElementName::PrivateFieldName(name))) @@ -4219,7 +4250,7 @@ impl<'alloc> AstBuilder<'alloc> { // ClassElement : `static` MethodDefinition pub fn class_element_static( &self, - static_token: arena::Box<'alloc, Token>, + static_token: arena::Box<'alloc, Token<'alloc>>, method: arena::Box<'alloc, MethodDefinition<'alloc>>, ) -> arena::Box<'alloc, arena::Vec<'alloc, arena::Box<'alloc, ClassElement<'alloc>>>> { let method_loc = method.get_loc(); @@ -4235,7 +4266,7 @@ impl<'alloc> AstBuilder<'alloc> { // ClassElement : `static` MethodDefinition pub fn class_element_static_field( &self, - _static_token: arena::Box<'alloc, Token>, + _static_token: arena::Box<'alloc, Token<'alloc>>, _field: arena::Box<'alloc, ClassElement<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Void>> { Err(ParseError::NotImplemented("class static field").into()) @@ -4251,14 +4282,14 @@ impl<'alloc> AstBuilder<'alloc> { // AsyncMethod : `async` PropertyName `(` UniqueFormalParameters `)` `{` AsyncFunctionBody `}` pub fn async_method( &mut self, - async_token: arena::Box<'alloc, Token>, + async_token: arena::Box<'alloc, Token<'alloc>>, name: arena::Box<'alloc, PropertyName<'alloc>>, - param_open_token: arena::Box<'alloc, Token>, + param_open_token: arena::Box<'alloc, Token<'alloc>>, mut params: arena::Box<'alloc, FormalParameters<'alloc>>, - param_close_token: arena::Box<'alloc, Token>, - body_open_token: arena::Box<'alloc, Token>, + param_close_token: arena::Box<'alloc, Token<'alloc>>, + body_open_token: arena::Box<'alloc, Token<'alloc>>, mut body: arena::Box<'alloc, FunctionBody<'alloc>>, - body_close_token: arena::Box<'alloc, Token>, + body_close_token: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, MethodDefinition<'alloc>>> { let param_open_loc = param_open_token.loc; let param_close_loc = param_close_token.loc; @@ -4284,7 +4315,7 @@ impl<'alloc> AstBuilder<'alloc> { // AwaitExpression : `await` UnaryExpression pub fn await_expr( &self, - await_token: arena::Box<'alloc, Token>, + await_token: arena::Box<'alloc, Token<'alloc>>, operand: arena::Box<'alloc, Expression<'alloc>>, ) -> arena::Box<'alloc, Expression<'alloc>> { let operand_loc = operand.get_loc(); @@ -4298,7 +4329,7 @@ impl<'alloc> AstBuilder<'alloc> { // AsyncArrowFunction : CoverCallExpressionAndAsyncArrowHead `=>` AsyncConciseBody pub fn async_arrow_function_bare( &mut self, - async_token: arena::Box<'alloc, Token>, + async_token: arena::Box<'alloc, Token<'alloc>>, identifier: arena::Box<'alloc, BindingIdentifier>, body: arena::Box<'alloc, ArrowExpressionBody<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Expression<'alloc>>> { @@ -4450,7 +4481,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn import_declaration( &self, _import_clause: Option>, - _module_specifier: arena::Box<'alloc, Token>, + _module_specifier: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Void>> { Err(ParseError::NotImplemented("import").into()) } @@ -4503,7 +4534,7 @@ impl<'alloc> AstBuilder<'alloc> { // ImportSpecifier : IdentifierName `as` ImportedBinding pub fn import_specifier_renaming( &self, - _original_name: arena::Box<'alloc, Token>, + _original_name: arena::Box<'alloc, Token<'alloc>>, _local_name: arena::Box<'alloc, BindingIdentifier>, ) -> Result<'alloc, arena::Box<'alloc, Void>> { Err(ParseError::NotImplemented("import").into()) @@ -4512,15 +4543,15 @@ impl<'alloc> AstBuilder<'alloc> { // ModuleSpecifier : StringLiteral pub fn module_specifier( &self, - _token: arena::Box<'alloc, Token>, - ) -> Result<'alloc, arena::Box<'alloc, Token>> { + _token: arena::Box<'alloc, Token<'alloc>>, + ) -> Result<'alloc, arena::Box<'alloc, Token<'alloc>>> { Err(ParseError::NotImplemented("import").into()) } // ExportDeclaration : `export` `*` FromClause `;` pub fn export_all_from( &self, - _module_specifier: arena::Box<'alloc, Token>, + _module_specifier: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Void>> { Err(ParseError::NotImplemented("export").into()) } @@ -4529,7 +4560,7 @@ impl<'alloc> AstBuilder<'alloc> { pub fn export_set_from( &self, _export_clause: arena::Box<'alloc, Void>, - _module_specifier: arena::Box<'alloc, Token>, + _module_specifier: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Void>> { Err(ParseError::NotImplemented("export").into()) } @@ -4600,7 +4631,7 @@ impl<'alloc> AstBuilder<'alloc> { // ExportSpecifier : IdentifierName pub fn export_specifier( &self, - _identifier: arena::Box<'alloc, Token>, + _identifier: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Void>> { Err(ParseError::NotImplemented("export").into()) } @@ -4608,8 +4639,8 @@ impl<'alloc> AstBuilder<'alloc> { // ExportSpecifier : IdentifierName `as` IdentifierName pub fn export_specifier_renaming( &self, - _local_name: arena::Box<'alloc, Token>, - _exported_name: arena::Box<'alloc, Token>, + _local_name: arena::Box<'alloc, Token<'alloc>>, + _exported_name: arena::Box<'alloc, Token<'alloc>>, ) -> Result<'alloc, arena::Box<'alloc, Void>> { Err(ParseError::NotImplemented("export").into()) } diff --git a/crates/generated_parser/src/early_error_checker.rs b/crates/generated_parser/src/early_error_checker.rs index 8d4e42361..4adfc7ef2 100644 --- a/crates/generated_parser/src/early_error_checker.rs +++ b/crates/generated_parser/src/early_error_checker.rs @@ -293,11 +293,15 @@ pub trait EarlyErrorChecker<'alloc> { // Check Early Error for BindingIdentifier and note binding info to the // stack. - fn on_binding_identifier(&mut self, token: &arena::Box<'alloc, Token>) -> Result<'alloc, ()> { + fn on_binding_identifier( + &mut self, + token: &arena::Box<'alloc, Token>, + token_atom: SourceAtomSetIndex, + ) -> Result<'alloc, ()> { let context = IdentifierEarlyErrorsContext::new(); - context.check_binding_identifier(token, &self.atoms().borrow())?; + context.check_binding_identifier(token, token_atom, &self.atoms().borrow())?; - let name = token.value.as_atom(); + let name = token_atom; let offset = token.loc.start; if let Some(info) = self.context_metadata_mut().last_binding() { @@ -314,17 +318,25 @@ pub trait EarlyErrorChecker<'alloc> { } // Check Early Error for IdentifierReference. - fn on_identifier_reference(&self, token: &arena::Box<'alloc, Token>) -> Result<'alloc, ()> { + fn on_identifier_reference( + &self, + token: &arena::Box<'alloc, Token>, + token_atom: SourceAtomSetIndex, + ) -> Result<'alloc, ()> { let context = IdentifierEarlyErrorsContext::new(); - context.check_identifier_reference(token, &self.atoms().borrow()) + context.check_identifier_reference(token, token_atom, &self.atoms().borrow()) } // Check Early Error for LabelIdentifier and note binding info to the // stack - fn on_label_identifier(&mut self, token: &arena::Box<'alloc, Token>) -> Result<'alloc, ()> { + fn on_label_identifier( + &mut self, + token: &arena::Box<'alloc, Token>, + token_atom: SourceAtomSetIndex, + ) -> Result<'alloc, ()> { let context = IdentifierEarlyErrorsContext::new(); - let name = token.value.as_atom(); + let name = token_atom; let offset = token.loc.start; if let Some(info) = self.context_metadata_mut().last_binding() { @@ -340,7 +352,7 @@ pub trait EarlyErrorChecker<'alloc> { kind: LabelKind::Other, }); - context.check_label_identifier(token, &self.atoms().borrow()) + context.check_label_identifier(token, token_atom, &self.atoms().borrow()) } /// Check Early Error for LabelledStatement. diff --git a/crates/generated_parser/src/early_errors.rs b/crates/generated_parser/src/early_errors.rs index a9d1fb891..7d49b20ba 100644 --- a/crates/generated_parser/src/early_errors.rs +++ b/crates/generated_parser/src/early_errors.rs @@ -83,36 +83,51 @@ impl IdentifierEarlyErrorsContext { } */ - fn is_arguments_identifier<'alloc>(token: &arena::Box<'alloc, Token>) -> bool { + fn is_arguments_identifier<'alloc>( + token: &arena::Box<'alloc, Token>, + token_atom: SourceAtomSetIndex, + ) -> bool { return (token.terminal_id == TerminalId::Name || token.terminal_id == TerminalId::NameWithEscape) - && token.value.as_atom() == CommonSourceAtomSetIndices::arguments(); + && token_atom == CommonSourceAtomSetIndices::arguments(); } - fn is_eval_identifier<'alloc>(token: &arena::Box<'alloc, Token>) -> bool { + fn is_eval_identifier<'alloc>( + token: &arena::Box<'alloc, Token>, + token_atom: SourceAtomSetIndex, + ) -> bool { return (token.terminal_id == TerminalId::Name || token.terminal_id == TerminalId::NameWithEscape) - && token.value.as_atom() == CommonSourceAtomSetIndices::eval(); + && token_atom == CommonSourceAtomSetIndices::eval(); } - fn is_yield_identifier<'alloc>(token: &arena::Box<'alloc, Token>) -> bool { + fn is_yield_identifier<'alloc>( + token: &arena::Box<'alloc, Token>, + token_atom: SourceAtomSetIndex, + ) -> bool { return token.terminal_id == TerminalId::Yield || (token.terminal_id == TerminalId::NameWithEscape - && token.value.as_atom() == CommonSourceAtomSetIndices::yield_()); + && token_atom == CommonSourceAtomSetIndices::yield_()); } - fn is_await_identifier<'alloc>(token: &arena::Box<'alloc, Token>) -> bool { + fn is_await_identifier<'alloc>( + token: &arena::Box<'alloc, Token>, + token_atom: SourceAtomSetIndex, + ) -> bool { return token.terminal_id == TerminalId::Await || (token.terminal_id == TerminalId::NameWithEscape - && token.value.as_atom() == CommonSourceAtomSetIndices::await_()); + && token_atom == CommonSourceAtomSetIndices::await_()); } pub fn check_binding_identifier<'alloc>( &self, token: &arena::Box<'alloc, Token>, + token_atom: SourceAtomSetIndex, atoms: &SourceAtomSet<'alloc>, ) -> EarlyErrorsResult<'alloc> { - if Self::is_arguments_identifier(token) || Self::is_eval_identifier(token) { + if Self::is_arguments_identifier(token, token_atom) + || Self::is_eval_identifier(token, token_atom) + { // Static Semantics: Early Errors // https://tc39.es/ecma262/#sec-identifiers-static-semantics-early-errors // @@ -122,7 +137,7 @@ impl IdentifierEarlyErrorsContext { // production is contained in strict mode code and the // StringValue of Identifier is "arguments" or "eval". if self.is_strict()? { - let name = atoms.get(token.value.as_atom()); + let name = atoms.get(token_atom); let offset = token.loc.start; return Err(ParseError::InvalidIdentifier(name.clone(), offset).into()); } @@ -130,7 +145,7 @@ impl IdentifierEarlyErrorsContext { return Ok(()); } - if Self::is_yield_identifier(token) { + if Self::is_yield_identifier(token, token_atom) { // BindingIdentifier : yield // // * It is a Syntax Error if this production has a [Yield] @@ -140,7 +155,7 @@ impl IdentifierEarlyErrorsContext { // return self.check_yield_common(); } - if Self::is_await_identifier(token) { + if Self::is_await_identifier(token, token_atom) { // BindingIdentifier : await // // * It is a Syntax Error if this production has an [Await] @@ -150,44 +165,47 @@ impl IdentifierEarlyErrorsContext { // return self.check_await_common(); } - self.check_identifier(token, atoms) + self.check_identifier(token, token_atom, atoms) } pub fn check_label_identifier<'alloc>( &self, token: &arena::Box<'alloc, Token>, + token_atom: SourceAtomSetIndex, atoms: &SourceAtomSet<'alloc>, ) -> EarlyErrorsResult<'alloc> { - if Self::is_yield_identifier(token) { - return self.check_yield_common(token, atoms); + if Self::is_yield_identifier(token, token_atom) { + return self.check_yield_common(token, token_atom, atoms); } - if Self::is_await_identifier(token) { - return self.check_await_common(token, atoms); + if Self::is_await_identifier(token, token_atom) { + return self.check_await_common(token, token_atom, atoms); } - self.check_identifier(token, atoms) + self.check_identifier(token, token_atom, atoms) } pub fn check_identifier_reference<'alloc>( &self, token: &arena::Box<'alloc, Token>, + token_atom: SourceAtomSetIndex, atoms: &SourceAtomSet<'alloc>, ) -> EarlyErrorsResult<'alloc> { - if Self::is_yield_identifier(token) { - return self.check_yield_common(token, atoms); + if Self::is_yield_identifier(token, token_atom) { + return self.check_yield_common(token, token_atom, atoms); } - if Self::is_await_identifier(token) { - return self.check_await_common(token, atoms); + if Self::is_await_identifier(token, token_atom) { + return self.check_await_common(token, token_atom, atoms); } - self.check_identifier(token, atoms) + self.check_identifier(token, token_atom, atoms) } fn check_yield_common<'alloc>( &self, _token: &arena::Box<'alloc, Token>, + _token_atom: SourceAtomSetIndex, _atoms: &SourceAtomSet<'alloc>, ) -> EarlyErrorsResult<'alloc> { // Static Semantics: Early Errors @@ -223,7 +241,7 @@ impl IdentifierEarlyErrorsContext { // // if self.is_strict()? { // return Err(ParseError::InvalidIdentifier( - // atoms.get(token.value.as_atom()), + // atoms.get(token_atom), // offset, // )); // } @@ -234,6 +252,7 @@ impl IdentifierEarlyErrorsContext { fn check_await_common<'alloc>( &self, _token: &arena::Box<'alloc, Token>, + _token_atom: SourceAtomSetIndex, _atoms: &SourceAtomSet<'alloc>, ) -> EarlyErrorsResult<'alloc> { // Static Semantics: Early Errors @@ -267,7 +286,7 @@ impl IdentifierEarlyErrorsContext { // // if self.is_module()? { // return Err(ParseError::InvalidIdentifier( - // atoms.get(token.value.as_atom()), + // atoms.get(token_atom), // offset, // )); // } @@ -328,11 +347,12 @@ impl IdentifierEarlyErrorsContext { fn check_identifier<'alloc>( &self, token: &arena::Box<'alloc, Token>, + token_atom: SourceAtomSetIndex, atoms: &SourceAtomSet<'alloc>, ) -> EarlyErrorsResult<'alloc> { match token.terminal_id { TerminalId::NameWithEscape => { - let name = token.value.as_atom(); + let name = token_atom; if Self::is_contextual_keyword_excluding_yield(name) { // Identifier : IdentifierName but not ReservedWord // @@ -346,7 +366,7 @@ impl IdentifierEarlyErrorsContext { // NOTE: "yield" case is handled in // `check_yield_common`. if self.is_strict()? { - let name = atoms.get(token.value.as_atom()); + let name = atoms.get(token_atom); let offset = token.loc.start; return Err(ParseError::InvalidIdentifier(name, offset).into()); } @@ -357,7 +377,7 @@ impl IdentifierEarlyErrorsContext { // IdentifierName is the same String value as the // StringValue of any ReservedWord except for yield // or await. - let name = atoms.get(token.value.as_atom()); + let name = atoms.get(token_atom); let offset = token.loc.start; return Err(ParseError::InvalidIdentifier(name, offset).into()); } @@ -379,7 +399,7 @@ impl IdentifierEarlyErrorsContext { // // NOTE: "yield" case is handled in `check_yield_common`. if self.is_strict()? { - let name = atoms.get(token.value.as_atom()); + let name = atoms.get(token_atom); let offset = token.loc.start; return Err(ParseError::InvalidIdentifier(name, offset).into()); } diff --git a/crates/generated_parser/src/error.rs b/crates/generated_parser/src/error.rs index 38e3ed77f..6924fbaa2 100644 --- a/crates/generated_parser/src/error.rs +++ b/crates/generated_parser/src/error.rs @@ -18,7 +18,7 @@ pub enum ParseError<'alloc> { // Generic syntax errors NotImplemented(&'static str), - SyntaxError(Token), + SyntaxError(Token<'alloc>), UnexpectedEnd, InvalidAssignmentTarget, InvalidParameter, diff --git a/crates/generated_parser/src/token.rs b/crates/generated_parser/src/token.rs index 8c87e3909..8d5f96943 100644 --- a/crates/generated_parser/src/token.rs +++ b/crates/generated_parser/src/token.rs @@ -1,17 +1,14 @@ use crate::parser_tables_generated::TerminalId; -use ast::source_atom_set::SourceAtomSetIndex; -use ast::source_slice_list::SourceSliceIndex; use ast::SourceLocation; #[derive(Clone, Debug, PartialEq)] -pub enum TokenValue { +pub enum TokenValue<'a> { None, Number(f64), - Atom(SourceAtomSetIndex), - Slice(SourceSliceIndex), + String(&'a str), } -impl TokenValue { +impl<'a> TokenValue<'a> { pub fn as_number(&self) -> f64 { match self { Self::Number(n) => *n, @@ -19,17 +16,10 @@ impl TokenValue { } } - pub fn as_atom(&self) -> SourceAtomSetIndex { + pub fn as_str(&self) -> &'a str { match self { - Self::Atom(index) => *index, - _ => panic!("expected atom"), - } - } - - pub fn as_slice(&self) -> SourceSliceIndex { - match self { - Self::Slice(index) => *index, - _ => panic!("expected atom"), + Self::String(s) => s, + _ => panic!("expected str"), } } } @@ -41,7 +31,7 @@ impl TokenValue { /// Tokens match the goal terminals of the ECMAScript lexical grammar; see /// . #[derive(Clone, Debug, PartialEq)] -pub struct Token { +pub struct Token<'a> { /// Token type. pub terminal_id: TerminalId, @@ -73,10 +63,10 @@ pub struct Token { /// /// For all other tokens (including template literal parts), the content is /// unspecified for now. TODO. - pub value: TokenValue, + pub value: TokenValue<'a>, } -impl Token { +impl<'a> Token<'a> { pub fn basic_token(terminal_id: TerminalId, loc: SourceLocation) -> Self { Self { terminal_id, diff --git a/crates/parser/src/lexer.rs b/crates/parser/src/lexer.rs index 224be0add..4fb7e14a7 100644 --- a/crates/parser/src/lexer.rs +++ b/crates/parser/src/lexer.rs @@ -4,31 +4,23 @@ use crate::numeric_value::{parse_float, parse_int, NumericLiteralBase}; use crate::parser::Parser; use crate::unicode::{is_id_continue, is_id_start}; use ast::arena; -use ast::source_atom_set::{CommonSourceAtomSetIndices, SourceAtomSet}; -use ast::source_slice_list::SourceSliceList; use ast::SourceLocation; use bumpalo::{collections::String, Bump}; use generated_parser::{ParseError, Result, TerminalId, Token, TokenValue}; -use std::cell::RefCell; use std::convert::TryFrom; -use std::rc::Rc; use std::str::Chars; pub struct Lexer<'alloc> { allocator: &'alloc Bump, /// Next token to be returned. - token: arena::Box<'alloc, Token>, + token: arena::Box<'alloc, Token<'alloc>>, /// Length of the input text, in UTF-8 bytes. source_length: usize, /// Iterator over the remaining not-yet-parsed input. chars: Chars<'alloc>, - - atoms: Rc>>, - - slices: Rc>>, } enum NumericResult { @@ -43,13 +35,8 @@ enum NumericResult { } impl<'alloc> Lexer<'alloc> { - pub fn new( - allocator: &'alloc Bump, - chars: Chars<'alloc>, - atoms: Rc>>, - slices: Rc>>, - ) -> Lexer<'alloc> { - Self::with_offset(allocator, chars, 0, atoms, slices) + pub fn new(allocator: &'alloc Bump, chars: Chars<'alloc>) -> Lexer<'alloc> { + Self::with_offset(allocator, chars, 0) } /// Create a lexer for a part of a JS script or module. `offset` is the @@ -59,8 +46,6 @@ impl<'alloc> Lexer<'alloc> { allocator: &'alloc Bump, chars: Chars<'alloc>, offset: usize, - atoms: Rc>>, - slices: Rc>>, ) -> Lexer<'alloc> { let source_length = offset + chars.as_str().len(); let mut token = arena::alloc(allocator, new_token()); @@ -70,8 +55,6 @@ impl<'alloc> Lexer<'alloc> { token, source_length, chars, - atoms, - slices, } } @@ -97,7 +80,7 @@ impl<'alloc> Lexer<'alloc> { &mut self, terminal_id: TerminalId, loc: SourceLocation, - value: TokenValue, + value: TokenValue<'alloc>, ) -> Result<'alloc, ()> { self.token.terminal_id = terminal_id; self.token.loc = loc; @@ -109,7 +92,7 @@ impl<'alloc> Lexer<'alloc> { pub fn next<'parser>( &mut self, parser: &Parser<'parser>, - ) -> Result<'alloc, arena::Box<'alloc, Token>> { + ) -> Result<'alloc, arena::Box<'alloc, Token<'alloc>>> { let mut next_token = arena::alloc_with(self.allocator, || new_token()); self.advance_impl(parser)?; std::mem::swap(&mut self.token, &mut next_token); @@ -126,7 +109,7 @@ impl<'alloc> Lexer<'alloc> { } /// Returns an empty token which is meant as a place holder to be mutated later. -fn new_token() -> Token { +fn new_token<'alloc>() -> Token<'alloc> { Token::basic_token(TerminalId::End, SourceLocation::default()) } @@ -416,15 +399,12 @@ impl<'alloc> Lexer<'alloc> { (TerminalId::NameWithEscape, self.string_to_token_value(text)) } else { match &text as &str { - "as" => ( - TerminalId::As, - TokenValue::Atom(CommonSourceAtomSetIndices::as_()), - ), + "as" => (TerminalId::As, TokenValue::String("as")), "async" => { /* ( TerminalId::Async, - TokenValue::Atom(CommonSourceAtomSetIndices::async_()), + TokenValue::String("async"), ), */ return Err(ParseError::NotImplemented( @@ -436,118 +416,43 @@ impl<'alloc> Lexer<'alloc> { /* ( TerminalId::Await, - TokenValue::Atom(CommonSourceAtomSetIndices::await_()), + TokenValue::String("await"), ), */ return Err( ParseError::NotImplemented("await cannot be handled in parser").into(), ); } - "break" => ( - TerminalId::Break, - TokenValue::Atom(CommonSourceAtomSetIndices::break_()), - ), - "case" => ( - TerminalId::Case, - TokenValue::Atom(CommonSourceAtomSetIndices::case()), - ), - "catch" => ( - TerminalId::Catch, - TokenValue::Atom(CommonSourceAtomSetIndices::catch()), - ), - "class" => ( - TerminalId::Class, - TokenValue::Atom(CommonSourceAtomSetIndices::class()), - ), - "const" => ( - TerminalId::Const, - TokenValue::Atom(CommonSourceAtomSetIndices::const_()), - ), - "continue" => ( - TerminalId::Continue, - TokenValue::Atom(CommonSourceAtomSetIndices::continue_()), - ), - "debugger" => ( - TerminalId::Debugger, - TokenValue::Atom(CommonSourceAtomSetIndices::debugger()), - ), - "default" => ( - TerminalId::Default, - TokenValue::Atom(CommonSourceAtomSetIndices::default()), - ), - "delete" => ( - TerminalId::Delete, - TokenValue::Atom(CommonSourceAtomSetIndices::delete()), - ), - "do" => ( - TerminalId::Do, - TokenValue::Atom(CommonSourceAtomSetIndices::do_()), - ), - "else" => ( - TerminalId::Else, - TokenValue::Atom(CommonSourceAtomSetIndices::else_()), - ), - "enum" => ( - TerminalId::Enum, - TokenValue::Atom(CommonSourceAtomSetIndices::enum_()), - ), - "export" => ( - TerminalId::Export, - TokenValue::Atom(CommonSourceAtomSetIndices::export()), - ), - "extends" => ( - TerminalId::Extends, - TokenValue::Atom(CommonSourceAtomSetIndices::extends()), - ), - "finally" => ( - TerminalId::Finally, - TokenValue::Atom(CommonSourceAtomSetIndices::finally()), - ), - "for" => ( - TerminalId::For, - TokenValue::Atom(CommonSourceAtomSetIndices::for_()), - ), - "from" => ( - TerminalId::From, - TokenValue::Atom(CommonSourceAtomSetIndices::from()), - ), - "function" => ( - TerminalId::Function, - TokenValue::Atom(CommonSourceAtomSetIndices::function()), - ), - "get" => ( - TerminalId::Get, - TokenValue::Atom(CommonSourceAtomSetIndices::get()), - ), - "if" => ( - TerminalId::If, - TokenValue::Atom(CommonSourceAtomSetIndices::if_()), - ), - "implements" => ( - TerminalId::Implements, - TokenValue::Atom(CommonSourceAtomSetIndices::implements()), - ), - "import" => ( - TerminalId::Import, - TokenValue::Atom(CommonSourceAtomSetIndices::import()), - ), - "in" => ( - TerminalId::In, - TokenValue::Atom(CommonSourceAtomSetIndices::in_()), - ), - "instanceof" => ( - TerminalId::Instanceof, - TokenValue::Atom(CommonSourceAtomSetIndices::instanceof()), - ), - "interface" => ( - TerminalId::Interface, - TokenValue::Atom(CommonSourceAtomSetIndices::interface()), - ), + "break" => (TerminalId::Break, TokenValue::String("break")), + "case" => (TerminalId::Case, TokenValue::String("case")), + "catch" => (TerminalId::Catch, TokenValue::String("catch")), + "class" => (TerminalId::Class, TokenValue::String("class")), + "const" => (TerminalId::Const, TokenValue::String("const")), + "continue" => (TerminalId::Continue, TokenValue::String("continue")), + "debugger" => (TerminalId::Debugger, TokenValue::String("debugger")), + "default" => (TerminalId::Default, TokenValue::String("default")), + "delete" => (TerminalId::Delete, TokenValue::String("delete")), + "do" => (TerminalId::Do, TokenValue::String("do")), + "else" => (TerminalId::Else, TokenValue::String("else")), + "enum" => (TerminalId::Enum, TokenValue::String("enum")), + "export" => (TerminalId::Export, TokenValue::String("export")), + "extends" => (TerminalId::Extends, TokenValue::String("extends")), + "finally" => (TerminalId::Finally, TokenValue::String("finally")), + "for" => (TerminalId::For, TokenValue::String("for")), + "from" => (TerminalId::From, TokenValue::String("from")), + "function" => (TerminalId::Function, TokenValue::String("function")), + "get" => (TerminalId::Get, TokenValue::String("get")), + "if" => (TerminalId::If, TokenValue::String("if")), + "implements" => (TerminalId::Implements, TokenValue::String("implements")), + "import" => (TerminalId::Import, TokenValue::String("import")), + "in" => (TerminalId::In, TokenValue::String("in")), + "instanceof" => (TerminalId::Instanceof, TokenValue::String("instanceof")), + "interface" => (TerminalId::Interface, TokenValue::String("interface")), "let" => { /* ( TerminalId::Let, - TokenValue::Atom(CommonSourceAtomSetIndices::let_()), + TokenValue::String("let"), ), */ return Err(ParseError::NotImplemented( @@ -555,109 +460,40 @@ impl<'alloc> Lexer<'alloc> { ) .into()); } - "new" => ( - TerminalId::New, - TokenValue::Atom(CommonSourceAtomSetIndices::new_()), - ), - "of" => ( - TerminalId::Of, - TokenValue::Atom(CommonSourceAtomSetIndices::of()), - ), - "package" => ( - TerminalId::Package, - TokenValue::Atom(CommonSourceAtomSetIndices::package()), - ), - "private" => ( - TerminalId::Private, - TokenValue::Atom(CommonSourceAtomSetIndices::private()), - ), - "protected" => ( - TerminalId::Protected, - TokenValue::Atom(CommonSourceAtomSetIndices::protected()), - ), - "public" => ( - TerminalId::Public, - TokenValue::Atom(CommonSourceAtomSetIndices::public()), - ), - "return" => ( - TerminalId::Return, - TokenValue::Atom(CommonSourceAtomSetIndices::return_()), - ), - "set" => ( - TerminalId::Set, - TokenValue::Atom(CommonSourceAtomSetIndices::set()), - ), - "static" => ( - TerminalId::Static, - TokenValue::Atom(CommonSourceAtomSetIndices::static_()), - ), - "super" => ( - TerminalId::Super, - TokenValue::Atom(CommonSourceAtomSetIndices::super_()), - ), - "switch" => ( - TerminalId::Switch, - TokenValue::Atom(CommonSourceAtomSetIndices::switch()), - ), - "target" => ( - TerminalId::Target, - TokenValue::Atom(CommonSourceAtomSetIndices::target()), - ), - "this" => ( - TerminalId::This, - TokenValue::Atom(CommonSourceAtomSetIndices::this()), - ), - "throw" => ( - TerminalId::Throw, - TokenValue::Atom(CommonSourceAtomSetIndices::throw()), - ), - "try" => ( - TerminalId::Try, - TokenValue::Atom(CommonSourceAtomSetIndices::try_()), - ), - "typeof" => ( - TerminalId::Typeof, - TokenValue::Atom(CommonSourceAtomSetIndices::typeof_()), - ), - "var" => ( - TerminalId::Var, - TokenValue::Atom(CommonSourceAtomSetIndices::var()), - ), - "void" => ( - TerminalId::Void, - TokenValue::Atom(CommonSourceAtomSetIndices::void()), - ), - "while" => ( - TerminalId::While, - TokenValue::Atom(CommonSourceAtomSetIndices::while_()), - ), - "with" => ( - TerminalId::With, - TokenValue::Atom(CommonSourceAtomSetIndices::with()), - ), + "new" => (TerminalId::New, TokenValue::String("new")), + "of" => (TerminalId::Of, TokenValue::String("of")), + "package" => (TerminalId::Package, TokenValue::String("package")), + "private" => (TerminalId::Private, TokenValue::String("private")), + "protected" => (TerminalId::Protected, TokenValue::String("protected")), + "public" => (TerminalId::Public, TokenValue::String("public")), + "return" => (TerminalId::Return, TokenValue::String("return")), + "set" => (TerminalId::Set, TokenValue::String("set")), + "static" => (TerminalId::Static, TokenValue::String("static")), + "super" => (TerminalId::Super, TokenValue::String("super")), + "switch" => (TerminalId::Switch, TokenValue::String("switch")), + "target" => (TerminalId::Target, TokenValue::String("target")), + "this" => (TerminalId::This, TokenValue::String("this")), + "throw" => (TerminalId::Throw, TokenValue::String("throw")), + "try" => (TerminalId::Try, TokenValue::String("try")), + "typeof" => (TerminalId::Typeof, TokenValue::String("typeof")), + "var" => (TerminalId::Var, TokenValue::String("var")), + "void" => (TerminalId::Void, TokenValue::String("void")), + "while" => (TerminalId::While, TokenValue::String("while")), + "with" => (TerminalId::With, TokenValue::String("with")), "yield" => { /* ( TerminalId::Yield, - TokenValue::Atom(CommonSourceAtomSetIndices::yield_()), + TokenValue::String("yield"), ), */ return Err( ParseError::NotImplemented("yield cannot be handled in parser").into(), ); } - "null" => ( - TerminalId::NullLiteral, - TokenValue::Atom(CommonSourceAtomSetIndices::null()), - ), - "true" => ( - TerminalId::BooleanLiteral, - TokenValue::Atom(CommonSourceAtomSetIndices::true_()), - ), - "false" => ( - TerminalId::BooleanLiteral, - TokenValue::Atom(CommonSourceAtomSetIndices::false_()), - ), + "null" => (TerminalId::NullLiteral, TokenValue::String("null")), + "true" => (TerminalId::BooleanLiteral, TokenValue::String("true")), + "false" => (TerminalId::BooleanLiteral, TokenValue::String("false")), _ => (TerminalId::Name, self.string_to_token_value(text)), } }; @@ -1378,7 +1214,7 @@ impl<'alloc> Lexer<'alloc> { Some(c @ '"') | Some(c @ '\'') => { if c == delimiter { - let value = self.slice_to_token_value(builder.finish_without_push(&self)); + let value = self.string_to_token_value(builder.finish_without_push(&self)); return self.set_result( TerminalId::StringLiteral, SourceLocation::new(offset, self.offset()), @@ -1502,7 +1338,7 @@ impl<'alloc> Lexer<'alloc> { // TODO: 12.2.8.2.4 and 12.2.8.2.5 Check that the body matches the // grammar defined in 21.2.1. - let value = self.slice_to_token_value(literal); + let value = self.string_to_token_value(literal); self.set_result( TerminalId::RegularExpressionLiteral, SourceLocation::new(offset, self.offset()), @@ -2214,14 +2050,8 @@ impl<'alloc> Lexer<'alloc> { ) } - fn string_to_token_value(&mut self, s: &'alloc str) -> TokenValue { - let index = self.atoms.borrow_mut().insert(s); - TokenValue::Atom(index) - } - - fn slice_to_token_value(&mut self, s: &'alloc str) -> TokenValue { - let index = self.slices.borrow_mut().push(s); - TokenValue::Slice(index) + fn string_to_token_value(&mut self, s: &'alloc str) -> TokenValue<'alloc> { + TokenValue::String(s) } fn numeric_result_to_advance_result( diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index e78744afa..4fa1417e8 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -73,7 +73,7 @@ fn parse<'alloc>( atoms: Rc>>, slices: Rc>>, ) -> Result<'alloc, StackValue<'alloc>> { - let mut tokens = Lexer::new(allocator, source.chars(), atoms.clone(), slices.clone()); + let mut tokens = Lexer::new(allocator, source.chars()); TABLES.check(); @@ -96,10 +96,10 @@ pub fn is_partial_script<'alloc>( slices: Rc>>, ) -> Result<'alloc, bool> { let mut parser = Parser::new( - AstBuilder::new(allocator, atoms.clone(), slices.clone()), + AstBuilder::new(allocator, atoms, slices), START_STATE_SCRIPT, ); - let mut tokens = Lexer::new(allocator, source.chars(), atoms, slices); + let mut tokens = Lexer::new(allocator, source.chars()); loop { let t = tokens.next(&parser)?; if t.terminal_id == TerminalId::End { diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index c9f12a56e..ea256c3b3 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -132,7 +132,7 @@ impl<'alloc> Parser<'alloc> { *self.state_stack.last().unwrap() } - pub fn write_token(&mut self, token: arena::Box<'alloc, Token>) -> Result<'alloc, ()> { + pub fn write_token(&mut self, token: arena::Box<'alloc, Token<'alloc>>) -> Result<'alloc, ()> { json_trace!({ "method": "write_token", "is_on_new_line": token.is_on_new_line, @@ -178,7 +178,7 @@ impl<'alloc> Parser<'alloc> { Ok(self.node_stack.pop().unwrap().value) } - pub(crate) fn parse_error(t: &Token) -> ParseError<'alloc> { + pub(crate) fn parse_error(t: &Token<'alloc>) -> ParseError<'alloc> { if t.terminal_id == TerminalId::End { ParseError::UnexpectedEnd } else { @@ -225,7 +225,7 @@ impl<'alloc> Parser<'alloc> { Err(ParseError::ParserCannotUnpackToken.into()) } - pub(crate) fn recover(t: &Token, error_code: ErrorCode) -> Result<'alloc, ()> { + pub(crate) fn recover(t: &Token<'alloc>, error_code: ErrorCode) -> Result<'alloc, ()> { match error_code { ErrorCode::Asi => { if t.is_on_new_line diff --git a/crates/parser/src/tests.rs b/crates/parser/src/tests.rs index 48bc5816b..e0da39b4f 100644 --- a/crates/parser/src/tests.rs +++ b/crates/parser/src/tests.rs @@ -138,18 +138,8 @@ fn assert_same_tokens<'alloc>(left: &str, right: &str) { let left_slices = Rc::new(RefCell::new(SourceSliceList::new())); let right_atoms = Rc::new(RefCell::new(SourceAtomSet::new())); let right_slices = Rc::new(RefCell::new(SourceSliceList::new())); - let mut left_lexer = Lexer::new( - allocator, - left.chars(), - left_atoms.clone(), - left_slices.clone(), - ); - let mut right_lexer = Lexer::new( - allocator, - right.chars(), - right_atoms.clone(), - right_slices.clone(), - ); + let mut left_lexer = Lexer::new(allocator, left.chars()); + let mut right_lexer = Lexer::new(allocator, right.chars()); let mut left_parser = Parser::new( AstBuilder::new(allocator, left_atoms, left_slices), @@ -193,7 +183,7 @@ fn assert_can_close_after<'alloc, T: IntoChunks<'alloc>>(code: T) { let buf = chunks_to_string(code); let atoms = Rc::new(RefCell::new(SourceAtomSet::new())); let slices = Rc::new(RefCell::new(SourceSliceList::new())); - let mut lexer = Lexer::new(allocator, buf.chars(), atoms.clone(), slices.clone()); + let mut lexer = Lexer::new(allocator, buf.chars()); let mut parser = Parser::new( AstBuilder::new(allocator, atoms, slices), generated_parser::START_STATE_SCRIPT,