Skip to content

Commit

Permalink
revert Literal from []byte to string
Browse files Browse the repository at this point in the history
  • Loading branch information
@gravataLonga
gravataLonga committed Jul 2, 2022
1 parent 9ef99b5 commit 7c514fc
Show file tree
Hide file tree
Showing 9 changed files with 131 additions and 60 deletions.
32 changes: 16 additions & 16 deletions ast/array_literal_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ import (

func TestArrayLiteral_String(t *testing.T) {
elements := []Expression{
&IntegerLiteral{Token: token.Token{Type: token.LBRACKET, Literal: []byte("1")}, Value: 1},
&FloatLiteral{Token: token.Token{Type: token.INT, Literal: []byte("2.2")}, Value: 2.2},
&StringLiteral{Token: token.Token{Type: token.STRING, Literal: []byte("3")}, Value: "3"},
&Boolean{Token: token.Token{Type: token.TRUE, Literal: []byte("True")}, Value: true},
&IntegerLiteral{Token: token.Token{Type: token.LBRACKET, Literal: "1"}, Value: 1},
&FloatLiteral{Token: token.Token{Type: token.INT, Literal: "2.2"}, Value: 2.2},
&StringLiteral{Token: token.Token{Type: token.STRING, Literal: "3"}, Value: "3"},
&Boolean{Token: token.Token{Type: token.TRUE, Literal: "True"}, Value: true},
}
arrLiteral := &ArrayLiteral{Token: token.Token{Type: token.LBRACKET, Literal: []byte("[")}, Elements: elements}
arrLiteral := &ArrayLiteral{Token: token.Token{Type: token.LBRACKET, Literal: "["}, Elements: elements}

if arrLiteral.String() != "[1, 2.2, 3, True]" {
t.Errorf("ArrayLiteral isnt equal to %s. Got: %s", "[1, 2, 3, True]", arrLiteral.String())
Expand All @@ -25,12 +25,12 @@ func TestArrayLiteral_String(t *testing.T) {

func BenchmarkArrayLiteral_String(b *testing.B) {
elements := []Expression{
&IntegerLiteral{Token: token.Token{Type: token.LBRACKET, Literal: []byte("1")}, Value: 1},
&FloatLiteral{Token: token.Token{Type: token.INT, Literal: []byte("2.2")}, Value: 2.2},
&StringLiteral{Token: token.Token{Type: token.STRING, Literal: []byte("3")}, Value: "3"},
&Boolean{Token: token.Token{Type: token.TRUE, Literal: []byte("True")}, Value: true},
&IntegerLiteral{Token: token.Token{Type: token.LBRACKET, Literal: "1"}, Value: 1},
&FloatLiteral{Token: token.Token{Type: token.INT, Literal: "2.2"}, Value: 2.2},
&StringLiteral{Token: token.Token{Type: token.STRING, Literal: "3"}, Value: "3"},
&Boolean{Token: token.Token{Type: token.TRUE, Literal: "True"}, Value: true},
}
arrLiteral := &ArrayLiteral{Token: token.Token{Type: token.LBRACKET, Literal: []byte("[")}, Elements: elements}
arrLiteral := &ArrayLiteral{Token: token.Token{Type: token.LBRACKET, Literal: "["}, Elements: elements}

for i := 0; i < b.N; i++ {
arrLiteral.String()
Expand All @@ -39,15 +39,15 @@ func BenchmarkArrayLiteral_String(b *testing.B) {

func BenchmarkArrayLiteral_StringWithHash(b *testing.B) {
elements := []Expression{
&IntegerLiteral{Token: token.Token{Type: token.LBRACKET, Literal: []byte("1")}, Value: 1},
&FloatLiteral{Token: token.Token{Type: token.INT, Literal: []byte("2.2")}, Value: 2.2},
&StringLiteral{Token: token.Token{Type: token.STRING, Literal: []byte("3")}, Value: "3"},
&Boolean{Token: token.Token{Type: token.TRUE, Literal: []byte("True")}, Value: true},
&IntegerLiteral{Token: token.Token{Type: token.LBRACKET, Literal: "1"}, Value: 1},
&FloatLiteral{Token: token.Token{Type: token.INT, Literal: "2.2"}, Value: 2.2},
&StringLiteral{Token: token.Token{Type: token.STRING, Literal: "3"}, Value: "3"},
&Boolean{Token: token.Token{Type: token.TRUE, Literal: "True"}, Value: true},
&HashLiteral{Pairs: map[Expression]Expression{
&IntegerLiteral{Token: token.Token{Type: token.LBRACKET, Literal: []byte("1")}, Value: 1}: &Boolean{Token: token.Token{Type: token.TRUE, Literal: []byte("True")}, Value: true},
&IntegerLiteral{Token: token.Token{Type: token.LBRACKET, Literal: "1"}, Value: 1}: &Boolean{Token: token.Token{Type: token.TRUE, Literal: "True"}, Value: true},
}},
}
arrLiteral := &ArrayLiteral{Token: token.Token{Type: token.LBRACKET, Literal: []byte("[")}, Elements: elements}
arrLiteral := &ArrayLiteral{Token: token.Token{Type: token.LBRACKET, Literal: "["}, Elements: elements}

for i := 0; i < b.N; i++ {
arrLiteral.String()
Expand Down
14 changes: 7 additions & 7 deletions ast/ast_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,25 +7,25 @@ import (

func createIdentifier(name string) *Identifier {
return &Identifier{
Token: token.Token{Type: token.IDENT, Literal: []byte(name)},
Token: token.Token{Type: token.IDENT, Literal: name},
Value: name,
}
}

func createVarStatement(name string, value Expression) *VarStatement {
return &VarStatement{
Token: token.Token{Type: token.VAR, Literal: []byte("var")},
Token: token.Token{Type: token.VAR, Literal: "var"},
Name: createIdentifier(name),
Value: value,
}
}

func createIntegerLiteral(value int64) Expression {
return &IntegerLiteral{Token: token.Token{Type: token.INT, Literal: []byte(strconv.FormatInt(value, 10))}, Value: value}
return &IntegerLiteral{Token: token.Token{Type: token.INT, Literal: strconv.FormatInt(value, 10)}, Value: value}
}

func createFloatLiteral(value float64) Expression {
return &FloatLiteral{Token: token.Token{Type: token.FLOAT, Literal: []byte(strconv.FormatFloat(value, 'f', -1, 64))}, Value: value}
return &FloatLiteral{Token: token.Token{Type: token.FLOAT, Literal: strconv.FormatFloat(value, 'f', -1, 64)}, Value: value}
}

func createBoolean(value bool) Expression {
Expand All @@ -36,16 +36,16 @@ func createBoolean(value bool) Expression {
tokenName = token.FALSE
}

return &Boolean{Token: token.Token{Type: tokenName, Literal: []byte(literal)}, Value: value}
return &Boolean{Token: token.Token{Type: tokenName, Literal: literal}, Value: value}
}

func createReturnStatement(returned Expression) Statement {
return &ReturnStatement{
Token: token.Token{Type: token.RETURN, Literal: []byte("return")},
Token: token.Token{Type: token.RETURN, Literal: "return"},
ReturnValue: returned,
}
}

func createBlockStatements(stmts []Statement) Statement {
return &BlockStatement{Token: token.Token{Type: token.LBRACE, Literal: []byte("{")}, Statements: stmts}
return &BlockStatement{Token: token.Token{Type: token.LBRACE, Literal: "{"}, Statements: stmts}
}
6 changes: 3 additions & 3 deletions ast/function_literal_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@ func TestFunctionLiteral_String(t *testing.T) {
createReturnStatement(createIntegerLiteral(tt.body)),
}

blockStatement := &BlockStatement{Token: token.Token{Type: token.LBRACE, Literal: []byte("{")}, Statements: stmts}
blockStatement := &BlockStatement{Token: token.Token{Type: token.LBRACE, Literal: "{"}, Statements: stmts}
argumentsIdentifier := []*Identifier{}
for _, arg := range tt.parameters {
integerLiteral := &Identifier{Token: token.Token{Type: token.INT, Literal: []byte(strconv.FormatInt(arg, 10))}, Value: strconv.FormatInt(arg, 10)}
integerLiteral := &Identifier{Token: token.Token{Type: token.INT, Literal: strconv.FormatInt(arg, 10)}, Value: strconv.FormatInt(arg, 10)}
argumentsIdentifier = append(argumentsIdentifier, integerLiteral)
}
fn := &FunctionLiteral{
Token: token.Token{Type: token.FUNCTION, Literal: []byte("function")},
Token: token.Token{Type: token.FUNCTION, Literal: "function"},
Parameters: argumentsIdentifier,
Body: blockStatement,
}
Expand Down
12 changes: 6 additions & 6 deletions ast/function_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,19 @@ func TestFunction_String(t *testing.T) {
for _, tt := range tests {

stmts := []Statement{
&ReturnStatement{Token: token.Token{Type: token.RETURN, Literal: []byte("return")}, ReturnValue: &IntegerLiteral{
Token: token.Token{Type: token.INT, Literal: []byte(strconv.FormatInt(tt.body, 10))}, Value: tt.body,
&ReturnStatement{Token: token.Token{Type: token.RETURN, Literal: "return"}, ReturnValue: &IntegerLiteral{
Token: token.Token{Type: token.INT, Literal: strconv.FormatInt(tt.body, 10)}, Value: tt.body,
}},
}
blockStatement := &BlockStatement{Token: token.Token{Type: token.LBRACE, Literal: []byte("{")}, Statements: stmts}
nameIdentifier := &Identifier{Token: token.Token{Type: token.IDENT, Literal: []byte("var")}, Value: tt.name}
blockStatement := &BlockStatement{Token: token.Token{Type: token.LBRACE, Literal: "{"}, Statements: stmts}
nameIdentifier := &Identifier{Token: token.Token{Type: token.IDENT, Literal: "var"}, Value: tt.name}
argumentsIdentifier := []*Identifier{}
for _, arg := range tt.parameters {
integerLiteral := &Identifier{Token: token.Token{Type: token.INT, Literal: []byte(strconv.FormatInt(arg, 10))}, Value: strconv.FormatInt(arg, 10)}
integerLiteral := &Identifier{Token: token.Token{Type: token.INT, Literal: strconv.FormatInt(arg, 10)}, Value: strconv.FormatInt(arg, 10)}
argumentsIdentifier = append(argumentsIdentifier, integerLiteral)
}
fn := &Function{
Token: token.Token{Type: token.FUNCTION, Literal: []byte("function")},
Token: token.Token{Type: token.FUNCTION, Literal: "function"},
Name: nameIdentifier,
Parameters: argumentsIdentifier,
Body: blockStatement,
Expand Down
12 changes: 8 additions & 4 deletions ast/integer_literal_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package ast

import (
"fmt"
"ninja/token"
"testing"
)
Expand All @@ -25,10 +26,13 @@ func TestIntegerLiteral_String(t *testing.T) {
}

for _, tt := range tests {
literal := &IntegerLiteral{Token: token.Token{Type: token.FLOAT, Literal: []byte(tt.expected)}, Value: tt.intValue}
t.Run(fmt.Sprintf("TestIntegerLiteral_String_%d", tt.intValue), func(t *testing.T) {
literal := &IntegerLiteral{Token: token.Token{Type: token.FLOAT, Literal: tt.expected}, Value: tt.intValue}

if literal.String() != tt.expected {
t.Errorf("IntegerLiteral.String() not match to %s. Got: %s", tt.expected, literal.String())
}
})

if literal.String() != tt.expected {
t.Errorf("IntegerLiteral.String() not match to %s. Got: %s", tt.expected, literal.String())
}
}
}
30 changes: 15 additions & 15 deletions benchmark/benchmarking.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,73 +4,73 @@ echo "======================="
echo "\n>>> SCRIPTING FIB(5);"

echo "### PHP"
time php ./benchmark/fib.php 5
time php ./fib.php 5
echo ""


echo "### GO"
time ./benchmark/fib-go 5
time ./fib-go 5
echo ""


echo "### NINJA"
time ./ninja-osx -e "function fib(n) { if (n < 2) { return n; } return fib(n-1) + fib(n-2); };fib(5)"
time ninja-lang -e "function fib(n) { if (n < 2) { return n; } return fib(n-1) + fib(n-2); };fib(5)"

echo "======================="
echo "\n>>> SCRIPTING FIB(10);"

echo "### PHP"
time php ./benchmark/fib.php 10
time php ./fib.php 10
echo ""

echo "### GO"
time ./benchmark/fib-go 10
time ./fib-go 10
echo ""

echo "### NINJA"
time ./ninja-osx -e "function fib(n) { if (n < 2) { return n; } return fib(n-1) + fib(n-2); };fib(10)"
time ninja-lang -e "function fib(n) { if (n < 2) { return n; } return fib(n-1) + fib(n-2); };fib(10)"

echo "======================="
echo "\n>>> SCRIPTING FIB(20);"

echo "### PHP"
time php ./benchmark/fib.php 20
time php ./fib.php 20
echo ""


echo "### GO"
time ./benchmark/fib-go 20
time ./fib-go 20
echo ""

echo "### NINJA"
time ./ninja-osx -e "function fib(n) { if (n < 2) { return n; } return fib(n-1) + fib(n-2); };fib(20)"
time ninja-lang -e "function fib(n) { if (n < 2) { return n; } return fib(n-1) + fib(n-2); };fib(20)"

echo "======================="
echo "\n>>> SCRIPTING FIB(25);"

echo "### PHP"
time php ./benchmark/fib.php 25
time php ./fib.php 25
echo ""


echo "### GO"
time ./benchmark/fib-go 25
time ./fib-go 25
echo ""

echo "### NINJA"
time ./ninja-osx -e "function fib(n) { if (n < 2) { return n; } return fib(n-1) + fib(n-2); };fib(25)"
time ninja-lang -e "function fib(n) { if (n < 2) { return n; } return fib(n-1) + fib(n-2); };fib(25)"

echo "======================="
echo "\n>>> SCRIPTING FIB(30);"

echo "### PHP"
time php ./benchmark/fib.php 30
time php ./fib.php 30
echo ""


echo "### GO"
time ./benchmark/fib-go 30
time ./fib-go 30
echo ""

echo "### NINJA"
time ./ninja-osx -e "function fib(n) { if (n < 2) { return n; } return fib(n-1) + fib(n-2); };fib(30)"
time ninja-lang -e "function fib(n) { if (n < 2) { return n; } return fib(n-1) + fib(n-2); };fib(30)"
56 changes: 48 additions & 8 deletions lexer/lexer.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package lexer

import (
"encoding/hex"
"io"
"ninja/token"
"strings"
"unicode/utf8"
)

Expand Down Expand Up @@ -39,8 +41,13 @@ func (l *Lexer) NextToken() token.Token {
case ';':
tok = l.newToken(token.SEMICOLON, []byte{l.ch})
case '"':
tok.Type = token.STRING
tok.Literal = runesToUTF8(l.readString())
str, err := l.readString()
if err != nil {
tok = l.newToken(token.ILLEGAL, []byte{l.ch})
} else {
tok.Type = token.STRING
tok.Literal = str
}
case '*':
tok = l.newToken(token.ASTERISK, []byte{l.ch})
case '/':
Expand Down Expand Up @@ -168,7 +175,7 @@ func (l *Lexer) keepTrackLineAndCharPosition() {

func (l *Lexer) newToken(tokenType token.TokenType, ch []byte) token.Token {
location := token.Location{Line: l.lineNumber + 1, Offset: l.characterPositionInLine}
return token.Token{Type: tokenType, Literal: ch, Location: location}
return token.Token{Type: tokenType, Literal: string(ch), Location: location}
}

func (l *Lexer) newTokenPeekOrDefault(tokenType token.TokenType, expectedPeek map[byte]token.TokenType) token.Token {
Expand Down Expand Up @@ -239,15 +246,48 @@ func (l *Lexer) peekChar() byte {
return l.input[l.readPosition]
}

func (l *Lexer) readString() []rune {
position := l.position + 1
func (l *Lexer) readString() (string, error) {
b := &strings.Builder{}
for {
l.readChar()
if l.ch == '"' {
break
// Support some basic escapes like \"
if l.ch == '\\' {
switch l.peekChar() {
case '"':
b.WriteByte('"')
case 'n':
b.WriteByte('\n')
case 'r':
b.WriteByte('\r')
case 't':
b.WriteByte('\t')
case '\\':
b.WriteByte('\\')
case 'x':
// Skip over the the '\\', 'x' and the next two bytes (hex)
l.readChar()
l.readChar()
prevCh := l.ch
l.readChar()
src := string([]byte{prevCh, l.ch})
dst, err := hex.DecodeString(src)
if err != nil {
return "", err
}
b.Write(dst)
continue
}
// Skip over the '\\' and the matched single escape char
l.readChar()
continue
} else {
if l.ch == '"' || l.ch == 0 {
break
}
}
b.WriteByte(l.ch)
}
return []rune(l.input[position:l.position])
return b.String(), nil
}

func runesToUTF8(rs []rune) []byte {
Expand Down
27 changes: 27 additions & 0 deletions lexer/lexer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,3 +163,30 @@ func TestStringAcceptUtf8Character(t *testing.T) {
}

}

func TestLexerReadString(t *testing.T) {
input := `"\"foo\"";"\x00\x0a\x7f";"\r\n\t"`

tests := []struct {
expectedType token.TokenType
expectedLiteral string
}{
{token.STRING, "\"foo\""},
{token.SEMICOLON, ";"},
{token.STRING, "\x00\n\u007f"},
{token.SEMICOLON, ";"},
{token.STRING, "\r\n\t"},
}
lexer := New(strings.NewReader(input))

for _, test := range tests {
tok := lexer.NextToken()
if tok.Type != test.expectedType {
t.Fatalf("token type wrong. expected=%q, got=%q", test.expectedType, tok.Type)
}

if string(tok.Literal) != test.expectedLiteral {
t.Fatalf("literal wrong. expected=%q, got=%q", test.expectedLiteral, tok.Literal)
}
}
}
2 changes: 1 addition & 1 deletion token/token.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ type Location struct {

type Token struct {
Type TokenType
Literal []byte
Literal string
Location
}

Expand Down

0 comments on commit 7c514fc

Please sign in to comment.