diff --git a/tokenizer/token.go b/tokenizer/token.go index f6b1258..0b2387a 100644 --- a/tokenizer/token.go +++ b/tokenizer/token.go @@ -210,7 +210,7 @@ func (t Token) String() string { return fmt.Sprintf("%d:%q (%d: %d, %d) %v [%d]", t.Index, t.Surface, t.Position, t.Start, t.End, t.Class, t.ID) } -// Equal returns true if tokens are equal. This function compares values other than the `Index` field. +// Equal returns true if tokens are equal. func (t Token) Equal(v Token) bool { return t.ID == v.ID && t.Class == v.Class &&