tokens.lem

open import Pervasives_extra
(* Tokens for Standard ML.  NB, not all of them are used in CakeML *)
type token =
| WhitespaceT of nat | NewlineT | LexErrorT
| HashT | LparT | RparT | StarT | CommaT | ArrowT | DotsT | ColonT | SealT
| SemicolonT | EqualsT | DarrowT | LbrackT | RbrackT | UnderbarT | LbraceT
| BarT | RbraceT | AndT | AndalsoT | AsT | CaseT | DatatypeT
| ElseT | EndT | EqtypeT | ExceptionT | FnT | FunT | HandleT | IfT
| InT | IncludeT | LetT | LocalT | OfT | OpT
| OpenT | OrelseT | RaiseT | RecT | RefT | SharingT | SigT | SignatureT | StructT
| StructureT | ThenT | TypeT | ValT | WhereT | WhileT | WithT | WithtypeT
| IntT of integer
| HexintT of string
| WordT of nat
| RealT of string
| StringT of string
| CharT of char
| TyvarT of string
| AlphaT of string
| SymbolT of string
| LongidT of string * string
| FFIT of string