Skip to content

Commit

Permalink
fmt
Browse files Browse the repository at this point in the history
  • Loading branch information
vcfxb committed Feb 24, 2024
1 parent 7f8919d commit bea8f04
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 25 deletions.
2 changes: 1 addition & 1 deletion wright/src/bin/wright.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use codespan_reporting::files::Files;
use std::{path::PathBuf, time::Instant};
use wright::{
filemap::{FileId, FileMap},
parser::lexer::{Lexer, token::Token},
parser::lexer::{token::Token, Lexer},
repl,
};

Expand Down
10 changes: 5 additions & 5 deletions wright/src/parser/fragment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,13 @@ impl<'src> Fragment<'src> {
}

/// Unsafe version of [`Fragment::split_at`]. Splits this [Fragment] into two subfragments,
/// where the left one contains the first `bytes` bytes of the fragment, and the right one
/// contains the rest.
///
/// where the left one contains the first `bytes` bytes of the fragment, and the right one
/// contains the rest.
///
/// # Safety
/// - Undefined Behavior occurs if `bytes` is greater than the length of the [Fragment].
/// - Undefined Behavior occurs if `bytes` is not on a UTF-8 character boundary.
/// - See [str::get_unchecked] for more details.
/// - Undefined Behavior occurs if `bytes` is not on a UTF-8 character boundary.
/// - See [str::get_unchecked] for more details.
pub unsafe fn split_at_unchecked(&self, bytes: usize) -> (Self, Self) {
let left: &str = self.inner.get_unchecked(..bytes);
let right: &str = self.inner.get_unchecked(bytes..);
Expand Down
13 changes: 8 additions & 5 deletions wright/src/parser/lexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use super::fragment::Fragment;
use std::iter::FusedIterator;
use std::str::Chars;
use std::{iter::Peekable, ptr};
use unicode_ident::{is_xid_continue, is_xid_start};
use token::{Token, TokenTy};
use unicode_ident::{is_xid_continue, is_xid_start};

pub mod token;
pub mod trivial;
Expand Down Expand Up @@ -97,15 +97,18 @@ impl<'src> Lexer<'src> {
}
}

/// Unsafe version of [Lexer::split_token].
///
/// Unsafe version of [Lexer::split_token].
///
/// # Safety:
/// - This function matches the safety guarantees of [Fragment::split_at_unchecked].
unsafe fn split_token_unchecked(&mut self, bytes: usize, kind: TokenTy) -> Token<'src> {
let (token_fragment, new_remaining_fragment) = self.remaining.split_at_unchecked(bytes);
self.remaining = new_remaining_fragment;

Token { variant: kind, fragment: token_fragment }
Token {
variant: kind,
fragment: token_fragment,
}
}

/// "Fork" this lexer, creating a new [`Lexer`] at the same position as this one that can be used for
Expand Down Expand Up @@ -341,7 +344,7 @@ impl<'src> Lexer<'src> {
token => return token,
}

// Handle a trivial token if there is one.
// Handle a trivial token if there is one.
if let Some(token) = trivial::try_consume_trivial_token(self) {
return Some(token);
}
Expand Down
4 changes: 2 additions & 2 deletions wright/src/parser/lexer/token.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Token models.
//! Token models.
use derive_more::Display;
use crate::parser::fragment::Fragment;
use derive_more::Display;

/// A token in wright source code.
#[derive(Debug, Display)]
Expand Down
28 changes: 16 additions & 12 deletions wright/src/parser/lexer/trivial.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
//! Trivial tokens and their implementation.
use super::{token::{Token, TokenTy}, Lexer};
use super::{
token::{Token, TokenTy},
Lexer,
};

/// Trivial tokens that are two ASCII characters and can be matched directly
/// against the input source code.
Expand Down Expand Up @@ -57,17 +60,18 @@ pub const SINGLE_ASCII_CHAR_TRIVIAL_TOKENS: &[(u8, TokenTy)] = &[
(b'%', TokenTy::Mod),
];


/// Attempt to consume a "trivial" token from the start of the [Lexer]'s [Lexer::remaining] fragment.
///
/// Leave the lexer unmodified if one is not available.
/// Attempt to consume a "trivial" token from the start of the [Lexer]'s [Lexer::remaining] fragment.
///
/// Leave the lexer unmodified if one is not available.
#[inline]
pub fn try_consume_trivial_token<'src>(lexer: &mut Lexer<'src>) -> Option<Token<'src>> {
// Get the number of bytes remaining, since we need at least 1 to parse anything.
let bytes_remaining: usize = lexer.bytes_remaining();

// No token if there are no bytes of source left.
if bytes_remaining == 0 { return None; }
// No token if there are no bytes of source left.
if bytes_remaining == 0 {
return None;
}

// Attempt to match any two-byte ASCII trivial tokens.
// This must be done before single-ascii byte tokens since matching is greedy.
Expand All @@ -79,25 +83,25 @@ pub fn try_consume_trivial_token<'src>(lexer: &mut Lexer<'src>) -> Option<Token<
// Match against each possible token pattern.
for (pattern, kind) in TWO_ASCII_TRIVIAL_TOKENS {
if bytes == *pattern {
// SAFETY: We have already done bounds checking, and this cannot be a character
// boundary since we just matched against ASCII characters.
// SAFETY: We have already done bounds checking, and this cannot be a character
// boundary since we just matched against ASCII characters.
return Some(unsafe { lexer.split_token_unchecked(2, *kind) });
}
}
}

// Do the same for single byte patterns.
// SAFETY: We checked that the number of bytes remaining is not 0 above.
// SAFETY: We checked that the number of bytes remaining is not 0 above.
let byte: &u8 = unsafe { lexer.remaining.inner.as_bytes().get_unchecked(0) };

for (pattern, kind) in SINGLE_ASCII_CHAR_TRIVIAL_TOKENS {
if byte == pattern {
// SAFETTY: If we matched, then the first byte is ASCII, and therefor we don't have to worry
// about bounds or unicode boundaries.
// about bounds or unicode boundaries.
return Some(unsafe { lexer.split_token_unchecked(1, *kind) });
}
}

// If nothing else has matched, there is no trivial token available.
// If nothing else has matched, there is no trivial token available.
None
}

0 comments on commit bea8f04

Please sign in to comment.