From b82b06d832fa5e624c02005b11cc0e9688bc1c2b Mon Sep 17 00:00:00 2001 From: Mikhail Tavarez Date: Tue, 9 Jul 2024 12:30:33 -0500 Subject: [PATCH] cleanup imports --- external/gojo/bufio/__init__.mojo | 2 +- external/gojo/bufio/bufio.mojo | 452 ++++---- external/gojo/bufio/scan.mojo | 239 ++-- external/gojo/builtins/__init__.mojo | 7 +- external/gojo/builtins/attributes.mojo | 124 ++- external/gojo/builtins/bytes.mojo | 46 +- external/gojo/builtins/errors.mojo | 2 +- external/gojo/builtins/list.mojo | 133 --- external/gojo/bytes/buffer.mojo | 662 +++++------ external/gojo/bytes/reader.mojo | 181 ++- external/gojo/fmt/fmt.mojo | 83 +- external/gojo/io/__init__.mojo | 363 +++++- external/gojo/io/file.mojo | 136 +++ external/gojo/io/io.mojo | 100 +- external/gojo/io/std.mojo | 54 + external/gojo/io/traits.mojo | 320 ------ external/gojo/net/__init__.mojo | 13 + external/gojo/net/address.mojo | 82 +- external/gojo/net/dial.mojo | 45 - external/gojo/net/fd.mojo | 64 +- external/gojo/net/ip.mojo | 142 +-- external/gojo/net/net.mojo | 130 --- external/gojo/net/socket.mojo | 385 ++++--- external/gojo/net/tcp.mojo | 263 +++-- external/gojo/net/udp.mojo | 210 ++++ external/gojo/strings/builder.mojo | 172 +-- external/gojo/strings/reader.mojo | 189 ++-- external/gojo/syscall/__init__.mojo | 62 ++ external/gojo/syscall/file.mojo | 63 +- external/gojo/syscall/net.mojo | 784 +++++++------ external/gojo/syscall/types.mojo | 63 -- external/gojo/unicode/__init__.mojo | 2 +- external/gojo/unicode/utf8/__init__.mojo | 3 +- external/gojo/unicode/utf8/runes.mojo | 327 +----- external/gojo/unicode/utf8/table.mojo | 1296 ++++++++++++++++++++++ external/gojo/unicode/utf8/width.mojo | 105 ++ external/hue/color.mojo | 34 +- external/hue/happy_palettegen.mojo | 2 +- external/hue/hsluv.mojo | 4 +- external/hue/math.mojo | 3 +- external/hue/soft_palettegen.mojo | 18 +- external/hue/warm_palettegen.mojo | 2 +- external/mist/__init__.mojo | 20 +- external/mist/ansi_colors.mojo | 517 +++++---- external/mist/color.mojo | 302 ++--- external/mist/profile.mojo | 100 +- external/mist/renderers.mojo | 64 +- external/mist/screen.mojo | 99 +- external/mist/style.mojo | 137 +-- external/morrow/_libc.mojo | 26 +- external/morrow/constants.mojo | 10 +- external/morrow/formatter.mojo | 34 +- external/morrow/morrow.mojo | 6 +- external/morrow/timedelta.mojo | 20 +- external/morrow/timezone.mojo | 8 +- external/morrow/util.mojo | 6 +- 56 files changed, 5079 insertions(+), 3637 deletions(-) delete mode 100644 external/gojo/builtins/list.mojo create mode 100644 external/gojo/io/file.mojo create mode 100644 external/gojo/io/std.mojo delete mode 100644 external/gojo/io/traits.mojo delete mode 100644 external/gojo/net/dial.mojo delete mode 100644 external/gojo/net/net.mojo create mode 100644 external/gojo/net/udp.mojo delete mode 100644 external/gojo/syscall/types.mojo create mode 100644 external/gojo/unicode/utf8/table.mojo create mode 100644 external/gojo/unicode/utf8/width.mojo diff --git a/external/gojo/bufio/__init__.mojo b/external/gojo/bufio/__init__.mojo index c501992..2028d55 100644 --- a/external/gojo/bufio/__init__.mojo +++ b/external/gojo/bufio/__init__.mojo @@ -1,2 +1,2 @@ -from .bufio import Reader, Writer, ReadWriter +from .bufio import Reader, Writer, ReadWriter, new_writer, new_reader, new_read_writer from .scan import Scanner, scan_words, scan_bytes, scan_lines diff --git a/external/gojo/bufio/bufio.mojo b/external/gojo/bufio/bufio.mojo index 92fc6b9..fa84947 100644 --- a/external/gojo/bufio/bufio.mojo +++ b/external/gojo/bufio/bufio.mojo @@ -1,12 +1,11 @@ -from math import max -from ..io import traits as io +from collections import InlineList +import ..io from ..builtins import copy, panic -from ..builtins.bytes import Byte, index_byte +from ..builtins.bytes import index_byte from ..strings import StringBuilder alias MIN_READ_BUFFER_SIZE = 16 alias MAX_CONSECUTIVE_EMPTY_READS = 100 -alias DEFAULT_BUF_SIZE = 4096 alias ERR_INVALID_UNREAD_BYTE = "bufio: invalid use of unread_byte" alias ERR_INVALID_UNREAD_RUNE = "bufio: invalid use of unread_rune" @@ -17,10 +16,11 @@ alias ERR_NEGATIVE_WRITE = "bufio: writer returned negative count from write" # buffered input -struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io.WriterTo): +# TODO: Uncomment write_to and write_buf once the bug with the trait's Span argument is fixed. +struct Reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](Sized, io.Reader, io.ByteReader, io.ByteScanner): """Implements buffering for an io.Reader object.""" - var buf: List[Byte] + var buf: InlineList[UInt8, size] var reader: R # reader provided by the client var read_pos: Int var write_pos: Int # buf read and write positions @@ -31,13 +31,16 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. fn __init__( inout self, owned reader: R, - buf: List[Byte] = List[Byte](capacity=DEFAULT_BUF_SIZE), + buf: InlineList[UInt8, size] = InlineList[UInt8, size](), read_pos: Int = 0, write_pos: Int = 0, last_byte: Int = -1, last_rune_size: Int = -1, ): - self.buf = buf + self.buf = InlineList[UInt8, size]() + for element in buf: + self.buf.append(element[]) + self.reader = reader^ self.read_pos = read_pos self.write_pos = write_pos @@ -46,7 +49,10 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. self.err = Error() fn __moveinit__(inout self, owned existing: Self): - self.buf = existing.buf^ + self.buf = InlineList[UInt8, size]() + for element in existing.buf: + self.buf.append(element[]) + self.reader = existing.reader^ self.read_pos = existing.read_pos self.write_pos = existing.write_pos @@ -71,12 +77,21 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. # return # # if self.buf == nil: - # # self.buf = make(List[Byte], DEFAULT_BUF_SIZE) + # # self.buf = make(InlineList[UInt8, io.BUFFER_SIZE], io.BUFFER_SIZE) # self.reset(self.buf, r) - fn reset(inout self, buf: List[Byte], owned reader: R): - self = Reader[R]( + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the internal data as a Span[UInt8].""" + return Span[UInt8, __lifetime_of(self)](array=self.buf._array) + + fn reset(inout self, buf: InlineList[UInt8, size], owned reader: R): + """Discards any buffered data, resets all state, and switches + the buffered reader to read from r. + Calling reset on the zero value of [Reader] initializes the internal buffer + to the default size. + Calling self.reset(b) (that is, resetting a [Reader] to itself) does nothing.""" + self = Reader[R, size]( buf=buf, reader=reader^, last_byte=-1, @@ -87,29 +102,32 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. """Reads a new chunk into the buffer.""" # Slide existing data to beginning. if self.read_pos > 0: - var current_capacity = self.buf.capacity - self.buf = self.buf[self.read_pos : self.write_pos] - self.buf.reserve(current_capacity) + var data_to_slide = self.as_bytes_slice()[self.read_pos : self.write_pos] + # TODO: Temp copying of elements until I figure out a better pattern or slice refs are added + for i in range(len(data_to_slide)): + self.buf[i] = data_to_slide[i] + + # self.buf.reserve(current_capacity) self.write_pos -= self.read_pos self.read_pos = 0 - # Compares to the length of the entire List[Byte] object, including 0 initialized positions. - # IE. var b = List[Byte](capacity=4096), then trying to write at b[4096] and onwards will fail. - if self.write_pos >= self.buf.capacity: + # Compares to the length of the entire InlineList[UInt8, io.BUFFER_SIZE] object, including 0 initialized positions. + # IE. var b = InlineList[UInt8, io.BUFFER_SIZE](capacity=4096), then trying to write at b[4096] and onwards will fail. + if self.write_pos >= io.BUFFER_SIZE: panic("bufio.Reader: tried to fill full buffer") # Read new data: try a limited number of times. var i: Int = MAX_CONSECUTIVE_EMPTY_READS while i > 0: - # TODO: Using temp until slicing can return a Reference - var temp = List[Byte](capacity=DEFAULT_BUF_SIZE) + # TODO: Using temp until slicing can return a Reference, does reading directly into a Span of self.buf work? + # Maybe we need to read into the end of the buffer. + var span = self.as_bytes_slice() var bytes_read: Int var err: Error - bytes_read, err = self.reader.read(temp) + bytes_read, err = self.reader._read(span, len(self.buf)) if bytes_read < 0: panic(ERR_NEGATIVE_READ) - bytes_read = copy(self.buf, temp, self.write_pos) self.write_pos += bytes_read if err: @@ -121,7 +139,7 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. i -= 1 - self.err = Error(io.ERR_NO_PROGRESS) + self.err = Error(str(io.ERR_NO_PROGRESS)) fn read_error(inout self) -> Error: if not self.err: @@ -131,7 +149,7 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. self.err = Error() return err - fn peek(inout self, number_of_bytes: Int) -> (List[Byte], Error): + fn peek(inout self, number_of_bytes: Int) -> (Span[UInt8, __lifetime_of(self)], Error): """Returns the next n bytes without advancing the reader. The bytes stop being valid at the next read call. If Peek returns fewer than n bytes, it also returns an error explaining why the read is short. The error is @@ -144,18 +162,18 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. number_of_bytes: The number of bytes to peek. """ if number_of_bytes < 0: - return List[Byte](), Error(ERR_NEGATIVE_COUNT) + return self.as_bytes_slice()[0:0], Error(ERR_NEGATIVE_COUNT) self.last_byte = -1 self.last_rune_size = -1 - while self.write_pos - self.read_pos < number_of_bytes and self.write_pos - self.read_pos < self.buf.capacity: - self.fill() # self.write_pos-self.read_pos < self.buf.capacity => buffer is not full + while self.write_pos - self.read_pos < number_of_bytes and self.write_pos - self.read_pos < io.BUFFER_SIZE: + self.fill() # self.write_pos-self.read_pos < self.capacity => buffer is not full - if number_of_bytes > self.buf.capacity: - return self.buf[self.read_pos : self.write_pos], Error(ERR_BUFFER_FULL) + if number_of_bytes > io.BUFFER_SIZE: + return self.as_bytes_slice()[self.read_pos : self.write_pos], Error(ERR_BUFFER_FULL) - # 0 <= n <= self.buf.capacity + # 0 <= n <= io.BUFFER_SIZE var err = Error() var available_space = self.write_pos - self.read_pos if available_space < number_of_bytes: @@ -164,7 +182,7 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. if not err: err = Error(ERR_BUFFER_FULL) - return self.buf[self.read_pos : self.read_pos + number_of_bytes], err + return self.as_bytes_slice()[self.read_pos : self.read_pos + number_of_bytes], err fn discard(inout self, number_of_bytes: Int) -> (Int, Error): """Discard skips the next n bytes, returning the number of bytes discarded. @@ -197,7 +215,7 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. if remain == 0: return number_of_bytes, Error() - fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): """Reads data into dest. It returns the number of bytes read into dest. The bytes are taken from at most one Read on the underlying [Reader], @@ -205,22 +223,19 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. To read exactly len(src) bytes, use io.ReadFull(b, src). If the underlying [Reader] can return a non-zero count with io.EOF, then this Read method can do so as well; see the [io.Reader] docs.""" - var space_available = dest.capacity - len(dest) - if space_available == 0: + if capacity == 0: if self.buffered() > 0: return 0, Error() return 0, self.read_error() var bytes_read: Int = 0 if self.read_pos == self.write_pos: - if space_available >= len(self.buf): + if capacity >= len(self.buf): # Large read, empty buffer. # Read directly into dest to avoid copy. var bytes_read: Int - var err: Error - bytes_read, err = self.reader.read(dest) + bytes_read, self.err = self.reader._read(dest, capacity) - self.err = err if bytes_read < 0: panic(ERR_NEGATIVE_READ) @@ -234,9 +249,9 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. # Do not use self.fill, which will loop. self.read_pos = 0 self.write_pos = 0 + var buf = self.as_bytes_slice() # TODO: I'm hoping this reads into self.data directly lol var bytes_read: Int - var err: Error - bytes_read, err = self.reader.read(self.buf) + bytes_read, self.err = self.reader._read(buf, len(buf)) if bytes_read < 0: panic(ERR_NEGATIVE_READ) @@ -247,23 +262,46 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. self.write_pos += bytes_read # copy as much as we can - # Note: if the slice panics here, it is probably because - # the underlying reader returned a bad count. See issue 49795. - bytes_read = copy(dest, self.buf[self.read_pos : self.write_pos]) + var source = self.as_bytes_slice()[self.read_pos : self.write_pos] + bytes_read = 0 + var start = len(dest) + var target = dest.unsafe_ptr() + for i in range(len(source)): + target[i + start] = source[i] + bytes_read += 1 + dest._len += bytes_read self.read_pos += bytes_read self.last_byte = int(self.buf[self.read_pos - 1]) self.last_rune_size = -1 return bytes_read, Error() - fn read_byte(inout self) -> (Byte, Error): + fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): + """Reads data into dest. + It returns the number of bytes read into dest. + The bytes are taken from at most one Read on the underlying [Reader], + hence n may be less than len(src). + To read exactly len(src) bytes, use io.ReadFull(b, src). + If the underlying [Reader] can return a non-zero count with io.EOF, + then this Read method can do so as well; see the [io.Reader] docs.""" + + var span = Span(dest) + + var bytes_read: Int + var err: Error + bytes_read, err = self._read(span, dest.capacity) + dest.size += bytes_read + + return bytes_read, err + + fn read_byte(inout self) -> (UInt8, Error): """Reads and returns a single byte from the internal buffer. If no byte is available, returns an error.""" self.last_rune_size = -1 while self.read_pos == self.write_pos: if self.err: - return Int8(0), self.read_error() + return UInt8(0), self.read_error() self.fill() # buffer is empty - var c = self.buf[self.read_pos] + var c = self.as_bytes_slice()[self.read_pos] self.read_pos += 1 self.last_byte = int(c) return c, Error() @@ -285,7 +323,7 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. # self.read_pos == 0 and self.write_pos == 0 self.write_pos = 1 - self.buf[self.read_pos] = self.last_byte + self.as_bytes_slice()[self.read_pos] = self.last_byte self.last_byte = -1 self.last_rune_size = -1 return Error() @@ -294,19 +332,19 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. # # rune and its size in bytes. If the encoded rune is invalid, it consumes one byte # # and returns unicode.ReplacementChar (U+FFFD) with a size of 1. # fn read_rune(inout self) (r rune, size int, err error): - # for self.read_pos+utf8.UTFMax > self.write_pos and !utf8.FullRune(self.buf[self.read_pos:self.write_pos]) and self.err == nil and self.write_pos-self.read_pos < self.buf.capacity: + # for self.read_pos+utf8.UTFMax > self.write_pos and !utf8.FullRune(self.as_bytes_slice()[self.read_pos:self.write_pos]) and self.err == nil and self.write_pos-self.read_pos < self.buf.capacity: # self.fill() # self.write_pos-self.read_pos < len(buf) => buffer is not full # self.last_rune_size = -1 # if self.read_pos == self.write_pos: # return 0, 0, self.read_poseadErr() - # r, size = rune(self.buf[self.read_pos]), 1 + # r, size = rune(self.as_bytes_slice()[self.read_pos]), 1 # if r >= utf8.RuneSelf: - # r, size = utf8.DecodeRune(self.buf[self.read_pos:self.write_pos]) + # r, size = utf8.DecodeRune(self.as_bytes_slice()[self.read_pos:self.write_pos]) # self.read_pos += size - # self.last_byte = int(self.buf[self.read_pos-1]) + # self.last_byte = int(self.as_bytes_slice()[self.read_pos-1]) # self.last_rune_size = size # return r, size, nil @@ -331,7 +369,7 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. """ return self.write_pos - self.read_pos - fn read_slice(inout self, delim: Int8) -> (List[Byte], Error): + fn read_slice(inout self, delim: UInt8) -> (Span[UInt8, __lifetime_of(self)], Error): """Reads until the first occurrence of delim in the input, returning a slice pointing at the bytes in the buffer. It includes the first occurrence of the delimiter. The bytes stop being valid at the next read. @@ -347,31 +385,31 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. delim: The delimiter to search for. Returns: - The List[Byte] from the internal buffer. + The Span[UInt8] from the internal buffer. """ var err = Error() var s = 0 # search start index - var line: List[Byte] = List[Byte](capacity=DEFAULT_BUF_SIZE) + var line: Span[UInt8, __lifetime_of(self)] while True: # Search buffer. - var i = index_byte(self.buf[self.read_pos + s : self.write_pos], delim) + var i = index_byte(self.as_bytes_slice()[self.read_pos + s : self.write_pos], delim) if i >= 0: i += s - line = self.buf[self.read_pos : self.read_pos + i + 1] + line = self.as_bytes_slice()[self.read_pos : self.read_pos + i + 1] self.read_pos += i + 1 break # Pending error? if self.err: - line = self.buf[self.read_pos : self.write_pos] + line = self.as_bytes_slice()[self.read_pos : self.write_pos] self.read_pos = self.write_pos err = self.read_error() break # Buffer full? - if self.buffered() >= self.buf.capacity: + if self.buffered() >= io.BUFFER_SIZE: self.read_pos = self.write_pos - line = self.buf + line = self.as_bytes_slice() err = Error(ERR_BUFFER_FULL) break @@ -386,7 +424,7 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. return line, err - fn read_line(inout self) raises -> (List[Byte], Bool): + fn read_line(inout self: Self) -> (List[UInt8], Bool): """Low-level line-reading primitive. Most callers should use [Reader.read_bytes]('\n') or [Reader.read_string]('\n') instead or use a [Scanner]. @@ -404,7 +442,7 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. (possibly a character belonging to the line end) even if that byte is not part of the line returned by read_line. """ - var line: List[Byte] + var line: Span[UInt8, __lifetime_of(self)] var err: Error line, err = self.read_slice(ord("\n")) @@ -415,14 +453,14 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. # Let the next call to read_line check for "\r\n". if self.read_pos == 0: # should be unreachable - raise Error("bufio: tried to rewind past start of buffer") + panic("bufio: tried to rewind past start of buffer") self.read_pos -= 1 line = line[: len(line) - 1] - return line, True + return List[UInt8](line), True if len(line) == 0: - return line, False + return List[UInt8](line), False if line[len(line) - 1] == ord("\n"): var drop = 1 @@ -431,9 +469,9 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. line = line[: len(line) - drop] - return line, False + return List[UInt8](line), False - fn collect_fragments(inout self, delim: Int8) -> (List[List[Byte]], List[Byte], Int, Error): + fn collect_fragments(inout self, delim: UInt8) -> (List[List[UInt8]], Span[UInt8, __lifetime_of(self)], Int, Error): """Reads until the first occurrence of delim in the input. It returns (slice of full buffers, remaining bytes before delim, total number of bytes in the combined first two elements, error). @@ -443,9 +481,9 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. """ # Use read_slice to look for delim, accumulating full buffers. var err = Error() - var full_buffers = List[List[Byte]]() + var full_buffers = List[List[UInt8]]() var total_len = 0 - var frag = List[Byte](capacity=4096) + var frag: Span[UInt8, __lifetime_of(self)] while True: frag, err = self.read_slice(delim) if not err: @@ -456,15 +494,15 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. err = read_slice_error break - # Make a copy of the buffer. - var buf = List[Byte](frag) + # Make a copy of the buffer Span. + var buf = List[UInt8](frag) full_buffers.append(buf) total_len += len(buf) total_len += len(frag) return full_buffers, frag, total_len, err - fn read_bytes(inout self, delim: Int8) -> (List[Byte], Error): + fn read_bytes(inout self, delim: UInt8) -> (List[UInt8], Error): """Reads until the first occurrence of delim in the input, returning a slice containing the data up to and including the delimiter. If read_bytes encounters an error before finding a delimiter, @@ -477,16 +515,16 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. delim: The delimiter to search for. Returns: - The List[Byte] from the internal buffer. + The List[UInt8] from the internal buffer. """ - var full: List[List[Byte]] - var frag: List[Byte] + var full: List[List[UInt8]] + var frag: Span[UInt8, __lifetime_of(self)] var n: Int var err: Error full, frag, n, err = self.collect_fragments(delim) # Allocate new buffer to hold the full pieces and the fragment. - var buf = List[Byte](capacity=n) + var buf = List[UInt8](capacity=n) n = 0 # copy full pieces and fragment in. @@ -498,7 +536,7 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. return buf, err - fn read_string(inout self, delim: Int8) -> (String, Error): + fn read_string(inout self, delim: UInt8) -> (String, Error): """Reads until the first occurrence of delim in the input, returning a string containing the data up to and including the delimiter. If read_string encounters an error before finding a delimiter, @@ -513,123 +551,109 @@ struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io. Returns: The String from the internal buffer. """ - var full: List[List[Byte]] - var frag: List[Byte] + var full: List[List[UInt8]] + var frag: Span[UInt8, __lifetime_of(self)] var n: Int var err: Error full, frag, n, err = self.collect_fragments(delim) # Allocate new buffer to hold the full pieces and the fragment. - var buf = StringBuilder(size=n) + var buf = StringBuilder(capacity=n) # copy full pieces and fragment in. for i in range(len(full)): var buffer = full[i] - _ = buf.write(buffer) + _ = buf.write(Span(buffer)) _ = buf.write(frag) return str(buf), err - fn write_to[W: io.Writer](inout self, inout writer: W) -> (Int64, Error): - """Writes the internal buffer to the writer. This may make multiple calls to the [Reader.Read] method of the underlying [Reader]. - If the underlying reader supports the [Reader.WriteTo] method, - this calls the underlying [Reader.WriteTo] without buffering. - write_to implements io.WriterTo. - - Args: - writer: The writer to write to. + # fn write_to[W: io.Writer](inout self, inout writer: W) -> (Int, Error): + # """Writes the internal buffer to the writer. This may make multiple calls to the [Reader.Read] method of the underlying [Reader]. + # If the underlying reader supports the [Reader.WriteTo] method, + # this calls the underlying [Reader.WriteTo] without buffering. + # write_to implements io.WriterTo. - Returns: - The number of bytes written. - """ - self.last_byte = -1 - self.last_rune_size = -1 + # Args: + # writer: The writer to write to. - var bytes_written: Int64 - var err: Error - bytes_written, err = self.write_buf(writer) - if err: - return bytes_written, err - - # internal buffer not full, fill before writing to writer - if (self.write_pos - self.read_pos) < self.buf.capacity: - self.fill() - - while self.read_pos < self.write_pos: - # self.read_pos < self.write_pos => buffer is not empty - var bw: Int64 - var err: Error - bw, err = self.write_buf(writer) - bytes_written += bw - - self.fill() # buffer is empty + # Returns: + # The number of bytes written. + # """ + # self.last_byte = -1 + # self.last_rune_size = -1 - return bytes_written, Error() + # var bytes_written: Int + # var err: Error + # bytes_written, err = self.write_buf(writer) + # if err: + # return bytes_written, err - fn write_buf[W: io.Writer](inout self, inout writer: W) -> (Int64, Error): - """Writes the [Reader]'s buffer to the writer. + # # internal buffer not full, fill before writing to writer + # if (self.write_pos - self.read_pos) < io.BUFFER_SIZE: + # self.fill() - Args: - writer: The writer to write to. + # while self.read_pos < self.write_pos: + # # self.read_pos < self.write_pos => buffer is not empty + # var bw: Int + # var err: Error + # bw, err = self.write_buf(writer) + # bytes_written += bw - Returns: - The number of bytes written. - """ - # Nothing to write - if self.read_pos == self.write_pos: - return Int64(0), Error() + # self.fill() # buffer is empty - # Write the buffer to the writer, if we hit EOF it's fine. That's not a failure condition. - var bytes_written: Int - var err: Error - bytes_written, err = writer.write(self.buf[self.read_pos : self.write_pos]) - if err: - return Int64(bytes_written), err + # return bytes_written, Error() - if bytes_written < 0: - panic(ERR_NEGATIVE_WRITE) + # fn write_buf[W: io.Writer](inout self, inout writer: W) -> (Int, Error): + # """Writes the [Reader]'s buffer to the writer. - self.read_pos += bytes_written - return Int64(bytes_written), Error() + # Args: + # writer: The writer to write to. + # Returns: + # The number of bytes written. + # """ + # # Nothing to write + # if self.read_pos == self.write_pos: + # return Int(0), Error() -# fn new_reader_size[R: io.Reader](owned reader: R, size: Int) -> Reader[R]: -# """Returns a new [Reader] whose buffer has at least the specified -# size. If the argument io.Reader is already a [Reader] with large enough -# size, it returns the underlying [Reader]. + # # Write the buffer to the writer, if we hit EOF it's fine. That's not a failure condition. + # var bytes_written: Int + # var err: Error + # var buf_to_write = self.as_bytes_slice()[self.read_pos : self.write_pos] + # bytes_written, err = writer.write(List[UInt8](buf_to_write)) + # if err: + # return bytes_written, err -# Args: -# reader: The reader to read from. -# size: The size of the buffer. + # if bytes_written < 0: + # panic(ERR_NEGATIVE_WRITE) -# Returns: -# The new [Reader]. -# """ -# # # Is it already a Reader? -# # b, ok := rd.(*Reader) -# # if ok and self.buf.capacity >= size: -# # return b + # self.read_pos += bytes_written + # return Int(bytes_written), Error() -# var r = Reader(reader ^) -# r.reset(List[Byte](capacity=max(size, MIN_READ_BUFFER_SIZE)), reader ^) -# return r +fn new_reader[R: io.Reader, size: Int = MIN_READ_BUFFER_SIZE](owned reader: R) -> Reader[R, size]: + """Returns a new [Reader] whose buffer has at least the specified + size. If the argument io.Reader is already a [Reader] with large enough + size, it returns the underlying [Reader]. -# fn new_reader[R: io.Reader](reader: R) -> Reader[R]: -# """Returns a new [Reader] whose buffer has the default size. + Args: + reader: The reader to read from. -# Args: -# reader: The reader to read from. + Params: + size: The size of the buffer. -# Returns: -# The new [Reader]. -# """ -# return new_reader_size(reader, DEFAULT_BUF_SIZE) + Returns: + The new [Reader]. + """ + var r = Reader[R, size](reader^) + return r^ # buffered output -# TODO: Reader and Writer maybe should not take ownership of the underlying reader/writer? Seems okay for now. -struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io.ReaderFrom): +struct Writer[W: io.Writer, size: Int = io.BUFFER_SIZE]( + Sized, io.Writer, io.ByteWriter, io.StringWriter, io.ReaderFrom +): """Implements buffering for an [io.Writer] object. # If an error occurs writing to a [Writer], no more data will be # accepted and all subsequent writes, and [Writer.flush], will return the error. @@ -637,7 +661,7 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io # [Writer.flush] method to guarantee all data has been forwarded to # the underlying [io.Writer].""" - var buf: List[Byte] + var buf: InlineList[UInt8, size] var bytes_written: Int var writer: W var err: Error @@ -645,16 +669,20 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io fn __init__( inout self, owned writer: W, - buf: List[Byte] = List[Byte](capacity=DEFAULT_BUF_SIZE), + buf: InlineList[UInt8, size] = InlineList[UInt8, size](), bytes_written: Int = 0, ): - self.buf = buf + self.buf = InlineList[UInt8, size]() + for element in buf: + self.buf.append(element[]) self.bytes_written = bytes_written self.writer = writer^ self.err = Error() fn __moveinit__(inout self, owned existing: Self): - self.buf = existing.buf^ + self.buf = InlineList[UInt8, size]() + for element in existing.buf: + self.buf.append(element[]) self.bytes_written = existing.bytes_written self.writer = existing.writer^ self.err = existing.err^ @@ -663,6 +691,10 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io """Returns the size of the underlying buffer in bytes.""" return len(self.buf) + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the internal data as a Span[UInt8].""" + return Span[UInt8, __lifetime_of(self)](array=self.buf._array) + fn reset(inout self, owned writer: W): """Discards any unflushed buffered data, clears any error, and resets b to write its output to w. @@ -680,7 +712,7 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io # return # if self.buf == nil: - # self.buf = make(List[Byte], DEFAULT_BUF_SIZE) + # self.buf = make(InlineList[UInt8, io.BUFFER_SIZE], io.BUFFER_SIZE) self.err = Error() self.bytes_written = 0 @@ -696,22 +728,28 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io return err var bytes_written: Int = 0 - bytes_written, err = self.writer.write(self.buf[0 : self.bytes_written]) + bytes_written, err = self.writer.write(self.as_bytes_slice()[0 : self.bytes_written]) # If the write was short, set a short write error and try to shift up the remaining bytes. if bytes_written < self.bytes_written and not err: - err = Error(io.ERR_SHORT_WRITE) + err = Error(str(io.ERR_SHORT_WRITE)) if err: if bytes_written > 0 and bytes_written < self.bytes_written: - _ = copy(self.buf, self.buf[bytes_written : self.bytes_written]) + # TODO: Temp copying of elements until I figure out a better pattern or slice refs are added + var temp = self.as_bytes_slice()[bytes_written : self.bytes_written] + for i in range(len(temp)): + if i > len(temp): + self.buf[i] = temp[i] + else: + self.buf.append(temp[i]) self.bytes_written -= bytes_written self.err = err return err # Reset the buffer - self.buf = List[Byte](capacity=self.buf.capacity) + self.buf = InlineList[UInt8, size]() self.bytes_written = 0 return err @@ -719,17 +757,6 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io """Returns how many bytes are unused in the buffer.""" return self.buf.capacity - len(self.buf) - fn available_buffer(self) raises -> List[Byte]: - """Returns an empty buffer with self.available() capacity. - This buffer is intended to be appended to and - passed to an immediately succeeding [Writer.write] call. - The buffer is only valid until the next write operation on self. - - Returns: - An empty buffer with self.available() capacity. - """ - return self.buf[self.bytes_written :][:0] - fn buffered(self) -> Int: """Returns the number of bytes that have been written into the current buffer. @@ -738,7 +765,7 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io """ return self.bytes_written - fn write(inout self, src: List[Byte]) -> (Int, Error): + fn write(inout self, src: Span[UInt8]) -> (Int, Error): """Writes the contents of src into the buffer. It returns the number of bytes written. If nn < len(src), it also returns an error explaining @@ -751,7 +778,7 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io The number of bytes written. """ var total_bytes_written: Int = 0 - var src_copy = src + var src_copy = src # TODO: Make a copy, maybe try a non owning Span var err = Error() while len(src_copy) > self.available() and not self.err: var bytes_written: Int = 0 @@ -761,7 +788,14 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io bytes_written, err = self.writer.write(src_copy) self.err = err else: - bytes_written = copy(self.buf, src_copy, self.bytes_written) + # TODO: Temp copying of elements until I figure out a better pattern or slice refs are added + for i in range(len(src_copy)): + if i + self.bytes_written > len(src_copy): + self.buf[i + self.bytes_written] = src_copy[i] + else: + self.buf.append(src_copy[i]) + bytes_written += 1 + self.bytes_written += bytes_written _ = self.flush() @@ -771,12 +805,19 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io if self.err: return total_bytes_written, self.err - var n = copy(self.buf, src_copy, self.bytes_written) + # TODO: Temp copying of elements until I figure out a better pattern or slice refs are added + var n = 0 + for i in range(len(src_copy)): + if i + self.bytes_written > len(src_copy): + self.buf[i + self.bytes_written] = src_copy[i] + else: + self.buf.append(src_copy[i]) + n += 1 self.bytes_written += n total_bytes_written += n return total_bytes_written, err - fn write_byte(inout self, src: Int8) -> (Int, Error): + fn write_byte(inout self, src: UInt8) -> (Int, Error): """Writes a single byte to the internal buffer. Args: @@ -818,7 +859,7 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io # # Can only happen if buffer is silly small. # return self.write_posriteString(string(r)) - # size = utf8.EncodeRune(self.buf[self.bytes_written:], r) + # size = utf8.EncodeRune(self.as_bytes_slice()[self.bytes_written:], r) # self.bytes_written += size # return size, nil @@ -834,9 +875,9 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io Returns: The number of bytes written. """ - return self.write(src.as_bytes()) + return self.write(src.as_bytes_slice()) - fn read_from[R: io.Reader](inout self, inout reader: R) -> (Int64, Error): + fn read_from[R: io.Reader](inout self, inout reader: R) -> (Int, Error): """Implements [io.ReaderFrom]. If the underlying writer supports the read_from method, this calls the underlying read_from. If there is buffered data and an underlying read_from, this fills @@ -849,10 +890,10 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io The number of bytes read. """ if self.err: - return Int64(0), self.err + return 0, self.err var bytes_read: Int = 0 - var total_bytes_written: Int64 = 0 + var total_bytes_written: Int = 0 var err = Error() while True: if self.available() == 0: @@ -862,27 +903,23 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io var nr = 0 while nr < MAX_CONSECUTIVE_EMPTY_READS: - # TODO: should really be using a slice that returns refs and not a copy. - # Read into remaining unused space in the buffer. We need to reserve capacity for the slice otherwise read will never hit EOF. - var sl = self.buf[self.bytes_written : len(self.buf)] - sl.reserve(self.buf.capacity) - bytes_read, err = reader.read(sl) - if bytes_read > 0: - bytes_read = copy(self.buf, sl, self.bytes_written) + # Read into remaining unused space in the buffer. + var buf = self.as_bytes_slice()[self.bytes_written : len(self.buf)] + bytes_read, err = reader._read(buf, self.bytes_written - len(self.buf)) if bytes_read != 0 or err: break nr += 1 if nr == MAX_CONSECUTIVE_EMPTY_READS: - return Int64(bytes_read), Error(io.ERR_NO_PROGRESS) + return bytes_read, io.ERR_NO_PROGRESS self.bytes_written += bytes_read - total_bytes_written += Int64(bytes_read) + total_bytes_written += bytes_read if err: break - if err and str(err) == io.EOF: + if err and str(err) == str(io.EOF): # If we filled the buffer exactly, flush preemptively. if self.available() == 0: err = self.flush() @@ -892,7 +929,7 @@ struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io return total_bytes_written, Error() -fn new_writer_size[W: io.Writer](owned writer: W, size: Int) -> Writer[W]: +fn new_writer[W: io.Writer, size: Int = io.BUFFER_SIZE](owned writer: W) -> Writer[W, size]: """Returns a new [Writer] whose buffer has at least the specified size. If the argument io.Writer is already a [Writer] with large enough size, it returns the underlying [Writer].""" @@ -901,24 +938,15 @@ fn new_writer_size[W: io.Writer](owned writer: W, size: Int) -> Writer[W]: # if ok and self.buf.capacity >= size: # return b - var buf_size = size - if buf_size <= 0: - buf_size = DEFAULT_BUF_SIZE + constrained[size > 0, "bufio: invalid buffer size. Must be greater than 0."]() - return Writer[W]( - buf=List[Byte](capacity=size), + return Writer[W, size]( + buf=InlineList[UInt8, size](), writer=writer^, bytes_written=0, ) -fn new_writer[W: io.Writer](owned writer: W) -> Writer[W]: - """Returns a new [Writer] whose buffer has the default size. - # If the argument io.Writer is already a [Writer] with large enough buffer size, - # it returns the underlying [Writer].""" - return new_writer_size[W](writer^, DEFAULT_BUF_SIZE) - - # buffered input and output struct ReadWriter[R: io.Reader, W: io.Writer](): """ReadWriter stores pointers to a [Reader] and a [Writer]. diff --git a/external/gojo/bufio/scan.mojo b/external/gojo/bufio/scan.mojo index 28489fc..e538d4d 100644 --- a/external/gojo/bufio/scan.mojo +++ b/external/gojo/bufio/scan.mojo @@ -1,15 +1,13 @@ -import math -from collections import Optional import ..io -from ..builtins import copy, panic, Error -from ..builtins.bytes import Byte, index_byte +from ..builtins import copy, panic +from ..builtins.bytes import index_byte from .bufio import MAX_CONSECUTIVE_EMPTY_READS alias MAX_INT: Int = 2147483647 -struct Scanner[R: io.Reader](): +struct Scanner[R: io.Reader, split: SplitFunction = scan_lines](): # The function to split the tokens. """Scanner provides a convenient Interface for reading data such as a file of newline-delimited lines of text. Successive calls to the [Scanner.Scan] method will step through the 'tokens' of a file, skipping @@ -27,10 +25,9 @@ struct Scanner[R: io.Reader](): on a reader, should use [bufio.Reader] instead.""" var reader: R # The reader provided by the client. - var split: SplitFunction # The function to split the tokens. var max_token_size: Int # Maximum size of a token; modified by tests. - var token: List[Byte] # Last token returned by split. - var buf: List[Byte] # buffer used as argument to split. + var token: List[UInt8] # Last token returned by split. + var buf: List[UInt8] # buffer used as argument to split. var start: Int # First non-processed byte in buf. var end: Int # End of data in buf. var empties: Int # Count of successive empty tokens. @@ -41,10 +38,9 @@ struct Scanner[R: io.Reader](): fn __init__( inout self, owned reader: R, - split: SplitFunction = scan_lines, max_token_size: Int = MAX_SCAN_TOKEN_SIZE, - token: List[Byte] = List[Byte](capacity=io.BUFFER_SIZE), - buf: List[Byte] = List[Byte](capacity=io.BUFFER_SIZE), + token: List[UInt8] = List[UInt8](capacity=io.BUFFER_SIZE), + buf: List[UInt8] = List[UInt8](capacity=io.BUFFER_SIZE), start: Int = 0, end: Int = 0, empties: Int = 0, @@ -52,7 +48,6 @@ struct Scanner[R: io.Reader](): done: Bool = False, ): self.reader = reader^ - self.split = split self.max_token_size = max_token_size self.token = token self.buf = buf @@ -63,7 +58,25 @@ struct Scanner[R: io.Reader](): self.done = done self.err = Error() - fn current_token_as_bytes(self) -> List[Byte]: + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the internal data as a Span[UInt8].""" + return Span[UInt8, __lifetime_of(self)](self.buf) + + fn current_token_as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the most recent token generated by a call to [Scanner.Scan]. + The underlying array may point to data that will be overwritten + by a subsequent call to Scan. It does no allocation. + """ + return Span[UInt8, __lifetime_of(self)](self.token) + + fn current_token_as_string_slice(ref [_]self) -> StringSlice[__lifetime_of(self)]: + """Returns the most recent token generated by a call to [Scanner.Scan]. + The underlying array may point to data that will be overwritten + by a subsequent call to Scan. It does no allocation. + """ + return StringSlice[__lifetime_of(self)](unsafe_from_utf8_ptr=self.token.unsafe_ptr(), len=len(self.token)) + + fn current_token_as_bytes(self) -> List[UInt8]: """Returns the most recent token generated by a call to [Scanner.Scan]. The underlying array may point to data that will be overwritten by a subsequent call to Scan. It does no allocation. @@ -73,9 +86,9 @@ struct Scanner[R: io.Reader](): fn current_token(self) -> String: """Returns the most recent token generated by a call to [Scanner.Scan] as a newly allocated string holding its bytes.""" - return String(self.token) + return self.current_token_as_string_slice() - fn scan(inout self) raises -> Bool: + fn scan(inout self) -> Bool: """Advances the [Scanner] to the next token, which will then be available through the [Scanner.current_token_as_bytes] or [Scanner.current_token] method. It returns False when there are no more tokens, either by reaching the end of the input or an error. @@ -96,14 +109,14 @@ struct Scanner[R: io.Reader](): # a chance to recover any remaining, possibly empty token. if (self.end > self.start) or self.err: var advance: Int - var token = List[Byte](capacity=io.BUFFER_SIZE) + var token = List[UInt8](capacity=io.BUFFER_SIZE) var err = Error() var at_eof = False if self.err: at_eof = True - advance, token, err = self.split(self.buf[self.start : self.end], at_eof) + advance, token, err = split(self.as_bytes_slice()[self.start : self.end], at_eof) if err: - if str(err) == ERR_FINAL_TOKEN: + if str(err) == str(ERR_FINAL_TOKEN): self.token = token self.done = True # When token is not nil, it means the scanning stops @@ -141,7 +154,7 @@ struct Scanner[R: io.Reader](): # First, shift data to beginning of buffer if there's lots of empty space # or space is needed. if self.start > 0 and (self.end == len(self.buf) or self.start > int(len(self.buf) / 2)): - _ = copy(self.buf, self.buf[self.start : self.end]) + _ = copy(self.buf, self.as_bytes_slice()[self.start : self.end]) self.end -= self.start self.start = 0 @@ -149,16 +162,16 @@ struct Scanner[R: io.Reader](): if self.end == len(self.buf): # Guarantee no overflow in the multiplication below. if len(self.buf) >= self.max_token_size or len(self.buf) > int(MAX_INT / 2): - self.set_err(Error(ERR_TOO_LONG)) + self.set_err((ERR_TOO_LONG)) return False var new_size = len(self.buf) * 2 if new_size == 0: new_size = START_BUF_SIZE - # Make a new List[Byte] buffer and copy the elements in - new_size = math.min(new_size, self.max_token_size) - var new_buf = List[Byte](capacity=new_size) + # Make a new List[UInt8] buffer and copy the elements in + new_size = min(new_size, self.max_token_size) + var new_buf = List[UInt8](capacity=new_size) _ = copy(new_buf, self.buf[self.start : self.end]) self.buf = new_buf self.end -= self.start @@ -169,15 +182,13 @@ struct Scanner[R: io.Reader](): # be extra careful: Scanner is for safe, simple jobs. var loop = 0 while True: + var buf = self.as_bytes_slice()[self.end :] + # Catch any reader errors and set the internal error field to that err instead of bubbling it up. var bytes_read: Int - var sl = self.buf[self.end : len(self.buf)] var err: Error - - # Catch any reader errors and set the internal error field to that err instead of bubbling it up. - bytes_read, err = self.reader.read(sl) - _ = copy(self.buf, sl, self.end) - if bytes_read < 0 or len(self.buf) - self.end < bytes_read: - self.set_err(Error(ERR_BAD_READ_COUNT)) + bytes_read, err = self.reader._read(buf, len(buf)) + if bytes_read < 0 or len(buf) - self.end < bytes_read: + self.set_err(ERR_BAD_READ_COUNT) break self.end += bytes_read @@ -191,7 +202,7 @@ struct Scanner[R: io.Reader](): loop += 1 if loop > MAX_CONSECUTIVE_EMPTY_READS: - self.set_err(Error(io.ERR_NO_PROGRESS)) + self.set_err(io.ERR_NO_PROGRESS) break fn set_err(inout self, err: Error): @@ -201,8 +212,8 @@ struct Scanner[R: io.Reader](): err: The error to set. """ if self.err: - var value = String(self.err) - if value == "" or value == io.EOF: + var value = str(self.err) + if value == "" or value == str(io.EOF): self.err = err else: self.err = err @@ -217,50 +228,16 @@ struct Scanner[R: io.Reader](): True if the advance was legal, False otherwise. """ if n < 0: - self.set_err(Error(ERR_NEGATIVE_ADVANCE)) + self.set_err(ERR_NEGATIVE_ADVANCE) return False if n > self.end - self.start: - self.set_err(Error(ERR_ADVANCE_TOO_FAR)) + self.set_err(ERR_ADVANCE_TOO_FAR) return False self.start += n return True - fn buffer(inout self, buf: List[Byte], max: Int) raises: - """Sets the initial buffer to use when scanning - and the maximum size of buffer that may be allocated during scanning. - The maximum token size must be less than the larger of max and cap(buf). - If max <= cap(buf), [Scanner.Scan] will use this buffer only and do no allocation. - - By default, [Scanner.Scan] uses an Internal buffer and sets the - maximum token size to [MAX_SCAN_TOKEN_SIZE]. - - buffer raises an Error if it is called after scanning has started. - - Args: - buf: The buffer to use when scanning. - max: The maximum size of buffer that may be allocated during scanning. - - Raises: - Error: If called after scanning has started. - """ - if self.scan_called: - raise Error("buffer called after Scan") - - # self.buf = buf[0:buf.capacity()] - self.max_token_size = max - - # # split sets the split function for the [Scanner]. - # # The default split function is [scan_lines]. - # # - # # split panics if it is called after scanning has started. - # fn split(inout self, split_function: SplitFunction) raises: - # if self.scan_called: - # raise Error("split called after Scan") - - # self.split = split_function - # SplitFunction is the signature of the split function used to tokenize the # input. The arguments are an initial substring of the remaining unprocessed @@ -286,13 +263,18 @@ struct Scanner[R: io.Reader](): # The function is never called with an empty data slice unless at_eof # is True. If at_eof is True, however, data may be non-empty and, # as always, holds unprocessed text. -alias SplitFunction = fn (data: List[Byte], at_eof: Bool) -> (Int, List[Byte], Error) +alias SplitFunction = fn (data: Span[UInt8], at_eof: Bool) -> ( + Int, + List[UInt8], + Error, +) -# # Errors returned by Scanner. +# Errors returned by Scanner. alias ERR_TOO_LONG = Error("bufio.Scanner: token too long") alias ERR_NEGATIVE_ADVANCE = Error("bufio.Scanner: SplitFunction returns negative advance count") alias ERR_ADVANCE_TOO_FAR = Error("bufio.Scanner: SplitFunction returns advance count beyond input") alias ERR_BAD_READ_COUNT = Error("bufio.Scanner: Read returned impossible count") + # ERR_FINAL_TOKEN is a special sentinel error value. It is Intended to be # returned by a split function to indicate that the scanning should stop # with no error. If the token being delivered with this error is not nil, @@ -321,55 +303,50 @@ fn new_scanner[R: io.Reader](owned reader: R) -> Scanner[R]: ###### split functions ###### -fn scan_bytes(data: List[Byte], at_eof: Bool) -> (Int, List[Byte], Error): - """Split function for a [Scanner] that returns each byte as a token.""" - if at_eof and data.capacity == 0: - return 0, List[Byte](), Error() +fn scan_bytes(data: Span[UInt8], at_eof: Bool) -> (Int, List[UInt8], Error): + """Returns each byte as a token. - return 1, data[0:1], Error() - - -# var errorRune = List[Byte](string(utf8.RuneError)) + Args: + data: The data to split. + at_eof: Whether the data is at the end of the file. -# # ScanRunes is a split function for a [Scanner] that returns each -# # UTF-8-encoded rune as a token. The sequence of runes returned is -# # equivalent to that from a range loop over the input as a string, which -# # means that erroneous UTF-8 encodings translate to U+FFFD = "\xef\xbf\xbd". -# # Because of the Scan Interface, this makes it impossible for the client to -# # distinguish correctly encoded replacement runes from encoding errors. -# fn ScanRunes(data List[Byte], at_eof Bool) (advance Int, token List[Byte], err error): -# if at_eof and data.capacity == 0: -# return 0, nil, nil + Returns: + The number of bytes to advance the input, token in bytes, and an error if one occurred. + """ + if at_eof and len(data) == 0: + return 0, List[UInt8](), Error() + return 1, List[UInt8](data[0:1]), Error() -# # Fast path 1: ASCII. -# if data[0] < utf8.RuneSelf: -# return 1, data[0:1], nil +# TODO: Fix this and uncomment it. +# fn scan_runes(data: Span[UInt8], at_eof: Bool) -> (Int, List[UInt8], Error): +# """Returns each UTF-8 encoded rune as a token. -# # Fast path 2: Correct UTF-8 decode without error. -# _, width := utf8.DecodeRune(data) -# if width > 1: -# # It's a valid encoding. Width cannot be one for a correctly encoded -# # non-ASCII rune. -# return width, data[0:width], nil +# Args: +# data: The data to split. +# at_eof: Whether the data is at the end of the file. +# Returns: +# The number of bytes to advance the input, token in bytes, and an error if one occurred. +# """ +# if at_eof and len(data) == 0: +# return 0, List[UInt8](), Error() -# # We know it's an error: we have width==1 and implicitly r==utf8.RuneError. -# # Is the error because there wasn't a full rune to be decoded? -# # FullRune distinguishes correctly between erroneous and incomplete encodings. -# if !at_eof and !utf8.FullRune(data): -# # Incomplete; get more bytes. -# return 0, nil, nil +# # Number of bytes of the current character +# var lhs = (SIMD[size=1].load(DTypePointer[DType.uint8](data.unsafe_ptr())) >> 7 == 0 * 1).cast[DType.uint8]() +# var rhs = countl_zero(~SIMD[size=1].load(DTypePointer[DType.uint8](data.unsafe_ptr()))) +# var char_length = int(lhs + rhs) +# # Copy N bytes into new pointer and construct List. +# var sp = UnsafePointer[UInt8].alloc(char_length) +# memcpy(sp, data.unsafe_ptr(), char_length) +# var result = List[UInt8](unsafe_pointer=sp, size=char_length, capacity=char_length) -# # We have a real UTF-8 encoding error. Return a properly encoded error rune -# # but advance only one byte. This matches the behavior of a range loop over -# # an incorrectly encoded string. -# return 1, errorRune, nil +# return char_length, result, Error() -fn drop_carriage_return(data: List[Byte]) -> List[Byte]: +fn drop_carriage_return(data: Span[UInt8]) -> List[UInt8]: """Drops a terminal \r from the data. Args: @@ -379,16 +356,14 @@ fn drop_carriage_return(data: List[Byte]) -> List[Byte]: The stripped data. """ # In the case of a \r ending without a \n, indexing on -1 doesn't work as it finds a null terminator instead of \r. - if data.capacity > 0 and data[data.capacity - 1] == ord("\r"): - return data[0 : data.capacity - 1] + if len(data) > 0 and data[-1] == ord("\r"): + return data[:-1] return data -# TODO: Doing modification of token and err in these split functions, so we don't have to return any memory only types as part of the return tuple. -fn scan_lines(data: List[Byte], at_eof: Bool) -> (Int, List[Byte], Error): - """Split function for a [Scanner] that returns each line of - text, stripped of any trailing end-of-line marker. The returned line may +fn scan_lines(data: Span[UInt8], at_eof: Bool) -> (Int, List[UInt8], Error): + """Returns each line of text, stripped of any trailing end-of-line marker. The returned line may be empty. The end-of-line marker is one optional carriage return followed by one mandatory newline. The last non-empty line of input will be returned even if it has no newline. @@ -396,11 +371,12 @@ fn scan_lines(data: List[Byte], at_eof: Bool) -> (Int, List[Byte], Error): Args: data: The data to split. at_eof: Whether the data is at the end of the file. + Returns: The number of bytes to advance the input. """ - if at_eof and data.capacity == 0: - return 0, List[Byte](), Error() + if at_eof and len(data) == 0: + return 0, List[UInt8](), Error() var i = index_byte(data, ord("\n")) if i >= 0: @@ -409,13 +385,13 @@ fn scan_lines(data: List[Byte], at_eof: Bool) -> (Int, List[Byte], Error): # If we're at EOF, we have a final, non-terminated line. Return it. # if at_eof: - return data.capacity, drop_carriage_return(data), Error() + return len(data), drop_carriage_return(data), Error() # Request more data. # return 0 -fn is_space(r: Int8) -> Bool: +fn is_space(r: UInt8) -> Bool: alias ALL_WHITESPACES: String = " \t\n\r\x0b\f" if chr(int(r)) in ALL_WHITESPACES: return True @@ -423,16 +399,21 @@ fn is_space(r: Int8) -> Bool: # TODO: Handle runes and utf8 decoding. For now, just assuming single byte length. -fn scan_words(data: List[Byte], at_eof: Bool) -> (Int, List[Byte], Error): - """Split function for a [Scanner] that returns each - space-separated word of text, with surrounding spaces deleted. It will - never return an empty string. The definition of space is set by - unicode.IsSpace. +fn scan_words(data: Span[UInt8], at_eof: Bool) -> (Int, List[UInt8], Error): + """Returns each space-separated word of text, with surrounding spaces deleted. It will + never return an empty string. + + Args: + data: The data to split. + at_eof: Whether the data is at the end of the file. + + Returns: + The number of bytes to advance the input, token in bytes, and an error if one occurred. """ # Skip leading spaces. var start = 0 var width = 0 - while start < data.capacity: + while start < len(data): width = len(data[0]) if not is_space(data[0]): break @@ -443,16 +424,16 @@ fn scan_words(data: List[Byte], at_eof: Bool) -> (Int, List[Byte], Error): var i = 0 width = 0 start = 0 - while i < data.capacity: + while i < len(data): width = len(data[i]) if is_space(data[i]): - return i + width, data[start:i], Error() + return i + width, List[UInt8](data[start:i]), Error() i += width # If we're at EOF, we have a final, non-empty, non-terminated word. Return it. - if at_eof and data.capacity > start: - return data.capacity, data[start:], Error() + if at_eof and len(data) > start: + return len(data), List[UInt8](data[start:]), Error() # Request more data. - return start, List[Byte](), Error() + return start, List[UInt8](), Error() diff --git a/external/gojo/builtins/__init__.mojo b/external/gojo/builtins/__init__.mojo index 3d1e11a..1d2e998 100644 --- a/external/gojo/builtins/__init__.mojo +++ b/external/gojo/builtins/__init__.mojo @@ -1,6 +1,3 @@ -from .bytes import Byte, index_byte, has_suffix, has_prefix, to_string -from .list import equals -from .attributes import cap, copy +from .bytes import index_byte, has_suffix, has_prefix, to_string +from .attributes import copy from .errors import exit, panic - -alias Rune = Int32 diff --git a/external/gojo/builtins/attributes.mojo b/external/gojo/builtins/attributes.mojo index 1787048..5202fff 100644 --- a/external/gojo/builtins/attributes.mojo +++ b/external/gojo/builtins/attributes.mojo @@ -1,3 +1,6 @@ +from collections import InlineList + + fn copy[T: CollectionElement](inout target: List[T], source: List[T], start: Int = 0) -> Int: """Copies the contents of source into target at the same index. Returns the number of bytes copied. Added a start parameter to specify the index to start copying into. @@ -22,10 +25,123 @@ fn copy[T: CollectionElement](inout target: List[T], source: List[T], start: Int return count -fn cap[T: CollectionElement](iterable: List[T]) -> Int: - """Returns the capacity of the List. +# fn copy[T: CollectionElement](inout target_span: Span[T], source_span: Span[T], start: Int = 0) -> Int: +# """Copies the contents of source into target at the same index. Returns the number of bytes copied. +# Added a start parameter to specify the index to start copying into. + +# Args: +# target_span: The buffer to copy into. +# source_span: The buffer to copy from. +# start: The index to start copying into. + +# Returns: +# The number of bytes copied. +# """ +# var count = 0 + +# for i in range(len(source_span)): +# target_span[i + start] = source_span[i] +# count += 1 + +# target_span._len += count +# return count + + +# fn copy[T: CollectionElementNew](inout target_span: Span[T], source: InlineList[T], start: Int = 0) -> Int: +# """Copies the contents of source into target at the same index. Returns the number of bytes copied. +# Added a start parameter to specify the index to start copying into. + +# Args: +# target_span: The buffer to copy into. +# source: The buffer to copy from. +# start: The index to start copying into. + +# Returns: +# The number of bytes copied. +# """ +# var count = 0 + +# for i in range(len(source)): +# target_span[i + start] = source[i] +# count += 1 + +# target_span._len += count +# return count + + +# fn copy[T: CollectionElementNew, T2: CollectionElement](inout list: InlineList[T], source: Span[T2], start: Int = 0) -> Int: +# """Copies the contents of source into target at the same index. Returns the number of bytes copied. +# Added a start parameter to specify the index to start copying into. + +# Args: +# list: The buffer to copy into. +# source: The buffer to copy from. +# start: The index to start copying into. + +# Returns: +# The number of bytes copied. +# """ +# var count = 0 + +# for i in range(len(source)): +# if i + start > len(list): +# list[i + start] = source[i] +# else: +# list.append(source[i]) +# count += 1 + +# return count + + +fn copy(target: UnsafePointer[UInt8], source: UnsafePointer[UInt8], source_length: Int, start: Int = 0) -> Int: + """Copies the contents of source into target at the same index. Returns the number of bytes copied. + Added a start parameter to specify the index to start copying into. + + Args: + target: The buffer to copy into. + source: The buffer to copy from. + source_length: The length of the source buffer. + start: The index to start copying into. + + Returns: + The number of bytes copied. + """ + var count = 0 + + for i in range(source_length): + target[i + start] = source[i] + count += 1 + + return count + + +fn copy( + inout target: List[UInt8], + source: DTypePointer[DType.uint8], + source_start: Int, + source_end: Int, + target_start: Int = 0, +) -> Int: + """Copies the contents of source into target at the same index. Returns the number of bytes copied. + Added a start parameter to specify the index to start copying into. Args: - iterable: The List to get the capacity of. + target: The buffer to copy into. + source: The buffer to copy from. + source_start: The index to start copying from. + source_end: The index to stop copying at. + target_start: The index to start copying into. + + Returns: + The number of bytes copied. """ - return iterable.capacity + var count = 0 + + for i in range(source_start, source_end): + if i + target_start > len(target): + target[i + target_start] = source[i] + else: + target.append(source[i]) + count += 1 + + return count diff --git a/external/gojo/builtins/bytes.mojo b/external/gojo/builtins/bytes.mojo index 0504d16..ff0dcb5 100644 --- a/external/gojo/builtins/bytes.mojo +++ b/external/gojo/builtins/bytes.mojo @@ -1,7 +1,13 @@ -from .list import equals +alias Byte = UInt8 -alias Byte = Int8 +fn equals(left: List[UInt8], right: List[UInt8]) -> Bool: + if len(left) != len(right): + return False + for i in range(len(left)): + if left[i] != right[i]: + return False + return True fn has_prefix(bytes: List[Byte], prefix: List[Byte]) -> Bool: @@ -44,7 +50,41 @@ fn index_byte(bytes: List[Byte], delim: Byte) -> Int: Returns: The index of the first occurrence of the byte delim. """ - var i = 0 + for i in range(len(bytes)): + if bytes[i] == delim: + return i + + return -1 + + +fn index_byte(bytes: DTypePointer[DType.uint8], size: Int, delim: Byte) -> Int: + """Return the index of the first occurrence of the byte delim. + + Args: + bytes: The DTypePointer[DType.int8] struct to search. + size: The size of the bytes pointer. + delim: The byte to search for. + + Returns: + The index of the first occurrence of the byte delim. + """ + for i in range(size): + if UInt8(bytes[i]) == delim: + return i + + return -1 + + +fn index_byte(bytes: Span[UInt8], delim: Byte) -> Int: + """Return the index of the first occurrence of the byte delim. + + Args: + bytes: The Span to search. + delim: The byte to search for. + + Returns: + The index of the first occurrence of the byte delim. + """ for i in range(len(bytes)): if bytes[i] == delim: return i diff --git a/external/gojo/builtins/errors.mojo b/external/gojo/builtins/errors.mojo index 1c1a7ba..0001ae7 100644 --- a/external/gojo/builtins/errors.mojo +++ b/external/gojo/builtins/errors.mojo @@ -8,5 +8,5 @@ fn panic[T: Stringable](message: T, code: Int = 1): message: The message to panic with. code: The exit code to panic with. """ - print("panic:", message) + print("panic:", str(message)) exit(code) diff --git a/external/gojo/builtins/list.mojo b/external/gojo/builtins/list.mojo deleted file mode 100644 index cb32504..0000000 --- a/external/gojo/builtins/list.mojo +++ /dev/null @@ -1,133 +0,0 @@ -fn equals(left: List[Int8], right: List[Int8]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[UInt8], right: List[UInt8]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[Int16], right: List[Int16]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[UInt16], right: List[UInt16]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[Int32], right: List[Int32]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[UInt32], right: List[UInt32]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[Int64], right: List[Int64]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[UInt64], right: List[UInt64]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[Int], right: List[Int]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[Float16], right: List[Float16]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[Float32], right: List[Float32]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[Float64], right: List[Float64]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[String], right: List[String]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[StringLiteral], right: List[StringLiteral]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True - - -fn equals(left: List[Bool], right: List[Bool]) -> Bool: - if len(left) != len(right): - return False - for i in range(len(left)): - if left[i] != right[i]: - return False - return True diff --git a/external/gojo/bytes/buffer.mojo b/external/gojo/bytes/buffer.mojo index 13f2df9..89c16ac 100644 --- a/external/gojo/bytes/buffer.mojo +++ b/external/gojo/bytes/buffer.mojo @@ -1,15 +1,6 @@ -from ..io import ( - Reader, - Writer, - ReadWriter, - ByteReader, - ByteWriter, - WriterTo, - StringWriter, - ReaderFrom, - BUFFER_SIZE, -) -from ..builtins import cap, copy, Byte, panic, index_byte +import ..io +from ..builtins import copy, panic, index_byte +from algorithm.memory import parallel_memcpy alias Rune = Int32 @@ -42,323 +33,156 @@ alias MIN_READ: Int = 512 # ERR_TOO_LARGE is passed to panic if memory cannot be allocated to store data in a buffer. alias ERR_TOO_LARGE = "buffer.Buffer: too large" alias ERR_NEGATIVE_READ = "buffer.Buffer: reader returned negative count from read" -alias ERR_SHORT_WRITE = "short write" +alias ERR_SHORTwrite = "short write" -@value struct Buffer( - Copyable, Stringable, Sized, - ReadWriter, - StringWriter, - ByteReader, - ByteWriter, - WriterTo, - ReaderFrom, + io.Reader, + io.Writer, + io.StringWriter, + io.ByteWriter, + io.ByteReader, ): - """A Buffer is a variable-sized buffer of bytes with [Buffer.read] and [Buffer.write] methods. - The zero value for Buffer is an empty buffer ready to use. - """ - - var buf: List[Byte] # contents are the bytes buf[off : len(buf)] - var off: Int # read at &buf[off], write at &buf[len(buf)] + var _data: UnsafePointer[UInt8] # contents are the bytes buf[off : len(buf)] + var _size: Int + var _capacity: Int + var offset: Int # read at &buf[off], write at &buf[len(buf)] var last_read: ReadOp # last read operation, so that unread* can work correctly. - fn __init__(inout self, owned buf: List[Byte]): - self.buf = buf - self.off = 0 + fn __init__(inout self, capacity: Int = io.BUFFER_SIZE): + self._capacity = capacity + self._size = 0 + self._data = UnsafePointer[UInt8]().alloc(capacity) + self.offset = 0 self.last_read = OP_INVALID - fn bytes(self) -> List[Byte]: - """Returns a slice of length self.buf.capacity holding the unread portion of the buffer. - The slice is valid for use only until the next buffer modification (that is, - only until the next call to a method like [Buffer.read], [Buffer.write], [Buffer.reset], or [Buffer.truncate]). - The slice aliases the buffer content at least until the next buffer modification, - so immediate changes to the slice will affect the result of future reads. - """ - return self.buf[self.off : len(self.buf)] - - # fn available_buffer(self) raises -> List[Byte]: - # """Returns an empty buffer with self.Available() capacity. - # This buffer is intended to be appended to and - # passed to an immediately succeeding [Buffer.write] call. - # The buffer is only valid until the next write operation on self. - # """ - # return self.buf[len(self.buf) :] - - fn __str__(self) -> String: - """Returns the contents of the unread portion of the buffer - as a string. If the [Buffer] is a nil pointer, it returns "". - - To build strings more efficiently, see the strings.Builder type. + fn __init__(inout self, owned buf: List[UInt8]): + self._capacity = buf.capacity + self._size = buf.size + self._data = buf.steal_data() + self.offset = 0 + self.last_read = OP_INVALID - Creates a copy of the readable buffer and returns it as a string. - """ - var valid_bytes = self.buf[self.off : len(self.buf)] + fn __init__(inout self, owned data: UnsafePointer[UInt8], capacity: Int, size: Int): + self._capacity = capacity + self._size = size + self._data = data + self.offset = 0 + self.last_read = OP_INVALID - valid_bytes.append(0) - return String(valid_bytes) - - fn empty(self) -> Bool: - """Reports whether the unread portion of the buffer is empty.""" - return len(self.buf) <= self.off + fn __moveinit__(inout self, owned other: Self): + self._data = other._data + self._size = other._size + self._capacity = other._capacity + self.offset = other.offset + self.last_read = other.last_read + other._data = UnsafePointer[UInt8]() + other._size = 0 + other._capacity = 0 + other.offset = 0 + other.last_read = OP_INVALID + + fn __del__(owned self): + if self._data: + self._data.free() fn __len__(self) -> Int: - """Returns the number of bytes of the unread portion of the buffer; - self.buf.capacity == len(self.List[Byte]()).""" - return len(self.buf) - self.off + """Returns the number of bytes of the unread portion of the buffer. + self._size - self.offset.""" + return self._size - self.offset - fn cap(self) -> Int: - """Cap returns the capacity of the buffer's underlying byte slice, that is, the - total space allocated for the buffer's data.""" - return cap(self.buf) + fn bytes_ptr(self) -> UnsafePointer[UInt8]: + """Returns a pointer holding the unread portion of the buffer.""" + return self._data.offset(self.offset) - fn available(self) -> Int: - """Returns how many bytes are unused in the buffer.""" - return self.buf.capacity - len(self.buf) + fn bytes(self) -> List[UInt8]: + """Returns a list of bytes holding a copy of the unread portion of the buffer.""" + var copy = UnsafePointer[UInt8]().alloc(self._size) + memcpy(copy, self._data.offset(self.offset), self._size) + return List[UInt8](unsafe_pointer=copy, size=self._size - self.offset, capacity=self._size - self.offset) - fn truncate(inout self, position: Int) raises: - """Discards all but the first n unread bytes from the buffer - but continues to use the same allocated storage. - It panics if position is negative or greater than the length of the buffer. + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the internal data as a Span[UInt8].""" + return Span[UInt8, __lifetime_of(self)](unsafe_ptr=self._data, len=self._size) - Args: - position: The position to truncate the buffer to. + fn as_string_slice(self) -> StringSlice[__lifetime_of(self)]: """ - if position == 0: - self.reset() - return - - self.last_read = OP_INVALID - if position < 0 or position > self.buf.capacity: - raise Error("buffer.Buffer: truncation out of range") - - self.buf = self.buf[: self.off + position] + Return a StringSlice view of the data owned by the builder. - fn reset(inout self): - """Resets the buffer to be empty, - but it retains the underlying storage for use by future writes. - reset is the same as [buffer.truncate](0).""" - self.buf = List[Byte](capacity=self.buf.capacity) - self.off = 0 - self.last_read = OP_INVALID - - fn try_grow_by_reslice(inout self, n: Int) -> (Int, Bool): - """Inlineable version of grow for the fast-case where the - internal buffer only needs to be resliced. - It returns the index where bytes should be written and whether it succeeded.""" - var buffer_already_used = len(self.buf) - - if n <= self.buf.capacity - buffer_already_used: - # FIXME: It seems like reslicing in go can extend the length of the slice. Doens't work like that for my get slice impl. - # Instead, just add bytes of len(n) to the end of the buffer for now. - # self.buf = self.buf[: l + n] - self.buf.reserve(self.buf.capacity + n) - return buffer_already_used, True - - return 0, False - - fn grow(inout self, n: Int) -> Int: - """Grows the buffer to guarantee space for n more bytes. - It returns the index where bytes should be written. - If the buffer can't grow it will panic with ERR_TOO_LARGE.""" - var write_at: Int = len(self.buf) - # If buffer is empty, reset to recover space. - if write_at == 0 and self.off != 0: - self.reset() - - # Try to grow by means of a reslice. - var i: Int - var ok: Bool - i, ok = self.try_grow_by_reslice(n) - if ok: - return i - - # If buffer length is 0 and elements being added is less than small_buffer_size, resize the buffer and write from the beginning. - if self.buf.capacity == 0 and n <= SMALL_BUFFER_SIZE: - self.buf.reserve(SMALL_BUFFER_SIZE) - return 0 - - var c = cap(self.buf) - if Float64(n) <= c / 2 - write_at: - # We can slide things down instead of allocating a new - # slice. We only need m+n <= c to slide, but - # we instead var capacity get twice as large so we - # don't spend all our time copying. - _ = copy(self.buf, self.buf[self.off :]) - elif c > MAX_INT - c - n: - panic(ERR_TOO_LARGE) - # TODO: Commented out this branch because growing the slice here and then at the end is redundant? - # else: - # # Add self.off to account for self.buf[:self.off] being sliced off the front. - # # var sl = self.buf[self.off :] - # # self.buf = self.grow_slice(sl, self.off + n) - - # Restore self.off and len(self.buf). - self.off = 0 - # FIXME: It seems like reslicing in go can extend the length of the slice. Doens't work like that for my get slice impl. - # Instead, just add bytes of len(n) to the end of the buffer for now. - # self.buf = self.buf[: m + n] - self.buf.reserve(self.buf.capacity + n) - return write_at - - fn Grow(inout self, n: Int): - """Grows the buffer's capacity, if necessary, to guarantee space for - another n bytes. After grow(n), at least n bytes can be written to the - buffer without another allocation. - If n is negative, grow will panic. - If the buffer can't grow it will panic with [ERR_TOO_LARGE]. + Returns: + The string representation of the string builder. Returns an empty string if the string builder is empty. """ - if n < 0: - panic("buffer.Buffer.Grow: negative count") - - var m = self.grow(n) - self.buf = self.buf[:m] + return StringSlice[__lifetime_of(self)](unsafe_from_utf8_ptr=self._data, len=self._size) - fn write(inout self, src: List[Byte]) -> (Int, Error): - """Appends the contents of p to the buffer, growing the buffer as - needed. The return value n is the length of p; err is always nil. If the - buffer becomes too large, write will panic with [ERR_TOO_LARGE]. + fn _resize(inout self, capacity: Int) -> None: + """ + Resizes the string builder buffer. Args: - src: The bytes to write to the buffer. - - Returns: - The number of bytes written to the buffer. + capacity: The new capacity of the string builder buffer. """ - self.last_read = OP_INVALID - var write_at: Int - var ok: Bool - write_at, ok = self.try_grow_by_reslice(len(src)) - if not ok: - write_at = self.grow(len(src)) + var new_data = UnsafePointer[UInt8]().alloc(capacity) + memcpy(new_data, self._data, self._size) + self._data.free() + self._data = new_data + self._capacity = capacity + + return None + + fn _resize_if_needed(inout self, bytes_to_add: Int): + # TODO: Handle the case where new_capacity is greater than MAX_INT. It should panic. + if bytes_to_add > self._capacity - self._size: + var new_capacity = int(self._capacity * 2) + if new_capacity < self._capacity + bytes_to_add: + new_capacity = self._capacity + bytes_to_add + self._resize(new_capacity) - var bytes_written = copy(self.buf, src, write_at) - return bytes_written, Error() - - fn write_string(inout self, src: String) -> (Int, Error): - """Appends the contents of s to the buffer, growing the buffer as - needed. The return value n is the length of s; err is always nil. If the - buffer becomes too large, write_string will panic with [ERR_TOO_LARGE]. - - Args: - src: The bytes to write to the buffer. + fn __str__(self) -> String: + """ + Converts the string builder to a string. Returns: - The number of bytes written to the buffer. + The string representation of the string builder. Returns an empty + string if the string builder is empty. """ - # self.last_read = OP_INVALID - # var write_at: Int - # var ok: Bool - # write_at, ok = self.try_grow_by_reslice(len(src)) - # if not ok: - # m = self.grow(len(src)) - # var b = self.buf[m:] - return self.write(src.as_bytes()) - - fn read_from[R: Reader](inout self, inout reader: R) -> (Int64, Error): - """Reads data from r until EOF and appends it to the buffer, growing - the buffer as needed. The return value n is the number of bytes read. Any - error except io.EOF encountered during the read is also returned. If the - buffer becomes too large, read_from will panic with [ERR_TOO_LARGE]. + return self.as_string_slice() - Args: - reader: The reader to read from. + @deprecated("Buffer.render() has been deprecated. Use Buffer.as_string_slice() instead.") + fn render(self) -> StringSlice[__lifetime_of(self)]: + """ + Return a StringSlice view of the data owned by the builder. Returns: - The number of bytes read from the reader. + The string representation of the string builder. Returns an empty string if the string builder is empty. """ - self.last_read = OP_INVALID - var total_bytes_read: Int64 = 0 - while True: - _ = self.grow(MIN_READ) + return self.as_string_slice() - var bytes_read: Int - var err: Error - bytes_read, err = reader.read(self.buf) - if bytes_read < 0: - panic(ERR_NEGATIVE_READ) + fn write(inout self, src: Span[UInt8]) -> (Int, Error): + """ + Appends a byte Span to the builder buffer. - total_bytes_read += bytes_read + Args: + src: The byte array to append. + """ + self._resize_if_needed(len(src)) - var err_message = str(err) - if err_message != "": - if err_message == io.EOF: - return total_bytes_read, Error() + memcpy(self._data.offset(self._size), src._data, len(src)) + self._size += len(src) - return total_bytes_read, err + return len(src), Error() - fn grow_slice(self, inout b: List[Byte], n: Int) -> List[Byte]: - """Grows b by n, preserving the original content of self. - If the allocation fails, it panics with ERR_TOO_LARGE. + fn write_string(inout self, src: String) -> (Int, Error): """ - # TODO(http:#golang.org/issue/51462): We should rely on the append-make - # pattern so that the compiler can call runtime.growslice. For example: - # return append(b, make(bytes, n)...) - # This avoids unnecessary zero-ing of the first b.capacity bytes of the - # allocated slice, but this pattern causes b to escape onto the heap. - # - # Instead use the append-make pattern with a nil slice to ensure that - # we allocate buffers rounded up to the closest size class. - var c = b.capacity + n # ensure enough space for n elements - if c < 2 * cap(b): - # The growth rate has historically always been 2x. In the future, - # we could rely purely on append to determine the growth rate. - c = 2 * cap(b) - - var resized_buffer = List[Byte](capacity=c) - _ = copy(resized_buffer, b) - # var b2: List[Byte] = List[Byte]() - # b2._vector.reserve(c) - - # # var b2 = append(bytes(nil), make(bytes, c)...) - # _ = copy(b2, b) - # return b2[:b.capacity] - # b._vector.reserve(c) - return resized_buffer[: b.capacity] - - fn write_to[W: Writer](inout self, inout writer: W) -> (Int64, Error): - """Writes data to w until the buffer is drained or an error occurs. - The return value n is the number of bytes written; it always fits into an - Int, but it is int64 to match the io.WriterTo trait. Any error - encountered during the write is also returned. + Appends a string to the builder buffer. Args: - writer: The writer to write to. - - Returns: - The number of bytes written to the writer. + src: The string to append. """ - self.last_read = OP_INVALID - var bytes_to_write = len(self.buf) - var total_bytes_written: Int64 = 0 - - if bytes_to_write > 0: - # TODO: Replace usage of this intermeidate slice when normal slicing, once slice references work. - var sl = self.buf[self.off : bytes_to_write] - var bytes_written: Int - var err: Error - bytes_written, err = writer.write(sl) - if bytes_written > bytes_to_write: - panic("bytes.Buffer.write_to: invalid write count") - - self.off += bytes_written - total_bytes_written = Int64(bytes_written) - - var err_message = str(err) - if err_message != "": - return total_bytes_written, err - - # all bytes should have been written, by definition of write method in io.Writer - if bytes_written != bytes_to_write: - return total_bytes_written, Error(ERR_SHORT_WRITE) - - # Buffer is now empty; reset. - self.reset() - return total_bytes_written, Error() - - fn write_byte(inout self, byte: Byte) -> (Int, Error): + return self.write(src.as_bytes_slice()) + + fn write_byte(inout self, byte: UInt8) -> (Int, Error): """Appends the byte c to the buffer, growing the buffer as needed. The returned error is always nil, but is included to match [bufio.Writer]'s write_byte. If the buffer becomes too large, write_byte will panic with @@ -371,37 +195,28 @@ struct Buffer( The number of bytes written to the buffer. """ self.last_read = OP_INVALID - var write_at: Int - var ok: Bool - write_at, ok = self.try_grow_by_reslice(1) - if not ok: - write_at = self.grow(1) - - _ = copy(self.buf, List[Byte](byte), write_at) - return write_at, Error() - - # fn write_rune(inout self, r: Rune) -> Int: - # """Appends the UTF-8 encoding of Unicode code point r to the - # buffer, returning its length and an error, which is always nil but is - # included to match [bufio.Writer]'s write_rune. The buffer is grown as needed; - # if it becomes too large, write_rune will panic with [ERR_TOO_LARGE]. - # """ - # # Compare as uint32 to correctly handle negative runes. - # if UInt32(r) < utf8.RuneSelf: - # self.write_byte(Byte(r)) - # return 1 + self._resize_if_needed(1) + self._data[self._size] = byte + self._size += 1 - # self.last_read = OP_INVALID - # var write_at: Int - # var ok: Bool - # write_at, ok = self.try_grow_by_reslice(utf8.UTFMax) - # if not ok: - # write_at = self.grow(utf8.UTFMax) + return 1, Error() - # self.buf = utf8.AppendRune(self.buf[:write_at], r) - # return len(self.buf) - write_at + fn empty(self) -> Bool: + """Reports whether the unread portion of the buffer is empty.""" + return self._size <= self.offset - fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + fn reset(inout self): + """Resets the buffer to be empty, + but it retains the underlying storage for use by future writes. + reset is the same as [buffer.truncate](0).""" + if self._data: + self._data.free() + self._data = UnsafePointer[UInt8]().alloc(self._capacity) + self._size = 0 + self.offset = 0 + self.last_read = OP_INVALID + + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): """Reads the next len(dest) bytes from the buffer or until the buffer is drained. The return value n is the number of bytes read. If the buffer has no data to return, err is io.EOF (unless len(dest) is zero); @@ -409,6 +224,7 @@ struct Buffer( Args: dest: The buffer to read into. + capacity: The capacity of the destination buffer. Returns: The number of bytes read from the buffer. @@ -417,97 +233,58 @@ struct Buffer( if self.empty(): # Buffer is empty, reset to recover space. self.reset() - if dest.capacity == 0: + # TODO: How to check if the span's pointer has 0 capacity? We want to return early if the span can't receive any data. + if capacity == 0: return 0, Error() - return 0, Error(io.EOF) + return 0, io.EOF + + # Copy the data of the internal buffer from offset to len(buf) into the destination buffer at the given index. + var bytes_to_read = self.as_bytes_slice()[self.offset :] + var bytes_read = copy(dest.unsafe_ptr(), bytes_to_read.unsafe_ptr(), source_length=len(bytes_to_read)) + dest._len += bytes_read + self.offset += bytes_read - var bytes_read = copy(dest, self.buf[self.off : len(self.buf)]) - self.off += bytes_read if bytes_read > 0: self.last_read = OP_READ return bytes_read, Error() - fn next(inout self, number_of_bytes: Int) raises -> List[Byte]: - """Returns a slice containing the next n bytes from the buffer, - advancing the buffer as if the bytes had been returned by [Buffer.read]. - If there are fewer than n bytes in the buffer, next returns the entire buffer. - The slice is only valid until the next call to a read or write method. + fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): + """Reads the next len(dest) bytes from the buffer or until the buffer + is drained. The return value n is the number of bytes read. If the + buffer has no data to return, err is io.EOF (unless len(dest) is zero); + otherwise it is nil. Args: - number_of_bytes: The number of bytes to read from the buffer. + dest: The buffer to read into. Returns: - A slice containing the next n bytes from the buffer. + The number of bytes read from the buffer. """ - self.last_read = OP_INVALID - var m = len(self) - var bytes_to_read = number_of_bytes - if bytes_to_read > m: - bytes_to_read = m + var span = Span(dest) - var data = self.buf[self.off : self.off + bytes_to_read] - self.off += bytes_to_read - if bytes_to_read > 0: - self.last_read = OP_READ + var bytes_read: Int + var err: Error + bytes_read, err = self._read(span, dest.capacity) + dest.size += bytes_read - return data + return bytes_read, err - fn read_byte(inout self) -> (Byte, Error): + fn read_byte(inout self) -> (UInt8, Error): """Reads and returns the next byte from the buffer. If no byte is available, it returns error io.EOF. """ if self.empty(): # Buffer is empty, reset to recover space. self.reset() - return Byte(0), Error(io.EOF) + return UInt8(0), io.EOF - var byte = self.buf[self.off] - self.off += 1 + var byte = self._data[self.offset] + self.offset += 1 self.last_read = OP_READ return byte, Error() - # read_rune reads and returns the next UTF-8-encoded - # Unicode code point from the buffer. - # If no bytes are available, the error returned is io.EOF. - # If the bytes are an erroneous UTF-8 encoding, it - # consumes one byte and returns U+FFFD, 1. - # fn read_rune(self) (r rune, size Int, err error) - # if self.empty() - # # Buffer is empty, reset to recover space. - # self.reset() - # return 0, 0, io.EOF - # - # c := self.buf[self.off] - # if c < utf8.RuneSelf - # self.off+= 1 - # self.last_read = OP_READ_RUNE1 - # return rune(c), 1, nil - # - # r, n := utf8.DecodeRune(self.buf[self.off:]) - # self.off += n - # self.last_read = ReadOp(n) - # return r, n, nil - # - - # unread_rune unreads the last rune returned by [Buffer.read_rune]. - # If the most recent read or write operation on the buffer was - # not a successful [Buffer.read_rune], unread_rune returns an error. (In this regard - # it is stricter than [Buffer.unread_byte], which will unread the last byte - # from any read operation.) - # fn unread_rune(self): - # if self.last_read <= OP_INVALID - # return errors.New("buffer.Buffer: unread_rune: previous operation was not a successful read_rune") - # - # if self.off >= Int(self.last_read) - # self.off -= Int(self.last_read) - # - # self.last_read = OP_INVALID - # return nil - - # var err_unread_byte = errors.New("buffer.Buffer: unread_byte: previous operation was not a successful read") - fn unread_byte(inout self) -> Error: """Unreads the last byte returned by the most recent successful read operation that read at least one byte. If a write has happened since @@ -518,12 +295,12 @@ struct Buffer( return Error("buffer.Buffer: unread_byte: previous operation was not a successful read") self.last_read = OP_INVALID - if self.off > 0: - self.off -= 1 + if self.offset > 0: + self.offset -= 1 return Error() - fn read_bytes(inout self, delim: Byte) -> (List[Byte], Error): + fn read_bytes(inout self, delim: UInt8) -> (List[UInt8], Error): """Reads until the first occurrence of delim in the input, returning a slice containing the data up to and including the delimiter. If read_bytes encounters an error before finding a delimiter, @@ -535,46 +312,42 @@ struct Buffer( delim: The delimiter to read until. Returns: - A List[Byte] struct containing the data up to and including the delimiter. + A List[UInt8] struct containing the data up to and including the delimiter. """ - var slice: List[Byte] + var slice: Span[UInt8, __lifetime_of(self)] var err: Error slice, err = self.read_slice(delim) - # return a copy of slice. The buffer's backing array may - # be overwritten by later calls. - var line = List[Byte](capacity=BUFFER_SIZE) - for i in range(len(slice)): - line.append(slice[i]) - return line, Error() + var bytes = List[UInt8](capacity=len(slice) + 1) + for byte in slice: + bytes.append(byte[]) + + return bytes, err - fn read_slice(inout self, delim: Byte) -> (List[Byte], Error): + fn read_slice(inout self, delim: UInt8) -> (Span[UInt8, __lifetime_of(self)], Error): """Like read_bytes but returns a reference to internal buffer data. Args: delim: The delimiter to read until. Returns: - A List[Byte] struct containing the data up to and including the delimiter. + A List[UInt8] struct containing the data up to and including the delimiter. """ - var at_eof = False - var i = index_byte(self.buf[self.off : len(self.buf)], delim) - var end = self.off + i + 1 + var i = index_byte(bytes=self.as_bytes_slice(), delim=delim) + var end = self.offset + i + 1 + var err = Error() if i < 0: - end = len(self.buf) - at_eof = True + end = self._size + err = Error(str(io.EOF)) - var line = self.buf[self.off : end] - self.off = end + var line = self.as_bytes_slice()[self.offset : end] + self.offset = end self.last_read = OP_READ - if at_eof: - return line, Error(io.EOF) + return line, err - return line, Error() - - fn read_string(inout self, delim: Byte) -> (String, Error): + fn read_string(inout self, delim: UInt8) -> (String, Error): """Reads until the first occurrence of delim in the input, returning a string containing the data up to and including the delimiter. If read_string encounters an error before finding a delimiter, @@ -588,14 +361,76 @@ struct Buffer( Returns: A string containing the data up to and including the delimiter. """ - var slice: List[Byte] + var bytes: List[UInt8] var err: Error - slice, err = self.read_slice(delim) - slice.append(0) - return String(slice), err + bytes, err = self.read_bytes(delim) + bytes.append(0) + + return String(bytes), err + + fn next(inout self, number_of_bytes: Int) -> Span[UInt8, __lifetime_of(self)]: + """Returns a slice containing the next n bytes from the buffer, + advancing the buffer as if the bytes had been returned by [Buffer.read]. + If there are fewer than n bytes in the buffer, next returns the entire buffer. + The slice is only valid until the next call to a read or write method. + + Args: + number_of_bytes: The number of bytes to read from the buffer. + + Returns: + A slice containing the next n bytes from the buffer. + """ + self.last_read = OP_INVALID + var bytes_remaining = len(self) + var bytes_to_read = number_of_bytes + if bytes_to_read > bytes_remaining: + bytes_to_read = bytes_remaining + + var data = self.as_bytes_slice()[self.offset : self.offset + bytes_to_read] + + self.offset += bytes_to_read + if bytes_to_read > 0: + self.last_read = OP_READ + + return data + + # fn write_to[W: io.Writer](inout self, inout writer: W) -> (Int, Error): + # """Writes data to w until the buffer is drained or an error occurs. + # The return value n is the number of bytes written; Any error + # encountered during the write is also returned. + + # Args: + # writer: The writer to write to. + + # Returns: + # The number of bytes written to the writer. + # """ + # self.last_read = OP_INVALID + # var bytes_towrite = len(self) + # var total_bytes_written: Int = 0 + + # if bytes_towrite > 0: + # var bytes_written: Int + # var err: Error + # bytes_written, err = writer.write(self.as_bytes_slice()[self.offset :]) + # if bytes_written > bytes_towrite: + # panic("bytes.Buffer.write_to: invalid write count") + + # self.offset += bytes_written + # total_bytes_written = bytes_written + # if err: + # return total_bytes_written, err + + # # all bytes should have been written, by definition of write method in io.Writer + # if bytes_written != bytes_towrite: + # return total_bytes_written, Error(ERR_SHORTwrite) + + # # Buffer is now empty; reset. + # self.reset() + # return total_bytes_written, Error() -fn new_buffer() -> Buffer: +fn new_buffer(capacity: Int = io.BUFFER_SIZE) -> Buffer: """Creates and initializes a new [Buffer] using buf as its` initial contents. The new [Buffer] takes ownership of buf, and the caller should not use buf after this call. new_buffer is intended to @@ -606,11 +441,11 @@ fn new_buffer() -> Buffer: In most cases, new([Buffer]) (or just declaring a [Buffer] variable) is sufficient to initialize a [Buffer]. """ - var b = List[Byte](capacity=BUFFER_SIZE) + var b = List[UInt8](capacity=capacity) return Buffer(b^) -fn new_buffer(owned buf: List[Byte]) -> Buffer: +fn new_buffer(owned buf: List[UInt8]) -> Buffer: """Creates and initializes a new [Buffer] using buf as its` initial contents. The new [Buffer] takes ownership of buf, and the caller should not use buf after this call. new_buffer is intended to @@ -644,5 +479,4 @@ fn new_buffer(owned s: String) -> Buffer: Returns: A new [Buffer] initialized with the provided string. """ - var bytes_buffer = List[Byte](s.as_bytes()) - return Buffer(bytes_buffer^) + return Buffer(s.as_bytes()) diff --git a/external/gojo/bytes/reader.mojo b/external/gojo/bytes/reader.mojo index 9539672..f640ec3 100644 --- a/external/gojo/bytes/reader.mojo +++ b/external/gojo/bytes/reader.mojo @@ -1,11 +1,10 @@ -from collections.optional import Optional -from ..builtins import cap, copy, Byte, panic +from ..builtins import copy, panic import ..io -@value +# TODO: Maybe try a non owning reader, but I'm concerned about the lifetime of the buffer. +# Is making it unsafe a good idea? The source data would need to be ensured to outlive the reader by the user. struct Reader( - Copyable, Sized, io.Reader, io.ReaderAt, @@ -21,50 +20,95 @@ struct Reader( The zero value for Reader operates like a Reader of an empty slice. """ - var buffer: List[Byte] - var index: Int64 # current reading index + var data: UnsafePointer[UInt8] # contents are the bytes buf[index : size] + var size: Int + var capacity: Int + var index: Int # current reading index var prev_rune: Int # index of previous rune; or < 0 + fn __init__(inout self, owned buffer: List[UInt8]): + """Initializes a new [Reader.Reader] struct.""" + self.capacity = buffer.capacity + self.size = buffer.size + self.data = buffer.steal_data() + self.index = 0 + self.prev_rune = -1 + + fn __moveinit__(inout self, owned other: Reader): + """Initializes a new [Reader.Reader] struct by moving the data from another [Reader.Reader] struct.""" + self.capacity = other.capacity + self.size = other.size + self.data = other.data + self.index = other.index + self.prev_rune = other.prev_rune + + other.data = UnsafePointer[UInt8]() + other.size = 0 + other.capacity = 0 + other.index = 0 + other.prev_rune = -1 + fn __len__(self) -> Int: - """len returns the number of bytes of the unread portion of the - slice.""" - if self.index >= len(self.buffer): - return 0 + """Returns the number of bytes of the unread portion of the slice.""" + return self.size - int(self.index) - return int(len(self.buffer) - self.index) + fn __del__(owned self): + if self.data: + self.data.free() - fn size(self) -> Int: - """Returns the original length of the underlying byte slice. - Size is the number of bytes available for reading via [Reader.ReadAt]. - The result is unaffected by any method calls except [Reader.Reset].""" - return len(self.buffer) + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the internal data as a Span[UInt8].""" + return Span[UInt8, __lifetime_of(self)](unsafe_ptr=self.data, len=self.size) - fn read(inout self, inout dest: List[Byte]) -> (Int, Error): - """Reads from the internal buffer into the dest List[Byte] struct. + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): + """Reads from the internal buffer into the dest List[UInt8] struct. Implements the [io.Reader] Interface. Args: - dest: The destination List[Byte] struct to read into. + dest: The destination Span[UInt8] struct to read into. + capacity: The capacity of the destination buffer. Returns: Int: The number of bytes read into dest.""" - if self.index >= len(self.buffer): - return 0, Error(io.EOF) + if self.index >= self.size: + return 0, io.EOF + + # Copy the data of the internal buffer from offset to len(buf) into the destination buffer at the given index. self.prev_rune = -1 - var unread_bytes = self.buffer[int(self.index) : len(self.buffer)] - var bytes_read = copy(dest, unread_bytes) + var bytes_to_write = self.as_bytes_slice()[self.index : self.size] + var bytes_written = copy(dest.unsafe_ptr(), bytes_to_write.unsafe_ptr(), len(bytes_to_write)) + dest._len += bytes_written + self.index += bytes_written + + return bytes_written, Error() + + fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): + """Reads from the internal buffer into the dest List[UInt8] struct. + Implements the [io.Reader] Interface. + + Args: + dest: The destination List[UInt8] struct to read into. + + Returns: + Int: The number of bytes read into dest.""" + var span = Span(dest) - self.index += bytes_read - return bytes_read, Error() + var bytes_read: Int + var err: Error + bytes_read, err = self._read(span, dest.capacity) + dest.size += bytes_read + + return bytes_read, err - fn read_at(self, inout dest: List[Byte], off: Int64) -> (Int, Error): + fn _read_at(self, inout dest: Span[UInt8], off: Int, capacity: Int) -> (Int, Error): """Reads len(dest) bytes into dest beginning at byte offset off. Implements the [io.ReaderAt] Interface. Args: - dest: The destination List[Byte] struct to read into. + dest: The destination Span[UInt8] struct to read into. off: The offset to start reading from. + capacity: The capacity of the destination buffer. Returns: Int: The number of bytes read into dest. @@ -73,23 +117,43 @@ struct Reader( if off < 0: return 0, Error("bytes.Reader.read_at: negative offset") - if off >= Int64(len(self.buffer)): - return 0, Error(io.EOF) + if off >= Int(self.size): + return 0, io.EOF - var unread_bytes = self.buffer[int(off) : len(self.buffer)] - var bytes_written = copy(dest, unread_bytes) + var unread_bytes = self.as_bytes_slice()[off : self.size] + var bytes_written = copy(dest.unsafe_ptr(), unread_bytes.unsafe_ptr(), len(unread_bytes)) if bytes_written < len(dest): - return 0, Error(io.EOF) + return 0, io.EOF return bytes_written, Error() - fn read_byte(inout self) -> (Byte, Error): + fn read_at(self, inout dest: List[UInt8], off: Int) -> (Int, Error): + """Reads len(dest) bytes into dest beginning at byte offset off. + Implements the [io.ReaderAt] Interface. + + Args: + dest: The destination List[UInt8] struct to read into. + off: The offset to start reading from. + + Returns: + Int: The number of bytes read into dest. + """ + var span = Span(dest) + + var bytes_read: Int + var err: Error + bytes_read, err = self._read_at(span, off, dest.capacity) + dest.size += bytes_read + + return bytes_read, err + + fn read_byte(inout self) -> (UInt8, Error): """Reads and returns a single byte from the internal buffer. Implements the [io.ByteReader] Interface.""" self.prev_rune = -1 - if self.index >= len(self.buffer): - return Int8(0), Error(io.EOF) + if self.index >= self.size: + return UInt8(0), io.EOF - var byte = self.buffer[int(self.index)] + var byte = self.data[self.index] self.index += 1 return byte, Error() @@ -107,7 +171,7 @@ struct Reader( # # read_rune implements the [io.RuneReader] Interface. # fn read_rune(self) (ch rune, size Int, err error): - # if self.index >= Int64(len(self.buffer)): + # if self.index >= Int(self.size): # self.prev_rune = -1 # return 0, 0, io.EOF @@ -117,7 +181,7 @@ struct Reader( # return rune(c), 1, nil # ch, size = utf8.DecodeRune(self.buffer[self.index:]) - # self.index += Int64(size) + # self.index += Int(size) # return # # unread_rune complements [Reader.read_rune] in implementing the [io.RuneScanner] Interface. @@ -128,13 +192,12 @@ struct Reader( # if self.prev_rune < 0: # return errors.New("bytes.Reader.unread_rune: previous operation was not read_rune") - # self.index = Int64(self.prev_rune) + # self.index = Int(self.prev_rune) # self.prev_rune = -1 # return nil - fn seek(inout self, offset: Int64, whence: Int) -> (Int64, Error): + fn seek(inout self, offset: Int, whence: Int) -> (Int, Error): """Moves the read position to the specified offset from the specified whence. - Implements the [io.Seeker] Interface. Args: offset: The offset to move to. @@ -144,24 +207,24 @@ struct Reader( The new position in which the next read will start from. """ self.prev_rune = -1 - var position: Int64 = 0 + var position: Int = 0 if whence == io.SEEK_START: position = offset elif whence == io.SEEK_CURRENT: position = self.index + offset elif whence == io.SEEK_END: - position = len(self.buffer) + offset + position = self.size + offset else: - return Int64(0), Error("bytes.Reader.seek: invalid whence") + return Int(0), Error("bytes.Reader.seek: invalid whence") if position < 0: - return Int64(0), Error("bytes.Reader.seek: negative position") + return Int(0), Error("bytes.Reader.seek: negative position") self.index = position return position, Error() - fn write_to[W: io.Writer](inout self, inout writer: W) -> (Int64, Error): + fn write_to[W: io.Writer](inout self, inout writer: W) -> (Int, Error): """Writes data to w until the buffer is drained or an error occurs. implements the [io.WriterTo] Interface. @@ -169,10 +232,10 @@ struct Reader( writer: The writer to write to. """ self.prev_rune = -1 - if self.index >= len(self.buffer): - return Int64(0), Error() + if self.index >= self.size: + return 0, Error() - var bytes = self.buffer[int(self.index) : len(self.buffer)] + var bytes = self.as_bytes_slice()[self.index : self.size] var write_count: Int var err: Error write_count, err = writer.write(bytes) @@ -181,36 +244,38 @@ struct Reader( self.index += write_count if write_count != len(bytes): - return Int64(write_count), Error(io.ERR_SHORT_WRITE) + return write_count, io.ERR_SHORT_WRITE - return Int64(write_count), Error() + return write_count, Error() - fn reset(inout self, buffer: List[Byte]): - """Resets the [Reader.Reader] to be reading from b. + fn reset(inout self, owned buffer: List[UInt8]): + """Resets the [Reader.Reader] to be reading from buffer. Args: buffer: The new buffer to read from. """ - self.buffer = buffer + self.capacity = buffer.capacity + self.size = buffer.size + self.data = buffer.steal_data() self.index = 0 self.prev_rune = -1 -fn new_reader(buffer: List[Byte]) -> Reader: +fn new_reader(owned buffer: List[UInt8]) -> Reader: """Returns a new [Reader.Reader] reading from b. Args: buffer: The new buffer to read from. """ - return Reader(buffer, 0, -1) + return Reader(buffer) -fn new_reader(buffer: String) -> Reader: +fn new_reader(owned buffer: String) -> Reader: """Returns a new [Reader.Reader] reading from b. Args: buffer: The new buffer to read from. """ - return Reader(buffer.as_bytes(), 0, -1) + return Reader(buffer.as_bytes()) diff --git a/external/gojo/fmt/fmt.mojo b/external/gojo/fmt/fmt.mojo index a44cdf3..5413c61 100644 --- a/external/gojo/fmt/fmt.mojo +++ b/external/gojo/fmt/fmt.mojo @@ -9,8 +9,6 @@ Boolean Integer %d base 10 %q a single-quoted character literal. -%x base 16, with lower-case letters for a-f -%X base 16, with upper-case letters for A-F Floating-point and complex constituents: %f decimal point but no exponent, e.g. 123.456 @@ -28,18 +26,17 @@ TODO: from utils.variant import Variant from math import floor -from ..builtins import Byte -alias Args = Variant[String, Int, Float64, Bool, List[Byte]] +alias Args = Variant[String, Int, Float64, Bool, List[UInt8]] fn replace_first(s: String, old: String, new: String) -> String: """Replace the first occurrence of a substring in a string. Args: - s: The original string - old: The substring to be replaced - new: The new substring + s: The original string. + old: The substring to be replaced. + new: The new substring. Returns: The string with the first occurrence of the old substring replaced by the new one. @@ -59,7 +56,7 @@ fn find_first_verb(s: String, verbs: List[String]) -> String: """Find the first occurrence of a verb in a string. Args: - s: The original string + s: The original string. verbs: The list of verbs to search for. Returns: @@ -77,34 +74,6 @@ fn find_first_verb(s: String, verbs: List[String]) -> String: return verb -alias BASE10_TO_BASE16 = List[String]("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f") - - -fn convert_base10_to_base16(value: Int) -> String: - """Converts a base 10 number to base 16. - - Args: - value: Base 10 number. - - Returns: - Base 16 number as a String. - """ - - var val: Float64 = 0.0 - var result: Float64 = value - var base16: String = "" - while result > 1: - var temp = result / 16 - var floor_result = floor(temp) - var remainder = temp - floor_result - result = floor_result - val = 16 * remainder - - base16 = BASE10_TO_BASE16[int(val)] + base16 - - return base16 - - fn format_string(format: String, arg: String) -> String: var verb = find_first_verb(format, List[String]("%s", "%q")) var arg_to_place = arg @@ -114,7 +83,7 @@ fn format_string(format: String, arg: String) -> String: return replace_first(format, String("%s"), arg) -fn format_bytes(format: String, arg: List[Byte]) -> String: +fn format_bytes(format: String, arg: List[UInt8]) -> String: var argument = arg if argument[-1] != 0: argument.append(0) @@ -123,20 +92,16 @@ fn format_bytes(format: String, arg: List[Byte]) -> String: fn format_integer(format: String, arg: Int) -> String: - var verb = find_first_verb(format, List[String]("%x", "%X", "%d", "%q")) - var arg_to_place = String(arg) - if verb == "%x": - arg_to_place = String(convert_base10_to_base16(arg)).lower() - elif verb == "%X": - arg_to_place = String(convert_base10_to_base16(arg)).upper() - elif verb == "%q": - arg_to_place = "'" + String(arg) + "'" + var verb = find_first_verb(format, List[String]("%d", "%q")) + var arg_to_place = str(arg) + if verb == "%q": + arg_to_place = "'" + str(arg) + "'" return replace_first(format, verb, arg_to_place) fn format_float(format: String, arg: Float64) -> String: - return replace_first(format, String("%f"), arg) + return replace_first(format, str("%f"), str(arg)) fn format_boolean(format: String, arg: Bool) -> String: @@ -162,15 +127,15 @@ fn sprintf(formatting: String, *args: Args) -> String: for i in range(len(args)): var argument = args[i] if argument.isa[String](): - text = format_string(text, argument.get[String]()[]) - elif argument.isa[List[Byte]](): - text = format_bytes(text, argument.get[List[Byte]]()[]) + text = format_string(text, argument[String]) + elif argument.isa[List[UInt8]](): + text = format_bytes(text, argument[List[UInt8]]) elif argument.isa[Int](): - text = format_integer(text, argument.get[Int]()[]) + text = format_integer(text, argument[Int]) elif argument.isa[Float64](): - text = format_float(text, argument.get[Float64]()[]) + text = format_float(text, argument[Float64]) elif argument.isa[Bool](): - text = format_boolean(text, argument.get[Bool]()[]) + text = format_boolean(text, argument[Bool]) return text @@ -204,16 +169,16 @@ fn printf(formatting: String, *args: Args) raises: for i in range(len(args)): var argument = args[i] if argument.isa[String](): - text = format_string(text, argument.get[String]()[]) - elif argument.isa[List[Byte]](): - text = format_bytes(text, argument.get[List[Byte]]()[]) + text = format_string(text, argument[String]) + elif argument.isa[List[UInt8]](): + text = format_bytes(text, argument[List[UInt8]]) elif argument.isa[Int](): - text = format_integer(text, argument.get[Int]()[]) + text = format_integer(text, argument[Int]) elif argument.isa[Float64](): - text = format_float(text, argument.get[Float64]()[]) + text = format_float(text, argument[Float64]) elif argument.isa[Bool](): - text = format_boolean(text, argument.get[Bool]()[]) + text = format_boolean(text, argument[Bool]) else: - raise Error("Unknown for argument #" + String(i)) + raise Error("Unknown for argument #" + str(i)) print(text) diff --git a/external/gojo/io/__init__.mojo b/external/gojo/io/__init__.mojo index 979bee7..e9bf48d 100644 --- a/external/gojo/io/__init__.mojo +++ b/external/gojo/io/__init__.mojo @@ -1,34 +1,331 @@ -from .traits import ( - Reader, - Writer, - Seeker, - Closer, - ReadWriter, - ReadCloser, - WriteCloser, - ReadWriteCloser, - ReadSeeker, - ReadSeekCloser, - WriteSeeker, - ReadWriteSeeker, - ReaderFrom, - WriterReadFrom, - WriterTo, - ReaderWriteTo, - ReaderAt, - WriterAt, - ByteReader, - ByteScanner, - ByteWriter, - RuneReader, - RuneScanner, - StringWriter, - SEEK_START, - SEEK_CURRENT, - SEEK_END, - ERR_SHORT_WRITE, - ERR_NO_PROGRESS, - ERR_SHORT_BUFFER, - EOF, -) from .io import write_string, read_at_least, read_full, read_all, BUFFER_SIZE +from .file import FileWrapper +from .std import STDWriter + + +alias Rune = Int32 + +# Package io provides basic interfaces to I/O primitives. +# Its primary job is to wrap existing implementations of such primitives, +# such as those in package os, into shared public interfaces that +# abstract the fntionality, plus some other related primitives. +# +# Because these interfaces and primitives wrap lower-level operations with +# various implementations, unless otherwise informed clients should not +# assume they are safe for parallel execution. +# Seek whence values. +alias SEEK_START = 0 # seek relative to the origin of the file +alias SEEK_CURRENT = 1 # seek relative to the current offset +alias SEEK_END = 2 # seek relative to the end + +# ERR_SHORT_WRITE means that a write accepted fewer bytes than requested +# but failed to return an explicit error. +alias ERR_SHORT_WRITE = Error("short write") + +# ERR_INVALID_WRITE means that a write returned an impossible count. +alias ERR_INVALID_WRITE = Error("invalid write result") + +# ERR_SHORT_BUFFER means that a read required a longer buffer than was provided. +alias ERR_SHORT_BUFFER = Error("short buffer") + +# EOF is the error returned by Read when no more input is available. +# (Read must return EOF itself, not an error wrapping EOF, +# because callers will test for EOF using ==.) +# fntions should return EOF only to signal a graceful end of input. +# If the EOF occurs unexpectedly in a structured data stream, +# the appropriate error is either [ERR_UNEXPECTED_EOF] or some other error +# giving more detail. +alias EOF = Error("EOF") + +# ERR_UNEXPECTED_EOF means that EOF was encountered in the +# middle of reading a fixed-size block or data structure. +alias ERR_UNEXPECTED_EOF = Error("unexpected EOF") + +# ERR_NO_PROGRESS is returned by some clients of a [Reader] when +# many calls to Read have failed to return any data or error, +# usually the sign of a broken [Reader] implementation. +alias ERR_NO_PROGRESS = Error("multiple Read calls return no data or error") + + +trait Reader(Movable): + """Reader is the trait that wraps the basic Read method. + + Read reads up to len(p) bytes into p. It returns the number of bytes + read (0 <= n <= len(p)) and any error encountered. Even if Read + returns n < len(p), it may use all of p as scratch space during the call. + If some data is available but not len(p) bytes, Read conventionally + returns what is available instead of waiting for more. + + When Read encounters an error or end-of-file condition after + successfully reading n > 0 bytes, it returns the number of + bytes read. It may return the (non-nil) error from the same call + or return the error (and n == 0) from a subsequent call. + An instance of this general case is that a Reader returning + a non-zero number of bytes at the end of the input stream may + return either err == EOF or err == nil. The next Read should + return 0, EOF. + + Callers should always process the n > 0 bytes returned before + considering the error err. Doing so correctly handles I/O errors + that happen after reading some bytes and also both of the + allowed EOF behaviors. + + If len(p) == 0, Read should always return n == 0. It may return a + non-nil error if some error condition is known, such as EOF. + + Implementations of Read are discouraged from returning a + zero byte count with a nil error, except when len(p) == 0. + Callers should treat a return of 0 and nil as indicating that + nothing happened; in particular it does not indicate EOF. + + Implementations must not retain p.""" + + fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): + ... + + fn _read(inout self, inout dest: Span[UInt8, _], capacity: Int) -> (Int, Error): + ... + + +trait Writer(Movable): + """Writer is the trait that wraps the basic Write method. + + Write writes len(p) bytes from p to the underlying data stream. + It returns the number of bytes written from p (0 <= n <= len(p)) + and any error encountered that caused the write to stop early. + Write must return a non-nil error if it returns n < len(p). + Write must not modify the slice data, even temporarily. + + Implementations must not retain p. + """ + + fn write(inout self, src: Span[UInt8, _]) -> (Int, Error): + ... + + +trait Closer(Movable): + """ + Closer is the trait that wraps the basic Close method. + + The behavior of Close after the first call is undefined. + Specific implementations may document their own behavior. + """ + + fn close(inout self) -> Error: + ... + + +trait Seeker(Movable): + """ + Seeker is the trait that wraps the basic Seek method. + + Seek sets the offset for the next Read or Write to offset, + interpreted according to whence: + [SEEK_START] means relative to the start of the file, + [SEEK_CURRENT] means relative to the current offset, and + [SEEK_END] means relative to the end + (for example, offset = -2 specifies the penultimate byte of the file). + Seek returns the new offset relative to the start of the + file or an error, if any. + + Seeking to an offset before the start of the file is an error. + Seeking to any positive offset may be allowed, but if the new offset exceeds + the size of the underlying object the behavior of subsequent I/O operations + is implementation-dependent. + """ + + fn seek(inout self, offset: Int, whence: Int) -> (Int, Error): + ... + + +trait ReadWriter(Reader, Writer): + ... + + +trait ReadCloser(Reader, Closer): + ... + + +trait WriteCloser(Writer, Closer): + ... + + +trait ReadWriteCloser(Reader, Writer, Closer): + ... + + +trait ReadSeeker(Reader, Seeker): + ... + + +trait ReadSeekCloser(Reader, Seeker, Closer): + ... + + +trait WriteSeeker(Writer, Seeker): + ... + + +trait ReadWriteSeeker(Reader, Writer, Seeker): + ... + + +trait ReaderFrom: + """ReaderFrom is the trait that wraps the ReadFrom method. + + ReadFrom reads data from r until EOF or error. + The return value n is the number of bytes read. + Any error except EOF encountered during the read is also returned. + + The [copy] function uses [ReaderFrom] if available.""" + + fn read_from[R: Reader](inout self, inout reader: R) -> (Int, Error): + ... + + +trait WriterReadFrom(Writer, ReaderFrom): + ... + + +trait WriterTo: + """WriterTo is the trait that wraps the WriteTo method. + + WriteTo writes data to w until there's no more data to write or + when an error occurs. The return value n is the number of bytes + written. Any error encountered during the write is also returned. + + The copy function uses WriterTo if available.""" + + fn write_to[W: Writer](inout self, inout writer: W) -> (Int, Error): + ... + + +trait ReaderWriteTo(Reader, WriterTo): + ... + + +trait ReaderAt: + """ReaderAt is the trait that wraps the basic ReadAt method. + + ReadAt reads len(p) bytes into p starting at offset off in the + underlying input source. It returns the number of bytes + read (0 <= n <= len(p)) and any error encountered. + + When ReadAt returns n < len(p), it returns a non-nil error + explaining why more bytes were not returned. In this respect, + ReadAt is stricter than Read. + + Even if ReadAt returns n < len(p), it may use all of p as scratch + space during the call. If some data is available but not len(p) bytes, + ReadAt blocks until either all the data is available or an error occurs. + In this respect ReadAt is different from Read. + + If the n = len(p) bytes returned by ReadAt are at the end of the + input source, ReadAt may return either err == EOF or err == nil. + + If ReadAt is reading from an input source with a seek offset, + ReadAt should not affect nor be affected by the underlying + seek offset. + + Clients of ReadAt can execute parallel ReadAt calls on the + same input source. + + Implementations must not retain p.""" + + fn read_at(self, inout dest: List[UInt8], off: Int) -> (Int, Error): + ... + + fn _read_at(self, inout dest: Span[UInt8], off: Int, capacity: Int) -> (Int, Error): + ... + + +trait WriterAt: + """WriterAt is the trait that wraps the basic WriteAt method. + + WriteAt writes len(p) bytes from p to the underlying data stream + at offset off. It returns the number of bytes written from p (0 <= n <= len(p)) + and any error encountered that caused the write to stop early. + WriteAt must return a non-nil error if it returns n < len(p). + + If WriteAt is writing to a destination with a seek offset, + WriteAt should not affect nor be affected by the underlying + seek offset. + + Clients of WriteAt can execute parallel WriteAt calls on the same + destination if the ranges do not overlap. + + Implementations must not retain p.""" + + fn _write_at(self, src: Span[UInt8], off: Int) -> (Int, Error): + ... + + fn write_at(self, src: List[UInt8], off: Int) -> (Int, Error): + ... + + +trait ByteReader: + """ByteReader is the trait that wraps the read_byte method. + + read_byte reads and returns the next byte from the input or + any error encountered. If read_byte returns an error, no input + byte was consumed, and the returned byte value is undefined. + + read_byte provides an efficient trait for byte-at-time + processing. A [Reader] that does not implement ByteReader + can be wrapped using bufio.NewReader to add this method.""" + + fn read_byte(inout self) -> (UInt8, Error): + ... + + +trait ByteScanner(ByteReader): + """ByteScanner is the trait that adds the unread_byte method to the + basic read_byte method. + + unread_byte causes the next call to read_byte to return the last byte read. + If the last operation was not a successful call to read_byte, unread_byte may + return an error, unread the last byte read (or the byte prior to the + last-unread byte), or (in implementations that support the [Seeker] trait) + seek to one byte before the current offset.""" + + fn unread_byte(inout self) -> Error: + ... + + +trait ByteWriter: + """ByteWriter is the trait that wraps the write_byte method.""" + + fn write_byte(inout self, byte: UInt8) -> (Int, Error): + ... + + +trait RuneReader: + """RuneReader is the trait that wraps the read_rune method. + + read_rune reads a single encoded Unicode character + and returns the rune and its size in bytes. If no character is + available, err will be set.""" + + fn read_rune(inout self) -> (Rune, Int): + ... + + +trait RuneScanner(RuneReader): + """RuneScanner is the trait that adds the unread_rune method to the + basic read_rune method. + + unread_rune causes the next call to read_rune to return the last rune read. + If the last operation was not a successful call to read_rune, unread_rune may + return an error, unread the last rune read (or the rune prior to the + last-unread rune), or (in implementations that support the [Seeker] trait) + seek to the start of the rune before the current offset.""" + + fn unread_rune(inout self) -> Rune: + ... + + +trait StringWriter: + """StringWriter is the trait that wraps the WriteString method.""" + + fn write_string(inout self, src: String) -> (Int, Error): + ... diff --git a/external/gojo/io/file.mojo b/external/gojo/io/file.mojo new file mode 100644 index 0000000..816c879 --- /dev/null +++ b/external/gojo/io/file.mojo @@ -0,0 +1,136 @@ +import ..io +from ..builtins import copy + + +struct FileWrapper(io.ReadWriteCloser, io.ByteReader): + var handle: FileHandle + + fn __init__(inout self, path: String, mode: String) raises: + self.handle = open(path, mode) + + fn __moveinit__(inout self, owned existing: Self): + self.handle = existing.handle^ + + fn __del__(owned self): + var err = self.close() + if err: + # TODO: __del__ can't raise, but there should be some fallback. + print(str(err)) + + fn close(inout self) -> Error: + try: + self.handle.close() + except e: + return e + + return Error() + + fn _read(inout self, inout dest: Span[UInt8, _], capacity: Int) -> (Int, Error): + """Read from the file handle into dest's pointer. + Pretty hacky way to force the filehandle read into the defined trait, and it's unsafe since we're + reading directly into the pointer. + """ + # var bytes_to_read = dest.capacity - len(dest) + var bytes_read: Int + var result: List[UInt8] + try: + result = self.handle.read_bytes() + bytes_read = len(result) + # TODO: Need to raise an Issue for this. Reading with pointer does not return an accurate count of bytes_read :( + # bytes_read = int(self.handle.read(DTypePointer[DType.uint8](dest.unsafe_ptr()) + dest.size)) + except e: + return 0, e + + var count = 0 + var target = dest.unsafe_ptr() + len(dest) + for i in range(len(result)): + target[i] = result[i] + count += 1 + dest._len += count + + if bytes_read == 0: + return bytes_read, io.EOF + + return bytes_read, Error() + + fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): + """Read from the file handle into dest's pointer. + Pretty hacky way to force the filehandle read into the defined trait, and it's unsafe since we're + reading directly into the pointer. + """ + # var bytes_to_read = dest.capacity - len(dest) + var bytes_read: Int + var result: List[UInt8] + try: + result = self.handle.read_bytes() + bytes_read = len(result) + # TODO: Need to raise an Issue for this. Reading with pointer does not return an accurate count of bytes_read :( + # bytes_read = int(self.handle.read(DTypePointer[DType.uint8](dest.unsafe_ptr()) + dest.size)) + except e: + return 0, e + + _ = copy(dest, result, len(dest)) + + if bytes_read == 0: + return bytes_read, io.EOF + + return bytes_read, Error() + + fn read_all(inout self) -> (List[UInt8], Error): + var bytes = List[UInt8](capacity=io.BUFFER_SIZE) + while True: + var temp = List[UInt8](capacity=io.BUFFER_SIZE) + _ = self.read(temp) + + # If new bytes will overflow the result, resize it. + if len(bytes) + len(temp) > bytes.capacity: + bytes.reserve(bytes.capacity * 2) + bytes.extend(temp) + + if len(temp) < io.BUFFER_SIZE: + return bytes, io.EOF + + fn read_byte(inout self) -> (UInt8, Error): + try: + var bytes: List[UInt8] + var err: Error + bytes, err = self.read_bytes(1) + return bytes[0], Error() + except e: + return UInt8(0), e + + fn read_bytes(inout self, size: Int = -1) raises -> (List[UInt8], Error): + try: + return self.handle.read_bytes(size), Error() + except e: + return List[UInt8](), e + + fn stream_until_delimiter(inout self, inout dest: List[UInt8], delimiter: UInt8, max_size: Int) -> Error: + var byte: UInt8 + var err = Error() + for _ in range(max_size): + byte, err = self.read_byte() + if err: + return err + + if byte == delimiter: + return err + dest.append(byte) + return Error("Stream too long") + + fn seek(inout self, offset: Int, whence: Int = 0) -> (Int, Error): + try: + var position = self.handle.seek(UInt64(offset), whence) + return int(position), Error() + except e: + return 0, e + + fn write(inout self, src: Span[UInt8]) -> (Int, Error): + if len(src) == 0: + return 0, Error("No data to write") + + try: + self.handle.write(src.unsafe_ptr()) + return len(src), io.EOF + except e: + return 0, Error(str(e)) diff --git a/external/gojo/io/io.mojo b/external/gojo/io/io.mojo index 338da66..8cf7df2 100644 --- a/external/gojo/io/io.mojo +++ b/external/gojo/io/io.mojo @@ -1,7 +1,4 @@ -from collections.optional import Optional -from ..builtins import cap, copy, Byte, Error, panic -from .traits import ERR_UNEXPECTED_EOF - +from ..builtins import copy, panic alias BUFFER_SIZE = 4096 @@ -18,7 +15,7 @@ fn write_string[W: Writer](inout writer: W, string: String) -> (Int, Error): Returns: The number of bytes written and an error, if any. """ - return writer.write(string.as_bytes()) + return writer.write(string.as_bytes_slice()) fn write_string[W: StringWriter](inout writer: W, string: String) -> (Int, Error): @@ -35,7 +32,7 @@ fn write_string[W: StringWriter](inout writer: W, string: String) -> (Int, Error return writer.write_string(string) -fn read_at_least[R: Reader](inout reader: R, inout dest: List[Byte], min: Int) -> (Int, Error): +fn read_at_least[R: Reader](inout reader: R, inout dest: List[UInt8], min: Int) -> (Int, Error): """Reads from r into buf until it has read at least min bytes. It returns the number of bytes copied and an error if fewer bytes were read. The error is EOF only if no bytes were read. @@ -54,7 +51,7 @@ fn read_at_least[R: Reader](inout reader: R, inout dest: List[Byte], min: Int) - The number of bytes read.""" var error = Error() if len(dest) < min: - return 0, Error(io.ERR_SHORT_BUFFER) + return 0, io.ERR_SHORT_BUFFER var total_bytes_read: Int = 0 while total_bytes_read < min and not error: @@ -66,12 +63,12 @@ fn read_at_least[R: Reader](inout reader: R, inout dest: List[Byte], min: Int) - error = Error() elif total_bytes_read > 0 and str(error): - error = Error(ERR_UNEXPECTED_EOF) + error = ERR_UNEXPECTED_EOF return total_bytes_read, error -fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error): +fn read_full[R: Reader](inout reader: R, inout dest: List[UInt8]) -> (Int, Error): """Reads exactly len(buf) bytes from r into buf. It returns the number of bytes copied and an error if fewer bytes were read. The error is EOF only if no bytes were read. @@ -83,7 +80,7 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) return read_at_least(reader, dest, len(dest)) -# fn copy_n[W: Writer, R: Reader](dst: W, src: R, n: Int64) raises -> Int64: +# fn copy_n[W: Writer, R: Reader](dst: W, src: R, n: Int) raises -> Int: # """Copies n bytes (or until an error) from src to dst. # It returns the number of bytes copied and the earliest # error encountered while copying. @@ -102,7 +99,7 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # return written -# fn copy[W: Writer, R: Reader](dst: W, src: R, n: Int64) -> Int64: +# fn copy[W: Writer, R: Reader](dst: W, src: R, n: Int) -> Int: # """copy copies from src to dst until either EOF is reached # on src or an error occurs. It returns the number of bytes # copied and the first error encountered while copying, if any. @@ -125,7 +122,7 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # # # # If either src implements [WriterTo] or dst implements [ReaderFrom], # # buf will not be used to perform the copy. -# fn CopyBuffer(dst Writer, src Reader, buf bytes) (written int64, err error) { +# fn CopyBuffer(dst Writer, src Reader, buf bytes) (written Int, err error) { # if buf != nil and len(buf) == 0 { # panic("empty buffer in CopyBuffer") # } @@ -133,7 +130,7 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # } -# fn copy_buffer[W: Writer, R: Reader](dst: W, src: R, buf: List[Byte]) raises -> Int64: +# fn copy_buffer[W: Writer, R: Reader](dst: W, src: R, buf: Span[UInt8]) raises -> Int: # """Actual implementation of copy and CopyBuffer. # if buf is nil, one is allocated. # """ @@ -146,24 +143,24 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # if nw < 0 or nr < nw: # nw = 0 -# var written = Int64(nw) +# var written = Int(nw) # if nr != nw: # raise Error(ERR_SHORT_WRITE) # return written -# fn copy_buffer[W: Writer, R: ReaderWriteTo](dst: W, src: R, buf: List[Byte]) -> Int64: +# fn copy_buffer[W: Writer, R: ReaderWriteTo](dst: W, src: R, buf: Span[UInt8]) -> Int: # return src.write_to(dst) -# fn copy_buffer[W: WriterReadFrom, R: Reader](dst: W, src: R, buf: List[Byte]) -> Int64: +# fn copy_buffer[W: WriterReadFrom, R: Reader](dst: W, src: R, buf: Span[UInt8]) -> Int: # return dst.read_from(src) # # LimitReader returns a Reader that reads from r # # but stops with EOF after n bytes. # # The underlying implementation is a *LimitedReader. -# fn LimitReader(r Reader, n int64) Reader { return &LimitedReader{r, n} } +# fn LimitReader(r Reader, n Int) Reader { return &LimitedReader{r, n} } # # A LimitedReader reads from R but limits the amount of # # data returned to just N bytes. Each call to Read @@ -171,31 +168,31 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # # Read returns EOF when N <= 0 or when the underlying R returns EOF. # struct LimitedReader(): # var R: Reader # underlying reader -# N int64 # max bytes remaining +# N Int # max bytes remaining # fn (l *LimitedReader) Read(p bytes) (n Int, err error) { # if l.N <= 0 { # return 0, EOF # } -# if int64(len(p)) > l.N { +# if Int(len(p)) > l.N { # p = p[0:l.N] # } # n, err = l.R.Read(p) -# l.N -= int64(n) +# l.N -= Int(n) # return # } # # NewSectionReader returns a [SectionReader] that reads from r # # starting at offset off and stops with EOF after n bytes. -# fn NewSectionReader(r ReaderAt, off int64, n int64) *SectionReader { -# var remaining int64 -# const maxint64 = 1<<63 - 1 -# if off <= maxint64-n { +# fn NewSectionReader(r ReaderAt, off Int, n Int) *SectionReader { +# var remaining Int +# const maxInt = 1<<63 - 1 +# if off <= maxInt-n { # remaining = n + off # } else { # # Overflow, with no way to return error. # # Assume we can read up to an offset of 1<<63 - 1. -# remaining = maxint64 +# remaining = maxInt # } # return &SectionReader{r, off, off, remaining, n} # } @@ -204,28 +201,28 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # # of an underlying [ReaderAt]. # type SectionReader struct { # r ReaderAt # constant after creation -# base int64 # constant after creation -# off int64 -# limit int64 # constant after creation -# n int64 # constant after creation +# base Int # constant after creation +# off Int +# limit Int # constant after creation +# n Int # constant after creation # } # fn (s *SectionReader) Read(p bytes) (n Int, err error) { # if s.off >= s.limit { # return 0, EOF # } -# if max := s.limit - s.off; int64(len(p)) > max { +# if max := s.limit - s.off; Int(len(p)) > max { # p = p[0:max] # } # n, err = s.r.ReadAt(p, s.off) -# s.off += int64(n) +# s.off += Int(n) # return # } # alias errWhence = "Seek: invalid whence" # alias errOffset = "Seek: invalid offset" -# fn (s *SectionReader) Seek(offset int64, whence Int) (int64, error) { +# fn (s *SectionReader) Seek(offset Int, whence Int) (Int, error) { # switch whence { # default: # return 0, errWhence @@ -243,12 +240,12 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # return offset - s.base, nil # } -# fn (s *SectionReader) ReadAt(p bytes, off int64) (n Int, err error) { +# fn (s *SectionReader) ReadAt(p bytes, off Int) (n Int, err error) { # if off < 0 or off >= s.capacity { # return 0, EOF # } # off += s.base -# if max := s.limit - off; int64(len(p)) > max { +# if max := s.limit - off; Int(len(p)) > max { # p = p[0:max] # n, err = s.r.ReadAt(p, off) # if err == nil { @@ -260,36 +257,36 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # } # # Size returns the size of the section in bytes. -# fn (s *SectionReader) Size() int64 { return s.limit - s.base } +# fn (s *SectionReader) Size() Int { return s.limit - s.base } # # Outer returns the underlying [ReaderAt] and offsets for the section. # # # # The returned values are the same that were passed to [NewSectionReader] # # when the [SectionReader] was created. -# fn (s *SectionReader) Outer() (r ReaderAt, off int64, n int64) { +# fn (s *SectionReader) Outer() (r ReaderAt, off Int, n Int) { # return s.r, s.base, s.n # } # # An OffsetWriter maps writes at offset base to offset base+off in the underlying writer. # type OffsetWriter struct { # w WriterAt -# base int64 # the original offset -# off int64 # the current offset +# base Int # the original offset +# off Int # the current offset # } # # NewOffsetWriter returns an [OffsetWriter] that writes to w # # starting at offset off. -# fn NewOffsetWriter(w WriterAt, off int64) *OffsetWriter { +# fn NewOffsetWriter(w WriterAt, off Int) *OffsetWriter { # return &OffsetWriter{w, off, off} # } # fn (o *OffsetWriter) Write(p bytes) (n Int, err error) { # n, err = o.w.WriteAt(p, o.off) -# o.off += int64(n) +# o.off += Int(n) # return # } -# fn (o *OffsetWriter) WriteAt(p bytes, off int64) (n Int, err error) { +# fn (o *OffsetWriter) WriteAt(p bytes, off Int) (n Int, err error) { # if off < 0 { # return 0, errOffset # } @@ -298,7 +295,7 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # return o.w.WriteAt(p, off) # } -# fn (o *OffsetWriter) Seek(offset int64, whence Int) (int64, error) { +# fn (o *OffsetWriter) Seek(offset Int, whence Int) (Int, error) { # switch whence { # default: # return 0, errWhence @@ -363,12 +360,12 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # }, # } -# fn (discard) ReadFrom(r Reader) (n int64, err error) { +# fn (discard) ReadFrom(r Reader) (n Int, err error) { # bufp := blackHolePool.Get().(*bytes) # readSize := 0 # for { # readSize, err = r.Read(*bufp) -# n += int64(readSize) +# n += Int(readSize) # if err != nil { # blackHolePool.Put(bufp) # if err == EOF { @@ -402,12 +399,13 @@ fn read_full[R: Reader](inout reader: R, inout dest: List[Byte]) -> (Int, Error) # fn (nopCloserWriterTo) Close() error { return nil } -# fn (c nopCloserWriterTo) WriteTo(w Writer) (n int64, err error) { +# fn (c nopCloserWriterTo) WriteTo(w Writer) (n Int, err error) { # return c.Reader.(WriterTo).WriteTo(w) # } -fn read_all[R: Reader](inout reader: R) -> (List[Byte], Error): +# TODO: read directly into dest +fn read_all[R: Reader](inout reader: R) -> (List[UInt8], Error): """Reads from r until an error or EOF and returns the data it read. A successful call returns err == nil, not err == EOF. Because ReadAll is defined to read from src until EOF, it does not treat an EOF from Read @@ -418,18 +416,16 @@ fn read_all[R: Reader](inout reader: R) -> (List[Byte], Error): Returns: The data read.""" - var dest = List[Byte](capacity=BUFFER_SIZE) - var index: Int = 0 + var dest = List[UInt8](capacity=BUFFER_SIZE) var at_eof: Bool = False while True: - var temp = List[Byte](capacity=BUFFER_SIZE) + var temp = List[UInt8](capacity=BUFFER_SIZE) var bytes_read: Int var err: Error bytes_read, err = reader.read(temp) - var err_message = str(err) - if err_message != "": - if err_message != EOF: + if str(err) != "": + if str(err) != str(EOF): return dest, err at_eof = True diff --git a/external/gojo/io/std.mojo b/external/gojo/io/std.mojo new file mode 100644 index 0000000..43fd563 --- /dev/null +++ b/external/gojo/io/std.mojo @@ -0,0 +1,54 @@ +import ..io + + +@value +struct STDWriter[file_descriptor: Int](Copyable, io.Writer, io.StringWriter): + """A writer for POSIX file descriptors.""" + + fn __init__(inout self): + constrained[ + file_descriptor == 1 or file_descriptor == 2, + "The STDWriter Struct is meant to write to STDOUT and STDERR. file_descriptor must be 1 or 2.", + ]() + + fn write(inout self, src: Span[UInt8]) -> (Int, Error): + """Writes the given bytes to the file descriptor. + + Args: + src: The bytes to write to the file descriptor. + + Returns: + The number of bytes written to the file descriptor. + """ + var write_count: Int = external_call["write", Int, Int32, UnsafePointer[UInt8], Int]( + file_descriptor, src.unsafe_ptr(), len(src) + ) + + if write_count == -1: + return 0, Error("Failed to write to file descriptor " + str(file_descriptor)) + + return write_count, Error() + + fn write_string(inout self, src: String) -> (Int, Error): + """Writes the given string to the file descriptor. + + Args: + src: The string to write to the file descriptor. + + Returns: + The number of bytes written to the file descriptor. + """ + return self.write(src.as_bytes_slice()) + + fn read_from[R: io.Reader](inout self, inout reader: R) -> (Int, Error): + """Reads from the given reader to a temporary buffer and writes to the file descriptor. + + Args: + reader: The reader to read from. + + Returns: + The number of bytes written to the file descriptor. + """ + var buffer = List[UInt8](capacity=io.BUFFER_SIZE) + _ = reader.read(buffer) + return self.write(Span(buffer)) diff --git a/external/gojo/io/traits.mojo b/external/gojo/io/traits.mojo deleted file mode 100644 index 0fa248c..0000000 --- a/external/gojo/io/traits.mojo +++ /dev/null @@ -1,320 +0,0 @@ -from collections.optional import Optional -from ..builtins import Byte, Error - -alias Rune = Int32 - -# Package io provides basic interfaces to I/O primitives. -# Its primary job is to wrap existing implementations of such primitives, -# such as those in package os, into shared public interfaces that -# abstract the fntionality, plus some other related primitives. -# -# Because these interfaces and primitives wrap lower-level operations with -# various implementations, unless otherwise informed clients should not -# assume they are safe for parallel execution. -# Seek whence values. -alias SEEK_START = 0 # seek relative to the origin of the file -alias SEEK_CURRENT = 1 # seek relative to the current offset -alias SEEK_END = 2 # seek relative to the end - -# ERR_SHORT_WRITE means that a write accepted fewer bytes than requested -# but failed to return an explicit error. -alias ERR_SHORT_WRITE = "short write" - -# ERR_INVALID_WRITE means that a write returned an impossible count. -alias ERR_INVALID_WRITE = "invalid write result" - -# ERR_SHORT_BUFFER means that a read required a longer buffer than was provided. -alias ERR_SHORT_BUFFER = "short buffer" - -# EOF is the error returned by Read when no more input is available. -# (Read must return EOF itself, not an error wrapping EOF, -# because callers will test for EOF using ==.) -# fntions should return EOF only to signal a graceful end of input. -# If the EOF occurs unexpectedly in a structured data stream, -# the appropriate error is either [ERR_UNEXPECTED_EOF] or some other error -# giving more detail. -alias EOF = "EOF" - -# ERR_UNEXPECTED_EOF means that EOF was encountered in the -# middle of reading a fixed-size block or data structure. -alias ERR_UNEXPECTED_EOF = "unexpected EOF" - -# ERR_NO_PROGRESS is returned by some clients of a [Reader] when -# many calls to Read have failed to return any data or error, -# usually the sign of a broken [Reader] implementation. -alias ERR_NO_PROGRESS = "multiple Read calls return no data or error" - - -trait Reader(Movable): - """Reader is the trait that wraps the basic Read method. - - Read reads up to len(p) bytes into p. It returns the number of bytes - read (0 <= n <= len(p)) and any error encountered. Even if Read - returns n < len(p), it may use all of p as scratch space during the call. - If some data is available but not len(p) bytes, Read conventionally - returns what is available instead of waiting for more. - - When Read encounters an error or end-of-file condition after - successfully reading n > 0 bytes, it returns the number of - bytes read. It may return the (non-nil) error from the same call - or return the error (and n == 0) from a subsequent call. - An instance of this general case is that a Reader returning - a non-zero number of bytes at the end of the input stream may - return either err == EOF or err == nil. The next Read should - return 0, EOF. - - Callers should always process the n > 0 bytes returned before - considering the error err. Doing so correctly handles I/O errors - that happen after reading some bytes and also both of the - allowed EOF behaviors. - - If len(p) == 0, Read should always return n == 0. It may return a - non-nil error if some error condition is known, such as EOF. - - Implementations of Read are discouraged from returning a - zero byte count with a nil error, except when len(p) == 0. - Callers should treat a return of 0 and nil as indicating that - nothing happened; in particular it does not indicate EOF. - - Implementations must not retain p.""" - - fn read(inout self, inout dest: List[Byte]) -> (Int, Error): - ... - - -trait Writer(Movable): - """Writer is the trait that wraps the basic Write method. - - Write writes len(p) bytes from p to the underlying data stream. - It returns the number of bytes written from p (0 <= n <= len(p)) - and any error encountered that caused the write to stop early. - Write must return a non-nil error if it returns n < len(p). - Write must not modify the slice data, even temporarily. - - Implementations must not retain p. - """ - - fn write(inout self, src: List[Byte]) -> (Int, Error): - ... - - -trait Closer(Movable): - """ - Closer is the trait that wraps the basic Close method. - - The behavior of Close after the first call is undefined. - Specific implementations may document their own behavior. - """ - - fn close(inout self) -> Error: - ... - - -trait Seeker(Movable): - """ - Seeker is the trait that wraps the basic Seek method. - - Seek sets the offset for the next Read or Write to offset, - interpreted according to whence: - [SEEK_START] means relative to the start of the file, - [SEEK_CURRENT] means relative to the current offset, and - [SEEK_END] means relative to the end - (for example, offset = -2 specifies the penultimate byte of the file). - Seek returns the new offset relative to the start of the - file or an error, if any. - - Seeking to an offset before the start of the file is an error. - Seeking to any positive offset may be allowed, but if the new offset exceeds - the size of the underlying object the behavior of subsequent I/O operations - is implementation-dependent. - """ - - fn seek(inout self, offset: Int64, whence: Int) -> (Int64, Error): - ... - - -trait ReadWriter(Reader, Writer): - ... - - -trait ReadCloser(Reader, Closer): - ... - - -trait WriteCloser(Writer, Closer): - ... - - -trait ReadWriteCloser(Reader, Writer, Closer): - ... - - -trait ReadSeeker(Reader, Seeker): - ... - - -trait ReadSeekCloser(Reader, Seeker, Closer): - ... - - -trait WriteSeeker(Writer, Seeker): - ... - - -trait ReadWriteSeeker(Reader, Writer, Seeker): - ... - - -trait ReaderFrom: - """ReaderFrom is the trait that wraps the ReadFrom method. - - ReadFrom reads data from r until EOF or error. - The return value n is the number of bytes read. - Any error except EOF encountered during the read is also returned. - - The [copy] function uses [ReaderFrom] if available.""" - - fn read_from[R: Reader](inout self, inout reader: R) -> (Int64, Error): - ... - - -trait WriterReadFrom(Writer, ReaderFrom): - ... - - -trait WriterTo: - """WriterTo is the trait that wraps the WriteTo method. - - WriteTo writes data to w until there's no more data to write or - when an error occurs. The return value n is the number of bytes - written. Any error encountered during the write is also returned. - - The copy function uses WriterTo if available.""" - - fn write_to[W: Writer](inout self, inout writer: W) -> (Int64, Error): - ... - - -trait ReaderWriteTo(Reader, WriterTo): - ... - - -trait ReaderAt: - """ReaderAt is the trait that wraps the basic ReadAt method. - - ReadAt reads len(p) bytes into p starting at offset off in the - underlying input source. It returns the number of bytes - read (0 <= n <= len(p)) and any error encountered. - - When ReadAt returns n < len(p), it returns a non-nil error - explaining why more bytes were not returned. In this respect, - ReadAt is stricter than Read. - - Even if ReadAt returns n < len(p), it may use all of p as scratch - space during the call. If some data is available but not len(p) bytes, - ReadAt blocks until either all the data is available or an error occurs. - In this respect ReadAt is different from Read. - - If the n = len(p) bytes returned by ReadAt are at the end of the - input source, ReadAt may return either err == EOF or err == nil. - - If ReadAt is reading from an input source with a seek offset, - ReadAt should not affect nor be affected by the underlying - seek offset. - - Clients of ReadAt can execute parallel ReadAt calls on the - same input source. - - Implementations must not retain p.""" - - fn read_at(self, inout dest: List[Byte], off: Int64) -> (Int, Error): - ... - - -trait WriterAt: - """WriterAt is the trait that wraps the basic WriteAt method. - - WriteAt writes len(p) bytes from p to the underlying data stream - at offset off. It returns the number of bytes written from p (0 <= n <= len(p)) - and any error encountered that caused the write to stop early. - WriteAt must return a non-nil error if it returns n < len(p). - - If WriteAt is writing to a destination with a seek offset, - WriteAt should not affect nor be affected by the underlying - seek offset. - - Clients of WriteAt can execute parallel WriteAt calls on the same - destination if the ranges do not overlap. - - Implementations must not retain p.""" - - fn write_at(self, src: List[Byte], off: Int64) -> (Int, Error): - ... - - -trait ByteReader: - """ByteReader is the trait that wraps the read_byte method. - - read_byte reads and returns the next byte from the input or - any error encountered. If read_byte returns an error, no input - byte was consumed, and the returned byte value is undefined. - - read_byte provides an efficient trait for byte-at-time - processing. A [Reader] that does not implement ByteReader - can be wrapped using bufio.NewReader to add this method.""" - - fn read_byte(inout self) -> (Byte, Error): - ... - - -trait ByteScanner(ByteReader): - """ByteScanner is the trait that adds the unread_byte method to the - basic read_byte method. - - unread_byte causes the next call to read_byte to return the last byte read. - If the last operation was not a successful call to read_byte, unread_byte may - return an error, unread the last byte read (or the byte prior to the - last-unread byte), or (in implementations that support the [Seeker] trait) - seek to one byte before the current offset.""" - - fn unread_byte(inout self) -> Error: - ... - - -trait ByteWriter: - """ByteWriter is the trait that wraps the write_byte method.""" - - fn write_byte(inout self, byte: Byte) -> (Int, Error): - ... - - -trait RuneReader: - """RuneReader is the trait that wraps the read_rune method. - - read_rune reads a single encoded Unicode character - and returns the rune and its size in bytes. If no character is - available, err will be set.""" - - fn read_rune(inout self) -> (Rune, Int): - ... - - -trait RuneScanner(RuneReader): - """RuneScanner is the trait that adds the unread_rune method to the - basic read_rune method. - - unread_rune causes the next call to read_rune to return the last rune read. - If the last operation was not a successful call to read_rune, unread_rune may - return an error, unread the last rune read (or the rune prior to the - last-unread rune), or (in implementations that support the [Seeker] trait) - seek to the start of the rune before the current offset.""" - - fn unread_rune(inout self) -> Rune: - ... - - -trait StringWriter: - """StringWriter is the trait that wraps the WriteString method.""" - - fn write_string(inout self, src: String) -> (Int, Error): - ... diff --git a/external/gojo/net/__init__.mojo b/external/gojo/net/__init__.mojo index 2587673..12b01d4 100644 --- a/external/gojo/net/__init__.mojo +++ b/external/gojo/net/__init__.mojo @@ -2,3 +2,16 @@ A good chunk of the leg work here came from the lightbug_http project! https://github.com/saviorand/lightbug_http/tree/main """ + +from .fd import FileDescriptor +from .socket import Socket +from .tcp import TCPConnection, TCPListener, listen_tcp, dial_tcp, TCPAddr +from .udp import UDPAddr, UDPConnection, listen_udp, dial_udp +from .address import NetworkType, Addr, HostPort +from .ip import get_ip_address, get_addr_info + + +# Time in nanoseconds +alias Duration = Int +alias DEFAULT_BUFFER_SIZE = 4096 +alias DEFAULT_TCP_KEEP_ALIVE = Duration(15 * 1000 * 1000 * 1000) # 15 seconds diff --git a/external/gojo/net/address.mojo b/external/gojo/net/address.mojo index 01bf25f..9278d9c 100644 --- a/external/gojo/net/address.mojo +++ b/external/gojo/net/address.mojo @@ -22,7 +22,7 @@ trait Addr(CollectionElement, Stringable): @value -struct TCPAddr(Addr): +struct BaseAddr: """Addr struct representing a TCP address. Args: @@ -35,29 +35,32 @@ struct TCPAddr(Addr): var port: Int var zone: String # IPv6 addressing zone - fn __init__(inout self): - self.ip = String("127.0.0.1") - self.port = 8000 - self.zone = "" - - fn __init__(inout self, ip: String, port: Int): + fn __init__(inout self, ip: String = "", port: Int = 0, zone: String = ""): self.ip = ip self.port = port - self.zone = "" + self.zone = zone + + fn __init__(inout self, other: TCPAddr): + self.ip = other.ip + self.port = other.port + self.zone = other.zone + + fn __init__(inout self, other: UDPAddr): + self.ip = other.ip + self.port = other.port + self.zone = other.zone fn __str__(self) -> String: if self.zone != "": - return join_host_port(String(self.ip) + "%" + self.zone, self.port) - return join_host_port(self.ip, self.port) + return join_host_port(self.ip + "%" + self.zone, str(self.port)) + return join_host_port(self.ip, str(self.port)) - fn network(self) -> String: - return NetworkType.tcp.value - -fn resolve_internet_addr(network: String, address: String) raises -> TCPAddr: +fn resolve_internet_addr(network: String, address: String) -> (TCPAddr, Error): var host: String = "" var port: String = "" var portnum: Int = 0 + var err = Error() if ( network == NetworkType.tcp.value or network == NetworkType.tcp4.value @@ -67,29 +70,33 @@ fn resolve_internet_addr(network: String, address: String) raises -> TCPAddr: or network == NetworkType.udp6.value ): if address != "": - var host_port = split_host_port(address) - host = host_port.host - port = host_port.port - portnum = atol(port.__str__()) + var result = split_host_port(address) + if result[1]: + return TCPAddr(), result[1] + + host = result[0].host + port = str(result[0].port) + portnum = result[0].port elif network == NetworkType.ip.value or network == NetworkType.ip4.value or network == NetworkType.ip6.value: if address != "": host = address elif network == NetworkType.unix.value: - raise Error("Unix addresses not supported yet") + return TCPAddr(), Error("Unix addresses not supported yet") else: - raise Error("unsupported network type: " + network) - return TCPAddr(host, portnum) + return TCPAddr(), Error("unsupported network type: " + network) + return TCPAddr(host, portnum), err -alias missingPortError = Error("missing port in address") -alias tooManyColonsError = Error("too many colons in address") +alias MISSING_PORT_ERROR = Error("missing port in address") +alias TOO_MANY_COLONS_ERROR = Error("too many colons in address") +@value struct HostPort(Stringable): var host: String var port: Int - fn __init__(inout self, host: String, port: Int): + fn __init__(inout self, host: String = "", port: Int = 0): self.host = host self.port = port @@ -103,7 +110,7 @@ fn join_host_port(host: String, port: String) -> String: return host + ":" + port -fn split_host_port(hostport: String) raises -> HostPort: +fn split_host_port(hostport: String) -> (HostPort, Error): var host: String = "" var port: String = "" var colon_index = hostport.rfind(":") @@ -111,35 +118,38 @@ fn split_host_port(hostport: String) raises -> HostPort: var k: Int = 0 if colon_index == -1: - raise missingPortError + return HostPort(), MISSING_PORT_ERROR if hostport[0] == "[": var end_bracket_index = hostport.find("]") if end_bracket_index == -1: - raise Error("missing ']' in address") + return HostPort(), Error("missing ']' in address") if end_bracket_index + 1 == len(hostport): - raise missingPortError + return HostPort(), MISSING_PORT_ERROR elif end_bracket_index + 1 == colon_index: host = hostport[1:end_bracket_index] j = 1 k = end_bracket_index + 1 else: if hostport[end_bracket_index + 1] == ":": - raise tooManyColonsError + return HostPort(), TOO_MANY_COLONS_ERROR else: - raise missingPortError + return HostPort(), MISSING_PORT_ERROR else: host = hostport[:colon_index] if host.find(":") != -1: - raise tooManyColonsError + return HostPort(), TOO_MANY_COLONS_ERROR if hostport[j:].find("[") != -1: - raise Error("unexpected '[' in address") + return HostPort(), Error("unexpected '[' in address") if hostport[k:].find("]") != -1: - raise Error("unexpected ']' in address") + return HostPort(), Error("unexpected ']' in address") port = hostport[colon_index + 1 :] if port == "": - raise missingPortError + return HostPort(), MISSING_PORT_ERROR if host == "": - raise Error("missing host") + return HostPort(), Error("missing host") - return HostPort(host, atol(port)) + try: + return HostPort(host, atol(port)), Error() + except e: + return HostPort(), e diff --git a/external/gojo/net/dial.mojo b/external/gojo/net/dial.mojo deleted file mode 100644 index 5effd65..0000000 --- a/external/gojo/net/dial.mojo +++ /dev/null @@ -1,45 +0,0 @@ -from .tcp import TCPAddr, TCPConnection, resolve_internet_addr -from .socket import Socket -from .address import split_host_port - - -@value -struct Dialer: - var local_address: TCPAddr - - fn dial(self, network: String, address: String) raises -> TCPConnection: - var tcp_addr = resolve_internet_addr(network, address) - var socket = Socket(local_address=self.local_address) - socket.connect(tcp_addr.ip, tcp_addr.port) - print(String("Connected to ") + socket.remote_address) - return TCPConnection(socket^) - - -fn dial_tcp(network: String, remote_address: TCPAddr) raises -> TCPConnection: - """Connects to the address on the named network. - - The network must be "tcp", "tcp4", or "tcp6". - Args: - network: The network type. - remote_address: The remote address to connect to. - - Returns: - The TCP connection. - """ - # TODO: Add conversion of domain name to ip address - return Dialer(remote_address).dial(network, remote_address.ip + ":" + str(remote_address.port)) - - -fn dial_tcp(network: String, remote_address: String) raises -> TCPConnection: - """Connects to the address on the named network. - - The network must be "tcp", "tcp4", or "tcp6". - Args: - network: The network type. - remote_address: The remote address to connect to. - - Returns: - The TCP connection. - """ - var address = split_host_port(remote_address) - return Dialer(TCPAddr(address.host, address.port)).dial(network, remote_address) diff --git a/external/gojo/net/fd.mojo b/external/gojo/net/fd.mojo index 6e4fc62..7d50ee3 100644 --- a/external/gojo/net/fd.mojo +++ b/external/gojo/net/fd.mojo @@ -1,34 +1,26 @@ -from collections.optional import Optional import ..io -from ..builtins import Byte -from ..syscall.file import close -from ..syscall.types import c_char -from ..syscall.net import ( +from ..syscall import ( recv, send, - strlen, + close, + FileDescriptorBase, ) alias O_RDWR = 0o2 -trait FileDescriptorBase(io.Reader, io.Writer, io.Closer): - ... - - struct FileDescriptor(FileDescriptorBase): var fd: Int var is_closed: Bool - # This takes ownership of a POSIX file descriptor. - fn __moveinit__(inout self, owned existing: Self): - self.fd = existing.fd - self.is_closed = existing.is_closed - fn __init__(inout self, fd: Int): self.fd = fd self.is_closed = False + fn __moveinit__(inout self, owned existing: Self): + self.fd = existing.fd + self.is_closed = existing.is_closed + fn __del__(owned self): if not self.is_closed: var err = self.close() @@ -44,33 +36,37 @@ struct FileDescriptor(FileDescriptorBase): self.is_closed = True return Error() - fn dup(self) -> Self: - """Duplicate the file descriptor.""" - var new_fd = external_call["dup", Int, Int](self.fd) - return Self(new_fd) - - # TODO: Need faster approach to copying data from the file descriptor to the buffer. - fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): """Receive data from the file descriptor and write it to the buffer provided.""" - var ptr = Pointer[UInt8]().alloc(dest.capacity) - var bytes_received = recv(self.fd, ptr, dest.capacity, 0) + var bytes_received = recv( + self.fd, + dest.unsafe_ptr() + len(dest), + capacity - len(dest), + 0, + ) + if bytes_received == 0: + return bytes_received, io.EOF + if bytes_received == -1: return 0, Error("Failed to receive message from socket.") + dest._len += bytes_received + + return bytes_received, Error() - var int8_ptr = ptr.bitcast[Int8]() - for i in range(bytes_received): - dest.append(int8_ptr[i]) + fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): + """Receive data from the file descriptor and write it to the buffer provided.""" + var span = Span(dest) - if bytes_received < dest.capacity: - return bytes_received, Error(io.EOF) + var bytes_read: Int + var err: Error + bytes_read, err = self._read(span, dest.capacity) + dest.size += bytes_read - return bytes_received, Error() + return bytes_read, err - fn write(inout self, src: List[Byte]) -> (Int, Error): + fn write(inout self, src: Span[UInt8]) -> (Int, Error): """Write data from the buffer to the file descriptor.""" - var header_pointer = Pointer[Int8](src.data.address).bitcast[UInt8]() - - var bytes_sent = send(self.fd, header_pointer, strlen(header_pointer), 0) + var bytes_sent = send(self.fd, src.unsafe_ptr(), len(src), 0) if bytes_sent == -1: return 0, Error("Failed to send message") diff --git a/external/gojo/net/ip.mojo b/external/gojo/net/ip.mojo index 76a56bd..7fe8dbe 100644 --- a/external/gojo/net/ip.mojo +++ b/external/gojo/net/ip.mojo @@ -1,17 +1,18 @@ from utils.variant import Variant +from utils.static_tuple import StaticTuple from sys.info import os_is_linux, os_is_macos -from ..syscall.types import ( +from ..syscall import ( c_int, c_char, c_void, c_uint, -) -from ..syscall.net import ( addrinfo, addrinfo_unix, - AF_INET, - SOCK_STREAM, - AI_PASSIVE, + AddressFamily, + AddressInformation, + SocketOptions, + SocketType, + ProtocolFamily, sockaddr, sockaddr_in, htons, @@ -21,75 +22,59 @@ from ..syscall.net import ( getaddrinfo, getaddrinfo_unix, gai_strerror, - to_char_ptr, - c_charptr_to_string, ) +from .address import HostPort alias AddrInfo = Variant[addrinfo, addrinfo_unix] fn get_addr_info(host: String) raises -> AddrInfo: - var status: Int32 = 0 if os_is_macos(): - var servinfo = Pointer[addrinfo]().alloc(1) - servinfo.store(addrinfo()) - var hints = addrinfo() - hints.ai_family = AF_INET - hints.ai_socktype = SOCK_STREAM - hints.ai_flags = AI_PASSIVE - - var host_ptr = to_char_ptr(host) + var servinfo = UnsafePointer[addrinfo]().alloc(1) + servinfo[0] = addrinfo() + var hints = addrinfo( + ai_family=AddressFamily.AF_INET, + ai_socktype=SocketType.SOCK_STREAM, + ai_flags=AddressInformation.AI_PASSIVE, + ) var status = getaddrinfo( - host_ptr, - Pointer[UInt8](), - Pointer.address_of(hints), - Pointer.address_of(servinfo), + host.unsafe_ptr(), + UnsafePointer[UInt8](), + UnsafePointer.address_of(hints), + UnsafePointer.address_of(servinfo), ) if status != 0: print("getaddrinfo failed to execute with status:", status) - var msg_ptr = gai_strerror(c_int(status)) - _ = external_call["printf", c_int, Pointer[c_char], Pointer[c_char]]( - to_char_ptr("gai_strerror: %s"), msg_ptr - ) - var msg = c_charptr_to_string(msg_ptr) - print("getaddrinfo error message: ", msg) if not servinfo: print("servinfo is null") raise Error("Failed to get address info. Pointer to addrinfo is null.") - return servinfo.load() + return servinfo.take_pointee() elif os_is_linux(): - var servinfo = Pointer[addrinfo_unix]().alloc(1) - servinfo.store(addrinfo_unix()) - var hints = addrinfo_unix() - hints.ai_family = AF_INET - hints.ai_socktype = SOCK_STREAM - hints.ai_flags = AI_PASSIVE - - var host_ptr = to_char_ptr(host) + var servinfo = UnsafePointer[addrinfo_unix]().alloc(1) + servinfo[0] = addrinfo_unix() + var hints = addrinfo_unix( + ai_family=AddressFamily.AF_INET, + ai_socktype=SocketType.SOCK_STREAM, + ai_flags=AddressInformation.AI_PASSIVE, + ) var status = getaddrinfo_unix( - host_ptr, - Pointer[UInt8](), - Pointer.address_of(hints), - Pointer.address_of(servinfo), + host.unsafe_ptr(), + UnsafePointer[UInt8](), + UnsafePointer.address_of(hints), + UnsafePointer.address_of(servinfo), ) if status != 0: print("getaddrinfo failed to execute with status:", status) - var msg_ptr = gai_strerror(c_int(status)) - _ = external_call["printf", c_int, Pointer[c_char], Pointer[c_char]]( - to_char_ptr("gai_strerror: %s"), msg_ptr - ) - var msg = c_charptr_to_string(msg_ptr) - print("getaddrinfo error message: ", msg) if not servinfo: print("servinfo is null") raise Error("Failed to get address info. Pointer to addrinfo is null.") - return servinfo.load() + return servinfo.take_pointee() else: raise Error("Windows is not supported yet! Sorry!") @@ -98,26 +83,26 @@ fn get_ip_address(host: String) raises -> String: """Get the IP address of a host.""" # Call getaddrinfo to get the IP address of the host. var result = get_addr_info(host) - var ai_addr: Pointer[sockaddr] + var ai_addr: UnsafePointer[sockaddr] var address_family: Int32 = 0 var address_length: UInt32 = 0 if result.isa[addrinfo](): - var addrinfo = result.get[addrinfo]() - ai_addr = addrinfo[].ai_addr - address_family = addrinfo[].ai_family - address_length = addrinfo[].ai_addrlen + var addrinfo = result[addrinfo] + ai_addr = addrinfo.ai_addr + address_family = addrinfo.ai_family + address_length = addrinfo.ai_addrlen else: - var addrinfo = result.get[addrinfo_unix]() - ai_addr = addrinfo[].ai_addr - address_family = addrinfo[].ai_family - address_length = addrinfo[].ai_addrlen + var addrinfo = result[addrinfo_unix] + ai_addr = addrinfo.ai_addr + address_family = addrinfo.ai_family + address_length = addrinfo.ai_addrlen if not ai_addr: print("ai_addr is null") raise Error("Failed to get IP address. getaddrinfo was called successfully, but ai_addr is null.") # Cast sockaddr struct to sockaddr_in struct and convert the binary IP to a string using inet_ntop. - var addr_in = ai_addr.bitcast[sockaddr_in]().load() + var addr_in = ai_addr.bitcast[sockaddr_in]().take_pointee() return convert_binary_ip_to_string(addr_in.sin_addr.s_addr, address_family, address_length).strip() @@ -131,12 +116,12 @@ fn convert_binary_port_to_int(port: UInt16) -> Int: fn convert_ip_to_binary(ip_address: String, address_family: Int) -> UInt32: - var ip_buffer = Pointer[c_void].alloc(4) - var status = inet_pton(address_family, to_char_ptr(ip_address), ip_buffer) + var ip_buffer = UnsafePointer[UInt8].alloc(4) + var status = inet_pton(address_family, ip_address.unsafe_ptr(), ip_buffer) if status == -1: print("Failed to convert IP address to binary") - return ip_buffer.bitcast[c_uint]().load() + return ip_buffer.bitcast[c_uint]().take_pointee() fn convert_binary_ip_to_string(owned ip_address: UInt32, address_family: Int32, address_length: UInt32) -> String: @@ -152,21 +137,20 @@ fn convert_binary_ip_to_string(owned ip_address: UInt32, address_family: Int32, """ # It seems like the len of the buffer depends on the length of the string IP. # Allocating 10 works for localhost (127.0.0.1) which I suspect is 9 bytes + 1 null terminator byte. So max should be 16 (15 + 1). - var ip_buffer = Pointer[c_void].alloc(16) - var ip_address_ptr = Pointer.address_of(ip_address).bitcast[c_void]() + var ip_buffer = UnsafePointer[c_void].alloc(16) + var ip_address_ptr = UnsafePointer.address_of(ip_address).bitcast[c_void]() _ = inet_ntop(address_family, ip_address_ptr, ip_buffer, 16) - var string_buf = ip_buffer.bitcast[Int8]() var index = 0 while True: - if string_buf[index] == 0: + if ip_buffer[index] == 0: break index += 1 - return StringRef(string_buf, index) + return StringRef(ip_buffer, index) -fn build_sockaddr_pointer(ip_address: String, port: Int, address_family: Int) -> Pointer[sockaddr]: +fn build_sockaddr_pointer(ip_address: String, port: Int, address_family: Int) -> UnsafePointer[sockaddr]: """Build a sockaddr pointer from an IP address and port number. https://learn.microsoft.com/en-us/windows/win32/winsock/sockaddr-2 https://learn.microsoft.com/en-us/windows/win32/api/ws2def/ns-ws2def-sockaddr_in. @@ -175,4 +159,28 @@ fn build_sockaddr_pointer(ip_address: String, port: Int, address_family: Int) -> var bin_ip = convert_ip_to_binary(ip_address, address_family) var ai = sockaddr_in(address_family, bin_port, bin_ip, StaticTuple[c_char, 8]()) - return Pointer[sockaddr_in].address_of(ai).bitcast[sockaddr]() + return UnsafePointer[sockaddr_in].address_of(ai).bitcast[sockaddr]() + + +fn convert_sockaddr_to_host_port(sockaddr: UnsafePointer[sockaddr]) -> (HostPort, Error): + """Casts a sockaddr pointer to a sockaddr_in pointer and converts the binary IP and port to a string and int respectively. + + Args: + sockaddr: The sockaddr pointer to convert. + + Returns: + A tuple containing the HostPort and an Error if any occurred,. + """ + if not sockaddr: + return HostPort(), Error("sockaddr is null, nothing to convert.") + + # Cast sockaddr struct to sockaddr_in to convert binary IP to string. + var addr_in = sockaddr.bitcast[sockaddr_in]().take_pointee() + + return ( + HostPort( + host=convert_binary_ip_to_string(addr_in.sin_addr.s_addr, AddressFamily.AF_INET, 16), + port=convert_binary_port_to_int(addr_in.sin_port), + ), + Error(), + ) diff --git a/external/gojo/net/net.mojo b/external/gojo/net/net.mojo deleted file mode 100644 index 324681a..0000000 --- a/external/gojo/net/net.mojo +++ /dev/null @@ -1,130 +0,0 @@ -from memory._arc import Arc -import ..io -from ..builtins import Byte -from .socket import Socket -from .address import Addr, TCPAddr - -alias DEFAULT_BUFFER_SIZE = 4096 - - -trait Conn(io.Writer, io.Reader, io.Closer): - fn __init__(inout self, owned socket: Socket): - ... - - """Conn is a generic stream-oriented network connection.""" - - fn local_address(self) -> TCPAddr: - """Returns the local network address, if known.""" - ... - - fn remote_address(self) -> TCPAddr: - """Returns the local network address, if known.""" - ... - - # fn set_deadline(self, t: time.Time) -> Error: - # """Sets the read and write deadlines associated - # with the connection. It is equivalent to calling both - # SetReadDeadline and SetWriteDeadline. - - # A deadline is an absolute time after which I/O operations - # fail instead of blocking. The deadline applies to all future - # and pending I/O, not just the immediately following call to - # read or write. After a deadline has been exceeded, the - # connection can be refreshed by setting a deadline in the future. - - # If the deadline is exceeded a call to read or write or to other - # I/O methods will return an error that wraps os.ErrDeadlineExceeded. - # This can be tested using errors.Is(err, os.ErrDeadlineExceeded). - # The error's Timeout method will return true, but note that there - # are other possible errors for which the Timeout method will - # return true even if the deadline has not been exceeded. - - # An idle timeout can be implemented by repeatedly extending - # the deadline after successful read or write calls. - - # A zero value for t means I/O operations will not time out.""" - # ... - - # fn set_read_deadline(self, t: time.Time) -> Error: - # """Sets the deadline for future read calls - # and any currently-blocked read call. - # A zero value for t means read will not time out.""" - # ... - - # fn set_write_deadline(self, t: time.Time) -> Error: - # """Sets the deadline for future write calls - # and any currently-blocked write call. - # Even if write times out, it may return n > 0, indicating that - # some of the data was successfully written. - # A zero value for t means write will not time out.""" - # ... - - -@value -struct Connection(Conn): - """Connection is a concrete generic stream-oriented network connection. - It is used as the internal connection for structs like TCPConnection. - - Args: - fd: The file descriptor of the connection. - """ - - var fd: Arc[Socket] - - fn __init__(inout self, owned socket: Socket): - self.fd = Arc(socket^) - - fn read(inout self, inout dest: List[Byte]) -> (Int, Error): - """Reads data from the underlying file descriptor. - - Args: - dest: The buffer to read data into. - - Returns: - The number of bytes read, or an error if one occurred. - """ - var bytes_written: Int = 0 - var err = Error() - bytes_written, err = self.fd[].read(dest) - if err: - if str(err) != io.EOF: - return 0, err - - return bytes_written, err - - fn write(inout self, src: List[Byte]) -> (Int, Error): - """Writes data to the underlying file descriptor. - - Args: - src: The buffer to read data into. - - Returns: - The number of bytes written, or an error if one occurred. - """ - var bytes_read: Int = 0 - var err = Error() - bytes_read, err = self.fd[].write(src) - if err: - return 0, err - - return bytes_read, err - - fn close(inout self) -> Error: - """Closes the underlying file descriptor. - - Returns: - An error if one occurred, or None if the file descriptor was closed successfully. - """ - return self.fd[].close() - - fn local_address(self) -> TCPAddr: - """Returns the local network address. - The Addr returned is shared by all invocations of local_address, so do not modify it. - """ - return self.fd[].local_address - - fn remote_address(self) -> TCPAddr: - """Returns the remote network address. - The Addr returned is shared by all invocations of remote_address, so do not modify it. - """ - return self.fd[].remote_address diff --git a/external/gojo/net/socket.mojo b/external/gojo/net/socket.mojo index 594cc23..a675d65 100644 --- a/external/gojo/net/socket.mojo +++ b/external/gojo/net/socket.mojo @@ -1,34 +1,28 @@ -from collections.optional import Optional -from ..builtins import Byte -from ..syscall.file import close -from ..syscall.types import ( - c_void, - c_uint, - c_char, - c_int, -) -from ..syscall.net import ( +from ..syscall import ( sockaddr, sockaddr_in, addrinfo, addrinfo_unix, socklen_t, + c_void, + c_uint, + c_char, + c_int, socket, connect, recv, + recvfrom, send, + sendto, shutdown, inet_pton, inet_ntoa, inet_ntop, - to_char_ptr, htons, ntohs, - strlen, getaddrinfo, getaddrinfo_unix, gai_strerror, - c_charptr_to_string, bind, listen, accept, @@ -36,21 +30,22 @@ from ..syscall.net import ( getsockopt, getsockname, getpeername, - AF_INET, - SOCK_STREAM, + AddressFamily, + AddressInformation, + SocketOptions, + SocketType, SHUT_RDWR, - AI_PASSIVE, SOL_SOCKET, - SO_REUSEADDR, - SO_RCVTIMEO, + close, ) from .fd import FileDescriptor, FileDescriptorBase from .ip import ( convert_binary_ip_to_string, build_sockaddr_pointer, convert_binary_port_to_int, + convert_sockaddr_to_host_port, ) -from .address import Addr, TCPAddr, HostPort +from .address import Addr, BaseAddr, HostPort alias SocketClosedError = Error("Socket: Socket is already closed") @@ -66,21 +61,21 @@ struct Socket(FileDescriptorBase): protocol: The protocol. """ - var sockfd: FileDescriptor + var fd: FileDescriptor var address_family: Int - var socket_type: UInt8 + var socket_type: Int32 var protocol: UInt8 - var local_address: TCPAddr - var remote_address: TCPAddr + var local_address: BaseAddr + var remote_address: BaseAddr var _closed: Bool var _is_connected: Bool fn __init__( inout self, - local_address: TCPAddr = TCPAddr(), - remote_address: TCPAddr = TCPAddr(), - address_family: Int = AF_INET, - socket_type: UInt8 = SOCK_STREAM, + local_address: BaseAddr = BaseAddr(), + remote_address: BaseAddr = BaseAddr(), + address_family: Int = AddressFamily.AF_INET, + socket_type: Int32 = SocketType.SOCK_STREAM, protocol: UInt8 = 0, ) raises: """Create a new socket object. @@ -96,10 +91,10 @@ struct Socket(FileDescriptorBase): self.socket_type = socket_type self.protocol = protocol - var fd = socket(address_family, SOCK_STREAM, 0) + var fd = socket(address_family, socket_type, 0) if fd == -1: raise Error("Socket creation error") - self.sockfd = FileDescriptor(int(fd)) + self.fd = FileDescriptor(int(fd)) self.local_address = local_address self.remote_address = remote_address self._closed = False @@ -109,10 +104,10 @@ struct Socket(FileDescriptorBase): inout self, fd: Int32, address_family: Int, - socket_type: UInt8, + socket_type: Int32, protocol: UInt8, - local_address: TCPAddr = TCPAddr(), - remote_address: TCPAddr = TCPAddr(), + local_address: BaseAddr = BaseAddr(), + remote_address: BaseAddr = BaseAddr(), ): """ Create a new socket object when you already have a socket file descriptor. Typically through socket.accept(). @@ -122,10 +117,10 @@ struct Socket(FileDescriptorBase): address_family: The address family of the socket. socket_type: The socket type. protocol: The protocol. - local_address: Local address of socket. - remote_address: Remote address of port. + local_address: The local address of the socket (local address if bound). + remote_address: The remote address of the socket (peer's address if connected). """ - self.sockfd = FileDescriptor(int(fd)) + self.fd = FileDescriptor(int(fd)) self.address_family = address_family self.socket_type = socket_type self.protocol = protocol @@ -135,7 +130,7 @@ struct Socket(FileDescriptorBase): self._is_connected = True fn __moveinit__(inout self, owned existing: Self): - self.sockfd = existing.sockfd^ + self.fd = existing.fd^ self.address_family = existing.address_family self.socket_type = existing.socket_type self.protocol = existing.protocol @@ -151,37 +146,59 @@ struct Socket(FileDescriptorBase): # if self._is_connected: # self.shutdown() # if not self._closed: - # self.close() + # var err = self.close() + # if err: + # raise err fn __del__(owned self): if self._is_connected: self.shutdown() if not self._closed: var err = self.close() - _ = self.sockfd.fd + _ = self.fd.fd if err: print("Failed to close socket during deletion:", str(err)) - @always_inline - fn accept(self) raises -> Self: + fn local_address_as_udp(self) -> UDPAddr: + return UDPAddr(self.local_address) + + fn local_address_as_tcp(self) -> TCPAddr: + return TCPAddr(self.local_address) + + fn remote_address_as_udp(self) -> UDPAddr: + return UDPAddr(self.remote_address) + + fn remote_address_as_tcp(self) -> TCPAddr: + return TCPAddr(self.remote_address) + + fn accept(self) raises -> Socket: """Accept a connection. The socket must be bound to an address and listening for connections. The return value is a connection where conn is a new socket object usable to send and receive data on the connection, and address is the address bound to the socket on the other end of the connection. """ - var their_addr_ptr = Pointer[sockaddr].alloc(1) + var remote_address_ptr = UnsafePointer[sockaddr].alloc(1) var sin_size = socklen_t(sizeof[socklen_t]()) - var new_sockfd = accept(self.sockfd.fd, their_addr_ptr, Pointer[socklen_t].address_of(sin_size)) - if new_sockfd == -1: + var new_fd = accept( + self.fd.fd, + remote_address_ptr, + UnsafePointer[socklen_t].address_of(sin_size), + ) + if new_fd == -1: raise Error("Failed to accept connection") - var remote = self.get_peer_name() - return Self( - new_sockfd, + var remote: HostPort + var err: Error + remote, err = convert_sockaddr_to_host_port(remote_address_ptr) + if err: + raise err + + return Socket( + new_fd, self.address_family, self.socket_type, self.protocol, self.local_address, - TCPAddr(remote.host, remote.port), + BaseAddr(remote.host, remote.port), ) fn listen(self, backlog: Int = 0) raises: @@ -193,17 +210,16 @@ struct Socket(FileDescriptorBase): var queued = backlog if backlog < 0: queued = 0 - if listen(self.sockfd.fd, queued) == -1: + if listen(self.fd.fd, queued) == -1: raise Error("Failed to listen for connections") - @always_inline fn bind(inout self, address: String, port: Int) raises: """Bind the socket to address. The socket must not already be bound. (The format of address depends on the address family). When a socket is created with Socket(), it exists in a name space (address family) but has no address assigned to it. bind() assigns the address specified by addr to the socket referred to - by the file descriptor sockfd. addrlen specifies the size, in + by the file descriptor fd. addrlen specifies the size, in bytes, of the address structure pointed to by addr. Traditionally, this operation is called 'assigning a name to a socket'. @@ -214,19 +230,17 @@ struct Socket(FileDescriptorBase): """ var sockaddr_pointer = build_sockaddr_pointer(address, port, self.address_family) - if bind(self.sockfd.fd, sockaddr_pointer, sizeof[sockaddr_in]()) == -1: - _ = shutdown(self.sockfd.fd, SHUT_RDWR) + if bind(self.fd.fd, sockaddr_pointer, sizeof[sockaddr_in]()) == -1: + _ = shutdown(self.fd.fd, SHUT_RDWR) raise Error("Binding socket failed. Wait a few seconds and try again?") var local = self.get_sock_name() - self.local_address = TCPAddr(local.host, local.port) + self.local_address = BaseAddr(local.host, local.port) - @always_inline fn file_no(self) -> Int32: """Return the file descriptor of the socket.""" - return self.sockfd.fd + return self.fd.fd - @always_inline fn get_sock_name(self) raises -> HostPort: """Return the address of the socket.""" if self._closed: @@ -234,45 +248,45 @@ struct Socket(FileDescriptorBase): # TODO: Add check to see if the socket is bound and error if not. - var local_address_ptr = Pointer[sockaddr].alloc(1) + var local_address_ptr = UnsafePointer[sockaddr].alloc(1) var local_address_ptr_size = socklen_t(sizeof[sockaddr]()) var status = getsockname( - self.sockfd.fd, + self.fd.fd, local_address_ptr, - Pointer[socklen_t].address_of(local_address_ptr_size), + UnsafePointer[socklen_t].address_of(local_address_ptr_size), ) if status == -1: raise Error("Socket.get_sock_name: Failed to get address of local socket.") - var addr_in = local_address_ptr.bitcast[sockaddr_in]().load() + var addr_in = local_address_ptr.bitcast[sockaddr_in]().take_pointee() return HostPort( - host=convert_binary_ip_to_string(addr_in.sin_addr.s_addr, AF_INET, 16), + host=convert_binary_ip_to_string(addr_in.sin_addr.s_addr, AddressFamily.AF_INET, 16), port=convert_binary_port_to_int(addr_in.sin_port), ) - fn get_peer_name(self) raises -> HostPort: + fn get_peer_name(self) -> (HostPort, Error): """Return the address of the peer connected to the socket.""" if self._closed: - raise SocketClosedError + return HostPort(), SocketClosedError # TODO: Add check to see if the socket is bound and error if not. - var remote_address_ptr = Pointer[sockaddr].alloc(1) + var remote_address_ptr = UnsafePointer[sockaddr].alloc(1) var remote_address_ptr_size = socklen_t(sizeof[sockaddr]()) var status = getpeername( - self.sockfd.fd, + self.fd.fd, remote_address_ptr, - Pointer[socklen_t].address_of(remote_address_ptr_size), + UnsafePointer[socklen_t].address_of(remote_address_ptr_size), ) if status == -1: - raise Error("Socket.get_peer_name: Failed to get address of remote socket.") + return HostPort(), Error("Socket.get_peer_name: Failed to get address of remote socket.") - # Cast sockaddr struct to sockaddr_in to convert binary IP to string. - var addr_in = remote_address_ptr.bitcast[sockaddr_in]().load() + var remote: HostPort + var err: Error + remote, err = convert_sockaddr_to_host_port(remote_address_ptr) + if err: + return HostPort(), err - return HostPort( - host=convert_binary_ip_to_string(addr_in.sin_addr.s_addr, AF_INET, 16), - port=convert_binary_port_to_int(addr_in.sin_port), - ) + return remote, Error() fn get_socket_option(self, option_name: Int) raises -> Int: """Return the value of the given socket option. @@ -280,11 +294,11 @@ struct Socket(FileDescriptorBase): Args: option_name: The socket option to get. """ - var option_value_pointer = Pointer[c_void].alloc(1) + var option_value_pointer = UnsafePointer[c_void].alloc(1) var option_len = socklen_t(sizeof[socklen_t]()) - var option_len_pointer = Pointer.address_of(option_len) + var option_len_pointer = UnsafePointer.address_of(option_len) var status = getsockopt( - self.sockfd.fd, + self.fd.fd, SOL_SOCKET, option_name, option_value_pointer, @@ -293,7 +307,7 @@ struct Socket(FileDescriptorBase): if status == -1: raise Error("Socket.get_sock_opt failed with status: " + str(status)) - return option_value_pointer.bitcast[Int]().load() + return option_value_pointer.bitcast[Int]().take_pointee() fn set_socket_option(self, option_name: Int, owned option_value: UInt8 = 1) raises: """Return the value of the given socket option. @@ -302,13 +316,19 @@ struct Socket(FileDescriptorBase): option_name: The socket option to set. option_value: The value to set the socket option to. """ - var option_value_pointer = Pointer[c_void].address_of(option_value) + var option_value_pointer = UnsafePointer[c_void].address_of(option_value) var option_len = sizeof[socklen_t]() - var status = setsockopt(self.sockfd.fd, SOL_SOCKET, option_name, option_value_pointer, option_len) + var status = setsockopt( + self.fd.fd, + SOL_SOCKET, + option_name, + option_value_pointer, + option_len, + ) if status == -1: raise Error("Socket.set_sock_opt failed with status: " + str(status)) - fn connect(inout self, address: String, port: Int) raises: + fn connect(inout self, address: String, port: Int) -> Error: """Connect to a remote socket at address. Args: @@ -317,14 +337,20 @@ struct Socket(FileDescriptorBase): """ var sockaddr_pointer = build_sockaddr_pointer(address, port, self.address_family) - if connect(self.sockfd.fd, sockaddr_pointer, sizeof[sockaddr_in]()) == -1: + if connect(self.fd.fd, sockaddr_pointer, sizeof[sockaddr_in]()) == -1: self.shutdown() - raise Error("Socket.connect: Failed to connect to the remote socket at: " + address + ":" + str(port)) + return Error("Socket.connect: Failed to connect to the remote socket at: " + address + ":" + str(port)) + + var remote: HostPort + var err: Error + remote, err = self.get_peer_name() + if err: + return err - var remote = self.get_peer_name() - self.remote_address = TCPAddr(remote.host, remote.port) + self.remote_address = BaseAddr(remote.host, remote.port) + return Error() - fn write(inout self: Self, src: List[Byte]) -> (Int, Error): + fn write(inout self: Self, src: Span[UInt8]) -> (Int, Error): """Send data to the socket. The socket must be connected to a remote socket. Args: @@ -333,42 +359,38 @@ struct Socket(FileDescriptorBase): Returns: The number of bytes sent. """ - var bytes_written: Int - var err: Error - bytes_written, err = self.sockfd.write(src) - if err: - return 0, err - - return bytes_written, Error() + return self.fd.write(src) - fn send_all(self, src: List[Byte], max_attempts: Int = 3) raises: + fn send_all(self, src: Span[UInt8], max_attempts: Int = 3) -> Error: """Send data to the socket. The socket must be connected to a remote socket. Args: src: The data to send. max_attempts: The maximum number of attempts to send the data. """ - var header_pointer = Pointer[Int8](src.data.address).bitcast[UInt8]() + var bytes_to_send = len(src) var total_bytes_sent = 0 var attempts = 0 # Try to send all the data in the buffer. If it did not send all the data, keep trying but start from the offset of the last successful send. while total_bytes_sent < len(src): if attempts > max_attempts: - raise Error("Failed to send message after " + String(max_attempts) + " attempts.") + return Error("Failed to send message after " + str(max_attempts) + " attempts.") var bytes_sent = send( - self.sockfd.fd, - header_pointer.offset(total_bytes_sent), - strlen(header_pointer.offset(total_bytes_sent)), + self.fd.fd, + src.unsafe_ptr() + total_bytes_sent, + bytes_to_send - total_bytes_sent, 0, ) if bytes_sent == -1: - raise Error("Failed to send message, wrote" + String(total_bytes_sent) + "bytes before failing.") + return Error("Failed to send message, wrote" + String(total_bytes_sent) + "bytes before failing.") total_bytes_sent += bytes_sent attempts += 1 - fn send_to(inout self, src: List[Byte], address: String, port: Int) raises -> Int: + return Error() + + fn send_to(inout self, src: Span[UInt8], address: String, port: Int) -> (Int, Error): """Send data to the a remote address by connecting to the remote socket before sending. The socket must be not already be connected to a remote socket. @@ -377,30 +399,141 @@ struct Socket(FileDescriptorBase): address: The IP address to connect to. port: The port number to connect to. """ - var header_pointer = Pointer[Int8](src.data.address).bitcast[UInt8]() - self.connect(address, port) - var bytes_written: Int + var bytes_sent = sendto( + self.fd.fd, + src.unsafe_ptr(), + len(src), + 0, + build_sockaddr_pointer(address, port, self.address_family), + sizeof[sockaddr_in](), + ) + + if bytes_sent == -1: + return 0, Error("Socket.send_to: Failed to send message to remote socket at: " + address + ":" + str(port)) + + return bytes_sent, Error() + + fn receive(inout self, size: Int = io.BUFFER_SIZE) -> (List[UInt8], Error): + """Receive data from the socket into the buffer with capacity of `size` bytes. + + Args: + size: The size of the buffer to receive data into. + + Returns: + The buffer with the received data, and an error if one occurred. + """ + var buffer = UnsafePointer[UInt8].alloc(size) + var bytes_received = recv( + self.fd.fd, + buffer, + size, + 0, + ) + if bytes_received == -1: + return List[UInt8](), Error("Socket.receive: Failed to receive message from socket.") + + var bytes = List[UInt8](unsafe_pointer=buffer, size=bytes_received, capacity=size) + if bytes_received < bytes.capacity: + return bytes, io.EOF + + return bytes, Error() + + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): + """Receive data from the socket into the buffer dest. Equivalent to recv_into(). + + Args: + dest: The buffer to read data into. + capacity: The capacity of the buffer. + + Returns: + The number of bytes read, and an error if one occurred. + """ + return self.fd._read(dest, capacity) + + fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): + """Receive data from the socket into the buffer dest. Equivalent to recv_into(). + + Args: + dest: The buffer to read data into. + + Returns: + The number of bytes read, and an error if one occurred. + """ + var span = Span(dest) + + var bytes_read: Int + var err: Error + bytes_read, err = self._read(span, dest.capacity) + dest.size += bytes_read + + return bytes_read, err + + fn receive_from(inout self, size: Int = io.BUFFER_SIZE) -> (List[UInt8], HostPort, Error): + """Receive data from the socket into the buffer dest. + + Args: + size: The size of the buffer to receive data into. + + Returns: + The number of bytes read, the remote address, and an error if one occurred. + """ + var remote_address_ptr = UnsafePointer[sockaddr].alloc(1) + var remote_address_ptr_size = socklen_t(sizeof[sockaddr]()) + var buffer = UnsafePointer[UInt8].alloc(size) + var bytes_received = recvfrom( + self.fd.fd, + buffer, + size, + 0, + remote_address_ptr, + UnsafePointer[socklen_t].address_of(remote_address_ptr_size), + ) + + if bytes_received == -1: + return List[UInt8](), HostPort(), Error("Failed to read from socket, received a -1 response.") + + var remote: HostPort var err: Error - bytes_written, err = self.write(src) + remote, err = convert_sockaddr_to_host_port(remote_address_ptr) if err: - raise err - return bytes_written + return List[UInt8](), HostPort(), err + + var bytes = List[UInt8](unsafe_pointer=buffer, size=bytes_received, capacity=size) + if bytes_received < bytes.capacity: + return bytes, remote, io.EOF - fn read(inout self, inout dest: List[Byte]) -> (Int, Error): - """Receive data from the socket.""" - # Not ideal since we can't use the pointer from the List[Byte] struct directly. So we use a temporary pointer to receive the data. - # Then we copy all the data over. - var bytes_written: Int + return bytes, remote, Error() + + fn receive_from_into(inout self, inout dest: List[UInt8]) -> (Int, HostPort, Error): + """Receive data from the socket into the buffer dest.""" + var remote_address_ptr = UnsafePointer[sockaddr].alloc(1) + var remote_address_ptr_size = socklen_t(sizeof[sockaddr]()) + var bytes_read = recvfrom( + self.fd.fd, + dest.unsafe_ptr() + dest.size, + dest.capacity - dest.size, + 0, + remote_address_ptr, + UnsafePointer[socklen_t].address_of(remote_address_ptr_size), + ) + dest.size += bytes_read + + if bytes_read == -1: + return 0, HostPort(), Error("Socket.receive_from_into: Failed to read from socket, received a -1 response.") + + var remote: HostPort var err: Error - bytes_written, err = self.sockfd.read(dest) + remote, err = convert_sockaddr_to_host_port(remote_address_ptr) if err: - if str(err) != "EOF": - return 0, err + return 0, HostPort(), err - return bytes_written, Error() + if bytes_read < dest.capacity: + return bytes_read, remote, io.EOF + + return bytes_read, remote, Error() fn shutdown(self): - _ = shutdown(self.sockfd.fd, SHUT_RDWR) + _ = shutdown(self.fd.fd, SHUT_RDWR) fn close(inout self) -> Error: """Mark the socket closed. @@ -408,7 +541,7 @@ struct Socket(FileDescriptorBase): The remote end will receive no more data (after queued data is flushed). """ self.shutdown() - var err = self.sockfd.close() + var err = self.fd.close() if err: return err @@ -416,17 +549,21 @@ struct Socket(FileDescriptorBase): return Error() # TODO: Trying to set timeout fails, but some other options don't? - # fn get_timeout(self) raises -> Seconds: + # fn get_timeout(self) raises -> Int: # """Return the timeout value for the socket.""" - # return self.get_socket_option(SO_RCVTIMEO) + # return self.get_socket_option(SocketOptions.SO_RCVTIMEO) - # fn set_timeout(self, owned duration: Seconds) raises: + # fn set_timeout(self, owned duration: Int) raises: # """Set the timeout value for the socket. # Args: # duration: Seconds - The timeout duration in seconds. # """ - # self.set_socket_option(SO_RCVTIMEO, duration) - - fn send_file(self, file: FileHandle, offset: Int = 0) raises: - self.send_all(file.read_bytes()) + # self.set_socket_option(SocketOptions.SO_RCVTIMEO, duration) + + fn send_file(self, file: FileHandle) -> Error: + try: + var bytes = file.read_bytes() + return self.send_all(Span(bytes)) + except e: + return e diff --git a/external/gojo/net/tcp.mojo b/external/gojo/net/tcp.mojo index 6a59db8..9822c41 100644 --- a/external/gojo/net/tcp.mojo +++ b/external/gojo/net/tcp.mojo @@ -1,109 +1,95 @@ -from ..builtins import Byte -from ..syscall.net import SO_REUSEADDR -from .net import Connection, Conn -from .address import TCPAddr, NetworkType, split_host_port +from collections import InlineList +from ..syscall import SocketOptions +from .address import NetworkType, split_host_port, join_host_port, BaseAddr, resolve_internet_addr, HostPort from .socket import Socket -# Time in nanoseconds -alias Duration = Int -alias DEFAULT_BUFFER_SIZE = 4096 -alias DEFAULT_TCP_KEEP_ALIVE = Duration(15 * 1000 * 1000 * 1000) # 15 seconds - - -fn resolve_internet_addr(network: String, address: String) raises -> TCPAddr: - var host: String = "" - var port: String = "" - var portnum: Int = 0 - if ( - network == NetworkType.tcp.value - or network == NetworkType.tcp4.value - or network == NetworkType.tcp6.value - or network == NetworkType.udp.value - or network == NetworkType.udp4.value - or network == NetworkType.udp6.value - ): - if address != "": - var host_port = split_host_port(address) - host = host_port.host - port = host_port.port - portnum = atol(port.__str__()) - elif network == NetworkType.ip.value or network == NetworkType.ip4.value or network == NetworkType.ip6.value: - if address != "": - host = address - elif network == NetworkType.unix.value: - raise Error("Unix addresses not supported yet") - else: - raise Error("unsupported network type: " + network) - return TCPAddr(host, portnum) - - -# TODO: For now listener is paired with TCP until we need to support -# more than one type of Connection or Listener @value -struct ListenConfig(CollectionElement): - var keep_alive: Duration +struct TCPAddr(Addr): + """Addr struct representing a TCP address. - fn listen(self, network: String, address: String) raises -> TCPListener: - var tcp_addr = resolve_internet_addr(network, address) - var socket = Socket(local_address=tcp_addr) - socket.bind(tcp_addr.ip, tcp_addr.port) - socket.set_socket_option(SO_REUSEADDR, 1) - socket.listen() - print(String("Listening on ") + socket.local_address) - return TCPListener(socket^, self, network, address) + Args: + ip: IP address. + port: Port number. + zone: IPv6 addressing zone. + """ + var ip: String + var port: Int + var zone: String # IPv6 addressing zone -trait Listener(Movable): - # Raising here because a Result[Optional[Connection], Error] is funky. - fn accept(self) raises -> Connection: - ... + fn __init__(inout self, ip: String = "127.0.0.1", port: Int = 8000, zone: String = ""): + self.ip = ip + self.port = port + self.zone = zone - fn close(inout self) -> Error: - ... + fn __init__(inout self, addr: BaseAddr): + self.ip = addr.ip + self.port = addr.port + self.zone = addr.zone - fn addr(self) raises -> TCPAddr: - ... + fn __str__(self) -> String: + if self.zone != "": + return join_host_port(str(self.ip) + "%" + self.zone, str(self.port)) + return join_host_port(self.ip, str(self.port)) + fn network(self) -> String: + return NetworkType.tcp.value -@value -struct TCPConnection(Conn): + +struct TCPConnection(Movable): """TCPConn is an implementation of the Conn interface for TCP network connections. Args: connection: The underlying Connection. """ - var _connection: Connection - - fn __init__(inout self, connection: Connection): - self._connection = connection + var socket: Socket fn __init__(inout self, owned socket: Socket): - self._connection = Connection(socket^) + self.socket = socket^ fn __moveinit__(inout self, owned existing: Self): - self._connection = existing._connection^ + self.socket = existing.socket^ - fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): """Reads data from the underlying file descriptor. Args: dest: The buffer to read data into. + capacity: The capacity of the destination buffer. Returns: The number of bytes read, or an error if one occurred. """ - var bytes_written: Int - var err: Error - bytes_written, err = self._connection.read(dest) + var bytes_read: Int + var err = Error() + bytes_read, err = self.socket._read(dest, capacity) if err: - if str(err) != io.EOF: - return 0, err + if str(err) != str(io.EOF): + return bytes_read, err + + return bytes_read, err + + fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): + """Reads data from the underlying file descriptor. + + Args: + dest: The buffer to read data into. - return bytes_written, Error() + Returns: + The number of bytes read, or an error if one occurred. + """ + var span = Span(dest) - fn write(inout self, src: List[Byte]) -> (Int, Error): + var bytes_read: Int + var err: Error + bytes_read, err = self._read(span, dest.capacity) + dest.size += bytes_read + + return bytes_read, err + + fn write(inout self, src: Span[UInt8]) -> (Int, Error): """Writes data to the underlying file descriptor. Args: @@ -112,13 +98,7 @@ struct TCPConnection(Conn): Returns: The number of bytes written, or an error if one occurred. """ - var bytes_written: Int - var err: Error - bytes_written, err = self._connection.write(src) - if err: - return 0, err - - return bytes_written, Error() + return self.socket.write(src) fn close(inout self) -> Error: """Closes the underlying file descriptor. @@ -126,7 +106,7 @@ struct TCPConnection(Conn): Returns: An error if one occurred, or None if the file descriptor was closed successfully. """ - return self._connection.close() + return self.socket.close() fn local_address(self) -> TCPAddr: """Returns the local network address. @@ -135,7 +115,7 @@ struct TCPConnection(Conn): Returns: The local network address. """ - return self._connection.local_address() + return self.socket.local_address_as_tcp() fn remote_address(self) -> TCPAddr: """Returns the remote network address. @@ -144,7 +124,7 @@ struct TCPConnection(Conn): Returns: The remote network address. """ - return self._connection.remote_address() + return self.socket.remote_address_as_tcp() fn listen_tcp(network: String, local_address: TCPAddr) raises -> TCPListener: @@ -154,7 +134,12 @@ fn listen_tcp(network: String, local_address: TCPAddr) raises -> TCPListener: network: The network type. local_address: The local address to listen on. """ - return ListenConfig(DEFAULT_TCP_KEEP_ALIVE).listen(network, local_address.ip + ":" + str(local_address.port)) + var socket = Socket() + socket.bind(local_address.ip, local_address.port) + socket.set_socket_option(SocketOptions.SO_REUSEADDR, 1) + socket.listen() + # print(str("Listening on ") + str(socket.local_address_as_tcp())) + return TCPListener(socket^, network, local_address) fn listen_tcp(network: String, local_address: String) raises -> TCPListener: @@ -164,44 +149,106 @@ fn listen_tcp(network: String, local_address: String) raises -> TCPListener: network: The network type. local_address: The address to listen on. The format is "host:port". """ - return ListenConfig(DEFAULT_TCP_KEEP_ALIVE).listen(network, local_address) + var tcp_addr: TCPAddr + var err: Error + tcp_addr, err = resolve_internet_addr(network, local_address) + if err: + raise err + return listen_tcp(network, tcp_addr) + +fn listen_tcp(network: String, host: String, port: Int) raises -> TCPListener: + """Creates a new TCP listener. -struct TCPListener(Listener): - var _file_descriptor: Socket - var listen_config: ListenConfig + Args: + network: The network type. + host: The address to listen on, in ipv4 format. + port: The port to listen on. + """ + return listen_tcp(network, TCPAddr(host, port)) + + +struct TCPListener: + var socket: Socket var network_type: String - var address: String + var address: TCPAddr fn __init__( inout self, - owned file_descriptor: Socket, - listen_config: ListenConfig, + owned socket: Socket, network_type: String, - address: String, + address: TCPAddr, ): - self._file_descriptor = file_descriptor^ - self.listen_config = listen_config + self.socket = socket^ self.network_type = network_type self.address = address fn __moveinit__(inout self, owned existing: Self): - self._file_descriptor = existing._file_descriptor^ - self.listen_config = existing.listen_config^ - self.network_type = existing.network_type - self.address = existing.address + self.socket = existing.socket^ + self.network_type = existing.network_type^ + self.address = existing.address^ + + fn accept(self) raises -> TCPConnection: + return TCPConnection(self.socket.accept()) - fn listen(self) raises -> Self: - return self.listen_config.listen(self.network_type, self.address) + fn close(inout self) -> Error: + return self.socket.close() - fn accept(self) raises -> Connection: - return Connection(self._file_descriptor.accept()) - fn accept_tcp(self) raises -> TCPConnection: - return TCPConnection(self._file_descriptor.accept()) +alias TCP_NETWORK_TYPES = InlineList[String, 3]("tcp", "tcp4", "tcp6") - fn close(inout self) -> Error: - return self._file_descriptor.close() - fn addr(self) raises -> TCPAddr: - return resolve_internet_addr(self.network_type, self.address) +fn dial_tcp(network: String, remote_address: TCPAddr) raises -> TCPConnection: + """Connects to the address on the named network. + + The network must be "tcp", "tcp4", or "tcp6". + Args: + network: The network type. + remote_address: The remote address to connect to. + + Returns: + The TCP connection. + """ + # TODO: Add conversion of domain name to ip address + if network not in TCP_NETWORK_TYPES: + raise Error("unsupported network type: " + network) + + var socket = Socket() + var err = socket.connect(remote_address.ip, remote_address.port) + if err: + raise err + return TCPConnection(socket^) + + +fn dial_tcp(network: String, remote_address: String) raises -> TCPConnection: + """Connects to the address on the named network. + + The network must be "tcp", "tcp4", or "tcp6". + Args: + network: The network type. + remote_address: The remote address to connect to. (The format is "host:port"). + + Returns: + The TCP connection. + """ + var remote: HostPort + var err: Error + remote, err = split_host_port(remote_address) + if err: + raise err + return dial_tcp(network, TCPAddr(remote.host, remote.port)) + + +fn dial_tcp(network: String, host: String, port: Int) raises -> TCPConnection: + """Connects to the address on the named network. + + The network must be "tcp", "tcp4", or "tcp6". + Args: + network: The network type. + host: The remote address to connect to in ipv4 format. + port: The remote port. + + Returns: + The TCP connection. + """ + return dial_tcp(network, TCPAddr(host, port)) diff --git a/external/gojo/net/udp.mojo b/external/gojo/net/udp.mojo new file mode 100644 index 0000000..ca2e710 --- /dev/null +++ b/external/gojo/net/udp.mojo @@ -0,0 +1,210 @@ +from collections import InlineList +from ..syscall import SocketOptions, SocketType +from .address import NetworkType, split_host_port, join_host_port, BaseAddr, resolve_internet_addr +from .socket import Socket + + +# TODO: Change ip to list of bytes +@value +struct UDPAddr(Addr): + """Represents the address of a UDP end point. + + Args: + ip: IP address. + port: Port number. + zone: IPv6 addressing zone. + """ + + var ip: String + var port: Int + var zone: String # IPv6 addressing zone + + fn __init__(inout self, ip: String = "127.0.0.1", port: Int = 8000, zone: String = ""): + self.ip = ip + self.port = port + self.zone = zone + + fn __init__(inout self, addr: BaseAddr): + self.ip = addr.ip + self.port = addr.port + self.zone = addr.zone + + fn __str__(self) -> String: + if self.zone != "": + return join_host_port(str(self.ip) + "%" + self.zone, str(self.port)) + return join_host_port(self.ip, str(self.port)) + + fn network(self) -> String: + return NetworkType.udp.value + + +struct UDPConnection(Movable): + """Implementation of the Conn interface for TCP network connections.""" + + var socket: Socket + + fn __init__(inout self, owned socket: Socket): + self.socket = socket^ + + fn __moveinit__(inout self, owned existing: Self): + self.socket = existing.socket^ + + fn read_from(inout self, inout dest: List[UInt8]) -> (Int, HostPort, Error): + """Reads data from the underlying file descriptor. + + Args: + dest: The buffer to read data into. + + Returns: + The number of bytes read, or an error if one occurred. + """ + var bytes_read: Int + var remote: HostPort + var err = Error() + bytes_read, remote, err = self.socket.receive_from_into(dest) + if err: + if str(err) != str(io.EOF): + return bytes_read, remote, err + + return bytes_read, remote, err + + fn write_to(inout self, src: Span[UInt8], address: UDPAddr) -> (Int, Error): + """Writes data to the underlying file descriptor. + + Args: + src: The buffer to read data into. + address: The remote peer address. + + Returns: + The number of bytes written, or an error if one occurred. + """ + return self.socket.send_to(src, address.ip, address.port) + + fn write_to(inout self, src: Span[UInt8], host: String, port: Int) -> (Int, Error): + """Writes data to the underlying file descriptor. + + Args: + src: The buffer to read data into. + host: The remote peer address in IPv4 format. + port: The remote peer port. + + Returns: + The number of bytes written, or an error if one occurred. + """ + return self.socket.send_to(src, host, port) + + fn close(inout self) -> Error: + """Closes the underlying file descriptor. + + Returns: + An error if one occurred, or None if the file descriptor was closed successfully. + """ + return self.socket.close() + + fn local_address(self) -> UDPAddr: + """Returns the local network address. + The Addr returned is shared by all invocations of local_address, so do not modify it. + + Returns: + The local network address. + """ + return self.socket.local_address_as_udp() + + fn remote_address(self) -> UDPAddr: + """Returns the remote network address. + The Addr returned is shared by all invocations of remote_address, so do not modify it. + + Returns: + The remote network address. + """ + return self.socket.remote_address_as_udp() + + +fn listen_udp(network: String, local_address: UDPAddr) raises -> UDPConnection: + """Creates a new UDP listener. + + Args: + network: The network type. + local_address: The local address to listen on. + """ + var socket = Socket(socket_type=SocketType.SOCK_DGRAM) + socket.bind(local_address.ip, local_address.port) + # print(str("Listening on ") + str(socket.local_address_as_udp())) + return UDPConnection(socket^) + + +fn listen_udp(network: String, local_address: String) raises -> UDPConnection: + """Creates a new UDP listener. + + Args: + network: The network type. + local_address: The address to listen on. The format is "host:port". + """ + var result = split_host_port(local_address) + return listen_udp(network, UDPAddr(result[0].host, result[0].port)) + + +fn listen_udp(network: String, host: String, port: Int) raises -> UDPConnection: + """Creates a new UDP listener. + + Args: + network: The network type. + host: The address to listen on in ipv4 format. + port: The port number. + """ + return listen_udp(network, UDPAddr(host, port)) + + +alias UDP_NETWORK_TYPES = InlineList[String, 3]("udp", "udp4", "udp6") + + +fn dial_udp(network: String, local_address: UDPAddr) raises -> UDPConnection: + """Connects to the address on the named network. + + The network must be "udp", "udp4", or "udp6". + Args: + network: The network type. + local_address: The local address. + + Returns: + The TCP connection. + """ + # TODO: Add conversion of domain name to ip address + if network not in UDP_NETWORK_TYPES: + raise Error("unsupported network type: " + network) + + var socket = Socket(local_address=BaseAddr(local_address), socket_type=SocketType.SOCK_DGRAM) + return UDPConnection(socket^) + + +fn dial_udp(network: String, local_address: String) raises -> UDPConnection: + """Connects to the address on the named network. + + The network must be "udp", "udp4", or "udp6". + Args: + network: The network type. + local_address: The local address to connect to. (The format is "host:port"). + + Returns: + The TCP connection. + """ + var result = split_host_port(local_address) + if result[1]: + raise result[1] + + return dial_udp(network, UDPAddr(result[0].host, result[0].port)) + + +fn dial_udp(network: String, host: String, port: Int) raises -> UDPConnection: + """Connects to the address on the named network. + + The network must be "udp", "udp4", or "udp6". + Args: + network: The network type. + host: The remote host in ipv4 format. + port: The remote port. + + Returns: + The TCP connection. + """ + return dial_udp(network, UDPAddr(host, port)) diff --git a/external/gojo/strings/builder.mojo b/external/gojo/strings/builder.mojo index e7a76ed..585a8c8 100644 --- a/external/gojo/strings/builder.mojo +++ b/external/gojo/strings/builder.mojo @@ -1,17 +1,19 @@ -# Adapted from https://github.com/maniartech/mojo-strings/blob/master/strings/builder.mojo -# Modified to use List[Int8] instead of List[String] - import ..io -from ..builtins import Byte -@value -struct StringBuilder(Stringable, Sized, io.Writer, io.ByteWriter, io.StringWriter): +struct StringBuilder[growth_factor: Float32 = 2]( + Stringable, + Sized, + io.Writer, + io.StringWriter, + io.ByteWriter, +): """ A string builder class that allows for efficient string management and concatenation. This class is useful when you need to build a string by appending multiple strings - together. It is around 20x faster than using the `+` operator to concatenate - strings because it avoids the overhead of creating and destroying many + together. The performance increase is not linear. Compared to string concatenation, + I've observed around 20-30x faster for writing and rending ~4KB and up to 2100x-2300x + for ~4MB. This is because it avoids the overhead of creating and destroying many intermediate strings and performs memcopy operations. The result is a more efficient when building larger string concatenations. It @@ -20,114 +22,116 @@ struct StringBuilder(Stringable, Sized, io.Writer, io.ByteWriter, io.StringWrite builder and appending the strings is not worth the performance gain. Example: - ``` - from strings.builder import StringBuilder - - var sb = StringBuilder() - sb.write_string("mojo") - sb.write_string("jojo") - print(sb) # mojojojo - ``` + ``` + from strings.builder import StringBuilder + + var sb = StringBuilder() + sb.write_string("Hello ") + sb.write_string("World!") + print(sb) # Hello World! + ``` """ - var _vector: List[Byte] + var _data: UnsafePointer[UInt8] + var _size: Int + var _capacity: Int + + fn __init__(inout self, *, capacity: Int = 4096): + constrained[growth_factor >= 1.25]() + self._data = UnsafePointer[UInt8]().alloc(capacity) + self._size = 0 + self._capacity = capacity + + fn __moveinit__(inout self, owned other: Self): + self._data = other._data + self._size = other._size + self._capacity = other._capacity + other._data = UnsafePointer[UInt8]() + other._size = 0 + other._capacity = 0 + + fn __del__(owned self): + if self._data: + self._data.free() - fn __init__(inout self, *, size: Int = 4096): - self._vector = List[Byte](capacity=size) + fn __len__(self) -> Int: + """Returns the length of the string builder.""" + return self._size fn __str__(self) -> String: """ Converts the string builder to a string. Returns: - The string representation of the string builder. Returns an empty - string if the string builder is empty. - """ - var copy = List[Byte](self._vector) - if copy[-1] != 0: - copy.append(0) - return String(copy) - - fn get_bytes(self) -> List[Int8]: + The string representation of the string builder. Returns an empty + string if the string builder is empty. """ - Returns a deepcopy of the byte array of the string builder. + var copy = UnsafePointer[UInt8]().alloc(self._size) + memcpy(copy, self._data, self._size) + return StringRef(copy, self._size) - Returns: - The byte array of the string builder. - """ - return List[Byte](self._vector) + fn as_bytes_slice(ref [_]self) -> Span[UInt8, __lifetime_of(self)]: + """Returns the internal data as a Span[UInt8].""" + return Span[UInt8, __lifetime_of(self)](unsafe_ptr=self._data, len=self._size) - fn get_null_terminated_bytes(self) -> List[Int8]: + fn render(ref [_]self) -> StringSlice[__lifetime_of(self)]: """ - Returns a deepcopy of the byte array of the string builder with a null terminator. + Return a StringSlice view of the _data owned by the builder. + Slightly faster than __str__, 10-20% faster in limited testing. Returns: - The byte array of the string builder with a null terminator. + The string representation of the string builder. Returns an empty string if the string builder is empty. """ - var copy = List[Byte](self._vector) - if copy[-1] != 0: - copy.append(0) - - return copy + return StringSlice[__lifetime_of(self)](unsafe_from_utf8_ptr=self._data, len=self._size) - fn write(inout self, src: List[Byte]) -> (Int, Error): + fn _resize(inout self, _capacity: Int) -> None: """ - Appends a byte array to the builder buffer. + Resizes the string builder buffer. Args: - src: The byte array to append. + _capacity: The new _capacity of the string builder buffer. """ - self._vector.extend(src) - return len(src), Error() + var new__data = UnsafePointer[UInt8]().alloc(_capacity) + memcpy(new__data, self._data, self._size) + self._data.free() + self._data = new__data + self._capacity = _capacity - fn write_byte(inout self, byte: Int8) -> (Int, Error): - """ - Appends a byte array to the builder buffer. + return None - Args: - byte: The byte array to append. - """ - self._vector.append(byte) - return 1, Error() + fn _resize_if_needed(inout self, bytes_to_add: Int): + """Resizes the buffer if the bytes to add exceeds the current capacity.""" + # TODO: Handle the case where new_capacity is greater than MAX_INT. It should panic. + if bytes_to_add > self._capacity - self._size: + var new_capacity = int(self._capacity * 2) + if new_capacity < self._capacity + bytes_to_add: + new_capacity = self._capacity + bytes_to_add + self._resize(new_capacity) - fn write_string(inout self, src: String) -> (Int, Error): + fn write(inout self, src: Span[UInt8]) -> (Int, Error): """ - Appends a string to the builder buffer. + Appends a byte Span to the builder buffer. Args: - src: The string to append. - """ - var string_buffer = src.as_bytes() - self._vector.extend(string_buffer) - return len(string_buffer), Error() - - fn __len__(self) -> Int: + src: The byte array to append. """ - Returns the length of the string builder. + self._resize_if_needed(len(src)) + memcpy(self._data.offset(self._size), src._data, len(src)) + self._size += len(src) - Returns: - The length of the string builder. - """ - return len(self._vector) + return len(src), Error() - fn __getitem__(self, index: Int) -> String: + fn write_string(inout self, src: String) -> (Int, Error): """ - Returns the string at the given index. + Appends a string to the builder buffer. Args: - index: The index of the string to return. - - Returns: - The string at the given index. - """ - return self._vector[index] - - fn __setitem__(inout self, index: Int, value: Int8): + src: The string to append. """ - Sets the string at the given index. + return self.write(src.as_bytes_slice()) - Args: - index: The index of the string to set. - value: The value to set. - """ - self._vector[index] = value + fn write_byte(inout self, byte: UInt8) -> (Int, Error): + self._resize_if_needed(1) + self._data[self._size] = byte + self._size += 1 + return 1, Error() diff --git a/external/gojo/strings/reader.mojo b/external/gojo/strings/reader.mojo index 8d8af28..a2019ff 100644 --- a/external/gojo/strings/reader.mojo +++ b/external/gojo/strings/reader.mojo @@ -1,15 +1,24 @@ import ..io -from ..builtins import Byte, copy, panic +from ..builtins import copy, panic @value -struct Reader(Sized, io.Reader, io.ReaderAt, io.ByteReader, io.ByteScanner, io.Seeker, io.WriterTo): +# TODO: Uncomment write_to and write_buf once the bug with the trait's Span argument is fixed. +struct Reader( + Sized, + io.Reader, + io.ReaderAt, + io.ByteReader, + io.ByteScanner, + io.Seeker, + # io.WriterTo, +): """A Reader that implements the [io.Reader], [io.ReaderAt], [io.ByteReader], [io.ByteScanner], [io.Seeker], and [io.WriterTo] traits by reading from a string. The zero value for Reader operates like a Reader of an empty string. """ var string: String - var read_pos: Int64 # current reading index + var read_pos: Int # current reading index var prev_rune: Int # index of previous rune; or < 0 fn __init__(inout self, string: String = ""): @@ -18,17 +27,13 @@ struct Reader(Sized, io.Reader, io.ReaderAt, io.ByteReader, io.ByteScanner, io.S self.prev_rune = -1 fn __len__(self) -> Int: - """Returns the number of bytes of the unread portion of the string. - - Returns: - int: the number of bytes of the unread portion of the string. - """ - if self.read_pos >= Int64(len(self.string)): + """Returns the number of bytes of the unread portion of the string.""" + if self.read_pos >= len(self.string): return 0 - return int(Int64(len(self.string)) - self.read_pos) + return len(self.string) - self.read_pos - fn size(self) -> Int64: + fn size(self) -> Int: """Returns the original length of the underlying string. size is the number of bytes available for reading via [Reader.read_at]. The returned value is always the same and is not affected by calls @@ -37,34 +42,56 @@ struct Reader(Sized, io.Reader, io.ReaderAt, io.ByteReader, io.ByteScanner, io.S Returns: The original length of the underlying string. """ - return Int64(len(self.string)) + return len(self.string) - fn read(inout self, inout dest: List[Byte]) -> (Int, Error): - """Reads from the underlying string into the provided List[Byte] object. + fn _read(inout self, inout dest: Span[UInt8], capacity: Int) -> (Int, Error): + """Reads from the underlying string into the provided List[UInt8] object. Implements the [io.Reader] trait. Args: - dest: The destination List[Byte] object to read into. + dest: The destination List[UInt8] object to read into. + capacity: The capacity of the destination List[UInt8] object. Returns: The number of bytes read into dest. """ - if self.read_pos >= Int64(len(self.string)): - return 0, Error(io.EOF) + if self.read_pos >= len(self.string): + return 0, io.EOF self.prev_rune = -1 - var bytes_written = copy(dest, self.string[int(self.read_pos) :].as_bytes()) - self.read_pos += Int64(bytes_written) + var bytes_to_read = self.string.as_bytes_slice()[self.read_pos :] + var bytes_written = copy(dest.unsafe_ptr(), bytes_to_read.unsafe_ptr(), len(bytes_to_read)) + dest._len += bytes_written + self.read_pos += bytes_written return bytes_written, Error() - fn read_at(self, inout dest: List[Byte], off: Int64) -> (Int, Error): - """Reads from the Reader into the dest List[Byte] starting at the offset off. + fn read(inout self, inout dest: List[UInt8]) -> (Int, Error): + """Reads from the underlying string into the provided List[UInt8] object. + Implements the [io.Reader] trait. + + Args: + dest: The destination List[UInt8] object to read into. + + Returns: + The number of bytes read into dest. + """ + var span = Span(dest) + + var bytes_read: Int + var err: Error + bytes_read, err = self._read(span, dest.capacity) + dest.size += bytes_read + + return bytes_read, err + + fn _read_at(self, inout dest: Span[UInt8], off: Int, capacity: Int) -> (Int, Error): + """Reads from the Reader into the dest List[UInt8] starting at the offset off. It returns the number of bytes read into dest and an error if any. - Implements the [io.ReaderAt] trait. Args: - dest: The destination List[Byte] object to read into. + dest: The destination List[UInt8] object to read into. off: The byte offset to start reading from. + capacity: The capacity of the destination List[UInt8] object. Returns: The number of bytes read into dest. @@ -73,35 +100,50 @@ struct Reader(Sized, io.Reader, io.ReaderAt, io.ByteReader, io.ByteScanner, io.S if off < 0: return 0, Error("strings.Reader.read_at: negative offset") - if off >= Int64(len(self.string)): - return 0, Error(io.EOF) + if off >= len(self.string): + return 0, io.EOF var error = Error() - var copied_elements_count = copy(dest, self.string[int(off) :].as_bytes()) + var bytes_to_read = self.string.as_bytes_slice()[off:] + var copied_elements_count = copy(dest.unsafe_ptr(), bytes_to_read.unsafe_ptr(), len(bytes_to_read)) + dest._len += copied_elements_count if copied_elements_count < len(dest): - error = Error(io.EOF) + error = Error(str(io.EOF)) - return copied_elements_count, Error() + return copied_elements_count, error - fn read_byte(inout self) -> (Byte, Error): - """Reads the next byte from the underlying string. - Implements the [io.ByteReader] trait. + fn read_at(self, inout dest: List[UInt8], off: Int) -> (Int, Error): + """Reads from the Reader into the dest List[UInt8] starting at the offset off. + It returns the number of bytes read into dest and an error if any. + + Args: + dest: The destination List[UInt8] object to read into. + off: The byte offset to start reading from. Returns: - The next byte from the underlying string. + The number of bytes read into dest. """ + var span = Span(dest) + + var bytes_read: Int + var err: Error + bytes_read, err = self._read_at(span, off, dest.capacity) + dest.size += bytes_read + + return bytes_read, err + + fn read_byte(inout self) -> (UInt8, Error): + """Reads the next byte from the underlying string.""" self.prev_rune = -1 - if self.read_pos >= Int64(len(self.string)): - return Byte(0), Error(io.EOF) + if self.read_pos >= len(self.string): + return UInt8(0), io.EOF - var b = self.string[int(self.read_pos)] + var b = self.string.as_bytes_slice()[self.read_pos] self.read_pos += 1 - return Byte(ord(b)), Error() + return UInt8(b), Error() fn unread_byte(inout self) -> Error: - """Unreads the last byte read. Only the most recent byte read can be unread. - Implements the [io.ByteScanner] trait. - """ + """Unreads the last byte read. Only the most recent byte read can be unread.""" if self.read_pos <= 0: return Error("strings.Reader.unread_byte: at beginning of string") @@ -112,7 +154,7 @@ struct Reader(Sized, io.Reader, io.ReaderAt, io.ByteReader, io.ByteScanner, io.S # # read_rune implements the [io.RuneReader] trait. # fn read_rune() (ch rune, size int, err error): - # if self.read_pos >= Int64(len(self.string)): + # if self.read_pos >= Int(len(self.string)): # self.prev_rune = -1 # return 0, 0, io.EOF @@ -122,7 +164,7 @@ struct Reader(Sized, io.Reader, io.ReaderAt, io.ByteReader, io.ByteScanner, io.S # return rune(c), 1, nil # ch, size = utf8.DecodeRuneInString(self.string[self.read_pos:]) - # self.read_pos += Int64(size) + # self.read_pos += Int(size) # return # # unread_rune implements the [io.RuneScanner] trait. @@ -133,13 +175,12 @@ struct Reader(Sized, io.Reader, io.ReaderAt, io.ByteReader, io.ByteScanner, io.S # if self.prev_rune < 0: # return errors.New("strings.Reader.unread_rune: previous operation was not read_rune") - # self.read_pos = Int64(self.prev_rune) + # self.read_pos = Int(self.prev_rune) # self.prev_rune = -1 # return nil - fn seek(inout self, offset: Int64, whence: Int) -> (Int64, Error): + fn seek(inout self, offset: Int, whence: Int) -> (Int, Error): """Seeks to a new position in the underlying string. The next read will start from that position. - Implements the [io.Seeker] trait. Args: offset: The offset to seek to. @@ -149,52 +190,52 @@ struct Reader(Sized, io.Reader, io.ReaderAt, io.ByteReader, io.ByteScanner, io.S The new position in the string. """ self.prev_rune = -1 - var position: Int64 = 0 + var position: Int = 0 if whence == io.SEEK_START: position = offset elif whence == io.SEEK_CURRENT: position = self.read_pos + offset elif whence == io.SEEK_END: - position = Int64(len(self.string)) + offset + position = Int(len(self.string)) + offset else: - return Int64(0), Error("strings.Reader.seek: invalid whence") + return Int(0), Error("strings.Reader.seek: invalid whence") if position < 0: - return Int64(0), Error("strings.Reader.seek: negative position") + return Int(0), Error("strings.Reader.seek: negative position") self.read_pos = position return position, Error() - fn write_to[W: io.Writer](inout self, inout writer: W) -> (Int64, Error): - """Writes the remaining portion of the underlying string to the provided writer. - Implements the [io.WriterTo] trait. + # fn write_to[W: io.Writer](inout self, inout writer: W) -> (Int, Error): + # """Writes the remaining portion of the underlying string to the provided writer. + # Implements the [io.WriterTo] trait. - Args: - writer: The writer to write the remaining portion of the string to. + # Args: + # writer: The writer to write the remaining portion of the string to. - Returns: - The number of bytes written to the writer. - """ - self.prev_rune = -1 - if self.read_pos >= Int64(len(self.string)): - return Int64(0), Error() + # Returns: + # The number of bytes written to the writer. + # """ + # self.prev_rune = -1 + # var err = Error() + # if self.read_pos >= len(self.string): + # return Int(0), err - var chunk_to_write = self.string[int(self.read_pos) :] - var bytes_written: Int - var err: Error - bytes_written, err = io.write_string(writer, chunk_to_write) - if bytes_written > len(chunk_to_write): - panic("strings.Reader.write_to: invalid write_string count") + # var chunk_to_write = self.string.as_bytes_slice()[self.read_pos :] + # var bytes_written: Int + # bytes_written, err = writer.write(chunk_to_write) + # if bytes_written > len(chunk_to_write): + # panic("strings.Reader.write_to: invalid write_string count") - self.read_pos += Int64(bytes_written) - if bytes_written != len(chunk_to_write) and not err: - err = Error(io.ERR_SHORT_WRITE) + # self.read_pos += bytes_written + # if bytes_written != len(chunk_to_write) and not err: + # err = Error(io.ERR_SHORT_WRITE) - return Int64(bytes_written), err + # return bytes_written, err - # TODO: How can I differentiate between the two write_to methods when the writer implements both traits? - # fn write_to[W: io.StringWriter](inout self, inout writer: W) raises -> Int64: + # # TODO: How can I differentiate between the two write_to methods when the writer implements both traits? + # fn write_to[W: io.StringWriter](inout self, inout writer: W) raises -> Int: # """Writes the remaining portion of the underlying string to the provided writer. # Implements the [io.WriterTo] trait. @@ -205,7 +246,7 @@ struct Reader(Sized, io.Reader, io.ReaderAt, io.ByteReader, io.ByteScanner, io.S # The number of bytes written to the writer. # """ # self.prev_rune = -1 - # if self.read_pos >= Int64(len(self.string)): + # if self.read_pos >= Int(len(self.string)): # return 0 # var chunk_to_write = self.string[self.read_pos:] @@ -213,11 +254,11 @@ struct Reader(Sized, io.Reader, io.ReaderAt, io.ByteReader, io.ByteScanner, io.S # if bytes_written > len(chunk_to_write): # raise Error("strings.Reader.write_to: invalid write_string count") - # self.read_pos += Int64(bytes_written) + # self.read_pos += Int(bytes_written) # if bytes_written != len(chunk_to_write): # raise Error(io.ERR_SHORT_WRITE) - # return Int64(bytes_written) + # return Int(bytes_written) fn reset(inout self, string: String): """Resets the [Reader] to be reading from the beginning of the provided string. diff --git a/external/gojo/syscall/__init__.mojo b/external/gojo/syscall/__init__.mojo index e69de29..c89fef0 100644 --- a/external/gojo/syscall/__init__.mojo +++ b/external/gojo/syscall/__init__.mojo @@ -0,0 +1,62 @@ +from .net import ( + FD, + SocketType, + AddressFamily, + ProtocolFamily, + SocketOptions, + AddressInformation, + send, + sendto, + recv, + recvfrom, + open, + addrinfo, + addrinfo_unix, + sockaddr, + sockaddr_in, + socklen_t, + socket, + connect, + htons, + ntohs, + inet_pton, + inet_ntop, + getaddrinfo, + getaddrinfo_unix, + gai_strerror, + shutdown, + inet_ntoa, + bind, + listen, + accept, + setsockopt, + getsockopt, + getsockname, + getpeername, + SHUT_RDWR, + SOL_SOCKET, +) +from .file import close, FileDescriptorBase + +# Adapted from https://github.com/crisadamo/mojo-Libc . Huge thanks to Cristian! +# C types +alias c_void = UInt8 +alias c_char = UInt8 +alias c_schar = Int8 +alias c_uchar = UInt8 +alias c_short = Int16 +alias c_ushort = UInt16 +alias c_int = Int32 +alias c_uint = UInt32 +alias c_long = Int64 +alias c_ulong = UInt64 +alias c_float = Float32 +alias c_double = Float64 + +# `Int` is known to be machine's width +alias c_size_t = Int +alias c_ssize_t = Int + +alias ptrdiff_t = Int64 +alias intptr_t = Int64 +alias uintptr_t = UInt64 diff --git a/external/gojo/syscall/file.mojo b/external/gojo/syscall/file.mojo index d4095a5..ef0427e 100644 --- a/external/gojo/syscall/file.mojo +++ b/external/gojo/syscall/file.mojo @@ -1,4 +1,5 @@ -from .types import c_int, c_char, c_void, c_size_t, c_ssize_t +trait FileDescriptorBase(io.Reader, io.Writer, io.Closer): + ... # --- ( File Related Syscalls & Structs )--------------------------------------- @@ -22,7 +23,7 @@ fn close(fildes: c_int) -> c_int: return external_call["close", c_int, c_int](fildes) -fn open[*T: AnyType](path: Pointer[c_char], oflag: c_int, *args: *T) -> c_int: +fn open[*T: AnyType](path: UnsafePointer[c_char], oflag: c_int) -> c_int: """Libc POSIX `open` function Reference: https://man7.org/linux/man-pages/man3/open.3p.html Fn signature: int open(const char *path, int oflag, ...). @@ -30,61 +31,13 @@ fn open[*T: AnyType](path: Pointer[c_char], oflag: c_int, *args: *T) -> c_int: Args: path: A pointer to a C string containing the path to open. oflag: The flags to open the file with. - args: The optional arguments. Returns: A File Descriptor or -1 in case of failure """ - return external_call["open", c_int, Pointer[c_char], c_int](path, oflag, args) # FnName, RetType # Args + return external_call["open", c_int, UnsafePointer[c_char], c_int](path, oflag) # FnName, RetType # Args -fn openat[*T: AnyType](fd: c_int, path: Pointer[c_char], oflag: c_int, *args: *T) -> c_int: - """Libc POSIX `open` function - Reference: https://man7.org/linux/man-pages/man3/open.3p.html - Fn signature: int openat(int fd, const char *path, int oflag, ...). - - Args: - fd: A File Descriptor. - path: A pointer to a C string containing the path to open. - oflag: The flags to open the file with. - args: The optional arguments. - Returns: - A File Descriptor or -1 in case of failure - """ - return external_call["openat", c_int, c_int, Pointer[c_char], c_int]( # FnName, RetType # Args - fd, path, oflag, args - ) - - -fn printf[*T: AnyType](format: Pointer[c_char], *args: *T) -> c_int: - """Libc POSIX `printf` function - Reference: https://man7.org/linux/man-pages/man3/fprintf.3p.html - Fn signature: int printf(const char *restrict format, ...). - - Args: format: A pointer to a C string containing the format. - args: The optional arguments. - Returns: The number of bytes written or -1 in case of failure. - """ - return external_call[ - "printf", - c_int, # FnName, RetType - Pointer[c_char], # Args - ](format, args) - - -fn sprintf[*T: AnyType](s: Pointer[c_char], format: Pointer[c_char], *args: *T) -> c_int: - """Libc POSIX `sprintf` function - Reference: https://man7.org/linux/man-pages/man3/fprintf.3p.html - Fn signature: int sprintf(char *restrict s, const char *restrict format, ...). - - Args: s: A pointer to a buffer to store the result. - format: A pointer to a C string containing the format. - args: The optional arguments. - Returns: The number of bytes written or -1 in case of failure. - """ - return external_call["sprintf", c_int, Pointer[c_char], Pointer[c_char]](s, format, args) # FnName, RetType # Args - - -fn read(fildes: c_int, buf: Pointer[c_void], nbyte: c_size_t) -> c_int: +fn read(fildes: c_int, buf: UnsafePointer[c_void], nbyte: c_size_t) -> c_int: """Libc POSIX `read` function Reference: https://man7.org/linux/man-pages/man3/read.3p.html Fn signature: sssize_t read(int fildes, void *buf, size_t nbyte). @@ -94,10 +47,10 @@ fn read(fildes: c_int, buf: Pointer[c_void], nbyte: c_size_t) -> c_int: nbyte: The number of bytes to read. Returns: The number of bytes read or -1 in case of failure. """ - return external_call["read", c_ssize_t, c_int, Pointer[c_void], c_size_t](fildes, buf, nbyte) + return external_call["read", c_ssize_t, c_int, UnsafePointer[c_void], c_size_t](fildes, buf, nbyte) -fn write(fildes: c_int, buf: Pointer[c_void], nbyte: c_size_t) -> c_int: +fn write(fildes: c_int, buf: UnsafePointer[c_void], nbyte: c_size_t) -> c_int: """Libc POSIX `write` function Reference: https://man7.org/linux/man-pages/man3/write.3p.html Fn signature: ssize_t write(int fildes, const void *buf, size_t nbyte). @@ -107,4 +60,4 @@ fn write(fildes: c_int, buf: Pointer[c_void], nbyte: c_size_t) -> c_int: nbyte: The number of bytes to write. Returns: The number of bytes written or -1 in case of failure. """ - return external_call["write", c_ssize_t, c_int, Pointer[c_void], c_size_t](fildes, buf, nbyte) + return external_call["write", c_ssize_t, c_int, UnsafePointer[c_void], c_size_t](fildes, buf, nbyte) diff --git a/external/gojo/syscall/net.mojo b/external/gojo/syscall/net.mojo index e396d3f..676b2da 100644 --- a/external/gojo/syscall/net.mojo +++ b/external/gojo/syscall/net.mojo @@ -1,5 +1,6 @@ -from .types import c_char, c_int, c_ushort, c_uint, c_void, c_size_t, c_ssize_t, strlen +from . import c_char, c_int, c_ushort, c_uint, c_size_t, c_ssize_t from .file import O_CLOEXEC, O_NONBLOCK +from utils.static_tuple import StaticTuple alias IPPROTO_IPV6 = 41 alias IPV6_V6ONLY = 26 @@ -7,64 +8,64 @@ alias EPROTONOSUPPORT = 93 # Adapted from https://github.com/gabrieldemarmiesse/mojo-stdlib-extensions/ . Huge thanks to Gabriel! -alias FD_STDIN: c_int = 0 -alias FD_STDOUT: c_int = 1 -alias FD_STDERR: c_int = 2 + +struct FD: + alias STDIN = 0 + alias STDOUT = 1 + alias STDERR = 2 + alias SUCCESS = 0 alias GRND_NONBLOCK: UInt8 = 1 -alias char_pointer = UnsafePointer[c_char] +alias char_pointer = UnsafePointer[UInt8] # --- ( error.h Constants )----------------------------------------------------- -alias EPERM = 1 -alias ENOENT = 2 -alias ESRCH = 3 -alias EINTR = 4 -alias EIO = 5 -alias ENXIO = 6 -alias E2BIG = 7 -alias ENOEXEC = 8 -alias EBADF = 9 -alias ECHILD = 10 -alias EAGAIN = 11 -alias ENOMEM = 12 -alias EACCES = 13 -alias EFAULT = 14 -alias ENOTBLK = 15 -alias EBUSY = 16 -alias EEXIST = 17 -alias EXDEV = 18 -alias ENODEV = 19 -alias ENOTDIR = 20 -alias EISDIR = 21 -alias EINVAL = 22 -alias ENFILE = 23 -alias EMFILE = 24 -alias ENOTTY = 25 -alias ETXTBSY = 26 -alias EFBIG = 27 -alias ENOSPC = 28 -alias ESPIPE = 29 -alias EROFS = 30 -alias EMLINK = 31 -alias EPIPE = 32 -alias EDOM = 33 -alias ERANGE = 34 -alias EWOULDBLOCK = EAGAIN - - -fn to_char_ptr(s: String) -> Pointer[c_char]: - """Only ASCII-based strings.""" - var ptr = Pointer[c_char]().alloc(len(s)) - for i in range(len(s)): - ptr.store(i, ord(s[i])) - return ptr - - -fn c_charptr_to_string(s: Pointer[c_char]) -> String: - return String(s.bitcast[Int8](), strlen(s)) +struct ErrnoConstants: + alias EPERM = 1 + alias ENOENT = 2 + alias ESRCH = 3 + alias EINTR = 4 + alias EIO = 5 + alias ENXIO = 6 + alias E2BIG = 7 + alias ENOEXEC = 8 + alias EBADF = 9 + alias ECHILD = 10 + alias EAGAIN = 11 + alias ENOMEM = 12 + alias EACCES = 13 + alias EFAULT = 14 + alias ENOTBLK = 15 + alias EBUSY = 16 + alias EEXIST = 17 + alias EXDEV = 18 + alias ENODEV = 19 + alias ENOTDIR = 20 + alias EISDIR = 21 + alias EINVAL = 22 + alias ENFILE = 23 + alias EMFILE = 24 + alias ENOTTY = 25 + alias ETXTBSY = 26 + alias EFBIG = 27 + alias ENOSPC = 28 + alias ESPIPE = 29 + alias EROFS = 30 + alias EMLINK = 31 + alias EPIPE = 32 + alias EDOM = 33 + alias ERANGE = 34 + alias EWOULDBLOCK = 11 + + +# fn to_char_ptr(s: String) -> UnsafePointer[UInt8]: +# """Only ASCII-based strings.""" +# var ptr = UnsafePointer[UInt8]().alloc(len(s)) +# for i in range(len(s)): +# ptr.store(i, ord(s[i])) +# return ptr fn cftob(val: c_int) -> Bool: @@ -78,117 +79,126 @@ alias socklen_t = c_uint alias in_addr_t = c_uint alias in_port_t = c_ushort + # Address Family Constants -alias AF_UNSPEC = 0 -alias AF_UNIX = 1 -alias AF_LOCAL = AF_UNIX -alias AF_INET = 2 -alias AF_AX25 = 3 -alias AF_IPX = 4 -alias AF_APPLETALK = 5 -alias AF_NETROM = 6 -alias AF_BRIDGE = 7 -alias AF_ATMPVC = 8 -alias AF_X25 = 9 -alias AF_INET6 = 10 -alias AF_ROSE = 11 -alias AF_DECnet = 12 -alias AF_NETBEUI = 13 -alias AF_SECURITY = 14 -alias AF_KEY = 15 -alias AF_NETLINK = 16 -alias AF_ROUTE = AF_NETLINK -alias AF_PACKET = 17 -alias AF_ASH = 18 -alias AF_ECONET = 19 -alias AF_ATMSVC = 20 -alias AF_RDS = 21 -alias AF_SNA = 22 -alias AF_IRDA = 23 -alias AF_PPPOX = 24 -alias AF_WANPIPE = 25 -alias AF_LLC = 26 -alias AF_CAN = 29 -alias AF_TIPC = 30 -alias AF_BLUETOOTH = 31 -alias AF_IUCV = 32 -alias AF_RXRPC = 33 -alias AF_ISDN = 34 -alias AF_PHONET = 35 -alias AF_IEEE802154 = 36 -alias AF_CAIF = 37 -alias AF_ALG = 38 -alias AF_NFC = 39 -alias AF_VSOCK = 40 -alias AF_KCM = 41 -alias AF_QIPCRTR = 42 -alias AF_MAX = 43 +struct AddressFamily: + alias AF_UNSPEC = 0 + alias AF_UNIX = 1 + alias AF_LOCAL = 1 + alias AF_INET = 2 + alias AF_AX25 = 3 + alias AF_IPX = 4 + alias AF_APPLETALK = 5 + alias AF_NETROM = 6 + alias AF_BRIDGE = 7 + alias AF_ATMPVC = 8 + alias AF_X25 = 9 + alias AF_INET6 = 10 + alias AF_ROSE = 11 + alias AF_DECnet = 12 + alias AF_NETBEUI = 13 + alias AF_SECURITY = 14 + alias AF_KEY = 15 + alias AF_NETLINK = 16 + alias AF_ROUTE = 16 + alias AF_PACKET = 17 + alias AF_ASH = 18 + alias AF_ECONET = 19 + alias AF_ATMSVC = 20 + alias AF_RDS = 21 + alias AF_SNA = 22 + alias AF_IRDA = 23 + alias AF_PPPOX = 24 + alias AF_WANPIPE = 25 + alias AF_LLC = 26 + alias AF_CAN = 29 + alias AF_TIPC = 30 + alias AF_BLUETOOTH = 31 + alias AF_IUCV = 32 + alias AF_RXRPC = 33 + alias AF_ISDN = 34 + alias AF_PHONET = 35 + alias AF_IEEE802154 = 36 + alias AF_CAIF = 37 + alias AF_ALG = 38 + alias AF_NFC = 39 + alias AF_VSOCK = 40 + alias AF_KCM = 41 + alias AF_QIPCRTR = 42 + alias AF_MAX = 43 + # Protocol family constants -alias PF_UNSPEC = AF_UNSPEC -alias PF_UNIX = AF_UNIX -alias PF_LOCAL = AF_LOCAL -alias PF_INET = AF_INET -alias PF_AX25 = AF_AX25 -alias PF_IPX = AF_IPX -alias PF_APPLETALK = AF_APPLETALK -alias PF_NETROM = AF_NETROM -alias PF_BRIDGE = AF_BRIDGE -alias PF_ATMPVC = AF_ATMPVC -alias PF_X25 = AF_X25 -alias PF_INET6 = AF_INET6 -alias PF_ROSE = AF_ROSE -alias PF_DECnet = AF_DECnet -alias PF_NETBEUI = AF_NETBEUI -alias PF_SECURITY = AF_SECURITY -alias PF_KEY = AF_KEY -alias PF_NETLINK = AF_NETLINK -alias PF_ROUTE = AF_ROUTE -alias PF_PACKET = AF_PACKET -alias PF_ASH = AF_ASH -alias PF_ECONET = AF_ECONET -alias PF_ATMSVC = AF_ATMSVC -alias PF_RDS = AF_RDS -alias PF_SNA = AF_SNA -alias PF_IRDA = AF_IRDA -alias PF_PPPOX = AF_PPPOX -alias PF_WANPIPE = AF_WANPIPE -alias PF_LLC = AF_LLC -alias PF_CAN = AF_CAN -alias PF_TIPC = AF_TIPC -alias PF_BLUETOOTH = AF_BLUETOOTH -alias PF_IUCV = AF_IUCV -alias PF_RXRPC = AF_RXRPC -alias PF_ISDN = AF_ISDN -alias PF_PHONET = AF_PHONET -alias PF_IEEE802154 = AF_IEEE802154 -alias PF_CAIF = AF_CAIF -alias PF_ALG = AF_ALG -alias PF_NFC = AF_NFC -alias PF_VSOCK = AF_VSOCK -alias PF_KCM = AF_KCM -alias PF_QIPCRTR = AF_QIPCRTR -alias PF_MAX = AF_MAX +struct ProtocolFamily: + alias PF_UNSPEC = AddressFamily.AF_UNSPEC + alias PF_UNIX = AddressFamily.AF_UNIX + alias PF_LOCAL = AddressFamily.AF_LOCAL + alias PF_INET = AddressFamily.AF_INET + alias PF_AX25 = AddressFamily.AF_AX25 + alias PF_IPX = AddressFamily.AF_IPX + alias PF_APPLETALK = AddressFamily.AF_APPLETALK + alias PF_NETROM = AddressFamily.AF_NETROM + alias PF_BRIDGE = AddressFamily.AF_BRIDGE + alias PF_ATMPVC = AddressFamily.AF_ATMPVC + alias PF_X25 = AddressFamily.AF_X25 + alias PF_INET6 = AddressFamily.AF_INET6 + alias PF_ROSE = AddressFamily.AF_ROSE + alias PF_DECnet = AddressFamily.AF_DECnet + alias PF_NETBEUI = AddressFamily.AF_NETBEUI + alias PF_SECURITY = AddressFamily.AF_SECURITY + alias PF_KEY = AddressFamily.AF_KEY + alias PF_NETLINK = AddressFamily.AF_NETLINK + alias PF_ROUTE = AddressFamily.AF_ROUTE + alias PF_PACKET = AddressFamily.AF_PACKET + alias PF_ASH = AddressFamily.AF_ASH + alias PF_ECONET = AddressFamily.AF_ECONET + alias PF_ATMSVC = AddressFamily.AF_ATMSVC + alias PF_RDS = AddressFamily.AF_RDS + alias PF_SNA = AddressFamily.AF_SNA + alias PF_IRDA = AddressFamily.AF_IRDA + alias PF_PPPOX = AddressFamily.AF_PPPOX + alias PF_WANPIPE = AddressFamily.AF_WANPIPE + alias PF_LLC = AddressFamily.AF_LLC + alias PF_CAN = AddressFamily.AF_CAN + alias PF_TIPC = AddressFamily.AF_TIPC + alias PF_BLUETOOTH = AddressFamily.AF_BLUETOOTH + alias PF_IUCV = AddressFamily.AF_IUCV + alias PF_RXRPC = AddressFamily.AF_RXRPC + alias PF_ISDN = AddressFamily.AF_ISDN + alias PF_PHONET = AddressFamily.AF_PHONET + alias PF_IEEE802154 = AddressFamily.AF_IEEE802154 + alias PF_CAIF = AddressFamily.AF_CAIF + alias PF_ALG = AddressFamily.AF_ALG + alias PF_NFC = AddressFamily.AF_NFC + alias PF_VSOCK = AddressFamily.AF_VSOCK + alias PF_KCM = AddressFamily.AF_KCM + alias PF_QIPCRTR = AddressFamily.AF_QIPCRTR + alias PF_MAX = AddressFamily.AF_MAX + # Socket Type constants -alias SOCK_STREAM = 1 -alias SOCK_DGRAM = 2 -alias SOCK_RAW = 3 -alias SOCK_RDM = 4 -alias SOCK_SEQPACKET = 5 -alias SOCK_DCCP = 6 -alias SOCK_PACKET = 10 -alias SOCK_CLOEXEC = O_CLOEXEC -alias SOCK_NONBLOCK = O_NONBLOCK +struct SocketType: + alias SOCK_STREAM = 1 + alias SOCK_DGRAM = 2 + alias SOCK_RAW = 3 + alias SOCK_RDM = 4 + alias SOCK_SEQPACKET = 5 + alias SOCK_DCCP = 6 + alias SOCK_PACKET = 10 + alias SOCK_CLOEXEC = O_CLOEXEC + alias SOCK_NONBLOCK = O_NONBLOCK + # Address Information -alias AI_PASSIVE = 1 -alias AI_CANONNAME = 2 -alias AI_NUMERICHOST = 4 -alias AI_V4MAPPED = 2048 -alias AI_ALL = 256 -alias AI_ADDRCONFIG = 1024 -alias AI_IDN = 64 +struct AddressInformation: + alias AI_PASSIVE = 1 + alias AI_CANONNAME = 2 + alias AI_NUMERICHOST = 4 + alias AI_V4MAPPED = 2048 + alias AI_ALL = 256 + alias AI_ADDRCONFIG = 1024 + alias AI_IDN = 64 + alias INET_ADDRSTRLEN = 16 alias INET6_ADDRSTRLEN = 46 @@ -199,86 +209,87 @@ alias SHUT_RDWR = 2 alias SOL_SOCKET = 65535 + # Socket Options -alias SO_DEBUG = 1 -alias SO_REUSEADDR = 4 -alias SO_TYPE = 4104 -alias SO_ERROR = 4103 -alias SO_DONTROUTE = 16 -alias SO_BROADCAST = 32 -alias SO_SNDBUF = 4097 -alias SO_RCVBUF = 4098 -alias SO_KEEPALIVE = 8 -alias SO_OOBINLINE = 256 -alias SO_LINGER = 128 -alias SO_REUSEPORT = 512 -alias SO_RCVLOWAT = 4100 -alias SO_SNDLOWAT = 4099 -alias SO_RCVTIMEO = 4102 -alias SO_SNDTIMEO = 4101 -alias SO_RCVTIMEO_OLD = 4102 -alias SO_SNDTIMEO_OLD = 4101 -alias SO_ACCEPTCONN = 2 - -# unsure of these socket options, they weren't available via python -alias SO_NO_CHECK = 11 -alias SO_PRIORITY = 12 -alias SO_BSDCOMPAT = 14 -alias SO_PASSCRED = 16 -alias SO_PEERCRED = 17 -alias SO_SECURITY_AUTHENTICATION = 22 -alias SO_SECURITY_ENCRYPTION_TRANSPORT = 23 -alias SO_SECURITY_ENCRYPTION_NETWORK = 24 -alias SO_BINDTODEVICE = 25 -alias SO_ATTACH_FILTER = 26 -alias SO_DETACH_FILTER = 27 -alias SO_GET_FILTER = SO_ATTACH_FILTER -alias SO_PEERNAME = 28 -alias SO_TIMESTAMP = 29 -alias SO_TIMESTAMP_OLD = 29 -alias SO_PEERSEC = 31 -alias SO_SNDBUFFORCE = 32 -alias SO_RCVBUFFORCE = 33 -alias SO_PASSSEC = 34 -alias SO_TIMESTAMPNS = 35 -alias SO_TIMESTAMPNS_OLD = 35 -alias SO_MARK = 36 -alias SO_TIMESTAMPING = 37 -alias SO_TIMESTAMPING_OLD = 37 -alias SO_PROTOCOL = 38 -alias SO_DOMAIN = 39 -alias SO_RXQ_OVFL = 40 -alias SO_WIFI_STATUS = 41 -alias SCM_WIFI_STATUS = SO_WIFI_STATUS -alias SO_PEEK_OFF = 42 -alias SO_NOFCS = 43 -alias SO_LOCK_FILTER = 44 -alias SO_SELECT_ERR_QUEUE = 45 -alias SO_BUSY_POLL = 46 -alias SO_MAX_PACING_RATE = 47 -alias SO_BPF_EXTENSIONS = 48 -alias SO_INCOMING_CPU = 49 -alias SO_ATTACH_BPF = 50 -alias SO_DETACH_BPF = SO_DETACH_FILTER -alias SO_ATTACH_REUSEPORT_CBPF = 51 -alias SO_ATTACH_REUSEPORT_EBPF = 52 -alias SO_CNX_ADVICE = 53 -alias SCM_TIMESTAMPING_OPT_STATS = 54 -alias SO_MEMINFO = 55 -alias SO_INCOMING_NAPI_ID = 56 -alias SO_COOKIE = 57 -alias SCM_TIMESTAMPING_PKTINFO = 58 -alias SO_PEERGROUPS = 59 -alias SO_ZEROCOPY = 60 -alias SO_TXTIME = 61 -alias SCM_TXTIME = SO_TXTIME -alias SO_BINDTOIFINDEX = 62 -alias SO_TIMESTAMP_NEW = 63 -alias SO_TIMESTAMPNS_NEW = 64 -alias SO_TIMESTAMPING_NEW = 65 -alias SO_RCVTIMEO_NEW = 66 -alias SO_SNDTIMEO_NEW = 67 -alias SO_DETACH_REUSEPORT_BPF = 68 +struct SocketOptions: + alias SO_DEBUG = 1 + alias SO_REUSEADDR = 4 + alias SO_TYPE = 4104 + alias SO_ERROR = 4103 + alias SO_DONTROUTE = 16 + alias SO_BROADCAST = 32 + alias SO_SNDBUF = 4097 + alias SO_RCVBUF = 4098 + alias SO_KEEPALIVE = 8 + alias SO_OOBINLINE = 256 + alias SO_LINGER = 128 + alias SO_REUSEPORT = 512 + alias SO_RCVLOWAT = 4100 + alias SO_SNDLOWAT = 4099 + alias SO_RCVTIMEO = 4102 + alias SO_SNDTIMEO = 4101 + alias SO_RCVTIMEO_OLD = 4102 + alias SO_SNDTIMEO_OLD = 4101 + alias SO_ACCEPTCONN = 2 + # unsure of these socket options, they weren't available via python + alias SO_NO_CHECK = 11 + alias SO_PRIORITY = 12 + alias SO_BSDCOMPAT = 14 + alias SO_PASSCRED = 16 + alias SO_PEERCRED = 17 + alias SO_SECURITY_AUTHENTICATION = 22 + alias SO_SECURITY_ENCRYPTION_TRANSPORT = 23 + alias SO_SECURITY_ENCRYPTION_NETWORK = 24 + alias SO_BINDTODEVICE = 25 + alias SO_ATTACH_FILTER = 26 + alias SO_DETACH_FILTER = 27 + alias SO_GET_FILTER = 26 + alias SO_PEERNAME = 28 + alias SO_TIMESTAMP = 29 + alias SO_TIMESTAMP_OLD = 29 + alias SO_PEERSEC = 31 + alias SO_SNDBUFFORCE = 32 + alias SO_RCVBUFFORCE = 33 + alias SO_PASSSEC = 34 + alias SO_TIMESTAMPNS = 35 + alias SO_TIMESTAMPNS_OLD = 35 + alias SO_MARK = 36 + alias SO_TIMESTAMPING = 37 + alias SO_TIMESTAMPING_OLD = 37 + alias SO_PROTOCOL = 38 + alias SO_DOMAIN = 39 + alias SO_RXQ_OVFL = 40 + alias SO_WIFI_STATUS = 41 + alias SCM_WIFI_STATUS = 41 + alias SO_PEEK_OFF = 42 + alias SO_NOFCS = 43 + alias SO_LOCK_FILTER = 44 + alias SO_SELECT_ERR_QUEUE = 45 + alias SO_BUSY_POLL = 46 + alias SO_MAX_PACING_RATE = 47 + alias SO_BPF_EXTENSIONS = 48 + alias SO_INCOMING_CPU = 49 + alias SO_ATTACH_BPF = 50 + alias SO_DETACH_BPF = 27 + alias SO_ATTACH_REUSEPORT_CBPF = 51 + alias SO_ATTACH_REUSEPORT_EBPF = 52 + alias SO_CNX_ADVICE = 53 + alias SCM_TIMESTAMPING_OPT_STATS = 54 + alias SO_MEMINFO = 55 + alias SO_INCOMING_NAPI_ID = 56 + alias SO_COOKIE = 57 + alias SCM_TIMESTAMPING_PKTINFO = 58 + alias SO_PEERGROUPS = 59 + alias SO_ZEROCOPY = 60 + alias SO_TXTIME = 61 + alias SCM_TXTIME = 61 + alias SO_BINDTOIFINDEX = 62 + alias SO_TIMESTAMP_NEW = 63 + alias SO_TIMESTAMPNS_NEW = 64 + alias SO_TIMESTAMPING_NEW = 65 + alias SO_RCVTIMEO_NEW = 66 + alias SO_SNDTIMEO_NEW = 67 + alias SO_DETACH_REUSEPORT_BPF = 68 # --- ( Network Related Structs )----------------------------------------------- @@ -333,12 +344,32 @@ struct addrinfo: var ai_socktype: c_int var ai_protocol: c_int var ai_addrlen: socklen_t - var ai_canonname: Pointer[c_char] - var ai_addr: Pointer[sockaddr] - var ai_next: Pointer[addrinfo] - - fn __init__() -> Self: - return Self(0, 0, 0, 0, 0, Pointer[c_char](), Pointer[sockaddr](), Pointer[addrinfo]()) + var ai_canonname: UnsafePointer[UInt8] + var ai_addr: UnsafePointer[sockaddr] + var ai_next: UnsafePointer[addrinfo] + + fn __init__( + inout self, + ai_flags: c_int = 0, + ai_family: c_int = 0, + ai_socktype: c_int = 0, + ai_protocol: c_int = 0, + ai_addrlen: socklen_t = 0, + ai_canonname: UnsafePointer[UInt8] = UnsafePointer[UInt8](), + ai_addr: UnsafePointer[sockaddr] = UnsafePointer[sockaddr](), + ai_next: UnsafePointer[addrinfo] = UnsafePointer[addrinfo](), + ): + self.ai_flags = ai_flags + self.ai_family = ai_family + self.ai_socktype = ai_socktype + self.ai_protocol = ai_protocol + self.ai_addrlen = ai_addrlen + self.ai_canonname = ai_canonname + self.ai_addr = ai_addr + self.ai_next = ai_next + + # fn __init__() -> Self: + # return Self(0, 0, 0, 0, 0, UnsafePointer[UInt8](), UnsafePointer[sockaddr](), UnsafePointer[addrinfo]()) @value @@ -354,12 +385,29 @@ struct addrinfo_unix: var ai_socktype: c_int var ai_protocol: c_int var ai_addrlen: socklen_t - var ai_addr: Pointer[sockaddr] - var ai_canonname: Pointer[c_char] - var ai_next: Pointer[addrinfo] - - fn __init__() -> Self: - return Self(0, 0, 0, 0, 0, Pointer[sockaddr](), Pointer[c_char](), Pointer[addrinfo]()) + var ai_addr: UnsafePointer[sockaddr] + var ai_canonname: UnsafePointer[UInt8] + var ai_next: UnsafePointer[addrinfo] + + fn __init__( + inout self, + ai_flags: c_int = 0, + ai_family: c_int = 0, + ai_socktype: c_int = 0, + ai_protocol: c_int = 0, + ai_addrlen: socklen_t = 0, + ai_canonname: UnsafePointer[UInt8] = UnsafePointer[UInt8](), + ai_addr: UnsafePointer[sockaddr] = UnsafePointer[sockaddr](), + ai_next: UnsafePointer[addrinfo] = UnsafePointer[addrinfo](), + ): + self.ai_flags = ai_flags + self.ai_family = ai_family + self.ai_socktype = ai_socktype + self.ai_protocol = ai_protocol + self.ai_addrlen = ai_addrlen + self.ai_canonname = ai_canonname + self.ai_addr = ai_addr + self.ai_next = ai_next # --- ( Network Related Syscalls & Structs )------------------------------------ @@ -409,7 +457,12 @@ fn ntohs(netshort: c_ushort) -> c_ushort: return external_call["ntohs", c_ushort, c_ushort](netshort) -fn inet_ntop(af: c_int, src: Pointer[c_void], dst: Pointer[c_char], size: socklen_t) -> Pointer[c_char]: +fn inet_ntop( + af: c_int, + src: UnsafePointer[UInt8], + dst: UnsafePointer[UInt8], + size: socklen_t, +) -> UnsafePointer[UInt8]: """Libc POSIX `inet_ntop` function Reference: https://man7.org/linux/man-pages/man3/inet_ntop.3p.html. Fn signature: const char *inet_ntop(int af, const void *restrict src, char *restrict dst, socklen_t size). @@ -425,15 +478,15 @@ fn inet_ntop(af: c_int, src: Pointer[c_void], dst: Pointer[c_char], size: sockle """ return external_call[ "inet_ntop", - Pointer[c_char], # FnName, RetType + UnsafePointer[UInt8], # FnName, RetType c_int, - Pointer[c_void], - Pointer[c_char], + UnsafePointer[UInt8], + UnsafePointer[UInt8], socklen_t, # Args ](af, src, dst, size) -fn inet_pton(af: c_int, src: Pointer[c_char], dst: Pointer[c_void]) -> c_int: +fn inet_pton(af: c_int, src: UnsafePointer[UInt8], dst: UnsafePointer[UInt8]) -> c_int: """Libc POSIX `inet_pton` function Reference: https://man7.org/linux/man-pages/man3/inet_ntop.3p.html Fn signature: int inet_pton(int af, const char *restrict src, void *restrict dst). @@ -447,12 +500,12 @@ fn inet_pton(af: c_int, src: Pointer[c_char], dst: Pointer[c_void]) -> c_int: "inet_pton", c_int, # FnName, RetType c_int, - Pointer[c_char], - Pointer[c_void], # Args + UnsafePointer[UInt8], + UnsafePointer[UInt8], # Args ](af, src, dst) -fn inet_addr(cp: Pointer[c_char]) -> in_addr_t: +fn inet_addr(cp: UnsafePointer[UInt8]) -> in_addr_t: """Libc POSIX `inet_addr` function Reference: https://man7.org/linux/man-pages/man3/inet_addr.3p.html Fn signature: in_addr_t inet_addr(const char *cp). @@ -460,10 +513,10 @@ fn inet_addr(cp: Pointer[c_char]) -> in_addr_t: Args: cp: A pointer to a string containing the address. Returns: The address in network byte order. """ - return external_call["inet_addr", in_addr_t, Pointer[c_char]](cp) + return external_call["inet_addr", in_addr_t, UnsafePointer[UInt8]](cp) -fn inet_ntoa(addr: in_addr) -> Pointer[c_char]: +fn inet_ntoa(addr: in_addr) -> UnsafePointer[UInt8]: """Libc POSIX `inet_ntoa` function Reference: https://man7.org/linux/man-pages/man3/inet_addr.3p.html Fn signature: char *inet_ntoa(struct in_addr in). @@ -471,7 +524,7 @@ fn inet_ntoa(addr: in_addr) -> Pointer[c_char]: Args: in: A pointer to a string containing the address. Returns: The address in network byte order. """ - return external_call["inet_ntoa", Pointer[c_char], in_addr](addr) + return external_call["inet_ntoa", UnsafePointer[UInt8], in_addr](addr) fn socket(domain: c_int, type: c_int, protocol: c_int) -> c_int: @@ -491,7 +544,7 @@ fn setsockopt( socket: c_int, level: c_int, option_name: c_int, - option_value: Pointer[c_void], + option_value: UnsafePointer[UInt8], option_len: socklen_t, ) -> c_int: """Libc POSIX `setsockopt` function @@ -512,7 +565,7 @@ fn setsockopt( c_int, c_int, c_int, - Pointer[c_void], + UnsafePointer[UInt8], socklen_t, # Args ](socket, level, option_name, option_value, option_len) @@ -521,8 +574,8 @@ fn getsockopt( socket: c_int, level: c_int, option_name: c_int, - option_value: Pointer[c_void], - option_len: Pointer[socklen_t], + option_value: UnsafePointer[UInt8], + option_len: UnsafePointer[socklen_t], ) -> c_int: """Libc POSIX `getsockopt` function Reference: https://man7.org/linux/man-pages/man3/getsockopt.3p.html @@ -532,7 +585,7 @@ fn getsockopt( level: The protocol level. option_name: The option to get. option_value: A pointer to the value to get. - option_len: Pointer to the size of the value. + option_len: DTypePointer to the size of the value. Returns: 0 on success, -1 on error. """ return external_call[ @@ -541,12 +594,16 @@ fn getsockopt( c_int, c_int, c_int, - Pointer[c_void], - Pointer[socklen_t], # Args + UnsafePointer[UInt8], + UnsafePointer[socklen_t], # Args ](socket, level, option_name, option_value, option_len) -fn getsockname(socket: c_int, address: Pointer[sockaddr], address_len: Pointer[socklen_t]) -> c_int: +fn getsockname( + socket: c_int, + address: UnsafePointer[sockaddr], + address_len: UnsafePointer[socklen_t], +) -> c_int: """Libc POSIX `getsockname` function Reference: https://man7.org/linux/man-pages/man3/getsockname.3p.html Fn signature: int getsockname(int socket, struct sockaddr *restrict address, socklen_t *restrict address_len). @@ -560,12 +617,16 @@ fn getsockname(socket: c_int, address: Pointer[sockaddr], address_len: Pointer[s "getsockname", c_int, # FnName, RetType c_int, - Pointer[sockaddr], - Pointer[socklen_t], # Args + UnsafePointer[sockaddr], + UnsafePointer[socklen_t], # Args ](socket, address, address_len) -fn getpeername(sockfd: c_int, addr: Pointer[sockaddr], address_len: Pointer[socklen_t]) -> c_int: +fn getpeername( + sockfd: c_int, + addr: UnsafePointer[sockaddr], + address_len: UnsafePointer[socklen_t], +) -> c_int: """Libc POSIX `getpeername` function Reference: https://man7.org/linux/man-pages/man2/getpeername.2.html Fn signature: int getpeername(int socket, struct sockaddr *restrict addr, socklen_t *restrict address_len). @@ -579,17 +640,17 @@ fn getpeername(sockfd: c_int, addr: Pointer[sockaddr], address_len: Pointer[sock "getpeername", c_int, # FnName, RetType c_int, - Pointer[sockaddr], - Pointer[socklen_t], # Args + UnsafePointer[sockaddr], + UnsafePointer[socklen_t], # Args ](sockfd, addr, address_len) -fn bind(socket: c_int, address: Pointer[sockaddr], address_len: socklen_t) -> c_int: +fn bind(socket: c_int, address: UnsafePointer[sockaddr], address_len: socklen_t) -> c_int: """Libc POSIX `bind` function Reference: https://man7.org/linux/man-pages/man3/bind.3p.html Fn signature: int bind(int socket, const struct sockaddr *address, socklen_t address_len). """ - return external_call["bind", c_int, c_int, Pointer[sockaddr], socklen_t]( # FnName, RetType # Args + return external_call["bind", c_int, c_int, UnsafePointer[sockaddr], socklen_t]( # FnName, RetType # Args socket, address, address_len ) @@ -606,7 +667,11 @@ fn listen(socket: c_int, backlog: c_int) -> c_int: return external_call["listen", c_int, c_int, c_int](socket, backlog) -fn accept(socket: c_int, address: Pointer[sockaddr], address_len: Pointer[socklen_t]) -> c_int: +fn accept( + socket: c_int, + address: UnsafePointer[sockaddr], + address_len: UnsafePointer[socklen_t], +) -> c_int: """Libc POSIX `accept` function Reference: https://man7.org/linux/man-pages/man3/accept.3p.html Fn signature: int accept(int socket, struct sockaddr *restrict address, socklen_t *restrict address_len). @@ -620,12 +685,12 @@ fn accept(socket: c_int, address: Pointer[sockaddr], address_len: Pointer[sockle "accept", c_int, # FnName, RetType c_int, - Pointer[sockaddr], - Pointer[socklen_t], # Args + UnsafePointer[sockaddr], + UnsafePointer[socklen_t], # Args ](socket, address, address_len) -fn connect(socket: c_int, address: Pointer[sockaddr], address_len: socklen_t) -> c_int: +fn connect(socket: c_int, address: UnsafePointer[sockaddr], address_len: socklen_t) -> c_int: """Libc POSIX `connect` function Reference: https://man7.org/linux/man-pages/man3/connect.3p.html Fn signature: int connect(int socket, const struct sockaddr *address, socklen_t address_len). @@ -635,27 +700,93 @@ fn connect(socket: c_int, address: Pointer[sockaddr], address_len: socklen_t) -> address_len: The size of the address. Returns: 0 on success, -1 on error. """ - return external_call["connect", c_int, c_int, Pointer[sockaddr], socklen_t]( # FnName, RetType # Args + return external_call["connect", c_int, c_int, UnsafePointer[sockaddr], socklen_t]( # FnName, RetType # Args socket, address, address_len ) -fn recv(socket: c_int, buffer: Pointer[c_void], length: c_size_t, flags: c_int) -> c_ssize_t: +fn recv( + socket: c_int, + buffer: UnsafePointer[UInt8], + length: c_size_t, + flags: c_int, +) -> c_ssize_t: """Libc POSIX `recv` function Reference: https://man7.org/linux/man-pages/man3/recv.3p.html Fn signature: ssize_t recv(int socket, void *buffer, size_t length, int flags). + + Args: + socket: Specifies the socket file descriptor. + buffer: Points to the buffer where the message should be stored. + length: Specifies the length in bytes of the buffer pointed to by the buffer argument. + flags: Specifies the type of message reception. + + Returns: + The number of bytes received or -1 in case of failure. + + Valid Flags: + MSG_PEEK: Peeks at an incoming message. The data is treated as unread and the next recvfrom() or similar function shall still return this data. + MSG_OOB: Requests out-of-band data. The significance and semantics of out-of-band data are protocol-specific. + MSG_WAITALL: On SOCK_STREAM sockets this requests that the function block until the full amount of data can be returned. The function may return the smaller amount of data if the socket is a message-based socket, if a signal is caught, if the connection is terminated, if MSG_PEEK was specified, or if an error is pending for the socket. """ return external_call[ "recv", - c_ssize_t, # FnName, RetType + c_ssize_t, c_int, - Pointer[c_void], + UnsafePointer[UInt8], c_size_t, - c_int, # Args + c_int, ](socket, buffer, length, flags) -fn send(socket: c_int, buffer: Pointer[c_void], length: c_size_t, flags: c_int) -> c_ssize_t: +fn recvfrom( + socket: c_int, + buffer: UnsafePointer[UInt8], + length: c_size_t, + flags: c_int, + address: UnsafePointer[sockaddr], + address_len: UnsafePointer[socklen_t], +) -> c_ssize_t: + """Libc POSIX `recvfrom` function + Reference: https://man7.org/linux/man-pages/man3/recvfrom.3p.html + Fn signature: ssize_t recvfrom(int socket, void *restrict buffer, size_t length, + int flags, struct sockaddr *restrict address, + socklen_t *restrict address_len). + + Args: + socket: Specifies the socket file descriptor. + buffer: Points to the buffer where the message should be stored. + length: Specifies the length in bytes of the buffer pointed to by the buffer argument. + flags: Specifies the type of message reception. + address: A null pointer, or points to a sockaddr structure in which the sending address is to be stored. + address_len: Either a null pointer, if address is a null pointer, or a pointer to a socklen_t object which on input specifies the length of the supplied sockaddr structure, and on output specifies the length of the stored address. + + Returns: + The number of bytes received or -1 in case of failure. + + Valid Flags: + MSG_PEEK: Peeks at an incoming message. The data is treated as unread and the next recvfrom() or similar function shall still return this data. + MSG_OOB: Requests out-of-band data. The significance and semantics of out-of-band data are protocol-specific. + MSG_WAITALL: On SOCK_STREAM sockets this requests that the function block until the full amount of data can be returned. The function may return the smaller amount of data if the socket is a message-based socket, if a signal is caught, if the connection is terminated, if MSG_PEEK was specified, or if an error is pending for the socket. + """ + return external_call[ + "recvfrom", + c_ssize_t, + c_int, + UnsafePointer[UInt8], + c_size_t, + c_int, + UnsafePointer[sockaddr], + UnsafePointer[socklen_t], + ](socket, buffer, length, flags, address, address_len) + + +fn send( + socket: c_int, + buffer: UnsafePointer[UInt8], + length: c_size_t, + flags: c_int, +) -> c_ssize_t: """Libc POSIX `send` function Reference: https://man7.org/linux/man-pages/man3/send.3p.html Fn signature: ssize_t send(int socket, const void *buffer, size_t length, int flags). @@ -670,12 +801,47 @@ fn send(socket: c_int, buffer: Pointer[c_void], length: c_size_t, flags: c_int) "send", c_ssize_t, # FnName, RetType c_int, - Pointer[c_void], + UnsafePointer[UInt8], c_size_t, c_int, # Args ](socket, buffer, length, flags) +fn sendto( + socket: c_int, + message: UnsafePointer[UInt8], + length: c_size_t, + flags: c_int, + dest_addr: UnsafePointer[sockaddr], + dest_len: socklen_t, +) -> c_ssize_t: + """Libc POSIX `sendto` function + Reference: https://man7.org/linux/man-pages/man3/sendto.3p.html + Fn signature: ssize_t sendto(int socket, const void *message, size_t length, + int flags, const struct sockaddr *dest_addr, + socklen_t dest_len). + + Args: + socket: Specifies the socket file descriptor. + message: Points to a buffer containing the message to be sent. + length: Specifies the size of the message in bytes. + flags: Specifies the type of message transmission. + dest_addr: Points to a sockaddr structure containing the destination address. + dest_len: Specifies the length of the sockaddr. + + Returns: + The number of bytes sent or -1 in case of failure. + + Valid Flags: + MSG_EOR: Terminates a record (if supported by the protocol). + MSG_OOB: Sends out-of-band data on sockets that support out-of-band data. The significance and semantics of out-of-band data are protocol-specific. + MSG_NOSIGNAL: Requests not to send the SIGPIPE signal if an attempt to send is made on a stream-oriented socket that is no longer connected. The [EPIPE] error shall still be returned. + """ + return external_call[ + "sendto", c_ssize_t, c_int, UnsafePointer[UInt8], c_size_t, c_int, UnsafePointer[sockaddr], socklen_t + ](socket, message, length, flags, dest_addr, dest_len) + + fn shutdown(socket: c_int, how: c_int) -> c_int: """Libc POSIX `shutdown` function Reference: https://man7.org/linux/man-pages/man3/shutdown.3p.html @@ -689,10 +855,10 @@ fn shutdown(socket: c_int, how: c_int) -> c_int: fn getaddrinfo( - nodename: Pointer[c_char], - servname: Pointer[c_char], - hints: Pointer[addrinfo], - res: Pointer[Pointer[addrinfo]], + nodename: UnsafePointer[UInt8], + servname: UnsafePointer[UInt8], + hints: UnsafePointer[addrinfo], + res: UnsafePointer[UnsafePointer[addrinfo]], ) -> c_int: """Libc POSIX `getaddrinfo` function Reference: https://man7.org/linux/man-pages/man3/getaddrinfo.3p.html @@ -701,18 +867,18 @@ fn getaddrinfo( return external_call[ "getaddrinfo", c_int, # FnName, RetType - Pointer[c_char], - Pointer[c_char], - Pointer[addrinfo], # Args - Pointer[Pointer[addrinfo]], # Args + UnsafePointer[UInt8], + UnsafePointer[UInt8], + UnsafePointer[addrinfo], # Args + UnsafePointer[UnsafePointer[addrinfo]], # Args ](nodename, servname, hints, res) fn getaddrinfo_unix( - nodename: Pointer[c_char], - servname: Pointer[c_char], - hints: Pointer[addrinfo_unix], - res: Pointer[Pointer[addrinfo_unix]], + nodename: UnsafePointer[UInt8], + servname: UnsafePointer[UInt8], + hints: UnsafePointer[addrinfo_unix], + res: UnsafePointer[UnsafePointer[addrinfo_unix]], ) -> c_int: """Libc POSIX `getaddrinfo` function Reference: https://man7.org/linux/man-pages/man3/getaddrinfo.3p.html @@ -721,14 +887,14 @@ fn getaddrinfo_unix( return external_call[ "getaddrinfo", c_int, # FnName, RetType - Pointer[c_char], - Pointer[c_char], - Pointer[addrinfo_unix], # Args - Pointer[Pointer[addrinfo_unix]], # Args + UnsafePointer[UInt8], + UnsafePointer[UInt8], + UnsafePointer[addrinfo_unix], # Args + UnsafePointer[UnsafePointer[addrinfo_unix]], # Args ](nodename, servname, hints, res) -fn gai_strerror(ecode: c_int) -> Pointer[c_char]: +fn gai_strerror(ecode: c_int) -> UnsafePointer[UInt8]: """Libc POSIX `gai_strerror` function Reference: https://man7.org/linux/man-pages/man3/gai_strerror.3p.html Fn signature: const char *gai_strerror(int ecode). @@ -736,14 +902,14 @@ fn gai_strerror(ecode: c_int) -> Pointer[c_char]: Args: ecode: The error code. Returns: A pointer to a string describing the error. """ - return external_call["gai_strerror", Pointer[c_char], c_int](ecode) # FnName, RetType # Args + return external_call["gai_strerror", UnsafePointer[UInt8], c_int](ecode) # FnName, RetType # Args -fn inet_pton(address_family: Int, address: String) -> Int: - var ip_buf_size = 4 - if address_family == AF_INET6: - ip_buf_size = 16 +# fn inet_pton(address_family: Int, address: String) -> Int: +# var ip_buf_size = 4 +# if address_family == AF_INET6: +# ip_buf_size = 16 - var ip_buf = Pointer[c_void].alloc(ip_buf_size) - var conv_status = inet_pton(rebind[c_int](address_family), to_char_ptr(address), ip_buf) - return int(ip_buf.bitcast[c_uint]().load()) +# var ip_buf = UnsafePointer[UInt8].alloc(ip_buf_size) +# var conv_status = inet_pton(rebind[c_int](address_family), to_char_ptr(address), ip_buf) +# return int(ip_buf.bitcast[c_uint]().load()) diff --git a/external/gojo/syscall/types.mojo b/external/gojo/syscall/types.mojo deleted file mode 100644 index 56693e7..0000000 --- a/external/gojo/syscall/types.mojo +++ /dev/null @@ -1,63 +0,0 @@ -@value -struct Str: - var vector: List[c_char] - - fn __init__(inout self, string: String): - self.vector = List[c_char](capacity=len(string) + 1) - for i in range(len(string)): - self.vector.append(ord(string[i])) - self.vector.append(0) - - fn __init__(inout self, size: Int): - self.vector = List[c_char]() - self.vector.resize(size + 1, 0) - - fn __len__(self) -> Int: - for i in range(len(self.vector)): - if self.vector[i] == 0: - return i - return -1 - - fn to_string(self, size: Int) -> String: - var result: String = "" - for i in range(size): - result += chr(int(self.vector[i])) - return result - - fn __enter__(owned self: Self) -> Self: - return self^ - - -fn strlen(s: Pointer[c_char]) -> c_size_t: - """Libc POSIX `strlen` function - Reference: https://man7.org/linux/man-pages/man3/strlen.3p.html - Fn signature: size_t strlen(const char *s). - - Args: s: A pointer to a C string. - Returns: The length of the string. - """ - return external_call["strlen", c_size_t, Pointer[c_char]](s) - - -# Adapted from https://github.com/crisadamo/mojo-Libc . Huge thanks to Cristian! -# C types -alias c_void = UInt8 -alias c_char = UInt8 -alias c_schar = Int8 -alias c_uchar = UInt8 -alias c_short = Int16 -alias c_ushort = UInt16 -alias c_int = Int32 -alias c_uint = UInt32 -alias c_long = Int64 -alias c_ulong = UInt64 -alias c_float = Float32 -alias c_double = Float64 - -# `Int` is known to be machine's width -alias c_size_t = Int -alias c_ssize_t = Int - -alias ptrdiff_t = Int64 -alias intptr_t = Int64 -alias uintptr_t = UInt64 diff --git a/external/gojo/unicode/__init__.mojo b/external/gojo/unicode/__init__.mojo index bd4cba6..109ae2d 100644 --- a/external/gojo/unicode/__init__.mojo +++ b/external/gojo/unicode/__init__.mojo @@ -1 +1 @@ -from .utf8 import string_iterator, rune_count_in_string +from .utf8 import rune_count_in_string, rune_width, string_width, Condition, DEFAULT_CONDITION diff --git a/external/gojo/unicode/utf8/__init__.mojo b/external/gojo/unicode/utf8/__init__.mojo index b8732ec..f905a8f 100644 --- a/external/gojo/unicode/utf8/__init__.mojo +++ b/external/gojo/unicode/utf8/__init__.mojo @@ -1,4 +1,5 @@ """Almost all of the actual implementation in this module was written by @mzaks (https://github.com/mzaks)! This would not be possible without his help. """ -from .runes import string_iterator, rune_count_in_string +from .runes import rune_count_in_string +from .width import string_width, rune_width, Condition, DEFAULT_CONDITION diff --git a/external/gojo/unicode/utf8/runes.mojo b/external/gojo/unicode/utf8/runes.mojo index 56da84b..7681570 100644 --- a/external/gojo/unicode/utf8/runes.mojo +++ b/external/gojo/unicode/utf8/runes.mojo @@ -2,312 +2,8 @@ This would not be possible without his help. """ -from ...builtins import Rune from algorithm.functional import vectorize -from memory.unsafe import DTypePointer from sys.info import simdwidthof -from math.bit import ctlz - - -# The default lowest and highest continuation byte. -alias locb = 0b10000000 -alias hicb = 0b10111111 -alias RUNE_SELF = 0x80 # Characters below RuneSelf are represented as themselves in a single byte - - -# acceptRange gives the range of valid values for the second byte in a UTF-8 -# sequence. -@value -struct AcceptRange(CollectionElement): - var lo: UInt8 # lowest value for second byte. - var hi: UInt8 # highest value for second byte. - - -# ACCEPT_RANGES has size 16 to avoid bounds checks in the code that uses it. -alias ACCEPT_RANGES = List[AcceptRange]( - AcceptRange(locb, hicb), - AcceptRange(0xA0, hicb), - AcceptRange(locb, 0x9F), - AcceptRange(0x90, hicb), - AcceptRange(locb, 0x8F), -) - -# These names of these constants are chosen to give nice alignment in the -# table below. The first nibble is an index into acceptRanges or F for -# special one-byte cases. The second nibble is the Rune length or the -# Status for the special one-byte case. -alias xx = 0xF1 # invalid: size 1 -alias as1 = 0xF0 # ASCII: size 1 -alias s1 = 0x02 # accept 0, size 2 -alias s2 = 0x13 # accept 1, size 3 -alias s3 = 0x03 # accept 0, size 3 -alias s4 = 0x23 # accept 2, size 3 -alias s5 = 0x34 # accept 3, size 4 -alias s6 = 0x04 # accept 0, size 4 -alias s7 = 0x44 # accept 4, size 4 - - -# first is information about the first byte in a UTF-8 sequence. -var first = List[UInt8]( - # 1 2 3 4 5 6 7 8 9 A B C D E F - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, # 0x00-0x0F - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, # 0x10-0x1F - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, # 0x20-0x2F - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, # 0x30-0x3F - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, # 0x40-0x4F - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, # 0x50-0x5F - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, # 0x60-0x6F - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, - as1, # 0x70-0x7F - # 1 2 3 4 5 6 7 8 9 A B C D E F - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, # 0x80-0x8F - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, # 0x90-0x9F - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, # 0xA0-0xAF - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, # 0xB0-0xBF - xx, - xx, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, # 0xC0-0xCF - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, - s1, # 0xD0-0xDF - s2, - s3, - s3, - s3, - s3, - s3, - s3, - s3, - s3, - s3, - s3, - s3, - s3, - s4, - s3, - s3, # 0xE0-0xEF - s5, - s6, - s6, - s6, - s7, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, - xx, # 0xF0-0xFF -) alias simd_width_u8 = simdwidthof[DType.uint8]() @@ -322,32 +18,13 @@ fn rune_count_in_string(s: String) -> Int: Returns: The number of runes in the string. """ - var p = s._as_ptr().bitcast[DType.uint8]() + var p = DTypePointer[DType.uint8](s.unsafe_ptr()) var string_byte_length = len(s) var result = 0 @parameter fn count[simd_width: Int](offset: Int): - result += int(((p.load[width=simd_width](offset) >> 6) != 0b10).cast[DType.uint8]().reduce_add()) + result += int(((SIMD[size=simd_width].load(p + offset) >> 6) != 0b10).cast[DType.uint8]().reduce_add()) vectorize[count, simd_width_u8](string_byte_length) return result - - -fn string_iterator(s: String, func: fn (String) -> None): - """Iterate over the runes in a string and call the given function with each rune. - - Args: - s: The string to iterate over. - func: The function to call with each rune. - """ - var bytes = len(s) - var p = s._as_ptr().bitcast[DType.uint8]() - while bytes > 0: - var char_length = int((p.load() >> 7 == 0).cast[DType.uint8]() * 1 + ctlz(~p.load())) - var sp = DTypePointer[DType.int8].alloc(char_length + 1) - memcpy(sp, p.bitcast[DType.int8](), char_length) - sp[char_length] = 0 - func(String(sp, char_length + 1)) - bytes -= char_length - p += char_length diff --git a/external/gojo/unicode/utf8/table.mojo b/external/gojo/unicode/utf8/table.mojo new file mode 100644 index 0000000..32717af --- /dev/null +++ b/external/gojo/unicode/utf8/table.mojo @@ -0,0 +1,1296 @@ +@register_passable("trivial") +struct Interval: + var first: UInt32 + var last: UInt32 + + fn __init__(inout self, first: UInt32, last: UInt32): + self.first = first + self.last = last + + +alias combining = List[Interval]( + Interval(0x0300, 0x036F), + Interval(0x0483, 0x0489), + Interval(0x07EB, 0x07F3), + Interval(0x0C00, 0x0C00), + Interval(0x0C04, 0x0C04), + Interval(0x0D00, 0x0D01), + Interval(0x135D, 0x135F), + Interval(0x1A7F, 0x1A7F), + Interval(0x1AB0, 0x1AC0), + Interval(0x1B6B, 0x1B73), + Interval(0x1DC0, 0x1DF9), + Interval(0x1DFB, 0x1DFF), + Interval(0x20D0, 0x20F0), + Interval(0x2CEF, 0x2CF1), + Interval(0x2DE0, 0x2DFF), + Interval(0x3099, 0x309A), + Interval(0xA66F, 0xA672), + Interval(0xA674, 0xA67D), + Interval(0xA69E, 0xA69F), + Interval(0xA6F0, 0xA6F1), + Interval(0xA8E0, 0xA8F1), + Interval(0xFE20, 0xFE2F), + Interval(0x101FD, 0x101FD), + Interval(0x10376, 0x1037A), + Interval(0x10EAB, 0x10EAC), + Interval(0x10F46, 0x10F50), + Interval(0x11300, 0x11301), + Interval(0x1133B, 0x1133C), + Interval(0x11366, 0x1136C), + Interval(0x11370, 0x11374), + Interval(0x16AF0, 0x16AF4), + Interval(0x1D165, 0x1D169), + Interval(0x1D16D, 0x1D172), + Interval(0x1D17B, 0x1D182), + Interval(0x1D185, 0x1D18B), + Interval(0x1D1AA, 0x1D1AD), + Interval(0x1D242, 0x1D244), + Interval(0x1E000, 0x1E006), + Interval(0x1E008, 0x1E018), + Interval(0x1E01B, 0x1E021), + Interval(0x1E023, 0x1E024), + Interval(0x1E026, 0x1E02A), + Interval(0x1E8D0, 0x1E8D6), +) + +alias doublewidth = List[Interval]( + Interval(0x1100, 0x115F), + Interval(0x231A, 0x231B), + Interval(0x2329, 0x232A), + Interval(0x23E9, 0x23EC), + Interval(0x23F0, 0x23F0), + Interval(0x23F3, 0x23F3), + Interval(0x25FD, 0x25FE), + Interval(0x2614, 0x2615), + Interval(0x2648, 0x2653), + Interval(0x267F, 0x267F), + Interval(0x2693, 0x2693), + Interval(0x26A1, 0x26A1), + Interval(0x26AA, 0x26AB), + Interval(0x26BD, 0x26BE), + Interval(0x26C4, 0x26C5), + Interval(0x26CE, 0x26CE), + Interval(0x26D4, 0x26D4), + Interval(0x26EA, 0x26EA), + Interval(0x26F2, 0x26F3), + Interval(0x26F5, 0x26F5), + Interval(0x26FA, 0x26FA), + Interval(0x26FD, 0x26FD), + Interval(0x2705, 0x2705), + Interval(0x270A, 0x270B), + Interval(0x2728, 0x2728), + Interval(0x274C, 0x274C), + Interval(0x274E, 0x274E), + Interval(0x2753, 0x2755), + Interval(0x2757, 0x2757), + Interval(0x2795, 0x2797), + Interval(0x27B0, 0x27B0), + Interval(0x27BF, 0x27BF), + Interval(0x2B1B, 0x2B1C), + Interval(0x2B50, 0x2B50), + Interval(0x2B55, 0x2B55), + Interval(0x2E80, 0x2E99), + Interval(0x2E9B, 0x2EF3), + Interval(0x2F00, 0x2FD5), + Interval(0x2FF0, 0x2FFB), + Interval(0x3000, 0x303E), + Interval(0x3041, 0x3096), + Interval(0x3099, 0x30FF), + Interval(0x3105, 0x312F), + Interval(0x3131, 0x318E), + Interval(0x3190, 0x31E3), + Interval(0x31F0, 0x321E), + Interval(0x3220, 0x3247), + Interval(0x3250, 0x4DBF), + Interval(0x4E00, 0xA48C), + Interval(0xA490, 0xA4C6), + Interval(0xA960, 0xA97C), + Interval(0xAC00, 0xD7A3), + Interval(0xF900, 0xFAFF), + Interval(0xFE10, 0xFE19), + Interval(0xFE30, 0xFE52), + Interval(0xFE54, 0xFE66), + Interval(0xFE68, 0xFE6B), + Interval(0xFF01, 0xFF60), + Interval(0xFFE0, 0xFFE6), + Interval(0x16FE0, 0x16FE4), + Interval(0x16FF0, 0x16FF1), + Interval(0x17000, 0x187F7), + Interval(0x18800, 0x18CD5), + Interval(0x18D00, 0x18D08), + Interval(0x1B000, 0x1B11E), + Interval(0x1B150, 0x1B152), + Interval(0x1B164, 0x1B167), + Interval(0x1B170, 0x1B2FB), + Interval(0x1F004, 0x1F004), + Interval(0x1F0CF, 0x1F0CF), + Interval(0x1F18E, 0x1F18E), + Interval(0x1F191, 0x1F19A), + Interval(0x1F200, 0x1F202), + Interval(0x1F210, 0x1F23B), + Interval(0x1F240, 0x1F248), + Interval(0x1F250, 0x1F251), + Interval(0x1F260, 0x1F265), + Interval(0x1F300, 0x1F320), + Interval(0x1F32D, 0x1F335), + Interval(0x1F337, 0x1F37C), + Interval(0x1F37E, 0x1F393), + Interval(0x1F3A0, 0x1F3CA), + Interval(0x1F3CF, 0x1F3D3), + Interval(0x1F3E0, 0x1F3F0), + Interval(0x1F3F4, 0x1F3F4), + Interval(0x1F3F8, 0x1F43E), + Interval(0x1F440, 0x1F440), + Interval(0x1F442, 0x1F4FC), + Interval(0x1F4FF, 0x1F53D), + Interval(0x1F54B, 0x1F54E), + Interval(0x1F550, 0x1F567), + Interval(0x1F57A, 0x1F57A), + Interval(0x1F595, 0x1F596), + Interval(0x1F5A4, 0x1F5A4), + Interval(0x1F5FB, 0x1F64F), + Interval(0x1F680, 0x1F6C5), + Interval(0x1F6CC, 0x1F6CC), + Interval(0x1F6D0, 0x1F6D2), + Interval(0x1F6D5, 0x1F6D7), + Interval(0x1F6EB, 0x1F6EC), + Interval(0x1F6F4, 0x1F6FC), + Interval(0x1F7E0, 0x1F7EB), + Interval(0x1F90C, 0x1F93A), + Interval(0x1F93C, 0x1F945), + Interval(0x1F947, 0x1F978), + Interval(0x1F97A, 0x1F9CB), + Interval(0x1F9CD, 0x1F9FF), + Interval(0x1FA70, 0x1FA74), + Interval(0x1FA78, 0x1FA7A), + Interval(0x1FA80, 0x1FA86), + Interval(0x1FA90, 0x1FAA8), + Interval(0x1FAB0, 0x1FAB6), + Interval(0x1FAC0, 0x1FAC2), + Interval(0x1FAD0, 0x1FAD6), + Interval(0x20000, 0x2FFFD), + Interval(0x30000, 0x3FFFD), +) + +alias ambiguous = List[Interval]( + Interval(0x00A1, 0x00A1), + Interval(0x00A4, 0x00A4), + Interval(0x00A7, 0x00A8), + Interval(0x00AA, 0x00AA), + Interval(0x00AD, 0x00AE), + Interval(0x00B0, 0x00B4), + Interval(0x00B6, 0x00BA), + Interval(0x00BC, 0x00BF), + Interval(0x00C6, 0x00C6), + Interval(0x00D0, 0x00D0), + Interval(0x00D7, 0x00D8), + Interval(0x00DE, 0x00E1), + Interval(0x00E6, 0x00E6), + Interval(0x00E8, 0x00EA), + Interval(0x00EC, 0x00ED), + Interval(0x00F0, 0x00F0), + Interval(0x00F2, 0x00F3), + Interval(0x00F7, 0x00FA), + Interval(0x00FC, 0x00FC), + Interval(0x00FE, 0x00FE), + Interval(0x0101, 0x0101), + Interval(0x0111, 0x0111), + Interval(0x0113, 0x0113), + Interval(0x011B, 0x011B), + Interval(0x0126, 0x0127), + Interval(0x012B, 0x012B), + Interval(0x0131, 0x0133), + Interval(0x0138, 0x0138), + Interval(0x013F, 0x0142), + Interval(0x0144, 0x0144), + Interval(0x0148, 0x014B), + Interval(0x014D, 0x014D), + Interval(0x0152, 0x0153), + Interval(0x0166, 0x0167), + Interval(0x016B, 0x016B), + Interval(0x01CE, 0x01CE), + Interval(0x01D0, 0x01D0), + Interval(0x01D2, 0x01D2), + Interval(0x01D4, 0x01D4), + Interval(0x01D6, 0x01D6), + Interval(0x01D8, 0x01D8), + Interval(0x01DA, 0x01DA), + Interval(0x01DC, 0x01DC), + Interval(0x0251, 0x0251), + Interval(0x0261, 0x0261), + Interval(0x02C4, 0x02C4), + Interval(0x02C7, 0x02C7), + Interval(0x02C9, 0x02CB), + Interval(0x02CD, 0x02CD), + Interval(0x02D0, 0x02D0), + Interval(0x02D8, 0x02DB), + Interval(0x02DD, 0x02DD), + Interval(0x02DF, 0x02DF), + Interval(0x0300, 0x036F), + Interval(0x0391, 0x03A1), + Interval(0x03A3, 0x03A9), + Interval(0x03B1, 0x03C1), + Interval(0x03C3, 0x03C9), + Interval(0x0401, 0x0401), + Interval(0x0410, 0x044F), + Interval(0x0451, 0x0451), + Interval(0x2010, 0x2010), + Interval(0x2013, 0x2016), + Interval(0x2018, 0x2019), + Interval(0x201C, 0x201D), + Interval(0x2020, 0x2022), + Interval(0x2024, 0x2027), + Interval(0x2030, 0x2030), + Interval(0x2032, 0x2033), + Interval(0x2035, 0x2035), + Interval(0x203B, 0x203B), + Interval(0x203E, 0x203E), + Interval(0x2074, 0x2074), + Interval(0x207F, 0x207F), + Interval(0x2081, 0x2084), + Interval(0x20AC, 0x20AC), + Interval(0x2103, 0x2103), + Interval(0x2105, 0x2105), + Interval(0x2109, 0x2109), + Interval(0x2113, 0x2113), + Interval(0x2116, 0x2116), + Interval(0x2121, 0x2122), + Interval(0x2126, 0x2126), + Interval(0x212B, 0x212B), + Interval(0x2153, 0x2154), + Interval(0x215B, 0x215E), + Interval(0x2160, 0x216B), + Interval(0x2170, 0x2179), + Interval(0x2189, 0x2189), + Interval(0x2190, 0x2199), + Interval(0x21B8, 0x21B9), + Interval(0x21D2, 0x21D2), + Interval(0x21D4, 0x21D4), + Interval(0x21E7, 0x21E7), + Interval(0x2200, 0x2200), + Interval(0x2202, 0x2203), + Interval(0x2207, 0x2208), + Interval(0x220B, 0x220B), + Interval(0x220F, 0x220F), + Interval(0x2211, 0x2211), + Interval(0x2215, 0x2215), + Interval(0x221A, 0x221A), + Interval(0x221D, 0x2220), + Interval(0x2223, 0x2223), + Interval(0x2225, 0x2225), + Interval(0x2227, 0x222C), + Interval(0x222E, 0x222E), + Interval(0x2234, 0x2237), + Interval(0x223C, 0x223D), + Interval(0x2248, 0x2248), + Interval(0x224C, 0x224C), + Interval(0x2252, 0x2252), + Interval(0x2260, 0x2261), + Interval(0x2264, 0x2267), + Interval(0x226A, 0x226B), + Interval(0x226E, 0x226F), + Interval(0x2282, 0x2283), + Interval(0x2286, 0x2287), + Interval(0x2295, 0x2295), + Interval(0x2299, 0x2299), + Interval(0x22A5, 0x22A5), + Interval(0x22BF, 0x22BF), + Interval(0x2312, 0x2312), + Interval(0x2460, 0x24E9), + Interval(0x24EB, 0x254B), + Interval(0x2550, 0x2573), + Interval(0x2580, 0x258F), + Interval(0x2592, 0x2595), + Interval(0x25A0, 0x25A1), + Interval(0x25A3, 0x25A9), + Interval(0x25B2, 0x25B3), + Interval(0x25B6, 0x25B7), + Interval(0x25BC, 0x25BD), + Interval(0x25C0, 0x25C1), + Interval(0x25C6, 0x25C8), + Interval(0x25CB, 0x25CB), + Interval(0x25CE, 0x25D1), + Interval(0x25E2, 0x25E5), + Interval(0x25EF, 0x25EF), + Interval(0x2605, 0x2606), + Interval(0x2609, 0x2609), + Interval(0x260E, 0x260F), + Interval(0x261C, 0x261C), + Interval(0x261E, 0x261E), + Interval(0x2640, 0x2640), + Interval(0x2642, 0x2642), + Interval(0x2660, 0x2661), + Interval(0x2663, 0x2665), + Interval(0x2667, 0x266A), + Interval(0x266C, 0x266D), + Interval(0x266F, 0x266F), + Interval(0x269E, 0x269F), + Interval(0x26BF, 0x26BF), + Interval(0x26C6, 0x26CD), + Interval(0x26CF, 0x26D3), + Interval(0x26D5, 0x26E1), + Interval(0x26E3, 0x26E3), + Interval(0x26E8, 0x26E9), + Interval(0x26EB, 0x26F1), + Interval(0x26F4, 0x26F4), + Interval(0x26F6, 0x26F9), + Interval(0x26FB, 0x26FC), + Interval(0x26FE, 0x26FF), + Interval(0x273D, 0x273D), + Interval(0x2776, 0x277F), + Interval(0x2B56, 0x2B59), + Interval(0x3248, 0x324F), + Interval(0xE000, 0xF8FF), + Interval(0xFE00, 0xFE0F), + Interval(0xFFFD, 0xFFFD), + Interval(0x1F100, 0x1F10A), + Interval(0x1F110, 0x1F12D), + Interval(0x1F130, 0x1F169), + Interval(0x1F170, 0x1F18D), + Interval(0x1F18F, 0x1F190), + Interval(0x1F19B, 0x1F1AC), + Interval(0xE0100, 0xE01EF), + Interval(0xF0000, 0xFFFFD), + Interval(0x100000, 0x10FFFD), +) + +alias narrow = List[Interval]( + Interval(0x0020, 0x007E), + Interval(0x00A2, 0x00A3), + Interval(0x00A5, 0x00A6), + Interval(0x00AC, 0x00AC), + Interval(0x00AF, 0x00AF), + Interval(0x27E6, 0x27ED), + Interval(0x2985, 0x2986), +) + +alias neutral = List[Interval]( + Interval(0x0000, 0x001F), + Interval(0x007F, 0x00A0), + Interval(0x00A9, 0x00A9), + Interval(0x00AB, 0x00AB), + Interval(0x00B5, 0x00B5), + Interval(0x00BB, 0x00BB), + Interval(0x00C0, 0x00C5), + Interval(0x00C7, 0x00CF), + Interval(0x00D1, 0x00D6), + Interval(0x00D9, 0x00DD), + Interval(0x00E2, 0x00E5), + Interval(0x00E7, 0x00E7), + Interval(0x00EB, 0x00EB), + Interval(0x00EE, 0x00EF), + Interval(0x00F1, 0x00F1), + Interval(0x00F4, 0x00F6), + Interval(0x00FB, 0x00FB), + Interval(0x00FD, 0x00FD), + Interval(0x00FF, 0x0100), + Interval(0x0102, 0x0110), + Interval(0x0112, 0x0112), + Interval(0x0114, 0x011A), + Interval(0x011C, 0x0125), + Interval(0x0128, 0x012A), + Interval(0x012C, 0x0130), + Interval(0x0134, 0x0137), + Interval(0x0139, 0x013E), + Interval(0x0143, 0x0143), + Interval(0x0145, 0x0147), + Interval(0x014C, 0x014C), + Interval(0x014E, 0x0151), + Interval(0x0154, 0x0165), + Interval(0x0168, 0x016A), + Interval(0x016C, 0x01CD), + Interval(0x01CF, 0x01CF), + Interval(0x01D1, 0x01D1), + Interval(0x01D3, 0x01D3), + Interval(0x01D5, 0x01D5), + Interval(0x01D7, 0x01D7), + Interval(0x01D9, 0x01D9), + Interval(0x01DB, 0x01DB), + Interval(0x01DD, 0x0250), + Interval(0x0252, 0x0260), + Interval(0x0262, 0x02C3), + Interval(0x02C5, 0x02C6), + Interval(0x02C8, 0x02C8), + Interval(0x02CC, 0x02CC), + Interval(0x02CE, 0x02CF), + Interval(0x02D1, 0x02D7), + Interval(0x02DC, 0x02DC), + Interval(0x02DE, 0x02DE), + Interval(0x02E0, 0x02FF), + Interval(0x0370, 0x0377), + Interval(0x037A, 0x037F), + Interval(0x0384, 0x038A), + Interval(0x038C, 0x038C), + Interval(0x038E, 0x0390), + Interval(0x03AA, 0x03B0), + Interval(0x03C2, 0x03C2), + Interval(0x03CA, 0x0400), + Interval(0x0402, 0x040F), + Interval(0x0450, 0x0450), + Interval(0x0452, 0x052F), + Interval(0x0531, 0x0556), + Interval(0x0559, 0x058A), + Interval(0x058D, 0x058F), + Interval(0x0591, 0x05C7), + Interval(0x05D0, 0x05EA), + Interval(0x05EF, 0x05F4), + Interval(0x0600, 0x061C), + Interval(0x061E, 0x070D), + Interval(0x070F, 0x074A), + Interval(0x074D, 0x07B1), + Interval(0x07C0, 0x07FA), + Interval(0x07FD, 0x082D), + Interval(0x0830, 0x083E), + Interval(0x0840, 0x085B), + Interval(0x085E, 0x085E), + Interval(0x0860, 0x086A), + Interval(0x08A0, 0x08B4), + Interval(0x08B6, 0x08C7), + Interval(0x08D3, 0x0983), + Interval(0x0985, 0x098C), + Interval(0x098F, 0x0990), + Interval(0x0993, 0x09A8), + Interval(0x09AA, 0x09B0), + Interval(0x09B2, 0x09B2), + Interval(0x09B6, 0x09B9), + Interval(0x09BC, 0x09C4), + Interval(0x09C7, 0x09C8), + Interval(0x09CB, 0x09CE), + Interval(0x09D7, 0x09D7), + Interval(0x09DC, 0x09DD), + Interval(0x09DF, 0x09E3), + Interval(0x09E6, 0x09FE), + Interval(0x0A01, 0x0A03), + Interval(0x0A05, 0x0A0A), + Interval(0x0A0F, 0x0A10), + Interval(0x0A13, 0x0A28), + Interval(0x0A2A, 0x0A30), + Interval(0x0A32, 0x0A33), + Interval(0x0A35, 0x0A36), + Interval(0x0A38, 0x0A39), + Interval(0x0A3C, 0x0A3C), + Interval(0x0A3E, 0x0A42), + Interval(0x0A47, 0x0A48), + Interval(0x0A4B, 0x0A4D), + Interval(0x0A51, 0x0A51), + Interval(0x0A59, 0x0A5C), + Interval(0x0A5E, 0x0A5E), + Interval(0x0A66, 0x0A76), + Interval(0x0A81, 0x0A83), + Interval(0x0A85, 0x0A8D), + Interval(0x0A8F, 0x0A91), + Interval(0x0A93, 0x0AA8), + Interval(0x0AAA, 0x0AB0), + Interval(0x0AB2, 0x0AB3), + Interval(0x0AB5, 0x0AB9), + Interval(0x0ABC, 0x0AC5), + Interval(0x0AC7, 0x0AC9), + Interval(0x0ACB, 0x0ACD), + Interval(0x0AD0, 0x0AD0), + Interval(0x0AE0, 0x0AE3), + Interval(0x0AE6, 0x0AF1), + Interval(0x0AF9, 0x0AFF), + Interval(0x0B01, 0x0B03), + Interval(0x0B05, 0x0B0C), + Interval(0x0B0F, 0x0B10), + Interval(0x0B13, 0x0B28), + Interval(0x0B2A, 0x0B30), + Interval(0x0B32, 0x0B33), + Interval(0x0B35, 0x0B39), + Interval(0x0B3C, 0x0B44), + Interval(0x0B47, 0x0B48), + Interval(0x0B4B, 0x0B4D), + Interval(0x0B55, 0x0B57), + Interval(0x0B5C, 0x0B5D), + Interval(0x0B5F, 0x0B63), + Interval(0x0B66, 0x0B77), + Interval(0x0B82, 0x0B83), + Interval(0x0B85, 0x0B8A), + Interval(0x0B8E, 0x0B90), + Interval(0x0B92, 0x0B95), + Interval(0x0B99, 0x0B9A), + Interval(0x0B9C, 0x0B9C), + Interval(0x0B9E, 0x0B9F), + Interval(0x0BA3, 0x0BA4), + Interval(0x0BA8, 0x0BAA), + Interval(0x0BAE, 0x0BB9), + Interval(0x0BBE, 0x0BC2), + Interval(0x0BC6, 0x0BC8), + Interval(0x0BCA, 0x0BCD), + Interval(0x0BD0, 0x0BD0), + Interval(0x0BD7, 0x0BD7), + Interval(0x0BE6, 0x0BFA), + Interval(0x0C00, 0x0C0C), + Interval(0x0C0E, 0x0C10), + Interval(0x0C12, 0x0C28), + Interval(0x0C2A, 0x0C39), + Interval(0x0C3D, 0x0C44), + Interval(0x0C46, 0x0C48), + Interval(0x0C4A, 0x0C4D), + Interval(0x0C55, 0x0C56), + Interval(0x0C58, 0x0C5A), + Interval(0x0C60, 0x0C63), + Interval(0x0C66, 0x0C6F), + Interval(0x0C77, 0x0C8C), + Interval(0x0C8E, 0x0C90), + Interval(0x0C92, 0x0CA8), + Interval(0x0CAA, 0x0CB3), + Interval(0x0CB5, 0x0CB9), + Interval(0x0CBC, 0x0CC4), + Interval(0x0CC6, 0x0CC8), + Interval(0x0CCA, 0x0CCD), + Interval(0x0CD5, 0x0CD6), + Interval(0x0CDE, 0x0CDE), + Interval(0x0CE0, 0x0CE3), + Interval(0x0CE6, 0x0CEF), + Interval(0x0CF1, 0x0CF2), + Interval(0x0D00, 0x0D0C), + Interval(0x0D0E, 0x0D10), + Interval(0x0D12, 0x0D44), + Interval(0x0D46, 0x0D48), + Interval(0x0D4A, 0x0D4F), + Interval(0x0D54, 0x0D63), + Interval(0x0D66, 0x0D7F), + Interval(0x0D81, 0x0D83), + Interval(0x0D85, 0x0D96), + Interval(0x0D9A, 0x0DB1), + Interval(0x0DB3, 0x0DBB), + Interval(0x0DBD, 0x0DBD), + Interval(0x0DC0, 0x0DC6), + Interval(0x0DCA, 0x0DCA), + Interval(0x0DCF, 0x0DD4), + Interval(0x0DD6, 0x0DD6), + Interval(0x0DD8, 0x0DDF), + Interval(0x0DE6, 0x0DEF), + Interval(0x0DF2, 0x0DF4), + Interval(0x0E01, 0x0E3A), + Interval(0x0E3F, 0x0E5B), + Interval(0x0E81, 0x0E82), + Interval(0x0E84, 0x0E84), + Interval(0x0E86, 0x0E8A), + Interval(0x0E8C, 0x0EA3), + Interval(0x0EA5, 0x0EA5), + Interval(0x0EA7, 0x0EBD), + Interval(0x0EC0, 0x0EC4), + Interval(0x0EC6, 0x0EC6), + Interval(0x0EC8, 0x0ECD), + Interval(0x0ED0, 0x0ED9), + Interval(0x0EDC, 0x0EDF), + Interval(0x0F00, 0x0F47), + Interval(0x0F49, 0x0F6C), + Interval(0x0F71, 0x0F97), + Interval(0x0F99, 0x0FBC), + Interval(0x0FBE, 0x0FCC), + Interval(0x0FCE, 0x0FDA), + Interval(0x1000, 0x10C5), + Interval(0x10C7, 0x10C7), + Interval(0x10CD, 0x10CD), + Interval(0x10D0, 0x10FF), + Interval(0x1160, 0x1248), + Interval(0x124A, 0x124D), + Interval(0x1250, 0x1256), + Interval(0x1258, 0x1258), + Interval(0x125A, 0x125D), + Interval(0x1260, 0x1288), + Interval(0x128A, 0x128D), + Interval(0x1290, 0x12B0), + Interval(0x12B2, 0x12B5), + Interval(0x12B8, 0x12BE), + Interval(0x12C0, 0x12C0), + Interval(0x12C2, 0x12C5), + Interval(0x12C8, 0x12D6), + Interval(0x12D8, 0x1310), + Interval(0x1312, 0x1315), + Interval(0x1318, 0x135A), + Interval(0x135D, 0x137C), + Interval(0x1380, 0x1399), + Interval(0x13A0, 0x13F5), + Interval(0x13F8, 0x13FD), + Interval(0x1400, 0x169C), + Interval(0x16A0, 0x16F8), + Interval(0x1700, 0x170C), + Interval(0x170E, 0x1714), + Interval(0x1720, 0x1736), + Interval(0x1740, 0x1753), + Interval(0x1760, 0x176C), + Interval(0x176E, 0x1770), + Interval(0x1772, 0x1773), + Interval(0x1780, 0x17DD), + Interval(0x17E0, 0x17E9), + Interval(0x17F0, 0x17F9), + Interval(0x1800, 0x180E), + Interval(0x1810, 0x1819), + Interval(0x1820, 0x1878), + Interval(0x1880, 0x18AA), + Interval(0x18B0, 0x18F5), + Interval(0x1900, 0x191E), + Interval(0x1920, 0x192B), + Interval(0x1930, 0x193B), + Interval(0x1940, 0x1940), + Interval(0x1944, 0x196D), + Interval(0x1970, 0x1974), + Interval(0x1980, 0x19AB), + Interval(0x19B0, 0x19C9), + Interval(0x19D0, 0x19DA), + Interval(0x19DE, 0x1A1B), + Interval(0x1A1E, 0x1A5E), + Interval(0x1A60, 0x1A7C), + Interval(0x1A7F, 0x1A89), + Interval(0x1A90, 0x1A99), + Interval(0x1AA0, 0x1AAD), + Interval(0x1AB0, 0x1AC0), + Interval(0x1B00, 0x1B4B), + Interval(0x1B50, 0x1B7C), + Interval(0x1B80, 0x1BF3), + Interval(0x1BFC, 0x1C37), + Interval(0x1C3B, 0x1C49), + Interval(0x1C4D, 0x1C88), + Interval(0x1C90, 0x1CBA), + Interval(0x1CBD, 0x1CC7), + Interval(0x1CD0, 0x1CFA), + Interval(0x1D00, 0x1DF9), + Interval(0x1DFB, 0x1F15), + Interval(0x1F18, 0x1F1D), + Interval(0x1F20, 0x1F45), + Interval(0x1F48, 0x1F4D), + Interval(0x1F50, 0x1F57), + Interval(0x1F59, 0x1F59), + Interval(0x1F5B, 0x1F5B), + Interval(0x1F5D, 0x1F5D), + Interval(0x1F5F, 0x1F7D), + Interval(0x1F80, 0x1FB4), + Interval(0x1FB6, 0x1FC4), + Interval(0x1FC6, 0x1FD3), + Interval(0x1FD6, 0x1FDB), + Interval(0x1FDD, 0x1FEF), + Interval(0x1FF2, 0x1FF4), + Interval(0x1FF6, 0x1FFE), + Interval(0x2000, 0x200F), + Interval(0x2011, 0x2012), + Interval(0x2017, 0x2017), + Interval(0x201A, 0x201B), + Interval(0x201E, 0x201F), + Interval(0x2023, 0x2023), + Interval(0x2028, 0x202F), + Interval(0x2031, 0x2031), + Interval(0x2034, 0x2034), + Interval(0x2036, 0x203A), + Interval(0x203C, 0x203D), + Interval(0x203F, 0x2064), + Interval(0x2066, 0x2071), + Interval(0x2075, 0x207E), + Interval(0x2080, 0x2080), + Interval(0x2085, 0x208E), + Interval(0x2090, 0x209C), + Interval(0x20A0, 0x20A8), + Interval(0x20AA, 0x20AB), + Interval(0x20AD, 0x20BF), + Interval(0x20D0, 0x20F0), + Interval(0x2100, 0x2102), + Interval(0x2104, 0x2104), + Interval(0x2106, 0x2108), + Interval(0x210A, 0x2112), + Interval(0x2114, 0x2115), + Interval(0x2117, 0x2120), + Interval(0x2123, 0x2125), + Interval(0x2127, 0x212A), + Interval(0x212C, 0x2152), + Interval(0x2155, 0x215A), + Interval(0x215F, 0x215F), + Interval(0x216C, 0x216F), + Interval(0x217A, 0x2188), + Interval(0x218A, 0x218B), + Interval(0x219A, 0x21B7), + Interval(0x21BA, 0x21D1), + Interval(0x21D3, 0x21D3), + Interval(0x21D5, 0x21E6), + Interval(0x21E8, 0x21FF), + Interval(0x2201, 0x2201), + Interval(0x2204, 0x2206), + Interval(0x2209, 0x220A), + Interval(0x220C, 0x220E), + Interval(0x2210, 0x2210), + Interval(0x2212, 0x2214), + Interval(0x2216, 0x2219), + Interval(0x221B, 0x221C), + Interval(0x2221, 0x2222), + Interval(0x2224, 0x2224), + Interval(0x2226, 0x2226), + Interval(0x222D, 0x222D), + Interval(0x222F, 0x2233), + Interval(0x2238, 0x223B), + Interval(0x223E, 0x2247), + Interval(0x2249, 0x224B), + Interval(0x224D, 0x2251), + Interval(0x2253, 0x225F), + Interval(0x2262, 0x2263), + Interval(0x2268, 0x2269), + Interval(0x226C, 0x226D), + Interval(0x2270, 0x2281), + Interval(0x2284, 0x2285), + Interval(0x2288, 0x2294), + Interval(0x2296, 0x2298), + Interval(0x229A, 0x22A4), + Interval(0x22A6, 0x22BE), + Interval(0x22C0, 0x2311), + Interval(0x2313, 0x2319), + Interval(0x231C, 0x2328), + Interval(0x232B, 0x23E8), + Interval(0x23ED, 0x23EF), + Interval(0x23F1, 0x23F2), + Interval(0x23F4, 0x2426), + Interval(0x2440, 0x244A), + Interval(0x24EA, 0x24EA), + Interval(0x254C, 0x254F), + Interval(0x2574, 0x257F), + Interval(0x2590, 0x2591), + Interval(0x2596, 0x259F), + Interval(0x25A2, 0x25A2), + Interval(0x25AA, 0x25B1), + Interval(0x25B4, 0x25B5), + Interval(0x25B8, 0x25BB), + Interval(0x25BE, 0x25BF), + Interval(0x25C2, 0x25C5), + Interval(0x25C9, 0x25CA), + Interval(0x25CC, 0x25CD), + Interval(0x25D2, 0x25E1), + Interval(0x25E6, 0x25EE), + Interval(0x25F0, 0x25FC), + Interval(0x25FF, 0x2604), + Interval(0x2607, 0x2608), + Interval(0x260A, 0x260D), + Interval(0x2610, 0x2613), + Interval(0x2616, 0x261B), + Interval(0x261D, 0x261D), + Interval(0x261F, 0x263F), + Interval(0x2641, 0x2641), + Interval(0x2643, 0x2647), + Interval(0x2654, 0x265F), + Interval(0x2662, 0x2662), + Interval(0x2666, 0x2666), + Interval(0x266B, 0x266B), + Interval(0x266E, 0x266E), + Interval(0x2670, 0x267E), + Interval(0x2680, 0x2692), + Interval(0x2694, 0x269D), + Interval(0x26A0, 0x26A0), + Interval(0x26A2, 0x26A9), + Interval(0x26AC, 0x26BC), + Interval(0x26C0, 0x26C3), + Interval(0x26E2, 0x26E2), + Interval(0x26E4, 0x26E7), + Interval(0x2700, 0x2704), + Interval(0x2706, 0x2709), + Interval(0x270C, 0x2727), + Interval(0x2729, 0x273C), + Interval(0x273E, 0x274B), + Interval(0x274D, 0x274D), + Interval(0x274F, 0x2752), + Interval(0x2756, 0x2756), + Interval(0x2758, 0x2775), + Interval(0x2780, 0x2794), + Interval(0x2798, 0x27AF), + Interval(0x27B1, 0x27BE), + Interval(0x27C0, 0x27E5), + Interval(0x27EE, 0x2984), + Interval(0x2987, 0x2B1A), + Interval(0x2B1D, 0x2B4F), + Interval(0x2B51, 0x2B54), + Interval(0x2B5A, 0x2B73), + Interval(0x2B76, 0x2B95), + Interval(0x2B97, 0x2C2E), + Interval(0x2C30, 0x2C5E), + Interval(0x2C60, 0x2CF3), + Interval(0x2CF9, 0x2D25), + Interval(0x2D27, 0x2D27), + Interval(0x2D2D, 0x2D2D), + Interval(0x2D30, 0x2D67), + Interval(0x2D6F, 0x2D70), + Interval(0x2D7F, 0x2D96), + Interval(0x2DA0, 0x2DA6), + Interval(0x2DA8, 0x2DAE), + Interval(0x2DB0, 0x2DB6), + Interval(0x2DB8, 0x2DBE), + Interval(0x2DC0, 0x2DC6), + Interval(0x2DC8, 0x2DCE), + Interval(0x2DD0, 0x2DD6), + Interval(0x2DD8, 0x2DDE), + Interval(0x2DE0, 0x2E52), + Interval(0x303F, 0x303F), + Interval(0x4DC0, 0x4DFF), + Interval(0xA4D0, 0xA62B), + Interval(0xA640, 0xA6F7), + Interval(0xA700, 0xA7BF), + Interval(0xA7C2, 0xA7CA), + Interval(0xA7F5, 0xA82C), + Interval(0xA830, 0xA839), + Interval(0xA840, 0xA877), + Interval(0xA880, 0xA8C5), + Interval(0xA8CE, 0xA8D9), + Interval(0xA8E0, 0xA953), + Interval(0xA95F, 0xA95F), + Interval(0xA980, 0xA9CD), + Interval(0xA9CF, 0xA9D9), + Interval(0xA9DE, 0xA9FE), + Interval(0xAA00, 0xAA36), + Interval(0xAA40, 0xAA4D), + Interval(0xAA50, 0xAA59), + Interval(0xAA5C, 0xAAC2), + Interval(0xAADB, 0xAAF6), + Interval(0xAB01, 0xAB06), + Interval(0xAB09, 0xAB0E), + Interval(0xAB11, 0xAB16), + Interval(0xAB20, 0xAB26), + Interval(0xAB28, 0xAB2E), + Interval(0xAB30, 0xAB6B), + Interval(0xAB70, 0xABED), + Interval(0xABF0, 0xABF9), + Interval(0xD7B0, 0xD7C6), + Interval(0xD7CB, 0xD7FB), + Interval(0xD800, 0xDFFF), + Interval(0xFB00, 0xFB06), + Interval(0xFB13, 0xFB17), + Interval(0xFB1D, 0xFB36), + Interval(0xFB38, 0xFB3C), + Interval(0xFB3E, 0xFB3E), + Interval(0xFB40, 0xFB41), + Interval(0xFB43, 0xFB44), + Interval(0xFB46, 0xFBC1), + Interval(0xFBD3, 0xFD3F), + Interval(0xFD50, 0xFD8F), + Interval(0xFD92, 0xFDC7), + Interval(0xFDF0, 0xFDFD), + Interval(0xFE20, 0xFE2F), + Interval(0xFE70, 0xFE74), + Interval(0xFE76, 0xFEFC), + Interval(0xFEFF, 0xFEFF), + Interval(0xFFF9, 0xFFFC), + Interval(0x10000, 0x1000B), + Interval(0x1000D, 0x10026), + Interval(0x10028, 0x1003A), + Interval(0x1003C, 0x1003D), + Interval(0x1003F, 0x1004D), + Interval(0x10050, 0x1005D), + Interval(0x10080, 0x100FA), + Interval(0x10100, 0x10102), + Interval(0x10107, 0x10133), + Interval(0x10137, 0x1018E), + Interval(0x10190, 0x1019C), + Interval(0x101A0, 0x101A0), + Interval(0x101D0, 0x101FD), + Interval(0x10280, 0x1029C), + Interval(0x102A0, 0x102D0), + Interval(0x102E0, 0x102FB), + Interval(0x10300, 0x10323), + Interval(0x1032D, 0x1034A), + Interval(0x10350, 0x1037A), + Interval(0x10380, 0x1039D), + Interval(0x1039F, 0x103C3), + Interval(0x103C8, 0x103D5), + Interval(0x10400, 0x1049D), + Interval(0x104A0, 0x104A9), + Interval(0x104B0, 0x104D3), + Interval(0x104D8, 0x104FB), + Interval(0x10500, 0x10527), + Interval(0x10530, 0x10563), + Interval(0x1056F, 0x1056F), + Interval(0x10600, 0x10736), + Interval(0x10740, 0x10755), + Interval(0x10760, 0x10767), + Interval(0x10800, 0x10805), + Interval(0x10808, 0x10808), + Interval(0x1080A, 0x10835), + Interval(0x10837, 0x10838), + Interval(0x1083C, 0x1083C), + Interval(0x1083F, 0x10855), + Interval(0x10857, 0x1089E), + Interval(0x108A7, 0x108AF), + Interval(0x108E0, 0x108F2), + Interval(0x108F4, 0x108F5), + Interval(0x108FB, 0x1091B), + Interval(0x1091F, 0x10939), + Interval(0x1093F, 0x1093F), + Interval(0x10980, 0x109B7), + Interval(0x109BC, 0x109CF), + Interval(0x109D2, 0x10A03), + Interval(0x10A05, 0x10A06), + Interval(0x10A0C, 0x10A13), + Interval(0x10A15, 0x10A17), + Interval(0x10A19, 0x10A35), + Interval(0x10A38, 0x10A3A), + Interval(0x10A3F, 0x10A48), + Interval(0x10A50, 0x10A58), + Interval(0x10A60, 0x10A9F), + Interval(0x10AC0, 0x10AE6), + Interval(0x10AEB, 0x10AF6), + Interval(0x10B00, 0x10B35), + Interval(0x10B39, 0x10B55), + Interval(0x10B58, 0x10B72), + Interval(0x10B78, 0x10B91), + Interval(0x10B99, 0x10B9C), + Interval(0x10BA9, 0x10BAF), + Interval(0x10C00, 0x10C48), + Interval(0x10C80, 0x10CB2), + Interval(0x10CC0, 0x10CF2), + Interval(0x10CFA, 0x10D27), + Interval(0x10D30, 0x10D39), + Interval(0x10E60, 0x10E7E), + Interval(0x10E80, 0x10EA9), + Interval(0x10EAB, 0x10EAD), + Interval(0x10EB0, 0x10EB1), + Interval(0x10F00, 0x10F27), + Interval(0x10F30, 0x10F59), + Interval(0x10FB0, 0x10FCB), + Interval(0x10FE0, 0x10FF6), + Interval(0x11000, 0x1104D), + Interval(0x11052, 0x1106F), + Interval(0x1107F, 0x110C1), + Interval(0x110CD, 0x110CD), + Interval(0x110D0, 0x110E8), + Interval(0x110F0, 0x110F9), + Interval(0x11100, 0x11134), + Interval(0x11136, 0x11147), + Interval(0x11150, 0x11176), + Interval(0x11180, 0x111DF), + Interval(0x111E1, 0x111F4), + Interval(0x11200, 0x11211), + Interval(0x11213, 0x1123E), + Interval(0x11280, 0x11286), + Interval(0x11288, 0x11288), + Interval(0x1128A, 0x1128D), + Interval(0x1128F, 0x1129D), + Interval(0x1129F, 0x112A9), + Interval(0x112B0, 0x112EA), + Interval(0x112F0, 0x112F9), + Interval(0x11300, 0x11303), + Interval(0x11305, 0x1130C), + Interval(0x1130F, 0x11310), + Interval(0x11313, 0x11328), + Interval(0x1132A, 0x11330), + Interval(0x11332, 0x11333), + Interval(0x11335, 0x11339), + Interval(0x1133B, 0x11344), + Interval(0x11347, 0x11348), + Interval(0x1134B, 0x1134D), + Interval(0x11350, 0x11350), + Interval(0x11357, 0x11357), + Interval(0x1135D, 0x11363), + Interval(0x11366, 0x1136C), + Interval(0x11370, 0x11374), + Interval(0x11400, 0x1145B), + Interval(0x1145D, 0x11461), + Interval(0x11480, 0x114C7), + Interval(0x114D0, 0x114D9), + Interval(0x11580, 0x115B5), + Interval(0x115B8, 0x115DD), + Interval(0x11600, 0x11644), + Interval(0x11650, 0x11659), + Interval(0x11660, 0x1166C), + Interval(0x11680, 0x116B8), + Interval(0x116C0, 0x116C9), + Interval(0x11700, 0x1171A), + Interval(0x1171D, 0x1172B), + Interval(0x11730, 0x1173F), + Interval(0x11800, 0x1183B), + Interval(0x118A0, 0x118F2), + Interval(0x118FF, 0x11906), + Interval(0x11909, 0x11909), + Interval(0x1190C, 0x11913), + Interval(0x11915, 0x11916), + Interval(0x11918, 0x11935), + Interval(0x11937, 0x11938), + Interval(0x1193B, 0x11946), + Interval(0x11950, 0x11959), + Interval(0x119A0, 0x119A7), + Interval(0x119AA, 0x119D7), + Interval(0x119DA, 0x119E4), + Interval(0x11A00, 0x11A47), + Interval(0x11A50, 0x11AA2), + Interval(0x11AC0, 0x11AF8), + Interval(0x11C00, 0x11C08), + Interval(0x11C0A, 0x11C36), + Interval(0x11C38, 0x11C45), + Interval(0x11C50, 0x11C6C), + Interval(0x11C70, 0x11C8F), + Interval(0x11C92, 0x11CA7), + Interval(0x11CA9, 0x11CB6), + Interval(0x11D00, 0x11D06), + Interval(0x11D08, 0x11D09), + Interval(0x11D0B, 0x11D36), + Interval(0x11D3A, 0x11D3A), + Interval(0x11D3C, 0x11D3D), + Interval(0x11D3F, 0x11D47), + Interval(0x11D50, 0x11D59), + Interval(0x11D60, 0x11D65), + Interval(0x11D67, 0x11D68), + Interval(0x11D6A, 0x11D8E), + Interval(0x11D90, 0x11D91), + Interval(0x11D93, 0x11D98), + Interval(0x11DA0, 0x11DA9), + Interval(0x11EE0, 0x11EF8), + Interval(0x11FB0, 0x11FB0), + Interval(0x11FC0, 0x11FF1), + Interval(0x11FFF, 0x12399), + Interval(0x12400, 0x1246E), + Interval(0x12470, 0x12474), + Interval(0x12480, 0x12543), + Interval(0x13000, 0x1342E), + Interval(0x13430, 0x13438), + Interval(0x14400, 0x14646), + Interval(0x16800, 0x16A38), + Interval(0x16A40, 0x16A5E), + Interval(0x16A60, 0x16A69), + Interval(0x16A6E, 0x16A6F), + Interval(0x16AD0, 0x16AED), + Interval(0x16AF0, 0x16AF5), + Interval(0x16B00, 0x16B45), + Interval(0x16B50, 0x16B59), + Interval(0x16B5B, 0x16B61), + Interval(0x16B63, 0x16B77), + Interval(0x16B7D, 0x16B8F), + Interval(0x16E40, 0x16E9A), + Interval(0x16F00, 0x16F4A), + Interval(0x16F4F, 0x16F87), + Interval(0x16F8F, 0x16F9F), + Interval(0x1BC00, 0x1BC6A), + Interval(0x1BC70, 0x1BC7C), + Interval(0x1BC80, 0x1BC88), + Interval(0x1BC90, 0x1BC99), + Interval(0x1BC9C, 0x1BCA3), + Interval(0x1D000, 0x1D0F5), + Interval(0x1D100, 0x1D126), + Interval(0x1D129, 0x1D1E8), + Interval(0x1D200, 0x1D245), + Interval(0x1D2E0, 0x1D2F3), + Interval(0x1D300, 0x1D356), + Interval(0x1D360, 0x1D378), + Interval(0x1D400, 0x1D454), + Interval(0x1D456, 0x1D49C), + Interval(0x1D49E, 0x1D49F), + Interval(0x1D4A2, 0x1D4A2), + Interval(0x1D4A5, 0x1D4A6), + Interval(0x1D4A9, 0x1D4AC), + Interval(0x1D4AE, 0x1D4B9), + Interval(0x1D4BB, 0x1D4BB), + Interval(0x1D4BD, 0x1D4C3), + Interval(0x1D4C5, 0x1D505), + Interval(0x1D507, 0x1D50A), + Interval(0x1D50D, 0x1D514), + Interval(0x1D516, 0x1D51C), + Interval(0x1D51E, 0x1D539), + Interval(0x1D53B, 0x1D53E), + Interval(0x1D540, 0x1D544), + Interval(0x1D546, 0x1D546), + Interval(0x1D54A, 0x1D550), + Interval(0x1D552, 0x1D6A5), + Interval(0x1D6A8, 0x1D7CB), + Interval(0x1D7CE, 0x1DA8B), + Interval(0x1DA9B, 0x1DA9F), + Interval(0x1DAA1, 0x1DAAF), + Interval(0x1E000, 0x1E006), + Interval(0x1E008, 0x1E018), + Interval(0x1E01B, 0x1E021), + Interval(0x1E023, 0x1E024), + Interval(0x1E026, 0x1E02A), + Interval(0x1E100, 0x1E12C), + Interval(0x1E130, 0x1E13D), + Interval(0x1E140, 0x1E149), + Interval(0x1E14E, 0x1E14F), + Interval(0x1E2C0, 0x1E2F9), + Interval(0x1E2FF, 0x1E2FF), + Interval(0x1E800, 0x1E8C4), + Interval(0x1E8C7, 0x1E8D6), + Interval(0x1E900, 0x1E94B), + Interval(0x1E950, 0x1E959), + Interval(0x1E95E, 0x1E95F), + Interval(0x1EC71, 0x1ECB4), + Interval(0x1ED01, 0x1ED3D), + Interval(0x1EE00, 0x1EE03), + Interval(0x1EE05, 0x1EE1F), + Interval(0x1EE21, 0x1EE22), + Interval(0x1EE24, 0x1EE24), + Interval(0x1EE27, 0x1EE27), + Interval(0x1EE29, 0x1EE32), + Interval(0x1EE34, 0x1EE37), + Interval(0x1EE39, 0x1EE39), + Interval(0x1EE3B, 0x1EE3B), + Interval(0x1EE42, 0x1EE42), + Interval(0x1EE47, 0x1EE47), + Interval(0x1EE49, 0x1EE49), + Interval(0x1EE4B, 0x1EE4B), + Interval(0x1EE4D, 0x1EE4F), + Interval(0x1EE51, 0x1EE52), + Interval(0x1EE54, 0x1EE54), + Interval(0x1EE57, 0x1EE57), + Interval(0x1EE59, 0x1EE59), + Interval(0x1EE5B, 0x1EE5B), + Interval(0x1EE5D, 0x1EE5D), + Interval(0x1EE5F, 0x1EE5F), + Interval(0x1EE61, 0x1EE62), + Interval(0x1EE64, 0x1EE64), + Interval(0x1EE67, 0x1EE6A), + Interval(0x1EE6C, 0x1EE72), + Interval(0x1EE74, 0x1EE77), + Interval(0x1EE79, 0x1EE7C), + Interval(0x1EE7E, 0x1EE7E), + Interval(0x1EE80, 0x1EE89), + Interval(0x1EE8B, 0x1EE9B), + Interval(0x1EEA1, 0x1EEA3), + Interval(0x1EEA5, 0x1EEA9), + Interval(0x1EEAB, 0x1EEBB), + Interval(0x1EEF0, 0x1EEF1), + Interval(0x1F000, 0x1F003), + Interval(0x1F005, 0x1F02B), + Interval(0x1F030, 0x1F093), + Interval(0x1F0A0, 0x1F0AE), + Interval(0x1F0B1, 0x1F0BF), + Interval(0x1F0C1, 0x1F0CE), + Interval(0x1F0D1, 0x1F0F5), + Interval(0x1F10B, 0x1F10F), + Interval(0x1F12E, 0x1F12F), + Interval(0x1F16A, 0x1F16F), + Interval(0x1F1AD, 0x1F1AD), + Interval(0x1F1E6, 0x1F1FF), + Interval(0x1F321, 0x1F32C), + Interval(0x1F336, 0x1F336), + Interval(0x1F37D, 0x1F37D), + Interval(0x1F394, 0x1F39F), + Interval(0x1F3CB, 0x1F3CE), + Interval(0x1F3D4, 0x1F3DF), + Interval(0x1F3F1, 0x1F3F3), + Interval(0x1F3F5, 0x1F3F7), + Interval(0x1F43F, 0x1F43F), + Interval(0x1F441, 0x1F441), + Interval(0x1F4FD, 0x1F4FE), + Interval(0x1F53E, 0x1F54A), + Interval(0x1F54F, 0x1F54F), + Interval(0x1F568, 0x1F579), + Interval(0x1F57B, 0x1F594), + Interval(0x1F597, 0x1F5A3), + Interval(0x1F5A5, 0x1F5FA), + Interval(0x1F650, 0x1F67F), + Interval(0x1F6C6, 0x1F6CB), + Interval(0x1F6CD, 0x1F6CF), + Interval(0x1F6D3, 0x1F6D4), + Interval(0x1F6E0, 0x1F6EA), + Interval(0x1F6F0, 0x1F6F3), + Interval(0x1F700, 0x1F773), + Interval(0x1F780, 0x1F7D8), + Interval(0x1F800, 0x1F80B), + Interval(0x1F810, 0x1F847), + Interval(0x1F850, 0x1F859), + Interval(0x1F860, 0x1F887), + Interval(0x1F890, 0x1F8AD), + Interval(0x1F8B0, 0x1F8B1), + Interval(0x1F900, 0x1F90B), + Interval(0x1F93B, 0x1F93B), + Interval(0x1F946, 0x1F946), + Interval(0x1FA00, 0x1FA53), + Interval(0x1FA60, 0x1FA6D), + Interval(0x1FB00, 0x1FB92), + Interval(0x1FB94, 0x1FBCA), + Interval(0x1FBF0, 0x1FBF9), + Interval(0xE0001, 0xE0001), + Interval(0xE0020, 0xE007F), +) + +alias emoji = List[Interval]( + Interval(0x203C, 0x203C), + Interval(0x2049, 0x2049), + Interval(0x2122, 0x2122), + Interval(0x2139, 0x2139), + Interval(0x2194, 0x2199), + Interval(0x21A9, 0x21AA), + Interval(0x231A, 0x231B), + Interval(0x2328, 0x2328), + Interval(0x2388, 0x2388), + Interval(0x23CF, 0x23CF), + Interval(0x23E9, 0x23F3), + Interval(0x23F8, 0x23FA), + Interval(0x24C2, 0x24C2), + Interval(0x25AA, 0x25AB), + Interval(0x25B6, 0x25B6), + Interval(0x25C0, 0x25C0), + Interval(0x25FB, 0x25FE), + Interval(0x2600, 0x2605), + Interval(0x2607, 0x2612), + Interval(0x2614, 0x2685), + Interval(0x2690, 0x2705), + Interval(0x2708, 0x2712), + Interval(0x2714, 0x2714), + Interval(0x2716, 0x2716), + Interval(0x271D, 0x271D), + Interval(0x2721, 0x2721), + Interval(0x2728, 0x2728), + Interval(0x2733, 0x2734), + Interval(0x2744, 0x2744), + Interval(0x2747, 0x2747), + Interval(0x274C, 0x274C), + Interval(0x274E, 0x274E), + Interval(0x2753, 0x2755), + Interval(0x2757, 0x2757), + Interval(0x2763, 0x2767), + Interval(0x2795, 0x2797), + Interval(0x27A1, 0x27A1), + Interval(0x27B0, 0x27B0), + Interval(0x27BF, 0x27BF), + Interval(0x2934, 0x2935), + Interval(0x2B05, 0x2B07), + Interval(0x2B1B, 0x2B1C), + Interval(0x2B50, 0x2B50), + Interval(0x2B55, 0x2B55), + Interval(0x3030, 0x3030), + Interval(0x303D, 0x303D), + Interval(0x3297, 0x3297), + Interval(0x3299, 0x3299), + Interval(0x1F000, 0x1F0FF), + Interval(0x1F10D, 0x1F10F), + Interval(0x1F12F, 0x1F12F), + Interval(0x1F16C, 0x1F171), + Interval(0x1F17E, 0x1F17F), + Interval(0x1F18E, 0x1F18E), + Interval(0x1F191, 0x1F19A), + Interval(0x1F1AD, 0x1F1E5), + Interval(0x1F201, 0x1F20F), + Interval(0x1F21A, 0x1F21A), + Interval(0x1F22F, 0x1F22F), + Interval(0x1F232, 0x1F23A), + Interval(0x1F23C, 0x1F23F), + Interval(0x1F249, 0x1F3FA), + Interval(0x1F400, 0x1F53D), + Interval(0x1F546, 0x1F64F), + Interval(0x1F680, 0x1F6FF), + Interval(0x1F774, 0x1F77F), + Interval(0x1F7D5, 0x1F7FF), + Interval(0x1F80C, 0x1F80F), + Interval(0x1F848, 0x1F84F), + Interval(0x1F85A, 0x1F85F), + Interval(0x1F888, 0x1F88F), + Interval(0x1F8AE, 0x1F8FF), + Interval(0x1F90C, 0x1F93A), + Interval(0x1F93C, 0x1F945), + Interval(0x1F947, 0x1FAFF), + Interval(0x1FC00, 0x1FFFD), +) + +alias private = List[Interval]( + Interval(0x00E000, 0x00F8FF), + Interval(0x0F0000, 0x0FFFFD), + Interval(0x100000, 0x10FFFD), +) + +alias nonprint = List[Interval]( + Interval(0x0000, 0x001F), + Interval(0x007F, 0x009F), + Interval(0x00AD, 0x00AD), + Interval(0x070F, 0x070F), + Interval(0x180B, 0x180E), + Interval(0x200B, 0x200F), + Interval(0x2028, 0x202E), + Interval(0x206A, 0x206F), + Interval(0xD800, 0xDFFF), + Interval(0xFEFF, 0xFEFF), + Interval(0xFFF9, 0xFFFB), + Interval(0xFFFE, 0xFFFF), +) diff --git a/external/gojo/unicode/utf8/width.mojo b/external/gojo/unicode/utf8/width.mojo new file mode 100644 index 0000000..5678d6e --- /dev/null +++ b/external/gojo/unicode/utf8/width.mojo @@ -0,0 +1,105 @@ +from .table import Interval, narrow, combining, doublewidth, ambiguous, emoji, nonprint + + +@value +struct Condition: + """Condition have flag EastAsianWidth whether the current locale is CJK or not.""" + + var east_asian_width: Bool + var strict_emoji_neutral: Bool + + fn rune_width(self, r: UInt32) -> Int: + """Returns the number of cells in r. + See http://www.unicode.org/reports/tr11/.""" + if r < 0 or r > 0x10FFFF: + return 0 + + if not self.east_asian_width: + if r < 0x20: + return 0 + # nonprint + elif (r >= 0x7F and r <= 0x9F) or r == 0xAD: + return 0 + elif r < 0x300: + return 1 + elif in_table(r, narrow): + return 1 + elif in_tables(r, nonprint, combining): + return 0 + elif in_table(r, doublewidth): + return 2 + else: + return 1 + else: + if in_tables(r, nonprint, combining): + return 0 + elif in_table(r, narrow): + return 1 + elif in_tables(r, ambiguous, doublewidth): + return 2 + elif in_table(r, ambiguous) or in_table(r, emoji): + return 2 + elif not self.strict_emoji_neutral and in_tables(r, ambiguous, emoji, narrow): + return 2 + else: + return 1 + + fn string_width(self, s: String) -> Int: + """Return width as you can see.""" + var width = 0 + for r in s: + width += self.rune_width(ord(String(r))) + return width + + +fn in_tables(r: UInt32, *ts: List[Interval]) -> Bool: + for t in ts: + if in_table(r, t[]): + return True + return False + + +fn in_table(r: UInt32, t: List[Interval]) -> Bool: + if r < t[0].first: + return False + + var bot = 0 + var top = len(t) - 1 + while top >= bot: + var mid = (bot + top) >> 1 + + if t[mid].last < r: + bot = mid + 1 + elif t[mid].first > r: + top = mid - 1 + else: + return True + + return False + + +alias DEFAULT_CONDITION = Condition(east_asian_width=False, strict_emoji_neutral=True) + + +fn string_width(s: String) -> Int: + """Return width as you can see. + + Args: + s: The string to calculate the width of. + + Returns: + The printable width of the string. + """ + return DEFAULT_CONDITION.string_width(s) + + +fn rune_width(rune: UInt32) -> Int: + """Return width as you can see. + + Args: + rune: The rune to calculate the width of. + + Returns: + The printable width of the rune. + """ + return DEFAULT_CONDITION.rune_width(rune) diff --git a/external/hue/color.mojo b/external/hue/color.mojo index b5e1600..dccdd99 100644 --- a/external/hue/color.mojo +++ b/external/hue/color.mojo @@ -136,12 +136,12 @@ struct Color(Stringable): fn almost_equal_rgb(self, c2: Self) -> Bool: """Check for equality between colors within the tolerance Delta (1/255).""" - return math.abs(self.R - c2.R) + math.abs(self.G - c2.G) + math.abs(self.B - c2.B) < 3.0 * Delta + return abs(self.R - c2.R) + abs(self.G - c2.G) + abs(self.B - c2.B) < 3.0 * Delta fn hsv(self) -> (Float64, Float64, Float64): """Hsv returns the Hue [0..360], Saturation and Value [0..1] of the color.""" - var min = math.min(math.min(self.R, self.G), self.B) - var v = math.max(math.max(self.R, self.G), self.B) + var min = min(min(self.R, self.G), self.B) + var v = max(max(self.R, self.G), self.B) var C = v - min var s = 0.0 @@ -151,7 +151,7 @@ struct Color(Stringable): var h = 0.0 # We use 0 instead of undefined as in wp. if min != v: if v == self.R: - h = math.mod((self.G - self.B) / C, 6.0) + h = (self.G - self.B) / C % 6.0 if v == self.G: h = (self.B - self.R) / C + 2.0 if v == self.B: @@ -163,8 +163,8 @@ struct Color(Stringable): fn hsl(self) -> (Float64, Float64, Float64): """Hsl returns the Hue [0..360], Saturation [0..1], and Luminance (lightness) [0..1] of the color.""" - var min = math.min(math.min(self.R, self.G), self.B) - var max = math.max(math.max(self.R, self.G), self.B) + var min = min(min(self.R, self.G), self.B) + var max = max(max(self.R, self.G), self.B) var l = (max + min) / 2.0 @@ -360,7 +360,7 @@ struct Color(Stringable): var cabmean = (cab1 + cab2) / 2 var p: Float64 = 25.0 - var g = 0.5 * (1 - math.sqrt(math.pow(cabmean, 7) / (math.pow(cabmean, 7) + math.pow(p, 7)))) + var g = 0.5 * (1 - math.sqrt((cabmean**7) / ((cabmean**7) + (p**7)))) var ap1 = (1 + g) * a1 var ap2 = (1 + g) * a2 var cp1 = math.sqrt(sq(ap1) + sq(b1)) @@ -396,7 +396,7 @@ struct Color(Stringable): var hpmean = hp1 + hp2 if cpProduct != 0: hpmean /= 2 - if math.abs(hp1 - hp2) > 180: + if abs(hp1 - hp2) > 180: if hp1 + hp2 < 360: hpmean += 180 else: @@ -406,7 +406,7 @@ struct Color(Stringable): 2 * hpmean * pi / 180 ) + 0.32 * math.cos((3 * hpmean + 6) * pi / 180) - 0.2 * math.cos((4 * hpmean - 63) * pi / 180) var deltaTheta = 30 * math.exp(-sq((hpmean - 275) / 25)) - var rc = 2 * math.sqrt(math.pow(cpmean, 7) / (math.pow(cpmean, 7) + math.pow(p, 7))) + var rc = 2 * math.sqrt((cpmean**7) / ((cpmean**7) + (p**7))) var sl = 1 + (0.015 * sq(lpmean - 50)) / math.sqrt(20 + sq(lpmean - 50)) var sc = 1 + 0.045 * cpmean var sh = 1 + 0.015 * cpmean * t @@ -563,8 +563,8 @@ fn interp_angle(a0: Float64, a1: Float64, t: Float64) -> Float64: """Utility used by Hxx color-spaces for interpolating between two angles in [0,360].""" # Based on the answer here: http://stackoverflow.com/a/14498790/2366315 # With potential proof that it works here: http://math.stackexchange.com/a/2144499 - var delta = math.mod(math.mod(a1 - a0, 360.0) + 540.0, 360.0) - 180.0 - return math.mod(a0 + t * delta + 360.0, 360.0) + var delta = ((((a1 - a0) % 360.0) + 540.0)) % 360.0 - 180.0 + return (a0 + t * delta + 360.0) % 360.0 ### HSV ### @@ -577,7 +577,7 @@ fn hsv(h: Float64, s: Float64, v: Float64) -> Color: """Hsv creates a new Color given a Hue in [0..360], a Saturation and a Value in [0..1].""" var hp = h / 60.0 var C = v * s - var X = C * (1.0 - math.abs(math.mod(hp, 2.0) - 1.0)) + var X = C * (1.0 - abs((hp % 2.0) - 1.0)) var m = v - C var r = 0.0 var g = 0.0 @@ -783,7 +783,7 @@ fn xyz_to_xyY_white_ref(X: Float64, Y: Float64, Z: Float64, wref: List[Float64]) var N = X + Y + Z var x = X var y = Y - if math.abs(N) < 1e-14: + if abs(N) < 1e-14: x = wref[0] / (wref[0] + wref[1] + wref[2]) y = wref[1] / (wref[0] + wref[1] + wref[2]) else: @@ -932,8 +932,8 @@ fn Luv_white_ref(l: Float64, u: Float64, v: Float64, wref: List[Float64]) -> Col fn lab_to_hcl(L: Float64, a: Float64, b: Float64) -> (Float64, Float64, Float64): var h = 0.0 - if math.abs(b - a) > 1e-4 and math.abs(a) > 1e-4: - var h = math.mod(57.29577951308232087721 * math.atan2(b, a) + 360.0, 360.0) # Rad2Deg + if abs(b - a) > 1e-4 and abs(a) > 1e-4: + h = (57.29577951308232087721 * math.atan2(b, a) + 360.0) % 360.0 # Rad2Deg var c = math.sqrt(sq(a) + sq(b)) var l = L @@ -1085,8 +1085,8 @@ fn xyz_to_Luv_white_ref(x: Float64, y: Float64, z: Float64, wref: List[Float64]) fn Luv_To_LuvLCh(L: Float64, u: Float64, v: Float64) -> (Float64, Float64, Float64): # Oops, floating point workaround necessary if u ~= v and both are very small (i.e. almost zero). var h: Float64 - if math.abs(v - u) > 1e-4 and math.abs(u) > 1e-4: - h = math.mod(57.29577951308232087721 * math.atan2(v, u) + 360.0, 360.0) # Rad2Deg + if abs(v - u) > 1e-4 and abs(u) > 1e-4: + h = (57.29577951308232087721 * math.atan2(v, u) + 360.0) % 360.0 # Rad2Deg else: h = 0.0 diff --git a/external/hue/happy_palettegen.mojo b/external/hue/happy_palettegen.mojo index 4678341..8bd0207 100644 --- a/external/hue/happy_palettegen.mojo +++ b/external/hue/happy_palettegen.mojo @@ -8,7 +8,7 @@ fn fast_happy_palette(colors_count: Int) -> List[Color]: evenly along their Hue. This is fast but not always pretty. If you've got time to spare, use Lab (the non-fast below).""" var colors = List[Color](capacity=colors_count) - for i in range(colors_count): + for _ in range(colors_count): colors.append(Color(0, 0, 0)) var i = 0 diff --git a/external/hue/hsluv.mojo b/external/hue/hsluv.mojo index 78a8aff..4097d12 100644 --- a/external/hue/hsluv.mojo +++ b/external/hue/hsluv.mojo @@ -27,11 +27,11 @@ fn HSLuvToLuvLCh(h: Float64, s: Float64, l: Float64) -> (Float64, Float64, Float var c: Float64 var max: Float64 - if l > 99.9999999 or l < 0.00000001: + if tmp_l > 99.9999999 or tmp_l < 0.00000001: c = 0.0 else: max = max_chroma_for_lh(l, h) - c = max / 100.0 * s + c = max / 100.0 * tmp_s # c is [-100..100], but for LCh it's supposed to be almost [-1..1] return clamp01(l / 100.0), c / 100.0, h diff --git a/external/hue/math.mojo b/external/hue/math.mojo index e86de58..93233fb 100644 --- a/external/hue/math.mojo +++ b/external/hue/math.mojo @@ -1,5 +1,4 @@ -from math import max, min -from math.limit import max_finite +from utils.numerics import max_finite fn cube(v: Float64) -> Float64: diff --git a/external/hue/soft_palettegen.mojo b/external/hue/soft_palettegen.mojo index 7f17d44..74a93c6 100644 --- a/external/hue/soft_palettegen.mojo +++ b/external/hue/soft_palettegen.mojo @@ -1,6 +1,6 @@ from collections.optional import Optional from random import random_si64 -from math.limit import inf +from utils.numerics import inf import math from .math import sq from .color import lab @@ -39,7 +39,7 @@ fn in_stack(haystack: List[lab_t], upto: Int, needle: lab_t) -> Bool: fn labs_2_cols(labs: List[lab_t]) -> List[Color]: var lab_count = len(labs) var cols = List[Color](capacity=lab_count) - for i in range(lab_count): + for _ in range(lab_count): cols.append(Color(0.0, 0.0, 0.0)) for i in range(lab_count): @@ -79,11 +79,7 @@ alias LAB_DELTA = 1e-6 fn lab_eq(lab1: lab_t, lab2: lab_t) -> Bool: - return ( - math.abs(lab1.L - lab2.L) < LAB_DELTA - and math.abs(lab1.A - lab2.A) < LAB_DELTA - and math.abs(lab1.B - lab2.B) < LAB_DELTA - ) + return abs(lab1.L - lab2.L) < LAB_DELTA and abs(lab1.A - lab2.A) < LAB_DELTA and abs(lab1.B - lab2.B) < LAB_DELTA fn soft_palette_ex(colors_count: Int, settings: SoftPaletteSettings) raises -> List[Color]: @@ -97,7 +93,7 @@ fn soft_palette_ex(colors_count: Int, settings: SoftPaletteSettings) raises -> L @always_inline fn check(col: lab_t) -> Bool: var c = lab(col.L, col.A, col.B) - return c.is_valid() and settings.check_color.take()(col.L, col.A, col.B) + return c.is_valid() and settings.check_color.value()[](col.L, col.A, col.B) # Sample the color space. These will be the points k-means is run on. var dl = 0.05 @@ -124,9 +120,9 @@ fn soft_palette_ex(colors_count: Int, settings: SoftPaletteSettings) raises -> L if len(samples) < colors_count: raise Error( String("palettegen: more colors requested ") - + colors_count + + str(colors_count) + " than samples available " - + len(samples) + + str(len(samples)) + " Your requested color count may be wrong, you might want to use many samples or your constraint fntion" " makes the valid color space too small" ) @@ -136,7 +132,7 @@ fn soft_palette_ex(colors_count: Int, settings: SoftPaletteSettings) raises -> L # We take the initial means out of the samples, so they are in fact medoids. # This helps us avoid infinite loops or arbitrary cutoffs with too restrictive constraints. var means = List[lab_t](capacity=colors_count) - for i in range(colors_count): + for _ in range(colors_count): means.append(lab_t(0.0, 0.0, 0.0)) var i = 0 diff --git a/external/hue/warm_palettegen.mojo b/external/hue/warm_palettegen.mojo index 623e7be..87c44a8 100644 --- a/external/hue/warm_palettegen.mojo +++ b/external/hue/warm_palettegen.mojo @@ -15,7 +15,7 @@ fn fast_warm_palette(colors_count: Int) -> List[Color]: A list of colors. """ var colors = List[Color](capacity=colors_count) - for i in range(colors_count): + for _ in range(colors_count): colors.append(Color(0, 0, 0)) var i = 0 diff --git a/external/mist/__init__.mojo b/external/mist/__init__.mojo index 459e17d..915cb0e 100644 --- a/external/mist/__init__.mojo +++ b/external/mist/__init__.mojo @@ -1,9 +1,21 @@ from .color import Color -from .style import TerminalStyle -from .profile import Profile, ASCII, ANSI, ANSI256, TRUE_COLOR, AnyColor, NoColor +from .style import Style, new_style +from .profile import ( + Profile, + ASCII, + ANSI, + ANSI256, + TRUE_COLOR, + ASCII_PROFILE, + ANSI_PROFILE, + ANSI256_PROFILE, + TRUE_COLOR_PROFILE, + AnyColor, + NoColor, +) from .renderers import ( - as_color, - with_background_color, + render_as_color, + render_with_background_color, red, green, blue, diff --git a/external/mist/ansi_colors.mojo b/external/mist/ansi_colors.mojo index 3bb781d..421c727 100644 --- a/external/mist/ansi_colors.mojo +++ b/external/mist/ansi_colors.mojo @@ -1,260 +1,259 @@ -# RGB values of ANSI colors (0-255). -alias ANSI_HEX_CODES = List[String]( - "#000000", - "#000000", - "#800000", - "#008000", - "#808000", - "#000080", - "#800080", - "#008080", - "#c0c0c0", - "#808080", - "#ff0000", - "#00ff00", - "#ffff00", - "#0000ff", - "#ff00ff", - "#00ffff", - "#ffffff", - "#000000", - "#00005f", - "#000087", - "#0000af", - "#0000d7", - "#0000ff", - "#005f00", - "#005f5f", - "#005f87", - "#005faf", - "#005fd7", - "#005fff", - "#008700", - "#00875f", - "#008787", - "#0087af", - "#0087d7", - "#0087ff", - "#00af00", - "#00af5f", - "#00af87", - "#00afaf", - "#00afd7", - "#00afff", - "#00d700", - "#00d75f", - "#00d787", - "#00d7af", - "#00d7d7", - "#00d7ff", - "#00ff00", - "#00ff5f", - "#00ff87", - "#00ffaf", - "#00ffd7", - "#00ffff", - "#5f0000", - "#5f005f", - "#5f0087", - "#5f00af", - "#5f00d7", - "#5f00ff", - "#5f5f00", - "#5f5f5f", - "#5f5f87", - "#5f5faf", - "#5f5fd7", - "#5f5fff", - "#5f8700", - "#5f875f", - "#5f8787", - "#5f87af", - "#5f87d7", - "#5f87ff", - "#5faf00", - "#5faf5f", - "#5faf87", - "#5fafaf", - "#5fafd7", - "#5fafff", - "#5fd700", - "#5fd75f", - "#5fd787", - "#5fd7af", - "#5fd7d7", - "#5fd7ff", - "#5fff00", - "#5fff5f", - "#5fff87", - "#5fffaf", - "#5fffd7", - "#5fffff", - "#870000", - "#87005f", - "#870087", - "#8700af", - "#8700d7", - "#8700ff", - "#875f00", - "#875f5f", - "#875f87", - "#875faf", - "#875fd7", - "#875fff", - "#878700", - "#87875f", - "#878787", - "#8787af", - "#8787d7", - "#8787ff", - "#87af00", - "#87af5f", - "#87af87", - "#87afaf", - "#87afd7", - "#87afff", - "#87d700", - "#87d75f", - "#87d787", - "#87d7af", - "#87d7d7", - "#87d7ff", - "#87ff00", - "#87ff5f", - "#87ff87", - "#87ffaf", - "#87ffd7", - "#87ffff", - "#af0000", - "#af005f", - "#af0087", - "#af00af", - "#af00d7", - "#af00ff", - "#af5f00", - "#af5f5f", - "#af5f87", - "#af5faf", - "#af5fd7", - "#af5fff", - "#af8700", - "#af875f", - "#af8787", - "#af87af", - "#af87d7", - "#af87ff", - "#afaf00", - "#afaf5f", - "#afaf87", - "#afafaf", - "#afafd7", - "#afafff", - "#afd700", - "#afd75f", - "#afd787", - "#afd7af", - "#afd7d7", - "#afd7ff", - "#afff00", - "#afff5f", - "#afff87", - "#afffaf", - "#afffd7", - "#afffff", - "#d70000", - "#d7005f", - "#d70087", - "#d700af", - "#d700d7", - "#d700ff", - "#d75f00", - "#d75f5f", - "#d75f87", - "#d75faf", - "#d75fd7", - "#d75fff", - "#d78700", - "#d7875f", - "#d78787", - "#d787af", - "#d787d7", - "#d787ff", - "#d7af00", - "#d7af5f", - "#d7af87", - "#d7afaf", - "#d7afd7", - "#d7afff", - "#d7d700", - "#d7d75f", - "#d7d787", - "#d7d7af", - "#d7d7d7", - "#d7d7ff", - "#d7ff00", - "#d7ff5f", - "#d7ff87", - "#d7ffaf", - "#d7ffd7", - "#d7ffff", - "#ff0000", - "#ff005f", - "#ff0087", - "#ff00af", - "#ff00d7", - "#ff00ff", - "#ff5f00", - "#ff5f5f", - "#ff5f87", - "#ff5faf", - "#ff5fd7", - "#ff5fff", - "#ff8700", - "#ff875f", - "#ff8787", - "#ff87af", - "#ff87d7", - "#ff87ff", - "#ffaf00", - "#ffaf5f", - "#ffaf87", - "#ffafaf", - "#ffafd7", - "#ffafff", - "#ffd700", - "#ffd75f", - "#ffd787", - "#ffd7af", - "#ffd7d7", - "#ffd7ff", - "#ffff00", - "#ffff5f", - "#ffff87", - "#ffffaf", - "#ffffd7", - "#ffffff", - "#080808", - "#121212", - "#1c1c1c", - "#262626", - "#303030", - "#3a3a3a", - "#444444", - "#4e4e4e", - "#585858", - "#626262", - "#6c6c6c", - "#767676", - "#808080", - "#8a8a8a", - "#949494", - "#9e9e9e", - "#a8a8a8", - "#b2b2b2", - "#bcbcbc", - "#c6c6c6", - "#d0d0d0", - "#dadada", - "#e4e4e4", - "#eeeeee", +alias ANSI_HEX_CODES = List[UInt32]( + 0x000000, + 0x800000, + 0x008000, + 0x808000, + 0x000080, + 0x800080, + 0x008080, + 0xC0C0C0, + 0x808080, + 0xFF0000, + 0x00FF00, + 0xFFFF00, + 0x0000FF, + 0xFF00FF, + 0x00FFFF, + 0xFFFFFF, + 0x000000, + 0x00005F, + 0x000087, + 0x0000AF, + 0x0000D7, + 0x0000FF, + 0x005F00, + 0x005F5F, + 0x005F87, + 0x005FAF, + 0x005FD7, + 0x005FFF, + 0x008700, + 0x00875F, + 0x008787, + 0x0087AF, + 0x0087D7, + 0x0087FF, + 0x00AF00, + 0x00AF5F, + 0x00AF87, + 0x00AFAF, + 0x00AFD7, + 0x00AFFF, + 0x00D700, + 0x00D75F, + 0x00D787, + 0x00D7AF, + 0x00D7D7, + 0x00D7FF, + 0x00FF00, + 0x00FF5F, + 0x00FF87, + 0x00FFAF, + 0x00FFD7, + 0x00FFFF, + 0x5F0000, + 0x5F005F, + 0x5F0087, + 0x5F00AF, + 0x5F00D7, + 0x5F00FF, + 0x5F5F00, + 0x5F5F5F, + 0x5F5F87, + 0x5F5FAF, + 0x5F5FD7, + 0x5F5FFF, + 0x5F8700, + 0x5F875F, + 0x5F8787, + 0x5F87AF, + 0x5F87D7, + 0x5F87FF, + 0x5FAF00, + 0x5FAF5F, + 0x5FAF87, + 0x5FAFAF, + 0x5FAFD7, + 0x5FAFFF, + 0x5FD700, + 0x5FD75F, + 0x5FD787, + 0x5FD7AF, + 0x5FD7D7, + 0x5FD7FF, + 0x5FFF00, + 0x5FFF5F, + 0x5FFF87, + 0x5FFFAF, + 0x5FFFD7, + 0x5FFFFF, + 0x870000, + 0x87005F, + 0x870087, + 0x8700AF, + 0x8700D7, + 0x8700FF, + 0x875F00, + 0x875F5F, + 0x875F87, + 0x875FAF, + 0x875FD7, + 0x875FFF, + 0x878700, + 0x87875F, + 0x878787, + 0x8787AF, + 0x8787D7, + 0x8787FF, + 0x87AF00, + 0x87AF5F, + 0x87AF87, + 0x87AFAF, + 0x87AFD7, + 0x87AFFF, + 0x87D700, + 0x87D75F, + 0x87D787, + 0x87D7AF, + 0x87D7D7, + 0x87D7FF, + 0x87FF00, + 0x87FF5F, + 0x87FF87, + 0x87FFAF, + 0x87FFD7, + 0x87FFFF, + 0xAF0000, + 0xAF005F, + 0xAF0087, + 0xAF00AF, + 0xAF00D7, + 0xAF00FF, + 0xAF5F00, + 0xAF5F5F, + 0xAF5F87, + 0xAF5FAF, + 0xAF5FD7, + 0xAF5FFF, + 0xAF8700, + 0xAF875F, + 0xAF8787, + 0xAF87AF, + 0xAF87D7, + 0xAF87FF, + 0xAFAF00, + 0xAFAF5F, + 0xAFAF87, + 0xAFAFAF, + 0xAFAFD7, + 0xAFAFFF, + 0xAFD700, + 0xAFD75F, + 0xAFD787, + 0xAFD7AF, + 0xAFD7D7, + 0xAFD7FF, + 0xAFFF00, + 0xAFFF5F, + 0xAFFF87, + 0xAFFFAF, + 0xAFFFD7, + 0xAFFFFF, + 0xD70000, + 0xD7005F, + 0xD70087, + 0xD700AF, + 0xD700D7, + 0xD700FF, + 0xD75F00, + 0xD75F5F, + 0xD75F87, + 0xD75FAF, + 0xD75FD7, + 0xD75FFF, + 0xD78700, + 0xD7875F, + 0xD78787, + 0xD787AF, + 0xD787D7, + 0xD787FF, + 0xD7AF00, + 0xD7AF5F, + 0xD7AF87, + 0xD7AFAF, + 0xD7AFD7, + 0xD7AFFF, + 0xD7D700, + 0xD7D75F, + 0xD7D787, + 0xD7D7AF, + 0xD7D7D7, + 0xD7D7FF, + 0xD7FF00, + 0xD7FF5F, + 0xD7FF87, + 0xD7FFAF, + 0xD7FFD7, + 0xD7FFFF, + 0xFF0000, + 0xFF005F, + 0xFF0087, + 0xFF00AF, + 0xFF00D7, + 0xFF00FF, + 0xFF5F00, + 0xFF5F5F, + 0xFF5F87, + 0xFF5FAF, + 0xFF5FD7, + 0xFF5FFF, + 0xFF8700, + 0xFF875F, + 0xFF8787, + 0xFF87AF, + 0xFF87D7, + 0xFF87FF, + 0xFFAF00, + 0xFFAF5F, + 0xFFAF87, + 0xFFAFAF, + 0xFFAFD7, + 0xFFAFFF, + 0xFFD700, + 0xFFD75F, + 0xFFD787, + 0xFFD7AF, + 0xFFD7D7, + 0xFFD7FF, + 0xFFFF00, + 0xFFFF5F, + 0xFFFF87, + 0xFFFFAF, + 0xFFFFD7, + 0xFFFFFF, + 0x080808, + 0x121212, + 0x1C1C1C, + 0x262626, + 0x303030, + 0x3A3A3A, + 0x444444, + 0x4E4E4E, + 0x585858, + 0x626262, + 0x6C6C6C, + 0x767676, + 0x808080, + 0x8A8A8A, + 0x949494, + 0x9E9E9E, + 0xA8A8A8, + 0xB2B2B2, + 0xBCBCBC, + 0xC6C6C6, + 0xD0D0D0, + 0xDADADA, + 0xE4E4E4, + 0xEEEEEE, ) +"""RGB values of ANSI colors (0-255).""" diff --git a/external/mist/color.mojo b/external/mist/color.mojo index fa455f7..fce0ecb 100644 --- a/external/mist/color.mojo +++ b/external/mist/color.mojo @@ -1,56 +1,54 @@ -from collections.dict import Dict, KeyElement -from utils.variant import Variant import external.hue -from external.hue.math import max_float64 from .ansi_colors import ANSI_HEX_CODES -@value -struct StringKey(KeyElement): - var s: String +# Workaround for str() not working at compile time due to using an external_call to c. +fn int_to_str(owned value: Int, base: Int = 10) -> String: + """Converts an integer to a string. - fn __init__(inout self, owned s: String): - self.s = s^ - - fn __init__(inout self, s: StringLiteral): - self.s = String(s) - - fn __hash__(self) -> Int: - return hash(self.s) + Args: + value: The integer to convert to a string. + base: The base to convert the integer to. - fn __eq__(self, other: Self) -> Bool: - return self.s == other.s + Returns: + The string representation of the integer. + """ + # Catch edge case of 0 + if value == 0: + return "0" + + var temp = List[UInt8]() + var i = 0 + while value > 0: + temp.append(ord(String("0123456789abcdef")[value % base])) + i += 1 + value /= 10 - fn __ne__(self, other: Self) -> Bool: - return self.s != other.s + var buffer = List[UInt8]() + for i in range(len(temp) - 1, -1, -1): + buffer.append(temp[i]) - fn __str__(self) -> String: - return self.s + buffer.append(0) + var result = String(buffer^) + return result -alias foreground = "38" -alias background = "48" +alias FOREGROUND = "38" +alias BACKGROUND = "48" alias AnyColor = Variant[NoColor, ANSIColor, ANSI256Color, RGBColor] -trait Equalable: - fn __eq__(self: Self, other: Self) -> Bool: - ... - - -trait NotEqualable: - fn __ne__(self: Self, other: Self) -> Bool: - ... - - -trait Color(Movable, Copyable, Equalable, NotEqualable, CollectionElement): +trait Color(EqualityComparable, CollectionElement): fn sequence(self, is_background: Bool) -> String: """Sequence returns the ANSI Sequence for the color.""" ... -@value +@register_passable("trivial") struct NoColor(Color, Stringable): + fn __init__(inout self): + pass + fn __eq__(self, other: NoColor) -> Bool: return True @@ -65,11 +63,14 @@ struct NoColor(Color, Stringable): return "" -@value +@register_passable("trivial") struct ANSIColor(Color, Stringable): """ANSIColor is a color (0-15) as defined by the ANSI Standard.""" - var value: Int + var value: UInt32 + + fn __init__(inout self, value: UInt32): + self.value = value fn __eq__(self, other: ANSIColor) -> Bool: return self.value == other.value @@ -88,26 +89,22 @@ struct ANSIColor(Color, Stringable): modifier += 10 if self.value < 8: - return String(modifier + self.value + 30) - else: - return String(modifier + self.value - 8 + 90) + return int_to_str(modifier + int(self.value) + 30) + return int_to_str(modifier + int(self.value) - 8 + 90) fn __str__(self) -> String: """String returns the ANSI Sequence for the color and the text.""" - return ANSI_HEX_CODES[self.value] - - fn convert_to_rgb(self) -> hue.Color: - """Converts an ANSI color to hue.Color by looking up the hex value and converting it.""" - var hex: String = ANSI_HEX_CODES[self.value] - - return hex_to_rgb(hex) + return ANSI_HEX_CODES[int(self.value)] -@value +@register_passable("trivial") struct ANSI256Color(Color, Stringable): """ANSI256Color is a color (16-255) as defined by the ANSI Standard.""" - var value: Int + var value: UInt32 + + fn __init__(inout self, value: UInt32): + self.value = value fn __eq__(self, other: ANSI256Color) -> Bool: return self.value == other.value @@ -121,99 +118,107 @@ struct ANSI256Color(Color, Stringable): Args: is_background: Whether the color is a background color. """ - var prefix: String = foreground + var prefix: String = FOREGROUND if is_background: - prefix = background + prefix = BACKGROUND - return prefix + ";5;" + String(self.value) + return prefix + ";5;" + int_to_str(int(self.value)) fn __str__(self) -> String: """String returns the ANSI Sequence for the color and the text.""" - return ANSI_HEX_CODES[self.value] - - fn convert_to_rgb(self) -> hue.Color: - """Converts an ANSI color to hue.Color by looking up the hex value and converting it.""" - var hex: String = ANSI_HEX_CODES[self.value] - - return hex_to_rgb(hex) - - -# fn convert_base10_to_base16(value: Int) -> String: -# """Converts a base 10 number to base 16.""" -# var sum: Int = value -# while value > 1: -# var remainder = sum % 16 -# sum = sum / 16 -# print(remainder, sum) - -# print(remainder * 16) + return ANSI_HEX_CODES[int(self.value)] + + +# // ansiToRGB converts an ANSI color to a 24-bit RGB color. +# // +# // r, g, b := ansiToRGB(57) +# func ansiToRGB(ansi uint32) (uint32, uint32, uint32) { +# // For out-of-range values return black. +# if ansi > 255 { +# return 0, 0, 0 +# } + +# // Low ANSI. +# if ansi < 16 { +# h, ok := lowANSI[ansi] +# if !ok { +# return 0, 0, 0 +# } +# r, g, b := hexToRGB(h) +# return r, g, b +# } + +# // Grays. +# if ansi > 231 { +# s := (ansi-232)*10 + 8 +# return s, s, s +# } + +# // ANSI256. +# n := ansi - 16 +# b := n % 6 +# g := (n - b) / 6 % 6 +# r := (n - b - g*6) / 36 % 6 +# for _, v := range []*uint32{&r, &g, &b} { +# if *v > 0 { +# c := *v*40 + 55 +# *v = c +# } +# } + +# return r, g, b +# } + + +fn ansi_to_rgb(ansi: UInt32) -> (UInt32, UInt32, UInt32): + """Converts an ANSI color to a 24-bit RGB color.""" + # For out-of-range values return black. + if ansi > 255: + return UInt32(0), UInt32(0), UInt32(0) + + # Low ANSI. + if ansi < 16: + var h = ANSI_HEX_CODES[int(ansi)] + return hex_to_rgb(h) + + # Grays. + if ansi > 231: + var s = (ansi - 232) * 10 + 8 + return s, s, s + + # ANSI256. + var n = ansi - 16 + var b = n % 6 + var g = (n - b) / 6 % 6 + var r = (n - b - g * 6) / 36 % 6 + var rgb = List[UInt32](r, g, b) + var v = rgb[0] + var i = 0 + while i < 3: + if v > 0: + var c = v * 40 + 55 + v = c + i += 1 + return r, g, b -fn convert_base16_to_base10(value: String) -> Int: - """Converts a base 16 number to base 10. - https://www.catalyst2.com/knowledgebase/dictionary/hexadecimal-base-16-numbers/#:~:text=To%20convert%20the%20hex%20number,16%20%2B%200%20%3D%2016). - Args: - value: Hexadecimal number. +fn hex_to_rgb(hex: UInt32) -> (UInt32, UInt32, UInt32): + """Converts a number in hexadecimal format to red, green, and blue values. - Returns: - Base 10 number. + `r, g, b = hex_to_rgb(0x0000FF)`. """ - var mapping = Dict[StringKey, Int]() - mapping["0"] = 0 - mapping["1"] = 1 - mapping["2"] = 2 - mapping["3"] = 3 - mapping["4"] = 4 - mapping["5"] = 5 - mapping["6"] = 6 - mapping["7"] = 7 - mapping["8"] = 8 - mapping["9"] = 9 - mapping["a"] = 10 - mapping["b"] = 11 - mapping["c"] = 12 - mapping["d"] = 13 - mapping["e"] = 14 - mapping["f"] = 15 - - # We assume mapping.find always returns a value considering the value passed in is a valid hex value - # and the mapping has all the values. - var length = len(value) - var total: Int = 0 - for i in range(length - 1, -1, -1): - var exponent = length - 1 - i - total += mapping.find(value[i]).value()[] * (16**exponent) - - return total - - -fn hex_to_rgb(value: String) -> hue.Color: - """Converts a hex color to hue.Color. + return hex >> 16, hex >> 8 & 0xFF, hex & 0xFF - Args: - value: Hex color value. - - Returns: - Color. - """ - var hex = value[1:] - var indices = List[Int](0, 2, 4) - var results = List[Int]() - for i in indices: - results.append(convert_base16_to_base10(hex[i[] : i[] + 2])) - return hue.Color(results[0], results[1], results[2]) - - -@value +@register_passable("trivial") struct RGBColor(Color): """RGBColor is a hex-encoded color, e.g. '#abcdef'.""" - var value: String + var value: UInt32 - fn __init__(inout self, value: String): - self.value = value.lower() + fn __init__(inout self, value: UInt32): + self.value = value fn __eq__(self, other: RGBColor) -> Bool: return self.value == other.value @@ -229,32 +234,40 @@ struct RGBColor(Color): """ var rgb = hex_to_rgb(self.value) - var prefix = foreground + var prefix = FOREGROUND if is_background: - prefix = background + prefix = BACKGROUND - return prefix + String(";2;") + String(int(rgb.R)) + ";" + String(int(rgb.G)) + ";" + String(int(rgb.B)) + return ( + prefix + + String(";2;") + + int_to_str(int(rgb[0])) + + ";" + + int_to_str(int(rgb[1])) + + ";" + + int_to_str(int(rgb[2])) + ) - fn convert_to_rgb(self) -> hue.Color: - """Converts the Hex code value to hue.Color.""" - return hex_to_rgb(self.value) - -fn ansi256_to_ansi(value: Int) -> ANSIColor: +fn ansi256_to_ansi(value: UInt32) -> ANSIColor: """Converts an ANSI256 color to an ANSI color. Args: value: ANSI256 color value. """ var r: Int = 0 - var md = max_float64 + var md = hue.math.max_float64 - var h = hex_to_rgb(ANSI_HEX_CODES[value]) + var h = hex_to_rgb(ANSI_HEX_CODES[int(value)]) var i: Int = 0 while i <= 15: - var hb = hex_to_rgb(ANSI_HEX_CODES[i]) - var d = h.distance_HSLuv(hb) + var hb = hex_to_rgb(ANSI_HEX_CODES[int(i)]) + var d = hue.Color( + h[0].cast[DType.float64](), h[1].cast[DType.float64](), h[2].cast[DType.float64]() + ).distance_HSLuv( + hue.Color(hb[0].cast[DType.float64](), hb[1].cast[DType.float64](), hb[2].cast[DType.float64]()) + ) if d < md: md = d @@ -270,8 +283,7 @@ fn v2ci(value: Float64) -> Int: return 0 elif value < 115: return 1 - else: - return int((value - 35) / 40) + return int((value - 35) / 40) fn hex_to_ansi256(color: hue.Color) -> ANSI256Color: @@ -288,19 +300,19 @@ fn hex_to_ansi256(color: hue.Color) -> ANSI256Color: var ci: Int = int((36 * r) + (6 * g) + b) # 0..215 # Calculate the represented colors back from the index - var i2cv = List[Int](0, 0x5F, 0x87, 0xAF, 0xD7, 0xFF) + alias i2cv = InlineArray[Int, 6](0, 0x5F, 0x87, 0xAF, 0xD7, 0xFF) var cr = i2cv[int(r)] # r/g/b, 0..255 each var cg = i2cv[int(g)] var cb = i2cv[int(b)] # Calculate the nearest 0-based gray index at 232..255 - var grayIdx: Int + var gray_index: Int var average = (r + g + b) / 3 if average > 238: - grayIdx = 23 + gray_index = 23 else: - grayIdx = int((average - 3) / 10) # 0..23 - var gv = 8 + 10 * grayIdx # same value for r/g/b, 0..255 + gray_index = int((average - 3) / 10) # 0..23 + var gv = 8 + 10 * gray_index # same value for r/g/b, 0..255 # Return the one which is nearer to the original input rgb value # Originall had / 255.0 for r, g, and b in each of these @@ -311,4 +323,4 @@ fn hex_to_ansi256(color: hue.Color) -> ANSI256Color: if color_dist <= gray_dist: return ANSI256Color(16 + ci) - return ANSI256Color(232 + grayIdx) + return ANSI256Color(232 + gray_index) diff --git a/external/mist/profile.mojo b/external/mist/profile.mojo index 97069a5..b2abc24 100644 --- a/external/mist/profile.mojo +++ b/external/mist/profile.mojo @@ -1,4 +1,5 @@ import os +import external.hue from .color import ( NoColor, ANSIColor, @@ -8,31 +9,30 @@ from .color import ( hex_to_ansi256, ansi256_to_ansi, hex_to_rgb, + ansi_to_rgb, + int_to_str, ) - -fn contains(vector: List[Int], value: Int) -> Bool: - for i in range(vector.size): - if vector[i] == value: - return True - return False - - alias TRUE_COLOR: Int = 0 alias ANSI256: Int = 1 alias ANSI: Int = 2 alias ASCII: Int = 3 +alias TRUE_COLOR_PROFILE = Profile(TRUE_COLOR) +alias ANSI256_PROFILE = Profile(ANSI256) +alias ANSI_PROFILE = Profile(ANSI) +alias ASCII_PROFILE = Profile(ASCII) + # TODO: UNIX systems only for now. Need to add Windows, POSIX, and SOLARIS support. -fn get_color_profile() -> Profile: +fn get_color_profile() -> Int: """Queries the terminal to determine the color profile it supports. ASCII, ANSI, ANSI256, or TRUE_COLOR. """ # if not o.isTTY(): # return Ascii if os.getenv("GOOGLE_CLOUD_SHELL", "false") == "true": - return Profile(TRUE_COLOR) + return TRUE_COLOR var term = os.getenv("TERM").lower() var color_term = os.getenv("COLORTERM").lower() @@ -44,54 +44,57 @@ fn get_color_profile() -> Profile: if term.startswith("screen"): # tmux supports TRUE_COLOR, screen only ANSI256 if os.getenv("TERM_PROGRAM") != "tmux": - return Profile(ANSI256) - return Profile(TRUE_COLOR) + return ANSI256 + return TRUE_COLOR elif color_term == "yes": pass elif color_term == "true": - return Profile(ANSI256) + return ANSI256 # TERM is used by most terminals to indicate color support. if term == "xterm-kitty" or term == "wezterm" or term == "xterm-ghostty": - return Profile(TRUE_COLOR) + return TRUE_COLOR elif term == "linux": - return Profile(ANSI) + return ANSI if "256color" in term: - return Profile(ANSI256) + return ANSI256 if "color" in term: - return Profile(ANSI) + return ANSI if "ansi" in term: - return Profile(ANSI) + return ANSI - return Profile(ASCII) + return ASCII -@value +@register_passable struct Profile: + alias valid = InlineArray[Int, 4](TRUE_COLOR, ANSI256, ANSI, ASCII) var value: Int - fn __init__(inout self, value: Int) -> None: + fn __init__(inout self, value: Int): """ Initialize a new profile with the given profile type. Args: value: The setting to use for this profile. Valid values: [TRUE_COLOR, ANSI256, ANSI, ASCII]. """ - var valid = List[Int](TRUE_COLOR, ANSI256, ANSI, ASCII) - if not contains(valid, value): + if value not in Self.valid: self.value = TRUE_COLOR return self.value = value - fn __init__(inout self) -> None: + fn __init__(inout self): """ Initialize a new profile with the given profile type. """ - self = get_color_profile() + self.value = get_color_profile() + + fn __copyinit__(inout self, other: Profile): + self.value = other.value fn convert(self, color: AnyColor) -> AnyColor: """Degrades a color based on the terminal profile. @@ -103,57 +106,44 @@ struct Profile: return NoColor() if color.isa[NoColor](): - return color.get[NoColor]()[] + return color[NoColor] elif color.isa[ANSIColor](): - return color.get[ANSIColor]()[] + return color[ANSIColor] elif color.isa[ANSI256Color](): if self.value == ANSI: - return ansi256_to_ansi(color.get[ANSIColor]()[].value) + return ansi256_to_ansi(color[ANSI256Color].value) - return color.get[ANSI256Color]()[] + return color[ANSI256Color] elif color.isa[RGBColor](): - var h = hex_to_rgb(color.get[RGBColor]()[].value) + var h = hex_to_rgb(color[RGBColor].value) if self.value != TRUE_COLOR: - var ansi256 = hex_to_ansi256(h) + var ansi256 = hex_to_ansi256( + hue.Color(h[0].cast[DType.float64](), h[1].cast[DType.float64](), h[2].cast[DType.float64]()) + ) if self.value == ANSI: return ansi256_to_ansi(ansi256.value) return ansi256 - return color.get[RGBColor]()[] + return color[RGBColor] # If it somehow gets here, just return No Color until I can figure out how to just return whatever color was passed in. - return color.get[NoColor]()[] + return color[NoColor] - fn color(self, value: String) -> AnyColor: + fn color(self, value: UInt32) -> AnyColor: """Color creates a Color from a string. Valid inputs are hex colors, as well as ANSI color codes (0-15, 16-255). If an invalid input is passed in, NoColor() is returned which will not apply any coloring. Args: value: The string to convert to a color. """ - if len(value) == 0: - return NoColor() - if self.value == ASCII: return NoColor() - if value[0] == "#": - var c = RGBColor(value) - return self.convert(c) - else: - var i = 0 - try: - i = atol(value) - except e: - return NoColor() - - if i < 16: - var c = ANSIColor(i) - return self.convert(c) - elif i < 256: - var c = ANSI256Color(i) - return self.convert(c) - - return NoColor() + if value < 16: + return self.convert(ANSIColor(value)) + elif value < 256: + return self.convert(ANSI256Color(value)) + + return self.convert(RGBColor(value)) diff --git a/external/mist/renderers.mojo b/external/mist/renderers.mojo index 58eae90..15c2b02 100644 --- a/external/mist/renderers.mojo +++ b/external/mist/renderers.mojo @@ -1,116 +1,116 @@ -from .style import TerminalStyle +from .style import Style, new_style from .profile import Profile -alias RED = "#E88388" -alias GREEN = "#A8CC8C" -alias YELLOW = "#DBAB79" -alias BLUE = "#71BEF2" -alias MAGENTA = "#D290E4" -alias CYAN = "#66C2CD" -alias GRAY = "#B9BFCA" +alias RED = 0xE88388 +alias GREEN = 0xA8CC8C +alias YELLOW = 0xDBAB79 +alias BLUE = 0x71BEF2 +alias MAGENTA = 0xD290E4 +alias CYAN = 0x66C2CD +alias GRAY = 0xB9BFCA # Convenience functions for quick style application -fn as_color(text: String, color: String) -> String: +fn render_as_color(text: String, color: UInt32) -> String: var profile = Profile() - return TerminalStyle.new().foreground(profile.color(color)).render(text) + return new_style(profile.value).foreground(color=profile.color(color)).render(text) fn red(text: String) -> String: """Apply red color to the text.""" - return as_color(text, RED) + return render_as_color(text, RED) fn green(text: String) -> String: """Apply green color to the text.""" - return as_color(text, GREEN) + return render_as_color(text, GREEN) fn yellow(text: String) -> String: """Apply yellow color to the text.""" - return as_color(text, YELLOW) + return render_as_color(text, YELLOW) fn blue(text: String) -> String: """Apply blue color to the text.""" - return as_color(text, BLUE) + return render_as_color(text, BLUE) fn magenta(text: String) -> String: """Apply magenta color to the text.""" - return as_color(text, MAGENTA) + return render_as_color(text, MAGENTA) fn cyan(text: String) -> String: """Apply cyan color to the text.""" - return as_color(text, CYAN) + return render_as_color(text, CYAN) fn gray(text: String) -> String: """Apply gray color to the text.""" - return as_color(text, GRAY) + return render_as_color(text, GRAY) -fn with_background_color(text: String, color: String) -> String: +fn render_with_background_color(text: String, color: UInt32) -> String: var profile = Profile() - return TerminalStyle.new().background(profile.color(color)).render(text) + return new_style().background(color=profile.color(color)).render(text) fn red_background(text: String) -> String: """Apply red background color to the text.""" - return with_background_color(text, RED) + return render_with_background_color(text, RED) fn green_background(text: String) -> String: """Apply green background color to the text.""" - return with_background_color(text, GREEN) + return render_with_background_color(text, GREEN) fn yellow_background(text: String) -> String: """Apply yellow background color to the text.""" - return with_background_color(text, YELLOW) + return render_with_background_color(text, YELLOW) fn blue_background(text: String) -> String: """Apply blue background color to the text.""" - return with_background_color(text, BLUE) + return render_with_background_color(text, BLUE) fn magenta_background(text: String) -> String: """Apply magenta background color to the text.""" - return with_background_color(text, MAGENTA) + return render_with_background_color(text, MAGENTA) fn cyan_background(text: String) -> String: """Apply cyan background color to the text.""" - return with_background_color(text, CYAN) + return render_with_background_color(text, CYAN) fn gray_background(text: String) -> String: """Apply gray background color to the text.""" - return with_background_color(text, GRAY) + return render_with_background_color(text, GRAY) fn bold(text: String) -> String: - return TerminalStyle.new().bold().render(text) + return new_style().bold().render(text) fn faint(text: String) -> String: - return TerminalStyle.new().faint().render(text) + return new_style().faint().render(text) fn italic(text: String) -> String: - return TerminalStyle.new().italic().render(text) + return new_style().italic().render(text) fn underline(text: String) -> String: - return TerminalStyle.new().underline().render(text) + return new_style().underline().render(text) fn overline(text: String) -> String: - return TerminalStyle.new().overline().render(text) + return new_style().overline().render(text) fn crossout(text: String) -> String: - return TerminalStyle.new().crossout().render(text) + return new_style().crossout().render(text) diff --git a/external/mist/screen.mojo b/external/mist/screen.mojo index 3208475..f10dc67 100644 --- a/external/mist/screen.mojo +++ b/external/mist/screen.mojo @@ -1,3 +1,4 @@ +from external.gojo.fmt import sprintf from .style import bel, csi, reset, osc from .color import AnyColor, NoColor, ANSIColor, ANSI256Color, RGBColor @@ -65,58 +66,6 @@ alias show_cursor_seq = "?25h" alias hide_cursor_seq = "?25l" -fn __string__mul__(input_string: String, n: UInt16) -> String: - var result: String = "" - for _ in range(n): - result += input_string - return result - - -fn replace(input_string: String, old: String, new: String, count: Int = -1) -> String: - if count == 0: - return input_string - - var output: String = "" - var start = 0 - var split_count = 0 - - for end in range(len(input_string) - len(old) + 1): - if input_string[end : end + len(old)] == old: - output += input_string[start:end] + new - start = end + len(old) - split_count += 1 - - if count >= 0 and split_count >= count and count >= 0: - break - - output += input_string[start:] - return output - - -fn sprintf(text: String, *strs: String) -> String: - var output: String = text - for i in range(len(strs)): - output = replace(output, "%s", strs[i], 1) - - return output - - -fn sprintf(text: String, *ints: UInt16) -> String: - var output: String = text - for i in range(len(ints)): - output = replace(output, "%d", String(ints[i]), 1) - - return output - - -fn sprintf(text: String, *floats: Float64) -> String: - var output: String = text - for i in range(len(floats)): - output = replace(output, "%d", String(floats[i]), 1) - - return output - - fn reset_terminal(): """Reset the terminal to its default style, removing any active styles.""" print(csi + reset + "m", end="") @@ -131,11 +80,11 @@ fn set_foreground_color(color: AnyColor): var c: String = "" if color.isa[ANSIColor](): - c = color.get[ANSIColor]()[].sequence(False) + c = color[ANSIColor].sequence(False) elif color.isa[ANSI256Color](): - c = color.get[ANSI256Color]()[].sequence(False) + c = color[ANSI256Color].sequence(False) elif color.isa[RGBColor](): - c = color.get[RGBColor]()[].sequence(False) + c = color[RGBColor].sequence(False) print(osc + set_foreground_color_seq, c, end="") @@ -150,11 +99,11 @@ fn set_background_color(color: AnyColor): if color.isa[NoColor](): pass elif color.isa[ANSIColor](): - c = color.get[ANSIColor]()[].sequence(True) + c = color[ANSIColor].sequence(True) elif color.isa[ANSI256Color](): - c = color.get[ANSI256Color]()[].sequence(True) + c = color[ANSI256Color].sequence(True) elif color.isa[RGBColor](): - c = color.get[RGBColor]()[].sequence(True) + c = color[RGBColor].sequence(True) print(osc + set_background_color_seq, c, end="") @@ -169,11 +118,11 @@ fn set_cursor_color(color: AnyColor): if color.isa[NoColor](): pass elif color.isa[ANSIColor](): - c = color.get[ANSIColor]()[].sequence(True) + c = color[ANSIColor].sequence(True) elif color.isa[ANSI256Color](): - c = color.get[ANSI256Color]()[].sequence(True) + c = color[ANSI256Color].sequence(True) elif color.isa[RGBColor](): - c = color.get[RGBColor]()[].sequence(True) + c = color[RGBColor].sequence(True) print(osc + set_cursor_color_seq, c, end="") @@ -189,8 +138,7 @@ fn save_screen(): fn alt_screen(): - """Switches to the alternate screen buffer. The former view can be restored with ExitAltScreen(). - """ + """Switches to the alternate screen buffer. The former view can be restored with ExitAltScreen().""" print(csi + alt_screen_seq, end="") @@ -205,7 +153,7 @@ fn clear_screen(): move_cursor(1, 1) -fn move_cursor(row: UInt16, column: UInt16): +fn move_cursor(row: UInt16, column: Int): """Moves the cursor to a given position. Args: @@ -216,8 +164,7 @@ fn move_cursor(row: UInt16, column: UInt16): fn hide_cursor(): - """TODO: Show and Hide cursor don't seem to work ATM. HideCursor hides the cursor. - """ + """TODO: Show and Hide cursor don't seem to work ATM. HideCursor hides the cursor.""" print(csi + hide_cursor_seq, end="") @@ -236,7 +183,7 @@ fn restore_cursor_position(): print(csi + restore_cursor_position_seq, end="") -fn cursor_up(n: UInt16): +fn cursor_up(n: Int): """Moves the cursor up a given number of lines. Args: @@ -245,7 +192,7 @@ fn cursor_up(n: UInt16): print(sprintf(csi + cursor_up_seq, n), end="") -fn cursor_down(n: UInt16): +fn cursor_down(n: Int): """Moves the cursor down a given number of lines. Args: @@ -254,7 +201,7 @@ fn cursor_down(n: UInt16): print(sprintf(csi + cursor_down_seq, n), end="") -fn cursor_forward(n: UInt16): +fn cursor_forward(n: Int): """Moves the cursor up a given number of lines. Args: @@ -263,7 +210,7 @@ fn cursor_forward(n: UInt16): print(sprintf(csi + cursor_forward_seq, n), end="") -fn cursor_back(n: UInt16): +fn cursor_back(n: Int): """Moves the cursor backwards a given number of cells. Args: @@ -272,7 +219,7 @@ fn cursor_back(n: UInt16): print(sprintf(csi + cursor_back_seq, n), end="") -fn cursor_next_line(n: UInt16): +fn cursor_next_line(n: Int): """Moves the cursor down a given number of lines and places it at the beginning of the line. Args: @@ -281,7 +228,7 @@ fn cursor_next_line(n: UInt16): print(sprintf(csi + cursor_next_line_seq, n), end="") -fn cursor_prev_line(n: UInt16): +fn cursor_prev_line(n: Int): """Moves the cursor up a given number of lines and places it at the beginning of the line. Args: @@ -305,7 +252,7 @@ fn clear_line_right(): print(csi + erase_line_right_seq, end="") -fn clear_lines(n: UInt16): +fn clear_lines(n: Int): """Clears a given number of lines. Args: @@ -313,7 +260,7 @@ fn clear_lines(n: UInt16): """ var clear_line = sprintf(csi + erase_line_seq, UInt16(2)) var cursor_up = sprintf(csi + cursor_up_seq, UInt16(1)) - var movement = __string__mul__(cursor_up + clear_line, n) + var movement = (cursor_up + clear_line) * n print(clear_line + movement, end="") @@ -327,7 +274,7 @@ fn change_scrolling_region(top: UInt16, bottom: UInt16): print(sprintf(csi + change_scrolling_region_seq, top, bottom), end="") -fn insert_lines(n: UInt16): +fn insert_lines(n: Int): """Inserts the given number of lines at the top of the scrollable region, pushing lines below down. @@ -337,7 +284,7 @@ fn insert_lines(n: UInt16): print(sprintf(csi + insert_line_seq, n), end="") -fn delete_lines(n: UInt16): +fn delete_lines(n: Int): """Deletes the given number of lines, pulling any lines in the scrollable region below up. diff --git a/external/mist/style.mojo b/external/mist/style.mojo index c1fc4a0..5eb06a3 100644 --- a/external/mist/style.mojo +++ b/external/mist/style.mojo @@ -1,3 +1,4 @@ +from external.gojo.strings import StringBuilder from .color import ( Color, NoColor, @@ -10,7 +11,6 @@ from .color import ( ansi256_to_ansi, ) from .profile import get_color_profile, ASCII -import time # Text formatting sequences alias reset = "0" @@ -35,26 +35,26 @@ alias clear = escape + "[2J" + escape + "[H" @value -struct TerminalStyle: - """TerminalStyle stores a list of styles to format text with. These styles are ANSI sequences which modify text (and control the terminal). +struct Style: + """Style stores a list of styles to format text with. These styles are ANSI sequences which modify text (and control the terminal). In reality, these styles are turning visual terminal features on and off around the text it's styling. This struct should be considered immutable and each style added returns a new instance of itself rather than modifying the struct in place. - It's recommended to use the `new` static method to create a new instance of TerminalStyle so that you can chain style methods together. + It's recommended to use `new_style()` function to create a new instance of Style so that you can chain style methods together. Example: - ``` - from mist import TerminalStyle + ```mojo + import mist - var style = TerminalStyle.new().foreground("#E88388").render("red") - print(style.render("Hello World")) - ``` + var style = mist.new_style().foreground(0xE88388) + print(style.render("Hello World")) + ``` """ var styles: List[String] var profile: Profile fn __init__(inout self, profile: Profile, *, styles: List[String] = List[String]()): - """Constructs a TerminalStyle. Use new instead of __init__ to chain function calls. + """Constructs a Style. Use new_style() instead of __init__ to chain function calls. Args: profile: The color profile to use for color conversion. @@ -64,7 +64,7 @@ struct TerminalStyle: self.profile = profile fn __init__(inout self, *, styles: List[String] = List[String]()): - """Constructs a TerminalStyle. Use new instead of __init__ to chain function calls. + """Constructs a Style. Use new_style() instead of __init__ to chain function calls. Args: styles: A list of ANSI styles to apply to the text. @@ -72,38 +72,15 @@ struct TerminalStyle: self.styles = styles self.profile = Profile() - @staticmethod - fn new(profile: Profile, *, styles: List[String] = List[String]()) -> Self: - """Constructs a TerminalStyle. Use new instead of __init__ to chain function calls. - - Args: - profile: The color profile to use for color conversion. - styles: A list of ANSI styles to apply to the text. - """ - return Self(profile, styles=styles) - - @staticmethod - fn new(styles: List[String] = List[String]()) -> Self: - """Constructs a TerminalStyle. Use new instead of __init__ to chain function calls. - - Args: - styles: A list of ANSI styles to apply to the text. - """ - return Self(styles=styles) - - fn copy(self) -> Self: - """Creates a deepcopy of Self and returns that. Immutability instead of mutating the object.""" - return Self(self.profile, styles=self.get_styles()) - fn _add_style(self, style: String) -> Self: """Creates a deepcopy of Self, adds a style to it's list of styles, and returns that. Immutability instead of mutating the object. Args: style: The ANSI style to add to the list of styles. """ - var new_styles = self.get_styles() - new_styles.append(style) - return Self(self.profile, styles=new_styles) + var new = self + new.styles.append(style) + return new fn get_styles(self) -> List[String]: """Return a deepcopy of the styles list.""" @@ -141,97 +118,72 @@ struct TerminalStyle: """Makes the text overlined when rendered.""" return self._add_style(overline) - fn background(self, color: AnyColor) -> Self: + fn background(self, *, color: AnyColor) -> Self: """Set the background color of the text when it's rendered. Args: color: The color value to set the background to. This can be a hex value, an ANSI color, or an RGB color. Returns: - A new TerminalStyle with the background color set. + A new Style with the background color set. """ if color.isa[NoColor](): return Self(self.profile, styles=self.styles) var sequence: String = "" if color.isa[ANSIColor](): - var c = color.get[ANSIColor]()[] + var c = color[ANSIColor] sequence = c.sequence(True) elif color.isa[ANSI256Color](): - var c = color.get[ANSI256Color]()[] + var c = color[ANSI256Color] sequence = c.sequence(True) elif color.isa[RGBColor](): - var c = color.get[RGBColor]()[] + var c = color[RGBColor] sequence = c.sequence(True) return self._add_style(sequence) - fn background(self, color_value: String) -> Self: - """Shorthand for using the style profile to set the background color of the text. - - Args: - color_value: The color value to set the background to. This can be a hex value, an ANSI color, or an RGB color. - - Returns: - A new TerminalStyle with the background color set. - """ - return self.background(self.profile.color(color_value)) - - fn background(self, color_value: StringLiteral) -> Self: + fn background(self, color_value: UInt32) -> Self: """Shorthand for using the style profile to set the background color of the text. Args: color_value: The color value to set the background to. This can be a hex value, an ANSI color, or an RGB color. Returns: - A new TerminalStyle with the background color set. + A new Style with the background color set. """ - return self.background(self.profile.color(color_value)) + return self.background(color=self.profile.color(color_value)) - fn foreground(self, color: AnyColor) -> Self: + fn foreground(self, *, color: AnyColor) -> Self: """Set the foreground color of the text. Args: color: The color value to set the foreground to. This can be a hex value, an ANSI color, or an RGB color. Returns: - A new TerminalStyle with the foreground color set. + A new Style with the foreground color set. """ if color.isa[NoColor](): return Self(self.profile, styles=self.styles) var sequence: String = "" if color.isa[ANSIColor](): - var c = color.get[ANSIColor]()[] - sequence = c.sequence(False) + sequence = color[ANSIColor].sequence(False) elif color.isa[ANSI256Color](): - var c = color.get[ANSI256Color]()[] - sequence = c.sequence(False) + sequence = color[ANSI256Color].sequence(False) elif color.isa[RGBColor](): - var c = color.get[RGBColor]()[] - sequence = c.sequence(False) + sequence = color[RGBColor].sequence(False) return self._add_style(sequence) - fn foreground(self, color_value: String) -> Self: + fn foreground(self, color_value: UInt32) -> Self: """Shorthand for using the style profile to set the foreground color of the text. Args: color_value: The color value to set the foreground to. This can be a hex value, an ANSI color, or an RGB color. Returns: - A new TerminalStyle with the foreground color set. + A new Style with the foreground color set. """ - return self.foreground(self.profile.color(color_value)) - - fn foreground(self, color_value: StringLiteral) -> Self: - """Shorthand for using the style profile to set the foreground color of the text. - - Args: - color_value: The color value to set the foreground to. This can be a hex value, an ANSI color, or an RGB color. - - Returns: - A new TerminalStyle with the foreground color set. - """ - return self.foreground(self.profile.color(color_value)) + return self.foreground(color=self.profile.color(color_value)) fn render(self, text: String) -> String: """Renders text with the styles applied to it. @@ -242,14 +194,35 @@ struct TerminalStyle: Returns: The text with the styles applied. """ - var start = time.now() if self.profile.value == ASCII: return text if len(self.styles) == 0: return text - var seq: String = "" + var builder = StringBuilder() + _ = builder.write_string(csi) for i in range(len(self.styles)): - seq = seq + ";" + self.styles[i] - return csi + seq + "m" + text + csi + reset + "m" + _ = builder.write_string(";") + _ = builder.write_string(self.styles[i]) + _ = builder.write_string("m") + _ = builder.write_string(text) + _ = builder.write_string(csi) + _ = builder.write_string(reset) + _ = builder.write_string("m") + + return builder.render() + + +fn new_style(profile: Optional[Int] = None) -> Style: + """Creates a new Style with no styles applied. + + Args: + profile: The color profile to use for color conversion. + + Returns: + A new Style with the given color profile. + """ + if profile: + return Style(profile.value()[]) + return Style() diff --git a/external/morrow/_libc.mojo b/external/morrow/_libc.mojo index f9af241..6b3a384 100644 --- a/external/morrow/_libc.mojo +++ b/external/morrow/_libc.mojo @@ -57,38 +57,30 @@ struct CTm: @always_inline fn c_gettimeofday() -> CTimeval: var tv = CTimeval() - var p_tv = Pointer[CTimeval].address_of(tv) - external_call["gettimeofday", NoneType, Pointer[CTimeval], Int32](p_tv, 0) + var p_tv = UnsafePointer[CTimeval].address_of(tv) + external_call["gettimeofday", NoneType, UnsafePointer[CTimeval], Int32](p_tv, 0) return tv @always_inline fn c_localtime(owned tv_sec: Int) -> CTm: - var p_tv_sec = Pointer[Int].address_of(tv_sec) - var tm = external_call["localtime", Pointer[CTm], Pointer[Int]](p_tv_sec).load() + var p_tv_sec = UnsafePointer[Int].address_of(tv_sec) + var tm = external_call["localtime", UnsafePointer[CTm], UnsafePointer[Int]](p_tv_sec).take_pointee() return tm @always_inline fn c_strptime(time_str: String, time_format: String) -> CTm: var tm = CTm() - var p_tm = Pointer[CTm].address_of(tm) - external_call["strptime", NoneType, Pointer[c_char], Pointer[c_char], Pointer[CTm]]( - to_char_ptr(time_str), to_char_ptr(time_format), p_tm + var p_tm = UnsafePointer[CTm].address_of(tm) + external_call["strptime", NoneType, UnsafePointer[c_char], UnsafePointer[c_char], UnsafePointer[CTm]]( + time_str.unsafe_ptr(), time_format.unsafe_ptr(), p_tm ) return tm @always_inline fn c_gmtime(owned tv_sec: Int) -> CTm: - var p_tv_sec = Pointer[Int].address_of(tv_sec) - var tm = external_call["gmtime", Pointer[CTm], Pointer[Int]](p_tv_sec).load() + var p_tv_sec = UnsafePointer[Int].address_of(tv_sec) + var tm = external_call["gmtime", UnsafePointer[CTm], UnsafePointer[Int]](p_tv_sec).take_pointee() return tm - - -fn to_char_ptr(s: String) -> Pointer[c_char]: - """Only ASCII-based strings.""" - var ptr = Pointer[c_char]().alloc(len(s)) - for i in range(len(s)): - ptr.store(i, ord(s[i])) - return ptr diff --git a/external/morrow/constants.mojo b/external/morrow/constants.mojo index f1b806f..3b9df2a 100644 --- a/external/morrow/constants.mojo +++ b/external/morrow/constants.mojo @@ -1,12 +1,12 @@ +from utils.static_tuple import StaticTuple + # todo: hardcode for tmp alias _MAX_TIMESTAMP: Int = 32503737600 alias MAX_TIMESTAMP = _MAX_TIMESTAMP alias MAX_TIMESTAMP_MS = MAX_TIMESTAMP * 1000 alias MAX_TIMESTAMP_US = MAX_TIMESTAMP * 1_000_000 -alias _DAYS_IN_MONTH = VariadicList[Int]( - -1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 -) +alias _DAYS_IN_MONTH = VariadicList[Int](-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) alias _DAYS_BEFORE_MONTH = VariadicList[Int]( -1, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 ) # -1 is a placeholder for indexing purposes. @@ -54,6 +54,4 @@ alias DAY_NAMES = StaticTuple[StringLiteral, 8]( "Saturday", "Sunday", ) -alias DAY_ABBREVIATIONS = StaticTuple[StringLiteral, 8]( - "", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" -) +alias DAY_ABBREVIATIONS = StaticTuple[StringLiteral, 8]("", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun") diff --git a/external/morrow/formatter.mojo b/external/morrow/formatter.mojo index a7541e5..5b385fc 100644 --- a/external/morrow/formatter.mojo +++ b/external/morrow/formatter.mojo @@ -73,30 +73,31 @@ struct _Formatter: if c == match_chr_ord: match_count += 1 else: - ret += self.replace_token(m, match_chr_ord, match_count) + ret += self.replace_token(m, str(match_chr_ord), match_count) match_chr_ord = c match_count = 1 if match_count == self._sub_chrs[c]: - ret += self.replace_token(m, match_chr_ord, match_count) + ret += self.replace_token(m, str(match_chr_ord), match_count) match_chr_ord = 0 else: if match_chr_ord > 0: - ret += self.replace_token(m, match_chr_ord, match_count) + ret += self.replace_token(m, str(match_chr_ord), match_count) match_chr_ord = 0 ret += s[i] if match_chr_ord > 0: - ret += self.replace_token(m, match_chr_ord, match_count) + ret += self.replace_token(m, str(match_chr_ord), match_count) return ret fn replace_token(self, m: Morrow, token: String, token_count: Int) raises -> String: - if token == _Y: + var token_bytes = ord(token) + if token_bytes == _Y: if token_count == 1: return "Y" if token_count == 2: return rjust(m.year, 4, "0")[2:4] if token_count == 4: return rjust(m.year, 4, "0") - elif token == _M: + elif token_bytes == _M: if token_count == 1: return String(m.month) if token_count == 2: @@ -105,17 +106,17 @@ struct _Formatter: return String(MONTH_ABBREVIATIONS[m.month]) if token_count == 4: return String(MONTH_NAMES[m.month]) - elif token == _D: + elif token_bytes == _D: if token_count == 1: return String(m.day) if token_count == 2: return rjust(m.day, 2, "0") - elif token == _H: + elif token_bytes == _H: if token_count == 1: return String(m.hour) if token_count == 2: return rjust(m.hour, 2, "0") - elif token == _h: + elif token_bytes == _h: var h_12 = m.hour if m.hour > 12: h_12 -= 12 @@ -123,17 +124,17 @@ struct _Formatter: return String(h_12) if token_count == 2: return rjust(h_12, 2, "0") - elif token == _m: + elif token_bytes == _m: if token_count == 1: return String(m.minute) if token_count == 2: return rjust(m.minute, 2, "0") - elif token == _s: + elif token_bytes == _s: if token_count == 1: return String(m.second) if token_count == 2: return rjust(m.second, 2, "0") - elif token == _S: + elif token_bytes == _S: if token_count == 1: return String(m.microsecond // 100000) if token_count == 2: @@ -146,14 +147,14 @@ struct _Formatter: return rjust(m.microsecond // 10, 5, "0") if token_count == 6: return rjust(m.microsecond, 6, "0") - elif token == _d: + elif token_bytes == _d: if token_count == 1: return String(m.isoweekday()) if token_count == 3: return String(DAY_ABBREVIATIONS[m.isoweekday()]) if token_count == 4: return String(DAY_NAMES[m.isoweekday()]) - elif token == _Z: + elif token_bytes == _Z: if token_count == 3: return UTC_TZ.name if m.tz.is_none() else m.tz.name var separator = "" if token_count == 1 else ":" @@ -161,10 +162,9 @@ struct _Formatter: return UTC_TZ.format(separator) else: return m.tz.format(separator) - - elif token == _a: + elif token_bytes == _a: return "am" if m.hour < 12 else "pm" - elif token == _A: + elif token_bytes == _A: return "AM" if m.hour < 12 else "PM" return "" diff --git a/external/morrow/morrow.mojo b/external/morrow/morrow.mojo index 223611a..ad9a84d 100644 --- a/external/morrow/morrow.mojo +++ b/external/morrow/morrow.mojo @@ -74,7 +74,7 @@ struct Morrow(StringableRaising): int(tm.tm_hour), int(tm.tm_min), int(tm.tm_sec), - t.tv_usec, + int(t.tv_usec), tz, ) return result @@ -82,13 +82,13 @@ struct Morrow(StringableRaising): @staticmethod fn fromtimestamp(timestamp: Float64) raises -> Self: var timestamp_ = normalize_timestamp(timestamp) - var t = CTimeval(int(timestamp)) + var t = CTimeval(int(timestamp_)) return Self._fromtimestamp(t, False) @staticmethod fn utcfromtimestamp(timestamp: Float64) raises -> Self: var timestamp_ = normalize_timestamp(timestamp) - var t = CTimeval(int(timestamp)) + var t = CTimeval(int(timestamp_)) return Self._fromtimestamp(t, True) @staticmethod diff --git a/external/morrow/timedelta.mojo b/external/morrow/timedelta.mojo index 0ddaa2c..5e96ccc 100644 --- a/external/morrow/timedelta.mojo +++ b/external/morrow/timedelta.mojo @@ -1,4 +1,3 @@ -from math import abs from .util import rjust alias SECONDS_OF_DAY = 24 * 3600 @@ -74,9 +73,7 @@ struct TimeDelta(Stringable): fn total_seconds(self) -> Float64: """Total seconds in the duration.""" - return ( - (self.days * 86400 + self.seconds) * 10**6 + self.microseconds - ) / 10**6 + return ((self.days * 86400 + self.seconds) * 10**6 + self.microseconds) / 10**6 @always_inline fn __add__(self, other: Self) -> Self: @@ -134,11 +131,7 @@ struct TimeDelta(Stringable): return Self(0, 0, r) fn __eq__(self, other: Self) -> Bool: - return ( - self.days == other.days - and self.seconds == other.seconds - and self.microseconds == other.microseconds - ) + return self.days == other.days and self.seconds == other.seconds and self.microseconds == other.microseconds @always_inline fn __le__(self, other: Self) -> Bool: @@ -147,10 +140,7 @@ struct TimeDelta(Stringable): elif self.days == other.days: if self.seconds < other.seconds: return True - elif ( - self.seconds == other.seconds - and self.microseconds <= other.microseconds - ): + elif self.seconds == other.seconds and self.microseconds <= other.microseconds: return True return False @@ -161,9 +151,7 @@ struct TimeDelta(Stringable): elif self.days == other.days: if self.seconds < other.seconds: return True - elif ( - self.seconds == other.seconds and self.microseconds < other.microseconds - ): + elif self.seconds == other.seconds and self.microseconds < other.microseconds: return True return False diff --git a/external/morrow/timezone.mojo b/external/morrow/timezone.mojo index 4da9916..bc65012 100644 --- a/external/morrow/timezone.mojo +++ b/external/morrow/timezone.mojo @@ -26,7 +26,7 @@ struct TimeZone(Stringable): @staticmethod fn local() -> TimeZone: var local_t = c_localtime(0) - return TimeZone(local_t.tm_gmtoff.to_int(), "local") + return TimeZone(int(local_t.tm_gmtoff), "local") @staticmethod fn from_utc(utc_str: String) raises -> TimeZone: @@ -40,11 +40,7 @@ struct TimeZone(Stringable): if utc_str[p] == "+" or utc_str[p] == "-": p += 1 - if ( - len(utc_str) < p + 2 - or not isdigit(ord(utc_str[p])) - or not isdigit(ord(utc_str[p + 1])) - ): + if len(utc_str) < p + 2 or not isdigit(ord(utc_str[p])) or not isdigit(ord(utc_str[p + 1])): raise Error("utc_str format is invalid") var hours: Int = atol(utc_str[p : p + 2]) p += 2 diff --git a/external/morrow/util.mojo b/external/morrow/util.mojo index e6a5718..2b57146 100644 --- a/external/morrow/util.mojo +++ b/external/morrow/util.mojo @@ -1,5 +1,3 @@ -from collections.vector import DynamicVector - from .constants import MAX_TIMESTAMP, MAX_TIMESTAMP_MS, MAX_TIMESTAMP_US from .constants import _DAYS_IN_MONTH, _DAYS_BEFORE_MONTH @@ -44,9 +42,7 @@ def normalize_timestamp(timestamp: Float64) -> Float64: elif timestamp < MAX_TIMESTAMP_US: timestamp /= 1_000_000 else: - raise Error( - "The specified timestamp " + String(timestamp) + "is too large." - ) + raise Error("The specified timestamp " + String(timestamp) + "is too large.") return timestamp