diff --git a/.github/labeler.yml b/.github/labeler.yml index 2c36a92c..378a6351 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -4,6 +4,10 @@ - changed-files: - any-glob-to-any-file: 'pkgs/args/**' +"package:convert": + - changed-files: + - any-glob-to-any-file: 'pkgs/convert/**' + "package:crypto": - changed-files: - any-glob-to-any-file: 'pkgs/crypto/**' diff --git a/.github/workflows/convert.yaml b/.github/workflows/convert.yaml new file mode 100644 index 00000000..3422c179 --- /dev/null +++ b/.github/workflows/convert.yaml @@ -0,0 +1,76 @@ +name: package:convert + +on: + # Run CI on pushes to the main branch, and on PRs against main. + push: + branches: [ main ] + paths: + - '.github/workflows/convert.yaml' + - 'pkgs/convert/**' + pull_request: + branches: [ main ] + paths: + - '.github/workflows/convert.yaml' + - 'pkgs/convert/**' + schedule: + - cron: "0 0 * * 0" +env: + PUB_ENVIRONMENT: bot.github + +defaults: + run: + working-directory: pkgs/convert/ + +jobs: + # Check code formatting and static analysis on a single OS (linux) + # against Dart dev. + analyze: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + sdk: [dev] + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 + - uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672 + with: + sdk: ${{ matrix.sdk }} + - id: install + name: Install dependencies + run: dart pub get + - name: Check formatting + run: dart format --output=none --set-exit-if-changed . + if: always() && steps.install.outcome == 'success' + - name: Analyze code + run: dart analyze --fatal-infos + if: always() && steps.install.outcome == 'success' + + # Run tests on a matrix consisting of two dimensions: + # 1. OS: ubuntu-latest, (macos-latest, windows-latest) + # 2. release sdk: dev + test: + needs: analyze + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # Add macos-latest and/or windows-latest if relevant for this package. + os: [ubuntu-latest] + sdk: [3.4, dev] + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 + - uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672 + with: + sdk: ${{ matrix.sdk }} + - id: install + name: Install dependencies + run: dart pub get + - name: Run VM tests + run: dart test --platform vm + if: always() && steps.install.outcome == 'success' + - name: Run Chrome tests + run: dart test --platform chrome + if: always() && steps.install.outcome == 'success' + - name: Run Chrome tests - wasm + run: dart test --platform chrome --compiler dart2wasm + if: always() && steps.install.outcome == 'success' diff --git a/README.md b/README.md index 4ff174b0..b067178e 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ This repository is home to various Dart packages under the [dart.dev](https://pu | Package | Description | Version | |---|---|---| | [args](pkgs/args/) | Library for defining parsers for parsing raw command-line arguments into a set of options and values. | [![pub package](https://img.shields.io/pub/v/args.svg)](https://pub.dev/packages/args) | +| [convert](pkgs/convert/) | Utilities for converting between data representations. | [![pub package](https://img.shields.io/pub/v/convert.svg)](https://pub.dev/packages/convert) | | [crypto](pkgs/crypto/) | Implementations of SHA, MD5, and HMAC cryptographic functions. | [![pub package](https://img.shields.io/pub/v/crypto.svg)](https://pub.dev/packages/crypto) | | [fixnum](pkgs/fixnum/) | Library for 32- and 64-bit signed fixed-width integers. | [![pub package](https://img.shields.io/pub/v/fixnum.svg)](https://pub.dev/packages/fixnum) | diff --git a/pkgs/convert/.gitignore b/pkgs/convert/.gitignore new file mode 100644 index 00000000..79f51c3d --- /dev/null +++ b/pkgs/convert/.gitignore @@ -0,0 +1,3 @@ +.dart_tool +.packages +pubspec.lock diff --git a/pkgs/convert/AUTHORS b/pkgs/convert/AUTHORS new file mode 100644 index 00000000..e8063a8c --- /dev/null +++ b/pkgs/convert/AUTHORS @@ -0,0 +1,6 @@ +# Below is a list of people and organizations that have contributed +# to the project. Names should be added to the list like so: +# +# Name/Organization + +Google Inc. diff --git a/pkgs/convert/CHANGELOG.md b/pkgs/convert/CHANGELOG.md new file mode 100644 index 00000000..8fc7ff5b --- /dev/null +++ b/pkgs/convert/CHANGELOG.md @@ -0,0 +1,75 @@ +## 3.1.2 + +- Require Dart 3.4 +- Add chunked decoding support (`startChunkedConversion`) for `CodePage` + encodings. +- Upper-cast the return type of the decoder from `List` to `Uint8List`. +- Move to `dart-lang/core` monorepo. + +## 3.1.1 + +- Require Dart 2.18 +- Fix a number of comment references. + +## 3.1.0 + +- Add a fixed-pattern DateTime formatter. See + [#210](https://github.com/dart-lang/intl/issues/210) in package:intl. + +## 3.0.2 + +- Fix bug in `CodePage` class. See issue + [#47](https://github.com/dart-lang/convert/issues/47). + +## 3.0.1 + +- Dependency clean-up. + +## 3.0.0 + +- Stable null safety release. +- Added `CodePage` class for single-byte `Encoding` implementations. + +## 2.1.1 + +- Fixed a DDC compilation regression for consumers using the Dart 1.x SDK that + was introduced in `2.1.0`. + +## 2.1.0 + +- Added an `IdentityCodec` which implements `Codec` for use as default + value for in functions accepting an optional `Codec` as parameter. + +## 2.0.2 + +- Set max SDK version to `<3.0.0`, and adjust other dependencies. + +## 2.0.1 + +- `PercentEncoder` no longer encodes digits. This follows the specified + behavior. + +## 2.0.0 + +**Note**: No new APIs have been added in 2.0.0. Packages that would use 2.0.0 as +a lower bound should use 1.0.0 instead—for example, `convert: ">=1.0.0 <3.0.0"`. + +- `HexDecoder`, `HexEncoder`, `PercentDecoder`, and `PercentEncoder` no longer + extend `ChunkedConverter`. + +## 1.1.1 + +- Fix all strong-mode warnings. + +## 1.1.0 + +- Add `AccumulatorSink`, `ByteAccumulatorSink`, and `StringAccumulatorSink` + classes for providing synchronous access to the output of chunked converters. + +## 1.0.1 + +- Small improvement in percent decoder efficiency. + +## 1.0.0 + +- Initial version diff --git a/pkgs/convert/LICENSE b/pkgs/convert/LICENSE new file mode 100644 index 00000000..633672ab --- /dev/null +++ b/pkgs/convert/LICENSE @@ -0,0 +1,27 @@ +Copyright 2015, the Dart project authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google LLC nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pkgs/convert/README.md b/pkgs/convert/README.md new file mode 100644 index 00000000..4efd4b0b --- /dev/null +++ b/pkgs/convert/README.md @@ -0,0 +1,9 @@ +[![Dart CI](https://github.com/dart-lang/core/actions/workflows/convert.yaml/badge.svg)](https://github.com/dart-lang/core/actions/workflows/convert.yaml) +[![pub package](https://img.shields.io/pub/v/convert.svg)](https://pub.dev/packages/convert) +[![package publisher](https://img.shields.io/pub/publisher/convert.svg)](https://pub.dev/packages/convert/publisher) + +Contains encoders and decoders for converting between different +data representations. It's the external counterpart of the +[`dart:convert`](https://api.dart.dev/dart-convert/dart-convert-library.html) +SDK library, and contains less-central APIs and APIs that need more flexible +versioning. diff --git a/pkgs/convert/analysis_options.yaml b/pkgs/convert/analysis_options.yaml new file mode 100644 index 00000000..20982383 --- /dev/null +++ b/pkgs/convert/analysis_options.yaml @@ -0,0 +1,24 @@ +# https://dart.dev/guides/language/analysis-options +include: package:dart_flutter_team_lints/analysis_options.yaml + +analyzer: + language: + strict-casts: true + strict-raw-types: true + +linter: + rules: + - avoid_private_typedef_functions + - avoid_redundant_argument_values + - avoid_unused_constructor_parameters + - avoid_void_async + - cancel_subscriptions + - literal_only_boolean_expressions + - missing_whitespace_between_adjacent_strings + - no_adjacent_strings_in_list + - no_runtimeType_toString + - package_api_docs + - prefer_const_declarations + - prefer_expression_function_bodies + - unnecessary_await_in_return + - use_string_buffers diff --git a/pkgs/convert/benchmark/fixed_datetime_formatter_benchmark.dart b/pkgs/convert/benchmark/fixed_datetime_formatter_benchmark.dart new file mode 100644 index 00000000..6f35c015 --- /dev/null +++ b/pkgs/convert/benchmark/fixed_datetime_formatter_benchmark.dart @@ -0,0 +1,23 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:benchmark_harness/benchmark_harness.dart'; +import 'package:convert/convert.dart'; + +/// Test the performance of [FixedDateTimeFormatter.decode]. +class DecodeBenchmark extends BenchmarkBase { + final fixedDateTimeFormatter = FixedDateTimeFormatter('YYYYMMDDhhmmss'); + DecodeBenchmark() : super('Parse 10k strings to DateTime'); + + @override + void run() { + for (var i = 0; i < 10000; i++) { + fixedDateTimeFormatter.decode('19960425050322'); + } + } +} + +void main() { + DecodeBenchmark().report(); +} diff --git a/pkgs/convert/example/example.dart b/pkgs/convert/example/example.dart new file mode 100644 index 00000000..50385914 --- /dev/null +++ b/pkgs/convert/example/example.dart @@ -0,0 +1,20 @@ +// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import 'package:convert/convert.dart'; + +void main(List args) { + // Creates a Codec that converts a UTF-8 strings to/from percent encoding + final fusedCodec = utf8.fuse(percent); + + final input = args.isNotEmpty ? args.first : 'ABC 123 @!('; + print(input); + final encodedMessage = fusedCodec.encode(input); + print(encodedMessage); + + final decodedMessage = fusedCodec.decode(encodedMessage); + assert(decodedMessage == input); +} diff --git a/pkgs/convert/lib/convert.dart b/pkgs/convert/lib/convert.dart new file mode 100644 index 00000000..ec5c878f --- /dev/null +++ b/pkgs/convert/lib/convert.dart @@ -0,0 +1,12 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +export 'src/accumulator_sink.dart'; +export 'src/byte_accumulator_sink.dart'; +export 'src/codepage.dart'; +export 'src/fixed_datetime_formatter.dart'; +export 'src/hex.dart'; +export 'src/identity_codec.dart'; +export 'src/percent.dart'; +export 'src/string_accumulator_sink.dart'; diff --git a/pkgs/convert/lib/src/accumulator_sink.dart b/pkgs/convert/lib/src/accumulator_sink.dart new file mode 100644 index 00000000..af55b10a --- /dev/null +++ b/pkgs/convert/lib/src/accumulator_sink.dart @@ -0,0 +1,40 @@ +// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:collection'; +import 'dart:convert'; + +/// A sink that provides access to all the [events] that have been passed to it. +/// +/// See also [ChunkedConversionSink.withCallback]. +class AccumulatorSink implements Sink { + /// An unmodifiable list of events passed to this sink so far. + List get events => UnmodifiableListView(_events); + final _events = []; + + /// Whether [close] has been called. + bool get isClosed => _isClosed; + var _isClosed = false; + + /// Removes all events from [events]. + /// + /// This can be used to avoid double-processing events. + void clear() { + _events.clear(); + } + + @override + void add(T event) { + if (_isClosed) { + throw StateError("Can't add to a closed sink."); + } + + _events.add(event); + } + + @override + void close() { + _isClosed = true; + } +} diff --git a/pkgs/convert/lib/src/byte_accumulator_sink.dart b/pkgs/convert/lib/src/byte_accumulator_sink.dart new file mode 100644 index 00000000..44cf1784 --- /dev/null +++ b/pkgs/convert/lib/src/byte_accumulator_sink.dart @@ -0,0 +1,56 @@ +// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:typed_data/typed_data.dart'; + +/// A sink that provides access to the concatenated bytes passed to it. +/// +/// See also [ByteConversionSink.withCallback]. +class ByteAccumulatorSink extends ByteConversionSinkBase { + /// The bytes accumulated so far. + /// + /// The returned [Uint8List] is viewing a shared buffer, so it should not be + /// changed and any bytes outside the view should not be accessed. + Uint8List get bytes => Uint8List.view(_buffer.buffer, 0, _buffer.length); + + final _buffer = Uint8Buffer(); + + /// Whether [close] has been called. + bool get isClosed => _isClosed; + var _isClosed = false; + + /// Removes all bytes from [bytes]. + /// + /// This can be used to avoid double-processing data. + void clear() { + _buffer.clear(); + } + + @override + void add(List chunk) { + if (_isClosed) { + throw StateError("Can't add to a closed sink."); + } + + _buffer.addAll(chunk); + } + + @override + void addSlice(List chunk, int start, int end, bool isLast) { + if (_isClosed) { + throw StateError("Can't add to a closed sink."); + } + + _buffer.addAll(chunk, start, end); + if (isLast) _isClosed = true; + } + + @override + void close() { + _isClosed = true; + } +} diff --git a/pkgs/convert/lib/src/charcodes.dart b/pkgs/convert/lib/src/charcodes.dart new file mode 100644 index 00000000..fc941484 --- /dev/null +++ b/pkgs/convert/lib/src/charcodes.dart @@ -0,0 +1,36 @@ +// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// Character `%`. +const int $percent = 0x25; + +/// Character `-`. +const int $dash = 0x2d; + +/// Character `.`. +const int $dot = 0x2e; + +/// Character `0`. +const int $0 = 0x30; + +/// Character `9`. +const int $9 = 0x39; + +/// Character `A`. +const int $A = 0x41; + +/// Character `_`. +const int $underscore = 0x5f; + +/// Character `a`. +const int $a = 0x61; + +/// Character `f`. +const int $f = 0x66; + +/// Character `z`. +const int $z = 0x7a; + +/// Character `~`. +const int $tilde = 0x7e; diff --git a/pkgs/convert/lib/src/codepage.dart b/pkgs/convert/lib/src/codepage.dart new file mode 100644 index 00000000..c298ff5a --- /dev/null +++ b/pkgs/convert/lib/src/codepage.dart @@ -0,0 +1,472 @@ +// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:typed_data'; + +/// The ISO-8859-2/Latin-2 (Eastern European) code page. +final CodePage latin2 = + CodePage._bmp('latin-2', '$_ascii$_noControls$_top8859_2'); + +/// The ISO-8859-3/Latin-3 (South European) code page. +final CodePage latin3 = + CodePage._bmp('latin-3', '$_ascii$_noControls$_top8859_3'); + +/// The ISO-8859-4/Latin-4 (North European) code page. +final CodePage latin4 = + CodePage._bmp('latin-4', '$_ascii$_noControls$_top8859_4'); + +/// The ISO-8859-5/Latin-Cyrillic code page. +final CodePage latinCyrillic = + CodePage._bmp('cyrillic', '$_ascii$_noControls$_top8859_5'); + +/// The ISO-8859-6/Latin-Arabic code page. +final CodePage latinArabic = + CodePage._bmp('arabic', '$_ascii$_noControls$_top8859_6'); + +/// The ISO-8859-7/Latin-Greek code page. +final CodePage latinGreek = + CodePage._bmp('greek', '$_ascii$_noControls$_top8859_7'); + +/// The ISO-8859-7/Latin-Hebrew code page. +final CodePage latinHebrew = + CodePage._bmp('hebrew', '$_ascii$_noControls$_top8859_8'); + +/// The ISO-8859-9/Latin-5 (Turkish) code page. +final CodePage latin5 = + CodePage._bmp('latin-5', '$_ascii$_noControls$_top8859_9'); + +/// The ISO-8859-10/Latin-6 (Nordic) code page. +final CodePage latin6 = + CodePage._bmp('latin-6', '$_ascii$_noControls$_top8859_10'); + +/// The ISO-8859-11/Latin-Thai code page. +final CodePage latinThai = + CodePage._bmp('tis620', '$_ascii$_noControls$_top8859_11'); + +/// The ISO-8859-13/Latin-6 (Baltic Rim) code page. +final CodePage latin7 = + CodePage._bmp('latin-7', '$_ascii$_noControls$_top8859_13'); + +/// The ISO-8859-14/Latin-8 (Celtic) code page. +final CodePage latin8 = + CodePage._bmp('latin-8', '$_ascii$_noControls$_top8859_14'); + +/// The ISO-8859-15/Latin-9 (Western European revised) code page. +final CodePage latin9 = + CodePage._bmp('latin-9', '$_ascii$_noControls$_top8859_15'); + +/// The ISO-8859-16/Latin-10 (South Eastern European) code page. +final CodePage latin10 = + CodePage._bmp('latin-10', '$_ascii$_noControls$_top8859_16'); + +/// Characters in ISO-8859-2 above the ASCII and top control characters. +const _top8859_2 = '\xa0Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\xadŽŻ°ą˛ł´ľśˇ¸šşťź˝žż' + 'ŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢß' + 'ŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙'; + +/// Characters in ISO-8859-3 above the ASCII and top control characters. +const _top8859_3 = '\xa0Ħ˘£\uFFFD¤Ĥ§¨İŞĞĴ\xad\uFFFDŻ°ħ²³´µĥ·¸ışğĵ½\uFFFDż' + 'ÀÁÂ\uFFFDÄĊĈÇÈÉÊËÌÍÎÏ\uFFFDÑÒÓÔĠÖ×ĜÙÚÛÜŬŜß' + 'àáâ\uFFFDäċĉçèéêëìíîï\uFFFDñòóôġö÷ĝùúûüŭŝ˙'; + +/// Characters in ISO-8859-4 above the ASCII and top control characters. +const _top8859_4 = '\xa0ĄĸŖ¤ĨĻ§¨ŠĒĢŦ\xadŽ¯°ą˛ŗ´ĩļˇ¸šēģŧŊžŋ' + 'ĀÁÂÃÄÅÆĮČÉĘËĖÍÎĪĐŅŌĶÔÕÖ×ØŲÚÛÜŨŪß' + 'āáâãäåæįčéęëėíîīđņōķôõö÷øųúûüũū˙'; + +/// Characters in ISO-8859-5 above the ASCII and top control characters. +const _top8859_5 = '\xa0ЁЂЃЄЅІЇЈЉЊЋЌ\xadЎЏАБВГДЕЖЗИЙКЛМНОП' + 'РСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмноп' + 'рстуфхцчшщъыьэюя№ёђѓєѕіїјљњћќ§ўџ'; + +/// Characters in ISO-8859-6 above the ASCII and top control characters. +const _top8859_6 = '\xa0\uFFFD\uFFFD\uFFFD¤\uFFFD\uFFFD\uFFFD' + '\uFFFD\uFFFD\uFFFD\uFFFD\u060c\xad\uFFFD\uFFFD' + '\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD' + '\uFFFD\uFFFD\uFFFD\u061b\uFFFD\uFFFD\uFFFD\u061f' + '\uFFFD\u0621\u0622\u0623\u0624\u0625\u0626\u0627' + '\u0628\u0629\u062a\u062b\u062c\u062d\u062e\u062f' + '\u0630\u0631\u0632\u0633\u0634\u0635\u0636\u0637' + '\u0638\u0639\u063a\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD' + '\u0640\u0641\u0642\u0643\u0644\u0645\u0646\u0647' + '\u0648\u0649\u064a\u064b\u064c\u064d\u064e\u064f' + '\u0650\u0651\u0652\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD' + '\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD'; + +/// Characters in ISO-8859-7 above the ASCII and top control characters. +const _top8859_7 = '\xa0‘’£€₯¦§¨©ͺ«¬\xad\uFFFD―°±²³΄΅Ά·ΈΉΊ»Ό½ΎΏ' + 'ΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡ\uFFFDΣΤΥΦΧΨΩΪΫάέήί' + 'ΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώ\uFFFD'; + +/// Characters in ISO-8859-8 above the ASCII and top control characters. +const _top8859_8 = '\xa0\uFFFD¢£¤¥¦§¨©×«¬\xad®¯°±²³´µ¶·¸¹÷»¼½¾\uFFFD' + '\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD' + '\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD' + '\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD' + '\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD‗' + '\u05d0\u05d1\u05d2\u05d3\u05d4\u05d5\u05d6\u05d7' + '\u05d8\u05d9\u05da\u05db\u05dc\u05dd\u05de\u05df' + '\u05e0\u05e1\u05e2\u05e3\u05e4\u05e5\u05e6\u05e7' + '\u05e8\u05e9\u05ea\uFFFD\uFFFD\u200e\u200f\uFFFD'; + +/// Characters in ISO-8859-9 above the ASCII and top control characters. +const _top8859_9 = '\xa0¡¢£¤¥¦§¨©ª«¬\xad®¯°±²³´µ¶·¸¹º»¼½¾¿' + 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏĞÑÒÓÔÕÖ×ØÙÚÛÜİŞß' + 'àáâãäåæçèéêëìíîïğñòóôõö÷øùúûüışÿ'; + +/// Characters in ISO-8859-10 above the ASCII and top control characters. +const _top8859_10 = '\xa0ĄĒĢĪĨĶ§ĻĐŠŦŽ\xadŪŊ°ąēģīĩķ·ļđšŧž―ūŋ' + 'ĀÁÂÃÄÅÆĮČÉĘËĖÍÎÏÐŅŌÓÔÕÖŨØŲÚÛÜÝÞß' + 'āáâãäåæįčéęëėíîïðņōóôõöũøųúûüýþĸ'; + +/// Characters in ISO-8859-11 above the ASCII and top control characters. +const _top8859_11 = '\xa0กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟ' + 'ภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู\uFFFD\uFFFD\uFFFD\uFFFD฿' + 'เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛\uFFFD\uFFFD\uFFFD\uFFFD'; + +/// Characters in ISO-8859-13 above the ASCII and top control characters. +const _top8859_13 = '\xa0”¢£¤„¦§Ø©Ŗ«¬\xad®Æ°±²³“µ¶·ø¹ŗ»¼½¾æ' + 'ĄĮĀĆÄÅĘĒČÉŹĖĢĶĪĻŠŃŅÓŌÕÖ×ŲŁŚŪÜŻŽß' + 'ąįāćäåęēčéźėģķīļšńņóōõö÷ųłśūüżž’'; + +/// Characters in ISO-8859-14 above the ASCII and top control characters. +const _top8859_14 = '\xa0Ḃḃ£ĊċḊ§Ẁ©ẂḋỲ\xad®ŸḞḟĠġṀṁ¶ṖẁṗẃṠỳẄẅṡ' + 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏŴÑÒÓÔÕÖṪØÙÚÛÜÝŶß' + 'àáâãäåæçèéêëìíîïŵñòóôõöṫøùúûüýŷÿ'; + +/// Characters in ISO-8859-15 above the ASCII and top control characters. +const _top8859_15 = '\xa0¡¢£€¥Š§š©ª«¬\xad®¯°±²³Žµ¶·ž¹º»ŒœŸ¿' + 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞß' + 'àáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ'; + +/// Characters in ISO-8859-16 above the ASCII and top control characters. +const _top8859_16 = '\xa0ĄąŁ€„Š§š©Ș«Ź\xadźŻ°±ČłŽ”¶·žčș»ŒœŸż' + 'ÀÁÂĂÄĆÆÇÈÉÊËÌÍÎÏĐŃÒÓÔŐÖŚŰÙÚÛÜĘȚß' + 'àáâăäćæçèéêëìíîïđńòóôőöśűùúûüęțÿ'; + +const _noControls = '\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD' + '\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD' + '\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD' + '\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD'; + +/// ASCII characters without control characters. Shared by many code pages. +const _ascii = '$_noControls' + // ignore: missing_whitespace_between_adjacent_strings + r""" !"#$%&'()*+,-./0123456789:;<=>?""" + r'@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_' + '`abcdefghijklmnopqrstuvwxyz{|}~\uFFFD'; + +/// A mapping between bytes and characters. +/// +/// A code page is a way to map bytes to character. +/// As such, it can only represent 256 different characters. +class CodePage extends Encoding { + @override + final CodePageDecoder decoder; + @override + final String name; + CodePageEncoder? _encoder; + + /// Creates a code page with the given name and characters. + /// + /// The [characters] string must contain 256 code points (runes) + /// in the order of the bytes representing them. + /// + /// Any byte not defined by the code page should have a + /// U+FFFD (invalid character) code point at its place in + /// [characters]. + /// + /// The name is used by [Encoding.name]. + factory CodePage(String name, String characters) = CodePage._general; + + /// Creates a code page with the characters of [characters]. + /// + /// The [characters] must contain precisely 256 characters (code points). + /// + /// A U+FFFD (invalid character) entry in [characters] means that the + /// corresponding byte does not have a definition in this code page. + CodePage._general(this.name, String characters) + : decoder = _createDecoder(characters); + + /// Creates a code page with characters from the basic multilingual plane. + /// + /// The basic multilingual plane (BMP) contains the first 65536 code points. + /// As such, each character can be represented by a single UTF-16 code unit, + /// which makes some operations more efficient. + /// + /// The [characters] must contain precisely 256 code points from the BMP + /// which means that it should have length 256 and not contain any surrogates. + /// + /// A U+FFFD (invalid character) entry in [characters] means that the + /// corresponding byte does not have a definition in this code page. + CodePage._bmp(this.name, String characters) + : decoder = _BmpCodePageDecoder(characters); + + /// The character associated with a particular byte in this code page. + /// + /// The [byte] must be in the range 0..255. + /// The returned value should be a Unicode scalar value + /// (a non-surrogate code point). + /// + /// If a code page does not have a defined character for a particular + /// byte, it should return the Unicode invalid character (U+FFFD) + /// instad. + int operator [](int byte) => decoder._char(byte); + + /// Encodes [input] using `encoder.convert`. + @override + Uint8List encode(String input, {int? invalidCharacter}) => + encoder.convert(input, invalidCharacter: invalidCharacter); + + /// Decodes [bytes] using `encoder.convert`. + @override + String decode(List bytes, {bool allowInvalid = false}) => + decoder.convert(bytes, allowInvalid: allowInvalid); + + @override + CodePageEncoder get encoder => _encoder ??= decoder._createEncoder(); +} + +/// A code page decoder, converts from bytes to characters. +/// +/// A code page assigns characters to a subset of byte values. +/// The decoder converts those bytes back to their characters. +abstract class CodePageDecoder implements Converter, String> { + /// Decodes a sequence of bytes into a string using a code page. + /// + /// The code page assigns one character to each byte. + /// Values in [input] must be bytes (integers in the range 0..255). + /// + /// If [allowInvalid] is true, non-byte values in [input], + /// or byte values not defined as a character in the code page, + /// are emitted as U+FFFD (the Unicode invalid character). + /// If not true, the bytes must be calid and defined characters. + @override + String convert(List input, {bool allowInvalid = false}); + + CodePageEncoder _createEncoder(); + int _char(int byte); +} + +/// Creates a decoder from [characters]. +/// +/// Recognizes if [characters] contains only characters in the BMP, +/// and creates a [_BmpCodePageDecoder] in that case. +CodePageDecoder _createDecoder(String characters) { + var result = Uint32List(256); + var i = 0; + var allChars = 0; + for (var char in characters.runes) { + if (i >= 256) { + throw ArgumentError.value( + characters, 'characters', 'Must contain 256 characters'); + } + result[i++] = char; + allChars |= char; + } + if (i < 256) { + throw ArgumentError.value( + characters, 'characters', 'Must contain 256 characters'); + } + if (allChars <= 0xFFFF) { + // It's in the BMP. + return _BmpCodePageDecoder(characters); + } + return _NonBmpCodePageDecoder._(result); +} + +/// An input [ByteConversionSink] for decoders where each input byte can be be +/// considered independantly. +class _CodePageDecoderSink extends ByteConversionSink { + final Sink _output; + final Converter, String> _decoder; + + _CodePageDecoderSink(this._output, this._decoder); + + @override + void add(List chunk) { + _output.add(_decoder.convert(chunk)); + } + + @override + void close() { + _output.close(); + } +} + +/// Code page with non-BMP characters. +class _NonBmpCodePageDecoder extends Converter, String> + implements CodePageDecoder { + final Uint32List _characters; + _NonBmpCodePageDecoder(String characters) : this._(_buildMapping(characters)); + _NonBmpCodePageDecoder._(this._characters); + + @override + int _char(int byte) => _characters[byte]; + + static Uint32List _buildMapping(String characters) { + var result = Uint32List(256); + var i = 0; + for (var char in characters.runes) { + if (i >= 256) { + throw ArgumentError.value( + characters, 'characters', 'Must contain 256 characters'); + } + result[i++] = char; + } + if (i < 256) { + throw ArgumentError.value( + characters, 'characters', 'Must contain 256 characters'); + } + return result; + } + + @override + CodePageEncoder _createEncoder() { + var result = {}; + for (var i = 0; i < 256; i++) { + var char = _characters[i]; + if (char != 0xFFFD) { + result[char] = i; + } + } + return CodePageEncoder._(result); + } + + @override + String convert(List input, {bool allowInvalid = false}) { + var buffer = Uint32List(input.length); + for (var i = 0; i < input.length; i++) { + var byte = input[i]; + if (byte & 0xff != byte) throw FormatException('Not a byte', input, i); + buffer[i] = _characters[byte]; + } + return String.fromCharCodes(buffer); + } + + @override + Sink> startChunkedConversion(Sink sink) => + _CodePageDecoderSink(sink, this); +} + +class _BmpCodePageDecoder extends Converter, String> + implements CodePageDecoder { + final String _characters; + _BmpCodePageDecoder(String characters) : _characters = characters { + if (characters.length != 256) { + throw ArgumentError.value(characters, 'characters', + 'Must contain 256 characters. Was ${characters.length}'); + } + } + + @override + int _char(int byte) => _characters.codeUnitAt(byte); + + @override + String convert(List bytes, {bool allowInvalid = false}) { + if (allowInvalid) return _convertAllowInvalid(bytes); + var count = bytes.length; + var codeUnits = Uint16List(count); + for (var i = 0; i < count; i++) { + var byte = bytes[i]; + if (byte != byte & 0xff) { + throw FormatException('Not a byte value', bytes, i); + } + var character = _characters.codeUnitAt(byte); + if (character == 0xFFFD) { + throw FormatException('Not defined in this code page', bytes, i); + } + codeUnits[i] = character; + } + return String.fromCharCodes(codeUnits); + } + + @override + Sink> startChunkedConversion(Sink sink) => + _CodePageDecoderSink(sink, this); + + String _convertAllowInvalid(List bytes) { + var count = bytes.length; + var codeUnits = Uint16List(count); + for (var i = 0; i < count; i++) { + var byte = bytes[i]; + int character; + if (byte == byte & 0xff) { + character = _characters.codeUnitAt(byte); + } else { + character = 0xFFFD; + } + codeUnits[i] = character; + } + return String.fromCharCodes(codeUnits); + } + + @override + CodePageEncoder _createEncoder() => CodePageEncoder._bmp(_characters); +} + +/// Encoder for a code page. +/// +/// Converts a string into bytes where each byte represents that character +/// according to the code page definition. +class CodePageEncoder extends Converter> { + final Map _encoding; + + CodePageEncoder._bmp(String characters) + : _encoding = _createBmpEncoding(characters); + + CodePageEncoder._(this._encoding); + + static Map _createBmpEncoding(String characters) { + var encoding = {}; + for (var i = 0; i < characters.length; i++) { + var char = characters.codeUnitAt(i); + if (char != 0xFFFD) encoding[characters.codeUnitAt(i)] = i; + } + return encoding; + } + + /// Converts input to the byte encoding in this code page. + /// + /// If [invalidCharacter] is supplied, it must be a byte value + /// (in the range 0..255). + /// + /// If [input] contains characters that are not available + /// in this code page, they are replaced by the [invalidCharacter] byte, + /// and then [invalidCharacter] must have been supplied. + @override + Uint8List convert(String input, {int? invalidCharacter}) { + if (invalidCharacter != null) { + RangeError.checkValueInInterval( + invalidCharacter, 0, 255, 'invalidCharacter'); + } + var count = input.length; + var result = Uint8List(count); + var j = 0; + for (var i = 0; i < count; i++) { + var char = input.codeUnitAt(i); + var byte = _encoding[char]; + nullCheck: + if (byte == null) { + // Check for surrogate. + var offset = i; + if (char & 0xFC00 == 0xD800 && i + 1 < count) { + var next = input.codeUnitAt(i + 1); + if ((next & 0xFC00) == 0xDC00) { + i = i + 1; + char = 0x10000 + ((char & 0x3ff) << 10) + (next & 0x3ff); + byte = _encoding[char]; + if (byte != null) break nullCheck; + } + } + byte = invalidCharacter ?? + (throw FormatException( + 'Not a character in this code page', input, offset)); + } + result[j++] = byte; + } + return Uint8List.sublistView(result, 0, j); + } +} diff --git a/pkgs/convert/lib/src/fixed_datetime_formatter.dart b/pkgs/convert/lib/src/fixed_datetime_formatter.dart new file mode 100644 index 00000000..fc0a58aa --- /dev/null +++ b/pkgs/convert/lib/src/fixed_datetime_formatter.dart @@ -0,0 +1,378 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// A formatter and parser for [DateTime] in a fixed format [String] pattern. +/// +/// For example, calling +/// `FixedDateTimeCodec('YYYYMMDDhhmmss').decodeToLocal('19960425050322')` has +/// the same result as calling `DateTime(1996, 4, 25, 5, 3, 22)`. +/// +/// The allowed characters are +/// * `Y` for “calendar year” +/// * `M` for “calendar month” +/// * `D` for “calendar day” +/// * `E` for “decade” +/// * `C` for “century” +/// * `h` for “clock hour” +/// * `m` for “clock minute” +/// * `s` for “clock second” +/// * `S` for “fractional clock second” +/// +/// Note: Negative years are not supported. +/// +/// Non-allowed characters in the format [pattern] are included when decoding a +/// string, in this case `YYYY kiwi MM` is the same format string as +/// `YYYY------MM`. When encoding a [DateTime], the non-format characters are in +/// the output verbatim. +/// +/// Note: this class differs from +/// [DateFormat](https://pub.dev/documentation/intl/latest/intl/DateFormat-class.html) +/// from [package:intl](https://pub.dev/packages/intl) in that here, the format +/// character count is interpreted literally. For example, using the format +/// string `YYY` to decode the string `996` would result in the same [DateTime] +/// as calling `DateTime(996)`, and the same format string used to encode the +/// `DateTime(1996)` would output only the three digits 996. +class FixedDateTimeFormatter { + static const _powersOfTen = [1, 10, 100, 1000, 10000, 100000]; + static const _validFormatCharacters = [ + _yearCode, + _monthCode, + _dayCode, + _decadeCode, + _centuryCode, + _hourCode, + _minuteCode, + _secondCode, + _fractionSecondCode, + ]; + static const _yearCode = 0x59; /*Y*/ + static const _monthCode = 0x4D; /*M*/ + static const _dayCode = 0x44; /*D*/ + static const _decadeCode = 0x45; /*E*/ + static const _centuryCode = 0x43; /*C*/ + static const _hourCode = 0x68; /*H*/ + static const _minuteCode = 0x6D; /*m*/ + static const _secondCode = 0x73; /*s*/ + static const _fractionSecondCode = 0x53; /*S*/ + + /// The format pattern string of this formatter. + final String pattern; + + /// Whether to create UTC [DateTime] objects when parsing. + /// + /// If not, the created [DateTime] objects are in the local time zone. + final bool isUtc; + + final _blocks = _ParsedFormatBlocks(); + + /// Creates a new [FixedDateTimeFormatter] with the provided [pattern]. + /// + /// The [pattern] interprets the characters mentioned in + /// [FixedDateTimeFormatter] to represent fields of a `DateTime` value. Other + /// characters are not special. If [isUtc] is set to false, the DateTime is + /// constructed with respect to the local timezone. + /// + /// There must at most be one sequence of each special character to ensure a + /// single source of truth when constructing the [DateTime], so a pattern of + /// `"CCCC-MM-DD, CC"` is invalid, because it has two separate `C` sequences. + FixedDateTimeFormatter(this.pattern, {this.isUtc = true}) { + int? currentCharacter; + var start = 0; + for (var i = 0; i < pattern.length; i++) { + var formatCharacter = pattern.codeUnitAt(i); + if (currentCharacter != formatCharacter) { + _blocks.saveBlock(currentCharacter, start, i); + if (_validFormatCharacters.contains(formatCharacter)) { + var hasSeenBefore = _blocks.formatCharacters.indexOf(formatCharacter); + if (hasSeenBefore > -1) { + throw FormatException( + "Pattern contains more than one '$formatCharacter' block.\n" + 'Previous occurrence at index ${_blocks.starts[hasSeenBefore]}', + pattern, + i); + } else { + start = i; + currentCharacter = formatCharacter; + } + } else { + currentCharacter = null; + } + } + } + _blocks.saveBlock(currentCharacter, start, pattern.length); + } + + /// Converts a [DateTime] to a [String] as specified by the [pattern]. + /// + /// The [DateTime.year] must not be negative. + String encode(DateTime dateTime) { + if (dateTime.year < 0) { + throw ArgumentError.value( + dateTime, + 'dateTime', + 'Year must not be negative', + ); + } + var buffer = StringBuffer(); + for (var i = 0; i < _blocks.length; i++) { + var start = _blocks.starts[i]; + var end = _blocks.ends[i]; + var length = end - start; + + var previousEnd = i > 0 ? _blocks.ends[i - 1] : 0; + if (previousEnd < start) { + buffer.write(pattern.substring(previousEnd, start)); + } + var formatCharacter = _blocks.formatCharacters[i]; + var number = _extractNumberFromDateTime( + formatCharacter, + dateTime, + length, + ); + if (number.length > length) { + number = number.substring(number.length - length); + } else if (length > number.length) { + number = number.padLeft(length, '0'); + } + buffer.write(number); + } + if (_blocks.length > 0) { + var lastEnd = _blocks.ends.last; + if (lastEnd < pattern.length) { + buffer.write(pattern.substring(lastEnd, pattern.length)); + } + } + return buffer.toString(); + } + + String _extractNumberFromDateTime( + int? formatCharacter, + DateTime dateTime, + int length, + ) { + int value; + switch (formatCharacter) { + case _yearCode: + value = dateTime.year; + break; + case _centuryCode: + value = dateTime.year ~/ 100; + break; + case _decadeCode: + value = dateTime.year ~/ 10; + break; + case _monthCode: + value = dateTime.month; + break; + case _dayCode: + value = dateTime.day; + break; + case _hourCode: + value = dateTime.hour; + break; + case _minuteCode: + value = dateTime.minute; + break; + case _secondCode: + value = dateTime.second; + break; + case _fractionSecondCode: + value = dateTime.millisecond; + switch (length) { + case 1: + value ~/= 100; + break; + case 2: + value ~/= 10; + break; + case 3: + break; + case 4: + value = value * 10 + dateTime.microsecond ~/ 100; + break; + case 5: + value = value * 100 + dateTime.microsecond ~/ 10; + break; + case 6: + value = value * 1000 + dateTime.microsecond; + break; + default: + throw AssertionError( + 'Unreachable, length is restricted to 6 in the constructor'); + } + break; + default: + throw AssertionError( + 'Unreachable, the key is checked in the constructor'); + } + return value.toString().padLeft(length, '0'); + } + + /// Parses [formattedDateTime] to a [DateTime] as specified by the [pattern]. + /// + /// Parts of a [DateTime] which are not mentioned in the pattern default to a + /// value of zero for time parts and year, and a value of 1 for day and month. + /// + /// Throws a [FormatException] if the [formattedDateTime] does not match the + /// [pattern]. + DateTime decode(String formattedDateTime) => + _decode(formattedDateTime, isUtc, true)!; + + /// Parses [formattedDateTime] to a [DateTime] as specified by the [pattern]. + /// + /// Parts of a [DateTime] which are not mentioned in the pattern default to a + /// value of zero for time parts and year, and a value of 1 for day and month. + /// + /// Returns the parsed value, or `null` if the [formattedDateTime] does not + /// match the [pattern]. + DateTime? tryDecode(String formattedDateTime) => + _decode(formattedDateTime, isUtc, false); + + DateTime? _decode( + String formattedDateTime, + bool isUtc, + bool throwOnError, + ) { + var year = 0; + var month = 1; + var day = 1; + var hour = 0; + var minute = 0; + var second = 0; + var microsecond = 0; + for (var i = 0; i < _blocks.length; i++) { + var formatCharacter = _blocks.formatCharacters[i]; + var number = _extractNumberFromString(formattedDateTime, i, throwOnError); + if (number != null) { + if (formatCharacter == _fractionSecondCode) { + // Special case, as we want fractional seconds to be the leading + // digits. + number *= _powersOfTen[6 - (_blocks.ends[i] - _blocks.starts[i])]; + } + switch (formatCharacter) { + case _yearCode: + year += number; + break; + case _centuryCode: + year += number * 100; + break; + case _decadeCode: + year += number * 10; + break; + case _monthCode: + month = number; + break; + case _dayCode: + day = number; + break; + case _hourCode: + hour = number; + break; + case _minuteCode: + minute = number; + break; + case _secondCode: + second = number; + break; + case _fractionSecondCode: + microsecond = number; + break; + } + } else { + return null; + } + } + if (isUtc) { + return DateTime.utc( + year, + month, + day, + hour, + minute, + second, + 0, + microsecond, + ); + } else { + return DateTime( + year, + month, + day, + hour, + minute, + second, + 0, + microsecond, + ); + } + } + + int? _extractNumberFromString( + String formattedDateTime, + int index, + bool throwOnError, + ) { + var parsed = tryParse( + formattedDateTime, + _blocks.starts[index], + _blocks.ends[index], + ); + if (parsed == null && throwOnError) { + throw FormatException( + 'Expected digits at ${formattedDateTime.substring( + _blocks.starts[index], + _blocks.ends[index], + )}', + formattedDateTime, + _blocks.starts[index], + ); + } + return parsed; + } + + int? tryParse(String formattedDateTime, int start, int end) { + var result = 0; + for (var i = start; i < end; i++) { + var digit = formattedDateTime.codeUnitAt(i) ^ 0x30; + if (digit <= 9) { + result = result * 10 + digit; + } else { + return null; + } + } + return result; + } +} + +class _ParsedFormatBlocks { + final formatCharacters = []; + final starts = []; + final ends = []; + + _ParsedFormatBlocks(); + + int get length => formatCharacters.length; + + void saveBlock(int? char, int start, int end) { + if (char != null) { + if (char == FixedDateTimeFormatter._fractionSecondCode && + end - start > 6) { + throw FormatException( + 'Fractional seconds can only be specified up to microseconds', + char, + start, + ); + } else if (end - start > 9) { + throw FormatException( + 'Length of a format char block cannot be larger than 9', + char, + start, + ); + } + formatCharacters.add(char); + starts.add(start); + ends.add(end); + } + } +} diff --git a/pkgs/convert/lib/src/hex.dart b/pkgs/convert/lib/src/hex.dart new file mode 100644 index 00000000..f3832406 --- /dev/null +++ b/pkgs/convert/lib/src/hex.dart @@ -0,0 +1,27 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import 'hex/decoder.dart'; +import 'hex/encoder.dart'; + +export 'hex/decoder.dart' hide hexDecoder; +export 'hex/encoder.dart' hide hexEncoder; + +/// The canonical instance of [HexCodec]. +const hex = HexCodec._(); + +/// A codec that converts byte arrays to and from hexadecimal strings, following +/// [the Base16 spec](https://tools.ietf.org/html/rfc4648#section-8). +/// +/// This should be used via the [hex] field. +class HexCodec extends Codec, String> { + @override + HexEncoder get encoder => hexEncoder; + @override + HexDecoder get decoder => hexDecoder; + + const HexCodec._(); +} diff --git a/pkgs/convert/lib/src/hex/decoder.dart b/pkgs/convert/lib/src/hex/decoder.dart new file mode 100644 index 00000000..3696d4d9 --- /dev/null +++ b/pkgs/convert/lib/src/hex/decoder.dart @@ -0,0 +1,181 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:typed_data'; + +import '../utils.dart'; + +/// The canonical instance of [HexDecoder]. +const hexDecoder = HexDecoder._(); + +/// A converter that decodes hexadecimal strings into byte arrays. +/// +/// Because two hexadecimal digits correspond to a single byte, this will throw +/// a [FormatException] if given an odd-length string. It will also throw a +/// [FormatException] if given a string containing non-hexadecimal code units. +class HexDecoder extends Converter> { + const HexDecoder._(); + + @override + Uint8List convert(String input) { + if (!input.length.isEven) { + throw FormatException( + 'Invalid input length, must be even.', + input, + input.length, + ); + } + + var bytes = Uint8List(input.length ~/ 2); + _decode(input.codeUnits, 0, input.length, bytes, 0); + return bytes; + } + + @override + StringConversionSink startChunkedConversion(Sink> sink) => + _HexDecoderSink(sink); +} + +/// A conversion sink for chunked hexadecimal decoding. +class _HexDecoderSink extends StringConversionSinkBase { + /// The underlying sink to which decoded byte arrays will be passed. + final Sink> _sink; + + /// The trailing digit from the previous string. + /// + /// This will be non-`null` if the most recent string had an odd number of + /// hexadecimal digits. Since it's the most significant digit, it's always a + /// multiple of 16. + int? _lastDigit; + + _HexDecoderSink(this._sink); + + @override + void addSlice(String string, int start, int end, bool isLast) { + RangeError.checkValidRange(start, end, string.length); + + if (start == end) { + if (isLast) _close(string, end); + return; + } + + var codeUnits = string.codeUnits; + Uint8List bytes; + int bytesStart; + if (_lastDigit == null) { + bytes = Uint8List((end - start) ~/ 2); + bytesStart = 0; + } else { + var hexPairs = (end - start - 1) ~/ 2; + bytes = Uint8List(1 + hexPairs); + bytes[0] = _lastDigit! + digitForCodeUnit(codeUnits, start); + start++; + bytesStart = 1; + } + + _lastDigit = _decode(codeUnits, start, end, bytes, bytesStart); + + _sink.add(bytes); + if (isLast) _close(string, end); + } + + @override + ByteConversionSink asUtf8Sink(bool allowMalformed) => + _HexDecoderByteSink(_sink); + + @override + void close() => _close(); + + /// Like [close], but includes [string] and [index] in the [FormatException] + /// if one is thrown. + void _close([String? string, int? index]) { + if (_lastDigit != null) { + throw FormatException( + 'Input ended with incomplete encoded byte.', string, index); + } + + _sink.close(); + } +} + +/// A conversion sink for chunked hexadecimal decoding from UTF-8 bytes. +class _HexDecoderByteSink extends ByteConversionSinkBase { + /// The underlying sink to which decoded byte arrays will be passed. + final Sink> _sink; + + /// The trailing digit from the previous string. + /// + /// This will be non-`null` if the most recent string had an odd number of + /// hexadecimal digits. Since it's the most significant digit, it's always a + /// multiple of 16. + int? _lastDigit; + + _HexDecoderByteSink(this._sink); + + @override + void add(List chunk) => addSlice(chunk, 0, chunk.length, false); + + @override + void addSlice(List chunk, int start, int end, bool isLast) { + RangeError.checkValidRange(start, end, chunk.length); + + if (start == end) { + if (isLast) _close(chunk, end); + return; + } + + Uint8List bytes; + int bytesStart; + if (_lastDigit == null) { + bytes = Uint8List((end - start) ~/ 2); + bytesStart = 0; + } else { + var hexPairs = (end - start - 1) ~/ 2; + bytes = Uint8List(1 + hexPairs); + bytes[0] = _lastDigit! + digitForCodeUnit(chunk, start); + start++; + bytesStart = 1; + } + + _lastDigit = _decode(chunk, start, end, bytes, bytesStart); + + _sink.add(bytes); + if (isLast) _close(chunk, end); + } + + @override + void close() => _close(); + + /// Like [close], but includes [chunk] and [index] in the [FormatException] + /// if one is thrown. + void _close([List? chunk, int? index]) { + if (_lastDigit != null) { + throw FormatException( + 'Input ended with incomplete encoded byte.', chunk, index); + } + + _sink.close(); + } +} + +/// Decodes [codeUnits] and writes the result into [destination]. +/// +/// This reads from [codeUnits] between [sourceStart] and [sourceEnd]. It writes +/// the result into [destination] starting at [destinationStart]. +/// +/// If there's a leftover digit at the end of the decoding, this returns that +/// digit. Otherwise it returns `null`. +int? _decode(List codeUnits, int sourceStart, int sourceEnd, + List destination, int destinationStart) { + var destinationIndex = destinationStart; + for (var i = sourceStart; i < sourceEnd - 1; i += 2) { + var firstDigit = digitForCodeUnit(codeUnits, i); + var secondDigit = digitForCodeUnit(codeUnits, i + 1); + destination[destinationIndex++] = 16 * firstDigit + secondDigit; + } + + if ((sourceEnd - sourceStart).isEven) return null; + return 16 * digitForCodeUnit(codeUnits, sourceEnd - 1); +} diff --git a/pkgs/convert/lib/src/hex/encoder.dart b/pkgs/convert/lib/src/hex/encoder.dart new file mode 100644 index 00000000..36d6c226 --- /dev/null +++ b/pkgs/convert/lib/src/hex/encoder.dart @@ -0,0 +1,92 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:typed_data'; + +import '../charcodes.dart'; + +/// The canonical instance of [HexEncoder]. +const hexEncoder = HexEncoder._(); + +/// A converter that encodes byte arrays into hexadecimal strings. +/// +/// This will throw a [RangeError] if the byte array has any digits that don't +/// fit in the gamut of a byte. +class HexEncoder extends Converter, String> { + const HexEncoder._(); + + @override + String convert(List input) => _convert(input, 0, input.length); + + @override + ByteConversionSink startChunkedConversion(Sink sink) => + _HexEncoderSink(sink); +} + +/// A conversion sink for chunked hexadecimal encoding. +class _HexEncoderSink extends ByteConversionSinkBase { + /// The underlying sink to which decoded byte arrays will be passed. + final Sink _sink; + + _HexEncoderSink(this._sink); + + @override + void add(List chunk) { + _sink.add(_convert(chunk, 0, chunk.length)); + } + + @override + void addSlice(List chunk, int start, int end, bool isLast) { + RangeError.checkValidRange(start, end, chunk.length); + _sink.add(_convert(chunk, start, end)); + if (isLast) _sink.close(); + } + + @override + void close() { + _sink.close(); + } +} + +String _convert(List bytes, int start, int end) { + // A Uint8List is more efficient than a StringBuffer given that we know that + // we're only emitting ASCII-compatible characters, and that we know the + // length ahead of time. + var buffer = Uint8List((end - start) * 2); + var bufferIndex = 0; + + // A bitwise OR of all bytes in [bytes]. This allows us to check for + // out-of-range bytes without adding more branches than necessary to the + // core loop. + var byteOr = 0; + for (var i = start; i < end; i++) { + var byte = bytes[i]; + byteOr |= byte; + + // The bitwise arithmetic here is equivalent to `byte ~/ 16` and `byte % 16` + // for valid byte values, but is easier for dart2js to optimize given that + // it can't prove that [byte] will always be positive. + buffer[bufferIndex++] = _codeUnitForDigit((byte & 0xF0) >> 4); + buffer[bufferIndex++] = _codeUnitForDigit(byte & 0x0F); + } + + if (byteOr >= 0 && byteOr <= 255) return String.fromCharCodes(buffer); + + // If there was an invalid byte, find it and throw an exception. + for (var i = start; i < end; i++) { + var byte = bytes[i]; + if (byte >= 0 && byte <= 0xff) continue; + throw FormatException( + "Invalid byte ${byte < 0 ? "-" : ""}0x${byte.abs().toRadixString(16)}.", + bytes, + i); + } + + throw StateError('unreachable'); +} + +/// Returns the ASCII/Unicode code unit corresponding to the hexadecimal digit +/// [digit]. +int _codeUnitForDigit(int digit) => digit < 10 ? digit + $0 : digit + $a - 10; diff --git a/pkgs/convert/lib/src/identity_codec.dart b/pkgs/convert/lib/src/identity_codec.dart new file mode 100644 index 00000000..0a2a0b52 --- /dev/null +++ b/pkgs/convert/lib/src/identity_codec.dart @@ -0,0 +1,36 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +class _IdentityConverter extends Converter { + _IdentityConverter(); + @override + T convert(T input) => input; +} + +/// A [Codec] that performs the identity conversion (changing nothing) in both +/// directions. +/// +/// The identity codec passes input directly to output in both directions. +/// This class can be used as a base when combining multiple codecs, +/// because fusing the identity codec with any other codec gives the other +/// codec back. +/// +/// Note, that when fused with another [Codec] the identity codec disppears. +class IdentityCodec extends Codec { + const IdentityCodec(); + + @override + Converter get decoder => _IdentityConverter(); + @override + Converter get encoder => _IdentityConverter(); + + /// Fuse with an other codec. + /// + /// Fusing with the identify converter is a no-op, so this always return + /// [other]. + @override + Codec fuse(Codec other) => other; +} diff --git a/pkgs/convert/lib/src/percent.dart b/pkgs/convert/lib/src/percent.dart new file mode 100644 index 00000000..ecf28670 --- /dev/null +++ b/pkgs/convert/lib/src/percent.dart @@ -0,0 +1,37 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import 'percent/decoder.dart'; +import 'percent/encoder.dart'; + +export 'percent/decoder.dart' hide percentDecoder; +export 'percent/encoder.dart' hide percentEncoder; + +/// The canonical instance of [PercentCodec]. +const percent = PercentCodec._(); + +// TODO(nweiz): Add flags to support generating and interpreting "+" as a space +// character. Also add an option for custom sets of unreserved characters. +/// A codec that converts byte arrays to and from percent-encoded (also known as +/// URL-encoded) strings according to +/// [RFC 3986](https://tools.ietf.org/html/rfc3986#section-2.1). +/// +/// [encoder] encodes all bytes other than ASCII letters, decimal digits, or one +/// of `-._~`. This matches the behavior of [Uri.encodeQueryComponent] except +/// that it doesn't encode `0x20` bytes to the `+` character. +/// +/// To be maximally flexible, [decoder] will decode any percent-encoded byte and +/// will allow any non-percent-encoded byte other than `%`. By default, it +/// interprets `+` as `0x2B` rather than `0x20` as emitted by +/// [Uri.encodeQueryComponent]. +class PercentCodec extends Codec, String> { + @override + PercentEncoder get encoder => percentEncoder; + @override + PercentDecoder get decoder => percentDecoder; + + const PercentCodec._(); +} diff --git a/pkgs/convert/lib/src/percent/decoder.dart b/pkgs/convert/lib/src/percent/decoder.dart new file mode 100644 index 00000000..ef9c8a8d --- /dev/null +++ b/pkgs/convert/lib/src/percent/decoder.dart @@ -0,0 +1,250 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import 'package:typed_data/typed_data.dart'; + +import '../charcodes.dart'; +import '../utils.dart'; + +/// The canonical instance of [PercentDecoder]. +const percentDecoder = PercentDecoder._(); + +const _lastPercent = -1; + +/// A converter that decodes percent-encoded strings into byte arrays. +/// +/// To be maximally flexible, this will decode any percent-encoded byte and +/// will allow any non-percent-encoded byte other than `%`. By default, it +/// interprets `+` as `0x2B` rather than `0x20` as emitted by +/// [Uri.encodeQueryComponent]. +/// +/// This will throw a [FormatException] if the input string has an incomplete +/// percent-encoding, or if it contains non-ASCII code units. +class PercentDecoder extends Converter> { + const PercentDecoder._(); + + @override + List convert(String input) { + var buffer = Uint8Buffer(); + var lastDigit = _decode(input.codeUnits, 0, input.length, buffer); + + if (lastDigit != null) { + throw FormatException( + 'Input ended with incomplete encoded byte.', input, input.length); + } + + return buffer.buffer.asUint8List(0, buffer.length); + } + + @override + StringConversionSink startChunkedConversion(Sink> sink) => + _PercentDecoderSink(sink); +} + +/// A conversion sink for chunked percent-encoded decoding. +class _PercentDecoderSink extends StringConversionSinkBase { + /// The underlying sink to which decoded byte arrays will be passed. + final Sink> _sink; + + /// The trailing digit from the previous string. + /// + /// This is `null` if the previous string ended with a complete + /// percent-encoded byte or a literal character. It's [_lastPercent] if the + /// most recent string ended with `%`. Otherwise, the most recent string ended + /// with a `%` followed by a hexadecimal digit, and this is that digit. Since + /// it's the most significant digit, it's always a multiple of 16. + int? _lastDigit; + + _PercentDecoderSink(this._sink); + + @override + void addSlice(String string, int start, int end, bool isLast) { + RangeError.checkValidRange(start, end, string.length); + + if (start == end) { + if (isLast) _close(string, end); + return; + } + + var buffer = Uint8Buffer(); + var codeUnits = string.codeUnits; + if (_lastDigit == _lastPercent) { + _lastDigit = 16 * digitForCodeUnit(codeUnits, start); + start++; + + if (start == end) { + if (isLast) _close(string, end); + return; + } + } + + if (_lastDigit != null) { + buffer.add(_lastDigit! + digitForCodeUnit(codeUnits, start)); + start++; + } + + _lastDigit = _decode(codeUnits, start, end, buffer); + + _sink.add(buffer.buffer.asUint8List(0, buffer.length)); + if (isLast) _close(string, end); + } + + @override + ByteConversionSink asUtf8Sink(bool allowMalformed) => + _PercentDecoderByteSink(_sink); + + @override + void close() => _close(); + + /// Like [close], but includes [string] and [index] in the [FormatException] + /// if one is thrown. + void _close([String? string, int? index]) { + if (_lastDigit != null) { + throw FormatException( + 'Input ended with incomplete encoded byte.', string, index); + } + + _sink.close(); + } +} + +/// A conversion sink for chunked percent-encoded decoding from UTF-8 bytes. +class _PercentDecoderByteSink extends ByteConversionSinkBase { + /// The underlying sink to which decoded byte arrays will be passed. + final Sink> _sink; + + /// The trailing digit from the previous string. + /// + /// This is `null` if the previous string ended with a complete + /// percent-encoded byte or a literal character. It's [_lastPercent] if the + /// most recent string ended with `%`. Otherwise, the most recent string ended + /// with a `%` followed by a hexadecimal digit, and this is that digit. Since + /// it's the most significant digit, it's always a multiple of 16. + int? _lastDigit; + + _PercentDecoderByteSink(this._sink); + + @override + void add(List chunk) => addSlice(chunk, 0, chunk.length, false); + + @override + void addSlice(List chunk, int start, int end, bool isLast) { + RangeError.checkValidRange(start, end, chunk.length); + + if (start == end) { + if (isLast) _close(chunk, end); + return; + } + + var buffer = Uint8Buffer(); + if (_lastDigit == _lastPercent) { + _lastDigit = 16 * digitForCodeUnit(chunk, start); + start++; + + if (start == end) { + if (isLast) _close(chunk, end); + return; + } + } + + if (_lastDigit != null) { + buffer.add(_lastDigit! + digitForCodeUnit(chunk, start)); + start++; + } + + _lastDigit = _decode(chunk, start, end, buffer); + + _sink.add(buffer.buffer.asUint8List(0, buffer.length)); + if (isLast) _close(chunk, end); + } + + @override + void close() => _close(); + + /// Like [close], but includes [chunk] and [index] in the [FormatException] + /// if one is thrown. + void _close([List? chunk, int? index]) { + if (_lastDigit != null) { + throw FormatException( + 'Input ended with incomplete encoded byte.', chunk, index); + } + + _sink.close(); + } +} + +/// Decodes [codeUnits] and writes the result into [buffer]. +/// +/// This reads from [codeUnits] between [start] and [end]. It writes +/// the result into [buffer] starting at [end]. +/// +/// If there's a leftover digit at the end of the decoding, this returns that +/// digit. Otherwise it returns `null`. +int? _decode(List codeUnits, int start, int end, Uint8Buffer buffer) { + // A bitwise OR of all code units in [codeUnits]. This allows us to check for + // out-of-range code units without adding more branches than necessary to the + // core loop. + var codeUnitOr = 0; + + // The beginning of the current slice of adjacent non-% characters. We can add + // all of these to the buffer at once. + var sliceStart = start; + for (var i = start; i < end; i++) { + // First, loop through non-% characters. + var codeUnit = codeUnits[i]; + if (codeUnits[i] != $percent) { + codeUnitOr |= codeUnit; + continue; + } + + // We found a %. The slice from `sliceStart` to `i` represents characters + // than can be copied to the buffer as-is. + if (i > sliceStart) { + _checkForInvalidCodeUnit(codeUnitOr, codeUnits, sliceStart, i); + buffer.addAll(codeUnits, sliceStart, i); + } + + // Now decode the percent-encoded byte and add it as well. + i++; + if (i >= end) return _lastPercent; + + var firstDigit = digitForCodeUnit(codeUnits, i); + i++; + if (i >= end) return 16 * firstDigit; + + var secondDigit = digitForCodeUnit(codeUnits, i); + buffer.add(16 * firstDigit + secondDigit); + + // The next iteration will look for non-% characters again. + sliceStart = i + 1; + } + + if (end > sliceStart) { + _checkForInvalidCodeUnit(codeUnitOr, codeUnits, sliceStart, end); + if (start == sliceStart) { + buffer.addAll(codeUnits); + } else { + buffer.addAll(codeUnits, sliceStart, end); + } + } + + return null; +} + +void _checkForInvalidCodeUnit( + int codeUnitOr, List codeUnits, int start, int end) { + if (codeUnitOr >= 0 && codeUnitOr <= 0x7f) return; + + for (var i = start; i < end; i++) { + var codeUnit = codeUnits[i]; + if (codeUnit >= 0 && codeUnit <= 0x7f) continue; + throw FormatException( + 'Non-ASCII code unit ' + "U+${codeUnit.toRadixString(16).padLeft(4, '0')}", + codeUnits, + i); + } +} diff --git a/pkgs/convert/lib/src/percent/encoder.dart b/pkgs/convert/lib/src/percent/encoder.dart new file mode 100644 index 00000000..b087b7a8 --- /dev/null +++ b/pkgs/convert/lib/src/percent/encoder.dart @@ -0,0 +1,109 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import '../charcodes.dart'; + +/// The canonical instance of [PercentEncoder]. +const percentEncoder = PercentEncoder._(); + +/// A converter that encodes byte arrays into percent-encoded strings. +/// +/// Encodes all bytes other than ASCII letters, decimal digits, or one +/// of `-._~`. This matches the behavior of [Uri.encodeQueryComponent] except +/// that it doesn't encode `0x20` bytes to the `+` character. +/// +/// This will throw a [RangeError] if the byte array has any digits that don't +/// fit in the gamut of a byte. +class PercentEncoder extends Converter, String> { + const PercentEncoder._(); + + @override + String convert(List input) => _convert(input, 0, input.length); + + @override + ByteConversionSink startChunkedConversion(Sink sink) => + _PercentEncoderSink(sink); +} + +/// A conversion sink for chunked percentadecimal encoding. +class _PercentEncoderSink extends ByteConversionSinkBase { + /// The underlying sink to which decoded byte arrays will be passed. + final Sink _sink; + + _PercentEncoderSink(this._sink); + + @override + void add(List chunk) { + _sink.add(_convert(chunk, 0, chunk.length)); + } + + @override + void addSlice(List chunk, int start, int end, bool isLast) { + RangeError.checkValidRange(start, end, chunk.length); + _sink.add(_convert(chunk, start, end)); + if (isLast) _sink.close(); + } + + @override + void close() { + _sink.close(); + } +} + +String _convert(List bytes, int start, int end) { + var buffer = StringBuffer(); + + // A bitwise OR of all bytes in [bytes]. This allows us to check for + // out-of-range bytes without adding more branches than necessary to the + // core loop. + var byteOr = 0; + for (var i = start; i < end; i++) { + var byte = bytes[i]; + byteOr |= byte; + + // If the byte is an uppercase letter, convert it to lowercase to check if + // it's unreserved. This works because uppercase letters in ASCII are + // exactly `0b100000 = 0x20` less than lowercase letters, so if we ensure + // that that bit is 1 we ensure that the letter is lowercase. + var letter = 0x20 | byte; + if ((letter >= $a && letter <= $z) || + (byte >= $0 && byte <= $9) || + byte == $dash || + byte == $dot || + byte == $underscore || + byte == $tilde) { + // Unreserved characters are safe to write as-is. + buffer.writeCharCode(byte); + continue; + } + + buffer.writeCharCode($percent); + + // The bitwise arithmetic here is equivalent to `byte ~/ 16` and `byte % 16` + // for valid byte values, but is easier for dart2js to optimize given that + // it can't prove that [byte] will always be positive. + buffer.writeCharCode(_codeUnitForDigit((byte & 0xF0) >> 4)); + buffer.writeCharCode(_codeUnitForDigit(byte & 0x0F)); + } + + if (byteOr >= 0 && byteOr <= 255) return buffer.toString(); + + // If there was an invalid byte, find it and throw an exception. + for (var i = start; i < end; i++) { + var byte = bytes[i]; + if (byte >= 0 && byte <= 0xff) continue; + throw FormatException( + "Invalid byte ${byte < 0 ? "-" : ""}0x${byte.abs().toRadixString(16)}.", + bytes, + i); + } + + throw StateError('unreachable'); +} + +/// Returns the ASCII/Unicode code unit corresponding to the hexadecimal digit +/// [digit]. +int _codeUnitForDigit(int digit) => digit < 10 ? digit + $0 : digit + $A - 10; diff --git a/pkgs/convert/lib/src/string_accumulator_sink.dart b/pkgs/convert/lib/src/string_accumulator_sink.dart new file mode 100644 index 00000000..9b07593f --- /dev/null +++ b/pkgs/convert/lib/src/string_accumulator_sink.dart @@ -0,0 +1,49 @@ +// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +/// A sink that provides access to the concatenated strings passed to it. +/// +/// See also [StringConversionSink.withCallback]. +class StringAccumulatorSink extends StringConversionSinkBase { + /// The string accumulated so far. + String get string => _buffer.toString(); + final _buffer = StringBuffer(); + + /// Whether [close] has been called. + bool get isClosed => _isClosed; + var _isClosed = false; + + /// Empties [string]. + /// + /// This can be used to avoid double-processing data. + void clear() { + _buffer.clear(); + } + + @override + void add(String str) { + if (_isClosed) { + throw StateError("Can't add to a closed sink."); + } + + _buffer.write(str); + } + + @override + void addSlice(String chunk, int start, int end, bool isLast) { + if (_isClosed) { + throw StateError("Can't add to a closed sink."); + } + + _buffer.write(chunk.substring(start, end)); + if (isLast) _isClosed = true; + } + + @override + void close() { + _isClosed = true; + } +} diff --git a/pkgs/convert/lib/src/utils.dart b/pkgs/convert/lib/src/utils.dart new file mode 100644 index 00000000..cfcc1273 --- /dev/null +++ b/pkgs/convert/lib/src/utils.dart @@ -0,0 +1,37 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'charcodes.dart'; + +/// Returns the digit (0 through 15) corresponding to the hexadecimal code unit +/// at index [index] in [codeUnits]. +/// +/// If the given code unit isn't valid hexadecimal, throws a [FormatException]. +int digitForCodeUnit(List codeUnits, int index) { + // If the code unit is a numeral, get its value. XOR works because 0 in ASCII + // is `0b110000` and the other numerals come after it in ascending order and + // take up at most four bits. + // + // We check for digits first because it ensures there's only a single branch + // for 10 out of 16 of the expected cases. We don't count the `digit >= 0` + // check because branch prediction will always work on it for valid data. + var codeUnit = codeUnits[index]; + var digit = $0 ^ codeUnit; + if (digit <= 9) { + if (digit >= 0) return digit; + } else { + // If the code unit is an uppercase letter, convert it to lowercase. This + // works because uppercase letters in ASCII are exactly `0b100000 = 0x20` + // less than lowercase letters, so if we ensure that that bit is 1 we ensure + // that the letter is lowercase. + var letter = 0x20 | codeUnit; + if ($a <= letter && letter <= $f) return letter - $a + 10; + } + + throw FormatException( + 'Invalid hexadecimal code unit ' + "U+${codeUnit.toRadixString(16).padLeft(4, '0')}.", + codeUnits, + index); +} diff --git a/pkgs/convert/pubspec.yaml b/pkgs/convert/pubspec.yaml new file mode 100644 index 00000000..97685ff3 --- /dev/null +++ b/pkgs/convert/pubspec.yaml @@ -0,0 +1,17 @@ +name: convert +version: 3.1.2 +description: >- + Utilities for converting between data representations. + Provides a number of Sink, Codec, Decoder, and Encoder types. +repository: https://github.com/dart-lang/core/tree/main/pkgs/convert + +environment: + sdk: ^3.4.0 + +dependencies: + typed_data: ^1.3.0 + +dev_dependencies: + benchmark_harness: ^2.2.0 + dart_flutter_team_lints: ^3.0.0 + test: ^1.17.0 diff --git a/pkgs/convert/test/accumulator_sink_test.dart b/pkgs/convert/test/accumulator_sink_test.dart new file mode 100644 index 00000000..6842d1c9 --- /dev/null +++ b/pkgs/convert/test/accumulator_sink_test.dart @@ -0,0 +1,54 @@ +// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:convert/convert.dart'; +import 'package:test/test.dart'; + +void main() { + late AccumulatorSink sink; + setUp(() { + sink = AccumulatorSink(); + }); + + test("provides access to events as they're added", () { + expect(sink.events, isEmpty); + + sink.add(1); + expect(sink.events, equals([1])); + + sink.add(2); + expect(sink.events, equals([1, 2])); + + sink.add(3); + expect(sink.events, equals([1, 2, 3])); + }); + + test('clear() clears the events', () { + sink + ..add(1) + ..add(2) + ..add(3); + expect(sink.events, equals([1, 2, 3])); + + sink.clear(); + expect(sink.events, isEmpty); + + sink + ..add(4) + ..add(5) + ..add(6); + expect(sink.events, equals([4, 5, 6])); + }); + + test('indicates whether the sink is closed', () { + expect(sink.isClosed, isFalse); + sink.close(); + expect(sink.isClosed, isTrue); + }); + + test("doesn't allow add() to be called after close()", () { + sink.close(); + expect(() => sink.add(1), throwsStateError); + }); +} diff --git a/pkgs/convert/test/byte_accumulator_sink_test.dart b/pkgs/convert/test/byte_accumulator_sink_test.dart new file mode 100644 index 00000000..8398ae24 --- /dev/null +++ b/pkgs/convert/test/byte_accumulator_sink_test.dart @@ -0,0 +1,56 @@ +// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:convert/convert.dart'; +import 'package:test/test.dart'; + +void main() { + late ByteAccumulatorSink sink; + setUp(() { + sink = ByteAccumulatorSink(); + }); + + test('provides access to the concatenated bytes', () { + expect(sink.bytes, isEmpty); + + sink.add([1, 2, 3]); + expect(sink.bytes, equals([1, 2, 3])); + + sink.addSlice([4, 5, 6, 7, 8], 1, 4, false); + expect(sink.bytes, equals([1, 2, 3, 5, 6, 7])); + }); + + test('clear() clears the bytes', () { + sink.add([1, 2, 3]); + expect(sink.bytes, equals([1, 2, 3])); + + sink.clear(); + expect(sink.bytes, isEmpty); + + sink.add([4, 5, 6]); + expect(sink.bytes, equals([4, 5, 6])); + }); + + test('indicates whether the sink is closed', () { + expect(sink.isClosed, isFalse); + sink.close(); + expect(sink.isClosed, isTrue); + }); + + test('indicates whether the sink is closed via addSlice', () { + expect(sink.isClosed, isFalse); + sink.addSlice([], 0, 0, true); + expect(sink.isClosed, isTrue); + }); + + test("doesn't allow add() to be called after close()", () { + sink.close(); + expect(() => sink.add([1]), throwsStateError); + }); + + test("doesn't allow addSlice() to be called after close()", () { + sink.close(); + expect(() => sink.addSlice([], 0, 0, false), throwsStateError); + }); +} diff --git a/pkgs/convert/test/codepage_test.dart b/pkgs/convert/test/codepage_test.dart new file mode 100644 index 00000000..cca75e7b --- /dev/null +++ b/pkgs/convert/test/codepage_test.dart @@ -0,0 +1,154 @@ +// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:core'; +import 'dart:typed_data'; + +import 'package:convert/convert.dart'; +import 'package:test/test.dart'; + +void main() { + var bytes = Uint8List.fromList([for (var i = 0; i < 256; i++) i]); + for (var cp in [ + latin2, + latin3, + latin4, + latin5, + latin6, + latin7, + latin8, + latin9, + latin10, + latinCyrillic, + latinGreek, + latinHebrew, + latinThai, + latinArabic + ]) { + group('${cp.name} codepage', () { + test('ascii compatible', () { + for (var byte = 0x20; byte < 0x7f; byte++) { + expect(cp[byte], byte); + } + }); + + test('bidirectional mapping', () { + // Maps both directions. + for (var byte = 0; byte < 256; byte++) { + var char = cp[byte]; + if (char != 0xFFFD) { + var string = String.fromCharCode(char); + expect(cp.encode(string), [byte]); + expect(cp.decode([byte]), string); + } + } + }); + + test('decode invalid characters not allowed', () { + expect(() => cp.decode([0xfffd]), throwsA(isA())); + }); + + test('decode invalid characters allowed', () { + // Decode works like operator[]. + expect(cp.decode(bytes, allowInvalid: true), + String.fromCharCodes([for (var i = 0; i < 256; i++) cp[i]])); + }); + + test('chunked conversion', () { + late final String decodedString; + final outputSink = StringConversionSink.withCallback( + (accumulated) => decodedString = accumulated); + final inputSink = cp.decoder.startChunkedConversion(outputSink); + final expected = StringBuffer(); + + for (var byte = 0; byte < 256; byte++) { + var char = cp[byte]; + if (char != 0xFFFD) { + inputSink.add([byte]); + expected.writeCharCode(char); + } + } + inputSink.close(); + expect(decodedString, expected.toString()); + }); + }); + } + test('latin-2 roundtrip', () { + // Data from http://www.columbia.edu/kermit/latin2.html + var latin2text = '\xa0Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\xadŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇ' + 'ÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙'; + expect(latin2.decode(latin2.encode(latin2text)), latin2text); + }); + + test('latin-3 roundtrip', () { + // Data from http://www.columbia.edu/kermit/latin3.html + var latin2text = '\xa0Ħ˘£¤\u{FFFD}Ĥ§¨İŞĞĴ\xad\u{FFFD}Ż°ħ²³´µĥ·¸ışğĵ½' + '\u{FFFD}żÀÁÂ\u{FFFD}ÄĊĈÇÈÉÊËÌÍÎÏ\u{FFFD}ÑÒÓÔĠÖ×ĜÙÚÛÜŬŜßàáâ' + '\u{FFFD}äċĉçèéêëìíîï\u{FFFD}ñòóôġö÷ĝùúûüŭŝ˙'; + var encoded = latin3.encode(latin2text, invalidCharacter: 0); + var decoded = latin3.decode(encoded, allowInvalid: true); + expect(decoded, latin2text); + }); + + group('Custom code page', () { + late final cp = CodePage('custom', "ABCDEF${"\uFFFD" * 250}"); + + test('simple encode', () { + var result = cp.encode('BADCAFE'); + expect(result, [1, 0, 3, 2, 0, 5, 4]); + }); + + test('unencodable character', () { + expect(() => cp.encode('GAD'), throwsFormatException); + }); + + test('unencodable character with invalidCharacter', () { + expect(cp.encode('GAD', invalidCharacter: 0x3F), [0x3F, 0, 3]); + }); + + test('simple decode', () { + expect(cp.decode([1, 0, 3, 2, 0, 5, 4]), 'BADCAFE'); + }); + + test('undecodable byte', () { + expect(() => cp.decode([6, 1, 255]), throwsFormatException); + }); + + test('undecodable byte with allowInvalid', () { + expect(cp.decode([6, 1, 255], allowInvalid: true), '\u{FFFD}B\u{FFFD}'); + }); + + test('chunked conversion', () { + late final String decodedString; + final outputSink = StringConversionSink.withCallback( + (accumulated) => decodedString = accumulated); + final inputSink = cp.decoder.startChunkedConversion(outputSink); + + inputSink + ..add([1]) + ..add([0]) + ..add([3]) + ..close(); + expect(decodedString, 'BAD'); + }); + + test('chunked conversion - byte conversion sink', () { + late final String decodedString; + final outputSink = StringConversionSink.withCallback( + (accumulated) => decodedString = accumulated); + final bytes = [1, 0, 3, 2, 0, 5, 4]; + + final inputSink = cp.decoder.startChunkedConversion(outputSink); + expect(inputSink, isA()); + + (inputSink as ByteConversionSink) + ..addSlice(bytes, 1, 3, false) + ..addSlice(bytes, 4, 5, false) + ..addSlice(bytes, 6, 6, true); + + expect(decodedString, 'ADA'); + }); + }); +} diff --git a/pkgs/convert/test/fixed_datetime_formatter_test.dart b/pkgs/convert/test/fixed_datetime_formatter_test.dart new file mode 100644 index 00000000..30e7ca91 --- /dev/null +++ b/pkgs/convert/test/fixed_datetime_formatter_test.dart @@ -0,0 +1,224 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:convert/src/fixed_datetime_formatter.dart'; +import 'package:test/test.dart'; + +void main() { + var noFractionalSeconds = DateTime.utc(0); + var skipWeb = { + 'js': const Skip( + 'Web does not support microseconds (see https://github.com/dart-lang/sdk/issues/44876)') + }; + // Testing `decode`. + test('Parse only year', () { + var time = FixedDateTimeFormatter('YYYY').decode('1996'); + expect(time, DateTime.utc(1996)); + }); + test('Escaped chars are ignored', () { + var time = FixedDateTimeFormatter('YYYY kiwi MM').decode('1996 rnad 01'); + expect(time, DateTime.utc(1996)); + }); + test('Parse two years throws', () { + expect(() => FixedDateTimeFormatter('YYYY YYYY'), throwsException); + }); + test('Parse year and century', () { + var time = FixedDateTimeFormatter('CCYY').decode('1996'); + expect(time, DateTime.utc(1996)); + }); + test('Parse year, decade and century', () { + var time = FixedDateTimeFormatter('CCEY').decode('1996'); + expect(time, DateTime.utc(1996)); + }); + test('Parse year, century, month', () { + var time = FixedDateTimeFormatter('CCYY MM').decode('1996 04'); + expect(time, DateTime.utc(1996, 4)); + }); + test('Parse year, century, month, day', () { + var time = FixedDateTimeFormatter('CCYY MM-DD').decode('1996 04-25'); + expect(time, DateTime.utc(1996, 4, 25)); + }); + test('Parse year, century, month, day, hour, minute, second', () { + var time = FixedDateTimeFormatter('CCYY MM-DD hh:mm:ss') + .decode('1996 04-25 05:03:22'); + expect(time, DateTime.utc(1996, 4, 25, 5, 3, 22)); + }); + test('Parse YYYYMMDDhhmmssSSS', () { + var time = + FixedDateTimeFormatter('YYYYMMDDhhmmssSSS').decode('19960425050322533'); + expect(time, DateTime.utc(1996, 4, 25, 5, 3, 22, 533)); + }); + test('Parse S 1/10 of a second', () { + var time = FixedDateTimeFormatter('S').decode('1'); + expect(time, noFractionalSeconds.add(const Duration(milliseconds: 100))); + }); + test('Parse SS 1/100 of a second', () { + var time = FixedDateTimeFormatter('SS').decode('01'); + expect(time, noFractionalSeconds.add(const Duration(milliseconds: 10))); + }); + test('Parse SSS a millisecond', () { + var time = FixedDateTimeFormatter('SSS').decode('001'); + expect(time, noFractionalSeconds.add(const Duration(milliseconds: 1))); + }); + test('Parse SSSSSS a microsecond', () { + var time = FixedDateTimeFormatter('SSSSSS').decode('000001'); + expect(time, noFractionalSeconds.add(const Duration(microseconds: 1))); + }, onPlatform: skipWeb); + test('Parse SSSSSS a millisecond', () { + var time = FixedDateTimeFormatter('SSSSSS').decode('001000'); + expect(time, noFractionalSeconds.add(const Duration(milliseconds: 1))); + }); + test('Parse SSSSSS a millisecond and a microsecond', () { + var time = FixedDateTimeFormatter('SSSSSS').decode('001001'); + expect( + time, + noFractionalSeconds.add(const Duration( + milliseconds: 1, + microseconds: 1, + ))); + }, onPlatform: skipWeb); + test('Parse ssSSSSSS a second and a microsecond', () { + var time = FixedDateTimeFormatter('ssSSSSSS').decode('01000001'); + expect( + time, + noFractionalSeconds.add(const Duration( + seconds: 1, + microseconds: 1, + ))); + }, onPlatform: skipWeb); + test('7 S throws', () { + expect( + () => FixedDateTimeFormatter('S' * 7), + throwsFormatException, + ); + }); + test('10 Y throws', () { + expect( + () => FixedDateTimeFormatter('Y' * 10), + throwsFormatException, + ); + }); + test('Parse hex year throws', () { + expect( + () => FixedDateTimeFormatter('YYYY').decode('0xAB'), + throwsFormatException, + ); + }); + // Testing `tryDecode`. + test('Try parse year', () { + var time = FixedDateTimeFormatter('YYYY').tryDecode('1996'); + expect(time, DateTime.utc(1996)); + }); + test('Try parse hex year returns null', () { + var time = FixedDateTimeFormatter('YYYY').tryDecode('0xAB'); + expect(time, null); + }); + test('Try parse invalid returns null', () { + var time = FixedDateTimeFormatter('YYYY').tryDecode('1x96'); + expect(time, null); + }); + // Testing `encode`. + test('Format simple', () { + var time = DateTime.utc(1996); + expect(FixedDateTimeFormatter('YYYY kiwi MM').encode(time), '1996 kiwi 01'); + }); + test('Format YYYYMMDDhhmmss', () { + var time = DateTime.utc(1996, 4, 25, 5, 3, 22); + expect( + FixedDateTimeFormatter('YYYYMMDDhhmmss').encode(time), + '19960425050322', + ); + }); + test('Format CCEY-MM', () { + var str = FixedDateTimeFormatter('CCEY-MM').encode(DateTime.utc(1996, 4)); + expect(str, '1996-04'); + }); + test('Format XCCEY-MMX', () { + var str = FixedDateTimeFormatter('XCCEY-MMX').encode(DateTime.utc(1996, 4)); + expect(str, 'X1996-04X'); + }); + test('Format S 1/10 of a second', () { + var str = FixedDateTimeFormatter('S') + .encode(noFractionalSeconds.add(const Duration(milliseconds: 100))); + expect(str, '1'); + }); + test('Format SS 1/100 of a second', () { + var str = FixedDateTimeFormatter('SS') + .encode(noFractionalSeconds.add(const Duration(milliseconds: 10))); + expect(str, '01'); + }); + test('Format SSS 1/100 of a second', () { + var str = FixedDateTimeFormatter('SSS') + .encode(noFractionalSeconds.add(const Duration(milliseconds: 10))); + expect(str, '010'); + }); + test('Format SSSS no fractions', () { + var str = FixedDateTimeFormatter('SSSS').encode(noFractionalSeconds); + expect(str, '0000'); + }); + test('Format SSSSSS no fractions', () { + var str = FixedDateTimeFormatter('SSSSSS').encode(noFractionalSeconds); + expect(str, '000000'); + }); + test('Format SSSS 1/10 of a second', () { + var str = FixedDateTimeFormatter('SSSS') + .encode(noFractionalSeconds.add(const Duration(milliseconds: 100))); + expect(str, '1000'); + }); + test('Format SSSS 1/100 of a second', () { + var str = FixedDateTimeFormatter('SSSS') + .encode(noFractionalSeconds.add(const Duration(milliseconds: 10))); + expect(str, '0100'); + }); + test('Format SSSS a millisecond', () { + var str = FixedDateTimeFormatter('SSSS') + .encode(noFractionalSeconds.add(const Duration(milliseconds: 1))); + expect(str, '0010'); + }); + test('Format SSSSSS a microsecond', () { + var str = FixedDateTimeFormatter('SSSSSS') + .encode(DateTime.utc(0, 1, 1, 0, 0, 0, 0, 1)); + expect(str, '000001'); + }, onPlatform: skipWeb); + test('Format SSSSSS a millisecond and a microsecond', () { + var dateTime = noFractionalSeconds.add(const Duration( + milliseconds: 1, + microseconds: 1, + )); + var str = FixedDateTimeFormatter('SSSSSS').encode(dateTime); + expect(str, '001001'); + }, onPlatform: skipWeb); + test('Format SSSSSS0 a microsecond', () { + var str = FixedDateTimeFormatter('SSSSSS0') + .encode(noFractionalSeconds.add(const Duration(microseconds: 1))); + expect(str, '0000010'); + }, onPlatform: skipWeb); + test('Format SSSSSS0 1/10 of a second', () { + var str = FixedDateTimeFormatter('SSSSSS0') + .encode(noFractionalSeconds.add(const Duration(milliseconds: 100))); + expect(str, '1000000'); + }); + test('Parse ssSSSSSS a second and a microsecond', () { + var dateTime = noFractionalSeconds.add(const Duration( + seconds: 1, + microseconds: 1, + )); + var str = FixedDateTimeFormatter('ssSSSSSS').encode(dateTime); + expect(str, '01000001'); + }, onPlatform: skipWeb); + test('Parse ssSSSSSS0 a second and a microsecond', () { + var dateTime = noFractionalSeconds.add(const Duration( + seconds: 1, + microseconds: 1, + )); + var str = FixedDateTimeFormatter('ssSSSSSS0').encode(dateTime); + expect(str, '010000010'); + }, onPlatform: skipWeb); + test('Parse negative year throws Error', () { + expect( + () => FixedDateTimeFormatter('YYYY').encode(DateTime(-1)), + throwsArgumentError, + ); + }); +} diff --git a/pkgs/convert/test/hex_test.dart b/pkgs/convert/test/hex_test.dart new file mode 100644 index 00000000..abd940f6 --- /dev/null +++ b/pkgs/convert/test/hex_test.dart @@ -0,0 +1,222 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; + +import 'package:convert/convert.dart'; +import 'package:test/test.dart'; + +void main() { + group('encoder', () { + test('converts byte arrays to hex', () { + expect(hex.encode([0x1a, 0xb2, 0x3c, 0xd4]), equals('1ab23cd4')); + expect(hex.encode([0x00, 0x01, 0xfe, 0xff]), equals('0001feff')); + }); + + group('with chunked conversion', () { + test('converts byte arrays to hex', () { + var results = []; + var controller = StreamController(sync: true); + controller.stream.listen(results.add); + var sink = hex.encoder.startChunkedConversion(controller.sink); + + sink.add([0x1a, 0xb2, 0x3c, 0xd4]); + expect(results, equals(['1ab23cd4'])); + + sink.add([0x00, 0x01, 0xfe, 0xff]); + expect(results, equals(['1ab23cd4', '0001feff'])); + }); + + test('handles empty and single-byte lists', () { + var results = []; + var controller = StreamController(sync: true); + controller.stream.listen(results.add); + var sink = hex.encoder.startChunkedConversion(controller.sink); + + sink.add([]); + expect(results, equals([''])); + + sink.add([0x00]); + expect(results, equals(['', '00'])); + + sink.add([]); + expect(results, equals(['', '00', ''])); + }); + }); + + test('rejects non-bytes', () { + expect(() => hex.encode([0x100]), throwsFormatException); + + var sink = + hex.encoder.startChunkedConversion(StreamController(sync: true)); + expect(() => sink.add([0x100]), throwsFormatException); + }); + }); + + group('decoder', () { + test('converts hex to byte arrays', () { + expect(hex.decode('1ab23cd4'), equals([0x1a, 0xb2, 0x3c, 0xd4])); + expect(hex.decode('0001feff'), equals([0x00, 0x01, 0xfe, 0xff])); + }); + + test('supports uppercase letters', () { + expect( + hex.decode('0123456789ABCDEFabcdef'), + equals([ + 0x01, + 0x23, + 0x45, + 0x67, + 0x89, + 0xab, + 0xcd, + 0xef, + 0xab, + 0xcd, + 0xef + ])); + }); + + group('with chunked conversion', () { + late List> results; + late StringConversionSink sink; + setUp(() { + results = []; + var controller = StreamController>(sync: true); + controller.stream.listen(results.add); + sink = hex.decoder.startChunkedConversion(controller.sink); + }); + + test('converts hex to byte arrays', () { + sink.add('1ab23cd4'); + expect( + results, + equals([ + [0x1a, 0xb2, 0x3c, 0xd4] + ])); + + sink.add('0001feff'); + expect( + results, + equals([ + [0x1a, 0xb2, 0x3c, 0xd4], + [0x00, 0x01, 0xfe, 0xff] + ])); + }); + + test('supports trailing digits split across chunks', () { + sink.add('1ab23'); + expect( + results, + equals([ + [0x1a, 0xb2] + ])); + + sink.add('cd'); + expect( + results, + equals([ + [0x1a, 0xb2], + [0x3c] + ])); + + sink.add('40001'); + expect( + results, + equals([ + [0x1a, 0xb2], + [0x3c], + [0xd4, 0x00, 0x01] + ])); + + sink.add('feff'); + expect( + results, + equals([ + [0x1a, 0xb2], + [0x3c], + [0xd4, 0x00, 0x01], + [0xfe, 0xff] + ])); + }); + + test('supports empty strings', () { + sink.add(''); + expect(results, isEmpty); + + sink.add('0'); + expect(results, equals([[]])); + + sink.add(''); + expect(results, equals([[]])); + + sink.add('0'); + expect( + results, + equals([ + [], + [0x00] + ])); + + sink.add(''); + expect( + results, + equals([ + [], + [0x00] + ])); + }); + + test('rejects odd length detected in close()', () { + sink.add('1ab23'); + expect( + results, + equals([ + [0x1a, 0xb2] + ])); + expect(() => sink.close(), throwsFormatException); + }); + + test('rejects odd length detected in addSlice()', () { + sink.addSlice('1ab23cd', 0, 5, false); + expect( + results, + equals([ + [0x1a, 0xb2] + ])); + + expect( + () => sink.addSlice('1ab23cd', 5, 7, true), throwsFormatException); + }); + }); + + group('rejects non-hex character', () { + for (var char in [ + 'g', + 'G', + '/', + ':', + '@', + '`', + '\x00', + '\u0141', + '\u{10041}' + ]) { + test('"$char"', () { + expect(() => hex.decode('a$char'), throwsFormatException); + expect(() => hex.decode('${char}a'), throwsFormatException); + + var sink = + hex.decoder.startChunkedConversion(StreamController(sync: true)); + expect(() => sink.add(char), throwsFormatException); + }); + } + }); + + test('rejects odd length detected in convert()', () { + expect(() => hex.decode('1ab23cd'), throwsFormatException); + }); + }); +} diff --git a/pkgs/convert/test/identity_codec_test.dart b/pkgs/convert/test/identity_codec_test.dart new file mode 100644 index 00000000..1fb841f3 --- /dev/null +++ b/pkgs/convert/test/identity_codec_test.dart @@ -0,0 +1,27 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'package:convert/convert.dart'; +import 'package:test/test.dart'; + +void main() { + group('IdentityCodec', () { + test('encode', () { + const codec = IdentityCodec(); + expect(codec.encode('hello-world'), equals('hello-world')); + }); + + test('decode', () { + const codec = IdentityCodec(); + expect(codec.decode('hello-world'), equals('hello-world')); + }); + + test('fuse', () { + const stringCodec = IdentityCodec(); + final utf8Strings = stringCodec.fuse(utf8); + expect(utf8Strings, equals(utf8)); + }); + }); +} diff --git a/pkgs/convert/test/percent_test.dart b/pkgs/convert/test/percent_test.dart new file mode 100644 index 00000000..4e0f605d --- /dev/null +++ b/pkgs/convert/test/percent_test.dart @@ -0,0 +1,246 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; + +import 'package:convert/convert.dart'; +import 'package:test/test.dart'; + +void main() { + group('encoder', () { + test("doesn't percent-encode unreserved characters", () { + var safeChars = 'abcdefghijklmnopqrstuvwxyz' + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + '0123456789-._~'; + expect(percent.encode([...safeChars.codeUnits]), equals(safeChars)); + }); + + test('percent-encodes reserved ASCII characters', () { + expect(percent.encode([...' `{@[,/^}\x7f\x00%'.codeUnits]), + equals('%20%60%7B%40%5B%2C%2F%5E%7D%7F%00%25')); + }); + + test('percent-encodes non-ASCII characters', () { + expect(percent.encode([0x80, 0xFF]), equals('%80%FF')); + }); + + test('mixes encoded and unencoded characters', () { + expect(percent.encode([...'a+b=\x80'.codeUnits]), equals('a%2Bb%3D%80')); + }); + + group('with chunked conversion', () { + test('percent-encodes byte arrays', () { + var results = []; + var controller = StreamController(sync: true); + controller.stream.listen(results.add); + var sink = percent.encoder.startChunkedConversion(controller.sink); + + sink.add([...'a+b=\x80'.codeUnits]); + expect(results, equals(['a%2Bb%3D%80'])); + + sink.add([0x00, 0x01, 0xfe, 0xff]); + expect(results, equals(['a%2Bb%3D%80', '%00%01%FE%FF'])); + }); + + test('handles empty and single-byte lists', () { + var results = []; + var controller = StreamController(sync: true); + controller.stream.listen(results.add); + var sink = percent.encoder.startChunkedConversion(controller.sink); + + sink.add([]); + expect(results, equals([''])); + + sink.add([0x00]); + expect(results, equals(['', '%00'])); + + sink.add([]); + expect(results, equals(['', '%00', ''])); + }); + }); + + test('rejects non-bytes', () { + expect(() => percent.encode([0x100]), throwsFormatException); + + var sink = + percent.encoder.startChunkedConversion(StreamController(sync: true)); + expect(() => sink.add([0x100]), throwsFormatException); + }); + }); + + group('decoder', () { + test('converts percent-encoded strings to byte arrays', () { + expect( + percent.decode('a%2Bb%3D%801'), equals([...'a+b=\x801'.codeUnits])); + }); + + test('supports lowercase letters', () { + expect(percent.decode('a%2bb%3d%80'), equals([...'a+b=\x80'.codeUnits])); + }); + + test('supports more aggressive encoding', () { + expect(percent.decode('%61%2E%5A'), equals([...'a.Z'.codeUnits])); + }); + + test('supports less aggressive encoding', () { + var chars = ' `{@[,/^}\x7F\x00'; + expect(percent.decode(chars), equals([...chars.codeUnits])); + }); + + group('with chunked conversion', () { + late List> results; + late StringConversionSink sink; + setUp(() { + results = []; + var controller = StreamController>(sync: true); + controller.stream.listen(results.add); + sink = percent.decoder.startChunkedConversion(controller.sink); + }); + + test('converts percent to byte arrays', () { + sink.add('a%2Bb%3D%801'); + expect( + results, + equals([ + [...'a+b=\x801'.codeUnits] + ])); + + sink.add('%00%01%FE%FF'); + expect( + results, + equals([ + [...'a+b=\x801'.codeUnits], + [0x00, 0x01, 0xfe, 0xff] + ])); + }); + + test('supports trailing percents and digits split across chunks', () { + sink.add('ab%'); + expect( + results, + equals([ + [...'ab'.codeUnits] + ])); + + sink.add('2'); + expect( + results, + equals([ + [...'ab'.codeUnits] + ])); + + sink.add('0cd%2'); + expect( + results, + equals([ + [...'ab'.codeUnits], + [...' cd'.codeUnits] + ])); + + sink.add('0'); + expect( + results, + equals([ + [...'ab'.codeUnits], + [...' cd'.codeUnits], + [...' '.codeUnits] + ])); + }); + + test('supports empty strings', () { + sink.add(''); + expect(results, isEmpty); + + sink.add('%'); + expect(results, equals([[]])); + + sink.add(''); + expect(results, equals([[]])); + + sink.add('2'); + expect(results, equals([[]])); + + sink.add(''); + expect(results, equals([[]])); + + sink.add('0'); + expect( + results, + equals([ + [], + [0x20] + ])); + }); + + test('rejects dangling % detected in close()', () { + sink.add('ab%'); + expect( + results, + equals([ + [...'ab'.codeUnits] + ])); + expect(() => sink.close(), throwsFormatException); + }); + + test('rejects dangling digit detected in close()', () { + sink.add('ab%2'); + expect( + results, + equals([ + [...'ab'.codeUnits] + ])); + expect(() => sink.close(), throwsFormatException); + }); + + test('rejects danging % detected in addSlice()', () { + sink.addSlice('ab%', 0, 3, false); + expect( + results, + equals([ + [...'ab'.codeUnits] + ])); + + expect(() => sink.addSlice('ab%', 0, 3, true), throwsFormatException); + }); + + test('rejects danging digit detected in addSlice()', () { + sink.addSlice('ab%2', 0, 3, false); + expect( + results, + equals([ + [...'ab'.codeUnits] + ])); + + expect(() => sink.addSlice('ab%2', 0, 3, true), throwsFormatException); + }); + }); + + group('rejects non-ASCII character', () { + for (var char in ['\u0141', '\u{10041}']) { + test('"$char"', () { + expect(() => percent.decode('a$char'), throwsFormatException); + expect(() => percent.decode('${char}a'), throwsFormatException); + + var sink = percent.decoder + .startChunkedConversion(StreamController(sync: true)); + expect(() => sink.add(char), throwsFormatException); + }); + } + }); + + test('rejects % followed by non-hex', () { + expect(() => percent.decode('%z2'), throwsFormatException); + expect(() => percent.decode('%2z'), throwsFormatException); + }); + + test('rejects dangling % detected in convert()', () { + expect(() => percent.decode('ab%'), throwsFormatException); + }); + + test('rejects dangling digit detected in convert()', () { + expect(() => percent.decode('ab%2'), throwsFormatException); + }); + }); +} diff --git a/pkgs/convert/test/string_accumulator_sink_test.dart b/pkgs/convert/test/string_accumulator_sink_test.dart new file mode 100644 index 00000000..8e3b7cb5 --- /dev/null +++ b/pkgs/convert/test/string_accumulator_sink_test.dart @@ -0,0 +1,56 @@ +// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:convert/convert.dart'; +import 'package:test/test.dart'; + +void main() { + late StringAccumulatorSink sink; + setUp(() { + sink = StringAccumulatorSink(); + }); + + test('provides access to the concatenated string', () { + expect(sink.string, isEmpty); + + sink.add('foo'); + expect(sink.string, equals('foo')); + + sink.addSlice(' bar baz', 1, 4, false); + expect(sink.string, equals('foobar')); + }); + + test('clear() clears the string', () { + sink.add('foo'); + expect(sink.string, equals('foo')); + + sink.clear(); + expect(sink.string, isEmpty); + + sink.add('bar'); + expect(sink.string, equals('bar')); + }); + + test('indicates whether the sink is closed', () { + expect(sink.isClosed, isFalse); + sink.close(); + expect(sink.isClosed, isTrue); + }); + + test('indicates whether the sink is closed via addSlice', () { + expect(sink.isClosed, isFalse); + sink.addSlice('', 0, 0, true); + expect(sink.isClosed, isTrue); + }); + + test("doesn't allow add() to be called after close()", () { + sink.close(); + expect(() => sink.add('x'), throwsStateError); + }); + + test("doesn't allow addSlice() to be called after close()", () { + sink.close(); + expect(() => sink.addSlice('', 0, 0, false), throwsStateError); + }); +}