diff --git a/components/core/src/clp/regex_utils/ErrorCode.cpp b/components/core/src/clp/regex_utils/ErrorCode.cpp index 112ede242..4e24e9a8b 100644 --- a/components/core/src/clp/regex_utils/ErrorCode.cpp +++ b/components/core/src/clp/regex_utils/ErrorCode.cpp @@ -72,6 +72,13 @@ auto ErrorCodeCategory::message(int ev) const -> string { case ErrorCode::UnmatchedParenthesis: return "Unmatched opening `(` or closing `)`."; + case ErrorCode::IncompleteCharsetStructure: + return "Unmatched closing `]` at the end of the string."; + + case ErrorCode::UnsupportedCharsetPattern: + return "Currently only supports character set that can be reduced to a single " + "character."; + default: return "(unrecognized error)"; } diff --git a/components/core/src/clp/regex_utils/ErrorCode.hpp b/components/core/src/clp/regex_utils/ErrorCode.hpp index 77a52cf58..9b4fbf8f2 100644 --- a/components/core/src/clp/regex_utils/ErrorCode.hpp +++ b/components/core/src/clp/regex_utils/ErrorCode.hpp @@ -21,6 +21,8 @@ enum class ErrorCode : uint8_t { IllegalDollarSign, IllegalEscapeSequence, UnmatchedParenthesis, + IncompleteCharsetStructure, + UnsupportedCharsetPattern, }; /** diff --git a/components/core/src/clp/regex_utils/constants.hpp b/components/core/src/clp/regex_utils/constants.hpp index 9833543fc..ff2eb5b10 100644 --- a/components/core/src/clp/regex_utils/constants.hpp +++ b/components/core/src/clp/regex_utils/constants.hpp @@ -44,6 +44,9 @@ constexpr auto cRegexEscapeSeqMetaCharsLut = create_char_bit_array("*+?|^$.{}[]( // The set of wildcard metacharacters that must remain escaped in the translated string to be // treated as a literal. constexpr auto cWildcardMetaCharsLut = create_char_bit_array("?*\\"); +// The set of metacharacters that can be preceded with an escape backslash in the regex character +// set to be treated as a literal. +constexpr auto cRegexCharsetEscapeSeqMetaCharsLut = create_char_bit_array("^-]\\"); } // namespace clp::regex_utils #endif // CLP_REGEX_UTILS_CONSTANTS_HPP diff --git a/components/core/src/clp/regex_utils/regex_translation_utils.cpp b/components/core/src/clp/regex_utils/regex_translation_utils.cpp index f26d70521..f1a987006 100644 --- a/components/core/src/clp/regex_utils/regex_translation_utils.cpp +++ b/components/core/src/clp/regex_utils/regex_translation_utils.cpp @@ -1,18 +1,22 @@ #include "regex_utils/regex_translation_utils.hpp" #include +#include #include #include #include #include +#include #include "regex_utils/constants.hpp" #include "regex_utils/ErrorCode.hpp" #include "regex_utils/RegexToWildcardTranslatorConfig.hpp" namespace clp::regex_utils { +using clp::string_utils::is_alphabet; using std::error_code; +using std::optional; using std::string; using std::string_view; @@ -31,6 +35,8 @@ class TranslatorState { * literally. *
  • Dot: Encountered a period `.`. Expecting wildcard expression.
    • *
  • Escaped: Encountered a backslash `\`. Expecting an escape sequence.
    • + *
  • Charset: Encountered an opening square bracket `[`. Expecting a character set.
    • + *
  • CharsetEscaped: Encountered an escape backslash in the character set.
    • *
  • End: Encountered a dollar sign `$`, meaning the regex string has reached the end * anchor.
    • * @@ -39,6 +45,8 @@ class TranslatorState { Normal = 0, Dot, Escaped, + Charset, + CharsetEscaped, End, }; @@ -48,12 +56,23 @@ class TranslatorState { // Getters [[nodiscard]] auto get_state() const -> RegexPatternState { return m_state; } + [[nodiscard]] auto get_charset_begin_it() const -> optional { + return m_charset_begin_it; + } + // Setters auto set_next_state(RegexPatternState const& state) -> void { m_state = state; } + auto set_charset_begin_it(string_view::const_iterator charset_begin_it) -> void { + m_charset_begin_it = charset_begin_it; + } + + auto invalidate_charset_begin_it() -> void { m_charset_begin_it.reset(); } + private: // Members RegexPatternState m_state{RegexPatternState::Normal}; + optional m_charset_begin_it; }; /** @@ -65,7 +84,7 @@ class TranslatorState { * @param[in, out] it The iterator that represents the current regex string scan position. May be * updated to advance or backtrack the scan position. * @param[out] wildcard_str The translated wildcard string. May or may not be updated. - * @param[in] config The translator config. + * @param[in] config The translator config predefined by the user. * @return clp::regex_utils::ErrorCode */ using StateTransitionFuncSig @@ -103,6 +122,24 @@ using StateTransitionFuncSig */ [[nodiscard]] StateTransitionFuncSig escaped_state_transition; +/** + * Reduces a regex character set into a single character so that the regex string is still + * translatable into a wildcard string. + * + * In most cases, only a trival character set containing a single character is reducable. However, + * if the output wildcard query will be analyzed in case-insensitive mode, character set patterns + * such as [aA] [Bb] are also reducable. Does not support empty charsets. + * On error, returns IncompleteCharsetStructure, UnsupportedCharsetPattern, or IllegalState. + */ +[[nodiscard]] StateTransitionFuncSig charset_state_transition; + +/** + * A transient state used to defer handling of escape sequences in a charset pattern. + * + * Allows the charset state to accurately capture the appearance of a closing bracket `]`. + */ +[[nodiscard]] StateTransitionFuncSig charset_escaped_state_transition; + /** * Disallows the appearances of other characters after encountering an end anchor in the string. */ @@ -114,6 +151,23 @@ using StateTransitionFuncSig */ [[nodiscard]] StateTransitionFuncSig final_state_cleanup; +/** + * Appends a single character as a literal to the wildcard string. + * + * If the literal is a metacharacter in the wildcard syntax, prepend the literal with an escape + * backslash. + * @param ch The literal to be appended. + * @param wildcard_str The wildcard string to be updated. + */ +auto append_char_to_wildcard(char ch, string& wildcard_str) -> void; + +/** + * @param ch0 + * @param ch1 + * @return Whether the given chars are the same, but with opposing letter cases, e.g. 'A' vs. 'a'. + */ +[[nodiscard]] auto is_same_char_opposite_case(char ch0, char ch1) -> bool; + auto normal_state_transition( TranslatorState& state, string_view::const_iterator& it, @@ -128,6 +182,10 @@ auto normal_state_transition( case cEscapeChar: state.set_next_state(TranslatorState::RegexPatternState::Escaped); break; + case '[': + state.set_charset_begin_it(it + 1); + state.set_next_state(TranslatorState::RegexPatternState::Charset); + break; case cRegexEndAnchor: state.set_next_state(TranslatorState::RegexPatternState::End); break; @@ -183,14 +241,72 @@ auto escaped_state_transition( if (false == cRegexEscapeSeqMetaCharsLut.at(ch)) { return ErrorCode::IllegalEscapeSequence; } - if (cWildcardMetaCharsLut.at(ch)) { - wildcard_str += cEscapeChar; + append_char_to_wildcard(ch, wildcard_str); + state.set_next_state(TranslatorState::RegexPatternState::Normal); + return ErrorCode::Success; +} + +auto charset_state_transition( + TranslatorState& state, + string_view::const_iterator& it, + string& wildcard_str, + RegexToWildcardTranslatorConfig const& config +) -> error_code { + auto const charset_begin_it_opt{state.get_charset_begin_it()}; + if (false == charset_begin_it_opt.has_value()) { + return ErrorCode::IllegalState; } - wildcard_str += ch; + string_view::const_iterator const charset_begin_it = charset_begin_it_opt.value(); + + auto const ch{*it}; + if (cEscapeChar == ch) { + state.set_next_state(TranslatorState::RegexPatternState::CharsetEscaped); + return ErrorCode::Success; + } + if (']' != ch) { + return ErrorCode::Success; + } + + auto const charset_len{it - charset_begin_it}; + if (0 == charset_len || charset_len > 2) { + return ErrorCode::UnsupportedCharsetPattern; + } + + auto const ch0{*charset_begin_it}; + auto const ch1{*(charset_begin_it + 1)}; + char parsed_char{}; + + if (1 == charset_len) { + if (cCharsetNegate == ch0 || cEscapeChar == ch0) { + return ErrorCode::UnsupportedCharsetPattern; + } + parsed_char = ch0; + } else { // 2 == charset_len + if (cEscapeChar == ch0 && cRegexCharsetEscapeSeqMetaCharsLut.at(ch1)) { + parsed_char = ch1; + } else if (config.case_insensitive_wildcard() && is_same_char_opposite_case(ch0, ch1)) { + parsed_char = ch0 > ch1 ? ch0 : ch1; // choose the lower case character + } else { + return ErrorCode::UnsupportedCharsetPattern; + } + } + + append_char_to_wildcard(parsed_char, wildcard_str); + state.invalidate_charset_begin_it(); state.set_next_state(TranslatorState::RegexPatternState::Normal); return ErrorCode::Success; } +auto charset_escaped_state_transition( + TranslatorState& state, + [[maybe_unused]] string_view::const_iterator& it, + [[maybe_unused]] string& wildcard_str, + [[maybe_unused]] RegexToWildcardTranslatorConfig const& config +) -> error_code { + state.set_next_state(TranslatorState::RegexPatternState::Charset); + return ErrorCode::Success; +} + auto end_state_transition( [[maybe_unused]] TranslatorState& state, string_view::const_iterator& it, @@ -215,6 +331,9 @@ auto final_state_cleanup( // multichar wildcard wildcard_str += cSingleCharWildcard; break; + case TranslatorState::RegexPatternState::Charset: + case TranslatorState::RegexPatternState::CharsetEscaped: + return ErrorCode::IncompleteCharsetStructure; default: break; } @@ -226,10 +345,27 @@ auto final_state_cleanup( } return ErrorCode::Success; } + +auto append_char_to_wildcard(char ch, string& wildcard_str) -> void { + if (cWildcardMetaCharsLut.at(ch)) { + wildcard_str += cEscapeChar; + } + wildcard_str += ch; +} + +auto is_same_char_opposite_case(char ch0, char ch1) -> bool { + int const upper_lower_case_ascii_offset{'a' - 'A'}; + return (is_alphabet(ch0) && is_alphabet(ch1) + && (((ch0 - ch1) == upper_lower_case_ascii_offset) + || ((ch1 - ch0) == upper_lower_case_ascii_offset))); +} } // namespace auto regex_to_wildcard(string_view regex_str) -> OUTCOME_V2_NAMESPACE::std_result { - return regex_to_wildcard(regex_str, {false, false}); + return regex_to_wildcard( + regex_str, + {/*case_insensitive_wildcard=*/false, /*add_prefix_suffix_wildcards=*/false} + ); } auto regex_to_wildcard(string_view regex_str, RegexToWildcardTranslatorConfig const& config) @@ -238,9 +374,9 @@ auto regex_to_wildcard(string_view regex_str, RegexToWildcardTranslatorConfig co return string{}; } - TranslatorState state; string_view::const_iterator it{regex_str.cbegin()}; string wildcard_str; + TranslatorState state; // If there is no starting anchor character, append multichar wildcard prefix if (cRegexStartAnchor == *it) { @@ -261,6 +397,12 @@ auto regex_to_wildcard(string_view regex_str, RegexToWildcardTranslatorConfig co case TranslatorState::RegexPatternState::Escaped: ec = escaped_state_transition(state, it, wildcard_str, config); break; + case TranslatorState::RegexPatternState::Charset: + ec = charset_state_transition(state, it, wildcard_str, config); + break; + case TranslatorState::RegexPatternState::CharsetEscaped: + ec = charset_escaped_state_transition(state, it, wildcard_str, config); + break; case TranslatorState::RegexPatternState::End: ec = end_state_transition(state, it, wildcard_str, config); break; diff --git a/components/core/tests/test-regex_utils.cpp b/components/core/tests/test-regex_utils.cpp index 9defd7d08..64af60318 100644 --- a/components/core/tests/test-regex_utils.cpp +++ b/components/core/tests/test-regex_utils.cpp @@ -41,9 +41,50 @@ TEST_CASE("regex_to_wildcard_escaped_metachar", "[regex_utils][re2wc][escaped_me ); } +TEST_CASE("regex_to_wildcard_charset", "[regex_utils][re2wc][charset]") { + REQUIRE((regex_to_wildcard("x[y]z").value() == "xyz")); + REQUIRE((regex_to_wildcard("x[\\^]z").value() == "x^z")); + REQUIRE((regex_to_wildcard("x[\\]]z").value() == "x]z")); + REQUIRE((regex_to_wildcard("x[-]z").value() == "x-z")); + REQUIRE((regex_to_wildcard("x[\\-]z").value() == "x-z")); + REQUIRE((regex_to_wildcard("x[\\\\]z").value() == "x\\\\z")); + REQUIRE((regex_to_wildcard("[a][b][\\^][-][\\-][\\]][\\\\][c][d]").value() == "ab^--]\\\\cd")); + + REQUIRE((regex_to_wildcard("x[]y").error() == ErrorCode::UnsupportedCharsetPattern)); + REQUIRE((regex_to_wildcard("x[a-z]y").error() == ErrorCode::UnsupportedCharsetPattern)); + REQUIRE((regex_to_wildcard("x[^^]y").error() == ErrorCode::UnsupportedCharsetPattern)); + REQUIRE((regex_to_wildcard("x[^0-9]y").error() == ErrorCode::UnsupportedCharsetPattern)); + REQUIRE((regex_to_wildcard("[xX][yY]").error() == ErrorCode::UnsupportedCharsetPattern)); + REQUIRE((regex_to_wildcard("ch:[a-zA-Z0-9]").error() == ErrorCode::UnsupportedCharsetPattern)); + + REQUIRE((regex_to_wildcard("[\\").error() == ErrorCode::IncompleteCharsetStructure)); + REQUIRE((regex_to_wildcard("[\\\\").error() == ErrorCode::IncompleteCharsetStructure)); + REQUIRE((regex_to_wildcard("[xX").error() == ErrorCode::IncompleteCharsetStructure)); + REQUIRE((regex_to_wildcard("ch:[a-zA-Z0-9").error() == ErrorCode::IncompleteCharsetStructure)); +} + +TEST_CASE("regex_to_wildcard_case_insensitive_config", "[regex_utils][re2wc][case_insensitive]") { + RegexToWildcardTranslatorConfig const config{/*case_insensitive_wildcard=*/true, false}; + REQUIRE((regex_to_wildcard("[xX][yY]", config).value() == "xy")); + REQUIRE((regex_to_wildcard("[Yy][Xx]", config).value() == "yx")); + REQUIRE((regex_to_wildcard("[aA][Bb][Cc]", config).value() == "abc")); + REQUIRE((regex_to_wildcard("[aA][Bb][\\^][-][\\]][Cc][dD]", config).value() == "ab^-]cd")); + + REQUIRE((regex_to_wildcard("[xX").error() == ErrorCode::IncompleteCharsetStructure)); + REQUIRE( + (regex_to_wildcard("[aA][Bb][^[-[\\[Cc[dD", config).error() + == ErrorCode::IncompleteCharsetStructure) + ); + REQUIRE((regex_to_wildcard("ch:[a-zA-Z0-9]").error() == ErrorCode::UnsupportedCharsetPattern)); + REQUIRE( + (regex_to_wildcard("[aA][Bb][^[-[\\[Cc[dD]", config).error() + == ErrorCode::UnsupportedCharsetPattern) + ); +} + TEST_CASE("regex_to_wildcard_anchor_config", "[regex_utils][re2wc][anchor_config]") { // Test anchors and prefix/suffix wildcards - RegexToWildcardTranslatorConfig const config{false, true}; + RegexToWildcardTranslatorConfig const config{false, /*add_prefix_suffix_wildcards=*/true}; REQUIRE(((regex_to_wildcard("^", config).value() == "*"))); REQUIRE((regex_to_wildcard("$", config).value() == "*")); REQUIRE((regex_to_wildcard("^xyz$", config).value() == "xyz")); diff --git a/docs/src/dev-guide/components-core/regex-utils.md b/docs/src/dev-guide/components-core/regex-utils.md index f7af037df..b79f0a729 100644 --- a/docs/src/dev-guide/components-core/regex-utils.md +++ b/docs/src/dev-guide/components-core/regex-utils.md @@ -77,14 +77,23 @@ For a detailed description on the options order and usage, see the * Escape sequences with alphanumeric characters are disallowed. * E.g. Special utility escape sequences `\Q`, `\E`, `\A` etc. and back references `\1` `\2` etc. cannot be translated. +* Character set + * Reduces a character set into a single character if possible. + * A trivial character set containing a single character or a single escaped metacharacter. + * E.g. `[a]` into `a`, `[\^]` into `^` + * If the `case_insensitive_wildcard` config is turned on, the translator can also reduce the + case-insensitive style character set patterns into a single lowercase character: + * E.g. `[aA]` into `a`, `[Bb]` into `b`, `[xX][Yy][zZ]` into `xyz` ### Custom configuration -The `RegexToWildcardTranslatorConfig` class objects are currently immutable once instantiated. The -constructor takes the following arguments in order: +The `RegexToWildcardTranslatorConfig` class objects are currently immutable once instantiated. By +default, all of the options are set to `false`. -* `case_insensitive_wildcard`: to be added later along with the character set translation - implementation. +The constructor takes the following option arguments in order: + +* `case_insensitive_wildcard`: see **Character set** bullet point in the + [Functionalities](#functionalities) section. * `add_prefix_suffix_wildcards`: in the absence of regex anchors, add prefix or suffix wildcards so the query becomes a substring query.