Update to version v3.31.0

.github/workflows/test.yml

@@ -209,7 +209,7 @@ jobs:
         run: |
           if [[ $OS == ubuntu* ]]; then
             sudo ./dependencies.sh
-            python3 -m pip install setuptools
+            python3 -m pip install setuptools build
           else
             ./dependencies.sh
           fi
@@ -238,7 +238,9 @@ jobs:
         with:
           repository: restream/reindexer-py
       - name: Install PyReindexer
-        run: sudo python3 setup.py install
+        run: |
+          python -m build
+          python -m pip install .
       - name: Test PyReindexer
         run: |
           cd pyreindexer

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v3.30.0"
+const ReindexerVersion = "v3.31.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

changelog.md

@@ -1,3 +1,29 @@
+# Version 3.31.0 (28.12.2024)
+## Core
+- [fea] Added fast path for index updates for some specific cases (when index switches type between `tree`/`hash`/`-` and/or changes it's `is_dense`-flag)
+- [fea] Allowed to update part of the composite index for some specific cases (when underlying index uses `fast path` update)
+- [fea] Added information about index/namespace cache hits into `#perfstats`-namespace
+- [fea] Slight comparators optimization (better inlining)
+- [fea] Allowed to update scalar-index to array-index and vice versa for the case, when namespace is empty
+- [fix] Fixed incorrect [EqualPosition's](readme.md#search-in-array-fields-with-matching-array-indexes) interaction with brackets optimizer in `SELECT`-queries. Now optimizer should not remove the brackets containing `EqualPosition`
+- [fix] Fixed crash in `ORDER BY` arithmetic sorting expresion with composite indexes
+- [fix] Fixed assertion on attempt to use 'null'-values with `=`, `IN()`, `<`, `>`, `<=`, `>=` and `RANGE()` operators
+- [fix] Added extra type validations for indexed document content during CJSON/JSON/MsgPack/Protobuf deserialization
+- [fix] Added JSON-validation for `SetObject`-method in `UPDATE`-queries
+
+## Fulltext
+- [fix] Fixed areas positions for `number search`
+- [fix] Removed some of the default stop-words
+
+## Go connector
+- [fea] Added support for empty `reindex`/`json` tags with default index/jsonpath-names (i.e. constructions like ``StrField string `reindex:"" json:""` ``)
+
+## Face
+- [fea] Added `splitter` field to the Full text index settings
+- [fix] Fixed wrong input field clearing on the cache settings page
+- [fix] Fixed sort order icon on the tables
+- [fix] Fixed data saving in JSON view on the Index settings page
+
 # Version 3.30.0 (29.11.2024)
 ## Core
 - [fea] Optimized memory layout for `-tuple`-index (this slightly reduces memory consumation for any `reindexer` database)

cjson/decoder.go

@@ -36,8 +36,7 @@ func fieldByTag(t reflect.Type, tag string) (result reflect.StructField, ok bool
 	}
 	for i := 0; i < t.NumField(); i++ {
 		result = t.Field(i)
-		if ftag := result.Tag.Get("json"); len(ftag) > 0 {
-			ftag, _ = splitStr(ftag, ',')
+		if ftag, _ := splitStr(result.Tag.Get("json"), ','); len(ftag) > 0 {
 			if tag == ftag {
 				return result, true
 			}

cpp_src/CMakeLists.txt

@@ -44,7 +44,7 @@ else()
   option(LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set(REINDEXER_VERSION_DEFAULT "3.30.0")
+set(REINDEXER_VERSION_DEFAULT "3.31.0")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")

cpp_src/core/cjson/cjsondecoder.cc

@@ -40,11 +40,12 @@ bool CJsonDecoder::decodeCJson(Payload& pl, Serializer& rdser, WrSerializer& wrs
 				const auto& fieldRef{pl.Type().Field(field)};
 				const KeyValueType fieldType{fieldRef.Type()};
 				if (tagType == TAG_ARRAY) {
+					const carraytag atag = rdser.GetCArrayTag();
+					const auto count = atag.Count();
 					if rx_unlikely (!fieldRef.IsArray()) {
 						throwUnexpectedArrayError(fieldRef);
 					}
-					const carraytag atag = rdser.GetCArrayTag();
-					const auto count = atag.Count();
+					validateArrayFieldRestrictions(fieldRef, count, "cjson");
 					const int ofs = pl.ResizeArray(field, count, true);
 					const TagType atagType = atag.Type();
 					if (atagType != TAG_OBJECT) {
@@ -61,6 +62,7 @@ bool CJsonDecoder::decodeCJson(Payload& pl, Serializer& rdser, WrSerializer& wrs
 					wrser.PutVarUint(count);
 				} else {
 					validateNonArrayFieldRestrictions(objectScalarIndexes_, pl, fieldRef, field, isInArray(), "cjson");
+					validateArrayFieldRestrictions(fieldRef, 1, "cjson");
 					objectScalarIndexes_.set(field);
 					pl.Set(field, cjsonValueToVariant(tagType, rdser, fieldType), true);
 					fieldType.EvaluateOneOf(

cpp_src/core/cjson/cjsontools.cc

@@ -207,6 +207,11 @@ void throwScalarMultipleEncodesError(const Payload& pl, const PayloadFieldType&
 	throw Error(errLogic, "Non-array field '%s' [%d] from '%s' can only be encoded once.", f.Name(), field, pl.Type().Name());
 }
 
+void throwUnexpectedArraySizeError(std::string_view parserName, const PayloadFieldType& f, int arraySize) {
+	throw Error(errParams, "%s array field '%s' for this index type must contain %d elements, but got %d", parserName, f.Name(),
+				f.ArrayDim(), arraySize);
+}
+
 static void dumpCjsonValue(TagType type, Serializer& cjson, std::ostream& dump) {
 	switch (type) {
 		case TAG_VARINT:

cpp_src/core/cjson/cjsontools.h

@@ -18,6 +18,7 @@ void skipCjsonTag(ctag tag, Serializer& rdser, std::array<unsigned, kMaxIndexes>
 
 [[noreturn]] void throwUnexpectedNestedArrayError(std::string_view parserName, const PayloadFieldType& f);
 [[noreturn]] void throwScalarMultipleEncodesError(const Payload& pl, const PayloadFieldType& f, int field);
+[[noreturn]] void throwUnexpectedArraySizeError(std::string_view parserName, const PayloadFieldType& f, int arraySize);
 RX_ALWAYS_INLINE void validateNonArrayFieldRestrictions(const ScalarIndexesSetT& scalarIndexes, const Payload& pl,
 														const PayloadFieldType& f, int field, bool isInArray, std::string_view parserName) {
 	if (!f.IsArray()) {
@@ -30,6 +31,14 @@ RX_ALWAYS_INLINE void validateNonArrayFieldRestrictions(const ScalarIndexesSetT&
 	}
 }
 
+RX_ALWAYS_INLINE void validateArrayFieldRestrictions(const PayloadFieldType& f, int arraySize, std::string_view parserName) {
+	if (f.IsArray()) {
+		if rx_unlikely (arraySize && f.ArrayDim() > 0 && f.ArrayDim() != arraySize) {
+			throwUnexpectedArraySizeError(parserName, f, arraySize);
+		}
+	}
+}
+
 void DumpCjson(Serializer& cjson, std::ostream& dump, const ConstPayload*, const TagsMatcher* = nullptr, std::string_view tab = "  ");
 inline void DumpCjson(Serializer&& cjson, std::ostream& dump, const ConstPayload* pl, const TagsMatcher* tm = nullptr,
 					  std::string_view tab = "  ") {
@@ -49,4 +58,21 @@ inline void DumpCjson(Serializer&& cjson, std::ostream& dump, const TagsMatcher*
 	DumpCjson(cjson, dump, tm, tab);
 }
 
+static inline Variant convertValueForPayload(Payload& pl, int field, Variant&& value, std::string_view source) {
+	if (field < 0) {
+		return value;
+	}
+
+	auto plFieldType = pl.Type().Field(field).Type();
+	if (plFieldType.IsSame(value.Type())) {
+		return value;
+	} else if ((plFieldType.IsNumeric() && value.Type().IsNumeric()) ||
+			   (plFieldType.Is<KeyValueType::Uuid>() && value.Type().Is<KeyValueType::String>())) {
+		return value.convert(pl.Type().Field(field).Type());
+	} else {
+		throw Error(errLogic, "Error parsing %s field '%s' - got %s, expected %s", source, pl.Type().Field(field).Name(),
+					value.Type().Name(), plFieldType.Name());
+	}
+}
+
 }  // namespace reindexer

cpp_src/core/cjson/jsondecoder.cc

@@ -52,6 +52,7 @@ void JsonDecoder::decodeJsonObject(Payload& pl, CJsonBuilder& builder, const gas
 						(void)subelem;
 						++count;
 					}
+					validateArrayFieldRestrictions(f, count, "json");
 					int pos = pl.ResizeArray(field, count, true);
 					for (auto& subelem : elem.value) {
 						pl.Set(field, pos++, jsonValue2Variant(subelem.value, f.Type(), f.Name()));
@@ -70,6 +71,7 @@ void JsonDecoder::decodeJsonObject(Payload& pl, CJsonBuilder& builder, const gas
 				case gason::JSON_TRUE:
 				case gason::JSON_FALSE: {
 					validateNonArrayFieldRestrictions(objectScalarIndexes_, pl, f, field, isInArray(), "json");
+					validateArrayFieldRestrictions(f, 1, "json");
 					objectScalarIndexes_.set(field);
 					Variant value = jsonValue2Variant(elem.value, f.Type(), f.Name());
 					builder.Ref(tagName, value, field);
@@ -150,7 +152,10 @@ class TagsPathGuard {
 
 void JsonDecoder::decodeJsonObject(const gason::JsonValue& root, CJsonBuilder& builder) {
 	for (const auto& elem : root) {
-		int tagName = tagsMatcher_.name2tag(elem.key, true);
+		const int tagName = tagsMatcher_.name2tag(elem.key, true);
+		if (tagName == 0) {
+			throw Error(errParseJson, "Unsupported JSON format. Unnamed field detected");
+		}
 		TagsPathGuard tagsPathGuard(tagsPath_, tagName);
 		decodeJson(nullptr, builder, elem.value, tagName, true);
 	}

cpp_src/core/cjson/msgpackdecoder.cc

@@ -11,22 +11,20 @@ template <typename T>
 void MsgPackDecoder::setValue(Payload& pl, CJsonBuilder& builder, const T& value, int tagName) {
 	int field = tm_.tags2field(tagsPath_.data(), tagsPath_.size());
 	if (field > 0) {
-		validateNonArrayFieldRestrictions(objectScalarIndexes_, pl, pl.Type().Field(field), field, isInArray(), "msgpack");
+		const auto& f = pl.Type().Field(field);
+		validateNonArrayFieldRestrictions(objectScalarIndexes_, pl, f, field, isInArray(), "msgpack");
+		if (!isInArray()) {
+			validateArrayFieldRestrictions(f, 1, "msgpack");
+		}
 		Variant val(value);
 		builder.Ref(tagName, val, field);
-		pl.Set(field, std::move(val), true);
+		pl.Set(field, convertValueForPayload(pl, field, std::move(val), "msgpack"));
 		objectScalarIndexes_.set(field);
 	} else {
 		builder.Put(tagName, value);
 	}
 }
 
-void MsgPackDecoder::iterateOverArray(const msgpack_object* begin, const msgpack_object* end, Payload& pl, CJsonBuilder& array) {
-	for (const msgpack_object* p = begin; p != end; ++p) {
-		decode(pl, array, *p, 0);
-	}
-}
-
 int MsgPackDecoder::decodeKeyToTag(const msgpack_object_kv& obj) {
 	using namespace std::string_view_literals;
 	switch (obj.key.type) {
@@ -95,11 +93,43 @@ void MsgPackDecoder::decode(Payload& pl, CJsonBuilder& builder, const msgpack_ob
 				if rx_unlikely (!f.IsArray()) {
 					throw Error(errLogic, "Error parsing msgpack field '%s' - got array, expected scalar %s", f.Name(), f.Type().Name());
 				}
-				auto& array = builder.ArrayRef(tagName, field, count);
-				iterateOverArray(begin, end, pl, array);
+				validateArrayFieldRestrictions(f, count, "msgpack");
+				int pos = pl.ResizeArray(field, count, true);
+				for (const msgpack_object* p = begin; p != end; ++p) {
+					pl.Set(field, pos++,
+						   convertValueForPayload(
+							   pl, field,
+							   [&] {
+								   switch (p->type) {
+									   case MSGPACK_OBJECT_BOOLEAN:
+										   return Variant{p->via.boolean};
+									   case MSGPACK_OBJECT_POSITIVE_INTEGER:
+										   return Variant{int64_t(p->via.u64)};
+									   case MSGPACK_OBJECT_NEGATIVE_INTEGER:
+										   return Variant{p->via.i64};
+									   case MSGPACK_OBJECT_FLOAT32:
+									   case MSGPACK_OBJECT_FLOAT64:
+										   return Variant{p->via.f64};
+									   case MSGPACK_OBJECT_STR:
+										   return Variant{p_string(reinterpret_cast<const l_msgpack_hdr*>(&p->via.str)), Variant::hold_t{}};
+									   case MSGPACK_OBJECT_NIL:
+									   case MSGPACK_OBJECT_ARRAY:
+									   case MSGPACK_OBJECT_MAP:
+									   case MSGPACK_OBJECT_BIN:
+									   case MSGPACK_OBJECT_EXT:
+									   default:
+										   throw Error(errParams, "Unsupported MsgPack array field type: %s(%d)", ToString(p->type),
+													   int(p->type));
+								   }
+							   }(),
+							   "msgpack"));
+				}
+				builder.ArrayRef(tagName, field, count);
 			} else {
 				auto array = builder.Array(tagName, type);
-				iterateOverArray(begin, end, pl, array);
+				for (const msgpack_object* p = begin; p != end; ++p) {
+					decode(pl, array, *p, 0);
+				}
 			}
 			break;
 		}

cpp_src/core/cjson/msgpackdecoder.h

@@ -19,7 +19,6 @@ class MsgPackDecoder {
 
 private:
 	void decode(Payload& pl, CJsonBuilder& builder, const msgpack_object& obj, int tagName);
-	void iterateOverArray(const msgpack_object* begin, const msgpack_object* end, Payload& pl, CJsonBuilder& builder);
 
 	int decodeKeyToTag(const msgpack_object_kv& obj);
 

cpp_src/core/cjson/protobufdecoder.cc

@@ -1,4 +1,5 @@
 #include "protobufdecoder.h"
+#include "core/cjson/cjsontools.h"
 #include "core/schema.h"
 #include "estl/protobufparser.h"
 
@@ -51,9 +52,10 @@ void ProtobufDecoder::setValue(Payload& pl, CJsonBuilder& builder, ProtobufValue
 		if (item.isArray) {
 			arraysStorage_.UpdateArraySize(item.tagName, field);
 		} else {
+			validateArrayFieldRestrictions(f, 1, "protobuf");
 			builder.Ref(item.tagName, value, field);
 		}
-		pl.Set(field, std::move(value), true);
+		pl.Set(field, convertValueForPayload(pl, field, std::move(value), "protobuf"), true);
 		objectScalarIndexes_.set(field);
 	} else {
 		if (item.isArray) {
@@ -78,13 +80,14 @@ Error ProtobufDecoder::decodeArray(Payload& pl, CJsonBuilder& builder, const Pro
 		if (packed) {
 			int count = 0;
 			while (!parser.IsEof()) {
-				pl.Set(field, parser.ReadArrayItem(item.itemType), true);
+				pl.Set(field, convertValueForPayload(pl, field, parser.ReadArrayItem(item.itemType), "protobuf"), true);
 				++count;
 			}
 			builder.ArrayRef(item.tagName, field, count);
 		} else {
 			setValue(pl, builder, item);
 		}
+		validateArrayFieldRestrictions(f, reinterpret_cast<PayloadFieldValue::Array*>(pl.Field(field).p_)->len, "protobuf");
 	} else {
 		CJsonBuilder& array = arraysStorage_.GetArray(item.tagName);
 		if (packed) {

cpp_src/core/defnsconfigs.h

@@ -12,7 +12,7 @@ constexpr char kConfigNamespace[] = "#config";
 constexpr char kActivityStatsNamespace[] = "#activitystats";
 constexpr char kClientsStatsNamespace[] = "#clientsstats";
 constexpr char kNsNameField[] = "name";
-const std::vector<std::string> kDefDBConfig = {
+constexpr std::string_view kDefDBConfig[] = {
 	R"json({
 		"type":"profiling",
 		"profiling":{
@@ -90,7 +90,7 @@ const std::vector<std::string> kDefDBConfig = {
 		}
 	})json"};
 
-const std::vector<NamespaceDef> kSystemNsDefs = {
+const NamespaceDef kSystemNsDefs[] = {
 	NamespaceDef(kConfigNamespace, StorageOpts().Enabled().CreateIfMissing().DropOnFileFormatError())
 		.AddIndex("type", "hash", "string", IndexOpts().PK()),
 	NamespaceDef(kPerfStatsNamespace, StorageOpts())

cpp_src/core/ft/config/ftfastconfig.cc

@@ -131,8 +131,8 @@ void FtFastConfig::parse(std::string_view json, const RHashMap<std::string, int>
 		const std::string splitterStr = toLower(root["splitter"].As<std::string>("fast"));
 		if (splitterStr == "fast") {
 			splitterType = Splitter::Fast;
-		} else if (splitterStr == "friso") {
-			splitterType = Splitter::Friso;
+		} else if (splitterStr == "friso" || splitterStr == "mmseg_cn") {
+			splitterType = Splitter::MMSegCN;
 		} else {
 			throw Error(errParseJson, "FtFastConfig: unknown splitter value: %s", splitterStr);
 		}
@@ -185,8 +185,8 @@ std::string FtFastConfig::GetJson(const fast_hash_map<std::string, int>& fields)
 		case Splitter::Fast:
 			jsonBuilder.Put("splitter", "fast");
 			break;
-		case Splitter::Friso:
-			jsonBuilder.Put("splitter", "friso");
+		case Splitter::MMSegCN:
+			jsonBuilder.Put("splitter", "mmseg_cn");
 			break;
 	}
 

cpp_src/core/ft/config/ftfastconfig.h

@@ -53,7 +53,7 @@ struct FtFastConfig : public BaseFTConfig {
 	int maxAreasInDoc = 5;
 	int maxTotalAreasToCache = -1;
 
-	enum class Splitter { Fast, Friso } splitterType = Splitter::Fast;
+	enum class Splitter { Fast, MMSegCN } splitterType = Splitter::Fast;
 
 	RVector<FtFastFieldConfig, 8> fieldsCfg;
 	enum class Optimization { CPU, Memory } optimization = Optimization::Memory;

cpp_src/core/ft/ft_fast/dataholder.cc

@@ -132,7 +132,7 @@ DataHolder<IdCont>::DataHolder(FtFastConfig* c) {
 	cfg_ = c;
 	if (cfg_->splitterType == FtFastConfig::Splitter::Fast) {
 		splitter_ = make_intrusive<FastTextSplitter>(cfg_->extraWordSymbols);
-	} else if (cfg_->splitterType == FtFastConfig::Splitter::Friso) {
+	} else if (cfg_->splitterType == FtFastConfig::Splitter::MMSegCN) {
 		splitter_ = make_intrusive<FrisoTextSplitter>();
 	} else {
 		assertrx_throw(false);

cpp_src/core/ft/ft_fast/dataholder.h

@@ -50,17 +50,16 @@ class PackedWordEntry {
 class WordEntry {
 public:
 	WordEntry() noexcept = default;
-	WordEntry(const IdRelSet& _vids, bool _virtualWord) : vids(_vids), virtualWord(_virtualWord) {}
+	WordEntry(const IdRelSet& _vids) : vids_(_vids) {}
 	WordEntry(const WordEntry&) = delete;
 	WordEntry(WordEntry&&) noexcept = default;
 	WordEntry& operator=(const WordEntry&) = delete;
 	WordEntry& operator=(WordEntry&&) noexcept = default;
 
 	// Explicit copy
-	WordEntry MakeCopy() const { return WordEntry(this->vids, this->virtualWord); }
+	WordEntry MakeCopy() const { return WordEntry(vids_); }
 
-	IdRelSet vids;
-	bool virtualWord = false;
+	IdRelSet vids_;
 };
 enum ProcessStatus { FullRebuild, RecommitLast, CreateNew };