Update to version v3.25.0

.github/workflows/test.yml

@@ -47,7 +47,7 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macos-latest]
+        os: [ubuntu-20.04, ubuntu-22.04, macos-13]
         include:
           - os: ubuntu-latest
             sanitizer: ASAN
@@ -91,7 +91,7 @@ jobs:
           make -j4
           STRIP=/bin/true cpack
       - name: 'C++ tests'
-        if: ${{ matrix.os == 'macos-latest' }}
+        if: ${{ matrix.os == 'macos-13' }}
         run: |
           echo "Running C++ directly in this job due to Action's problem with artifacts transition for macos-11 and macos-12 runners"
           cd build
@@ -111,7 +111,7 @@ jobs:
   test:
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macos-latest]
+        os: [ubuntu-20.04, ubuntu-22.04, macos-13]
         test: ['C++', 'GO']
         include:
           - os: ubuntu-latest
@@ -140,18 +140,24 @@ jobs:
       TEST: ${{matrix.test}}
     steps:
       - name: Checkout repository
-        if: ${{ matrix.os != 'macos-latest' || matrix.test == 'GO' }}
+        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
         uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        if: ${{ matrix.os == 'macos-13' && matrix.test == 'GO' }}
+        with:
+          go-version: '1.22.x'
+          check-latest: true
       - name: Download ${{matrix.os}}${{matrix.sanitizer}} Artifacts
-        if: ${{ matrix.os != 'macos-latest' || matrix.test == 'GO' }}
+        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
         uses: actions/download-artifact@v4
         with:
           name: ${{matrix.os}}${{matrix.sanitizer}}
       - name: 'Untar Artifacts'
-        if: ${{ matrix.os != 'macos-latest' || matrix.test == 'GO' }}
+        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
         run: tar -xvf artifacts.tar
       - name: Prepare Environment
-        if: ${{ matrix.os != 'macos-latest' || matrix.test == 'GO' }}
+        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
         env:
           OS: ${{matrix.os}}
         run: |
@@ -166,7 +172,7 @@ jobs:
             ./.github/workflows/install_gtest_parallel.sh
           fi
       - name: Tests
-        if: ${{ matrix.os != 'macos-latest' || matrix.test == 'GO' }}
+        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
         run: |
           if [[ $TEST == 'GO' ]]; then
             if [[ -z "${{matrix.sanitizer}}" ]]; then

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v3.24.0"
+const ReindexerVersion = "v3.25.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

changelog.md

@@ -1,3 +1,45 @@
+# Version 3.25.0 (30.05.2024)
+
+## Core
+- [fea] Added field `max_iterations_idset_preresult` into `#config`-namespace. This field allows to configure `preselect` limit for the `JOIN`-queries
+- [fea] Added `WHERE`-values deduplication for the composite indexes in `SELECT`-queries
+- [fea] Added scalar fields support for the `array_remove` and `array_remove_once` update-functions
+- [fix] Fixed whitespaces handling in the `UPDATE` SQL-queries with JSON-objects
+
+## Fulltext
+- [fea] Optimized fulltext index building (index rebuild works ~30% faster than in `v3.24.0`)
+- [fea] Added gujarati UTF-8 subset to the internal locale
+- [fix] Fixed snippet and highlight on the queries with large `merge_limit` (>65k) and typos
+
+## Replication
+- [fix] Fixed replicated `TagsMatcher`'s behavior on the follower-nodes after leader's `schema` modification
+- [fix] Fixed online replication for `DeleteMeta`-calls
+
+## Reindexer server
+- [fix] Fixed errors in [swagger.yml](https://github.com/Restream/reindexer/blob/master/cpp_src/server/contrib/server.yml)
+
+## CPP-client
+- [fix] Fixed `SetSchema`-call
+
+## Ported
+- [fea] Ported the comparators improvements/optimizations from [v4.15.0](https://github.com/Restream/reindexer/releases/tag/v4.15.0)
+
+## Face
+- [fea] Added UUID to the PK options
+- [fea] Added the "Minimum preselect size for optimization of inner join by injection of filters" field to the NS config 
+- [fea] Increased the max of the position_boost field
+- [fea] Changed the fonts to the system defaults
+- [fea] Moved the field descriptions to the tooltips on the DB Config page
+- [fix] Fixed the pagination issue after item insertion
+- [fix] Fixed the issue related to the Aggregation result displaying
+- [fix] Fixed icons and tabs layout on the NS page
+- [fix] Fixed Explain tab layout on the SQL result page
+- [fix] Fixed "Tags" input layout in the Index config modal 
+- [fix] Fixed placeholder layout for the "Forced sort values" field and filer condition on the Query Builder page
+- [fix] Fixed console issue that appeared on the SQL -> Explain query
+- [fix] Fixed incorrect message about the empty result of the Explain operation
+- [fix] Fixed Pin NS button
+
 # Version 3.24.0 (12.04.2024)
 
 ## Core

cpp_src/CMakeLists.txt

@@ -43,7 +43,7 @@ else()
   option (LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set (REINDEXER_VERSION_DEFAULT "3.24.0")
+set (REINDEXER_VERSION_DEFAULT "3.25.0")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")

cpp_src/client/coroqueryresults.cc

@@ -3,8 +3,6 @@
 #include "core/cjson/baseencoder.h"
 #include "core/keyvalue/p_string.h"
 #include "net/cproto/coroclientconnection.h"
-#include "server/rpcqrwatcher.h"
-#include "tools/logger.h"
 
 namespace reindexer {
 namespace client {

cpp_src/client/cororeindexer.cc

@@ -1,7 +1,6 @@
 #include "client/cororeindexer.h"
 #include "client/cororpcclient.h"
 #include "tools/cpucheck.h"
-#include "tools/logger.h"
 
 namespace reindexer {
 namespace client {

cpp_src/client/cororpcclient.cc

@@ -422,7 +422,9 @@ Error CoroRPCClient::SubscribeUpdates(IUpdatesObserver* observer, const UpdatesF
 }
 
 Error CoroRPCClient::UnsubscribeUpdates(IUpdatesObserver* observer) {
-	observers_.Delete(observer);
+	if (auto err = observers_.Delete(observer); !err.ok()) {
+		return err;
+	}
 	return subscribeImpl(!observers_.Empty());
 }
 
@@ -546,11 +548,17 @@ void CoroRPCClient::resubRoutine() {
 		loop_->granular_sleep(kSubscriptionCheckInterval, kCoroSleepGranularity, [this] { return terminate_; });
 		if (subscribed_) {
 			if (observers_.Empty()) {
-				subscribeImpl(false);
+				auto err = subscribeImpl(false);
+				if (!err.ok()) {
+					logPrintf(LogError, "[RPC-client]: Unsub error (observers array is empty): %s", err.what());
+				}
 			}
 		} else {
 			if (!observers_.Empty()) {
-				subscribeImpl(true);
+				auto err = subscribeImpl(true);
+				if (!err.ok()) {
+					logPrintf(LogError, "[RPC-client]: Resub error (observers array is empty): %s", err.what());
+				}
 			}
 		}
 	}

cpp_src/client/cororpcclient.h

@@ -8,15 +8,13 @@
 #include "client/item.h"
 #include "client/namespace.h"
 #include "client/reindexerconfig.h"
-#include "core/keyvalue/p_string.h"
 #include "core/namespacedef.h"
 #include "core/query/query.h"
 #include "coroutine/waitgroup.h"
 #include "estl/fast_hash_map.h"
 #include "net/cproto/coroclientconnection.h"
 #include "replicator/updatesobserver.h"
 #include "tools/errors.h"
-#include "urlparser/urlparser.h"
 
 namespace reindexer {
 

cpp_src/client/item.h

@@ -38,7 +38,7 @@ class Item {
 	/// If Item is in *Unsafe Mode*, then Item will not store slice, but just keep pointer to data in slice,
 	/// application *MUST* hold slice until end of life of Item
 	/// @param slice - data slice with CJson
-	[[nodiscard]] Error FromCJSON(std::string_view slice) &noexcept;
+	Error FromCJSON(std::string_view slice) &noexcept;
 	void FromCJSONImpl(std::string_view slice) &;
 	/// Serialize item to CJSON.<br>
 	/// If Item is in *Unfafe Mode*, then returned slice is allocated in temporary buffer, and can be invalidated by any next operation with

cpp_src/client/itemimpl.h

@@ -3,10 +3,8 @@
 #include <deque>
 #include <vector>
 #include "core/cjson/tagsmatcher.h"
-#include "core/keyvalue/key_string.h"
 #include "core/keyvalue/variant.h"
 #include "core/payload/payloadiface.h"
-#include "gason/gason.h"
 #include "tools/serializer.h"
 
 namespace reindexer {

cpp_src/client/queryresults.cc

@@ -3,7 +3,6 @@
 #include "core/cjson/baseencoder.h"
 #include "core/keyvalue/p_string.h"
 #include "net/cproto/clientconnection.h"
-#include "tools/logger.h"
 
 namespace reindexer {
 namespace client {

cpp_src/client/reindexer.cc

@@ -1,6 +1,5 @@
 #include "client/reindexer.h"
 #include "client/rpcclient.h"
-#include "tools/logger.h"
 
 namespace reindexer {
 namespace client {

cpp_src/client/resultserializer.cc

@@ -1,6 +1,5 @@
 #include "resultserializer.h"
 #include "core/payload/payloadtypeimpl.h"
-#include "vendor/msgpack/msgpack.h"
 
 namespace reindexer {
 namespace client {
@@ -39,14 +38,18 @@ void ResultSerializer::GetRawQueryParams(ResultSerializer::QueryParams& ret, con
 		}
 		std::string_view data = GetSlice();
 		switch (tag) {
-			case QueryResultAggregation:
-				ret.aggResults.push_back({});
+			case QueryResultAggregation: {
+				auto& aggRes = ret.aggResults.emplace_back();
+				Error err;
 				if ((ret.flags & kResultsFormatMask) == kResultsMsgPack) {
-					ret.aggResults.back().FromMsgPack(giftStr(data));
+					err = aggRes.FromMsgPack(giftStr(data));
 				} else {
-					ret.aggResults.back().FromJSON(giftStr(data));
+					err = aggRes.FromJSON(giftStr(data));
 				}
-				break;
+				if (!err.ok()) {
+					throw err;
+				}
+			} break;
 			case QueryResultExplain:
 				ret.explainResults = std::string(data);
 				break;

cpp_src/client/rpcclient.cc

@@ -291,11 +291,14 @@ Error RPCClient::modifyItemAsync(std::string_view nsName, Item* item, int mode,
 					// Rebuild item with new state
 					auto newItem = NewItem(ns);
 					Error err = newItem.FromJSON(item->impl_->GetJSON());
+					if (err.ok()) return ctx.cmpl()(err);
 					newItem.SetPrecepts(item->impl_->GetPrecepts());
 					*item = std::move(newItem);
-					modifyItemAsync(ns, item, mode, conn, timeout, ctx);
+					err = modifyItemAsync(ns, item, mode, conn, timeout, ctx);
+					if (err.ok()) return ctx.cmpl()(err);
 				});
-				selectImpl(Query(ns).Limit(0), *qr, conn, netTimeout, ctxCmpl);
+				auto err = selectImpl(Query(ns).Limit(0), *qr, conn, netTimeout, ctxCmpl);
+				if (err.ok()) return ctx.cmpl()(err);
 			} else
 				try {
 					auto args = ret.GetArgs(2);
@@ -593,7 +596,9 @@ Error RPCClient::SubscribeUpdates(IUpdatesObserver* observer, const UpdatesFilte
 }
 
 Error RPCClient::UnsubscribeUpdates(IUpdatesObserver* observer) {
-	observers_.Delete(observer);
+	if (auto err = observers_.Delete(observer); !err.ok()) {
+		return err;
+	}
 	return subscribeImpl(!observers_.Empty());
 }
 
@@ -729,34 +734,41 @@ void RPCClient::onUpdates(net::cproto::RPCAnswer& ans, cproto::ClientConnection*
 			delayedUpdates_.emplace_back(std::move(ans));
 
 			QueryResults* qr = new QueryResults;
-			Select(Query(std::string(nsName)).Limit(0), *qr,
-				   InternalRdxContext(
-					   nullptr,
-					   [this, qr, conn](const Error& err) {
-						   delete qr;
-						   // If there are delayed updates then send them to client
-						   auto uq = std::move(delayedUpdates_);
-						   delayedUpdates_.clear();
-
-						   if (!err.ok() || serialDelays_ > 1) {
-							   // This update was already dealyed, but was not able to synchronize tagsmatcher.
-							   // Such situation usually means, that master's namespace was recreated and must be synchronized via force
-							   // sync.
-							   // Current fix is suboptimal and in some cases even incorrect (but still better, than previous
-							   // implementation) - proper fix requires some versioning info about namespaces, which exists
-							   // in v4 only
-							   std::string_view nsName(std::string_view(uq.front().GetArgs(1)[1]));
-							   logPrintf(
-								   LogWarning,
-								   "[repl:%s] Unable to sync tags matcher via online-replication (err: '%s'). Calling UpdatesLost fallback",
-								   nsName, err.what());
-							   serialDelays_ = 0;
-							   observers_.OnUpdatesLost(nsName);
-						   } else {
-							   for (auto& a1 : uq) onUpdates(a1, conn);
-						   }
-					   }),
-				   conn);
+			auto err = Select(
+				Query(std::string(nsName)).Limit(0), *qr,
+				InternalRdxContext(
+					nullptr,
+					[this, qr, conn](const Error& err) {
+						delete qr;
+						// If there are delayed updates then send them to client
+						auto uq = std::move(delayedUpdates_);
+						delayedUpdates_.clear();
+
+						if (!err.ok() || serialDelays_ > 1) {
+							// This update was already delayed, but was not able to synchronize tagsmatcher.
+							// Such situation usually means, that master's namespace was recreated and must be synchronized via force
+							// sync.
+							// Current fix is suboptimal and in some cases even incorrect (but still better, than previous
+							// implementation) - proper fix requires some versioning info about namespaces, which exists
+							// in v4 only
+							std::string_view nsName(std::string_view(uq.front().GetArgs(1)[1]));
+							logPrintf(
+								LogWarning,
+								"[repl:%s] Unable to sync tags matcher via online-replication (err: '%s'). Calling UpdatesLost fallback",
+								nsName, err.what());
+							serialDelays_ = 0;
+							observers_.OnUpdatesLost(nsName);
+						} else {
+							for (auto& a1 : uq) onUpdates(a1, conn);
+						}
+					}),
+				conn);
+			if (!err.ok()) {
+				logPrintf(LogWarning,
+						  "[repl:%s] Unable to sync tags matcher via online-replication (select err: '%s'). Calling UpdatesLost fallback",
+						  err.what());
+				observers_.OnUpdatesLost(nsName);
+			}
 			return;
 		} else {
 			// We have bundled tagsMatcher

cpp_src/client/rpcclient.h

@@ -12,15 +12,13 @@
 #include "client/queryresults.h"
 #include "client/reindexerconfig.h"
 #include "client/transaction.h"
-#include "core/keyvalue/p_string.h"
 #include "core/namespacedef.h"
 #include "core/query/query.h"
 #include "estl/fast_hash_map.h"
 #include "estl/shared_mutex.h"
 #include "net/cproto/clientconnection.h"
 #include "replicator/updatesobserver.h"
 #include "tools/errors.h"
-#include "urlparser/urlparser.h"
 
 namespace reindexer {