CBL-5558 : Vector Search Tests (#529)

* CBL-5558 : Vector Search Tests * Wrote vector search tests per the test spec. * Disabled the vector search tests from running automatically as we need the infrastructure in place for that. * Updated Predictive Model API to return FLSliceResult instead of FLDict for CBL-C. This will be different from the other plaforms. The main result is that when returnning a dict, users do not have a way to know when it’s time to release any nested dicts/arrays created for their output dictionary. LiteCore also has the same API for the same reason. * Fixed PQ Encoding bugs.
couchbase · Apr 11, 2024 · 812a552 · 812a552
1 parent 2def36d
commit 812a552
Show file tree

Hide file tree

Showing 14 changed files with 1,458 additions and 47 deletions.
diff --git a/include/cbl/CBLPrediction.h b/include/cbl/CBLPrediction.h
@@ -31,13 +31,13 @@ typedef struct {
     /** Called from within a query (or document indexing) to run the prediction.
         @param context  The value of the CBLPredictiveModel's `context` field.
         @param input  The input dictionary from the query.
-        @return The output dictionary of the prediction function or NULL if there is no output.
-        @note The output dictionary will be automatically released after it's being consumed.
+        @return The output of the prediction function, encoded as a Fleece dictionary, or null if there is no output.
+        @note A null \ref FLSliceResult can be created by calling FLSliceResult_CreateWith(nullptr, 0).
         @warning This function must be "pure": given the same input parameters it must always
-                 produce the same output (otherwise indexes or queries may be messed up).
-                 It MUST NOT alter the database or any documents, nor run a query: either of
-                 those are very likely to cause a crash. */
-    FLDict _cbl_nullable (* _cbl_nonnull prediction)(void* _cbl_nullable context, FLDict input);
+                produce the same output (otherwise indexes or queries may be messed up).
+                It MUST NOT alter the database or any documents, nor run a query: either of
+                those are very likely to cause a crash. */
+    FLSliceResult (* _cbl_nonnull prediction)(void* _cbl_nullable context, FLDict input);
 
     /** Called if the model is unregistered, so it can release resources. */
     void (*_cbl_nullable unregistered)(void* context);

diff --git a/include/cbl/CouchbaseLite.h b/include/cbl/CouchbaseLite.h
@@ -26,6 +26,7 @@
 #include "CBLEncryptable.h"
 #include "CBLLog.h"
 #include "CBLPlatform.h"
+#include "CBLPrediction.h"
 #include "CBLQuery.h"
 #include "CBLReplicator.h"
 #include "CBLScope.h"
diff --git a/src/CBLPrediction.cc b/src/CBLPrediction.cc
@@ -28,8 +28,7 @@ namespace cbl_internal {
     void PredictiveModel::registerModel(const slice name, const CBLPredictiveModel& model) {
         auto prediction = [](void* context, FLDict input, C4Database *db, C4Error *outError) {
             auto m = (PredictiveModel*)context;
-            FLDict output = m->_model.prediction(m->_model.context, input);
-            return C4SliceResult(m->encodeOutput(Dict(output)));
+            return m->_model.prediction(m->_model.context, input);
         };
 
         auto unregistered = [](void* context) {
@@ -55,16 +54,6 @@ namespace cbl_internal {
         auto nameStr = name.asString();
         c4pred_unregisterModel(nameStr.c_str());
     }
-
-    alloc_slice PredictiveModel::encodeOutput(Dict dict) {
-        if (!dict) {
-            return nullslice;
-        }
-
-        Encoder enc;
-        enc.writeValue(dict);
-        return enc.finish();
-    }
 }
 
 #endif
diff --git a/src/CBLVectorIndexConfig.hh b/src/CBLVectorIndexConfig.hh
@@ -71,7 +71,7 @@ namespace cbl_internal {
         }
 
     private:
-        C4VectorEncoding _encoding { kC4VectorEncodingSQ };
+        C4VectorEncoding _encoding { kC4VectorEncodingPQ };
     };
 }
 

diff --git a/src/exports/CBL_EE_Exports.txt b/src/exports/CBL_EE_Exports.txt
@@ -27,13 +27,16 @@ CBLEncryptable_CreateWithDict
 CBLEncryptable_Properties
 CBLEncryptable_Value
 
-FLDict_IsEncryptableValue
-FLDict_GetEncryptableValue
-FLSlot_SetEncryptableValue
+CBL_RegisterPredictiveModel
+CBL_UnregisterPredictiveModel
 
 CBL_SetExtensionPath
 CBLCollection_CreateVectorIndex
 CBLVectorEncoding_CreateNone
 CBLVectorEncoding_CreateProductQuantizer
 CBLVectorEncoding_CreateScalarQuantizer
 CBLVectorEncoding_Free
+
+FLDict_IsEncryptableValue
+FLDict_GetEncryptableValue
+FLSlot_SetEncryptableValue
diff --git a/src/exports/generated/CBL_EE.def b/src/exports/generated/CBL_EE.def
@@ -20,15 +20,17 @@ CBLEncryptable_CreateWithArray
 CBLEncryptable_CreateWithDict
 CBLEncryptable_Properties
 CBLEncryptable_Value
-FLDict_IsEncryptableValue
-FLDict_GetEncryptableValue
-FLSlot_SetEncryptableValue
+CBL_RegisterPredictiveModel
+CBL_UnregisterPredictiveModel
 CBL_SetExtensionPath
 CBLCollection_CreateVectorIndex
 CBLVectorEncoding_CreateNone
 CBLVectorEncoding_CreateProductQuantizer
 CBLVectorEncoding_CreateScalarQuantizer
 CBLVectorEncoding_Free
+FLDict_IsEncryptableValue
+FLDict_GetEncryptableValue
+FLSlot_SetEncryptableValue
 CBL_Retain
 CBL_Release
 CBL_InstanceCount

diff --git a/src/exports/generated/CBL_EE.exp b/src/exports/generated/CBL_EE.exp
@@ -28,15 +28,17 @@ _CBLEncryptable_CreateWithArray
 _CBLEncryptable_CreateWithDict
 _CBLEncryptable_Properties
 _CBLEncryptable_Value
-_FLDict_IsEncryptableValue
-_FLDict_GetEncryptableValue
-_FLSlot_SetEncryptableValue
+_CBL_RegisterPredictiveModel
+_CBL_UnregisterPredictiveModel
 _CBL_SetExtensionPath
 _CBLCollection_CreateVectorIndex
 _CBLVectorEncoding_CreateNone
 _CBLVectorEncoding_CreateProductQuantizer
 _CBLVectorEncoding_CreateScalarQuantizer
 _CBLVectorEncoding_Free
+_FLDict_IsEncryptableValue
+_FLDict_GetEncryptableValue
+_FLSlot_SetEncryptableValue
 _CBL_Retain
 _CBL_Release
 _CBL_InstanceCount

diff --git a/src/exports/generated/CBL_EE.gnu b/src/exports/generated/CBL_EE.gnu
@@ -18,15 +18,17 @@ CBL_C {
 		CBLEncryptable_CreateWithDict;
 		CBLEncryptable_Properties;
 		CBLEncryptable_Value;
-		FLDict_IsEncryptableValue;
-		FLDict_GetEncryptableValue;
-		FLSlot_SetEncryptableValue;
+		CBL_RegisterPredictiveModel;
+		CBL_UnregisterPredictiveModel;
 		CBL_SetExtensionPath;
 		CBLCollection_CreateVectorIndex;
 		CBLVectorEncoding_CreateNone;
 		CBLVectorEncoding_CreateProductQuantizer;
 		CBLVectorEncoding_CreateScalarQuantizer;
 		CBLVectorEncoding_Free;
+		FLDict_IsEncryptableValue;
+		FLDict_GetEncryptableValue;
+		FLSlot_SetEncryptableValue;
 		CBL_Retain;
 		CBL_Release;
 		CBL_InstanceCount;

diff --git a/src/exports/generated/CBL_EE_Android.gnu b/src/exports/generated/CBL_EE_Android.gnu
@@ -19,15 +19,17 @@ CBL_C {
 		CBLEncryptable_CreateWithDict;
 		CBLEncryptable_Properties;
 		CBLEncryptable_Value;
-		FLDict_IsEncryptableValue;
-		FLDict_GetEncryptableValue;
-		FLSlot_SetEncryptableValue;
+		CBL_RegisterPredictiveModel;
+		CBL_UnregisterPredictiveModel;
 		CBL_SetExtensionPath;
 		CBLCollection_CreateVectorIndex;
 		CBLVectorEncoding_CreateNone;
 		CBLVectorEncoding_CreateProductQuantizer;
 		CBLVectorEncoding_CreateScalarQuantizer;
 		CBLVectorEncoding_Free;
+		FLDict_IsEncryptableValue;
+		FLDict_GetEncryptableValue;
+		FLSlot_SetEncryptableValue;
 		CBL_Retain;
 		CBL_Release;
 		CBL_InstanceCount;

diff --git a/test/CBLTest.cc b/test/CBLTest.cc
@@ -108,7 +108,7 @@ alloc_slice CBLTest::databaseDir() {
     return sDatabaseDir;
 }
 
-slice const CBLTest::kDatabaseName = "CBLtest";
+slice const CBLTest::kDatabaseName = "CBLTest";
 
 CBLDatabaseConfiguration CBLTest::databaseConfig() {
     // One-time setup:
@@ -202,7 +202,7 @@ void CBLTest_Cpp::createDocumentInDefault(std::string docID, std::string propert
 
 #pragma mark - Test Utils C :
 
-string GetTestFilePath(const std::string &filename) {
+string GetAssetFilePath(const std::string &filename) {
     static string sTestFilesPath;
     if (sTestFilesPath.empty()) {
 #ifdef __APPLE__
@@ -284,7 +284,7 @@ unsigned ImportJSONLines(string filename, CBLDatabase* database) {
 }
 
 unsigned ImportJSONLines(string filename, CBLCollection* collection) {
-    auto path = GetTestFilePath(filename);
+    auto path = GetAssetFilePath(filename);
     CBL_Log(kCBLLogDomainDatabase, kCBLLogInfo, "Reading %s ...  ", path.c_str());
     CBLError error {};
     unsigned numDocs = 0;
@@ -405,6 +405,21 @@ void PurgeAllDocs(CBLCollection* collection) {
     CBLQuery_Release(query);
 }
 
+CBLQuery* CreateQuery(CBLDatabase* db, std::string sql) {
+    int errPos;
+    CBLError error {};
+    auto query = CBLDatabase_CreateQuery(db, kCBLN1QLLanguage, slice(sql), &errPos, &error);
+    REQUIRE(query);
+    return query;
+}
+
+int CountResults(CBLResultSet *results) {
+    int n = 0;
+    while (CBLResultSet_Next(results))
+        ++n;
+    return n;
+}
+
 #pragma mark - Test Utils C++:
 
 void createNumberedDocsWithPrefix(cbl::Collection& collection, unsigned n, const std::string& idprefix, unsigned start) {

diff --git a/test/CBLTest.hh b/test/CBLTest.hh
@@ -93,7 +93,7 @@ public:
     CBLCollection *defaultCollection {nullptr};
 };
 
-std::string GetTestFilePath(const std::string &filename);
+std::string GetAssetFilePath(const std::string &filename);
 
 #ifdef COUCHBASE_ENTERPRISE
 std::string GetExtensionPath();
@@ -123,6 +123,9 @@ void createNumberedDocsWithPrefix(CBLCollection *col, unsigned n, const std::str
 
 void PurgeAllDocs(CBLCollection* collection);
 
+CBLQuery* CreateQuery(CBLDatabase* db, std::string sql);
+
+int CountResults(CBLResultSet *results);
 
 // RAII utility to suppress reporting C++ exceptions (or breaking at them, in the Xcode debugger.)
 // Declare an instance when testing something that's expected to throw an exception internally.
-Original file line number
+Diff line change
@@ Expand Up / @@ -71,7 +71,7 @@ namespace cbl_internal { @@
             }
         private:
-            C4VectorEncoding _encoding { kC4VectorEncodingSQ };
+            C4VectorEncoding _encoding { kC4VectorEncodingPQ };
         };
     }
@@ Expand Down @@