Support nullable field and default value (#440)

Signed-off-by: wayblink <[email protected]>

core/backup_impl_create_backup.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/golang/protobuf/proto"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/milvus-io/milvus-sdk-go/v2/entity"
 	"github.com/samber/lo"
@@ -236,7 +237,7 @@ func (b *BackupContext) backupCollectionPrepare(ctx context.Context, backupInfo
 	}
 	fields := make([]*backuppb.FieldSchema, 0)
 	for _, field := range completeCollection.Schema.Fields {
-		fields = append(fields, &backuppb.FieldSchema{
+		fieldBak := &backuppb.FieldSchema{
 			FieldID:        field.ID,
 			Name:           field.Name,
 			IsPrimaryKey:   field.PrimaryKey,
@@ -247,8 +248,18 @@ func (b *BackupContext) backupCollectionPrepare(ctx context.Context, backupInfo
 			IndexParams:    utils.MapToKVPair(field.IndexParams),
 			IsDynamic:      field.IsDynamic,
 			IsPartitionKey: field.IsPartitionKey,
+			Nullable:       field.Nullable,
 			ElementType:    backuppb.DataType(field.ElementType),
-		})
+		}
+		defaultValue := field.DefaultValue
+		if defaultValue != nil {
+			bytes, err := proto.Marshal(field.DefaultValue)
+			if err != nil {
+				return err
+			}
+			fieldBak.DefaultValueProto = string(bytes)
+		}
+		fields = append(fields, fieldBak)
 	}
 	schema := &backuppb.CollectionSchema{
 		Name:               completeCollection.Schema.CollectionName,

core/backup_impl_restore_backup.go

@@ -8,7 +8,9 @@ import (
 	"time"
 
 	"github.com/cockroachdb/errors"
+	"github.com/golang/protobuf/proto"
 	jsoniter "github.com/json-iterator/go"
+	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
 	gomilvus "github.com/milvus-io/milvus-sdk-go/v2/client"
 	"github.com/milvus-io/milvus-sdk-go/v2/entity"
 	"github.com/samber/lo"
@@ -420,7 +422,7 @@ func (b *BackupContext) executeRestoreCollectionTask(ctx context.Context, backup
 	fields := make([]*entity.Field, 0)
 	hasPartitionKey := false
 	for _, field := range task.GetCollBackup().GetSchema().GetFields() {
-		fields = append(fields, &entity.Field{
+		fieldRestore := &entity.Field{
 			ID:             field.GetFieldID(),
 			Name:           field.GetName(),
 			PrimaryKey:     field.GetIsPrimaryKey(),
@@ -431,8 +433,19 @@ func (b *BackupContext) executeRestoreCollectionTask(ctx context.Context, backup
 			IndexParams:    utils.KvPairsMap(field.GetIndexParams()),
 			IsDynamic:      field.GetIsDynamic(),
 			IsPartitionKey: field.GetIsPartitionKey(),
+			Nullable:       field.GetNullable(),
 			ElementType:    entity.FieldType(field.GetElementType()),
-		})
+		}
+		if field.DefaultValueProto != "" {
+			defaultValue := &schemapb.ValueField{}
+			err := proto.Unmarshal([]byte(field.DefaultValueProto), defaultValue)
+			if err != nil {
+				return nil, err
+			}
+			fieldRestore.DefaultValue = defaultValue
+		}
+
+		fields = append(fields, fieldRestore)
 		if field.GetIsPartitionKey() {
 			hasPartitionKey = true
 		}

core/proto/backup.proto

@@ -451,6 +451,8 @@ message FieldSchema {
   ValueField default_value = 11; // default_value only support scalars except array and json for now
   bool is_dynamic = 12; // mark whether this field is the dynamic field
   bool is_partition_key = 13; // enable logic partitions
+  bool nullable = 14;
+  string default_value_proto = 15; // json not support marshall oneof, so use proto string
 }
 
 /**

docs/docs.go

@@ -625,6 +625,9 @@ const docTemplate = `{
                 "default_value": {
                     "$ref": "#/definitions/backuppb.ValueField"
                 },
+                "default_value_proto": {
+                    "type": "string"
+                },
                 "description": {
                     "type": "string"
                 },
@@ -652,6 +655,9 @@ const docTemplate = `{
                 "name": {
                     "type": "string"
                 },
+                "nullable": {
+                    "type": "boolean"
+                },
                 "state": {
                     "$ref": "#/definitions/backuppb.FieldState"
                 },

docs/swagger.json

@@ -617,6 +617,9 @@
                 "default_value": {
                     "$ref": "#/definitions/backuppb.ValueField"
                 },
+                "default_value_proto": {
+                    "type": "string"
+                },
                 "description": {
                     "type": "string"
                 },
@@ -644,6 +647,9 @@
                 "name": {
                     "type": "string"
                 },
+                "nullable": {
+                    "type": "boolean"
+                },
                 "state": {
                     "$ref": "#/definitions/backuppb.FieldState"
                 },

docs/swagger.yaml

@@ -268,6 +268,8 @@ definitions:
         $ref: '#/definitions/backuppb.DataType'
       default_value:
         $ref: '#/definitions/backuppb.ValueField'
+      default_value_proto:
+        type: string
       description:
         type: string
       element_type:
@@ -286,6 +288,8 @@ definitions:
         type: boolean
       name:
         type: string
+      nullable:
+        type: boolean
       state:
         $ref: '#/definitions/backuppb.FieldState'
       type_params:

example/null_support/prepare_data.py

@@ -0,0 +1,31 @@
+from pymilvus import CollectionSchema, FieldSchema, Collection, connections, DataType, Partition, utility
+import numpy as np
+import random
+import time
+
+connections.connect()
+dim = 128
+int64_field = FieldSchema(name="int64", dtype=DataType.INT64, is_primary=True)
+double_field = FieldSchema(name="nullableFid", dtype=DataType.DOUBLE, nullable=True, is_primary=False, is_clustering_key=True)
+int32_field = FieldSchema(name="int32", dtype=DataType.INT64, default_value=10)
+string_field = FieldSchema(name="string", dtype=DataType.VARCHAR, max_length=1000, default_value="10")
+float_vector = FieldSchema(name="float_vector", dtype=DataType.FLOAT_VECTOR, dim=dim, mmap_enabled=True)
+schema = CollectionSchema(fields=[int64_field, double_field, int32_field, string_field, float_vector])
+utility.drop_collection("null_test")
+collection = Collection("null_test", schema=schema)
+res = collection.schema
+print(res)
+
+
+nb = 10000
+slice = 100
+for i in range(int(nb/slice)):
+    vectors = [[random.random() for _ in range(dim)] for _ in range(slice)]
+    data = [[j for j in range(i*slice,(i+1)*slice)], [None for _ in range(slice)],[], ["1" for _ in range(slice)], vectors]
+    collection.insert(data=data)
+    print("inserted %d %d" %(i, slice))
+
+
+collection.flush()
+res = collection.num_entities
+print(res)

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76
 	//github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3
-	github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240821030256-0c339b63d265
+	github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20241009111120-8008f14f2c16
 	github.com/minio/minio-go/v7 v7.0.61
 	github.com/pkg/errors v0.9.1
 	github.com/samber/lo v1.39.0
@@ -33,14 +33,14 @@ require (
 	go.uber.org/atomic v1.10.0
 	go.uber.org/zap v1.17.0
 	golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
-	golang.org/x/sync v0.3.0
+	golang.org/x/sync v0.8.0
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
 	google.golang.org/grpc v1.48.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
-require github.com/milvus-io/milvus-proto/go-api/v2 v2.4.10-0.20240819025435-512e3b98866a
+require github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240909041258-8f8ca67816cd
 
 require (
 	cloud.google.com/go v0.81.0 // indirect
@@ -112,5 +112,3 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
-
-// replace github.com/milvus-io/milvus-sdk-go/v2 => github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta4.0.20240814024750-1d41342d2c27

go.sum

@@ -378,10 +378,10 @@ github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpe
 github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.4.10-0.20240819025435-512e3b98866a h1:0B/8Fo66D8Aa23Il0yrQvg1KKz92tE/BJ5BvkUxxAAk=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.4.10-0.20240819025435-512e3b98866a/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
-github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240821030256-0c339b63d265 h1:HxVtvxRflwbsKcGyIEr+2FCktUmm5Ga0oLSq7k0Rlg0=
-github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240821030256-0c339b63d265/go.mod h1:hCubTFcavPtRXyW7CpspzczPvIfq9k/2GB3/qrJkPUA=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240909041258-8f8ca67816cd h1:x0b0+foTe23sKcVFseR1DE8+BB08EH6ViiRHaz8PEik=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240909041258-8f8ca67816cd/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
+github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20241009111120-8008f14f2c16 h1:zsA/xt/HJ3HkWUgazbbkoIR8S5mcxAtypvoSKsemhMw=
+github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20241009111120-8008f14f2c16/go.mod h1:TdzShm5isV4F6kvrd+9V/CKowNIX+H+jvybF0WWts0Y=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI=
@@ -699,8 +699,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=