sync code (#263)

secretflow · Apr 16, 2024 · df9c78f · df9c78f
1 parent b234298
commit df9c78f
Show file tree

Hide file tree

Showing 31 changed files with 525 additions and 340 deletions.
diff --git a/.circleci/regtest-config.yml b/.circleci/regtest-config.yml
@@ -51,6 +51,7 @@ jobs:
             mkdir -p image
             cp docker/scql-ubuntu.Dockerfile image
             docker run -d -it --name scql-dev -v $(pwd):/home/admin/dev/ -w /home/admin/dev --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true --entrypoint=bash secretflow/scql-ci:latest
+            docker exec -it scql-dev bash -c "pip install numpy"
             docker exec -it scql-dev bash -c "make && bazel build //engine/exe:scqlengine -c opt --ui_event_filters=-info,-debug,-warning" 
 
             MACHINE_TYPE=`arch`

diff --git a/.circleci/release-config.yml b/.circleci/release-config.yml
@@ -57,6 +57,7 @@ jobs:
       - run:
           name: "Build"
           command: |
+            pip install numpy
             make && bazel build //engine/exe:scqlengine -c opt --ui_event_filters=-info,-debug,-warning
       - run:
           name: "Copy binary"

diff --git a/.circleci/unittest-config.yml b/.circleci/unittest-config.yml
@@ -70,6 +70,7 @@ jobs:
           command: |
             set +e
             declare -i test_status
+            pip install numpy
             bazel test //engine/... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors | tee test_result.log; test_status=${PIPESTATUS[0]}
 
             sh ../devtools/rename_junit_xml.sh
@@ -129,7 +130,7 @@ jobs:
           command: |
             set +e
             declare -i test_status
-
+            pip install numpy
             bazel test //engine/... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors| tee test_result.log; test_status=${PIPESTATUS[0]}
 
             sh ../devtools/rename_junit_xml.sh

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Staging
 
+## [0.6.0] - 2024-04-15
+
 ### Added
 
 - Support for RSA key pairs in SCQLBroker.
@@ -33,7 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
- - Added support for HTTP data source router.
+ - Added support for HTTP data source router. 
 
 ### Changed
 

diff --git a/README.md b/README.md
@@ -37,6 +37,10 @@ docker exec -it scql-dev-$(whoami) bash
 
 
 ```sh
+# prerequisite
+# spu needs numpy
+pip install numpy
+
 # build SCQL engine as release
 bazel build //engine/exe:scqlengine -c opt
 

diff --git a/WORKSPACE b/WORKSPACE
@@ -39,21 +39,9 @@ load("@psi//bazel:repositories.bzl", "psi_deps")
 
 psi_deps()
 
-# grpc
-load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
-
-grpc_deps()
-
-# Not mentioned in official docs... mentioned here https://github.com/grpc/grpc/issues/20511
-load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
-
-grpc_extra_deps()
-
-load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains")
-
-rules_proto_grpc_toolchains()
+load("@rules_python//python:repositories.bzl", "py_repositories")
 
-rules_proto_grpc_repos()
+py_repositories()
 
 #
 # heu
@@ -62,10 +50,6 @@ load("@com_alipay_sf_heu//third_party/bazel_cpp:repositories.bzl", "heu_cpp_deps
 
 heu_cpp_deps()
 
-load("@rules_python//python:repositories.bzl", "py_repositories")
-
-py_repositories()
-
 load(
     "@rules_foreign_cc//foreign_cc:repositories.bzl",
     "rules_foreign_cc_dependencies",
@@ -77,21 +61,40 @@ rules_foreign_cc_dependencies(
     register_preinstalled_tools = True,
 )
 
-load("@xla//third_party/llvm:workspace.bzl", llvm = "repo")
+load("@xla//:workspace4.bzl", "xla_workspace4")
+
+xla_workspace4()
+
+load("@xla//:workspace3.bzl", "xla_workspace3")
 
-llvm("llvm-raw")
+xla_workspace3()
 
-load("@xla//third_party/llvm:setup.bzl", "llvm_setup")
+load("@xla//:workspace2.bzl", "xla_workspace2")
 
-llvm_setup("llvm-project")
+xla_workspace2()
+
+load("@xla//:workspace1.bzl", "xla_workspace1")
+
+xla_workspace1()
+
+load("@xla//:workspace0.bzl", "xla_workspace0")
+
+xla_workspace0()
+
+# grpc
+load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
+
+grpc_deps()
+
+load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains")
+
+rules_proto_grpc_toolchains()
+
+rules_proto_grpc_repos()
 
 #
 # boost
 #
 load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps")
 
 boost_deps()
-
-
-
-
diff --git a/docker/build.sh b/docker/build.sh
@@ -126,6 +126,8 @@ version=$(grep "version" $SCRIPT_DIR/version.txt | awk -F'"' '{print $2}')
 version+=$(date '+%Y%m%d-%H:%M:%S')
 version+=".$(git rev-parse --short HEAD)"
 echo "binary version: ${version}"
+# install deps
+docker exec -it ${container_id} bash -c "pip install numpy"
 # build engine binary
 docker exec -it ${container_id} bash -c "cd /home/admin/dev && sed -i "s/SCQL_VERSION/$version/g" engine/exe/version.h && bazel build //engine/exe:scqlengine -c opt"
 # build scdbserver + scdbclient binary

diff --git a/docker/version.txt b/docker/version.txt
@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version = "0.5.0b2"
+version = "0.6.0b1"
diff --git a/docs/api/broker.swagger.yaml b/docs/api/broker.swagger.yaml
@@ -1481,6 +1481,25 @@ definitions:
     properties:
       status:
         $ref: '#/definitions/scql.pb.Status'
+  spu.CheetahConfig:
+    type: object
+    properties:
+      disable_matmul_pack:
+        type: boolean
+        title: disable the ciphertext packing for matmul
+      enable_mul_lsb_error:
+        type: boolean
+        title: allow least significant bits error for point-wise mul
+      ot_kind:
+        $ref: '#/definitions/spu.CheetahOtKind'
+        title: Setup for cheetah ot
+  spu.CheetahOtKind:
+    type: string
+    enum:
+      - YACL_Ferret
+      - YACL_Softspoken
+      - EMP_Ferret
+    default: YACL_Ferret
   spu.FieldType:
     type: string
     enum:
@@ -1637,6 +1656,9 @@ definitions:
       ttp_beaver_config:
         $ref: '#/definitions/spu.TTPBeaverConfig'
         description: TrustedThirdParty configs.
+      cheetah_2pc_config:
+        $ref: '#/definitions/spu.CheetahConfig'
+        description: Cheetah 2PC configs.
       trunc_allow_msb_error:
         type: boolean
         description: |-
@@ -1695,13 +1717,15 @@ definitions:
       - LOG_DEFAULT
       - LOG_PADE
       - LOG_NEWTON
+      - LOG_MINMAX
     default: LOG_DEFAULT
     description: |-
       The logarithm approximation method.
 
        - LOG_DEFAULT: Implementation defined.
        - LOG_PADE: The pade approximation.
        - LOG_NEWTON: The newton approximation.
+       - LOG_MINMAX: The minmax approximation.
   spu.RuntimeConfig.SigmoidMode:
     type: string
     enum:

diff --git a/engine/bazel/engine_deps.bzl b/engine/bazel/engine_deps.bzl
@@ -40,56 +40,56 @@ def engine_deps():
     _org_postgres()
 
 def _secretflow_deps():
-    SPU_COMMIT = "3ccae529b70b37f7312e1171eab7e5acc16ff136"
-
-    #PSI_COMMIT = "47aaeb9c9ccd92deb8c150b2d55302ac0070f723"
-    #HEU_COMMIT = "afa15a0ad009cb5d5e40bd1dce885b9e4d472083"
-    KUSCIA_COMMIT = "1979d1f4f17db5c2bd6c57be7a690e88fa9ce7ed"
+    # SPU_COMMIT = "3ccae529b70b37f7312e1171eab7e5acc16ff136"
+    # PSI_COMMIT = "47aaeb9c9ccd92deb8c150b2d55302ac0070f723"
+    # HEU_COMMIT = "afa15a0ad009cb5d5e40bd1dce885b9e4d472083"
+    # KUSCIA_COMMIT = "1979d1f4f17db5c2bd6c57be7a690e88fa9ce7ed"
 
     maybe(
         http_archive,
         name = "spulib",
         urls = [
-            "https://github.com/secretflow/spu/archive/%s.tar.gz" % SPU_COMMIT,
-            # "https://github.com/secretflow/spu/archive/refs/tags/0.8.0dev20240112.tar.gz",
+            # "https://github.com/secretflow/spu/archive/%s.tar.gz" % SPU_COMMIT,
+            "https://github.com/secretflow/spu/archive/refs/tags/0.9.0b1.tar.gz",
         ],
-        strip_prefix = "spu-%s" % SPU_COMMIT,
-        # strip_prefix = "spu-0.8.0dev20240112",
-        sha256 = "33131793625b68775af03df6ac11eb3af45c34a71b9f1bb258e86487d0d20841",
+        # strip_prefix = "spu-%s" % SPU_COMMIT,
+        strip_prefix = "spu-0.9.0b1",
+        sha256 = "1f6220a43dda542b15b7673d8a03b10ef9f62d63053e33ef2182b6c61f498a1f",
     )
+
     maybe(
         http_archive,
         name = "psi",
         urls = [
             #"https://github.com/secretflow/psi/archive/%s.tar.gz" % PSI_COMMIT,
-            "https://github.com/secretflow/psi/archive/refs/tags/v0.3.0.dev240222.tar.gz",
+            "https://github.com/secretflow/psi/archive/refs/tags/v0.4.0.dev240401.tar.gz",
         ],
-        #strip_prefix = "psi-%s" % PSI_COMMIT,
-        strip_prefix = "psi-0.3.0.dev240222",
-        sha256 = "a7319040510a1581741f05ac4b31e3d887ba8ba4766154736f96d76970d00de5",
+        # strip_prefix = "psi-%s" % PSI_COMMIT,
+        strip_prefix = "psi-0.4.0.dev240401",
+        sha256 = "bc91e5c635fc94f865004e61e3896eb334d76549c1125fbc98caf8c6b3a82463",
     )
 
     maybe(
         http_archive,
         name = "com_alipay_sf_heu",
         urls = [
-            #"https://github.com/secretflow/heu/archive/%s.tar.gz" % HEU_COMMIT,
-            "https://github.com/secretflow/heu/archive/tags/v0.5.0b0.tar.gz",
+            # "https://github.com/secretflow/heu/archive/%s.tar.gz" % HEU_COMMIT,
+            "https://github.com/secretflow/heu/archive/tags/0.5.1.dev20240402.tar.gz",
         ],
-        #strip_prefix = "heu-%s" % HEU_COMMIT,
-        strip_prefix = "heu-tags-v0.5.0b0",
-        sha256 = "ca9391e16cc84e4a763f953f997d42530e75fc89ae26e345b2d3ec3e9fc63364",
+        # strip_prefix = "heu-%s" % HEU_COMMIT,
+        strip_prefix = "heu-tags-0.5.1.dev20240402",
+        sha256 = "3a672039b559edbe3133025880074207829720a06b57183a7a3d25a5ec84a84b",
     )
 
     maybe(
         http_archive,
         name = "kuscia",
         urls = [
-            "https://github.com/secretflow/kuscia/archive/%s.tar.gz" % KUSCIA_COMMIT,
-            # "https://github.com/secretflow/kuscia/archive/refs/tags/0.6.0.dev240115.tar.gz",
+            # "https://github.com/secretflow/kuscia/archive/%s.tar.gz" % KUSCIA_COMMIT,
+            "https://github.com/secretflow/kuscia/archive/refs/tags/v0.7.0b0.tar.gz",
         ],
-        strip_prefix = "kuscia-%s" % KUSCIA_COMMIT,
-        sha256 = "96aae6c878b7567455fe97ac96e8fd42d180e30cbd10de18dd86e12ac083b346",
+        strip_prefix = "kuscia-0.7.0b0",
+        sha256 = "76e396f9b148ec741e3c938d0a54ce9e91709466254d2f6effc8a4d50a77ff97",
     )
 
 def _org_apache_arrow():

diff --git a/engine/core/type.cc b/engine/core/type.cc
@@ -106,7 +106,7 @@ spu::PtType ArrowDataTypeToSpuPtType(
   spu::PtType pt;
   switch (dtype->id()) {
     case arrow::Type::BOOL:
-      pt = spu::PT_BOOL;
+      pt = spu::PT_I1;
       break;
     case arrow::Type::UINT8:
       pt = spu::PT_U8;
@@ -177,7 +177,7 @@ std::shared_ptr<arrow::DataType> SpuPtTypeToArrowDataType(spu::PtType pt_type) {
     case spu::PT_F64:
       dt = arrow::float64();
       break;
-    case spu::PT_BOOL:
+    case spu::PT_I1:
       dt = arrow::boolean();
       break;
     default:
@@ -204,7 +204,7 @@ spu::PtType DataTypeToSpuPtType(pb::PrimitiveDataType dtype) {
       pt = spu::PT_I64;
       break;
     case pb::PrimitiveDataType::BOOL:
-      pt = spu::PT_BOOL;
+      pt = spu::PT_I1;
       break;
     case pb::PrimitiveDataType::FLOAT32:
       pt = spu::PT_F32;

diff --git a/engine/operator/in.cc b/engine/operator/in.cc
@@ -32,7 +32,7 @@
 #include "psi/ecdh/ecdh_oprf_psi.h"
 #include "psi/ecdh/ecdh_psi.h"
 #include "psi/utils/ec_point_store.h"
-#include "yacl/crypto/utils/rand.h"
+#include "yacl/crypto/rand/rand.h"
 
 #include "engine/audit/audit_log.h"
 #include "engine/core/arrow_helper.h"

diff --git a/engine/operator/join.cc b/engine/operator/join.cc
@@ -31,7 +31,7 @@
 #include "psi/cryptor/cryptor_selector.h"
 #include "psi/ecdh/ecdh_oprf_psi.h"
 #include "psi/ecdh/ecdh_psi.h"
-#include "yacl/crypto/utils/rand.h"
+#include "yacl/crypto/rand/rand.h"
 
 #include "engine/audit/audit_log.h"
 #include "engine/core/primitive_builder.h"

diff --git a/engine/operator/make_private_test.cc b/engine/operator/make_private_test.cc
@@ -63,10 +63,14 @@ INSTANTIATE_TEST_SUITE_P(
                                              "[1.234, 0, -1.5, 2.75]")),
                      test::NamedTensor(
                          "z", TensorFromJSON(arrow::large_utf8(),
-                                             R"json(["X", "Y", "ZZZ"])json"))},
+                                             R"json(["X", "Y", "ZZZ"])json")),
+                     test::NamedTensor(
+                         "k", TensorFromJSON(
+                                  arrow::boolean(),
+                                  R"json([true, false, false, false])json"))},
                 .input_status = pb::TENSORSTATUS_SECRET,
                 .reveal_to = "alice",
-                .output_names = {"x_hat", "y_hat", "z_hat"}},
+                .output_names = {"x_hat", "y_hat", "z_hat", "k_hat"}},
             MakePrivateTestCase{
                 .inputs = {test::NamedTensor(
                                "x",

diff --git a/engine/util/ndarray_to_arrow.cc b/engine/util/ndarray_to_arrow.cc
@@ -100,7 +100,7 @@ class NdArrayConverter {
           "NdArrayConverter doesn't support strided arrays");
     }
 
-    if (pt_type_ == spu::PT_BOOL) {
+    if (pt_type_ == spu::PT_I1) {
       int64_t nbytes = arrow::bit_util::BytesForBits(length_);
       ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateBuffer(nbytes, pool_));
 

diff --git a/engine/util/psi_helper.cc b/engine/util/psi_helper.cc
@@ -30,7 +30,7 @@
 #include "arrow/compute/api.h"
 #include "gflags/gflags.h"
 #include "msgpack.hpp"
-#include "yacl/crypto/utils/rand.h"
+#include "yacl/crypto/rand/rand.h"
 
 #include "engine/core/arrow_helper.h"
 #include "engine/core/primitive_builder.h"