feat(sgx): update gramine version to 1.3 bigDL for hdfs error

deploy/scripts/sgx/enclave_env.sh

@@ -25,8 +25,8 @@ function make_custom_env() {
     export GRPC_VERBOSITY=ERROR
     export GRPC_POLL_STRATEGY=epoll1
     export TF_CPP_MIN_LOG_LEVEL=1
-    export TF_GRPC_SGX_RA_TLS_ENABLE=off
-    export FL_GRPC_SGX_RA_TLS_ENABLE=off
+    export TF_GRPC_SGX_RA_TLS_ENABLE=on
+    export FL_GRPC_SGX_RA_TLS_ENABLE=on
     export TF_DISABLE_MKL=0
     export TF_ENABLE_MKL_NATIVE_FORMAT=1
     export parallel_num_threads=$1
@@ -35,10 +35,16 @@ function make_custom_env() {
     export GRPC_SERVER_CHANNEL_THREADS=4
     export KMP_SETTINGS=1
     export KMP_BLOCKTIME=0
+    export HADOOP_HOME=${HADOOP_HOME:-/opt/tiger/yarn_deploy/hadoop_current}
+    export PATH=$PATH:${HADOOP_HOME}/bin
+    export JAVA_HOME=/opt/tiger/jdk/openjdk-1.8.0_265
+    export LD_LIBRARY_PATH=${HADOOP_HOME}/lib/native:${JAVA_HOME}/jre/lib/amd64/server:${LD_LIBRARY_PATH}
+    export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$($HADOOP_HOME/bin/hadoop classpath --glob)
     export MR_ENCLAVE=`get_env mr_enclave`
     export MR_SIGNER=`get_env mr_signer`
     export ISV_PROD_ID=`get_env isv_prod_id`
     export ISV_SVN=`get_env isv_svn`
+    export RA_TLS_ALLOW_OUTDATED_TCB_INSECURE=1
     # network proxy
     unset http_proxy https_proxy
     jq ' .sgx_mrs[0].mr_enclave = ''"'`get_env mr_enclave`'" | .sgx_mrs[0].mr_signer = ''"'`get_env mr_signer`'" ' \
@@ -50,17 +56,21 @@ function generate_token() {
     ./generate.sh
     mkdir -p /app/sgx/token/
     cp python.sig /app/sgx/token/
-    cp python.token /app/sgx/token/
     cp python.manifest.sgx /app/sgx/token/
+    cp python.token /app/sgx/token/
+    cp python.manifest /app/sgx/token/
     cd -
 }
 
+if [ -n "$PCCS_IP" ]; then
+        sed -i "s|PCCS_URL=https://[^ ]*|PCCS_URL=https://pccs_url:8081/sgx/certification/v3/|" /etc/sgx_default_qcnl.conf
+        echo >> /etc/hosts
+        echo "$PCCS_IP   pccs_url" | tee -a /etc/hosts
+elif [ -n "$PCCS_URL" ]; then
+        sed -i "s|PCCS_URL=[^ ]*|PCCS_URL=$PCCS_URL|" /etc/sgx_default_qcnl.conf
+fi
 
-# 为站内不同临时设置，后续改为环境变量区分，参考文档‘隐私计算支持sgx’
-sed -i 's|PCCS_URL=https://[^ ]*|PCCS_URL=https://pccs_url:8081/sgx/certification/v3/|' /etc/sgx_default_qcnl.conf
 sed -i 's/USE_SECURE_CERT=TRUE/USE_SECURE_CERT=FALSE/' /etc/sgx_default_qcnl.conf
-echo >> /etc/hosts
-echo "10.137.29.200   pccs_url" | tee -a /etc/hosts
 mkdir -p /data
 
 generate_token

deploy/scripts/sgx/run_trainer_worker_sgx.sh

@@ -19,12 +19,19 @@ source ~/.env
 export CUDA_VISIBLE_DEVICES=
 export MODEL_NAME=${APPLICATION_ID}
 
+LISTEN_PORT=50051
+if [[ -n "${PORT0}" ]]; then
+  LISTEN_PORT=${PORT0}
+fi
+
+echo $LISTEN_PORT > /pod-data/listen_port
+
 cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./
 source /app/deploy/scripts/hdfs_common.sh || true
 source /app/deploy/scripts/pre_start_hook.sh || true
 source /app/deploy/scripts/env_to_args.sh
 
-PEER_ADDR=$SERVICE_ID
+PEER_ADDR="localhost:${PROXY_LOCAL_PORT}"
 
 if [[ -n "${CODE_KEY}" ]]; then
   pull_code ${CODE_KEY} $PWD
@@ -79,11 +86,6 @@ print(json.dumps({'clusterSpec': cluster_spec}))
 """`
 fi
 
-LISTEN_PORT=50051
-if [[ -n "${PORT0}" ]]; then
-  LISTEN_PORT=${PORT0}
-fi
-
 make_custom_env 4
 source /root/start_aesm_service.sh
 

fedlearner-sgx-dev.dockerfile

@@ -45,24 +45,25 @@ ENV GRAMINEDIR=/gramine
 ENV SGX_DCAP_VERSION=DCAP_1.11
 # ENV GRAPHENE_VERSION=master
 # ENV GRAMINE_VERSION=497847c0353a13c9e83c0ec4c0cbe99f11d4a75d
-ENV GRAMINE_VERSION=c662f63bba76736e6d5122a866da762efd1978c1
+ENV GRAMINE_VERSION=devel-v1.3.1-2023-07-13
 ENV ISGX_DRIVER_PATH=${GRAMINEDIR}/driver
-ENV SGX_SIGNER_KEY=${GRAMINEDIR}/Pal/src/host/Linux-SGX/signer/enclave-key.pem
+ENV SGX_SIGNER_KEY=/root/.config/gramine/enclave-key.pem
 ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
 ENV WERROR=1
 ENV SGX=1
 
 # https://gramine.readthedocs.io/en/latest/building.html
 # golang is needed by grpc/BoringSSL
+RUN apt-get update
 RUN apt-get install -y gawk bison python3-click python3-jinja2 golang ninja-build
 RUN apt-get install -y libcurl4-openssl-dev libprotobuf-c-dev python3-protobuf protobuf-c-compiler
-RUN apt-get install -y libgmp-dev libmpfr-dev libmpc-dev libisl-dev
+RUN apt-get install -y libgmp-dev libmpfr-dev libmpc-dev libisl-dev nasm protobuf-compiler
 
 RUN ln -s /usr/bin/python3 /usr/bin/python \
     && pip3 install --upgrade pip \
-    && pip3 install toml meson
+    && pip3 install toml meson pyelftools
 
-RUN git clone https://github.com/gramineproject/gramine.git ${GRAMINEDIR} \
+RUN git clone https://github.com/analytics-zoo/gramine ${GRAMINEDIR} \
     && cd ${GRAMINEDIR} \
     && git checkout ${GRAMINE_VERSION}
 
@@ -75,16 +76,16 @@ RUN cd ${GRAMINEDIR} \
     && git apply *.diff
 
 # https://gramine.readthedocs.io/en/latest/quickstart.html#quick-start-with-sgx-support
-RUN openssl genrsa -3 -out ${SGX_SIGNER_KEY} 3072
+RUN mkdir -p /root/.config/gramine/ && openssl genrsa -3 -out ${SGX_SIGNER_KEY} 3072
 RUN cd ${GRAMINEDIR} \
     && LD_LIBRARY_PATH="" meson setup build/ --buildtype=release -Dprefix=${INSTALL_PREFIX} -Ddirect=enabled -Dsgx=enabled -Ddcap=enabled -Dsgx_driver=dcap1.10 -Dsgx_driver_include_path=${ISGX_DRIVER_PATH}/driver/linux/include \
     && LD_LIBRARY_PATH="" ninja -C build/ \
     && LD_LIBRARY_PATH="" ninja -C build/ install
 
 # Install mbedtls
 RUN cd ${GRAMINEDIR}/build/subprojects/mbedtls-mbedtls* \
-    && cp -r `find . -name "*_gramine.a"` ${INSTALL_PREFIX}/lib \
-    && cp -r ${GRAMINEDIR}/subprojects/mbedtls-mbedtls*/include ${INSTALL_PREFIX}
+    && cp -r `find . -maxdepth 1 -name "*_gramine.a"` ${INSTALL_PREFIX}/lib \
+    && cp -r ${GRAMINEDIR}/subprojects/mbedtls-mbedtls*/mbedtls-mbedtls*/include ${INSTALL_PREFIX}
 
 # Install cJSON
 RUN cd ${GRAMINEDIR}/subprojects/cJSON* \
@@ -126,6 +127,7 @@ RUN apt-get install -y libmysqlclient-dev
 COPY sgx/grpc/common ${GRPC_PATH}
 COPY sgx/grpc/v1.38.1 ${GRPC_PATH}
 
+RUN pip3 install 'cython==0.29.36'
 RUN ${GRPC_PATH}/build_python.sh
 
 # Build tensorflow
@@ -136,7 +138,7 @@ RUN cd ${TF_BUILD_PATH} \
 
 ARG TF_BUILD_CFG="--config=numa --config=mkl --config=mkl_threadpool --copt=-march=native --copt=-O3 --cxxopt=-march=native --cxxopt=-O3 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0"
 RUN cd ${TF_BUILD_PATH} \
-    && bazel build -c opt ${TF_BUILD_CFG} //tensorflow/tools/pip_package:build_pip_package \
+    && bazel build --local_ram_resources=2048 -c opt ${TF_BUILD_CFG} //tensorflow/tools/pip_package:build_pip_package \
     && bazel-bin/tensorflow/tools/pip_package/build_pip_package ${TF_BUILD_OUTPUT}
 
 # Build and install fedlearner

fedlearner-sgx-release.dockerfile

@@ -12,6 +12,10 @@ RUN unset PWD HOSTNAME http_proxy https_proxy
 
 RUN env && env > ~/.env && sed -i "s/^/export ${i}\t&/g" ~/.env && echo "source ~/.env" >> ~/.bashrc
 
+RUN rm -rf /gramine/driver && rm -rf /gramine/build
+
 FROM scratch
 
 COPY --from=builder / /
+
+RUN mv /fedlearner /app

requirements.txt

@@ -1,17 +1,18 @@
-tensorflow==2.4.2
-tensorflow_io==0.17.1
 setuptools==41.0.0
+cityhash
 pylint==2.4.4
 jinja2
 grpcio-tools
 etcd3
+influxdb
 peewee
 apsw
 configparser
 prettytable
 kubernetes
 scipy
 gmpy2
+cityhash
 scikit-learn
 pycryptodomex
 rsa
@@ -21,14 +22,12 @@ guppy3
 psutil
 sqlalchemy==1.2.19
 mysqlclient
+leveldb
 prison==0.1.3
 matplotlib
-leveldb
-pytz
-cityhash
 flatten_dict
 pyspark==3.0.2
 pandas==1.1.5
 opentelemetry-api==1.10.0
 opentelemetry-sdk==1.10.0
-opentelemetry-exporter-otlp==1.10.0
+opentelemetry-exporter-otlp==1.10.0

sgx/gramine/CI-Examples/generate-token/Makefile

@@ -3,7 +3,7 @@
 GRAMINEDIR ?= ../..
 SGX_SIGNER_KEY ?= $(GRAMINEDIR)/Pal/src/host/Linux-SGX/signer/enclave-key.pem
 
-include $(GRAMINEDIR)/Scripts/Makefile.configs
+ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine)
 
 ifeq ($(DEBUG),1)
 GRAPHENE_LOG_LEVEL = debug
@@ -14,7 +14,7 @@ endif
 .PHONY: all
 all: python.manifest
 ifeq ($(SGX),1)
-all: python.manifest.sgx python.sig python.token
+all: python.manifest python.manifest.sgx python.sig python.token
 endif
 
 ################################ fedlearner MANIFEST ###############################

sgx/gramine/CI-Examples/generate-token/generate.sh

@@ -2,7 +2,8 @@
 set -x
 
 shopt -s expand_aliases
-alias make_logfilter="grep \"mr_enclave\|mr_signer\|isv_prod_id\|isv_svn\""
+alias make_logfilter="grep -v 'measured'"
+alias runtime_logfilter="grep -v 'FUTEX|measured|memory entry|cleaning up|async event|shim_exit'"
 
 rm -rf *.log
 make clean && make | make_logfilter