diff --git a/deploy/scripts/sgx/enclave_env.sh b/deploy/scripts/sgx/enclave_env.sh index b08613b75..b79e78424 100755 --- a/deploy/scripts/sgx/enclave_env.sh +++ b/deploy/scripts/sgx/enclave_env.sh @@ -25,8 +25,8 @@ function make_custom_env() { export GRPC_VERBOSITY=ERROR export GRPC_POLL_STRATEGY=epoll1 export TF_CPP_MIN_LOG_LEVEL=1 - export TF_GRPC_SGX_RA_TLS_ENABLE=off - export FL_GRPC_SGX_RA_TLS_ENABLE=off + export TF_GRPC_SGX_RA_TLS_ENABLE=on + export FL_GRPC_SGX_RA_TLS_ENABLE=on export TF_DISABLE_MKL=0 export TF_ENABLE_MKL_NATIVE_FORMAT=1 export parallel_num_threads=$1 @@ -35,10 +35,16 @@ function make_custom_env() { export GRPC_SERVER_CHANNEL_THREADS=4 export KMP_SETTINGS=1 export KMP_BLOCKTIME=0 + export HADOOP_HOME=${HADOOP_HOME:-/opt/tiger/yarn_deploy/hadoop_current} + export PATH=$PATH:${HADOOP_HOME}/bin + export JAVA_HOME=/opt/tiger/jdk/openjdk-1.8.0_265 + export LD_LIBRARY_PATH=${HADOOP_HOME}/lib/native:${JAVA_HOME}/jre/lib/amd64/server:${LD_LIBRARY_PATH} + export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$($HADOOP_HOME/bin/hadoop classpath --glob) export MR_ENCLAVE=`get_env mr_enclave` export MR_SIGNER=`get_env mr_signer` export ISV_PROD_ID=`get_env isv_prod_id` export ISV_SVN=`get_env isv_svn` + export RA_TLS_ALLOW_OUTDATED_TCB_INSECURE=1 # network proxy unset http_proxy https_proxy jq ' .sgx_mrs[0].mr_enclave = ''"'`get_env mr_enclave`'" | .sgx_mrs[0].mr_signer = ''"'`get_env mr_signer`'" ' \ @@ -50,17 +56,21 @@ function generate_token() { ./generate.sh mkdir -p /app/sgx/token/ cp python.sig /app/sgx/token/ - cp python.token /app/sgx/token/ cp python.manifest.sgx /app/sgx/token/ + cp python.token /app/sgx/token/ + cp python.manifest /app/sgx/token/ cd - } +if [ -n "$PCCS_IP" ]; then + sed -i "s|PCCS_URL=https://[^ ]*|PCCS_URL=https://pccs_url:8081/sgx/certification/v3/|" /etc/sgx_default_qcnl.conf + echo >> /etc/hosts + echo "$PCCS_IP pccs_url" | tee -a /etc/hosts +elif [ -n "$PCCS_URL" ]; then + sed -i "s|PCCS_URL=[^ ]*|PCCS_URL=$PCCS_URL|" /etc/sgx_default_qcnl.conf +fi -# 为站内不同临时设置,后续改为环境变量区分,参考文档‘隐私计算支持sgx’ -sed -i 's|PCCS_URL=https://[^ ]*|PCCS_URL=https://pccs_url:8081/sgx/certification/v3/|' /etc/sgx_default_qcnl.conf sed -i 's/USE_SECURE_CERT=TRUE/USE_SECURE_CERT=FALSE/' /etc/sgx_default_qcnl.conf -echo >> /etc/hosts -echo "10.137.29.200 pccs_url" | tee -a /etc/hosts mkdir -p /data generate_token diff --git a/deploy/scripts/sgx/run_trainer_worker_sgx.sh b/deploy/scripts/sgx/run_trainer_worker_sgx.sh index f25511c35..0baa4e451 100755 --- a/deploy/scripts/sgx/run_trainer_worker_sgx.sh +++ b/deploy/scripts/sgx/run_trainer_worker_sgx.sh @@ -19,12 +19,19 @@ source ~/.env export CUDA_VISIBLE_DEVICES= export MODEL_NAME=${APPLICATION_ID} +LISTEN_PORT=50051 +if [[ -n "${PORT0}" ]]; then + LISTEN_PORT=${PORT0} +fi + +echo $LISTEN_PORT > /pod-data/listen_port + cp /app/sgx/gramine/CI-Examples/tensorflow_io.py ./ source /app/deploy/scripts/hdfs_common.sh || true source /app/deploy/scripts/pre_start_hook.sh || true source /app/deploy/scripts/env_to_args.sh -PEER_ADDR=$SERVICE_ID +PEER_ADDR="localhost:${PROXY_LOCAL_PORT}" if [[ -n "${CODE_KEY}" ]]; then pull_code ${CODE_KEY} $PWD @@ -79,11 +86,6 @@ print(json.dumps({'clusterSpec': cluster_spec})) """` fi -LISTEN_PORT=50051 -if [[ -n "${PORT0}" ]]; then - LISTEN_PORT=${PORT0} -fi - make_custom_env 4 source /root/start_aesm_service.sh diff --git a/fedlearner-sgx-dev.dockerfile b/fedlearner-sgx-dev.dockerfile index a2d201d35..763fc2119 100644 --- a/fedlearner-sgx-dev.dockerfile +++ b/fedlearner-sgx-dev.dockerfile @@ -45,24 +45,25 @@ ENV GRAMINEDIR=/gramine ENV SGX_DCAP_VERSION=DCAP_1.11 # ENV GRAPHENE_VERSION=master # ENV GRAMINE_VERSION=497847c0353a13c9e83c0ec4c0cbe99f11d4a75d -ENV GRAMINE_VERSION=c662f63bba76736e6d5122a866da762efd1978c1 +ENV GRAMINE_VERSION=devel-v1.3.1-2023-07-13 ENV ISGX_DRIVER_PATH=${GRAMINEDIR}/driver -ENV SGX_SIGNER_KEY=${GRAMINEDIR}/Pal/src/host/Linux-SGX/signer/enclave-key.pem +ENV SGX_SIGNER_KEY=/root/.config/gramine/enclave-key.pem ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 ENV WERROR=1 ENV SGX=1 # https://gramine.readthedocs.io/en/latest/building.html # golang is needed by grpc/BoringSSL +RUN apt-get update RUN apt-get install -y gawk bison python3-click python3-jinja2 golang ninja-build RUN apt-get install -y libcurl4-openssl-dev libprotobuf-c-dev python3-protobuf protobuf-c-compiler -RUN apt-get install -y libgmp-dev libmpfr-dev libmpc-dev libisl-dev +RUN apt-get install -y libgmp-dev libmpfr-dev libmpc-dev libisl-dev nasm protobuf-compiler RUN ln -s /usr/bin/python3 /usr/bin/python \ && pip3 install --upgrade pip \ - && pip3 install toml meson + && pip3 install toml meson pyelftools -RUN git clone https://github.com/gramineproject/gramine.git ${GRAMINEDIR} \ +RUN git clone https://github.com/analytics-zoo/gramine ${GRAMINEDIR} \ && cd ${GRAMINEDIR} \ && git checkout ${GRAMINE_VERSION} @@ -75,7 +76,7 @@ RUN cd ${GRAMINEDIR} \ && git apply *.diff # https://gramine.readthedocs.io/en/latest/quickstart.html#quick-start-with-sgx-support -RUN openssl genrsa -3 -out ${SGX_SIGNER_KEY} 3072 +RUN mkdir -p /root/.config/gramine/ && openssl genrsa -3 -out ${SGX_SIGNER_KEY} 3072 RUN cd ${GRAMINEDIR} \ && LD_LIBRARY_PATH="" meson setup build/ --buildtype=release -Dprefix=${INSTALL_PREFIX} -Ddirect=enabled -Dsgx=enabled -Ddcap=enabled -Dsgx_driver=dcap1.10 -Dsgx_driver_include_path=${ISGX_DRIVER_PATH}/driver/linux/include \ && LD_LIBRARY_PATH="" ninja -C build/ \ @@ -83,8 +84,8 @@ RUN cd ${GRAMINEDIR} \ # Install mbedtls RUN cd ${GRAMINEDIR}/build/subprojects/mbedtls-mbedtls* \ - && cp -r `find . -name "*_gramine.a"` ${INSTALL_PREFIX}/lib \ - && cp -r ${GRAMINEDIR}/subprojects/mbedtls-mbedtls*/include ${INSTALL_PREFIX} + && cp -r `find . -maxdepth 1 -name "*_gramine.a"` ${INSTALL_PREFIX}/lib \ + && cp -r ${GRAMINEDIR}/subprojects/mbedtls-mbedtls*/mbedtls-mbedtls*/include ${INSTALL_PREFIX} # Install cJSON RUN cd ${GRAMINEDIR}/subprojects/cJSON* \ @@ -126,6 +127,7 @@ RUN apt-get install -y libmysqlclient-dev COPY sgx/grpc/common ${GRPC_PATH} COPY sgx/grpc/v1.38.1 ${GRPC_PATH} +RUN pip3 install 'cython==0.29.36' RUN ${GRPC_PATH}/build_python.sh # Build tensorflow @@ -136,7 +138,7 @@ RUN cd ${TF_BUILD_PATH} \ ARG TF_BUILD_CFG="--config=numa --config=mkl --config=mkl_threadpool --copt=-march=native --copt=-O3 --cxxopt=-march=native --cxxopt=-O3 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0" RUN cd ${TF_BUILD_PATH} \ - && bazel build -c opt ${TF_BUILD_CFG} //tensorflow/tools/pip_package:build_pip_package \ + && bazel build --local_ram_resources=2048 -c opt ${TF_BUILD_CFG} //tensorflow/tools/pip_package:build_pip_package \ && bazel-bin/tensorflow/tools/pip_package/build_pip_package ${TF_BUILD_OUTPUT} # Build and install fedlearner diff --git a/fedlearner-sgx-release.dockerfile b/fedlearner-sgx-release.dockerfile index a5b4d9b13..bcf3a8156 100644 --- a/fedlearner-sgx-release.dockerfile +++ b/fedlearner-sgx-release.dockerfile @@ -12,6 +12,10 @@ RUN unset PWD HOSTNAME http_proxy https_proxy RUN env && env > ~/.env && sed -i "s/^/export ${i}\t&/g" ~/.env && echo "source ~/.env" >> ~/.bashrc +RUN rm -rf /gramine/driver && rm -rf /gramine/build + FROM scratch COPY --from=builder / / + +RUN mv /fedlearner /app \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e3369912b..fb4e955c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -tensorflow==2.4.2 -tensorflow_io==0.17.1 setuptools==41.0.0 +cityhash pylint==2.4.4 jinja2 grpcio-tools etcd3 +influxdb peewee apsw configparser @@ -12,6 +12,7 @@ prettytable kubernetes scipy gmpy2 +cityhash scikit-learn pycryptodomex rsa @@ -21,14 +22,12 @@ guppy3 psutil sqlalchemy==1.2.19 mysqlclient +leveldb prison==0.1.3 matplotlib -leveldb -pytz -cityhash flatten_dict pyspark==3.0.2 pandas==1.1.5 opentelemetry-api==1.10.0 opentelemetry-sdk==1.10.0 -opentelemetry-exporter-otlp==1.10.0 +opentelemetry-exporter-otlp==1.10.0 \ No newline at end of file diff --git a/sgx/gramine/CI-Examples/generate-token/Makefile b/sgx/gramine/CI-Examples/generate-token/Makefile index 78e6ba12c..e10d9635e 100644 --- a/sgx/gramine/CI-Examples/generate-token/Makefile +++ b/sgx/gramine/CI-Examples/generate-token/Makefile @@ -3,7 +3,7 @@ GRAMINEDIR ?= ../.. SGX_SIGNER_KEY ?= $(GRAMINEDIR)/Pal/src/host/Linux-SGX/signer/enclave-key.pem -include $(GRAMINEDIR)/Scripts/Makefile.configs +ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine) ifeq ($(DEBUG),1) GRAPHENE_LOG_LEVEL = debug @@ -14,7 +14,7 @@ endif .PHONY: all all: python.manifest ifeq ($(SGX),1) -all: python.manifest.sgx python.sig python.token +all: python.manifest python.manifest.sgx python.sig python.token endif ################################ fedlearner MANIFEST ############################### diff --git a/sgx/gramine/CI-Examples/generate-token/generate.sh b/sgx/gramine/CI-Examples/generate-token/generate.sh index e299c6f09..529666ec8 100755 --- a/sgx/gramine/CI-Examples/generate-token/generate.sh +++ b/sgx/gramine/CI-Examples/generate-token/generate.sh @@ -2,7 +2,8 @@ set -x shopt -s expand_aliases -alias make_logfilter="grep \"mr_enclave\|mr_signer\|isv_prod_id\|isv_svn\"" +alias make_logfilter="grep -v 'measured'" +alias runtime_logfilter="grep -v 'FUTEX|measured|memory entry|cleaning up|async event|shim_exit'" rm -rf *.log make clean && make | make_logfilter diff --git a/sgx/gramine/CI-Examples/generate-token/python.manifest.template b/sgx/gramine/CI-Examples/generate-token/python.manifest.template index ee5188b81..95fc436b0 100644 --- a/sgx/gramine/CI-Examples/generate-token/python.manifest.template +++ b/sgx/gramine/CI-Examples/generate-token/python.manifest.template @@ -2,63 +2,40 @@ libos.entrypoint = "{{ entrypoint }}" libos.check_invalid_pointers = false loader.preload = "file:{{ gramine.libos }}" -loader.log_level = "{{ log_level }}" -# loader.log_file = "" +loader.log_level = "none" +loader.entrypoint = "file:{{ gramine.libos }}" loader.pal_internal_mem_size = "200M" -loader.insecure__use_cmdline_argv = true +loader.insecure__use_cmdline_argv = true loader.insecure__use_host_env = true -loader.env.LD_LIBRARY_PATH = "/opt/tiger/yarn_deploy/hadoop_current/lib/native:/opt/tiger/jdk/jdk1.8/jre/lib/amd64/server:{{ python.stdlib }}/lib:/lib:{{ arch_libdir }}:/usr/local/lib:/usr/local/{{ arch_libdir }}:/usr/lib:/usr/{{ arch_libdir }}" +loader.env.LD_LIBRARY_PATH = "/opt/tiger/yarn_deploy/hadoop_current/lib/native:/opt/tiger/jdk/openjdk-1.8.0_265/jre/lib/amd64/server:/opt/tiger/jdk/openjdk-1.8.0_265/jre/lib/amd64/jli/:/opt/tiger/jdk/openjdk-1.8.0_265/jre/lib/amd64:{{ python.stdlib }}/lib:/lib:{{ arch_libdir }}:/usr/local/lib:/usr/local/{{ arch_libdir }}:/usr/lib:/usr/{{ arch_libdir }}" loader.env.CLASSPATH = "/opt/tiger/jdk/jdk1.8/lib/dt.jar:/opt/tiger/jdk/jdk1.8/lib/tools.jar:/opt/tiger/yarn_deploy/hadoop_current/conf:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/curator-client-2.7.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/okhttp-3.8.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/xmlenc-0.52.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jasper-runtime-5.5.23.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/netty-3.6.10.Final.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/infsecclient-1.4.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/junit-4.11.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-math3-3.1.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jackson-core-2.6.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jsp-api-2.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/gcs-connector-hadoop2-1.9.5-shaded.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-io-2.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-lang-2.6.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/apacheds-i18n-2.0.0-M15.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/asm-3.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/metrics4j-1.0.27.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/dnsjava-2.1.7.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/metrics-core-4.0.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/okio-1.13.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-common-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jackson-dataformat-yaml-2.11.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/hadoop-annotations-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/protobuf-java-2.5.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jackson-datatype-jsr310-2.11.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/curator-framework-2.7.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-policy-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-httpclient-3.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-el-1.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/httpclient-4.2.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-codec-1.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-logging-1.1.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/curator-recipes-2.7.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jsch-0.1.54.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/logredactor-1.0.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-apiextensions-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jasper-compiler-5.5.23.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/log4j-1.2.17.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/zti-issuer-helper-java-1.0.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/snakeyaml-1.26.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-networking-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/btrace-1.0.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/javax.annotation-api-1.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-beanutils-1.7.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/gson-2.2.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/zookeeper-3.4.5-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/paranamer-2.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-events-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/httpcore-4.2.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/java-xmlbuilder-0.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jackson-xc-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-beanutils-core-1.8.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/avro-1.7.6-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jackson-jaxrs-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-cli-1.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/java-jwt-3.11.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-0.0.20.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/hadoop-auth-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-batch-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jetty-util-6.1.26.cloudera.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jaxb-impl-2.2.3-1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-storageclass-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/zjsonpatch-0.3.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jackson-core-asl-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/zti-jwt-helper-java-1.0.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-flowcontrol-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-node-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-coordination-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jackson-mapper-asl-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/caffeine-2.6.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jaxb-api-2.2.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/generex-1.0.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-configuration-1.6.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-apps-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/netty-all-4.1.51.Final.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-rbac-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jersey-core-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/thrift-client-pool-java-1.3.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/servlet-api-2.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-client-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/automaton-1.11-8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-collections-3.2.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/guava-11.0.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-compress-1.4.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/ufs.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jackson-annotations-2.6.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/infsecclient-1.4.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/dps-1.3.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/api-util-1.0.0-M20.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/btrace-1.0.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-autoscaling-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jets3t-0.9.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-extensions-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jersey-server-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jwt-1.0.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/slf4j-api-1.7.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-discovery-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/htrace-core-3.0.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/snappy-java-1.1.2.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-certificates-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jersey-json-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-digester-1.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/activation-1.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/oci-hdfs-full-2.7.2.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jettison-1.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jackson-databind-2.6.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jetty-6.1.26.cloudera.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-core-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/commons-net-3.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jakarta.xml.bind-api-2.3.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-scheduling-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/libthrift-0.9.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/apacheds-kerberos-codec-2.0.0-M15.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/logging-interceptor-3.12.12.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/zti-jwt-java-1.0.16.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-admissionregistration-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/mockito-all-1.8.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/dps-2.0.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jakarta.activation-api-1.2.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/kubernetes-model-metrics-5.10.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jackson-module-jaxb-annotations-2.11.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/stax-api-1.0-2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/hamcrest-core-1.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/jsr305-3.0.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/api-asn1-api-1.0.0-M20.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/lib/xz-1.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/hadoop-nfs-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/hadoop-lzo-0.4.20-SNAPSHOT.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/hadoop-zstd-1.0.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/databus4j-1.2.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/hadoop-common-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/hadoop-brotli-0.0.1-SNAPSHOT.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/hadoop-common-2.6.0-cdh5.4.4-tests.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/hadoop-xz-1.5-byted.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/jniloader-1.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/common/json-serde-1.3-jar-with-dependencies.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/xmlenc-0.52.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/jasper-runtime-5.5.23.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/netty-3.6.10.Final.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/commons-pool2-2.4.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/jsp-api-2.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/commons-io-2.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/commons-lang-2.6.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/asm-3.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/commons-daemon-1.0.13.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/protobuf-java-2.5.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/commons-el-1.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/commons-codec-1.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/commons-logging-1.1.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/log4j-1.2.17.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/commons-cli-1.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/jetty-util-6.1.26.cloudera.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/jackson-core-asl-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/jedis-2.9.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/jackson-mapper-asl-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/leveldbjni-all-1.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/jersey-core-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/servlet-api-2.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/guava-11.0.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/jersey-server-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/htrace-core-3.0.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/jetty-6.1.26.cloudera.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/lib/jsr305-3.0.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/hadoop-hdfs-nfs-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/hadoop-hdfs-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/hadoop-qlimiter-client-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/bookkeeper-stats-api-4.5.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/hadoop-hdfs-2.6.0-cdh5.4.4-tests.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/hadoop-aws-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/hadoop-hdfs-bkjournal-2.6.0-cdh5.4.4-bd1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/bec.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/hdfs/hadoop-bytedance-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/commons-io-2.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/commons-lang-2.6.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/asm-3.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/protobuf-java-2.5.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jersey-guice-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/commons-httpclient-3.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/commons-codec-1.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/commons-logging-1.1.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/bytekv4j-1.2.14.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/log4j-1.2.17.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/guice-servlet-3.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/zookeeper-3.4.5-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/guice-3.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jackson-xc-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jackson-jaxrs-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/commons-cli-1.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jetty-util-6.1.26.cloudera.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jaxb-impl-2.2.3-1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jackson-core-asl-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/javax.inject-1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jackson-mapper-asl-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/aopalliance-1.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jaxb-api-2.2.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/leveldbjni-all-1.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jersey-core-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/servlet-api-2.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/commons-collections-3.2.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/guava-11.0.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/commons-compress-1.4.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/hadoop-yarn-server-resourcemanager-netquota-2.6.0-cdh5.4.4-bd1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jersey-server-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jersey-json-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/activation-1.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jettison-1.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jetty-6.1.26.cloudera.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jline-2.11.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jersey-client-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/stax-api-1.0-2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/jsr305-3.0.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/lib/xz-1.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-server-common-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-applications-distributedshell-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-server-resourcemanager-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-applications-unmanaged-am-launcher-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-server-applicationhistoryservice-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-server-web-proxy-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-common-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-applications-distributedshell-reslake-client-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-registry-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-api-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-server-nodemanager-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-server-tests-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-client-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/yarn/hadoop-yarn-server-nodemanager-containerd-shaded-2.6.0-cdh5.4.4-bd1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/netty-3.6.10.Final.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/junit-4.11.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/commons-io-2.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/asm-3.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/hadoop-annotations-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/protobuf-java-2.5.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/jersey-guice-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/log4j-1.2.17.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/guice-servlet-3.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/paranamer-2.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/guice-3.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/avro-1.7.6-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/jackson-core-asl-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/javax.inject-1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/jackson-mapper-asl-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/aopalliance-1.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/leveldbjni-all-1.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/jersey-core-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/commons-compress-1.4.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/jersey-server-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/snappy-java-1.1.2.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/hamcrest-core-1.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/lib/xz-1.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/hadoop-mapreduce-client-app-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/hadoop-mapreduce-client-core-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/hadoop-mapreduce-client-shuffle-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/hadoop-mapreduce-client-nativetask-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-2.6.0-cdh5.4.4-tests.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/hadoop-mapreduce-client-common-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/hadoop-mapreduce-client-hs-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/hadoop-4mc.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/mapreduce/hadoop-mapreduce-client-hs-plugins-2.6.0-cdh5.4.4.jar::/opt/tiger/tez_deploy/conf:/opt/tiger/tez_deploy/tez/*:/opt/tiger/tez_deploy/tez/lib/*:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/curator-client-2.7.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/okhttp-3.8.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/xmlenc-0.52.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jasper-runtime-5.5.23.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-gridmix-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/junit-4.11.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-math3-3.1.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jackson-core-2.6.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jsp-api-2.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-io-2.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-lang-2.6.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/apacheds-i18n-2.0.0-M15.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-datajoin-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/asm-3.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/metrics4j-1.0.27.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/metrics-core-3.0.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/okio-1.13.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-aliyun-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/protobuf-java-2.5.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/curator-framework-2.7.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-httpclient-3.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-el-1.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/httpclient-4.2.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-codec-1.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-logging-1.1.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/curator-recipes-2.7.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jasper-compiler-5.5.23.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/log4j-1.2.17.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/javax.annotation-api-1.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-beanutils-1.7.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/gson-2.2.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/zookeeper-3.4.5-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/paranamer-2.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/httpcore-4.2.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-ant-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jsch-0.1.42.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/java-xmlbuilder-0.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jackson-xc-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-beanutils-core-1.8.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/avro-1.7.6-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jackson-jaxrs-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-cli-1.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-0.0.20.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-auth-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jetty-util-6.1.26.cloudera.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jaxb-impl-2.2.3-1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/infsecclient-1.2.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jackson-core-asl-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/pystream-0.0.1-SNAPSHOT.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jackson-mapper-asl-1.8.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jaxb-api-2.2.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-sls-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-configuration-1.6.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jersey-core-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/microsoft-windowsazure-storage-sdk-0.6.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/servlet-api-2.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-collections-3.2.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-aws-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/guava-11.0.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-compress-1.4.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jackson-annotations-2.6.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-azure-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/api-util-1.0.0-M20.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-rumen-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-extras-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jets3t-0.9.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jersey-server-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jwt-1.0.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-streaming-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/htrace-core-3.0.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/snappy-java-1.1.2.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-archives-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-openstack-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jersey-json-1.9.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-digester-1.8.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/aws-java-sdk-1.7.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/activation-1.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jettison-1.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jackson-databind-2.6.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jetty-6.1.26.cloudera.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jdom-1.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/commons-net-3.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/libthrift-0.9.2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/apacheds-kerberos-codec-2.0.0-M15.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hadoop-distcp-2.6.0-cdh5.4.4.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/mockito-all-1.8.5.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/aliyun-sdk-oss-2.4.1.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/stax-api-1.0-2.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/dps-1.2.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/hamcrest-core-1.3.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/jsr305-3.0.0.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/api-asn1-api-1.0.0-M20.jar:/opt/tiger/yarn_deploy/hadoop_current/share/hadoop/tools/lib/xz-1.0.jar" loader.env.SECRET_PROVISION_CONSTRUCTOR = "1" loader.env.SECRET_PROVISION_SET_PF_KEY = "1" sys.enable_sigterm_injection = true +sys.enable_extra_runtime_domain_names_conf = true sys.stack.size = "2M" -fs.mount.lib.type = "chroot" -fs.mount.lib.path = "/lib" -fs.mount.lib.uri = "file:{{ gramine.runtimedir() }}" - -fs.mount.lib2.type = "chroot" -fs.mount.lib2.path = "{{ arch_libdir }}" -fs.mount.lib2.uri = "file:{{ arch_libdir }}" - -fs.mount.usr.type = "chroot" -fs.mount.usr.path = "/usr" -fs.mount.usr.uri = "file:/usr" - -fs.mount.pyhome.type = "chroot" -fs.mount.pyhome.path = "{{ python.stdlib }}" -fs.mount.pyhome.uri = "file:{{ python.stdlib }}" - -fs.mount.pydisthome.type = "chroot" -fs.mount.pydisthome.path = "{{ python.distlib }}" -fs.mount.pydisthome.uri = "file:{{ python.distlib }}" - -fs.mount.tmp.type = "chroot" -fs.mount.tmp.path = "/tmp" -fs.mount.tmp.uri = "file:/tmp" - -# Do not use "/proc" in tensorflow -# fs.mount.proc.type = "chroot" -# fs.mount.proc.path = "/proc" -# fs.mount.proc.uri = "file:/proc" - -fs.mount.etc.type = "chroot" -fs.mount.etc.path = "/etc" -fs.mount.etc.uri = "file:/etc" - -fs.mount.keras.type = "chroot" -fs.mount.keras.path = "/root/.keras/keras.json" -fs.mount.keras.uri = "file:/root/.keras/keras.json" - -fs.mount.opt.type = "chroot" -fs.mount.opt.path = "/opt" -fs.mount.opt.uri = "file:/opt" +fs.mounts = [ + { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, + { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" }, + { path = "/usr", uri = "file:/usr" }, + { path = "{{ python.stdlib }}", uri = "file:{{ python.stdlib }}" }, + { path = "{{ python.distlib }}", uri = "file:{{ python.distlib }}" }, + { path = "/opt", uri = "file:/opt" }, + { path = "/etc", uri = "file:/etc" }, + { path = "/tmp", uri = "file:/tmp" }, + { path = "/bin", uri = "file:/bin" }, + { path = "/data", uri = "file:/data" }, + { path = "/proc/net", uri = "file:/proc/net" }, + { path = "/proc/stat", uri = "file:/proc/stat" }, + { path = "/proc/self/cmdline", uri = "file:/proc/self/cmdline" }, + { path = "/root/.keras/keras.json", uri = "file:/root/.keras/keras.json" } +] # Note that Gramine also creates an internal # thread for handling inter-process communication (IPC), and potentially another @@ -66,7 +43,7 @@ fs.mount.opt.uri = "file:/opt" # the application can create is (sgx.thread_num - 2). sgx.debug = false -sgx.remote_attestation = true +sgx.remote_attestation = "dcap" sgx.preheat_enclave = false sgx.nonpie_binary = true sgx.enable_stats = false @@ -89,16 +66,11 @@ sgx.allowed_files = [ "file:tensorflow_io.py", "file:main.py", "file:config.py", - "file:/opt/tiger/jdk/jdk1.8/", - "file:/opt/tiger/yarn_deploy/hadoop_current/", + "file:/opt/tiger/", "file:{{ python.stdlib }}/", "file:{{ python.distlib }}/", - "file:/usr/local/lib/python3.6/dist-packages/", "file:/usr/lib/ssl/openssl.cnf", "file:/usr/lib/gcc", - "file:/usr/bin/ld", - "file:/usr/bin/gcc", - "file:/usr/bin/objdump", "file:/etc/ethers", "file:/etc/hosts", "file:/etc/group", @@ -110,12 +82,17 @@ sgx.allowed_files = [ "file:/etc/nsswitch.conf", "file:/etc/sgx_default_qcnl.conf", "file:/etc/ssl/certs/ca-certificates.crt", + "file:/etc/tce_dynamic/identity.token", "file:/proc/stat", + "file:/proc/net/", + "file:/proc/self/cmdline", "file:/tmp/", "file:/root/.keras/keras.json", "file:dynamic_config.json", + "file:/usr/bin/", + "file:/usr/local/lib", + "file:/lib/", + "file:/bin/", + "file:/data/" ] -#sgx.protected_files = [ -# "file:model", -#] diff --git a/sgx/gramine/patches/ipv6_support.diff b/sgx/gramine/patches/ipv6_support.diff new file mode 100644 index 000000000..75885af19 --- /dev/null +++ b/sgx/gramine/patches/ipv6_support.diff @@ -0,0 +1,166 @@ +diff --git a/libos/src/net/ip.c b/libos/src/net/ip.c +index 20278149..d335ff5b 100644 +--- a/libos/src/net/ip.c ++++ b/libos/src/net/ip.c +@@ -318,6 +318,12 @@ static int set_ipv6_option(struct libos_handle* handle, int optname, void* optva + case IPV6_RECVERR: + /* See the comment in `set_ipv4_option` for why we handle it this way. */ + return 0; ++ case IP_MULTICAST_ALL: ++ attr.socket.ip_multicast_all = !!*(int*)optval; ++ break; ++ case IPV6_MULTICAST_HOPS: ++ attr.socket.ipv6_multicast_hops = !!*(int*)optval; ++ break; + default: + return -ENOPROTOOPT; + } +@@ -529,6 +535,12 @@ static int get_ipv6_option(struct libos_handle* handle, int optname, void* optva + case IPV6_V6ONLY: + val = attr.socket.ipv6_v6only; + break; ++ case IP_MULTICAST_ALL: ++ val = attr.socket.ip_multicast_all; ++ break; ++ case IPV6_MULTICAST_HOPS: ++ val = attr.socket.ipv6_multicast_hops; ++ break; + default: + return -ENOPROTOOPT; + } +diff --git a/pal/include/pal/pal.h b/pal/include/pal/pal.h +index e4f98c01..4cae9ea9 100644 +--- a/pal/include/pal/pal.h ++++ b/pal/include/pal/pal.h +@@ -480,6 +480,8 @@ typedef struct _PAL_STREAM_ATTR { + bool tcp_cork; + bool tcp_nodelay; + bool ipv6_v6only; ++ bool ip_multicast_all; ++ bool ipv6_multicast_hops; + } socket; + }; + } PAL_STREAM_ATTR; +diff --git a/pal/src/host/linux-sgx/pal_host.h b/pal/src/host/linux-sgx/pal_host.h +index b6e00d09..258cbc4c 100644 +--- a/pal/src/host/linux-sgx/pal_host.h ++++ b/pal/src/host/linux-sgx/pal_host.h +@@ -104,6 +104,8 @@ typedef struct { + bool tcp_cork; + bool tcp_nodelay; + bool ipv6_v6only; ++ bool ip_multicast_all; ++ bool ipv6_multicast_hops; + } sock; + + struct { +diff --git a/pal/src/host/linux-sgx/pal_sockets.c b/pal/src/host/linux-sgx/pal_sockets.c +index 08560ba9..cceb6dd6 100644 +--- a/pal/src/host/linux-sgx/pal_sockets.c ++++ b/pal/src/host/linux-sgx/pal_sockets.c +@@ -95,6 +95,8 @@ static PAL_HANDLE create_sock_handle(int fd, enum pal_socket_domain domain, + handle->sock.tcp_cork = false; + handle->sock.tcp_nodelay = false; + handle->sock.ipv6_v6only = false; ++ handle->sock.ip_multicast_all = false; ++ handle->sock.ipv6_multicast_hops = false; + + return handle; + } +@@ -320,6 +322,8 @@ static int attrquerybyhdl(PAL_HANDLE handle, PAL_STREAM_ATTR* attr) { + attr->socket.tcp_cork = handle->sock.tcp_cork; + attr->socket.tcp_nodelay = handle->sock.tcp_nodelay; + attr->socket.ipv6_v6only = handle->sock.ipv6_v6only; ++ attr->socket.ip_multicast_all = handle->sock.ip_multicast_all; ++ attr->socket.ipv6_multicast_hops = handle->sock.ipv6_multicast_hops; + + return 0; + }; +@@ -441,6 +445,25 @@ static int attrsetbyhdl_common(PAL_HANDLE handle, PAL_STREAM_ATTR* attr) { + handle->sock.ipv6_v6only = attr->socket.ipv6_v6only; + } + ++ if (attr->socket.ip_multicast_all != handle->sock.ip_multicast_all) { ++ int val = attr->socket.ip_multicast_all; ++ int ret = ocall_setsockopt(handle->sock.fd, IPPROTO_IPV6, IP_MULTICAST_ALL, &val, sizeof(val)); ++ if (ret < 0) { ++ return unix_to_pal_error(ret); ++ } ++ handle->sock.ip_multicast_all = attr->socket.ip_multicast_all; ++ } ++ ++ if (attr->socket.ipv6_multicast_hops != handle->sock.ipv6_multicast_hops) { ++ int val = attr->socket.ipv6_multicast_hops; ++ int ret = ocall_setsockopt(handle->sock.fd, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, &val, ++ sizeof(val)); ++ if (ret < 0) { ++ return unix_to_pal_error(ret); ++ } ++ handle->sock.ipv6_multicast_hops = attr->socket.ipv6_multicast_hops; ++ } ++ + return 0; + } + +diff --git a/pal/src/host/linux/pal_host.h b/pal/src/host/linux/pal_host.h +index 155d72cd..89678877 100644 +--- a/pal/src/host/linux/pal_host.h ++++ b/pal/src/host/linux/pal_host.h +@@ -81,6 +81,8 @@ typedef struct { + bool tcp_cork; + bool tcp_nodelay; + bool ipv6_v6only; ++ bool ip_multicast_all; ++ bool ipv6_multicast_hops; + } sock; + + struct { +diff --git a/pal/src/host/linux/pal_sockets.c b/pal/src/host/linux/pal_sockets.c +index 7757ffc2..ee48843b 100644 +--- a/pal/src/host/linux/pal_sockets.c ++++ b/pal/src/host/linux/pal_sockets.c +@@ -78,6 +78,8 @@ static PAL_HANDLE create_sock_handle(int fd, enum pal_socket_domain domain, + handle->sock.tcp_cork = false; + handle->sock.tcp_nodelay = false; + handle->sock.ipv6_v6only = false; ++ handle->sock.ip_multicast_all = false; ++ handle->sock.ipv6_multicast_hops = false; + + return handle; + } +@@ -351,6 +353,8 @@ static int attrquerybyhdl(PAL_HANDLE handle, PAL_STREAM_ATTR* attr) { + attr->socket.tcp_cork = handle->sock.tcp_cork; + attr->socket.tcp_nodelay = handle->sock.tcp_nodelay; + attr->socket.ipv6_v6only = handle->sock.ipv6_v6only; ++ attr->socket.ip_multicast_all = handle->sock.ip_multicast_all; ++ attr->socket.ipv6_multicast_hops = handle->sock.ipv6_multicast_hops; + + return 0; + }; +@@ -487,6 +491,26 @@ static int attrsetbyhdl_common(PAL_HANDLE handle, PAL_STREAM_ATTR* attr) { + handle->sock.ipv6_v6only = attr->socket.ipv6_v6only; + } + ++ if (attr->socket.ip_multicast_all != handle->sock.ip_multicast_all) { ++ int val = attr->socket.ip_multicast_all; ++ int ret = ++ DO_SYSCALL(setsockopt, handle->sock.fd, IPPROTO_IPV6, IP_MULTICAST_ALL, &val, sizeof(val)); ++ if (ret < 0) { ++ return unix_to_pal_error(ret); ++ } ++ handle->sock.ip_multicast_all = attr->socket.ip_multicast_all; ++ } ++ ++ if (attr->socket.ipv6_multicast_hops != handle->sock.ipv6_multicast_hops) { ++ int val = attr->socket.ipv6_multicast_hops; ++ int ret = DO_SYSCALL(setsockopt, handle->sock.fd, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, &val, ++ sizeof(val)); ++ if (ret < 0) { ++ return unix_to_pal_error(ret); ++ } ++ handle->sock.ipv6_multicast_hops = attr->socket.ipv6_multicast_hops; ++ } ++ + return 0; + } + diff --git a/sgx/gramine/patches/pf_rename.diff b/sgx/gramine/patches/pf_rename.diff deleted file mode 100644 index 6d0b1231f..000000000 --- a/sgx/gramine/patches/pf_rename.diff +++ /dev/null @@ -1,250 +0,0 @@ -diff --git a/LibOS/shim/src/fs/chroot/fs.c b/LibOS/shim/src/fs/chroot/fs.c -index d67816a59..f0c0feefd 100644 ---- a/LibOS/shim/src/fs/chroot/fs.c -+++ b/LibOS/shim/src/fs/chroot/fs.c -@@ -200,14 +200,15 @@ static int chroot_lookup(struct shim_dentry* dent) { - return ret; - } - --/* Open a temporary read-only PAL handle for a file (used by `unlink` etc.) */ --static int chroot_temp_open(struct shim_dentry* dent, mode_t type, PAL_HANDLE* out_palhdl) { -+/* Open a temporary PAL handle for a file (used by `rename`, `unlink` etc.) */ -+static int chroot_temp_open(struct shim_dentry* dent, mode_t type, int pal_options, -+ PAL_HANDLE* out_palhdl) { - char* uri; - int ret = chroot_dentry_uri(dent, type, &uri); - if (ret < 0) - return ret; - -- ret = DkStreamOpen(uri, PAL_ACCESS_RDONLY, /*share_flags=*/0, /*create=*/0, /*options=*/0, -+ ret = DkStreamOpen(uri, PAL_ACCESS_RDONLY, /*share_flags=*/0, /*create=*/0, pal_options, - out_palhdl); - free(uri); - return pal_to_unix_errno(ret); -@@ -522,7 +523,7 @@ static int chroot_readdir(struct shim_dentry* dent, readdir_callback_t callback, - char* buf = NULL; - size_t buf_size = READDIR_BUF_SIZE; - -- ret = chroot_temp_open(dent, S_IFDIR, &palhdl); -+ ret = chroot_temp_open(dent, S_IFDIR, /*pal_options=*/0, &palhdl); - if (ret < 0) - return ret; - -@@ -584,7 +585,7 @@ static int chroot_unlink(struct shim_dentry* dir, struct shim_dentry* dent) { - lock(&dent->lock); - - PAL_HANDLE palhdl; -- ret = chroot_temp_open(dent, dent->type, &palhdl); -+ ret = chroot_temp_open(dent, dent->type, /*pal_options=*/0, &palhdl); - if (ret < 0) - goto out; - -@@ -638,7 +639,7 @@ static int chroot_rename(struct shim_dentry* old, struct shim_dentry* new) { - goto out; - - PAL_HANDLE palhdl; -- ret = chroot_temp_open(old, old->type, &palhdl); -+ ret = chroot_temp_open(old, old->type, PAL_OPTION_RENAME, &palhdl); - if (ret < 0) - goto out; - -@@ -677,7 +678,7 @@ static int chroot_chmod(struct shim_dentry* dent, mode_t perm) { - lock(&dent->inode->lock); - - PAL_HANDLE palhdl; -- ret = chroot_temp_open(dent, dent->type, &palhdl); -+ ret = chroot_temp_open(dent, dent->type, /*pal_options=*/0, &palhdl); - if (ret < 0) - goto out; - -diff --git a/Pal/include/host/Linux-common/pal_flags_conv.h b/Pal/include/host/Linux-common/pal_flags_conv.h -index f7c4a919f..556c10f6a 100644 ---- a/Pal/include/host/Linux-common/pal_flags_conv.h -+++ b/Pal/include/host/Linux-common/pal_flags_conv.h -@@ -57,7 +57,7 @@ static inline int PAL_CREATE_TO_LINUX_OPEN(int create) { - } - - static inline int PAL_OPTION_TO_LINUX_OPEN(int options) { -- assert(WITHIN_MASK(options, PAL_OPTION_CLOEXEC | PAL_OPTION_NONBLOCK)); -+ assert(WITHIN_MASK(options, PAL_OPTION_CLOEXEC | PAL_OPTION_NONBLOCK | PAL_OPTION_RENAME)); - return (options & PAL_OPTION_CLOEXEC ? O_CLOEXEC : 0) | - (options & PAL_OPTION_NONBLOCK ? O_NONBLOCK : 0); - } -diff --git a/Pal/include/pal/pal.h b/Pal/include/pal/pal.h -index f4a8d176c..1f8b379f6 100644 ---- a/Pal/include/pal/pal.h -+++ b/Pal/include/pal/pal.h -@@ -297,8 +297,9 @@ enum PAL_OPTION { - PAL_OPTION_CLOEXEC = 1, - PAL_OPTION_EFD_SEMAPHORE = 2, /*!< specific to `eventfd` syscall */ - PAL_OPTION_NONBLOCK = 4, -+ PAL_OPTION_RENAME = 8, /*!< specific to `rename` syscall */ - -- PAL_OPTION_MASK = 7, -+ PAL_OPTION_MASK = 15, - }; - - #define WITHIN_MASK(val, mask) (((val) | (mask)) == (mask)) -diff --git a/Pal/src/host/Linux-SGX/db_files.c b/Pal/src/host/Linux-SGX/db_files.c -index dc7c88198..6c38135a6 100644 ---- a/Pal/src/host/Linux-SGX/db_files.c -+++ b/Pal/src/host/Linux-SGX/db_files.c -@@ -122,6 +122,13 @@ static int file_open(PAL_HANDLE* handle, const char* type, const char* uri, int - pf_mode = PF_FILE_MODE_READ | PF_FILE_MODE_WRITE; - } - -+ /* The file is being opened for renaming. We will need to update the metadata in the file, -+ * so open with RDWR mode with necessary share permissions. */ -+ if (pal_options & PAL_OPTION_RENAME) { -+ pf_mode = PF_FILE_MODE_READ | PF_FILE_MODE_WRITE; -+ flags = O_RDWR; -+ } -+ - if ((pf_mode & PF_FILE_MODE_WRITE) && pf->writable_fd >= 0) { - log_warning("file_open(%s): disallowing concurrent writable handle", - hdl->file.realpath); -@@ -788,22 +795,74 @@ static int file_rename(PAL_HANDLE handle, const char* type, const char* uri) { - if (strcmp(type, URI_TYPE_FILE)) - return -PAL_ERROR_INVAL; - -- char* tmp = strdup(uri); -- if (!tmp) -+ char* new_path = strdup(uri); -+ if (!new_path) - return -PAL_ERROR_NOMEM; - -+ struct protected_file* pf = find_protected_file_handle(handle); -+ -+ /* TODO: Handle the case of renaming a file that has a file handle already open */ -+ if (pf) { -+ size_t normpath_size = strlen(uri) + 1; -+ char* new_normpath = (char*)calloc(1, normpath_size); -+ -+ if (!new_normpath) { -+ free(new_path); -+ return -PAL_ERROR_NOMEM; -+ } -+ -+ if (get_norm_path(uri, new_normpath, &normpath_size) < 0) { -+ log_warning("Could not normalize path (%s)", uri); -+ free(new_normpath); -+ free(new_path); -+ return -PAL_ERROR_DENIED; -+ } -+ -+ if (!get_protected_file(new_normpath)) { -+ log_warning("New path during rename is not specified in 'sgx.protected_files' (%s)", new_normpath); -+ free(new_normpath); -+ free(new_path); -+ return -PAL_ERROR_DENIED; -+ } -+ -+ /* update the metadata of the protected file */ -+ pf_status_t pf_ret = pf_rename(pf->context, new_normpath); -+ -+ free(new_normpath); -+ -+ if (PF_FAILURE(pf_ret)) { -+ log_warning("pf_rename failed: %s", pf_strerror(pf_ret)); -+ free(new_path); -+ return -PAL_ERROR_DENIED; -+ } -+ } -+ - int ret = ocall_rename(handle->file.realpath, uri); - if (ret < 0) { -- free(tmp); -+ free(new_path); -+ if (pf) { -+ /* restore the original file name in pf metadata */ -+ pf_status_t pf_ret = pf_rename(pf->context, handle->file.realpath); -+ if (PF_FAILURE(pf_ret)) { -+ log_warning("Rename failed: %s, the file might be unusable", pf_strerror(pf_ret)); -+ } -+ } - return unix_to_pal_error(ret); - } - -+ if (pf) { -+ ret = pf_file_close(pf, handle); -+ if (ret < 0) { -+ log_warning("pf_file_close failed during rename"); -+ } -+ } -+ - /* initial realpath is part of handle object and will be freed with it */ - if (handle->file.realpath && handle->file.realpath != (void*)handle + HANDLE_SIZE(file)) { - free((void*)handle->file.realpath); - } - -- handle->file.realpath = tmp; -+ handle->file.realpath = new_path; - return 0; - } - -diff --git a/Pal/src/host/Linux-SGX/protected-files/protected_files.c b/Pal/src/host/Linux-SGX/protected-files/protected_files.c -index 8ab14cd52..d88860bca 100644 ---- a/Pal/src/host/Linux-SGX/protected-files/protected_files.c -+++ b/Pal/src/host/Linux-SGX/protected-files/protected_files.c -@@ -372,6 +372,20 @@ static bool ipf_init_new_file(pf_context_t* pf, const char* path) { - return true; - } - -+static bool ipf_rename_file(pf_context_t* pf, const char* new_path) { -+ if (strlen(new_path) > PATH_MAX_SIZE - 1) { -+ pf->last_error = PF_STATUS_PATH_TOO_LONG; -+ return false; -+ } -+ -+ memset(&pf->encrypted_part_plain.path, 0, sizeof(pf->encrypted_part_plain.path)); -+ memcpy(pf->encrypted_part_plain.path, new_path, strlen(new_path) + 1); -+ -+ pf->need_writing = true; -+ -+ return true; -+} -+ - static bool ipf_close(pf_context_t* pf) { - void* data; - bool retval = true; -@@ -1320,6 +1334,19 @@ pf_status_t pf_flush(pf_context_t* pf) { - return PF_STATUS_SUCCESS; - } - -+pf_status_t pf_rename(pf_context_t* pf, const char* new_path) { -+ if (!g_initialized) -+ return PF_STATUS_UNINITIALIZED; -+ -+ if (!ipf_rename_file(pf, new_path)) -+ return pf->last_error; -+ -+ if (!ipf_internal_flush(pf)) -+ return pf->last_error; -+ -+ return PF_STATUS_SUCCESS; -+} -+ - pf_status_t pf_get_handle(pf_context_t* pf, pf_handle_t* handle) { - if (!g_initialized) - return PF_STATUS_UNINITIALIZED; -diff --git a/Pal/src/host/Linux-SGX/protected-files/protected_files.h b/Pal/src/host/Linux-SGX/protected-files/protected_files.h -index 7dd840663..a7b020507 100644 ---- a/Pal/src/host/Linux-SGX/protected-files/protected_files.h -+++ b/Pal/src/host/Linux-SGX/protected-files/protected_files.h -@@ -275,4 +275,16 @@ pf_status_t pf_get_handle(pf_context_t* pf, pf_handle_t* handle); - */ - pf_status_t pf_flush(pf_context_t* pf); - -+/*! -+ * \brief Update the path in the metadata during a rename -+ * -+ * \param [in] pf PF context -+ * \param [in] new_path Renamed path -+ * \return PF status -+ * \details For protected files, the file name including the path is stored in the encrypted -+ * metadata which is verified against the actual path during open. So, during a rename -+ * we need to update the metadata with the new path. -+ */ -+pf_status_t pf_rename(pf_context_t* pf, const char* new_path); -+ - #endif /* PROTECTED_FILES_H_ */ diff --git a/sgx/gramine/patches/ra_tls.diff b/sgx/gramine/patches/ra_tls.diff new file mode 100644 index 000000000..c7dd5fa90 --- /dev/null +++ b/sgx/gramine/patches/ra_tls.diff @@ -0,0 +1,13 @@ +diff --git a/tools/sgx/ra-tls/ra_tls.h b/tools/sgx/ra-tls/ra_tls.h +index cb5a7805..3968f3a2 100644 +--- a/tools/sgx/ra-tls/ra_tls.h ++++ b/tools/sgx/ra-tls/ra_tls.h +@@ -61,7 +61,7 @@ int verify_quote_body_against_envvar_measurements(const sgx_quote_body_t* quote_ + __attribute__ ((visibility("hidden"))) + int ra_tls_verify_callback(void* data, mbedtls_x509_crt* crt, int depth, uint32_t* flags); + +-__attribute__ ((visibility("hidden"))) ++__attribute__ ((visibility("default"))) + int ra_tls_create_key_and_crt(mbedtls_pk_context* key, mbedtls_x509_crt* crt); + + /*! diff --git a/sgx/gramine/patches/ra_tls_verify_dcap.diff b/sgx/gramine/patches/ra_tls_verify_dcap.diff deleted file mode 100644 index 51e728669..000000000 --- a/sgx/gramine/patches/ra_tls_verify_dcap.diff +++ /dev/null @@ -1,102 +0,0 @@ -diff --git a/Pal/src/host/Linux-SGX/tools/ra-tls/ra_tls_verify_dcap.c b/Pal/src/host/Linux-SGX/tools/ra-tls/ra_tls_verify_dcap.c -index 8ce518da..4c854410 100644 ---- a/Pal/src/host/Linux-SGX/tools/ra-tls/ra_tls_verify_dcap.c -+++ b/Pal/src/host/Linux-SGX/tools/ra-tls/ra_tls_verify_dcap.c -@@ -24,6 +24,8 @@ - #include - #include - #include -+#include -+#include - - #include "attestation.h" - #include "ra_tls.h" -@@ -96,8 +98,10 @@ int ra_tls_verify_callback(void* data, mbedtls_x509_crt* crt, int depth, uint32_ - size_t quote_size; - ret = find_oid(crt->v3_ext.p, crt->v3_ext.len, quote_oid, quote_oid_len, (uint8_t**)"e, - "e_size); -- if (ret < 0) -+ if (ret < 0) { -+ printf("find_oid failed"); - goto out; -+ } - - if (quote_size < sizeof(*quote)) { - ret = MBEDTLS_ERR_X509_INVALID_EXTENSIONS; -@@ -106,8 +110,11 @@ int ra_tls_verify_callback(void* data, mbedtls_x509_crt* crt, int depth, uint32_ - - /* compare public key's hash from cert against quote's report_data */ - ret = cmp_crt_pk_against_quote_report_data(crt, quote); -- if (ret < 0) -+ if (ret < 0) { -+ printf("cmp_crt_pk_against_quote_report_data failed"); -+ ret = MBEDTLS_ERR_X509_BAD_INPUT_DATA; - goto out; -+ } - - /* prepare user-supplied verification parameters "allow outdated TCB"/"allow debug enclave" */ - bool allow_outdated_tcb = getenv_allow_outdated_tcb(); -@@ -116,12 +123,14 @@ int ra_tls_verify_callback(void* data, mbedtls_x509_crt* crt, int depth, uint32_ - /* call into libsgx_dcap_quoteverify to verify ECDSA/based SGX quote */ - ret = sgx_qv_get_quote_supplemental_data_size(&supplemental_data_size); - if (ret) { -+ printf("sgx_qv_get_quote_supplemental_data_size failed"); - ret = MBEDTLS_ERR_X509_FATAL_ERROR; - goto out; - } - - supplemental_data = (uint8_t*)malloc(supplemental_data_size); - if (!supplemental_data) { -+ printf("malloc failed"); - ret = MBEDTLS_ERR_X509_ALLOC_FAILED; - goto out; - } -@@ -140,6 +149,7 @@ int ra_tls_verify_callback(void* data, mbedtls_x509_crt* crt, int depth, uint32_ - /*p_qve_report_info=*/NULL, supplemental_data_size, - supplemental_data); - if (ret) { -+ printf("sgx_qv_verify_quote failed"); - ret = MBEDTLS_ERR_X509_CERT_VERIFY_FAILED; - goto out; - } -@@ -159,6 +169,7 @@ int ra_tls_verify_callback(void* data, mbedtls_x509_crt* crt, int depth, uint32_ - case SGX_QL_QV_RESULT_REVOKED: - case SGX_QL_QV_RESULT_UNSPECIFIED: - default: -+ printf("verification_result meet unknown failure\n"); - ret = MBEDTLS_ERR_X509_CERT_VERIFY_FAILED; - break; - } -@@ -166,6 +177,7 @@ int ra_tls_verify_callback(void* data, mbedtls_x509_crt* crt, int depth, uint32_ - /* verify enclave attributes from the SGX quote */ - ret = verify_quote_enclave_attributes(quote, allow_debug_enclave); - if (ret < 0) { -+ printf("verify_quote_enclave_attributes failed"); - ret = MBEDTLS_ERR_X509_CERT_VERIFY_FAILED; - goto out; - } -@@ -177,9 +189,15 @@ int ra_tls_verify_callback(void* data, mbedtls_x509_crt* crt, int depth, uint32_ - (const char*)"e->report_body.mr_signer, - (const char*)"e->report_body.isv_prod_id, - (const char*)"e->report_body.isv_svn); -+ if (ret < 0) { -+ printf("g_verify_measurements_cb failed"); -+ } - } else { - /* use default logic to verify measurements */ - ret = verify_quote_against_envvar_measurements(quote, quote_size); -+ if (ret < 0) { -+ printf("verify_quote_against_envvar_measurements failed"); -+ } - } - if (ret < 0) { - ret = MBEDTLS_ERR_X509_CERT_VERIFY_FAILED; -@@ -189,5 +207,8 @@ int ra_tls_verify_callback(void* data, mbedtls_x509_crt* crt, int depth, uint32_ - ret = 0; - out: - free(supplemental_data); -+ if (ret != 0) { -+ printf(", code: %d, error: %s\n", ret, mbedtls_high_level_strerr(ret)); -+ } - return ret; - } diff --git a/sgx/grpc/v1.38.1/src/cpp/sgx/sgx_ra_tls_utils.h b/sgx/grpc/v1.38.1/src/cpp/sgx/sgx_ra_tls_utils.h index bb187b84b..e060f4377 100644 --- a/sgx/grpc/v1.38.1/src/cpp/sgx/sgx_ra_tls_utils.h +++ b/sgx/grpc/v1.38.1/src/cpp/sgx/sgx_ra_tls_utils.h @@ -48,8 +48,6 @@ namespace grpc { namespace sgx { -#include -#include #include #include #include diff --git a/sgx/tf/tensorflow/core/distributed_runtime/rpc/grpc_sgx_ra_tls_utils.h b/sgx/tf/tensorflow/core/distributed_runtime/rpc/grpc_sgx_ra_tls_utils.h index fabb290c4..394609f7e 100644 --- a/sgx/tf/tensorflow/core/distributed_runtime/rpc/grpc_sgx_ra_tls_utils.h +++ b/sgx/tf/tensorflow/core/distributed_runtime/rpc/grpc_sgx_ra_tls_utils.h @@ -40,8 +40,6 @@ namespace grpc { namespace sgx { -#include -#include #include #include #include