Unable to inject fault (jvm) on VMware NSX Appliance

[anujmittal14]

# Environment Details, steps to reproduce and Error Information

root@nsx-wdc-99:~# sudo update-alternatives --config java
There are 2 choices for the alternative java (providing /usr/bin/java).

Selection Path Priority Status

• 0 /usr/lib/jvm/openjdk-java17-runtime-amd64/bin/java 21700102 auto mode
  1 /usr/lib/jvm/openjdk-java17-amd64/bin/java 21700102 manual mode
  2 /usr/lib/jvm/openjdk-java17-runtime-amd64/bin/java 21700102 manual mode

Press to keep the current choice[*], or type selection number: 0

root@nsx-wdc-99:~# java --version
openjdk 17.0.10 2024-01-16 LTS
OpenJDK Runtime Environment (build 17.0.10+13-LTS)
OpenJDK 64-Bit Server VM (build 17.0.10+13-LTS, mixed mode, sharing)

root@nsx-wdc-99:~# cat /etc/issue.net
Ubuntu 22.04.4 LTS

root@nsx-wdc-99:~# python get-pip.py
Collecting pip
Downloading pip-24.2-py3-none-any.whl.metadata (3.6 kB)
Collecting wheel
Downloading wheel-0.44.0-py3-none-any.whl.metadata (2.3 kB)
Downloading pip-24.2-py3-none-any.whl (1.8 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.8/1.8 MB 23.0 MB/s eta 0:00:00
Downloading wheel-0.44.0-py3-none-any.whl (67 kB)
Installing collected packages: wheel, pip
Successfully installed pip-24.2 wheel-0.44.0
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.

root@nsx-wdc-99:~# pip install virtualenv
Collecting virtualenv
Downloading virtualenv-20.26.4-py3-none-any.whl.metadata (4.5 kB)
Collecting distlib<1,>=0.3.7 (from virtualenv)
Downloading distlib-0.3.8-py2.py3-none-any.whl.metadata (5.1 kB)
Collecting filelock<4,>=3.12.2 (from virtualenv)
Downloading filelock-3.16.0-py3-none-any.whl.metadata (3.0 kB)
Collecting platformdirs<5,>=3.9.1 (from virtualenv)
Downloading platformdirs-4.3.3-py3-none-any.whl.metadata (11 kB)
Downloading virtualenv-20.26.4-py3-none-any.whl (6.0 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 6.0/6.0 MB 30.8 MB/s eta 0:00:00
Downloading distlib-0.3.8-py2.py3-none-any.whl (468 kB)
Downloading filelock-3.16.0-py3-none-any.whl (16 kB)
Downloading platformdirs-4.3.3-py3-none-any.whl (18 kB)
Installing collected packages: distlib, platformdirs, filelock, virtualenv
Successfully installed distlib-0.3.8 filelock-3.16.0 platformdirs-4.3.3 virtualenv-20.26.4
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.

root@nsx-wdc-99:~# virtualenv chaosblade-env
created virtual environment CPython3.10.12.final.0-64 in 613ms
creator CPython3Posix(dest=/root/chaosblade-env, clear=False, no_vcs_ignore=False, global=False)
seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/root/.local/share/virtualenv)
added seed packages: pip==24.2, setuptools==74.1.2, wheel==0.44.0
activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator

root@nsx-wdc-99:~# ls
chaosblade-env get-pip.py

root@nsx-wdc-99:# cd chaosblade-env/bin/
root@nsx-wdc-99:/chaosblade-env/bin# ls
activate activate.fish activate.ps1 pip pip3 python python3.10 wheel-3.10 wheel3.10
activate.csh activate.nu activate_this.py pip-3.10 pip3.10 python3 wheel wheel3

root@nsx-wdc-99:/chaosblade-env/bin# source activate
(chaosblade-env) root@nsx-wdc-99:/chaosblade-env/bin#

(chaosblade-env) root@nsx-wdc-99:~# wget https://github.com/chaosblade-io/chaosblade/releases/download/v1.7.2/chaosblade-1.7.2-linux-amd64.tar.gz

chaosblade-1.7.2-linux-amd64.tar.gz 100%[============================================================================>] 54.72M 19.3MB/s in 2.8s

2024-09-16 04:33:28 (19.3 MB/s) - ‘chaosblade-1.7.2-linux-amd64.tar.gz’ saved [57380052/57380052]

(chaosblade-env) root@nsx-wdc-99:~# ls
chaosblade-1.7.2-linux-amd64.tar.gz chaosblade-env get-pip.py

(chaosblade-env) root@nsx-wdc-99:~# tar -xvf chaosblade-1.7.2-linux-amd64.tar.gz && cd chaosblade-1.7.2/

(chaosblade-env) root@nsx-wdc-99:~# ls
chaosblade-1.7.2 chaosblade-1.7.2-linux-amd64.tar.gz chaosblade-env get-pip.py

(chaosblade-env) root@nsx-wdc-99:~/chaosblade-1.7.2# ./blade version
version: 1.7.2
env: #1 SMP Wed Jan 25 16:41:43 UTC 2023 x86_64
build-time: Fri May 19 03:23:24 UTC 2023

now try to inject jvm fault on 'corfu' process whose user is 'corfu'

(chaosblade-env) root@nsx-wdc-99:~# ps -ef|grep 'corfu_oom'

corfu 75689 75668 10 04:15 ? 00:05:49 /usr/lib/jvm/openjdk-java17-runtime-amd64/bin/java -Dlogback.configurationFile=/usr/share/corfu/conf/logback.prod.xml -Xlog:gc*:/dev/shm/corfu.jvm.gc.9000.log:time,uptime,level:filecount=10,filesize=5m -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/image/core/corfu_oom.hprof -XX:OnOutOfMemoryError=/sbin/dumpcore.py tanuki /image/core/corfu_oom.hprof /run/corfu/corfu-jvm.pid -Djdk.nio.maxCachedBufferSize=1048576 -Dio.netty.recycler.maxCapacityPerThread=0 -XX:-ParallelRefProcEnabled -Djava.io.tmpdir=/image/corfu-server/temp -Dnsx-service-type=corfu-server.service --add-opens=java.base/sun.security.provider=ALL-UNNAMED --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED -Xms1920m -Xmx1920m -Djava.library.path=../lib -classpath ../lib/wrapper.jar:/usr/share/corfu/lib/cmdlets-4.1.20240827191428.4582.1-shaded.jar:/usr/lib/jvm/openjdk-java17-runtime-amd64/vmware/lib/bc-fips-1.0.2.4.jar:/usr/lib/jvm/openjdk-java17-runtime-amd64/vmware/lib/bcpg-fips-1.0.7.1.jar:/usr/lib/jvm/openjdk-java17-runtime-amd64/vmware/lib/bcpkix-fips-1.0.7.jar -Dwrapper.key=8C0QWINQN-6LbH6cq9aqOAel-0ZQrbei -Dwrapper.port=32004 -Dwrapper.jvm.port.min=31000 -Dwrapper.jvm.port.max=31999 -Dwrapper.disable_console_input=TRUE -Dwrapper.pid=75668 -Dwrapper.version=3.5.41-pro -Dwrapper.native_library=wrapper -Dwrapper.arch=x86 -Dwrapper.service=TRUE -Dwrapper.cpu.timeout=10 -Dwrapper.jvmid=1 -Dwrapper.lang.domain=wrapper -Dwrapper.lang.folder=../lang org.tanukisoftware.wrapper.WrapperSimpleApp org.corfudb.infrastructure.CorfuServer -l /config -a 10.196.196.99 -c 0.3 -k 5000000 -d DEBUG --enable-tls --enable-tls-mutual-auth -u /config/cluster-manager/corfu/private/keystore.jks -f /config/cluster-manager/corfu/private/keystore.password -r /config/cluster-manager/corfu/public/truststore.jks -w /config/cluster-manager/corfu/public/truststore.password --log-size-quota-percentage=45 --metrics 9000 --compactor-script=/usr/share/corfu/scripts/compactor_runner.py --compactor-config=/usr/share/corfu/conf/corfu-compactor-config.yml --compaction-trigger-freq-ms=900000 --run-compactor-as-root --tls-protocols=TLSv1.3,TLSv1.2 --network-interface-version=IPv4 --tls-ciphers=TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256

**(chaosblade-env) root@nsx-wdc-99:~/chaosblade-1.7.2# ./blade create jvm cpufullload --pid 75689 **

{"code":63063,"success":false,"error":"/bin/sh -c sudo -u corfu /usr/lib/jvm/openjdk-java17-amd64/bin/java -Xms128M -Xmx128M -Xnoclassgc -ea -Xbootclasspath/a:/root/chaosblade-1.7.2/lib/sandbox/tools.jar -jar /root/chaosblade-1.7.2/lib/sandbox/lib/sandbox-core.jar 75689 \"/root/chaosblade-1.7.2/lib/sandbox/lib/sandbox-agent.jar\" \"home=/root/chaosblade-1.7.2/lib/sandbox;token=178946325929;server.ip=127.0.0.1;server.port=38323;namespace=chaosblade\": cmd exec failed, err: Error: Unable to access jarfile /root/chaosblade-1.7.2/lib/sandbox/lib/sandbox-core.jar\n exit status 1","result":"e0fb23e525e9dfc2"}

all requird 'jars' are available

(chaosblade-env) root@nsx-wdc-99:~/chaosblade-1.7.2# ls /root/chaosblade-1.7.2/lib/sandbox/lib/sandbox-core.jar
/root/chaosblade-1.7.2/lib/sandbox/lib/sandbox-core.jar

(chaosblade-env) root@nsx-wdc-99:~/chaosblade-1.7.2# ls /root/chaosblade-1.7.2/lib/sandbox/lib/sandbox-agent.jar
/root/chaosblade-1.7.2/lib/sandbox/lib/sandbox-agent.jar

(chaosblade-env) root@nsx-wdc-99:~/chaosblade-1.7.2# ls /root/chaosblade-1.7.2/lib/sandbox/lib/sandbox-agent.jar
/root/chaosblade-1.7.2/lib/sandbox/lib/sandbox-agent.jar

if i try to run command using 'root' user

(chaosblade-env) root@nsx-wdc-99:~/chaosblade-1.7.2# /bin/sh -c "sudo -u root /usr/lib/jvm/openjdk-java17-amd64/bin/java -Xms128M -Xmx128M -Xnoclassgc -ea -Xbootclasspath/a:/root/chaosblade-1.7.2/lib/sandbox/tools.jar -jar /root/chaosblade-1.7.2/lib/sandbox/lib/sandbox-core.jar 75689 "/root/chaosblade-1.7.2/lib/sandbox/lib/sandbox-agent.jar" "home=/root/chaosblade-1.7.2/lib/sandbox;token=193312609329;server.ip=127.0.0.1;server.port=43145;namespace=chaosblade""
java.util.ServiceConfigurationError: com.sun.tools.attach.spi.AttachProvider: Provider sun.tools.attach.LinuxAttachProvider not found
at java.base/java.util.ServiceLoader.fail(ServiceLoader.java:593)
at java.base/java.util.ServiceLoader$LazyClassPathLookupIterator.nextProviderClass(ServiceLoader.java:1219)
at java.base/java.util.ServiceLoader$LazyClassPathLookupIterator.hasNextService(ServiceLoader.java:1228)
at java.base/java.util.ServiceLoader$LazyClassPathLookupIterator.hasNext(ServiceLoader.java:1273)
at java.base/java.util.ServiceLoader$2.hasNext(ServiceLoader.java:1309)
at java.base/java.util.ServiceLoader$3.hasNext(ServiceLoader.java:1393)
at jdk.attach/com.sun.tools.attach.spi.AttachProvider.providers(AttachProvider.java:260)
at jdk.attach/com.sun.tools.attach.VirtualMachine.attach(VirtualMachine.java:200)
at com.alibaba.jvm.sandbox.core.CoreLauncher.attachAgent(CoreLauncher.java:59)
at com.alibaba.jvm.sandbox.core.CoreLauncher.(CoreLauncher.java:20)
at com.alibaba.jvm.sandbox.core.CoreLauncher.main(CoreLauncher.java:43)
com.sun.tools.attach.AgentLoadException: Agent JAR not found or no Agent-Class attribute
at jdk.attach/sun.tools.attach.HotSpotVirtualMachine.loadAgent(HotSpotVirtualMachine.java:160)
at com.alibaba.jvm.sandbox.core.CoreLauncher.attachAgent(CoreLauncher.java:61)
at com.alibaba.jvm.sandbox.core.CoreLauncher.(CoreLauncher.java:20)
at com.alibaba.jvm.sandbox.core.CoreLauncher.main(CoreLauncher.java:43)
sandbox load jvm failed : Agent JAR not found or no Agent-Class attribute
-bash: server.ip=127.0.0.1: command not found
-bash: server.port=43145: command not found

Could you please help fix this issue here?

Also I tried to inject the same fault './blade create jvm cpufullload --pid' using choasblade v174 but
'./blade version' command failed with (core dump)

attaching core-dump file.

core-chaosblade-v174.tar.gz

[anujmittal14]

please let me know, if you need more information

[McGaosai]

你好，我也遇到这个问题，已经解决了，工具默认使用jdk的用户去访问sandbox，指定root也没用，你的jdk用户是corfu，他是没权限访问/root的，你可以把chaosblade工具安装在其他目录下