diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh index 400d0a23834a5..25feca4768bd6 100755 --- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh +++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh @@ -32,36 +32,87 @@ processCount=8 ## SECTION: TEST COMBINATION METHODS -runHNSOAuthTest() +runHNSOAuthDFSTest() { accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath) PROPERTIES=("fs.azure.account.auth.type") VALUES=("OAuth") - triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" + triggerRun "HNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" } -runHNSSharedKeyTest() +runHNSSharedKeyDFSTest() { accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath) PROPERTIES=("fs.azure.account.auth.type") VALUES=("SharedKey") - triggerRun "HNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" + triggerRun "HNS-SharedKey-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" } -runNonHNSSharedKeyTest() +runNonHNSSharedKeyDFSTest() { accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) PROPERTIES=("fs.azure.account.auth.type") VALUES=("SharedKey") - triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" + triggerRun "NonHNS-SharedKey-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" } -runAppendBlobHNSOAuthTest() +runAppendBlobHNSOAuthDFSTest() { accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath) PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled") VALUES=("OAuth" "true") - triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" + triggerRun "AppendBlob-HNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" +} + +runNonHNSSharedKeyBlobTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) + fnsBlobConfigFileCheck "$accountName" + PROPERTIES=("fs.azure.account.auth.type") + VALUES=("SharedKey") + triggerRun "NonHNS-SharedKey-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers" +} + +runNonHNSOAuthDFSTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) + PROPERTIES=("fs.azure.account.auth.type") + VALUES=("OAuth") + triggerRun "NonHNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" +} + +runNonHNSOAuthBlobTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) + fnsBlobConfigFileCheck "$accountName" + PROPERTIES=("fs.azure.account.auth.type") + VALUES=("OAuth") + triggerRun "NonHNS-OAuth-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers" +} + +runAppendBlobNonHNSOAuthBlobTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) + fnsBlobConfigFileCheck "$accountName" + PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled") + VALUES=("OAuth" "true") + triggerRun "AppendBlob-NonHNS-OAuth-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers" +} + +runHNSOAuthDFSIngressBlobTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath) + PROPERTIES=("fs.azure.account.auth.type" "fs.azure.ingress.service.type") + VALUES=("OAuth" "blob") + triggerRun "HNS-Oauth-DFS-IngressBlob" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" +} + +runNonHNSOAuthDFSIngressBlobTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) + PROPERTIES=("fs.azure.account.auth.type" "fs.azure.ingress.service.type") + VALUES=("OAuth" "blob") + triggerRun "NonHNS-OAuth-DFS-IngressBlob" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" } runTest=false @@ -126,27 +177,52 @@ do esac done + ## SECTION: COMBINATION DEFINITIONS AND TRIGGER echo ' ' echo 'Set the active test combination to run the action:' -select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit +select combo in HNS-OAuth-DFS HNS-SharedKey-DFS NonHNS-SharedKey-DFS AppendBlob-HNS-OAuth-DFS NonHNS-SharedKey-Blob NonHNS-OAuth-DFS NonHNS-OAuth-Blob AppendBlob-NonHNS-OAuth-Blob HNS-Oauth-DFS-IngressBlob NonHNS-Oauth-DFS-IngressBlob AllCombinationsTestRun Quit do case $combo in - HNS-OAuth) - runHNSOAuthTest + HNS-OAuth-DFS) + runHNSOAuthDFSTest + break + ;; + HNS-SharedKey-DFS) + runHNSSharedKeyDFSTest + break + ;; + NonHNS-SharedKey-DFS) + runNonHNSSharedKeyDFSTest + break + ;; + AppendBlob-HNS-OAuth-DFS) + runAppendBlobHNSOAuthDFSTest + break + ;; + NonHNS-SharedKey-Blob) + runNonHNSSharedKeyBlobTest + break + ;; + NonHNS-OAuth-DFS) + runNonHNSOAuthDFSTest + break + ;; + NonHNS-OAuth-Blob) + runNonHNSOAuthBlobTest break ;; - HNS-SharedKey) - runHNSSharedKeyTest + AppendBlob-NonHNS-OAuth-Blob) + runAppendBlobNonHNSOAuthBlobTest break ;; - nonHNS-SharedKey) - runNonHNSSharedKeyTest + HNS-Oauth-DFS-IngressBlob) + runHNSOAuthDFSIngressBlobTest break ;; - AppendBlob-HNS-OAuth) - runAppendBlobHNSOAuthTest + NonHNS-Oauth-DFS-IngressBlob) + runNonHNSOAuthDFSIngressBlobTest break ;; AllCombinationsTestRun) @@ -155,10 +231,16 @@ do logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST." break fi - runHNSOAuthTest - runHNSSharedKeyTest - runNonHNSSharedKeyTest - runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always + runHNSOAuthDFSTest + runHNSSharedKeyDFSTest + runNonHNSSharedKeyDFSTest + runAppendBlobHNSOAuthDFSTest + runNonHNSSharedKeyBlobTest + runNonHNSOAuthDFSTest + runNonHNSOAuthBlobTest + runAppendBlobNonHNSOAuthBlobTest + runHNSOAuthDFSIngressBlobTest + runNonHNSOAuthDFSIngressBlobTest break ;; Quit) diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh index 8933b81145de1..e63e12bf8f05e 100644 --- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh +++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh @@ -45,6 +45,24 @@ ENDTIME=$(date +%s) outputFormatOn="\033[0;95m" outputFormatOff="\033[0m" +fnsBlobConfigFileCheck() { + baseFileName=$1 + targetWord=".dfs.core.windows.net" + replacementWord=".blob.core.windows.net" + accountSettingsDir="src/test/resources/accountSettings/" + accountConfigFileSuffix="_settings.xml" + sourceFilePath="${accountSettingsDir}${baseFileName}${accountConfigFileSuffix}" + targetFilePath="${accountSettingsDir}${baseFileName}_blob${accountConfigFileSuffix}" + + if [ ! -f "$targetFilePath" ]; then + cp "$sourceFilePath" "$targetFilePath" + sed -i "s/$targetWord/$replacementWord/g" "$targetFilePath" + echo "File created and word replaced." + else + echo "File already exists." + fi +} + triggerRun() { echo ' '