Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updating test scripts #130

Merged
merged 3 commits into from
Jul 19, 2024
Merged

Updating test scripts #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5c839d8
Updating test scripts
manika137 Jul 17, 2024
7fbf876
Merge branch 'wasbDepCodeReview' of github.com:ABFSDriver/AbfsHadoop …
manika137 Jul 17, 2024
7c46823
minor changes
manika137 Jul 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 103 additions & 21 deletions hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,36 +32,87 @@ processCount=8

## SECTION: TEST COMBINATION METHODS

runHNSOAuthTest()
runHNSOAuthDFSTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
PROPERTIES=("fs.azure.account.auth.type")
VALUES=("OAuth")
triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
triggerRun "HNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
}

runHNSSharedKeyTest()
runHNSSharedKeyDFSTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
PROPERTIES=("fs.azure.account.auth.type")
VALUES=("SharedKey")
triggerRun "HNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
triggerRun "HNS-SharedKey-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
}

runNonHNSSharedKeyTest()
runNonHNSSharedKeyDFSTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
PROPERTIES=("fs.azure.account.auth.type")
VALUES=("SharedKey")
triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
triggerRun "NonHNS-SharedKey-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
}

runAppendBlobHNSOAuthTest()
runAppendBlobHNSOAuthDFSTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
VALUES=("OAuth" "true")
triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
triggerRun "AppendBlob-HNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
}

runNonHNSSharedKeyBlobTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
fnsBlobConfigFileCheck "$accountName"
PROPERTIES=("fs.azure.account.auth.type")
VALUES=("SharedKey")
triggerRun "NonHNS-SharedKey-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does ${accountName}_blob works ?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems to be working...

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it's creating new file if not already created.
It's showing the same file name in abfs-test-combinations-config.xml for the required combo

}

runNonHNSOAuthDFSTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
PROPERTIES=("fs.azure.account.auth.type")
VALUES=("OAuth")
triggerRun "NonHNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
}

runNonHNSOAuthBlobTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
fnsBlobConfigFileCheck "$accountName"
PROPERTIES=("fs.azure.account.auth.type")
VALUES=("OAuth")
triggerRun "NonHNS-OAuth-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers"
}

runAppendBlobNonHNSOAuthBlobTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
fnsBlobConfigFileCheck "$accountName"
PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
VALUES=("OAuth" "true")
triggerRun "AppendBlob-NonHNS-OAuth-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers"
}

runHNSOAuthDFSIngressBlobTest()
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Blob ingress on an hns account, dont think it should happen. @anmolanmol1234, what you feel.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should work @saxenapranav. We have added support to handle it

{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
PROPERTIES=("fs.azure.account.auth.type" "fs.azure.ingress.service.type")
VALUES=("OAuth" "blob")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this should be DFS or BLOB. Not in the uppercase. Reason being, in the code, it resolves from enum AbfsServiceType which has the values in uppercase.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think Anuj has added support for it being case insensitive, right @anujmodi2021 ?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was supposed to be case insensitive but looks like it's not.
May be that's why ingress related tests are failing.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will fix this and make it case insensitive, meanwhile @manika137 can you trigger a ingressBlob combination after changing this setting value from blob to BLOB. to confirm this hypothesis.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Worked

triggerRun "HNS-Oauth-DFS-IngressBlob" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
}

runNonHNSOAuthDFSIngressBlobTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
PROPERTIES=("fs.azure.account.auth.type" "fs.azure.ingress.service.type")
VALUES=("OAuth" "blob")
triggerRun "NonHNS-OAuth-DFS-IngressBlob" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
}

runTest=false
Expand Down Expand Up @@ -126,27 +177,52 @@ do
esac
done


## SECTION: COMBINATION DEFINITIONS AND TRIGGER

echo ' '
echo 'Set the active test combination to run the action:'
select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit
select combo in HNS-OAuth-DFS HNS-SharedKey-DFS NonHNS-SharedKey-DFS AppendBlob-HNS-OAuth-DFS NonHNS-SharedKey-Blob NonHNS-OAuth-DFS NonHNS-OAuth-Blob AppendBlob-NonHNS-OAuth-Blob HNS-Oauth-DFS-IngressBlob NonHNS-Oauth-DFS-IngressBlob AllCombinationsTestRun Quit
do
case $combo in
HNS-OAuth)
runHNSOAuthTest
HNS-OAuth-DFS)
runHNSOAuthDFSTest
break
;;
HNS-SharedKey-DFS)
runHNSSharedKeyDFSTest
break
;;
NonHNS-SharedKey-DFS)
runNonHNSSharedKeyDFSTest
break
;;
AppendBlob-HNS-OAuth-DFS)
runAppendBlobHNSOAuthDFSTest
break
;;
NonHNS-SharedKey-Blob)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

indentation is not proper

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Corrected

runNonHNSSharedKeyBlobTest
break
;;
NonHNS-OAuth-DFS)
runNonHNSOAuthDFSTest
break
;;
NonHNS-OAuth-Blob)
runNonHNSOAuthBlobTest
break
;;
HNS-SharedKey)
runHNSSharedKeyTest
AppendBlob-NonHNS-OAuth-Blob)
runAppendBlobNonHNSOAuthBlobTest
break
;;
nonHNS-SharedKey)
runNonHNSSharedKeyTest
HNS-Oauth-DFS-IngressBlob)
runHNSOAuthDFSIngressBlobTest
break
;;
AppendBlob-HNS-OAuth)
runAppendBlobHNSOAuthTest
NonHNS-Oauth-DFS-IngressBlob)
runNonHNSOAuthDFSIngressBlobTest
break
;;
AllCombinationsTestRun)
Expand All @@ -155,10 +231,16 @@ do
logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST."
break
fi
runHNSOAuthTest
runHNSSharedKeyTest
runNonHNSSharedKeyTest
runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always
runHNSOAuthDFSTest
runHNSSharedKeyDFSTest
runNonHNSSharedKeyDFSTest
runAppendBlobHNSOAuthDFSTest
runNonHNSSharedKeyBlobTest
runNonHNSOAuthDFSTest
runNonHNSOAuthBlobTest
runAppendBlobNonHNSOAuthBlobTest
runHNSOAuthDFSIngressBlobTest
runNonHNSOAuthDFSIngressBlobTest
break
;;
Quit)
Expand Down
18 changes: 18 additions & 0 deletions hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,24 @@ ENDTIME=$(date +%s)
outputFormatOn="\033[0;95m"
outputFormatOff="\033[0m"

fnsBlobConfigFileCheck() {
baseFileName=$1
targetWord=".dfs.core.windows.net"
replacementWord=".blob.core.windows.net"
accountSettingsDir="src/test/resources/accountSettings/"
accountConfigFileSuffix="_settings.xml"
sourceFilePath="${accountSettingsDir}${baseFileName}${accountConfigFileSuffix}"
targetFilePath="${accountSettingsDir}${baseFileName}_blob${accountConfigFileSuffix}"

if [ ! -f "$targetFilePath" ]; then
cp "$sourceFilePath" "$targetFilePath"
sed -i "s/$targetWord/$replacementWord/g" "$targetFilePath"
echo "File created and word replaced."
else
echo "File already exists."
fi
}

triggerRun()
{
echo ' '
Expand Down
Loading