diff --git a/tests/Oryx.Integration.Tests/DotNetCoreEndToEndTests.cs b/tests/Oryx.Integration.Tests/DotNetCoreEndToEndTests.cs index 1422d35ded..63c238c6fc 100644 --- a/tests/Oryx.Integration.Tests/DotNetCoreEndToEndTests.cs +++ b/tests/Oryx.Integration.Tests/DotNetCoreEndToEndTests.cs @@ -13,6 +13,7 @@ namespace Microsoft.Oryx.Integration.Tests { + [Trait("category", "dotnetcore")] public class DotNetCoreEndToEndTests : PlatformEndToEndTestsBase { private const int HostPort = Constants.DotNetCoreEndToEndTestsPort; diff --git a/tests/Oryx.Integration.Tests/MySqlIntegrationTests.cs b/tests/Oryx.Integration.Tests/MySqlIntegrationTests.cs index 6279696c6a..ca59ed2587 100644 --- a/tests/Oryx.Integration.Tests/MySqlIntegrationTests.cs +++ b/tests/Oryx.Integration.Tests/MySqlIntegrationTests.cs @@ -10,7 +10,7 @@ namespace Microsoft.Oryx.Integration.Tests { - [Trait("Category", "DB")] + [Trait("category", "db")] [Trait("db", "mysql")] public class MySqlIntegrationTests : DatabaseTestsBase, IClassFixture { diff --git a/tests/Oryx.Integration.Tests/NodeEndToEndTests.cs b/tests/Oryx.Integration.Tests/NodeEndToEndTests.cs index efdb2fb7ae..975bfc2aaa 100644 --- a/tests/Oryx.Integration.Tests/NodeEndToEndTests.cs +++ b/tests/Oryx.Integration.Tests/NodeEndToEndTests.cs @@ -770,7 +770,7 @@ await EndToEndTestHelper.BuildRunAndAssertAppAsync( }); } - [Fact] + [Fact(Skip = "#824174: Sync the Node Go startup code with the C# 'run-script' code")] public async Task Node_CreateReactAppSample_singleImage() { // Arrange @@ -856,7 +856,7 @@ await EndToEndTestHelper.BuildRunAndAssertAppAsync( }); } - [Fact] + [Fact(Skip = "#824174: Sync the Node Go startup code with the C# 'run-script' code")] public async Task CanBuildAndRun_NodeExpressApp_UsingSingleImage_AndCustomStartupCommandOnly() { // Arrange diff --git a/tests/Oryx.Integration.Tests/PhpEndToEndTests.cs b/tests/Oryx.Integration.Tests/PhpEndToEndTests.cs index cbc8afa6e2..8afd90d3e6 100644 --- a/tests/Oryx.Integration.Tests/PhpEndToEndTests.cs +++ b/tests/Oryx.Integration.Tests/PhpEndToEndTests.cs @@ -14,6 +14,7 @@ namespace Microsoft.Oryx.Integration.Tests { + [Trait("category", "php")] public class PhpEndToEndTests : PlatformEndToEndTestsBase { private const int HostPort = Constants.PhpEndToEndTestsPort; diff --git a/tests/Oryx.Integration.Tests/PostgreSqlIntegrationTests.cs b/tests/Oryx.Integration.Tests/PostgreSqlIntegrationTests.cs index 74b3905067..bd7597b451 100644 --- a/tests/Oryx.Integration.Tests/PostgreSqlIntegrationTests.cs +++ b/tests/Oryx.Integration.Tests/PostgreSqlIntegrationTests.cs @@ -10,7 +10,7 @@ namespace Microsoft.Oryx.Integration.Tests { - [Trait("Category", "DB")] + [Trait("category", "db")] [Trait("db", "postgres")] public class PostgreSqlIntegrationTests : DatabaseTestsBase, IClassFixture { diff --git a/tests/Oryx.Integration.Tests/PythonEndToEndTests.cs b/tests/Oryx.Integration.Tests/PythonEndToEndTests.cs index 555ed3c64d..0ae9d3aaff 100644 --- a/tests/Oryx.Integration.Tests/PythonEndToEndTests.cs +++ b/tests/Oryx.Integration.Tests/PythonEndToEndTests.cs @@ -13,6 +13,7 @@ namespace Microsoft.Oryx.Integration.Tests { + [Trait("category", "python")] public class PythonEndToEndTests : PlatformEndToEndTestsBase { private const int HostPort = Constants.PythonEndToEndTestsPort; diff --git a/tests/Oryx.Integration.Tests/SqlServerIntegrationTests.cs b/tests/Oryx.Integration.Tests/SqlServerIntegrationTests.cs index c23d67e520..97a20dc74a 100644 --- a/tests/Oryx.Integration.Tests/SqlServerIntegrationTests.cs +++ b/tests/Oryx.Integration.Tests/SqlServerIntegrationTests.cs @@ -14,7 +14,7 @@ namespace Microsoft.Oryx.Integration.Tests //// Commenting out the tests here as the test fixture is still run and we want to avoid that as stopping the //// SQL Server container on the build agent fails sometimes requiring a reboot of the agent machine. - //[Trait("Category", "DB")] + //[Trait("category", "db")] //[Trait("db", "sqlserver")] //public class SqlServerIntegrationTests : DatabaseTestsBase, IClassFixture //{ diff --git a/vsts/pipelines/_buildParallel.yml b/vsts/pipelines/_buildParallel.yml index 7fae081a66..bad89e89bb 100644 --- a/vsts/pipelines/_buildParallel.yml +++ b/vsts/pipelines/_buildParallel.yml @@ -85,19 +85,11 @@ steps: condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) - task: ms-devlabs.utilitytasks.task-Shellpp.Shell++@0 - displayName: 'Pull and Retag recently built oryx build image' + displayName: 'Pull and Retag recently built oryx build and runtime images' inputs: type: FilePath scriptPath: ./vsts/scripts/pullandretag.sh - args: $(System.ArtifactsDirectory)/drop/images/build-images-acr.txt - condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) - -- task: ms-devlabs.utilitytasks.task-Shellpp.Shell++@0 - displayName: 'Pull and Retag recently built oryx runtime images' - inputs: - type: FilePath - scriptPath: ./vsts/scripts/pullandretag.sh - args: $(System.ArtifactsDirectory)/drop/images/runtime-images-acr.txt + args: $(System.ArtifactsDirectory)/drop/images/build-images-acr.txt $(System.ArtifactsDirectory)/drop/images/runtime-images-acr.txt condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) - task: ms-devlabs.utilitytasks.task-Shellpp.Shell++@0 diff --git a/vsts/pipelines/ci.yml b/vsts/pipelines/ci.yml index c65d24bacc..14b679479a 100644 --- a/vsts/pipelines/ci.yml +++ b/vsts/pipelines/ci.yml @@ -42,4 +42,4 @@ jobs: displayName: 'Set variables' - template: _buildTemplate.yml -trigger: none +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/nightly.yml b/vsts/pipelines/nightly.yml index 7ff80c44e6..e9bd52a733 100644 --- a/vsts/pipelines/nightly.yml +++ b/vsts/pipelines/nightly.yml @@ -48,8 +48,8 @@ jobs: displayName: 'Set variables' - template: _buildParallel.yml -- job: Job_IntegrationTests - displayName: Test Integration +- job: Job_PythonIntegrationTests + displayName: Run Python Integration Tests dependsOn: - Job_BuildImage - Job_RuntimeImages @@ -62,7 +62,74 @@ jobs: echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" echo "##vso[task.setvariable variable=TestBuildImages;]false" echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]Category!=DB" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=python" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushToDockerHub;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + displayName: 'Set variables' + - template: _buildParallel.yml + +- job: Job_DotNetCoreIntegrationTests + displayName: Run DotNetCore Integration Tests + dependsOn: + - Job_BuildImage + - Job_RuntimeImages + pool: + name: OryxLinux + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=dotnetcore" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushToDockerHub;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + displayName: 'Set variables' + - template: _buildParallel.yml + +- job: Job_NodeIntegrationTests + displayName: Run NodeJs Integration Tests + dependsOn: + - Job_BuildImage + - Job_RuntimeImages + pool: + name: OryxLinux + timeoutInMinutes: 150 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=node" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushToDockerHub;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + displayName: 'Set variables' + - template: _buildParallel.yml + +- job: Job_PhpIntegrationTests + displayName: Run Php Integration Tests + dependsOn: + - Job_BuildImage + - Job_RuntimeImages + pool: + name: OryxLinux + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=php" echo "##vso[task.setvariable variable=TestIntegration;]true" echo "##vso[task.setvariable variable=PushBuildImages;]false" echo "##vso[task.setvariable variable=PushRuntimeImages;]false" @@ -72,7 +139,7 @@ jobs: - template: _buildParallel.yml - job: Job_DbIntegrationTests - displayName: Test Database Integration + displayName: Run Database Integration Tests dependsOn: - Job_BuildImage - Job_RuntimeImages @@ -84,7 +151,7 @@ jobs: echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" echo "##vso[task.setvariable variable=TestBuildImages;]false" echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]Category=DB" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=db" echo "##vso[task.setvariable variable=TestIntegration;]true" echo "##vso[task.setvariable variable=PushBuildImages;]false" echo "##vso[task.setvariable variable=PushRuntimeImages;]false" @@ -93,4 +160,23 @@ jobs: displayName: 'Set variables' - template: _buildParallel.yml +- job: Job_ENDIntegrationTests + displayName: End of Integration Tests + dependsOn: + - Job_BuildImage + - Job_RuntimeImages + - Job_PythonIntegrationTests + - Job_PhpIntegrationTests + - Job_NodeIntegrationTests + - Job_DotNetCoreIntegrationTests + - Job_DbIntegrationTests + pool: + name: OryxLinux + steps: + - task: ms-devlabs.utilitytasks.task-Shellpp.Shell++@0 + displayName: 'End of Integration Tests' + inputs: + type: InlineScript + script: 'echo "Integration tests ended"' + trigger: none \ No newline at end of file diff --git a/vsts/scripts/dockerCleanup.sh b/vsts/scripts/dockerCleanup.sh index a48323c77d..de95956578 100755 --- a/vsts/scripts/dockerCleanup.sh +++ b/vsts/scripts/dockerCleanup.sh @@ -7,6 +7,10 @@ declare -r BUILD_NUMBER="$BUILD_BUILDNUMBER" declare -r DOCKER_SYSTEM_PRUNE="${ORYX_DOCKER_SYSTEM_PRUNE:-false}" +echo +echo "Printing all running containers and stopped containers" +echo +docker ps -a echo echo "Kill all running containers and delete all stopped containers" echo diff --git a/vsts/scripts/pullandretag.sh b/vsts/scripts/pullandretag.sh index c08d7c8616..53fa8d8c1a 100644 --- a/vsts/scripts/pullandretag.sh +++ b/vsts/scripts/pullandretag.sh @@ -5,23 +5,60 @@ # -------------------------------------------------------------------------------------------- set -euo pipefail +# $1 > buildimage-acr.txt +# $2 > runtime-images-acr.txt +declare imagefilter="oryxdevmcr.azurecr.io/public/oryx" -while read sourceImage; do +echo "Build image filter is set" +while read buildImage; do # Always use specific build number based tag and then use the same tag to create a 'latest' tag and push it - if [[ $sourceImage != *:latest ]]; then - echo "Pulling the source image $sourceImage ..." - docker pull "$sourceImage" | sed 's/^/ /' + if [[ $buildImage != *:latest ]]; then + echo "Pulling the build image $buildImage ..." + docker pull "$buildImage" | sed 's/^/ /' - # Trim the build number tag and append the '':latest' to end of it - newtag="${sourceImage%:*}:latest" + # Trim the build number tag and append the '':latest' to end of it + newtag="${buildImage%:*}:latest" - # Replace the ACR registry repository name with a name that the tests know about - newtag=$(echo "$newtag" | sed 's,oryxdevmcr.azurecr.io/public/oryx,oryxdevms,g') + # Replace the ACR registry repository name with a name that the tests know about + newtag=$(echo "$newtag" | sed 's,oryxdevmcr.azurecr.io/public/oryx,oryxdevms,g') - echo - echo "Tagging the source image with tag $newtag ..." - docker tag "$sourceImage" "$newtag" | sed 's/^/ /' - echo - echo ------------------------------------------------------------------------------- + echo + echo "Tagging the source image with tag $newtag ..." + docker tag "$buildImage" "$newtag" | sed 's/^/ /' + echo + echo ------------------------------------------------------------------------------- fi done <"$1" + +# Extract language string from string (e.g extract 'python' from 'category=python') +if [ -n "$TESTINTEGRATIONCASEFILTER" ];then + # For DB tests we want all the runtime images to be present at thae agent machine + if [[ "$TESTINTEGRATIONCASEFILTER" != *db* ]];then + imagefilter=$(echo $TESTINTEGRATIONCASEFILTER | cut -d'=' -f 2) + fi +fi + +# Always convert filter for runtime images to lower case +echo "Runtime image filter is set for "$imagefilter + +while read sourceImage; do + # Always use specific build number based tag and then use the same tag to create a 'latest' tag and push it + if [[ "$sourceImage" != *:latest ]]; then + if [[ "$sourceImage" == *"$imagefilter"* ]]; then + echo "Pulling the runtime image $sourceImage ..." + docker pull "$sourceImage" | sed 's/^/ /' + + # Trim the build number tag and append the '':latest' to end of it + newtag="${sourceImage%:*}:latest" + + # Replace the ACR registry repository name with a name that the tests know about + newtag=$(echo "$newtag" | sed 's,oryxdevmcr.azurecr.io/public/oryx,oryxdevms,g') + + echo + echo "Tagging the source image with tag $newtag ..." + docker tag "$sourceImage" "$newtag" | sed 's/^/ /' + echo + echo ------------------------------------------------------------------------------- + fi + fi +done <"$2" \ No newline at end of file