From 86c5121b68eaaa8853d242de64e580e1311f5f69 Mon Sep 17 00:00:00 2001 From: samik Date: Thu, 26 Oct 2023 23:27:31 +0530 Subject: [PATCH] Add SCM ApplicationManager --- .../services/ApplicationLifecycleService.java | 4 +- .../app/sourcecontrol/ApplicationManager.java | 52 ----- .../LocalApplicationManager.java | 105 ++++++++++ .../app/sourcecontrol/PullAppsOperation.java | 26 ++- .../RemoteApplicationManager.java | 118 +++++++++++ .../LocalApplicationDetailFetcher.java | 6 +- .../RemoteApplicationDetailFetcher.java | 11 +- .../sourcecontrol/ApplicationManagerTest.java | 196 ++++++++++++++++++ .../sourcecontrol/PullAppsOperationTest.java | 3 +- .../io/cdap/cdap/proto/app/AppVersion.java | 6 + .../io/cdap/cdap/proto/id/ApplicationId.java | 5 + .../sourcecontrol/ApplicationManager.java | 84 ++++++++ .../sourcecontrol/SourceControlException.java | 4 + 13 files changed, 551 insertions(+), 69 deletions(-) delete mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/ApplicationManager.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/LocalApplicationManager.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/RemoteApplicationManager.java create mode 100644 cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/sourcecontrol/ApplicationManagerTest.java create mode 100644 cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/ApplicationManager.java diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java index 8913988c746a..eb837b2a1cc0 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java @@ -86,15 +86,15 @@ import io.cdap.cdap.internal.capability.CapabilityNotAvailableException; import io.cdap.cdap.internal.capability.CapabilityReader; import io.cdap.cdap.internal.profile.AdminEventPublisher; -import io.cdap.cdap.messaging.spi.MessagingService; import io.cdap.cdap.messaging.context.MultiThreadMessagingContext; +import io.cdap.cdap.messaging.spi.MessagingService; import io.cdap.cdap.proto.ApplicationDetail; import io.cdap.cdap.proto.PluginInstanceDetail; import io.cdap.cdap.proto.ProgramType; import io.cdap.cdap.proto.app.AppVersion; import io.cdap.cdap.proto.app.MarkLatestAppsRequest; -import io.cdap.cdap.proto.app.UpdateSourceControlMetaRequest; import io.cdap.cdap.proto.app.UpdateMultiSourceControlMetaReqeust; +import io.cdap.cdap.proto.app.UpdateSourceControlMetaRequest; import io.cdap.cdap.proto.artifact.AppRequest; import io.cdap.cdap.proto.artifact.ArtifactSortOrder; import io.cdap.cdap.proto.artifact.ChangeDetail; diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/ApplicationManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/ApplicationManager.java deleted file mode 100644 index 30902b625eb0..000000000000 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/ApplicationManager.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright © 2023 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.cdap.internal.app.sourcecontrol; - -import io.cdap.cdap.proto.id.ApplicationId; -import io.cdap.cdap.proto.id.ApplicationReference; -import io.cdap.cdap.sourcecontrol.SourceControlException; -import io.cdap.cdap.sourcecontrol.operationrunner.PullAppResponse; -import java.util.Collection; - -/** - * Provides various helper methods for source control operations to operate on applications. - * Would be implemented for both running in app-fabric and running in workers. - */ -public interface ApplicationManager { - - /** - * Deploys a given app with the given pull app details. - * - * @param appRef the {@link ApplicationReference} for the app to be deployed - * @param pullDetails {@link PullAppResponse} which includes the app spec and the git hash. - * @return The {@link ApplicationId} for the deployed version - * @throws SourceControlException for any failure. We wrap all failures to - * {@link SourceControlException} - */ - ApplicationId deployApp(ApplicationReference appRef, PullAppResponse pullDetails) - throws SourceControlException; - - /** - * Mark the given list of app-versions as the latest. Only the latest version for any app is - * runnable. - * - * @param appIds List of {@link ApplicationId} to be marked latest - * @throws SourceControlException for any failure. We wrap all failures to - * {@link SourceControlException} - */ - void markAppVersionsLatest(Collection appIds) throws SourceControlException; -} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/LocalApplicationManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/LocalApplicationManager.java new file mode 100644 index 000000000000..c55818b9496c --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/LocalApplicationManager.java @@ -0,0 +1,105 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.sourcecontrol; + +import com.google.inject.Inject; +import io.cdap.cdap.common.ApplicationNotFoundException; +import io.cdap.cdap.common.BadRequestException; +import io.cdap.cdap.common.NotFoundException; +import io.cdap.cdap.common.app.RunIds; +import io.cdap.cdap.internal.app.services.ApplicationLifecycleService; +import io.cdap.cdap.metadata.LocalApplicationDetailFetcher; +import io.cdap.cdap.proto.ApplicationDetail; +import io.cdap.cdap.proto.app.AppVersion; +import io.cdap.cdap.proto.app.MarkLatestAppsRequest; +import io.cdap.cdap.proto.app.UpdateMultiSourceControlMetaReqeust; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ApplicationReference; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.proto.sourcecontrol.SourceControlMeta; +import io.cdap.cdap.security.spi.authorization.UnauthorizedException; +import io.cdap.cdap.sourcecontrol.ApplicationManager; +import io.cdap.cdap.sourcecontrol.SourceControlException; +import io.cdap.cdap.sourcecontrol.operationrunner.PullAppResponse; +import java.io.IOException; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Local implementation of {@link ApplicationManager} to fetch/update data while running in + * app-fabric. + */ +public class LocalApplicationManager implements ApplicationManager { + + private final ApplicationLifecycleService appLifeCycleService; + private final LocalApplicationDetailFetcher appDetailsFetcher; + + private static final Logger LOG = LoggerFactory.getLogger(LocalApplicationManager.class); + + + @Inject + LocalApplicationManager(ApplicationLifecycleService appLifeCycleService, + LocalApplicationDetailFetcher fetcher) { + this.appLifeCycleService = appLifeCycleService; + this.appDetailsFetcher = fetcher; + } + + + @Override + public ApplicationId deployApp(ApplicationReference appRef, PullAppResponse pullDetails) + throws Exception { + String versionId = RunIds.generate().getId(); + ApplicationId appId = appRef.app(versionId); + + AppRequest appRequest = pullDetails.getAppRequest(); + SourceControlMeta sourceControlMeta = new SourceControlMeta( + pullDetails.getApplicationFileHash()); + + LOG.info("Start to deploy app {} in namespace {} without marking latest", + appId.getApplication(), appId.getParent()); + + appLifeCycleService.deployApp(appId, appRequest, sourceControlMeta, x -> { + }, true); + return appId; + } + + @Override + public void markAppVersionsLatest(NamespaceId namespace, List apps) + throws SourceControlException, ApplicationNotFoundException, BadRequestException, IOException { + MarkLatestAppsRequest request = new MarkLatestAppsRequest(apps); + LOG.info("Marking latest in namespace {} : {}", namespace, apps); + appLifeCycleService.markAppsAsLatest(namespace, request); + } + + @Override + public void updateSourceControlMeta(NamespaceId namespace, + UpdateMultiSourceControlMetaReqeust metas) + throws SourceControlException, BadRequestException, IOException { + appLifeCycleService.updateSourceControlMeta(namespace, metas); + } + + @Override + public ApplicationDetail get(ApplicationReference appRef) + throws IOException, NotFoundException, UnauthorizedException { + return appDetailsFetcher.get(appRef); + } + +} + + diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/PullAppsOperation.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/PullAppsOperation.java index 6c9ff33afc6d..394425128f96 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/PullAppsOperation.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/PullAppsOperation.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; +import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.NotFoundException; import io.cdap.cdap.internal.operation.LongRunningOperation; import io.cdap.cdap.internal.operation.LongRunningOperationContext; @@ -30,9 +31,11 @@ import io.cdap.cdap.proto.operation.OperationResource; import io.cdap.cdap.proto.operation.OperationResourceScopedError; import io.cdap.cdap.proto.sourcecontrol.RepositoryConfig; +import io.cdap.cdap.sourcecontrol.ApplicationManager; import io.cdap.cdap.sourcecontrol.SourceControlException; import io.cdap.cdap.sourcecontrol.operationrunner.InMemorySourceControlOperationRunner; import io.cdap.cdap.sourcecontrol.operationrunner.MultiPullAppOperationRequest; +import java.io.IOException; import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -88,13 +91,17 @@ public ListenableFuture> run(LongRunningOperationContext scmOpRunner.pull(pullReq, response -> { appTobeDeployed.set(new ApplicationReference(context.getRunId().getNamespace(), response.getApplicationName())); - ApplicationId deployedVersion = applicationManager.deployApp( - appTobeDeployed.get(), response - ); - deployed.add(deployedVersion); - context.updateOperationResources(getResources()); + try { + ApplicationId deployedVersion = applicationManager.deployApp( + appTobeDeployed.get(), response + ); + deployed.add(deployedVersion); + context.updateOperationResources(getResources()); + } catch (Exception e) { + throw new SourceControlException(e); + } }); - } catch (NotFoundException | SourceControlException e) { + } catch (Exception e) { throw new OperationException( "Failed to deploy applications", appTobeDeployed.get() != null ? ImmutableList.of( @@ -105,8 +112,11 @@ public ListenableFuture> run(LongRunningOperationContext try { // all deployed versions are marked latest atomically - applicationManager.markAppVersionsLatest(deployed); - } catch (SourceControlException e) { + applicationManager.markAppVersionsLatest( + context.getRunId().getNamespaceId(), + deployed.stream().map(ApplicationId::getAppVersion).collect(Collectors.toList()) + ); + } catch (BadRequestException | NotFoundException | IOException e) { throw new OperationException( "Failed to mark applications latest", Collections.emptySet() diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/RemoteApplicationManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/RemoteApplicationManager.java new file mode 100644 index 000000000000..e1f7e6f3c54e --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/sourcecontrol/RemoteApplicationManager.java @@ -0,0 +1,118 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.sourcecontrol; + +import com.google.gson.Gson; +import com.google.inject.Inject; +import io.cdap.cdap.common.NotFoundException; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.http.DefaultHttpRequestConfig; +import io.cdap.cdap.common.internal.remote.RemoteClient; +import io.cdap.cdap.common.internal.remote.RemoteClientFactory; +import io.cdap.cdap.metadata.RemoteApplicationDetailFetcher; +import io.cdap.cdap.proto.ApplicationDetail; +import io.cdap.cdap.proto.ApplicationRecord; +import io.cdap.cdap.proto.app.AppVersion; +import io.cdap.cdap.proto.app.MarkLatestAppsRequest; +import io.cdap.cdap.proto.app.UpdateMultiSourceControlMetaReqeust; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ApplicationReference; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.security.spi.authorization.UnauthorizedException; +import io.cdap.cdap.sourcecontrol.ApplicationManager; +import io.cdap.cdap.sourcecontrol.SourceControlException; +import io.cdap.cdap.sourcecontrol.operationrunner.PullAppResponse; +import io.cdap.common.http.HttpMethod; +import io.cdap.common.http.HttpRequest; +import io.cdap.common.http.HttpResponse; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.List; + +/** + * Remote implementation of {@link ApplicationManager} which calls app-fabric apis. + */ +public class RemoteApplicationManager implements ApplicationManager { + + private final RemoteClient remoteClient; + private final RemoteApplicationDetailFetcher appDetailsFetcher; + + private static final Gson GSON = new Gson(); + + @Inject + RemoteApplicationManager(RemoteClientFactory remoteClientFactory, + RemoteApplicationDetailFetcher appDetailsFetcher) { + this.remoteClient = remoteClientFactory.createRemoteClient( + Constants.Service.APP_FABRIC_HTTP, + new DefaultHttpRequestConfig(false), + Constants.Gateway.INTERNAL_API_VERSION_3); + this.appDetailsFetcher = appDetailsFetcher; + } + + @Override + public ApplicationId deployApp(ApplicationReference appRef, PullAppResponse pullDetails) + throws SourceControlException, NotFoundException, IOException { + String url = String.format("namespaces/%s/apps/%s?skipMarkingLatest=true", + appRef.getNamespace(), appRef.getApplication()); + HttpRequest.Builder requestBuilder = remoteClient.requestBuilder(HttpMethod.PUT, url) + .withBody(GSON.toJson(pullDetails.getAppRequest())); + HttpResponse httpResponse; + httpResponse = execute(requestBuilder.build()); + ApplicationRecord response = GSON.fromJson(httpResponse.getResponseBodyAsString(), + ApplicationRecord.class); + return appRef.app(response.getAppVersion()); + } + + @Override + public void markAppVersionsLatest(NamespaceId namespace, List apps) + throws SourceControlException, NotFoundException, IOException { + MarkLatestAppsRequest markLatestRequest = new MarkLatestAppsRequest(apps); + String url = String.format("namespaces/%s/apps/markLatest", namespace.getEntityName()); + HttpRequest.Builder requestBuilder = remoteClient.requestBuilder(HttpMethod.POST, url) + .withBody(GSON.toJson(markLatestRequest)); + execute(requestBuilder.build()); + } + + @Override + public void updateSourceControlMeta(NamespaceId namespace, + UpdateMultiSourceControlMetaReqeust metas) + throws SourceControlException, NotFoundException, IOException { + String url = String.format("namespaces/%s/apps/updateSourceControlMeta", + namespace.getEntityName()); + HttpRequest.Builder requestBuilder = remoteClient.requestBuilder(HttpMethod.POST, url) + .withBody(GSON.toJson(metas)); + execute(requestBuilder.build()); + } + + @Override + public ApplicationDetail get(ApplicationReference appRef) + throws IOException, NotFoundException, UnauthorizedException { + return appDetailsFetcher.get(appRef); + } + + private HttpResponse execute(HttpRequest request) + throws SourceControlException, NotFoundException, IOException { + HttpResponse httpResponse = remoteClient.execute(request); + if (httpResponse.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { + throw new NotFoundException(httpResponse.getResponseBodyAsString()); + } + if (httpResponse.getResponseCode() != HttpURLConnection.HTTP_OK) { + throw new SourceControlException(httpResponse.getResponseBodyAsString()); + } + return httpResponse; + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/LocalApplicationDetailFetcher.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/LocalApplicationDetailFetcher.java index 1a17a505ac84..6817bb627e8e 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/LocalApplicationDetailFetcher.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/LocalApplicationDetailFetcher.java @@ -50,9 +50,9 @@ public LocalApplicationDetailFetcher(ApplicationLifecycleService applicationLife * * @param appRef the versionless id of the application * @return {@link ApplicationDetail} for the given application - * @throws IOException if failed to get {@link ApplicationDetail} for the given {@link - * ApplicationId} - * @throws NotFoundException if the given the given application doesn't exist + * @throws IOException if failed to get {@link ApplicationDetail} for the given + * {@link ApplicationId} + * @throws NotFoundException if the given application doesn't exist */ @Override public ApplicationDetail get(ApplicationReference appRef) throws IOException, NotFoundException { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/RemoteApplicationDetailFetcher.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/RemoteApplicationDetailFetcher.java index 16730e40053b..ffaca22b62aa 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/RemoteApplicationDetailFetcher.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/RemoteApplicationDetailFetcher.java @@ -43,7 +43,7 @@ import java.util.function.Consumer; /** - * Fetch application detail via internal REST API calls + * Fetch application detail via internal REST API calls. */ public class RemoteApplicationDetailFetcher implements ApplicationDetailFetcher { @@ -59,6 +59,11 @@ public class RemoteApplicationDetailFetcher implements ApplicationDetailFetcher private final RemoteClient remoteClient; + /** + * Default constructor. + * + * @param remoteClientFactory factory to create remote client + */ @Inject public RemoteApplicationDetailFetcher(RemoteClientFactory remoteClientFactory) { this.remoteClient = remoteClientFactory.createRemoteClient( @@ -68,7 +73,7 @@ public RemoteApplicationDetailFetcher(RemoteClientFactory remoteClientFactory) { } /** - * Get the application detail for the given application reference + * Get the application detail for the given application reference. */ public ApplicationDetail get(ApplicationReference appRef) throws IOException, NotFoundException, UnauthorizedException { @@ -81,7 +86,7 @@ public ApplicationDetail get(ApplicationReference appRef) } /** - * Scans all application details in the given namespace + * Scans all application details in the given namespace. */ @Override public void scan(String namespace, Consumer consumer, Integer batchSize) diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/sourcecontrol/ApplicationManagerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/sourcecontrol/ApplicationManagerTest.java new file mode 100644 index 000000000000..f33ed743fea7 --- /dev/null +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/sourcecontrol/ApplicationManagerTest.java @@ -0,0 +1,196 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.sourcecontrol; + +import io.cdap.cdap.AllProgramsApp; +import io.cdap.cdap.api.artifact.ArtifactSummary; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.id.Id; +import io.cdap.cdap.internal.app.services.http.AppFabricTestBase; +import io.cdap.cdap.proto.ApplicationDetail; +import io.cdap.cdap.proto.app.AppVersion; +import io.cdap.cdap.proto.app.UpdateMultiSourceControlMetaReqeust; +import io.cdap.cdap.proto.app.UpdateSourceControlMetaRequest; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ApplicationReference; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.sourcecontrol.ApplicationManager; +import io.cdap.cdap.sourcecontrol.operationrunner.PullAppResponse; +import io.cdap.common.http.HttpResponse; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * Tests for {@link RemoteApplicationManager} and {@link LocalApplicationManager} + */ +@RunWith(Parameterized.class) +public class ApplicationManagerTest extends AppFabricTestBase { + + private enum ApplicationManagerType { + LOCAL, + REMOTE, + } + + private final ApplicationManagerType managerType; + + private static final String namespace = TEST_NAMESPACE1; + private static final Id.Artifact artifact = Id.Artifact.from(new Id.Namespace(namespace), + AllProgramsApp.class.getSimpleName(), "1.0.0-SNAPSHOT"); + private static final AppRequest request = new AppRequest<>( + ArtifactSummary.from(artifact.toArtifactId())); + + public ApplicationManagerTest(ApplicationManagerType type) { + this.managerType = type; + } + + @Parameterized.Parameters + public static Collection parameters() { + return Arrays.asList(new Object[][]{ + {ApplicationManagerType.LOCAL}, + {ApplicationManagerType.REMOTE}, + }); + } + + @SuppressWarnings("checkstyle:MissingSwitchDefault") + private ApplicationManager getApplicationManager(ApplicationManagerType type) { + if (type == ApplicationManagerType.LOCAL) { + return AppFabricTestBase.getInjector().getInstance(LocalApplicationManager.class); + } + return AppFabricTestBase.getInjector().getInstance(RemoteApplicationManager.class); + } + + + @Before + public void setUp() throws Exception { + HttpResponse response = addAppArtifact(artifact, AllProgramsApp.class); + Assert.assertEquals(200, response.getResponseCode()); + } + + @Test + public void testDeployApp() throws Exception { + ApplicationManager manager = getApplicationManager(managerType); + ApplicationReference appRef = new ApplicationReference(namespace, AllProgramsApp.NAME); + + // Deploy the application + ApplicationId deployedAppId = manager.deployApp( + appRef, + new PullAppResponse<>(AllProgramsApp.NAME, "originalHash", request) + ); + + // fetch and validate the application version is created + HttpResponse response = doGet(getVersionedAPIPath( + String.format("apps/%s/versions/%s", deployedAppId.getApplication(), + deployedAppId.getVersion()), + Constants.Gateway.API_VERSION_3_TOKEN, namespace)); + + Assert.assertEquals(200, response.getResponseCode()); + + ApplicationDetail detail = GSON.fromJson(response.getResponseBodyAsString(), + ApplicationDetail.class); + Assert.assertEquals(false, detail.getChange().getLatest()); + + // Delete the application + Assert.assertEquals( + 200, + doDelete(getVersionedAPIPath("apps/", + Constants.Gateway.API_VERSION_3_TOKEN, namespace)).getResponseCode()); + } + + @Test + public void testMarkAppVersionsLatest() throws Exception { + ApplicationManager manager = getApplicationManager(managerType); + ApplicationReference appRef = new ApplicationReference(namespace, AllProgramsApp.NAME); + + // Deploy the application + ApplicationId deployedAppId = manager.deployApp( + appRef, + new PullAppResponse<>(AllProgramsApp.NAME, "originalHash", request) + ); + + // mark the application as latest + manager.markAppVersionsLatest(new NamespaceId(namespace), Collections.singletonList( + new AppVersion(deployedAppId.getApplication(), deployedAppId.getVersion()))); + + // fetch and validate the application version is created + // fetch and validate the application version is created + HttpResponse response = doGet(getVersionedAPIPath( + String.format("apps/%s/versions/%s", deployedAppId.getApplication(), + deployedAppId.getVersion()), + Constants.Gateway.API_VERSION_3_TOKEN, namespace)); + + Assert.assertEquals(200, response.getResponseCode()); + + ApplicationDetail detail = GSON.fromJson(response.getResponseBodyAsString(), + ApplicationDetail.class); + Assert.assertEquals(true, detail.getChange().getLatest()); + + // Delete the application + Assert.assertEquals( + 200, + doDelete(getVersionedAPIPath("apps/", + Constants.Gateway.API_VERSION_3_TOKEN, namespace)).getResponseCode()); + } + + @Test + public void testUpdateSourceControlMeta() throws Exception { + ApplicationManager manager = getApplicationManager(managerType); + ApplicationReference appRef = new ApplicationReference(namespace, AllProgramsApp.NAME); + + // Deploy the application + ApplicationId deployedAppId = manager.deployApp( + appRef, + new PullAppResponse<>(AllProgramsApp.NAME, "originalHash", request) + ); + + UpdateMultiSourceControlMetaReqeust request = new UpdateMultiSourceControlMetaReqeust( + Collections.singletonList(new UpdateSourceControlMetaRequest( + deployedAppId.getApplication(), + deployedAppId.getVersion(), + "updatedHash" + )) + ); + + // update the source control meta + manager.updateSourceControlMeta(new NamespaceId(namespace), request); + + // fetch and validate the application version is created + // fetch and validate the application version is created + HttpResponse response = doGet(getVersionedAPIPath( + String.format("apps/%s/versions/%s", deployedAppId.getApplication(), + deployedAppId.getVersion()), + Constants.Gateway.API_VERSION_3_TOKEN, namespace)); + + Assert.assertEquals(200, response.getResponseCode()); + + ApplicationDetail detail = GSON.fromJson(response.getResponseBodyAsString(), + ApplicationDetail.class); + Assert.assertEquals("updatedHash", detail.getSourceControlMeta().getFileHash()); + + // Delete the application + Assert.assertEquals( + 200, + doDelete(getVersionedAPIPath("apps/", + Constants.Gateway.API_VERSION_3_TOKEN, namespace)).getResponseCode()); + } +} diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/sourcecontrol/PullAppsOperationTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/sourcecontrol/PullAppsOperationTest.java index 865d7a3287e7..168e7e66daf5 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/sourcecontrol/PullAppsOperationTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/sourcecontrol/PullAppsOperationTest.java @@ -31,6 +31,7 @@ import io.cdap.cdap.proto.id.ApplicationReference; import io.cdap.cdap.proto.id.OperationRunId; import io.cdap.cdap.proto.operation.OperationResource; +import io.cdap.cdap.sourcecontrol.ApplicationManager; import io.cdap.cdap.sourcecontrol.RepositoryManager; import io.cdap.cdap.sourcecontrol.RepositoryManagerFactory; import io.cdap.cdap.sourcecontrol.SourceControlException; @@ -170,7 +171,7 @@ public void testRunFailedWhenMarkingLatest() throws Exception { ).when(mockManager).deployApp(Mockito.any(), Mockito.any()); Mockito.doThrow(new SourceControlException("")).when(mockManager) - .markAppVersionsLatest(Mockito.any()); + .markAppVersionsLatest(Mockito.any(), Mockito.any()); try { operation.run(context).get(); diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/app/AppVersion.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/app/AppVersion.java index 2fed3a6df954..c920224f2560 100644 --- a/cdap-proto/src/main/java/io/cdap/cdap/proto/app/AppVersion.java +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/app/AppVersion.java @@ -22,6 +22,7 @@ * Contains an app name and version. */ public class AppVersion { + private final String name; private final String appVersion; @@ -55,4 +56,9 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(name, appVersion); } + + @Override + public String toString() { + return String.format("%s-%s", name, appVersion); + } } diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/id/ApplicationId.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/id/ApplicationId.java index 0a8035d5054c..49a8d7314f57 100644 --- a/cdap-proto/src/main/java/io/cdap/cdap/proto/id/ApplicationId.java +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/id/ApplicationId.java @@ -17,6 +17,7 @@ import io.cdap.cdap.api.metadata.MetadataEntity; import io.cdap.cdap.proto.ProgramType; +import io.cdap.cdap.proto.app.AppVersion; import io.cdap.cdap.proto.element.EntityType; import java.util.Arrays; import java.util.Collections; @@ -80,6 +81,10 @@ public ApplicationReference getAppReference() { return new ApplicationReference(namespace, application); } + public AppVersion getAppVersion() { + return new AppVersion(application, version); + } + public ProgramId program(ProgramType type, String program) { return new ProgramId(this, type, program); } diff --git a/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/ApplicationManager.java b/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/ApplicationManager.java new file mode 100644 index 000000000000..a4541bb5c78d --- /dev/null +++ b/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/ApplicationManager.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.sourcecontrol; + +import io.cdap.cdap.common.BadRequestException; +import io.cdap.cdap.common.NotFoundException; +import io.cdap.cdap.proto.ApplicationDetail; +import io.cdap.cdap.proto.app.AppVersion; +import io.cdap.cdap.proto.app.UpdateMultiSourceControlMetaReqeust; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ApplicationReference; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.security.spi.authorization.UnauthorizedException; +import io.cdap.cdap.sourcecontrol.operationrunner.PullAppResponse; +import java.io.IOException; +import java.util.List; + +/** + * Provides various helper methods for source control operations to operate on applications. Would + * be implemented for both running in app-fabric and running in workers. + */ +public interface ApplicationManager { + + /** + * Deploys a given app with the given pull app details. + * + * @param appRef the {@link ApplicationReference} for the app to be deployed + * @param pullDetails {@link PullAppResponse} which includes the app spec and the git hash. + * @return The {@link ApplicationId} for the deployed version + * @throws SourceControlException for any failure. We wrap all failures to + * {@link SourceControlException} + */ + ApplicationId deployApp(ApplicationReference appRef, PullAppResponse pullDetails) + throws Exception; + + /** + * Mark the given list of app-versions as the latest. Only the latest version for any app is + * runnable. + * + * @param namespace of the apps to be marked latest + * @param apps List of {@link AppVersion} to be marked latest + * @throws SourceControlException for any failure. We wrap all failures to + * {@link SourceControlException} + */ + void markAppVersionsLatest(NamespaceId namespace, List apps) + throws NotFoundException, IOException, BadRequestException; + + /** + * Update the source control metadata of the given applications. + * + * @param namespace for the apps to be updated + * @param metas to be updated + * @throws SourceControlException for any failure. We wrap all failures to + * {@link SourceControlException} + */ + void updateSourceControlMeta(NamespaceId namespace, UpdateMultiSourceControlMetaReqeust metas) + throws NotFoundException, IOException, BadRequestException; + + /** + * Get the application detail for the given application reference. + * + * @param appRef the versionless ID of the application + * @return the detail of the given application + * @throws IOException if failed to get {@code ApplicationDetail} + * @throws NotFoundException if the application or namespace identified by the supplied id + * doesn't exist + */ + ApplicationDetail get(ApplicationReference appRef) + throws IOException, NotFoundException, UnauthorizedException; +} diff --git a/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/SourceControlException.java b/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/SourceControlException.java index c9b370a0b67d..3e77eca8055c 100644 --- a/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/SourceControlException.java +++ b/cdap-source-control/src/main/java/io/cdap/cdap/sourcecontrol/SourceControlException.java @@ -29,4 +29,8 @@ public SourceControlException(String message, Throwable cause) { public SourceControlException(String message) { super(message); } + + public SourceControlException(Throwable cause) { + super(cause); + } }