From a4a7a4860761938e2ea8b24649ff53311df2a940 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Thu, 6 Jun 2024 11:30:00 -0400 Subject: [PATCH] Refactor backgrounds tasks for better visibility (#926) --- .apibuilder/.tracked_files | 10 +- .apibuilder/config | 7 +- api/app/actors/Bindings.scala | 11 +- api/app/actors/EmailActor.scala | 131 ---- api/app/actors/Emails.scala | 39 +- api/app/actors/GeneratorServiceActor.scala | 50 -- api/app/actors/MainActor.scala | 127 ---- api/app/actors/PeriodicActor.scala | 53 ++ api/app/actors/TaskActor.scala | 230 +------ api/app/actors/TaskDispatchActor.scala | 63 ++ api/app/actors/UserActor.scala | 32 - api/app/db/ApplicationsDao.scala | 33 +- api/app/db/EmailVerificationsDao.scala | 18 +- api/app/db/InternalMigrationsDao.scala | 97 --- api/app/db/InternalTasksDao.scala | 59 ++ api/app/db/MembershipRequestsDao.scala | 26 +- api/app/db/MembershipsDao.scala | 15 +- api/app/db/PasswordResetRequestsDao.scala | 23 +- api/app/db/TasksDao.scala | 189 ------ api/app/db/UsersDao.scala | 35 +- api/app/db/VersionsDao.scala | 55 +- .../PsqlApibuilderMigrationsDao.scala | 494 -------------- .../db/generated/PsqlApibuilderTasksDao.scala | 614 ++++++++++++++++++ api/app/db/generators/ServicesDao.scala | 28 +- ...ctiveApibuilderInternalV0Conversions.scala | 92 --- ...ollectiveApibuilderInternalV0Parsers.scala | 97 --- api/app/invariants/CheckInvariants.scala | 44 ++ api/app/invariants/Invariants.scala | 12 + api/app/invariants/PurgeInvariants.scala | 18 + api/app/invariants/TaskInvariants.scala | 20 + .../CleanupDeletionsProcessor.scala} | 52 +- api/app/processor/DiffVersionProcessor.scala | 134 ++++ api/app/processor/EmailProcessor.scala | 146 +++++ .../IndexApplicationProcessor.scala} | 23 +- .../processor/MigrateVersionProcessor.scala | 22 + .../ScheduleMigrateVersionsProcessor.scala | 57 ++ ...heduleSyncGeneratorServicesProcessor.scala | 37 ++ .../SyncGeneratorServiceProcessor.scala | 118 ++++ api/app/processor/TaskActorCompanion.scala | 38 ++ .../TaskDispatchActorCompanion.scala | 25 + api/app/processor/TaskProcessor.scala | 159 +++++ api/app/processor/UserCreatedProcessor.scala | 31 + api/app/util/GeneratorServiceUtil.scala | 2 + api/app/views/emails/errors.scala.html | 7 - api/conf/base.conf | 22 +- api/test/actors/TaskActorSpec.scala | 17 + api/test/db/EmailVerificationsDaoSpec.scala | 7 +- api/test/db/InternalMigrationsDaoSpec.scala | 46 -- api/test/db/TasksDaoSpec.scala | 232 ------- api/test/db/VersionValidatorSpec.scala | 2 +- api/test/helpers/AsyncHelpers.scala | 16 + api/test/invariants/CheckInvariantsSpec.scala | 13 + .../CleanupDeletionsProcessorSpec.scala} | 22 +- .../IndexApplicationProcessorSpec.scala} | 32 +- api/test/util/Daos.scala | 6 +- dao/spec/psql-apibuilder.json | 37 ++ ...collectiveApibuilderInternalV0Models.scala | 263 -------- .../ApicollectiveApibuilderTaskV0Client.scala | 605 +++++++++++++++++ ...collectiveApibuilderTaskV0MockClient.scala | 57 ++ spec/apibuilder-internal.json | 51 -- spec/apibuilder-task.json | 100 +++ 61 files changed, 2740 insertions(+), 2361 deletions(-) delete mode 100644 api/app/actors/EmailActor.scala delete mode 100644 api/app/actors/GeneratorServiceActor.scala delete mode 100644 api/app/actors/MainActor.scala create mode 100644 api/app/actors/PeriodicActor.scala create mode 100644 api/app/actors/TaskDispatchActor.scala delete mode 100644 api/app/actors/UserActor.scala delete mode 100644 api/app/db/InternalMigrationsDao.scala create mode 100644 api/app/db/InternalTasksDao.scala delete mode 100644 api/app/db/TasksDao.scala delete mode 100644 api/app/db/generated/PsqlApibuilderMigrationsDao.scala create mode 100644 api/app/db/generated/PsqlApibuilderTasksDao.scala delete mode 100644 api/app/generated/ApicollectiveApibuilderInternalV0Conversions.scala delete mode 100644 api/app/generated/ApicollectiveApibuilderInternalV0Parsers.scala create mode 100644 api/app/invariants/CheckInvariants.scala create mode 100644 api/app/invariants/Invariants.scala create mode 100644 api/app/invariants/PurgeInvariants.scala create mode 100644 api/app/invariants/TaskInvariants.scala rename api/app/{util/ProcessDeletes.scala => processor/CleanupDeletionsProcessor.scala} (64%) create mode 100644 api/app/processor/DiffVersionProcessor.scala create mode 100644 api/app/processor/EmailProcessor.scala rename api/app/{actors/Search.scala => processor/IndexApplicationProcessor.scala} (77%) create mode 100644 api/app/processor/MigrateVersionProcessor.scala create mode 100644 api/app/processor/ScheduleMigrateVersionsProcessor.scala create mode 100644 api/app/processor/ScheduleSyncGeneratorServicesProcessor.scala create mode 100644 api/app/processor/SyncGeneratorServiceProcessor.scala create mode 100644 api/app/processor/TaskActorCompanion.scala create mode 100644 api/app/processor/TaskDispatchActorCompanion.scala create mode 100644 api/app/processor/TaskProcessor.scala create mode 100644 api/app/processor/UserCreatedProcessor.scala delete mode 100644 api/app/views/emails/errors.scala.html create mode 100644 api/test/actors/TaskActorSpec.scala delete mode 100644 api/test/db/InternalMigrationsDaoSpec.scala delete mode 100644 api/test/db/TasksDaoSpec.scala create mode 100644 api/test/helpers/AsyncHelpers.scala create mode 100644 api/test/invariants/CheckInvariantsSpec.scala rename api/test/{util/ProcessDeletesSpec.scala => processor/CleanupDeletionsProcessorSpec.scala} (83%) rename api/test/{actors/SearchSpec.scala => processor/IndexApplicationProcessorSpec.scala} (76%) delete mode 100644 generated/app/ApicollectiveApibuilderInternalV0Models.scala create mode 100644 generated/app/ApicollectiveApibuilderTaskV0Client.scala create mode 100644 generated/app/ApicollectiveApibuilderTaskV0MockClient.scala delete mode 100644 spec/apibuilder-internal.json create mode 100644 spec/apibuilder-task.json diff --git a/.apibuilder/.tracked_files b/.apibuilder/.tracked_files index d2c00a101..0c3e6d82c 100644 --- a/.apibuilder/.tracked_files +++ b/.apibuilder/.tracked_files @@ -3,7 +3,6 @@ apicollective: apibuilder-api: anorm_2_8_parsers: - api/app/generated/ApicollectiveApibuilderApiV0Conversions.scala - - api/app/generated/ApicollectiveApibuilderApiV0Parsers.scala play_2_8_client: - generated/app/ApicollectiveApibuilderApiV0Client.scala play_2_x_routes: @@ -14,13 +13,11 @@ apicollective: apibuilder-common: anorm_2_8_parsers: - api/app/generated/ApicollectiveApibuilderCommonV0Conversions.scala - - api/app/generated/ApicollectiveApibuilderCommonV0Parsers.scala play_2_x_json: - generated/app/ApicollectiveApibuilderCommonV0Models.scala apibuilder-generator: anorm_2_8_parsers: - api/app/generated/ApicollectiveApibuilderGeneratorV0Conversions.scala - - api/app/generated/ApicollectiveApibuilderGeneratorV0Parsers.scala play_2_8_client: - generated/app/ApicollectiveApibuilderGeneratorV0Client.scala play_2_8_mock_client: @@ -28,14 +25,17 @@ apicollective: apibuilder-internal: anorm_2_8_parsers: - api/app/generated/ApicollectiveApibuilderInternalV0Conversions.scala - - api/app/generated/ApicollectiveApibuilderInternalV0Parsers.scala play_2_x_json: - generated/app/ApicollectiveApibuilderInternalV0Models.scala apibuilder-spec: anorm_2_8_parsers: - api/app/generated/ApicollectiveApibuilderSpecV0Conversions.scala - - api/app/generated/ApicollectiveApibuilderSpecV0Parsers.scala play_2_8_client: - generated/app/ApicollectiveApibuilderSpecV0Client.scala play_2_8_mock_client: - generated/app/ApicollectiveApibuilderSpecV0MockClient.scala + apibuilder-task: + play_2_8_client: + - generated/app/ApicollectiveApibuilderTaskV0Client.scala + play_2_8_mock_client: + - generated/app/ApicollectiveApibuilderTaskV0MockClient.scala diff --git a/.apibuilder/config b/.apibuilder/config index bdb07c04f..acbaffe79 100644 --- a/.apibuilder/config +++ b/.apibuilder/config @@ -27,8 +27,9 @@ code: play_2_8_client: generated/app play_2_8_mock_client: generated/app anorm_2_8_parsers: api/app/generated - apibuilder-internal: + apibuilder-task: version: latest generators: - play_2_x_json: generated/app - anorm_2_8_parsers: api/app/generated + play_2_8_client: generated/app + play_2_8_mock_client: generated/app + diff --git a/api/app/actors/Bindings.scala b/api/app/actors/Bindings.scala index 5a8f25832..584934fe3 100644 --- a/api/app/actors/Bindings.scala +++ b/api/app/actors/Bindings.scala @@ -5,10 +5,11 @@ import play.api.libs.concurrent.AkkaGuiceSupport class ActorsModule extends AbstractModule with AkkaGuiceSupport { override def configure = { - bindActor[MainActor]("main-actor") - bindActor[GeneratorServiceActor]("generator-service-actor") - bindActor[EmailActor]("email-actor") - bindActor[TaskActor]("task-actor") - bindActor[UserActor]("user-actor") + bindActor[PeriodicActor]("PeriodicActor") + bindActor[TaskDispatchActor]( + "TaskDispatchActor", + _.withDispatcher("task-context-dispatcher") + ) + bindActorFactory[TaskActor, actors.TaskActor.Factory] } } diff --git a/api/app/actors/EmailActor.scala b/api/app/actors/EmailActor.scala deleted file mode 100644 index 9d706496a..000000000 --- a/api/app/actors/EmailActor.scala +++ /dev/null @@ -1,131 +0,0 @@ -package actors - -import akka.actor.{Actor, ActorLogging, ActorSystem} -import db._ -import io.apibuilder.api.v0.models.Publication -import lib.{AppConfig, EmailUtil, Person, Role} - -import java.util.UUID - -object EmailActor { - - object Messages { - case class EmailVerificationCreated(guid: UUID) - case class MembershipCreated(guid: UUID) - case class MembershipRequestCreated(guid: UUID) - case class MembershipRequestAccepted(organizationGuid: UUID, userGuid: UUID, role: Role) - case class MembershipRequestDeclined(organizationGuid: UUID, userGuid: UUID, role: Role) - case class PasswordResetRequestCreated(guid: UUID) - case class ApplicationCreated(guid: UUID) - } - -} - -@javax.inject.Singleton -class EmailActor @javax.inject.Inject() ( - system: ActorSystem, - appConfig: AppConfig, - applicationsDao: db.ApplicationsDao, - email: EmailUtil, - emails: Emails, - emailVerificationsDao: db.EmailVerificationsDao, - membershipsDao: db.MembershipsDao, - membershipRequestsDao: db.MembershipRequestsDao, - organizationsDao: OrganizationsDao, - passwordResetRequestsDao: db.PasswordResetRequestsDao, - usersDao: UsersDao -) extends Actor with ActorLogging with ErrorHandler { - - def receive = { - - case m @ EmailActor.Messages.MembershipRequestCreated(guid) => withVerboseErrorHandler(m) { - membershipRequestsDao.findByGuid(Authorization.All, guid).foreach { request => - emails.deliver( - context = Emails.Context.OrganizationAdmin, - org = request.organization, - publication = Publication.MembershipRequestsCreate, - subject = s"${request.organization.name}: Membership Request from ${request.user.email}", - body = views.html.emails.membershipRequestCreated(appConfig, request).toString - ) - } - } - - case m @ EmailActor.Messages.MembershipRequestAccepted(organizationGuid, userGuid, role) => withVerboseErrorHandler(m) { - organizationsDao.findByGuid(Authorization.All, organizationGuid).foreach { org => - usersDao.findByGuid(userGuid).foreach { user => - email.sendHtml( - to = Person(user), - subject = s"Welcome to ${org.name}", - body = views.html.emails.membershipRequestAccepted(org, user, role).toString - ) - } - } - } - - case m @ EmailActor.Messages.MembershipRequestDeclined(organizationGuid, userGuid, role) => withVerboseErrorHandler(m) { - organizationsDao.findByGuid(Authorization.All, organizationGuid).foreach { org => - usersDao.findByGuid(userGuid).foreach { user => - email.sendHtml( - to = Person(user), - subject = s"Your Membership Request to join ${org.name} was declined", - body = views.html.emails.membershipRequestDeclined(org, user, role).toString - ) - } - } - } - - case m @ EmailActor.Messages.MembershipCreated(guid) => withVerboseErrorHandler(m) { - membershipsDao.findByGuid(Authorization.All, guid).foreach { membership => - emails.deliver( - context = Emails.Context.OrganizationAdmin, - org = membership.organization, - publication = Publication.MembershipsCreate, - subject = s"${membership.organization.name}: ${membership.user.email} has joined as ${membership.role}", - body = views.html.emails.membershipCreated(appConfig, membership).toString - ) - } - } - - case m @ EmailActor.Messages.ApplicationCreated(guid) => withVerboseErrorHandler(m) { - applicationsDao.findByGuid(Authorization.All, guid).foreach { application => - organizationsDao.findAll(Authorization.All, application = Some(application)).foreach { org => - emails.deliver( - context = Emails.Context.OrganizationMember, - org = org, - publication = Publication.ApplicationsCreate, - subject = s"${org.name}: New Application Created - ${application.name}", - body = views.html.emails.applicationCreated(appConfig, org, application).toString - ) - } - } - } - - case m @ EmailActor.Messages.PasswordResetRequestCreated(guid) => withVerboseErrorHandler(m) { - passwordResetRequestsDao.findByGuid(guid).foreach { request => - usersDao.findByGuid(request.userGuid).foreach { user => - email.sendHtml( - to = Person(user), - subject = s"Reset your password", - body = views.html.emails.passwordResetRequestCreated(appConfig, request.token).toString - ) - } - } - } - - case m @ EmailActor.Messages.EmailVerificationCreated(guid) => withVerboseErrorHandler(m) { - emailVerificationsDao.findByGuid(guid).foreach { verification => - usersDao.findByGuid(verification.userGuid).foreach { user => - email.sendHtml( - to = Person(email = verification.email, name = user.name), - subject = s"Verify your email address", - body = views.html.emails.emailVerificationCreated(appConfig, verification).toString - ) - } - } - } - - case m: Any => logUnhandledMessage(m) - } - -} - diff --git a/api/app/actors/Emails.scala b/api/app/actors/Emails.scala index 0a7adb03c..153209729 100644 --- a/api/app/actors/Emails.scala +++ b/api/app/actors/Emails.scala @@ -1,10 +1,11 @@ package actors -import io.apibuilder.api.v0.models._ import db.{ApplicationsDao, Authorization, MembershipsDao, SubscriptionsDao} -import javax.inject.{Inject, Singleton} +import io.apibuilder.api.v0.models._ import lib._ -import play.api.{Logger, Logging} +import play.api.Logging + +import javax.inject.{Inject, Singleton} object Emails { @@ -29,7 +30,6 @@ object Emails { @Singleton class Emails @Inject() ( - appConfig: AppConfig, email: EmailUtil, applicationsDao: ApplicationsDao, membershipsDao: MembershipsDao, @@ -46,11 +46,13 @@ class Emails @Inject() ( implicit filter: Subscription => Boolean = { _ => true } ): Unit = { eachSubscription(context, org, publication, { subscription => - email.sendHtml( - to = Person(subscription.user), - subject = subject, - body = body - ) + if (filter(subscription)) { + email.sendHtml( + to = Person(subscription.user), + subject = subject, + body = body + ) + } }) } @@ -108,23 +110,4 @@ class Emails @Inject() ( } } - def sendErrors( - subject: String, - errors: Seq[String] - ): Unit = { - errors match { - case Nil => {} - case _ => { - val body = views.html.emails.errors(errors).toString - appConfig.sendErrorsTo.foreach { emailAddress => - email.sendHtml( - to = Person(emailAddress), - subject = subject, - body = body - ) - } - } - } - } - } diff --git a/api/app/actors/GeneratorServiceActor.scala b/api/app/actors/GeneratorServiceActor.scala deleted file mode 100644 index bb72698c6..000000000 --- a/api/app/actors/GeneratorServiceActor.scala +++ /dev/null @@ -1,50 +0,0 @@ -package actors - -import akka.actor.{Actor, ActorLogging, ActorSystem} -import db.generators.ServicesDao -import play.api.Mode -import util.GeneratorServiceUtil - -import java.util.UUID -import javax.inject.Inject -import scala.concurrent.ExecutionContext -import scala.concurrent.duration._ - -sealed trait GeneratorServiceActorMessage -object GeneratorServiceActorMessage { - - case class GeneratorServiceCreated(guid: UUID) extends GeneratorServiceActorMessage - case object SyncAll extends GeneratorServiceActorMessage - -} - -@javax.inject.Singleton -class GeneratorServiceActor @javax.inject.Inject() ( - system: ActorSystem, - processor: GeneratorServiceActorProcessor, -) extends Actor with ActorLogging with ErrorHandler { - - private[this] implicit val ec: ExecutionContext = system.dispatchers.lookup("generator-service-actor-context") - - system.scheduler.scheduleAtFixedRate(1.hour, 1.hour, self, GeneratorServiceActorMessage.SyncAll) - - def receive: Receive = { - case m: GeneratorServiceActorMessage => processor.processMessage(m) - case other => logUnhandledMessage(other) - } - -} - -class GeneratorServiceActorProcessor @Inject() ( - servicesDao: ServicesDao, - util: GeneratorServiceUtil -) { - def processMessage(msg: GeneratorServiceActorMessage)(implicit ec: ExecutionContext): Unit = { - import GeneratorServiceActorMessage._ - msg match { - case GeneratorServiceCreated(guid) => util.sync(guid) - case SyncAll => util.syncAll() - } - } -} - diff --git a/api/app/actors/MainActor.scala b/api/app/actors/MainActor.scala deleted file mode 100644 index 8f7be2e56..000000000 --- a/api/app/actors/MainActor.scala +++ /dev/null @@ -1,127 +0,0 @@ -package actors - -import akka.actor._ -import db.InternalMigrationsDao -import lib.Role -import play.api.Mode -import util.ProcessDeletes - -import java.util.UUID -import scala.concurrent.ExecutionContext -import scala.concurrent.duration.{FiniteDuration, SECONDS} - -object MainActor { - - object Messages { - case class EmailVerificationCreated(guid: UUID) - case class MembershipRequestCreated(guid: UUID) - case class MembershipRequestAccepted(organizationGuid: UUID, userGuid: UUID, role: Role) - case class MembershipRequestDeclined(organizationGuid: UUID, userGuid: UUID, role: Role) - case class MembershipCreated(guid: UUID) - - case class PasswordResetRequestCreated(guid: UUID) - case class ApplicationCreated(guid: UUID) - case class UserCreated(guid: UUID) - - case class TaskCreated(guid: UUID) - case class GeneratorServiceCreated(guid: UUID) - } -} - -@javax.inject.Singleton -class MainActor @javax.inject.Inject() ( - app: play.api.Application, - system: ActorSystem, - internalMigrationsDao: InternalMigrationsDao, - processDeletes: ProcessDeletes, - @javax.inject.Named("email-actor") emailActor: akka.actor.ActorRef, - @javax.inject.Named("generator-service-actor") generatorServiceActor: akka.actor.ActorRef, - @javax.inject.Named("task-actor") taskActor: akka.actor.ActorRef, - @javax.inject.Named("user-actor") userActor: akka.actor.ActorRef -) extends Actor with ActorLogging with ErrorHandler { - - private[this] implicit val ec: ExecutionContext = system.dispatchers.lookup("main-actor-context") - - private[this] case object QueueVersionsToMigrate - private[this] case object MigrateVersions - private[this] case object CleanupDeletedApplications - - private[this] def scheduleOnce(msg: Any)(implicit delay: FiniteDuration = FiniteDuration(10, SECONDS)): Unit = { - system.scheduler.scheduleOnce(delay) { - self ! msg - } - } - - scheduleOnce(QueueVersionsToMigrate) - scheduleOnce(CleanupDeletedApplications) - - def receive = akka.event.LoggingReceive { - - case m @ MainActor.Messages.MembershipRequestCreated(guid) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.MembershipRequestCreated(guid) - } - - case m @ MainActor.Messages.MembershipRequestAccepted(organizationGuid, userGuid, role) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.MembershipRequestAccepted(organizationGuid, userGuid, role) - } - - case m @ MainActor.Messages.MembershipRequestDeclined(organizationGuid, userGuid, role) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.MembershipRequestDeclined(organizationGuid, userGuid, role) - } - - case m @ MainActor.Messages.MembershipCreated(guid) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.MembershipCreated(guid) - } - - case m @ MainActor.Messages.ApplicationCreated(guid) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.ApplicationCreated(guid) - } - - case m @ MainActor.Messages.TaskCreated(guid) => withVerboseErrorHandler(m) { - taskActor ! TaskActor.Messages.Created(guid) - } - - case m @ MainActor.Messages.GeneratorServiceCreated(guid) => withVerboseErrorHandler(m) { - generatorServiceActor ! GeneratorServiceActorMessage.GeneratorServiceCreated(guid) - } - - case m @ MainActor.Messages.EmailVerificationCreated(guid) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.EmailVerificationCreated(guid) - } - - case m @ MainActor.Messages.PasswordResetRequestCreated(guid) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.PasswordResetRequestCreated(guid) - } - - case m @ MainActor.Messages.UserCreated(guid) => withVerboseErrorHandler(m) { - userActor ! UserActor.Messages.UserCreated(guid) - } - - case m @ QueueVersionsToMigrate => withVerboseErrorHandler(m) { - app.mode match { - case Mode.Test => // No-op - case Mode.Prod | Mode.Dev => internalMigrationsDao.queueVersions() - } - } - - case m @ MigrateVersions => withVerboseErrorHandler(m) { - println(s"DEBUG_MigrateVersions - STARTING") - app.mode match { - case Mode.Test => // No-op - case Mode.Prod | Mode.Dev => { - if (internalMigrationsDao.migrateBatch(60)) { - scheduleOnce(MigrateVersions)(FiniteDuration(20, SECONDS)) - } - } - } - } - - case m @ CleanupDeletedApplications => withVerboseErrorHandler(m) { - println(s"DEBUG_CleanupDeletedApplications - STARTING") - processDeletes.all() - scheduleOnce(MigrateVersions) - } - - case m: Any => logUnhandledMessage(m) - } -} diff --git a/api/app/actors/PeriodicActor.scala b/api/app/actors/PeriodicActor.scala new file mode 100644 index 000000000..825786a1c --- /dev/null +++ b/api/app/actors/PeriodicActor.scala @@ -0,0 +1,53 @@ +package actors + +import akka.actor.{Actor, ActorLogging, Cancellable} +import db.InternalTasksDao +import io.apibuilder.task.v0.models.TaskType +import play.api.{Environment, Mode} + +import javax.inject.Inject +import scala.concurrent.ExecutionContext +import scala.concurrent.duration.{FiniteDuration, HOURS} + +class PeriodicActor @Inject() ( + tasksDao: InternalTasksDao, + env: Environment +) extends Actor with ActorLogging with ErrorHandler { + + private[this] implicit val ec: ExecutionContext = context.system.dispatchers.lookup("periodic-actor-context") + + private[this] case class UpsertTask(typ: TaskType) + + private[this] def schedule( + taskType: TaskType, + interval: FiniteDuration + )(implicit + initialInterval: FiniteDuration = interval + ): Cancellable = { + val finalInitial = env.mode match { + case Mode.Test => FiniteDuration(24, HOURS) + case _ => initialInterval + } + context.system.scheduler.scheduleWithFixedDelay(finalInitial, interval, self, UpsertTask(taskType)) + } + + private[this] val cancellables: Seq[Cancellable] = { + import TaskType._ + Seq( + schedule(CleanupDeletions, FiniteDuration(1, HOURS)), + schedule(ScheduleMigrateVersions, FiniteDuration(12, HOURS)), + schedule(ScheduleSyncGeneratorServices, FiniteDuration(1, HOURS)), + ) + } + + override def postStop(): Unit = { + cancellables.foreach(_.cancel()) + super.postStop() + } + + override def receive: Receive = { + case UpsertTask(taskType) => tasksDao.queue(taskType, "periodic") + case other => logUnhandledMessage(other) + } + +} diff --git a/api/app/actors/TaskActor.scala b/api/app/actors/TaskActor.scala index 5f504473f..23fd73f5c 100644 --- a/api/app/actors/TaskActor.scala +++ b/api/app/actors/TaskActor.scala @@ -1,231 +1,29 @@ package actors -import akka.actor.{Actor, ActorLogging, ActorSystem} -import io.apibuilder.api.v0.models.{Application, Diff, DiffBreaking, DiffNonBreaking, DiffUndefinedType, Organization, Publication, Version} -import io.apibuilder.internal.v0.models.{Task, TaskDataDiffVersion, TaskDataIndexApplication, TaskDataUndefinedType} -import db.{ApplicationsDao, Authorization, ChangesDao, OrganizationsDao, TasksDao, UsersDao, VersionsDao, WatchesDao} -import lib.{AppConfig, ServiceDiff, Text} +import akka.actor.{Actor, ActorLogging} +import com.google.inject.assistedinject.Assisted +import io.apibuilder.task.v0.models.TaskType +import processor.TaskActorCompanion -import java.util.UUID -import org.joda.time.DateTime -import play.twirl.api.Html - -import scala.concurrent.ExecutionContext -import scala.concurrent.duration._ -import scala.util.{Failure, Success, Try} +import javax.inject.Inject object TaskActor { - - object Messages { - case class Created(guid: UUID) - case object RestartDroppedTasks - case object PurgeOldTasks - case object NotifyFailed + case object Process + trait Factory { + def apply( + @Assisted("type") `type`: TaskType + ): Actor } - } -@javax.inject.Singleton -class TaskActor @javax.inject.Inject() ( - system: ActorSystem, - appConfig: AppConfig, - applicationsDao: ApplicationsDao, - changesDao: ChangesDao, - emails: Emails, - organizationsDao: OrganizationsDao, - search: Search, - tasksDao: TasksDao, - usersDao: UsersDao, - versionsDao: VersionsDao, - watchesDao: WatchesDao +class TaskActor @Inject() ( + @Assisted("type") `type`: TaskType, + companion: TaskActorCompanion ) extends Actor with ActorLogging with ErrorHandler { - private[this] implicit val ec: ExecutionContext = system.dispatchers.lookup("task-actor-context") - - private[this] val NumberDaysBeforePurge = 30 - private[this] case class Process(guid: UUID) - - system.scheduler.scheduleAtFixedRate(1.hour, 1.hour, self, TaskActor.Messages.RestartDroppedTasks) - system.scheduler.scheduleAtFixedRate(1.day, 1.day, self, TaskActor.Messages.NotifyFailed) - system.scheduler.scheduleAtFixedRate(1.day, 1.day, self, TaskActor.Messages.PurgeOldTasks) - def receive: Receive = { - - case m @ TaskActor.Messages.Created(guid) => withVerboseErrorHandler(m) { - self ! Process(guid) - } - - case m @ Process(guid) => withVerboseErrorHandler(m) { - tasksDao.findByGuid(guid).foreach { task => - tasksDao.incrementNumberAttempts(usersDao.AdminUser, task) - - task.data match { - case TaskDataDiffVersion(oldVersionGuid, newVersionGuid) => { - processTask(task, Try(diffVersion(oldVersionGuid, newVersionGuid))) - } - - case TaskDataIndexApplication(applicationGuid) => { - processTask(task, Try(search.indexApplication(applicationGuid))) - } - - case TaskDataUndefinedType(desc) => { - tasksDao.recordError(usersDao.AdminUser, task, "Task actor got an undefined data type: " + desc) - } - } - } - } - - case m @ TaskActor.Messages.RestartDroppedTasks => withVerboseErrorHandler(m) { - tasksDao.findAll( - nOrFewerAttempts = Some(2), - createdOnOrBefore = Some(DateTime.now.minusMinutes(1)) - ).foreach { task => - self ! Process(task.guid) - } - } - - case m @ TaskActor.Messages.NotifyFailed => withVerboseErrorHandler(m) { - val errors = tasksDao.findAll( - nOrMoreAttempts = Some(2), - isDeleted = Some(false), - createdOnOrAfter = Some(DateTime.now.minusDays(3)) - ).map { task => - val errorType = task.data match { - case TaskDataDiffVersion(a, b) => s"TaskDataDiffVersion($a, $b)" - case TaskDataIndexApplication(guid) => s"TaskDataIndexApplication($guid)" - case TaskDataUndefinedType(desc) => s"TaskDataUndefinedType($desc)" - } - - val errorMsg = Text.truncate(task.lastError.getOrElse("No information on error"), 500) - s"$errorType task ${task.guid}: $errorMsg" - } - emails.sendErrors( - subject = "One or more tasks failed", - errors = errors - ) - } - - case m @ TaskActor.Messages.PurgeOldTasks => withVerboseErrorHandler(m) { - tasksDao.findAll( - isDeleted = Some(true), - deletedAtLeastNDaysAgo = Some(NumberDaysBeforePurge) - ).foreach { task => - tasksDao.purge(task) - } - } - + case TaskActor.Process => companion.process(`type`) case m: Any => logUnhandledMessage(m) } - private[this] def diffVersion(oldVersionGuid: UUID, newVersionGuid: UUID): Unit = { - versionsDao.findByGuid(Authorization.All, oldVersionGuid, isDeleted = None).foreach { oldVersion => - versionsDao.findByGuid(Authorization.All, newVersionGuid).foreach { newVersion => - ServiceDiff(oldVersion.service, newVersion.service).differences match { - case Nil => { - // No-op - } - case diffs => { - changesDao.upsert( - createdBy = usersDao.AdminUser, - fromVersion = oldVersion, - toVersion = newVersion, - differences = diffs - ) - versionUpdated(newVersion, diffs) - versionUpdatedMaterialOnly(newVersion, diffs) - } - } - } - } - } - - private[this] def versionUpdated( - version: Version, - diffs: Seq[Diff], - ): Unit = { - if (diffs.nonEmpty) { - sendVersionUpsertedEmail( - publication = Publication.VersionsCreate, - version = version, - diffs = diffs, - ) { (org, application, breakingDiffs, nonBreakingDiffs) => - views.html.emails.versionUpserted( - appConfig, - org, - application, - version, - breakingDiffs = breakingDiffs, - nonBreakingDiffs = nonBreakingDiffs - ) - } - } - } - - private[this] def versionUpdatedMaterialOnly( - version: Version, - diffs: Seq[Diff], - ): Unit = { - val filtered = diffs.filter(_.isMaterial) - if (filtered.nonEmpty) { - sendVersionUpsertedEmail( - publication = Publication.VersionsMaterialChange, - version = version, - diffs = filtered, - ) { (org, application, breakingDiffs, nonBreakingDiffs) => - views.html.emails.versionUpserted( - appConfig, - org, - application, - version, - breakingDiffs = breakingDiffs, - nonBreakingDiffs = nonBreakingDiffs - ) - } - } - } - - private[this] def sendVersionUpsertedEmail( - publication: Publication, - version: Version, - diffs: Seq[Diff], - )( - generateBody: (Organization, Application, Seq[Diff], Seq[Diff]) => Html, - ): Unit = { - val (breakingDiffs, nonBreakingDiffs) = diffs.partition { - case _: DiffBreaking => true - case _: DiffNonBreaking => false - case _: DiffUndefinedType => true - } - - applicationsDao.findAll(Authorization.All, version = Some(version), limit = 1).foreach { application => - organizationsDao.findAll(Authorization.All, application = Some(application), limit = 1).foreach { org => - emails.deliver( - context = Emails.Context.Application(application), - org = org, - publication = publication, - subject = s"${org.name}/${application.name}:${version.version} Updated", - body = generateBody(org, application, breakingDiffs, nonBreakingDiffs).toString - ) { subscription => - watchesDao.findAll( - Authorization.All, - application = Some(application), - userGuid = Some(subscription.user.guid), - limit = 1 - ).nonEmpty - } - } - } - } - - def processTask[T](task: Task, attempt: Try[T]): Unit = { - attempt match { - case Success(_) => { - tasksDao.softDelete(usersDao.AdminUser, task) - } - case Failure(ex) => { - tasksDao.recordError(usersDao.AdminUser, task, ex) - } - } - } - } diff --git a/api/app/actors/TaskDispatchActor.scala b/api/app/actors/TaskDispatchActor.scala new file mode 100644 index 000000000..974e9127b --- /dev/null +++ b/api/app/actors/TaskDispatchActor.scala @@ -0,0 +1,63 @@ +package actors + +import akka.actor._ +import io.apibuilder.task.v0.models.TaskType +import play.api.libs.concurrent.InjectedActorSupport +import processor.TaskDispatchActorCompanion + +import javax.inject.{Inject, Singleton} +import scala.concurrent.ExecutionContext +import scala.concurrent.duration.{FiniteDuration, SECONDS} + +@Singleton +class TaskDispatchActor @Inject() ( + factory: TaskActor.Factory, + companion: TaskDispatchActorCompanion +) extends Actor with ActorLogging with ErrorHandler with InjectedActorSupport { + private[this] val ec: ExecutionContext = context.system.dispatchers.lookup("task-context") + private[this] case object Process + + private[this] val actors = scala.collection.mutable.Map[TaskType, ActorRef]() + + private[this] def schedule(message: Any, interval: FiniteDuration): Cancellable = { + context.system.scheduler.scheduleWithFixedDelay(FiniteDuration(1, SECONDS), interval, self, message)(ec) + } + + private[this] val cancellables: Seq[Cancellable] = { + Seq( + schedule(Process, FiniteDuration(2, SECONDS)) + ) + } + + override def postStop(): Unit = { + cancellables.foreach(_.cancel()) + super.postStop() + } + + override def receive: Receive = { + case Process => process() + case other => logUnhandledMessage(other) + } + + private[this] def process(): Unit = { + companion.typesWithWork.foreach { typ => + upsertActor(typ) ! TaskActor.Process + } + } + + private[this] def upsertActor(typ: TaskType): ActorRef = { + actors.getOrElse( + typ, { + val name = s"task$typ" + val ref = injectedChild( + factory(typ), + name = name, + _.withDispatcher("task-context-dispatcher") + ) + actors += (typ -> ref) + ref + } + ) + } + +} diff --git a/api/app/actors/UserActor.scala b/api/app/actors/UserActor.scala deleted file mode 100644 index e84589aa7..000000000 --- a/api/app/actors/UserActor.scala +++ /dev/null @@ -1,32 +0,0 @@ -package actors - -import akka.actor.{Actor, ActorLogging} -import db.UsersDao - -import java.util.UUID -import javax.inject.{Inject, Singleton} - -object UserActor { - - object Messages { - case class UserCreated(guid: UUID) - } - -} - -@Singleton -class UserActor @Inject() ( - usersDao: UsersDao -) extends Actor with ActorLogging with ErrorHandler { - - def receive = { - - case m @ UserActor.Messages.UserCreated(guid) => withVerboseErrorHandler(m) { - usersDao.processUserCreated(guid) - } - - case m: Any => logUnhandledMessage(m) - - } - -} diff --git a/api/app/db/ApplicationsDao.scala b/api/app/db/ApplicationsDao.scala index fad67096e..f7c6217db 100644 --- a/api/app/db/ApplicationsDao.scala +++ b/api/app/db/ApplicationsDao.scala @@ -2,20 +2,21 @@ package db import anorm._ import io.apibuilder.api.v0.models.{AppSortBy, Application, ApplicationForm, Error, MoveForm, Organization, SortOrder, User, Version, Visibility} -import io.apibuilder.internal.v0.models.TaskDataIndexApplication +import io.apibuilder.task.v0.models.{EmailDataApplicationCreated, TaskType} import io.flow.postgresql.Query import lib.{UrlKey, Validation} import play.api.db._ +import processor.EmailProcessorQueue import java.util.UUID -import javax.inject.{Inject, Named, Singleton} +import javax.inject.{Inject, Singleton} @Singleton class ApplicationsDao @Inject() ( - @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, + emailQueue: EmailProcessorQueue, organizationsDao: OrganizationsDao, - tasksDao: TasksDao + tasksDao: InternalTasksDao, ) { private[this] val dbHelpers = DbHelpers(db, "applications") @@ -163,7 +164,7 @@ class ApplicationsDao @Inject() ( val errors = validate(org, form, Some(app)) assert(errors.isEmpty, errors.map(_.message).mkString(" ")) - withTasks(updatedBy, app.guid, { implicit c => + withTasks(app.guid, { implicit c => SQL(UpdateQuery).on( "guid" -> app.guid, "name" -> form.name.trim, @@ -194,7 +195,7 @@ class ApplicationsDao @Inject() ( organizationsDao.findByKey(Authorization.All, form.orgKey) match { case None => sys.error(s"Could not find organization with key[${form.orgKey}]") case Some(newOrg) => { - withTasks(updatedBy, app.guid, { implicit c => + withTasks(app.guid, { implicit c => SQL(InsertMoveQuery).on( "guid" -> UUID.randomUUID, "application_guid" -> app.guid, @@ -228,7 +229,7 @@ class ApplicationsDao @Inject() ( "User[${user.guid}] not authorized to update app[${app.key}]" ) - withTasks(updatedBy, app.guid, { implicit c => + withTasks(app.guid, { implicit c => SQL(UpdateVisibilityQuery).on( "guid" -> app.guid, "visibility" -> visibility.toString, @@ -252,7 +253,7 @@ class ApplicationsDao @Inject() ( val guid = UUID.randomUUID val key = form.key.getOrElse(UrlKey.generate(form.name)) - withTasks(createdBy, guid, { implicit c => + withTasks(guid, { implicit c => SQL(InsertQuery).on( "guid" -> guid, "organization_guid" -> org.guid, @@ -263,26 +264,22 @@ class ApplicationsDao @Inject() ( "created_by_guid" -> createdBy.guid, "updated_by_guid" -> createdBy.guid ).execute() + emailQueue.queueWithConnection(c, EmailDataApplicationCreated(guid)) }) - mainActor ! actors.MainActor.Messages.ApplicationCreated(guid) - findAll(Authorization.All, orgKey = Some(org.key), key = Some(key)).headOption.getOrElse { sys.error("Failed to create application") } } def softDelete(deletedBy: User, application: Application): Unit = { - withTasks(deletedBy, application.guid, { c => + withTasks(application.guid, { c => dbHelpers.delete(c, deletedBy.guid, application.guid) }) } def canUserUpdate(user: User, app: Application): Boolean = { - findAll(Authorization.User(user.guid), key = Some(app.key)).headOption match { - case None => false - case Some(a) => true - } + findAll(Authorization.User(user.guid), key = Some(app.key)).nonEmpty } private[db] def findByOrganizationAndName(authorization: Authorization, org: Organization, name: String): Option[Application] = { @@ -359,15 +356,13 @@ class ApplicationsDao @Inject() ( } private[this] def withTasks( - user: User, guid: UUID, f: java.sql.Connection => Unit ): Unit = { - val taskGuid = db.withTransaction { implicit c => + db.withTransaction { implicit c => f(c) - tasksDao.insert(c, user, TaskDataIndexApplication(guid)) + tasksDao.queueWithConnection(c, TaskType.IndexApplication, guid.toString) } - mainActor ! actors.MainActor.Messages.TaskCreated(taskGuid) } } \ No newline at end of file diff --git a/api/app/db/EmailVerificationsDao.scala b/api/app/db/EmailVerificationsDao.scala index 13828f9a3..9cdcbef70 100644 --- a/api/app/db/EmailVerificationsDao.scala +++ b/api/app/db/EmailVerificationsDao.scala @@ -1,14 +1,17 @@ package db +import anorm.JodaParameterMetaData._ +import anorm._ import io.apibuilder.api.v0.models.User +import io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated import io.flow.postgresql.Query import lib.{Role, TokenGenerator} -import anorm._ -import anorm.JodaParameterMetaData._ -import javax.inject.{Inject, Named, Singleton} +import org.joda.time.DateTime import play.api.db._ +import processor.EmailProcessorQueue + import java.util.UUID -import org.joda.time.DateTime +import javax.inject.{Inject, Singleton} case class EmailVerification( guid: UUID, @@ -20,8 +23,8 @@ case class EmailVerification( @Singleton class EmailVerificationsDao @Inject() ( - @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, + emailQueue: EmailProcessorQueue, emailVerificationConfirmationsDao: EmailVerificationConfirmationsDao, membershipRequestsDao: MembershipRequestsDao, organizationsDao: OrganizationsDao @@ -56,7 +59,7 @@ class EmailVerificationsDao @Inject() ( def create(createdBy: User, user: User, email: String): EmailVerification = { val guid = UUID.randomUUID - db.withConnection { implicit c => + db.withTransaction { implicit c => SQL(InsertQuery).on( "guid" -> guid, "user_guid" -> user.guid, @@ -65,10 +68,9 @@ class EmailVerificationsDao @Inject() ( "expires_at" -> DateTime.now.plusHours(HoursUntilTokenExpires), "created_by_guid" -> createdBy.guid ).execute() + emailQueue.queueWithConnection(c, EmailDataEmailVerificationCreated(guid)) } - mainActor ! actors.MainActor.Messages.EmailVerificationCreated(guid) - findByGuid(guid).getOrElse { sys.error("Failed to create email verification") } diff --git a/api/app/db/InternalMigrationsDao.scala b/api/app/db/InternalMigrationsDao.scala deleted file mode 100644 index 494b23a2e..000000000 --- a/api/app/db/InternalMigrationsDao.scala +++ /dev/null @@ -1,97 +0,0 @@ -package db - -import anorm._ -import cats.data.Validated.{Invalid, Valid} -import db.generated.{MigrationForm, MigrationsDao} -import io.flow.postgresql.Query -import lib.Constants -import org.joda.time.DateTime -import play.api.db._ - -import javax.inject.Inject -import scala.util.{Failure, Success, Try} - -/** - * TODO: Test this class - * Setup a separate actor to: - * - delete if service_version < latest - * - attempt migration for num_attempts = 0 - * - record errors - */ -object Migration { - val ServiceVersionNumber: String = io.apibuilder.spec.v0.Constants.Version.toLowerCase -} - -class InternalMigrationsDao @Inject()( - @NamedDatabase("default") db: Database, - migrationsDao: MigrationsDao, - versionsDao: VersionsDao -) { - - private[this] val VersionsNeedingUpgrade = Query( - """ - |select v.guid - | from versions v - | join applications apps on apps.guid = v.application_guid and apps.deleted_at is null - | left join migrations m on m.version_guid = v.guid - | where v.deleted_at is null - | and m.id is null - | and not exists ( - | select 1 - | from cache.services - | where services.deleted_at is null - | and services.version_guid = v.guid - | and services.version = {service_version} - | ) - |limit 250 - |""".stripMargin - ).bind("service_version", Migration.ServiceVersionNumber) - - def queueVersions(): Unit = { - val versionGuids = db.withConnection { implicit c => - VersionsNeedingUpgrade - .as(SqlParser.get[_root_.java.util.UUID](1).*) - } - if (versionGuids.nonEmpty) { - migrationsDao.insertBatch(Constants.DefaultUserGuid, versionGuids.map { vGuid => - MigrationForm( - versionGuid = vGuid, - numAttempts = 0, - errors = None - ) - }) - queueVersions() - } - } - - def migrateBatch(limit: Long): Boolean = { - println(s"[${DateTime.now}] DEBUG STARTING migrateBatch($limit)") - val all = migrationsDao.findAll( - numAttempts = Some(0), - limit = Some(limit) - ) - all.foreach { migration => - Try { - migrateOne(migration) - } match { - case Success(_) => () - case Failure(ex) => setErrors(migration, Seq(ex.getMessage)) - } - } - all.nonEmpty - } - - private[this] def migrateOne(migration: _root_.db.generated.Migration): Unit = { - versionsDao.migrateVersionGuid(migration.versionGuid) match { - case Valid(_) => migrationsDao.delete(Constants.DefaultUserGuid, migration) - case Invalid(e) => setErrors(migration, e.toNonEmptyList.toList) - } - } - - private[this] def setErrors(migration: _root_.db.generated.Migration, errors: Seq[String]): Unit = { - migrationsDao.update(Constants.DefaultUserGuid, migration, migration.form.copy( - numAttempts = migration.numAttempts + 1, - errors = Some(errors) - )) - } -} diff --git a/api/app/db/InternalTasksDao.scala b/api/app/db/InternalTasksDao.scala new file mode 100644 index 000000000..6f87a0b1a --- /dev/null +++ b/api/app/db/InternalTasksDao.scala @@ -0,0 +1,59 @@ +package db + +import db.generated.{Task, TaskForm} +import io.apibuilder.task.v0.models.TaskType +import lib.Constants +import org.joda.time.DateTime +import play.api.libs.json.{JsValue, Json} + +import java.sql.Connection +import java.util.UUID +import javax.inject.Inject + +class InternalTasksDao @Inject() ( + dao: generated.TasksDao +) { + def findByTypeIdAndType(typeId: String, typ: TaskType): Option[Task] = { + dao.findByTypeIdAndType(typeId, typ.toString) + } + + // TODO: Use a bulk insert for this method + def queueBatch(typ: TaskType, ids: Seq[String]): Unit = { + ids.foreach { id => + queue(typ, id) + } + } + + def queue(typ: TaskType, id: String, organizationGuid: Option[UUID] = None, data: JsValue = Json.obj()): Unit = { + dao.db.withConnection { c => + queueWithConnection(c, typ, id, organizationGuid = organizationGuid, data = data) + } + } + + def queueWithConnection( + c: Connection, + typ: TaskType, + id: String, + organizationGuid: Option[UUID] = None, + data: JsValue = Json.obj() + ): Unit = { + if (dao.findByTypeIdAndTypeWithConnection(c, id, typ.toString).isEmpty) { + dao.upsertByTypeIdAndType( + c, + Constants.DefaultUserGuid, + TaskForm( + id = s"$typ:$id", + `type` = typ.toString, + typeId = id, + organizationGuid = organizationGuid, + data = data, + numAttempts = 0, + nextAttemptAt = DateTime.now, + errors = None, + stacktrace = None + ) + ) + } + } + +} diff --git a/api/app/db/MembershipRequestsDao.scala b/api/app/db/MembershipRequestsDao.scala index bc36b70c4..e949510c9 100644 --- a/api/app/db/MembershipRequestsDao.scala +++ b/api/app/db/MembershipRequestsDao.scala @@ -1,17 +1,20 @@ package db +import anorm._ import io.apibuilder.api.v0.models.{MembershipRequest, Organization, User} +import io.apibuilder.task.v0.models.{EmailDataMembershipRequestAccepted, EmailDataMembershipRequestCreated, EmailDataMembershipRequestDeclined} import io.flow.postgresql.Query import lib.Role -import anorm._ -import javax.inject.{Inject, Named, Singleton} import play.api.db._ +import processor.EmailProcessorQueue + import java.util.UUID +import javax.inject.{Inject, Singleton} @Singleton class MembershipRequestsDao @Inject() ( - @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, + emailQueue: EmailProcessorQueue, membershipsDao: MembershipsDao, organizationLogsDao: OrganizationLogsDao ) { @@ -59,7 +62,7 @@ class MembershipRequestsDao @Inject() ( private[db] def create(createdBy: User, organization: Organization, user: User, role: Role): MembershipRequest = { val guid = UUID.randomUUID - db.withConnection { implicit c => + db.withTransaction { implicit c => SQL(InsertQuery).on( "guid" -> guid, "organization_guid" -> organization.guid, @@ -67,10 +70,9 @@ class MembershipRequestsDao @Inject() ( "role" -> role.key, "created_by_guid" -> createdBy.guid ).execute() + emailQueue.queueWithConnection(c, EmailDataMembershipRequestCreated(guid)) } - mainActor ! actors.MainActor.Messages.MembershipRequestCreated(guid) - findByGuid(Authorization.All, guid).getOrElse { sys.error("Failed to create membership_request") } @@ -101,13 +103,12 @@ class MembershipRequestsDao @Inject() ( sys.error(s"Invalid role[${request.role}]") } - db.withTransaction { implicit conn => + db.withTransaction { implicit c => organizationLogsDao.create(createdBy, request.organization, message) - dbHelpers.delete(conn, createdBy, request.guid) + dbHelpers.delete(c, createdBy, request.guid) membershipsDao.upsert(createdBy, request.organization, request.user, r) + emailQueue.queueWithConnection(c, EmailDataMembershipRequestAccepted(request.organization.guid, request.user.guid, r.toString)) } - - mainActor ! actors.MainActor.Messages.MembershipRequestAccepted(request.organization.guid, request.user.guid, r) } /** @@ -121,12 +122,11 @@ class MembershipRequestsDao @Inject() ( } val message = s"Declined membership request for ${request.user.email} to join as ${r.name}" - db.withTransaction { implicit conn => + db.withTransaction { implicit c => organizationLogsDao.create(createdBy.guid, request.organization, message) softDelete(createdBy, request) + emailQueue.queueWithConnection(c, EmailDataMembershipRequestDeclined(request.organization.guid, request.user.guid, r.toString)) } - - mainActor ! actors.MainActor.Messages.MembershipRequestDeclined(request.organization.guid, request.user.guid, r) } private[this] def assertUserCanReview(user: User, request: MembershipRequest): Unit = { diff --git a/api/app/db/MembershipsDao.scala b/api/app/db/MembershipsDao.scala index f8f640f66..91df2fa90 100644 --- a/api/app/db/MembershipsDao.scala +++ b/api/app/db/MembershipsDao.scala @@ -1,19 +1,22 @@ package db +import anorm._ import io.apibuilder.api.v0.models.{Membership, Organization, User} import io.apibuilder.common.v0.models.{Audit, ReferenceGuid} +import io.apibuilder.task.v0.models.EmailDataMembershipCreated import io.flow.postgresql.Query import lib.Role -import anorm._ -import javax.inject.{Inject, Named, Singleton} +import org.joda.time.DateTime import play.api.db._ +import processor.EmailProcessorQueue + import java.util.UUID -import org.joda.time.DateTime +import javax.inject.{Inject, Singleton} @Singleton class MembershipsDao @Inject() ( - @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, + emailQueue: EmailProcessorQueue, subscriptionsDao: SubscriptionsDao ) { @@ -67,7 +70,7 @@ class MembershipsDao @Inject() ( } private[db] def create(createdBy: UUID, organization: Organization, user: User, role: Role): Membership = { - db.withConnection { implicit c => + db.withTransaction { implicit c => create(c, createdBy, organization, user, role) } } @@ -94,7 +97,7 @@ class MembershipsDao @Inject() ( "created_by_guid" -> createdBy ).execute() - mainActor ! actors.MainActor.Messages.MembershipCreated(membership.guid) + emailQueue.queueWithConnection(c, EmailDataMembershipCreated(membership.guid)) membership } diff --git a/api/app/db/PasswordResetRequestsDao.scala b/api/app/db/PasswordResetRequestsDao.scala index 4f37c0be6..00bad985d 100644 --- a/api/app/db/PasswordResetRequestsDao.scala +++ b/api/app/db/PasswordResetRequestsDao.scala @@ -1,15 +1,17 @@ package db -import io.apibuilder.api.v0.models.User -import lib.TokenGenerator -import anorm._ import anorm.JodaParameterMetaData._ -import java.util.UUID -import javax.inject.{Inject, Named, Singleton} - -import play.api.db._ +import anorm._ +import io.apibuilder.api.v0.models.User +import io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated import io.flow.postgresql.Query +import lib.TokenGenerator import org.joda.time.DateTime +import play.api.db._ +import processor.EmailProcessorQueue + +import java.util.UUID +import javax.inject.{Inject, Singleton} case class PasswordReset( guid: UUID, @@ -20,8 +22,8 @@ case class PasswordReset( @Singleton class PasswordResetRequestsDao @Inject() ( - @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, + emailQueue: EmailProcessorQueue, userPasswordsDao: UserPasswordsDao, usersDao: UsersDao ) { @@ -48,7 +50,7 @@ class PasswordResetRequestsDao @Inject() ( def create(createdBy: Option[User], user: User): PasswordReset = { val guid = UUID.randomUUID - db.withConnection { implicit c => + db.withTransaction { implicit c => SQL(InsertQuery).on( "guid" -> guid, "user_guid" -> user.guid, @@ -56,10 +58,9 @@ class PasswordResetRequestsDao @Inject() ( "expires_at" -> DateTime.now.plusHours(HoursUntilTokenExpires), "created_by_guid" -> createdBy.getOrElse(user).guid ).execute() + emailQueue.queueWithConnection(c, EmailDataPasswordResetRequestCreated(guid)) } - mainActor ! actors.MainActor.Messages.PasswordResetRequestCreated(guid) - findByGuid(guid).getOrElse { sys.error("Failed to create password reset") } diff --git a/api/app/db/TasksDao.scala b/api/app/db/TasksDao.scala deleted file mode 100644 index 5557f12e5..000000000 --- a/api/app/db/TasksDao.scala +++ /dev/null @@ -1,189 +0,0 @@ -package db - -import anorm._ -import io.apibuilder.internal.v0.models.{Task, TaskData} -import io.apibuilder.internal.v0.models.json._ -import io.apibuilder.api.v0.models.User -import io.flow.postgresql.Query -import javax.inject.{Inject, Named, Singleton} -import org.joda.time.DateTime -import play.api.db._ -import play.api.libs.json._ -import java.util.UUID - -/** - * A task represents a bit of work that should be completed asynchronously. Example tasks include: - * - * - generating a diff between two versions of a service - * - updates a search index for a particular service - * - * Tasks provide for the following semantics: - * - transactionally record to execute a task - * - at least once execution - * - near real time execution - * - mostly sequential execution of tasks (but not guaranteed) - * - * The implementation works by: - * - caller inserts a task transactionally - * - to trigger real time execution, send a TaskCreated message to - * the Main Actor - send this AFTER the transaction commits or the - * actor may not see the task - * - periodically, the task system picks up tasks that have not - * been processed and executes them - */ -@Singleton -class TasksDao @Inject() ( - @NamedDatabase("default") db: Database, - @Named("main-actor") mainActor: akka.actor.ActorRef -) { - - private[this] val dbHelpers = DbHelpers(db, "tasks") - - private[this] val BaseQuery = Query(""" - select tasks.guid, - tasks.data::text, - tasks.number_attempts, - tasks.last_error - from tasks - """) - - val IncrementNumberAttemptsQuery = """ - update tasks - set number_attempts = number_attempts + 1, - updated_by_guid = {updated_by_guid}::uuid - where guid = {guid}::uuid - """ - - val RecordErrorQuery = """ - update tasks - set last_error = {last_error}, - updated_by_guid = {updated_by_guid}::uuid - where guid = {guid}::uuid - """ - - private[this] val InsertQuery = """ - insert into tasks - (guid, data, created_by_guid, updated_by_guid) - values - ({guid}::uuid, {data}::json, {created_by_guid}::uuid, {updated_by_guid}::uuid) - """ - - private[this] val PurgeQuery = """ - delete from tasks where guid = {guid}::uuid - """ - - def create(createdBy: User, data: TaskData): UUID = { - val taskGuid = db.withConnection { implicit c => - insert(c, createdBy, data) - } - mainActor ! actors.MainActor.Messages.TaskCreated(taskGuid) - taskGuid - } - - private[db] def insert(implicit c: java.sql.Connection, createdBy: User, data: TaskData): UUID = { - val guid = UUID.randomUUID - - SQL(InsertQuery).on( - "guid" -> guid, - "data" -> Json.toJson(data).toString, - "created_by_guid" -> createdBy.guid, - "updated_by_guid" -> createdBy.guid - ).execute() - - guid - } - - def softDelete(deletedBy: User, task: Task): Unit = { - dbHelpers.delete(deletedBy, task.guid) - } - - def purge(task: Task): Unit = { - db.withConnection { implicit c => - SQL(PurgeQuery).on("guid" -> task.guid).execute() - } - } - - def incrementNumberAttempts(user: User, task: Task): Unit = { - db.withConnection { implicit c => - SQL(IncrementNumberAttemptsQuery).on( - "guid" -> task.guid, - "updated_by_guid" -> user.guid - ).execute() - } - } - - def recordError(user: User, task: Task, error: Throwable): Unit = { - val sw = new java.io.StringWriter - error.printStackTrace(new java.io.PrintWriter(sw)) - recordError(user, task, sw.toString) - } - - def recordError(user: User, task: Task, error: String): Unit = { - db.withConnection { implicit c => - SQL(RecordErrorQuery).on( - "guid" -> task.guid, - "last_error" -> error, - "updated_by_guid" -> user.guid - ).execute() - } - } - - def findByGuid(guid: UUID): Option[Task] = { - findAll(guid = Some(guid), limit = 1).headOption - } - - def findAll( - guid: Option[UUID] = None, - nOrFewerAttempts: Option[Long] = None, - nOrMoreAttempts: Option[Long] = None, - createdOnOrBefore: Option[DateTime] = None, - createdOnOrAfter: Option[DateTime] = None, - isDeleted: Option[Boolean] = Some(false), - deletedAtLeastNDaysAgo: Option[Long] = None, - limit: Long = 25, - offset: Long = 0 - ): Seq[Task] = { - deletedAtLeastNDaysAgo.foreach { _ => - isDeleted match { - case None | Some(true) => // no-op - case Some(false) => sys.error("When filtering by deletedAtLeastNDaysAgo, you must also set isDeleted") - } - } - - db.withConnection { implicit c => - BaseQuery. - equals("tasks.guid", guid). - lessThanOrEquals("tasks.number_attempts", nOrFewerAttempts). - greaterThanOrEquals("tasks.number_attempts", nOrMoreAttempts). - lessThanOrEquals("tasks.created_at", createdOnOrBefore). - greaterThanOrEquals("tasks.created_at", createdOnOrAfter). - and(isDeleted.map { - Filters.isDeleted("tasks", _) - }). - lessThanOrEquals("tasks.deleted_at", deletedAtLeastNDaysAgo.map(days => DateTime.now.minusDays(days.toInt))). - orderBy("tasks.number_attempts, tasks.created_at"). - limit(limit). - offset(offset). - anormSql().as( - parser().* - ) - } - } - - private[this] def parser(): RowParser[Task] = { - SqlParser.get[UUID]("guid") ~ - SqlParser.str("data") ~ - SqlParser.long("number_attempts") ~ - SqlParser.str("last_error").? map { - case guid ~ data ~ numberAttempts ~ lastError => { - Task( - guid = guid, - data = Json.parse(data).as[TaskData], - numberAttempts = numberAttempts, - lastError = lastError - ) - } - } - } - -} diff --git a/api/app/db/UsersDao.scala b/api/app/db/UsersDao.scala index 2dba70fc0..4a990f79d 100644 --- a/api/app/db/UsersDao.scala +++ b/api/app/db/UsersDao.scala @@ -1,22 +1,21 @@ package db +import anorm._ import io.apibuilder.api.v0.models.{Error, User, UserForm, UserUpdateForm} +import io.apibuilder.task.v0.models.TaskType import io.flow.postgresql.Query -import lib.{Constants, Misc, Role, UrlKey, Validation} -import anorm._ -import javax.inject.{Inject, Named, Singleton} - +import lib.{Constants, Misc, UrlKey, Validation} import play.api.db._ -import java.util.UUID - import play.api.inject.Injector +import java.util.UUID +import javax.inject.{Inject, Singleton} import scala.annotation.tailrec import scala.util.{Failure, Success, Try} object UsersDao { - val AdminUserEmails: Seq[String] = Seq("admin@apibuilder.io") + private val AdminUserEmails: Seq[String] = Seq("admin@apibuilder.io") val AdminUserGuid: UUID = UUID.fromString("f3973f60-be9f-11e3-b1b6-0800200c9a66") @@ -25,11 +24,10 @@ object UsersDao { @Singleton class UsersDao @Inject() ( @NamedDatabase("default") db: Database, - @Named("main-actor") mainActor: akka.actor.ActorRef, injector: Injector, emailVerificationsDao: EmailVerificationsDao, - organizationsDao: OrganizationsDao, - userPasswordsDao: UserPasswordsDao + userPasswordsDao: UserPasswordsDao, + internalTasksDao: InternalTasksDao, ) { // TODO: Inject directly - here because of circular references @@ -153,7 +151,7 @@ class UsersDao @Inject() ( gravatarId: Option[String] ): User = { val nickname = generateNickname(login) - val guid = db.withConnection { implicit c => + val guid = db.withTransaction { implicit c => doInsert( nickname = nickname, email = email, @@ -163,8 +161,6 @@ class UsersDao @Inject() ( ) } - mainActor ! actors.MainActor.Messages.UserCreated(guid) - findByGuid(guid).getOrElse { sys.error("Failed to create user") } @@ -197,8 +193,6 @@ class UsersDao @Inject() ( id } - mainActor ! actors.MainActor.Messages.UserCreated(guid) - findByGuid(guid).getOrElse { sys.error("Failed to create user") } @@ -224,16 +218,9 @@ class UsersDao @Inject() ( "updated_by_guid" -> Constants.DefaultUserGuid.toString ).execute() - guid - } + internalTasksDao.queueWithConnection(c, TaskType.UserCreated, guid.toString) - def processUserCreated(guid: UUID): Unit = { - findByGuid(guid).foreach { user => - organizationsDao.findAllByEmailDomain(user.email).foreach { org => - membershipRequestsDao.upsert(user, org, user, Role.Member) - } - emailVerificationsDao.create(user, user, user.email) - } + guid } def findByToken(token: String): Option[User] = { diff --git a/api/app/db/VersionsDao.scala b/api/app/db/VersionsDao.scala index 18346dbba..34d7c13be 100644 --- a/api/app/db/VersionsDao.scala +++ b/api/app/db/VersionsDao.scala @@ -8,24 +8,25 @@ import cats.implicits._ import core.{ServiceFetcher, VersionMigration} import io.apibuilder.api.v0.models._ import io.apibuilder.common.v0.models.{Audit, Reference} -import io.apibuilder.internal.v0.models.{TaskDataDiffVersion, TaskDataIndexApplication} import io.apibuilder.spec.v0.models.Service import io.apibuilder.spec.v0.models.json._ +import io.apibuilder.task.v0.models._ +import io.apibuilder.task.v0.models.json._ import io.flow.postgresql.Query import lib.{ServiceConfiguration, ServiceUri, ValidatedHelpers, VersionTag} import play.api.Logger import play.api.db._ import play.api.libs.json._ +import processor.MigrateVersion import java.util.UUID -import javax.inject.{Inject, Named} +import javax.inject.Inject class VersionsDao @Inject() ( @NamedDatabase("default") db: Database, - @Named("main-actor") mainActor: akka.actor.ActorRef, applicationsDao: ApplicationsDao, originalsDao: OriginalsDao, - tasksDao: TasksDao, + tasksDao: InternalTasksDao, usersDao: UsersDao, organizationsDao: OrganizationsDao, serviceParser: ServiceParser @@ -106,19 +107,15 @@ class VersionsDao @Inject() ( limit = 1 ).headOption - val (guid, taskGuid) = db.withTransaction { implicit c => + val guid = db.withTransaction { implicit c => val versionGuid = doCreate(c, user, application, version, original, service) - val taskGuid = latestVersion.map { v => - createDiffTask(user, v.guid, versionGuid) + latestVersion.foreach { v => + createDiffTask(v.guid, versionGuid) } - (versionGuid, taskGuid) + versionGuid } - taskGuid.foreach { guid => - mainActor ! actors.MainActor.Messages.TaskCreated(guid) - } - - findAll(Authorization.All, guid = Some(guid), limit = 1).headOption.getOrElse { + findByGuid(Authorization.All, guid).getOrElse { sys.error("Failed to create version") } } @@ -161,27 +158,29 @@ class VersionsDao @Inject() ( } private[this] def createDiffTask( - user: User, oldVersionGuid: UUID, newVersionGuid: UUID + oldVersionGuid: UUID, + newVersionGuid: UUID ) ( implicit c: java.sql.Connection - ): UUID = { - tasksDao.insert(c, user, TaskDataDiffVersion(oldVersionGuid = oldVersionGuid, newVersionGuid = newVersionGuid)) + ): Unit = { + tasksDao.queueWithConnection( + c, + TaskType.DiffVersion, + newVersionGuid.toString, + data = Json.toJson(DiffVersionData(oldVersionGuid = oldVersionGuid, newVersionGuid = newVersionGuid)) + ) } def replace(user: User, version: Version, application: Application, original: Original, service: Service): Version = { - val (versionGuid, taskGuids) = db.withTransaction { implicit c => + val versionGuid = db.withTransaction { implicit c => softDelete(user, version) val versionGuid = doCreate(c, user, application, version.version, original, service) - val diffTaskGuid = createDiffTask(user, version.guid, versionGuid) - val indexTaskGuid = tasksDao.insert(c, user, TaskDataIndexApplication(application.guid)) - (versionGuid, Seq(diffTaskGuid, indexTaskGuid)) - } - - taskGuids.foreach { taskGuid => - mainActor ! actors.MainActor.Messages.TaskCreated(taskGuid) + createDiffTask(version.guid, versionGuid) + tasksDao.queueWithConnection(c, TaskType.IndexApplication, application.guid.toString) + versionGuid } - findAll(Authorization.All, guid = Some(versionGuid), limit = 1).headOption.getOrElse { + findByGuid(Authorization.All, versionGuid).getOrElse { sys.error(s"Failed to replace version[${version.guid}]") } } @@ -370,7 +369,7 @@ class VersionsDao @Inject() ( orgNamespace = org.namespace, version = versionName ) - logger.info(s"Migrating $orgKey/$applicationKey/$versionName versionGuid[$versionGuid] to latest API Builder spec version[${Migration.ServiceVersionNumber}] (with serviceConfig=$serviceConfig)") + logger.info(s"Migrating $orgKey/$applicationKey/$versionName versionGuid[$versionGuid] to latest API Builder spec version[${MigrateVersion.ServiceVersionNumber}] (with serviceConfig=$serviceConfig)") val validator = OriginalValidator( config = serviceConfig, @@ -395,7 +394,7 @@ class VersionsDao @Inject() ( ): Unit = { SQL(SoftDeleteServiceByVersionGuidAndVersionNumberQuery).on( "version_guid" -> versionGuid, - "version" -> Migration.ServiceVersionNumber, + "version" -> MigrateVersion.ServiceVersionNumber, "user_guid" -> user.guid ).execute() } @@ -409,7 +408,7 @@ class VersionsDao @Inject() ( SQL(InsertServiceQuery).on( "guid" -> UUID.randomUUID, "version_guid" -> versionGuid, - "version" -> Migration.ServiceVersionNumber, + "version" -> MigrateVersion.ServiceVersionNumber, "json" -> Json.toJson(service).as[JsObject].toString.trim, "user_guid" -> user.guid ).execute() diff --git a/api/app/db/generated/PsqlApibuilderMigrationsDao.scala b/api/app/db/generated/PsqlApibuilderMigrationsDao.scala deleted file mode 100644 index 1fa057fda..000000000 --- a/api/app/db/generated/PsqlApibuilderMigrationsDao.scala +++ /dev/null @@ -1,494 +0,0 @@ -package db.generated - -import anorm._ -import io.flow.postgresql.{OrderBy, Query} -import java.sql.Connection -import java.util.UUID -import javax.inject.{Inject, Singleton} -import org.joda.time.DateTime -import play.api.db.Database -import play.api.libs.json.Json -import util.IdGenerator - -case class Migration( - id: String, - versionGuid: UUID, - numAttempts: Long, - errors: Option[Seq[String]], - updatedByGuid: String, - createdAt: DateTime, - updatedAt: DateTime -) { - - lazy val form: MigrationForm = MigrationForm( - versionGuid = versionGuid, - numAttempts = numAttempts, - errors = errors - ) - -} - -case class MigrationForm( - versionGuid: UUID, - numAttempts: Long, - errors: Option[Seq[String]] -) - -object MigrationsTable { - val Schema: String = "public" - val Name: String = "migrations" - val QualifiedName: String = s"$Schema.$Name" - - object Columns { - val Id: String = "id" - val VersionGuid: String = "version_guid" - val NumAttempts: String = "num_attempts" - val Errors: String = "errors" - val UpdatedByGuid: String = "updated_by_guid" - val CreatedAt: String = "created_at" - val UpdatedAt: String = "updated_at" - val HashCode: String = "hash_code" - val all: List[String] = List(Id, VersionGuid, NumAttempts, Errors, UpdatedByGuid, CreatedAt, UpdatedAt, HashCode) - } -} - -trait BaseMigrationsDao { - - def db: Database - - private[this] val BaseQuery = Query(""" - | select migrations.id, - | migrations.version_guid, - | migrations.num_attempts, - | migrations.errors::text as errors_text, - | migrations.updated_by_guid, - | migrations.created_at, - | migrations.updated_at, - | migrations.hash_code - | from migrations - """.stripMargin) - - def findById(id: String): Option[Migration] = { - db.withConnection { c => - findByIdWithConnection(c, id) - } - } - - def findByIdWithConnection(c: java.sql.Connection, id: String): Option[Migration] = { - findAllWithConnection(c, ids = Some(Seq(id)), limit = Some(1L), orderBy = None).headOption - } - - def iterateAll( - ids: Option[Seq[String]] = None, - versionGuid: Option[UUID] = None, - numAttempts: Option[Long] = None, - numAttemptsGreaterThanOrEquals: Option[Long] = None, - numAttemptsGreaterThan: Option[Long] = None, - numAttemptsLessThanOrEquals: Option[Long] = None, - numAttemptsLessThan: Option[Long] = None, - numAttemptses: Option[Seq[Long]] = None, - createdAt: Option[DateTime] = None, - createdAtGreaterThanOrEquals: Option[DateTime] = None, - createdAtGreaterThan: Option[DateTime] = None, - createdAtLessThanOrEquals: Option[DateTime] = None, - createdAtLessThan: Option[DateTime] = None, - numAttemptsCreatedAts: Option[Seq[(Long, DateTime)]] = None, - pageSize: Long = 2000L, - ) ( - implicit customQueryModifier: Query => Query = { q => q } - ): Iterator[Migration] = { - def iterate(lastValue: Option[Migration]): Iterator[Migration] = { - val page = findAll( - ids = ids, - versionGuid = versionGuid, - numAttempts = numAttempts, - numAttemptsGreaterThanOrEquals = numAttemptsGreaterThanOrEquals, - numAttemptsGreaterThan = numAttemptsGreaterThan, - numAttemptsLessThanOrEquals = numAttemptsLessThanOrEquals, - numAttemptsLessThan = numAttemptsLessThan, - numAttemptses = numAttemptses, - createdAt = createdAt, - createdAtGreaterThanOrEquals = createdAtGreaterThanOrEquals, - createdAtGreaterThan = createdAtGreaterThan, - createdAtLessThanOrEquals = createdAtLessThanOrEquals, - createdAtLessThan = createdAtLessThan, - numAttemptsCreatedAts = numAttemptsCreatedAts, - limit = Some(pageSize), - orderBy = Some(OrderBy("migrations.id")), - ) { q => customQueryModifier(q).greaterThan("migrations.id", lastValue.map(_.id)) } - - page.lastOption match { - case None => Iterator.empty - case lastValue => page.iterator ++ iterate(lastValue) - } - } - - iterate(None) - } - - def findAll( - ids: Option[Seq[String]] = None, - versionGuid: Option[UUID] = None, - numAttempts: Option[Long] = None, - numAttemptsGreaterThanOrEquals: Option[Long] = None, - numAttemptsGreaterThan: Option[Long] = None, - numAttemptsLessThanOrEquals: Option[Long] = None, - numAttemptsLessThan: Option[Long] = None, - numAttemptses: Option[Seq[Long]] = None, - createdAt: Option[DateTime] = None, - createdAtGreaterThanOrEquals: Option[DateTime] = None, - createdAtGreaterThan: Option[DateTime] = None, - createdAtLessThanOrEquals: Option[DateTime] = None, - createdAtLessThan: Option[DateTime] = None, - numAttemptsCreatedAts: Option[Seq[(Long, DateTime)]] = None, - limit: Option[Long], - offset: Long = 0, - orderBy: Option[OrderBy] = Some(OrderBy("migrations.id")) - ) ( - implicit customQueryModifier: Query => Query = { q => q } - ): Seq[Migration] = { - db.withConnection { c => - findAllWithConnection( - c, - ids = ids, - versionGuid = versionGuid, - numAttempts = numAttempts, - numAttemptsGreaterThanOrEquals = numAttemptsGreaterThanOrEquals, - numAttemptsGreaterThan = numAttemptsGreaterThan, - numAttemptsLessThanOrEquals = numAttemptsLessThanOrEquals, - numAttemptsLessThan = numAttemptsLessThan, - numAttemptses = numAttemptses, - createdAt = createdAt, - createdAtGreaterThanOrEquals = createdAtGreaterThanOrEquals, - createdAtGreaterThan = createdAtGreaterThan, - createdAtLessThanOrEquals = createdAtLessThanOrEquals, - createdAtLessThan = createdAtLessThan, - numAttemptsCreatedAts = numAttemptsCreatedAts, - limit = limit, - offset = offset, - orderBy = orderBy - )(customQueryModifier) - } - } - - def findAllWithConnection( - c: java.sql.Connection, - ids: Option[Seq[String]] = None, - versionGuid: Option[UUID] = None, - numAttempts: Option[Long] = None, - numAttemptsGreaterThanOrEquals: Option[Long] = None, - numAttemptsGreaterThan: Option[Long] = None, - numAttemptsLessThanOrEquals: Option[Long] = None, - numAttemptsLessThan: Option[Long] = None, - numAttemptses: Option[Seq[Long]] = None, - createdAt: Option[DateTime] = None, - createdAtGreaterThanOrEquals: Option[DateTime] = None, - createdAtGreaterThan: Option[DateTime] = None, - createdAtLessThanOrEquals: Option[DateTime] = None, - createdAtLessThan: Option[DateTime] = None, - numAttemptsCreatedAts: Option[Seq[(Long, DateTime)]] = None, - limit: Option[Long], - offset: Long = 0, - orderBy: Option[OrderBy] = Some(OrderBy("migrations.id")) - ) ( - implicit customQueryModifier: Query => Query = { q => q } - ): Seq[Migration] = { - customQueryModifier(BaseQuery). - optionalIn("migrations.id", ids). - equals("migrations.version_guid", versionGuid). - equals("migrations.num_attempts", numAttempts). - greaterThanOrEquals("migrations.num_attempts", numAttemptsGreaterThanOrEquals). - greaterThan("migrations.num_attempts", numAttemptsGreaterThan). - lessThanOrEquals("migrations.num_attempts", numAttemptsLessThanOrEquals). - lessThan("migrations.num_attempts", numAttemptsLessThan). - optionalIn("migrations.num_attempts", numAttemptses). - equals("migrations.created_at", createdAt). - greaterThanOrEquals("migrations.created_at", createdAtGreaterThanOrEquals). - greaterThan("migrations.created_at", createdAtGreaterThan). - lessThanOrEquals("migrations.created_at", createdAtLessThanOrEquals). - lessThan("migrations.created_at", createdAtLessThan). - optionalIn2(("migrations.num_attempts", "migrations.created_at"), numAttemptsCreatedAts). - optionalLimit(limit). - offset(offset). - orderBy(orderBy.flatMap(_.sql)). - as(MigrationsDao.parser.*)(c) - } - -} - -object MigrationsDao { - - val parser: RowParser[Migration] = { - SqlParser.str("id") ~ - SqlParser.get[UUID]("version_guid") ~ - SqlParser.long("num_attempts") ~ - SqlParser.str("errors_text").? ~ - SqlParser.str("updated_by_guid") ~ - SqlParser.get[DateTime]("created_at") ~ - SqlParser.get[DateTime]("updated_at") map { - case id ~ versionGuid ~ numAttempts ~ errors ~ updatedByGuid ~ createdAt ~ updatedAt => Migration( - id = id, - versionGuid = versionGuid, - numAttempts = numAttempts, - errors = errors.map { text => Json.parse(text).as[Seq[String]] }, - updatedByGuid = updatedByGuid, - createdAt = createdAt, - updatedAt = updatedAt - ) - } - } - -} - -@Singleton -class MigrationsDao @Inject() ( - override val db: Database -) extends BaseMigrationsDao { - - private[this] val idGenerator = util.IdGenerator() - - def randomId(): String = idGenerator.randomId() - - private[this] val InsertQuery = Query(""" - | insert into migrations - | (id, version_guid, num_attempts, errors, updated_by_guid, hash_code) - | values - | ({id}, {version_guid}::uuid, {num_attempts}::bigint, {errors}::json, {updated_by_guid}, {hash_code}::bigint) - """.stripMargin) - - private[this] val UpdateQuery = Query(""" - | update migrations - | set version_guid = {version_guid}::uuid, - | num_attempts = {num_attempts}::bigint, - | errors = {errors}::json, - | updated_by_guid = {updated_by_guid}, - | hash_code = {hash_code}::bigint - | where id = {id} - | and migrations.hash_code != {hash_code}::bigint - """.stripMargin) - - private[this] def bindQuery(query: Query, form: MigrationForm): Query = { - query. - bind("version_guid", form.versionGuid). - bind("num_attempts", form.numAttempts). - bind("errors", form.errors.map { v => Json.toJson(v) }). - bind("hash_code", form.hashCode()) - } - - private[this] def toNamedParameter(updatedBy: UUID, id: String, form: MigrationForm): Seq[NamedParameter] = { - Seq( - scala.Symbol("id") -> id, - scala.Symbol("version_guid") -> form.versionGuid, - scala.Symbol("num_attempts") -> form.numAttempts, - scala.Symbol("errors") -> form.errors.map { v => Json.toJson(v).toString }, - scala.Symbol("updated_by_guid") -> updatedBy, - scala.Symbol("hash_code") -> form.hashCode() - ) - } - - def insert(updatedBy: UUID, form: MigrationForm): String = { - db.withConnection { c => - insert(c, updatedBy, form) - } - } - - def insert(c: Connection, updatedBy: UUID, form: MigrationForm): String = { - val id = randomId() - bindQuery(InsertQuery, form). - bind("id", id). - bind("updated_by_guid", updatedBy). - anormSql.execute()(c) - id - } - - def insertBatch(updatedBy: UUID, forms: Seq[MigrationForm]): Seq[String] = { - db.withConnection { c => - insertBatchWithConnection(c, updatedBy, forms) - } - } - - def insertBatchWithConnection(c: Connection, updatedBy: UUID, forms: Seq[MigrationForm]): Seq[String] = { - if (forms.nonEmpty) { - val ids = forms.map(_ => randomId()) - val params = ids.zip(forms).map { case (id, form) => toNamedParameter(updatedBy, id, form) } - BatchSql(InsertQuery.sql(), params.head, params.tail: _*).execute()(c) - ids - } else { - Nil - } - } - - def updateIfChangedById(updatedBy: UUID, id: String, form: MigrationForm): Unit = { - if (!findById(id).map(_.form).contains(form)) { - updateById(updatedBy, id, form) - } - } - - def updateById(updatedBy: UUID, id: String, form: MigrationForm): Unit = { - db.withConnection { c => - updateById(c, updatedBy, id, form) - } - } - - def updateById(c: Connection, updatedBy: UUID, id: String, form: MigrationForm): Unit = { - bindQuery(UpdateQuery, form). - bind("id", id). - bind("updated_by_guid", updatedBy). - anormSql.execute()(c) - () - } - - def update(updatedBy: UUID, existing: Migration, form: MigrationForm): Unit = { - db.withConnection { c => - update(c, updatedBy, existing, form) - } - } - - def update(c: Connection, updatedBy: UUID, existing: Migration, form: MigrationForm): Unit = { - updateById(c, updatedBy, existing.id, form) - } - - def updateBatch(updatedBy: UUID, idsAndForms: Seq[(String, MigrationForm)]): Unit = { - db.withConnection { c => - updateBatchWithConnection(c, updatedBy, idsAndForms) - } - } - - def updateBatchWithConnection(c: Connection, updatedBy: UUID, idsAndForms: Seq[(String, MigrationForm)]): Unit = { - if (idsAndForms.nonEmpty) { - val params = idsAndForms.map { case (id, form) => toNamedParameter(updatedBy, id, form) } - BatchSql(UpdateQuery.sql(), params.head, params.tail: _*).execute()(c) - () - } - } - - def delete(deletedBy: UUID, migration: Migration): Unit = { - db.withConnection { c => - delete(c, deletedBy, migration) - } - } - - def delete(c: Connection, deletedBy: UUID, migration: Migration): Unit = { - deleteById(c, deletedBy, migration.id) - } - - def deleteById(deletedBy: UUID, id: String): Unit = { - db.withConnection { c => - deleteById(c, deletedBy, id) - } - } - - def deleteById(c: Connection, deletedBy: UUID, id: String): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .equals("id", id) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByIds(deletedBy: UUID, ids: Seq[String]): Unit = { - db.withConnection { c => - deleteAllByIds(c, deletedBy, ids) - } - } - - def deleteAllByIds(c: Connection, deletedBy: UUID, ids: Seq[String]): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .in("id", ids) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByNumAttempts(deletedBy: UUID, numAttempts: Long): Unit = { - db.withConnection { c => - deleteAllByNumAttempts(c, deletedBy, numAttempts) - } - } - - def deleteAllByNumAttempts(c: Connection, deletedBy: UUID, numAttempts: Long): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .equals("num_attempts", numAttempts) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByNumAttemptses(deletedBy: UUID, numAttemptses: Seq[Long]): Unit = { - db.withConnection { c => - deleteAllByNumAttemptses(c, deletedBy, numAttemptses) - } - } - - def deleteAllByNumAttemptses(c: Connection, deletedBy: UUID, numAttemptses: Seq[Long]): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .in("num_attempts", numAttemptses) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByNumAttemptsAndCreatedAt(deletedBy: UUID, numAttempts: Long, createdAt: DateTime): Unit = { - db.withConnection { c => - deleteAllByNumAttemptsAndCreatedAt(c, deletedBy, numAttempts, createdAt) - } - } - - def deleteAllByNumAttemptsAndCreatedAt(c: Connection, deletedBy: UUID, numAttempts: Long, createdAt: DateTime): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .equals("num_attempts", numAttempts) - .equals("created_at", createdAt) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByNumAttemptsAndCreatedAts(deletedBy: UUID, numAttempts: Long, createdAts: Seq[DateTime]): Unit = { - db.withConnection { c => - deleteAllByNumAttemptsAndCreatedAts(c, deletedBy, numAttempts, createdAts) - } - } - - def deleteAllByNumAttemptsAndCreatedAts(c: Connection, deletedBy: UUID, numAttempts: Long, createdAts: Seq[DateTime]): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .equals("num_attempts", numAttempts) - .in("created_at", createdAts) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByVersionGuid(deletedBy: UUID, versionGuid: UUID): Unit = { - db.withConnection { c => - deleteAllByVersionGuid(c, deletedBy, versionGuid) - } - } - - def deleteAllByVersionGuid(c: Connection, deletedBy: UUID, versionGuid: UUID): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .equals("version_guid", versionGuid) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByVersionGuids(deletedBy: UUID, versionGuids: Seq[UUID]): Unit = { - db.withConnection { c => - deleteAllByVersionGuids(c, deletedBy, versionGuids) - } - } - - def deleteAllByVersionGuids(c: Connection, deletedBy: UUID, versionGuids: Seq[UUID]): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .in("version_guid", versionGuids) - .anormSql.executeUpdate()(c) - () - } - - def setJournalDeletedByUserId(c: Connection, deletedBy: UUID): Unit = { - anorm.SQL(s"SET journal.deleted_by_user_id = '${deletedBy}'").executeUpdate()(c) - () - } - -} \ No newline at end of file diff --git a/api/app/db/generated/PsqlApibuilderTasksDao.scala b/api/app/db/generated/PsqlApibuilderTasksDao.scala new file mode 100644 index 000000000..612e41ddd --- /dev/null +++ b/api/app/db/generated/PsqlApibuilderTasksDao.scala @@ -0,0 +1,614 @@ +package db.generated + +import anorm.JodaParameterMetaData._ +import anorm._ +import io.flow.postgresql.{OrderBy, Query} +import java.sql.Connection +import java.util.UUID +import javax.inject.{Inject, Singleton} +import org.joda.time.DateTime +import play.api.db.Database +import play.api.libs.json.{JsValue, Json} + +case class Task( + id: String, + `type`: String, + typeId: String, + organizationGuid: Option[UUID], + numAttempts: Int, + nextAttemptAt: DateTime, + errors: Option[Seq[String]], + stacktrace: Option[String], + data: JsValue, + updatedByGuid: String, + createdAt: DateTime, + updatedAt: DateTime +) { + + lazy val form: TaskForm = TaskForm( + id = id, + `type` = `type`, + typeId = typeId, + organizationGuid = organizationGuid, + numAttempts = numAttempts, + nextAttemptAt = nextAttemptAt, + errors = errors, + stacktrace = stacktrace, + data = data + ) + +} + +case class TaskForm( + id: String, + `type`: String, + typeId: String, + organizationGuid: Option[UUID], + numAttempts: Int, + nextAttemptAt: DateTime, + errors: Option[Seq[String]], + stacktrace: Option[String], + data: JsValue +) + +object TasksTable { + val Schema: String = "public" + val Name: String = "tasks" + val QualifiedName: String = s"$Schema.$Name" + + object Columns { + val Id: String = "id" + val Type: String = "type" + val TypeId: String = "type_id" + val OrganizationGuid: String = "organization_guid" + val NumAttempts: String = "num_attempts" + val NextAttemptAt: String = "next_attempt_at" + val Errors: String = "errors" + val Stacktrace: String = "stacktrace" + val Data: String = "data" + val UpdatedByGuid: String = "updated_by_guid" + val CreatedAt: String = "created_at" + val UpdatedAt: String = "updated_at" + val HashCode: String = "hash_code" + val all: List[String] = List(Id, Type, TypeId, OrganizationGuid, NumAttempts, NextAttemptAt, Errors, Stacktrace, Data, UpdatedByGuid, CreatedAt, UpdatedAt, HashCode) + } +} + +trait BaseTasksDao { + + def db: Database + + private[this] val BaseQuery = Query(""" + | select tasks.id, + | tasks.type, + | tasks.type_id, + | tasks.organization_guid, + | tasks.num_attempts, + | tasks.next_attempt_at, + | tasks.errors::text as errors_text, + | tasks.stacktrace, + | tasks.data::text as data_text, + | tasks.updated_by_guid, + | tasks.created_at, + | tasks.updated_at, + | tasks.hash_code + | from tasks + """.stripMargin) + + def findById(id: String): Option[Task] = { + db.withConnection { c => + findByIdWithConnection(c, id) + } + } + + def findByIdWithConnection(c: java.sql.Connection, id: String): Option[Task] = { + findAllWithConnection(c, ids = Some(Seq(id)), limit = Some(1L), orderBy = None).headOption + } + + def findByTypeIdAndType(typeId: String, `type`: String): Option[Task] = { + db.withConnection { c => + findByTypeIdAndTypeWithConnection(c, typeId, `type`) + } + } + + def findByTypeIdAndTypeWithConnection(c: java.sql.Connection, typeId: String, `type`: String): Option[Task] = { + findAllWithConnection(c, typeId = Some(typeId), `type` = Some(`type`), limit = Some(1L), orderBy = None).headOption + } + + def iterateAll( + ids: Option[Seq[String]] = None, + typeId: Option[String] = None, + typeIds: Option[Seq[String]] = None, + `type`: Option[String] = None, + types: Option[Seq[String]] = None, + numAttempts: Option[Int] = None, + numAttemptsGreaterThanOrEquals: Option[Int] = None, + numAttemptsGreaterThan: Option[Int] = None, + numAttemptsLessThanOrEquals: Option[Int] = None, + numAttemptsLessThan: Option[Int] = None, + nextAttemptAt: Option[DateTime] = None, + nextAttemptAtGreaterThanOrEquals: Option[DateTime] = None, + nextAttemptAtGreaterThan: Option[DateTime] = None, + nextAttemptAtLessThanOrEquals: Option[DateTime] = None, + nextAttemptAtLessThan: Option[DateTime] = None, + numAttemptsNextAttemptAts: Option[Seq[(Int, DateTime)]] = None, + typeIdTypes: Option[Seq[(String, String)]] = None, + pageSize: Long = 2000L, + ) ( + implicit customQueryModifier: Query => Query = { q => q } + ): Iterator[Task] = { + def iterate(lastValue: Option[Task]): Iterator[Task] = { + val page = findAll( + ids = ids, + typeId = typeId, + typeIds = typeIds, + `type` = `type`, + types = types, + numAttempts = numAttempts, + numAttemptsGreaterThanOrEquals = numAttemptsGreaterThanOrEquals, + numAttemptsGreaterThan = numAttemptsGreaterThan, + numAttemptsLessThanOrEquals = numAttemptsLessThanOrEquals, + numAttemptsLessThan = numAttemptsLessThan, + nextAttemptAt = nextAttemptAt, + nextAttemptAtGreaterThanOrEquals = nextAttemptAtGreaterThanOrEquals, + nextAttemptAtGreaterThan = nextAttemptAtGreaterThan, + nextAttemptAtLessThanOrEquals = nextAttemptAtLessThanOrEquals, + nextAttemptAtLessThan = nextAttemptAtLessThan, + numAttemptsNextAttemptAts = numAttemptsNextAttemptAts, + typeIdTypes = typeIdTypes, + limit = Some(pageSize), + orderBy = Some(OrderBy("tasks.id")), + ) { q => customQueryModifier(q).greaterThan("tasks.id", lastValue.map(_.id)) } + + page.lastOption match { + case None => Iterator.empty + case lastValue => page.iterator ++ iterate(lastValue) + } + } + + iterate(None) + } + + def findAll( + ids: Option[Seq[String]] = None, + typeId: Option[String] = None, + typeIds: Option[Seq[String]] = None, + `type`: Option[String] = None, + types: Option[Seq[String]] = None, + numAttempts: Option[Int] = None, + numAttemptsGreaterThanOrEquals: Option[Int] = None, + numAttemptsGreaterThan: Option[Int] = None, + numAttemptsLessThanOrEquals: Option[Int] = None, + numAttemptsLessThan: Option[Int] = None, + nextAttemptAt: Option[DateTime] = None, + nextAttemptAtGreaterThanOrEquals: Option[DateTime] = None, + nextAttemptAtGreaterThan: Option[DateTime] = None, + nextAttemptAtLessThanOrEquals: Option[DateTime] = None, + nextAttemptAtLessThan: Option[DateTime] = None, + numAttemptsNextAttemptAts: Option[Seq[(Int, DateTime)]] = None, + typeIdTypes: Option[Seq[(String, String)]] = None, + limit: Option[Long], + offset: Long = 0, + orderBy: Option[OrderBy] = Some(OrderBy("tasks.id")) + ) ( + implicit customQueryModifier: Query => Query = { q => q } + ): Seq[Task] = { + db.withConnection { c => + findAllWithConnection( + c, + ids = ids, + typeId = typeId, + typeIds = typeIds, + `type` = `type`, + types = types, + numAttempts = numAttempts, + numAttemptsGreaterThanOrEquals = numAttemptsGreaterThanOrEquals, + numAttemptsGreaterThan = numAttemptsGreaterThan, + numAttemptsLessThanOrEquals = numAttemptsLessThanOrEquals, + numAttemptsLessThan = numAttemptsLessThan, + nextAttemptAt = nextAttemptAt, + nextAttemptAtGreaterThanOrEquals = nextAttemptAtGreaterThanOrEquals, + nextAttemptAtGreaterThan = nextAttemptAtGreaterThan, + nextAttemptAtLessThanOrEquals = nextAttemptAtLessThanOrEquals, + nextAttemptAtLessThan = nextAttemptAtLessThan, + numAttemptsNextAttemptAts = numAttemptsNextAttemptAts, + typeIdTypes = typeIdTypes, + limit = limit, + offset = offset, + orderBy = orderBy + )(customQueryModifier) + } + } + + def findAllWithConnection( + c: java.sql.Connection, + ids: Option[Seq[String]] = None, + typeId: Option[String] = None, + typeIds: Option[Seq[String]] = None, + `type`: Option[String] = None, + types: Option[Seq[String]] = None, + numAttempts: Option[Int] = None, + numAttemptsGreaterThanOrEquals: Option[Int] = None, + numAttemptsGreaterThan: Option[Int] = None, + numAttemptsLessThanOrEquals: Option[Int] = None, + numAttemptsLessThan: Option[Int] = None, + nextAttemptAt: Option[DateTime] = None, + nextAttemptAtGreaterThanOrEquals: Option[DateTime] = None, + nextAttemptAtGreaterThan: Option[DateTime] = None, + nextAttemptAtLessThanOrEquals: Option[DateTime] = None, + nextAttemptAtLessThan: Option[DateTime] = None, + numAttemptsNextAttemptAts: Option[Seq[(Int, DateTime)]] = None, + typeIdTypes: Option[Seq[(String, String)]] = None, + limit: Option[Long], + offset: Long = 0, + orderBy: Option[OrderBy] = Some(OrderBy("tasks.id")) + ) ( + implicit customQueryModifier: Query => Query = { q => q } + ): Seq[Task] = { + customQueryModifier(BaseQuery). + optionalIn("tasks.id", ids). + equals("tasks.type_id", typeId). + optionalIn("tasks.type_id", typeIds). + equals("tasks.type", `type`). + optionalIn("tasks.type", types). + equals("tasks.num_attempts", numAttempts). + greaterThanOrEquals("tasks.num_attempts", numAttemptsGreaterThanOrEquals). + greaterThan("tasks.num_attempts", numAttemptsGreaterThan). + lessThanOrEquals("tasks.num_attempts", numAttemptsLessThanOrEquals). + lessThan("tasks.num_attempts", numAttemptsLessThan). + equals("tasks.next_attempt_at", nextAttemptAt). + greaterThanOrEquals("tasks.next_attempt_at", nextAttemptAtGreaterThanOrEquals). + greaterThan("tasks.next_attempt_at", nextAttemptAtGreaterThan). + lessThanOrEquals("tasks.next_attempt_at", nextAttemptAtLessThanOrEquals). + lessThan("tasks.next_attempt_at", nextAttemptAtLessThan). + optionalIn2(("tasks.num_attempts", "tasks.next_attempt_at"), numAttemptsNextAttemptAts). + optionalIn2(("tasks.type_id", "tasks.type"), typeIdTypes). + optionalLimit(limit). + offset(offset). + orderBy(orderBy.flatMap(_.sql)). + as(TasksDao.parser.*)(c) + } + +} + +object TasksDao { + + val parser: RowParser[Task] = { + SqlParser.str("id") ~ + SqlParser.str("type") ~ + SqlParser.str("type_id") ~ + SqlParser.get[UUID]("organization_guid").? ~ + SqlParser.int("num_attempts") ~ + SqlParser.get[DateTime]("next_attempt_at") ~ + SqlParser.str("errors_text").? ~ + SqlParser.str("stacktrace").? ~ + SqlParser.str("data_text") ~ + SqlParser.str("updated_by_guid") ~ + SqlParser.get[DateTime]("created_at") ~ + SqlParser.get[DateTime]("updated_at") map { + case id ~ type_ ~ typeId ~ organizationGuid ~ numAttempts ~ nextAttemptAt ~ errors ~ stacktrace ~ data ~ updatedByGuid ~ createdAt ~ updatedAt => Task( + id = id, + `type` = type_, + typeId = typeId, + organizationGuid = organizationGuid, + numAttempts = numAttempts, + nextAttemptAt = nextAttemptAt, + errors = errors.map { text => Json.parse(text).as[Seq[String]] }, + stacktrace = stacktrace, + data = Json.parse(data), + updatedByGuid = updatedByGuid, + createdAt = createdAt, + updatedAt = updatedAt + ) + } + } + +} + +@Singleton +class TasksDao @Inject() ( + override val db: Database +) extends BaseTasksDao { + + private[this] val UpsertQuery = Query(""" + | insert into tasks + | (id, type, type_id, organization_guid, num_attempts, next_attempt_at, errors, stacktrace, data, updated_by_guid, hash_code) + | values + | ({id}, {type}, {type_id}, {organization_guid}::uuid, {num_attempts}::int, {next_attempt_at}::timestamptz, {errors}::json, {stacktrace}, {data}::json, {updated_by_guid}, {hash_code}::bigint) + | on conflict (type_id, type) + | do update + | set organization_guid = {organization_guid}::uuid, + | num_attempts = {num_attempts}::int, + | next_attempt_at = {next_attempt_at}::timestamptz, + | errors = {errors}::json, + | stacktrace = {stacktrace}, + | data = {data}::json, + | updated_by_guid = {updated_by_guid}, + | hash_code = {hash_code}::bigint + | where tasks.hash_code != {hash_code}::bigint + | returning id + """.stripMargin) + + private[this] val UpdateQuery = Query(""" + | update tasks + | set type = {type}, + | type_id = {type_id}, + | organization_guid = {organization_guid}::uuid, + | num_attempts = {num_attempts}::int, + | next_attempt_at = {next_attempt_at}::timestamptz, + | errors = {errors}::json, + | stacktrace = {stacktrace}, + | data = {data}::json, + | updated_by_guid = {updated_by_guid}, + | hash_code = {hash_code}::bigint + | where id = {id} + | and tasks.hash_code != {hash_code}::bigint + """.stripMargin) + + private[this] def bindQuery(query: Query, form: TaskForm): Query = { + query. + bind("type", form.`type`). + bind("type_id", form.typeId). + bind("organization_guid", form.organizationGuid). + bind("num_attempts", form.numAttempts). + bind("next_attempt_at", form.nextAttemptAt). + bind("errors", form.errors.map { v => Json.toJson(v) }). + bind("stacktrace", form.stacktrace). + bind("data", form.data). + bind("hash_code", form.hashCode()) + } + + private[this] def toNamedParameter(updatedBy: UUID, form: TaskForm): Seq[NamedParameter] = { + Seq( + scala.Symbol("id") -> form.id, + scala.Symbol("type") -> form.`type`, + scala.Symbol("type_id") -> form.typeId, + scala.Symbol("organization_guid") -> form.organizationGuid, + scala.Symbol("num_attempts") -> form.numAttempts, + scala.Symbol("next_attempt_at") -> form.nextAttemptAt, + scala.Symbol("errors") -> form.errors.map { v => Json.toJson(v).toString }, + scala.Symbol("stacktrace") -> form.stacktrace, + scala.Symbol("data") -> form.data.toString, + scala.Symbol("updated_by_guid") -> updatedBy, + scala.Symbol("hash_code") -> form.hashCode() + ) + } + + def upsertIfChangedByTypeIdAndType(updatedBy: UUID, form: TaskForm): Unit = { + if (!findByTypeIdAndType(form.typeId, form.`type`).map(_.form).contains(form)) { + upsertByTypeIdAndType(updatedBy, form) + } + } + + def upsertByTypeIdAndType(updatedBy: UUID, form: TaskForm): Unit = { + db.withConnection { c => + upsertByTypeIdAndType(c, updatedBy, form) + } + } + + def upsertByTypeIdAndType(c: Connection, updatedBy: UUID, form: TaskForm): Unit = { + bindQuery(UpsertQuery, form). + bind("id", form.id). + bind("updated_by_guid", updatedBy). + anormSql.execute()(c) + () + } + + def upsertBatchByTypeIdAndType(updatedBy: UUID, forms: Seq[TaskForm]): Unit = { + db.withConnection { c => + upsertBatchByTypeIdAndType(c, updatedBy, forms) + } + } + + def upsertBatchByTypeIdAndType(c: Connection, updatedBy: UUID, forms: Seq[TaskForm]): Unit = { + if (forms.nonEmpty) { + val params = forms.map(toNamedParameter(updatedBy, _)) + BatchSql(UpsertQuery.sql(), params.head, params.tail: _*).execute()(c) + () + } + } + + def updateIfChangedById(updatedBy: UUID, id: String, form: TaskForm): Unit = { + if (!findById(id).map(_.form).contains(form)) { + updateById(updatedBy, id, form) + } + } + + def updateById(updatedBy: UUID, id: String, form: TaskForm): Unit = { + db.withConnection { c => + updateById(c, updatedBy, id, form) + } + } + + def updateById(c: Connection, updatedBy: UUID, id: String, form: TaskForm): Unit = { + bindQuery(UpdateQuery, form). + bind("id", id). + bind("updated_by_guid", updatedBy). + anormSql.execute()(c) + () + } + + def update(updatedBy: UUID, existing: Task, form: TaskForm): Unit = { + db.withConnection { c => + update(c, updatedBy, existing, form) + } + } + + def update(c: Connection, updatedBy: UUID, existing: Task, form: TaskForm): Unit = { + updateById(c, updatedBy, existing.id, form) + } + + def updateBatch(updatedBy: UUID, forms: Seq[TaskForm]): Unit = { + db.withConnection { c => + updateBatchWithConnection(c, updatedBy, forms) + } + } + + def updateBatchWithConnection(c: Connection, updatedBy: UUID, forms: Seq[TaskForm]): Unit = { + if (forms.nonEmpty) { + val params = forms.map(toNamedParameter(updatedBy, _)) + BatchSql(UpdateQuery.sql(), params.head, params.tail: _*).execute()(c) + () + } + } + + def delete(deletedBy: UUID, task: Task): Unit = { + db.withConnection { c => + delete(c, deletedBy, task) + } + } + + def delete(c: Connection, deletedBy: UUID, task: Task): Unit = { + deleteById(c, deletedBy, task.id) + } + + def deleteById(deletedBy: UUID, id: String): Unit = { + db.withConnection { c => + deleteById(c, deletedBy, id) + } + } + + def deleteById(c: Connection, deletedBy: UUID, id: String): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("id", id) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByIds(deletedBy: UUID, ids: Seq[String]): Unit = { + db.withConnection { c => + deleteAllByIds(c, deletedBy, ids) + } + } + + def deleteAllByIds(c: Connection, deletedBy: UUID, ids: Seq[String]): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .in("id", ids) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByNumAttempts(deletedBy: UUID, numAttempts: Int): Unit = { + db.withConnection { c => + deleteAllByNumAttempts(c, deletedBy, numAttempts) + } + } + + def deleteAllByNumAttempts(c: Connection, deletedBy: UUID, numAttempts: Int): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("num_attempts", numAttempts) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByNumAttemptses(deletedBy: UUID, numAttemptses: Seq[Int]): Unit = { + db.withConnection { c => + deleteAllByNumAttemptses(c, deletedBy, numAttemptses) + } + } + + def deleteAllByNumAttemptses(c: Connection, deletedBy: UUID, numAttemptses: Seq[Int]): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .in("num_attempts", numAttemptses) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByNumAttemptsAndNextAttemptAt(deletedBy: UUID, numAttempts: Int, nextAttemptAt: DateTime): Unit = { + db.withConnection { c => + deleteAllByNumAttemptsAndNextAttemptAt(c, deletedBy, numAttempts, nextAttemptAt) + } + } + + def deleteAllByNumAttemptsAndNextAttemptAt(c: Connection, deletedBy: UUID, numAttempts: Int, nextAttemptAt: DateTime): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("num_attempts", numAttempts) + .equals("next_attempt_at", nextAttemptAt) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByNumAttemptsAndNextAttemptAts(deletedBy: UUID, numAttempts: Int, nextAttemptAts: Seq[DateTime]): Unit = { + db.withConnection { c => + deleteAllByNumAttemptsAndNextAttemptAts(c, deletedBy, numAttempts, nextAttemptAts) + } + } + + def deleteAllByNumAttemptsAndNextAttemptAts(c: Connection, deletedBy: UUID, numAttempts: Int, nextAttemptAts: Seq[DateTime]): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("num_attempts", numAttempts) + .in("next_attempt_at", nextAttemptAts) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByTypeId(deletedBy: UUID, typeId: String): Unit = { + db.withConnection { c => + deleteAllByTypeId(c, deletedBy, typeId) + } + } + + def deleteAllByTypeId(c: Connection, deletedBy: UUID, typeId: String): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("type_id", typeId) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByTypeIds(deletedBy: UUID, typeIds: Seq[String]): Unit = { + db.withConnection { c => + deleteAllByTypeIds(c, deletedBy, typeIds) + } + } + + def deleteAllByTypeIds(c: Connection, deletedBy: UUID, typeIds: Seq[String]): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .in("type_id", typeIds) + .anormSql.executeUpdate()(c) + () + } + + def deleteByTypeIdAndType(deletedBy: UUID, typeId: String, `type`: String): Unit = { + db.withConnection { c => + deleteByTypeIdAndType(c, deletedBy, typeId, `type`) + } + } + + def deleteByTypeIdAndType(c: Connection, deletedBy: UUID, typeId: String, `type`: String): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("type_id", typeId) + .equals("type", `type`) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByTypeIdAndTypes(deletedBy: UUID, typeId: String, types: Seq[String]): Unit = { + db.withConnection { c => + deleteAllByTypeIdAndTypes(c, deletedBy, typeId, types) + } + } + + def deleteAllByTypeIdAndTypes(c: Connection, deletedBy: UUID, typeId: String, types: Seq[String]): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("type_id", typeId) + .in("type", types) + .anormSql.executeUpdate()(c) + () + } + + def setJournalDeletedByUserId(c: Connection, deletedBy: UUID): Unit = { + anorm.SQL(s"SET journal.deleted_by_user_id = '${deletedBy}'").executeUpdate()(c) + () + } + +} \ No newline at end of file diff --git a/api/app/db/generators/ServicesDao.scala b/api/app/db/generators/ServicesDao.scala index 4bfc0b1dc..8f5b4da5d 100644 --- a/api/app/db/generators/ServicesDao.scala +++ b/api/app/db/generators/ServicesDao.scala @@ -1,23 +1,22 @@ package db.generators -import io.apibuilder.api.v0.models.{GeneratorService, GeneratorServiceForm} -import db._ -import io.apibuilder.api.v0.models.{Error, User} +import anorm._ import core.Util -import javax.inject.{Inject, Singleton} - +import db._ +import io.apibuilder.api.v0.models.{Error, GeneratorService, GeneratorServiceForm, User} +import io.apibuilder.task.v0.models.TaskType +import io.flow.postgresql.Query import lib.{Pager, Validation} -import anorm._ import play.api.db._ -import java.util.UUID -import io.flow.postgresql.Query +import java.util.UUID +import javax.inject.{Inject, Singleton} @Singleton class ServicesDao @Inject() ( @NamedDatabase("default") db: Database, - @javax.inject.Named("main-actor") mainActor: akka.actor.ActorRef, - generatorsDao: GeneratorsDao + generatorsDao: GeneratorsDao, + internalTasksDao: InternalTasksDao ) { private[this] val dbHelpers = DbHelpers(db, "generators.services") @@ -45,7 +44,7 @@ class ServicesDao @Inject() ( uri = Some(form.uri.trim) ).headOption match { case None => Nil - case Some(uri) => { + case Some(_) => { Seq(s"URI[${form.uri.trim}] already exists") } } @@ -62,16 +61,15 @@ class ServicesDao @Inject() ( val guid = UUID.randomUUID - db.withConnection { implicit c => + db.withTransaction { implicit c => SQL(InsertQuery).on( "guid" -> guid, "uri" -> form.uri.trim, "created_by_guid" -> user.guid ).execute() + internalTasksDao.queueWithConnection(c, TaskType.SyncGeneratorService, guid.toString) } - mainActor ! actors.MainActor.Messages.GeneratorServiceCreated(guid) - findByGuid(Authorization.All, guid).getOrElse { sys.error("Failed to create service") } @@ -81,7 +79,7 @@ class ServicesDao @Inject() ( * Also will soft delete all generators for this service */ def softDelete(deletedBy: User, service: GeneratorService): Unit = { - Pager.eachPage { offset => + Pager.eachPage { _ => // Note we do not include offset in the query as each iteration // deletes records which will then NOT show up in the next loop generatorsDao.findAll( diff --git a/api/app/generated/ApicollectiveApibuilderInternalV0Conversions.scala b/api/app/generated/ApicollectiveApibuilderInternalV0Conversions.scala deleted file mode 100644 index 79c8f0e40..000000000 --- a/api/app/generated/ApicollectiveApibuilderInternalV0Conversions.scala +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Generated by API Builder - https://www.apibuilder.io - * Service version: 0.15.81 - * apibuilder 0.15.33 app.apibuilder.io/apicollective/apibuilder-internal/latest/anorm_2_8_parsers - */ -package io.apibuilder.internal.v0.anorm.conversions { - - import anorm.{Column, MetaDataItem, TypeDoesNotMatch} - import play.api.libs.json.{JsArray, JsObject, JsValue} - import scala.util.{Failure, Success, Try} - import play.api.libs.json.JodaReads._ - - /** - * Conversions to collections of objects using JSON. - */ - object Util { - - def parser[T]( - f: play.api.libs.json.JsValue => T - ) = anorm.Column.nonNull { (value, meta) => - val MetaDataItem(columnName, nullable, clazz) = meta - value match { - case json: org.postgresql.util.PGobject => parseJson(f, columnName.qualified, json.getValue) - case json: java.lang.String => parseJson(f, columnName.qualified, json) - case _=> { - Left( - TypeDoesNotMatch( - s"Column[${columnName.qualified}] error converting $value to Json. Expected instance of type[org.postgresql.util.PGobject] and not[${value.asInstanceOf[AnyRef].getClass}]" - ) - ) - } - - - } - } - - private[this] def parseJson[T](f: play.api.libs.json.JsValue => T, columnName: String, value: String) = { - Try { - f( - play.api.libs.json.Json.parse(value) - ) - } match { - case Success(result) => Right(result) - case Failure(ex) => Left( - TypeDoesNotMatch( - s"Column[$columnName] error parsing json $value: $ex" - ) - ) - } - } - - } - - object Types { - import io.apibuilder.internal.v0.models.json._ - implicit val columnToSeqApibuilderInternalTask: Column[Seq[_root_.io.apibuilder.internal.v0.models.Task]] = Util.parser { _.as[Seq[_root_.io.apibuilder.internal.v0.models.Task]] } - implicit val columnToMapApibuilderInternalTask: Column[Map[String, _root_.io.apibuilder.internal.v0.models.Task]] = Util.parser { _.as[Map[String, _root_.io.apibuilder.internal.v0.models.Task]] } - implicit val columnToSeqApibuilderInternalTaskDataDiffVersion: Column[Seq[_root_.io.apibuilder.internal.v0.models.TaskDataDiffVersion]] = Util.parser { _.as[Seq[_root_.io.apibuilder.internal.v0.models.TaskDataDiffVersion]] } - implicit val columnToMapApibuilderInternalTaskDataDiffVersion: Column[Map[String, _root_.io.apibuilder.internal.v0.models.TaskDataDiffVersion]] = Util.parser { _.as[Map[String, _root_.io.apibuilder.internal.v0.models.TaskDataDiffVersion]] } - implicit val columnToSeqApibuilderInternalTaskDataIndexApplication: Column[Seq[_root_.io.apibuilder.internal.v0.models.TaskDataIndexApplication]] = Util.parser { _.as[Seq[_root_.io.apibuilder.internal.v0.models.TaskDataIndexApplication]] } - implicit val columnToMapApibuilderInternalTaskDataIndexApplication: Column[Map[String, _root_.io.apibuilder.internal.v0.models.TaskDataIndexApplication]] = Util.parser { _.as[Map[String, _root_.io.apibuilder.internal.v0.models.TaskDataIndexApplication]] } - implicit val columnToSeqApibuilderInternalTaskData: Column[Seq[_root_.io.apibuilder.internal.v0.models.TaskData]] = Util.parser { _.as[Seq[_root_.io.apibuilder.internal.v0.models.TaskData]] } - implicit val columnToMapApibuilderInternalTaskData: Column[Map[String, _root_.io.apibuilder.internal.v0.models.TaskData]] = Util.parser { _.as[Map[String, _root_.io.apibuilder.internal.v0.models.TaskData]] } - } - - object Standard { - implicit val columnToJsObject: Column[play.api.libs.json.JsObject] = Util.parser { _.as[play.api.libs.json.JsObject] } - implicit val columnToSeqBoolean: Column[Seq[Boolean]] = Util.parser { _.as[Seq[Boolean]] } - implicit val columnToMapBoolean: Column[Map[String, Boolean]] = Util.parser { _.as[Map[String, Boolean]] } - implicit val columnToSeqDouble: Column[Seq[Double]] = Util.parser { _.as[Seq[Double]] } - implicit val columnToMapDouble: Column[Map[String, Double]] = Util.parser { _.as[Map[String, Double]] } - implicit val columnToSeqInt: Column[Seq[Int]] = Util.parser { _.as[Seq[Int]] } - implicit val columnToMapInt: Column[Map[String, Int]] = Util.parser { _.as[Map[String, Int]] } - implicit val columnToSeqLong: Column[Seq[Long]] = Util.parser { _.as[Seq[Long]] } - implicit val columnToMapLong: Column[Map[String, Long]] = Util.parser { _.as[Map[String, Long]] } - implicit val columnToSeqLocalDate: Column[Seq[_root_.org.joda.time.LocalDate]] = Util.parser { _.as[Seq[_root_.org.joda.time.LocalDate]] } - implicit val columnToMapLocalDate: Column[Map[String, _root_.org.joda.time.LocalDate]] = Util.parser { _.as[Map[String, _root_.org.joda.time.LocalDate]] } - implicit val columnToSeqDateTime: Column[Seq[_root_.org.joda.time.DateTime]] = Util.parser { _.as[Seq[_root_.org.joda.time.DateTime]] } - implicit val columnToMapDateTime: Column[Map[String, _root_.org.joda.time.DateTime]] = Util.parser { _.as[Map[String, _root_.org.joda.time.DateTime]] } - implicit val columnToSeqBigDecimal: Column[Seq[BigDecimal]] = Util.parser { _.as[Seq[BigDecimal]] } - implicit val columnToMapBigDecimal: Column[Map[String, BigDecimal]] = Util.parser { _.as[Map[String, BigDecimal]] } - implicit val columnToSeqJsObject: Column[Seq[_root_.play.api.libs.json.JsObject]] = Util.parser { _.as[Seq[_root_.play.api.libs.json.JsObject]] } - implicit val columnToMapJsObject: Column[Map[String, _root_.play.api.libs.json.JsObject]] = Util.parser { _.as[Map[String, _root_.play.api.libs.json.JsObject]] } - implicit val columnToSeqJsValue: Column[Seq[_root_.play.api.libs.json.JsValue]] = Util.parser { _.as[Seq[_root_.play.api.libs.json.JsValue]] } - implicit val columnToMapJsValue: Column[Map[String, _root_.play.api.libs.json.JsValue]] = Util.parser { _.as[Map[String, _root_.play.api.libs.json.JsValue]] } - implicit val columnToSeqString: Column[Seq[String]] = Util.parser { _.as[Seq[String]] } - implicit val columnToMapString: Column[Map[String, String]] = Util.parser { _.as[Map[String, String]] } - implicit val columnToSeqUUID: Column[Seq[_root_.java.util.UUID]] = Util.parser { _.as[Seq[_root_.java.util.UUID]] } - implicit val columnToMapUUID: Column[Map[String, _root_.java.util.UUID]] = Util.parser { _.as[Map[String, _root_.java.util.UUID]] } - } - -} \ No newline at end of file diff --git a/api/app/generated/ApicollectiveApibuilderInternalV0Parsers.scala b/api/app/generated/ApicollectiveApibuilderInternalV0Parsers.scala deleted file mode 100644 index 9177e546b..000000000 --- a/api/app/generated/ApicollectiveApibuilderInternalV0Parsers.scala +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Generated by API Builder - https://www.apibuilder.io - * Service version: 0.15.81 - * apibuilder 0.15.33 app.apibuilder.io/apicollective/apibuilder-internal/latest/anorm_2_8_parsers - */ -import anorm._ - -package io.apibuilder.internal.v0.anorm.parsers { - - import io.apibuilder.internal.v0.anorm.conversions.Standard._ - - import io.apibuilder.internal.v0.anorm.conversions.Types._ - - object Task { - - def parserWithPrefix(prefix: String, sep: String = "_"): RowParser[io.apibuilder.internal.v0.models.Task] = parser(prefixOpt = Some(s"$prefix$sep")) - - def parser( - guid: String = "guid", - dataPrefix: String = "data", - numberAttempts: String = "number_attempts", - lastError: String = "last_error", - prefixOpt: Option[String] = None - ): RowParser[io.apibuilder.internal.v0.models.Task] = { - SqlParser.get[_root_.java.util.UUID](prefixOpt.getOrElse("") + guid) ~ - io.apibuilder.internal.v0.anorm.parsers.TaskData.parserWithPrefix(prefixOpt.getOrElse("") + dataPrefix) ~ - SqlParser.long(prefixOpt.getOrElse("") + numberAttempts) ~ - SqlParser.str(prefixOpt.getOrElse("") + lastError).? map { - case guid ~ data ~ numberAttempts ~ lastError => { - io.apibuilder.internal.v0.models.Task( - guid = guid, - data = data, - numberAttempts = numberAttempts, - lastError = lastError - ) - } - } - } - - } - - object TaskDataDiffVersion { - - def parserWithPrefix(prefix: String, sep: String = "_"): RowParser[io.apibuilder.internal.v0.models.TaskDataDiffVersion] = parser(prefixOpt = Some(s"$prefix$sep")) - - def parser( - oldVersionGuid: String = "old_version_guid", - newVersionGuid: String = "new_version_guid", - prefixOpt: Option[String] = None - ): RowParser[io.apibuilder.internal.v0.models.TaskDataDiffVersion] = { - SqlParser.get[_root_.java.util.UUID](prefixOpt.getOrElse("") + oldVersionGuid) ~ - SqlParser.get[_root_.java.util.UUID](prefixOpt.getOrElse("") + newVersionGuid) map { - case oldVersionGuid ~ newVersionGuid => { - io.apibuilder.internal.v0.models.TaskDataDiffVersion( - oldVersionGuid = oldVersionGuid, - newVersionGuid = newVersionGuid - ) - } - } - } - - } - - object TaskDataIndexApplication { - - def parserWithPrefix(prefix: String, sep: String = "_"): RowParser[io.apibuilder.internal.v0.models.TaskDataIndexApplication] = parser(prefixOpt = Some(s"$prefix$sep")) - - def parser( - applicationGuid: String = "application_guid", - prefixOpt: Option[String] = None - ): RowParser[io.apibuilder.internal.v0.models.TaskDataIndexApplication] = { - SqlParser.get[_root_.java.util.UUID](prefixOpt.getOrElse("") + applicationGuid) map { - case applicationGuid => { - io.apibuilder.internal.v0.models.TaskDataIndexApplication( - applicationGuid = applicationGuid - ) - } - } - } - - } - - object TaskData { - - def parserWithPrefix(prefix: String, sep: String = "_") = { - io.apibuilder.internal.v0.anorm.parsers.TaskDataIndexApplication.parser(prefixOpt = Some(s"$prefix$sep")) | - io.apibuilder.internal.v0.anorm.parsers.TaskDataDiffVersion.parser(prefixOpt = Some(s"$prefix$sep")) - } - - def parser() = { - io.apibuilder.internal.v0.anorm.parsers.TaskDataIndexApplication.parser() | - io.apibuilder.internal.v0.anorm.parsers.TaskDataDiffVersion.parser() - } - - } - -} \ No newline at end of file diff --git a/api/app/invariants/CheckInvariants.scala b/api/app/invariants/CheckInvariants.scala new file mode 100644 index 000000000..ee0f539cc --- /dev/null +++ b/api/app/invariants/CheckInvariants.scala @@ -0,0 +1,44 @@ +package invariants + +import anorm.SqlParser +import play.api.db.Database + +import javax.inject.Inject + +/** Sync all invoices with a balance + */ +class CheckInvariants @Inject() ( + invariants: Invariants, + database: Database, +) { + + def process(): Unit = { + val results = invariants.all.map { i => + val count = database.withConnection { c => + i.query.as(SqlParser.long(1).*)(c).headOption.getOrElse(0L) + } + InvariantResult(i, count) + } + sendResults(results) + } + + private[this] case class InvariantResult(invariant: Invariant, count: Long) + private[this] def sendResults(results: Seq[InvariantResult]): Unit = { + val (noErrors, withErrors) = results.partition(_.count == 0) + + println(s"# Invariants checked with no errors: ${noErrors.length}") + if (withErrors.nonEmpty) { + val subject = if (withErrors.length == 1) { + "1 Error" + } else { + s"${withErrors.length} Errors" + } + println(subject) + withErrors.foreach { e => + println(s"${e.invariant.name}: ${e.count}") + println(s"${e.invariant.query.interpolate()}") + println("") + } + } + } +} diff --git a/api/app/invariants/Invariants.scala b/api/app/invariants/Invariants.scala new file mode 100644 index 000000000..b85f31b63 --- /dev/null +++ b/api/app/invariants/Invariants.scala @@ -0,0 +1,12 @@ +package invariants + +import io.flow.postgresql.Query + +import javax.inject.Inject + +case class Invariant(name: String, query: Query) + +class Invariants @Inject() { + val all: Seq[Invariant] = + TaskInvariants.all ++ PurgeInvariants.all +} diff --git a/api/app/invariants/PurgeInvariants.scala b/api/app/invariants/PurgeInvariants.scala new file mode 100644 index 000000000..a297d6efe --- /dev/null +++ b/api/app/invariants/PurgeInvariants.scala @@ -0,0 +1,18 @@ +package invariants + +import io.flow.postgresql.Query + +object PurgeInvariants { + private[this] val Tables = Seq( + "organizations", "applications", "versions" + ) + + val all: Seq[Invariant] = Tables.map { t => + Invariant( + s"old_deleted_records_purged_from_$t", + Query( + s"select count(*) from $t where deleted_at < now() - interval '1 year'" + ) + ) + } +} diff --git a/api/app/invariants/TaskInvariants.scala b/api/app/invariants/TaskInvariants.scala new file mode 100644 index 000000000..26918d534 --- /dev/null +++ b/api/app/invariants/TaskInvariants.scala @@ -0,0 +1,20 @@ +package invariants + +import io.flow.postgresql.Query + +object TaskInvariants { + val all: Seq[Invariant] = Seq( + Invariant( + "tasks_not_attempted", + Query(""" + |select count(*) from tasks where num_attempts=0 and created_at < now() - interval '1 hour' + |""".stripMargin) + ), + Invariant( + "tasks_not_completed_in_12_hours", + Query(""" + |select count(*) from tasks where num_attempts>0 and created_at < now() - interval '12 hour' + |""".stripMargin) + ) + ) +} diff --git a/api/app/util/ProcessDeletes.scala b/api/app/processor/CleanupDeletionsProcessor.scala similarity index 64% rename from api/app/util/ProcessDeletes.scala rename to api/app/processor/CleanupDeletionsProcessor.scala index 50797cead..906d77a1a 100644 --- a/api/app/util/ProcessDeletes.scala +++ b/api/app/processor/CleanupDeletionsProcessor.scala @@ -1,12 +1,15 @@ -package util +package processor +import cats.data.ValidatedNec +import cats.implicits._ import db.UsersDao +import io.apibuilder.task.v0.models.TaskType import io.flow.postgresql.Query import play.api.db.Database import javax.inject.Inject -object ProcessDeletes { +object DeleteMetadata { val OrganizationSoft: Seq[String] = Seq( "public.applications", "public.membership_requests", @@ -16,7 +19,7 @@ object ProcessDeletes { "public.subscriptions" ) val OrganizationHard: Seq[String] = Seq( - "public.organization_logs", "search.items" + "public.organization_logs", "public.tasks", "search.items" ) val ApplicationSoft: Seq[String] = Seq( "public.application_moves", "public.changes", "public.versions", "public.watches" @@ -25,30 +28,33 @@ object ProcessDeletes { val VersionSoft: Seq[String] = Seq( "cache.services", "public.originals" ) - val VersionHard: Seq[String] = Seq("public.migrations") + val VersionHard: Seq[String] = Nil } -class ProcessDeletes @Inject() ( - db: Database, - usersDao: UsersDao - ) { - import ProcessDeletes._ +class CleanupDeletionsProcessor @Inject()( + args: TaskProcessorArgs, + db: Database, + usersDao: UsersDao +) extends TaskProcessor(args, TaskType.CleanupDeletions) { + import DeleteMetadata._ - def all(): Unit = { + override def processRecord(id: String): ValidatedNec[String, Unit] = { organizations() applications() versions() + ().validNec } - private[util] def organizations(): Unit = { + + private[processor] def organizations(): Unit = { OrganizationSoft.foreach { table => exec( s""" - |update $table - | set deleted_at=now(),deleted_by_guid={deleted_by_guid}::uuid - | where deleted_at is null - | and organization_guid in (select guid from organizations where deleted_at is not null) - |""".stripMargin + |update $table + | set deleted_at=now(),deleted_by_guid={deleted_by_guid}::uuid + | where deleted_at is null + | and organization_guid in (select guid from organizations where deleted_at is not null) + |""".stripMargin ) } @@ -62,15 +68,15 @@ class ProcessDeletes @Inject() ( } } - private[util] def applications(): Unit = { + private[processor] def applications(): Unit = { ApplicationSoft.foreach { table => exec( s""" - |update $table - | set deleted_at=now(),deleted_by_guid={deleted_by_guid}::uuid - | where deleted_at is null - | and application_guid in (select guid from applications where deleted_at is not null) - |""".stripMargin + |update $table + | set deleted_at=now(),deleted_by_guid={deleted_by_guid}::uuid + | where deleted_at is null + | and application_guid in (select guid from applications where deleted_at is not null) + |""".stripMargin ) } @@ -84,7 +90,7 @@ class ProcessDeletes @Inject() ( } } - private[util] def versions(): Unit = { + private[processor] def versions(): Unit = { VersionSoft.foreach { table => exec( s""" diff --git a/api/app/processor/DiffVersionProcessor.scala b/api/app/processor/DiffVersionProcessor.scala new file mode 100644 index 000000000..9f03a334e --- /dev/null +++ b/api/app/processor/DiffVersionProcessor.scala @@ -0,0 +1,134 @@ +package processor + +import actors.Emails +import cats.data.ValidatedNec +import cats.implicits._ +import db._ +import io.apibuilder.api.v0.models._ +import io.apibuilder.task.v0.models.json._ +import io.apibuilder.task.v0.models.{DiffVersionData, TaskType} +import lib.{AppConfig, ServiceDiff} +import play.twirl.api.Html + +import java.util.UUID +import javax.inject.Inject + +class DiffVersionProcessor @Inject()( + args: TaskProcessorArgs, + appConfig: AppConfig, + usersDao: UsersDao, + applicationsDao: ApplicationsDao, + organizationsDao: OrganizationsDao, + emails: Emails, + changesDao: ChangesDao, + versionsDao: VersionsDao, + watchesDao: WatchesDao, +) extends TaskProcessorWithData[DiffVersionData](args, TaskType.DiffVersion) { + + override def processRecord(id: String, data: DiffVersionData): ValidatedNec[String, Unit] = { + diffVersion(data.oldVersionGuid, data.newVersionGuid) + ().validNec + } + + private[this] def diffVersion(oldVersionGuid: UUID, newVersionGuid: UUID): Unit = { + versionsDao.findByGuid(Authorization.All, oldVersionGuid, isDeleted = None).foreach { oldVersion => + versionsDao.findByGuid(Authorization.All, newVersionGuid).foreach { newVersion => + ServiceDiff(oldVersion.service, newVersion.service).differences match { + case Nil => { + // No-op + } + case diffs => { + changesDao.upsert( + createdBy = usersDao.AdminUser, + fromVersion = oldVersion, + toVersion = newVersion, + differences = diffs + ) + versionUpdated(newVersion, diffs) + versionUpdatedMaterialOnly(newVersion, diffs) + } + } + } + } + } + + private[this] def versionUpdated( + version: Version, + diffs: Seq[Diff], + ): Unit = { + if (diffs.nonEmpty) { + sendVersionUpsertedEmail( + publication = Publication.VersionsCreate, + version = version, + diffs = diffs, + ) { (org, application, breakingDiffs, nonBreakingDiffs) => + views.html.emails.versionUpserted( + appConfig, + org, + application, + version, + breakingDiffs = breakingDiffs, + nonBreakingDiffs = nonBreakingDiffs + ) + } + } + } + + private[this] def versionUpdatedMaterialOnly( + version: Version, + diffs: Seq[Diff], + ): Unit = { + val filtered = diffs.filter(_.isMaterial) + if (filtered.nonEmpty) { + sendVersionUpsertedEmail( + publication = Publication.VersionsMaterialChange, + version = version, + diffs = filtered, + ) { (org, application, breakingDiffs, nonBreakingDiffs) => + views.html.emails.versionUpserted( + appConfig, + org, + application, + version, + breakingDiffs = breakingDiffs, + nonBreakingDiffs = nonBreakingDiffs + ) + } + } + } + + private[this] def sendVersionUpsertedEmail( + publication: Publication, + version: Version, + diffs: Seq[Diff], + )( + generateBody: (Organization, Application, Seq[Diff], Seq[Diff]) => Html, + ): Unit = { + val (breakingDiffs, nonBreakingDiffs) = diffs.partition { + case _: DiffBreaking => true + case _: DiffNonBreaking => false + case _: DiffUndefinedType => true + } + + applicationsDao.findAll(Authorization.All, version = Some(version), limit = 1).foreach { application => + organizationsDao.findAll(Authorization.All, application = Some(application), limit = 1).foreach { org => + emails.deliver( + context = Emails.Context.Application(application), + org = org, + publication = publication, + subject = s"${org.name}/${application.name}:${version.version} Updated", + body = generateBody(org, application, breakingDiffs, nonBreakingDiffs).toString + ) { subscription => + watchesDao.findAll( + Authorization.All, + application = Some(application), + userGuid = Some(subscription.user.guid), + limit = 1 + ).nonEmpty + } + } + } + } + +} + diff --git a/api/app/processor/EmailProcessor.scala b/api/app/processor/EmailProcessor.scala new file mode 100644 index 000000000..59ed14fe0 --- /dev/null +++ b/api/app/processor/EmailProcessor.scala @@ -0,0 +1,146 @@ +package processor + +import actors.Emails +import cats.data.ValidatedNec +import cats.implicits._ +import db.{Authorization, InternalTasksDao, OrganizationsDao, UsersDao} +import io.apibuilder.api.v0.models.Publication +import io.apibuilder.task.v0.models._ +import io.apibuilder.task.v0.models.json._ +import lib.{AppConfig, EmailUtil, Person, Role} +import play.api.libs.json.Json + +import java.sql.Connection +import java.util.UUID +import javax.inject.Inject + +class EmailProcessorQueue @Inject() ( + internalTasksDao: InternalTasksDao + ) { + def queueWithConnection(c: Connection, data: EmailData): Unit = { + val dataJson = Json.toJson(data) + internalTasksDao.queue(TaskType.Email, id = Json.asciiStringify(dataJson), data = dataJson) + } +} + +class EmailProcessor @Inject()( + args: TaskProcessorArgs, + appConfig: AppConfig, + applicationsDao: db.ApplicationsDao, + email: EmailUtil, + emails: Emails, + emailVerificationsDao: db.EmailVerificationsDao, + membershipsDao: db.MembershipsDao, + membershipRequestsDao: db.MembershipRequestsDao, + organizationsDao: OrganizationsDao, + passwordResetRequestsDao: db.PasswordResetRequestsDao, + usersDao: UsersDao, +) extends TaskProcessorWithData[EmailData](args, TaskType.Email) { + + override def processRecord(id: String, data: EmailData): ValidatedNec[String, Unit] = { + data match { + case EmailDataApplicationCreated(guid) => applicationCreated(guid).validNec + case EmailDataEmailVerificationCreated(guid) => emailVerificationCreated(guid).validNec + case EmailDataMembershipCreated(guid) => membershipCreated(guid).validNec + case EmailDataMembershipRequestCreated(guid) => membershipRequestCreated(guid).validNec + case EmailDataMembershipRequestAccepted(orgGuid, userGuid, role) => membershipRequestAccepted(orgGuid, userGuid, role) + case EmailDataMembershipRequestDeclined(orgGuid, userGuid, role) => membershipRequestDeclined(orgGuid, userGuid, role) + case EmailDataPasswordResetRequestCreated(guid) => passwordResetRequestCreated(guid).validNec + case EmailDataUndefinedType(description) => s"Invalid email data type '$description'".invalidNec + } + } + + private[this] def applicationCreated(applicationGuid: UUID): Unit = { + applicationsDao.findByGuid(Authorization.All, applicationGuid).foreach { application => + organizationsDao.findByGuid(Authorization.All, application.guid).foreach { org => + emails.deliver( + context = Emails.Context.OrganizationMember, + org = org, + publication = Publication.ApplicationsCreate, + subject = s"${org.name}: New Application Created - ${application.name}", + body = views.html.emails.applicationCreated(appConfig, org, application).toString + ) + } + } + } + + private[this] def emailVerificationCreated(guid: UUID): Unit = { + emailVerificationsDao.findByGuid(guid).foreach { verification => + usersDao.findByGuid(verification.userGuid).foreach { user => + email.sendHtml( + to = Person(email = verification.email, name = user.name), + subject = "Verify your email address", + body = views.html.emails.emailVerificationCreated(appConfig, verification).toString + ) + } + } + } + + private[this] def membershipCreated(guid: UUID): Unit = { + membershipsDao.findByGuid(Authorization.All, guid).foreach { membership => + emails.deliver( + context = Emails.Context.OrganizationAdmin, + org = membership.organization, + publication = Publication.MembershipsCreate, + subject = s"${membership.organization.name}: ${membership.user.email} has joined as ${membership.role}", + body = views.html.emails.membershipCreated(appConfig, membership).toString + ) + } + } + + private[this] def membershipRequestCreated(guid: UUID): Unit = { + membershipRequestsDao.findByGuid(Authorization.All, guid).foreach { request => + emails.deliver( + context = Emails.Context.OrganizationAdmin, + org = request.organization, + publication = Publication.MembershipRequestsCreate, + subject = s"${request.organization.name}: Membership Request from ${request.user.email}", + body = views.html.emails.membershipRequestCreated(appConfig, request).toString + ) + } + } + + private[this] def validateRole(role: String): ValidatedNec[String, Role] = { + Role.fromString(role).toValidNec(s"Invalid role '$role'") + } + + private[this] def membershipRequestAccepted(orgGuid: UUID, userGuid: UUID, role: String): ValidatedNec[String, Unit] = { + validateRole(role).map { vRole => + organizationsDao.findByGuid(Authorization.All, orgGuid).foreach { org => + usersDao.findByGuid(userGuid).foreach { user => + email.sendHtml( + to = Person(user), + subject = s"Welcome to ${org.name}", + body = views.html.emails.membershipRequestAccepted(org, user, vRole).toString + ) + } + } + } + } + + private[this] def membershipRequestDeclined(orgGuid: UUID, userGuid: UUID, role: String): ValidatedNec[String, Unit] = { + validateRole(role).map { vRole => + organizationsDao.findByGuid(Authorization.All, orgGuid).foreach { org => + usersDao.findByGuid(userGuid).foreach { user => + email.sendHtml( + to = Person(user), + subject = s"Your Membership Request to join ${org.name} was declined", + body = views.html.emails.membershipRequestDeclined(org, user, vRole).toString + ) + } + } + } + } + + private[this] def passwordResetRequestCreated(guid: UUID): Unit = { + passwordResetRequestsDao.findByGuid(guid).foreach { request => + usersDao.findByGuid(request.userGuid).foreach { user => + email.sendHtml( + to = Person(user), + subject = "Reset your password", + body = views.html.emails.passwordResetRequestCreated(appConfig, request.token).toString + ) + } + } + } +} \ No newline at end of file diff --git a/api/app/actors/Search.scala b/api/app/processor/IndexApplicationProcessor.scala similarity index 77% rename from api/app/actors/Search.scala rename to api/app/processor/IndexApplicationProcessor.scala index 4ad6b0228..00445c704 100644 --- a/api/app/actors/Search.scala +++ b/api/app/processor/IndexApplicationProcessor.scala @@ -1,19 +1,24 @@ -package actors +package processor +import cats.implicits._ +import cats.data.ValidatedNec +import db.{ApplicationsDao, Authorization, ItemsDao, OrganizationsDao} import io.apibuilder.api.v0.models.{Application, ApplicationSummary, Organization} import io.apibuilder.common.v0.models.Reference -import db.{ApplicationsDao, Authorization, ItemsDao, OrganizationsDao} +import io.apibuilder.task.v0.models.TaskType + import java.util.UUID -import javax.inject.{Inject, Singleton} +import javax.inject.Inject -@Singleton -class Search @Inject() ( + +class IndexApplicationProcessor @Inject()( + args: TaskProcessorArgs, applicationsDao: ApplicationsDao, itemsDao: ItemsDao, organizationsDao: OrganizationsDao -) { +) extends TaskProcessorWithGuid(args, TaskType.IndexApplication) { - def indexApplication(applicationGuid: UUID): Unit = { + override def processRecord(applicationGuid: UUID): ValidatedNec[String, Unit] = { getInfo(applicationGuid) match { case Some((org, app)) => { val content = s"""${app.name} ${app.key} ${app.description.getOrElse("")}""".trim.toLowerCase @@ -33,6 +38,7 @@ class Search @Inject() ( itemsDao.delete(applicationGuid) } } + ().validNec } private[this] def getInfo(applicationGuid: UUID): Option[(Organization, Application)] = { @@ -42,5 +48,4 @@ class Search @Inject() ( } } } - -} +} \ No newline at end of file diff --git a/api/app/processor/MigrateVersionProcessor.scala b/api/app/processor/MigrateVersionProcessor.scala new file mode 100644 index 000000000..d2f15fbe9 --- /dev/null +++ b/api/app/processor/MigrateVersionProcessor.scala @@ -0,0 +1,22 @@ +package processor + +import cats.data.ValidatedNec +import db.VersionsDao +import io.apibuilder.task.v0.models.TaskType + +import java.util.UUID +import javax.inject.Inject + +object MigrateVersion { + val ServiceVersionNumber: String = io.apibuilder.spec.v0.Constants.Version.toLowerCase +} + +class MigrateVersionProcessor @Inject()( + args: TaskProcessorArgs, + versionsDao: VersionsDao +) extends TaskProcessorWithGuid(args, TaskType.MigrateVersion) { + + override def processRecord(versionGuid: UUID): ValidatedNec[String, Unit] = { + versionsDao.migrateVersionGuid(versionGuid) + } +} \ No newline at end of file diff --git a/api/app/processor/ScheduleMigrateVersionsProcessor.scala b/api/app/processor/ScheduleMigrateVersionsProcessor.scala new file mode 100644 index 000000000..d62480f19 --- /dev/null +++ b/api/app/processor/ScheduleMigrateVersionsProcessor.scala @@ -0,0 +1,57 @@ +package processor + +import anorm.SqlParser +import cats.data.ValidatedNec +import cats.implicits._ +import db.InternalTasksDao +import io.apibuilder.task.v0.models.TaskType +import io.flow.postgresql.Query +import play.api.db.{Database, NamedDatabase} + +import javax.inject.Inject +import scala.annotation.tailrec + + +class ScheduleMigrateVersionsProcessor @Inject()( + args: TaskProcessorArgs, + @NamedDatabase("default") db: Database, + internalTasksDao: InternalTasksDao, +) extends TaskProcessor(args, TaskType.ScheduleMigrateVersions) { + + override def processRecord(id: String): ValidatedNec[String, Unit] = { + scheduleMigrationTasks().validNec + } + + + private[this] val VersionsNeedingUpgrade = Query( + """ + |select v.guid + | from versions v + | join applications apps on apps.guid = v.application_guid and apps.deleted_at is null + | left join tasks t on t.type = {task_type} and t.type_id::uuid = v.guid + | where v.deleted_at is null + | and t.id is null + | and not exists ( + | select 1 + | from cache.services + | where services.deleted_at is null + | and services.version_guid = v.guid + | and services.version = {service_version} + | ) + |limit 250 + |""".stripMargin + ).bind("service_version", MigrateVersion.ServiceVersionNumber) + .bind("task_type", TaskType.MigrateVersion.toString) + + @tailrec + private[this] def scheduleMigrationTasks(): Unit = { + val versionGuids = db.withConnection { implicit c => + VersionsNeedingUpgrade + .as(SqlParser.get[_root_.java.util.UUID](1).*) + } + if (versionGuids.nonEmpty) { + internalTasksDao.queueBatch(TaskType.MigrateVersion, versionGuids.map(_.toString)) + scheduleMigrationTasks() + } + } +} \ No newline at end of file diff --git a/api/app/processor/ScheduleSyncGeneratorServicesProcessor.scala b/api/app/processor/ScheduleSyncGeneratorServicesProcessor.scala new file mode 100644 index 000000000..24eb1e4ab --- /dev/null +++ b/api/app/processor/ScheduleSyncGeneratorServicesProcessor.scala @@ -0,0 +1,37 @@ +package processor + +import cats.data.ValidatedNec +import cats.implicits._ +import db.generators.ServicesDao +import db.{Authorization, InternalTasksDao} +import io.apibuilder.task.v0.models.TaskType +import lib.ValidatedHelpers + +import javax.inject.Inject +import scala.annotation.tailrec + + +class ScheduleSyncGeneratorServicesProcessor @Inject()( + args: TaskProcessorArgs, + servicesDao: ServicesDao, + internalTasksDao: InternalTasksDao +) extends TaskProcessor(args, TaskType.ScheduleSyncGeneratorServices) with ValidatedHelpers { + + override def processRecord(id: String): ValidatedNec[String, Unit] = { + doSyncAll(pageSize = 200, offset = 0).validNec + } + + @tailrec + private[this] def doSyncAll(pageSize: Long, offset: Long): Unit = { + val all = servicesDao.findAll( + Authorization.All, + limit = pageSize, + offset = offset + ).map(_.guid.toString) + + if (all.nonEmpty) { + internalTasksDao.queueBatch(TaskType.SyncGeneratorService, all) + doSyncAll(pageSize, offset + all.length) + } + } +} \ No newline at end of file diff --git a/api/app/processor/SyncGeneratorServiceProcessor.scala b/api/app/processor/SyncGeneratorServiceProcessor.scala new file mode 100644 index 000000000..2562022f7 --- /dev/null +++ b/api/app/processor/SyncGeneratorServiceProcessor.scala @@ -0,0 +1,118 @@ +package processor + +import akka.actor.ActorSystem +import cats.data.Validated.{Invalid, Valid} +import cats.data.ValidatedNec +import cats.implicits._ +import db.generators.{GeneratorsDao, ServicesDao} +import db.{Authorization, UsersDao} +import io.apibuilder.api.v0.models.{GeneratorForm, GeneratorService} +import io.apibuilder.generator.v0.interfaces.Client +import io.apibuilder.generator.v0.models.Generator +import io.apibuilder.task.v0.models.TaskType +import lib.{Pager, ValidatedHelpers} +import modules.clients.GeneratorClientFactory +import play.api.Logger + +import java.util.UUID +import javax.inject.Inject +import scala.annotation.tailrec +import scala.concurrent.duration.{FiniteDuration, SECONDS} +import scala.concurrent.{Await, ExecutionContext} +import scala.util.{Failure, Success, Try} + + +class SyncGeneratorServiceProcessor @Inject()( + args: TaskProcessorArgs, + system: ActorSystem, + servicesDao: ServicesDao, + generatorsDao: GeneratorsDao, + usersDao: UsersDao, + generatorClientFactory: GeneratorClientFactory +) extends TaskProcessorWithGuid(args, TaskType.SyncGeneratorService) with ValidatedHelpers { + + private[this] val log: Logger = Logger(this.getClass) + private[this] val ec: ExecutionContext = system.dispatchers.lookup("generator-service-sync-context") + + override def processRecord(guid: UUID): ValidatedNec[String, Unit] = { + syncAll()(ec).validNec + } + + private[this] def syncAll(pageSize: Long = 200)(implicit ec: scala.concurrent.ExecutionContext): Unit = { + Pager.eachPage { offset => + servicesDao.findAll( + Authorization.All, + limit = pageSize, + offset = offset + ) + } { service => + Try { + sync(service, pageSize = pageSize) + } match { + case Success(_) => { + log.info(s"[GeneratorServiceActor] Service[${service.guid}] at uri[${service.uri}] synced") + } + case Failure(ex) => { + ex match { + case _ => log.error(s"[GeneratorServiceActor] Service[${service.guid}] at uri[${service.uri}] failed to sync: ${ex.getMessage}", ex) + } + } + } + } + } + + private[this] def sync( + service: GeneratorService, + pageSize: Long = 200 + ) ( + implicit ec: scala.concurrent.ExecutionContext + ): Unit = { + doSync( + client = generatorClientFactory.instance(service.uri), + service = service, + pageSize = pageSize, + offset = 0, + resolved = Nil + ) + } + + @tailrec + private[this] def doSync( + client: Client, + service: GeneratorService, + pageSize: Long, + offset: Int, + resolved: List[Generator] + )( + implicit ec: scala.concurrent.ExecutionContext + ): Unit = { + val newGenerators = Await.result( + client.generators.get(limit = pageSize.toInt, offset = offset), + FiniteDuration(30, SECONDS) + ).filterNot { g => resolved.exists(_.key == g.key) } + + if (newGenerators.nonEmpty) { + storeGenerators(service, newGenerators) + doSync(client, service, pageSize = pageSize, offset + pageSize.toInt, resolved ++ newGenerators) + } + } + + private[this] def storeGenerators(service: GeneratorService, generators: Seq[Generator]): Unit = { + sequenceUnique { + generators.map { gen => + generatorsDao.upsert( + usersDao.AdminUser, + GeneratorForm( + serviceGuid = service.guid, + generator = gen + ) + ) + } + } match { + case Invalid(errors) => { + log.error(s"Error fetching generators for service[${service.guid}] uri[${service.uri}]: " + formatErrors(errors)) + } + case Valid(_) => // no-op + } + } +} \ No newline at end of file diff --git a/api/app/processor/TaskActorCompanion.scala b/api/app/processor/TaskActorCompanion.scala new file mode 100644 index 000000000..702597948 --- /dev/null +++ b/api/app/processor/TaskActorCompanion.scala @@ -0,0 +1,38 @@ +package processor + +import io.apibuilder.task.v0.models.TaskType + +import javax.inject.Inject + +class TaskActorCompanion @Inject() ( + indexApplication: IndexApplicationProcessor, + diffVersion: DiffVersionProcessor, + cleanupDeletions: CleanupDeletionsProcessor, + scheduleMigrateVersions: ScheduleMigrateVersionsProcessor, + migrateVersion: MigrateVersionProcessor, + userCreated: UserCreatedProcessor, + scheduleSyncGeneratorServices: ScheduleSyncGeneratorServicesProcessor, + syncGeneratorService: SyncGeneratorServiceProcessor, + email: EmailProcessor +) { + + def process(typ: TaskType): Unit = { + lookup(typ).process() + } + + private[this] def lookup(typ: TaskType): BaseTaskProcessor = { + import TaskType._ + typ match { + case IndexApplication => indexApplication + case CleanupDeletions => cleanupDeletions + case DiffVersion => diffVersion + case MigrateVersion => migrateVersion + case ScheduleMigrateVersions => scheduleMigrateVersions + case UserCreated => userCreated + case ScheduleSyncGeneratorServices => scheduleSyncGeneratorServices + case SyncGeneratorService => syncGeneratorService + case Email => email + case UNDEFINED(_) => sys.error(s"Undefined task type '$typ") + } + } +} diff --git a/api/app/processor/TaskDispatchActorCompanion.scala b/api/app/processor/TaskDispatchActorCompanion.scala new file mode 100644 index 000000000..dcdd7f908 --- /dev/null +++ b/api/app/processor/TaskDispatchActorCompanion.scala @@ -0,0 +1,25 @@ +package processor + +import anorm.SqlParser +import io.apibuilder.task.v0.models.TaskType +import io.flow.postgresql.Query +import play.api.db.Database + +import javax.inject.Inject + +class TaskDispatchActorCompanion @Inject() ( + database: Database +) { + private[this] val TypesQuery = Query( + "select distinct type from tasks where next_attempt_at <= now()" + ) + + def typesWithWork: Seq[TaskType] = { + database + .withConnection { c => + TypesQuery.as(SqlParser.str(1).*)(c) + } + .flatMap(TaskType.fromString) + } + +} diff --git a/api/app/processor/TaskProcessor.scala b/api/app/processor/TaskProcessor.scala new file mode 100644 index 000000000..d866c7bf8 --- /dev/null +++ b/api/app/processor/TaskProcessor.scala @@ -0,0 +1,159 @@ +package processor + +import cats.data.Validated.{Invalid, Valid} +import cats.data.ValidatedNec +import cats.implicits._ +import db.generated.{Task, TaskForm, TasksDao} +import io.apibuilder.task.v0.models.TaskType +import io.flow.postgresql.OrderBy +import lib.Constants +import org.joda.time.DateTime +import play.api.libs.json.{JsObject, JsValue, Reads, Writes} +import play.libs.exception.ExceptionUtils + +import java.sql.Connection +import java.util.UUID +import javax.inject.Inject +import scala.util.{Failure, Success, Try} + +class TaskProcessorArgs @Inject() ( + val dao: TasksDao +) {} + +abstract class TaskProcessor( + args: TaskProcessorArgs, + typ: TaskType +) extends BaseTaskProcessor(args, typ) { + def processRecord(id: String): ValidatedNec[String, Unit] + + override final def processTask(task: Task): ValidatedNec[String, Unit] = { + processRecord(task.typeId) + } + +} + +abstract class TaskProcessorWithGuid( + args: TaskProcessorArgs, + typ: TaskType + ) extends BaseTaskProcessor(args, typ) { + + def processRecord(guid: UUID): ValidatedNec[String, Unit] + + override def processTask(task: Task): ValidatedNec[String, Unit] = { + validateGuid(task.typeId).andThen(processRecord) + } + + protected def validateGuid(value: String): ValidatedNec[String, UUID] = { + Try { + UUID.fromString(value) + } match { + case Success(v) => v.validNec + case Failure(_) => s"Invalid guid '$value'".invalidNec + } + } +} + +abstract class TaskProcessorWithData[T]( + args: TaskProcessorArgs, + typ: TaskType +)(implicit reads: Reads[T], writes: Writes[T]) + extends BaseTaskProcessor(args, typ) { + + def processRecord(id: String, data: T): ValidatedNec[String, Unit] + + override final def processTask(task: Task): ValidatedNec[String, Unit] = { + parseData(task.data).andThen { d => + processRecord(task.typeId, d) + } + } + + private[this] def parseData(data: JsValue): ValidatedNec[String, T] = { + data.asOpt[T] match { + case None => "Failed to parse data".invalidNec + case Some(instance) => instance.validNec + } + } + +} + +abstract class BaseTaskProcessor( + args: TaskProcessorArgs, + typ: TaskType +) { + + private[this] val Limit = 100 + + final def process(): Unit = { + args.dao + .findAll( + `type` = Some(typ.toString), + nextAttemptAtLessThanOrEquals = Some(DateTime.now), + limit = Some(Limit), + orderBy = Some(OrderBy("num_attempts, next_attempt_at")) + ) + .foreach(processRecordSafe) + } + + def processTask(task: Task): ValidatedNec[String, Unit] + + private[this] def processRecordSafe(task: Task): Unit = { + Try { + processTask(task) + } match { + case Success(r) => + r match { + case Invalid(errors) => { + setErrors(task, errors.toList, stacktrace = None) + } + case Valid(_) => args.dao.delete(Constants.DefaultUserGuid, task) + } + case Failure(e) => { + setErrors(task, Seq("ERROR: " + e.getMessage), stacktrace = Some(ExceptionUtils.getStackTrace(e))) + } + } + } + + private[this] def setErrors(task: Task, errors: Seq[String], stacktrace: Option[String]): Unit = { + val numAttempts = task.numAttempts + 1 + + args.dao.update( + Constants.DefaultUserGuid, + task, + task.form.copy( + numAttempts = numAttempts, + nextAttemptAt = computeNextAttemptAt(task), + errors = Some(errors.distinct), + stacktrace = stacktrace + ) + ) + } + + protected def computeNextAttemptAt(task: Task): DateTime = { + DateTime.now.plusMinutes(5 * (task.numAttempts + 1)) + } + + final protected def makeInitialTaskForm( + typeId: String, + organizationGuid: Option[UUID], + data: JsObject + ): TaskForm = { + TaskForm( + id = s"$typ:$typeId", + `type` = typ.toString, + typeId = typeId, + organizationGuid = organizationGuid, + data = data, + numAttempts = 0, + nextAttemptAt = DateTime.now, + errors = None, + stacktrace = None + ) + } + + final protected def insertIfNew(c: Connection, form: TaskForm): Unit = { + if (args.dao.findByTypeIdAndTypeWithConnection(c, form.typeId, form.`type`).isEmpty) { + args.dao.upsertByTypeIdAndType(c, Constants.DefaultUserGuid, form) + } + } + +} diff --git a/api/app/processor/UserCreatedProcessor.scala b/api/app/processor/UserCreatedProcessor.scala new file mode 100644 index 000000000..83184a45d --- /dev/null +++ b/api/app/processor/UserCreatedProcessor.scala @@ -0,0 +1,31 @@ +package processor + +import cats.implicits._ +import cats.data.ValidatedNec +import db._ +import io.apibuilder.task.v0.models.TaskType +import lib.Role + +import java.util.UUID +import javax.inject.Inject + + +class UserCreatedProcessor @Inject()( + args: TaskProcessorArgs, + usersDao: UsersDao, + organizationsDao: OrganizationsDao, + membershipRequestsDao: MembershipRequestsDao, + emailVerificationsDao: EmailVerificationsDao, +) extends TaskProcessorWithGuid(args, TaskType.UserCreated) { + + override def processRecord(userGuid: UUID): ValidatedNec[String, Unit] = { + usersDao.findByGuid(userGuid).foreach { user => + organizationsDao.findAllByEmailDomain(user.email).foreach { org => + membershipRequestsDao.upsert(user, org, user, Role.Member) + } + emailVerificationsDao.create(user, user, user.email) + } + ().validNec + } + +} \ No newline at end of file diff --git a/api/app/util/GeneratorServiceUtil.scala b/api/app/util/GeneratorServiceUtil.scala index 87ede88ea..849131830 100644 --- a/api/app/util/GeneratorServiceUtil.scala +++ b/api/app/util/GeneratorServiceUtil.scala @@ -12,6 +12,7 @@ import play.api.Logger import java.util.UUID import javax.inject.Inject +import scala.annotation.tailrec import scala.concurrent.Await import scala.concurrent.duration.{FiniteDuration, SECONDS} import scala.util.{Failure, Success, Try} @@ -67,6 +68,7 @@ class GeneratorServiceUtil @Inject() ( ) } + @tailrec private[this] def doSync( client: Client, service: GeneratorService, diff --git a/api/app/views/emails/errors.scala.html b/api/app/views/emails/errors.scala.html deleted file mode 100644 index 625d887a4..000000000 --- a/api/app/views/emails/errors.scala.html +++ /dev/null @@ -1,7 +0,0 @@ -@(errors: Seq[String]) - - diff --git a/api/conf/base.conf b/api/conf/base.conf index d5042e19f..590bdd73d 100644 --- a/api/conf/base.conf +++ b/api/conf/base.conf @@ -39,38 +39,44 @@ akka { } -main-actor-context { +task-context { fork-join-executor { parallelism-factor = 2.0 parallelism-max = 5 } } -email-actor-context { +periodic-actor-context { fork-join-executor { parallelism-factor = 2.0 - parallelism-max = 5 + parallelism-max = 2 } } -generator-service-actor-context { +task-context-dispatcher { fork-join-executor { - parallelism-factor = 1.0 + parallelism-factor = 2.0 parallelism-max = 2 } } -task-actor-context { +email-actor-context { + fork-join-executor { + parallelism-factor = 2.0 + parallelism-max = 5 + } +} + +generator-service-sync-context { fork-join-executor { parallelism-factor = 1.0 parallelism-max = 2 } } -user-actor-context { +task-actor-context { fork-join-executor { parallelism-factor = 1.0 parallelism-max = 2 } } -git.version = 0.16.19 diff --git a/api/test/actors/TaskActorSpec.scala b/api/test/actors/TaskActorSpec.scala new file mode 100644 index 000000000..b1a7c8b6c --- /dev/null +++ b/api/test/actors/TaskActorSpec.scala @@ -0,0 +1,17 @@ +package actors + +import db.Authorization +import helpers.AsyncHelpers +import org.scalatestplus.play.PlaySpec +import org.scalatestplus.play.guice.GuiceOneAppPerSuite + +class TaskActorSpec extends PlaySpec with GuiceOneAppPerSuite with AsyncHelpers with db.Helpers { + + "run" in { + val app = createApplication() + eventuallyInNSeconds(10) { + itemsDao.findByGuid(Authorization.All, app.guid).value + } + } + +} diff --git a/api/test/db/EmailVerificationsDaoSpec.scala b/api/test/db/EmailVerificationsDaoSpec.scala index 705f4746e..32fc91d4c 100644 --- a/api/test/db/EmailVerificationsDaoSpec.scala +++ b/api/test/db/EmailVerificationsDaoSpec.scala @@ -1,13 +1,14 @@ package db import java.util.UUID - import io.apibuilder.api.v0.models.UserForm import org.scalatestplus.play.PlaySpec import org.scalatestplus.play.guice.GuiceOneAppPerSuite +import processor.UserCreatedProcessor class EmailVerificationsDaoSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { + def userCreatedProcessor: UserCreatedProcessor = injector.instanceOf[UserCreatedProcessor] def emailVerificationConfirmationsDao: EmailVerificationConfirmationsDao = injector.instanceOf[db.EmailVerificationConfirmationsDao] "upsert" in { @@ -112,8 +113,8 @@ class EmailVerificationsDaoSpec extends PlaySpec with GuiceOneAppPerSuite with d password = "testing" )) - usersDao.processUserCreated(user.guid) - usersDao.processUserCreated(nonMatchingUser.guid) + userCreatedProcessor.processRecord(user.guid) + userCreatedProcessor.processRecord(nonMatchingUser.guid) membershipsDao.isUserMember(user, org) must be(false) membershipsDao.isUserMember(nonMatchingUser, org) must be(false) diff --git a/api/test/db/InternalMigrationsDaoSpec.scala b/api/test/db/InternalMigrationsDaoSpec.scala deleted file mode 100644 index 87b6a720f..000000000 --- a/api/test/db/InternalMigrationsDaoSpec.scala +++ /dev/null @@ -1,46 +0,0 @@ -package db - -import db.generated.MigrationsDao -import io.flow.postgresql.Query -import org.scalatestplus.play.PlaySpec -import org.scalatestplus.play.guice.GuiceOneAppPerSuite - -import java.util.UUID - -class InternalMigrationsDaoSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers with DbUtils { - - private[this] def internalMigrationsDao: InternalMigrationsDao = injector.instanceOf[InternalMigrationsDao] - private[this] def generatedMigrationsDao: MigrationsDao = injector.instanceOf[MigrationsDao] - - private[this] def deleteCachedServices(versionGuid: UUID): Unit = { - execute( - Query("update cache.services set deleted_at=now(), deleted_by_guid={user_guid}::uuid") - .equals("version_guid", versionGuid) - .bind("user_guid", testUser.guid) - ) - } - - "queueVersions" in { - val version = createVersion() - deleteCachedServices(version.guid) - def count = generatedMigrationsDao.findAll(versionGuid = Some(version.guid), limit = None).length - - count mustBe 0 - internalMigrationsDao.queueVersions() - count mustBe 1 - } - - "migrateBatch" in { - val version = createVersion() - deleteCachedServices(version.guid) - - def exists = versionsDao.findByGuid(Authorization.All, version.guid).isDefined - - exists mustBe false - - internalMigrationsDao.queueVersions() - internalMigrationsDao.migrateBatch(1000L) - - exists mustBe true - } -} \ No newline at end of file diff --git a/api/test/db/TasksDaoSpec.scala b/api/test/db/TasksDaoSpec.scala deleted file mode 100644 index 927d3c521..000000000 --- a/api/test/db/TasksDaoSpec.scala +++ /dev/null @@ -1,232 +0,0 @@ -package db - -import java.util.UUID - -import anorm._ -import io.apibuilder.api.v0.models.User -import io.apibuilder.internal.v0.models.{Task, TaskDataDiffVersion} -import org.joda.time.DateTime -import org.postgresql.util.PSQLException -import org.scalatestplus.play.PlaySpec -import org.scalatestplus.play.guice.GuiceOneAppPerSuite -import play.api.db._ - -class TasksDaoSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { - - private[this] def setDeletedAt(task: Task, days: Int): Unit = { - val query = s""" - update tasks set deleted_at = timezone('utc', now()) - interval '$days days' where guid = {guid}::uuid - """ - - injector.instanceOf[DBApi].database("default").withConnection { implicit c => - SQL(query).on("guid" -> task.guid).execute() - } - } - - private[this] lazy val user: User = createRandomUser() - - private[this] def createTaskDataDiffVersion( - oldGuid: UUID = UUID.randomUUID, - newGuid: UUID = UUID.randomUUID, - numberAttempts: Int = 0 - ): Task = { - val guid = injector.instanceOf[DBApi].database("default").withConnection { implicit c => - tasksDao.insert(c, user, TaskDataDiffVersion(oldGuid, newGuid)) - } - - val task = tasksDao.findByGuid(guid).getOrElse { - sys.error("failed to find task") - } - - (0 to numberAttempts).foreach { _ => - tasksDao.incrementNumberAttempts(user, task) - } - - tasksDao.findByGuid(guid).getOrElse { - sys.error("failed to create task") - } - } - - "findByGuid" in { - val oldGuid = UUID.randomUUID - val newGuid = UUID.randomUUID - createTaskDataDiffVersion(oldGuid, newGuid).data must be(TaskDataDiffVersion(oldGuid, newGuid)) - } - - "softDelete" in { - val task = createTaskDataDiffVersion() - tasksDao.softDelete(user, task) - tasksDao.findByGuid(task.guid) must be(None) - } - - "incrementNumberAttempts" in { - val task = createTaskDataDiffVersion() - val original = task.numberAttempts - tasksDao.incrementNumberAttempts(user, task) - tasksDao.findByGuid(task.guid).getOrElse { - sys.error("failed to find task") - }.numberAttempts must be(original + 1) - } - - "recordError" in { - val task = createTaskDataDiffVersion() - tasksDao.recordError(user, task, "Test") - tasksDao.findByGuid(task.guid).getOrElse { - sys.error("failed to find task") - }.lastError must be(Some("Test")) - } - - "findAll" must { - - "nOrFewerAttempts" in { - val task = createTaskDataDiffVersion(numberAttempts = 2) - - tasksDao.findAll( - guid = Some(task.guid), - nOrFewerAttempts = Some(task.numberAttempts) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - nOrFewerAttempts = Some(task.numberAttempts - 1) - ).map(_.guid) must be(Nil) - } - - "nOrMoreAttempts" in { - val task = createTaskDataDiffVersion(numberAttempts = 2) - - tasksDao.findAll( - guid = Some(task.guid), - nOrMoreAttempts = Some(task.numberAttempts) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - nOrMoreAttempts = Some(task.numberAttempts + 1) - ).map(_.guid) must be(Nil) - } - - "nOrMoreMinutesOld" in { - val task = createTaskDataDiffVersion() - - tasksDao.findAll( - guid = Some(task.guid), - createdOnOrBefore = Some(DateTime.now.plusHours(1)) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - createdOnOrBefore = Some(DateTime.now.minusHours(1)) - ).map(_.guid) must be(Nil) - } - - "nOrMoreMinutesYoung" in { - val task = createTaskDataDiffVersion() - - tasksDao.findAll( - guid = Some(task.guid), - createdOnOrAfter = Some(DateTime.now.minusHours(1)) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - createdOnOrAfter = Some(DateTime.now.plusHours(1)) - ).map(_.guid) must be(Nil) - } - - "isDeleted" in { - val task = createTaskDataDiffVersion() - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = Some(false) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = Some(true) - ).map(_.guid) must be(Nil) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.softDelete(user, task) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = Some(false) - ).map(_.guid) must be(Nil) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = Some(true) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None - ).map(_.guid) must be(Seq(task.guid)) - - } - - "deletedAtLeastNDaysAgo" in { - val task = createTaskDataDiffVersion() - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None, - deletedAtLeastNDaysAgo = Some(0) - ) must be(Nil) - - tasksDao.softDelete(user, task) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None, - deletedAtLeastNDaysAgo = Some(90) - ).map(_.guid) must be(Nil) - - setDeletedAt(task, 89) - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None, - deletedAtLeastNDaysAgo = Some(90) - ) must be(Nil) - - setDeletedAt(task, 91) - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None, - deletedAtLeastNDaysAgo = Some(90) - ).map(_.guid) must be(Seq(task.guid)) - } - } - - "purge" must { - - "raises error if recently deleted" in { - val task = createTaskDataDiffVersion() - tasksDao.softDelete(user, task) - val ex = intercept[PSQLException] { - tasksDao.purge(task) - } - println(ex.getMessage) - ex.getMessage.contains("ERROR: Physical deletes on this table can occur only after 1 month of deleting the records") must be(true) - } - - "purges if old" in { - val task = createTaskDataDiffVersion() - tasksDao.softDelete(user, task) - setDeletedAt(task, 45) - tasksDao.purge(task) - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None - ) must be(Nil) - } - - } - -} diff --git a/api/test/db/VersionValidatorSpec.scala b/api/test/db/VersionValidatorSpec.scala index 3dd57c55d..19d5187da 100644 --- a/api/test/db/VersionValidatorSpec.scala +++ b/api/test/db/VersionValidatorSpec.scala @@ -7,7 +7,7 @@ import org.scalatestplus.play.guice.GuiceOneAppPerSuite class VersionValidatorSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { - def versionValidator = injector.instanceOf[VersionValidator] + private[this] def versionValidator = injector.instanceOf[VersionValidator] "validates user is a member of the organization to create an application" in { val org = createOrganization() diff --git a/api/test/helpers/AsyncHelpers.scala b/api/test/helpers/AsyncHelpers.scala new file mode 100644 index 000000000..964cdb79a --- /dev/null +++ b/api/test/helpers/AsyncHelpers.scala @@ -0,0 +1,16 @@ +package helpers + +import lib.TestHelper +import org.scalatest.concurrent.Eventually +import org.scalatest.concurrent.PatienceConfiguration.Timeout +import org.scalatest.time.{Seconds, Span} + +trait AsyncHelpers extends TestHelper with Eventually { + + def eventuallyInNSeconds[T](n: Long = 3)(f: => T): T = { + eventually(Timeout(Span(n, Seconds))) { + f + } + } + +} diff --git a/api/test/invariants/CheckInvariantsSpec.scala b/api/test/invariants/CheckInvariantsSpec.scala new file mode 100644 index 000000000..b0e51179f --- /dev/null +++ b/api/test/invariants/CheckInvariantsSpec.scala @@ -0,0 +1,13 @@ +package invariants + +import org.scalatestplus.play.PlaySpec +import org.scalatestplus.play.guice.GuiceOneAppPerSuite + +class CheckInvariantsSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { + + private def checkInvariants: CheckInvariants = injector.instanceOf[CheckInvariants] + + "process" in { + checkInvariants.process() + } +} diff --git a/api/test/util/ProcessDeletesSpec.scala b/api/test/processor/CleanupDeletionsProcessorSpec.scala similarity index 83% rename from api/test/util/ProcessDeletesSpec.scala rename to api/test/processor/CleanupDeletionsProcessorSpec.scala index 94fbf8280..31369fee7 100644 --- a/api/test/util/ProcessDeletesSpec.scala +++ b/api/test/processor/CleanupDeletionsProcessorSpec.scala @@ -1,4 +1,4 @@ -package util +package processor import anorm.SqlParser import db.Helpers @@ -10,9 +10,9 @@ import play.api.db.Database import java.util.UUID -class ProcessDeletesSpec extends PlaySpec with GuiceOneAppPerSuite with Helpers { +class CleanupDeletionsProcessorSpec extends PlaySpec with GuiceOneAppPerSuite with Helpers { - private[this] def processDeletes: ProcessDeletes = app.injector.instanceOf[ProcessDeletes] + private[this] def processor: CleanupDeletionsProcessor = app.injector.instanceOf[CleanupDeletionsProcessor] private[this] def database: Database = app.injector.instanceOf[Database] private[this] def isDeleted(table: String, guid: UUID): Boolean = { @@ -39,7 +39,7 @@ class ProcessDeletesSpec extends PlaySpec with GuiceOneAppPerSuite with Helpers isAppDeleted(app) mustBe false isAppDeleted(appDeleted) mustBe false - processDeletes.organizations() + processor.organizations() isAppDeleted(app) mustBe false isAppDeleted(appDeleted) mustBe true } @@ -61,7 +61,7 @@ class ProcessDeletesSpec extends PlaySpec with GuiceOneAppPerSuite with Helpers isVersionDeleted(version) mustBe false isVersionDeleted(versionDeleted) mustBe false - processDeletes.applications() + processor.applications() isVersionDeleted(version) mustBe false isVersionDeleted(versionDeleted) mustBe true } @@ -102,28 +102,28 @@ class ProcessDeletesSpec extends PlaySpec with GuiceOneAppPerSuite with Helpers "Organization" must { "soft" in { - getTablesSoft("organization_guid") mustBe ProcessDeletes.OrganizationSoft + getTablesSoft("organization_guid") mustBe DeleteMetadata.OrganizationSoft } "hard" in { - getTablesHard("organization_guid") mustBe ProcessDeletes.OrganizationHard + getTablesHard("organization_guid") mustBe DeleteMetadata.OrganizationHard } } "Application" must { "soft" in { - getTablesSoft("application_guid") mustBe ProcessDeletes.ApplicationSoft + getTablesSoft("application_guid") mustBe DeleteMetadata.ApplicationSoft } "hard" in { - getTablesHard("application_guid") mustBe ProcessDeletes.ApplicationHard + getTablesHard("application_guid") mustBe DeleteMetadata.ApplicationHard } } "Version" must { "soft" in { - getTablesSoft("version_guid") mustBe ProcessDeletes.VersionSoft + getTablesSoft("version_guid") mustBe DeleteMetadata.VersionSoft } "hard" in { - getTablesHard("version_guid") mustBe ProcessDeletes.VersionHard + getTablesHard("version_guid") mustBe DeleteMetadata.VersionHard } } } diff --git a/api/test/actors/SearchSpec.scala b/api/test/processor/IndexApplicationProcessorSpec.scala similarity index 76% rename from api/test/actors/SearchSpec.scala rename to api/test/processor/IndexApplicationProcessorSpec.scala index a3b02b2ba..f8b6ec79b 100644 --- a/api/test/actors/SearchSpec.scala +++ b/api/test/processor/IndexApplicationProcessorSpec.scala @@ -1,12 +1,21 @@ -package actors - -import java.util.UUID +package processor import db.Authorization +import lib.TestHelper import org.scalatestplus.play.PlaySpec import org.scalatestplus.play.guice.GuiceOneAppPerSuite -class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { +import java.util.UUID + +class IndexApplicationProcessorSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers with TestHelper { + + private[this] def processor = injector.instanceOf[IndexApplicationProcessor] + + private[this] def indexApplication(guid: UUID): Unit = { + expectValid { + processor.processRecord(guid) + } + } "indexApplication" must { @@ -14,7 +23,7 @@ class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { val description = UUID.randomUUID.toString val form = createApplicationForm().copy(description = Some(description)) val app = createApplication(form = form) - search.indexApplication(app.guid) + indexApplication(app.guid) Seq( app.name, @@ -31,8 +40,7 @@ class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { "on create" in { val app = createApplication() - - search.indexApplication(app.guid) + indexApplication(app.guid) itemsDao.findAll(Authorization.All, guid = Some(app.guid) @@ -42,8 +50,7 @@ class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { "on update" in { val form = createApplicationForm() val app = createApplication(form = form) - - search.indexApplication(app.guid) + indexApplication(app.guid) val newName = app.name + "2" @@ -52,8 +59,7 @@ class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { app = app, form = form.copy(name = newName) ) - - search.indexApplication(app.guid) + indexApplication(app.guid) val existing = applicationsDao.findByGuid(Authorization.All, app.guid).get @@ -65,10 +71,10 @@ class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { "on delete" in { val app = createApplication() - search.indexApplication(app.guid) + indexApplication(app.guid) applicationsDao.softDelete(testUser, app) - search.indexApplication(app.guid) + indexApplication(app.guid) itemsDao.findAll(Authorization.All, guid = Some(app.guid)) must be(Nil) } diff --git a/api/test/util/Daos.scala b/api/test/util/Daos.scala index 651f38e22..537b3caae 100644 --- a/api/test/util/Daos.scala +++ b/api/test/util/Daos.scala @@ -1,9 +1,9 @@ package util -import actors.{Emails, Search} -import db.{ApplicationsDao, AttributesDao, ChangesDao, EmailVerificationsDao, ItemsDao, MembershipRequestsDao, MembershipsDao, OrganizationAttributeValuesDao, OrganizationDomainsDao, OrganizationLogsDao, OrganizationsDao, OriginalsDao, PasswordResetRequestsDao, SubscriptionsDao, TasksDao, TokensDao, UserPasswordsDao, UsersDao, VersionsDao} +import actors.Emails import db.generated.SessionsDao import db.generators.{GeneratorsDao, ServicesDao} +import db._ import lib.DatabaseServiceFetcher import play.api.Application import play.api.inject.Injector @@ -31,7 +31,6 @@ trait Daos { def sessionsDao: SessionsDao = injector.instanceOf[SessionsDao] def subscriptionsDao: SubscriptionsDao = injector.instanceOf[db.SubscriptionsDao] - def tasksDao: TasksDao = injector.instanceOf[db.TasksDao] def tokensDao: TokensDao = injector.instanceOf[db.TokensDao] def userPasswordsDao: UserPasswordsDao = injector.instanceOf[db.UserPasswordsDao] def versionsDao: VersionsDao = injector.instanceOf[db.VersionsDao] @@ -40,7 +39,6 @@ trait Daos { def generatorsDao: GeneratorsDao = injector.instanceOf[db.generators.GeneratorsDao] def emails: Emails = injector.instanceOf[actors.Emails] - def search: Search = injector.instanceOf[actors.Search] def sessionHelper: SessionHelper = injector.instanceOf[SessionHelper] } diff --git a/dao/spec/psql-apibuilder.json b/dao/spec/psql-apibuilder.json index 14c9c84ec..6115a9571 100644 --- a/dao/spec/psql-apibuilder.json +++ b/dao/spec/psql-apibuilder.json @@ -70,6 +70,43 @@ } } ] + }, + + "task": { + "fields": [ + { "name": "id", "type": "string" }, + { "name": "type", "type": "string" }, + { "name": "type_id", "type": "string" }, + { "name": "organization_guid", "type": "uuid", "required": false }, + { "name": "num_attempts", "type": "integer", "minimum": 0, "default": 0 }, + { "name": "next_attempt_at", "type": "date-time-iso8601" }, + { "name": "errors", "type": "[string]", "required": false }, + { "name": "stacktrace", "type": "string", "required": false }, + { "name": "data", "type": "json" } + ], + "attributes": [ + { + "name": "scala", + "value": { + "package": "db.generated", + "dao_user_class": "java.util.UUID", + "order_by": { "optional": true } + } + }, + { + "name": "psql", + "value": { + "pkey": "id", + "authorization": { "type": "disabled" }, + "on_conflict": { + "fields": ["type_id", "type"] + }, + "indexes": [ + { "fields": ["num_attempts", "next_attempt_at"] } + ] + } + } + ] } } } diff --git a/generated/app/ApicollectiveApibuilderInternalV0Models.scala b/generated/app/ApicollectiveApibuilderInternalV0Models.scala deleted file mode 100644 index 3255d2e0d..000000000 --- a/generated/app/ApicollectiveApibuilderInternalV0Models.scala +++ /dev/null @@ -1,263 +0,0 @@ -/** - * Generated by API Builder - https://www.apibuilder.io - * Service version: 0.15.81 - * apibuilder 0.15.33 app.apibuilder.io/apicollective/apibuilder-internal/latest/play_2_x_json - */ -package io.apibuilder.internal.v0.models { - - sealed trait TaskData extends _root_.scala.Product with _root_.scala.Serializable - /** - * @param numberAttempts Records the number of times we have attempted to run this task. Commonly we - * increment number attempts, process the task, and if succeeds we then delete the - * task. If it fails, we update last_error. This allows us to retry a task say - * twice; after which we no longer process the task (can notify an admin of the - * error). - */ - - final case class Task( - guid: _root_.java.util.UUID, - data: io.apibuilder.internal.v0.models.TaskData, - numberAttempts: Long = 0L, - lastError: _root_.scala.Option[String] = None - ) - - final case class TaskDataDiffVersion( - oldVersionGuid: _root_.java.util.UUID, - newVersionGuid: _root_.java.util.UUID - ) extends TaskData - - final case class TaskDataIndexApplication( - applicationGuid: _root_.java.util.UUID - ) extends TaskData - - /** - * Provides future compatibility in clients - in the future, when a type is added - * to the union TaskData, it will need to be handled in the client code. This - * implementation will deserialize these future types as an instance of this class. - * - * @param description Information about the type that we received that is undefined in this version of - * the client. - */ - - final case class TaskDataUndefinedType( - description: String - ) extends TaskData - -} - -package io.apibuilder.internal.v0.models { - - package object json { - import play.api.libs.json.__ - import play.api.libs.json.JsString - import play.api.libs.json.Writes - import play.api.libs.functional.syntax._ - import io.apibuilder.internal.v0.models.json._ - - private[v0] implicit val jsonReadsUUID: play.api.libs.json.Reads[_root_.java.util.UUID] = __.read[String].map { str => - _root_.java.util.UUID.fromString(str) - } - - private[v0] implicit val jsonWritesUUID: play.api.libs.json.Writes[_root_.java.util.UUID] = (x: _root_.java.util.UUID) => play.api.libs.json.JsString(x.toString) - - private[v0] implicit val jsonReadsJodaDateTime: play.api.libs.json.Reads[_root_.org.joda.time.DateTime] = __.read[String].map { str => - _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseDateTime(str) - } - - private[v0] implicit val jsonWritesJodaDateTime: play.api.libs.json.Writes[_root_.org.joda.time.DateTime] = (x: _root_.org.joda.time.DateTime) => { - play.api.libs.json.JsString(_root_.org.joda.time.format.ISODateTimeFormat.dateTime.print(x)) - } - - private[v0] implicit val jsonReadsJodaLocalDate: play.api.libs.json.Reads[_root_.org.joda.time.LocalDate] = __.read[String].map { str => - _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseLocalDate(str) - } - - private[v0] implicit val jsonWritesJodaLocalDate: play.api.libs.json.Writes[_root_.org.joda.time.LocalDate] = (x: _root_.org.joda.time.LocalDate) => { - play.api.libs.json.JsString(_root_.org.joda.time.format.ISODateTimeFormat.date.print(x)) - } - - implicit def jsonReadsApibuilderInternalTask: play.api.libs.json.Reads[Task] = { - for { - guid <- (__ \ "guid").read[_root_.java.util.UUID] - data <- (__ \ "data").read[io.apibuilder.internal.v0.models.TaskData] - numberAttempts <- (__ \ "number_attempts").read[Long] - lastError <- (__ \ "last_error").readNullable[String] - } yield Task(guid, data, numberAttempts, lastError) - } - - def jsObjectTask(obj: io.apibuilder.internal.v0.models.Task): play.api.libs.json.JsObject = { - play.api.libs.json.Json.obj( - "guid" -> play.api.libs.json.JsString(obj.guid.toString), - "data" -> jsObjectTaskData(obj.data), - "number_attempts" -> play.api.libs.json.JsNumber(obj.numberAttempts) - ) ++ (obj.lastError match { - case None => play.api.libs.json.Json.obj() - case Some(x) => play.api.libs.json.Json.obj("last_error" -> play.api.libs.json.JsString(x)) - }) - } - - implicit def jsonWritesApibuilderInternalTask: play.api.libs.json.Writes[Task] = { - (obj: io.apibuilder.internal.v0.models.Task) => { - jsObjectTask(obj) - } - } - - implicit def jsonReadsApibuilderInternalTaskDataDiffVersion: play.api.libs.json.Reads[TaskDataDiffVersion] = { - for { - oldVersionGuid <- (__ \ "old_version_guid").read[_root_.java.util.UUID] - newVersionGuid <- (__ \ "new_version_guid").read[_root_.java.util.UUID] - } yield TaskDataDiffVersion(oldVersionGuid, newVersionGuid) - } - - def jsObjectTaskDataDiffVersion(obj: io.apibuilder.internal.v0.models.TaskDataDiffVersion): play.api.libs.json.JsObject = { - play.api.libs.json.Json.obj( - "old_version_guid" -> play.api.libs.json.JsString(obj.oldVersionGuid.toString), - "new_version_guid" -> play.api.libs.json.JsString(obj.newVersionGuid.toString) - ) - } - - implicit def jsonWritesApibuilderInternalTaskDataDiffVersion: play.api.libs.json.Writes[TaskDataDiffVersion] = { - (obj: io.apibuilder.internal.v0.models.TaskDataDiffVersion) => { - jsObjectTaskDataDiffVersion(obj) - } - } - - implicit def jsonReadsApibuilderInternalTaskDataIndexApplication: play.api.libs.json.Reads[TaskDataIndexApplication] = { - (__ \ "application_guid").read[_root_.java.util.UUID].map { x => new TaskDataIndexApplication(applicationGuid = x) } - } - - def jsObjectTaskDataIndexApplication(obj: io.apibuilder.internal.v0.models.TaskDataIndexApplication): play.api.libs.json.JsObject = { - play.api.libs.json.Json.obj( - "application_guid" -> play.api.libs.json.JsString(obj.applicationGuid.toString) - ) - } - - implicit def jsonWritesApibuilderInternalTaskDataIndexApplication: play.api.libs.json.Writes[TaskDataIndexApplication] = { - (obj: io.apibuilder.internal.v0.models.TaskDataIndexApplication) => { - jsObjectTaskDataIndexApplication(obj) - } - } - - implicit def jsonReadsApibuilderInternalTaskData: play.api.libs.json.Reads[TaskData] = { - ( - (__ \ "task_data_index_application").read(jsonReadsApibuilderInternalTaskDataIndexApplication).asInstanceOf[play.api.libs.json.Reads[TaskData]] - orElse - (__ \ "task_data_diff_version").read(jsonReadsApibuilderInternalTaskDataDiffVersion).asInstanceOf[play.api.libs.json.Reads[TaskData]] - orElse - play.api.libs.json.Reads(jsValue => play.api.libs.json.JsSuccess(TaskDataUndefinedType(jsValue.toString))).asInstanceOf[play.api.libs.json.Reads[TaskData]] - ) - } - - def jsObjectTaskData(obj: io.apibuilder.internal.v0.models.TaskData): play.api.libs.json.JsObject = { - obj match { - case x: io.apibuilder.internal.v0.models.TaskDataIndexApplication => play.api.libs.json.Json.obj("task_data_index_application" -> jsObjectTaskDataIndexApplication(x)) - case x: io.apibuilder.internal.v0.models.TaskDataDiffVersion => play.api.libs.json.Json.obj("task_data_diff_version" -> jsObjectTaskDataDiffVersion(x)) - case x: io.apibuilder.internal.v0.models.TaskDataUndefinedType => sys.error(s"The type[io.apibuilder.internal.v0.models.TaskDataUndefinedType] should never be serialized") - } - } - - implicit def jsonWritesApibuilderInternalTaskData: play.api.libs.json.Writes[TaskData] = { - (obj: io.apibuilder.internal.v0.models.TaskData) => { - jsObjectTaskData(obj) - } - } - } -} - -package io.apibuilder.internal.v0 { - - object Bindables { - - import play.api.mvc.{PathBindable, QueryStringBindable} - - // import models directly for backwards compatibility with prior versions of the generator - import Core._ - - object Core { - implicit def pathBindableDateTimeIso8601(implicit stringBinder: QueryStringBindable[String]): PathBindable[_root_.org.joda.time.DateTime] = ApibuilderPathBindable(ApibuilderTypes.dateTimeIso8601) - implicit def queryStringBindableDateTimeIso8601(implicit stringBinder: QueryStringBindable[String]): QueryStringBindable[_root_.org.joda.time.DateTime] = ApibuilderQueryStringBindable(ApibuilderTypes.dateTimeIso8601) - - implicit def pathBindableDateIso8601(implicit stringBinder: QueryStringBindable[String]): PathBindable[_root_.org.joda.time.LocalDate] = ApibuilderPathBindable(ApibuilderTypes.dateIso8601) - implicit def queryStringBindableDateIso8601(implicit stringBinder: QueryStringBindable[String]): QueryStringBindable[_root_.org.joda.time.LocalDate] = ApibuilderQueryStringBindable(ApibuilderTypes.dateIso8601) - } - - trait ApibuilderTypeConverter[T] { - - def convert(value: String): T - - def convert(value: T): String - - def example: T - - def validValues: Seq[T] = Nil - - def errorMessage(key: String, value: String, ex: java.lang.Exception): String = { - val base = s"Invalid value '$value' for parameter '$key'. " - validValues.toList match { - case Nil => base + "Ex: " + convert(example) - case values => base + ". Valid values are: " + values.mkString("'", "', '", "'") - } - } - } - - object ApibuilderTypes { - val dateTimeIso8601: ApibuilderTypeConverter[_root_.org.joda.time.DateTime] = new ApibuilderTypeConverter[_root_.org.joda.time.DateTime] { - override def convert(value: String): _root_.org.joda.time.DateTime = _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseDateTime(value) - override def convert(value: _root_.org.joda.time.DateTime): String = _root_.org.joda.time.format.ISODateTimeFormat.dateTime.print(value) - override def example: _root_.org.joda.time.DateTime = _root_.org.joda.time.DateTime.now - } - - val dateIso8601: ApibuilderTypeConverter[_root_.org.joda.time.LocalDate] = new ApibuilderTypeConverter[_root_.org.joda.time.LocalDate] { - override def convert(value: String): _root_.org.joda.time.LocalDate = _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseLocalDate(value) - override def convert(value: _root_.org.joda.time.LocalDate): String = _root_.org.joda.time.format.ISODateTimeFormat.date.print(value) - override def example: _root_.org.joda.time.LocalDate = _root_.org.joda.time.LocalDate.now - } - } - - final case class ApibuilderQueryStringBindable[T]( - converters: ApibuilderTypeConverter[T] - ) extends QueryStringBindable[T] { - - override def bind(key: String, params: Map[String, Seq[String]]): _root_.scala.Option[_root_.scala.Either[String, T]] = { - params.getOrElse(key, Nil).headOption.map { v => - try { - Right( - converters.convert(v) - ) - } catch { - case ex: java.lang.Exception => Left( - converters.errorMessage(key, v, ex) - ) - } - } - } - - override def unbind(key: String, value: T): String = { - s"$key=${converters.convert(value)}" - } - } - - final case class ApibuilderPathBindable[T]( - converters: ApibuilderTypeConverter[T] - ) extends PathBindable[T] { - - override def bind(key: String, value: String): _root_.scala.Either[String, T] = { - try { - Right( - converters.convert(value) - ) - } catch { - case ex: java.lang.Exception => Left( - converters.errorMessage(key, value, ex) - ) - } - } - - override def unbind(key: String, value: T): String = { - converters.convert(value) - } - } - - } - -} diff --git a/generated/app/ApicollectiveApibuilderTaskV0Client.scala b/generated/app/ApicollectiveApibuilderTaskV0Client.scala new file mode 100644 index 000000000..dad800f0b --- /dev/null +++ b/generated/app/ApicollectiveApibuilderTaskV0Client.scala @@ -0,0 +1,605 @@ +/** + * Generated by API Builder - https://www.apibuilder.io + * Service version: 0.16.24 + * apibuilder app.apibuilder.io/apicollective/apibuilder-task/latest/play_2_8_client + */ +package io.apibuilder.task.v0.models { + + sealed trait EmailData extends _root_.scala.Product with _root_.scala.Serializable + final case class DiffVersionData( + oldVersionGuid: _root_.java.util.UUID, + newVersionGuid: _root_.java.util.UUID + ) + + final case class EmailDataApplicationCreated( + applicationGuid: _root_.java.util.UUID + ) extends EmailData + + final case class EmailDataEmailVerificationCreated( + guid: _root_.java.util.UUID + ) extends EmailData + + final case class EmailDataMembershipCreated( + guid: _root_.java.util.UUID + ) extends EmailData + + final case class EmailDataMembershipRequestAccepted( + organizationGuid: _root_.java.util.UUID, + userGuid: _root_.java.util.UUID, + role: String + ) extends EmailData + + final case class EmailDataMembershipRequestCreated( + guid: _root_.java.util.UUID + ) extends EmailData + + final case class EmailDataMembershipRequestDeclined( + organizationGuid: _root_.java.util.UUID, + userGuid: _root_.java.util.UUID, + role: String + ) extends EmailData + + final case class EmailDataPasswordResetRequestCreated( + guid: _root_.java.util.UUID + ) extends EmailData + + /** + * Provides future compatibility in clients - in the future, when a type is added + * to the union EmailData, it will need to be handled in the client code. This + * implementation will deserialize these future types as an instance of this class. + * + * @param description Information about the type that we received that is undefined in this version of + * the client. + */ + + final case class EmailDataUndefinedType( + description: String + ) extends EmailData + sealed trait TaskType extends _root_.scala.Product with _root_.scala.Serializable + + object TaskType { + + case object Email extends TaskType { override def toString = "email" } + case object IndexApplication extends TaskType { override def toString = "index_application" } + case object CleanupDeletions extends TaskType { override def toString = "cleanup_deletions" } + case object ScheduleMigrateVersions extends TaskType { override def toString = "schedule_migrate_versions" } + case object MigrateVersion extends TaskType { override def toString = "migrate_version" } + case object ScheduleSyncGeneratorServices extends TaskType { + override def toString = "schedule_sync_generator_services" + } + case object SyncGeneratorService extends TaskType { override def toString = "sync_generator_service" } + case object DiffVersion extends TaskType { override def toString = "diff_version" } + case object UserCreated extends TaskType { override def toString = "user_created" } + /** + * UNDEFINED captures values that are sent either in error or + * that were added by the server after this library was + * generated. We want to make it easy and obvious for users of + * this library to handle this case gracefully. + * + * We use all CAPS for the variable name to avoid collisions + * with the camel cased values above. + */ + final case class UNDEFINED(override val toString: String) extends TaskType + + /** + * all returns a list of all the valid, known values. We use + * lower case to avoid collisions with the camel cased values + * above. + */ + val all: scala.List[TaskType] = scala.List(Email, IndexApplication, CleanupDeletions, ScheduleMigrateVersions, MigrateVersion, ScheduleSyncGeneratorServices, SyncGeneratorService, DiffVersion, UserCreated) + + private[this] + val byName: Map[String, TaskType] = all.map(x => x.toString.toLowerCase -> x).toMap + + def apply(value: String): TaskType = fromString(value).getOrElse(UNDEFINED(value)) + + def fromString(value: String): _root_.scala.Option[TaskType] = byName.get(value.toLowerCase) + + } + +} + +package io.apibuilder.task.v0.models { + + package object json { + import play.api.libs.json.__ + import play.api.libs.json.JsString + import play.api.libs.json.Writes + import play.api.libs.functional.syntax._ + import io.apibuilder.task.v0.models.json._ + + private[v0] implicit val jsonReadsUUID: play.api.libs.json.Reads[_root_.java.util.UUID] = __.read[String].map { str => + _root_.java.util.UUID.fromString(str) + } + + private[v0] implicit val jsonWritesUUID: play.api.libs.json.Writes[_root_.java.util.UUID] = (x: _root_.java.util.UUID) => play.api.libs.json.JsString(x.toString) + + private[v0] implicit val jsonReadsJodaDateTime: play.api.libs.json.Reads[_root_.org.joda.time.DateTime] = __.read[String].map { str => + _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseDateTime(str) + } + + private[v0] implicit val jsonWritesJodaDateTime: play.api.libs.json.Writes[_root_.org.joda.time.DateTime] = (x: _root_.org.joda.time.DateTime) => { + play.api.libs.json.JsString(_root_.org.joda.time.format.ISODateTimeFormat.dateTime.print(x)) + } + + private[v0] implicit val jsonReadsJodaLocalDate: play.api.libs.json.Reads[_root_.org.joda.time.LocalDate] = __.read[String].map { str => + _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseLocalDate(str) + } + + private[v0] implicit val jsonWritesJodaLocalDate: play.api.libs.json.Writes[_root_.org.joda.time.LocalDate] = (x: _root_.org.joda.time.LocalDate) => { + play.api.libs.json.JsString(_root_.org.joda.time.format.ISODateTimeFormat.date.print(x)) + } + + implicit val jsonReadsApibuilderTaskTaskType: play.api.libs.json.Reads[io.apibuilder.task.v0.models.TaskType] = new play.api.libs.json.Reads[io.apibuilder.task.v0.models.TaskType] { + def reads(js: play.api.libs.json.JsValue): play.api.libs.json.JsResult[io.apibuilder.task.v0.models.TaskType] = { + js match { + case v: play.api.libs.json.JsString => play.api.libs.json.JsSuccess(io.apibuilder.task.v0.models.TaskType(v.value)) + case _ => { + (js \ "value").validate[String] match { + case play.api.libs.json.JsSuccess(v, _) => play.api.libs.json.JsSuccess(io.apibuilder.task.v0.models.TaskType(v)) + case err: play.api.libs.json.JsError => + (js \ "task_type").validate[String] match { + case play.api.libs.json.JsSuccess(v, _) => play.api.libs.json.JsSuccess(io.apibuilder.task.v0.models.TaskType(v)) + case err: play.api.libs.json.JsError => err + } + } + } + } + } + } + + def jsonWritesApibuilderTaskTaskType(obj: io.apibuilder.task.v0.models.TaskType) = { + play.api.libs.json.JsString(obj.toString) + } + + def jsObjectTaskType(obj: io.apibuilder.task.v0.models.TaskType) = { + play.api.libs.json.Json.obj("value" -> play.api.libs.json.JsString(obj.toString)) + } + + implicit def jsonWritesApibuilderTaskTaskType: play.api.libs.json.Writes[TaskType] = { + (obj: io.apibuilder.task.v0.models.TaskType) => { + jsonWritesApibuilderTaskTaskType(obj) + } + } + + implicit def jsonReadsApibuilderTaskDiffVersionData: play.api.libs.json.Reads[DiffVersionData] = { + for { + oldVersionGuid <- (__ \ "old_version_guid").read[_root_.java.util.UUID] + newVersionGuid <- (__ \ "new_version_guid").read[_root_.java.util.UUID] + } yield DiffVersionData(oldVersionGuid, newVersionGuid) + } + + def jsObjectDiffVersionData(obj: io.apibuilder.task.v0.models.DiffVersionData): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "old_version_guid" -> play.api.libs.json.JsString(obj.oldVersionGuid.toString), + "new_version_guid" -> play.api.libs.json.JsString(obj.newVersionGuid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskDiffVersionData: play.api.libs.json.Writes[DiffVersionData] = { + (obj: io.apibuilder.task.v0.models.DiffVersionData) => { + jsObjectDiffVersionData(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataApplicationCreated: play.api.libs.json.Reads[EmailDataApplicationCreated] = { + (__ \ "application_guid").read[_root_.java.util.UUID].map { x => new EmailDataApplicationCreated(applicationGuid = x) } + } + + def jsObjectEmailDataApplicationCreated(obj: io.apibuilder.task.v0.models.EmailDataApplicationCreated): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "application_guid" -> play.api.libs.json.JsString(obj.applicationGuid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataApplicationCreated: play.api.libs.json.Writes[EmailDataApplicationCreated] = { + (obj: io.apibuilder.task.v0.models.EmailDataApplicationCreated) => { + jsObjectEmailDataApplicationCreated(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataEmailVerificationCreated: play.api.libs.json.Reads[EmailDataEmailVerificationCreated] = { + (__ \ "guid").read[_root_.java.util.UUID].map { x => new EmailDataEmailVerificationCreated(guid = x) } + } + + def jsObjectEmailDataEmailVerificationCreated(obj: io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "guid" -> play.api.libs.json.JsString(obj.guid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataEmailVerificationCreated: play.api.libs.json.Writes[EmailDataEmailVerificationCreated] = { + (obj: io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated) => { + jsObjectEmailDataEmailVerificationCreated(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataMembershipCreated: play.api.libs.json.Reads[EmailDataMembershipCreated] = { + (__ \ "guid").read[_root_.java.util.UUID].map { x => new EmailDataMembershipCreated(guid = x) } + } + + def jsObjectEmailDataMembershipCreated(obj: io.apibuilder.task.v0.models.EmailDataMembershipCreated): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "guid" -> play.api.libs.json.JsString(obj.guid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataMembershipCreated: play.api.libs.json.Writes[EmailDataMembershipCreated] = { + (obj: io.apibuilder.task.v0.models.EmailDataMembershipCreated) => { + jsObjectEmailDataMembershipCreated(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataMembershipRequestAccepted: play.api.libs.json.Reads[EmailDataMembershipRequestAccepted] = { + for { + organizationGuid <- (__ \ "organization_guid").read[_root_.java.util.UUID] + userGuid <- (__ \ "user_guid").read[_root_.java.util.UUID] + role <- (__ \ "role").read[String] + } yield EmailDataMembershipRequestAccepted(organizationGuid, userGuid, role) + } + + def jsObjectEmailDataMembershipRequestAccepted(obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestAccepted): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "organization_guid" -> play.api.libs.json.JsString(obj.organizationGuid.toString), + "user_guid" -> play.api.libs.json.JsString(obj.userGuid.toString), + "role" -> play.api.libs.json.JsString(obj.role) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataMembershipRequestAccepted: play.api.libs.json.Writes[EmailDataMembershipRequestAccepted] = { + (obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestAccepted) => { + jsObjectEmailDataMembershipRequestAccepted(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataMembershipRequestCreated: play.api.libs.json.Reads[EmailDataMembershipRequestCreated] = { + (__ \ "guid").read[_root_.java.util.UUID].map { x => new EmailDataMembershipRequestCreated(guid = x) } + } + + def jsObjectEmailDataMembershipRequestCreated(obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestCreated): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "guid" -> play.api.libs.json.JsString(obj.guid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataMembershipRequestCreated: play.api.libs.json.Writes[EmailDataMembershipRequestCreated] = { + (obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestCreated) => { + jsObjectEmailDataMembershipRequestCreated(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataMembershipRequestDeclined: play.api.libs.json.Reads[EmailDataMembershipRequestDeclined] = { + for { + organizationGuid <- (__ \ "organization_guid").read[_root_.java.util.UUID] + userGuid <- (__ \ "user_guid").read[_root_.java.util.UUID] + role <- (__ \ "role").read[String] + } yield EmailDataMembershipRequestDeclined(organizationGuid, userGuid, role) + } + + def jsObjectEmailDataMembershipRequestDeclined(obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestDeclined): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "organization_guid" -> play.api.libs.json.JsString(obj.organizationGuid.toString), + "user_guid" -> play.api.libs.json.JsString(obj.userGuid.toString), + "role" -> play.api.libs.json.JsString(obj.role) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataMembershipRequestDeclined: play.api.libs.json.Writes[EmailDataMembershipRequestDeclined] = { + (obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestDeclined) => { + jsObjectEmailDataMembershipRequestDeclined(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataPasswordResetRequestCreated: play.api.libs.json.Reads[EmailDataPasswordResetRequestCreated] = { + (__ \ "guid").read[_root_.java.util.UUID].map { x => new EmailDataPasswordResetRequestCreated(guid = x) } + } + + def jsObjectEmailDataPasswordResetRequestCreated(obj: io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "guid" -> play.api.libs.json.JsString(obj.guid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataPasswordResetRequestCreated: play.api.libs.json.Writes[EmailDataPasswordResetRequestCreated] = { + (obj: io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated) => { + jsObjectEmailDataPasswordResetRequestCreated(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailData: play.api.libs.json.Reads[EmailData] = { + ( + (__ \ "email_data_application_created").read(jsonReadsApibuilderTaskEmailDataApplicationCreated).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_email_verification_created").read(jsonReadsApibuilderTaskEmailDataEmailVerificationCreated).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_membership_created").read(jsonReadsApibuilderTaskEmailDataMembershipCreated).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_membership_request_created").read(jsonReadsApibuilderTaskEmailDataMembershipRequestCreated).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_membership_request_accepted").read(jsonReadsApibuilderTaskEmailDataMembershipRequestAccepted).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_membership_request_declined").read(jsonReadsApibuilderTaskEmailDataMembershipRequestDeclined).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_password_reset_request_created").read(jsonReadsApibuilderTaskEmailDataPasswordResetRequestCreated).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + play.api.libs.json.Reads(jsValue => play.api.libs.json.JsSuccess(EmailDataUndefinedType(jsValue.toString))).asInstanceOf[play.api.libs.json.Reads[EmailData]] + ) + } + + def jsObjectEmailData(obj: io.apibuilder.task.v0.models.EmailData): play.api.libs.json.JsObject = { + obj match { + case x: io.apibuilder.task.v0.models.EmailDataApplicationCreated => play.api.libs.json.Json.obj("email_data_application_created" -> jsObjectEmailDataApplicationCreated(x)) + case x: io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated => play.api.libs.json.Json.obj("email_data_email_verification_created" -> jsObjectEmailDataEmailVerificationCreated(x)) + case x: io.apibuilder.task.v0.models.EmailDataMembershipCreated => play.api.libs.json.Json.obj("email_data_membership_created" -> jsObjectEmailDataMembershipCreated(x)) + case x: io.apibuilder.task.v0.models.EmailDataMembershipRequestCreated => play.api.libs.json.Json.obj("email_data_membership_request_created" -> jsObjectEmailDataMembershipRequestCreated(x)) + case x: io.apibuilder.task.v0.models.EmailDataMembershipRequestAccepted => play.api.libs.json.Json.obj("email_data_membership_request_accepted" -> jsObjectEmailDataMembershipRequestAccepted(x)) + case x: io.apibuilder.task.v0.models.EmailDataMembershipRequestDeclined => play.api.libs.json.Json.obj("email_data_membership_request_declined" -> jsObjectEmailDataMembershipRequestDeclined(x)) + case x: io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated => play.api.libs.json.Json.obj("email_data_password_reset_request_created" -> jsObjectEmailDataPasswordResetRequestCreated(x)) + case x: io.apibuilder.task.v0.models.EmailDataUndefinedType => sys.error(s"The type[io.apibuilder.task.v0.models.EmailDataUndefinedType] should never be serialized") + } + } + + implicit def jsonWritesApibuilderTaskEmailData: play.api.libs.json.Writes[EmailData] = { + (obj: io.apibuilder.task.v0.models.EmailData) => { + jsObjectEmailData(obj) + } + } + } +} + +package io.apibuilder.task.v0 { + + object Bindables { + + import play.api.mvc.{PathBindable, QueryStringBindable} + + // import models directly for backwards compatibility with prior versions of the generator + import Core._ + import Models._ + + object Core { + implicit def pathBindableDateTimeIso8601(implicit stringBinder: QueryStringBindable[String]): PathBindable[_root_.org.joda.time.DateTime] = ApibuilderPathBindable(ApibuilderTypes.dateTimeIso8601) + implicit def queryStringBindableDateTimeIso8601(implicit stringBinder: QueryStringBindable[String]): QueryStringBindable[_root_.org.joda.time.DateTime] = ApibuilderQueryStringBindable(ApibuilderTypes.dateTimeIso8601) + + implicit def pathBindableDateIso8601(implicit stringBinder: QueryStringBindable[String]): PathBindable[_root_.org.joda.time.LocalDate] = ApibuilderPathBindable(ApibuilderTypes.dateIso8601) + implicit def queryStringBindableDateIso8601(implicit stringBinder: QueryStringBindable[String]): QueryStringBindable[_root_.org.joda.time.LocalDate] = ApibuilderQueryStringBindable(ApibuilderTypes.dateIso8601) + } + + object Models { + import io.apibuilder.task.v0.models._ + + val taskTypeConverter: ApibuilderTypeConverter[io.apibuilder.task.v0.models.TaskType] = new ApibuilderTypeConverter[io.apibuilder.task.v0.models.TaskType] { + override def convert(value: String): io.apibuilder.task.v0.models.TaskType = io.apibuilder.task.v0.models.TaskType(value) + override def convert(value: io.apibuilder.task.v0.models.TaskType): String = value.toString + override def example: io.apibuilder.task.v0.models.TaskType = io.apibuilder.task.v0.models.TaskType.Email + override def validValues: Seq[io.apibuilder.task.v0.models.TaskType] = io.apibuilder.task.v0.models.TaskType.all + } + implicit def pathBindableTaskType(implicit stringBinder: QueryStringBindable[String]): PathBindable[io.apibuilder.task.v0.models.TaskType] = ApibuilderPathBindable(taskTypeConverter) + implicit def queryStringBindableTaskType(implicit stringBinder: QueryStringBindable[String]): QueryStringBindable[io.apibuilder.task.v0.models.TaskType] = ApibuilderQueryStringBindable(taskTypeConverter) + } + + trait ApibuilderTypeConverter[T] { + + def convert(value: String): T + + def convert(value: T): String + + def example: T + + def validValues: Seq[T] = Nil + + def errorMessage(key: String, value: String, ex: java.lang.Exception): String = { + val base = s"Invalid value '$value' for parameter '$key'. " + validValues.toList match { + case Nil => base + "Ex: " + convert(example) + case values => base + ". Valid values are: " + values.mkString("'", "', '", "'") + } + } + } + + object ApibuilderTypes { + val dateTimeIso8601: ApibuilderTypeConverter[_root_.org.joda.time.DateTime] = new ApibuilderTypeConverter[_root_.org.joda.time.DateTime] { + override def convert(value: String): _root_.org.joda.time.DateTime = _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseDateTime(value) + override def convert(value: _root_.org.joda.time.DateTime): String = _root_.org.joda.time.format.ISODateTimeFormat.dateTime.print(value) + override def example: _root_.org.joda.time.DateTime = _root_.org.joda.time.DateTime.now + } + + val dateIso8601: ApibuilderTypeConverter[_root_.org.joda.time.LocalDate] = new ApibuilderTypeConverter[_root_.org.joda.time.LocalDate] { + override def convert(value: String): _root_.org.joda.time.LocalDate = _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseLocalDate(value) + override def convert(value: _root_.org.joda.time.LocalDate): String = _root_.org.joda.time.format.ISODateTimeFormat.date.print(value) + override def example: _root_.org.joda.time.LocalDate = _root_.org.joda.time.LocalDate.now + } + } + + final case class ApibuilderQueryStringBindable[T]( + converters: ApibuilderTypeConverter[T] + ) extends QueryStringBindable[T] { + + override def bind(key: String, params: Map[String, Seq[String]]): _root_.scala.Option[_root_.scala.Either[String, T]] = { + params.getOrElse(key, Nil).headOption.map { v => + try { + Right( + converters.convert(v) + ) + } catch { + case ex: java.lang.Exception => Left( + converters.errorMessage(key, v, ex) + ) + } + } + } + + override def unbind(key: String, value: T): String = { + s"$key=${converters.convert(value)}" + } + } + + final case class ApibuilderPathBindable[T]( + converters: ApibuilderTypeConverter[T] + ) extends PathBindable[T] { + + override def bind(key: String, value: String): _root_.scala.Either[String, T] = { + try { + Right( + converters.convert(value) + ) + } catch { + case ex: java.lang.Exception => Left( + converters.errorMessage(key, value, ex) + ) + } + } + + override def unbind(key: String, value: T): String = { + converters.convert(value) + } + } + + } + +} + + +package io.apibuilder.task.v0 { + + object Constants { + + val Namespace = "io.apibuilder.task.v0" + val UserAgent = "apibuilder app.apibuilder.io/apicollective/apibuilder-task/latest/play_2_8_client" + val Version = "0.16.24" + val VersionMajor = 0 + + } + + class Client( + ws: play.api.libs.ws.WSClient, + val baseUrl: String, + auth: scala.Option[io.apibuilder.task.v0.Authorization] = None, + defaultHeaders: Seq[(String, String)] = Nil + ) extends interfaces.Client { + import io.apibuilder.task.v0.models.json._ + + private[this] val logger = play.api.Logger("io.apibuilder.task.v0.Client") + + logger.info(s"Initializing io.apibuilder.task.v0.Client for url $baseUrl") + + + + + + def _requestHolder(path: String): play.api.libs.ws.WSRequest = { + + val holder = ws.url(baseUrl + path).addHttpHeaders( + "User-Agent" -> Constants.UserAgent, + "X-Apidoc-Version" -> Constants.Version, + "X-Apidoc-Version-Major" -> Constants.VersionMajor.toString + ).addHttpHeaders(defaultHeaders : _*) + auth.fold(holder) { + case Authorization.Basic(username, password) => { + holder.withAuth(username, password.getOrElse(""), play.api.libs.ws.WSAuthScheme.BASIC) + } + case a => sys.error("Invalid authorization scheme[" + a.getClass + "]") + } + } + + def _logRequest(method: String, req: play.api.libs.ws.WSRequest): play.api.libs.ws.WSRequest = { + val queryComponents = for { + (name, values) <- req.queryString + value <- values + } yield s"$name=$value" + val url = s"${req.url}${queryComponents.mkString("?", "&", "")}" + auth.fold(logger.info(s"curl -X $method '$url'")) { _ => + logger.info(s"curl -X $method -u '[REDACTED]:' '$url'") + } + req + } + + def _executeRequest( + method: String, + path: String, + queryParameters: Seq[(String, String)] = Nil, + requestHeaders: Seq[(String, String)] = Nil, + body: Option[play.api.libs.json.JsValue] = None + ): scala.concurrent.Future[play.api.libs.ws.WSResponse] = { + method.toUpperCase match { + case "GET" => { + _logRequest("GET", _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)).get() + } + case "POST" => { + _logRequest("POST", _requestHolder(path).addHttpHeaders(_withJsonContentType(requestHeaders):_*).addQueryStringParameters(queryParameters:_*)).post(body.getOrElse(play.api.libs.json.Json.obj())) + } + case "PUT" => { + _logRequest("PUT", _requestHolder(path).addHttpHeaders(_withJsonContentType(requestHeaders):_*).addQueryStringParameters(queryParameters:_*)).put(body.getOrElse(play.api.libs.json.Json.obj())) + } + case "PATCH" => { + _logRequest("PATCH", _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)).patch(body.getOrElse(play.api.libs.json.Json.obj())) + } + case "DELETE" => { + _logRequest("DELETE", _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)).delete() + } + case "HEAD" => { + _logRequest("HEAD", _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)).head() + } + case "OPTIONS" => { + _logRequest("OPTIONS", _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)).options() + } + case _ => { + _logRequest(method, _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)) + sys.error("Unsupported method[%s]".format(method)) + } + } + } + + /** + * Adds a Content-Type: application/json header unless the specified requestHeaders + * already contain a Content-Type header + */ + def _withJsonContentType(headers: Seq[(String, String)]): Seq[(String, String)] = { + headers.find { _._1.toUpperCase == "CONTENT-TYPE" } match { + case None => headers ++ Seq("Content-Type" -> "application/json; charset=UTF-8") + case Some(_) => headers + } + } + + } + + object Client { + + def parseJson[T]( + className: String, + r: play.api.libs.ws.WSResponse, + f: (play.api.libs.json.JsValue => play.api.libs.json.JsResult[T]) + ): T = { + f(play.api.libs.json.Json.parse(r.body)) match { + case play.api.libs.json.JsSuccess(x, _) => x + case play.api.libs.json.JsError(errors) => { + throw io.apibuilder.task.v0.errors.FailedRequest(r.status, s"Invalid json for class[" + className + "]: " + errors.mkString(" ")) + } + } + } + + } + + sealed trait Authorization extends _root_.scala.Product with _root_.scala.Serializable + object Authorization { + final case class Basic(username: String, password: Option[String] = None) extends Authorization + } + + package interfaces { + + trait Client { + def baseUrl: String + + } + + } + + + + package errors { + + final case class FailedRequest(responseCode: Int, message: String, requestUri: Option[_root_.java.net.URI] = None) extends _root_.java.lang.Exception(s"HTTP $responseCode: $message") + + } + +} \ No newline at end of file diff --git a/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala b/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala new file mode 100644 index 000000000..3d2a23648 --- /dev/null +++ b/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala @@ -0,0 +1,57 @@ +/** + * Generated by API Builder - https://www.apibuilder.io + * Service version: 0.16.24 + * apibuilder app.apibuilder.io/apicollective/apibuilder-task/latest/play_2_8_mock_client + */ +package io.apibuilder.task.v0.mock { + + object Factories { + + def randomString(length: Int = 24): String = { + _root_.scala.util.Random.alphanumeric.take(length).mkString + } + + def makeTaskType(): io.apibuilder.task.v0.models.TaskType = io.apibuilder.task.v0.models.TaskType.Email + + def makeDiffVersionData(): io.apibuilder.task.v0.models.DiffVersionData = io.apibuilder.task.v0.models.DiffVersionData( + oldVersionGuid = _root_.java.util.UUID.randomUUID, + newVersionGuid = _root_.java.util.UUID.randomUUID + ) + + def makeEmailDataApplicationCreated(): io.apibuilder.task.v0.models.EmailDataApplicationCreated = io.apibuilder.task.v0.models.EmailDataApplicationCreated( + applicationGuid = _root_.java.util.UUID.randomUUID + ) + + def makeEmailDataEmailVerificationCreated(): io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated = io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated( + guid = _root_.java.util.UUID.randomUUID + ) + + def makeEmailDataMembershipCreated(): io.apibuilder.task.v0.models.EmailDataMembershipCreated = io.apibuilder.task.v0.models.EmailDataMembershipCreated( + guid = _root_.java.util.UUID.randomUUID + ) + + def makeEmailDataMembershipRequestAccepted(): io.apibuilder.task.v0.models.EmailDataMembershipRequestAccepted = io.apibuilder.task.v0.models.EmailDataMembershipRequestAccepted( + organizationGuid = _root_.java.util.UUID.randomUUID, + userGuid = _root_.java.util.UUID.randomUUID, + role = Factories.randomString(24) + ) + + def makeEmailDataMembershipRequestCreated(): io.apibuilder.task.v0.models.EmailDataMembershipRequestCreated = io.apibuilder.task.v0.models.EmailDataMembershipRequestCreated( + guid = _root_.java.util.UUID.randomUUID + ) + + def makeEmailDataMembershipRequestDeclined(): io.apibuilder.task.v0.models.EmailDataMembershipRequestDeclined = io.apibuilder.task.v0.models.EmailDataMembershipRequestDeclined( + organizationGuid = _root_.java.util.UUID.randomUUID, + userGuid = _root_.java.util.UUID.randomUUID, + role = Factories.randomString(24) + ) + + def makeEmailDataPasswordResetRequestCreated(): io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated = io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated( + guid = _root_.java.util.UUID.randomUUID + ) + + def makeEmailData(): io.apibuilder.task.v0.models.EmailData = io.apibuilder.task.v0.mock.Factories.makeEmailDataApplicationCreated() + + } + +} \ No newline at end of file diff --git a/spec/apibuilder-internal.json b/spec/apibuilder-internal.json deleted file mode 100644 index dca2feda7..000000000 --- a/spec/apibuilder-internal.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "name": "apibuilder internal", - "description": "Internal models used in the implementation of apibuilder", - - "info": { - "contact": { - "name": "Michael Bryzek", - "email": "mbryzek@alum.mit.edu", - "url": "http://twitter.com/mbryzek" - }, - "license": { - "name": "MIT", - "url": "http://opensource.org/licenses/MIT" - } - }, - - "unions": { - "task_data": { - "types": [ - { "type": "task_data_index_application", "description": "Task to update the search index following a change in application" }, - { "type": "task_data_diff_version", "description": "Task triggered whenever a version changes. Diffs the service to record what actually changed" } - ] - } - }, - - "models": { - - "task": { - "fields": [ - { "name": "guid", "type": "uuid" }, - { "name": "data", "type": "task_data" }, - { "name": "number_attempts", "type": "long", "default": 0, "description": "Records the number of times we have attempted to run this task. Commonly we increment number attempts, process the task, and if succeeds we then delete the task. If it fails, we update last_error. This allows us to retry a task say twice; after which we no longer process the task (can notify an admin of the error)." }, - { "name": "last_error", "type": "string", "required": false } - ] - }, - - "task_data_index_application": { - "fields": [ - { "name": "application_guid", "type": "uuid" } - ] - }, - - "task_data_diff_version": { - "fields": [ - { "name": "old_version_guid", "type": "uuid" }, - { "name": "new_version_guid", "type": "uuid" } - ] - } - } - -} diff --git a/spec/apibuilder-task.json b/spec/apibuilder-task.json new file mode 100644 index 000000000..9f7c0f5c6 --- /dev/null +++ b/spec/apibuilder-task.json @@ -0,0 +1,100 @@ +{ + "name": "apibuilder task", + + "info": { + "contact": { + "name": "Michael Bryzek", + "email": "mbryzek@alum.mit.edu", + "url": "http://twitter.com/mbryzek" + }, + "license": { + "name": "MIT", + "url": "http://opensource.org/licenses/MIT" + } + }, + + "enums": { + "task_type": { + "values": [ + { "name": "email" }, + { "name": "index_application" }, + { "name": "cleanup_deletions" }, + { "name": "schedule_migrate_versions" }, + { "name": "migrate_version" }, + { "name": "schedule_sync_generator_services" }, + { "name": "sync_generator_service" }, + { "name": "diff_version" }, + { "name": "user_created" } + ] + } + }, + + "unions": { + "email_data": { + "types": [ + { "type": "email_data_application_created" }, + { "type": "email_data_email_verification_created" }, + { "type": "email_data_membership_created" }, + { "type": "email_data_membership_request_created" }, + { "type": "email_data_membership_request_accepted" }, + { "type": "email_data_membership_request_declined" }, + { "type": "email_data_password_reset_request_created" } + ] + } + }, + + "models": { + "diff_version_data": { + "fields": [ + { "name": "old_version_guid", "type": "uuid" }, + { "name": "new_version_guid", "type": "uuid" } + ] + }, + + "email_data_application_created": { + "fields": [ + { "name": "application_guid", "type": "uuid" } + ] + }, + + "email_data_email_verification_created": { + "fields": [ + { "name": "guid", "type": "uuid" } + ] + }, + + "email_data_membership_created": { + "fields": [ + { "name": "guid", "type": "uuid" } + ] + }, + + "email_data_membership_request_created": { + "fields": [ + { "name": "guid", "type": "uuid" } + ] + }, + + "email_data_membership_request_accepted": { + "fields": [ + { "name": "organization_guid", "type": "uuid" }, + { "name": "user_guid", "type": "uuid" }, + { "name": "role", "type": "string" } + ] + }, + + "email_data_membership_request_declined": { + "fields": [ + { "name": "organization_guid", "type": "uuid" }, + { "name": "user_guid", "type": "uuid" }, + { "name": "role", "type": "string" } + ] + }, + + "email_data_password_reset_request_created": { + "fields": [ + { "name": "guid", "type": "uuid" } + ] + } + } +}