From 3c2bf59b38f5752894b12439f9ae98d298d0a74b Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:08:16 -0700 Subject: [PATCH 01/32] wip --- api/app/actors/TaskActor.scala | 231 ------- api/app/db/TasksDao.scala | 189 ------ .../db/generated/PsqlApibuilderTasksDao.scala | 616 ++++++++++++++++++ dao/spec/psql-apibuilder.json | 37 ++ 4 files changed, 653 insertions(+), 420 deletions(-) delete mode 100644 api/app/actors/TaskActor.scala delete mode 100644 api/app/db/TasksDao.scala create mode 100644 api/app/db/generated/PsqlApibuilderTasksDao.scala diff --git a/api/app/actors/TaskActor.scala b/api/app/actors/TaskActor.scala deleted file mode 100644 index 5f504473f..000000000 --- a/api/app/actors/TaskActor.scala +++ /dev/null @@ -1,231 +0,0 @@ -package actors - -import akka.actor.{Actor, ActorLogging, ActorSystem} -import io.apibuilder.api.v0.models.{Application, Diff, DiffBreaking, DiffNonBreaking, DiffUndefinedType, Organization, Publication, Version} -import io.apibuilder.internal.v0.models.{Task, TaskDataDiffVersion, TaskDataIndexApplication, TaskDataUndefinedType} -import db.{ApplicationsDao, Authorization, ChangesDao, OrganizationsDao, TasksDao, UsersDao, VersionsDao, WatchesDao} -import lib.{AppConfig, ServiceDiff, Text} - -import java.util.UUID -import org.joda.time.DateTime -import play.twirl.api.Html - -import scala.concurrent.ExecutionContext -import scala.concurrent.duration._ -import scala.util.{Failure, Success, Try} - -object TaskActor { - - object Messages { - case class Created(guid: UUID) - case object RestartDroppedTasks - case object PurgeOldTasks - case object NotifyFailed - } - -} - -@javax.inject.Singleton -class TaskActor @javax.inject.Inject() ( - system: ActorSystem, - appConfig: AppConfig, - applicationsDao: ApplicationsDao, - changesDao: ChangesDao, - emails: Emails, - organizationsDao: OrganizationsDao, - search: Search, - tasksDao: TasksDao, - usersDao: UsersDao, - versionsDao: VersionsDao, - watchesDao: WatchesDao -) extends Actor with ActorLogging with ErrorHandler { - - private[this] implicit val ec: ExecutionContext = system.dispatchers.lookup("task-actor-context") - - private[this] val NumberDaysBeforePurge = 30 - private[this] case class Process(guid: UUID) - - system.scheduler.scheduleAtFixedRate(1.hour, 1.hour, self, TaskActor.Messages.RestartDroppedTasks) - system.scheduler.scheduleAtFixedRate(1.day, 1.day, self, TaskActor.Messages.NotifyFailed) - system.scheduler.scheduleAtFixedRate(1.day, 1.day, self, TaskActor.Messages.PurgeOldTasks) - - def receive: Receive = { - - case m @ TaskActor.Messages.Created(guid) => withVerboseErrorHandler(m) { - self ! Process(guid) - } - - case m @ Process(guid) => withVerboseErrorHandler(m) { - tasksDao.findByGuid(guid).foreach { task => - tasksDao.incrementNumberAttempts(usersDao.AdminUser, task) - - task.data match { - case TaskDataDiffVersion(oldVersionGuid, newVersionGuid) => { - processTask(task, Try(diffVersion(oldVersionGuid, newVersionGuid))) - } - - case TaskDataIndexApplication(applicationGuid) => { - processTask(task, Try(search.indexApplication(applicationGuid))) - } - - case TaskDataUndefinedType(desc) => { - tasksDao.recordError(usersDao.AdminUser, task, "Task actor got an undefined data type: " + desc) - } - } - } - } - - case m @ TaskActor.Messages.RestartDroppedTasks => withVerboseErrorHandler(m) { - tasksDao.findAll( - nOrFewerAttempts = Some(2), - createdOnOrBefore = Some(DateTime.now.minusMinutes(1)) - ).foreach { task => - self ! Process(task.guid) - } - } - - case m @ TaskActor.Messages.NotifyFailed => withVerboseErrorHandler(m) { - val errors = tasksDao.findAll( - nOrMoreAttempts = Some(2), - isDeleted = Some(false), - createdOnOrAfter = Some(DateTime.now.minusDays(3)) - ).map { task => - val errorType = task.data match { - case TaskDataDiffVersion(a, b) => s"TaskDataDiffVersion($a, $b)" - case TaskDataIndexApplication(guid) => s"TaskDataIndexApplication($guid)" - case TaskDataUndefinedType(desc) => s"TaskDataUndefinedType($desc)" - } - - val errorMsg = Text.truncate(task.lastError.getOrElse("No information on error"), 500) - s"$errorType task ${task.guid}: $errorMsg" - } - emails.sendErrors( - subject = "One or more tasks failed", - errors = errors - ) - } - - case m @ TaskActor.Messages.PurgeOldTasks => withVerboseErrorHandler(m) { - tasksDao.findAll( - isDeleted = Some(true), - deletedAtLeastNDaysAgo = Some(NumberDaysBeforePurge) - ).foreach { task => - tasksDao.purge(task) - } - } - - case m: Any => logUnhandledMessage(m) - } - - private[this] def diffVersion(oldVersionGuid: UUID, newVersionGuid: UUID): Unit = { - versionsDao.findByGuid(Authorization.All, oldVersionGuid, isDeleted = None).foreach { oldVersion => - versionsDao.findByGuid(Authorization.All, newVersionGuid).foreach { newVersion => - ServiceDiff(oldVersion.service, newVersion.service).differences match { - case Nil => { - // No-op - } - case diffs => { - changesDao.upsert( - createdBy = usersDao.AdminUser, - fromVersion = oldVersion, - toVersion = newVersion, - differences = diffs - ) - versionUpdated(newVersion, diffs) - versionUpdatedMaterialOnly(newVersion, diffs) - } - } - } - } - } - - private[this] def versionUpdated( - version: Version, - diffs: Seq[Diff], - ): Unit = { - if (diffs.nonEmpty) { - sendVersionUpsertedEmail( - publication = Publication.VersionsCreate, - version = version, - diffs = diffs, - ) { (org, application, breakingDiffs, nonBreakingDiffs) => - views.html.emails.versionUpserted( - appConfig, - org, - application, - version, - breakingDiffs = breakingDiffs, - nonBreakingDiffs = nonBreakingDiffs - ) - } - } - } - - private[this] def versionUpdatedMaterialOnly( - version: Version, - diffs: Seq[Diff], - ): Unit = { - val filtered = diffs.filter(_.isMaterial) - if (filtered.nonEmpty) { - sendVersionUpsertedEmail( - publication = Publication.VersionsMaterialChange, - version = version, - diffs = filtered, - ) { (org, application, breakingDiffs, nonBreakingDiffs) => - views.html.emails.versionUpserted( - appConfig, - org, - application, - version, - breakingDiffs = breakingDiffs, - nonBreakingDiffs = nonBreakingDiffs - ) - } - } - } - - private[this] def sendVersionUpsertedEmail( - publication: Publication, - version: Version, - diffs: Seq[Diff], - )( - generateBody: (Organization, Application, Seq[Diff], Seq[Diff]) => Html, - ): Unit = { - val (breakingDiffs, nonBreakingDiffs) = diffs.partition { - case _: DiffBreaking => true - case _: DiffNonBreaking => false - case _: DiffUndefinedType => true - } - - applicationsDao.findAll(Authorization.All, version = Some(version), limit = 1).foreach { application => - organizationsDao.findAll(Authorization.All, application = Some(application), limit = 1).foreach { org => - emails.deliver( - context = Emails.Context.Application(application), - org = org, - publication = publication, - subject = s"${org.name}/${application.name}:${version.version} Updated", - body = generateBody(org, application, breakingDiffs, nonBreakingDiffs).toString - ) { subscription => - watchesDao.findAll( - Authorization.All, - application = Some(application), - userGuid = Some(subscription.user.guid), - limit = 1 - ).nonEmpty - } - } - } - } - - def processTask[T](task: Task, attempt: Try[T]): Unit = { - attempt match { - case Success(_) => { - tasksDao.softDelete(usersDao.AdminUser, task) - } - case Failure(ex) => { - tasksDao.recordError(usersDao.AdminUser, task, ex) - } - } - } - -} diff --git a/api/app/db/TasksDao.scala b/api/app/db/TasksDao.scala deleted file mode 100644 index 5557f12e5..000000000 --- a/api/app/db/TasksDao.scala +++ /dev/null @@ -1,189 +0,0 @@ -package db - -import anorm._ -import io.apibuilder.internal.v0.models.{Task, TaskData} -import io.apibuilder.internal.v0.models.json._ -import io.apibuilder.api.v0.models.User -import io.flow.postgresql.Query -import javax.inject.{Inject, Named, Singleton} -import org.joda.time.DateTime -import play.api.db._ -import play.api.libs.json._ -import java.util.UUID - -/** - * A task represents a bit of work that should be completed asynchronously. Example tasks include: - * - * - generating a diff between two versions of a service - * - updates a search index for a particular service - * - * Tasks provide for the following semantics: - * - transactionally record to execute a task - * - at least once execution - * - near real time execution - * - mostly sequential execution of tasks (but not guaranteed) - * - * The implementation works by: - * - caller inserts a task transactionally - * - to trigger real time execution, send a TaskCreated message to - * the Main Actor - send this AFTER the transaction commits or the - * actor may not see the task - * - periodically, the task system picks up tasks that have not - * been processed and executes them - */ -@Singleton -class TasksDao @Inject() ( - @NamedDatabase("default") db: Database, - @Named("main-actor") mainActor: akka.actor.ActorRef -) { - - private[this] val dbHelpers = DbHelpers(db, "tasks") - - private[this] val BaseQuery = Query(""" - select tasks.guid, - tasks.data::text, - tasks.number_attempts, - tasks.last_error - from tasks - """) - - val IncrementNumberAttemptsQuery = """ - update tasks - set number_attempts = number_attempts + 1, - updated_by_guid = {updated_by_guid}::uuid - where guid = {guid}::uuid - """ - - val RecordErrorQuery = """ - update tasks - set last_error = {last_error}, - updated_by_guid = {updated_by_guid}::uuid - where guid = {guid}::uuid - """ - - private[this] val InsertQuery = """ - insert into tasks - (guid, data, created_by_guid, updated_by_guid) - values - ({guid}::uuid, {data}::json, {created_by_guid}::uuid, {updated_by_guid}::uuid) - """ - - private[this] val PurgeQuery = """ - delete from tasks where guid = {guid}::uuid - """ - - def create(createdBy: User, data: TaskData): UUID = { - val taskGuid = db.withConnection { implicit c => - insert(c, createdBy, data) - } - mainActor ! actors.MainActor.Messages.TaskCreated(taskGuid) - taskGuid - } - - private[db] def insert(implicit c: java.sql.Connection, createdBy: User, data: TaskData): UUID = { - val guid = UUID.randomUUID - - SQL(InsertQuery).on( - "guid" -> guid, - "data" -> Json.toJson(data).toString, - "created_by_guid" -> createdBy.guid, - "updated_by_guid" -> createdBy.guid - ).execute() - - guid - } - - def softDelete(deletedBy: User, task: Task): Unit = { - dbHelpers.delete(deletedBy, task.guid) - } - - def purge(task: Task): Unit = { - db.withConnection { implicit c => - SQL(PurgeQuery).on("guid" -> task.guid).execute() - } - } - - def incrementNumberAttempts(user: User, task: Task): Unit = { - db.withConnection { implicit c => - SQL(IncrementNumberAttemptsQuery).on( - "guid" -> task.guid, - "updated_by_guid" -> user.guid - ).execute() - } - } - - def recordError(user: User, task: Task, error: Throwable): Unit = { - val sw = new java.io.StringWriter - error.printStackTrace(new java.io.PrintWriter(sw)) - recordError(user, task, sw.toString) - } - - def recordError(user: User, task: Task, error: String): Unit = { - db.withConnection { implicit c => - SQL(RecordErrorQuery).on( - "guid" -> task.guid, - "last_error" -> error, - "updated_by_guid" -> user.guid - ).execute() - } - } - - def findByGuid(guid: UUID): Option[Task] = { - findAll(guid = Some(guid), limit = 1).headOption - } - - def findAll( - guid: Option[UUID] = None, - nOrFewerAttempts: Option[Long] = None, - nOrMoreAttempts: Option[Long] = None, - createdOnOrBefore: Option[DateTime] = None, - createdOnOrAfter: Option[DateTime] = None, - isDeleted: Option[Boolean] = Some(false), - deletedAtLeastNDaysAgo: Option[Long] = None, - limit: Long = 25, - offset: Long = 0 - ): Seq[Task] = { - deletedAtLeastNDaysAgo.foreach { _ => - isDeleted match { - case None | Some(true) => // no-op - case Some(false) => sys.error("When filtering by deletedAtLeastNDaysAgo, you must also set isDeleted") - } - } - - db.withConnection { implicit c => - BaseQuery. - equals("tasks.guid", guid). - lessThanOrEquals("tasks.number_attempts", nOrFewerAttempts). - greaterThanOrEquals("tasks.number_attempts", nOrMoreAttempts). - lessThanOrEquals("tasks.created_at", createdOnOrBefore). - greaterThanOrEquals("tasks.created_at", createdOnOrAfter). - and(isDeleted.map { - Filters.isDeleted("tasks", _) - }). - lessThanOrEquals("tasks.deleted_at", deletedAtLeastNDaysAgo.map(days => DateTime.now.minusDays(days.toInt))). - orderBy("tasks.number_attempts, tasks.created_at"). - limit(limit). - offset(offset). - anormSql().as( - parser().* - ) - } - } - - private[this] def parser(): RowParser[Task] = { - SqlParser.get[UUID]("guid") ~ - SqlParser.str("data") ~ - SqlParser.long("number_attempts") ~ - SqlParser.str("last_error").? map { - case guid ~ data ~ numberAttempts ~ lastError => { - Task( - guid = guid, - data = Json.parse(data).as[TaskData], - numberAttempts = numberAttempts, - lastError = lastError - ) - } - } - } - -} diff --git a/api/app/db/generated/PsqlApibuilderTasksDao.scala b/api/app/db/generated/PsqlApibuilderTasksDao.scala new file mode 100644 index 000000000..d61d6a4a0 --- /dev/null +++ b/api/app/db/generated/PsqlApibuilderTasksDao.scala @@ -0,0 +1,616 @@ +package db.generated + +import anorm.JodaParameterMetaData._ +import anorm._ +import io.flow.postgresql.{OrderBy, Query} +import java.sql.Connection +import javax.inject.{Inject, Singleton} +import org.joda.time.DateTime +import play.api.db.Database +import play.api.libs.json.{JsValue, Json} + +case class Task( + id: String, + `type`: String, + typeId: String, + organizationGuid: Option[String], + numAttempts: Int, + nextAttemptAt: DateTime, + errors: Option[Seq[String]], + stacktrace: Option[String], + data: JsValue, + updatedByUserId: String, + createdAt: DateTime, + updatedAt: DateTime +) { + + lazy val form: TaskForm = TaskForm( + id = id, + `type` = `type`, + typeId = typeId, + organizationGuid = organizationGuid, + numAttempts = numAttempts, + nextAttemptAt = nextAttemptAt, + errors = errors, + stacktrace = stacktrace, + data = data + ) + +} + +case class TaskForm( + id: String, + `type`: String, + typeId: String, + organizationGuid: Option[String], + numAttempts: Int, + nextAttemptAt: DateTime, + errors: Option[Seq[String]], + stacktrace: Option[String], + data: JsValue +) + +object TasksTable { + val Schema: String = "public" + val Name: String = "tasks" + val QualifiedName: String = s"$Schema.$Name" + + object Columns { + val Id: String = "id" + val Type: String = "type" + val TypeId: String = "type_id" + val OrganizationGuid: String = "organization_guid" + val NumAttempts: String = "num_attempts" + val NextAttemptAt: String = "next_attempt_at" + val Errors: String = "errors" + val Stacktrace: String = "stacktrace" + val Data: String = "data" + val UpdatedByUserId: String = "updated_by_user_id" + val CreatedAt: String = "created_at" + val UpdatedAt: String = "updated_at" + val HashCode: String = "hash_code" + val all: List[String] = List(Id, Type, TypeId, OrganizationGuid, NumAttempts, NextAttemptAt, Errors, Stacktrace, Data, UpdatedByUserId, CreatedAt, UpdatedAt, HashCode) + } +} + +trait BaseTasksDao { + + def db: Database + + private[this] val BaseQuery = Query(""" + | select tasks.id, + | tasks.type, + | tasks.type_id, + | tasks.organization_guid, + | tasks.num_attempts, + | tasks.next_attempt_at, + | tasks.errors::text as errors_text, + | tasks.stacktrace, + | tasks.data::text as data_text, + | tasks.updated_by_user_id, + | tasks.created_at, + | tasks.updated_at, + | tasks.hash_code + | from tasks + """.stripMargin) + + def findById(id: String): Option[Task] = { + db.withConnection { c => + findByIdWithConnection(c, id) + } + } + + def findByIdWithConnection(c: java.sql.Connection, id: String): Option[Task] = { + findAllWithConnection(c, ids = Some(Seq(id)), limit = Some(1L), orderBy = None).headOption + } + + def findByTypeIdAndType(typeId: String, `type`: String): Option[Task] = { + db.withConnection { c => + findByTypeIdAndTypeWithConnection(c, typeId, `type`) + } + } + + def findByTypeIdAndTypeWithConnection(c: java.sql.Connection, typeId: String, `type`: String): Option[Task] = { + findAllWithConnection(c, typeId = Some(typeId), `type` = Some(`type`), limit = Some(1L), orderBy = None).headOption + } + + def iterateAll( + ids: Option[Seq[String]] = None, + typeId: Option[String] = None, + typeIds: Option[Seq[String]] = None, + `type`: Option[String] = None, + types: Option[Seq[String]] = None, + numAttempts: Option[Int] = None, + numAttemptsGreaterThanOrEquals: Option[Int] = None, + numAttemptsGreaterThan: Option[Int] = None, + numAttemptsLessThanOrEquals: Option[Int] = None, + numAttemptsLessThan: Option[Int] = None, + nextAttemptAt: Option[DateTime] = None, + nextAttemptAtGreaterThanOrEquals: Option[DateTime] = None, + nextAttemptAtGreaterThan: Option[DateTime] = None, + nextAttemptAtLessThanOrEquals: Option[DateTime] = None, + nextAttemptAtLessThan: Option[DateTime] = None, + numAttemptsNextAttemptAts: Option[Seq[(Int, DateTime)]] = None, + typeIdTypes: Option[Seq[(String, String)]] = None, + pageSize: Long = 2000L, + ) ( + implicit customQueryModifier: Query => Query = { q => q } + ): Iterator[Task] = { + def iterate(lastValue: Option[Task]): Iterator[Task] = { + val page = findAll( + ids = ids, + typeId = typeId, + typeIds = typeIds, + `type` = `type`, + types = types, + numAttempts = numAttempts, + numAttemptsGreaterThanOrEquals = numAttemptsGreaterThanOrEquals, + numAttemptsGreaterThan = numAttemptsGreaterThan, + numAttemptsLessThanOrEquals = numAttemptsLessThanOrEquals, + numAttemptsLessThan = numAttemptsLessThan, + nextAttemptAt = nextAttemptAt, + nextAttemptAtGreaterThanOrEquals = nextAttemptAtGreaterThanOrEquals, + nextAttemptAtGreaterThan = nextAttemptAtGreaterThan, + nextAttemptAtLessThanOrEquals = nextAttemptAtLessThanOrEquals, + nextAttemptAtLessThan = nextAttemptAtLessThan, + numAttemptsNextAttemptAts = numAttemptsNextAttemptAts, + typeIdTypes = typeIdTypes, + limit = Some(pageSize), + orderBy = Some(OrderBy("tasks.id")), + ) { q => customQueryModifier(q).greaterThan("tasks.id", lastValue.map(_.id)) } + + page.lastOption match { + case None => Iterator.empty + case lastValue => page.iterator ++ iterate(lastValue) + } + } + + iterate(None) + } + + def findAll( + ids: Option[Seq[String]] = None, + typeId: Option[String] = None, + typeIds: Option[Seq[String]] = None, + `type`: Option[String] = None, + types: Option[Seq[String]] = None, + numAttempts: Option[Int] = None, + numAttemptsGreaterThanOrEquals: Option[Int] = None, + numAttemptsGreaterThan: Option[Int] = None, + numAttemptsLessThanOrEquals: Option[Int] = None, + numAttemptsLessThan: Option[Int] = None, + nextAttemptAt: Option[DateTime] = None, + nextAttemptAtGreaterThanOrEquals: Option[DateTime] = None, + nextAttemptAtGreaterThan: Option[DateTime] = None, + nextAttemptAtLessThanOrEquals: Option[DateTime] = None, + nextAttemptAtLessThan: Option[DateTime] = None, + numAttemptsNextAttemptAts: Option[Seq[(Int, DateTime)]] = None, + typeIdTypes: Option[Seq[(String, String)]] = None, + limit: Option[Long], + offset: Long = 0, + orderBy: Option[OrderBy] = Some(OrderBy("tasks.id")) + ) ( + implicit customQueryModifier: Query => Query = { q => q } + ): Seq[Task] = { + db.withConnection { c => + findAllWithConnection( + c, + ids = ids, + typeId = typeId, + typeIds = typeIds, + `type` = `type`, + types = types, + numAttempts = numAttempts, + numAttemptsGreaterThanOrEquals = numAttemptsGreaterThanOrEquals, + numAttemptsGreaterThan = numAttemptsGreaterThan, + numAttemptsLessThanOrEquals = numAttemptsLessThanOrEquals, + numAttemptsLessThan = numAttemptsLessThan, + nextAttemptAt = nextAttemptAt, + nextAttemptAtGreaterThanOrEquals = nextAttemptAtGreaterThanOrEquals, + nextAttemptAtGreaterThan = nextAttemptAtGreaterThan, + nextAttemptAtLessThanOrEquals = nextAttemptAtLessThanOrEquals, + nextAttemptAtLessThan = nextAttemptAtLessThan, + numAttemptsNextAttemptAts = numAttemptsNextAttemptAts, + typeIdTypes = typeIdTypes, + limit = limit, + offset = offset, + orderBy = orderBy + )(customQueryModifier) + } + } + + def findAllWithConnection( + c: java.sql.Connection, + ids: Option[Seq[String]] = None, + typeId: Option[String] = None, + typeIds: Option[Seq[String]] = None, + `type`: Option[String] = None, + types: Option[Seq[String]] = None, + numAttempts: Option[Int] = None, + numAttemptsGreaterThanOrEquals: Option[Int] = None, + numAttemptsGreaterThan: Option[Int] = None, + numAttemptsLessThanOrEquals: Option[Int] = None, + numAttemptsLessThan: Option[Int] = None, + nextAttemptAt: Option[DateTime] = None, + nextAttemptAtGreaterThanOrEquals: Option[DateTime] = None, + nextAttemptAtGreaterThan: Option[DateTime] = None, + nextAttemptAtLessThanOrEquals: Option[DateTime] = None, + nextAttemptAtLessThan: Option[DateTime] = None, + numAttemptsNextAttemptAts: Option[Seq[(Int, DateTime)]] = None, + typeIdTypes: Option[Seq[(String, String)]] = None, + limit: Option[Long], + offset: Long = 0, + orderBy: Option[OrderBy] = Some(OrderBy("tasks.id")) + ) ( + implicit customQueryModifier: Query => Query = { q => q } + ): Seq[Task] = { + customQueryModifier(BaseQuery). + optionalIn("tasks.id", ids). + equals("tasks.type_id", typeId). + optionalIn("tasks.type_id", typeIds). + equals("tasks.type", `type`). + optionalIn("tasks.type", types). + equals("tasks.num_attempts", numAttempts). + greaterThanOrEquals("tasks.num_attempts", numAttemptsGreaterThanOrEquals). + greaterThan("tasks.num_attempts", numAttemptsGreaterThan). + lessThanOrEquals("tasks.num_attempts", numAttemptsLessThanOrEquals). + lessThan("tasks.num_attempts", numAttemptsLessThan). + equals("tasks.next_attempt_at", nextAttemptAt). + greaterThanOrEquals("tasks.next_attempt_at", nextAttemptAtGreaterThanOrEquals). + greaterThan("tasks.next_attempt_at", nextAttemptAtGreaterThan). + lessThanOrEquals("tasks.next_attempt_at", nextAttemptAtLessThanOrEquals). + lessThan("tasks.next_attempt_at", nextAttemptAtLessThan). + optionalIn2(("tasks.num_attempts", "tasks.next_attempt_at"), numAttemptsNextAttemptAts). + optionalIn2(("tasks.type_id", "tasks.type"), typeIdTypes). + optionalLimit(limit). + offset(offset). + orderBy(orderBy.flatMap(_.sql)). + as(TasksDao.parser.*)(c) + } + +} + +object TasksDao { + + val parser: RowParser[Task] = { + SqlParser.str("id") ~ + SqlParser.str("type") ~ + SqlParser.str("type_id") ~ + SqlParser.str("organization_guid").? ~ + SqlParser.int("num_attempts") ~ + SqlParser.get[DateTime]("next_attempt_at") ~ + SqlParser.str("errors_text").? ~ + SqlParser.str("stacktrace").? ~ + SqlParser.str("data_text") ~ + SqlParser.str("updated_by_user_id") ~ + SqlParser.get[DateTime]("created_at") ~ + SqlParser.get[DateTime]("updated_at") map { + case id ~ type_ ~ typeId ~ organizationGuid ~ numAttempts ~ nextAttemptAt ~ errors ~ stacktrace ~ data ~ updatedByUserId ~ createdAt ~ updatedAt => Task( + id = id, + `type` = type_, + typeId = typeId, + organizationGuid = organizationGuid, + numAttempts = numAttempts, + nextAttemptAt = nextAttemptAt, + errors = errors.map { text => Json.parse(text).as[Seq[String]] }, + stacktrace = stacktrace, + data = Json.parse(data), + updatedByUserId = updatedByUserId, + createdAt = createdAt, + updatedAt = updatedAt + ) + } + } + +} + +@Singleton +class TasksDao @Inject() ( + override val db: Database +) extends BaseTasksDao { + + private[this] val UpsertQuery = Query(""" + | insert into tasks + | (id, type, type_id, organization_guid, num_attempts, next_attempt_at, errors, stacktrace, data, updated_by_user_id, hash_code) + | values + | ({id}, {type}, {type_id}, {organization_guid}, {num_attempts}::int, {next_attempt_at}::timestamptz, {errors}::json, {stacktrace}, {data}::json, {updated_by_user_id}, {hash_code}::bigint) + | on conflict (type_id, type) + | do update + | set organization_guid = {organization_guid}, + | num_attempts = {num_attempts}::int, + | next_attempt_at = {next_attempt_at}::timestamptz, + | errors = {errors}::json, + | stacktrace = {stacktrace}, + | data = {data}::json, + | updated_by_user_id = {updated_by_user_id}, + | hash_code = {hash_code}::bigint + | where tasks.hash_code != {hash_code}::bigint + | returning id + """.stripMargin) + + private[this] val UpdateQuery = Query(""" + | update tasks + | set type = {type}, + | type_id = {type_id}, + | organization_guid = {organization_guid}, + | num_attempts = {num_attempts}::int, + | next_attempt_at = {next_attempt_at}::timestamptz, + | errors = {errors}::json, + | stacktrace = {stacktrace}, + | data = {data}::json, + | updated_by_user_id = {updated_by_user_id}, + | hash_code = {hash_code}::bigint + | where id = {id} + | and tasks.hash_code != {hash_code}::bigint + """.stripMargin) + + private[this] def bindQuery(query: Query, form: TaskForm): Query = { + query. + bind("type", form.`type`). + bind("type_id", form.typeId). + bind("organization_guid", form.organizationGuid). + bind("num_attempts", form.numAttempts). + bind("next_attempt_at", form.nextAttemptAt). + bind("errors", form.errors.map { v => Json.toJson(v) }). + bind("stacktrace", form.stacktrace). + bind("data", form.data). + bind("hash_code", form.hashCode()) + } + + private[this] def toNamedParameter(updatedBy: String, form: TaskForm): Seq[NamedParameter] = { + Seq( + scala.Symbol("id") -> form.id, + scala.Symbol("type") -> form.`type`, + scala.Symbol("type_id") -> form.typeId, + scala.Symbol("organization_guid") -> form.organizationGuid, + scala.Symbol("num_attempts") -> form.numAttempts, + scala.Symbol("next_attempt_at") -> form.nextAttemptAt, + scala.Symbol("errors") -> form.errors.map { v => Json.toJson(v).toString }, + scala.Symbol("stacktrace") -> form.stacktrace, + scala.Symbol("data") -> form.data.toString, + scala.Symbol("updated_by_user_id") -> updatedBy, + scala.Symbol("hash_code") -> form.hashCode() + ) + } + + def upsertIfChangedByTypeIdAndType(updatedBy: String, form: TaskForm): Unit = { + if (!findByTypeIdAndType(form.typeId, form.`type`).map(_.form).contains(form)) { + upsertByTypeIdAndType(updatedBy, form) + } + } + + def upsertByTypeIdAndType(updatedBy: String, form: TaskForm): Unit = { + db.withConnection { c => + upsertByTypeIdAndType(c, updatedBy, form) + } + } + + def upsertByTypeIdAndType(c: Connection, updatedBy: String, form: TaskForm): Unit = { + bindQuery(UpsertQuery, form). + bind("id", form.id). + bind("updated_by_user_id", updatedBy). + anormSql.execute()(c) + () + } + + def upsertBatchByTypeIdAndType(updatedBy: String, forms: Seq[TaskForm]): Unit = { + db.withConnection { c => + upsertBatchByTypeIdAndType(c, updatedBy, forms) + } + } + + def upsertBatchByTypeIdAndType(c: Connection, updatedBy: String, forms: Seq[TaskForm]): Unit = { + if (forms.nonEmpty) { + val params = forms.map(toNamedParameter(updatedBy, _)) + BatchSql(UpsertQuery.sql(), params.head, params.tail: _*).execute()(c) + () + } + } + + def updateIfChangedById(updatedBy: String, id: String, form: TaskForm): Unit = { + if (!findById(id).map(_.form).contains(form)) { + updateById(updatedBy, id, form) + } + } + + def updateById(updatedBy: String, id: String, form: TaskForm): Unit = { + db.withConnection { c => + updateById(c, updatedBy, id, form) + } + } + + def updateById(c: Connection, updatedBy: String, id: String, form: TaskForm): Unit = { + bindQuery(UpdateQuery, form). + bind("id", id). + bind("updated_by_user_id", updatedBy). + anormSql.execute()(c) + () + } + + def update(updatedBy: String, existing: Task, form: TaskForm): Unit = { + db.withConnection { c => + update(c, updatedBy, existing, form) + } + } + + def update(c: Connection, updatedBy: String, existing: Task, form: TaskForm): Unit = { + updateById(c, updatedBy, existing.id, form) + } + + def updateBatch(updatedBy: String, forms: Seq[TaskForm]): Unit = { + db.withConnection { c => + updateBatchWithConnection(c, updatedBy, forms) + } + } + + def updateBatchWithConnection(c: Connection, updatedBy: String, forms: Seq[TaskForm]): Unit = { + if (forms.nonEmpty) { + val params = forms.map(toNamedParameter(updatedBy, _)) + BatchSql(UpdateQuery.sql(), params.head, params.tail: _*).execute()(c) + () + } + } + + def delete(deletedBy: String, task: Task): Unit = { + db.withConnection { c => + delete(c, deletedBy, task) + } + } + + def delete(c: Connection, deletedBy: String, task: Task): Unit = { + deleteById(c, deletedBy, task.id) + } + + def deleteById(deletedBy: String, id: String): Unit = { + db.withConnection { c => + deleteById(c, deletedBy, id) + } + } + + def deleteById(c: Connection, deletedBy: String, id: String): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("id", id) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByIds(deletedBy: String, ids: Seq[String]): Unit = { + db.withConnection { c => + deleteAllByIds(c, deletedBy, ids) + } + } + + def deleteAllByIds(c: Connection, deletedBy: String, ids: Seq[String]): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .in("id", ids) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByNumAttempts(deletedBy: String, numAttempts: Int): Unit = { + db.withConnection { c => + deleteAllByNumAttempts(c, deletedBy, numAttempts) + } + } + + def deleteAllByNumAttempts(c: Connection, deletedBy: String, numAttempts: Int): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("num_attempts", numAttempts) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByNumAttemptses(deletedBy: String, numAttemptses: Seq[Int]): Unit = { + db.withConnection { c => + deleteAllByNumAttemptses(c, deletedBy, numAttemptses) + } + } + + def deleteAllByNumAttemptses(c: Connection, deletedBy: String, numAttemptses: Seq[Int]): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .in("num_attempts", numAttemptses) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByNumAttemptsAndNextAttemptAt(deletedBy: String, numAttempts: Int, nextAttemptAt: DateTime): Unit = { + db.withConnection { c => + deleteAllByNumAttemptsAndNextAttemptAt(c, deletedBy, numAttempts, nextAttemptAt) + } + } + + def deleteAllByNumAttemptsAndNextAttemptAt(c: Connection, deletedBy: String, numAttempts: Int, nextAttemptAt: DateTime): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("num_attempts", numAttempts) + .equals("next_attempt_at", nextAttemptAt) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByNumAttemptsAndNextAttemptAts(deletedBy: String, numAttempts: Int, nextAttemptAts: Seq[DateTime]): Unit = { + db.withConnection { c => + deleteAllByNumAttemptsAndNextAttemptAts(c, deletedBy, numAttempts, nextAttemptAts) + } + } + + def deleteAllByNumAttemptsAndNextAttemptAts(c: Connection, deletedBy: String, numAttempts: Int, nextAttemptAts: Seq[DateTime]): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("num_attempts", numAttempts) + .in("next_attempt_at", nextAttemptAts) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByTypeId(deletedBy: String, typeId: String): Unit = { + db.withConnection { c => + deleteAllByTypeId(c, deletedBy, typeId) + } + } + + def deleteAllByTypeId(c: Connection, deletedBy: String, typeId: String): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("type_id", typeId) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByTypeIds(deletedBy: String, typeIds: Seq[String]): Unit = { + db.withConnection { c => + deleteAllByTypeIds(c, deletedBy, typeIds) + } + } + + def deleteAllByTypeIds(c: Connection, deletedBy: String, typeIds: Seq[String]): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .in("type_id", typeIds) + .anormSql.executeUpdate()(c) + () + } + + def deleteByTypeIdAndType(deletedBy: String, typeId: String, `type`: String): Unit = { + db.withConnection { c => + deleteByTypeIdAndType(c, deletedBy, typeId, `type`) + } + } + + def deleteByTypeIdAndType(c: Connection, deletedBy: String, typeId: String, `type`: String): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("type_id", typeId) + .equals("type", `type`) + .anormSql.executeUpdate()(c) + () + } + + def deleteAllByTypeIdAndTypes(deletedBy: String, typeId: String, types: Seq[String]): Unit = { + db.withConnection { c => + deleteAllByTypeIdAndTypes(c, deletedBy, typeId, types) + } + } + + def deleteAllByTypeIdAndTypes(c: Connection, deletedBy: String, typeId: String, types: Seq[String]): Unit = { + setJournalDeletedByUserId(c, deletedBy) + Query("delete from tasks") + .equals("type_id", typeId) + .in("type", types) + .anormSql.executeUpdate()(c) + () + } + + private[this] val ValidCharacters: Set[String] = "_-,.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789".split("").toSet + private[this] def isSafe(value: String): Boolean = value.trim.split("").forall(ValidCharacters.contains) + def setJournalDeletedByUserId(c: Connection, deletedBy: String): Unit = { + assert(isSafe(deletedBy), s"Value '${deletedBy}' contains unsafe characters") + anorm.SQL(s"SET journal.deleted_by_user_id = '${deletedBy}'").executeUpdate()(c) + () + } + +} \ No newline at end of file diff --git a/dao/spec/psql-apibuilder.json b/dao/spec/psql-apibuilder.json index 14c9c84ec..b6ec0e2d0 100644 --- a/dao/spec/psql-apibuilder.json +++ b/dao/spec/psql-apibuilder.json @@ -70,6 +70,43 @@ } } ] + }, + + "task": { + "fields": [ + { "name": "id", "type": "string" }, + { "name": "type", "type": "string" }, + { "name": "type_id", "type": "string" }, + { "name": "organization_guid", "type": "string", "required": false }, + { "name": "num_attempts", "type": "integer", "minimum": 0, "default": 0 }, + { "name": "next_attempt_at", "type": "date-time-iso8601" }, + { "name": "errors", "type": "[string]", "required": false }, + { "name": "stacktrace", "type": "string", "required": false }, + { "name": "data", "type": "json" } + ], + "attributes": [ + { + "name": "scala", + "value": { + "package": "db.generated", + "dao_user_class": "String", + "order_by": { "optional": true } + } + }, + { + "name": "psql", + "value": { + "pkey": "id", + "authorization": { "type": "disabled" }, + "on_conflict": { + "fields": ["type_id", "type"] + }, + "indexes": [ + { "fields": ["num_attempts", "next_attempt_at"] } + ] + } + } + ] } } } From 8808c3d0c76a6eac7fc31c4926c8dfd11f61b5f9 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:23:59 -0700 Subject: [PATCH 02/32] wip --- api/app/actors/TaskActor.scala | 28 ++++ api/app/actors/TaskDispatchActor.scala | 62 +++++++ .../db/generated/PsqlApibuilderTasksDao.scala | 114 +++++++------ api/app/processor/TaskActorCompanion.scala | 31 ++++ .../TaskDispatchActorCompanion.scala | 24 +++ api/app/processor/TaskProcessor.scala | 156 ++++++++++++++++++ api/app/processor/TaskType.scala | 15 ++ api/conf/base.conf | 14 ++ api/test/actors/TaskActorSpec.scala | 15 ++ dao/spec/psql-apibuilder.json | 4 +- 10 files changed, 403 insertions(+), 60 deletions(-) create mode 100644 api/app/actors/TaskActor.scala create mode 100644 api/app/actors/TaskDispatchActor.scala create mode 100644 api/app/processor/TaskActorCompanion.scala create mode 100644 api/app/processor/TaskDispatchActorCompanion.scala create mode 100644 api/app/processor/TaskProcessor.scala create mode 100644 api/app/processor/TaskType.scala create mode 100644 api/test/actors/TaskActorSpec.scala diff --git a/api/app/actors/TaskActor.scala b/api/app/actors/TaskActor.scala new file mode 100644 index 000000000..493d088ce --- /dev/null +++ b/api/app/actors/TaskActor.scala @@ -0,0 +1,28 @@ +package actors + +import akka.actor.{Actor, ActorLogging} +import com.google.inject.assistedinject.Assisted +import processor.{TaskActorCompanion, TaskType} + +import javax.inject.Inject + +object TaskActor { + case object Process + trait Factory { + def apply( + @Assisted("type") `type`: TaskType + ): Actor + } +} + +class TaskActor @Inject() ( + @Assisted("type") `type`: TaskType, + companion: TaskActorCompanion +) extends Actor with ActorLogging with ErrorHandler { + + def receive: Receive = { + case TaskActor.Process => companion.process(`type`) + case m: Any => logUnhandledMessage(m) + } + +} diff --git a/api/app/actors/TaskDispatchActor.scala b/api/app/actors/TaskDispatchActor.scala new file mode 100644 index 000000000..224353ef9 --- /dev/null +++ b/api/app/actors/TaskDispatchActor.scala @@ -0,0 +1,62 @@ +package actors + +import akka.actor._ +import play.api.libs.concurrent.InjectedActorSupport +import processor.{TaskDispatchActorCompanion, TaskType} + +import javax.inject.{Inject, Singleton} +import scala.concurrent.ExecutionContext +import scala.concurrent.duration.{FiniteDuration, SECONDS} + +@Singleton +class TaskDispatchActor @Inject() ( + factory: TaskActor.Factory, + companion: TaskDispatchActorCompanion +) extends Actor with ActorLogging with ErrorHandler with InjectedActorSupport { + private[this] val ec: ExecutionContext = context.system.dispatchers.lookup("task-context") + private[this] case object Process + + private[this] val actors = scala.collection.mutable.Map[TaskType, ActorRef]() + + private[this] def schedule(message: Any, interval: FiniteDuration): Cancellable = { + context.system.scheduler.scheduleWithFixedDelay(FiniteDuration(1, SECONDS), interval, self, message)(ec) + } + + private[this] val cancellables: Seq[Cancellable] = { + Seq( + schedule(Process, FiniteDuration(2, SECONDS)) + ) + } + + override def postStop(): Unit = { + cancellables.foreach(_.cancel()) + super.postStop() + } + + override def receive: Receive = { + case Process => process() + case other => logUnhandledMessage(other) + } + + private[this] def process(): Unit = { + companion.typesWithWork.foreach { typ => + upsertActor(typ) ! TaskActor.Process + } + } + + private[this] def upsertActor(typ: TaskType): ActorRef = { + actors.getOrElse( + typ, { + val name = s"task$typ" + val ref = injectedChild( + factory(typ), + name = name, + _.withDispatcher("task-context-dispatch") + ) + actors += (typ -> ref) + ref + } + ) + } + +} diff --git a/api/app/db/generated/PsqlApibuilderTasksDao.scala b/api/app/db/generated/PsqlApibuilderTasksDao.scala index d61d6a4a0..612e41ddd 100644 --- a/api/app/db/generated/PsqlApibuilderTasksDao.scala +++ b/api/app/db/generated/PsqlApibuilderTasksDao.scala @@ -4,6 +4,7 @@ import anorm.JodaParameterMetaData._ import anorm._ import io.flow.postgresql.{OrderBy, Query} import java.sql.Connection +import java.util.UUID import javax.inject.{Inject, Singleton} import org.joda.time.DateTime import play.api.db.Database @@ -13,13 +14,13 @@ case class Task( id: String, `type`: String, typeId: String, - organizationGuid: Option[String], + organizationGuid: Option[UUID], numAttempts: Int, nextAttemptAt: DateTime, errors: Option[Seq[String]], stacktrace: Option[String], data: JsValue, - updatedByUserId: String, + updatedByGuid: String, createdAt: DateTime, updatedAt: DateTime ) { @@ -42,7 +43,7 @@ case class TaskForm( id: String, `type`: String, typeId: String, - organizationGuid: Option[String], + organizationGuid: Option[UUID], numAttempts: Int, nextAttemptAt: DateTime, errors: Option[Seq[String]], @@ -65,11 +66,11 @@ object TasksTable { val Errors: String = "errors" val Stacktrace: String = "stacktrace" val Data: String = "data" - val UpdatedByUserId: String = "updated_by_user_id" + val UpdatedByGuid: String = "updated_by_guid" val CreatedAt: String = "created_at" val UpdatedAt: String = "updated_at" val HashCode: String = "hash_code" - val all: List[String] = List(Id, Type, TypeId, OrganizationGuid, NumAttempts, NextAttemptAt, Errors, Stacktrace, Data, UpdatedByUserId, CreatedAt, UpdatedAt, HashCode) + val all: List[String] = List(Id, Type, TypeId, OrganizationGuid, NumAttempts, NextAttemptAt, Errors, Stacktrace, Data, UpdatedByGuid, CreatedAt, UpdatedAt, HashCode) } } @@ -87,7 +88,7 @@ trait BaseTasksDao { | tasks.errors::text as errors_text, | tasks.stacktrace, | tasks.data::text as data_text, - | tasks.updated_by_user_id, + | tasks.updated_by_guid, | tasks.created_at, | tasks.updated_at, | tasks.hash_code @@ -276,16 +277,16 @@ object TasksDao { SqlParser.str("id") ~ SqlParser.str("type") ~ SqlParser.str("type_id") ~ - SqlParser.str("organization_guid").? ~ + SqlParser.get[UUID]("organization_guid").? ~ SqlParser.int("num_attempts") ~ SqlParser.get[DateTime]("next_attempt_at") ~ SqlParser.str("errors_text").? ~ SqlParser.str("stacktrace").? ~ SqlParser.str("data_text") ~ - SqlParser.str("updated_by_user_id") ~ + SqlParser.str("updated_by_guid") ~ SqlParser.get[DateTime]("created_at") ~ SqlParser.get[DateTime]("updated_at") map { - case id ~ type_ ~ typeId ~ organizationGuid ~ numAttempts ~ nextAttemptAt ~ errors ~ stacktrace ~ data ~ updatedByUserId ~ createdAt ~ updatedAt => Task( + case id ~ type_ ~ typeId ~ organizationGuid ~ numAttempts ~ nextAttemptAt ~ errors ~ stacktrace ~ data ~ updatedByGuid ~ createdAt ~ updatedAt => Task( id = id, `type` = type_, typeId = typeId, @@ -295,7 +296,7 @@ object TasksDao { errors = errors.map { text => Json.parse(text).as[Seq[String]] }, stacktrace = stacktrace, data = Json.parse(data), - updatedByUserId = updatedByUserId, + updatedByGuid = updatedByGuid, createdAt = createdAt, updatedAt = updatedAt ) @@ -311,18 +312,18 @@ class TasksDao @Inject() ( private[this] val UpsertQuery = Query(""" | insert into tasks - | (id, type, type_id, organization_guid, num_attempts, next_attempt_at, errors, stacktrace, data, updated_by_user_id, hash_code) + | (id, type, type_id, organization_guid, num_attempts, next_attempt_at, errors, stacktrace, data, updated_by_guid, hash_code) | values - | ({id}, {type}, {type_id}, {organization_guid}, {num_attempts}::int, {next_attempt_at}::timestamptz, {errors}::json, {stacktrace}, {data}::json, {updated_by_user_id}, {hash_code}::bigint) + | ({id}, {type}, {type_id}, {organization_guid}::uuid, {num_attempts}::int, {next_attempt_at}::timestamptz, {errors}::json, {stacktrace}, {data}::json, {updated_by_guid}, {hash_code}::bigint) | on conflict (type_id, type) | do update - | set organization_guid = {organization_guid}, + | set organization_guid = {organization_guid}::uuid, | num_attempts = {num_attempts}::int, | next_attempt_at = {next_attempt_at}::timestamptz, | errors = {errors}::json, | stacktrace = {stacktrace}, | data = {data}::json, - | updated_by_user_id = {updated_by_user_id}, + | updated_by_guid = {updated_by_guid}, | hash_code = {hash_code}::bigint | where tasks.hash_code != {hash_code}::bigint | returning id @@ -332,13 +333,13 @@ class TasksDao @Inject() ( | update tasks | set type = {type}, | type_id = {type_id}, - | organization_guid = {organization_guid}, + | organization_guid = {organization_guid}::uuid, | num_attempts = {num_attempts}::int, | next_attempt_at = {next_attempt_at}::timestamptz, | errors = {errors}::json, | stacktrace = {stacktrace}, | data = {data}::json, - | updated_by_user_id = {updated_by_user_id}, + | updated_by_guid = {updated_by_guid}, | hash_code = {hash_code}::bigint | where id = {id} | and tasks.hash_code != {hash_code}::bigint @@ -357,7 +358,7 @@ class TasksDao @Inject() ( bind("hash_code", form.hashCode()) } - private[this] def toNamedParameter(updatedBy: String, form: TaskForm): Seq[NamedParameter] = { + private[this] def toNamedParameter(updatedBy: UUID, form: TaskForm): Seq[NamedParameter] = { Seq( scala.Symbol("id") -> form.id, scala.Symbol("type") -> form.`type`, @@ -368,38 +369,38 @@ class TasksDao @Inject() ( scala.Symbol("errors") -> form.errors.map { v => Json.toJson(v).toString }, scala.Symbol("stacktrace") -> form.stacktrace, scala.Symbol("data") -> form.data.toString, - scala.Symbol("updated_by_user_id") -> updatedBy, + scala.Symbol("updated_by_guid") -> updatedBy, scala.Symbol("hash_code") -> form.hashCode() ) } - def upsertIfChangedByTypeIdAndType(updatedBy: String, form: TaskForm): Unit = { + def upsertIfChangedByTypeIdAndType(updatedBy: UUID, form: TaskForm): Unit = { if (!findByTypeIdAndType(form.typeId, form.`type`).map(_.form).contains(form)) { upsertByTypeIdAndType(updatedBy, form) } } - def upsertByTypeIdAndType(updatedBy: String, form: TaskForm): Unit = { + def upsertByTypeIdAndType(updatedBy: UUID, form: TaskForm): Unit = { db.withConnection { c => upsertByTypeIdAndType(c, updatedBy, form) } } - def upsertByTypeIdAndType(c: Connection, updatedBy: String, form: TaskForm): Unit = { + def upsertByTypeIdAndType(c: Connection, updatedBy: UUID, form: TaskForm): Unit = { bindQuery(UpsertQuery, form). bind("id", form.id). - bind("updated_by_user_id", updatedBy). + bind("updated_by_guid", updatedBy). anormSql.execute()(c) () } - def upsertBatchByTypeIdAndType(updatedBy: String, forms: Seq[TaskForm]): Unit = { + def upsertBatchByTypeIdAndType(updatedBy: UUID, forms: Seq[TaskForm]): Unit = { db.withConnection { c => upsertBatchByTypeIdAndType(c, updatedBy, forms) } } - def upsertBatchByTypeIdAndType(c: Connection, updatedBy: String, forms: Seq[TaskForm]): Unit = { + def upsertBatchByTypeIdAndType(c: Connection, updatedBy: UUID, forms: Seq[TaskForm]): Unit = { if (forms.nonEmpty) { val params = forms.map(toNamedParameter(updatedBy, _)) BatchSql(UpsertQuery.sql(), params.head, params.tail: _*).execute()(c) @@ -407,43 +408,43 @@ class TasksDao @Inject() ( } } - def updateIfChangedById(updatedBy: String, id: String, form: TaskForm): Unit = { + def updateIfChangedById(updatedBy: UUID, id: String, form: TaskForm): Unit = { if (!findById(id).map(_.form).contains(form)) { updateById(updatedBy, id, form) } } - def updateById(updatedBy: String, id: String, form: TaskForm): Unit = { + def updateById(updatedBy: UUID, id: String, form: TaskForm): Unit = { db.withConnection { c => updateById(c, updatedBy, id, form) } } - def updateById(c: Connection, updatedBy: String, id: String, form: TaskForm): Unit = { + def updateById(c: Connection, updatedBy: UUID, id: String, form: TaskForm): Unit = { bindQuery(UpdateQuery, form). bind("id", id). - bind("updated_by_user_id", updatedBy). + bind("updated_by_guid", updatedBy). anormSql.execute()(c) () } - def update(updatedBy: String, existing: Task, form: TaskForm): Unit = { + def update(updatedBy: UUID, existing: Task, form: TaskForm): Unit = { db.withConnection { c => update(c, updatedBy, existing, form) } } - def update(c: Connection, updatedBy: String, existing: Task, form: TaskForm): Unit = { + def update(c: Connection, updatedBy: UUID, existing: Task, form: TaskForm): Unit = { updateById(c, updatedBy, existing.id, form) } - def updateBatch(updatedBy: String, forms: Seq[TaskForm]): Unit = { + def updateBatch(updatedBy: UUID, forms: Seq[TaskForm]): Unit = { db.withConnection { c => updateBatchWithConnection(c, updatedBy, forms) } } - def updateBatchWithConnection(c: Connection, updatedBy: String, forms: Seq[TaskForm]): Unit = { + def updateBatchWithConnection(c: Connection, updatedBy: UUID, forms: Seq[TaskForm]): Unit = { if (forms.nonEmpty) { val params = forms.map(toNamedParameter(updatedBy, _)) BatchSql(UpdateQuery.sql(), params.head, params.tail: _*).execute()(c) @@ -451,23 +452,23 @@ class TasksDao @Inject() ( } } - def delete(deletedBy: String, task: Task): Unit = { + def delete(deletedBy: UUID, task: Task): Unit = { db.withConnection { c => delete(c, deletedBy, task) } } - def delete(c: Connection, deletedBy: String, task: Task): Unit = { + def delete(c: Connection, deletedBy: UUID, task: Task): Unit = { deleteById(c, deletedBy, task.id) } - def deleteById(deletedBy: String, id: String): Unit = { + def deleteById(deletedBy: UUID, id: String): Unit = { db.withConnection { c => deleteById(c, deletedBy, id) } } - def deleteById(c: Connection, deletedBy: String, id: String): Unit = { + def deleteById(c: Connection, deletedBy: UUID, id: String): Unit = { setJournalDeletedByUserId(c, deletedBy) Query("delete from tasks") .equals("id", id) @@ -475,13 +476,13 @@ class TasksDao @Inject() ( () } - def deleteAllByIds(deletedBy: String, ids: Seq[String]): Unit = { + def deleteAllByIds(deletedBy: UUID, ids: Seq[String]): Unit = { db.withConnection { c => deleteAllByIds(c, deletedBy, ids) } } - def deleteAllByIds(c: Connection, deletedBy: String, ids: Seq[String]): Unit = { + def deleteAllByIds(c: Connection, deletedBy: UUID, ids: Seq[String]): Unit = { setJournalDeletedByUserId(c, deletedBy) Query("delete from tasks") .in("id", ids) @@ -489,13 +490,13 @@ class TasksDao @Inject() ( () } - def deleteAllByNumAttempts(deletedBy: String, numAttempts: Int): Unit = { + def deleteAllByNumAttempts(deletedBy: UUID, numAttempts: Int): Unit = { db.withConnection { c => deleteAllByNumAttempts(c, deletedBy, numAttempts) } } - def deleteAllByNumAttempts(c: Connection, deletedBy: String, numAttempts: Int): Unit = { + def deleteAllByNumAttempts(c: Connection, deletedBy: UUID, numAttempts: Int): Unit = { setJournalDeletedByUserId(c, deletedBy) Query("delete from tasks") .equals("num_attempts", numAttempts) @@ -503,13 +504,13 @@ class TasksDao @Inject() ( () } - def deleteAllByNumAttemptses(deletedBy: String, numAttemptses: Seq[Int]): Unit = { + def deleteAllByNumAttemptses(deletedBy: UUID, numAttemptses: Seq[Int]): Unit = { db.withConnection { c => deleteAllByNumAttemptses(c, deletedBy, numAttemptses) } } - def deleteAllByNumAttemptses(c: Connection, deletedBy: String, numAttemptses: Seq[Int]): Unit = { + def deleteAllByNumAttemptses(c: Connection, deletedBy: UUID, numAttemptses: Seq[Int]): Unit = { setJournalDeletedByUserId(c, deletedBy) Query("delete from tasks") .in("num_attempts", numAttemptses) @@ -517,13 +518,13 @@ class TasksDao @Inject() ( () } - def deleteAllByNumAttemptsAndNextAttemptAt(deletedBy: String, numAttempts: Int, nextAttemptAt: DateTime): Unit = { + def deleteAllByNumAttemptsAndNextAttemptAt(deletedBy: UUID, numAttempts: Int, nextAttemptAt: DateTime): Unit = { db.withConnection { c => deleteAllByNumAttemptsAndNextAttemptAt(c, deletedBy, numAttempts, nextAttemptAt) } } - def deleteAllByNumAttemptsAndNextAttemptAt(c: Connection, deletedBy: String, numAttempts: Int, nextAttemptAt: DateTime): Unit = { + def deleteAllByNumAttemptsAndNextAttemptAt(c: Connection, deletedBy: UUID, numAttempts: Int, nextAttemptAt: DateTime): Unit = { setJournalDeletedByUserId(c, deletedBy) Query("delete from tasks") .equals("num_attempts", numAttempts) @@ -532,13 +533,13 @@ class TasksDao @Inject() ( () } - def deleteAllByNumAttemptsAndNextAttemptAts(deletedBy: String, numAttempts: Int, nextAttemptAts: Seq[DateTime]): Unit = { + def deleteAllByNumAttemptsAndNextAttemptAts(deletedBy: UUID, numAttempts: Int, nextAttemptAts: Seq[DateTime]): Unit = { db.withConnection { c => deleteAllByNumAttemptsAndNextAttemptAts(c, deletedBy, numAttempts, nextAttemptAts) } } - def deleteAllByNumAttemptsAndNextAttemptAts(c: Connection, deletedBy: String, numAttempts: Int, nextAttemptAts: Seq[DateTime]): Unit = { + def deleteAllByNumAttemptsAndNextAttemptAts(c: Connection, deletedBy: UUID, numAttempts: Int, nextAttemptAts: Seq[DateTime]): Unit = { setJournalDeletedByUserId(c, deletedBy) Query("delete from tasks") .equals("num_attempts", numAttempts) @@ -547,13 +548,13 @@ class TasksDao @Inject() ( () } - def deleteAllByTypeId(deletedBy: String, typeId: String): Unit = { + def deleteAllByTypeId(deletedBy: UUID, typeId: String): Unit = { db.withConnection { c => deleteAllByTypeId(c, deletedBy, typeId) } } - def deleteAllByTypeId(c: Connection, deletedBy: String, typeId: String): Unit = { + def deleteAllByTypeId(c: Connection, deletedBy: UUID, typeId: String): Unit = { setJournalDeletedByUserId(c, deletedBy) Query("delete from tasks") .equals("type_id", typeId) @@ -561,13 +562,13 @@ class TasksDao @Inject() ( () } - def deleteAllByTypeIds(deletedBy: String, typeIds: Seq[String]): Unit = { + def deleteAllByTypeIds(deletedBy: UUID, typeIds: Seq[String]): Unit = { db.withConnection { c => deleteAllByTypeIds(c, deletedBy, typeIds) } } - def deleteAllByTypeIds(c: Connection, deletedBy: String, typeIds: Seq[String]): Unit = { + def deleteAllByTypeIds(c: Connection, deletedBy: UUID, typeIds: Seq[String]): Unit = { setJournalDeletedByUserId(c, deletedBy) Query("delete from tasks") .in("type_id", typeIds) @@ -575,13 +576,13 @@ class TasksDao @Inject() ( () } - def deleteByTypeIdAndType(deletedBy: String, typeId: String, `type`: String): Unit = { + def deleteByTypeIdAndType(deletedBy: UUID, typeId: String, `type`: String): Unit = { db.withConnection { c => deleteByTypeIdAndType(c, deletedBy, typeId, `type`) } } - def deleteByTypeIdAndType(c: Connection, deletedBy: String, typeId: String, `type`: String): Unit = { + def deleteByTypeIdAndType(c: Connection, deletedBy: UUID, typeId: String, `type`: String): Unit = { setJournalDeletedByUserId(c, deletedBy) Query("delete from tasks") .equals("type_id", typeId) @@ -590,13 +591,13 @@ class TasksDao @Inject() ( () } - def deleteAllByTypeIdAndTypes(deletedBy: String, typeId: String, types: Seq[String]): Unit = { + def deleteAllByTypeIdAndTypes(deletedBy: UUID, typeId: String, types: Seq[String]): Unit = { db.withConnection { c => deleteAllByTypeIdAndTypes(c, deletedBy, typeId, types) } } - def deleteAllByTypeIdAndTypes(c: Connection, deletedBy: String, typeId: String, types: Seq[String]): Unit = { + def deleteAllByTypeIdAndTypes(c: Connection, deletedBy: UUID, typeId: String, types: Seq[String]): Unit = { setJournalDeletedByUserId(c, deletedBy) Query("delete from tasks") .equals("type_id", typeId) @@ -605,10 +606,7 @@ class TasksDao @Inject() ( () } - private[this] val ValidCharacters: Set[String] = "_-,.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789".split("").toSet - private[this] def isSafe(value: String): Boolean = value.trim.split("").forall(ValidCharacters.contains) - def setJournalDeletedByUserId(c: Connection, deletedBy: String): Unit = { - assert(isSafe(deletedBy), s"Value '${deletedBy}' contains unsafe characters") + def setJournalDeletedByUserId(c: Connection, deletedBy: UUID): Unit = { anorm.SQL(s"SET journal.deleted_by_user_id = '${deletedBy}'").executeUpdate()(c) () } diff --git a/api/app/processor/TaskActorCompanion.scala b/api/app/processor/TaskActorCompanion.scala new file mode 100644 index 000000000..22c7d5dc3 --- /dev/null +++ b/api/app/processor/TaskActorCompanion.scala @@ -0,0 +1,31 @@ +package processor + +import cats.implicits._ +import cats.data.ValidatedNec +import db.generated.Task + +import javax.inject.Inject + +class TaskActorCompanion @Inject() ( + noopProcessor: NoopProcessor +) { + + def process(typ: TaskType): Unit = { + lookup(typ).process() + } + + private[this] def lookup(typ: TaskType): BaseTaskProcessor = { + import TaskType._ + typ match { + case Noop => noopProcessor + } + } +} + +class NoopProcessor @Inject() ( + args: TaskProcessorArgs + ) extends BaseTaskProcessor(args, TaskType.Noop) { + override def processTask(task: Task): ValidatedNec[String, Unit] = { + ().validNec + } +} \ No newline at end of file diff --git a/api/app/processor/TaskDispatchActorCompanion.scala b/api/app/processor/TaskDispatchActorCompanion.scala new file mode 100644 index 000000000..a3e59d49c --- /dev/null +++ b/api/app/processor/TaskDispatchActorCompanion.scala @@ -0,0 +1,24 @@ +package processor + +import anorm.SqlParser +import io.flow.postgresql.Query +import play.api.db.Database + +import javax.inject.Inject + +class TaskDispatchActorCompanion @Inject() ( + database: Database +) { + private[this] val TypesQuery = Query( + "select distinct type from tasks where next_attempt_at <= now()" + ) + + def typesWithWork: Seq[TaskType] = { + database + .withConnection { c => + TypesQuery.as(SqlParser.str(1).*)(c) + } + .map(TaskType.fromString).flatMap(_.toOption) + } + +} diff --git a/api/app/processor/TaskProcessor.scala b/api/app/processor/TaskProcessor.scala new file mode 100644 index 000000000..8ac52240a --- /dev/null +++ b/api/app/processor/TaskProcessor.scala @@ -0,0 +1,156 @@ +package processor + +import cats.data.Validated.{Invalid, Valid} +import cats.data.ValidatedNec +import cats.implicits._ +import db.generated.{Task, TaskForm, TasksDao} +import io.flow.postgresql.OrderBy +import lib.Constants +import org.joda.time.DateTime +import play.api.libs.json.{JsObject, JsValue, Json, Reads} +import play.libs.exception.ExceptionUtils + +import java.sql.Connection +import java.util.UUID +import javax.inject.Inject +import scala.util.{Failure, Success, Try} + +class TaskProcessorArgs @Inject() ( + val dao: TasksDao +) {} + +abstract class TaskProcessor( + args: TaskProcessorArgs, + typ: TaskType +) extends BaseTaskProcessor(args, typ) { + def processRecord(id: String): ValidatedNec[String, Unit] + + override final def processTask(task: Task): ValidatedNec[String, Unit] = { + processRecord(task.typeId) + } + + final def queue(typeId: String, organizationGuid: Option[UUID]): Unit = { + args.dao.db.withConnection { c => + queue(c, typeId, organizationGuid = organizationGuid) + } + } + + final def queue(c: Connection, typeId: String, organizationGuid: Option[UUID]): Unit = { + insertIfNew(c, makeInitialTaskForm(typeId, organizationGuid, Json.obj())) + } +} + +abstract class TaskProcessorWithData[T]( + args: TaskProcessorArgs, + typ: TaskType +)(implicit reads: Reads[T]) + extends BaseTaskProcessor(args, typ) { + + def processRecord(id: String, data: T): ValidatedNec[String, Unit] + + override final def processTask(task: Task): ValidatedNec[String, Unit] = { + parseData(task.data).andThen { d => + processRecord(task.typeId, d) + } + } + + private[this] def parseData(data: JsValue): ValidatedNec[String, T] = { + data.asOpt[T] match { + case None => "Failed to parse data".invalidNec + case Some(instance) => instance.validNec + } + } + + final def queue(typeId: String, organizationGuid: Option[UUID], data: JsObject): Unit = { + args.dao.db.withConnection { c => + queue(c, typeId, organizationGuid, data) + } + } + + final def queue(c: Connection, typeId: String, organizationGuid: Option[UUID], data: JsObject): Unit = { + insertIfNew(c, makeInitialTaskForm(typeId, organizationGuid, data)) + } + +} + +abstract class BaseTaskProcessor( + args: TaskProcessorArgs, + typ: TaskType +) { + + private[this] val Limit = 100 + + final def process(): Unit = { + args.dao + .findAll( + `type` = Some(typ.toString), + nextAttemptAtLessThanOrEquals = Some(DateTime.now), + limit = Some(Limit), + orderBy = Some(OrderBy("num_attempts, next_attempt_at")) + ) + .foreach(processRecordSafe) + } + + def processTask(task: Task): ValidatedNec[String, Unit] + + private[this] def processRecordSafe(task: Task): Unit = { + Try { + processTask(task) + } match { + case Success(r) => + r match { + case Invalid(errors) => { + setErrors(task, errors.toList, stacktrace = None) + } + case Valid(_) => args.dao.delete(Constants.DefaultUserGuid, task) + } + case Failure(e) => { + setErrors(task, Seq("ERROR: " + e.getMessage), stacktrace = Some(ExceptionUtils.getStackTrace(e))) + } + } + } + + private[this] def setErrors(task: Task, errors: Seq[String], stacktrace: Option[String]): Unit = { + val numAttempts = task.numAttempts + 1 + + args.dao.update( + Constants.DefaultUserGuid, + task, + task.form.copy( + numAttempts = numAttempts, + nextAttemptAt = computeNextAttemptAt(task), + errors = Some(errors.distinct), + stacktrace = stacktrace + ) + ) + } + + protected def computeNextAttemptAt(task: Task): DateTime = { + DateTime.now.plusMinutes(5 * (task.numAttempts + 1)) + } + + final protected def makeInitialTaskForm( + typeId: String, + organizationGuid: Option[UUID], + data: JsObject + ): TaskForm = { + TaskForm( + id = s"$typ:$typeId", + `type` = typ.toString, + typeId = typeId, + organizationGuid = organizationGuid, + data = data, + numAttempts = 0, + nextAttemptAt = DateTime.now, + errors = None, + stacktrace = None + ) + } + + final protected def insertIfNew(c: Connection, form: TaskForm): Unit = { + if (args.dao.findByTypeIdAndTypeWithConnection(c, form.typeId, form.`type`).isEmpty) { + args.dao.upsertByTypeIdAndType(c, Constants.DefaultUserGuid, form) + } + } + +} diff --git a/api/app/processor/TaskType.scala b/api/app/processor/TaskType.scala new file mode 100644 index 000000000..3bdd34313 --- /dev/null +++ b/api/app/processor/TaskType.scala @@ -0,0 +1,15 @@ +package processor + +import cats.implicits._ +import cats.data.ValidatedNec + +sealed trait TaskType +object TaskType { + case object Noop extends TaskType { override def toString = "noop" } + val all: Seq[TaskType] = Seq(Noop) + + private[this] val byString = all.map { t => t.toString.toLowerCase -> t }.toMap + def fromString(value: String): ValidatedNec[String, TaskType] = { + byString.get(value.trim.toLowerCase()).toValidNec(s"Invalid task type '$value'") + } +} \ No newline at end of file diff --git a/api/conf/base.conf b/api/conf/base.conf index d5042e19f..7698a3b26 100644 --- a/api/conf/base.conf +++ b/api/conf/base.conf @@ -46,6 +46,20 @@ main-actor-context { } } +task-context { + fork-join-executor { + parallelism-factor = 2.0 + parallelism-max = 5 + } +} + +task-context-dispatcher { + fork-join-executor { + parallelism-factor = 2.0 + parallelism-max = 2 + } +} + email-actor-context { fork-join-executor { parallelism-factor = 2.0 diff --git a/api/test/actors/TaskActorSpec.scala b/api/test/actors/TaskActorSpec.scala new file mode 100644 index 000000000..de664066d --- /dev/null +++ b/api/test/actors/TaskActorSpec.scala @@ -0,0 +1,15 @@ +package actors + +import helpers.{AsyncHelpers, DbHelpers, DefaultAppSpec, ExportHelpers} +import util.DbAuthorization + +class TaskActorSpec extends DefaultAppSpec with AsyncHelpers with ExportHelpers with DbHelpers { + + "processes export" in { + val exp = createDefaultExport() + eventuallyInNSeconds(10) { + internalExportsDao.findById(DbAuthorization.All, exp.id).value.db.processedAt.value + } + } + +} diff --git a/dao/spec/psql-apibuilder.json b/dao/spec/psql-apibuilder.json index b6ec0e2d0..6115a9571 100644 --- a/dao/spec/psql-apibuilder.json +++ b/dao/spec/psql-apibuilder.json @@ -77,7 +77,7 @@ { "name": "id", "type": "string" }, { "name": "type", "type": "string" }, { "name": "type_id", "type": "string" }, - { "name": "organization_guid", "type": "string", "required": false }, + { "name": "organization_guid", "type": "uuid", "required": false }, { "name": "num_attempts", "type": "integer", "minimum": 0, "default": 0 }, { "name": "next_attempt_at", "type": "date-time-iso8601" }, { "name": "errors", "type": "[string]", "required": false }, @@ -89,7 +89,7 @@ "name": "scala", "value": { "package": "db.generated", - "dao_user_class": "String", + "dao_user_class": "java.util.UUID", "order_by": { "optional": true } } }, From 2ba4a566cc888fbd8b2e0d2f579547cf85997015 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:26:20 -0700 Subject: [PATCH 03/32] wip --- .apibuilder/config | 5 - api/app/db/ApplicationsDao.scala | 4 +- api/app/db/InternalTasksDao.scala | 52 ++++ ...ctiveApibuilderInternalV0Conversions.scala | 92 ------ ...ollectiveApibuilderInternalV0Parsers.scala | 97 ------- ...collectiveApibuilderInternalV0Models.scala | 263 ------------------ spec/apibuilder-internal.json | 51 ---- 7 files changed, 54 insertions(+), 510 deletions(-) create mode 100644 api/app/db/InternalTasksDao.scala delete mode 100644 api/app/generated/ApicollectiveApibuilderInternalV0Conversions.scala delete mode 100644 api/app/generated/ApicollectiveApibuilderInternalV0Parsers.scala delete mode 100644 generated/app/ApicollectiveApibuilderInternalV0Models.scala delete mode 100644 spec/apibuilder-internal.json diff --git a/.apibuilder/config b/.apibuilder/config index bdb07c04f..cfc091d56 100644 --- a/.apibuilder/config +++ b/.apibuilder/config @@ -27,8 +27,3 @@ code: play_2_8_client: generated/app play_2_8_mock_client: generated/app anorm_2_8_parsers: api/app/generated - apibuilder-internal: - version: latest - generators: - play_2_x_json: generated/app - anorm_2_8_parsers: api/app/generated diff --git a/api/app/db/ApplicationsDao.scala b/api/app/db/ApplicationsDao.scala index fad67096e..613c85c6a 100644 --- a/api/app/db/ApplicationsDao.scala +++ b/api/app/db/ApplicationsDao.scala @@ -15,7 +15,7 @@ class ApplicationsDao @Inject() ( @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, organizationsDao: OrganizationsDao, - tasksDao: TasksDao + tasksDao: InternalTasksDao ) { private[this] val dbHelpers = DbHelpers(db, "applications") @@ -365,7 +365,7 @@ class ApplicationsDao @Inject() ( ): Unit = { val taskGuid = db.withTransaction { implicit c => f(c) - tasksDao.insert(c, user, TaskDataIndexApplication(guid)) + tasksDao.queueWithConnection(c, user, TaskDataIndexApplication(guid)) } mainActor ! actors.MainActor.Messages.TaskCreated(taskGuid) } diff --git a/api/app/db/InternalTasksDao.scala b/api/app/db/InternalTasksDao.scala new file mode 100644 index 000000000..05249c4f1 --- /dev/null +++ b/api/app/db/InternalTasksDao.scala @@ -0,0 +1,52 @@ +package db + +import db.generated.{Task, TaskForm} +import lib.Constants +import org.joda.time.DateTime +import play.api.libs.json.{JsValue, Json} +import processor.TaskType + +import java.sql.Connection +import java.util.UUID +import javax.inject.Inject + +class InternalTasksDao @Inject() ( + dao: generated.TasksDao +) { + def findByTypeIdAndType(typeId: String, typ: TaskType): Option[Task] = { + dao.findByTypeIdAndType(typeId, typ.toString) + } + + def queue(typ: TaskType, id: String, organizationGuid: Option[UUID], data: JsValue = Json.obj()): Unit = { + dao.db.withConnection { c => + queueWithConnection(c, typ, id, organizationGuid = organizationGuid, data = data) + } + } + + def queueWithConnection( + c: Connection, + typ: TaskType, + id: String, + organizationGuid: Option[UUID], + data: JsValue = Json.obj() + ): Unit = { + if (dao.findByTypeIdAndTypeWithConnection(c, id, typ.toString).isEmpty) { + dao.upsertByTypeIdAndType( + c, + Constants.DefaultUserGuid, + TaskForm( + id = s"$typ:$id", + `type` = typ.toString, + typeId = id, + organizationGuid = organizationGuid, + data = data, + numAttempts = 0, + nextAttemptAt = DateTime.now, + errors = None, + stacktrace = None + ) + ) + } + } + +} diff --git a/api/app/generated/ApicollectiveApibuilderInternalV0Conversions.scala b/api/app/generated/ApicollectiveApibuilderInternalV0Conversions.scala deleted file mode 100644 index 79c8f0e40..000000000 --- a/api/app/generated/ApicollectiveApibuilderInternalV0Conversions.scala +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Generated by API Builder - https://www.apibuilder.io - * Service version: 0.15.81 - * apibuilder 0.15.33 app.apibuilder.io/apicollective/apibuilder-internal/latest/anorm_2_8_parsers - */ -package io.apibuilder.internal.v0.anorm.conversions { - - import anorm.{Column, MetaDataItem, TypeDoesNotMatch} - import play.api.libs.json.{JsArray, JsObject, JsValue} - import scala.util.{Failure, Success, Try} - import play.api.libs.json.JodaReads._ - - /** - * Conversions to collections of objects using JSON. - */ - object Util { - - def parser[T]( - f: play.api.libs.json.JsValue => T - ) = anorm.Column.nonNull { (value, meta) => - val MetaDataItem(columnName, nullable, clazz) = meta - value match { - case json: org.postgresql.util.PGobject => parseJson(f, columnName.qualified, json.getValue) - case json: java.lang.String => parseJson(f, columnName.qualified, json) - case _=> { - Left( - TypeDoesNotMatch( - s"Column[${columnName.qualified}] error converting $value to Json. Expected instance of type[org.postgresql.util.PGobject] and not[${value.asInstanceOf[AnyRef].getClass}]" - ) - ) - } - - - } - } - - private[this] def parseJson[T](f: play.api.libs.json.JsValue => T, columnName: String, value: String) = { - Try { - f( - play.api.libs.json.Json.parse(value) - ) - } match { - case Success(result) => Right(result) - case Failure(ex) => Left( - TypeDoesNotMatch( - s"Column[$columnName] error parsing json $value: $ex" - ) - ) - } - } - - } - - object Types { - import io.apibuilder.internal.v0.models.json._ - implicit val columnToSeqApibuilderInternalTask: Column[Seq[_root_.io.apibuilder.internal.v0.models.Task]] = Util.parser { _.as[Seq[_root_.io.apibuilder.internal.v0.models.Task]] } - implicit val columnToMapApibuilderInternalTask: Column[Map[String, _root_.io.apibuilder.internal.v0.models.Task]] = Util.parser { _.as[Map[String, _root_.io.apibuilder.internal.v0.models.Task]] } - implicit val columnToSeqApibuilderInternalTaskDataDiffVersion: Column[Seq[_root_.io.apibuilder.internal.v0.models.TaskDataDiffVersion]] = Util.parser { _.as[Seq[_root_.io.apibuilder.internal.v0.models.TaskDataDiffVersion]] } - implicit val columnToMapApibuilderInternalTaskDataDiffVersion: Column[Map[String, _root_.io.apibuilder.internal.v0.models.TaskDataDiffVersion]] = Util.parser { _.as[Map[String, _root_.io.apibuilder.internal.v0.models.TaskDataDiffVersion]] } - implicit val columnToSeqApibuilderInternalTaskDataIndexApplication: Column[Seq[_root_.io.apibuilder.internal.v0.models.TaskDataIndexApplication]] = Util.parser { _.as[Seq[_root_.io.apibuilder.internal.v0.models.TaskDataIndexApplication]] } - implicit val columnToMapApibuilderInternalTaskDataIndexApplication: Column[Map[String, _root_.io.apibuilder.internal.v0.models.TaskDataIndexApplication]] = Util.parser { _.as[Map[String, _root_.io.apibuilder.internal.v0.models.TaskDataIndexApplication]] } - implicit val columnToSeqApibuilderInternalTaskData: Column[Seq[_root_.io.apibuilder.internal.v0.models.TaskData]] = Util.parser { _.as[Seq[_root_.io.apibuilder.internal.v0.models.TaskData]] } - implicit val columnToMapApibuilderInternalTaskData: Column[Map[String, _root_.io.apibuilder.internal.v0.models.TaskData]] = Util.parser { _.as[Map[String, _root_.io.apibuilder.internal.v0.models.TaskData]] } - } - - object Standard { - implicit val columnToJsObject: Column[play.api.libs.json.JsObject] = Util.parser { _.as[play.api.libs.json.JsObject] } - implicit val columnToSeqBoolean: Column[Seq[Boolean]] = Util.parser { _.as[Seq[Boolean]] } - implicit val columnToMapBoolean: Column[Map[String, Boolean]] = Util.parser { _.as[Map[String, Boolean]] } - implicit val columnToSeqDouble: Column[Seq[Double]] = Util.parser { _.as[Seq[Double]] } - implicit val columnToMapDouble: Column[Map[String, Double]] = Util.parser { _.as[Map[String, Double]] } - implicit val columnToSeqInt: Column[Seq[Int]] = Util.parser { _.as[Seq[Int]] } - implicit val columnToMapInt: Column[Map[String, Int]] = Util.parser { _.as[Map[String, Int]] } - implicit val columnToSeqLong: Column[Seq[Long]] = Util.parser { _.as[Seq[Long]] } - implicit val columnToMapLong: Column[Map[String, Long]] = Util.parser { _.as[Map[String, Long]] } - implicit val columnToSeqLocalDate: Column[Seq[_root_.org.joda.time.LocalDate]] = Util.parser { _.as[Seq[_root_.org.joda.time.LocalDate]] } - implicit val columnToMapLocalDate: Column[Map[String, _root_.org.joda.time.LocalDate]] = Util.parser { _.as[Map[String, _root_.org.joda.time.LocalDate]] } - implicit val columnToSeqDateTime: Column[Seq[_root_.org.joda.time.DateTime]] = Util.parser { _.as[Seq[_root_.org.joda.time.DateTime]] } - implicit val columnToMapDateTime: Column[Map[String, _root_.org.joda.time.DateTime]] = Util.parser { _.as[Map[String, _root_.org.joda.time.DateTime]] } - implicit val columnToSeqBigDecimal: Column[Seq[BigDecimal]] = Util.parser { _.as[Seq[BigDecimal]] } - implicit val columnToMapBigDecimal: Column[Map[String, BigDecimal]] = Util.parser { _.as[Map[String, BigDecimal]] } - implicit val columnToSeqJsObject: Column[Seq[_root_.play.api.libs.json.JsObject]] = Util.parser { _.as[Seq[_root_.play.api.libs.json.JsObject]] } - implicit val columnToMapJsObject: Column[Map[String, _root_.play.api.libs.json.JsObject]] = Util.parser { _.as[Map[String, _root_.play.api.libs.json.JsObject]] } - implicit val columnToSeqJsValue: Column[Seq[_root_.play.api.libs.json.JsValue]] = Util.parser { _.as[Seq[_root_.play.api.libs.json.JsValue]] } - implicit val columnToMapJsValue: Column[Map[String, _root_.play.api.libs.json.JsValue]] = Util.parser { _.as[Map[String, _root_.play.api.libs.json.JsValue]] } - implicit val columnToSeqString: Column[Seq[String]] = Util.parser { _.as[Seq[String]] } - implicit val columnToMapString: Column[Map[String, String]] = Util.parser { _.as[Map[String, String]] } - implicit val columnToSeqUUID: Column[Seq[_root_.java.util.UUID]] = Util.parser { _.as[Seq[_root_.java.util.UUID]] } - implicit val columnToMapUUID: Column[Map[String, _root_.java.util.UUID]] = Util.parser { _.as[Map[String, _root_.java.util.UUID]] } - } - -} \ No newline at end of file diff --git a/api/app/generated/ApicollectiveApibuilderInternalV0Parsers.scala b/api/app/generated/ApicollectiveApibuilderInternalV0Parsers.scala deleted file mode 100644 index 9177e546b..000000000 --- a/api/app/generated/ApicollectiveApibuilderInternalV0Parsers.scala +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Generated by API Builder - https://www.apibuilder.io - * Service version: 0.15.81 - * apibuilder 0.15.33 app.apibuilder.io/apicollective/apibuilder-internal/latest/anorm_2_8_parsers - */ -import anorm._ - -package io.apibuilder.internal.v0.anorm.parsers { - - import io.apibuilder.internal.v0.anorm.conversions.Standard._ - - import io.apibuilder.internal.v0.anorm.conversions.Types._ - - object Task { - - def parserWithPrefix(prefix: String, sep: String = "_"): RowParser[io.apibuilder.internal.v0.models.Task] = parser(prefixOpt = Some(s"$prefix$sep")) - - def parser( - guid: String = "guid", - dataPrefix: String = "data", - numberAttempts: String = "number_attempts", - lastError: String = "last_error", - prefixOpt: Option[String] = None - ): RowParser[io.apibuilder.internal.v0.models.Task] = { - SqlParser.get[_root_.java.util.UUID](prefixOpt.getOrElse("") + guid) ~ - io.apibuilder.internal.v0.anorm.parsers.TaskData.parserWithPrefix(prefixOpt.getOrElse("") + dataPrefix) ~ - SqlParser.long(prefixOpt.getOrElse("") + numberAttempts) ~ - SqlParser.str(prefixOpt.getOrElse("") + lastError).? map { - case guid ~ data ~ numberAttempts ~ lastError => { - io.apibuilder.internal.v0.models.Task( - guid = guid, - data = data, - numberAttempts = numberAttempts, - lastError = lastError - ) - } - } - } - - } - - object TaskDataDiffVersion { - - def parserWithPrefix(prefix: String, sep: String = "_"): RowParser[io.apibuilder.internal.v0.models.TaskDataDiffVersion] = parser(prefixOpt = Some(s"$prefix$sep")) - - def parser( - oldVersionGuid: String = "old_version_guid", - newVersionGuid: String = "new_version_guid", - prefixOpt: Option[String] = None - ): RowParser[io.apibuilder.internal.v0.models.TaskDataDiffVersion] = { - SqlParser.get[_root_.java.util.UUID](prefixOpt.getOrElse("") + oldVersionGuid) ~ - SqlParser.get[_root_.java.util.UUID](prefixOpt.getOrElse("") + newVersionGuid) map { - case oldVersionGuid ~ newVersionGuid => { - io.apibuilder.internal.v0.models.TaskDataDiffVersion( - oldVersionGuid = oldVersionGuid, - newVersionGuid = newVersionGuid - ) - } - } - } - - } - - object TaskDataIndexApplication { - - def parserWithPrefix(prefix: String, sep: String = "_"): RowParser[io.apibuilder.internal.v0.models.TaskDataIndexApplication] = parser(prefixOpt = Some(s"$prefix$sep")) - - def parser( - applicationGuid: String = "application_guid", - prefixOpt: Option[String] = None - ): RowParser[io.apibuilder.internal.v0.models.TaskDataIndexApplication] = { - SqlParser.get[_root_.java.util.UUID](prefixOpt.getOrElse("") + applicationGuid) map { - case applicationGuid => { - io.apibuilder.internal.v0.models.TaskDataIndexApplication( - applicationGuid = applicationGuid - ) - } - } - } - - } - - object TaskData { - - def parserWithPrefix(prefix: String, sep: String = "_") = { - io.apibuilder.internal.v0.anorm.parsers.TaskDataIndexApplication.parser(prefixOpt = Some(s"$prefix$sep")) | - io.apibuilder.internal.v0.anorm.parsers.TaskDataDiffVersion.parser(prefixOpt = Some(s"$prefix$sep")) - } - - def parser() = { - io.apibuilder.internal.v0.anorm.parsers.TaskDataIndexApplication.parser() | - io.apibuilder.internal.v0.anorm.parsers.TaskDataDiffVersion.parser() - } - - } - -} \ No newline at end of file diff --git a/generated/app/ApicollectiveApibuilderInternalV0Models.scala b/generated/app/ApicollectiveApibuilderInternalV0Models.scala deleted file mode 100644 index 3255d2e0d..000000000 --- a/generated/app/ApicollectiveApibuilderInternalV0Models.scala +++ /dev/null @@ -1,263 +0,0 @@ -/** - * Generated by API Builder - https://www.apibuilder.io - * Service version: 0.15.81 - * apibuilder 0.15.33 app.apibuilder.io/apicollective/apibuilder-internal/latest/play_2_x_json - */ -package io.apibuilder.internal.v0.models { - - sealed trait TaskData extends _root_.scala.Product with _root_.scala.Serializable - /** - * @param numberAttempts Records the number of times we have attempted to run this task. Commonly we - * increment number attempts, process the task, and if succeeds we then delete the - * task. If it fails, we update last_error. This allows us to retry a task say - * twice; after which we no longer process the task (can notify an admin of the - * error). - */ - - final case class Task( - guid: _root_.java.util.UUID, - data: io.apibuilder.internal.v0.models.TaskData, - numberAttempts: Long = 0L, - lastError: _root_.scala.Option[String] = None - ) - - final case class TaskDataDiffVersion( - oldVersionGuid: _root_.java.util.UUID, - newVersionGuid: _root_.java.util.UUID - ) extends TaskData - - final case class TaskDataIndexApplication( - applicationGuid: _root_.java.util.UUID - ) extends TaskData - - /** - * Provides future compatibility in clients - in the future, when a type is added - * to the union TaskData, it will need to be handled in the client code. This - * implementation will deserialize these future types as an instance of this class. - * - * @param description Information about the type that we received that is undefined in this version of - * the client. - */ - - final case class TaskDataUndefinedType( - description: String - ) extends TaskData - -} - -package io.apibuilder.internal.v0.models { - - package object json { - import play.api.libs.json.__ - import play.api.libs.json.JsString - import play.api.libs.json.Writes - import play.api.libs.functional.syntax._ - import io.apibuilder.internal.v0.models.json._ - - private[v0] implicit val jsonReadsUUID: play.api.libs.json.Reads[_root_.java.util.UUID] = __.read[String].map { str => - _root_.java.util.UUID.fromString(str) - } - - private[v0] implicit val jsonWritesUUID: play.api.libs.json.Writes[_root_.java.util.UUID] = (x: _root_.java.util.UUID) => play.api.libs.json.JsString(x.toString) - - private[v0] implicit val jsonReadsJodaDateTime: play.api.libs.json.Reads[_root_.org.joda.time.DateTime] = __.read[String].map { str => - _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseDateTime(str) - } - - private[v0] implicit val jsonWritesJodaDateTime: play.api.libs.json.Writes[_root_.org.joda.time.DateTime] = (x: _root_.org.joda.time.DateTime) => { - play.api.libs.json.JsString(_root_.org.joda.time.format.ISODateTimeFormat.dateTime.print(x)) - } - - private[v0] implicit val jsonReadsJodaLocalDate: play.api.libs.json.Reads[_root_.org.joda.time.LocalDate] = __.read[String].map { str => - _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseLocalDate(str) - } - - private[v0] implicit val jsonWritesJodaLocalDate: play.api.libs.json.Writes[_root_.org.joda.time.LocalDate] = (x: _root_.org.joda.time.LocalDate) => { - play.api.libs.json.JsString(_root_.org.joda.time.format.ISODateTimeFormat.date.print(x)) - } - - implicit def jsonReadsApibuilderInternalTask: play.api.libs.json.Reads[Task] = { - for { - guid <- (__ \ "guid").read[_root_.java.util.UUID] - data <- (__ \ "data").read[io.apibuilder.internal.v0.models.TaskData] - numberAttempts <- (__ \ "number_attempts").read[Long] - lastError <- (__ \ "last_error").readNullable[String] - } yield Task(guid, data, numberAttempts, lastError) - } - - def jsObjectTask(obj: io.apibuilder.internal.v0.models.Task): play.api.libs.json.JsObject = { - play.api.libs.json.Json.obj( - "guid" -> play.api.libs.json.JsString(obj.guid.toString), - "data" -> jsObjectTaskData(obj.data), - "number_attempts" -> play.api.libs.json.JsNumber(obj.numberAttempts) - ) ++ (obj.lastError match { - case None => play.api.libs.json.Json.obj() - case Some(x) => play.api.libs.json.Json.obj("last_error" -> play.api.libs.json.JsString(x)) - }) - } - - implicit def jsonWritesApibuilderInternalTask: play.api.libs.json.Writes[Task] = { - (obj: io.apibuilder.internal.v0.models.Task) => { - jsObjectTask(obj) - } - } - - implicit def jsonReadsApibuilderInternalTaskDataDiffVersion: play.api.libs.json.Reads[TaskDataDiffVersion] = { - for { - oldVersionGuid <- (__ \ "old_version_guid").read[_root_.java.util.UUID] - newVersionGuid <- (__ \ "new_version_guid").read[_root_.java.util.UUID] - } yield TaskDataDiffVersion(oldVersionGuid, newVersionGuid) - } - - def jsObjectTaskDataDiffVersion(obj: io.apibuilder.internal.v0.models.TaskDataDiffVersion): play.api.libs.json.JsObject = { - play.api.libs.json.Json.obj( - "old_version_guid" -> play.api.libs.json.JsString(obj.oldVersionGuid.toString), - "new_version_guid" -> play.api.libs.json.JsString(obj.newVersionGuid.toString) - ) - } - - implicit def jsonWritesApibuilderInternalTaskDataDiffVersion: play.api.libs.json.Writes[TaskDataDiffVersion] = { - (obj: io.apibuilder.internal.v0.models.TaskDataDiffVersion) => { - jsObjectTaskDataDiffVersion(obj) - } - } - - implicit def jsonReadsApibuilderInternalTaskDataIndexApplication: play.api.libs.json.Reads[TaskDataIndexApplication] = { - (__ \ "application_guid").read[_root_.java.util.UUID].map { x => new TaskDataIndexApplication(applicationGuid = x) } - } - - def jsObjectTaskDataIndexApplication(obj: io.apibuilder.internal.v0.models.TaskDataIndexApplication): play.api.libs.json.JsObject = { - play.api.libs.json.Json.obj( - "application_guid" -> play.api.libs.json.JsString(obj.applicationGuid.toString) - ) - } - - implicit def jsonWritesApibuilderInternalTaskDataIndexApplication: play.api.libs.json.Writes[TaskDataIndexApplication] = { - (obj: io.apibuilder.internal.v0.models.TaskDataIndexApplication) => { - jsObjectTaskDataIndexApplication(obj) - } - } - - implicit def jsonReadsApibuilderInternalTaskData: play.api.libs.json.Reads[TaskData] = { - ( - (__ \ "task_data_index_application").read(jsonReadsApibuilderInternalTaskDataIndexApplication).asInstanceOf[play.api.libs.json.Reads[TaskData]] - orElse - (__ \ "task_data_diff_version").read(jsonReadsApibuilderInternalTaskDataDiffVersion).asInstanceOf[play.api.libs.json.Reads[TaskData]] - orElse - play.api.libs.json.Reads(jsValue => play.api.libs.json.JsSuccess(TaskDataUndefinedType(jsValue.toString))).asInstanceOf[play.api.libs.json.Reads[TaskData]] - ) - } - - def jsObjectTaskData(obj: io.apibuilder.internal.v0.models.TaskData): play.api.libs.json.JsObject = { - obj match { - case x: io.apibuilder.internal.v0.models.TaskDataIndexApplication => play.api.libs.json.Json.obj("task_data_index_application" -> jsObjectTaskDataIndexApplication(x)) - case x: io.apibuilder.internal.v0.models.TaskDataDiffVersion => play.api.libs.json.Json.obj("task_data_diff_version" -> jsObjectTaskDataDiffVersion(x)) - case x: io.apibuilder.internal.v0.models.TaskDataUndefinedType => sys.error(s"The type[io.apibuilder.internal.v0.models.TaskDataUndefinedType] should never be serialized") - } - } - - implicit def jsonWritesApibuilderInternalTaskData: play.api.libs.json.Writes[TaskData] = { - (obj: io.apibuilder.internal.v0.models.TaskData) => { - jsObjectTaskData(obj) - } - } - } -} - -package io.apibuilder.internal.v0 { - - object Bindables { - - import play.api.mvc.{PathBindable, QueryStringBindable} - - // import models directly for backwards compatibility with prior versions of the generator - import Core._ - - object Core { - implicit def pathBindableDateTimeIso8601(implicit stringBinder: QueryStringBindable[String]): PathBindable[_root_.org.joda.time.DateTime] = ApibuilderPathBindable(ApibuilderTypes.dateTimeIso8601) - implicit def queryStringBindableDateTimeIso8601(implicit stringBinder: QueryStringBindable[String]): QueryStringBindable[_root_.org.joda.time.DateTime] = ApibuilderQueryStringBindable(ApibuilderTypes.dateTimeIso8601) - - implicit def pathBindableDateIso8601(implicit stringBinder: QueryStringBindable[String]): PathBindable[_root_.org.joda.time.LocalDate] = ApibuilderPathBindable(ApibuilderTypes.dateIso8601) - implicit def queryStringBindableDateIso8601(implicit stringBinder: QueryStringBindable[String]): QueryStringBindable[_root_.org.joda.time.LocalDate] = ApibuilderQueryStringBindable(ApibuilderTypes.dateIso8601) - } - - trait ApibuilderTypeConverter[T] { - - def convert(value: String): T - - def convert(value: T): String - - def example: T - - def validValues: Seq[T] = Nil - - def errorMessage(key: String, value: String, ex: java.lang.Exception): String = { - val base = s"Invalid value '$value' for parameter '$key'. " - validValues.toList match { - case Nil => base + "Ex: " + convert(example) - case values => base + ". Valid values are: " + values.mkString("'", "', '", "'") - } - } - } - - object ApibuilderTypes { - val dateTimeIso8601: ApibuilderTypeConverter[_root_.org.joda.time.DateTime] = new ApibuilderTypeConverter[_root_.org.joda.time.DateTime] { - override def convert(value: String): _root_.org.joda.time.DateTime = _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseDateTime(value) - override def convert(value: _root_.org.joda.time.DateTime): String = _root_.org.joda.time.format.ISODateTimeFormat.dateTime.print(value) - override def example: _root_.org.joda.time.DateTime = _root_.org.joda.time.DateTime.now - } - - val dateIso8601: ApibuilderTypeConverter[_root_.org.joda.time.LocalDate] = new ApibuilderTypeConverter[_root_.org.joda.time.LocalDate] { - override def convert(value: String): _root_.org.joda.time.LocalDate = _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseLocalDate(value) - override def convert(value: _root_.org.joda.time.LocalDate): String = _root_.org.joda.time.format.ISODateTimeFormat.date.print(value) - override def example: _root_.org.joda.time.LocalDate = _root_.org.joda.time.LocalDate.now - } - } - - final case class ApibuilderQueryStringBindable[T]( - converters: ApibuilderTypeConverter[T] - ) extends QueryStringBindable[T] { - - override def bind(key: String, params: Map[String, Seq[String]]): _root_.scala.Option[_root_.scala.Either[String, T]] = { - params.getOrElse(key, Nil).headOption.map { v => - try { - Right( - converters.convert(v) - ) - } catch { - case ex: java.lang.Exception => Left( - converters.errorMessage(key, v, ex) - ) - } - } - } - - override def unbind(key: String, value: T): String = { - s"$key=${converters.convert(value)}" - } - } - - final case class ApibuilderPathBindable[T]( - converters: ApibuilderTypeConverter[T] - ) extends PathBindable[T] { - - override def bind(key: String, value: String): _root_.scala.Either[String, T] = { - try { - Right( - converters.convert(value) - ) - } catch { - case ex: java.lang.Exception => Left( - converters.errorMessage(key, value, ex) - ) - } - } - - override def unbind(key: String, value: T): String = { - converters.convert(value) - } - } - - } - -} diff --git a/spec/apibuilder-internal.json b/spec/apibuilder-internal.json deleted file mode 100644 index dca2feda7..000000000 --- a/spec/apibuilder-internal.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "name": "apibuilder internal", - "description": "Internal models used in the implementation of apibuilder", - - "info": { - "contact": { - "name": "Michael Bryzek", - "email": "mbryzek@alum.mit.edu", - "url": "http://twitter.com/mbryzek" - }, - "license": { - "name": "MIT", - "url": "http://opensource.org/licenses/MIT" - } - }, - - "unions": { - "task_data": { - "types": [ - { "type": "task_data_index_application", "description": "Task to update the search index following a change in application" }, - { "type": "task_data_diff_version", "description": "Task triggered whenever a version changes. Diffs the service to record what actually changed" } - ] - } - }, - - "models": { - - "task": { - "fields": [ - { "name": "guid", "type": "uuid" }, - { "name": "data", "type": "task_data" }, - { "name": "number_attempts", "type": "long", "default": 0, "description": "Records the number of times we have attempted to run this task. Commonly we increment number attempts, process the task, and if succeeds we then delete the task. If it fails, we update last_error. This allows us to retry a task say twice; after which we no longer process the task (can notify an admin of the error)." }, - { "name": "last_error", "type": "string", "required": false } - ] - }, - - "task_data_index_application": { - "fields": [ - { "name": "application_guid", "type": "uuid" } - ] - }, - - "task_data_diff_version": { - "fields": [ - { "name": "old_version_guid", "type": "uuid" }, - { "name": "new_version_guid", "type": "uuid" } - ] - } - } - -} From 195d47536161c519add822209dba9ca19f5dc073 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:29:14 -0700 Subject: [PATCH 04/32] wip --- api/app/db/ApplicationsDao.scala | 3 ++- .../processor/IndexApplicationProcessor.scala | 22 +++++++++++++++++++ api/app/processor/TaskActorCompanion.scala | 12 ++-------- api/app/processor/TaskProcessor.scala | 9 ++++++++ api/app/processor/TaskType.scala | 2 +- 5 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 api/app/processor/IndexApplicationProcessor.scala diff --git a/api/app/db/ApplicationsDao.scala b/api/app/db/ApplicationsDao.scala index 613c85c6a..fda6f6e2c 100644 --- a/api/app/db/ApplicationsDao.scala +++ b/api/app/db/ApplicationsDao.scala @@ -6,6 +6,7 @@ import io.apibuilder.internal.v0.models.TaskDataIndexApplication import io.flow.postgresql.Query import lib.{UrlKey, Validation} import play.api.db._ +import processor.TaskType import java.util.UUID import javax.inject.{Inject, Named, Singleton} @@ -365,7 +366,7 @@ class ApplicationsDao @Inject() ( ): Unit = { val taskGuid = db.withTransaction { implicit c => f(c) - tasksDao.queueWithConnection(c, user, TaskDataIndexApplication(guid)) + tasksDao.queueWithConnection(c, TaskType.IndexApplication, guid.toString) } mainActor ! actors.MainActor.Messages.TaskCreated(taskGuid) } diff --git a/api/app/processor/IndexApplicationProcessor.scala b/api/app/processor/IndexApplicationProcessor.scala new file mode 100644 index 000000000..d500a5e9e --- /dev/null +++ b/api/app/processor/IndexApplicationProcessor.scala @@ -0,0 +1,22 @@ +package processor + +import cats.data.ValidatedNec +import cats.implicits._ +import db.generated.Task + +import java.util.UUID +import javax.inject.Inject + + +class IndexApplicationProcessor @Inject()( + args: TaskProcessorArgs, + ) extends BaseTaskProcessor(args, TaskType.Noop) { + + override def processTask(task: Task): ValidatedNec[String, Unit] = { + validateGuid(task.typeId).andThen(processApplicationGuid) + } + + def processApplicationGuid(guid: UUID): ValidatedNec[String, Unit] = { + + } +} \ No newline at end of file diff --git a/api/app/processor/TaskActorCompanion.scala b/api/app/processor/TaskActorCompanion.scala index 22c7d5dc3..b23b71a34 100644 --- a/api/app/processor/TaskActorCompanion.scala +++ b/api/app/processor/TaskActorCompanion.scala @@ -7,7 +7,7 @@ import db.generated.Task import javax.inject.Inject class TaskActorCompanion @Inject() ( - noopProcessor: NoopProcessor + indexApplication: IndexApplicationProcessor ) { def process(typ: TaskType): Unit = { @@ -17,15 +17,7 @@ class TaskActorCompanion @Inject() ( private[this] def lookup(typ: TaskType): BaseTaskProcessor = { import TaskType._ typ match { - case Noop => noopProcessor + case IndexApplication => indexApplication } } } - -class NoopProcessor @Inject() ( - args: TaskProcessorArgs - ) extends BaseTaskProcessor(args, TaskType.Noop) { - override def processTask(task: Task): ValidatedNec[String, Unit] = { - ().validNec - } -} \ No newline at end of file diff --git a/api/app/processor/TaskProcessor.scala b/api/app/processor/TaskProcessor.scala index 8ac52240a..b2545a3bc 100644 --- a/api/app/processor/TaskProcessor.scala +++ b/api/app/processor/TaskProcessor.scala @@ -129,6 +129,15 @@ abstract class BaseTaskProcessor( DateTime.now.plusMinutes(5 * (task.numAttempts + 1)) } + protected final def validateGuid(value: String): ValidatedNec[String, UUID] = { + Try { + UUID.fromString(value) + } match { + case Success(v) => v.validNec + case Failure(_) => s"Invalid guid '$value'".invalidNec + } + } + final protected def makeInitialTaskForm( typeId: String, organizationGuid: Option[UUID], diff --git a/api/app/processor/TaskType.scala b/api/app/processor/TaskType.scala index 3bdd34313..ddda7c55a 100644 --- a/api/app/processor/TaskType.scala +++ b/api/app/processor/TaskType.scala @@ -5,7 +5,7 @@ import cats.data.ValidatedNec sealed trait TaskType object TaskType { - case object Noop extends TaskType { override def toString = "noop" } + case object IndexApplication extends TaskType { override def toString = "index_application" } val all: Seq[TaskType] = Seq(Noop) private[this] val byString = all.map { t => t.toString.toLowerCase -> t }.toMap From 3c2fc04e2fb6fc26eff83d52345c37eb21c162e1 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:32:37 -0700 Subject: [PATCH 05/32] wip --- api/app/actors/Search.scala | 46 ------------------- .../processor/IndexApplicationProcessor.scala | 40 ++++++++++++++-- api/test/util/Daos.scala | 2 +- 3 files changed, 37 insertions(+), 51 deletions(-) delete mode 100644 api/app/actors/Search.scala diff --git a/api/app/actors/Search.scala b/api/app/actors/Search.scala deleted file mode 100644 index 4ad6b0228..000000000 --- a/api/app/actors/Search.scala +++ /dev/null @@ -1,46 +0,0 @@ -package actors - -import io.apibuilder.api.v0.models.{Application, ApplicationSummary, Organization} -import io.apibuilder.common.v0.models.Reference -import db.{ApplicationsDao, Authorization, ItemsDao, OrganizationsDao} -import java.util.UUID -import javax.inject.{Inject, Singleton} - -@Singleton -class Search @Inject() ( - applicationsDao: ApplicationsDao, - itemsDao: ItemsDao, - organizationsDao: OrganizationsDao -) { - - def indexApplication(applicationGuid: UUID): Unit = { - getInfo(applicationGuid) match { - case Some((org, app)) => { - val content = s"""${app.name} ${app.key} ${app.description.getOrElse("")}""".trim.toLowerCase - itemsDao.upsert( - guid = app.guid, - detail = ApplicationSummary( - guid = app.guid, - organization = Reference(guid = org.guid, key = org.key), - key = app.key - ), - label = s"${org.key}/${app.key}", - description = app.description, - content = content - ) - } - case None => { - itemsDao.delete(applicationGuid) - } - } - } - - private[this] def getInfo(applicationGuid: UUID): Option[(Organization, Application)] = { - applicationsDao.findByGuid(Authorization.All, applicationGuid).flatMap { application => - organizationsDao.findAll(Authorization.All, application = Some(application), limit = 1).headOption.map { org => - (org, application) - } - } - } - -} diff --git a/api/app/processor/IndexApplicationProcessor.scala b/api/app/processor/IndexApplicationProcessor.scala index d500a5e9e..68d486a38 100644 --- a/api/app/processor/IndexApplicationProcessor.scala +++ b/api/app/processor/IndexApplicationProcessor.scala @@ -1,8 +1,10 @@ package processor import cats.data.ValidatedNec -import cats.implicits._ import db.generated.Task +import db.{ApplicationsDao, Authorization, ItemsDao, OrganizationsDao} +import io.apibuilder.api.v0.models.{Application, ApplicationSummary, Organization} +import io.apibuilder.common.v0.models.Reference import java.util.UUID import javax.inject.Inject @@ -10,13 +12,43 @@ import javax.inject.Inject class IndexApplicationProcessor @Inject()( args: TaskProcessorArgs, - ) extends BaseTaskProcessor(args, TaskType.Noop) { + applicationsDao: ApplicationsDao, + itemsDao: ItemsDao, + organizationsDao: OrganizationsDao + + ) extends BaseTaskProcessor(args, TaskType.IndexApplication) { override def processTask(task: Task): ValidatedNec[String, Unit] = { - validateGuid(task.typeId).andThen(processApplicationGuid) + validateGuid(task.typeId).map(processApplicationGuid) } - def processApplicationGuid(guid: UUID): ValidatedNec[String, Unit] = { + def processApplicationGuid(applicationGuid: UUID): Unit = { + getInfo(applicationGuid) match { + case Some((org, app)) => { + val content = s"""${app.name} ${app.key} ${app.description.getOrElse("")}""".trim.toLowerCase + itemsDao.upsert( + guid = app.guid, + detail = ApplicationSummary( + guid = app.guid, + organization = Reference(guid = org.guid, key = org.key), + key = app.key + ), + label = s"${org.key}/${app.key}", + description = app.description, + content = content + ) + } + case None => { + itemsDao.delete(applicationGuid) + } + } + } + private[this] def getInfo(applicationGuid: UUID): Option[(Organization, Application)] = { + applicationsDao.findByGuid(Authorization.All, applicationGuid).flatMap { application => + organizationsDao.findAll(Authorization.All, application = Some(application), limit = 1).headOption.map { org => + (org, application) + } + } } } \ No newline at end of file diff --git a/api/test/util/Daos.scala b/api/test/util/Daos.scala index 651f38e22..91c7bff35 100644 --- a/api/test/util/Daos.scala +++ b/api/test/util/Daos.scala @@ -1,6 +1,6 @@ package util -import actors.{Emails, Search} +import actors.Emails import db.{ApplicationsDao, AttributesDao, ChangesDao, EmailVerificationsDao, ItemsDao, MembershipRequestsDao, MembershipsDao, OrganizationAttributeValuesDao, OrganizationDomainsDao, OrganizationLogsDao, OrganizationsDao, OriginalsDao, PasswordResetRequestsDao, SubscriptionsDao, TasksDao, TokensDao, UserPasswordsDao, UsersDao, VersionsDao} import db.generated.SessionsDao import db.generators.{GeneratorsDao, ServicesDao} From dade8870e5ad1cb1e09d2fc4fa9178643405d2ef Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:34:06 -0700 Subject: [PATCH 06/32] wip --- api/app/actors/MainActor.scala | 4 ---- api/app/db/ApplicationsDao.scala | 15 ++++++--------- api/app/db/InternalTasksDao.scala | 2 +- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/api/app/actors/MainActor.scala b/api/app/actors/MainActor.scala index 8f7be2e56..bdaf59076 100644 --- a/api/app/actors/MainActor.scala +++ b/api/app/actors/MainActor.scala @@ -77,10 +77,6 @@ class MainActor @javax.inject.Inject() ( emailActor ! EmailActor.Messages.ApplicationCreated(guid) } - case m @ MainActor.Messages.TaskCreated(guid) => withVerboseErrorHandler(m) { - taskActor ! TaskActor.Messages.Created(guid) - } - case m @ MainActor.Messages.GeneratorServiceCreated(guid) => withVerboseErrorHandler(m) { generatorServiceActor ! GeneratorServiceActorMessage.GeneratorServiceCreated(guid) } diff --git a/api/app/db/ApplicationsDao.scala b/api/app/db/ApplicationsDao.scala index fda6f6e2c..1859e5736 100644 --- a/api/app/db/ApplicationsDao.scala +++ b/api/app/db/ApplicationsDao.scala @@ -2,7 +2,6 @@ package db import anorm._ import io.apibuilder.api.v0.models.{AppSortBy, Application, ApplicationForm, Error, MoveForm, Organization, SortOrder, User, Version, Visibility} -import io.apibuilder.internal.v0.models.TaskDataIndexApplication import io.flow.postgresql.Query import lib.{UrlKey, Validation} import play.api.db._ @@ -164,7 +163,7 @@ class ApplicationsDao @Inject() ( val errors = validate(org, form, Some(app)) assert(errors.isEmpty, errors.map(_.message).mkString(" ")) - withTasks(updatedBy, app.guid, { implicit c => + withTasks(app.guid, { implicit c => SQL(UpdateQuery).on( "guid" -> app.guid, "name" -> form.name.trim, @@ -195,7 +194,7 @@ class ApplicationsDao @Inject() ( organizationsDao.findByKey(Authorization.All, form.orgKey) match { case None => sys.error(s"Could not find organization with key[${form.orgKey}]") case Some(newOrg) => { - withTasks(updatedBy, app.guid, { implicit c => + withTasks(app.guid, { implicit c => SQL(InsertMoveQuery).on( "guid" -> UUID.randomUUID, "application_guid" -> app.guid, @@ -229,7 +228,7 @@ class ApplicationsDao @Inject() ( "User[${user.guid}] not authorized to update app[${app.key}]" ) - withTasks(updatedBy, app.guid, { implicit c => + withTasks(app.guid, { implicit c => SQL(UpdateVisibilityQuery).on( "guid" -> app.guid, "visibility" -> visibility.toString, @@ -253,7 +252,7 @@ class ApplicationsDao @Inject() ( val guid = UUID.randomUUID val key = form.key.getOrElse(UrlKey.generate(form.name)) - withTasks(createdBy, guid, { implicit c => + withTasks(guid, { implicit c => SQL(InsertQuery).on( "guid" -> guid, "organization_guid" -> org.guid, @@ -274,7 +273,7 @@ class ApplicationsDao @Inject() ( } def softDelete(deletedBy: User, application: Application): Unit = { - withTasks(deletedBy, application.guid, { c => + withTasks(application.guid, { c => dbHelpers.delete(c, deletedBy.guid, application.guid) }) } @@ -360,15 +359,13 @@ class ApplicationsDao @Inject() ( } private[this] def withTasks( - user: User, guid: UUID, f: java.sql.Connection => Unit ): Unit = { - val taskGuid = db.withTransaction { implicit c => + db.withTransaction { implicit c => f(c) tasksDao.queueWithConnection(c, TaskType.IndexApplication, guid.toString) } - mainActor ! actors.MainActor.Messages.TaskCreated(taskGuid) } } \ No newline at end of file diff --git a/api/app/db/InternalTasksDao.scala b/api/app/db/InternalTasksDao.scala index 05249c4f1..d1eace52f 100644 --- a/api/app/db/InternalTasksDao.scala +++ b/api/app/db/InternalTasksDao.scala @@ -27,7 +27,7 @@ class InternalTasksDao @Inject() ( c: Connection, typ: TaskType, id: String, - organizationGuid: Option[UUID], + organizationGuid: Option[UUID] = None, data: JsValue = Json.obj() ): Unit = { if (dao.findByTypeIdAndTypeWithConnection(c, id, typ.toString).isEmpty) { From f20c038ab60e502891104777ad1e655114834840 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:39:54 -0700 Subject: [PATCH 07/32] wip --- .apibuilder/.tracked_files | 10 +- .apibuilder/config | 6 + api/app/db/VersionsDao.scala | 8 +- .../processor/IndexApplicationProcessor.scala | 1 - .../ApicollectiveApibuilderTaskV0Client.scala | 309 ++++++++++++++++++ ...collectiveApibuilderTaskV0MockClient.scala | 21 ++ spec/apibuilder-task.json | 24 ++ 7 files changed, 370 insertions(+), 9 deletions(-) create mode 100644 generated/app/ApicollectiveApibuilderTaskV0Client.scala create mode 100644 generated/app/ApicollectiveApibuilderTaskV0MockClient.scala create mode 100644 spec/apibuilder-task.json diff --git a/.apibuilder/.tracked_files b/.apibuilder/.tracked_files index d2c00a101..0c3e6d82c 100644 --- a/.apibuilder/.tracked_files +++ b/.apibuilder/.tracked_files @@ -3,7 +3,6 @@ apicollective: apibuilder-api: anorm_2_8_parsers: - api/app/generated/ApicollectiveApibuilderApiV0Conversions.scala - - api/app/generated/ApicollectiveApibuilderApiV0Parsers.scala play_2_8_client: - generated/app/ApicollectiveApibuilderApiV0Client.scala play_2_x_routes: @@ -14,13 +13,11 @@ apicollective: apibuilder-common: anorm_2_8_parsers: - api/app/generated/ApicollectiveApibuilderCommonV0Conversions.scala - - api/app/generated/ApicollectiveApibuilderCommonV0Parsers.scala play_2_x_json: - generated/app/ApicollectiveApibuilderCommonV0Models.scala apibuilder-generator: anorm_2_8_parsers: - api/app/generated/ApicollectiveApibuilderGeneratorV0Conversions.scala - - api/app/generated/ApicollectiveApibuilderGeneratorV0Parsers.scala play_2_8_client: - generated/app/ApicollectiveApibuilderGeneratorV0Client.scala play_2_8_mock_client: @@ -28,14 +25,17 @@ apicollective: apibuilder-internal: anorm_2_8_parsers: - api/app/generated/ApicollectiveApibuilderInternalV0Conversions.scala - - api/app/generated/ApicollectiveApibuilderInternalV0Parsers.scala play_2_x_json: - generated/app/ApicollectiveApibuilderInternalV0Models.scala apibuilder-spec: anorm_2_8_parsers: - api/app/generated/ApicollectiveApibuilderSpecV0Conversions.scala - - api/app/generated/ApicollectiveApibuilderSpecV0Parsers.scala play_2_8_client: - generated/app/ApicollectiveApibuilderSpecV0Client.scala play_2_8_mock_client: - generated/app/ApicollectiveApibuilderSpecV0MockClient.scala + apibuilder-task: + play_2_8_client: + - generated/app/ApicollectiveApibuilderTaskV0Client.scala + play_2_8_mock_client: + - generated/app/ApicollectiveApibuilderTaskV0MockClient.scala diff --git a/.apibuilder/config b/.apibuilder/config index cfc091d56..acbaffe79 100644 --- a/.apibuilder/config +++ b/.apibuilder/config @@ -27,3 +27,9 @@ code: play_2_8_client: generated/app play_2_8_mock_client: generated/app anorm_2_8_parsers: api/app/generated + apibuilder-task: + version: latest + generators: + play_2_8_client: generated/app + play_2_8_mock_client: generated/app + diff --git a/api/app/db/VersionsDao.scala b/api/app/db/VersionsDao.scala index 18346dbba..d7002dd96 100644 --- a/api/app/db/VersionsDao.scala +++ b/api/app/db/VersionsDao.scala @@ -8,14 +8,16 @@ import cats.implicits._ import core.{ServiceFetcher, VersionMigration} import io.apibuilder.api.v0.models._ import io.apibuilder.common.v0.models.{Audit, Reference} -import io.apibuilder.internal.v0.models.{TaskDataDiffVersion, TaskDataIndexApplication} import io.apibuilder.spec.v0.models.Service import io.apibuilder.spec.v0.models.json._ +import io.apibuilder.task.v0.models._ +import io.apibuilder.task.v0.models.json._ import io.flow.postgresql.Query import lib.{ServiceConfiguration, ServiceUri, ValidatedHelpers, VersionTag} import play.api.Logger import play.api.db._ import play.api.libs.json._ +import processor.TaskType import java.util.UUID import javax.inject.{Inject, Named} @@ -25,7 +27,7 @@ class VersionsDao @Inject() ( @Named("main-actor") mainActor: akka.actor.ActorRef, applicationsDao: ApplicationsDao, originalsDao: OriginalsDao, - tasksDao: TasksDao, + tasksDao: InternalTasksDao, usersDao: UsersDao, organizationsDao: OrganizationsDao, serviceParser: ServiceParser @@ -165,7 +167,7 @@ class VersionsDao @Inject() ( ) ( implicit c: java.sql.Connection ): UUID = { - tasksDao.insert(c, user, TaskDataDiffVersion(oldVersionGuid = oldVersionGuid, newVersionGuid = newVersionGuid)) + tasksDao.queueWithConnection(c, TaskType.DiffVersion, data = Json.toJson(DiffVersionData(oldVersionGuid = oldVersionGuid, newVersionGuid = newVersionGuid))) } def replace(user: User, version: Version, application: Application, original: Original, service: Service): Version = { diff --git a/api/app/processor/IndexApplicationProcessor.scala b/api/app/processor/IndexApplicationProcessor.scala index 68d486a38..ec47e36a4 100644 --- a/api/app/processor/IndexApplicationProcessor.scala +++ b/api/app/processor/IndexApplicationProcessor.scala @@ -15,7 +15,6 @@ class IndexApplicationProcessor @Inject()( applicationsDao: ApplicationsDao, itemsDao: ItemsDao, organizationsDao: OrganizationsDao - ) extends BaseTaskProcessor(args, TaskType.IndexApplication) { override def processTask(task: Task): ValidatedNec[String, Unit] = { diff --git a/generated/app/ApicollectiveApibuilderTaskV0Client.scala b/generated/app/ApicollectiveApibuilderTaskV0Client.scala new file mode 100644 index 000000000..e5ecd91f8 --- /dev/null +++ b/generated/app/ApicollectiveApibuilderTaskV0Client.scala @@ -0,0 +1,309 @@ +/** + * Generated by API Builder - https://www.apibuilder.io + * Service version: 0.16.24 + * apibuilder app.apibuilder.io/apicollective/apibuilder-task/latest/play_2_8_client + */ +package io.apibuilder.task.v0.models { + + final case class DiffVersionData( + oldVersionGuid: _root_.java.util.UUID, + newVersionGuid: _root_.java.util.UUID + ) + +} + +package io.apibuilder.task.v0.models { + + package object json { + import play.api.libs.json.__ + import play.api.libs.json.JsString + import play.api.libs.json.Writes + import play.api.libs.functional.syntax._ + import io.apibuilder.task.v0.models.json._ + + private[v0] implicit val jsonReadsUUID: play.api.libs.json.Reads[_root_.java.util.UUID] = __.read[String].map { str => + _root_.java.util.UUID.fromString(str) + } + + private[v0] implicit val jsonWritesUUID: play.api.libs.json.Writes[_root_.java.util.UUID] = (x: _root_.java.util.UUID) => play.api.libs.json.JsString(x.toString) + + private[v0] implicit val jsonReadsJodaDateTime: play.api.libs.json.Reads[_root_.org.joda.time.DateTime] = __.read[String].map { str => + _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseDateTime(str) + } + + private[v0] implicit val jsonWritesJodaDateTime: play.api.libs.json.Writes[_root_.org.joda.time.DateTime] = (x: _root_.org.joda.time.DateTime) => { + play.api.libs.json.JsString(_root_.org.joda.time.format.ISODateTimeFormat.dateTime.print(x)) + } + + private[v0] implicit val jsonReadsJodaLocalDate: play.api.libs.json.Reads[_root_.org.joda.time.LocalDate] = __.read[String].map { str => + _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseLocalDate(str) + } + + private[v0] implicit val jsonWritesJodaLocalDate: play.api.libs.json.Writes[_root_.org.joda.time.LocalDate] = (x: _root_.org.joda.time.LocalDate) => { + play.api.libs.json.JsString(_root_.org.joda.time.format.ISODateTimeFormat.date.print(x)) + } + + implicit def jsonReadsApibuilderTaskDiffVersionData: play.api.libs.json.Reads[DiffVersionData] = { + for { + oldVersionGuid <- (__ \ "old_version_guid").read[_root_.java.util.UUID] + newVersionGuid <- (__ \ "new_version_guid").read[_root_.java.util.UUID] + } yield DiffVersionData(oldVersionGuid, newVersionGuid) + } + + def jsObjectDiffVersionData(obj: io.apibuilder.task.v0.models.DiffVersionData): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "old_version_guid" -> play.api.libs.json.JsString(obj.oldVersionGuid.toString), + "new_version_guid" -> play.api.libs.json.JsString(obj.newVersionGuid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskDiffVersionData: play.api.libs.json.Writes[DiffVersionData] = { + (obj: io.apibuilder.task.v0.models.DiffVersionData) => { + jsObjectDiffVersionData(obj) + } + } + } +} + +package io.apibuilder.task.v0 { + + object Bindables { + + import play.api.mvc.{PathBindable, QueryStringBindable} + + // import models directly for backwards compatibility with prior versions of the generator + import Core._ + + object Core { + implicit def pathBindableDateTimeIso8601(implicit stringBinder: QueryStringBindable[String]): PathBindable[_root_.org.joda.time.DateTime] = ApibuilderPathBindable(ApibuilderTypes.dateTimeIso8601) + implicit def queryStringBindableDateTimeIso8601(implicit stringBinder: QueryStringBindable[String]): QueryStringBindable[_root_.org.joda.time.DateTime] = ApibuilderQueryStringBindable(ApibuilderTypes.dateTimeIso8601) + + implicit def pathBindableDateIso8601(implicit stringBinder: QueryStringBindable[String]): PathBindable[_root_.org.joda.time.LocalDate] = ApibuilderPathBindable(ApibuilderTypes.dateIso8601) + implicit def queryStringBindableDateIso8601(implicit stringBinder: QueryStringBindable[String]): QueryStringBindable[_root_.org.joda.time.LocalDate] = ApibuilderQueryStringBindable(ApibuilderTypes.dateIso8601) + } + + trait ApibuilderTypeConverter[T] { + + def convert(value: String): T + + def convert(value: T): String + + def example: T + + def validValues: Seq[T] = Nil + + def errorMessage(key: String, value: String, ex: java.lang.Exception): String = { + val base = s"Invalid value '$value' for parameter '$key'. " + validValues.toList match { + case Nil => base + "Ex: " + convert(example) + case values => base + ". Valid values are: " + values.mkString("'", "', '", "'") + } + } + } + + object ApibuilderTypes { + val dateTimeIso8601: ApibuilderTypeConverter[_root_.org.joda.time.DateTime] = new ApibuilderTypeConverter[_root_.org.joda.time.DateTime] { + override def convert(value: String): _root_.org.joda.time.DateTime = _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseDateTime(value) + override def convert(value: _root_.org.joda.time.DateTime): String = _root_.org.joda.time.format.ISODateTimeFormat.dateTime.print(value) + override def example: _root_.org.joda.time.DateTime = _root_.org.joda.time.DateTime.now + } + + val dateIso8601: ApibuilderTypeConverter[_root_.org.joda.time.LocalDate] = new ApibuilderTypeConverter[_root_.org.joda.time.LocalDate] { + override def convert(value: String): _root_.org.joda.time.LocalDate = _root_.org.joda.time.format.ISODateTimeFormat.dateTimeParser.parseLocalDate(value) + override def convert(value: _root_.org.joda.time.LocalDate): String = _root_.org.joda.time.format.ISODateTimeFormat.date.print(value) + override def example: _root_.org.joda.time.LocalDate = _root_.org.joda.time.LocalDate.now + } + } + + final case class ApibuilderQueryStringBindable[T]( + converters: ApibuilderTypeConverter[T] + ) extends QueryStringBindable[T] { + + override def bind(key: String, params: Map[String, Seq[String]]): _root_.scala.Option[_root_.scala.Either[String, T]] = { + params.getOrElse(key, Nil).headOption.map { v => + try { + Right( + converters.convert(v) + ) + } catch { + case ex: java.lang.Exception => Left( + converters.errorMessage(key, v, ex) + ) + } + } + } + + override def unbind(key: String, value: T): String = { + s"$key=${converters.convert(value)}" + } + } + + final case class ApibuilderPathBindable[T]( + converters: ApibuilderTypeConverter[T] + ) extends PathBindable[T] { + + override def bind(key: String, value: String): _root_.scala.Either[String, T] = { + try { + Right( + converters.convert(value) + ) + } catch { + case ex: java.lang.Exception => Left( + converters.errorMessage(key, value, ex) + ) + } + } + + override def unbind(key: String, value: T): String = { + converters.convert(value) + } + } + + } + +} + + +package io.apibuilder.task.v0 { + + object Constants { + + val Namespace = "io.apibuilder.task.v0" + val UserAgent = "apibuilder app.apibuilder.io/apicollective/apibuilder-task/latest/play_2_8_client" + val Version = "0.16.24" + val VersionMajor = 0 + + } + + class Client( + ws: play.api.libs.ws.WSClient, + val baseUrl: String, + auth: scala.Option[io.apibuilder.task.v0.Authorization] = None, + defaultHeaders: Seq[(String, String)] = Nil + ) extends interfaces.Client { + import io.apibuilder.task.v0.models.json._ + + private[this] val logger = play.api.Logger("io.apibuilder.task.v0.Client") + + logger.info(s"Initializing io.apibuilder.task.v0.Client for url $baseUrl") + + + + + + def _requestHolder(path: String): play.api.libs.ws.WSRequest = { + + val holder = ws.url(baseUrl + path).addHttpHeaders( + "User-Agent" -> Constants.UserAgent, + "X-Apidoc-Version" -> Constants.Version, + "X-Apidoc-Version-Major" -> Constants.VersionMajor.toString + ).addHttpHeaders(defaultHeaders : _*) + auth.fold(holder) { + case Authorization.Basic(username, password) => { + holder.withAuth(username, password.getOrElse(""), play.api.libs.ws.WSAuthScheme.BASIC) + } + case a => sys.error("Invalid authorization scheme[" + a.getClass + "]") + } + } + + def _logRequest(method: String, req: play.api.libs.ws.WSRequest): play.api.libs.ws.WSRequest = { + val queryComponents = for { + (name, values) <- req.queryString + value <- values + } yield s"$name=$value" + val url = s"${req.url}${queryComponents.mkString("?", "&", "")}" + auth.fold(logger.info(s"curl -X $method '$url'")) { _ => + logger.info(s"curl -X $method -u '[REDACTED]:' '$url'") + } + req + } + + def _executeRequest( + method: String, + path: String, + queryParameters: Seq[(String, String)] = Nil, + requestHeaders: Seq[(String, String)] = Nil, + body: Option[play.api.libs.json.JsValue] = None + ): scala.concurrent.Future[play.api.libs.ws.WSResponse] = { + method.toUpperCase match { + case "GET" => { + _logRequest("GET", _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)).get() + } + case "POST" => { + _logRequest("POST", _requestHolder(path).addHttpHeaders(_withJsonContentType(requestHeaders):_*).addQueryStringParameters(queryParameters:_*)).post(body.getOrElse(play.api.libs.json.Json.obj())) + } + case "PUT" => { + _logRequest("PUT", _requestHolder(path).addHttpHeaders(_withJsonContentType(requestHeaders):_*).addQueryStringParameters(queryParameters:_*)).put(body.getOrElse(play.api.libs.json.Json.obj())) + } + case "PATCH" => { + _logRequest("PATCH", _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)).patch(body.getOrElse(play.api.libs.json.Json.obj())) + } + case "DELETE" => { + _logRequest("DELETE", _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)).delete() + } + case "HEAD" => { + _logRequest("HEAD", _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)).head() + } + case "OPTIONS" => { + _logRequest("OPTIONS", _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)).options() + } + case _ => { + _logRequest(method, _requestHolder(path).addHttpHeaders(requestHeaders:_*).addQueryStringParameters(queryParameters:_*)) + sys.error("Unsupported method[%s]".format(method)) + } + } + } + + /** + * Adds a Content-Type: application/json header unless the specified requestHeaders + * already contain a Content-Type header + */ + def _withJsonContentType(headers: Seq[(String, String)]): Seq[(String, String)] = { + headers.find { _._1.toUpperCase == "CONTENT-TYPE" } match { + case None => headers ++ Seq("Content-Type" -> "application/json; charset=UTF-8") + case Some(_) => headers + } + } + + } + + object Client { + + def parseJson[T]( + className: String, + r: play.api.libs.ws.WSResponse, + f: (play.api.libs.json.JsValue => play.api.libs.json.JsResult[T]) + ): T = { + f(play.api.libs.json.Json.parse(r.body)) match { + case play.api.libs.json.JsSuccess(x, _) => x + case play.api.libs.json.JsError(errors) => { + throw io.apibuilder.task.v0.errors.FailedRequest(r.status, s"Invalid json for class[" + className + "]: " + errors.mkString(" ")) + } + } + } + + } + + sealed trait Authorization extends _root_.scala.Product with _root_.scala.Serializable + object Authorization { + final case class Basic(username: String, password: Option[String] = None) extends Authorization + } + + package interfaces { + + trait Client { + def baseUrl: String + + } + + } + + + + package errors { + + final case class FailedRequest(responseCode: Int, message: String, requestUri: Option[_root_.java.net.URI] = None) extends _root_.java.lang.Exception(s"HTTP $responseCode: $message") + + } + +} \ No newline at end of file diff --git a/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala b/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala new file mode 100644 index 000000000..073a33b07 --- /dev/null +++ b/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala @@ -0,0 +1,21 @@ +/** + * Generated by API Builder - https://www.apibuilder.io + * Service version: 0.16.24 + * apibuilder app.apibuilder.io/apicollective/apibuilder-task/latest/play_2_8_mock_client + */ +package io.apibuilder.task.v0.mock { + + object Factories { + + def randomString(length: Int = 24): String = { + _root_.scala.util.Random.alphanumeric.take(length).mkString + } + + def makeDiffVersionData(): io.apibuilder.task.v0.models.DiffVersionData = io.apibuilder.task.v0.models.DiffVersionData( + oldVersionGuid = _root_.java.util.UUID.randomUUID, + newVersionGuid = _root_.java.util.UUID.randomUUID + ) + + } + +} \ No newline at end of file diff --git a/spec/apibuilder-task.json b/spec/apibuilder-task.json new file mode 100644 index 000000000..833812170 --- /dev/null +++ b/spec/apibuilder-task.json @@ -0,0 +1,24 @@ +{ + "name": "apibuilder task", + + "info": { + "contact": { + "name": "Michael Bryzek", + "email": "mbryzek@alum.mit.edu", + "url": "http://twitter.com/mbryzek" + }, + "license": { + "name": "MIT", + "url": "http://opensource.org/licenses/MIT" + } + }, + + "models": { + "diff_version_data": { + "fields": [ + { "name": "old_version_guid", "type": "uuid" }, + { "name": "new_version_guid", "type": "uuid" } + ] + } + } +} From 147f96672382a7370d6768088ed2bb4b56741646 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:47:22 -0700 Subject: [PATCH 08/32] wip --- api/app/actors/MainActor.scala | 4 +- api/app/actors/TaskActor.scala | 3 +- api/app/db/InternalTasksDao.scala | 2 +- api/app/db/VersionsDao.scala | 17 ++-- api/app/processor/DiffVersionProcessor.scala | 18 +++++ .../processor/IndexApplicationProcessor.scala | 20 +++-- api/app/processor/TaskActorCompanion.scala | 8 +- api/app/processor/TaskProcessor.scala | 31 +++++--- api/app/processor/TaskType.scala | 15 ---- .../ApicollectiveApibuilderTaskV0Client.scala | 78 +++++++++++++++++++ ...collectiveApibuilderTaskV0MockClient.scala | 2 + spec/apibuilder-task.json | 9 +++ 12 files changed, 158 insertions(+), 49 deletions(-) create mode 100644 api/app/processor/DiffVersionProcessor.scala delete mode 100644 api/app/processor/TaskType.scala diff --git a/api/app/actors/MainActor.scala b/api/app/actors/MainActor.scala index bdaf59076..17146d564 100644 --- a/api/app/actors/MainActor.scala +++ b/api/app/actors/MainActor.scala @@ -23,7 +23,6 @@ object MainActor { case class ApplicationCreated(guid: UUID) case class UserCreated(guid: UUID) - case class TaskCreated(guid: UUID) case class GeneratorServiceCreated(guid: UUID) } } @@ -36,7 +35,6 @@ class MainActor @javax.inject.Inject() ( processDeletes: ProcessDeletes, @javax.inject.Named("email-actor") emailActor: akka.actor.ActorRef, @javax.inject.Named("generator-service-actor") generatorServiceActor: akka.actor.ActorRef, - @javax.inject.Named("task-actor") taskActor: akka.actor.ActorRef, @javax.inject.Named("user-actor") userActor: akka.actor.ActorRef ) extends Actor with ActorLogging with ErrorHandler { @@ -55,7 +53,7 @@ class MainActor @javax.inject.Inject() ( scheduleOnce(QueueVersionsToMigrate) scheduleOnce(CleanupDeletedApplications) - def receive = akka.event.LoggingReceive { + def receive: Receive = akka.event.LoggingReceive { case m @ MainActor.Messages.MembershipRequestCreated(guid) => withVerboseErrorHandler(m) { emailActor ! EmailActor.Messages.MembershipRequestCreated(guid) diff --git a/api/app/actors/TaskActor.scala b/api/app/actors/TaskActor.scala index 493d088ce..23fd73f5c 100644 --- a/api/app/actors/TaskActor.scala +++ b/api/app/actors/TaskActor.scala @@ -2,7 +2,8 @@ package actors import akka.actor.{Actor, ActorLogging} import com.google.inject.assistedinject.Assisted -import processor.{TaskActorCompanion, TaskType} +import io.apibuilder.task.v0.models.TaskType +import processor.TaskActorCompanion import javax.inject.Inject diff --git a/api/app/db/InternalTasksDao.scala b/api/app/db/InternalTasksDao.scala index d1eace52f..1bea116cd 100644 --- a/api/app/db/InternalTasksDao.scala +++ b/api/app/db/InternalTasksDao.scala @@ -1,10 +1,10 @@ package db import db.generated.{Task, TaskForm} +import io.apibuilder.task.v0.models.TaskType import lib.Constants import org.joda.time.DateTime import play.api.libs.json.{JsValue, Json} -import processor.TaskType import java.sql.Connection import java.util.UUID diff --git a/api/app/db/VersionsDao.scala b/api/app/db/VersionsDao.scala index d7002dd96..2560cfe03 100644 --- a/api/app/db/VersionsDao.scala +++ b/api/app/db/VersionsDao.scala @@ -17,7 +17,6 @@ import lib.{ServiceConfiguration, ServiceUri, ValidatedHelpers, VersionTag} import play.api.Logger import play.api.db._ import play.api.libs.json._ -import processor.TaskType import java.util.UUID import javax.inject.{Inject, Named} @@ -111,7 +110,7 @@ class VersionsDao @Inject() ( val (guid, taskGuid) = db.withTransaction { implicit c => val versionGuid = doCreate(c, user, application, version, original, service) val taskGuid = latestVersion.map { v => - createDiffTask(user, v.guid, versionGuid) + createDiffTask(v.guid, versionGuid) } (versionGuid, taskGuid) } @@ -163,11 +162,19 @@ class VersionsDao @Inject() ( } private[this] def createDiffTask( - user: User, oldVersionGuid: UUID, newVersionGuid: UUID + oldVersionGuid: UUID, + newVersionGuid: UUID ) ( implicit c: java.sql.Connection - ): UUID = { - tasksDao.queueWithConnection(c, TaskType.DiffVersion, data = Json.toJson(DiffVersionData(oldVersionGuid = oldVersionGuid, newVersionGuid = newVersionGuid))) + ): Unit = { + tasksDao.queueWithConnection( + c, + TaskType.DiffVersion, + newVersionGuid.toString, + data = Json.toJson( + DiffVersionData(oldVersionGuid = oldVersionGuid, newVersionGuid = newVersionGuid) + ) + ) } def replace(user: User, version: Version, application: Application, original: Original, service: Service): Version = { diff --git a/api/app/processor/DiffVersionProcessor.scala b/api/app/processor/DiffVersionProcessor.scala new file mode 100644 index 000000000..adf2990fd --- /dev/null +++ b/api/app/processor/DiffVersionProcessor.scala @@ -0,0 +1,18 @@ +package processor + +import cats.data.ValidatedNec +import cats.implicits._ +import io.apibuilder.task.v0.models.{DiffVersionData, TaskType} + +import javax.inject.Inject + + +class DiffVersionProcessor @Inject()( + args: TaskProcessorArgs, +) extends TaskProcessorWithData[DiffVersionData](args, TaskType.DiffVersion) { + + override def processRecord(id: String, data: DiffVersionData): ValidatedNec[String, Unit] = { + "TODO".invalidNec + } + +} \ No newline at end of file diff --git a/api/app/processor/IndexApplicationProcessor.scala b/api/app/processor/IndexApplicationProcessor.scala index ec47e36a4..00445c704 100644 --- a/api/app/processor/IndexApplicationProcessor.scala +++ b/api/app/processor/IndexApplicationProcessor.scala @@ -1,27 +1,24 @@ package processor +import cats.implicits._ import cats.data.ValidatedNec -import db.generated.Task import db.{ApplicationsDao, Authorization, ItemsDao, OrganizationsDao} import io.apibuilder.api.v0.models.{Application, ApplicationSummary, Organization} import io.apibuilder.common.v0.models.Reference +import io.apibuilder.task.v0.models.TaskType import java.util.UUID import javax.inject.Inject class IndexApplicationProcessor @Inject()( - args: TaskProcessorArgs, - applicationsDao: ApplicationsDao, - itemsDao: ItemsDao, - organizationsDao: OrganizationsDao - ) extends BaseTaskProcessor(args, TaskType.IndexApplication) { + args: TaskProcessorArgs, + applicationsDao: ApplicationsDao, + itemsDao: ItemsDao, + organizationsDao: OrganizationsDao +) extends TaskProcessorWithGuid(args, TaskType.IndexApplication) { - override def processTask(task: Task): ValidatedNec[String, Unit] = { - validateGuid(task.typeId).map(processApplicationGuid) - } - - def processApplicationGuid(applicationGuid: UUID): Unit = { + override def processRecord(applicationGuid: UUID): ValidatedNec[String, Unit] = { getInfo(applicationGuid) match { case Some((org, app)) => { val content = s"""${app.name} ${app.key} ${app.description.getOrElse("")}""".trim.toLowerCase @@ -41,6 +38,7 @@ class IndexApplicationProcessor @Inject()( itemsDao.delete(applicationGuid) } } + ().validNec } private[this] def getInfo(applicationGuid: UUID): Option[(Organization, Application)] = { diff --git a/api/app/processor/TaskActorCompanion.scala b/api/app/processor/TaskActorCompanion.scala index b23b71a34..e807b2cee 100644 --- a/api/app/processor/TaskActorCompanion.scala +++ b/api/app/processor/TaskActorCompanion.scala @@ -1,13 +1,12 @@ package processor -import cats.implicits._ -import cats.data.ValidatedNec -import db.generated.Task +import io.apibuilder.task.v0.models.TaskType import javax.inject.Inject class TaskActorCompanion @Inject() ( - indexApplication: IndexApplicationProcessor + indexApplication: IndexApplicationProcessor, + diffVersion: DiffVersionProcessor, ) { def process(typ: TaskType): Unit = { @@ -18,6 +17,7 @@ class TaskActorCompanion @Inject() ( import TaskType._ typ match { case IndexApplication => indexApplication + case DiffVersion => diffVersion } } } diff --git a/api/app/processor/TaskProcessor.scala b/api/app/processor/TaskProcessor.scala index b2545a3bc..bc769e45a 100644 --- a/api/app/processor/TaskProcessor.scala +++ b/api/app/processor/TaskProcessor.scala @@ -4,6 +4,7 @@ import cats.data.Validated.{Invalid, Valid} import cats.data.ValidatedNec import cats.implicits._ import db.generated.{Task, TaskForm, TasksDao} +import io.apibuilder.task.v0.models.TaskType import io.flow.postgresql.OrderBy import lib.Constants import org.joda.time.DateTime @@ -40,6 +41,27 @@ abstract class TaskProcessor( } } +abstract class TaskProcessorWithGuid( + args: TaskProcessorArgs, + typ: TaskType + ) extends TaskProcessor(args, typ) { + + def processRecord(guid: UUID): ValidatedNec[String, Unit] + + override final def processRecord(id: String): ValidatedNec[String, Unit] = { + validateGuid(id).andThen(processRecord) + } + + private[this] def validateGuid(value: String): ValidatedNec[String, UUID] = { + Try { + UUID.fromString(value) + } match { + case Success(v) => v.validNec + case Failure(_) => s"Invalid guid '$value'".invalidNec + } + } +} + abstract class TaskProcessorWithData[T]( args: TaskProcessorArgs, typ: TaskType @@ -129,15 +151,6 @@ abstract class BaseTaskProcessor( DateTime.now.plusMinutes(5 * (task.numAttempts + 1)) } - protected final def validateGuid(value: String): ValidatedNec[String, UUID] = { - Try { - UUID.fromString(value) - } match { - case Success(v) => v.validNec - case Failure(_) => s"Invalid guid '$value'".invalidNec - } - } - final protected def makeInitialTaskForm( typeId: String, organizationGuid: Option[UUID], diff --git a/api/app/processor/TaskType.scala b/api/app/processor/TaskType.scala deleted file mode 100644 index ddda7c55a..000000000 --- a/api/app/processor/TaskType.scala +++ /dev/null @@ -1,15 +0,0 @@ -package processor - -import cats.implicits._ -import cats.data.ValidatedNec - -sealed trait TaskType -object TaskType { - case object IndexApplication extends TaskType { override def toString = "index_application" } - val all: Seq[TaskType] = Seq(Noop) - - private[this] val byString = all.map { t => t.toString.toLowerCase -> t }.toMap - def fromString(value: String): ValidatedNec[String, TaskType] = { - byString.get(value.trim.toLowerCase()).toValidNec(s"Invalid task type '$value'") - } -} \ No newline at end of file diff --git a/generated/app/ApicollectiveApibuilderTaskV0Client.scala b/generated/app/ApicollectiveApibuilderTaskV0Client.scala index e5ecd91f8..27765783e 100644 --- a/generated/app/ApicollectiveApibuilderTaskV0Client.scala +++ b/generated/app/ApicollectiveApibuilderTaskV0Client.scala @@ -9,6 +9,38 @@ package io.apibuilder.task.v0.models { oldVersionGuid: _root_.java.util.UUID, newVersionGuid: _root_.java.util.UUID ) + sealed trait TaskType extends _root_.scala.Product with _root_.scala.Serializable + + object TaskType { + + case object IndexApplication extends TaskType { override def toString = "index_application" } + case object DiffVersion extends TaskType { override def toString = "diff_version" } + /** + * UNDEFINED captures values that are sent either in error or + * that were added by the server after this library was + * generated. We want to make it easy and obvious for users of + * this library to handle this case gracefully. + * + * We use all CAPS for the variable name to avoid collisions + * with the camel cased values above. + */ + final case class UNDEFINED(override val toString: String) extends TaskType + + /** + * all returns a list of all the valid, known values. We use + * lower case to avoid collisions with the camel cased values + * above. + */ + val all: scala.List[TaskType] = scala.List(IndexApplication, DiffVersion) + + private[this] + val byName: Map[String, TaskType] = all.map(x => x.toString.toLowerCase -> x).toMap + + def apply(value: String): TaskType = fromString(value).getOrElse(UNDEFINED(value)) + + def fromString(value: String): _root_.scala.Option[TaskType] = byName.get(value.toLowerCase) + + } } @@ -43,6 +75,38 @@ package io.apibuilder.task.v0.models { play.api.libs.json.JsString(_root_.org.joda.time.format.ISODateTimeFormat.date.print(x)) } + implicit val jsonReadsApibuilderTaskTaskType: play.api.libs.json.Reads[io.apibuilder.task.v0.models.TaskType] = new play.api.libs.json.Reads[io.apibuilder.task.v0.models.TaskType] { + def reads(js: play.api.libs.json.JsValue): play.api.libs.json.JsResult[io.apibuilder.task.v0.models.TaskType] = { + js match { + case v: play.api.libs.json.JsString => play.api.libs.json.JsSuccess(io.apibuilder.task.v0.models.TaskType(v.value)) + case _ => { + (js \ "value").validate[String] match { + case play.api.libs.json.JsSuccess(v, _) => play.api.libs.json.JsSuccess(io.apibuilder.task.v0.models.TaskType(v)) + case err: play.api.libs.json.JsError => + (js \ "task_type").validate[String] match { + case play.api.libs.json.JsSuccess(v, _) => play.api.libs.json.JsSuccess(io.apibuilder.task.v0.models.TaskType(v)) + case err: play.api.libs.json.JsError => err + } + } + } + } + } + } + + def jsonWritesApibuilderTaskTaskType(obj: io.apibuilder.task.v0.models.TaskType) = { + play.api.libs.json.JsString(obj.toString) + } + + def jsObjectTaskType(obj: io.apibuilder.task.v0.models.TaskType) = { + play.api.libs.json.Json.obj("value" -> play.api.libs.json.JsString(obj.toString)) + } + + implicit def jsonWritesApibuilderTaskTaskType: play.api.libs.json.Writes[TaskType] = { + (obj: io.apibuilder.task.v0.models.TaskType) => { + jsonWritesApibuilderTaskTaskType(obj) + } + } + implicit def jsonReadsApibuilderTaskDiffVersionData: play.api.libs.json.Reads[DiffVersionData] = { for { oldVersionGuid <- (__ \ "old_version_guid").read[_root_.java.util.UUID] @@ -73,6 +137,7 @@ package io.apibuilder.task.v0 { // import models directly for backwards compatibility with prior versions of the generator import Core._ + import Models._ object Core { implicit def pathBindableDateTimeIso8601(implicit stringBinder: QueryStringBindable[String]): PathBindable[_root_.org.joda.time.DateTime] = ApibuilderPathBindable(ApibuilderTypes.dateTimeIso8601) @@ -82,6 +147,19 @@ package io.apibuilder.task.v0 { implicit def queryStringBindableDateIso8601(implicit stringBinder: QueryStringBindable[String]): QueryStringBindable[_root_.org.joda.time.LocalDate] = ApibuilderQueryStringBindable(ApibuilderTypes.dateIso8601) } + object Models { + import io.apibuilder.task.v0.models._ + + val taskTypeConverter: ApibuilderTypeConverter[io.apibuilder.task.v0.models.TaskType] = new ApibuilderTypeConverter[io.apibuilder.task.v0.models.TaskType] { + override def convert(value: String): io.apibuilder.task.v0.models.TaskType = io.apibuilder.task.v0.models.TaskType(value) + override def convert(value: io.apibuilder.task.v0.models.TaskType): String = value.toString + override def example: io.apibuilder.task.v0.models.TaskType = io.apibuilder.task.v0.models.TaskType.IndexApplication + override def validValues: Seq[io.apibuilder.task.v0.models.TaskType] = io.apibuilder.task.v0.models.TaskType.all + } + implicit def pathBindableTaskType(implicit stringBinder: QueryStringBindable[String]): PathBindable[io.apibuilder.task.v0.models.TaskType] = ApibuilderPathBindable(taskTypeConverter) + implicit def queryStringBindableTaskType(implicit stringBinder: QueryStringBindable[String]): QueryStringBindable[io.apibuilder.task.v0.models.TaskType] = ApibuilderQueryStringBindable(taskTypeConverter) + } + trait ApibuilderTypeConverter[T] { def convert(value: String): T diff --git a/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala b/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala index 073a33b07..5504cf9f8 100644 --- a/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala +++ b/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala @@ -11,6 +11,8 @@ package io.apibuilder.task.v0.mock { _root_.scala.util.Random.alphanumeric.take(length).mkString } + def makeTaskType(): io.apibuilder.task.v0.models.TaskType = io.apibuilder.task.v0.models.TaskType.IndexApplication + def makeDiffVersionData(): io.apibuilder.task.v0.models.DiffVersionData = io.apibuilder.task.v0.models.DiffVersionData( oldVersionGuid = _root_.java.util.UUID.randomUUID, newVersionGuid = _root_.java.util.UUID.randomUUID diff --git a/spec/apibuilder-task.json b/spec/apibuilder-task.json index 833812170..d3a59fecc 100644 --- a/spec/apibuilder-task.json +++ b/spec/apibuilder-task.json @@ -13,6 +13,15 @@ } }, + "enums": { + "task_type": { + "values": [ + { "name": "index_application" }, + { "name": "diff_version" } + ] + } + }, + "models": { "diff_version_data": { "fields": [ From 7bc68c310d1aec6e48189a9415d362256c6433bf Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:50:38 -0700 Subject: [PATCH 09/32] wip --- api/app/processor/DiffVersionProcessor.scala | 122 ++++++++++++++++++- 1 file changed, 119 insertions(+), 3 deletions(-) diff --git a/api/app/processor/DiffVersionProcessor.scala b/api/app/processor/DiffVersionProcessor.scala index adf2990fd..9f03a334e 100644 --- a/api/app/processor/DiffVersionProcessor.scala +++ b/api/app/processor/DiffVersionProcessor.scala @@ -1,18 +1,134 @@ package processor +import actors.Emails import cats.data.ValidatedNec import cats.implicits._ +import db._ +import io.apibuilder.api.v0.models._ +import io.apibuilder.task.v0.models.json._ import io.apibuilder.task.v0.models.{DiffVersionData, TaskType} +import lib.{AppConfig, ServiceDiff} +import play.twirl.api.Html +import java.util.UUID import javax.inject.Inject - class DiffVersionProcessor @Inject()( args: TaskProcessorArgs, + appConfig: AppConfig, + usersDao: UsersDao, + applicationsDao: ApplicationsDao, + organizationsDao: OrganizationsDao, + emails: Emails, + changesDao: ChangesDao, + versionsDao: VersionsDao, + watchesDao: WatchesDao, ) extends TaskProcessorWithData[DiffVersionData](args, TaskType.DiffVersion) { override def processRecord(id: String, data: DiffVersionData): ValidatedNec[String, Unit] = { - "TODO".invalidNec + diffVersion(data.oldVersionGuid, data.newVersionGuid) + ().validNec + } + + private[this] def diffVersion(oldVersionGuid: UUID, newVersionGuid: UUID): Unit = { + versionsDao.findByGuid(Authorization.All, oldVersionGuid, isDeleted = None).foreach { oldVersion => + versionsDao.findByGuid(Authorization.All, newVersionGuid).foreach { newVersion => + ServiceDiff(oldVersion.service, newVersion.service).differences match { + case Nil => { + // No-op + } + case diffs => { + changesDao.upsert( + createdBy = usersDao.AdminUser, + fromVersion = oldVersion, + toVersion = newVersion, + differences = diffs + ) + versionUpdated(newVersion, diffs) + versionUpdatedMaterialOnly(newVersion, diffs) + } + } + } + } } -} \ No newline at end of file + private[this] def versionUpdated( + version: Version, + diffs: Seq[Diff], + ): Unit = { + if (diffs.nonEmpty) { + sendVersionUpsertedEmail( + publication = Publication.VersionsCreate, + version = version, + diffs = diffs, + ) { (org, application, breakingDiffs, nonBreakingDiffs) => + views.html.emails.versionUpserted( + appConfig, + org, + application, + version, + breakingDiffs = breakingDiffs, + nonBreakingDiffs = nonBreakingDiffs + ) + } + } + } + + private[this] def versionUpdatedMaterialOnly( + version: Version, + diffs: Seq[Diff], + ): Unit = { + val filtered = diffs.filter(_.isMaterial) + if (filtered.nonEmpty) { + sendVersionUpsertedEmail( + publication = Publication.VersionsMaterialChange, + version = version, + diffs = filtered, + ) { (org, application, breakingDiffs, nonBreakingDiffs) => + views.html.emails.versionUpserted( + appConfig, + org, + application, + version, + breakingDiffs = breakingDiffs, + nonBreakingDiffs = nonBreakingDiffs + ) + } + } + } + + private[this] def sendVersionUpsertedEmail( + publication: Publication, + version: Version, + diffs: Seq[Diff], + )( + generateBody: (Organization, Application, Seq[Diff], Seq[Diff]) => Html, + ): Unit = { + val (breakingDiffs, nonBreakingDiffs) = diffs.partition { + case _: DiffBreaking => true + case _: DiffNonBreaking => false + case _: DiffUndefinedType => true + } + + applicationsDao.findAll(Authorization.All, version = Some(version), limit = 1).foreach { application => + organizationsDao.findAll(Authorization.All, application = Some(application), limit = 1).foreach { org => + emails.deliver( + context = Emails.Context.Application(application), + org = org, + publication = publication, + subject = s"${org.name}/${application.name}:${version.version} Updated", + body = generateBody(org, application, breakingDiffs, nonBreakingDiffs).toString + ) { subscription => + watchesDao.findAll( + Authorization.All, + application = Some(application), + userGuid = Some(subscription.user.guid), + limit = 1 + ).nonEmpty + } + } + } + } + +} + From 852f9526a85a9c743c559f230e9575fbb306f39a Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:56:22 -0700 Subject: [PATCH 10/32] wip --- api/app/actors/TaskDispatchActor.scala | 3 +- api/app/db/ApplicationsDao.scala | 6 +-- api/app/db/VersionsDao.scala | 41 +++++++------------ .../TaskDispatchActorCompanion.scala | 3 +- api/app/processor/TaskProcessor.scala | 16 +++++--- 5 files changed, 32 insertions(+), 37 deletions(-) diff --git a/api/app/actors/TaskDispatchActor.scala b/api/app/actors/TaskDispatchActor.scala index 224353ef9..cdce5ea02 100644 --- a/api/app/actors/TaskDispatchActor.scala +++ b/api/app/actors/TaskDispatchActor.scala @@ -1,8 +1,9 @@ package actors import akka.actor._ +import io.apibuilder.task.v0.models.TaskType import play.api.libs.concurrent.InjectedActorSupport -import processor.{TaskDispatchActorCompanion, TaskType} +import processor.TaskDispatchActorCompanion import javax.inject.{Inject, Singleton} import scala.concurrent.ExecutionContext diff --git a/api/app/db/ApplicationsDao.scala b/api/app/db/ApplicationsDao.scala index 1859e5736..ab844daa4 100644 --- a/api/app/db/ApplicationsDao.scala +++ b/api/app/db/ApplicationsDao.scala @@ -5,7 +5,7 @@ import io.apibuilder.api.v0.models.{AppSortBy, Application, ApplicationForm, Err import io.flow.postgresql.Query import lib.{UrlKey, Validation} import play.api.db._ -import processor.TaskType +import processor.IndexApplicationProcessor import java.util.UUID import javax.inject.{Inject, Named, Singleton} @@ -15,7 +15,7 @@ class ApplicationsDao @Inject() ( @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, organizationsDao: OrganizationsDao, - tasksDao: InternalTasksDao + indexApplicationProcessor: IndexApplicationProcessor, ) { private[this] val dbHelpers = DbHelpers(db, "applications") @@ -364,7 +364,7 @@ class ApplicationsDao @Inject() ( ): Unit = { db.withTransaction { implicit c => f(c) - tasksDao.queueWithConnection(c, TaskType.IndexApplication, guid.toString) + indexApplicationProcessor.queue(c, guid) } } diff --git a/api/app/db/VersionsDao.scala b/api/app/db/VersionsDao.scala index 2560cfe03..b13a51d65 100644 --- a/api/app/db/VersionsDao.scala +++ b/api/app/db/VersionsDao.scala @@ -11,22 +11,22 @@ import io.apibuilder.common.v0.models.{Audit, Reference} import io.apibuilder.spec.v0.models.Service import io.apibuilder.spec.v0.models.json._ import io.apibuilder.task.v0.models._ -import io.apibuilder.task.v0.models.json._ import io.flow.postgresql.Query import lib.{ServiceConfiguration, ServiceUri, ValidatedHelpers, VersionTag} import play.api.Logger import play.api.db._ import play.api.libs.json._ +import processor.{DiffVersionProcessor, IndexApplicationProcessor} import java.util.UUID -import javax.inject.{Inject, Named} +import javax.inject.Inject class VersionsDao @Inject() ( @NamedDatabase("default") db: Database, - @Named("main-actor") mainActor: akka.actor.ActorRef, applicationsDao: ApplicationsDao, originalsDao: OriginalsDao, - tasksDao: InternalTasksDao, + diffVersionProcessor: DiffVersionProcessor, + indexApplicationProcessor: IndexApplicationProcessor, usersDao: UsersDao, organizationsDao: OrganizationsDao, serviceParser: ServiceParser @@ -107,19 +107,15 @@ class VersionsDao @Inject() ( limit = 1 ).headOption - val (guid, taskGuid) = db.withTransaction { implicit c => + val guid = db.withTransaction { implicit c => val versionGuid = doCreate(c, user, application, version, original, service) - val taskGuid = latestVersion.map { v => + latestVersion.foreach { v => createDiffTask(v.guid, versionGuid) } - (versionGuid, taskGuid) + versionGuid } - taskGuid.foreach { guid => - mainActor ! actors.MainActor.Messages.TaskCreated(guid) - } - - findAll(Authorization.All, guid = Some(guid), limit = 1).headOption.getOrElse { + findByGuid(Authorization.All, guid).getOrElse { sys.error("Failed to create version") } } @@ -167,30 +163,23 @@ class VersionsDao @Inject() ( ) ( implicit c: java.sql.Connection ): Unit = { - tasksDao.queueWithConnection( + diffVersionProcessor.queue( c, - TaskType.DiffVersion, newVersionGuid.toString, - data = Json.toJson( - DiffVersionData(oldVersionGuid = oldVersionGuid, newVersionGuid = newVersionGuid) - ) + data = DiffVersionData(oldVersionGuid = oldVersionGuid, newVersionGuid = newVersionGuid) ) } def replace(user: User, version: Version, application: Application, original: Original, service: Service): Version = { - val (versionGuid, taskGuids) = db.withTransaction { implicit c => + val versionGuid = db.withTransaction { implicit c => softDelete(user, version) val versionGuid = doCreate(c, user, application, version.version, original, service) - val diffTaskGuid = createDiffTask(user, version.guid, versionGuid) - val indexTaskGuid = tasksDao.insert(c, user, TaskDataIndexApplication(application.guid)) - (versionGuid, Seq(diffTaskGuid, indexTaskGuid)) - } - - taskGuids.foreach { taskGuid => - mainActor ! actors.MainActor.Messages.TaskCreated(taskGuid) + createDiffTask(version.guid, versionGuid) + indexApplicationProcessor.queue(c, application.guid) + versionGuid } - findAll(Authorization.All, guid = Some(versionGuid), limit = 1).headOption.getOrElse { + findByGuid(Authorization.All, versionGuid).getOrElse { sys.error(s"Failed to replace version[${version.guid}]") } } diff --git a/api/app/processor/TaskDispatchActorCompanion.scala b/api/app/processor/TaskDispatchActorCompanion.scala index a3e59d49c..dcdd7f908 100644 --- a/api/app/processor/TaskDispatchActorCompanion.scala +++ b/api/app/processor/TaskDispatchActorCompanion.scala @@ -1,6 +1,7 @@ package processor import anorm.SqlParser +import io.apibuilder.task.v0.models.TaskType import io.flow.postgresql.Query import play.api.db.Database @@ -18,7 +19,7 @@ class TaskDispatchActorCompanion @Inject() ( .withConnection { c => TypesQuery.as(SqlParser.str(1).*)(c) } - .map(TaskType.fromString).flatMap(_.toOption) + .flatMap(TaskType.fromString) } } diff --git a/api/app/processor/TaskProcessor.scala b/api/app/processor/TaskProcessor.scala index bc769e45a..b0745f139 100644 --- a/api/app/processor/TaskProcessor.scala +++ b/api/app/processor/TaskProcessor.scala @@ -8,7 +8,7 @@ import io.apibuilder.task.v0.models.TaskType import io.flow.postgresql.OrderBy import lib.Constants import org.joda.time.DateTime -import play.api.libs.json.{JsObject, JsValue, Json, Reads} +import play.api.libs.json.{JsObject, JsValue, Json, Reads, Writes} import play.libs.exception.ExceptionUtils import java.sql.Connection @@ -36,7 +36,7 @@ abstract class TaskProcessor( } } - final def queue(c: Connection, typeId: String, organizationGuid: Option[UUID]): Unit = { + final def queue(c: Connection, typeId: String, organizationGuid: Option[UUID] = None): Unit = { insertIfNew(c, makeInitialTaskForm(typeId, organizationGuid, Json.obj())) } } @@ -48,6 +48,10 @@ abstract class TaskProcessorWithGuid( def processRecord(guid: UUID): ValidatedNec[String, Unit] + final def queue(c: Connection, typeId: UUID, organizationGuid: Option[UUID] = None): Unit = { + insertIfNew(c, makeInitialTaskForm(typeId.toString, organizationGuid, Json.obj())) + } + override final def processRecord(id: String): ValidatedNec[String, Unit] = { validateGuid(id).andThen(processRecord) } @@ -65,7 +69,7 @@ abstract class TaskProcessorWithGuid( abstract class TaskProcessorWithData[T]( args: TaskProcessorArgs, typ: TaskType -)(implicit reads: Reads[T]) +)(implicit reads: Reads[T], writes: Writes[T]) extends BaseTaskProcessor(args, typ) { def processRecord(id: String, data: T): ValidatedNec[String, Unit] @@ -83,14 +87,14 @@ abstract class TaskProcessorWithData[T]( } } - final def queue(typeId: String, organizationGuid: Option[UUID], data: JsObject): Unit = { + final def queue(typeId: String, organizationGuid: Option[UUID] = None, data: T): Unit = { args.dao.db.withConnection { c => queue(c, typeId, organizationGuid, data) } } - final def queue(c: Connection, typeId: String, organizationGuid: Option[UUID], data: JsObject): Unit = { - insertIfNew(c, makeInitialTaskForm(typeId, organizationGuid, data)) + final def queue(c: Connection, typeId: String, organizationGuid: Option[UUID] = None, data: T): Unit = { + insertIfNew(c, makeInitialTaskForm(typeId, organizationGuid, Json.toJson(data).asInstanceOf[JsObject])) } } From 074034c9eae539bf7c06101dd13c8ce2d64c9fa8 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:58:37 -0700 Subject: [PATCH 11/32] wip --- api/app/db/VersionsDao.scala | 2 +- api/app/processor/TaskActorCompanion.scala | 1 + api/app/processor/TaskProcessor.scala | 13 ++----------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/api/app/db/VersionsDao.scala b/api/app/db/VersionsDao.scala index b13a51d65..89f24a610 100644 --- a/api/app/db/VersionsDao.scala +++ b/api/app/db/VersionsDao.scala @@ -163,7 +163,7 @@ class VersionsDao @Inject() ( ) ( implicit c: java.sql.Connection ): Unit = { - diffVersionProcessor.queue( + diffVersionProcessor.queueWithConnection( c, newVersionGuid.toString, data = DiffVersionData(oldVersionGuid = oldVersionGuid, newVersionGuid = newVersionGuid) diff --git a/api/app/processor/TaskActorCompanion.scala b/api/app/processor/TaskActorCompanion.scala index e807b2cee..68d2976b7 100644 --- a/api/app/processor/TaskActorCompanion.scala +++ b/api/app/processor/TaskActorCompanion.scala @@ -18,6 +18,7 @@ class TaskActorCompanion @Inject() ( typ match { case IndexApplication => indexApplication case DiffVersion => diffVersion + case UNDEFINED(_) => sys.error(s"Undefined task type '$typ") } } } diff --git a/api/app/processor/TaskProcessor.scala b/api/app/processor/TaskProcessor.scala index b0745f139..a49838c95 100644 --- a/api/app/processor/TaskProcessor.scala +++ b/api/app/processor/TaskProcessor.scala @@ -30,15 +30,6 @@ abstract class TaskProcessor( processRecord(task.typeId) } - final def queue(typeId: String, organizationGuid: Option[UUID]): Unit = { - args.dao.db.withConnection { c => - queue(c, typeId, organizationGuid = organizationGuid) - } - } - - final def queue(c: Connection, typeId: String, organizationGuid: Option[UUID] = None): Unit = { - insertIfNew(c, makeInitialTaskForm(typeId, organizationGuid, Json.obj())) - } } abstract class TaskProcessorWithGuid( @@ -89,11 +80,11 @@ abstract class TaskProcessorWithData[T]( final def queue(typeId: String, organizationGuid: Option[UUID] = None, data: T): Unit = { args.dao.db.withConnection { c => - queue(c, typeId, organizationGuid, data) + queueWithConnection(c, typeId, organizationGuid, data) } } - final def queue(c: Connection, typeId: String, organizationGuid: Option[UUID] = None, data: T): Unit = { + final def queueWithConnection(c: Connection, typeId: String, organizationGuid: Option[UUID] = None, data: T): Unit = { insertIfNew(c, makeInitialTaskForm(typeId, organizationGuid, Json.toJson(data).asInstanceOf[JsObject])) } From 0ada0d5bbbb10ee373462fcf526d193c6cc8d108 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 18:59:35 -0700 Subject: [PATCH 12/32] wip --- api/app/actors/Bindings.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/app/actors/Bindings.scala b/api/app/actors/Bindings.scala index 5a8f25832..13ff12acc 100644 --- a/api/app/actors/Bindings.scala +++ b/api/app/actors/Bindings.scala @@ -8,7 +8,11 @@ class ActorsModule extends AbstractModule with AkkaGuiceSupport { bindActor[MainActor]("main-actor") bindActor[GeneratorServiceActor]("generator-service-actor") bindActor[EmailActor]("email-actor") - bindActor[TaskActor]("task-actor") bindActor[UserActor]("user-actor") + bindActor[TaskDispatchActor]( + "TaskDispatchActor", + _.withDispatcher("task-context-dispatcher") + ) + bindActorFactory[TaskActor, actors.TaskActor.Factory] } } From b2467e338302dd2e84fc91f470b46df1bc38e0b2 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:04:56 -0700 Subject: [PATCH 13/32] wip --- api/test/actors/TaskActorSpec.scala | 14 +- api/test/db/TasksDaoSpec.scala | 232 ------------------ .../IndexApplicationProcessorSpec.scala} | 32 ++- api/test/util/Daos.scala | 4 +- 4 files changed, 28 insertions(+), 254 deletions(-) delete mode 100644 api/test/db/TasksDaoSpec.scala rename api/test/{actors/SearchSpec.scala => processor/IndexApplicationProcessorSpec.scala} (76%) diff --git a/api/test/actors/TaskActorSpec.scala b/api/test/actors/TaskActorSpec.scala index de664066d..b1a7c8b6c 100644 --- a/api/test/actors/TaskActorSpec.scala +++ b/api/test/actors/TaskActorSpec.scala @@ -1,14 +1,16 @@ package actors -import helpers.{AsyncHelpers, DbHelpers, DefaultAppSpec, ExportHelpers} -import util.DbAuthorization +import db.Authorization +import helpers.AsyncHelpers +import org.scalatestplus.play.PlaySpec +import org.scalatestplus.play.guice.GuiceOneAppPerSuite -class TaskActorSpec extends DefaultAppSpec with AsyncHelpers with ExportHelpers with DbHelpers { +class TaskActorSpec extends PlaySpec with GuiceOneAppPerSuite with AsyncHelpers with db.Helpers { - "processes export" in { - val exp = createDefaultExport() + "run" in { + val app = createApplication() eventuallyInNSeconds(10) { - internalExportsDao.findById(DbAuthorization.All, exp.id).value.db.processedAt.value + itemsDao.findByGuid(Authorization.All, app.guid).value } } diff --git a/api/test/db/TasksDaoSpec.scala b/api/test/db/TasksDaoSpec.scala deleted file mode 100644 index 927d3c521..000000000 --- a/api/test/db/TasksDaoSpec.scala +++ /dev/null @@ -1,232 +0,0 @@ -package db - -import java.util.UUID - -import anorm._ -import io.apibuilder.api.v0.models.User -import io.apibuilder.internal.v0.models.{Task, TaskDataDiffVersion} -import org.joda.time.DateTime -import org.postgresql.util.PSQLException -import org.scalatestplus.play.PlaySpec -import org.scalatestplus.play.guice.GuiceOneAppPerSuite -import play.api.db._ - -class TasksDaoSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { - - private[this] def setDeletedAt(task: Task, days: Int): Unit = { - val query = s""" - update tasks set deleted_at = timezone('utc', now()) - interval '$days days' where guid = {guid}::uuid - """ - - injector.instanceOf[DBApi].database("default").withConnection { implicit c => - SQL(query).on("guid" -> task.guid).execute() - } - } - - private[this] lazy val user: User = createRandomUser() - - private[this] def createTaskDataDiffVersion( - oldGuid: UUID = UUID.randomUUID, - newGuid: UUID = UUID.randomUUID, - numberAttempts: Int = 0 - ): Task = { - val guid = injector.instanceOf[DBApi].database("default").withConnection { implicit c => - tasksDao.insert(c, user, TaskDataDiffVersion(oldGuid, newGuid)) - } - - val task = tasksDao.findByGuid(guid).getOrElse { - sys.error("failed to find task") - } - - (0 to numberAttempts).foreach { _ => - tasksDao.incrementNumberAttempts(user, task) - } - - tasksDao.findByGuid(guid).getOrElse { - sys.error("failed to create task") - } - } - - "findByGuid" in { - val oldGuid = UUID.randomUUID - val newGuid = UUID.randomUUID - createTaskDataDiffVersion(oldGuid, newGuid).data must be(TaskDataDiffVersion(oldGuid, newGuid)) - } - - "softDelete" in { - val task = createTaskDataDiffVersion() - tasksDao.softDelete(user, task) - tasksDao.findByGuid(task.guid) must be(None) - } - - "incrementNumberAttempts" in { - val task = createTaskDataDiffVersion() - val original = task.numberAttempts - tasksDao.incrementNumberAttempts(user, task) - tasksDao.findByGuid(task.guid).getOrElse { - sys.error("failed to find task") - }.numberAttempts must be(original + 1) - } - - "recordError" in { - val task = createTaskDataDiffVersion() - tasksDao.recordError(user, task, "Test") - tasksDao.findByGuid(task.guid).getOrElse { - sys.error("failed to find task") - }.lastError must be(Some("Test")) - } - - "findAll" must { - - "nOrFewerAttempts" in { - val task = createTaskDataDiffVersion(numberAttempts = 2) - - tasksDao.findAll( - guid = Some(task.guid), - nOrFewerAttempts = Some(task.numberAttempts) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - nOrFewerAttempts = Some(task.numberAttempts - 1) - ).map(_.guid) must be(Nil) - } - - "nOrMoreAttempts" in { - val task = createTaskDataDiffVersion(numberAttempts = 2) - - tasksDao.findAll( - guid = Some(task.guid), - nOrMoreAttempts = Some(task.numberAttempts) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - nOrMoreAttempts = Some(task.numberAttempts + 1) - ).map(_.guid) must be(Nil) - } - - "nOrMoreMinutesOld" in { - val task = createTaskDataDiffVersion() - - tasksDao.findAll( - guid = Some(task.guid), - createdOnOrBefore = Some(DateTime.now.plusHours(1)) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - createdOnOrBefore = Some(DateTime.now.minusHours(1)) - ).map(_.guid) must be(Nil) - } - - "nOrMoreMinutesYoung" in { - val task = createTaskDataDiffVersion() - - tasksDao.findAll( - guid = Some(task.guid), - createdOnOrAfter = Some(DateTime.now.minusHours(1)) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - createdOnOrAfter = Some(DateTime.now.plusHours(1)) - ).map(_.guid) must be(Nil) - } - - "isDeleted" in { - val task = createTaskDataDiffVersion() - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = Some(false) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = Some(true) - ).map(_.guid) must be(Nil) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.softDelete(user, task) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = Some(false) - ).map(_.guid) must be(Nil) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = Some(true) - ).map(_.guid) must be(Seq(task.guid)) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None - ).map(_.guid) must be(Seq(task.guid)) - - } - - "deletedAtLeastNDaysAgo" in { - val task = createTaskDataDiffVersion() - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None, - deletedAtLeastNDaysAgo = Some(0) - ) must be(Nil) - - tasksDao.softDelete(user, task) - - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None, - deletedAtLeastNDaysAgo = Some(90) - ).map(_.guid) must be(Nil) - - setDeletedAt(task, 89) - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None, - deletedAtLeastNDaysAgo = Some(90) - ) must be(Nil) - - setDeletedAt(task, 91) - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None, - deletedAtLeastNDaysAgo = Some(90) - ).map(_.guid) must be(Seq(task.guid)) - } - } - - "purge" must { - - "raises error if recently deleted" in { - val task = createTaskDataDiffVersion() - tasksDao.softDelete(user, task) - val ex = intercept[PSQLException] { - tasksDao.purge(task) - } - println(ex.getMessage) - ex.getMessage.contains("ERROR: Physical deletes on this table can occur only after 1 month of deleting the records") must be(true) - } - - "purges if old" in { - val task = createTaskDataDiffVersion() - tasksDao.softDelete(user, task) - setDeletedAt(task, 45) - tasksDao.purge(task) - tasksDao.findAll( - guid = Some(task.guid), - isDeleted = None - ) must be(Nil) - } - - } - -} diff --git a/api/test/actors/SearchSpec.scala b/api/test/processor/IndexApplicationProcessorSpec.scala similarity index 76% rename from api/test/actors/SearchSpec.scala rename to api/test/processor/IndexApplicationProcessorSpec.scala index a3b02b2ba..f8b6ec79b 100644 --- a/api/test/actors/SearchSpec.scala +++ b/api/test/processor/IndexApplicationProcessorSpec.scala @@ -1,12 +1,21 @@ -package actors - -import java.util.UUID +package processor import db.Authorization +import lib.TestHelper import org.scalatestplus.play.PlaySpec import org.scalatestplus.play.guice.GuiceOneAppPerSuite -class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { +import java.util.UUID + +class IndexApplicationProcessorSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers with TestHelper { + + private[this] def processor = injector.instanceOf[IndexApplicationProcessor] + + private[this] def indexApplication(guid: UUID): Unit = { + expectValid { + processor.processRecord(guid) + } + } "indexApplication" must { @@ -14,7 +23,7 @@ class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { val description = UUID.randomUUID.toString val form = createApplicationForm().copy(description = Some(description)) val app = createApplication(form = form) - search.indexApplication(app.guid) + indexApplication(app.guid) Seq( app.name, @@ -31,8 +40,7 @@ class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { "on create" in { val app = createApplication() - - search.indexApplication(app.guid) + indexApplication(app.guid) itemsDao.findAll(Authorization.All, guid = Some(app.guid) @@ -42,8 +50,7 @@ class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { "on update" in { val form = createApplicationForm() val app = createApplication(form = form) - - search.indexApplication(app.guid) + indexApplication(app.guid) val newName = app.name + "2" @@ -52,8 +59,7 @@ class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { app = app, form = form.copy(name = newName) ) - - search.indexApplication(app.guid) + indexApplication(app.guid) val existing = applicationsDao.findByGuid(Authorization.All, app.guid).get @@ -65,10 +71,10 @@ class SearchSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { "on delete" in { val app = createApplication() - search.indexApplication(app.guid) + indexApplication(app.guid) applicationsDao.softDelete(testUser, app) - search.indexApplication(app.guid) + indexApplication(app.guid) itemsDao.findAll(Authorization.All, guid = Some(app.guid)) must be(Nil) } diff --git a/api/test/util/Daos.scala b/api/test/util/Daos.scala index 91c7bff35..537b3caae 100644 --- a/api/test/util/Daos.scala +++ b/api/test/util/Daos.scala @@ -1,9 +1,9 @@ package util import actors.Emails -import db.{ApplicationsDao, AttributesDao, ChangesDao, EmailVerificationsDao, ItemsDao, MembershipRequestsDao, MembershipsDao, OrganizationAttributeValuesDao, OrganizationDomainsDao, OrganizationLogsDao, OrganizationsDao, OriginalsDao, PasswordResetRequestsDao, SubscriptionsDao, TasksDao, TokensDao, UserPasswordsDao, UsersDao, VersionsDao} import db.generated.SessionsDao import db.generators.{GeneratorsDao, ServicesDao} +import db._ import lib.DatabaseServiceFetcher import play.api.Application import play.api.inject.Injector @@ -31,7 +31,6 @@ trait Daos { def sessionsDao: SessionsDao = injector.instanceOf[SessionsDao] def subscriptionsDao: SubscriptionsDao = injector.instanceOf[db.SubscriptionsDao] - def tasksDao: TasksDao = injector.instanceOf[db.TasksDao] def tokensDao: TokensDao = injector.instanceOf[db.TokensDao] def userPasswordsDao: UserPasswordsDao = injector.instanceOf[db.UserPasswordsDao] def versionsDao: VersionsDao = injector.instanceOf[db.VersionsDao] @@ -40,7 +39,6 @@ trait Daos { def generatorsDao: GeneratorsDao = injector.instanceOf[db.generators.GeneratorsDao] def emails: Emails = injector.instanceOf[actors.Emails] - def search: Search = injector.instanceOf[actors.Search] def sessionHelper: SessionHelper = injector.instanceOf[SessionHelper] } From 3e97b240430473e5005374a6afc4ceea3869304a Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:07:22 -0700 Subject: [PATCH 14/32] wip --- api/app/actors/TaskDispatchActor.scala | 2 +- api/app/db/ApplicationsDao.scala | 6 +++--- api/app/db/VersionsDao.scala | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/app/actors/TaskDispatchActor.scala b/api/app/actors/TaskDispatchActor.scala index cdce5ea02..974e9127b 100644 --- a/api/app/actors/TaskDispatchActor.scala +++ b/api/app/actors/TaskDispatchActor.scala @@ -52,7 +52,7 @@ class TaskDispatchActor @Inject() ( val ref = injectedChild( factory(typ), name = name, - _.withDispatcher("task-context-dispatch") + _.withDispatcher("task-context-dispatcher") ) actors += (typ -> ref) ref diff --git a/api/app/db/ApplicationsDao.scala b/api/app/db/ApplicationsDao.scala index ab844daa4..82b6eca26 100644 --- a/api/app/db/ApplicationsDao.scala +++ b/api/app/db/ApplicationsDao.scala @@ -2,10 +2,10 @@ package db import anorm._ import io.apibuilder.api.v0.models.{AppSortBy, Application, ApplicationForm, Error, MoveForm, Organization, SortOrder, User, Version, Visibility} +import io.apibuilder.task.v0.models.TaskType import io.flow.postgresql.Query import lib.{UrlKey, Validation} import play.api.db._ -import processor.IndexApplicationProcessor import java.util.UUID import javax.inject.{Inject, Named, Singleton} @@ -15,7 +15,7 @@ class ApplicationsDao @Inject() ( @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, organizationsDao: OrganizationsDao, - indexApplicationProcessor: IndexApplicationProcessor, + tasksDao: InternalTasksDao, ) { private[this] val dbHelpers = DbHelpers(db, "applications") @@ -364,7 +364,7 @@ class ApplicationsDao @Inject() ( ): Unit = { db.withTransaction { implicit c => f(c) - indexApplicationProcessor.queue(c, guid) + tasksDao.queueWithConnection(c, TaskType.IndexApplication, guid.toString) } } diff --git a/api/app/db/VersionsDao.scala b/api/app/db/VersionsDao.scala index 89f24a610..8c2a1681e 100644 --- a/api/app/db/VersionsDao.scala +++ b/api/app/db/VersionsDao.scala @@ -11,12 +11,12 @@ import io.apibuilder.common.v0.models.{Audit, Reference} import io.apibuilder.spec.v0.models.Service import io.apibuilder.spec.v0.models.json._ import io.apibuilder.task.v0.models._ +import io.apibuilder.task.v0.models.json._ import io.flow.postgresql.Query import lib.{ServiceConfiguration, ServiceUri, ValidatedHelpers, VersionTag} import play.api.Logger import play.api.db._ import play.api.libs.json._ -import processor.{DiffVersionProcessor, IndexApplicationProcessor} import java.util.UUID import javax.inject.Inject @@ -25,8 +25,7 @@ class VersionsDao @Inject() ( @NamedDatabase("default") db: Database, applicationsDao: ApplicationsDao, originalsDao: OriginalsDao, - diffVersionProcessor: DiffVersionProcessor, - indexApplicationProcessor: IndexApplicationProcessor, + tasksDao: InternalTasksDao, usersDao: UsersDao, organizationsDao: OrganizationsDao, serviceParser: ServiceParser @@ -163,10 +162,11 @@ class VersionsDao @Inject() ( ) ( implicit c: java.sql.Connection ): Unit = { - diffVersionProcessor.queueWithConnection( + tasksDao.queueWithConnection( c, + TaskType.DiffVersion, newVersionGuid.toString, - data = DiffVersionData(oldVersionGuid = oldVersionGuid, newVersionGuid = newVersionGuid) + data = Json.toJson(DiffVersionData(oldVersionGuid = oldVersionGuid, newVersionGuid = newVersionGuid)) ) } @@ -175,7 +175,7 @@ class VersionsDao @Inject() ( softDelete(user, version) val versionGuid = doCreate(c, user, application, version.version, original, service) createDiffTask(version.guid, versionGuid) - indexApplicationProcessor.queue(c, application.guid) + tasksDao.queueWithConnection(c, TaskType.IndexApplication, application.guid.toString) versionGuid } From aac1f11a1eba9e11894feb277688df66fe72a0b3 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:07:49 -0700 Subject: [PATCH 15/32] wip --- api/test/helpers/AsyncHelpers.scala | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 api/test/helpers/AsyncHelpers.scala diff --git a/api/test/helpers/AsyncHelpers.scala b/api/test/helpers/AsyncHelpers.scala new file mode 100644 index 000000000..964cdb79a --- /dev/null +++ b/api/test/helpers/AsyncHelpers.scala @@ -0,0 +1,16 @@ +package helpers + +import lib.TestHelper +import org.scalatest.concurrent.Eventually +import org.scalatest.concurrent.PatienceConfiguration.Timeout +import org.scalatest.time.{Seconds, Span} + +trait AsyncHelpers extends TestHelper with Eventually { + + def eventuallyInNSeconds[T](n: Long = 3)(f: => T): T = { + eventually(Timeout(Span(n, Seconds))) { + f + } + } + +} From b62a656b0345995fc062e065af5626edc765f1ba Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:14:58 -0700 Subject: [PATCH 16/32] wip --- api/app/actors/MainActor.scala | 11 +--- .../CleanupDeletionsProcessor.scala} | 50 +++++++++++-------- api/app/processor/TaskActorCompanion.scala | 2 + api/app/processor/TaskProcessor.scala | 22 ++------ .../CleanupDeletionsProcessorSpec.scala} | 22 ++++---- .../ApicollectiveApibuilderTaskV0Client.scala | 3 +- spec/apibuilder-task.json | 1 + 7 files changed, 49 insertions(+), 62 deletions(-) rename api/app/{util/ProcessDeletes.scala => processor/CleanupDeletionsProcessor.scala} (65%) rename api/test/{util/ProcessDeletesSpec.scala => processor/CleanupDeletionsProcessorSpec.scala} (83%) diff --git a/api/app/actors/MainActor.scala b/api/app/actors/MainActor.scala index 17146d564..207135230 100644 --- a/api/app/actors/MainActor.scala +++ b/api/app/actors/MainActor.scala @@ -4,7 +4,6 @@ import akka.actor._ import db.InternalMigrationsDao import lib.Role import play.api.Mode -import util.ProcessDeletes import java.util.UUID import scala.concurrent.ExecutionContext @@ -32,7 +31,6 @@ class MainActor @javax.inject.Inject() ( app: play.api.Application, system: ActorSystem, internalMigrationsDao: InternalMigrationsDao, - processDeletes: ProcessDeletes, @javax.inject.Named("email-actor") emailActor: akka.actor.ActorRef, @javax.inject.Named("generator-service-actor") generatorServiceActor: akka.actor.ActorRef, @javax.inject.Named("user-actor") userActor: akka.actor.ActorRef @@ -42,7 +40,6 @@ class MainActor @javax.inject.Inject() ( private[this] case object QueueVersionsToMigrate private[this] case object MigrateVersions - private[this] case object CleanupDeletedApplications private[this] def scheduleOnce(msg: Any)(implicit delay: FiniteDuration = FiniteDuration(10, SECONDS)): Unit = { system.scheduler.scheduleOnce(delay) { @@ -51,7 +48,7 @@ class MainActor @javax.inject.Inject() ( } scheduleOnce(QueueVersionsToMigrate) - scheduleOnce(CleanupDeletedApplications) + scheduleOnce(MigrateVersions) def receive: Receive = akka.event.LoggingReceive { @@ -110,12 +107,6 @@ class MainActor @javax.inject.Inject() ( } } - case m @ CleanupDeletedApplications => withVerboseErrorHandler(m) { - println(s"DEBUG_CleanupDeletedApplications - STARTING") - processDeletes.all() - scheduleOnce(MigrateVersions) - } - case m: Any => logUnhandledMessage(m) } } diff --git a/api/app/util/ProcessDeletes.scala b/api/app/processor/CleanupDeletionsProcessor.scala similarity index 65% rename from api/app/util/ProcessDeletes.scala rename to api/app/processor/CleanupDeletionsProcessor.scala index 50797cead..93faa847e 100644 --- a/api/app/util/ProcessDeletes.scala +++ b/api/app/processor/CleanupDeletionsProcessor.scala @@ -1,12 +1,15 @@ -package util +package processor +import cats.data.ValidatedNec +import cats.implicits._ import db.UsersDao +import io.apibuilder.task.v0.models.TaskType import io.flow.postgresql.Query import play.api.db.Database import javax.inject.Inject -object ProcessDeletes { +object DeleteMetadata { val OrganizationSoft: Seq[String] = Seq( "public.applications", "public.membership_requests", @@ -16,7 +19,7 @@ object ProcessDeletes { "public.subscriptions" ) val OrganizationHard: Seq[String] = Seq( - "public.organization_logs", "search.items" + "public.organization_logs", "public.tasks", "search.items" ) val ApplicationSoft: Seq[String] = Seq( "public.application_moves", "public.changes", "public.versions", "public.watches" @@ -28,27 +31,30 @@ object ProcessDeletes { val VersionHard: Seq[String] = Seq("public.migrations") } -class ProcessDeletes @Inject() ( - db: Database, - usersDao: UsersDao - ) { - import ProcessDeletes._ +class CleanupDeletionsProcessor @Inject()( + args: TaskProcessorArgs, + db: Database, + usersDao: UsersDao +) extends TaskProcessor(args, TaskType.CleanupDeletions) { + import DeleteMetadata._ - def all(): Unit = { + override def processRecord(id: String): ValidatedNec[String, Unit] = { organizations() applications() versions() + ().validNec } - private[util] def organizations(): Unit = { + + private[processor] def organizations(): Unit = { OrganizationSoft.foreach { table => exec( s""" - |update $table - | set deleted_at=now(),deleted_by_guid={deleted_by_guid}::uuid - | where deleted_at is null - | and organization_guid in (select guid from organizations where deleted_at is not null) - |""".stripMargin + |update $table + | set deleted_at=now(),deleted_by_guid={deleted_by_guid}::uuid + | where deleted_at is null + | and organization_guid in (select guid from organizations where deleted_at is not null) + |""".stripMargin ) } @@ -62,15 +68,15 @@ class ProcessDeletes @Inject() ( } } - private[util] def applications(): Unit = { + private[processor] def applications(): Unit = { ApplicationSoft.foreach { table => exec( s""" - |update $table - | set deleted_at=now(),deleted_by_guid={deleted_by_guid}::uuid - | where deleted_at is null - | and application_guid in (select guid from applications where deleted_at is not null) - |""".stripMargin + |update $table + | set deleted_at=now(),deleted_by_guid={deleted_by_guid}::uuid + | where deleted_at is null + | and application_guid in (select guid from applications where deleted_at is not null) + |""".stripMargin ) } @@ -84,7 +90,7 @@ class ProcessDeletes @Inject() ( } } - private[util] def versions(): Unit = { + private[processor] def versions(): Unit = { VersionSoft.foreach { table => exec( s""" diff --git a/api/app/processor/TaskActorCompanion.scala b/api/app/processor/TaskActorCompanion.scala index 68d2976b7..4454c50d1 100644 --- a/api/app/processor/TaskActorCompanion.scala +++ b/api/app/processor/TaskActorCompanion.scala @@ -7,6 +7,7 @@ import javax.inject.Inject class TaskActorCompanion @Inject() ( indexApplication: IndexApplicationProcessor, diffVersion: DiffVersionProcessor, + cleanupDeletions: CleanupDeletionsProcessor ) { def process(typ: TaskType): Unit = { @@ -17,6 +18,7 @@ class TaskActorCompanion @Inject() ( import TaskType._ typ match { case IndexApplication => indexApplication + case CleanupDeletions => cleanupDeletions case DiffVersion => diffVersion case UNDEFINED(_) => sys.error(s"Undefined task type '$typ") } diff --git a/api/app/processor/TaskProcessor.scala b/api/app/processor/TaskProcessor.scala index a49838c95..9aea180cd 100644 --- a/api/app/processor/TaskProcessor.scala +++ b/api/app/processor/TaskProcessor.scala @@ -8,7 +8,7 @@ import io.apibuilder.task.v0.models.TaskType import io.flow.postgresql.OrderBy import lib.Constants import org.joda.time.DateTime -import play.api.libs.json.{JsObject, JsValue, Json, Reads, Writes} +import play.api.libs.json.{JsObject, JsValue, Reads, Writes} import play.libs.exception.ExceptionUtils import java.sql.Connection @@ -35,16 +35,12 @@ abstract class TaskProcessor( abstract class TaskProcessorWithGuid( args: TaskProcessorArgs, typ: TaskType - ) extends TaskProcessor(args, typ) { + ) extends BaseTaskProcessor(args, typ) { def processRecord(guid: UUID): ValidatedNec[String, Unit] - final def queue(c: Connection, typeId: UUID, organizationGuid: Option[UUID] = None): Unit = { - insertIfNew(c, makeInitialTaskForm(typeId.toString, organizationGuid, Json.obj())) - } - - override final def processRecord(id: String): ValidatedNec[String, Unit] = { - validateGuid(id).andThen(processRecord) + override def processTask(task: Task): ValidatedNec[String, Unit] = { + validateGuid(task.typeId).andThen(processRecord) } private[this] def validateGuid(value: String): ValidatedNec[String, UUID] = { @@ -78,16 +74,6 @@ abstract class TaskProcessorWithData[T]( } } - final def queue(typeId: String, organizationGuid: Option[UUID] = None, data: T): Unit = { - args.dao.db.withConnection { c => - queueWithConnection(c, typeId, organizationGuid, data) - } - } - - final def queueWithConnection(c: Connection, typeId: String, organizationGuid: Option[UUID] = None, data: T): Unit = { - insertIfNew(c, makeInitialTaskForm(typeId, organizationGuid, Json.toJson(data).asInstanceOf[JsObject])) - } - } abstract class BaseTaskProcessor( diff --git a/api/test/util/ProcessDeletesSpec.scala b/api/test/processor/CleanupDeletionsProcessorSpec.scala similarity index 83% rename from api/test/util/ProcessDeletesSpec.scala rename to api/test/processor/CleanupDeletionsProcessorSpec.scala index 94fbf8280..31369fee7 100644 --- a/api/test/util/ProcessDeletesSpec.scala +++ b/api/test/processor/CleanupDeletionsProcessorSpec.scala @@ -1,4 +1,4 @@ -package util +package processor import anorm.SqlParser import db.Helpers @@ -10,9 +10,9 @@ import play.api.db.Database import java.util.UUID -class ProcessDeletesSpec extends PlaySpec with GuiceOneAppPerSuite with Helpers { +class CleanupDeletionsProcessorSpec extends PlaySpec with GuiceOneAppPerSuite with Helpers { - private[this] def processDeletes: ProcessDeletes = app.injector.instanceOf[ProcessDeletes] + private[this] def processor: CleanupDeletionsProcessor = app.injector.instanceOf[CleanupDeletionsProcessor] private[this] def database: Database = app.injector.instanceOf[Database] private[this] def isDeleted(table: String, guid: UUID): Boolean = { @@ -39,7 +39,7 @@ class ProcessDeletesSpec extends PlaySpec with GuiceOneAppPerSuite with Helpers isAppDeleted(app) mustBe false isAppDeleted(appDeleted) mustBe false - processDeletes.organizations() + processor.organizations() isAppDeleted(app) mustBe false isAppDeleted(appDeleted) mustBe true } @@ -61,7 +61,7 @@ class ProcessDeletesSpec extends PlaySpec with GuiceOneAppPerSuite with Helpers isVersionDeleted(version) mustBe false isVersionDeleted(versionDeleted) mustBe false - processDeletes.applications() + processor.applications() isVersionDeleted(version) mustBe false isVersionDeleted(versionDeleted) mustBe true } @@ -102,28 +102,28 @@ class ProcessDeletesSpec extends PlaySpec with GuiceOneAppPerSuite with Helpers "Organization" must { "soft" in { - getTablesSoft("organization_guid") mustBe ProcessDeletes.OrganizationSoft + getTablesSoft("organization_guid") mustBe DeleteMetadata.OrganizationSoft } "hard" in { - getTablesHard("organization_guid") mustBe ProcessDeletes.OrganizationHard + getTablesHard("organization_guid") mustBe DeleteMetadata.OrganizationHard } } "Application" must { "soft" in { - getTablesSoft("application_guid") mustBe ProcessDeletes.ApplicationSoft + getTablesSoft("application_guid") mustBe DeleteMetadata.ApplicationSoft } "hard" in { - getTablesHard("application_guid") mustBe ProcessDeletes.ApplicationHard + getTablesHard("application_guid") mustBe DeleteMetadata.ApplicationHard } } "Version" must { "soft" in { - getTablesSoft("version_guid") mustBe ProcessDeletes.VersionSoft + getTablesSoft("version_guid") mustBe DeleteMetadata.VersionSoft } "hard" in { - getTablesHard("version_guid") mustBe ProcessDeletes.VersionHard + getTablesHard("version_guid") mustBe DeleteMetadata.VersionHard } } } diff --git a/generated/app/ApicollectiveApibuilderTaskV0Client.scala b/generated/app/ApicollectiveApibuilderTaskV0Client.scala index 27765783e..ff29f9969 100644 --- a/generated/app/ApicollectiveApibuilderTaskV0Client.scala +++ b/generated/app/ApicollectiveApibuilderTaskV0Client.scala @@ -14,6 +14,7 @@ package io.apibuilder.task.v0.models { object TaskType { case object IndexApplication extends TaskType { override def toString = "index_application" } + case object CleanupDeletions extends TaskType { override def toString = "cleanup_deletions" } case object DiffVersion extends TaskType { override def toString = "diff_version" } /** * UNDEFINED captures values that are sent either in error or @@ -31,7 +32,7 @@ package io.apibuilder.task.v0.models { * lower case to avoid collisions with the camel cased values * above. */ - val all: scala.List[TaskType] = scala.List(IndexApplication, DiffVersion) + val all: scala.List[TaskType] = scala.List(IndexApplication, CleanupDeletions, DiffVersion) private[this] val byName: Map[String, TaskType] = all.map(x => x.toString.toLowerCase -> x).toMap diff --git a/spec/apibuilder-task.json b/spec/apibuilder-task.json index d3a59fecc..b9faeef7f 100644 --- a/spec/apibuilder-task.json +++ b/spec/apibuilder-task.json @@ -17,6 +17,7 @@ "task_type": { "values": [ { "name": "index_application" }, + { "name": "cleanup_deletions" }, { "name": "diff_version" } ] } From a0e74702f32428370ff81187b6ed0d79ef7242d4 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:17:46 -0700 Subject: [PATCH 17/32] wip --- api/app/actors/Bindings.scala | 1 + api/app/actors/MainActor.scala | 5 --- api/app/actors/PeriodicActor.scala | 51 ++++++++++++++++++++++++++++++ api/app/db/InternalTasksDao.scala | 2 +- api/conf/base.conf | 7 ++++ 5 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 api/app/actors/PeriodicActor.scala diff --git a/api/app/actors/Bindings.scala b/api/app/actors/Bindings.scala index 13ff12acc..8c2f2848a 100644 --- a/api/app/actors/Bindings.scala +++ b/api/app/actors/Bindings.scala @@ -6,6 +6,7 @@ import play.api.libs.concurrent.AkkaGuiceSupport class ActorsModule extends AbstractModule with AkkaGuiceSupport { override def configure = { bindActor[MainActor]("main-actor") + bindActor[PeriodicActor]("PeriodicActor") bindActor[GeneratorServiceActor]("generator-service-actor") bindActor[EmailActor]("email-actor") bindActor[UserActor]("user-actor") diff --git a/api/app/actors/MainActor.scala b/api/app/actors/MainActor.scala index 207135230..9bccad163 100644 --- a/api/app/actors/MainActor.scala +++ b/api/app/actors/MainActor.scala @@ -38,17 +38,12 @@ class MainActor @javax.inject.Inject() ( private[this] implicit val ec: ExecutionContext = system.dispatchers.lookup("main-actor-context") - private[this] case object QueueVersionsToMigrate - private[this] case object MigrateVersions - private[this] def scheduleOnce(msg: Any)(implicit delay: FiniteDuration = FiniteDuration(10, SECONDS)): Unit = { system.scheduler.scheduleOnce(delay) { self ! msg } } - scheduleOnce(QueueVersionsToMigrate) - scheduleOnce(MigrateVersions) def receive: Receive = akka.event.LoggingReceive { diff --git a/api/app/actors/PeriodicActor.scala b/api/app/actors/PeriodicActor.scala new file mode 100644 index 000000000..4154a5b07 --- /dev/null +++ b/api/app/actors/PeriodicActor.scala @@ -0,0 +1,51 @@ +package actors + +import akka.actor.{Actor, ActorLogging, Cancellable} +import db.InternalTasksDao +import io.apibuilder.task.v0.models.TaskType +import play.api.{Environment, Mode} + +import javax.inject.Inject +import scala.concurrent.ExecutionContext +import scala.concurrent.duration.{FiniteDuration, HOURS} + +class PeriodicActor @Inject() ( + tasksDao: InternalTasksDao, + env: Environment +) extends Actor with ActorLogging with ErrorHandler { + + private[this] implicit val ec: ExecutionContext = context.system.dispatchers.lookup("periodic-actor-context") + + private[this] case class UpsertTask(typ: TaskType) + + private[this] def schedule( + taskType: TaskType, + interval: FiniteDuration + )(implicit + initialInterval: FiniteDuration = interval + ): Cancellable = { + val finalInitial = env.mode match { + case Mode.Test => FiniteDuration(24, HOURS) + case _ => initialInterval + } + context.system.scheduler.scheduleWithFixedDelay(finalInitial, interval, self, UpsertTask(taskType)) + } + + private[this] val cancellables: Seq[Cancellable] = { + import TaskType._ + Seq( + schedule(CleanupDeletions, FiniteDuration(1, HOURS)), + ) + } + + override def postStop(): Unit = { + cancellables.foreach(_.cancel()) + super.postStop() + } + + override def receive: Receive = { + case UpsertTask(taskType) => tasksDao.queue(taskType, "periodic") + case other => logUnhandledMessage(other) + } + +} diff --git a/api/app/db/InternalTasksDao.scala b/api/app/db/InternalTasksDao.scala index 1bea116cd..4d402c851 100644 --- a/api/app/db/InternalTasksDao.scala +++ b/api/app/db/InternalTasksDao.scala @@ -17,7 +17,7 @@ class InternalTasksDao @Inject() ( dao.findByTypeIdAndType(typeId, typ.toString) } - def queue(typ: TaskType, id: String, organizationGuid: Option[UUID], data: JsValue = Json.obj()): Unit = { + def queue(typ: TaskType, id: String, organizationGuid: Option[UUID] = None, data: JsValue = Json.obj()): Unit = { dao.db.withConnection { c => queueWithConnection(c, typ, id, organizationGuid = organizationGuid, data = data) } diff --git a/api/conf/base.conf b/api/conf/base.conf index 7698a3b26..dc0ba7fd4 100644 --- a/api/conf/base.conf +++ b/api/conf/base.conf @@ -53,6 +53,13 @@ task-context { } } +periodic-actor-context { + fork-join-executor { + parallelism-factor = 2.0 + parallelism-max = 2 + } +} + task-context-dispatcher { fork-join-executor { parallelism-factor = 2.0 From b4dec4e0c9b7dcdf4739e90e4c2b127d01020f53 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:27:10 -0700 Subject: [PATCH 18/32] wip --- api/app/actors/MainActor.scala | 23 - api/app/actors/PeriodicActor.scala | 1 + api/app/db/InternalMigrationsDao.scala | 97 ---- .../PsqlApibuilderMigrationsDao.scala | 494 ------------------ .../processor/MigrateVersionProcessor.scala | 19 + .../ScheduleMigrateVersionsProcessor.scala | 59 +++ .../ApicollectiveApibuilderTaskV0Client.scala | 4 +- spec/apibuilder-task.json | 2 + 8 files changed, 84 insertions(+), 615 deletions(-) delete mode 100644 api/app/db/InternalMigrationsDao.scala delete mode 100644 api/app/db/generated/PsqlApibuilderMigrationsDao.scala create mode 100644 api/app/processor/MigrateVersionProcessor.scala create mode 100644 api/app/processor/ScheduleMigrateVersionsProcessor.scala diff --git a/api/app/actors/MainActor.scala b/api/app/actors/MainActor.scala index 9bccad163..18ae8c3bb 100644 --- a/api/app/actors/MainActor.scala +++ b/api/app/actors/MainActor.scala @@ -1,9 +1,7 @@ package actors import akka.actor._ -import db.InternalMigrationsDao import lib.Role -import play.api.Mode import java.util.UUID import scala.concurrent.ExecutionContext @@ -28,9 +26,7 @@ object MainActor { @javax.inject.Singleton class MainActor @javax.inject.Inject() ( - app: play.api.Application, system: ActorSystem, - internalMigrationsDao: InternalMigrationsDao, @javax.inject.Named("email-actor") emailActor: akka.actor.ActorRef, @javax.inject.Named("generator-service-actor") generatorServiceActor: akka.actor.ActorRef, @javax.inject.Named("user-actor") userActor: akka.actor.ActorRef @@ -83,25 +79,6 @@ class MainActor @javax.inject.Inject() ( userActor ! UserActor.Messages.UserCreated(guid) } - case m @ QueueVersionsToMigrate => withVerboseErrorHandler(m) { - app.mode match { - case Mode.Test => // No-op - case Mode.Prod | Mode.Dev => internalMigrationsDao.queueVersions() - } - } - - case m @ MigrateVersions => withVerboseErrorHandler(m) { - println(s"DEBUG_MigrateVersions - STARTING") - app.mode match { - case Mode.Test => // No-op - case Mode.Prod | Mode.Dev => { - if (internalMigrationsDao.migrateBatch(60)) { - scheduleOnce(MigrateVersions)(FiniteDuration(20, SECONDS)) - } - } - } - } - case m: Any => logUnhandledMessage(m) } } diff --git a/api/app/actors/PeriodicActor.scala b/api/app/actors/PeriodicActor.scala index 4154a5b07..bfa6c1abb 100644 --- a/api/app/actors/PeriodicActor.scala +++ b/api/app/actors/PeriodicActor.scala @@ -35,6 +35,7 @@ class PeriodicActor @Inject() ( import TaskType._ Seq( schedule(CleanupDeletions, FiniteDuration(1, HOURS)), + schedule(ScheduleMigrateVersions, FiniteDuration(12, HOURS)), ) } diff --git a/api/app/db/InternalMigrationsDao.scala b/api/app/db/InternalMigrationsDao.scala deleted file mode 100644 index 494b23a2e..000000000 --- a/api/app/db/InternalMigrationsDao.scala +++ /dev/null @@ -1,97 +0,0 @@ -package db - -import anorm._ -import cats.data.Validated.{Invalid, Valid} -import db.generated.{MigrationForm, MigrationsDao} -import io.flow.postgresql.Query -import lib.Constants -import org.joda.time.DateTime -import play.api.db._ - -import javax.inject.Inject -import scala.util.{Failure, Success, Try} - -/** - * TODO: Test this class - * Setup a separate actor to: - * - delete if service_version < latest - * - attempt migration for num_attempts = 0 - * - record errors - */ -object Migration { - val ServiceVersionNumber: String = io.apibuilder.spec.v0.Constants.Version.toLowerCase -} - -class InternalMigrationsDao @Inject()( - @NamedDatabase("default") db: Database, - migrationsDao: MigrationsDao, - versionsDao: VersionsDao -) { - - private[this] val VersionsNeedingUpgrade = Query( - """ - |select v.guid - | from versions v - | join applications apps on apps.guid = v.application_guid and apps.deleted_at is null - | left join migrations m on m.version_guid = v.guid - | where v.deleted_at is null - | and m.id is null - | and not exists ( - | select 1 - | from cache.services - | where services.deleted_at is null - | and services.version_guid = v.guid - | and services.version = {service_version} - | ) - |limit 250 - |""".stripMargin - ).bind("service_version", Migration.ServiceVersionNumber) - - def queueVersions(): Unit = { - val versionGuids = db.withConnection { implicit c => - VersionsNeedingUpgrade - .as(SqlParser.get[_root_.java.util.UUID](1).*) - } - if (versionGuids.nonEmpty) { - migrationsDao.insertBatch(Constants.DefaultUserGuid, versionGuids.map { vGuid => - MigrationForm( - versionGuid = vGuid, - numAttempts = 0, - errors = None - ) - }) - queueVersions() - } - } - - def migrateBatch(limit: Long): Boolean = { - println(s"[${DateTime.now}] DEBUG STARTING migrateBatch($limit)") - val all = migrationsDao.findAll( - numAttempts = Some(0), - limit = Some(limit) - ) - all.foreach { migration => - Try { - migrateOne(migration) - } match { - case Success(_) => () - case Failure(ex) => setErrors(migration, Seq(ex.getMessage)) - } - } - all.nonEmpty - } - - private[this] def migrateOne(migration: _root_.db.generated.Migration): Unit = { - versionsDao.migrateVersionGuid(migration.versionGuid) match { - case Valid(_) => migrationsDao.delete(Constants.DefaultUserGuid, migration) - case Invalid(e) => setErrors(migration, e.toNonEmptyList.toList) - } - } - - private[this] def setErrors(migration: _root_.db.generated.Migration, errors: Seq[String]): Unit = { - migrationsDao.update(Constants.DefaultUserGuid, migration, migration.form.copy( - numAttempts = migration.numAttempts + 1, - errors = Some(errors) - )) - } -} diff --git a/api/app/db/generated/PsqlApibuilderMigrationsDao.scala b/api/app/db/generated/PsqlApibuilderMigrationsDao.scala deleted file mode 100644 index 1fa057fda..000000000 --- a/api/app/db/generated/PsqlApibuilderMigrationsDao.scala +++ /dev/null @@ -1,494 +0,0 @@ -package db.generated - -import anorm._ -import io.flow.postgresql.{OrderBy, Query} -import java.sql.Connection -import java.util.UUID -import javax.inject.{Inject, Singleton} -import org.joda.time.DateTime -import play.api.db.Database -import play.api.libs.json.Json -import util.IdGenerator - -case class Migration( - id: String, - versionGuid: UUID, - numAttempts: Long, - errors: Option[Seq[String]], - updatedByGuid: String, - createdAt: DateTime, - updatedAt: DateTime -) { - - lazy val form: MigrationForm = MigrationForm( - versionGuid = versionGuid, - numAttempts = numAttempts, - errors = errors - ) - -} - -case class MigrationForm( - versionGuid: UUID, - numAttempts: Long, - errors: Option[Seq[String]] -) - -object MigrationsTable { - val Schema: String = "public" - val Name: String = "migrations" - val QualifiedName: String = s"$Schema.$Name" - - object Columns { - val Id: String = "id" - val VersionGuid: String = "version_guid" - val NumAttempts: String = "num_attempts" - val Errors: String = "errors" - val UpdatedByGuid: String = "updated_by_guid" - val CreatedAt: String = "created_at" - val UpdatedAt: String = "updated_at" - val HashCode: String = "hash_code" - val all: List[String] = List(Id, VersionGuid, NumAttempts, Errors, UpdatedByGuid, CreatedAt, UpdatedAt, HashCode) - } -} - -trait BaseMigrationsDao { - - def db: Database - - private[this] val BaseQuery = Query(""" - | select migrations.id, - | migrations.version_guid, - | migrations.num_attempts, - | migrations.errors::text as errors_text, - | migrations.updated_by_guid, - | migrations.created_at, - | migrations.updated_at, - | migrations.hash_code - | from migrations - """.stripMargin) - - def findById(id: String): Option[Migration] = { - db.withConnection { c => - findByIdWithConnection(c, id) - } - } - - def findByIdWithConnection(c: java.sql.Connection, id: String): Option[Migration] = { - findAllWithConnection(c, ids = Some(Seq(id)), limit = Some(1L), orderBy = None).headOption - } - - def iterateAll( - ids: Option[Seq[String]] = None, - versionGuid: Option[UUID] = None, - numAttempts: Option[Long] = None, - numAttemptsGreaterThanOrEquals: Option[Long] = None, - numAttemptsGreaterThan: Option[Long] = None, - numAttemptsLessThanOrEquals: Option[Long] = None, - numAttemptsLessThan: Option[Long] = None, - numAttemptses: Option[Seq[Long]] = None, - createdAt: Option[DateTime] = None, - createdAtGreaterThanOrEquals: Option[DateTime] = None, - createdAtGreaterThan: Option[DateTime] = None, - createdAtLessThanOrEquals: Option[DateTime] = None, - createdAtLessThan: Option[DateTime] = None, - numAttemptsCreatedAts: Option[Seq[(Long, DateTime)]] = None, - pageSize: Long = 2000L, - ) ( - implicit customQueryModifier: Query => Query = { q => q } - ): Iterator[Migration] = { - def iterate(lastValue: Option[Migration]): Iterator[Migration] = { - val page = findAll( - ids = ids, - versionGuid = versionGuid, - numAttempts = numAttempts, - numAttemptsGreaterThanOrEquals = numAttemptsGreaterThanOrEquals, - numAttemptsGreaterThan = numAttemptsGreaterThan, - numAttemptsLessThanOrEquals = numAttemptsLessThanOrEquals, - numAttemptsLessThan = numAttemptsLessThan, - numAttemptses = numAttemptses, - createdAt = createdAt, - createdAtGreaterThanOrEquals = createdAtGreaterThanOrEquals, - createdAtGreaterThan = createdAtGreaterThan, - createdAtLessThanOrEquals = createdAtLessThanOrEquals, - createdAtLessThan = createdAtLessThan, - numAttemptsCreatedAts = numAttemptsCreatedAts, - limit = Some(pageSize), - orderBy = Some(OrderBy("migrations.id")), - ) { q => customQueryModifier(q).greaterThan("migrations.id", lastValue.map(_.id)) } - - page.lastOption match { - case None => Iterator.empty - case lastValue => page.iterator ++ iterate(lastValue) - } - } - - iterate(None) - } - - def findAll( - ids: Option[Seq[String]] = None, - versionGuid: Option[UUID] = None, - numAttempts: Option[Long] = None, - numAttemptsGreaterThanOrEquals: Option[Long] = None, - numAttemptsGreaterThan: Option[Long] = None, - numAttemptsLessThanOrEquals: Option[Long] = None, - numAttemptsLessThan: Option[Long] = None, - numAttemptses: Option[Seq[Long]] = None, - createdAt: Option[DateTime] = None, - createdAtGreaterThanOrEquals: Option[DateTime] = None, - createdAtGreaterThan: Option[DateTime] = None, - createdAtLessThanOrEquals: Option[DateTime] = None, - createdAtLessThan: Option[DateTime] = None, - numAttemptsCreatedAts: Option[Seq[(Long, DateTime)]] = None, - limit: Option[Long], - offset: Long = 0, - orderBy: Option[OrderBy] = Some(OrderBy("migrations.id")) - ) ( - implicit customQueryModifier: Query => Query = { q => q } - ): Seq[Migration] = { - db.withConnection { c => - findAllWithConnection( - c, - ids = ids, - versionGuid = versionGuid, - numAttempts = numAttempts, - numAttemptsGreaterThanOrEquals = numAttemptsGreaterThanOrEquals, - numAttemptsGreaterThan = numAttemptsGreaterThan, - numAttemptsLessThanOrEquals = numAttemptsLessThanOrEquals, - numAttemptsLessThan = numAttemptsLessThan, - numAttemptses = numAttemptses, - createdAt = createdAt, - createdAtGreaterThanOrEquals = createdAtGreaterThanOrEquals, - createdAtGreaterThan = createdAtGreaterThan, - createdAtLessThanOrEquals = createdAtLessThanOrEquals, - createdAtLessThan = createdAtLessThan, - numAttemptsCreatedAts = numAttemptsCreatedAts, - limit = limit, - offset = offset, - orderBy = orderBy - )(customQueryModifier) - } - } - - def findAllWithConnection( - c: java.sql.Connection, - ids: Option[Seq[String]] = None, - versionGuid: Option[UUID] = None, - numAttempts: Option[Long] = None, - numAttemptsGreaterThanOrEquals: Option[Long] = None, - numAttemptsGreaterThan: Option[Long] = None, - numAttemptsLessThanOrEquals: Option[Long] = None, - numAttemptsLessThan: Option[Long] = None, - numAttemptses: Option[Seq[Long]] = None, - createdAt: Option[DateTime] = None, - createdAtGreaterThanOrEquals: Option[DateTime] = None, - createdAtGreaterThan: Option[DateTime] = None, - createdAtLessThanOrEquals: Option[DateTime] = None, - createdAtLessThan: Option[DateTime] = None, - numAttemptsCreatedAts: Option[Seq[(Long, DateTime)]] = None, - limit: Option[Long], - offset: Long = 0, - orderBy: Option[OrderBy] = Some(OrderBy("migrations.id")) - ) ( - implicit customQueryModifier: Query => Query = { q => q } - ): Seq[Migration] = { - customQueryModifier(BaseQuery). - optionalIn("migrations.id", ids). - equals("migrations.version_guid", versionGuid). - equals("migrations.num_attempts", numAttempts). - greaterThanOrEquals("migrations.num_attempts", numAttemptsGreaterThanOrEquals). - greaterThan("migrations.num_attempts", numAttemptsGreaterThan). - lessThanOrEquals("migrations.num_attempts", numAttemptsLessThanOrEquals). - lessThan("migrations.num_attempts", numAttemptsLessThan). - optionalIn("migrations.num_attempts", numAttemptses). - equals("migrations.created_at", createdAt). - greaterThanOrEquals("migrations.created_at", createdAtGreaterThanOrEquals). - greaterThan("migrations.created_at", createdAtGreaterThan). - lessThanOrEquals("migrations.created_at", createdAtLessThanOrEquals). - lessThan("migrations.created_at", createdAtLessThan). - optionalIn2(("migrations.num_attempts", "migrations.created_at"), numAttemptsCreatedAts). - optionalLimit(limit). - offset(offset). - orderBy(orderBy.flatMap(_.sql)). - as(MigrationsDao.parser.*)(c) - } - -} - -object MigrationsDao { - - val parser: RowParser[Migration] = { - SqlParser.str("id") ~ - SqlParser.get[UUID]("version_guid") ~ - SqlParser.long("num_attempts") ~ - SqlParser.str("errors_text").? ~ - SqlParser.str("updated_by_guid") ~ - SqlParser.get[DateTime]("created_at") ~ - SqlParser.get[DateTime]("updated_at") map { - case id ~ versionGuid ~ numAttempts ~ errors ~ updatedByGuid ~ createdAt ~ updatedAt => Migration( - id = id, - versionGuid = versionGuid, - numAttempts = numAttempts, - errors = errors.map { text => Json.parse(text).as[Seq[String]] }, - updatedByGuid = updatedByGuid, - createdAt = createdAt, - updatedAt = updatedAt - ) - } - } - -} - -@Singleton -class MigrationsDao @Inject() ( - override val db: Database -) extends BaseMigrationsDao { - - private[this] val idGenerator = util.IdGenerator() - - def randomId(): String = idGenerator.randomId() - - private[this] val InsertQuery = Query(""" - | insert into migrations - | (id, version_guid, num_attempts, errors, updated_by_guid, hash_code) - | values - | ({id}, {version_guid}::uuid, {num_attempts}::bigint, {errors}::json, {updated_by_guid}, {hash_code}::bigint) - """.stripMargin) - - private[this] val UpdateQuery = Query(""" - | update migrations - | set version_guid = {version_guid}::uuid, - | num_attempts = {num_attempts}::bigint, - | errors = {errors}::json, - | updated_by_guid = {updated_by_guid}, - | hash_code = {hash_code}::bigint - | where id = {id} - | and migrations.hash_code != {hash_code}::bigint - """.stripMargin) - - private[this] def bindQuery(query: Query, form: MigrationForm): Query = { - query. - bind("version_guid", form.versionGuid). - bind("num_attempts", form.numAttempts). - bind("errors", form.errors.map { v => Json.toJson(v) }). - bind("hash_code", form.hashCode()) - } - - private[this] def toNamedParameter(updatedBy: UUID, id: String, form: MigrationForm): Seq[NamedParameter] = { - Seq( - scala.Symbol("id") -> id, - scala.Symbol("version_guid") -> form.versionGuid, - scala.Symbol("num_attempts") -> form.numAttempts, - scala.Symbol("errors") -> form.errors.map { v => Json.toJson(v).toString }, - scala.Symbol("updated_by_guid") -> updatedBy, - scala.Symbol("hash_code") -> form.hashCode() - ) - } - - def insert(updatedBy: UUID, form: MigrationForm): String = { - db.withConnection { c => - insert(c, updatedBy, form) - } - } - - def insert(c: Connection, updatedBy: UUID, form: MigrationForm): String = { - val id = randomId() - bindQuery(InsertQuery, form). - bind("id", id). - bind("updated_by_guid", updatedBy). - anormSql.execute()(c) - id - } - - def insertBatch(updatedBy: UUID, forms: Seq[MigrationForm]): Seq[String] = { - db.withConnection { c => - insertBatchWithConnection(c, updatedBy, forms) - } - } - - def insertBatchWithConnection(c: Connection, updatedBy: UUID, forms: Seq[MigrationForm]): Seq[String] = { - if (forms.nonEmpty) { - val ids = forms.map(_ => randomId()) - val params = ids.zip(forms).map { case (id, form) => toNamedParameter(updatedBy, id, form) } - BatchSql(InsertQuery.sql(), params.head, params.tail: _*).execute()(c) - ids - } else { - Nil - } - } - - def updateIfChangedById(updatedBy: UUID, id: String, form: MigrationForm): Unit = { - if (!findById(id).map(_.form).contains(form)) { - updateById(updatedBy, id, form) - } - } - - def updateById(updatedBy: UUID, id: String, form: MigrationForm): Unit = { - db.withConnection { c => - updateById(c, updatedBy, id, form) - } - } - - def updateById(c: Connection, updatedBy: UUID, id: String, form: MigrationForm): Unit = { - bindQuery(UpdateQuery, form). - bind("id", id). - bind("updated_by_guid", updatedBy). - anormSql.execute()(c) - () - } - - def update(updatedBy: UUID, existing: Migration, form: MigrationForm): Unit = { - db.withConnection { c => - update(c, updatedBy, existing, form) - } - } - - def update(c: Connection, updatedBy: UUID, existing: Migration, form: MigrationForm): Unit = { - updateById(c, updatedBy, existing.id, form) - } - - def updateBatch(updatedBy: UUID, idsAndForms: Seq[(String, MigrationForm)]): Unit = { - db.withConnection { c => - updateBatchWithConnection(c, updatedBy, idsAndForms) - } - } - - def updateBatchWithConnection(c: Connection, updatedBy: UUID, idsAndForms: Seq[(String, MigrationForm)]): Unit = { - if (idsAndForms.nonEmpty) { - val params = idsAndForms.map { case (id, form) => toNamedParameter(updatedBy, id, form) } - BatchSql(UpdateQuery.sql(), params.head, params.tail: _*).execute()(c) - () - } - } - - def delete(deletedBy: UUID, migration: Migration): Unit = { - db.withConnection { c => - delete(c, deletedBy, migration) - } - } - - def delete(c: Connection, deletedBy: UUID, migration: Migration): Unit = { - deleteById(c, deletedBy, migration.id) - } - - def deleteById(deletedBy: UUID, id: String): Unit = { - db.withConnection { c => - deleteById(c, deletedBy, id) - } - } - - def deleteById(c: Connection, deletedBy: UUID, id: String): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .equals("id", id) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByIds(deletedBy: UUID, ids: Seq[String]): Unit = { - db.withConnection { c => - deleteAllByIds(c, deletedBy, ids) - } - } - - def deleteAllByIds(c: Connection, deletedBy: UUID, ids: Seq[String]): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .in("id", ids) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByNumAttempts(deletedBy: UUID, numAttempts: Long): Unit = { - db.withConnection { c => - deleteAllByNumAttempts(c, deletedBy, numAttempts) - } - } - - def deleteAllByNumAttempts(c: Connection, deletedBy: UUID, numAttempts: Long): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .equals("num_attempts", numAttempts) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByNumAttemptses(deletedBy: UUID, numAttemptses: Seq[Long]): Unit = { - db.withConnection { c => - deleteAllByNumAttemptses(c, deletedBy, numAttemptses) - } - } - - def deleteAllByNumAttemptses(c: Connection, deletedBy: UUID, numAttemptses: Seq[Long]): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .in("num_attempts", numAttemptses) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByNumAttemptsAndCreatedAt(deletedBy: UUID, numAttempts: Long, createdAt: DateTime): Unit = { - db.withConnection { c => - deleteAllByNumAttemptsAndCreatedAt(c, deletedBy, numAttempts, createdAt) - } - } - - def deleteAllByNumAttemptsAndCreatedAt(c: Connection, deletedBy: UUID, numAttempts: Long, createdAt: DateTime): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .equals("num_attempts", numAttempts) - .equals("created_at", createdAt) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByNumAttemptsAndCreatedAts(deletedBy: UUID, numAttempts: Long, createdAts: Seq[DateTime]): Unit = { - db.withConnection { c => - deleteAllByNumAttemptsAndCreatedAts(c, deletedBy, numAttempts, createdAts) - } - } - - def deleteAllByNumAttemptsAndCreatedAts(c: Connection, deletedBy: UUID, numAttempts: Long, createdAts: Seq[DateTime]): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .equals("num_attempts", numAttempts) - .in("created_at", createdAts) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByVersionGuid(deletedBy: UUID, versionGuid: UUID): Unit = { - db.withConnection { c => - deleteAllByVersionGuid(c, deletedBy, versionGuid) - } - } - - def deleteAllByVersionGuid(c: Connection, deletedBy: UUID, versionGuid: UUID): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .equals("version_guid", versionGuid) - .anormSql.executeUpdate()(c) - () - } - - def deleteAllByVersionGuids(deletedBy: UUID, versionGuids: Seq[UUID]): Unit = { - db.withConnection { c => - deleteAllByVersionGuids(c, deletedBy, versionGuids) - } - } - - def deleteAllByVersionGuids(c: Connection, deletedBy: UUID, versionGuids: Seq[UUID]): Unit = { - setJournalDeletedByUserId(c, deletedBy) - Query("delete from migrations") - .in("version_guid", versionGuids) - .anormSql.executeUpdate()(c) - () - } - - def setJournalDeletedByUserId(c: Connection, deletedBy: UUID): Unit = { - anorm.SQL(s"SET journal.deleted_by_user_id = '${deletedBy}'").executeUpdate()(c) - () - } - -} \ No newline at end of file diff --git a/api/app/processor/MigrateVersionProcessor.scala b/api/app/processor/MigrateVersionProcessor.scala new file mode 100644 index 000000000..df41e0783 --- /dev/null +++ b/api/app/processor/MigrateVersionProcessor.scala @@ -0,0 +1,19 @@ +package processor + +import cats.data.ValidatedNec +import db.VersionsDao +import io.apibuilder.task.v0.models.TaskType + +import java.util.UUID +import javax.inject.Inject + + +class MigrateVersionProcessor @Inject()( + args: TaskProcessorArgs, + versionsDao: VersionsDao +) extends TaskProcessorWithGuid(args, TaskType.MigrateVersion) { + + override def processRecord(versionGuid: UUID): ValidatedNec[String, Unit] = { + versionsDao.migrateVersionGuid(versionGuid) + } +} \ No newline at end of file diff --git a/api/app/processor/ScheduleMigrateVersionsProcessor.scala b/api/app/processor/ScheduleMigrateVersionsProcessor.scala new file mode 100644 index 000000000..2db076576 --- /dev/null +++ b/api/app/processor/ScheduleMigrateVersionsProcessor.scala @@ -0,0 +1,59 @@ +package processor + +import anorm.SqlParser +import cats.data.ValidatedNec +import cats.implicits._ +import db.{InternalTasksDao, Migration, VersionsDao} +import io.apibuilder.task.v0.models.TaskType +import io.flow.postgresql.Query +import play.api.db.{Database, NamedDatabase} + +import javax.inject.Inject +import scala.annotation.tailrec + + +class ScheduleMigrateVersionsProcessor @Inject()( + args: TaskProcessorArgs, + @NamedDatabase("default") db: Database, + internalTasksDao: InternalTasksDao, +) extends TaskProcessor(args, TaskType.ScheduleMigrateVersions) { + + override def processRecord(id: String): ValidatedNec[String, Unit] = { + scheduleMigrationTasks().validNec + } + + + private[this] val VersionsNeedingUpgrade = Query( + """ + |select v.guid + | from versions v + | join applications apps on apps.guid = v.application_guid and apps.deleted_at is null + | left join tasks t on t.type = {task_type} and t.type_id::uuid = v.guid + | where v.deleted_at is null + | and t.id is null + | and not exists ( + | select 1 + | from cache.services + | where services.deleted_at is null + | and services.version_guid = v.guid + | and services.version = {service_version} + | ) + |limit 250 + |""".stripMargin + ).bind("service_version", Migration.ServiceVersionNumber) + .bind("task_type", TaskType.MigrateVersion.toString) + + @tailrec + private[this] def scheduleMigrationTasks(): Unit = { + val versionGuids = db.withConnection { implicit c => + VersionsNeedingUpgrade + .as(SqlParser.get[_root_.java.util.UUID](1).*) + } + if (versionGuids.nonEmpty) { + versionGuids.foreach { vGuid => + internalTasksDao.queue(TaskType.MigrateVersion, vGuid.toString) + } + scheduleMigrationTasks() + } + } +} \ No newline at end of file diff --git a/generated/app/ApicollectiveApibuilderTaskV0Client.scala b/generated/app/ApicollectiveApibuilderTaskV0Client.scala index ff29f9969..dad56c2bd 100644 --- a/generated/app/ApicollectiveApibuilderTaskV0Client.scala +++ b/generated/app/ApicollectiveApibuilderTaskV0Client.scala @@ -15,6 +15,8 @@ package io.apibuilder.task.v0.models { case object IndexApplication extends TaskType { override def toString = "index_application" } case object CleanupDeletions extends TaskType { override def toString = "cleanup_deletions" } + case object ScheduleMigrateVersions extends TaskType { override def toString = "schedule_migrate_versions" } + case object MigrateVersion extends TaskType { override def toString = "migrate_version" } case object DiffVersion extends TaskType { override def toString = "diff_version" } /** * UNDEFINED captures values that are sent either in error or @@ -32,7 +34,7 @@ package io.apibuilder.task.v0.models { * lower case to avoid collisions with the camel cased values * above. */ - val all: scala.List[TaskType] = scala.List(IndexApplication, CleanupDeletions, DiffVersion) + val all: scala.List[TaskType] = scala.List(IndexApplication, CleanupDeletions, ScheduleMigrateVersions, MigrateVersion, DiffVersion) private[this] val byName: Map[String, TaskType] = all.map(x => x.toString.toLowerCase -> x).toMap diff --git a/spec/apibuilder-task.json b/spec/apibuilder-task.json index b9faeef7f..8a5b3b6f0 100644 --- a/spec/apibuilder-task.json +++ b/spec/apibuilder-task.json @@ -18,6 +18,8 @@ "values": [ { "name": "index_application" }, { "name": "cleanup_deletions" }, + { "name": "schedule_migrate_versions" }, + { "name": "migrate_version" }, { "name": "diff_version" } ] } From ae0429b3caa562404b986f83f3f87b356503da9b Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:28:20 -0700 Subject: [PATCH 19/32] wip --- api/app/db/VersionsDao.scala | 7 ++++--- api/app/processor/MigrateVersionProcessor.scala | 3 +++ api/app/processor/ScheduleMigrateVersionsProcessor.scala | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/api/app/db/VersionsDao.scala b/api/app/db/VersionsDao.scala index 8c2a1681e..34d7c13be 100644 --- a/api/app/db/VersionsDao.scala +++ b/api/app/db/VersionsDao.scala @@ -17,6 +17,7 @@ import lib.{ServiceConfiguration, ServiceUri, ValidatedHelpers, VersionTag} import play.api.Logger import play.api.db._ import play.api.libs.json._ +import processor.MigrateVersion import java.util.UUID import javax.inject.Inject @@ -368,7 +369,7 @@ class VersionsDao @Inject() ( orgNamespace = org.namespace, version = versionName ) - logger.info(s"Migrating $orgKey/$applicationKey/$versionName versionGuid[$versionGuid] to latest API Builder spec version[${Migration.ServiceVersionNumber}] (with serviceConfig=$serviceConfig)") + logger.info(s"Migrating $orgKey/$applicationKey/$versionName versionGuid[$versionGuid] to latest API Builder spec version[${MigrateVersion.ServiceVersionNumber}] (with serviceConfig=$serviceConfig)") val validator = OriginalValidator( config = serviceConfig, @@ -393,7 +394,7 @@ class VersionsDao @Inject() ( ): Unit = { SQL(SoftDeleteServiceByVersionGuidAndVersionNumberQuery).on( "version_guid" -> versionGuid, - "version" -> Migration.ServiceVersionNumber, + "version" -> MigrateVersion.ServiceVersionNumber, "user_guid" -> user.guid ).execute() } @@ -407,7 +408,7 @@ class VersionsDao @Inject() ( SQL(InsertServiceQuery).on( "guid" -> UUID.randomUUID, "version_guid" -> versionGuid, - "version" -> Migration.ServiceVersionNumber, + "version" -> MigrateVersion.ServiceVersionNumber, "json" -> Json.toJson(service).as[JsObject].toString.trim, "user_guid" -> user.guid ).execute() diff --git a/api/app/processor/MigrateVersionProcessor.scala b/api/app/processor/MigrateVersionProcessor.scala index df41e0783..d2f15fbe9 100644 --- a/api/app/processor/MigrateVersionProcessor.scala +++ b/api/app/processor/MigrateVersionProcessor.scala @@ -7,6 +7,9 @@ import io.apibuilder.task.v0.models.TaskType import java.util.UUID import javax.inject.Inject +object MigrateVersion { + val ServiceVersionNumber: String = io.apibuilder.spec.v0.Constants.Version.toLowerCase +} class MigrateVersionProcessor @Inject()( args: TaskProcessorArgs, diff --git a/api/app/processor/ScheduleMigrateVersionsProcessor.scala b/api/app/processor/ScheduleMigrateVersionsProcessor.scala index 2db076576..920a221b4 100644 --- a/api/app/processor/ScheduleMigrateVersionsProcessor.scala +++ b/api/app/processor/ScheduleMigrateVersionsProcessor.scala @@ -3,7 +3,7 @@ package processor import anorm.SqlParser import cats.data.ValidatedNec import cats.implicits._ -import db.{InternalTasksDao, Migration, VersionsDao} +import db.InternalTasksDao import io.apibuilder.task.v0.models.TaskType import io.flow.postgresql.Query import play.api.db.{Database, NamedDatabase} @@ -40,7 +40,7 @@ class ScheduleMigrateVersionsProcessor @Inject()( | ) |limit 250 |""".stripMargin - ).bind("service_version", Migration.ServiceVersionNumber) + ).bind("service_version", MigrateVersion.ServiceVersionNumber) .bind("task_type", TaskType.MigrateVersion.toString) @tailrec From 608878dc12ae0d1930eab80a4b5d4012640a91e1 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:29:08 -0700 Subject: [PATCH 20/32] wip --- api/app/processor/TaskActorCompanion.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/app/processor/TaskActorCompanion.scala b/api/app/processor/TaskActorCompanion.scala index 4454c50d1..dc9ba5ce9 100644 --- a/api/app/processor/TaskActorCompanion.scala +++ b/api/app/processor/TaskActorCompanion.scala @@ -7,7 +7,9 @@ import javax.inject.Inject class TaskActorCompanion @Inject() ( indexApplication: IndexApplicationProcessor, diffVersion: DiffVersionProcessor, - cleanupDeletions: CleanupDeletionsProcessor + cleanupDeletions: CleanupDeletionsProcessor, + scheduleMigrateVersions: ScheduleMigrateVersionsProcessor, + migrateVersion: MigrateVersionProcessor ) { def process(typ: TaskType): Unit = { @@ -20,6 +22,8 @@ class TaskActorCompanion @Inject() ( case IndexApplication => indexApplication case CleanupDeletions => cleanupDeletions case DiffVersion => diffVersion + case MigrateVersion => migrateVersion + case ScheduleMigrateVersions => scheduleMigrateVersions case UNDEFINED(_) => sys.error(s"Undefined task type '$typ") } } From a04b12da89df16d891e0e74c7408a1551bc40a81 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:29:54 -0700 Subject: [PATCH 21/32] wip --- api/app/actors/MainActor.scala | 12 ------ api/conf/base.conf | 7 ---- api/test/db/InternalMigrationsDaoSpec.scala | 46 --------------------- 3 files changed, 65 deletions(-) delete mode 100644 api/test/db/InternalMigrationsDaoSpec.scala diff --git a/api/app/actors/MainActor.scala b/api/app/actors/MainActor.scala index 18ae8c3bb..fed669ac5 100644 --- a/api/app/actors/MainActor.scala +++ b/api/app/actors/MainActor.scala @@ -4,8 +4,6 @@ import akka.actor._ import lib.Role import java.util.UUID -import scala.concurrent.ExecutionContext -import scala.concurrent.duration.{FiniteDuration, SECONDS} object MainActor { @@ -26,21 +24,11 @@ object MainActor { @javax.inject.Singleton class MainActor @javax.inject.Inject() ( - system: ActorSystem, @javax.inject.Named("email-actor") emailActor: akka.actor.ActorRef, @javax.inject.Named("generator-service-actor") generatorServiceActor: akka.actor.ActorRef, @javax.inject.Named("user-actor") userActor: akka.actor.ActorRef ) extends Actor with ActorLogging with ErrorHandler { - private[this] implicit val ec: ExecutionContext = system.dispatchers.lookup("main-actor-context") - - private[this] def scheduleOnce(msg: Any)(implicit delay: FiniteDuration = FiniteDuration(10, SECONDS)): Unit = { - system.scheduler.scheduleOnce(delay) { - self ! msg - } - } - - def receive: Receive = akka.event.LoggingReceive { case m @ MainActor.Messages.MembershipRequestCreated(guid) => withVerboseErrorHandler(m) { diff --git a/api/conf/base.conf b/api/conf/base.conf index dc0ba7fd4..7479eb3cf 100644 --- a/api/conf/base.conf +++ b/api/conf/base.conf @@ -39,13 +39,6 @@ akka { } -main-actor-context { - fork-join-executor { - parallelism-factor = 2.0 - parallelism-max = 5 - } -} - task-context { fork-join-executor { parallelism-factor = 2.0 diff --git a/api/test/db/InternalMigrationsDaoSpec.scala b/api/test/db/InternalMigrationsDaoSpec.scala deleted file mode 100644 index 87b6a720f..000000000 --- a/api/test/db/InternalMigrationsDaoSpec.scala +++ /dev/null @@ -1,46 +0,0 @@ -package db - -import db.generated.MigrationsDao -import io.flow.postgresql.Query -import org.scalatestplus.play.PlaySpec -import org.scalatestplus.play.guice.GuiceOneAppPerSuite - -import java.util.UUID - -class InternalMigrationsDaoSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers with DbUtils { - - private[this] def internalMigrationsDao: InternalMigrationsDao = injector.instanceOf[InternalMigrationsDao] - private[this] def generatedMigrationsDao: MigrationsDao = injector.instanceOf[MigrationsDao] - - private[this] def deleteCachedServices(versionGuid: UUID): Unit = { - execute( - Query("update cache.services set deleted_at=now(), deleted_by_guid={user_guid}::uuid") - .equals("version_guid", versionGuid) - .bind("user_guid", testUser.guid) - ) - } - - "queueVersions" in { - val version = createVersion() - deleteCachedServices(version.guid) - def count = generatedMigrationsDao.findAll(versionGuid = Some(version.guid), limit = None).length - - count mustBe 0 - internalMigrationsDao.queueVersions() - count mustBe 1 - } - - "migrateBatch" in { - val version = createVersion() - deleteCachedServices(version.guid) - - def exists = versionsDao.findByGuid(Authorization.All, version.guid).isDefined - - exists mustBe false - - internalMigrationsDao.queueVersions() - internalMigrationsDao.migrateBatch(1000L) - - exists mustBe true - } -} \ No newline at end of file From 76d6788797c2155af203d6dd586c84879f6cdeb1 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:31:42 -0700 Subject: [PATCH 22/32] wip --- api/app/actors/Bindings.scala | 1 - api/app/actors/MainActor.scala | 6 ---- api/app/actors/UserActor.scala | 32 ------------------- api/app/processor/UserCreatedProcessor.scala | 19 +++++++++++ api/conf/base.conf | 8 ----- .../ApicollectiveApibuilderTaskV0Client.scala | 3 +- spec/apibuilder-task.json | 3 +- 7 files changed, 23 insertions(+), 49 deletions(-) delete mode 100644 api/app/actors/UserActor.scala create mode 100644 api/app/processor/UserCreatedProcessor.scala diff --git a/api/app/actors/Bindings.scala b/api/app/actors/Bindings.scala index 8c2f2848a..a04bbdb8b 100644 --- a/api/app/actors/Bindings.scala +++ b/api/app/actors/Bindings.scala @@ -9,7 +9,6 @@ class ActorsModule extends AbstractModule with AkkaGuiceSupport { bindActor[PeriodicActor]("PeriodicActor") bindActor[GeneratorServiceActor]("generator-service-actor") bindActor[EmailActor]("email-actor") - bindActor[UserActor]("user-actor") bindActor[TaskDispatchActor]( "TaskDispatchActor", _.withDispatcher("task-context-dispatcher") diff --git a/api/app/actors/MainActor.scala b/api/app/actors/MainActor.scala index fed669ac5..b1cf2e16b 100644 --- a/api/app/actors/MainActor.scala +++ b/api/app/actors/MainActor.scala @@ -16,7 +16,6 @@ object MainActor { case class PasswordResetRequestCreated(guid: UUID) case class ApplicationCreated(guid: UUID) - case class UserCreated(guid: UUID) case class GeneratorServiceCreated(guid: UUID) } @@ -26,7 +25,6 @@ object MainActor { class MainActor @javax.inject.Inject() ( @javax.inject.Named("email-actor") emailActor: akka.actor.ActorRef, @javax.inject.Named("generator-service-actor") generatorServiceActor: akka.actor.ActorRef, - @javax.inject.Named("user-actor") userActor: akka.actor.ActorRef ) extends Actor with ActorLogging with ErrorHandler { def receive: Receive = akka.event.LoggingReceive { @@ -63,10 +61,6 @@ class MainActor @javax.inject.Inject() ( emailActor ! EmailActor.Messages.PasswordResetRequestCreated(guid) } - case m @ MainActor.Messages.UserCreated(guid) => withVerboseErrorHandler(m) { - userActor ! UserActor.Messages.UserCreated(guid) - } - case m: Any => logUnhandledMessage(m) } } diff --git a/api/app/actors/UserActor.scala b/api/app/actors/UserActor.scala deleted file mode 100644 index e84589aa7..000000000 --- a/api/app/actors/UserActor.scala +++ /dev/null @@ -1,32 +0,0 @@ -package actors - -import akka.actor.{Actor, ActorLogging} -import db.UsersDao - -import java.util.UUID -import javax.inject.{Inject, Singleton} - -object UserActor { - - object Messages { - case class UserCreated(guid: UUID) - } - -} - -@Singleton -class UserActor @Inject() ( - usersDao: UsersDao -) extends Actor with ActorLogging with ErrorHandler { - - def receive = { - - case m @ UserActor.Messages.UserCreated(guid) => withVerboseErrorHandler(m) { - usersDao.processUserCreated(guid) - } - - case m: Any => logUnhandledMessage(m) - - } - -} diff --git a/api/app/processor/UserCreatedProcessor.scala b/api/app/processor/UserCreatedProcessor.scala new file mode 100644 index 000000000..42979bb14 --- /dev/null +++ b/api/app/processor/UserCreatedProcessor.scala @@ -0,0 +1,19 @@ +package processor + +import cats.data.ValidatedNec +import db.UsersDao +import io.apibuilder.task.v0.models.TaskType + +import javax.inject.Inject + + +class UserCreatedProcessor @Inject()( + args: TaskProcessorArgs, + usersDao: UsersDao +) extends TaskProcessorWithGuid(args, TaskType.UserCreated) { + + override def processRecord(userGuid: UUID): ValidatedNec[String, Unit] = { + usersDao.processUserCreated(userGuid) + } + +} \ No newline at end of file diff --git a/api/conf/base.conf b/api/conf/base.conf index 7479eb3cf..01ff9ecf1 100644 --- a/api/conf/base.conf +++ b/api/conf/base.conf @@ -80,11 +80,3 @@ task-actor-context { parallelism-max = 2 } } - -user-actor-context { - fork-join-executor { - parallelism-factor = 1.0 - parallelism-max = 2 - } -} -git.version = 0.16.19 diff --git a/generated/app/ApicollectiveApibuilderTaskV0Client.scala b/generated/app/ApicollectiveApibuilderTaskV0Client.scala index dad56c2bd..e4bd8950e 100644 --- a/generated/app/ApicollectiveApibuilderTaskV0Client.scala +++ b/generated/app/ApicollectiveApibuilderTaskV0Client.scala @@ -18,6 +18,7 @@ package io.apibuilder.task.v0.models { case object ScheduleMigrateVersions extends TaskType { override def toString = "schedule_migrate_versions" } case object MigrateVersion extends TaskType { override def toString = "migrate_version" } case object DiffVersion extends TaskType { override def toString = "diff_version" } + case object UserCreated extends TaskType { override def toString = "user_created" } /** * UNDEFINED captures values that are sent either in error or * that were added by the server after this library was @@ -34,7 +35,7 @@ package io.apibuilder.task.v0.models { * lower case to avoid collisions with the camel cased values * above. */ - val all: scala.List[TaskType] = scala.List(IndexApplication, CleanupDeletions, ScheduleMigrateVersions, MigrateVersion, DiffVersion) + val all: scala.List[TaskType] = scala.List(IndexApplication, CleanupDeletions, ScheduleMigrateVersions, MigrateVersion, DiffVersion, UserCreated) private[this] val byName: Map[String, TaskType] = all.map(x => x.toString.toLowerCase -> x).toMap diff --git a/spec/apibuilder-task.json b/spec/apibuilder-task.json index 8a5b3b6f0..a25991ad9 100644 --- a/spec/apibuilder-task.json +++ b/spec/apibuilder-task.json @@ -20,7 +20,8 @@ { "name": "cleanup_deletions" }, { "name": "schedule_migrate_versions" }, { "name": "migrate_version" }, - { "name": "diff_version" } + { "name": "diff_version" }, + { "name": "user_created" } ] } }, From decc73cdf0c08b01bd3c146dc7e49df07b6c7f47 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:35:44 -0700 Subject: [PATCH 23/32] wip --- api/app/db/UsersDao.scala | 35 ++++++-------------- api/app/processor/TaskActorCompanion.scala | 4 ++- api/app/processor/UserCreatedProcessor.scala | 18 ++++++++-- api/test/db/EmailVerificationsDaoSpec.scala | 7 ++-- 4 files changed, 33 insertions(+), 31 deletions(-) diff --git a/api/app/db/UsersDao.scala b/api/app/db/UsersDao.scala index 2dba70fc0..4a990f79d 100644 --- a/api/app/db/UsersDao.scala +++ b/api/app/db/UsersDao.scala @@ -1,22 +1,21 @@ package db +import anorm._ import io.apibuilder.api.v0.models.{Error, User, UserForm, UserUpdateForm} +import io.apibuilder.task.v0.models.TaskType import io.flow.postgresql.Query -import lib.{Constants, Misc, Role, UrlKey, Validation} -import anorm._ -import javax.inject.{Inject, Named, Singleton} - +import lib.{Constants, Misc, UrlKey, Validation} import play.api.db._ -import java.util.UUID - import play.api.inject.Injector +import java.util.UUID +import javax.inject.{Inject, Singleton} import scala.annotation.tailrec import scala.util.{Failure, Success, Try} object UsersDao { - val AdminUserEmails: Seq[String] = Seq("admin@apibuilder.io") + private val AdminUserEmails: Seq[String] = Seq("admin@apibuilder.io") val AdminUserGuid: UUID = UUID.fromString("f3973f60-be9f-11e3-b1b6-0800200c9a66") @@ -25,11 +24,10 @@ object UsersDao { @Singleton class UsersDao @Inject() ( @NamedDatabase("default") db: Database, - @Named("main-actor") mainActor: akka.actor.ActorRef, injector: Injector, emailVerificationsDao: EmailVerificationsDao, - organizationsDao: OrganizationsDao, - userPasswordsDao: UserPasswordsDao + userPasswordsDao: UserPasswordsDao, + internalTasksDao: InternalTasksDao, ) { // TODO: Inject directly - here because of circular references @@ -153,7 +151,7 @@ class UsersDao @Inject() ( gravatarId: Option[String] ): User = { val nickname = generateNickname(login) - val guid = db.withConnection { implicit c => + val guid = db.withTransaction { implicit c => doInsert( nickname = nickname, email = email, @@ -163,8 +161,6 @@ class UsersDao @Inject() ( ) } - mainActor ! actors.MainActor.Messages.UserCreated(guid) - findByGuid(guid).getOrElse { sys.error("Failed to create user") } @@ -197,8 +193,6 @@ class UsersDao @Inject() ( id } - mainActor ! actors.MainActor.Messages.UserCreated(guid) - findByGuid(guid).getOrElse { sys.error("Failed to create user") } @@ -224,16 +218,9 @@ class UsersDao @Inject() ( "updated_by_guid" -> Constants.DefaultUserGuid.toString ).execute() - guid - } + internalTasksDao.queueWithConnection(c, TaskType.UserCreated, guid.toString) - def processUserCreated(guid: UUID): Unit = { - findByGuid(guid).foreach { user => - organizationsDao.findAllByEmailDomain(user.email).foreach { org => - membershipRequestsDao.upsert(user, org, user, Role.Member) - } - emailVerificationsDao.create(user, user, user.email) - } + guid } def findByToken(token: String): Option[User] = { diff --git a/api/app/processor/TaskActorCompanion.scala b/api/app/processor/TaskActorCompanion.scala index dc9ba5ce9..e715e674a 100644 --- a/api/app/processor/TaskActorCompanion.scala +++ b/api/app/processor/TaskActorCompanion.scala @@ -9,7 +9,8 @@ class TaskActorCompanion @Inject() ( diffVersion: DiffVersionProcessor, cleanupDeletions: CleanupDeletionsProcessor, scheduleMigrateVersions: ScheduleMigrateVersionsProcessor, - migrateVersion: MigrateVersionProcessor + migrateVersion: MigrateVersionProcessor, + userCreated: UserCreatedProcessor ) { def process(typ: TaskType): Unit = { @@ -24,6 +25,7 @@ class TaskActorCompanion @Inject() ( case DiffVersion => diffVersion case MigrateVersion => migrateVersion case ScheduleMigrateVersions => scheduleMigrateVersions + case UserCreated => userCreated case UNDEFINED(_) => sys.error(s"Undefined task type '$typ") } } diff --git a/api/app/processor/UserCreatedProcessor.scala b/api/app/processor/UserCreatedProcessor.scala index 42979bb14..83184a45d 100644 --- a/api/app/processor/UserCreatedProcessor.scala +++ b/api/app/processor/UserCreatedProcessor.scala @@ -1,19 +1,31 @@ package processor +import cats.implicits._ import cats.data.ValidatedNec -import db.UsersDao +import db._ import io.apibuilder.task.v0.models.TaskType +import lib.Role +import java.util.UUID import javax.inject.Inject class UserCreatedProcessor @Inject()( args: TaskProcessorArgs, - usersDao: UsersDao + usersDao: UsersDao, + organizationsDao: OrganizationsDao, + membershipRequestsDao: MembershipRequestsDao, + emailVerificationsDao: EmailVerificationsDao, ) extends TaskProcessorWithGuid(args, TaskType.UserCreated) { override def processRecord(userGuid: UUID): ValidatedNec[String, Unit] = { - usersDao.processUserCreated(userGuid) + usersDao.findByGuid(userGuid).foreach { user => + organizationsDao.findAllByEmailDomain(user.email).foreach { org => + membershipRequestsDao.upsert(user, org, user, Role.Member) + } + emailVerificationsDao.create(user, user, user.email) + } + ().validNec } } \ No newline at end of file diff --git a/api/test/db/EmailVerificationsDaoSpec.scala b/api/test/db/EmailVerificationsDaoSpec.scala index 705f4746e..32fc91d4c 100644 --- a/api/test/db/EmailVerificationsDaoSpec.scala +++ b/api/test/db/EmailVerificationsDaoSpec.scala @@ -1,13 +1,14 @@ package db import java.util.UUID - import io.apibuilder.api.v0.models.UserForm import org.scalatestplus.play.PlaySpec import org.scalatestplus.play.guice.GuiceOneAppPerSuite +import processor.UserCreatedProcessor class EmailVerificationsDaoSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { + def userCreatedProcessor: UserCreatedProcessor = injector.instanceOf[UserCreatedProcessor] def emailVerificationConfirmationsDao: EmailVerificationConfirmationsDao = injector.instanceOf[db.EmailVerificationConfirmationsDao] "upsert" in { @@ -112,8 +113,8 @@ class EmailVerificationsDaoSpec extends PlaySpec with GuiceOneAppPerSuite with d password = "testing" )) - usersDao.processUserCreated(user.guid) - usersDao.processUserCreated(nonMatchingUser.guid) + userCreatedProcessor.processRecord(user.guid) + userCreatedProcessor.processRecord(nonMatchingUser.guid) membershipsDao.isUserMember(user, org) must be(false) membershipsDao.isUserMember(nonMatchingUser, org) must be(false) From c17b0750c042198dc9699a814da60e90c4f92e1c Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:36:49 -0700 Subject: [PATCH 24/32] wip --- api/app/actors/EmailActor.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api/app/actors/EmailActor.scala b/api/app/actors/EmailActor.scala index 9d706496a..ed28b6875 100644 --- a/api/app/actors/EmailActor.scala +++ b/api/app/actors/EmailActor.scala @@ -1,6 +1,6 @@ package actors -import akka.actor.{Actor, ActorLogging, ActorSystem} +import akka.actor.{Actor, ActorLogging} import db._ import io.apibuilder.api.v0.models.Publication import lib.{AppConfig, EmailUtil, Person, Role} @@ -23,7 +23,6 @@ object EmailActor { @javax.inject.Singleton class EmailActor @javax.inject.Inject() ( - system: ActorSystem, appConfig: AppConfig, applicationsDao: db.ApplicationsDao, email: EmailUtil, @@ -36,7 +35,7 @@ class EmailActor @javax.inject.Inject() ( usersDao: UsersDao ) extends Actor with ActorLogging with ErrorHandler { - def receive = { + def receive: Receive = { case m @ EmailActor.Messages.MembershipRequestCreated(guid) => withVerboseErrorHandler(m) { membershipRequestsDao.findByGuid(Authorization.All, guid).foreach { request => @@ -117,7 +116,7 @@ class EmailActor @javax.inject.Inject() ( usersDao.findByGuid(verification.userGuid).foreach { user => email.sendHtml( to = Person(email = verification.email, name = user.name), - subject = s"Verify your email address", + subject = "Verify your email address", body = views.html.emails.emailVerificationCreated(appConfig, verification).toString ) } From c70eb10ee695d3448d88344680ee73d63751d386 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:38:17 -0700 Subject: [PATCH 25/32] wip --- api/app/actors/GeneratorServiceActor.scala | 3 --- api/app/processor/CleanupDeletionsProcessor.scala | 2 +- api/app/util/GeneratorServiceUtil.scala | 2 ++ 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/api/app/actors/GeneratorServiceActor.scala b/api/app/actors/GeneratorServiceActor.scala index bb72698c6..051960a02 100644 --- a/api/app/actors/GeneratorServiceActor.scala +++ b/api/app/actors/GeneratorServiceActor.scala @@ -1,8 +1,6 @@ package actors import akka.actor.{Actor, ActorLogging, ActorSystem} -import db.generators.ServicesDao -import play.api.Mode import util.GeneratorServiceUtil import java.util.UUID @@ -36,7 +34,6 @@ class GeneratorServiceActor @javax.inject.Inject() ( } class GeneratorServiceActorProcessor @Inject() ( - servicesDao: ServicesDao, util: GeneratorServiceUtil ) { def processMessage(msg: GeneratorServiceActorMessage)(implicit ec: ExecutionContext): Unit = { diff --git a/api/app/processor/CleanupDeletionsProcessor.scala b/api/app/processor/CleanupDeletionsProcessor.scala index 93faa847e..906d77a1a 100644 --- a/api/app/processor/CleanupDeletionsProcessor.scala +++ b/api/app/processor/CleanupDeletionsProcessor.scala @@ -28,7 +28,7 @@ object DeleteMetadata { val VersionSoft: Seq[String] = Seq( "cache.services", "public.originals" ) - val VersionHard: Seq[String] = Seq("public.migrations") + val VersionHard: Seq[String] = Nil } class CleanupDeletionsProcessor @Inject()( diff --git a/api/app/util/GeneratorServiceUtil.scala b/api/app/util/GeneratorServiceUtil.scala index 87ede88ea..849131830 100644 --- a/api/app/util/GeneratorServiceUtil.scala +++ b/api/app/util/GeneratorServiceUtil.scala @@ -12,6 +12,7 @@ import play.api.Logger import java.util.UUID import javax.inject.Inject +import scala.annotation.tailrec import scala.concurrent.Await import scala.concurrent.duration.{FiniteDuration, SECONDS} import scala.util.{Failure, Success, Try} @@ -67,6 +68,7 @@ class GeneratorServiceUtil @Inject() ( ) } + @tailrec private[this] def doSync( client: Client, service: GeneratorService, From 520128cc549986e9430457691c3a27453018732c Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 19:55:27 -0700 Subject: [PATCH 26/32] wip --- api/app/actors/Bindings.scala | 1 - api/app/actors/GeneratorServiceActor.scala | 47 ------- api/app/actors/MainActor.scala | 7 -- api/app/actors/PeriodicActor.scala | 1 + api/app/db/InternalTasksDao.scala | 7 ++ api/app/db/generators/ServicesDao.scala | 28 ++--- .../ScheduleMigrateVersionsProcessor.scala | 4 +- ...heduleSyncGeneratorServicesProcessor.scala | 37 ++++++ .../SyncGeneratorServiceProcessor.scala | 118 ++++++++++++++++++ api/app/processor/TaskActorCompanion.scala | 6 +- api/conf/base.conf | 2 +- .../ApicollectiveApibuilderTaskV0Client.scala | 6 +- spec/apibuilder-task.json | 2 + 13 files changed, 190 insertions(+), 76 deletions(-) delete mode 100644 api/app/actors/GeneratorServiceActor.scala create mode 100644 api/app/processor/ScheduleSyncGeneratorServicesProcessor.scala create mode 100644 api/app/processor/SyncGeneratorServiceProcessor.scala diff --git a/api/app/actors/Bindings.scala b/api/app/actors/Bindings.scala index a04bbdb8b..ab02df448 100644 --- a/api/app/actors/Bindings.scala +++ b/api/app/actors/Bindings.scala @@ -7,7 +7,6 @@ class ActorsModule extends AbstractModule with AkkaGuiceSupport { override def configure = { bindActor[MainActor]("main-actor") bindActor[PeriodicActor]("PeriodicActor") - bindActor[GeneratorServiceActor]("generator-service-actor") bindActor[EmailActor]("email-actor") bindActor[TaskDispatchActor]( "TaskDispatchActor", diff --git a/api/app/actors/GeneratorServiceActor.scala b/api/app/actors/GeneratorServiceActor.scala deleted file mode 100644 index 051960a02..000000000 --- a/api/app/actors/GeneratorServiceActor.scala +++ /dev/null @@ -1,47 +0,0 @@ -package actors - -import akka.actor.{Actor, ActorLogging, ActorSystem} -import util.GeneratorServiceUtil - -import java.util.UUID -import javax.inject.Inject -import scala.concurrent.ExecutionContext -import scala.concurrent.duration._ - -sealed trait GeneratorServiceActorMessage -object GeneratorServiceActorMessage { - - case class GeneratorServiceCreated(guid: UUID) extends GeneratorServiceActorMessage - case object SyncAll extends GeneratorServiceActorMessage - -} - -@javax.inject.Singleton -class GeneratorServiceActor @javax.inject.Inject() ( - system: ActorSystem, - processor: GeneratorServiceActorProcessor, -) extends Actor with ActorLogging with ErrorHandler { - - private[this] implicit val ec: ExecutionContext = system.dispatchers.lookup("generator-service-actor-context") - - system.scheduler.scheduleAtFixedRate(1.hour, 1.hour, self, GeneratorServiceActorMessage.SyncAll) - - def receive: Receive = { - case m: GeneratorServiceActorMessage => processor.processMessage(m) - case other => logUnhandledMessage(other) - } - -} - -class GeneratorServiceActorProcessor @Inject() ( - util: GeneratorServiceUtil -) { - def processMessage(msg: GeneratorServiceActorMessage)(implicit ec: ExecutionContext): Unit = { - import GeneratorServiceActorMessage._ - msg match { - case GeneratorServiceCreated(guid) => util.sync(guid) - case SyncAll => util.syncAll() - } - } -} - diff --git a/api/app/actors/MainActor.scala b/api/app/actors/MainActor.scala index b1cf2e16b..9891fc1bb 100644 --- a/api/app/actors/MainActor.scala +++ b/api/app/actors/MainActor.scala @@ -16,15 +16,12 @@ object MainActor { case class PasswordResetRequestCreated(guid: UUID) case class ApplicationCreated(guid: UUID) - - case class GeneratorServiceCreated(guid: UUID) } } @javax.inject.Singleton class MainActor @javax.inject.Inject() ( @javax.inject.Named("email-actor") emailActor: akka.actor.ActorRef, - @javax.inject.Named("generator-service-actor") generatorServiceActor: akka.actor.ActorRef, ) extends Actor with ActorLogging with ErrorHandler { def receive: Receive = akka.event.LoggingReceive { @@ -49,10 +46,6 @@ class MainActor @javax.inject.Inject() ( emailActor ! EmailActor.Messages.ApplicationCreated(guid) } - case m @ MainActor.Messages.GeneratorServiceCreated(guid) => withVerboseErrorHandler(m) { - generatorServiceActor ! GeneratorServiceActorMessage.GeneratorServiceCreated(guid) - } - case m @ MainActor.Messages.EmailVerificationCreated(guid) => withVerboseErrorHandler(m) { emailActor ! EmailActor.Messages.EmailVerificationCreated(guid) } diff --git a/api/app/actors/PeriodicActor.scala b/api/app/actors/PeriodicActor.scala index bfa6c1abb..825786a1c 100644 --- a/api/app/actors/PeriodicActor.scala +++ b/api/app/actors/PeriodicActor.scala @@ -36,6 +36,7 @@ class PeriodicActor @Inject() ( Seq( schedule(CleanupDeletions, FiniteDuration(1, HOURS)), schedule(ScheduleMigrateVersions, FiniteDuration(12, HOURS)), + schedule(ScheduleSyncGeneratorServices, FiniteDuration(1, HOURS)), ) } diff --git a/api/app/db/InternalTasksDao.scala b/api/app/db/InternalTasksDao.scala index 4d402c851..6f87a0b1a 100644 --- a/api/app/db/InternalTasksDao.scala +++ b/api/app/db/InternalTasksDao.scala @@ -17,6 +17,13 @@ class InternalTasksDao @Inject() ( dao.findByTypeIdAndType(typeId, typ.toString) } + // TODO: Use a bulk insert for this method + def queueBatch(typ: TaskType, ids: Seq[String]): Unit = { + ids.foreach { id => + queue(typ, id) + } + } + def queue(typ: TaskType, id: String, organizationGuid: Option[UUID] = None, data: JsValue = Json.obj()): Unit = { dao.db.withConnection { c => queueWithConnection(c, typ, id, organizationGuid = organizationGuid, data = data) diff --git a/api/app/db/generators/ServicesDao.scala b/api/app/db/generators/ServicesDao.scala index 4bfc0b1dc..8f5b4da5d 100644 --- a/api/app/db/generators/ServicesDao.scala +++ b/api/app/db/generators/ServicesDao.scala @@ -1,23 +1,22 @@ package db.generators -import io.apibuilder.api.v0.models.{GeneratorService, GeneratorServiceForm} -import db._ -import io.apibuilder.api.v0.models.{Error, User} +import anorm._ import core.Util -import javax.inject.{Inject, Singleton} - +import db._ +import io.apibuilder.api.v0.models.{Error, GeneratorService, GeneratorServiceForm, User} +import io.apibuilder.task.v0.models.TaskType +import io.flow.postgresql.Query import lib.{Pager, Validation} -import anorm._ import play.api.db._ -import java.util.UUID -import io.flow.postgresql.Query +import java.util.UUID +import javax.inject.{Inject, Singleton} @Singleton class ServicesDao @Inject() ( @NamedDatabase("default") db: Database, - @javax.inject.Named("main-actor") mainActor: akka.actor.ActorRef, - generatorsDao: GeneratorsDao + generatorsDao: GeneratorsDao, + internalTasksDao: InternalTasksDao ) { private[this] val dbHelpers = DbHelpers(db, "generators.services") @@ -45,7 +44,7 @@ class ServicesDao @Inject() ( uri = Some(form.uri.trim) ).headOption match { case None => Nil - case Some(uri) => { + case Some(_) => { Seq(s"URI[${form.uri.trim}] already exists") } } @@ -62,16 +61,15 @@ class ServicesDao @Inject() ( val guid = UUID.randomUUID - db.withConnection { implicit c => + db.withTransaction { implicit c => SQL(InsertQuery).on( "guid" -> guid, "uri" -> form.uri.trim, "created_by_guid" -> user.guid ).execute() + internalTasksDao.queueWithConnection(c, TaskType.SyncGeneratorService, guid.toString) } - mainActor ! actors.MainActor.Messages.GeneratorServiceCreated(guid) - findByGuid(Authorization.All, guid).getOrElse { sys.error("Failed to create service") } @@ -81,7 +79,7 @@ class ServicesDao @Inject() ( * Also will soft delete all generators for this service */ def softDelete(deletedBy: User, service: GeneratorService): Unit = { - Pager.eachPage { offset => + Pager.eachPage { _ => // Note we do not include offset in the query as each iteration // deletes records which will then NOT show up in the next loop generatorsDao.findAll( diff --git a/api/app/processor/ScheduleMigrateVersionsProcessor.scala b/api/app/processor/ScheduleMigrateVersionsProcessor.scala index 920a221b4..d62480f19 100644 --- a/api/app/processor/ScheduleMigrateVersionsProcessor.scala +++ b/api/app/processor/ScheduleMigrateVersionsProcessor.scala @@ -50,9 +50,7 @@ class ScheduleMigrateVersionsProcessor @Inject()( .as(SqlParser.get[_root_.java.util.UUID](1).*) } if (versionGuids.nonEmpty) { - versionGuids.foreach { vGuid => - internalTasksDao.queue(TaskType.MigrateVersion, vGuid.toString) - } + internalTasksDao.queueBatch(TaskType.MigrateVersion, versionGuids.map(_.toString)) scheduleMigrationTasks() } } diff --git a/api/app/processor/ScheduleSyncGeneratorServicesProcessor.scala b/api/app/processor/ScheduleSyncGeneratorServicesProcessor.scala new file mode 100644 index 000000000..24eb1e4ab --- /dev/null +++ b/api/app/processor/ScheduleSyncGeneratorServicesProcessor.scala @@ -0,0 +1,37 @@ +package processor + +import cats.data.ValidatedNec +import cats.implicits._ +import db.generators.ServicesDao +import db.{Authorization, InternalTasksDao} +import io.apibuilder.task.v0.models.TaskType +import lib.ValidatedHelpers + +import javax.inject.Inject +import scala.annotation.tailrec + + +class ScheduleSyncGeneratorServicesProcessor @Inject()( + args: TaskProcessorArgs, + servicesDao: ServicesDao, + internalTasksDao: InternalTasksDao +) extends TaskProcessor(args, TaskType.ScheduleSyncGeneratorServices) with ValidatedHelpers { + + override def processRecord(id: String): ValidatedNec[String, Unit] = { + doSyncAll(pageSize = 200, offset = 0).validNec + } + + @tailrec + private[this] def doSyncAll(pageSize: Long, offset: Long): Unit = { + val all = servicesDao.findAll( + Authorization.All, + limit = pageSize, + offset = offset + ).map(_.guid.toString) + + if (all.nonEmpty) { + internalTasksDao.queueBatch(TaskType.SyncGeneratorService, all) + doSyncAll(pageSize, offset + all.length) + } + } +} \ No newline at end of file diff --git a/api/app/processor/SyncGeneratorServiceProcessor.scala b/api/app/processor/SyncGeneratorServiceProcessor.scala new file mode 100644 index 000000000..2562022f7 --- /dev/null +++ b/api/app/processor/SyncGeneratorServiceProcessor.scala @@ -0,0 +1,118 @@ +package processor + +import akka.actor.ActorSystem +import cats.data.Validated.{Invalid, Valid} +import cats.data.ValidatedNec +import cats.implicits._ +import db.generators.{GeneratorsDao, ServicesDao} +import db.{Authorization, UsersDao} +import io.apibuilder.api.v0.models.{GeneratorForm, GeneratorService} +import io.apibuilder.generator.v0.interfaces.Client +import io.apibuilder.generator.v0.models.Generator +import io.apibuilder.task.v0.models.TaskType +import lib.{Pager, ValidatedHelpers} +import modules.clients.GeneratorClientFactory +import play.api.Logger + +import java.util.UUID +import javax.inject.Inject +import scala.annotation.tailrec +import scala.concurrent.duration.{FiniteDuration, SECONDS} +import scala.concurrent.{Await, ExecutionContext} +import scala.util.{Failure, Success, Try} + + +class SyncGeneratorServiceProcessor @Inject()( + args: TaskProcessorArgs, + system: ActorSystem, + servicesDao: ServicesDao, + generatorsDao: GeneratorsDao, + usersDao: UsersDao, + generatorClientFactory: GeneratorClientFactory +) extends TaskProcessorWithGuid(args, TaskType.SyncGeneratorService) with ValidatedHelpers { + + private[this] val log: Logger = Logger(this.getClass) + private[this] val ec: ExecutionContext = system.dispatchers.lookup("generator-service-sync-context") + + override def processRecord(guid: UUID): ValidatedNec[String, Unit] = { + syncAll()(ec).validNec + } + + private[this] def syncAll(pageSize: Long = 200)(implicit ec: scala.concurrent.ExecutionContext): Unit = { + Pager.eachPage { offset => + servicesDao.findAll( + Authorization.All, + limit = pageSize, + offset = offset + ) + } { service => + Try { + sync(service, pageSize = pageSize) + } match { + case Success(_) => { + log.info(s"[GeneratorServiceActor] Service[${service.guid}] at uri[${service.uri}] synced") + } + case Failure(ex) => { + ex match { + case _ => log.error(s"[GeneratorServiceActor] Service[${service.guid}] at uri[${service.uri}] failed to sync: ${ex.getMessage}", ex) + } + } + } + } + } + + private[this] def sync( + service: GeneratorService, + pageSize: Long = 200 + ) ( + implicit ec: scala.concurrent.ExecutionContext + ): Unit = { + doSync( + client = generatorClientFactory.instance(service.uri), + service = service, + pageSize = pageSize, + offset = 0, + resolved = Nil + ) + } + + @tailrec + private[this] def doSync( + client: Client, + service: GeneratorService, + pageSize: Long, + offset: Int, + resolved: List[Generator] + )( + implicit ec: scala.concurrent.ExecutionContext + ): Unit = { + val newGenerators = Await.result( + client.generators.get(limit = pageSize.toInt, offset = offset), + FiniteDuration(30, SECONDS) + ).filterNot { g => resolved.exists(_.key == g.key) } + + if (newGenerators.nonEmpty) { + storeGenerators(service, newGenerators) + doSync(client, service, pageSize = pageSize, offset + pageSize.toInt, resolved ++ newGenerators) + } + } + + private[this] def storeGenerators(service: GeneratorService, generators: Seq[Generator]): Unit = { + sequenceUnique { + generators.map { gen => + generatorsDao.upsert( + usersDao.AdminUser, + GeneratorForm( + serviceGuid = service.guid, + generator = gen + ) + ) + } + } match { + case Invalid(errors) => { + log.error(s"Error fetching generators for service[${service.guid}] uri[${service.uri}]: " + formatErrors(errors)) + } + case Valid(_) => // no-op + } + } +} \ No newline at end of file diff --git a/api/app/processor/TaskActorCompanion.scala b/api/app/processor/TaskActorCompanion.scala index e715e674a..6438a4ae3 100644 --- a/api/app/processor/TaskActorCompanion.scala +++ b/api/app/processor/TaskActorCompanion.scala @@ -10,7 +10,9 @@ class TaskActorCompanion @Inject() ( cleanupDeletions: CleanupDeletionsProcessor, scheduleMigrateVersions: ScheduleMigrateVersionsProcessor, migrateVersion: MigrateVersionProcessor, - userCreated: UserCreatedProcessor + userCreated: UserCreatedProcessor, + scheduleSyncGeneratorServices: ScheduleSyncGeneratorServicesProcessor, + syncGeneratorService: SyncGeneratorServiceProcessor ) { def process(typ: TaskType): Unit = { @@ -26,6 +28,8 @@ class TaskActorCompanion @Inject() ( case MigrateVersion => migrateVersion case ScheduleMigrateVersions => scheduleMigrateVersions case UserCreated => userCreated + case ScheduleSyncGeneratorServices => scheduleSyncGeneratorServices + case SyncGeneratorService => syncGeneratorService case UNDEFINED(_) => sys.error(s"Undefined task type '$typ") } } diff --git a/api/conf/base.conf b/api/conf/base.conf index 01ff9ecf1..590bdd73d 100644 --- a/api/conf/base.conf +++ b/api/conf/base.conf @@ -67,7 +67,7 @@ email-actor-context { } } -generator-service-actor-context { +generator-service-sync-context { fork-join-executor { parallelism-factor = 1.0 parallelism-max = 2 diff --git a/generated/app/ApicollectiveApibuilderTaskV0Client.scala b/generated/app/ApicollectiveApibuilderTaskV0Client.scala index e4bd8950e..5d9ef8230 100644 --- a/generated/app/ApicollectiveApibuilderTaskV0Client.scala +++ b/generated/app/ApicollectiveApibuilderTaskV0Client.scala @@ -17,6 +17,10 @@ package io.apibuilder.task.v0.models { case object CleanupDeletions extends TaskType { override def toString = "cleanup_deletions" } case object ScheduleMigrateVersions extends TaskType { override def toString = "schedule_migrate_versions" } case object MigrateVersion extends TaskType { override def toString = "migrate_version" } + case object ScheduleSyncGeneratorServices extends TaskType { + override def toString = "schedule_sync_generator_services" + } + case object SyncGeneratorService extends TaskType { override def toString = "sync_generator_service" } case object DiffVersion extends TaskType { override def toString = "diff_version" } case object UserCreated extends TaskType { override def toString = "user_created" } /** @@ -35,7 +39,7 @@ package io.apibuilder.task.v0.models { * lower case to avoid collisions with the camel cased values * above. */ - val all: scala.List[TaskType] = scala.List(IndexApplication, CleanupDeletions, ScheduleMigrateVersions, MigrateVersion, DiffVersion, UserCreated) + val all: scala.List[TaskType] = scala.List(IndexApplication, CleanupDeletions, ScheduleMigrateVersions, MigrateVersion, ScheduleSyncGeneratorServices, SyncGeneratorService, DiffVersion, UserCreated) private[this] val byName: Map[String, TaskType] = all.map(x => x.toString.toLowerCase -> x).toMap diff --git a/spec/apibuilder-task.json b/spec/apibuilder-task.json index a25991ad9..e07ab9722 100644 --- a/spec/apibuilder-task.json +++ b/spec/apibuilder-task.json @@ -20,6 +20,8 @@ { "name": "cleanup_deletions" }, { "name": "schedule_migrate_versions" }, { "name": "migrate_version" }, + { "name": "schedule_sync_generator_services" }, + { "name": "sync_generator_service" }, { "name": "diff_version" }, { "name": "user_created" } ] From 5dbca96451b61562408300f26c46ea7b8b2bffae Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 20:16:04 -0700 Subject: [PATCH 27/32] wip --- api/app/actors/Bindings.scala | 2 - api/app/actors/EmailActor.scala | 130 ----------- api/app/actors/MainActor.scala | 59 ----- api/app/processor/EmailProcessor.scala | 136 +++++++++++ api/app/processor/TaskProcessor.scala | 2 +- .../ApicollectiveApibuilderTaskV0Client.scala | 214 +++++++++++++++++- ...collectiveApibuilderTaskV0MockClient.scala | 36 ++- spec/apibuilder-task.json | 61 +++++ 8 files changed, 445 insertions(+), 195 deletions(-) delete mode 100644 api/app/actors/EmailActor.scala delete mode 100644 api/app/actors/MainActor.scala create mode 100644 api/app/processor/EmailProcessor.scala diff --git a/api/app/actors/Bindings.scala b/api/app/actors/Bindings.scala index ab02df448..584934fe3 100644 --- a/api/app/actors/Bindings.scala +++ b/api/app/actors/Bindings.scala @@ -5,9 +5,7 @@ import play.api.libs.concurrent.AkkaGuiceSupport class ActorsModule extends AbstractModule with AkkaGuiceSupport { override def configure = { - bindActor[MainActor]("main-actor") bindActor[PeriodicActor]("PeriodicActor") - bindActor[EmailActor]("email-actor") bindActor[TaskDispatchActor]( "TaskDispatchActor", _.withDispatcher("task-context-dispatcher") diff --git a/api/app/actors/EmailActor.scala b/api/app/actors/EmailActor.scala deleted file mode 100644 index ed28b6875..000000000 --- a/api/app/actors/EmailActor.scala +++ /dev/null @@ -1,130 +0,0 @@ -package actors - -import akka.actor.{Actor, ActorLogging} -import db._ -import io.apibuilder.api.v0.models.Publication -import lib.{AppConfig, EmailUtil, Person, Role} - -import java.util.UUID - -object EmailActor { - - object Messages { - case class EmailVerificationCreated(guid: UUID) - case class MembershipCreated(guid: UUID) - case class MembershipRequestCreated(guid: UUID) - case class MembershipRequestAccepted(organizationGuid: UUID, userGuid: UUID, role: Role) - case class MembershipRequestDeclined(organizationGuid: UUID, userGuid: UUID, role: Role) - case class PasswordResetRequestCreated(guid: UUID) - case class ApplicationCreated(guid: UUID) - } - -} - -@javax.inject.Singleton -class EmailActor @javax.inject.Inject() ( - appConfig: AppConfig, - applicationsDao: db.ApplicationsDao, - email: EmailUtil, - emails: Emails, - emailVerificationsDao: db.EmailVerificationsDao, - membershipsDao: db.MembershipsDao, - membershipRequestsDao: db.MembershipRequestsDao, - organizationsDao: OrganizationsDao, - passwordResetRequestsDao: db.PasswordResetRequestsDao, - usersDao: UsersDao -) extends Actor with ActorLogging with ErrorHandler { - - def receive: Receive = { - - case m @ EmailActor.Messages.MembershipRequestCreated(guid) => withVerboseErrorHandler(m) { - membershipRequestsDao.findByGuid(Authorization.All, guid).foreach { request => - emails.deliver( - context = Emails.Context.OrganizationAdmin, - org = request.organization, - publication = Publication.MembershipRequestsCreate, - subject = s"${request.organization.name}: Membership Request from ${request.user.email}", - body = views.html.emails.membershipRequestCreated(appConfig, request).toString - ) - } - } - - case m @ EmailActor.Messages.MembershipRequestAccepted(organizationGuid, userGuid, role) => withVerboseErrorHandler(m) { - organizationsDao.findByGuid(Authorization.All, organizationGuid).foreach { org => - usersDao.findByGuid(userGuid).foreach { user => - email.sendHtml( - to = Person(user), - subject = s"Welcome to ${org.name}", - body = views.html.emails.membershipRequestAccepted(org, user, role).toString - ) - } - } - } - - case m @ EmailActor.Messages.MembershipRequestDeclined(organizationGuid, userGuid, role) => withVerboseErrorHandler(m) { - organizationsDao.findByGuid(Authorization.All, organizationGuid).foreach { org => - usersDao.findByGuid(userGuid).foreach { user => - email.sendHtml( - to = Person(user), - subject = s"Your Membership Request to join ${org.name} was declined", - body = views.html.emails.membershipRequestDeclined(org, user, role).toString - ) - } - } - } - - case m @ EmailActor.Messages.MembershipCreated(guid) => withVerboseErrorHandler(m) { - membershipsDao.findByGuid(Authorization.All, guid).foreach { membership => - emails.deliver( - context = Emails.Context.OrganizationAdmin, - org = membership.organization, - publication = Publication.MembershipsCreate, - subject = s"${membership.organization.name}: ${membership.user.email} has joined as ${membership.role}", - body = views.html.emails.membershipCreated(appConfig, membership).toString - ) - } - } - - case m @ EmailActor.Messages.ApplicationCreated(guid) => withVerboseErrorHandler(m) { - applicationsDao.findByGuid(Authorization.All, guid).foreach { application => - organizationsDao.findAll(Authorization.All, application = Some(application)).foreach { org => - emails.deliver( - context = Emails.Context.OrganizationMember, - org = org, - publication = Publication.ApplicationsCreate, - subject = s"${org.name}: New Application Created - ${application.name}", - body = views.html.emails.applicationCreated(appConfig, org, application).toString - ) - } - } - } - - case m @ EmailActor.Messages.PasswordResetRequestCreated(guid) => withVerboseErrorHandler(m) { - passwordResetRequestsDao.findByGuid(guid).foreach { request => - usersDao.findByGuid(request.userGuid).foreach { user => - email.sendHtml( - to = Person(user), - subject = s"Reset your password", - body = views.html.emails.passwordResetRequestCreated(appConfig, request.token).toString - ) - } - } - } - - case m @ EmailActor.Messages.EmailVerificationCreated(guid) => withVerboseErrorHandler(m) { - emailVerificationsDao.findByGuid(guid).foreach { verification => - usersDao.findByGuid(verification.userGuid).foreach { user => - email.sendHtml( - to = Person(email = verification.email, name = user.name), - subject = "Verify your email address", - body = views.html.emails.emailVerificationCreated(appConfig, verification).toString - ) - } - } - } - - case m: Any => logUnhandledMessage(m) - } - -} - diff --git a/api/app/actors/MainActor.scala b/api/app/actors/MainActor.scala deleted file mode 100644 index 9891fc1bb..000000000 --- a/api/app/actors/MainActor.scala +++ /dev/null @@ -1,59 +0,0 @@ -package actors - -import akka.actor._ -import lib.Role - -import java.util.UUID - -object MainActor { - - object Messages { - case class EmailVerificationCreated(guid: UUID) - case class MembershipRequestCreated(guid: UUID) - case class MembershipRequestAccepted(organizationGuid: UUID, userGuid: UUID, role: Role) - case class MembershipRequestDeclined(organizationGuid: UUID, userGuid: UUID, role: Role) - case class MembershipCreated(guid: UUID) - - case class PasswordResetRequestCreated(guid: UUID) - case class ApplicationCreated(guid: UUID) - } -} - -@javax.inject.Singleton -class MainActor @javax.inject.Inject() ( - @javax.inject.Named("email-actor") emailActor: akka.actor.ActorRef, -) extends Actor with ActorLogging with ErrorHandler { - - def receive: Receive = akka.event.LoggingReceive { - - case m @ MainActor.Messages.MembershipRequestCreated(guid) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.MembershipRequestCreated(guid) - } - - case m @ MainActor.Messages.MembershipRequestAccepted(organizationGuid, userGuid, role) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.MembershipRequestAccepted(organizationGuid, userGuid, role) - } - - case m @ MainActor.Messages.MembershipRequestDeclined(organizationGuid, userGuid, role) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.MembershipRequestDeclined(organizationGuid, userGuid, role) - } - - case m @ MainActor.Messages.MembershipCreated(guid) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.MembershipCreated(guid) - } - - case m @ MainActor.Messages.ApplicationCreated(guid) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.ApplicationCreated(guid) - } - - case m @ MainActor.Messages.EmailVerificationCreated(guid) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.EmailVerificationCreated(guid) - } - - case m @ MainActor.Messages.PasswordResetRequestCreated(guid) => withVerboseErrorHandler(m) { - emailActor ! EmailActor.Messages.PasswordResetRequestCreated(guid) - } - - case m: Any => logUnhandledMessage(m) - } -} diff --git a/api/app/processor/EmailProcessor.scala b/api/app/processor/EmailProcessor.scala new file mode 100644 index 000000000..0601eb8d9 --- /dev/null +++ b/api/app/processor/EmailProcessor.scala @@ -0,0 +1,136 @@ +package processor + +import actors.Emails +import cats.data.ValidatedNec +import cats.implicits._ +import db.{Authorization, OrganizationsDao, UsersDao} +import io.apibuilder.api.v0.models.Publication +import io.apibuilder.task.v0.models._ +import io.apibuilder.task.v0.models.json._ +import lib.{AppConfig, EmailUtil, Person, Role} + +import java.util.UUID +import javax.inject.Inject + + +class EmailProcessor @Inject()( + args: TaskProcessorArgs, + appConfig: AppConfig, + applicationsDao: db.ApplicationsDao, + email: EmailUtil, + emails: Emails, + emailVerificationsDao: db.EmailVerificationsDao, + membershipsDao: db.MembershipsDao, + membershipRequestsDao: db.MembershipRequestsDao, + organizationsDao: OrganizationsDao, + passwordResetRequestsDao: db.PasswordResetRequestsDao, + usersDao: UsersDao, +) extends TaskProcessorWithData[EmailData](args, TaskType.Email) { + + override def processRecord(id: String, data: EmailData): ValidatedNec[String, Unit] = { + data match { + case EmailDataApplicationCreated(guid) => applicationCreated(guid).validNec + case EmailDataEmailVerificationCreated(guid) => emailVerificationCreated(guid).validNec + case EmailDataMembershipCreated(guid) => membershipCreated(guid).validNec + case EmailDataMembershipRequestCreated(guid) => membershipRequestCreated(guid).validNec + case EmailDataMembershipRequestAccepted(orgGuid, userGuid, role) => membershipRequestAccepted(orgGuid, userGuid, role) + case EmailDataMembershipRequestDeclined(orgGuid, userGuid, role) => membershipRequestDeclined(orgGuid, userGuid, role) + case EmailDataPasswordResetRequestCreated(guid) => passwordResetRequestCreated(guid).validNec + case EmailDataUndefinedType(description) => s"Invalid email data type '$description'".invalidNec + } + } + + private[this] def applicationCreated(applicationGuid: UUID): Unit = { + applicationsDao.findByGuid(Authorization.All, applicationGuid).foreach { application => + organizationsDao.findByGuid(Authorization.All, application.guid).foreach { org => + emails.deliver( + context = Emails.Context.OrganizationMember, + org = org, + publication = Publication.ApplicationsCreate, + subject = s"${org.name}: New Application Created - ${application.name}", + body = views.html.emails.applicationCreated(appConfig, org, application).toString + ) + } + } + } + + private[this] def emailVerificationCreated(guid: UUID): Unit = { + emailVerificationsDao.findByGuid(guid).foreach { verification => + usersDao.findByGuid(verification.userGuid).foreach { user => + email.sendHtml( + to = Person(email = verification.email, name = user.name), + subject = "Verify your email address", + body = views.html.emails.emailVerificationCreated(appConfig, verification).toString + ) + } + } + } + + private[this] def membershipCreated(guid: UUID): Unit = { + membershipsDao.findByGuid(Authorization.All, guid).foreach { membership => + emails.deliver( + context = Emails.Context.OrganizationAdmin, + org = membership.organization, + publication = Publication.MembershipsCreate, + subject = s"${membership.organization.name}: ${membership.user.email} has joined as ${membership.role}", + body = views.html.emails.membershipCreated(appConfig, membership).toString + ) + } + } + + private[this] def membershipRequestCreated(guid: UUID): Unit = { + membershipRequestsDao.findByGuid(Authorization.All, guid).foreach { request => + emails.deliver( + context = Emails.Context.OrganizationAdmin, + org = request.organization, + publication = Publication.MembershipRequestsCreate, + subject = s"${request.organization.name}: Membership Request from ${request.user.email}", + body = views.html.emails.membershipRequestCreated(appConfig, request).toString + ) + } + } + + private[this] def validateRole(role: String): ValidatedNec[String, Role] = { + Role.fromString(role).toValidNec(s"Invalid role '$role'") + } + + private[this] def membershipRequestAccepted(orgGuid: UUID, userGuid: UUID, role: String): ValidatedNec[String, Unit] = { + validateRole(role).map { vRole => + organizationsDao.findByGuid(Authorization.All, orgGuid).foreach { org => + usersDao.findByGuid(userGuid).foreach { user => + email.sendHtml( + to = Person(user), + subject = s"Welcome to ${org.name}", + body = views.html.emails.membershipRequestAccepted(org, user, vRole).toString + ) + } + } + } + } + + private[this] def membershipRequestDeclined(orgGuid: UUID, userGuid: UUID, role: String): ValidatedNec[String, Unit] = { + validateRole(role).map { vRole => + organizationsDao.findByGuid(Authorization.All, orgGuid).foreach { org => + usersDao.findByGuid(userGuid).foreach { user => + email.sendHtml( + to = Person(user), + subject = s"Your Membership Request to join ${org.name} was declined", + body = views.html.emails.membershipRequestDeclined(org, user, vRole).toString + ) + } + } + } + } + + private[this] def passwordResetRequestCreated(guid: UUID): Unit = { + passwordResetRequestsDao.findByGuid(guid).foreach { request => + usersDao.findByGuid(request.userGuid).foreach { user => + email.sendHtml( + to = Person(user), + subject = "Reset your password", + body = views.html.emails.passwordResetRequestCreated(appConfig, request.token).toString + ) + } + } + } +} \ No newline at end of file diff --git a/api/app/processor/TaskProcessor.scala b/api/app/processor/TaskProcessor.scala index 9aea180cd..d866c7bf8 100644 --- a/api/app/processor/TaskProcessor.scala +++ b/api/app/processor/TaskProcessor.scala @@ -43,7 +43,7 @@ abstract class TaskProcessorWithGuid( validateGuid(task.typeId).andThen(processRecord) } - private[this] def validateGuid(value: String): ValidatedNec[String, UUID] = { + protected def validateGuid(value: String): ValidatedNec[String, UUID] = { Try { UUID.fromString(value) } match { diff --git a/generated/app/ApicollectiveApibuilderTaskV0Client.scala b/generated/app/ApicollectiveApibuilderTaskV0Client.scala index 5d9ef8230..dad800f0b 100644 --- a/generated/app/ApicollectiveApibuilderTaskV0Client.scala +++ b/generated/app/ApicollectiveApibuilderTaskV0Client.scala @@ -5,14 +5,61 @@ */ package io.apibuilder.task.v0.models { + sealed trait EmailData extends _root_.scala.Product with _root_.scala.Serializable final case class DiffVersionData( oldVersionGuid: _root_.java.util.UUID, newVersionGuid: _root_.java.util.UUID ) + + final case class EmailDataApplicationCreated( + applicationGuid: _root_.java.util.UUID + ) extends EmailData + + final case class EmailDataEmailVerificationCreated( + guid: _root_.java.util.UUID + ) extends EmailData + + final case class EmailDataMembershipCreated( + guid: _root_.java.util.UUID + ) extends EmailData + + final case class EmailDataMembershipRequestAccepted( + organizationGuid: _root_.java.util.UUID, + userGuid: _root_.java.util.UUID, + role: String + ) extends EmailData + + final case class EmailDataMembershipRequestCreated( + guid: _root_.java.util.UUID + ) extends EmailData + + final case class EmailDataMembershipRequestDeclined( + organizationGuid: _root_.java.util.UUID, + userGuid: _root_.java.util.UUID, + role: String + ) extends EmailData + + final case class EmailDataPasswordResetRequestCreated( + guid: _root_.java.util.UUID + ) extends EmailData + + /** + * Provides future compatibility in clients - in the future, when a type is added + * to the union EmailData, it will need to be handled in the client code. This + * implementation will deserialize these future types as an instance of this class. + * + * @param description Information about the type that we received that is undefined in this version of + * the client. + */ + + final case class EmailDataUndefinedType( + description: String + ) extends EmailData sealed trait TaskType extends _root_.scala.Product with _root_.scala.Serializable object TaskType { + case object Email extends TaskType { override def toString = "email" } case object IndexApplication extends TaskType { override def toString = "index_application" } case object CleanupDeletions extends TaskType { override def toString = "cleanup_deletions" } case object ScheduleMigrateVersions extends TaskType { override def toString = "schedule_migrate_versions" } @@ -39,7 +86,7 @@ package io.apibuilder.task.v0.models { * lower case to avoid collisions with the camel cased values * above. */ - val all: scala.List[TaskType] = scala.List(IndexApplication, CleanupDeletions, ScheduleMigrateVersions, MigrateVersion, ScheduleSyncGeneratorServices, SyncGeneratorService, DiffVersion, UserCreated) + val all: scala.List[TaskType] = scala.List(Email, IndexApplication, CleanupDeletions, ScheduleMigrateVersions, MigrateVersion, ScheduleSyncGeneratorServices, SyncGeneratorService, DiffVersion, UserCreated) private[this] val byName: Map[String, TaskType] = all.map(x => x.toString.toLowerCase -> x).toMap @@ -134,6 +181,169 @@ package io.apibuilder.task.v0.models { jsObjectDiffVersionData(obj) } } + + implicit def jsonReadsApibuilderTaskEmailDataApplicationCreated: play.api.libs.json.Reads[EmailDataApplicationCreated] = { + (__ \ "application_guid").read[_root_.java.util.UUID].map { x => new EmailDataApplicationCreated(applicationGuid = x) } + } + + def jsObjectEmailDataApplicationCreated(obj: io.apibuilder.task.v0.models.EmailDataApplicationCreated): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "application_guid" -> play.api.libs.json.JsString(obj.applicationGuid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataApplicationCreated: play.api.libs.json.Writes[EmailDataApplicationCreated] = { + (obj: io.apibuilder.task.v0.models.EmailDataApplicationCreated) => { + jsObjectEmailDataApplicationCreated(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataEmailVerificationCreated: play.api.libs.json.Reads[EmailDataEmailVerificationCreated] = { + (__ \ "guid").read[_root_.java.util.UUID].map { x => new EmailDataEmailVerificationCreated(guid = x) } + } + + def jsObjectEmailDataEmailVerificationCreated(obj: io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "guid" -> play.api.libs.json.JsString(obj.guid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataEmailVerificationCreated: play.api.libs.json.Writes[EmailDataEmailVerificationCreated] = { + (obj: io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated) => { + jsObjectEmailDataEmailVerificationCreated(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataMembershipCreated: play.api.libs.json.Reads[EmailDataMembershipCreated] = { + (__ \ "guid").read[_root_.java.util.UUID].map { x => new EmailDataMembershipCreated(guid = x) } + } + + def jsObjectEmailDataMembershipCreated(obj: io.apibuilder.task.v0.models.EmailDataMembershipCreated): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "guid" -> play.api.libs.json.JsString(obj.guid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataMembershipCreated: play.api.libs.json.Writes[EmailDataMembershipCreated] = { + (obj: io.apibuilder.task.v0.models.EmailDataMembershipCreated) => { + jsObjectEmailDataMembershipCreated(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataMembershipRequestAccepted: play.api.libs.json.Reads[EmailDataMembershipRequestAccepted] = { + for { + organizationGuid <- (__ \ "organization_guid").read[_root_.java.util.UUID] + userGuid <- (__ \ "user_guid").read[_root_.java.util.UUID] + role <- (__ \ "role").read[String] + } yield EmailDataMembershipRequestAccepted(organizationGuid, userGuid, role) + } + + def jsObjectEmailDataMembershipRequestAccepted(obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestAccepted): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "organization_guid" -> play.api.libs.json.JsString(obj.organizationGuid.toString), + "user_guid" -> play.api.libs.json.JsString(obj.userGuid.toString), + "role" -> play.api.libs.json.JsString(obj.role) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataMembershipRequestAccepted: play.api.libs.json.Writes[EmailDataMembershipRequestAccepted] = { + (obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestAccepted) => { + jsObjectEmailDataMembershipRequestAccepted(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataMembershipRequestCreated: play.api.libs.json.Reads[EmailDataMembershipRequestCreated] = { + (__ \ "guid").read[_root_.java.util.UUID].map { x => new EmailDataMembershipRequestCreated(guid = x) } + } + + def jsObjectEmailDataMembershipRequestCreated(obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestCreated): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "guid" -> play.api.libs.json.JsString(obj.guid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataMembershipRequestCreated: play.api.libs.json.Writes[EmailDataMembershipRequestCreated] = { + (obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestCreated) => { + jsObjectEmailDataMembershipRequestCreated(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataMembershipRequestDeclined: play.api.libs.json.Reads[EmailDataMembershipRequestDeclined] = { + for { + organizationGuid <- (__ \ "organization_guid").read[_root_.java.util.UUID] + userGuid <- (__ \ "user_guid").read[_root_.java.util.UUID] + role <- (__ \ "role").read[String] + } yield EmailDataMembershipRequestDeclined(organizationGuid, userGuid, role) + } + + def jsObjectEmailDataMembershipRequestDeclined(obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestDeclined): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "organization_guid" -> play.api.libs.json.JsString(obj.organizationGuid.toString), + "user_guid" -> play.api.libs.json.JsString(obj.userGuid.toString), + "role" -> play.api.libs.json.JsString(obj.role) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataMembershipRequestDeclined: play.api.libs.json.Writes[EmailDataMembershipRequestDeclined] = { + (obj: io.apibuilder.task.v0.models.EmailDataMembershipRequestDeclined) => { + jsObjectEmailDataMembershipRequestDeclined(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailDataPasswordResetRequestCreated: play.api.libs.json.Reads[EmailDataPasswordResetRequestCreated] = { + (__ \ "guid").read[_root_.java.util.UUID].map { x => new EmailDataPasswordResetRequestCreated(guid = x) } + } + + def jsObjectEmailDataPasswordResetRequestCreated(obj: io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated): play.api.libs.json.JsObject = { + play.api.libs.json.Json.obj( + "guid" -> play.api.libs.json.JsString(obj.guid.toString) + ) + } + + implicit def jsonWritesApibuilderTaskEmailDataPasswordResetRequestCreated: play.api.libs.json.Writes[EmailDataPasswordResetRequestCreated] = { + (obj: io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated) => { + jsObjectEmailDataPasswordResetRequestCreated(obj) + } + } + + implicit def jsonReadsApibuilderTaskEmailData: play.api.libs.json.Reads[EmailData] = { + ( + (__ \ "email_data_application_created").read(jsonReadsApibuilderTaskEmailDataApplicationCreated).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_email_verification_created").read(jsonReadsApibuilderTaskEmailDataEmailVerificationCreated).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_membership_created").read(jsonReadsApibuilderTaskEmailDataMembershipCreated).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_membership_request_created").read(jsonReadsApibuilderTaskEmailDataMembershipRequestCreated).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_membership_request_accepted").read(jsonReadsApibuilderTaskEmailDataMembershipRequestAccepted).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_membership_request_declined").read(jsonReadsApibuilderTaskEmailDataMembershipRequestDeclined).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + (__ \ "email_data_password_reset_request_created").read(jsonReadsApibuilderTaskEmailDataPasswordResetRequestCreated).asInstanceOf[play.api.libs.json.Reads[EmailData]] + orElse + play.api.libs.json.Reads(jsValue => play.api.libs.json.JsSuccess(EmailDataUndefinedType(jsValue.toString))).asInstanceOf[play.api.libs.json.Reads[EmailData]] + ) + } + + def jsObjectEmailData(obj: io.apibuilder.task.v0.models.EmailData): play.api.libs.json.JsObject = { + obj match { + case x: io.apibuilder.task.v0.models.EmailDataApplicationCreated => play.api.libs.json.Json.obj("email_data_application_created" -> jsObjectEmailDataApplicationCreated(x)) + case x: io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated => play.api.libs.json.Json.obj("email_data_email_verification_created" -> jsObjectEmailDataEmailVerificationCreated(x)) + case x: io.apibuilder.task.v0.models.EmailDataMembershipCreated => play.api.libs.json.Json.obj("email_data_membership_created" -> jsObjectEmailDataMembershipCreated(x)) + case x: io.apibuilder.task.v0.models.EmailDataMembershipRequestCreated => play.api.libs.json.Json.obj("email_data_membership_request_created" -> jsObjectEmailDataMembershipRequestCreated(x)) + case x: io.apibuilder.task.v0.models.EmailDataMembershipRequestAccepted => play.api.libs.json.Json.obj("email_data_membership_request_accepted" -> jsObjectEmailDataMembershipRequestAccepted(x)) + case x: io.apibuilder.task.v0.models.EmailDataMembershipRequestDeclined => play.api.libs.json.Json.obj("email_data_membership_request_declined" -> jsObjectEmailDataMembershipRequestDeclined(x)) + case x: io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated => play.api.libs.json.Json.obj("email_data_password_reset_request_created" -> jsObjectEmailDataPasswordResetRequestCreated(x)) + case x: io.apibuilder.task.v0.models.EmailDataUndefinedType => sys.error(s"The type[io.apibuilder.task.v0.models.EmailDataUndefinedType] should never be serialized") + } + } + + implicit def jsonWritesApibuilderTaskEmailData: play.api.libs.json.Writes[EmailData] = { + (obj: io.apibuilder.task.v0.models.EmailData) => { + jsObjectEmailData(obj) + } + } } } @@ -161,7 +371,7 @@ package io.apibuilder.task.v0 { val taskTypeConverter: ApibuilderTypeConverter[io.apibuilder.task.v0.models.TaskType] = new ApibuilderTypeConverter[io.apibuilder.task.v0.models.TaskType] { override def convert(value: String): io.apibuilder.task.v0.models.TaskType = io.apibuilder.task.v0.models.TaskType(value) override def convert(value: io.apibuilder.task.v0.models.TaskType): String = value.toString - override def example: io.apibuilder.task.v0.models.TaskType = io.apibuilder.task.v0.models.TaskType.IndexApplication + override def example: io.apibuilder.task.v0.models.TaskType = io.apibuilder.task.v0.models.TaskType.Email override def validValues: Seq[io.apibuilder.task.v0.models.TaskType] = io.apibuilder.task.v0.models.TaskType.all } implicit def pathBindableTaskType(implicit stringBinder: QueryStringBindable[String]): PathBindable[io.apibuilder.task.v0.models.TaskType] = ApibuilderPathBindable(taskTypeConverter) diff --git a/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala b/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala index 5504cf9f8..3d2a23648 100644 --- a/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala +++ b/generated/app/ApicollectiveApibuilderTaskV0MockClient.scala @@ -11,13 +11,47 @@ package io.apibuilder.task.v0.mock { _root_.scala.util.Random.alphanumeric.take(length).mkString } - def makeTaskType(): io.apibuilder.task.v0.models.TaskType = io.apibuilder.task.v0.models.TaskType.IndexApplication + def makeTaskType(): io.apibuilder.task.v0.models.TaskType = io.apibuilder.task.v0.models.TaskType.Email def makeDiffVersionData(): io.apibuilder.task.v0.models.DiffVersionData = io.apibuilder.task.v0.models.DiffVersionData( oldVersionGuid = _root_.java.util.UUID.randomUUID, newVersionGuid = _root_.java.util.UUID.randomUUID ) + def makeEmailDataApplicationCreated(): io.apibuilder.task.v0.models.EmailDataApplicationCreated = io.apibuilder.task.v0.models.EmailDataApplicationCreated( + applicationGuid = _root_.java.util.UUID.randomUUID + ) + + def makeEmailDataEmailVerificationCreated(): io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated = io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated( + guid = _root_.java.util.UUID.randomUUID + ) + + def makeEmailDataMembershipCreated(): io.apibuilder.task.v0.models.EmailDataMembershipCreated = io.apibuilder.task.v0.models.EmailDataMembershipCreated( + guid = _root_.java.util.UUID.randomUUID + ) + + def makeEmailDataMembershipRequestAccepted(): io.apibuilder.task.v0.models.EmailDataMembershipRequestAccepted = io.apibuilder.task.v0.models.EmailDataMembershipRequestAccepted( + organizationGuid = _root_.java.util.UUID.randomUUID, + userGuid = _root_.java.util.UUID.randomUUID, + role = Factories.randomString(24) + ) + + def makeEmailDataMembershipRequestCreated(): io.apibuilder.task.v0.models.EmailDataMembershipRequestCreated = io.apibuilder.task.v0.models.EmailDataMembershipRequestCreated( + guid = _root_.java.util.UUID.randomUUID + ) + + def makeEmailDataMembershipRequestDeclined(): io.apibuilder.task.v0.models.EmailDataMembershipRequestDeclined = io.apibuilder.task.v0.models.EmailDataMembershipRequestDeclined( + organizationGuid = _root_.java.util.UUID.randomUUID, + userGuid = _root_.java.util.UUID.randomUUID, + role = Factories.randomString(24) + ) + + def makeEmailDataPasswordResetRequestCreated(): io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated = io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated( + guid = _root_.java.util.UUID.randomUUID + ) + + def makeEmailData(): io.apibuilder.task.v0.models.EmailData = io.apibuilder.task.v0.mock.Factories.makeEmailDataApplicationCreated() + } } \ No newline at end of file diff --git a/spec/apibuilder-task.json b/spec/apibuilder-task.json index e07ab9722..9f7c0f5c6 100644 --- a/spec/apibuilder-task.json +++ b/spec/apibuilder-task.json @@ -16,6 +16,7 @@ "enums": { "task_type": { "values": [ + { "name": "email" }, { "name": "index_application" }, { "name": "cleanup_deletions" }, { "name": "schedule_migrate_versions" }, @@ -28,12 +29,72 @@ } }, + "unions": { + "email_data": { + "types": [ + { "type": "email_data_application_created" }, + { "type": "email_data_email_verification_created" }, + { "type": "email_data_membership_created" }, + { "type": "email_data_membership_request_created" }, + { "type": "email_data_membership_request_accepted" }, + { "type": "email_data_membership_request_declined" }, + { "type": "email_data_password_reset_request_created" } + ] + } + }, + "models": { "diff_version_data": { "fields": [ { "name": "old_version_guid", "type": "uuid" }, { "name": "new_version_guid", "type": "uuid" } ] + }, + + "email_data_application_created": { + "fields": [ + { "name": "application_guid", "type": "uuid" } + ] + }, + + "email_data_email_verification_created": { + "fields": [ + { "name": "guid", "type": "uuid" } + ] + }, + + "email_data_membership_created": { + "fields": [ + { "name": "guid", "type": "uuid" } + ] + }, + + "email_data_membership_request_created": { + "fields": [ + { "name": "guid", "type": "uuid" } + ] + }, + + "email_data_membership_request_accepted": { + "fields": [ + { "name": "organization_guid", "type": "uuid" }, + { "name": "user_guid", "type": "uuid" }, + { "name": "role", "type": "string" } + ] + }, + + "email_data_membership_request_declined": { + "fields": [ + { "name": "organization_guid", "type": "uuid" }, + { "name": "user_guid", "type": "uuid" }, + { "name": "role", "type": "string" } + ] + }, + + "email_data_password_reset_request_created": { + "fields": [ + { "name": "guid", "type": "uuid" } + ] } } } From c80be6d8bc9e8de22ce52816e3ed24ab01d5c2fa Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Wed, 5 Jun 2024 23:42:13 -0400 Subject: [PATCH 28/32] wip --- api/app/db/ApplicationsDao.scala | 15 +++++-------- api/app/db/EmailVerificationsDao.scala | 18 ++++++++------- api/app/db/MembershipRequestsDao.scala | 26 +++++++++++----------- api/app/db/MembershipsDao.scala | 15 ++++++++----- api/app/db/PasswordResetRequestsDao.scala | 23 ++++++++++--------- api/app/processor/EmailProcessor.scala | 12 +++++++++- api/app/processor/TaskActorCompanion.scala | 4 +++- 7 files changed, 64 insertions(+), 49 deletions(-) diff --git a/api/app/db/ApplicationsDao.scala b/api/app/db/ApplicationsDao.scala index 82b6eca26..f7c6217db 100644 --- a/api/app/db/ApplicationsDao.scala +++ b/api/app/db/ApplicationsDao.scala @@ -2,18 +2,19 @@ package db import anorm._ import io.apibuilder.api.v0.models.{AppSortBy, Application, ApplicationForm, Error, MoveForm, Organization, SortOrder, User, Version, Visibility} -import io.apibuilder.task.v0.models.TaskType +import io.apibuilder.task.v0.models.{EmailDataApplicationCreated, TaskType} import io.flow.postgresql.Query import lib.{UrlKey, Validation} import play.api.db._ +import processor.EmailProcessorQueue import java.util.UUID -import javax.inject.{Inject, Named, Singleton} +import javax.inject.{Inject, Singleton} @Singleton class ApplicationsDao @Inject() ( - @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, + emailQueue: EmailProcessorQueue, organizationsDao: OrganizationsDao, tasksDao: InternalTasksDao, ) { @@ -263,10 +264,9 @@ class ApplicationsDao @Inject() ( "created_by_guid" -> createdBy.guid, "updated_by_guid" -> createdBy.guid ).execute() + emailQueue.queueWithConnection(c, EmailDataApplicationCreated(guid)) }) - mainActor ! actors.MainActor.Messages.ApplicationCreated(guid) - findAll(Authorization.All, orgKey = Some(org.key), key = Some(key)).headOption.getOrElse { sys.error("Failed to create application") } @@ -279,10 +279,7 @@ class ApplicationsDao @Inject() ( } def canUserUpdate(user: User, app: Application): Boolean = { - findAll(Authorization.User(user.guid), key = Some(app.key)).headOption match { - case None => false - case Some(a) => true - } + findAll(Authorization.User(user.guid), key = Some(app.key)).nonEmpty } private[db] def findByOrganizationAndName(authorization: Authorization, org: Organization, name: String): Option[Application] = { diff --git a/api/app/db/EmailVerificationsDao.scala b/api/app/db/EmailVerificationsDao.scala index 13828f9a3..9cdcbef70 100644 --- a/api/app/db/EmailVerificationsDao.scala +++ b/api/app/db/EmailVerificationsDao.scala @@ -1,14 +1,17 @@ package db +import anorm.JodaParameterMetaData._ +import anorm._ import io.apibuilder.api.v0.models.User +import io.apibuilder.task.v0.models.EmailDataEmailVerificationCreated import io.flow.postgresql.Query import lib.{Role, TokenGenerator} -import anorm._ -import anorm.JodaParameterMetaData._ -import javax.inject.{Inject, Named, Singleton} +import org.joda.time.DateTime import play.api.db._ +import processor.EmailProcessorQueue + import java.util.UUID -import org.joda.time.DateTime +import javax.inject.{Inject, Singleton} case class EmailVerification( guid: UUID, @@ -20,8 +23,8 @@ case class EmailVerification( @Singleton class EmailVerificationsDao @Inject() ( - @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, + emailQueue: EmailProcessorQueue, emailVerificationConfirmationsDao: EmailVerificationConfirmationsDao, membershipRequestsDao: MembershipRequestsDao, organizationsDao: OrganizationsDao @@ -56,7 +59,7 @@ class EmailVerificationsDao @Inject() ( def create(createdBy: User, user: User, email: String): EmailVerification = { val guid = UUID.randomUUID - db.withConnection { implicit c => + db.withTransaction { implicit c => SQL(InsertQuery).on( "guid" -> guid, "user_guid" -> user.guid, @@ -65,10 +68,9 @@ class EmailVerificationsDao @Inject() ( "expires_at" -> DateTime.now.plusHours(HoursUntilTokenExpires), "created_by_guid" -> createdBy.guid ).execute() + emailQueue.queueWithConnection(c, EmailDataEmailVerificationCreated(guid)) } - mainActor ! actors.MainActor.Messages.EmailVerificationCreated(guid) - findByGuid(guid).getOrElse { sys.error("Failed to create email verification") } diff --git a/api/app/db/MembershipRequestsDao.scala b/api/app/db/MembershipRequestsDao.scala index bc36b70c4..e949510c9 100644 --- a/api/app/db/MembershipRequestsDao.scala +++ b/api/app/db/MembershipRequestsDao.scala @@ -1,17 +1,20 @@ package db +import anorm._ import io.apibuilder.api.v0.models.{MembershipRequest, Organization, User} +import io.apibuilder.task.v0.models.{EmailDataMembershipRequestAccepted, EmailDataMembershipRequestCreated, EmailDataMembershipRequestDeclined} import io.flow.postgresql.Query import lib.Role -import anorm._ -import javax.inject.{Inject, Named, Singleton} import play.api.db._ +import processor.EmailProcessorQueue + import java.util.UUID +import javax.inject.{Inject, Singleton} @Singleton class MembershipRequestsDao @Inject() ( - @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, + emailQueue: EmailProcessorQueue, membershipsDao: MembershipsDao, organizationLogsDao: OrganizationLogsDao ) { @@ -59,7 +62,7 @@ class MembershipRequestsDao @Inject() ( private[db] def create(createdBy: User, organization: Organization, user: User, role: Role): MembershipRequest = { val guid = UUID.randomUUID - db.withConnection { implicit c => + db.withTransaction { implicit c => SQL(InsertQuery).on( "guid" -> guid, "organization_guid" -> organization.guid, @@ -67,10 +70,9 @@ class MembershipRequestsDao @Inject() ( "role" -> role.key, "created_by_guid" -> createdBy.guid ).execute() + emailQueue.queueWithConnection(c, EmailDataMembershipRequestCreated(guid)) } - mainActor ! actors.MainActor.Messages.MembershipRequestCreated(guid) - findByGuid(Authorization.All, guid).getOrElse { sys.error("Failed to create membership_request") } @@ -101,13 +103,12 @@ class MembershipRequestsDao @Inject() ( sys.error(s"Invalid role[${request.role}]") } - db.withTransaction { implicit conn => + db.withTransaction { implicit c => organizationLogsDao.create(createdBy, request.organization, message) - dbHelpers.delete(conn, createdBy, request.guid) + dbHelpers.delete(c, createdBy, request.guid) membershipsDao.upsert(createdBy, request.organization, request.user, r) + emailQueue.queueWithConnection(c, EmailDataMembershipRequestAccepted(request.organization.guid, request.user.guid, r.toString)) } - - mainActor ! actors.MainActor.Messages.MembershipRequestAccepted(request.organization.guid, request.user.guid, r) } /** @@ -121,12 +122,11 @@ class MembershipRequestsDao @Inject() ( } val message = s"Declined membership request for ${request.user.email} to join as ${r.name}" - db.withTransaction { implicit conn => + db.withTransaction { implicit c => organizationLogsDao.create(createdBy.guid, request.organization, message) softDelete(createdBy, request) + emailQueue.queueWithConnection(c, EmailDataMembershipRequestDeclined(request.organization.guid, request.user.guid, r.toString)) } - - mainActor ! actors.MainActor.Messages.MembershipRequestDeclined(request.organization.guid, request.user.guid, r) } private[this] def assertUserCanReview(user: User, request: MembershipRequest): Unit = { diff --git a/api/app/db/MembershipsDao.scala b/api/app/db/MembershipsDao.scala index f8f640f66..91df2fa90 100644 --- a/api/app/db/MembershipsDao.scala +++ b/api/app/db/MembershipsDao.scala @@ -1,19 +1,22 @@ package db +import anorm._ import io.apibuilder.api.v0.models.{Membership, Organization, User} import io.apibuilder.common.v0.models.{Audit, ReferenceGuid} +import io.apibuilder.task.v0.models.EmailDataMembershipCreated import io.flow.postgresql.Query import lib.Role -import anorm._ -import javax.inject.{Inject, Named, Singleton} +import org.joda.time.DateTime import play.api.db._ +import processor.EmailProcessorQueue + import java.util.UUID -import org.joda.time.DateTime +import javax.inject.{Inject, Singleton} @Singleton class MembershipsDao @Inject() ( - @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, + emailQueue: EmailProcessorQueue, subscriptionsDao: SubscriptionsDao ) { @@ -67,7 +70,7 @@ class MembershipsDao @Inject() ( } private[db] def create(createdBy: UUID, organization: Organization, user: User, role: Role): Membership = { - db.withConnection { implicit c => + db.withTransaction { implicit c => create(c, createdBy, organization, user, role) } } @@ -94,7 +97,7 @@ class MembershipsDao @Inject() ( "created_by_guid" -> createdBy ).execute() - mainActor ! actors.MainActor.Messages.MembershipCreated(membership.guid) + emailQueue.queueWithConnection(c, EmailDataMembershipCreated(membership.guid)) membership } diff --git a/api/app/db/PasswordResetRequestsDao.scala b/api/app/db/PasswordResetRequestsDao.scala index 4f37c0be6..00bad985d 100644 --- a/api/app/db/PasswordResetRequestsDao.scala +++ b/api/app/db/PasswordResetRequestsDao.scala @@ -1,15 +1,17 @@ package db -import io.apibuilder.api.v0.models.User -import lib.TokenGenerator -import anorm._ import anorm.JodaParameterMetaData._ -import java.util.UUID -import javax.inject.{Inject, Named, Singleton} - -import play.api.db._ +import anorm._ +import io.apibuilder.api.v0.models.User +import io.apibuilder.task.v0.models.EmailDataPasswordResetRequestCreated import io.flow.postgresql.Query +import lib.TokenGenerator import org.joda.time.DateTime +import play.api.db._ +import processor.EmailProcessorQueue + +import java.util.UUID +import javax.inject.{Inject, Singleton} case class PasswordReset( guid: UUID, @@ -20,8 +22,8 @@ case class PasswordReset( @Singleton class PasswordResetRequestsDao @Inject() ( - @Named("main-actor") mainActor: akka.actor.ActorRef, @NamedDatabase("default") db: Database, + emailQueue: EmailProcessorQueue, userPasswordsDao: UserPasswordsDao, usersDao: UsersDao ) { @@ -48,7 +50,7 @@ class PasswordResetRequestsDao @Inject() ( def create(createdBy: Option[User], user: User): PasswordReset = { val guid = UUID.randomUUID - db.withConnection { implicit c => + db.withTransaction { implicit c => SQL(InsertQuery).on( "guid" -> guid, "user_guid" -> user.guid, @@ -56,10 +58,9 @@ class PasswordResetRequestsDao @Inject() ( "expires_at" -> DateTime.now.plusHours(HoursUntilTokenExpires), "created_by_guid" -> createdBy.getOrElse(user).guid ).execute() + emailQueue.queueWithConnection(c, EmailDataPasswordResetRequestCreated(guid)) } - mainActor ! actors.MainActor.Messages.PasswordResetRequestCreated(guid) - findByGuid(guid).getOrElse { sys.error("Failed to create password reset") } diff --git a/api/app/processor/EmailProcessor.scala b/api/app/processor/EmailProcessor.scala index 0601eb8d9..59ed14fe0 100644 --- a/api/app/processor/EmailProcessor.scala +++ b/api/app/processor/EmailProcessor.scala @@ -3,15 +3,25 @@ package processor import actors.Emails import cats.data.ValidatedNec import cats.implicits._ -import db.{Authorization, OrganizationsDao, UsersDao} +import db.{Authorization, InternalTasksDao, OrganizationsDao, UsersDao} import io.apibuilder.api.v0.models.Publication import io.apibuilder.task.v0.models._ import io.apibuilder.task.v0.models.json._ import lib.{AppConfig, EmailUtil, Person, Role} +import play.api.libs.json.Json +import java.sql.Connection import java.util.UUID import javax.inject.Inject +class EmailProcessorQueue @Inject() ( + internalTasksDao: InternalTasksDao + ) { + def queueWithConnection(c: Connection, data: EmailData): Unit = { + val dataJson = Json.toJson(data) + internalTasksDao.queue(TaskType.Email, id = Json.asciiStringify(dataJson), data = dataJson) + } +} class EmailProcessor @Inject()( args: TaskProcessorArgs, diff --git a/api/app/processor/TaskActorCompanion.scala b/api/app/processor/TaskActorCompanion.scala index 6438a4ae3..702597948 100644 --- a/api/app/processor/TaskActorCompanion.scala +++ b/api/app/processor/TaskActorCompanion.scala @@ -12,7 +12,8 @@ class TaskActorCompanion @Inject() ( migrateVersion: MigrateVersionProcessor, userCreated: UserCreatedProcessor, scheduleSyncGeneratorServices: ScheduleSyncGeneratorServicesProcessor, - syncGeneratorService: SyncGeneratorServiceProcessor + syncGeneratorService: SyncGeneratorServiceProcessor, + email: EmailProcessor ) { def process(typ: TaskType): Unit = { @@ -30,6 +31,7 @@ class TaskActorCompanion @Inject() ( case UserCreated => userCreated case ScheduleSyncGeneratorServices => scheduleSyncGeneratorServices case SyncGeneratorService => syncGeneratorService + case Email => email case UNDEFINED(_) => sys.error(s"Undefined task type '$typ") } } From 21bd608f49ce0dae5b7d13e27b186d1154d5d030 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Thu, 6 Jun 2024 00:02:28 -0400 Subject: [PATCH 29/32] wip --- api/app/actors/Emails.scala | 39 ++++++--------------- api/app/invariants/CheckInvariants.scala | 43 ++++++++++++++++++++++++ api/app/invariants/Invariants.scala | 12 +++++++ api/app/invariants/PurgeInvariants.scala | 18 ++++++++++ api/app/invariants/TaskInvariants.scala | 20 +++++++++++ api/app/views/emails/errors.scala.html | 7 ---- api/test/db/VersionValidatorSpec.scala | 2 +- 7 files changed, 105 insertions(+), 36 deletions(-) create mode 100644 api/app/invariants/CheckInvariants.scala create mode 100644 api/app/invariants/Invariants.scala create mode 100644 api/app/invariants/PurgeInvariants.scala create mode 100644 api/app/invariants/TaskInvariants.scala delete mode 100644 api/app/views/emails/errors.scala.html diff --git a/api/app/actors/Emails.scala b/api/app/actors/Emails.scala index 0a7adb03c..153209729 100644 --- a/api/app/actors/Emails.scala +++ b/api/app/actors/Emails.scala @@ -1,10 +1,11 @@ package actors -import io.apibuilder.api.v0.models._ import db.{ApplicationsDao, Authorization, MembershipsDao, SubscriptionsDao} -import javax.inject.{Inject, Singleton} +import io.apibuilder.api.v0.models._ import lib._ -import play.api.{Logger, Logging} +import play.api.Logging + +import javax.inject.{Inject, Singleton} object Emails { @@ -29,7 +30,6 @@ object Emails { @Singleton class Emails @Inject() ( - appConfig: AppConfig, email: EmailUtil, applicationsDao: ApplicationsDao, membershipsDao: MembershipsDao, @@ -46,11 +46,13 @@ class Emails @Inject() ( implicit filter: Subscription => Boolean = { _ => true } ): Unit = { eachSubscription(context, org, publication, { subscription => - email.sendHtml( - to = Person(subscription.user), - subject = subject, - body = body - ) + if (filter(subscription)) { + email.sendHtml( + to = Person(subscription.user), + subject = subject, + body = body + ) + } }) } @@ -108,23 +110,4 @@ class Emails @Inject() ( } } - def sendErrors( - subject: String, - errors: Seq[String] - ): Unit = { - errors match { - case Nil => {} - case _ => { - val body = views.html.emails.errors(errors).toString - appConfig.sendErrorsTo.foreach { emailAddress => - email.sendHtml( - to = Person(emailAddress), - subject = subject, - body = body - ) - } - } - } - } - } diff --git a/api/app/invariants/CheckInvariants.scala b/api/app/invariants/CheckInvariants.scala new file mode 100644 index 000000000..549244c8a --- /dev/null +++ b/api/app/invariants/CheckInvariants.scala @@ -0,0 +1,43 @@ +package invariants + +import anorm.SqlParser +import play.api.db.Database + +import javax.inject.Inject + +/** Sync all invoices with a balance + */ +class CheckInvariants @Inject() ( + invariants: Invariants, + database: Database, +) { + + def process(): Unit = { + val results = invariants.all.map { i => + val count = database.withConnection { c => + i.query.as(SqlParser.long(1).*)(c).headOption.getOrElse(0L) + } + InvariantResult(i, count) + } + sendResults(results) + } + + private[this] case class InvariantResult(invariant: Invariant, count: Long) + private[this] def sendResults(results: Seq[InvariantResult]): Unit = { + val (_, withErrors) = results.partition(_.count == 0) + + if (withErrors.nonEmpty) { + val subject = if (withErrors.length == 1) { + "1 Error" + } else { + s"${withErrors.length} Errors" + } + println(subject) + withErrors.foreach { e => + println(s"${e.invariant.name}: ${e.count}") + println(s"${e.invariant.query.interpolate()}") + println("") + } + } + } +} diff --git a/api/app/invariants/Invariants.scala b/api/app/invariants/Invariants.scala new file mode 100644 index 000000000..b85f31b63 --- /dev/null +++ b/api/app/invariants/Invariants.scala @@ -0,0 +1,12 @@ +package invariants + +import io.flow.postgresql.Query + +import javax.inject.Inject + +case class Invariant(name: String, query: Query) + +class Invariants @Inject() { + val all: Seq[Invariant] = + TaskInvariants.all ++ PurgeInvariants.all +} diff --git a/api/app/invariants/PurgeInvariants.scala b/api/app/invariants/PurgeInvariants.scala new file mode 100644 index 000000000..baefa4407 --- /dev/null +++ b/api/app/invariants/PurgeInvariants.scala @@ -0,0 +1,18 @@ +package invariants + +import io.flow.postgresql.Query + +object PurgeInvariants { + private[this] val Tables = Seq( + "organizations", "applications", "versions" + ) + + val all: Seq[Invariant] = Tables.map { t => + Invariant( + s"old_deleted_records_purged_from_$t", + Query( + s"select count(*) from $t where deleted_at < now() - interval '3 months'" + ) + ) + } +} diff --git a/api/app/invariants/TaskInvariants.scala b/api/app/invariants/TaskInvariants.scala new file mode 100644 index 000000000..26918d534 --- /dev/null +++ b/api/app/invariants/TaskInvariants.scala @@ -0,0 +1,20 @@ +package invariants + +import io.flow.postgresql.Query + +object TaskInvariants { + val all: Seq[Invariant] = Seq( + Invariant( + "tasks_not_attempted", + Query(""" + |select count(*) from tasks where num_attempts=0 and created_at < now() - interval '1 hour' + |""".stripMargin) + ), + Invariant( + "tasks_not_completed_in_12_hours", + Query(""" + |select count(*) from tasks where num_attempts>0 and created_at < now() - interval '12 hour' + |""".stripMargin) + ) + ) +} diff --git a/api/app/views/emails/errors.scala.html b/api/app/views/emails/errors.scala.html deleted file mode 100644 index 625d887a4..000000000 --- a/api/app/views/emails/errors.scala.html +++ /dev/null @@ -1,7 +0,0 @@ -@(errors: Seq[String]) - -
    - @errors.map { error => -
  • @error
    • - } -
diff --git a/api/test/db/VersionValidatorSpec.scala b/api/test/db/VersionValidatorSpec.scala index 3dd57c55d..19d5187da 100644 --- a/api/test/db/VersionValidatorSpec.scala +++ b/api/test/db/VersionValidatorSpec.scala @@ -7,7 +7,7 @@ import org.scalatestplus.play.guice.GuiceOneAppPerSuite class VersionValidatorSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { - def versionValidator = injector.instanceOf[VersionValidator] + private[this] def versionValidator = injector.instanceOf[VersionValidator] "validates user is a member of the organization to create an application" in { val org = createOrganization() From 6263686a5bb26290cea61aa41a8d685fe1a9f800 Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Thu, 6 Jun 2024 00:03:17 -0400 Subject: [PATCH 30/32] wip --- api/app/invariants/CheckInvariants.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/app/invariants/CheckInvariants.scala b/api/app/invariants/CheckInvariants.scala index 549244c8a..2c9bd5a64 100644 --- a/api/app/invariants/CheckInvariants.scala +++ b/api/app/invariants/CheckInvariants.scala @@ -15,7 +15,9 @@ class CheckInvariants @Inject() ( def process(): Unit = { val results = invariants.all.map { i => val count = database.withConnection { c => - i.query.as(SqlParser.long(1).*)(c).headOption.getOrElse(0L) + i.query + .withDebugging() + .as(SqlParser.long(1).*)(c).headOption.getOrElse(0L) } InvariantResult(i, count) } @@ -24,8 +26,9 @@ class CheckInvariants @Inject() ( private[this] case class InvariantResult(invariant: Invariant, count: Long) private[this] def sendResults(results: Seq[InvariantResult]): Unit = { - val (_, withErrors) = results.partition(_.count == 0) + val (noErrors, withErrors) = results.partition(_.count == 0) + println(s"# Invariants checked with no errors: ${noErrors.length}") if (withErrors.nonEmpty) { val subject = if (withErrors.length == 1) { "1 Error" From 9a00f21c89cc606e599af1e954e8f3cb2afe808d Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Thu, 6 Jun 2024 00:04:13 -0400 Subject: [PATCH 31/32] wip --- api/app/invariants/CheckInvariants.scala | 4 +--- api/app/invariants/PurgeInvariants.scala | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/api/app/invariants/CheckInvariants.scala b/api/app/invariants/CheckInvariants.scala index 2c9bd5a64..ee0f539cc 100644 --- a/api/app/invariants/CheckInvariants.scala +++ b/api/app/invariants/CheckInvariants.scala @@ -15,9 +15,7 @@ class CheckInvariants @Inject() ( def process(): Unit = { val results = invariants.all.map { i => val count = database.withConnection { c => - i.query - .withDebugging() - .as(SqlParser.long(1).*)(c).headOption.getOrElse(0L) + i.query.as(SqlParser.long(1).*)(c).headOption.getOrElse(0L) } InvariantResult(i, count) } diff --git a/api/app/invariants/PurgeInvariants.scala b/api/app/invariants/PurgeInvariants.scala index baefa4407..a297d6efe 100644 --- a/api/app/invariants/PurgeInvariants.scala +++ b/api/app/invariants/PurgeInvariants.scala @@ -11,7 +11,7 @@ object PurgeInvariants { Invariant( s"old_deleted_records_purged_from_$t", Query( - s"select count(*) from $t where deleted_at < now() - interval '3 months'" + s"select count(*) from $t where deleted_at < now() - interval '1 year'" ) ) } From 01596c87f841ea1a53a7dcc32aa6f6d05c50f78c Mon Sep 17 00:00:00 2001 From: Michael Bryzek Date: Thu, 6 Jun 2024 00:04:17 -0400 Subject: [PATCH 32/32] wip --- api/test/invariants/CheckInvariantsSpec.scala | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 api/test/invariants/CheckInvariantsSpec.scala diff --git a/api/test/invariants/CheckInvariantsSpec.scala b/api/test/invariants/CheckInvariantsSpec.scala new file mode 100644 index 000000000..b0e51179f --- /dev/null +++ b/api/test/invariants/CheckInvariantsSpec.scala @@ -0,0 +1,13 @@ +package invariants + +import org.scalatestplus.play.PlaySpec +import org.scalatestplus.play.guice.GuiceOneAppPerSuite + +class CheckInvariantsSpec extends PlaySpec with GuiceOneAppPerSuite with db.Helpers { + + private def checkInvariants: CheckInvariants = injector.instanceOf[CheckInvariants] + + "process" in { + checkInvariants.process() + } +}