From 3c3d9436d21d14b089751891122972e1ff7494ab Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 20 Nov 2022 17:21:41 -0800 Subject: [PATCH 01/39] feat: allow calling multiople workflows --- .../aws-runtime/src/injected/actions.ts | 4 +- .../aws-runtime/src/injected/workflow.ts | 4 +- .../@eventual/compiler/src/esbuild-plugin.ts | 10 +- packages/@eventual/core/src/chain.ts | 5 + packages/@eventual/core/src/eventual.ts | 38 +++---- packages/@eventual/core/src/index.ts | 2 +- packages/@eventual/core/src/interpret.ts | 42 +++++++- packages/@eventual/core/src/workflow.ts | 101 +++++++++++------- .../@eventual/core/test/interpret.test.ts | 16 +-- 9 files changed, 142 insertions(+), 80 deletions(-) diff --git a/packages/@eventual/aws-runtime/src/injected/actions.ts b/packages/@eventual/aws-runtime/src/injected/actions.ts index 5cd272c46..4c6b626ae 100644 --- a/packages/@eventual/aws-runtime/src/injected/actions.ts +++ b/packages/@eventual/aws-runtime/src/injected/actions.ts @@ -1,3 +1,3 @@ -import { eventual } from "@eventual/core"; +import { workflow } from "@eventual/core"; -export default eventual(async () => {}); +export default workflow(async () => {}); diff --git a/packages/@eventual/aws-runtime/src/injected/workflow.ts b/packages/@eventual/aws-runtime/src/injected/workflow.ts index 5cd272c46..4c6b626ae 100644 --- a/packages/@eventual/aws-runtime/src/injected/workflow.ts +++ b/packages/@eventual/aws-runtime/src/injected/workflow.ts @@ -1,3 +1,3 @@ -import { eventual } from "@eventual/core"; +import { workflow } from "@eventual/core"; -export default eventual(async () => {}); +export default workflow(async () => {}); diff --git a/packages/@eventual/compiler/src/esbuild-plugin.ts b/packages/@eventual/compiler/src/esbuild-plugin.ts index a2adcdf3b..001b3b1b4 100644 --- a/packages/@eventual/compiler/src/esbuild-plugin.ts +++ b/packages/@eventual/compiler/src/esbuild-plugin.ts @@ -57,13 +57,13 @@ class OuterVisitor extends Visitor { public visitCallExpression(call: CallExpression): Expression { if ( isEventualCallee(call.callee) && - call.arguments.length === 1 && - (call.arguments[0]?.expression.type === "ArrowFunctionExpression" || - call.arguments[0]?.expression.type === "FunctionExpression") && - !call.arguments[0].expression.generator + call.arguments.length === 2 && + (call.arguments[1]?.expression.type === "ArrowFunctionExpression" || + call.arguments[1]?.expression.type === "FunctionExpression") && + !call.arguments[1].expression.generator ) { this.foundEventual = true; - return new InnerVisitor().visitExpression(call.arguments[0].expression); + return new InnerVisitor().visitExpression(call.arguments[1].expression); } return super.visitCallExpression(call); } diff --git a/packages/@eventual/core/src/chain.ts b/packages/@eventual/core/src/chain.ts index 2feea5abd..6daa4e8f6 100644 --- a/packages/@eventual/core/src/chain.ts +++ b/packages/@eventual/core/src/chain.ts @@ -18,6 +18,11 @@ export interface Chain extends Program { awaiting?: Eventual; } +export function chain Program>(definition: F): F { + return ((...args: Parameters) => + registerChain(definition(...args))) as any; +} + export function createChain(program: Program): Chain { (program as any)[EventualSymbol] = EventualKind.Chain; return program as Chain; diff --git a/packages/@eventual/core/src/eventual.ts b/packages/@eventual/core/src/eventual.ts index 19e6c4523..e23a44104 100644 --- a/packages/@eventual/core/src/eventual.ts +++ b/packages/@eventual/core/src/eventual.ts @@ -1,27 +1,15 @@ -import { Program } from "./interpret.js"; -import { registerChain, Chain } from "./chain.js"; -import { ActivityCall } from "./activity-call.js"; -import { AwaitAll } from "./await-all.js"; - -export function eventual Promise>( - func: F -): (...args: Parameters) => Program>>; - -export function eventual Program>( - func: F -): (...args: Parameters) => Chain>>; - -export function eventual any>(func: F): F { - return ((...args: any[]) => { - const generator = func(...args); - return registerChain(generator); - }) as any; -} - -type Resolved = T extends Program - ? Resolved +import type { ActivityCall } from "./activity-call.js"; +import type { AwaitAll } from "./await-all.js"; +import type { Chain } from "./chain.js"; +import type { Program } from "./interpret.js"; +import type { WorkflowCall } from "./workflow.js"; + +export type AwaitedEventual = T extends Promise + ? Awaited + : T extends Program + ? AwaitedEventual : T extends Eventual - ? Resolved + ? AwaitedEventual : T; export const EventualSymbol = Symbol.for("eventual:Eventual"); @@ -30,6 +18,7 @@ export enum EventualKind { AwaitAll = 0, ActivityCall = 1, Chain = 2, + WorkflowCall = 3, } export function isEventual(a: any): a is Eventual { @@ -39,7 +28,8 @@ export function isEventual(a: any): a is Eventual { export type Eventual = | ActivityCall | AwaitAll - | Chain; + | Chain + | WorkflowCall; export namespace Eventual { /** diff --git a/packages/@eventual/core/src/index.ts b/packages/@eventual/core/src/index.ts index 45d567717..8019f0271 100644 --- a/packages/@eventual/core/src/index.ts +++ b/packages/@eventual/core/src/index.ts @@ -2,8 +2,8 @@ export * from "./activity.js"; export * from "./await-all.js"; export * from "./chain.js"; export * from "./command.js"; -export * from "./events.js"; export * from "./error.js"; +export * from "./events.js"; export * from "./eventual.js"; export * from "./execution.js"; export * from "./interpret.js"; diff --git a/packages/@eventual/core/src/interpret.ts b/packages/@eventual/core/src/interpret.ts index 6dcc47a97..224fe627e 100644 --- a/packages/@eventual/core/src/interpret.ts +++ b/packages/@eventual/core/src/interpret.ts @@ -6,10 +6,15 @@ import { ActivityCompleted, ActivityFailed, ActivityScheduled, + filterEvents, HistoryEvent, + HistoryStateEvents, isActivityCompleted, isActivityFailed, isActivityScheduled, + isHistoryEvent, + isWorkflowStarted, + WorkflowEventType, } from "./events.js"; import { collectActivities } from "./global.js"; import { @@ -23,6 +28,7 @@ import { import { createChain, isChain, Chain } from "./chain.js"; import { assertNever } from "./util.js"; import { Command } from "./command.js"; +import { isWorkflowCall } from "./workflow.js"; export interface WorkflowResult { /** @@ -39,6 +45,40 @@ export interface WorkflowResult { export type Program = Generator; +export interface AdvanceWorkflowResult extends WorkflowResult { + history: HistoryStateEvents[]; +} + +/** + * Advance a workflow using previous history, new events, and a program. + */ +export function advance( + program: (input: any) => Program, + historyEvents: HistoryStateEvents[], + taskEvents: HistoryStateEvents[] +): AdvanceWorkflowResult { + // historical events and incoming events will be fed into the workflow to resume/progress state + const inputEvents = filterEvents( + historyEvents, + taskEvents + ); + + const startEvent = inputEvents.find(isWorkflowStarted); + + if (!startEvent) { + throw new DeterminismError( + `No ${WorkflowEventType.WorkflowStarted} found.` + ); + } + + // execute workflow + const interpretEvents = inputEvents.filter(isHistoryEvent); + return { + ...interpret(program(startEvent.input), interpretEvents), + history: inputEvents, + }; +} + /** * Interprets a workflow program */ @@ -263,7 +303,7 @@ export function interpret( } function tryResolveResult(activity: Eventual): Result | undefined { - if (isActivityCall(activity)) { + if (isActivityCall(activity) || isWorkflowCall(activity)) { return activity.result; } else if (isChain(activity)) { if (activity.result) { diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index 48cfb7a3b..dcdd3ea93 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -1,43 +1,70 @@ -import { DeterminismError } from "./error.js"; import { - filterEvents, - HistoryStateEvents, - isHistoryEvent, - isWorkflowStarted, - WorkflowEventType, -} from "./events.js"; -import { interpret, Program, WorkflowResult } from "./interpret.js"; - -interface ProgressWorkflowResult extends WorkflowResult { - history: HistoryStateEvents[]; + AwaitedEventual, + Eventual, + EventualKind, + EventualSymbol, +} from "./eventual.js"; +import { registerActivity } from "./global.js"; +import type { Program } from "./interpret.js"; +import type { Result } from "./result.js"; + +export interface ExecutionHandle { + /** + * ID of the workflow execution. + */ + executionId: string; } /** - * Progress a workflow using previous history, new events, and a program. + * A {@link Workflow} is a long-running process that orchestrates calls + * to other services in a durable and observable way. */ -export function progressWorkflow( - program: (input: any) => Program, - historyEvents: HistoryStateEvents[], - taskEvents: HistoryStateEvents[] -): ProgressWorkflowResult { - // historical events and incoming events will be fed into the workflow to resume/progress state - const inputEvents = filterEvents( - historyEvents, - taskEvents - ); - - const startEvent = inputEvents.find(isWorkflowStarted); - - if (!startEvent) { - throw new DeterminismError( - `No ${WorkflowEventType.WorkflowStarted} found.` - ); - } - - // execute workflow - const interpretEvents = inputEvents.filter(isHistoryEvent); - return { - ...interpret(program(startEvent.input), interpretEvents), - history: inputEvents, - }; +export interface Workflow any> { + id: string; + /** + * Invokes + */ + (...args: Parameters): ReturnType; + /** + * Starts an execution of this {@link Workflow} without waiting for the response. + * + * @returns a {@link ExecutionHandle} with the `executionId`. + */ + startExecution(...args: Parameters): Promise; + + /** + * @internal - this is the internal DSL representation that produces a {@link Program} instead of a Promise. + */ + definition: ( + ...args: Parameters + ) => Program>>; +} + +export function workflow Promise | Program>( + id: string, + definition: F +): Workflow { + const workflow: Workflow = ((...args: any[]) => + registerActivity({ + [EventualSymbol]: EventualKind.WorkflowCall, + id, + args, + })) as any; + + // TODO: + // workflow.start = function start(...args) {}; + + workflow.definition = definition as Workflow["definition"]; // safe to cast because we rely on transformer (it is always the generator API) + return workflow; +} + +export function isWorkflowCall(a: Eventual): a is WorkflowCall { + return a[EventualSymbol] === EventualKind.WorkflowCall; +} + +export interface WorkflowCall { + [EventualSymbol]: EventualKind.WorkflowCall; + id: string; + args: any[]; + result?: Result; } diff --git a/packages/@eventual/core/test/interpret.test.ts b/packages/@eventual/core/test/interpret.test.ts index 6c6e031fc..1675b8ef8 100644 --- a/packages/@eventual/core/test/interpret.test.ts +++ b/packages/@eventual/core/test/interpret.test.ts @@ -2,7 +2,6 @@ import "jest"; import { interpret, - eventual, Eventual, Result, WorkflowEventType, @@ -14,6 +13,7 @@ import { } from "../src/index.js"; import { createActivityCall } from "../src/activity-call.js"; import { DeterminismError } from "../src/error.js"; +import { chain } from "../src/chain.js"; function* myWorkflow(event: any): Program { try { @@ -112,7 +112,7 @@ test("should wait if partial results", () => { test("should return result of inner function", () => { function* workflow() { - const inner = eventual(function* () { + const inner = chain(function* () { return "foo"; }); return yield* inner(); @@ -126,7 +126,7 @@ test("should return result of inner function", () => { test("should await an un-awaited returned Activity", () => { function* workflow() { - const inner = eventual(function* () { + const inner = chain(function* () { return "foo"; }); return inner(); @@ -141,7 +141,7 @@ test("should await an un-awaited returned Activity", () => { test("should await an un-awaited returned AwaitAll", () => { function* workflow() { let i = 0; - const inner = eventual(function* () { + const inner = chain(function* () { return `foo-${i++}`; }); return Eventual.all([inner(), inner()]); @@ -157,7 +157,7 @@ test("should support Eventual.all of function calls", () => { function* workflow(items: string[]) { return Eventual.all( items.map( - eventual(function* (item): Program { + chain(function* (item): Program { return yield createActivityCall("process-item", [item]); }) ) @@ -188,7 +188,7 @@ test("should have left-to-right determinism semantics for Eventual.all", () => { return Eventual.all([ createActivityCall("before", ["before"]), ...items.map( - eventual(function* (item) { + chain(function* (item) { yield createActivityCall("inside", [item]); }) ), @@ -254,7 +254,7 @@ test("throw error within nested function", () => { try { yield* Eventual.all( items.map( - eventual(function* (item) { + chain(function* (item) { const result = yield createActivityCall("inside", [item]); if (result === "bad") { @@ -392,7 +392,7 @@ test("generator function returns an ActivityCall", () => { return yield* sub(); } - const sub = eventual(function* () { + const sub = chain(function* () { return createActivityCall("call-a", []); }); From 74631a6a9983bb5b3f54a3d32b2e8e20b5dec422 Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 20 Nov 2022 18:13:55 -0800 Subject: [PATCH 02/39] feat: update transpiler --- apps/test-app-runtime/src/my-workflow.ts | 8 +- .../@eventual/compiler/src/esbuild-plugin.ts | 119 +++++++++++++++--- .../compiler/test-files/open-account.ts | 3 +- .../compiler/test-files/workflow.mts | 2 +- .../@eventual/compiler/test-files/workflow.ts | 2 +- .../__snapshots__/esbuild-plugin.test.ts.snap | 34 ++--- packages/@eventual/core/src/eventual.ts | 8 +- packages/@eventual/core/src/workflow.ts | 28 ++++- 8 files changed, 155 insertions(+), 49 deletions(-) diff --git a/apps/test-app-runtime/src/my-workflow.ts b/apps/test-app-runtime/src/my-workflow.ts index cf6947976..107f99617 100644 --- a/apps/test-app-runtime/src/my-workflow.ts +++ b/apps/test-app-runtime/src/my-workflow.ts @@ -1,11 +1,11 @@ import { activity, workflow } from "@eventual/core"; -const hello = activity("hello", async (name: string) => { - return `hello ${name}`; -}); - export default workflow("my-workflow", async ({ name }: { name: string }) => { const result = await hello(name); console.log(result); return `you said ${result}`; }); + +const hello = activity("hello", async (name: string) => { + return `hello ${name}`; +}); diff --git a/packages/@eventual/compiler/src/esbuild-plugin.ts b/packages/@eventual/compiler/src/esbuild-plugin.ts index 001b3b1b4..7c6078aa4 100644 --- a/packages/@eventual/compiler/src/esbuild-plugin.ts +++ b/packages/@eventual/compiler/src/esbuild-plugin.ts @@ -1,5 +1,7 @@ import esBuild from "esbuild"; import { + Node, + Argument, ArrowFunctionExpression, AwaitExpression, CallExpression, @@ -10,6 +12,7 @@ import { parseFile, print, Span, + StringLiteral, TsType, } from "@swc/core"; import path from "path"; @@ -49,24 +52,31 @@ const supportedPromiseFunctions: string[] = [ ]; class OuterVisitor extends Visitor { + readonly inner = new InnerVisitor(); + public foundEventual = false; - public visitTsType(n: TsType): TsType { - return n; - } public visitCallExpression(call: CallExpression): Expression { - if ( - isEventualCallee(call.callee) && - call.arguments.length === 2 && - (call.arguments[1]?.expression.type === "ArrowFunctionExpression" || - call.arguments[1]?.expression.type === "FunctionExpression") && - !call.arguments[1].expression.generator - ) { + if (isWorkflowCall(call)) { this.foundEventual = true; - return new InnerVisitor().visitExpression(call.arguments[1].expression); + + return { + ...call, + arguments: [ + call.arguments[0], + { + spread: call.arguments[1].spread, + expression: this.inner.visitWorkflow(call.arguments[1].expression), + }, + ], + }; } return super.visitCallExpression(call); } + + public visitTsType(n: TsType): TsType { + return n; + } } export class InnerVisitor extends Visitor { @@ -74,6 +84,49 @@ export class InnerVisitor extends Visitor { return n; } + public visitWorkflow( + workflow: FunctionExpression | ArrowFunctionExpression + ): FunctionExpression { + return { + type: "FunctionExpression", + generator: true, + span: workflow.span, + async: false, + identifier: + workflow.type === "FunctionExpression" + ? workflow.identifier + : undefined, + decorators: + workflow.type === "FunctionExpression" + ? workflow.decorators + : undefined, + body: workflow.body + ? workflow.body.type === "BlockStatement" + ? this.visitBlockStatement(workflow.body) + : { + type: "BlockStatement", + span: getSpan(workflow.body), + stmts: [ + { + type: "ReturnStatement", + span: getSpan(workflow.body), + argument: this.visitExpression(workflow.body), + }, + ], + } + : undefined, + params: workflow.params.map((p) => + p.type === "Parameter" + ? this.visitParameter(p) + : { + pat: this.visitPattern(p), + span: getSpan(p), + type: "Parameter", + } + ), + }; + } + public visitAwaitExpression(awaitExpr: AwaitExpression): Expression { return { type: "YieldExpression", @@ -93,7 +146,7 @@ export class InnerVisitor extends Visitor { if ( supportedPromiseFunctions.includes(call.callee.property.value as any) ) { - call.callee.object.value = "Eventual"; + call.callee.object.value = "$Eventual"; } } return super.visitCallExpression(call); @@ -102,7 +155,7 @@ export class InnerVisitor extends Visitor { public visitFunctionExpression( funcExpr: FunctionExpression ): FunctionExpression { - return this.wrapEventual({ + return this.createChain({ ...funcExpr, async: false, generator: true, @@ -114,10 +167,10 @@ export class InnerVisitor extends Visitor { public visitArrowFunctionExpression( funcExpr: ArrowFunctionExpression ): Expression { - return this.wrapEventual(funcExpr); + return this.createChain(funcExpr); } - private wrapEventual( + private createChain( funcExpr: FunctionExpression | ArrowFunctionExpression ): CallExpression { const call: CallExpression = { @@ -125,7 +178,7 @@ export class InnerVisitor extends Visitor { span: funcExpr.span, callee: { type: "Identifier", - value: "eventual", + value: "$eventual", optional: false, span: funcExpr.span, }, @@ -173,8 +226,9 @@ export class InnerVisitor extends Visitor { } } -function getSpan(expr: Expression): Span { +function getSpan(expr: Node): Span { if ("span" in expr) { + // @ts-ignore return expr.span; } else { // this is only true for JSXExpressions which we should not encounter @@ -182,12 +236,37 @@ function getSpan(expr: Expression): Span { } } -function isEventualCallee(callee: CallExpression["callee"]) { +/** + * A heuristic for identifying a {@link CallExpression} that is a call + * to the eventual.workflow utility: + * + * 1. must be a function call with exactly 2 arguments + * 2. first argument is a string literal + * 3. second argument is a FunctionExpression or ArrowFunctionExpression + * 4. callee is an identifier `"workflow"` or `.workflow` + */ +function isWorkflowCall(call: CallExpression): call is CallExpression & { + arguments: [ + Argument & { expression: StringLiteral }, + Argument & { expression: FunctionExpression | ArrowFunctionExpression } + ]; +} { + return ( + isWorkflowCallee(call.callee) && + call.arguments.length === 2 && + call.arguments[0]?.expression.type === "StringLiteral" && + (call.arguments[1]?.expression.type === "ArrowFunctionExpression" || + call.arguments[1]?.expression.type === "FunctionExpression") && + !call.arguments[1].expression.generator + ); +} + +function isWorkflowCallee(callee: CallExpression["callee"]) { return ( - (callee.type === "Identifier" && callee.value === "eventual") || + (callee.type === "Identifier" && callee.value === "workflow") || (callee.type === "MemberExpression" && callee.property.type === "Identifier" && - callee.property.value === "eventual") + callee.property.value === "workflow") ); } diff --git a/packages/@eventual/compiler/test-files/open-account.ts b/packages/@eventual/compiler/test-files/open-account.ts index 3ca5f90e0..7a57c0ad2 100644 --- a/packages/@eventual/compiler/test-files/open-account.ts +++ b/packages/@eventual/compiler/test-files/open-account.ts @@ -1,6 +1,7 @@ // @ts-nocheck -export default eventual( +export default workflow( + "open-account", async ({ accountId, address, email, bankDetails }: OpenAccountRequest) => { const rollbacks: RollbackHandler[] = []; diff --git a/packages/@eventual/compiler/test-files/workflow.mts b/packages/@eventual/compiler/test-files/workflow.mts index 3a59d2580..f20e27871 100644 --- a/packages/@eventual/compiler/test-files/workflow.mts +++ b/packages/@eventual/compiler/test-files/workflow.mts @@ -2,7 +2,7 @@ const doWork = activity("doWork", async (input: any) => input); -export default eventual(async (input) => { +export default workflow("workflow", async (input) => { const items = await doWork(input); await Promise.all( diff --git a/packages/@eventual/compiler/test-files/workflow.ts b/packages/@eventual/compiler/test-files/workflow.ts index 3a59d2580..f20e27871 100644 --- a/packages/@eventual/compiler/test-files/workflow.ts +++ b/packages/@eventual/compiler/test-files/workflow.ts @@ -2,7 +2,7 @@ const doWork = activity("doWork", async (input: any) => input); -export default eventual(async (input) => { +export default workflow("workflow", async (input) => { const items = await doWork(input); await Promise.all( diff --git a/packages/@eventual/compiler/test/__snapshots__/esbuild-plugin.test.ts.snap b/packages/@eventual/compiler/test/__snapshots__/esbuild-plugin.test.ts.snap index 8761617ee..26f5f8638 100644 --- a/packages/@eventual/compiler/test/__snapshots__/esbuild-plugin.test.ts.snap +++ b/packages/@eventual/compiler/test/__snapshots__/esbuild-plugin.test.ts.snap @@ -13,21 +13,21 @@ exports[`esbuild-plugin mts workflow 1`] = ` (() => { // test-files/workflow.mts var doWork = activity("doWork", async (input) => input); - var workflow_default = eventual(function* (input) { + var workflow_default = workflow("workflow", function* (input) { const items = yield doWork(input); - yield Eventual.all(items.map(eventual(function* (item) { + yield $Eventual.all(items.map($eventual(function* (item) { yield doWork(item); }))); - yield Eventual.all(items.map(eventual(function* (item) { + yield $Eventual.all(items.map($eventual(function* (item) { yield doWork(item); }))); - yield Eventual.allSettled(items.map(eventual(function* (item) { + yield $Eventual.allSettled(items.map($eventual(function* (item) { yield doWork(item); }))); - yield Eventual.any(items.map(eventual(function* (item) { + yield $Eventual.any(items.map($eventual(function* (item) { yield doWork(item); }))); - yield Eventual.race(items.map(eventual(function* (item) { + yield $Eventual.race(items.map($eventual(function* (item) { yield doWork(item); }))); }); @@ -39,7 +39,7 @@ exports[`esbuild-plugin open-account 1`] = ` ""use strict"; (() => { // test-files/open-account.ts - var open_account_default = eventual(function* ({ accountId, address, email, bankDetails }) { + var open_account_default = workflow("open-account", function* ({ accountId, address, email, bankDetails }) { const rollbacks = []; try { yield createAccount(accountId); @@ -49,19 +49,19 @@ exports[`esbuild-plugin open-account 1`] = ` } try { yield addAddress(accountId, address); - rollbacks.push(eventual(function* () { + rollbacks.push($eventual(function* () { return removeAddress(accountId); })); yield addEmail(accountId, email); - rollbacks.push(eventual(function* () { + rollbacks.push($eventual(function* () { return removeEmail(accountId); })); yield addBankAccount(accountId, bankDetails); - rollbacks.push(eventual(function* () { + rollbacks.push($eventual(function* () { return removeBankAccount(accountId); })); } catch (err) { - yield Eventual.all(rollbacks.map(eventual(function* (rollback) { + yield $Eventual.all(rollbacks.map($eventual(function* (rollback) { return rollback(); }))); } @@ -87,21 +87,21 @@ exports[`esbuild-plugin ts workflow 1`] = ` ""use strict"; (() => { var doWork = activity("doWork", async (input) => input); - var workflow_default = eventual(function* (input) { + var workflow_default = workflow("workflow", function* (input) { const items = yield doWork(input); - yield Eventual.all(items.map(eventual(function* (item) { + yield $Eventual.all(items.map($eventual(function* (item) { yield doWork(item); }))); - yield Eventual.all(items.map(eventual(function* (item) { + yield $Eventual.all(items.map($eventual(function* (item) { yield doWork(item); }))); - yield Eventual.allSettled(items.map(eventual(function* (item) { + yield $Eventual.allSettled(items.map($eventual(function* (item) { yield doWork(item); }))); - yield Eventual.any(items.map(eventual(function* (item) { + yield $Eventual.any(items.map($eventual(function* (item) { yield doWork(item); }))); - yield Eventual.race(items.map(eventual(function* (item) { + yield $Eventual.race(items.map($eventual(function* (item) { yield doWork(item); }))); }); diff --git a/packages/@eventual/core/src/eventual.ts b/packages/@eventual/core/src/eventual.ts index e23a44104..2cb5f1c8c 100644 --- a/packages/@eventual/core/src/eventual.ts +++ b/packages/@eventual/core/src/eventual.ts @@ -1,6 +1,6 @@ import type { ActivityCall } from "./activity-call.js"; import type { AwaitAll } from "./await-all.js"; -import type { Chain } from "./chain.js"; +import { chain, Chain } from "./chain.js"; import type { Program } from "./interpret.js"; import type { WorkflowCall } from "./workflow.js"; @@ -55,5 +55,9 @@ export namespace Eventual { } } +// the below globals are required by the transformer + +// @ts-ignore +global.$eventual = chain; // @ts-ignore -global.Eventual = Eventual; +global.$Eventual = Eventual; diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index bea1860d3..a60b8de95 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -7,7 +7,6 @@ import { import { registerActivity } from "./global.js"; import type { Program } from "./interpret.js"; import type { Result } from "./result.js"; - import { Context, WorkflowContext } from "./context.js"; import { DeterminismError } from "./error.js"; import { @@ -53,6 +52,26 @@ export interface Workflow< ) => Program>>; } +/** + * Creates and registers a long-running workflow. + * + * Example: + * ```ts + * import { activity, workflow } from "@eventual/core"; + * + * export default workflow("my-workflow", async ({ name }: { name: string }) => { + * const result = await hello(name); + * console.log(result); + * return `you said ${result}`; + * }); + * + * const hello = activity("hello", async (name: string) => { + * return `hello ${name}`; + * }); + * ``` + * @param id a globally unique ID for this workflow. + * @param definition the workflow definition. + */ export function workflow Promise | Program>( id: string, definition: F @@ -75,6 +94,9 @@ export function isWorkflowCall(a: Eventual): a is WorkflowCall { return a[EventualSymbol] === EventualKind.WorkflowCall; } +/** + * An {@link Eventual} representing an awaited call to a {@link Workflow}. + */ export interface WorkflowCall { [EventualSymbol]: EventualKind.WorkflowCall; id: string; @@ -82,7 +104,7 @@ export interface WorkflowCall { result?: Result; } -export interface AdvanceWorkflowResult extends WorkflowResult { +export interface ProgressWorkflowResult extends WorkflowResult { history: HistoryStateEvents[]; } @@ -95,7 +117,7 @@ export function progressWorkflow( taskEvents: HistoryStateEvents[], workflowContext: WorkflowContext, executionId: string -): AdvanceWorkflowResult { +): ProgressWorkflowResult { // historical events and incoming events will be fed into the workflow to resume/progress state const inputEvents = filterEvents( historyEvents, From 7034f8d655369c65a4572c3d2100686076ec3e78 Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 20 Nov 2022 23:28:04 -0800 Subject: [PATCH 03/39] feat: implement plumbing --- apps/test-app/src/app.ts | 6 +- packages/@eventual/aws-cdk/src/api.ts | 8 +- packages/@eventual/aws-cdk/src/index.ts | 2 +- .../aws-cdk/src/{workflow.ts => service.ts} | 8 +- .../@eventual/aws-runtime/src/activity.ts | 5 +- .../aws-runtime/src/clients/create.ts | 2 + .../src/clients/workflow-client.ts | 38 +++- .../src/clients/workflow-runtime-client.ts | 120 ++++++++++++- packages/@eventual/aws-runtime/src/env.ts | 2 - .../src/handlers/activity-worker.ts | 3 +- .../aws-runtime/src/handlers/orchestrator.ts | 50 ++++-- packages/@eventual/core/src/activity-call.ts | 18 +- packages/@eventual/core/src/command.ts | 25 ++- packages/@eventual/core/src/context.ts | 4 + packages/@eventual/core/src/events.ts | 164 +++++++++++++----- packages/@eventual/core/src/interpret.ts | 36 ++-- packages/@eventual/core/src/tasks.ts | 1 - packages/@eventual/core/src/util.ts | 2 +- packages/@eventual/core/src/workflow.ts | 33 ++-- .../@eventual/core/test/interpret.test.ts | 65 ++++--- packages/@eventual/core/tsconfig.json | 3 +- 21 files changed, 436 insertions(+), 159 deletions(-) rename packages/@eventual/aws-cdk/src/{workflow.ts => service.ts} (96%) diff --git a/apps/test-app/src/app.ts b/apps/test-app/src/app.ts index c19aaf088..fcba75e87 100644 --- a/apps/test-app/src/app.ts +++ b/apps/test-app/src/app.ts @@ -1,5 +1,5 @@ import { App, aws_dynamodb, Stack } from "aws-cdk-lib"; -import { EventualApi, Workflow } from "@eventual/aws-cdk"; +import { EventualApi, Service } from "@eventual/aws-cdk"; const app = new App(); @@ -13,7 +13,7 @@ const accountTable = new aws_dynamodb.Table(stack, "Accounts", { billingMode: aws_dynamodb.BillingMode.PAY_PER_REQUEST, }); -const openAccount = new Workflow(stack, "OpenAccount", { +const openAccount = new Service(stack, "OpenAccount", { entry: require.resolve("test-app-runtime/lib/open-account.js"), name: "open-account", environment: { @@ -23,7 +23,7 @@ const openAccount = new Workflow(stack, "OpenAccount", { accountTable.grantReadWriteData(openAccount); -const myWorkflow = new Workflow(stack, "workflow1", { +const myWorkflow = new Service(stack, "workflow1", { name: "my-workflow", entry: require.resolve("test-app-runtime/lib/my-workflow.js"), }); diff --git a/packages/@eventual/aws-cdk/src/api.ts b/packages/@eventual/aws-cdk/src/api.ts index d896d2d65..ba600f8a8 100644 --- a/packages/@eventual/aws-cdk/src/api.ts +++ b/packages/@eventual/aws-cdk/src/api.ts @@ -12,10 +12,10 @@ import { } from "aws-cdk-lib"; import { Construct } from "constructs"; import path from "path"; -import { Workflow } from "./workflow"; +import { Service } from "./service"; export interface EventualApiProps { - workflows: Workflow[]; + workflows: Service[]; } interface RouteMapping { @@ -35,9 +35,9 @@ export class EventualApi extends Construct { WORKFLOWS: JSON.stringify( Object.fromEntries( props.workflows.map((w) => [ - w.workflowName, + w.serviceName, { - name: w.workflowName, + name: w.serviceName, tableName: w.table.tableName, workflowQueueUrl: w.workflowQueue.queueUrl, executionHistoryBucket: w.history.bucketName, diff --git a/packages/@eventual/aws-cdk/src/index.ts b/packages/@eventual/aws-cdk/src/index.ts index f0bcf9765..63592207e 100644 --- a/packages/@eventual/aws-cdk/src/index.ts +++ b/packages/@eventual/aws-cdk/src/index.ts @@ -1,2 +1,2 @@ -export * from "./workflow"; +export * from "./service"; export * from "./api"; diff --git a/packages/@eventual/aws-cdk/src/workflow.ts b/packages/@eventual/aws-cdk/src/service.ts similarity index 96% rename from packages/@eventual/aws-cdk/src/workflow.ts rename to packages/@eventual/aws-cdk/src/service.ts index 8cacc8f65..61403213f 100644 --- a/packages/@eventual/aws-cdk/src/workflow.ts +++ b/packages/@eventual/aws-cdk/src/service.ts @@ -35,8 +35,8 @@ export interface WorkflowProps { }; } -export class Workflow extends Construct implements IGrantable { - public readonly workflowName: string; +export class Service extends Construct implements IGrantable { + public readonly serviceName: string; /** * S3 bucket that contains events necessary to replay a workflow execution. * @@ -77,7 +77,7 @@ export class Workflow extends Construct implements IGrantable { constructor(scope: Construct, id: string, props: WorkflowProps) { super(scope, id); - this.workflowName = props.name ?? Names.uniqueResourceName(this, {}); + this.serviceName = props.name ?? Names.uniqueResourceName(this, {}); // ExecutionHistoryBucket this.history = new Bucket(this, "History", { @@ -159,7 +159,6 @@ export class Workflow extends Construct implements IGrantable { [ENV_NAMES.WORKFLOW_QUEUE_URL]: this.workflowQueue.queueUrl, [ENV_NAMES.ACTIVITY_LOCK_TABLE_NAME]: this.locksTable.tableName, [ENV_NAMES.EVENTUAL_WORKER]: "1", - [ENV_NAMES.WORKFLOW_NAME]: this.workflowName, ...(props.environment ?? {}), }, // retry attempts should be handled with a new request and a new retry count in accordance with the user's retry policy. @@ -182,7 +181,6 @@ export class Workflow extends Construct implements IGrantable { [ENV_NAMES.EXECUTION_HISTORY_BUCKET]: this.history.bucketName, [ENV_NAMES.TABLE_NAME]: this.table.tableName, [ENV_NAMES.WORKFLOW_QUEUE_URL]: this.workflowQueue.queueUrl, - [ENV_NAMES.WORKFLOW_NAME]: this.workflowName, }, events: [ new SqsEventSource(this.workflowQueue, { diff --git a/packages/@eventual/aws-runtime/src/activity.ts b/packages/@eventual/aws-runtime/src/activity.ts index 1d4ea29d1..3cff99804 100644 --- a/packages/@eventual/aws-runtime/src/activity.ts +++ b/packages/@eventual/aws-runtime/src/activity.ts @@ -1,8 +1,9 @@ -import { Command } from "@eventual/core"; +import { ScheduleActivityCommand } from "@eventual/core"; export interface ActivityWorkerRequest { scheduledTime: string; + workflowName: string; executionId: string; - command: Command; + command: ScheduleActivityCommand; retry: number; } diff --git a/packages/@eventual/aws-runtime/src/clients/create.ts b/packages/@eventual/aws-runtime/src/clients/create.ts index 86ea7fea9..58028f7ae 100644 --- a/packages/@eventual/aws-runtime/src/clients/create.ts +++ b/packages/@eventual/aws-runtime/src/clients/create.ts @@ -79,5 +79,7 @@ export const createWorkflowRuntimeClient = /*@__PURE__*/ memoize( lambda: lambda(), activityWorkerFunctionName: activityWorkerFunctionName ?? env.activityWorkerFunctionName(), + sqs: sqs(), + workflowQueueUrl: env.workflowQueueUrl(), }) ); diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts index ca6dd4dd4..c7c67743d 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts @@ -15,6 +15,26 @@ import { import { ulid } from "ulidx"; import { ExecutionHistoryClient } from "./execution-history-client.js"; +export interface StartWorkflowRequest { + /** + * Name of the workflow execution. + * + * Only one workflow can exist for an ID. Requests to start a workflow + * with the name of an existing workflow will fail. + * + * @default - a unique name is generated. + */ + name?: string; + /** + * Input payload for the workflow function. + */ + input?: any; + /** + * ID of the parent execution if this is a child workflow + */ + parentId?: string; +} + export interface WorkflowClientProps { readonly dynamo: DynamoDBClient; readonly tableName: string; @@ -35,13 +55,18 @@ export class WorkflowClient { public async startWorkflow({ name: _name, input, - }: { name?: string; input?: any } = {}) { + parentId, + }: StartWorkflowRequest = {}) { const name = _name ?? ulid(); + if (name.includes("/")) { + throw new Error(`name cannot contains reserved character '/'`); + } const executionId = `execution_${name}`; console.log("execution input:", input); await this.props.dynamo.send( new PutItemCommand({ + TableName: this.props.tableName, Item: { pk: { S: ExecutionRecord.PRIMARY_KEY }, sk: { S: ExecutionRecord.sortKey(executionId) }, @@ -49,8 +74,8 @@ export class WorkflowClient { name: { S: name }, status: { S: ExecutionStatus.IN_PROGRESS }, startTime: { S: new Date().toISOString() }, + ...(parentId ? { parentId: { S: parentId } } : {}), }, - TableName: this.props.tableName, }) ); @@ -60,7 +85,10 @@ export class WorkflowClient { { type: WorkflowEventType.WorkflowStarted, input, - context: { name }, + context: { + name, + parentId, + }, } ); @@ -73,11 +101,9 @@ export class WorkflowClient { executionId: string, ...events: HistoryStateEvents[] ) { - const id = ulid(); // send workflow task to workflow queue const workflowTask: SQSWorkflowTaskMessage = { task: { - id, executionId, events, }, @@ -89,7 +115,7 @@ export class WorkflowClient { QueueUrl: this.props.workflowQueueUrl, MessageGroupId: executionId, // just de-dupe with itself - MessageDeduplicationId: `${executionId}_${id}`, + MessageDeduplicationId: `${executionId}_${ulid()}`, }) ); } diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index fbf782daa..ec2a36fa2 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -17,25 +17,35 @@ import { } from "@aws-sdk/client-lambda"; import { ExecutionStatus, - Command, HistoryStateEvents, CompleteExecution, FailedExecution, Execution, + ScheduleActivityCommand, + WorkflowEventType, } from "@eventual/core"; import { createExecutionFromResult, ExecutionRecord, + SQSWorkflowTaskMessage, } from "./workflow-client.js"; import { ActivityWorkerRequest } from "../activity.js"; +import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; export interface WorkflowRuntimeClientProps { readonly lambda: LambdaClient; readonly activityWorkerFunctionName: string; readonly dynamo: DynamoDBClient; readonly s3: S3Client; + readonly sqs: SQSClient; readonly executionHistoryBucket: string; readonly tableName: string; + readonly workflowQueueUrl: string; +} + +export interface CompleteExecutionRequest { + executionId: string; + result?: any; } export class WorkflowRuntimeClient { @@ -77,10 +87,10 @@ export class WorkflowRuntimeClient { return { bytes: content.length }; } - async completeExecution( - executionId: string, - result?: any - ): Promise { + async completeExecution({ + executionId, + result, + }: CompleteExecutionRequest): Promise { const executionResult = await this.props.dynamo.send( new UpdateItemCommand({ Key: { @@ -106,6 +116,10 @@ export class WorkflowRuntimeClient { }) ); + if (isChildExecutionId(executionId)) { + await this.completeChildExecution(executionId, result); + } + return createExecutionFromResult( executionResult.Attributes as unknown as ExecutionRecord ) as CompleteExecution; @@ -142,11 +156,62 @@ export class WorkflowRuntimeClient { }) ); + if (isChildExecutionId(executionId)) { + await this.completeChildExecution(executionId, error, message); + } + return createExecutionFromResult( executionResult.Attributes as unknown as ExecutionRecord ) as FailedExecution; } + private async completeChildExecution( + executionId: ChildExecutionId, + result: any + ): Promise; + + private async completeChildExecution( + executionId: ChildExecutionId, + error: string, + message: string + ): Promise; + + private async completeChildExecution( + executionId: ChildExecutionId, + ...args: [result: any] | [error: string, message: string] + ) { + const { parentExecutionId, seq } = parseChildExecutionId(executionId); + const workflowTask: SQSWorkflowTaskMessage = { + task: { + executionId: parentExecutionId, + events: [ + { + seq, + timestamp: new Date().toISOString(), + ...(args.length === 1 + ? { + type: WorkflowEventType.ChildWorkflowCompleted, + result: args[0], + } + : { + type: WorkflowEventType.ChildWorkflowFailed, + error: args[0], + message: args[1], + }), + }, + ], + }, + }; + await this.props.sqs.send( + new SendMessageCommand({ + QueueUrl: this.props.workflowQueueUrl, + MessageBody: JSON.stringify(workflowTask), + MessageGroupId: parentExecutionId, + MessageDeduplicationId: `${executionId}/complete`, + }) + ); + } + async getExecutions(): Promise { const executions = await this.props.dynamo.send( new QueryCommand({ @@ -162,9 +227,14 @@ export class WorkflowRuntimeClient { ); } - async scheduleActivity(executionId: string, command: Command) { + async scheduleActivity( + workflowName: string, + executionId: string, + command: ScheduleActivityCommand + ) { const request: ActivityWorkerRequest = { scheduledTime: new Date().toISOString(), + workflowName, executionId, command, retry: 0, @@ -194,3 +264,41 @@ async function historyEntryToEvents( function formatExecutionHistoryKey(executionId: string) { return `executionHistory/${executionId}`; } + +/** + * A child workflow execution's Id is encoded as follows: + * ``` + * {parentExecutionId}/{seq} + * ``` + * Child workflow's can be multiple levels deep: + * ``` + * {rootExecutionId}/{seq-0}/../{seq-n} + * ``` + * + * TODO: define a depth limit based on key limit maximums. + * TODO: can we use a hash so that nested execution ids are bounded in length? + */ +type ChildExecutionId = `${string}/${number}`; + +function isChildExecutionId( + executionId: string +): executionId is ChildExecutionId { + return executionId.split("/").length >= 2; +} + +function parseChildExecutionId(executionId: ChildExecutionId): { + parentExecutionId: string; + seq: number; +} { + const lastSlash = executionId.lastIndexOf("/"); + const parentExecutionId = executionId.slice(0, lastSlash); + const seqStr = executionId.slice(lastSlash + 1 /* +1 to skip past the '/' */); + const seq = parseInt(seqStr!, 10); + if (isNaN(seq)) { + throw new Error(`invalid sequence number ${seqStr}`); + } + return { + parentExecutionId, + seq, + }; +} diff --git a/packages/@eventual/aws-runtime/src/env.ts b/packages/@eventual/aws-runtime/src/env.ts index a965977a8..64ecb266c 100644 --- a/packages/@eventual/aws-runtime/src/env.ts +++ b/packages/@eventual/aws-runtime/src/env.ts @@ -8,7 +8,6 @@ export namespace ENV_NAMES { export const ACTIVITY_WORKER_FUNCTION_NAME = "EVENTUAL_ACTIVITY_WORKER_FUNCTION_NAME"; export const ACTIVITY_LOCK_TABLE_NAME = "EVENTUAL_ACTIVITY_LOCK_TABLE_NAME"; - export const WORKFLOW_NAME = "EVENTUAL_WORKFLOW_NAME"; /** * A flag that determines if a function is an activity worker. * @@ -34,4 +33,3 @@ export const activityWorkerFunctionName = () => tryGetEnv(ENV_NAMES.ACTIVITY_WORKER_FUNCTION_NAME); export const activityLockTableName = () => tryGetEnv(ENV_NAMES.ACTIVITY_LOCK_TABLE_NAME); -export const workflowName = () => tryGetEnv(ENV_NAMES.WORKFLOW_NAME); diff --git a/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts b/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts index e56953d8f..acd2ed32b 100644 --- a/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts @@ -16,7 +16,6 @@ import { } from "../clients/index.js"; import { metricScope, Unit } from "aws-embedded-metrics"; import { timed } from "../metrics/utils.js"; -import { workflowName } from "../env.js"; import { ActivityMetrics, MetricsCommon } from "../metrics/constants.js"; const activityRuntimeClient = createActivityRuntimeClient(); @@ -30,7 +29,7 @@ export const activityWorker = (): Handler => { metrics.setNamespace(MetricsCommon.EventualNamespace); metrics.putDimensions({ ActivityName: request.command.name, - WorkflowName: workflowName(), + WorkflowName: request.workflowName, }); // the time from the workflow emitting the activity scheduled command // to the request being seen. diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index 73fa7343d..3ef539024 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -18,21 +18,26 @@ import { isCompleteExecution, progressWorkflow, Workflow, + isScheduleActivityCommand, + isStartWorkflowCommand, + assertNever, + ChildWorkflowScheduled, } from "@eventual/core"; import { SQSWorkflowTaskMessage } from "../clients/workflow-client.js"; import { createExecutionHistoryClient, + createWorkflowClient, createWorkflowRuntimeClient, } from "../clients/index.js"; import { SQSHandler, SQSRecord } from "aws-lambda"; import { createMetricsLogger, Unit } from "aws-embedded-metrics"; import { timed, timedSync } from "../metrics/utils.js"; -import { workflowName } from "../env.js"; import { MetricsCommon, OrchestratorMetrics } from "../metrics/constants.js"; import { WorkflowContext } from "@eventual/core"; const executionHistoryClient = createExecutionHistoryClient(); const workflowRuntimeClient = createWorkflowRuntimeClient(); +const workflowClient = createWorkflowClient(); /** * Creates an entrypoint function for orchestrating a workflow. @@ -97,7 +102,7 @@ async function orchestrateExecution( metrics.resetDimensions(false); metrics.setNamespace(MetricsCommon.EventualNamespace); metrics.setDimensions({ - [MetricsCommon.WorkflowNameDimension]: workflowName(), + [MetricsCommon.WorkflowNameDimension]: workflow.name, }); const events = sqsRecordsToEvents(records); const start = new Date(); @@ -147,7 +152,7 @@ async function orchestrateExecution( ); const workflowContext: WorkflowContext = { - name: workflowName(), + name: workflow.name, }; const { @@ -257,7 +262,10 @@ async function orchestrateExecution( metrics, OrchestratorMetrics.ExecutionStatusUpdateDuration, () => - workflowRuntimeClient.completeExecution(executionId, result.value) + workflowRuntimeClient.completeExecution({ + executionId, + result: result.value, + }) ); logExecutionCompleteMetrics(execution); } @@ -312,13 +320,33 @@ async function orchestrateExecution( // register command events return await Promise.all( commands.map(async (command) => { - await workflowRuntimeClient.scheduleActivity(executionId, command); - - return createEvent({ - type: WorkflowEventType.ActivityScheduled, - seq: command.seq, - name: command.name, - }); + if (isScheduleActivityCommand(command)) { + await workflowRuntimeClient.scheduleActivity( + workflow.name, + executionId, + command + ); + + return createEvent({ + type: WorkflowEventType.ActivityScheduled, + seq: command.seq, + name: command.name, + }); + } else if (isStartWorkflowCommand(command)) { + await workflowClient.startWorkflow({ + name: `${command.name}_${executionId}_${command.seq}`, + input: command.input, + }); + + return createEvent({ + type: WorkflowEventType.ChildWorkflowScheduled, + seq: command.seq, + name: command.name, + input: command.input, + }); + } else { + return assertNever(command, `unknown command type`); + } }) ); } diff --git a/packages/@eventual/core/src/activity-call.ts b/packages/@eventual/core/src/activity-call.ts index d287c0af5..bc8b5acb9 100644 --- a/packages/@eventual/core/src/activity-call.ts +++ b/packages/@eventual/core/src/activity-call.ts @@ -14,22 +14,10 @@ export interface ActivityCall { result?: Resolved | Failed; } -export function createActivityCall( - name: string, - args: any[], - seq?: number -): ActivityCall { - const command: ActivityCall = { +export function createActivityCall(name: string, args: any[]): ActivityCall { + return registerActivity({ [EventualSymbol]: EventualKind.ActivityCall, - seq, name, args, - }; - if (seq !== undefined) { - // if seq is passed in, then this Command is assumed to be in a dev environment - // so - do not register it - return command; - } else { - return registerActivity(command); - } + }); } diff --git a/packages/@eventual/core/src/command.ts b/packages/@eventual/core/src/command.ts index 6645abd4f..0946d55dd 100644 --- a/packages/@eventual/core/src/command.ts +++ b/packages/@eventual/core/src/command.ts @@ -1,11 +1,34 @@ +import { EventualKind } from "./eventual.js"; + /** * A command is an action taken to start or emit something. * * Current: Schedule Activity * Future: Emit Signal, Start Workflow, etc */ -export interface Command { +export type Command = ScheduleActivityCommand | StartWorkflowCommand; + +interface BaseCommand { seq: number; name: string; +} + +export function isScheduleActivityCommand( + a: Command +): a is ScheduleActivityCommand { + return a.kind === EventualKind.ActivityCall; +} + +export interface ScheduleActivityCommand extends BaseCommand { + kind: EventualKind.ActivityCall; args: any[]; } + +export function isStartWorkflowCommand(a: Command): a is StartWorkflowCommand { + return a.kind === EventualKind.WorkflowCall; +} + +export interface StartWorkflowCommand extends BaseCommand { + kind: EventualKind.WorkflowCall; + input?: any; +} diff --git a/packages/@eventual/core/src/context.ts b/packages/@eventual/core/src/context.ts index 90639b051..1737df2e0 100644 --- a/packages/@eventual/core/src/context.ts +++ b/packages/@eventual/core/src/context.ts @@ -10,6 +10,10 @@ export interface ExecutionContext { * Unique name of the execution, optionally provided in the startWorkflow call. */ name: string; + /** + * ID of the parent execution if this is a child workflow + */ + parentId?: string; /** * The ISO 8601 UTC time the execution started. */ diff --git a/packages/@eventual/core/src/events.ts b/packages/@eventual/core/src/events.ts index 8f45b2803..04cdc2038 100644 --- a/packages/@eventual/core/src/events.ts +++ b/packages/@eventual/core/src/events.ts @@ -1,4 +1,5 @@ import { ExecutionContext } from "./context.js"; +import { or } from "./util.js"; export interface BaseEvent { type: WorkflowEventType; @@ -19,6 +20,9 @@ export enum WorkflowEventType { WorkflowCompleted = "WorkflowCompleted", WorkflowFailed = "WorkflowFailed", WorkflowStarted = "WorkflowStarted", + ChildWorkflowScheduled = "ChildWorkflowScheduled", + ChildWorkflowCompleted = "ChildWorkflowCompleted", + ChildWorkflowFailed = "ChildWorkflowFailed", } /** @@ -27,12 +31,18 @@ export enum WorkflowEventType { export type HistoryEvent = | ActivityScheduled | ActivityCompleted - | ActivityFailed; + | ActivityFailed + | ChildWorkflowScheduled + | ChildWorkflowCompleted + | ChildWorkflowFailed; /** * Events that we save into history. */ -export type HistoryStateEvents = HistoryEvent | WorkflowStarted; +export type HistoryStateEvents = + | HistoryEvent + | WorkflowStarted + | ChildWorkflowScheduled; /** * Events generated by the engine that represent the in-order state of the workflow. @@ -41,17 +51,87 @@ export type WorkflowEvent = | ActivityCompleted | ActivityFailed | ActivityScheduled + | ChildWorkflowScheduled + | ChildWorkflowCompleted + | ChildWorkflowFailed | WorkflowTaskCompleted | WorkflowTaskStarted | WorkflowCompleted | WorkflowFailed | WorkflowStarted; +export type ScheduledEvent = ActivityScheduled | ChildWorkflowScheduled; +export type CompletedEvent = ActivityCompleted | ChildWorkflowCompleted; +export type FailedEvent = ActivityFailed | ChildWorkflowFailed; + export interface WorkflowStarted extends BaseEvent { type: WorkflowEventType.WorkflowStarted; + /** + * Input payload for the workflow function. + */ input?: any; context: Omit; } +export interface WorkflowTaskStarted extends BaseEvent { + type: WorkflowEventType.WorkflowTaskStarted; + /** + * An execution ID of the parent workflow execution that + * started this workflow if this is a child workflow. + */ + parent?: string; +} + +export interface ActivityScheduled extends HistoryEventBase { + type: WorkflowEventType.ActivityScheduled; + name: string; +} + +export interface ActivityCompleted extends HistoryEventBase { + type: WorkflowEventType.ActivityCompleted; + // the time from being scheduled until the activity completes. + duration: number; + result: any; +} + +export interface ActivityFailed extends HistoryEventBase { + type: WorkflowEventType.ActivityFailed; + error: string; + // the time from being scheduled until the activity completes. + duration: number; + message: string; +} + +export interface WorkflowTaskCompleted extends BaseEvent { + type: WorkflowEventType.WorkflowTaskCompleted; +} + +export interface WorkflowCompleted extends BaseEvent { + type: WorkflowEventType.WorkflowCompleted; + output: any; +} + +export interface WorkflowFailed extends BaseEvent { + type: WorkflowEventType.WorkflowFailed; + error: string; + message: string; +} + +export interface ChildWorkflowScheduled extends HistoryEventBase { + type: WorkflowEventType.ChildWorkflowScheduled; + name: string; + input: any; +} + +export interface ChildWorkflowCompleted extends HistoryEventBase { + type: WorkflowEventType.ChildWorkflowCompleted; + result: any; +} + +export interface ChildWorkflowFailed extends HistoryEventBase { + type: WorkflowEventType.ChildWorkflowFailed; + error: string; + message: string; +} export function isWorkflowStarted( event: WorkflowEvent @@ -59,53 +139,38 @@ export function isWorkflowStarted( return event.type === WorkflowEventType.WorkflowStarted; } -export interface WorkflowTaskStarted extends BaseEvent { - type: WorkflowEventType.WorkflowTaskStarted; -} - export function isTaskStarted( event: WorkflowEvent ): event is WorkflowTaskStarted { return event.type === WorkflowEventType.WorkflowTaskStarted; } -export interface ActivityScheduled extends HistoryEventBase { - type: WorkflowEventType.ActivityScheduled; - name: string; -} - export function isActivityScheduled( event: WorkflowEvent ): event is ActivityScheduled { return event.type === WorkflowEventType.ActivityScheduled; } -export interface ActivityCompleted extends HistoryEventBase { - type: WorkflowEventType.ActivityCompleted; - // the time from being scheduled until the activity completes. - duration: number; - result: any; -} - export function isActivityCompleted( event: WorkflowEvent ): event is ActivityCompleted { return event.type === WorkflowEventType.ActivityCompleted; } -export interface ActivityFailed extends HistoryEventBase { - type: WorkflowEventType.ActivityFailed; - error: string; - // the time from being scheduled until the activity completes. - duration: number; - message: string; +export function isTaskCompleted( + event: WorkflowEvent +): event is WorkflowTaskCompleted { + return event.type === WorkflowEventType.WorkflowTaskCompleted; } export function isHistoryEvent(event: WorkflowEvent): event is HistoryEvent { return ( isActivityCompleted(event) || isActivityFailed(event) || - isActivityScheduled(event) + isActivityScheduled(event) || + isChildWorkflowCompleted(event) || + isChildWorkflowFailed(event) || + isChildWorkflowScheduled(event) ); } @@ -115,39 +180,46 @@ export function isActivityFailed( return event.type === WorkflowEventType.ActivityFailed; } -export interface WorkflowTaskCompleted extends BaseEvent { - type: WorkflowEventType.WorkflowTaskCompleted; -} - -export function isTaskCompleted( - event: WorkflowEvent -): event is WorkflowTaskCompleted { - return event.type === WorkflowEventType.WorkflowTaskCompleted; -} - -export interface WorkflowCompleted extends BaseEvent { - type: WorkflowEventType.WorkflowCompleted; - output: any; -} - export function isWorkflowCompleted( event: WorkflowEvent ): event is WorkflowCompleted { return event.type === WorkflowEventType.WorkflowCompleted; } -export interface WorkflowFailed extends BaseEvent { - type: WorkflowEventType.WorkflowFailed; - error: string; - message: string; -} - export function isWorkflowFailed( event: WorkflowEvent ): event is WorkflowFailed { return event.type === WorkflowEventType.WorkflowFailed; } +export function isChildWorkflowScheduled( + event: WorkflowEvent +): event is ChildWorkflowScheduled { + return event.type === WorkflowEventType.ChildWorkflowScheduled; +} +export function isChildWorkflowCompleted( + event: WorkflowEvent +): event is ChildWorkflowCompleted { + return event.type === WorkflowEventType.ChildWorkflowCompleted; +} +export function isChildWorkflowFailed( + event: WorkflowEvent +): event is ChildWorkflowFailed { + return event.type === WorkflowEventType.ChildWorkflowFailed; +} + +export const isScheduledEvent = or( + isActivityScheduled, + isChildWorkflowScheduled +); + +export const isCompletedEvent = or( + isActivityCompleted, + isChildWorkflowCompleted +); + +export const isFailedEvent = or(isActivityFailed, isChildWorkflowFailed); + export function assertEventType( event: any, type: T["type"] diff --git a/packages/@eventual/core/src/interpret.ts b/packages/@eventual/core/src/interpret.ts index 5a6d92f36..56ad36076 100644 --- a/packages/@eventual/core/src/interpret.ts +++ b/packages/@eventual/core/src/interpret.ts @@ -1,15 +1,19 @@ -import { Eventual, isEventual } from "./eventual.js"; +import { Eventual, EventualSymbol, isEventual } from "./eventual.js"; import { isAwaitAll } from "./await-all.js"; import { ActivityCall, isActivityCall } from "./activity-call.js"; import { DeterminismError } from "./error.js"; import { ActivityCompleted, ActivityFailed, - ActivityScheduled, + ChildWorkflowCompleted, + ChildWorkflowFailed, HistoryEvent, - isActivityCompleted, - isActivityFailed, isActivityScheduled, + isChildWorkflowScheduled, + isCompletedEvent, + isFailedEvent, + isScheduledEvent, + ScheduledEvent, } from "./events.js"; import { collectActivities } from "./global.js"; import { @@ -81,17 +85,14 @@ export function interpret( let event; // run the event loop one event at a time, ensuring deterministic execution. - while ((event = peek()) && peekForward(isActivityScheduled)) { + while ((event = peek()) && peekForward(isScheduledEvent)) { pop(); - if (isActivityCompleted(event) || isActivityFailed(event)) { + if (isCompletedEvent(event) || isFailedEvent(event)) { commitCompletionEvent(event, true); - } else if (isActivityScheduled(event)) { + } else if (isScheduledEvent(event)) { const calls = advance(true) ?? []; - const events = [ - event, - ...takeWhile(calls.length - 1, isActivityScheduled), - ]; + const events = [event, ...takeWhile(calls.length - 1, isScheduledEvent)]; if (events.length !== calls.length) { throw new DeterminismError(); } @@ -114,7 +115,7 @@ export function interpret( // if the history's tail contains completed events, e.g. [...scheduled, ...completed] // then we need to apply the completions, resume chains and schedule any produced activity calls while ((event = pop())) { - if (isActivityScheduled(event)) { + if (isActivityScheduled(event) || isChildWorkflowScheduled(event)) { // it should be impossible to receive a scheduled event // -> because the tail of history can only contain completion events // -> scheduled events stored in history should correspond to activity calls @@ -137,6 +138,7 @@ export function interpret( return { result, commands: calls.map((call) => ({ + kind: call[EventualSymbol], args: call.args, name: call.name, seq: call.seq!, @@ -295,7 +297,11 @@ export function interpret( } function commitCompletionEvent( - event: ActivityCompleted | ActivityFailed, + event: + | ActivityCompleted + | ActivityFailed + | ChildWorkflowCompleted + | ChildWorkflowFailed, isReplay: boolean ) { const call = callTable[event.seq]; @@ -305,13 +311,13 @@ export function interpret( if (isReplay && call.result && !isPending(call.result)) { throw new DeterminismError(); } - call.result = isActivityCompleted(event) + call.result = isCompletedEvent(event) ? Result.resolved(event.result) : Result.failed(event.error); } } -function isCorresponding(event: ActivityScheduled, call: ActivityCall) { +function isCorresponding(event: ScheduledEvent, call: ActivityCall) { return ( event.seq === call.seq && event.name === call.name // TODO: also validate arguments diff --git a/packages/@eventual/core/src/tasks.ts b/packages/@eventual/core/src/tasks.ts index 379ac311d..51965a2e6 100644 --- a/packages/@eventual/core/src/tasks.ts +++ b/packages/@eventual/core/src/tasks.ts @@ -7,6 +7,5 @@ import { HistoryStateEvents, WorkflowEvent } from "./events.js"; */ export interface WorkflowTask { executionId: string; - id: string; events: HistoryStateEvents[]; } diff --git a/packages/@eventual/core/src/util.ts b/packages/@eventual/core/src/util.ts index c5c1b5066..a378c9168 100644 --- a/packages/@eventual/core/src/util.ts +++ b/packages/@eventual/core/src/util.ts @@ -17,6 +17,6 @@ export function not( export function or a is any)[]>( ...conditions: F -): (a: any) => a is F extends (a: any) => a is infer T ? T : never { +): (a: any) => a is F[number] extends (a: any) => a is infer T ? T : never { return ((a: any) => conditions.some((cond) => cond(a))) as any; } diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index a60b8de95..6c57d146c 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -32,9 +32,18 @@ export interface ExecutionHandle { export interface Workflow< F extends (...args: any[]) => any = (...args: any[]) => any > { - id: string; /** - * Invokes + * Globally unique ID of this {@link Workflow}. + */ + name: string; + /** + * Invokes the {@link Workflow} from within another workflow. + * + * This can only be called from within another workflow because it's not possible + * to wait for completion synchronously - it relies on the event-driven environment + * of a workflow execution. + * + * To start a workflow from another environment, use {@link start}. */ (...args: Parameters): ReturnType; /** @@ -42,8 +51,7 @@ export interface Workflow< * * @returns a {@link ExecutionHandle} with the `executionId`. */ - startExecution(...args: Parameters): Promise; - + start(...args: Parameters): Promise; /** * @internal - this is the internal DSL representation that produces a {@link Program} instead of a Promise. */ @@ -69,23 +77,24 @@ export interface Workflow< * return `hello ${name}`; * }); * ``` - * @param id a globally unique ID for this workflow. + * @param name a globally unique ID for this workflow. * @param definition the workflow definition. */ -export function workflow Promise | Program>( - id: string, +export function workflow Promise | Program>( + name: string, definition: F ): Workflow { const workflow: Workflow = ((...args: any[]) => registerActivity({ [EventualSymbol]: EventualKind.WorkflowCall, - id, + name, args, })) as any; - // TODO: - // workflow.start = function start(...args) {}; - + workflow.start = async function (..._args: Parameters) { + // TODO: get a client and submit execution + throw new Error("not implemented"); + }; workflow.definition = definition as Workflow["definition"]; // safe to cast because we rely on transformer (it is always the generator API) return workflow; } @@ -99,7 +108,7 @@ export function isWorkflowCall(a: Eventual): a is WorkflowCall { */ export interface WorkflowCall { [EventualSymbol]: EventualKind.WorkflowCall; - id: string; + name: string; args: any[]; result?: Result; } diff --git a/packages/@eventual/core/test/interpret.test.ts b/packages/@eventual/core/test/interpret.test.ts index 1675b8ef8..f446a5460 100644 --- a/packages/@eventual/core/test/interpret.test.ts +++ b/packages/@eventual/core/test/interpret.test.ts @@ -10,10 +10,12 @@ import { ActivityScheduled, ActivityFailed, Program, + ScheduleActivityCommand, + EventualKind, } from "../src/index.js"; -import { createActivityCall } from "../src/activity-call.js"; import { DeterminismError } from "../src/error.js"; import { chain } from "../src/chain.js"; +import { createActivityCall } from "../src/activity-call.js"; function* myWorkflow(event: any): Program { try { @@ -37,7 +39,7 @@ const event = "hello world"; test("no history", () => { expect(interpret(myWorkflow(event), [])).toMatchObject({ - commands: [createActivityCall("my-activity", [event], 0)], + commands: [createScheduledActivityCommand("my-activity", [event], 0)], }); }); @@ -58,9 +60,9 @@ test("should continue with result of completed Activity", () => { ]) ).toMatchObject({ commands: [ - createActivityCall("my-activity-0", [event], 1), - createActivityCall("my-activity-1", [event], 2), - createActivityCall("my-activity-2", [event], 3), + createScheduledActivityCommand("my-activity-0", [event], 1), + createScheduledActivityCommand("my-activity-1", [event], 2), + createScheduledActivityCommand("my-activity-2", [event], 3), ], }); }); @@ -72,7 +74,7 @@ test("should catch error of failed Activity", () => { failed("error", 0), ]) ).toMatchObject({ - commands: [createActivityCall("handle-error", ["error"], 1)], + commands: [createScheduledActivityCommand("handle-error", ["error"], 1)], }); }); @@ -166,8 +168,8 @@ test("should support Eventual.all of function calls", () => { expect(interpret(workflow(["a", "b"]), [])).toMatchObject({ commands: [ - createActivityCall("process-item", ["a"], 0), - createActivityCall("process-item", ["b"], 1), + createScheduledActivityCommand("process-item", ["a"], 0), + createScheduledActivityCommand("process-item", ["b"], 1), ], }); @@ -199,10 +201,10 @@ test("should have left-to-right determinism semantics for Eventual.all", () => { const result = interpret(workflow(["a", "b"]), []); expect(result).toMatchObject({ commands: [ - createActivityCall("before", ["before"], 0), - createActivityCall("inside", ["a"], 1), - createActivityCall("inside", ["b"], 2), - createActivityCall("after", ["after"], 3), + createScheduledActivityCommand("before", ["before"], 0), + createScheduledActivityCommand("inside", ["a"], 1), + createScheduledActivityCommand("inside", ["b"], 2), + createScheduledActivityCommand("after", ["after"], 3), ], }); }); @@ -218,12 +220,12 @@ test("try-catch-finally with yield in catch", () => { } } expect(interpret(workflow(), [])).toMatchObject({ - commands: [createActivityCall("catch", [], 0)], + commands: [createScheduledActivityCommand("catch", [], 0)], }); expect( interpret(workflow(), [scheduled("catch", 0), completed(undefined, 0)]) ).toMatchObject({ - commands: [createActivityCall("finally", [], 1)], + commands: [createScheduledActivityCommand("finally", [], 1)], }); }); @@ -243,8 +245,8 @@ test("try-catch-finally with dangling promise in catch", () => { ) ).toMatchObject({ commands: [ - createActivityCall("catch", [], 0), - createActivityCall("finally", [], 1), + createScheduledActivityCommand("catch", [], 0), + createScheduledActivityCommand("finally", [], 1), ], }); }); @@ -275,8 +277,8 @@ test("throw error within nested function", () => { WorkflowResult >{ commands: [ - createActivityCall("inside", ["good"], 0), - createActivityCall("inside", ["bad"], 1), + createScheduledActivityCommand("inside", ["good"], 0), + createScheduledActivityCommand("inside", ["bad"], 1), ], }); expect( @@ -287,7 +289,7 @@ test("throw error within nested function", () => { completed("bad", 1), ]) ).toMatchObject({ - commands: [createActivityCall("catch", [], 2)], + commands: [createScheduledActivityCommand("catch", [], 2)], }); expect( interpret(workflow(["good", "bad"]), [ @@ -299,7 +301,7 @@ test("throw error within nested function", () => { completed("catch", 2), ]) ).toMatchObject({ - commands: [createActivityCall("finally", [], 3)], + commands: [createScheduledActivityCommand("finally", [], 3)], }); expect( interpret(workflow(["good", "bad"]), [ @@ -335,8 +337,8 @@ test("properly evaluate yield* of sub-programs", () => { expect(interpret(workflow(), [])).toMatchObject({ commands: [ // - createActivityCall("a", [], 0), - createActivityCall("b", [], 1), + createScheduledActivityCommand("a", [], 0), + createScheduledActivityCommand("b", [], 1), ], }); @@ -368,8 +370,8 @@ test("properly evaluate yield of Eventual.all", () => { expect(interpret(workflow(), [])).toMatchObject({ commands: [ // - createActivityCall("a", [], 0), - createActivityCall("b", [], 1), + createScheduledActivityCommand("a", [], 0), + createScheduledActivityCommand("b", [], 1), ], }); @@ -397,7 +399,7 @@ test("generator function returns an ActivityCall", () => { }); expect(interpret(workflow(), [])).toMatchObject({ - commands: [createActivityCall("call-a", [], 0)], + commands: [createScheduledActivityCommand("call-a", [], 0)], }); expect( interpret(workflow(), [scheduled("call-a", 0), completed("result", 0)]) @@ -407,6 +409,19 @@ test("generator function returns an ActivityCall", () => { }); }); +function createScheduledActivityCommand( + name: string, + args: any[], + seq: number +): ScheduleActivityCommand { + return { + kind: EventualKind.ActivityCall, + seq, + name, + args, + }; +} + function completed(result: any, seq: number): ActivityCompleted { return { type: WorkflowEventType.ActivityCompleted, diff --git a/packages/@eventual/core/tsconfig.json b/packages/@eventual/core/tsconfig.json index f0802c119..f08a0a25f 100644 --- a/packages/@eventual/core/tsconfig.json +++ b/packages/@eventual/core/tsconfig.json @@ -9,7 +9,8 @@ "inlineSourceMap": true, "rootDir": "src", "typeRoots": ["./node_modules/@types"], - "allowJs": true + "allowJs": true, + "stripInternal": true }, "include": ["src", "src/package.json"], "exclude": ["lib", "node_modules"], From 4ede43a68e939344e329aa124a06e52894ece51f Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 20 Nov 2022 23:36:18 -0800 Subject: [PATCH 04/39] chore: rename start to scheduled --- .../@eventual/aws-runtime/src/handlers/orchestrator.ts | 4 ++-- packages/@eventual/core/src/command.ts | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index 3ef539024..975c3e0d0 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -19,7 +19,7 @@ import { progressWorkflow, Workflow, isScheduleActivityCommand, - isStartWorkflowCommand, + isScheduleWorkflowCommand, assertNever, ChildWorkflowScheduled, } from "@eventual/core"; @@ -332,7 +332,7 @@ async function orchestrateExecution( seq: command.seq, name: command.name, }); - } else if (isStartWorkflowCommand(command)) { + } else if (isScheduleWorkflowCommand(command)) { await workflowClient.startWorkflow({ name: `${command.name}_${executionId}_${command.seq}`, input: command.input, diff --git a/packages/@eventual/core/src/command.ts b/packages/@eventual/core/src/command.ts index 0946d55dd..e28a39d1a 100644 --- a/packages/@eventual/core/src/command.ts +++ b/packages/@eventual/core/src/command.ts @@ -6,7 +6,7 @@ import { EventualKind } from "./eventual.js"; * Current: Schedule Activity * Future: Emit Signal, Start Workflow, etc */ -export type Command = ScheduleActivityCommand | StartWorkflowCommand; +export type Command = ScheduleActivityCommand | ScheduleWorkflowCommand; interface BaseCommand { seq: number; @@ -24,11 +24,13 @@ export interface ScheduleActivityCommand extends BaseCommand { args: any[]; } -export function isStartWorkflowCommand(a: Command): a is StartWorkflowCommand { +export function isScheduleWorkflowCommand( + a: Command +): a is ScheduleWorkflowCommand { return a.kind === EventualKind.WorkflowCall; } -export interface StartWorkflowCommand extends BaseCommand { +export interface ScheduleWorkflowCommand extends BaseCommand { kind: EventualKind.WorkflowCall; input?: any; } From 72bdf69a97dfafa548f2634cafa80fe43afa8946 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 21 Nov 2022 00:06:05 -0800 Subject: [PATCH 05/39] feat: add workflowName to StartWorkflowRequest --- apps/test-app/src/app.ts | 10 ++++---- packages/@eventual/aws-cdk/src/api.ts | 10 ++++---- .../src/clients/workflow-client.ts | 23 ++++++++++++------- .../src/functions/start-workflow.ts | 1 + .../src/handlers/api/executions/new.ts | 13 +++++++---- .../aws-runtime/src/handlers/orchestrator.ts | 3 ++- packages/@eventual/core/src/events.ts | 4 ++++ packages/@eventual/core/test/workflow.test.ts | 1 + 8 files changed, 41 insertions(+), 24 deletions(-) diff --git a/apps/test-app/src/app.ts b/apps/test-app/src/app.ts index fcba75e87..5ab5f95f8 100644 --- a/apps/test-app/src/app.ts +++ b/apps/test-app/src/app.ts @@ -1,5 +1,5 @@ import { App, aws_dynamodb, Stack } from "aws-cdk-lib"; -import { EventualApi, Service } from "@eventual/aws-cdk"; +import * as eventual from "@eventual/aws-cdk"; const app = new App(); @@ -13,7 +13,7 @@ const accountTable = new aws_dynamodb.Table(stack, "Accounts", { billingMode: aws_dynamodb.BillingMode.PAY_PER_REQUEST, }); -const openAccount = new Service(stack, "OpenAccount", { +const openAccount = new eventual.Service(stack, "OpenAccount", { entry: require.resolve("test-app-runtime/lib/open-account.js"), name: "open-account", environment: { @@ -23,11 +23,11 @@ const openAccount = new Service(stack, "OpenAccount", { accountTable.grantReadWriteData(openAccount); -const myWorkflow = new Service(stack, "workflow1", { +const myWorkflow = new eventual.Service(stack, "workflow1", { name: "my-workflow", entry: require.resolve("test-app-runtime/lib/my-workflow.js"), }); -new EventualApi(stack, "api", { - workflows: [myWorkflow, openAccount], +new eventual.EventualApi(stack, "api", { + services: [myWorkflow, openAccount], }); diff --git a/packages/@eventual/aws-cdk/src/api.ts b/packages/@eventual/aws-cdk/src/api.ts index ba600f8a8..2ef3e1f60 100644 --- a/packages/@eventual/aws-cdk/src/api.ts +++ b/packages/@eventual/aws-cdk/src/api.ts @@ -15,7 +15,7 @@ import path from "path"; import { Service } from "./service"; export interface EventualApiProps { - workflows: Service[]; + services: Service[]; } interface RouteMapping { @@ -34,7 +34,7 @@ export class EventualApi extends Construct { const environment = { WORKFLOWS: JSON.stringify( Object.fromEntries( - props.workflows.map((w) => [ + props.services.map((w) => [ w.serviceName, { name: w.serviceName, @@ -99,7 +99,7 @@ export class EventualApi extends Construct { methods: [HttpMethod.POST], entry: "executions/new.js", config: (fn) => { - props.workflows.forEach((w) => { + props.services.forEach((w) => { w.table.grantReadWriteData(fn); w.workflowQueue.grantSendMessages(fn); }); @@ -109,7 +109,7 @@ export class EventualApi extends Construct { methods: [HttpMethod.GET], entry: "executions/list.js", config: (fn) => { - props.workflows.forEach((w) => { + props.services.forEach((w) => { w.table.grantReadWriteData(fn); w.workflowQueue.grantSendMessages(fn); }); @@ -121,7 +121,7 @@ export class EventualApi extends Construct { methods: [HttpMethod.GET], entry: "executions/history.js", config: (fn) => { - props.workflows.forEach((w) => { + props.services.forEach((w) => { w.table.grantReadData(fn); }); }, diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts index c7c67743d..2b635c2f2 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts @@ -24,7 +24,11 @@ export interface StartWorkflowRequest { * * @default - a unique name is generated. */ - name?: string; + executionName?: string; + /** + * Name of the workflow to execute. + */ + workflowName: string; /** * Input payload for the workflow function. */ @@ -53,15 +57,17 @@ export class WorkflowClient { * @returns */ public async startWorkflow({ - name: _name, + executionName = ulid(), + workflowName, input, parentId, - }: StartWorkflowRequest = {}) { - const name = _name ?? ulid(); - if (name.includes("/")) { + }: StartWorkflowRequest) { + if (executionName.includes("/")) { throw new Error(`name cannot contains reserved character '/'`); } - const executionId = `execution_${name}`; + // TODO: validate workflowName + + const executionId = `execution_${executionName}`; console.log("execution input:", input); await this.props.dynamo.send( @@ -71,7 +77,7 @@ export class WorkflowClient { pk: { S: ExecutionRecord.PRIMARY_KEY }, sk: { S: ExecutionRecord.sortKey(executionId) }, id: { S: executionId }, - name: { S: name }, + name: { S: executionName }, status: { S: ExecutionStatus.IN_PROGRESS }, startTime: { S: new Date().toISOString() }, ...(parentId ? { parentId: { S: parentId } } : {}), @@ -85,8 +91,9 @@ export class WorkflowClient { { type: WorkflowEventType.WorkflowStarted, input, + workflowName, context: { - name, + name: executionName, parentId, }, } diff --git a/packages/@eventual/aws-runtime/src/functions/start-workflow.ts b/packages/@eventual/aws-runtime/src/functions/start-workflow.ts index 0a061da72..a8ff3d600 100644 --- a/packages/@eventual/aws-runtime/src/functions/start-workflow.ts +++ b/packages/@eventual/aws-runtime/src/functions/start-workflow.ts @@ -5,6 +5,7 @@ const workflowClient = createWorkflowClient(); export interface StartWorkflowRequest { name: string; + workflowName: string; input: any; } diff --git a/packages/@eventual/aws-runtime/src/handlers/api/executions/new.ts b/packages/@eventual/aws-runtime/src/handlers/api/executions/new.ts index 433383fee..5f854b698 100644 --- a/packages/@eventual/aws-runtime/src/handlers/api/executions/new.ts +++ b/packages/@eventual/aws-runtime/src/handlers/api/executions/new.ts @@ -8,20 +8,23 @@ import { createWorkflowClient } from "../../../clients/create"; * @returns */ export async function handler(event: APIGatewayProxyEventV2) { - const name = event.pathParameters?.name; - if (!name) { + const workflowName = event.pathParameters?.name; + if (!workflowName) { return { statusCode: 400, body: `Missing workflow name` }; } - const workflow = workflows[name]; + const workflow = workflows[workflowName]; if (!workflow) { return { statusCode: 400, - body: `Workflow ${name} does not exist!`, + body: `Workflow ${workflowName} does not exist!`, }; } const workflowClient = createWorkflowClient(workflow); return { - executionId: await workflowClient.startWorkflow({ input: event.body }), + executionId: await workflowClient.startWorkflow({ + workflowName, + input: event.body, + }), }; } diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index 975c3e0d0..102206f98 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -334,7 +334,8 @@ async function orchestrateExecution( }); } else if (isScheduleWorkflowCommand(command)) { await workflowClient.startWorkflow({ - name: `${command.name}_${executionId}_${command.seq}`, + workflowName: command.name, + executionName: `${command.name}_${executionId}_${command.seq}`, input: command.input, }); diff --git a/packages/@eventual/core/src/events.ts b/packages/@eventual/core/src/events.ts index 04cdc2038..d0f561ef7 100644 --- a/packages/@eventual/core/src/events.ts +++ b/packages/@eventual/core/src/events.ts @@ -66,6 +66,10 @@ export type FailedEvent = ActivityFailed | ChildWorkflowFailed; export interface WorkflowStarted extends BaseEvent { type: WorkflowEventType.WorkflowStarted; + /** + * Name of the workflow to execute. + */ + workflowName: string; /** * Input payload for the workflow function. */ diff --git a/packages/@eventual/core/test/workflow.test.ts b/packages/@eventual/core/test/workflow.test.ts index a1972c699..d11ab2b8e 100644 --- a/packages/@eventual/core/test/workflow.test.ts +++ b/packages/@eventual/core/test/workflow.test.ts @@ -16,6 +16,7 @@ function* myWorkflow(event: any): Program { const started1: WorkflowStarted = { type: WorkflowEventType.WorkflowStarted, + workflowName: "workflowName", id: "1", input: `""`, timestamp: "", From a5b85e772ee777bffe74d11805f2e11541f00389 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 16:38:29 -0800 Subject: [PATCH 06/39] stash --- apps/test-app-runtime/src/open-account.ts | 24 +++++++++++-------- apps/test-app/src/app.ts | 2 +- .../src/clients/workflow-runtime-client.ts | 11 --------- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/apps/test-app-runtime/src/open-account.ts b/apps/test-app-runtime/src/open-account.ts index 0b31095a5..26c259c75 100644 --- a/apps/test-app-runtime/src/open-account.ts +++ b/apps/test-app-runtime/src/open-account.ts @@ -33,18 +33,22 @@ interface OpenAccountRequest { type RollbackHandler = () => Promise; -export default workflow( - "open-account", - async ({ accountId, address, email, bankDetails }: OpenAccountRequest) => { - const rollbacks: RollbackHandler[] = []; +export default workflow("open-account", async (request: OpenAccountRequest) => { + try { + await createAccount(request.accountId); + } catch (err) { + console.error(err); + throw err; + } - try { - await createAccount(accountId); - } catch (err) { - console.error(err); - throw err; - } + await associateAccountInformation(request); +}); +// sub-workflow for testing purposes +export const associateAccountInformation = workflow( + "associate", + async ({ accountId, address, email, bankDetails }: OpenAccountRequest) => { + const rollbacks: RollbackHandler[] = []; try { await addAddress(accountId, address); rollbacks.push(async () => removeAddress(accountId)); diff --git a/apps/test-app/src/app.ts b/apps/test-app/src/app.ts index 5ab5f95f8..a6870dfa3 100644 --- a/apps/test-app/src/app.ts +++ b/apps/test-app/src/app.ts @@ -3,7 +3,7 @@ import * as eventual from "@eventual/aws-cdk"; const app = new App(); -const stack = new Stack(app, "test-eventual"); +const stack = new Stack(app, "test-eventual-sam"); const accountTable = new aws_dynamodb.Table(stack, "Accounts", { partitionKey: { diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index ec2a36fa2..924ae43f6 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -165,17 +165,6 @@ export class WorkflowRuntimeClient { ) as FailedExecution; } - private async completeChildExecution( - executionId: ChildExecutionId, - result: any - ): Promise; - - private async completeChildExecution( - executionId: ChildExecutionId, - error: string, - message: string - ): Promise; - private async completeChildExecution( executionId: ChildExecutionId, ...args: [result: any] | [error: string, message: string] From e646ba3a49b527708afb83c3e1f4f922a3b42453 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 20:16:34 -0800 Subject: [PATCH 07/39] stash --- apps/test-app-runtime/src/open-account.ts | 1 + packages/@eventual/aws-cdk/src/service.ts | 2 +- .../src/clients/workflow-client.ts | 7 +- .../src/functions/start-workflow.ts | 11 +- .../src/handlers/api/middleware.ts | 3 +- .../aws-runtime/src/handlers/orchestrator.ts | 2 +- packages/@eventual/core/src/interpret.ts | 33 ++- packages/@eventual/core/src/workflow.ts | 14 +- .../@eventual/core/test/interpret.test.ts | 227 +++++++++++++----- 9 files changed, 214 insertions(+), 86 deletions(-) diff --git a/apps/test-app-runtime/src/open-account.ts b/apps/test-app-runtime/src/open-account.ts index 26c259c75..a8ff9cb45 100644 --- a/apps/test-app-runtime/src/open-account.ts +++ b/apps/test-app-runtime/src/open-account.ts @@ -72,6 +72,7 @@ const dynamo = memoize(() => ); const createAccount = activity("createAccount", async (accountId: string) => { + console.log("processing", accountId); await dynamo().send( new PutCommand({ TableName, diff --git a/packages/@eventual/aws-cdk/src/service.ts b/packages/@eventual/aws-cdk/src/service.ts index a66c92ce2..55fc023e1 100644 --- a/packages/@eventual/aws-cdk/src/service.ts +++ b/packages/@eventual/aws-cdk/src/service.ts @@ -119,7 +119,7 @@ export class Service extends Construct implements IGrantable { "--conditions": "module,import,require", }, metafile: true, - externalModules: ["@aws-sdk", "aws-sdk"], + externalModules: ["aws-sdk"], }, environment: { [ENV_NAMES.TABLE_NAME]: this.table.tableName, diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts index 2b635c2f2..8c73822fa 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts @@ -62,12 +62,7 @@ export class WorkflowClient { input, parentId, }: StartWorkflowRequest) { - if (executionName.includes("/")) { - throw new Error(`name cannot contains reserved character '/'`); - } - // TODO: validate workflowName - - const executionId = `execution_${executionName}`; + const executionId = executionName; console.log("execution input:", input); await this.props.dynamo.send( diff --git a/packages/@eventual/aws-runtime/src/functions/start-workflow.ts b/packages/@eventual/aws-runtime/src/functions/start-workflow.ts index a8ff3d600..f3dc60744 100644 --- a/packages/@eventual/aws-runtime/src/functions/start-workflow.ts +++ b/packages/@eventual/aws-runtime/src/functions/start-workflow.ts @@ -1,14 +1,11 @@ import { Handler } from "aws-lambda"; -import { createWorkflowClient } from "../clients/index.js"; +import { + createWorkflowClient, + StartWorkflowRequest, +} from "../clients/index.js"; const workflowClient = createWorkflowClient(); -export interface StartWorkflowRequest { - name: string; - workflowName: string; - input: any; -} - export interface StartWorkflowResponse { executionId: string; } diff --git a/packages/@eventual/aws-runtime/src/handlers/api/middleware.ts b/packages/@eventual/aws-runtime/src/handlers/api/middleware.ts index 7c1197c30..5728380d2 100644 --- a/packages/@eventual/aws-runtime/src/handlers/api/middleware.ts +++ b/packages/@eventual/aws-runtime/src/handlers/api/middleware.ts @@ -1,4 +1,5 @@ import { MiddlewareObj } from "@middy/core"; +import util from "util"; /** * A middy middleware to handle crashed api lambdas, emitting the lambda's error in the response body @@ -8,7 +9,7 @@ export const errorMiddleware: MiddlewareObj = { onError: (req) => { return { statusCode: 500, - body: JSON.stringify({ error: req.error }, undefined, 2), + body: JSON.stringify({ error: util.inspect(req.error) }, undefined, 2), }; }, }; diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index e0b4c7635..85f97ca30 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -342,7 +342,7 @@ async function orchestrateExecution( } else if (isScheduleWorkflowCommand(command)) { await workflowClient.startWorkflow({ workflowName: command.name, - executionName: `${command.name}_${executionId}_${command.seq}`, + executionName: `${executionId}/${command.seq}`, input: command.input, }); diff --git a/packages/@eventual/core/src/interpret.ts b/packages/@eventual/core/src/interpret.ts index 56ad36076..71f29beb4 100644 --- a/packages/@eventual/core/src/interpret.ts +++ b/packages/@eventual/core/src/interpret.ts @@ -27,7 +27,7 @@ import { import { createChain, isChain, Chain } from "./chain.js"; import { assertNever } from "./util.js"; import { Command } from "./command.js"; -import { isWorkflowCall } from "./workflow.js"; +import { isWorkflowCall, WorkflowCall } from "./workflow.js"; export interface WorkflowResult { /** @@ -51,7 +51,7 @@ export function interpret( program: Program, history: HistoryEvent[] ): WorkflowResult> { - const callTable: Record = {}; + const callTable: Record = {}; const mainChain = createChain(program); const activeChains = new Set([mainChain]); @@ -138,15 +138,24 @@ export function interpret( return { result, commands: calls.map((call) => ({ - kind: call[EventualSymbol], - args: call.args, + ...(isActivityCall(call) + ? { + kind: call[EventualSymbol], + args: call.args, + } + : { + kind: call[EventualSymbol], + input: call.input, + }), name: call.name, seq: call.seq!, })), }; - function advance(isReplay: boolean): ActivityCall[] | undefined { - let calls: ActivityCall[] | undefined; + function advance( + isReplay: boolean + ): (ActivityCall | WorkflowCall)[] | undefined { + let calls: (ActivityCall | WorkflowCall)[] | undefined; let madeProgress: boolean; do { madeProgress = false; @@ -180,7 +189,7 @@ export function interpret( function tryAdvanceChain( chain: Chain, isReplay: boolean - ): ActivityCall[] | undefined { + ): (ActivityCall | WorkflowCall)[] | undefined { if (chain.awaiting === undefined) { // this is the first time the chain is running, so wake it with an undefined input return advanceChain(chain, undefined); @@ -220,7 +229,7 @@ export function interpret( function advanceChain( chain: Chain, result: Resolved | Failed | undefined - ): ActivityCall[] { + ): (ActivityCall | WorkflowCall)[] { try { const iterResult = result === undefined || isResolved(result) @@ -248,12 +257,13 @@ export function interpret( chain.awaiting = iterResult.value; } } catch (err) { + console.error(chain, err); activeChains.delete(chain); chain.result = Result.failed(err); } return collectActivities().flatMap((activity) => { - if (isActivityCall(activity)) { + if (isActivityCall(activity) || isWorkflowCall(activity)) { return [activity]; } else if (isChain(activity)) { activeChains.add(activity); @@ -317,7 +327,10 @@ export function interpret( } } -function isCorresponding(event: ScheduledEvent, call: ActivityCall) { +function isCorresponding( + event: ScheduledEvent, + call: ActivityCall | WorkflowCall +) { return ( event.seq === call.seq && event.name === call.name // TODO: also validate arguments diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index 6c57d146c..3abfc1d17 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -84,11 +84,11 @@ export function workflow Promise | Program>( name: string, definition: F ): Workflow { - const workflow: Workflow = ((...args: any[]) => + const workflow: Workflow = ((input?: any) => registerActivity({ [EventualSymbol]: EventualKind.WorkflowCall, name, - args, + input, })) as any; workflow.start = async function (..._args: Parameters) { @@ -109,8 +109,9 @@ export function isWorkflowCall(a: Eventual): a is WorkflowCall { export interface WorkflowCall { [EventualSymbol]: EventualKind.WorkflowCall; name: string; - args: any[]; + input: any; result?: Result; + seq?: number; } export interface ProgressWorkflowResult extends WorkflowResult { @@ -121,7 +122,7 @@ export interface ProgressWorkflowResult extends WorkflowResult { * Advance a workflow using previous history, new events, and a program. */ export function progressWorkflow( - program: (...args: any[]) => Program, + program: Workflow, historyEvents: HistoryStateEvents[], taskEvents: HistoryStateEvents[], workflowContext: WorkflowContext, @@ -153,7 +154,10 @@ export function progressWorkflow( // execute workflow const interpretEvents = inputEvents.filter(isHistoryEvent); return { - ...interpret(program(startEvent.input, context), interpretEvents), + ...interpret( + program.definition(startEvent.input, context), + interpretEvents + ), history: inputEvents, }; } diff --git a/packages/@eventual/core/test/interpret.test.ts b/packages/@eventual/core/test/interpret.test.ts index f446a5460..b1a9832ed 100644 --- a/packages/@eventual/core/test/interpret.test.ts +++ b/packages/@eventual/core/test/interpret.test.ts @@ -12,6 +12,11 @@ import { Program, ScheduleActivityCommand, EventualKind, + workflow, + ScheduleWorkflowCommand, + ChildWorkflowScheduled, + ChildWorkflowCompleted, + ChildWorkflowFailed, } from "../src/index.js"; import { DeterminismError } from "../src/error.js"; import { chain } from "../src/chain.js"; @@ -47,7 +52,7 @@ test("determinism error if no corresponding ActivityScheduled", () => { expect(() => interpret(myWorkflow(event), [ // error: completed event should be after a scheduled event - completed("result", 0), + activityCompleted("result", 0), ]) ).toThrow(expect.any(DeterminismError)); }); @@ -55,8 +60,8 @@ test("determinism error if no corresponding ActivityScheduled", () => { test("should continue with result of completed Activity", () => { expect( interpret(myWorkflow(event), [ - scheduled("my-activity", 0), - completed("result", 0), + activityScheduled("my-activity", 0), + activityCompleted("result", 0), ]) ).toMatchObject({ commands: [ @@ -70,8 +75,8 @@ test("should continue with result of completed Activity", () => { test("should catch error of failed Activity", () => { expect( interpret(myWorkflow(event), [ - scheduled("my-activity", 0), - failed("error", 0), + activityScheduled("my-activity", 0), + activityFailed("error", 0), ]) ).toMatchObject({ commands: [createScheduledActivityCommand("handle-error", ["error"], 1)], @@ -81,14 +86,14 @@ test("should catch error of failed Activity", () => { test("should return final result", () => { expect( interpret(myWorkflow(event), [ - scheduled("my-activity", 0), - completed("result", 0), - scheduled("my-activity-0", 1), - scheduled("my-activity-1", 2), - scheduled("my-activity-2", 3), - completed("result-0", 1), - completed("result-1", 2), - completed("result-2", 3), + activityScheduled("my-activity", 0), + activityCompleted("result", 0), + activityScheduled("my-activity-0", 1), + activityScheduled("my-activity-1", 2), + activityScheduled("my-activity-2", 3), + activityCompleted("result-0", 1), + activityCompleted("result-1", 2), + activityCompleted("result-2", 3), ]) ).toMatchObject({ result: Result.resolved(["result", ["result-1", "result-2"]]), @@ -99,13 +104,13 @@ test("should return final result", () => { test("should wait if partial results", () => { expect( interpret(myWorkflow(event), [ - scheduled("my-activity", 0), - completed("result", 0), - scheduled("my-activity-0", 1), - scheduled("my-activity-1", 2), - scheduled("my-activity-2", 3), - completed("result-0", 1), - completed("result-1", 2), + activityScheduled("my-activity", 0), + activityCompleted("result", 0), + activityScheduled("my-activity-0", 1), + activityScheduled("my-activity-1", 2), + activityScheduled("my-activity-2", 3), + activityCompleted("result-0", 1), + activityCompleted("result-1", 2), ]) ).toMatchObject({ commands: [], @@ -175,10 +180,10 @@ test("should support Eventual.all of function calls", () => { expect( interpret(workflow(["a", "b"]), [ - scheduled("process-item", 0), - scheduled("process-item", 1), - completed("A", 0), - completed("B", 1), + activityScheduled("process-item", 0), + activityScheduled("process-item", 1), + activityCompleted("A", 0), + activityCompleted("B", 1), ]) ).toMatchObject({ result: Result.resolved(["A", "B"]), @@ -223,7 +228,10 @@ test("try-catch-finally with yield in catch", () => { commands: [createScheduledActivityCommand("catch", [], 0)], }); expect( - interpret(workflow(), [scheduled("catch", 0), completed(undefined, 0)]) + interpret(workflow(), [ + activityScheduled("catch", 0), + activityCompleted(undefined, 0), + ]) ).toMatchObject({ commands: [createScheduledActivityCommand("finally", [], 1)], }); @@ -283,36 +291,36 @@ test("throw error within nested function", () => { }); expect( interpret(workflow(["good", "bad"]), [ - scheduled("inside", 0), - scheduled("inside", 1), - completed("good", 0), - completed("bad", 1), + activityScheduled("inside", 0), + activityScheduled("inside", 1), + activityCompleted("good", 0), + activityCompleted("bad", 1), ]) ).toMatchObject({ commands: [createScheduledActivityCommand("catch", [], 2)], }); expect( interpret(workflow(["good", "bad"]), [ - scheduled("inside", 0), - scheduled("inside", 1), - completed("good", 0), - completed("bad", 1), - scheduled("catch", 2), - completed("catch", 2), + activityScheduled("inside", 0), + activityScheduled("inside", 1), + activityCompleted("good", 0), + activityCompleted("bad", 1), + activityScheduled("catch", 2), + activityCompleted("catch", 2), ]) ).toMatchObject({ commands: [createScheduledActivityCommand("finally", [], 3)], }); expect( interpret(workflow(["good", "bad"]), [ - scheduled("inside", 0), - scheduled("inside", 1), - completed("good", 0), - completed("bad", 1), - scheduled("catch", 2), - completed("catch", 2), - scheduled("finally", 3), - completed("finally", 3), + activityScheduled("inside", 0), + activityScheduled("inside", 1), + activityCompleted("good", 0), + activityCompleted("bad", 1), + activityScheduled("catch", 2), + activityCompleted("catch", 2), + activityScheduled("finally", 3), + activityCompleted("finally", 3), ]) ).toMatchObject({ result: Result.resolved("returned in finally"), @@ -344,10 +352,10 @@ test("properly evaluate yield* of sub-programs", () => { expect( interpret(workflow(), [ - scheduled("a", 0), - scheduled("b", 1), - completed("a", 0), - completed("b", 1), + activityScheduled("a", 0), + activityScheduled("b", 1), + activityCompleted("a", 0), + activityCompleted("b", 1), ]) ).toMatchObject({ result: Result.resolved(["a", "b"]), @@ -378,10 +386,10 @@ test("properly evaluate yield of Eventual.all", () => { expect( // @ts-ignore interpret(workflow(), [ - scheduled("a", 0), - scheduled("b", 1), - completed("a", 0), - completed("b", 1), + activityScheduled("a", 0), + activityScheduled("b", 1), + activityCompleted("a", 0), + activityCompleted("b", 1), ]) ).toMatchObject({ result: Result.resolved(["a", "b"]), @@ -402,13 +410,80 @@ test("generator function returns an ActivityCall", () => { commands: [createScheduledActivityCommand("call-a", [], 0)], }); expect( - interpret(workflow(), [scheduled("call-a", 0), completed("result", 0)]) + interpret(workflow(), [ + activityScheduled("call-a", 0), + activityCompleted("result", 0), + ]) ).toMatchObject({ result: Result.resolved("result"), commands: [], }); }); +test("workflow calling other workflow", () => { + const wf1 = workflow("wf1", function* () { + yield createActivityCall("call-a", []); + }); + // @ts-ignore + const wf2 = workflow("wf2", function* () { + // @ts-ignore + const result = yield wf1(); + yield createActivityCall("call-b", []); + return result; + }); + + expect(interpret(wf2.definition(), [])).toMatchObject({ + commands: [createScheduledWorkflowCommand("wf1", undefined, 0)], + }); + + expect( + interpret(wf2.definition(), [workflowScheduled("wf1", 0)]) + ).toMatchObject({ + commands: [], + }); + + expect( + interpret(wf2.definition(), [ + workflowScheduled("wf1", 0), + workflowCompleted("result", 0), + ]) + ).toMatchObject({ + commands: [createScheduledActivityCommand("call-b", [], 1)], + }); + + expect( + interpret(wf2.definition(), [ + workflowScheduled("wf1", 0), + workflowCompleted("result", 0), + activityScheduled("call-b", 1), + ]) + ).toMatchObject({ + commands: [], + }); + + expect( + interpret(wf2.definition(), [ + workflowScheduled("wf1", 0), + workflowCompleted("result", 0), + activityScheduled("call-b", 1), + activityCompleted(undefined, 1), + ]) + ).toMatchObject({ + result: Result.resolved("result"), + commands: [], + }); + + expect( + interpret(wf2.definition(), [ + workflowScheduled("wf1", 0), + workflowFailed("error", 0), + ]) + ).toMatchObject({ + result: Result.failed("error"), + commands: [], + }); +}); + function createScheduledActivityCommand( name: string, args: any[], @@ -422,7 +497,20 @@ function createScheduledActivityCommand( }; } -function completed(result: any, seq: number): ActivityCompleted { +function createScheduledWorkflowCommand( + name: string, + input: any, + seq: number +): ScheduleWorkflowCommand { + return { + kind: EventualKind.WorkflowCall, + seq, + name, + input, + }; +} + +function activityCompleted(result: any, seq: number): ActivityCompleted { return { type: WorkflowEventType.ActivityCompleted, duration: 0, @@ -432,7 +520,16 @@ function completed(result: any, seq: number): ActivityCompleted { }; } -function failed(error: any, seq: number): ActivityFailed { +function workflowCompleted(result: any, seq: number): ChildWorkflowCompleted { + return { + type: WorkflowEventType.ChildWorkflowCompleted, + result, + seq, + timestamp: new Date(0).toISOString(), + }; +} + +function activityFailed(error: any, seq: number): ActivityFailed { return { type: WorkflowEventType.ActivityFailed, duration: 0, @@ -443,7 +540,17 @@ function failed(error: any, seq: number): ActivityFailed { }; } -function scheduled(name: string, seq: number): ActivityScheduled { +function workflowFailed(error: any, seq: number): ChildWorkflowFailed { + return { + type: WorkflowEventType.ChildWorkflowFailed, + error, + message: "message", + seq, + timestamp: new Date(0).toISOString(), + }; +} + +function activityScheduled(name: string, seq: number): ActivityScheduled { return { type: WorkflowEventType.ActivityScheduled, name, @@ -451,3 +558,13 @@ function scheduled(name: string, seq: number): ActivityScheduled { timestamp: new Date(0).toISOString(), }; } + +function workflowScheduled(name: string, seq: number): ChildWorkflowScheduled { + return { + type: WorkflowEventType.ChildWorkflowScheduled, + name, + seq, + timestamp: new Date(0).toISOString(), + input: undefined, + }; +} From 2fedfbb9608e06546ab76e89570885ef8129d715 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 20:47:13 -0800 Subject: [PATCH 08/39] fix: lookup workflowName from dynamodb --- packages/@eventual/aws-cdk/src/service.ts | 1 + packages/@eventual/aws-runtime/package.json | 3 +- .../src/clients/workflow-client.ts | 59 +++++++--- .../src/clients/workflow-runtime-client.ts | 107 ++++++++++-------- .../aws-runtime/src/entry/orchestrator.ts | 4 +- .../aws-runtime/src/handlers/orchestrator.ts | 19 +++- packages/@eventual/core/src/workflow.ts | 10 ++ packages/@eventual/core/test/workflow.test.ts | 6 +- pnpm-lock.yaml | 3 +- 9 files changed, 134 insertions(+), 78 deletions(-) diff --git a/packages/@eventual/aws-cdk/src/service.ts b/packages/@eventual/aws-cdk/src/service.ts index 55fc023e1..018e4b3f5 100644 --- a/packages/@eventual/aws-cdk/src/service.ts +++ b/packages/@eventual/aws-cdk/src/service.ts @@ -90,6 +90,7 @@ export class Service extends Construct implements IGrantable { fifo: true, fifoThroughputLimit: FifoThroughputLimit.PER_MESSAGE_GROUP_ID, deduplicationScope: DeduplicationScope.MESSAGE_GROUP, + contentBasedDeduplication: true, }); // Table - History, Executions, ExecutionData diff --git a/packages/@eventual/aws-runtime/package.json b/packages/@eventual/aws-runtime/package.json index 7744aac5f..cfb2007b7 100644 --- a/packages/@eventual/aws-runtime/package.json +++ b/packages/@eventual/aws-runtime/package.json @@ -21,7 +21,8 @@ "aws-lambda": "^1.0.7", "fast-equals": "^4.0.3", "micro-memoize": "^4.0.11", - "ulidx": "^0.3.0" + "ulidx": "^0.3.0", + "lru-cache": "7.14.1" }, "peerDependencies": { "@aws-sdk/client-dynamodb": "^3.208.0", diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts index 8c73822fa..757b1b97e 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts @@ -36,7 +36,11 @@ export interface StartWorkflowRequest { /** * ID of the parent execution if this is a child workflow */ - parentId?: string; + parentExecutionId?: string; + /** + * Sequence ID of this execution if this is a child workflow + */ + seq?: number; } export interface WorkflowClientProps { @@ -60,7 +64,8 @@ export class WorkflowClient { executionName = ulid(), workflowName, input, - parentId, + parentExecutionId, + seq, }: StartWorkflowRequest) { const executionId = executionName; console.log("execution input:", input); @@ -73,9 +78,15 @@ export class WorkflowClient { sk: { S: ExecutionRecord.sortKey(executionId) }, id: { S: executionId }, name: { S: executionName }, + workflowName: { S: workflowName }, status: { S: ExecutionStatus.IN_PROGRESS }, startTime: { S: new Date().toISOString() }, - ...(parentId ? { parentId: { S: parentId } } : {}), + ...(parentExecutionId + ? { + parentExecutionId: { S: parentExecutionId }, + seq: { N: seq!.toString(10) }, + } + : {}), }, }) ); @@ -89,7 +100,7 @@ export class WorkflowClient { workflowName, context: { name: executionName, - parentId, + parentId: parentExecutionId, }, } ); @@ -116,8 +127,6 @@ export class WorkflowClient { MessageBody: JSON.stringify(workflowTask), QueueUrl: this.props.workflowQueueUrl, MessageGroupId: executionId, - // just de-dupe with itself - MessageDeduplicationId: `${executionId}_${ulid()}`, }) ); } @@ -127,22 +136,36 @@ export interface SQSWorkflowTaskMessage { task: WorkflowTask; } -export interface ExecutionRecord { - pk: { S: typeof ExecutionRecord.PRIMARY_KEY }; - sk: { S: `${typeof ExecutionRecord.SORT_KEY_PREFIX}${string}` }; - result?: AttributeValue.SMember; - id: AttributeValue.SMember; - status: { S: ExecutionStatus }; - startTime: AttributeValue.SMember; - endTime?: AttributeValue.SMember; - error?: AttributeValue.SMember; - message?: AttributeValue.SMember; -} +export type ExecutionRecord = + | { + pk: { S: typeof ExecutionRecord.PRIMARY_KEY }; + sk: { S: `${typeof ExecutionRecord.SORT_KEY_PREFIX}${string}` }; + result?: AttributeValue.SMember; + id: AttributeValue.SMember; + status: { S: ExecutionStatus }; + startTime: AttributeValue.SMember; + name: AttributeValue.SMember; + workflowName: AttributeValue.SMember; + endTime?: AttributeValue.SMember; + error?: AttributeValue.SMember; + message?: AttributeValue.SMember; + } & ( + | { + parentExecutionId: AttributeValue.SMember; + seq: AttributeValue.NMember; + } + | { + parentExecutionId?: never; + seq?: never; + } + ); export namespace ExecutionRecord { export const PRIMARY_KEY = "Execution"; export const SORT_KEY_PREFIX = `Execution$`; - export function sortKey(executionId: string) { + export function sortKey( + executionId: string + ): `${typeof SORT_KEY_PREFIX}${typeof executionId}` { return `${SORT_KEY_PREFIX}${executionId}`; } } diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index 924ae43f6..4493d259b 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -1,5 +1,7 @@ import { + AttributeValue, DynamoDBClient, + GetItemCommand, QueryCommand, UpdateItemCommand, } from "@aws-sdk/client-dynamodb"; @@ -32,6 +34,8 @@ import { import { ActivityWorkerRequest } from "../activity.js"; import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; +import LRUCache from "lru-cache"; + export interface WorkflowRuntimeClientProps { readonly lambda: LambdaClient; readonly activityWorkerFunctionName: string; @@ -49,6 +53,10 @@ export interface CompleteExecutionRequest { } export class WorkflowRuntimeClient { + private workflowNameCache = new LRUCache({ + max: 1000, + }); + constructor(private props: WorkflowRuntimeClientProps) {} async getHistory(executionId: string) { @@ -70,6 +78,31 @@ export class WorkflowRuntimeClient { } } + async getWorkflowName(executionId: string): Promise { + let workflowName = this.workflowNameCache.get(executionId); + if (workflowName !== undefined) { + return workflowName; + } + const response = await this.props.dynamo.send( + new GetItemCommand({ + TableName: this.props.tableName, + Key: { + pk: { S: ExecutionRecord.PRIMARY_KEY }, + sk: { S: ExecutionRecord.sortKey(executionId) }, + }, + AttributesToGet: ["workflowName"], + }) + ); + if (response.Item === undefined) { + return undefined; + } + workflowName = response.Item.workflowName?.S; + if (workflowName) { + this.workflowNameCache.set(executionId, workflowName); + } + return workflowName; + } + // TODO: etag async updateHistory( executionId: string, @@ -116,13 +149,17 @@ export class WorkflowRuntimeClient { }) ); - if (isChildExecutionId(executionId)) { - await this.completeChildExecution(executionId, result); + const record = executionResult.Attributes as unknown as ExecutionRecord; + if (record.parentExecutionId) { + await this.completeChildExecution( + executionId, + record.parentExecutionId, + record.seq, + result + ); } - return createExecutionFromResult( - executionResult.Attributes as unknown as ExecutionRecord - ) as CompleteExecution; + return createExecutionFromResult(record) as CompleteExecution; } async failExecution( @@ -156,8 +193,15 @@ export class WorkflowRuntimeClient { }) ); - if (isChildExecutionId(executionId)) { - await this.completeChildExecution(executionId, error, message); + const record = executionResult.Attributes as unknown as ExecutionRecord; + if (record.parentExecutionId) { + await this.completeChildExecution( + executionId, + record.parentExecutionId, + record.seq, + error, + message + ); } return createExecutionFromResult( @@ -166,16 +210,17 @@ export class WorkflowRuntimeClient { } private async completeChildExecution( - executionId: ChildExecutionId, + executionId: string, + parentExecutionId: AttributeValue.SMember, + seq: AttributeValue.NMember, ...args: [result: any] | [error: string, message: string] ) { - const { parentExecutionId, seq } = parseChildExecutionId(executionId); const workflowTask: SQSWorkflowTaskMessage = { task: { - executionId: parentExecutionId, + executionId: parentExecutionId.S, events: [ { - seq, + seq: parseInt(seq.N, 10), timestamp: new Date().toISOString(), ...(args.length === 1 ? { @@ -195,7 +240,7 @@ export class WorkflowRuntimeClient { new SendMessageCommand({ QueueUrl: this.props.workflowQueueUrl, MessageBody: JSON.stringify(workflowTask), - MessageGroupId: parentExecutionId, + MessageGroupId: parentExecutionId.S, MessageDeduplicationId: `${executionId}/complete`, }) ); @@ -253,41 +298,3 @@ async function historyEntryToEvents( function formatExecutionHistoryKey(executionId: string) { return `executionHistory/${executionId}`; } - -/** - * A child workflow execution's Id is encoded as follows: - * ``` - * {parentExecutionId}/{seq} - * ``` - * Child workflow's can be multiple levels deep: - * ``` - * {rootExecutionId}/{seq-0}/../{seq-n} - * ``` - * - * TODO: define a depth limit based on key limit maximums. - * TODO: can we use a hash so that nested execution ids are bounded in length? - */ -type ChildExecutionId = `${string}/${number}`; - -function isChildExecutionId( - executionId: string -): executionId is ChildExecutionId { - return executionId.split("/").length >= 2; -} - -function parseChildExecutionId(executionId: ChildExecutionId): { - parentExecutionId: string; - seq: number; -} { - const lastSlash = executionId.lastIndexOf("/"); - const parentExecutionId = executionId.slice(0, lastSlash); - const seqStr = executionId.slice(lastSlash + 1 /* +1 to skip past the '/' */); - const seq = parseInt(seqStr!, 10); - if (isNaN(seq)) { - throw new Error(`invalid sequence number ${seqStr}`); - } - return { - parentExecutionId, - seq, - }; -} diff --git a/packages/@eventual/aws-runtime/src/entry/orchestrator.ts b/packages/@eventual/aws-runtime/src/entry/orchestrator.ts index 3426b27f4..cc928918d 100644 --- a/packages/@eventual/aws-runtime/src/entry/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/entry/orchestrator.ts @@ -1,4 +1,4 @@ -import workflow from "@eventual/injected/workflow"; +import "@eventual/injected/workflow"; import { orchestrator } from "../handlers/orchestrator.js"; -export default orchestrator(workflow); +export default orchestrator(); diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index 85f97ca30..125a9fe40 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -22,6 +22,7 @@ import { isScheduleWorkflowCommand, assertNever, ChildWorkflowScheduled, + lookupWorkflow, } from "@eventual/core"; import { SQSWorkflowTaskMessage } from "../clients/workflow-client.js"; import { @@ -44,7 +45,7 @@ const workflowClient = createWorkflowClient(); /** * Creates an entrypoint function for orchestrating a workflow. */ -export function orchestrator(workflow: Workflow): SQSHandler { +export function orchestrator(): SQSHandler { return middy(async (event: SQSEvent) => { logger.debug("Handle workflowQueue records"); // if a polling request @@ -65,8 +66,20 @@ export function orchestrator(workflow: Workflow): SQSHandler { // for each execution id const results = await promiseAllSettledPartitioned( Object.entries(eventsByExecutionId), - async ([executionId, records]) => - orchestrateExecution(workflow, executionId, records) + async ([executionId, records]) => { + const workflowName = await workflowRuntimeClient.getWorkflowName( + executionId + ); + if (workflowName === undefined) { + throw new Error(`execution ID '${executionId}' does not exist`); + } + const workflow = lookupWorkflow(workflowName); + if (workflow === undefined) { + throw new Error(`no such workflow with name '${workflowName}'`); + } + // TODO: get workflow from execution id + return orchestrateExecution(workflow, executionId, records); + } ); logger.debug( diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index 3abfc1d17..756726740 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -60,6 +60,12 @@ export interface Workflow< ) => Program>>; } +const workflows = new Map(); + +export function lookupWorkflow(name: string): Workflow | undefined { + return workflows.get(name); +} + /** * Creates and registers a long-running workflow. * @@ -84,6 +90,9 @@ export function workflow Promise | Program>( name: string, definition: F ): Workflow { + if (workflows.has(name)) { + throw new Error(`workflow with name '${name}' already exists`); + } const workflow: Workflow = ((input?: any) => registerActivity({ [EventualSymbol]: EventualKind.WorkflowCall, @@ -96,6 +105,7 @@ export function workflow Promise | Program>( throw new Error("not implemented"); }; workflow.definition = definition as Workflow["definition"]; // safe to cast because we rely on transformer (it is always the generator API) + workflows.set(name, workflow); return workflow; } diff --git a/packages/@eventual/core/test/workflow.test.ts b/packages/@eventual/core/test/workflow.test.ts index d11ab2b8e..6c6e92d74 100644 --- a/packages/@eventual/core/test/workflow.test.ts +++ b/packages/@eventual/core/test/workflow.test.ts @@ -6,13 +6,13 @@ import { WorkflowEventType, WorkflowStarted, } from "../src/events"; -import { progressWorkflow } from "../src/workflow"; +import { progressWorkflow, workflow } from "../src/workflow"; import { WorkflowContext } from "../src/context"; -function* myWorkflow(event: any): Program { +const myWorkflow = workflow("myWorkflow", function* (event: any): Program { yield createActivityCall("my-activity", [event]); yield createActivityCall("my-activity", [event]); -} +}); const started1: WorkflowStarted = { type: WorkflowEventType.WorkflowStarted, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 51ec54bf0..8680f0d16 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -144,6 +144,7 @@ importers: aws-lambda: ^1.0.7 fast-equals: ^4.0.3 jest: ^29 + lru-cache: 7.14.1 micro-memoize: ^4.0.11 ts-jest: ^29 ts-node: ^10.9.1 @@ -157,6 +158,7 @@ importers: aws-embedded-metrics: 4.0.0 aws-lambda: 1.0.7 fast-equals: 4.0.3 + lru-cache: 7.14.1 micro-memoize: 4.0.11 ulidx: 0.3.0 devDependencies: @@ -6762,7 +6764,6 @@ packages: /lru-cache/7.14.1: resolution: {integrity: sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA==} engines: {node: '>=12'} - dev: true /make-dir/2.1.0: resolution: {integrity: sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==} From 5869a8b474dc3d5eeef7e340c435710c2baf56e8 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 20:52:24 -0800 Subject: [PATCH 09/39] fix: pass through parent execution id and seq --- packages/@eventual/aws-runtime/src/handlers/orchestrator.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index 125a9fe40..78edd36ca 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -355,8 +355,9 @@ async function orchestrateExecution( } else if (isScheduleWorkflowCommand(command)) { await workflowClient.startWorkflow({ workflowName: command.name, - executionName: `${executionId}/${command.seq}`, input: command.input, + parentExecutionId: executionId, + seq: command.seq, }); return createEvent({ From d6b88ee51fdba02bba9a603ed6df8ff5219e32fb Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 21:09:52 -0800 Subject: [PATCH 10/39] fix: encode workflow name in ID --- packages/@eventual/aws-runtime/package.json | 3 +- .../src/clients/workflow-client.ts | 3 +- .../src/clients/workflow-runtime-client.ts | 32 ------------------- .../@eventual/aws-runtime/src/execution-id.ts | 19 +++++++++++ .../aws-runtime/src/handlers/orchestrator.ts | 8 +++-- pnpm-lock.yaml | 3 +- 6 files changed, 28 insertions(+), 40 deletions(-) create mode 100644 packages/@eventual/aws-runtime/src/execution-id.ts diff --git a/packages/@eventual/aws-runtime/package.json b/packages/@eventual/aws-runtime/package.json index cfb2007b7..7744aac5f 100644 --- a/packages/@eventual/aws-runtime/package.json +++ b/packages/@eventual/aws-runtime/package.json @@ -21,8 +21,7 @@ "aws-lambda": "^1.0.7", "fast-equals": "^4.0.3", "micro-memoize": "^4.0.11", - "ulidx": "^0.3.0", - "lru-cache": "7.14.1" + "ulidx": "^0.3.0" }, "peerDependencies": { "@aws-sdk/client-dynamodb": "^3.208.0", diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts index 757b1b97e..258e387ae 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts @@ -12,6 +12,7 @@ import { WorkflowEventType, HistoryStateEvents, } from "@eventual/core"; +import { formatExecutionId } from "../execution-id.js"; import { ulid } from "ulidx"; import { ExecutionHistoryClient } from "./execution-history-client.js"; @@ -67,7 +68,7 @@ export class WorkflowClient { parentExecutionId, seq, }: StartWorkflowRequest) { - const executionId = executionName; + const executionId = formatExecutionId(workflowName, executionName); console.log("execution input:", input); await this.props.dynamo.send( diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index 4493d259b..6b9118489 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -1,7 +1,6 @@ import { AttributeValue, DynamoDBClient, - GetItemCommand, QueryCommand, UpdateItemCommand, } from "@aws-sdk/client-dynamodb"; @@ -34,8 +33,6 @@ import { import { ActivityWorkerRequest } from "../activity.js"; import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; -import LRUCache from "lru-cache"; - export interface WorkflowRuntimeClientProps { readonly lambda: LambdaClient; readonly activityWorkerFunctionName: string; @@ -53,10 +50,6 @@ export interface CompleteExecutionRequest { } export class WorkflowRuntimeClient { - private workflowNameCache = new LRUCache({ - max: 1000, - }); - constructor(private props: WorkflowRuntimeClientProps) {} async getHistory(executionId: string) { @@ -78,31 +71,6 @@ export class WorkflowRuntimeClient { } } - async getWorkflowName(executionId: string): Promise { - let workflowName = this.workflowNameCache.get(executionId); - if (workflowName !== undefined) { - return workflowName; - } - const response = await this.props.dynamo.send( - new GetItemCommand({ - TableName: this.props.tableName, - Key: { - pk: { S: ExecutionRecord.PRIMARY_KEY }, - sk: { S: ExecutionRecord.sortKey(executionId) }, - }, - AttributesToGet: ["workflowName"], - }) - ); - if (response.Item === undefined) { - return undefined; - } - workflowName = response.Item.workflowName?.S; - if (workflowName) { - this.workflowNameCache.set(executionId, workflowName); - } - return workflowName; - } - // TODO: etag async updateHistory( executionId: string, diff --git a/packages/@eventual/aws-runtime/src/execution-id.ts b/packages/@eventual/aws-runtime/src/execution-id.ts new file mode 100644 index 000000000..43fa1ec64 --- /dev/null +++ b/packages/@eventual/aws-runtime/src/execution-id.ts @@ -0,0 +1,19 @@ +export type ExecutionID< + WorkflowName extends string = string, + ID extends string = string +> = `${WorkflowName}/${ID}`; + +export function isExecutionId(a: any): a is ExecutionID { + return typeof a === "string" && a.split("/").length === 2; +} + +export function parseWorkflowName(executionId: ExecutionID): string { + return executionId.split("/")[0]!; +} + +export function formatExecutionId< + WorkflowName extends string, + ID extends string +>(workflowName: string, id: string): ExecutionID { + return `${workflowName}/${id}` as ExecutionID; +} diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index 78edd36ca..7cef3f21c 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -37,6 +37,7 @@ import { MetricsCommon, OrchestratorMetrics } from "../metrics/constants.js"; import middy from "@middy/core"; import { logger, loggerMiddlewares } from "../logger.js"; import { WorkflowContext } from "@eventual/core"; +import { isExecutionId, parseWorkflowName } from "src/execution-id.js"; const executionHistoryClient = createExecutionHistoryClient(); const workflowRuntimeClient = createWorkflowRuntimeClient(); @@ -67,9 +68,10 @@ export function orchestrator(): SQSHandler { const results = await promiseAllSettledPartitioned( Object.entries(eventsByExecutionId), async ([executionId, records]) => { - const workflowName = await workflowRuntimeClient.getWorkflowName( - executionId - ); + if (!isExecutionId(executionId)) { + throw new Error(`invalid ExecutionID: '${executionId}'`); + } + const workflowName = parseWorkflowName(executionId); if (workflowName === undefined) { throw new Error(`execution ID '${executionId}' does not exist`); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8680f0d16..51ec54bf0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -144,7 +144,6 @@ importers: aws-lambda: ^1.0.7 fast-equals: ^4.0.3 jest: ^29 - lru-cache: 7.14.1 micro-memoize: ^4.0.11 ts-jest: ^29 ts-node: ^10.9.1 @@ -158,7 +157,6 @@ importers: aws-embedded-metrics: 4.0.0 aws-lambda: 1.0.7 fast-equals: 4.0.3 - lru-cache: 7.14.1 micro-memoize: 4.0.11 ulidx: 0.3.0 devDependencies: @@ -6764,6 +6762,7 @@ packages: /lru-cache/7.14.1: resolution: {integrity: sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA==} engines: {node: '>=12'} + dev: true /make-dir/2.1.0: resolution: {integrity: sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==} From 61c33ae2717934e2a6dd4f641be7e3d3ef9896ca Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 21:23:41 -0800 Subject: [PATCH 11/39] chore: feedback --- .../aws-runtime/src/clients/create.ts | 2 +- .../src/clients/workflow-runtime-client.ts | 52 ++++++------------- 2 files changed, 18 insertions(+), 36 deletions(-) diff --git a/packages/@eventual/aws-runtime/src/clients/create.ts b/packages/@eventual/aws-runtime/src/clients/create.ts index 58028f7ae..db572f142 100644 --- a/packages/@eventual/aws-runtime/src/clients/create.ts +++ b/packages/@eventual/aws-runtime/src/clients/create.ts @@ -80,6 +80,6 @@ export const createWorkflowRuntimeClient = /*@__PURE__*/ memoize( activityWorkerFunctionName: activityWorkerFunctionName ?? env.activityWorkerFunctionName(), sqs: sqs(), - workflowQueueUrl: env.workflowQueueUrl(), + workflowClient: createWorkflowClient(), }) ); diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index 6b9118489..79f879688 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -28,10 +28,10 @@ import { import { createExecutionFromResult, ExecutionRecord, - SQSWorkflowTaskMessage, + WorkflowClient, } from "./workflow-client.js"; import { ActivityWorkerRequest } from "../activity.js"; -import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; +import { SQSClient } from "@aws-sdk/client-sqs"; export interface WorkflowRuntimeClientProps { readonly lambda: LambdaClient; @@ -41,7 +41,7 @@ export interface WorkflowRuntimeClientProps { readonly sqs: SQSClient; readonly executionHistoryBucket: string; readonly tableName: string; - readonly workflowQueueUrl: string; + readonly workflowClient: WorkflowClient; } export interface CompleteExecutionRequest { @@ -120,7 +120,6 @@ export class WorkflowRuntimeClient { const record = executionResult.Attributes as unknown as ExecutionRecord; if (record.parentExecutionId) { await this.completeChildExecution( - executionId, record.parentExecutionId, record.seq, result @@ -164,7 +163,6 @@ export class WorkflowRuntimeClient { const record = executionResult.Attributes as unknown as ExecutionRecord; if (record.parentExecutionId) { await this.completeChildExecution( - executionId, record.parentExecutionId, record.seq, error, @@ -178,40 +176,24 @@ export class WorkflowRuntimeClient { } private async completeChildExecution( - executionId: string, parentExecutionId: AttributeValue.SMember, seq: AttributeValue.NMember, ...args: [result: any] | [error: string, message: string] ) { - const workflowTask: SQSWorkflowTaskMessage = { - task: { - executionId: parentExecutionId.S, - events: [ - { - seq: parseInt(seq.N, 10), - timestamp: new Date().toISOString(), - ...(args.length === 1 - ? { - type: WorkflowEventType.ChildWorkflowCompleted, - result: args[0], - } - : { - type: WorkflowEventType.ChildWorkflowFailed, - error: args[0], - message: args[1], - }), - }, - ], - }, - }; - await this.props.sqs.send( - new SendMessageCommand({ - QueueUrl: this.props.workflowQueueUrl, - MessageBody: JSON.stringify(workflowTask), - MessageGroupId: parentExecutionId.S, - MessageDeduplicationId: `${executionId}/complete`, - }) - ); + await this.props.workflowClient.submitWorkflowTask(parentExecutionId.S, { + seq: parseInt(seq.N, 10), + timestamp: new Date().toISOString(), + ...(args.length === 1 + ? { + type: WorkflowEventType.ChildWorkflowCompleted, + result: args[0], + } + : { + type: WorkflowEventType.ChildWorkflowFailed, + error: args[0], + message: args[1], + }), + }); } async getExecutions(): Promise { From b64d0b1d512b717f2c1c78ceea6ad765c4cfae3e Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 22:31:09 -0800 Subject: [PATCH 12/39] fix: make completion idempotent --- .../aws-runtime/src/clients/workflow-runtime-client.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index 79f879688..69e01cea9 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -100,16 +100,14 @@ export class WorkflowRuntimeClient { }, TableName: this.props.tableName, UpdateExpression: result - ? "SET #status=:complete, #result=:result, endTime=:endTime" - : "SET #status=:complete, endTime=:endTime", - ConditionExpression: "#status=:in_progress", + ? "SET #status=:complete, #result=:result, endTime=if_not_exists(endTime,:endTime)" + : "SET #status=:complete, endTime=if_not_exists(endTime,:endTime)", ExpressionAttributeNames: { "#status": "status", ...(result ? { "#result": "result" } : {}), }, ExpressionAttributeValues: { ":complete": { S: ExecutionStatus.COMPLETE }, - ":in_progress": { S: ExecutionStatus.IN_PROGRESS }, ":endTime": { S: new Date().toISOString() }, ...(result ? { ":result": { S: JSON.stringify(result) } } : {}), }, @@ -142,8 +140,7 @@ export class WorkflowRuntimeClient { }, TableName: this.props.tableName, UpdateExpression: - "SET #status=:failed, #error=:error, #message=:message, endTime=:endTime", - ConditionExpression: "#status=:in_progress", + "SET #status=:failed, #error=:error, #message=:message, endTime=if_not_exists(endTime,:endTime)", ExpressionAttributeNames: { "#status": "status", "#error": "error", @@ -151,7 +148,6 @@ export class WorkflowRuntimeClient { }, ExpressionAttributeValues: { ":failed": { S: ExecutionStatus.FAILED }, - ":in_progress": { S: ExecutionStatus.IN_PROGRESS }, ":endTime": { S: new Date().toISOString() }, ":error": { S: error }, ":message": { S: message }, From e0d1e2055e05264dbf81d9c1f252e66d5e071a4b Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 22:38:02 -0800 Subject: [PATCH 13/39] fix: feedback --- .../aws-runtime/src/clients/workflow-runtime-client.ts | 6 +----- .../@eventual/aws-runtime/src/handlers/activity-worker.ts | 2 +- packages/@eventual/aws-runtime/src/handlers/orchestrator.ts | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index 69e01cea9..0d55a7ec0 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -31,14 +31,12 @@ import { WorkflowClient, } from "./workflow-client.js"; import { ActivityWorkerRequest } from "../activity.js"; -import { SQSClient } from "@aws-sdk/client-sqs"; export interface WorkflowRuntimeClientProps { readonly lambda: LambdaClient; readonly activityWorkerFunctionName: string; readonly dynamo: DynamoDBClient; readonly s3: S3Client; - readonly sqs: SQSClient; readonly executionHistoryBucket: string; readonly tableName: string; readonly workflowClient: WorkflowClient; @@ -166,9 +164,7 @@ export class WorkflowRuntimeClient { ); } - return createExecutionFromResult( - executionResult.Attributes as unknown as ExecutionRecord - ) as FailedExecution; + return createExecutionFromResult(record) as FailedExecution; } private async completeChildExecution( diff --git a/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts b/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts index 69c51546e..874f4b9ef 100644 --- a/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts @@ -18,7 +18,7 @@ import { metricScope, Unit } from "aws-embedded-metrics"; import { timed } from "../metrics/utils.js"; import { ActivityMetrics, MetricsCommon } from "../metrics/constants.js"; import middy from "@middy/core"; -import { logger, loggerMiddlewares } from "src/logger.js"; +import { logger, loggerMiddlewares } from "../logger.js"; const activityRuntimeClient = createActivityRuntimeClient(); const executionHistoryClient = createExecutionHistoryClient(); diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index 7cef3f21c..e373ef5e9 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -37,7 +37,7 @@ import { MetricsCommon, OrchestratorMetrics } from "../metrics/constants.js"; import middy from "@middy/core"; import { logger, loggerMiddlewares } from "../logger.js"; import { WorkflowContext } from "@eventual/core"; -import { isExecutionId, parseWorkflowName } from "src/execution-id.js"; +import { isExecutionId, parseWorkflowName } from "../execution-id.js"; const executionHistoryClient = createExecutionHistoryClient(); const workflowRuntimeClient = createWorkflowRuntimeClient(); From e65ebcc330904a774f917ee79b6fd4f8d6b4f280 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 22:39:55 -0800 Subject: [PATCH 14/39] fix: tsc --- packages/@eventual/aws-runtime/src/clients/create.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/@eventual/aws-runtime/src/clients/create.ts b/packages/@eventual/aws-runtime/src/clients/create.ts index db572f142..058555ea7 100644 --- a/packages/@eventual/aws-runtime/src/clients/create.ts +++ b/packages/@eventual/aws-runtime/src/clients/create.ts @@ -79,7 +79,6 @@ export const createWorkflowRuntimeClient = /*@__PURE__*/ memoize( lambda: lambda(), activityWorkerFunctionName: activityWorkerFunctionName ?? env.activityWorkerFunctionName(), - sqs: sqs(), workflowClient: createWorkflowClient(), }) ); From a3226f143b2ce1958e4fe57d9765d36d37cd6e64 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 23:05:15 -0800 Subject: [PATCH 15/39] fix: add context into workflow --- packages/@eventual/core/src/workflow.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index 756726740..5c6f7411c 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -30,7 +30,10 @@ export interface ExecutionHandle { * to other services in a durable and observable way. */ export interface Workflow< - F extends (...args: any[]) => any = (...args: any[]) => any + F extends (input: any, context: Context) => any = ( + input: any, + context: Context + ) => any > { /** * Globally unique ID of this {@link Workflow}. @@ -45,7 +48,7 @@ export interface Workflow< * * To start a workflow from another environment, use {@link start}. */ - (...args: Parameters): ReturnType; + (input: Parameters[0]): ReturnType; /** * Starts an execution of this {@link Workflow} without waiting for the response. * From bee8e8329ad2ad9019324f407b3384b2d1e7055d Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 23:05:40 -0800 Subject: [PATCH 16/39] fix: add context into workflow --- packages/@eventual/core/src/workflow.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index 5c6f7411c..d1900514e 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -89,10 +89,9 @@ export function lookupWorkflow(name: string): Workflow | undefined { * @param name a globally unique ID for this workflow. * @param definition the workflow definition. */ -export function workflow Promise | Program>( - name: string, - definition: F -): Workflow { +export function workflow< + F extends (input: any, context: Context) => Promise | Program +>(name: string, definition: F): Workflow { if (workflows.has(name)) { throw new Error(`workflow with name '${name}' already exists`); } From 22cd195bde256b5bb9ea3edae8ac77fcbf8825cf Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 23:07:06 -0800 Subject: [PATCH 17/39] docs --- packages/@eventual/compiler/src/esbuild-plugin.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/@eventual/compiler/src/esbuild-plugin.ts b/packages/@eventual/compiler/src/esbuild-plugin.ts index 7c6078aa4..ca4836688 100644 --- a/packages/@eventual/compiler/src/esbuild-plugin.ts +++ b/packages/@eventual/compiler/src/esbuild-plugin.ts @@ -60,12 +60,16 @@ class OuterVisitor extends Visitor { if (isWorkflowCall(call)) { this.foundEventual = true; + // workflow("id", async () => { .. }) return { ...call, arguments: [ + // workflow name, e.g. "id" call.arguments[0], { spread: call.arguments[1].spread, + // transform the function into a generator + // e.g. async () => { .. } becomes function*() { .. } expression: this.inner.visitWorkflow(call.arguments[1].expression), }, ], From ffb24b577bb725e02fec369026b1b75ee9e96320 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 23:10:04 -0800 Subject: [PATCH 18/39] feat: workflow handler type --- packages/@eventual/core/src/workflow.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index d1900514e..865d46a0b 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -25,16 +25,16 @@ export interface ExecutionHandle { executionId: string; } +export type WorkflowHandler = ( + input: any, + context: Context +) => Promise | Program; + /** * A {@link Workflow} is a long-running process that orchestrates calls * to other services in a durable and observable way. */ -export interface Workflow< - F extends (input: any, context: Context) => any = ( - input: any, - context: Context - ) => any -> { +export interface Workflow { /** * Globally unique ID of this {@link Workflow}. */ @@ -89,9 +89,10 @@ export function lookupWorkflow(name: string): Workflow | undefined { * @param name a globally unique ID for this workflow. * @param definition the workflow definition. */ -export function workflow< - F extends (input: any, context: Context) => Promise | Program ->(name: string, definition: F): Workflow { +export function workflow( + name: string, + definition: F +): Workflow { if (workflows.has(name)) { throw new Error(`workflow with name '${name}' already exists`); } From d6d2c91b34537cf5e44e69e5b5f0b9855cf056e9 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 23:13:57 -0800 Subject: [PATCH 19/39] fix tests --- packages/@eventual/core/src/workflow.ts | 17 ------------- .../@eventual/core/test/interpret.test.ts | 24 ++++++++++++++----- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index 865d46a0b..45f86990e 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -18,13 +18,6 @@ import { } from "./events.js"; import { interpret, WorkflowResult } from "./interpret.js"; -export interface ExecutionHandle { - /** - * ID of the workflow execution. - */ - executionId: string; -} - export type WorkflowHandler = ( input: any, context: Context @@ -49,12 +42,6 @@ export interface Workflow { * To start a workflow from another environment, use {@link start}. */ (input: Parameters[0]): ReturnType; - /** - * Starts an execution of this {@link Workflow} without waiting for the response. - * - * @returns a {@link ExecutionHandle} with the `executionId`. - */ - start(...args: Parameters): Promise; /** * @internal - this is the internal DSL representation that produces a {@link Program} instead of a Promise. */ @@ -103,10 +90,6 @@ export function workflow( input, })) as any; - workflow.start = async function (..._args: Parameters) { - // TODO: get a client and submit execution - throw new Error("not implemented"); - }; workflow.definition = definition as Workflow["definition"]; // safe to cast because we rely on transformer (it is always the generator API) workflows.set(name, workflow); return workflow; diff --git a/packages/@eventual/core/test/interpret.test.ts b/packages/@eventual/core/test/interpret.test.ts index b1a9832ed..28767d854 100644 --- a/packages/@eventual/core/test/interpret.test.ts +++ b/packages/@eventual/core/test/interpret.test.ts @@ -17,6 +17,7 @@ import { ChildWorkflowScheduled, ChildWorkflowCompleted, ChildWorkflowFailed, + Context, } from "../src/index.js"; import { DeterminismError } from "../src/error.js"; import { chain } from "../src/chain.js"; @@ -432,18 +433,29 @@ test("workflow calling other workflow", () => { return result; }); - expect(interpret(wf2.definition(), [])).toMatchObject({ + const context: Context = { + workflow: { + name: "wf1", + }, + execution: { + id: "123", + name: "wf1#123", + startTime: "", + }, + }; + + expect(interpret(wf2.definition(undefined, context), [])).toMatchObject({ commands: [createScheduledWorkflowCommand("wf1", undefined, 0)], }); expect( - interpret(wf2.definition(), [workflowScheduled("wf1", 0)]) + interpret(wf2.definition(undefined, context), [workflowScheduled("wf1", 0)]) ).toMatchObject({ commands: [], }); expect( - interpret(wf2.definition(), [ + interpret(wf2.definition(undefined, context), [ workflowScheduled("wf1", 0), workflowCompleted("result", 0), ]) @@ -452,7 +464,7 @@ test("workflow calling other workflow", () => { }); expect( - interpret(wf2.definition(), [ + interpret(wf2.definition(undefined, context), [ workflowScheduled("wf1", 0), workflowCompleted("result", 0), activityScheduled("call-b", 1), @@ -462,7 +474,7 @@ test("workflow calling other workflow", () => { }); expect( - interpret(wf2.definition(), [ + interpret(wf2.definition(undefined, context), [ workflowScheduled("wf1", 0), workflowCompleted("result", 0), activityScheduled("call-b", 1), @@ -474,7 +486,7 @@ test("workflow calling other workflow", () => { }); expect( - interpret(wf2.definition(), [ + interpret(wf2.definition(undefined, context), [ workflowScheduled("wf1", 0), workflowFailed("error", 0), ]) From 9ba9559f026e3cb8f4ab777269e21af5560eee26 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 22 Nov 2022 23:16:24 -0800 Subject: [PATCH 20/39] chore --- packages/@eventual/core/src/workflow.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index 45f86990e..79f19d591 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -46,7 +46,8 @@ export interface Workflow { * @internal - this is the internal DSL representation that produces a {@link Program} instead of a Promise. */ definition: ( - ...args: Parameters + input: Parameters[0], + context: Context ) => Program>>; } From feb6f0217b0ac0c3e5cb42364115de9ca8302181 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 23 Nov 2022 01:01:27 -0800 Subject: [PATCH 21/39] fix: --- .../src/clients/workflow-runtime-client.ts | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index ace1078d1..820cffff1 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -1,5 +1,4 @@ import { - AttributeValue, DynamoDBClient, QueryCommand, UpdateItemCommand, @@ -126,9 +125,9 @@ export class WorkflowRuntimeClient { const record = executionResult.Attributes as unknown as ExecutionRecord; if (record.parentExecutionId) { - await this.completeChildExecution( - record.parentExecutionId, - record.seq, + await this.reportCompletionToParent( + record.parentExecutionId.S, + record.seq.N, result ); } @@ -167,9 +166,9 @@ export class WorkflowRuntimeClient { const record = executionResult.Attributes as unknown as ExecutionRecord; if (record.parentExecutionId) { - await this.completeChildExecution( - record.parentExecutionId, - record.seq, + await this.reportCompletionToParent( + record.parentExecutionId.S, + record.seq.N, error, message ); @@ -178,13 +177,13 @@ export class WorkflowRuntimeClient { return createExecutionFromResult(record) as FailedExecution; } - private async completeChildExecution( - parentExecutionId: AttributeValue.SMember, - seq: AttributeValue.NMember, + private async reportCompletionToParent( + parentExecutionId: string, + seq: string, ...args: [result: any] | [error: string, message: string] ) { - await this.props.workflowClient.submitWorkflowTask(parentExecutionId.S, { - seq: parseInt(seq.N, 10), + await this.props.workflowClient.submitWorkflowTask(parentExecutionId, { + seq: parseInt(seq, 10), timestamp: new Date().toISOString(), ...(args.length === 1 ? { From 9382151b98062103cee6e321c1719aee43eca015 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 23 Nov 2022 01:36:26 -0800 Subject: [PATCH 22/39] bug still exists --- apps/test-app/package.json | 1 + package.json | 1 + .../aws-runtime/src/clients/workflow-runtime-client.ts | 1 + packages/@eventual/aws-runtime/src/handlers/orchestrator.ts | 1 + turbo.json | 4 ++++ 5 files changed, 8 insertions(+) diff --git a/apps/test-app/package.json b/apps/test-app/package.json index cf34c26fb..572c58261 100644 --- a/apps/test-app/package.json +++ b/apps/test-app/package.json @@ -6,6 +6,7 @@ "build": "cdk synth", "cdk": "cdk", "deploy": "cdk deploy --require-approval=never", + "hotswap": "cdk deploy --hotswap", "eventual": "eventual", "start-my-workflow": "eventual start my-workflow --input '{\"name\": \"world\"}' --tail" }, diff --git a/package.json b/package.json index 412c83f41..fed204c2f 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "clean:deep": "git clean -fqdx .", "clean:tsbuildinfo": "find . -name tsconfig.tsbuildinfo -exec rm {} \\;", "deploy": "turbo run deploy", + "hotswap": "turbo run hotswap", "dev": "turbo run dev --parallel", "lint": "turbo run lint", "prepare": "husky install", diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index 820cffff1..92247239e 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -100,6 +100,7 @@ export class WorkflowRuntimeClient { executionId, result, }: CompleteExecutionRequest): Promise { + console.log("COMPLETE", { executionId, result }); const executionResult = await this.props.dynamo.send( new UpdateItemCommand({ Key: { diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index 119ac33cc..bf0d3757b 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -122,6 +122,7 @@ async function orchestrateExecution( executionId: string, records: SQSRecord[] ) { + console.log(executionId, records); const executionLogger = logger.createChild({ persistentLogAttributes: { executionId }, }); diff --git a/turbo.json b/turbo.json index 7156901ab..8cc363589 100644 --- a/turbo.json +++ b/turbo.json @@ -34,6 +34,10 @@ "dependsOn": ["^export", "^build"], "cache": false }, + "hotswap": { + "dependsOn": ["^export", "^build"], + "cache": false + }, "synth": { "dependsOn": ["^export"] } From 67aaec7b43facb3da16740ada342a4e2b457d3a2 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 23 Nov 2022 03:40:56 -0800 Subject: [PATCH 23/39] feat: barebones of cloudflare --- apps/test-app-cloudflare/package.json | 17 + apps/test-app-cloudflare/src/index.ts | 30 + apps/test-app-cloudflare/tsconfig.json | 15 + apps/test-app-cloudflare/wrangler.toml | 3 + apps/test-app/package.json | 1 - .../cloudflare-runtime/CONTRIBUTING.MD | 8 + .../@eventual/cloudflare-runtime/README.md | 1 + .../@eventual/cloudflare-runtime/package.json | 76 ++ .../@eventual/cloudflare-runtime/src/index.ts | 1 + .../cloudflare-runtime/src/orchestrator.ts | 1 + .../cloudflare-runtime/tsconfig.cjs.json | 20 + .../cloudflare-runtime/tsconfig.json | 21 + .../cloudflare-runtime/tsconfig.test.json | 17 + pnpm-lock.yaml | 776 ++++++++++++++++++ tsconfig.json | 3 + 15 files changed, 989 insertions(+), 1 deletion(-) create mode 100644 apps/test-app-cloudflare/package.json create mode 100644 apps/test-app-cloudflare/src/index.ts create mode 100644 apps/test-app-cloudflare/tsconfig.json create mode 100644 apps/test-app-cloudflare/wrangler.toml create mode 100644 packages/@eventual/cloudflare-runtime/CONTRIBUTING.MD create mode 100644 packages/@eventual/cloudflare-runtime/README.md create mode 100644 packages/@eventual/cloudflare-runtime/package.json create mode 100644 packages/@eventual/cloudflare-runtime/src/index.ts create mode 100644 packages/@eventual/cloudflare-runtime/src/orchestrator.ts create mode 100644 packages/@eventual/cloudflare-runtime/tsconfig.cjs.json create mode 100644 packages/@eventual/cloudflare-runtime/tsconfig.json create mode 100644 packages/@eventual/cloudflare-runtime/tsconfig.test.json diff --git a/apps/test-app-cloudflare/package.json b/apps/test-app-cloudflare/package.json new file mode 100644 index 000000000..44710a7e1 --- /dev/null +++ b/apps/test-app-cloudflare/package.json @@ -0,0 +1,17 @@ +{ + "name": "test-app-cloudflare", + "version": "0.0.0", + "dependencies": { + "@eventual/cloudflare-runtime": "workspace:^" + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20221111.1", + "typescript": "^4.9.3", + "wrangler": "2.4.4" + }, + "private": true, + "scripts": { + "start": "wrangler dev", + "deploy": "wrangler publish" + } +} diff --git a/apps/test-app-cloudflare/src/index.ts b/apps/test-app-cloudflare/src/index.ts new file mode 100644 index 000000000..c13ce44af --- /dev/null +++ b/apps/test-app-cloudflare/src/index.ts @@ -0,0 +1,30 @@ +/** + * Welcome to Cloudflare Workers! This is your first worker. + * + * - Run `wrangler dev src/index.ts` in your terminal to start a development server + * - Open a browser tab at http://localhost:8787/ to see your worker in action + * - Run `wrangler publish src/index.ts --name my-worker` to publish your worker + * + * Learn more at https://developers.cloudflare.com/workers/ + */ + +export interface Env { + // Example binding to KV. Learn more at https://developers.cloudflare.com/workers/runtime-apis/kv/ + // MY_KV_NAMESPACE: KVNamespace; + // + // Example binding to Durable Object. Learn more at https://developers.cloudflare.com/workers/runtime-apis/durable-objects/ + // MY_DURABLE_OBJECT: DurableObjectNamespace; + // + // Example binding to R2. Learn more at https://developers.cloudflare.com/workers/runtime-apis/r2/ + // MY_BUCKET: R2Bucket; +} + +export default { + async fetch( + request: Request, + env: Env, + ctx: ExecutionContext + ): Promise { + return new Response("Hello World!"); + }, +}; diff --git a/apps/test-app-cloudflare/tsconfig.json b/apps/test-app-cloudflare/tsconfig.json new file mode 100644 index 000000000..6007cf593 --- /dev/null +++ b/apps/test-app-cloudflare/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig-base", + "compilerOptions": { + "outDir": "lib", + "declaration": true, + "inlineSourceMap": true, + "rootDir": "src", + "types": ["@cloudflare/workers-types"], + "typeRoots": ["./node_modules/@types"], + "allowJs": true + }, + "include": ["src"], + "exclude": ["lib", "node_modules"], + "references": [{ "path": "../../packages/@eventual/core" }] +} diff --git a/apps/test-app-cloudflare/wrangler.toml b/apps/test-app-cloudflare/wrangler.toml new file mode 100644 index 000000000..074fb298f --- /dev/null +++ b/apps/test-app-cloudflare/wrangler.toml @@ -0,0 +1,3 @@ +name = "test-app-cloudflare" +main = "src/index.ts" +compatibility_date = "2022-11-23" diff --git a/apps/test-app/package.json b/apps/test-app/package.json index 572c58261..12d867312 100644 --- a/apps/test-app/package.json +++ b/apps/test-app/package.json @@ -3,7 +3,6 @@ "version": "0.0.0", "main": "lib/index.js", "scripts": { - "build": "cdk synth", "cdk": "cdk", "deploy": "cdk deploy --require-approval=never", "hotswap": "cdk deploy --hotswap", diff --git a/packages/@eventual/cloudflare-runtime/CONTRIBUTING.MD b/packages/@eventual/cloudflare-runtime/CONTRIBUTING.MD new file mode 100644 index 000000000..c1bf38e4c --- /dev/null +++ b/packages/@eventual/cloudflare-runtime/CONTRIBUTING.MD @@ -0,0 +1,8 @@ +## ESM + +This package supports both ESM and CommonJS. + +1. Have separate `tsconfig*.json` files, one for esm and one for cjs. These output to `lib/esm` and `lib/cjs` respectively. +2. Define `exports` in `package.json` to allow `esbuild` and other tools that support `conditions` to pick the right path. +3. Define `main` and `module` in `package.json` to allow for legacy tools (ex: older versions of `typescript`) to find the right path. +4. Place a `package.json` with `type:module` set into `lib/esm` to support node natively running an esm module. Node only seems to use the `type` field, unlike typescript and esbuild when running. \ No newline at end of file diff --git a/packages/@eventual/cloudflare-runtime/README.md b/packages/@eventual/cloudflare-runtime/README.md new file mode 100644 index 000000000..081fdf4c2 --- /dev/null +++ b/packages/@eventual/cloudflare-runtime/README.md @@ -0,0 +1 @@ +# Eventual \ No newline at end of file diff --git a/packages/@eventual/cloudflare-runtime/package.json b/packages/@eventual/cloudflare-runtime/package.json new file mode 100644 index 000000000..036a8a6c4 --- /dev/null +++ b/packages/@eventual/cloudflare-runtime/package.json @@ -0,0 +1,76 @@ +{ + "name": "@eventual/cloudflare-runtime", + "exports": { + ".": { + "import": "./lib/esm/index.js", + "require": "./lib/cjs/index.js" + } + }, + "main": "./lib/cjs/index.js", + "module": "./lib/esm/index.js", + "version": "0.0.0", + "scripts": { + "test": "jest --passWithNoTests", + "start": "wrangler dev", + "deploy": "wrangler publish" + }, + "dependencies": { + "@aws-lambda-powertools/logger": "^1.4.1", + "@eventual/core": "workspace:^", + "@middy/core": "^3.6.2", + "@middy/error-logger": "^3.6.2", + "aws-embedded-metrics": "^4.0.0", + "aws-lambda": "^1.0.7", + "fast-equals": "^4.0.3", + "micro-memoize": "^4.0.11", + "ulidx": "^0.3.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.208.0", + "@aws-sdk/client-lambda": "^3.208.0", + "@aws-sdk/client-s3": "^3.208.0", + "@aws-sdk/client-sqs": "^3.208.0", + "@aws-sdk/client-scheduler": "^3.208.0", + "@types/aws-lambda": "8.10.108" + }, + "devDependencies": { + "@aws-sdk/client-dynamodb": "3.214.0", + "@aws-sdk/client-lambda": "^3.213.0", + "@aws-sdk/client-s3": "3.213.0", + "@aws-sdk/client-scheduler": "3.213.0", + "@aws-sdk/client-sqs": "3.213.0", + "@cloudflare/workers-types": "^4.20221111.1", + "@types/aws-lambda": "8.10.108", + "@types/jest": "^29", + "@types/node": "^16", + "jest": "^29", + "ts-jest": "^29", + "ts-node": "^10.9.1", + "typescript": "^4.9.3", + "wrangler": "^2.4.4" + }, + "jest": { + "extensionsToTreatAsEsm": [ + ".ts" + ], + "roots": [ + "/src/" + ], + "transform": { + "^.+\\.tsx?$": [ + "ts-jest", + { + "tsconfig": "tsconfig.test.json", + "useESM": true + } + ] + }, + "moduleNameMapper": { + "@eventual/injected/(.*)": "/src/injected/$1", + "^(\\.{1,2}/.*)\\.js$": "$1" + } + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/@eventual/cloudflare-runtime/src/index.ts b/packages/@eventual/cloudflare-runtime/src/index.ts new file mode 100644 index 000000000..29dd4e0f4 --- /dev/null +++ b/packages/@eventual/cloudflare-runtime/src/index.ts @@ -0,0 +1 @@ +export * from "./orchestrator"; diff --git a/packages/@eventual/cloudflare-runtime/src/orchestrator.ts b/packages/@eventual/cloudflare-runtime/src/orchestrator.ts new file mode 100644 index 000000000..50fd308c7 --- /dev/null +++ b/packages/@eventual/cloudflare-runtime/src/orchestrator.ts @@ -0,0 +1 @@ +export const TODO = "TODO"; diff --git a/packages/@eventual/cloudflare-runtime/tsconfig.cjs.json b/packages/@eventual/cloudflare-runtime/tsconfig.cjs.json new file mode 100644 index 000000000..7d636905f --- /dev/null +++ b/packages/@eventual/cloudflare-runtime/tsconfig.cjs.json @@ -0,0 +1,20 @@ +{ + "extends": "../../../tsconfig-base", + "compilerOptions": { + "module": "CommonJS", + "target": "ESNext", + "baseUrl": ".", + "outDir": "lib/cjs", + "declaration": true, + "inlineSourceMap": true, + "rootDir": "src", + "typeRoots": ["./node_modules/@types"], + "allowJs": true, + "paths": { + "@eventual/injected/*": ["./src/injected/*"] + } + }, + "include": ["src"], + "exclude": ["lib", "node_modules", "src/package.json"], + "references": [{ "path": "../core/tsconfig.cjs.json" }] +} diff --git a/packages/@eventual/cloudflare-runtime/tsconfig.json b/packages/@eventual/cloudflare-runtime/tsconfig.json new file mode 100644 index 000000000..535c28fe2 --- /dev/null +++ b/packages/@eventual/cloudflare-runtime/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../../tsconfig-base", + "compilerOptions": { + "module": "esnext", + "target": "esnext", + "moduleResolution": "node", + "baseUrl": ".", + "outDir": "lib/esm", + "declaration": true, + "inlineSourceMap": true, + "rootDir": "src", + "typeRoots": ["./node_modules/@types"], + "allowJs": true, + "paths": { + "@eventual/injected/*": ["./src/injected/*"] + } + }, + "include": ["src", "src/package.json"], + "exclude": ["lib", "node_modules"], + "references": [{ "path": "../core" }] +} diff --git a/packages/@eventual/cloudflare-runtime/tsconfig.test.json b/packages/@eventual/cloudflare-runtime/tsconfig.test.json new file mode 100644 index 000000000..5935e3a6a --- /dev/null +++ b/packages/@eventual/cloudflare-runtime/tsconfig.test.json @@ -0,0 +1,17 @@ +{ + "extends": "../../../tsconfig-base", + "compilerOptions": { + "module": "esnext", + "target": "esnext", + "moduleResolution": "node", + "declaration": true, + "inlineSourceMap": true, + "rootDir": ".", + "typeRoots": ["./node_modules/@types"], + "allowJs": true, + "noEmit": true + }, + "include": ["src", "test"], + "exclude": ["lib", "node_modules"], + "references": [{ "path": "../core" }] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 17a904c0c..e70dfce80 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,6 +53,19 @@ importers: ts-node: 10.9.1_fvpuwgkpfe3dm3hnpcpbcxmb3y typescript: 4.9.3 + apps/test-app-cloudflare: + specifiers: + '@cloudflare/workers-types': ^4.20221111.1 + '@eventual/cloudflare-runtime': workspace:^ + typescript: ^4.9.3 + wrangler: 2.4.4 + dependencies: + '@eventual/cloudflare-runtime': link:../../packages/@eventual/cloudflare-runtime + devDependencies: + '@cloudflare/workers-types': 4.20221111.1 + typescript: 4.9.3 + wrangler: 2.4.4 + apps/test-app-runtime: specifiers: '@aws-sdk/client-dynamodb': ^3.214.0 @@ -237,6 +250,57 @@ importers: ts-node: 10.9.1_72os6jwxu2zrt2v7mxnztv2e74 typescript: 4.9.3 + packages/@eventual/cloudflare-runtime: + specifiers: + '@aws-lambda-powertools/logger': ^1.4.1 + '@aws-sdk/client-dynamodb': 3.214.0 + '@aws-sdk/client-lambda': ^3.213.0 + '@aws-sdk/client-s3': 3.213.0 + '@aws-sdk/client-scheduler': 3.213.0 + '@aws-sdk/client-sqs': 3.213.0 + '@cloudflare/workers-types': ^4.20221111.1 + '@eventual/core': workspace:^ + '@middy/core': ^3.6.2 + '@middy/error-logger': ^3.6.2 + '@types/aws-lambda': 8.10.108 + '@types/jest': ^29 + '@types/node': ^16 + aws-embedded-metrics: ^4.0.0 + aws-lambda: ^1.0.7 + fast-equals: ^4.0.3 + jest: ^29 + micro-memoize: ^4.0.11 + ts-jest: ^29 + ts-node: ^10.9.1 + typescript: ^4.9.3 + ulidx: ^0.3.0 + wrangler: ^2.4.4 + dependencies: + '@aws-lambda-powertools/logger': 1.4.1 + '@eventual/core': link:../core + '@middy/core': 3.6.2 + '@middy/error-logger': 3.6.2 + aws-embedded-metrics: 4.0.0 + aws-lambda: 1.0.7 + fast-equals: 4.0.3 + micro-memoize: 4.0.11 + ulidx: 0.3.0 + devDependencies: + '@aws-sdk/client-dynamodb': 3.214.0 + '@aws-sdk/client-lambda': 3.213.0 + '@aws-sdk/client-s3': 3.213.0 + '@aws-sdk/client-scheduler': 3.213.0 + '@aws-sdk/client-sqs': 3.213.0 + '@cloudflare/workers-types': 4.20221111.1 + '@types/aws-lambda': 8.10.108 + '@types/jest': 29.2.2 + '@types/node': 16.18.3 + jest: 29.3.1_dnlfjp7n5lpfgnj4digwzn5fhe + ts-jest: 29.0.3_lg7llryyssof5ag2ezy5wawx7m + ts-node: 10.9.1_fvpuwgkpfe3dm3hnpcpbcxmb3y + typescript: 4.9.3 + wrangler: 2.4.4 + packages/@eventual/compiler: specifiers: '@eventual/aws-runtime': workspace:^ @@ -1932,6 +1996,16 @@ packages: resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} dev: true + /@cloudflare/kv-asset-handler/0.2.0: + resolution: {integrity: sha512-MVbXLbTcAotOPUj0pAMhVtJ+3/kFkwJqc5qNOleOZTv6QkZZABDMS21dSrSlVswEHwrpWC03e4fWytjqKvuE2A==} + dependencies: + mime: 3.0.0 + dev: true + + /@cloudflare/workers-types/4.20221111.1: + resolution: {integrity: sha512-BNV2wN8V6Zduvo7UzxcdjBbLQ906D2KhS804PDufLgx/sanGJCHVJMOIaLvS/b61JKtot1U7P/l1fjrjZ7/E3A==} + dev: true + /@cspotcode/source-map-support/0.8.1: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -1943,6 +2017,24 @@ packages: resolution: {integrity: sha512-Dl6MPPVXxzWsSQxIaV0sOpAx/B8r7RYUO5/GWe7GhG9v9P4QfZ1cgPSq+SoF0QJFhu9G9TmtPfRLHPWzL73GpQ==} dev: false + /@esbuild-plugins/node-globals-polyfill/0.1.1_esbuild@0.14.51: + resolution: {integrity: sha512-MR0oAA+mlnJWrt1RQVQ+4VYuRJW/P2YmRTv1AsplObyvuBMnPHiizUF95HHYiSsMGLhyGtWufaq2XQg6+iurBg==} + peerDependencies: + esbuild: '*' + dependencies: + esbuild: 0.14.51 + dev: true + + /@esbuild-plugins/node-modules-polyfill/0.1.4_esbuild@0.14.51: + resolution: {integrity: sha512-uZbcXi0zbmKC/050p3gJnne5Qdzw8vkXIv+c2BW0Lsc1ji1SkrxbKPUy5Efr0blbTu1SL8w4eyfpnSdPg3G0Qg==} + peerDependencies: + esbuild: '*' + dependencies: + esbuild: 0.14.51 + escape-string-regexp: 4.0.0 + rollup-plugin-node-polyfills: 0.2.1 + dev: true + /@esbuild/android-arm/0.15.14: resolution: {integrity: sha512-+Rb20XXxRGisNu2WmNKk+scpanb7nL5yhuI1KR9wQFiC43ddPj/V1fmNyzlFC9bKiG4mYzxW7egtoHVcynr+OA==} engines: {node: '>=12'} @@ -1968,6 +2060,10 @@ packages: engines: {node: '>=6.9.0'} dev: true + /@iarna/toml/2.2.5: + resolution: {integrity: sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==} + dev: true + /@isaacs/string-locale-compare/1.1.0: resolution: {integrity: sha512-SQ7Kzhh9+D+ZW9MA0zkYv3VXhIDNx+LzM6EJ+/65I3QY+enU6Itte7E5XX7EWrqLW2FN4n06GWzBnPoC3th2aQ==} dev: true @@ -3036,6 +3132,177 @@ packages: engines: {node: '>=14'} dev: false + /@miniflare/cache/2.10.0: + resolution: {integrity: sha512-nzEqFVPnD7Yf0HMDv7gCPpf4NSXfjhc+zg3gSwUS4Dad5bWV10B1ujTZW6HxQulW3CBHIg616mTjXIiaimVuEQ==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/core': 2.10.0 + '@miniflare/shared': 2.10.0 + http-cache-semantics: 4.1.0 + undici: 5.9.1 + dev: true + + /@miniflare/cli-parser/2.10.0: + resolution: {integrity: sha512-NAiCtqlHTUKCmV+Jl9af+ixGmMhiGhIyIfr/vCdbismNEBxEsrQGg3sQYTNfvCkdHtODurQqayQreFq21OuEow==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/shared': 2.10.0 + kleur: 4.1.5 + dev: true + + /@miniflare/core/2.10.0: + resolution: {integrity: sha512-Jx1M5oXQua0jzsJVdZSq07baVRmGC/6JkglrPQGAlZ7gQ1sunVZzq9fjxFqj0bqfEuYS0Wy6+lvK4rOAHISIjw==} + engines: {node: '>=16.13'} + dependencies: + '@iarna/toml': 2.2.5 + '@miniflare/queues': 2.10.0 + '@miniflare/shared': 2.10.0 + '@miniflare/watcher': 2.10.0 + busboy: 1.6.0 + dotenv: 10.0.0 + kleur: 4.1.5 + set-cookie-parser: 2.5.1 + undici: 5.9.1 + urlpattern-polyfill: 4.0.3 + dev: true + + /@miniflare/d1/2.10.0: + resolution: {integrity: sha512-mOYZSmpTthH0tmFTQ+O9G0Q+iDAd7oiUtoIBianlKa9QiqYAoO7EBUPy6kUgDHXapOcN5Ri1u3J5UTpxXvw3qg==} + engines: {node: '>=16.7'} + dependencies: + '@miniflare/core': 2.10.0 + '@miniflare/shared': 2.10.0 + dev: true + + /@miniflare/durable-objects/2.10.0: + resolution: {integrity: sha512-gU45f52gveFtCasm0ixYnt0mHI1lHrPomtmF+89oZGKBzOqUfO5diDs6wmoRSnovOWZCwtmwQGRoorAQN7AmoA==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/core': 2.10.0 + '@miniflare/shared': 2.10.0 + '@miniflare/storage-memory': 2.10.0 + undici: 5.9.1 + dev: true + + /@miniflare/html-rewriter/2.10.0: + resolution: {integrity: sha512-hCdG99L8+Ros4dn3B5H37PlQPBH0859EoRslzNTd4jzGIkwdiawpJvrvesL8056GjbUjeJN1zh7OPBRuMgyGLw==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/core': 2.10.0 + '@miniflare/shared': 2.10.0 + html-rewriter-wasm: 0.4.1 + undici: 5.9.1 + dev: true + + /@miniflare/http-server/2.10.0: + resolution: {integrity: sha512-cm6hwkONucll93yoY8dteMp//Knvmb7n6zAgeHrtuNYKn//lAL6bRY//VLTttrMmfWxZFi1C7WpOeCv8Mn6/ug==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/core': 2.10.0 + '@miniflare/shared': 2.10.0 + '@miniflare/web-sockets': 2.10.0 + kleur: 4.1.5 + selfsigned: 2.1.1 + undici: 5.9.1 + ws: 8.11.0 + youch: 2.2.2 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: true + + /@miniflare/kv/2.10.0: + resolution: {integrity: sha512-3+u1lO77FnlS0lQ6b1VgM1E/ZgQ/zy/FU+SdBG5LUOIiv3x522VYHOApeJLnSEo0KtZUB22Ni0fWQM6DgpaREg==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/shared': 2.10.0 + dev: true + + /@miniflare/queues/2.10.0: + resolution: {integrity: sha512-WKdO6qI9rfS96KlCjazzPFf+qj6DPov4vONyf18+jzbRjRJh/xwWSk1/1h5A+gDPwVNG8TsNRPh9DW5OKBGNjw==} + engines: {node: '>=16.7'} + dependencies: + '@miniflare/shared': 2.10.0 + dev: true + + /@miniflare/r2/2.10.0: + resolution: {integrity: sha512-uC1CCWbwM1t8DdpZgrveg6+CkZLfTq+wUMqs20BC5rCT8u8UyRv6ZVRQ7pTPiswLyt1oYDTXsZJK7tjV0U0zew==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/shared': 2.10.0 + undici: 5.9.1 + dev: true + + /@miniflare/runner-vm/2.10.0: + resolution: {integrity: sha512-oTsHitQdQ1B1kT3G/6n9AEXsMd/sT1D8tLGzc7Xr79ZrxYxwRO0ATF3cdkxk4dUjUqg/RUqvOJV4YjJGyqvctg==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/shared': 2.10.0 + dev: true + + /@miniflare/scheduler/2.10.0: + resolution: {integrity: sha512-eGt2cZFE/yo585nT8xINQwdbTotZfeRIh6FUWmZkbva1i5SW0zTiOojr5a95vAGBF3TzwWGsUuzJpLhBB69a/g==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/core': 2.10.0 + '@miniflare/shared': 2.10.0 + cron-schedule: 3.0.6 + dev: true + + /@miniflare/shared/2.10.0: + resolution: {integrity: sha512-GDSweEhJ3nNtStGm6taZGUNytM0QTQ/sjZSedAKyF1/aHRaZUcD9cuKAMgIbSpKfvgGdLMNS7Bhd8jb249TO7g==} + engines: {node: '>=16.13'} + dependencies: + '@types/better-sqlite3': 7.6.2 + kleur: 4.1.5 + npx-import: 1.1.4 + picomatch: 2.3.1 + dev: true + + /@miniflare/sites/2.10.0: + resolution: {integrity: sha512-1NVAT6+JS2OubL+pOOR5E/6MMddxQHWMi/yIDSumyyfXmj7Sm7n5dE1FvNPetggMP4f8+AjoyT9AYvdd1wkspQ==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/kv': 2.10.0 + '@miniflare/shared': 2.10.0 + '@miniflare/storage-file': 2.10.0 + dev: true + + /@miniflare/storage-file/2.10.0: + resolution: {integrity: sha512-K/cRIWiTl4+Z+VO6tl4VfuYXA3NLJgvGPV+BCRYD7uTKuPYHqDMErtD1BI1I7nc3WJhwIXfzJrAR3XXhSKKWQQ==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/shared': 2.10.0 + '@miniflare/storage-memory': 2.10.0 + dev: true + + /@miniflare/storage-memory/2.10.0: + resolution: {integrity: sha512-ZATU+qZtJ9yG0umgTrOEUi9SU//YyDb8nYXMgqT4JHODYA3RTz1SyyiQSOOz589upJPdu1LN+0j8W24WGRwwxQ==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/shared': 2.10.0 + dev: true + + /@miniflare/watcher/2.10.0: + resolution: {integrity: sha512-X9CFYYyszfSYDzs07KhbWC2i08Dpyh3D60fPonYZcoZAfa5h9eATHUdRGvNCdax7awYp4b8bvU8upAI//OPlMg==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/shared': 2.10.0 + dev: true + + /@miniflare/web-sockets/2.10.0: + resolution: {integrity: sha512-W+PrapdQqNEEFeD+amENgPQWcETGDp7OEh6JAoSzCRhHA0OoMe8DG0xb5a5+2FjGW/J7FFKsv84wkURpmFT4dQ==} + engines: {node: '>=16.13'} + dependencies: + '@miniflare/core': 2.10.0 + '@miniflare/shared': 2.10.0 + undici: 5.9.1 + ws: 8.11.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: true + /@nodelib/fs.scandir/2.1.5: resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -3549,6 +3816,12 @@ packages: '@babel/types': 7.20.2 dev: true + /@types/better-sqlite3/7.6.2: + resolution: {integrity: sha512-RgmaapusqTq6IMAr4McMyAsC6RshYTCjXCnzwVV59WctUxC8bNPyUfT9t5F81lKcU41lLurhjqjoMHfauzfqGg==} + dependencies: + '@types/node': 18.11.9 + dev: true + /@types/fs-extra/9.0.13: resolution: {integrity: sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==} dependencies: @@ -3620,6 +3893,10 @@ packages: /@types/ps-tree/1.1.2: resolution: {integrity: sha512-ZREFYlpUmPQJ0esjxoG1fMvB2HNaD3z+mjqdSosZvd3RalncI9NEur73P8ZJz4YQdL64CmV1w0RuqoRUlhQRBw==} + /@types/stack-trace/0.0.29: + resolution: {integrity: sha512-TgfOX+mGY/NyNxJLIbDWrO9DjGoVSW9+aB8H2yy1fy32jsvxijhmyJI9fDFgvz3YP4lvJaq9DzdR/M1bOgVc9g==} + dev: true + /@types/stack-utils/2.0.1: resolution: {integrity: sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==} dev: true @@ -4048,6 +4325,10 @@ packages: readable-stream: 3.6.0 dev: false + /blake3-wasm/2.1.5: + resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} + dev: true + /bowser/2.11.0: resolution: {integrity: sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==} @@ -4128,6 +4409,13 @@ packages: semver: 7.3.8 dev: true + /busboy/1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + dependencies: + streamsearch: 1.1.0 + dev: true + /byte-size/7.0.1: resolution: {integrity: sha512-crQdqyCwhokxwV1UyDzLZanhkugAgft7vt0qbbdt60C6Zf3CAiGmtUCylbtYwrU6loOUw3euGrNtW1J651ot1A==} engines: {node: '>=10'} @@ -4541,6 +4829,11 @@ packages: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} dev: true + /cookie/0.4.2: + resolution: {integrity: sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==} + engines: {node: '>= 0.6'} + dev: true + /core-util-is/1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} dev: true @@ -4560,6 +4853,10 @@ packages: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} dev: true + /cron-schedule/3.0.6: + resolution: {integrity: sha512-izfGgKyzzIyLaeb1EtZ3KbglkS6AKp9cv7LxmiyoOu+fXfol1tQDC0Cof0enVZGNtudTHW+3lfuW9ZkLQss4Wg==} + dev: true + /cross-spawn/7.0.3: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} @@ -4776,6 +5073,15 @@ packages: is-arrayish: 0.2.1 dev: true + /esbuild-android-64/0.14.51: + resolution: {integrity: sha512-6FOuKTHnC86dtrKDmdSj2CkcKF8PnqkaIXqvgydqfJmqBazCPdw+relrMlhGjkvVdiiGV70rpdnyFmA65ekBCQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + requiresBuild: true + dev: true + optional: true + /esbuild-android-64/0.15.14: resolution: {integrity: sha512-HuilVIb4rk9abT4U6bcFdU35UHOzcWVGLSjEmC58OVr96q5UiRqzDtWjPlCMugjhgUGKEs8Zf4ueIvYbOStbIg==} engines: {node: '>=12'} @@ -4784,6 +5090,15 @@ packages: requiresBuild: true optional: true + /esbuild-android-arm64/0.14.51: + resolution: {integrity: sha512-vBtp//5VVkZWmYYvHsqBRCMMi1MzKuMIn5XDScmnykMTu9+TD9v0NMEDqQxvtFToeYmojdo5UCV2vzMQWJcJ4A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + requiresBuild: true + dev: true + optional: true + /esbuild-android-arm64/0.15.14: resolution: {integrity: sha512-/QnxRVxsR2Vtf3XottAHj7hENAMW2wCs6S+OZcAbc/8nlhbAL/bCQRCVD78VtI5mdwqWkVi3wMqM94kScQCgqg==} engines: {node: '>=12'} @@ -4792,6 +5107,15 @@ packages: requiresBuild: true optional: true + /esbuild-darwin-64/0.14.51: + resolution: {integrity: sha512-YFmXPIOvuagDcwCejMRtCDjgPfnDu+bNeh5FU2Ryi68ADDVlWEpbtpAbrtf/lvFTWPexbgyKgzppNgsmLPr8PA==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + /esbuild-darwin-64/0.15.14: resolution: {integrity: sha512-ToNuf1uifu8hhwWvoZJGCdLIX/1zpo8cOGnT0XAhDQXiKOKYaotVNx7pOVB1f+wHoWwTLInrOmh3EmA7Fd+8Vg==} engines: {node: '>=12'} @@ -4800,6 +5124,15 @@ packages: requiresBuild: true optional: true + /esbuild-darwin-arm64/0.14.51: + resolution: {integrity: sha512-juYD0QnSKwAMfzwKdIF6YbueXzS6N7y4GXPDeDkApz/1RzlT42mvX9jgNmyOlWKN7YzQAYbcUEJmZJYQGdf2ow==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + /esbuild-darwin-arm64/0.15.14: resolution: {integrity: sha512-KgGP+y77GszfYJgceO0Wi/PiRtYo5y2Xo9rhBUpxTPaBgWDJ14gqYN0+NMbu+qC2fykxXaipHxN4Scaj9tUS1A==} engines: {node: '>=12'} @@ -4808,6 +5141,15 @@ packages: requiresBuild: true optional: true + /esbuild-freebsd-64/0.14.51: + resolution: {integrity: sha512-cLEI/aXjb6vo5O2Y8rvVSQ7smgLldwYY5xMxqh/dQGfWO+R1NJOFsiax3IS4Ng300SVp7Gz3czxT6d6qf2cw0g==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + requiresBuild: true + dev: true + optional: true + /esbuild-freebsd-64/0.15.14: resolution: {integrity: sha512-xr0E2n5lyWw3uFSwwUXHc0EcaBDtsal/iIfLioflHdhAe10KSctV978Te7YsfnsMKzcoGeS366+tqbCXdqDHQA==} engines: {node: '>=12'} @@ -4816,6 +5158,15 @@ packages: requiresBuild: true optional: true + /esbuild-freebsd-arm64/0.14.51: + resolution: {integrity: sha512-TcWVw/rCL2F+jUgRkgLa3qltd5gzKjIMGhkVybkjk6PJadYInPtgtUBp1/hG+mxyigaT7ib+od1Xb84b+L+1Mg==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + requiresBuild: true + dev: true + optional: true + /esbuild-freebsd-arm64/0.15.14: resolution: {integrity: sha512-8XH96sOQ4b1LhMlO10eEWOjEngmZ2oyw3pW4o8kvBcpF6pULr56eeYVP5radtgw54g3T8nKHDHYEI5AItvskZg==} engines: {node: '>=12'} @@ -4824,6 +5175,15 @@ packages: requiresBuild: true optional: true + /esbuild-linux-32/0.14.51: + resolution: {integrity: sha512-RFqpyC5ChyWrjx8Xj2K0EC1aN0A37H6OJfmUXIASEqJoHcntuV3j2Efr9RNmUhMfNE6yEj2VpYuDteZLGDMr0w==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + requiresBuild: true + dev: true + optional: true + /esbuild-linux-32/0.15.14: resolution: {integrity: sha512-6ssnvwaTAi8AzKN8By2V0nS+WF5jTP7SfuK6sStGnDP7MCJo/4zHgM9oE1eQTS2jPmo3D673rckuCzRlig+HMA==} engines: {node: '>=12'} @@ -4832,6 +5192,15 @@ packages: requiresBuild: true optional: true + /esbuild-linux-64/0.14.51: + resolution: {integrity: sha512-dxjhrqo5i7Rq6DXwz5v+MEHVs9VNFItJmHBe1CxROWNf4miOGoQhqSG8StStbDkQ1Mtobg6ng+4fwByOhoQoeA==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /esbuild-linux-64/0.15.14: resolution: {integrity: sha512-ONySx3U0wAJOJuxGUlXBWxVKFVpWv88JEv0NZ6NlHknmDd1yCbf4AEdClSgLrqKQDXYywmw4gYDvdLsS6z0hcw==} engines: {node: '>=12'} @@ -4840,6 +5209,15 @@ packages: requiresBuild: true optional: true + /esbuild-linux-arm/0.14.51: + resolution: {integrity: sha512-LsJynDxYF6Neg7ZC7748yweCDD+N8ByCv22/7IAZglIEniEkqdF4HCaa49JNDLw1UQGlYuhOB8ZT/MmcSWzcWg==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: true + optional: true + /esbuild-linux-arm/0.15.14: resolution: {integrity: sha512-D2LImAIV3QzL7lHURyCHBkycVFbKwkDb1XEUWan+2fb4qfW7qAeUtul7ZIcIwFKZgPcl+6gKZmvLgPSj26RQ2Q==} engines: {node: '>=12'} @@ -4848,6 +5226,15 @@ packages: requiresBuild: true optional: true + /esbuild-linux-arm64/0.14.51: + resolution: {integrity: sha512-D9rFxGutoqQX3xJPxqd6o+kvYKeIbM0ifW2y0bgKk5HPgQQOo2k9/2Vpto3ybGYaFPCE5qTGtqQta9PoP6ZEzw==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /esbuild-linux-arm64/0.15.14: resolution: {integrity: sha512-kle2Ov6a1e5AjlHlMQl1e+c4myGTeggrRzArQFmWp6O6JoqqB9hT+B28EW4tjFWgV/NxUq46pWYpgaWXsXRPAg==} engines: {node: '>=12'} @@ -4856,6 +5243,15 @@ packages: requiresBuild: true optional: true + /esbuild-linux-mips64le/0.14.51: + resolution: {integrity: sha512-vS54wQjy4IinLSlb5EIlLoln8buh1yDgliP4CuEHumrPk4PvvP4kTRIG4SzMXm6t19N0rIfT4bNdAxzJLg2k6A==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + requiresBuild: true + dev: true + optional: true + /esbuild-linux-mips64le/0.15.14: resolution: {integrity: sha512-FVdMYIzOLXUq+OE7XYKesuEAqZhmAIV6qOoYahvUp93oXy0MOVTP370ECbPfGXXUdlvc0TNgkJa3YhEwyZ6MRA==} engines: {node: '>=12'} @@ -4864,6 +5260,15 @@ packages: requiresBuild: true optional: true + /esbuild-linux-ppc64le/0.14.51: + resolution: {integrity: sha512-xcdd62Y3VfGoyphNP/aIV9LP+RzFw5M5Z7ja+zdpQHHvokJM7d0rlDRMN+iSSwvUymQkqZO+G/xjb4/75du8BQ==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /esbuild-linux-ppc64le/0.15.14: resolution: {integrity: sha512-2NzH+iuzMDA+jjtPjuIz/OhRDf8tzbQ1tRZJI//aT25o1HKc0reMMXxKIYq/8nSHXiJSnYV4ODzTiv45s+h73w==} engines: {node: '>=12'} @@ -4872,6 +5277,15 @@ packages: requiresBuild: true optional: true + /esbuild-linux-riscv64/0.14.51: + resolution: {integrity: sha512-syXHGak9wkAnFz0gMmRBoy44JV0rp4kVCEA36P5MCeZcxFq8+fllBC2t6sKI23w3qd8Vwo9pTADCgjTSf3L3rA==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /esbuild-linux-riscv64/0.15.14: resolution: {integrity: sha512-VqxvutZNlQxmUNS7Ac+aczttLEoHBJ9e3OYGqnULrfipRvG97qLrAv9EUY9iSrRKBqeEbSvS9bSfstZqwz0T4Q==} engines: {node: '>=12'} @@ -4880,6 +5294,15 @@ packages: requiresBuild: true optional: true + /esbuild-linux-s390x/0.14.51: + resolution: {integrity: sha512-kFAJY3dv+Wq8o28K/C7xkZk/X34rgTwhknSsElIqoEo8armCOjMJ6NsMxm48KaWY2h2RUYGtQmr+RGuUPKBhyw==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + requiresBuild: true + dev: true + optional: true + /esbuild-linux-s390x/0.15.14: resolution: {integrity: sha512-+KVHEUshX5n6VP6Vp/AKv9fZIl5kr2ph8EUFmQUJnDpHwcfTSn2AQgYYm0HTBR2Mr4d0Wlr0FxF/Cs5pbFgiOw==} engines: {node: '>=12'} @@ -4888,6 +5311,15 @@ packages: requiresBuild: true optional: true + /esbuild-netbsd-64/0.14.51: + resolution: {integrity: sha512-ZZBI7qrR1FevdPBVHz/1GSk1x5GDL/iy42Zy8+neEm/HA7ma+hH/bwPEjeHXKWUDvM36CZpSL/fn1/y9/Hb+1A==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + requiresBuild: true + dev: true + optional: true + /esbuild-netbsd-64/0.15.14: resolution: {integrity: sha512-6D/dr17piEgevIm1xJfZP2SjB9Z+g8ERhNnBdlZPBWZl+KSPUKLGF13AbvC+nzGh8IxOH2TyTIdRMvKMP0nEzQ==} engines: {node: '>=12'} @@ -4896,6 +5328,15 @@ packages: requiresBuild: true optional: true + /esbuild-openbsd-64/0.14.51: + resolution: {integrity: sha512-7R1/p39M+LSVQVgDVlcY1KKm6kFKjERSX1lipMG51NPcspJD1tmiZSmmBXoY5jhHIu6JL1QkFDTx94gMYK6vfA==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + requiresBuild: true + dev: true + optional: true + /esbuild-openbsd-64/0.15.14: resolution: {integrity: sha512-rREQBIlMibBetgr2E9Lywt2Qxv2ZdpmYahR4IUlAQ1Efv/A5gYdO0/VIN3iowDbCNTLxp0bb57Vf0LFcffD6kA==} engines: {node: '>=12'} @@ -4914,6 +5355,15 @@ packages: jsonfile: 6.1.0 dev: false + /esbuild-sunos-64/0.14.51: + resolution: {integrity: sha512-HoHaCswHxLEYN8eBTtyO0bFEWvA3Kdb++hSQ/lLG7TyKF69TeSG0RNoBRAs45x/oCeWaTDntEZlYwAfQlhEtJA==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + requiresBuild: true + dev: true + optional: true + /esbuild-sunos-64/0.15.14: resolution: {integrity: sha512-DNVjSp/BY4IfwtdUAvWGIDaIjJXY5KI4uD82+15v6k/w7px9dnaDaJJ2R6Mu+KCgr5oklmFc0KjBjh311Gxl9Q==} engines: {node: '>=12'} @@ -4922,6 +5372,15 @@ packages: requiresBuild: true optional: true + /esbuild-windows-32/0.14.51: + resolution: {integrity: sha512-4rtwSAM35A07CBt1/X8RWieDj3ZUHQqUOaEo5ZBs69rt5WAFjP4aqCIobdqOy4FdhYw1yF8Z0xFBTyc9lgPtEg==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: true + optional: true + /esbuild-windows-32/0.15.14: resolution: {integrity: sha512-pHBWrcA+/oLgvViuG9FO3kNPO635gkoVrRQwe6ZY1S0jdET07xe2toUvQoJQ8KT3/OkxqUasIty5hpuKFLD+eg==} engines: {node: '>=12'} @@ -4930,6 +5389,15 @@ packages: requiresBuild: true optional: true + /esbuild-windows-64/0.14.51: + resolution: {integrity: sha512-HoN/5HGRXJpWODprGCgKbdMvrC3A2gqvzewu2eECRw2sYxOUoh2TV1tS+G7bHNapPGI79woQJGV6pFH7GH7qnA==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /esbuild-windows-64/0.15.14: resolution: {integrity: sha512-CszIGQVk/P8FOS5UgAH4hKc9zOaFo69fe+k1rqgBHx3CSK3Opyk5lwYriIamaWOVjBt7IwEP6NALz+tkVWdFog==} engines: {node: '>=12'} @@ -4938,6 +5406,15 @@ packages: requiresBuild: true optional: true + /esbuild-windows-arm64/0.14.51: + resolution: {integrity: sha512-JQDqPjuOH7o+BsKMSddMfmVJXrnYZxXDHsoLHc0xgmAZkOOCflRmC43q31pk79F9xuyWY45jDBPolb5ZgGOf9g==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /esbuild-windows-arm64/0.15.14: resolution: {integrity: sha512-KW9W4psdZceaS9A7Jsgl4WialOznSURvqX/oHZk3gOP7KbjtHLSsnmSvNdzagGJfxbAe30UVGXRe8q8nDsOSQw==} engines: {node: '>=12'} @@ -4946,6 +5423,34 @@ packages: requiresBuild: true optional: true + /esbuild/0.14.51: + resolution: {integrity: sha512-+CvnDitD7Q5sT7F+FM65sWkF8wJRf+j9fPcprxYV4j+ohmzVj2W7caUqH2s5kCaCJAfcAICjSlKhDCcvDpU7nw==} + engines: {node: '>=12'} + hasBin: true + requiresBuild: true + optionalDependencies: + esbuild-android-64: 0.14.51 + esbuild-android-arm64: 0.14.51 + esbuild-darwin-64: 0.14.51 + esbuild-darwin-arm64: 0.14.51 + esbuild-freebsd-64: 0.14.51 + esbuild-freebsd-arm64: 0.14.51 + esbuild-linux-32: 0.14.51 + esbuild-linux-64: 0.14.51 + esbuild-linux-arm: 0.14.51 + esbuild-linux-arm64: 0.14.51 + esbuild-linux-mips64le: 0.14.51 + esbuild-linux-ppc64le: 0.14.51 + esbuild-linux-riscv64: 0.14.51 + esbuild-linux-s390x: 0.14.51 + esbuild-netbsd-64: 0.14.51 + esbuild-openbsd-64: 0.14.51 + esbuild-sunos-64: 0.14.51 + esbuild-windows-32: 0.14.51 + esbuild-windows-64: 0.14.51 + esbuild-windows-arm64: 0.14.51 + dev: true + /esbuild/0.15.14: resolution: {integrity: sha512-pJN8j42fvWLFWwSMG4luuupl2Me7mxciUOsMegKvwCmhEbJ2covUdFnihxm0FMIBV+cbwbtMoHgMCCI+pj1btQ==} engines: {node: '>=12'} @@ -4989,6 +5494,11 @@ packages: engines: {node: '>=8'} dev: true + /escape-string-regexp/4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + dev: true + /esprima/4.0.1: resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} engines: {node: '>=4'} @@ -5005,6 +5515,10 @@ packages: engines: {node: '>=4.0'} dev: true + /estree-walker/0.6.1: + resolution: {integrity: sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==} + dev: true + /event-stream/3.3.4: resolution: {integrity: sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==} dependencies: @@ -5525,6 +6039,10 @@ packages: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} dev: true + /html-rewriter-wasm/0.4.1: + resolution: {integrity: sha512-lNovG8CMCCmcVB1Q7xggMSf7tqPCijZXaH4gL6iE8BFghdQCbaY5Met9i1x2Ex8m/cZHDUtXK9H6/znKamRP8Q==} + dev: true + /http-cache-semantics/4.1.0: resolution: {integrity: sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==} dev: true @@ -6515,6 +7033,7 @@ packages: /js-yaml/4.1.0: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} + hasBin: true dependencies: argparse: 2.0.1 dev: true @@ -6542,6 +7061,7 @@ packages: /json5/1.0.1: resolution: {integrity: sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==} + hasBin: true dependencies: minimist: 1.2.7 dev: true @@ -6588,6 +7108,11 @@ packages: engines: {node: '>=6'} dev: true + /kleur/4.1.5: + resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} + engines: {node: '>=6'} + dev: true + /ky-universal/0.11.0_ky@0.32.2: resolution: {integrity: sha512-65KyweaWvk+uKKkCrfAf+xqN2/epw1IJDtlyCPxYffFCMR8u1sp2U65NtWpnozYfZxQ6IUzIlvUcw+hQ82U2Xw==} engines: {node: '>=14.16'} @@ -6832,6 +7357,12 @@ packages: engines: {node: '>=12'} dev: true + /magic-string/0.25.9: + resolution: {integrity: sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==} + dependencies: + sourcemap-codec: 1.4.8 + dev: true + /make-dir/2.1.0: resolution: {integrity: sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==} engines: {node: '>=6'} @@ -6943,6 +7474,12 @@ packages: mime-db: 1.52.0 dev: true + /mime/3.0.0: + resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==} + engines: {node: '>=10.0.0'} + hasBin: true + dev: true + /mimic-fn/2.1.0: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} @@ -6957,6 +7494,48 @@ packages: engines: {node: '>=4'} dev: true + /miniflare/2.10.0: + resolution: {integrity: sha512-WPveqChVDdmDGv+wFqXjFqEZlZ5/aBlAKX37h/e4TAjl2XsK5nPfQATP8jZXwNDEC5iE29bYZymOqeZkp+t7OA==} + engines: {node: '>=16.13'} + hasBin: true + peerDependencies: + '@miniflare/storage-redis': 2.10.0 + cron-schedule: ^3.0.4 + ioredis: ^4.27.9 + peerDependenciesMeta: + '@miniflare/storage-redis': + optional: true + cron-schedule: + optional: true + ioredis: + optional: true + dependencies: + '@miniflare/cache': 2.10.0 + '@miniflare/cli-parser': 2.10.0 + '@miniflare/core': 2.10.0 + '@miniflare/d1': 2.10.0 + '@miniflare/durable-objects': 2.10.0 + '@miniflare/html-rewriter': 2.10.0 + '@miniflare/http-server': 2.10.0 + '@miniflare/kv': 2.10.0 + '@miniflare/queues': 2.10.0 + '@miniflare/r2': 2.10.0 + '@miniflare/runner-vm': 2.10.0 + '@miniflare/scheduler': 2.10.0 + '@miniflare/shared': 2.10.0 + '@miniflare/sites': 2.10.0 + '@miniflare/storage-file': 2.10.0 + '@miniflare/storage-memory': 2.10.0 + '@miniflare/web-sockets': 2.10.0 + kleur: 4.1.5 + semiver: 1.1.0 + source-map-support: 0.5.21 + undici: 5.9.1 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: true + /minimatch/3.0.5: resolution: {integrity: sha512-tUpxzX0VAzJHjLu0xUfFv1gwVp9ba3IOuRAVH2EGuRW8a5emA2FlACLqiT/lDVtS1W+TGNwqz3sWaNyLgDJWuw==} dependencies: @@ -7091,10 +7670,21 @@ packages: minimatch: 3.1.2 dev: true + /mustache/4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + dev: true + /mute-stream/0.0.8: resolution: {integrity: sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==} dev: true + /nanoid/3.3.4: + resolution: {integrity: sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + dev: true + /natural-compare/1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} dev: true @@ -7145,6 +7735,11 @@ packages: formdata-polyfill: 4.0.10 dev: false + /node-forge/1.3.1: + resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} + engines: {node: '>= 6.13.0'} + dev: true + /node-gyp-build/4.5.0: resolution: {integrity: sha512-2iGbaQBV+ITgCz76ZEjmhUKAKVf7xfY1sRl4UiKQspfZMH2h06SyhNsnSVy50cwkFQDGLyif6m/6uFXHkOZ6rg==} dev: true @@ -7332,6 +7927,15 @@ packages: set-blocking: 2.0.0 dev: true + /npx-import/1.1.4: + resolution: {integrity: sha512-3ShymTWOgqGyNlh5lMJAejLuIv3W1K3fbI5Ewc6YErZU3Sp0PqsNs8UIU1O8z5+KVl/Du5ag56Gza9vdorGEoA==} + dependencies: + execa: 6.1.0 + parse-package-name: 1.0.0 + semver: 7.3.8 + validate-npm-package-name: 4.0.0 + dev: true + /nx/15.0.13: resolution: {integrity: sha512-5mJGWz91B9/sxzLjXdD+pmZTel54NeNNxFDis8OhtGDn6eRZ25qWsZNDgzqIDtwKn3c9gThAMHU4XH2OTgWUnA==} hasBin: true @@ -7618,6 +8222,10 @@ packages: lines-and-columns: 1.2.4 dev: true + /parse-package-name/1.0.0: + resolution: {integrity: sha512-kBeTUtcj+SkyfaW4+KBe0HtsloBJ/mKTPoxpVdA57GZiPerREsUWJOhVj9anXweFiJkm5y8FG1sxFZkZ0SN6wg==} + dev: true + /parse-path/7.0.0: resolution: {integrity: sha512-Euf9GG8WT9CdqwuWJGdf3RkUcTBArppHABkO7Lm8IzRQp0e2r/kkFnmhu4TSK30Wcu5rVAZLmfPKSBBi9tWFog==} dependencies: @@ -7658,6 +8266,10 @@ packages: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} dev: true + /path-to-regexp/6.2.1: + resolution: {integrity: sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==} + dev: true + /path-type/3.0.0: resolution: {integrity: sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==} engines: {node: '>=4'} @@ -8009,6 +8621,27 @@ packages: glob: 7.2.3 dev: true + /rollup-plugin-inject/3.0.2: + resolution: {integrity: sha512-ptg9PQwzs3orn4jkgXJ74bfs5vYz1NCZlSQMBUA0wKcGp5i5pA1AO3fOUEte8enhGUC+iapTCzEWw2jEFFUO/w==} + deprecated: This package has been deprecated and is no longer maintained. Please use @rollup/plugin-inject. + dependencies: + estree-walker: 0.6.1 + magic-string: 0.25.9 + rollup-pluginutils: 2.8.2 + dev: true + + /rollup-plugin-node-polyfills/0.2.1: + resolution: {integrity: sha512-4kCrKPTJ6sK4/gLL/U5QzVT8cxJcofO0OU74tnB19F40cmuAKSzH5/siithxlofFEjwvw1YAhPmbvGNA6jEroA==} + dependencies: + rollup-plugin-inject: 3.0.2 + dev: true + + /rollup-pluginutils/2.8.2: + resolution: {integrity: sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==} + dependencies: + estree-walker: 0.6.1 + dev: true + /run-async/2.4.1: resolution: {integrity: sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==} engines: {node: '>=0.12.0'} @@ -8040,6 +8673,18 @@ packages: resolution: {integrity: sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==} dev: false + /selfsigned/2.1.1: + resolution: {integrity: sha512-GSL3aowiF7wa/WtSFwnUrludWFoNhftq8bUkH9pkzjpN2XSPOAYEgg6e0sS9s0rZwgJzJiQRPU18A6clnoW5wQ==} + engines: {node: '>=10'} + dependencies: + node-forge: 1.3.1 + dev: true + + /semiver/1.1.0: + resolution: {integrity: sha512-QNI2ChmuioGC1/xjyYwyZYADILWyW6AmS1UH6gDj/SFUUUS4MBAWs/7mxnkRPc/F4iHezDP+O8t0dO8WHiEOdg==} + engines: {node: '>=6'} + dev: true + /semver/5.7.1: resolution: {integrity: sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==} hasBin: true @@ -8047,11 +8692,13 @@ packages: /semver/6.3.0: resolution: {integrity: sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==} + hasBin: true dev: true /semver/7.3.4: resolution: {integrity: sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==} engines: {node: '>=10'} + hasBin: true dependencies: lru-cache: 6.0.0 dev: true @@ -8066,6 +8713,10 @@ packages: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} dev: true + /set-cookie-parser/2.5.1: + resolution: {integrity: sha512-1jeBGaKNGdEq4FgIrORu/N570dwoPYio8lSoYLWmX7sQ//0JY08Xh9o5pBcgmHQ/MbsYp/aZnOe1s1lIsbLprQ==} + dev: true + /shallow-clone/3.0.1: resolution: {integrity: sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==} engines: {node: '>=8'} @@ -8172,11 +8823,27 @@ packages: source-map: 0.6.1 dev: true + /source-map-support/0.5.21: + resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} + dependencies: + buffer-from: 1.1.2 + source-map: 0.6.1 + dev: true + /source-map/0.6.1: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} dev: true + /source-map/0.7.4: + resolution: {integrity: sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==} + engines: {node: '>= 8'} + dev: true + + /sourcemap-codec/1.4.8: + resolution: {integrity: sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==} + dev: true + /spdx-correct/3.1.1: resolution: {integrity: sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==} dependencies: @@ -8226,6 +8893,10 @@ packages: minipass: 3.3.4 dev: true + /stack-trace/0.0.10: + resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==} + dev: true + /stack-utils/2.0.6: resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} engines: {node: '>=10'} @@ -8238,6 +8909,11 @@ packages: dependencies: duplexer: 0.1.2 + /streamsearch/1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + dev: true + /string-argv/0.3.1: resolution: {integrity: sha512-a1uQGz7IyVy9YwhqjZIZu1c8JO8dNIe20xBmSS6qu9kv++k3JGzCVmprbNN5Kn+BgzD5E7YYwg1CcjuJMRNsvg==} engines: {node: '>=0.6.19'} @@ -8531,6 +9207,41 @@ packages: yargs-parser: 21.1.1 dev: true + /ts-jest/29.0.3_lg7llryyssof5ag2ezy5wawx7m: + resolution: {integrity: sha512-Ibygvmuyq1qp/z3yTh9QTwVVAbFdDy/+4BtIQR2sp6baF2SJU/8CKK/hhnGIDY2L90Az2jIqTwZPnN2p+BweiQ==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + hasBin: true + peerDependencies: + '@babel/core': '>=7.0.0-beta.0 <8' + '@jest/types': ^29.0.0 + babel-jest: ^29.0.0 + esbuild: '*' + jest: ^29.0.0 + typescript: '>=4.3' + peerDependenciesMeta: + '@babel/core': + optional: true + '@jest/types': + optional: true + babel-jest: + optional: true + esbuild: + optional: true + dependencies: + '@babel/core': 7.20.2 + bs-logger: 0.2.6 + esbuild: 0.14.51 + fast-json-stable-stringify: 2.1.0 + jest: 29.3.1_dnlfjp7n5lpfgnj4digwzn5fhe + jest-util: 29.3.1 + json5: 2.2.1 + lodash.memoize: 4.1.2 + make-error: 1.3.6 + semver: 7.3.8 + typescript: 4.9.3 + yargs-parser: 21.1.1 + dev: true + /ts-node/10.9.1_72os6jwxu2zrt2v7mxnztv2e74: resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} hasBin: true @@ -8727,6 +9438,11 @@ packages: layerr: 0.1.2 dev: false + /undici/5.9.1: + resolution: {integrity: sha512-6fB3a+SNnWEm4CJbgo0/CWR8RGcOCQP68SF4X0mxtYTq2VNN8T88NYrWVBAeSX+zb7bny2dx2iYhP3XHi00omg==} + engines: {node: '>=12.18'} + dev: true + /unique-filename/2.0.1: resolution: {integrity: sha512-ODWHtkkdx3IAR+veKxFV+VBkUMcN+FaqzUUd7IZzt+0zhDZFPFxhlqwPF3YQvMHx1TD0tdgYl+kuPnJ8E6ql7A==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} @@ -8772,6 +9488,10 @@ packages: querystring: 0.2.0 dev: false + /urlpattern-polyfill/4.0.3: + resolution: {integrity: sha512-DOE84vZT2fEcl9gqCUTcnAw5ZY5Id55ikUcziSUntuEFL3pRvavg5kwDmTEUJkeCHInTlV/HexFomgYnzO5kdQ==} + dev: true + /util-deprecate/1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -8902,6 +9622,36 @@ packages: resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} dev: true + /wrangler/2.4.4: + resolution: {integrity: sha512-Of3O/7RzIcGWGmt7dd5JevvP419De55smr4Hi07REKt9oXYhMNeaFy4wX35fHlv5e0pVCyGB3Fna8mI4Ib2pew==} + engines: {node: '>=16.13.0'} + hasBin: true + dependencies: + '@cloudflare/kv-asset-handler': 0.2.0 + '@esbuild-plugins/node-globals-polyfill': 0.1.1_esbuild@0.14.51 + '@esbuild-plugins/node-modules-polyfill': 0.1.4_esbuild@0.14.51 + '@miniflare/core': 2.10.0 + '@miniflare/d1': 2.10.0 + '@miniflare/durable-objects': 2.10.0 + blake3-wasm: 2.1.5 + chokidar: 3.5.3 + esbuild: 0.14.51 + miniflare: 2.10.0 + nanoid: 3.3.4 + path-to-regexp: 6.2.1 + selfsigned: 2.1.1 + source-map: 0.7.4 + xxhash-wasm: 1.0.2 + optionalDependencies: + fsevents: 2.3.2 + transitivePeerDependencies: + - '@miniflare/storage-redis' + - bufferutil + - cron-schedule + - ioredis + - utf-8-validate + dev: true + /wrap-ansi/6.2.0: resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} engines: {node: '>=8'} @@ -8981,6 +9731,19 @@ packages: write-json-file: 3.2.0 dev: true + /ws/8.11.0: + resolution: {integrity: sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ^5.0.2 + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: true + /xml2js/0.4.19: resolution: {integrity: sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==} dependencies: @@ -8998,6 +9761,10 @@ packages: engines: {node: '>=0.4'} dev: true + /xxhash-wasm/1.0.2: + resolution: {integrity: sha512-ibF0Or+FivM9lNrg+HGJfVX8WJqgo+kCLDc4vx6xMeTce7Aj+DLttKbxxRR/gNLSAelRc1omAPlJ77N/Jem07A==} + dev: true + /y18n/5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -9061,6 +9828,15 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + /youch/2.2.2: + resolution: {integrity: sha512-/FaCeG3GkuJwaMR34GHVg0l8jCbafZLHiFowSjqLlqhC6OMyf2tPJBu8UirF7/NI9X/R5ai4QfEKUCOxMAGxZQ==} + dependencies: + '@types/stack-trace': 0.0.29 + cookie: 0.4.2 + mustache: 4.2.0 + stack-trace: 0.0.10 + dev: true + /zx/7.1.1: resolution: {integrity: sha512-5YlTO2AJ+Ku2YuZKSSSqnUKuagcM/f/j4LmHs15O84Ch80Z9gzR09ZK3gR7GV+rc8IFpz2H/XNFtFVmj31yrZA==} engines: {node: '>= 16.0.0'} diff --git a/tsconfig.json b/tsconfig.json index 94a92fd91..98862884d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,9 @@ { "files": [], "references": [ + { + "path": "packages/@eventual/cloudflare-runtime" + }, { "path": "packages/@eventual/compiler" }, From a8af1208419908a02142af33709af3989d806153 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 23 Nov 2022 12:27:26 -0800 Subject: [PATCH 24/39] fix: stash --- .../src/clients/workflow-client.ts | 2 +- .../aws-runtime/src/handlers/orchestrator.ts | 125 +++++++++--------- .../{aws-runtime => core}/src/execution-id.ts | 0 packages/@eventual/core/src/index.ts | 3 +- 4 files changed, 66 insertions(+), 64 deletions(-) rename packages/@eventual/{aws-runtime => core}/src/execution-id.ts (100%) diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts index 73d12b4ec..ad42d5a85 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts @@ -7,6 +7,7 @@ import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; import { Execution, ExecutionStatus, + formatExecutionId, HistoryStateEvent, WorkflowEventType, WorkflowStarted, @@ -14,7 +15,6 @@ import { } from "@eventual/core"; import { StartWorkflowRequest } from "src/types.js"; import { ulid } from "ulidx"; -import { formatExecutionId } from "../execution-id.js"; import { ExecutionHistoryClient } from "./execution-history-client.js"; export interface WorkflowClientProps { diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index bf0d3757b..6d40a7c07 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -8,6 +8,7 @@ import { FailedExecution, HistoryStateEvent, isCompleteExecution, + isExecutionId, isFailed, isResolved, isResult, @@ -17,6 +18,7 @@ import { isSleepForCommand, isSleepUntilCommand, lookupWorkflow, + parseWorkflowName, progressWorkflow, Workflow, WorkflowCompleted, @@ -39,7 +41,6 @@ import { createWorkflowRuntimeClient, } from "../clients/index.js"; import { SQSWorkflowTaskMessage } from "../clients/workflow-client.js"; -import { isExecutionId, parseWorkflowName } from "../execution-id.js"; import { logger, loggerMiddlewares } from "../logger.js"; import { MetricsCommon, OrchestratorMetrics } from "../metrics/constants.js"; import { timed, timedSync } from "../metrics/utils.js"; @@ -64,65 +65,71 @@ export function orchestrator(): SQSHandler { // batch by execution id const eventsByExecutionId = groupBy( event.Records, - (r) => r.attributes.MessageGroupId! + (r) => r.attributes.MessageGroupId!, + sqsRecordToEvents ); - logger.info( - "Found execution ids: " + Object.keys(eventsByExecutionId).join(", ") - ); + orchestrate(eventsByExecutionId); + }).use(loggerMiddlewares); +} - // for each execution id - const results = await promiseAllSettledPartitioned( - Object.entries(eventsByExecutionId), - async ([executionId, records]) => { - if (!isExecutionId(executionId)) { - throw new Error(`invalid ExecutionID: '${executionId}'`); - } - const workflowName = parseWorkflowName(executionId); - if (workflowName === undefined) { - throw new Error(`execution ID '${executionId}' does not exist`); - } - const workflow = lookupWorkflow(workflowName); - if (workflow === undefined) { - throw new Error(`no such workflow with name '${workflowName}'`); - } - // TODO: get workflow from execution id - return orchestrateExecution(workflow, executionId, records); +async function orchestrate( + eventsByExecutionId: Record +) { + logger.info( + "Found execution ids: " + Object.keys(eventsByExecutionId).join(", ") + ); + + // for each execution id + const results = await promiseAllSettledPartitioned( + Object.entries(eventsByExecutionId), + async ([executionId, records]) => { + if (!isExecutionId(executionId)) { + throw new Error(`invalid ExecutionID: '${executionId}'`); } - ); - - logger.debug( - "Executions succeeded: " + - results.fulfilled.map(([[executionId]]) => executionId).join(",") - ); - - if (results.rejected.length > 0) { - logger.error( - "Executions failed: \n" + - results.rejected - .map(([[executionId], error]) => `${executionId}: ${error}`) - .join("\n") - ); + const workflowName = parseWorkflowName(executionId); + if (workflowName === undefined) { + throw new Error(`execution ID '${executionId}' does not exist`); + } + const workflow = lookupWorkflow(workflowName); + if (workflow === undefined) { + throw new Error(`no such workflow with name '${workflowName}'`); + } + // TODO: get workflow from execution id + return orchestrateExecution(workflow, executionId, records); } - - const failedMessageIds = results.rejected.flatMap( - ([[, records]]) => records.map((r) => r.messageId) ?? [] + ); + + logger.debug( + "Executions succeeded: " + + results.fulfilled.map(([[executionId]]) => executionId).join(",") + ); + + if (results.rejected.length > 0) { + logger.error( + "Executions failed: \n" + + results.rejected + .map(([[executionId], error]) => `${executionId}: ${error}`) + .join("\n") ); + } - return { - batchItemFailures: failedMessageIds.map((r) => ({ - itemIdentifier: r, - })), - }; - }).use(loggerMiddlewares); + const failedMessageIds = results.rejected.flatMap( + ([[, records]]) => records.map((r) => r.messageId) ?? [] + ); + + return { + batchItemFailures: failedMessageIds.map((r) => ({ + itemIdentifier: r, + })), + }; } async function orchestrateExecution( workflow: Workflow, executionId: string, - records: SQSRecord[] + events: HistoryStateEvent[] ) { - console.log(executionId, records); const executionLogger = logger.createChild({ persistentLogAttributes: { executionId }, }); @@ -132,13 +139,12 @@ async function orchestrateExecution( metrics.setDimensions({ [MetricsCommon.WorkflowNameDimension]: workflow.name, }); - const events = sqsRecordsToEvents(records); const start = new Date(); try { // number of events that came from the workflow task metrics.setProperty(OrchestratorMetrics.TaskEvents, events.length); // number of workflow tasks that are being processed in the batch (max: 10) - metrics.setProperty(OrchestratorMetrics.AggregatedTasks, records.length); + // metrics.setProperty(OrchestratorMetrics.AggregatedTasks, records.length); /** Events to be written to the history table at the end of the workflow task */ const newEvents: WorkflowEvent[] = []; @@ -150,9 +156,7 @@ async function orchestrateExecution( ); // length of time the oldest SQS record was in the queue. const maxTaskAge = Math.max( - ...records.map( - (r) => new Date().getTime() - Number(r.attributes.SentTimestamp) - ) + ...events.map((r) => new Date().getTime() - Number(r.timestamp)) ); metrics.putMetric( OrchestratorMetrics.MaxTaskAge, @@ -413,11 +417,7 @@ async function orchestrateExecution( } } -function sqsRecordsToEvents(sqsRecords: SQSRecord[]) { - return sqsRecords.flatMap(sqsRecordToEvents); -} - -function sqsRecordToEvents(sqsRecord: SQSRecord) { +function sqsRecordToEvents(sqsRecord: SQSRecord): HistoryStateEvent[] { const message = JSON.parse(sqsRecord.body) as SQSWorkflowTaskMessage; return message.task.events; @@ -441,15 +441,16 @@ function logEventMetrics( } } -function groupBy( +function groupBy( items: T[], - extract: (item: T) => string -): Record { - return items.reduce((obj: Record, r) => { + extract: (item: T) => string, + map?: (item: T) => U[] +): Record { + return items.reduce((obj: Record, r) => { const id = extract(r); return { ...obj, - [id]: [...(obj[id] || []), r], + [id]: [...(obj[id] || []), ...(map ? map(r) : [r as any as U])], }; }, {}); } diff --git a/packages/@eventual/aws-runtime/src/execution-id.ts b/packages/@eventual/core/src/execution-id.ts similarity index 100% rename from packages/@eventual/aws-runtime/src/execution-id.ts rename to packages/@eventual/core/src/execution-id.ts diff --git a/packages/@eventual/core/src/index.ts b/packages/@eventual/core/src/index.ts index 6b2905bdd..3d0a36d88 100644 --- a/packages/@eventual/core/src/index.ts +++ b/packages/@eventual/core/src/index.ts @@ -1,11 +1,12 @@ export * from "./activity.js"; export * from "./await-all.js"; -export * from "./context.js"; export * from "./chain.js"; export * from "./command.js"; +export * from "./context.js"; export * from "./error.js"; export * from "./events.js"; export * from "./eventual.js"; +export * from "./execution-id.js"; export * from "./execution.js"; export * from "./interpret.js"; export * from "./result.js"; From 528043cb36a3aff0173a269582a1c4c9313d0c4e Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 24 Nov 2022 15:29:45 -0800 Subject: [PATCH 25/39] feat: set up webhook function url with itty router --- packages/@eventual/aws-cdk/src/service.ts | 27 ++++++++++++++++ packages/@eventual/aws-runtime/package.json | 8 +++-- packages/@eventual/aws-runtime/src/env.ts | 4 +++ .../src/handlers/webhook-handler.ts | 32 +++++++++++++++++++ packages/@eventual/core/package.json | 4 +++ packages/@eventual/core/src/hook.ts | 15 +++++++++ packages/@eventual/core/src/index.ts | 3 +- pnpm-lock.yaml | 8 +++++ 8 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts create mode 100644 packages/@eventual/core/src/hook.ts diff --git a/packages/@eventual/aws-cdk/src/service.ts b/packages/@eventual/aws-cdk/src/service.ts index 110ccb605..891a9fa44 100644 --- a/packages/@eventual/aws-cdk/src/service.ts +++ b/packages/@eventual/aws-cdk/src/service.ts @@ -106,6 +106,11 @@ export class Service extends Construct implements IGrantable { */ public readonly dlq: Queue; + /** + * A Lambda Function URL endpoint for accepting inbound webhook requests. + */ + public readonly webhookEndpoint: IFunction; + readonly grantPrincipal: IPrincipal; constructor(scope: Construct, id: string, props: WorkflowProps) { @@ -306,6 +311,28 @@ export class Service extends Construct implements IGrantable { ], }); + this.webhookEndpoint = new NodejsFunction(this, "WebhookEndpoint", { + entry: path.join( + require.resolve("@eventual/aws-runtime"), + "../../esm/handlers/webhook-handler.js" + ), + handler: "handle", + runtime: Runtime.NODEJS_16_X, + architecture: Architecture.ARM_64, + bundling: { + mainFields: ["module", "main"], + esbuildArgs: { + "--conditions": "module,import,require", + }, + metafile: true, + }, + environment: { + [ENV_NAMES.TABLE_NAME]: this.table.tableName, + [ENV_NAMES.WORKFLOW_QUEUE_URL]: this.workflowQueue.queueUrl, + [ENV_NAMES.EVENTUAL_WEBHOOK]: "1", + }, + }); + this.timerQueue.grantSendMessages(this.scheduleForwarder); this.timerQueue.grantSendMessages(this.orchestrator); diff --git a/packages/@eventual/aws-runtime/package.json b/packages/@eventual/aws-runtime/package.json index c405d3337..04e76f922 100644 --- a/packages/@eventual/aws-runtime/package.json +++ b/packages/@eventual/aws-runtime/package.json @@ -27,19 +27,21 @@ "@aws-sdk/client-dynamodb": "^3.208.0", "@aws-sdk/client-lambda": "^3.208.0", "@aws-sdk/client-s3": "^3.208.0", - "@aws-sdk/client-sqs": "^3.208.0", "@aws-sdk/client-scheduler": "^3.208.0", - "@types/aws-lambda": "8.10.108" + "@aws-sdk/client-sqs": "^3.208.0", + "@types/aws-lambda": "8.10.108", + "itty-router": "^2.6.6" }, "devDependencies": { "@aws-sdk/client-dynamodb": "3.214.0", "@aws-sdk/client-lambda": "^3.213.0", "@aws-sdk/client-s3": "3.213.0", - "@aws-sdk/client-sqs": "3.213.0", "@aws-sdk/client-scheduler": "3.213.0", + "@aws-sdk/client-sqs": "3.213.0", "@types/aws-lambda": "8.10.108", "@types/jest": "^29", "@types/node": "^16", + "itty-router": "2.6.6", "jest": "^29", "ts-jest": "^29", "ts-node": "^10.9.1", diff --git a/packages/@eventual/aws-runtime/src/env.ts b/packages/@eventual/aws-runtime/src/env.ts index 5bf428495..e964e6ce2 100644 --- a/packages/@eventual/aws-runtime/src/env.ts +++ b/packages/@eventual/aws-runtime/src/env.ts @@ -21,6 +21,10 @@ export namespace ENV_NAMES { * Activity calls behave different based on their context. */ export const EVENTUAL_WORKER = "EVENTUAL_WORKER"; + /** + * A flag that determines if a function is the webhook endpoint. + */ + export const EVENTUAL_WEBHOOK = "EVENTUAL_WEBHOOK"; } export function tryGetEnv(name: string) { diff --git a/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts new file mode 100644 index 000000000..f6b7edb7b --- /dev/null +++ b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts @@ -0,0 +1,32 @@ +import { APIGatewayProxyEventV2, APIGatewayProxyResultV2 } from "aws-lambda"; +import { getHooks } from "@eventual/core"; +import itty from "itty-router"; +import { createWorkflowClient } from "src/clients"; + +const router = itty.Router({}); + +getHooks().forEach((hook) => hook(router)); + +// TODO: plu +const workflowClient = createWorkflowClient(); + +export async function handle( + event: APIGatewayProxyEventV2 +): Promise { + const response: Response = await router.handle({ + method: event.requestContext.http.method, + // TODO: is this right? + url: event.requestContext.http.path, + params: event.pathParameters as itty.Obj, + query: event.queryStringParameters as itty.Obj, + }); + + const headers: Record = {}; + response.headers.forEach((value, key) => (headers[key] = value)); + return { + headers, + statusCode: response.status, + body: Buffer.from(await response.arrayBuffer()).toString("base64"), + isBase64Encoded: true, + }; +} diff --git a/packages/@eventual/core/package.json b/packages/@eventual/core/package.json index b407ef876..477749144 100644 --- a/packages/@eventual/core/package.json +++ b/packages/@eventual/core/package.json @@ -15,7 +15,11 @@ "scripts": { "test": "jest" }, + "peerDependencies": { + "itty-router": "^2.6.6" + }, "devDependencies": { + "itty-router": "2.6.6", "@types/jest": "^29", "@types/node": "^16", "jest": "^29", diff --git a/packages/@eventual/core/src/hook.ts b/packages/@eventual/core/src/hook.ts new file mode 100644 index 000000000..a4e4f7966 --- /dev/null +++ b/packages/@eventual/core/src/hook.ts @@ -0,0 +1,15 @@ +import itty from "itty-router"; + +const hooks: Hook[] = []; + +export type Hook = ( + router: itty.Router +) => void; + +export function hook(setup: Hook) { + hooks.push(setup); +} + +export function getHooks() { + return [...hooks]; +} diff --git a/packages/@eventual/core/src/index.ts b/packages/@eventual/core/src/index.ts index 6b2905bdd..45753a1d9 100644 --- a/packages/@eventual/core/src/index.ts +++ b/packages/@eventual/core/src/index.ts @@ -1,12 +1,13 @@ export * from "./activity.js"; export * from "./await-all.js"; -export * from "./context.js"; export * from "./chain.js"; export * from "./command.js"; +export * from "./context.js"; export * from "./error.js"; export * from "./events.js"; export * from "./eventual.js"; export * from "./execution.js"; +export * from "./hook.js"; export * from "./interpret.js"; export * from "./result.js"; export * from "./sleep.js"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index efc1fb3e8..4614f8c18 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -146,6 +146,7 @@ importers: aws-embedded-metrics: ^4.0.0 aws-lambda: ^1.0.7 fast-equals: ^4.0.3 + itty-router: 2.6.6 jest: ^29 micro-memoize: ^4.0.11 ts-jest: ^29 @@ -171,6 +172,7 @@ importers: '@types/aws-lambda': 8.10.108 '@types/jest': 29.2.2 '@types/node': 16.18.3 + itty-router: 2.6.6 jest: 29.3.1_dnlfjp7n5lpfgnj4digwzn5fhe ts-jest: 29.0.3_6crhf7ajeizammv76u753sn6i4 ts-node: 10.9.1_fvpuwgkpfe3dm3hnpcpbcxmb3y @@ -270,6 +272,7 @@ importers: specifiers: '@types/jest': ^29 '@types/node': ^16 + itty-router: 2.6.6 jest: ^29 ts-jest: ^29 ts-node: ^10.9.1 @@ -278,6 +281,7 @@ importers: devDependencies: '@types/jest': 29.2.2 '@types/node': 16.18.3 + itty-router: 2.6.6 jest: 29.3.1_dnlfjp7n5lpfgnj4digwzn5fhe ts-jest: 29.0.3_6crhf7ajeizammv76u753sn6i4 ts-node: 10.9.1_fvpuwgkpfe3dm3hnpcpbcxmb3y @@ -5917,6 +5921,10 @@ packages: istanbul-lib-report: 3.0.0 dev: true + /itty-router/2.6.6: + resolution: {integrity: sha512-hIPHtXGymCX7Lzb2I4G6JgZFE4QEEQwst9GORK7sMYUpJvLfy4yZJr95r04e8DzoAnj6HcxM2m4TbK+juu+18g==} + dev: true + /jake/10.8.5: resolution: {integrity: sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw==} engines: {node: '>=10'} From 4c94d8a70172f9c05c09ca9c5005166e915c1c86 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 25 Nov 2022 02:31:13 -0800 Subject: [PATCH 26/39] feat: runtime client interfaces and webhook router --- apps/test-app-runtime/src/open-account.ts | 10 ++- packages/@eventual/aws-cdk/src/service.ts | 54 +++++++------ .../src/clients/activity-runtime-client.ts | 8 +- .../aws-runtime/src/clients/create.ts | 4 +- .../src/clients/execution-history-client.ts | 4 +- .../aws-runtime/src/clients/timer-client.ts | 3 +- .../src/clients/workflow-client.ts | 7 +- .../src/clients/workflow-runtime-client.ts | 3 +- .../src/functions/start-workflow.ts | 2 +- .../src/handlers/webhook-handler.ts | 16 ++-- packages/@eventual/aws-runtime/src/types.ts | 27 ------- packages/@eventual/core/package.json | 6 +- packages/@eventual/core/src/index.ts | 1 + .../src/runtime/activity-runtime-client.ts | 18 +++++ .../src/runtime/execution-history-client.ts | 38 +++++++++ packages/@eventual/core/src/runtime/index.ts | 5 ++ .../core/src/runtime/timer-client.ts | 78 +++++++++++++++++++ .../core/src/runtime/workflow-client.ts | 78 +++++++++++++++++++ .../src/runtime/workflow-runtime-client.ts | 53 +++++++++++++ packages/@eventual/core/src/workflow.ts | 56 ++++++++++--- pnpm-lock.yaml | 4 +- 21 files changed, 394 insertions(+), 81 deletions(-) delete mode 100644 packages/@eventual/aws-runtime/src/types.ts create mode 100644 packages/@eventual/core/src/runtime/activity-runtime-client.ts create mode 100644 packages/@eventual/core/src/runtime/execution-history-client.ts create mode 100644 packages/@eventual/core/src/runtime/index.ts create mode 100644 packages/@eventual/core/src/runtime/timer-client.ts create mode 100644 packages/@eventual/core/src/runtime/workflow-client.ts create mode 100644 packages/@eventual/core/src/runtime/workflow-runtime-client.ts diff --git a/apps/test-app-runtime/src/open-account.ts b/apps/test-app-runtime/src/open-account.ts index a8ff9cb45..05236ae0c 100644 --- a/apps/test-app-runtime/src/open-account.ts +++ b/apps/test-app-runtime/src/open-account.ts @@ -1,4 +1,4 @@ -import { activity, workflow } from "@eventual/core"; +import { activity, hook, workflow } from "@eventual/core"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; import { @@ -65,6 +65,13 @@ export const associateAccountInformation = workflow( } ); +// register a web hook API route +hook((api) => { + api.get("/hello", async () => { + return new Response("hello"); + }); +}); + const TableName = process.env.TABLE_NAME!; const dynamo = memoize(() => @@ -72,7 +79,6 @@ const dynamo = memoize(() => ); const createAccount = activity("createAccount", async (accountId: string) => { - console.log("processing", accountId); await dynamo().send( new PutCommand({ TableName, diff --git a/packages/@eventual/aws-cdk/src/service.ts b/packages/@eventual/aws-cdk/src/service.ts index 891a9fa44..1e43704f1 100644 --- a/packages/@eventual/aws-cdk/src/service.ts +++ b/packages/@eventual/aws-cdk/src/service.ts @@ -5,6 +5,7 @@ import { Code, IFunction, Runtime, + FunctionUrlAuthType, } from "aws-cdk-lib/aws-lambda"; import { Construct } from "constructs"; import { Bucket, IBucket } from "aws-cdk-lib/aws-s3"; @@ -311,28 +312,6 @@ export class Service extends Construct implements IGrantable { ], }); - this.webhookEndpoint = new NodejsFunction(this, "WebhookEndpoint", { - entry: path.join( - require.resolve("@eventual/aws-runtime"), - "../../esm/handlers/webhook-handler.js" - ), - handler: "handle", - runtime: Runtime.NODEJS_16_X, - architecture: Architecture.ARM_64, - bundling: { - mainFields: ["module", "main"], - esbuildArgs: { - "--conditions": "module,import,require", - }, - metafile: true, - }, - environment: { - [ENV_NAMES.TABLE_NAME]: this.table.tableName, - [ENV_NAMES.WORKFLOW_QUEUE_URL]: this.workflowQueue.queueUrl, - [ENV_NAMES.EVENTUAL_WEBHOOK]: "1", - }, - }); - this.timerQueue.grantSendMessages(this.scheduleForwarder); this.timerQueue.grantSendMessages(this.orchestrator); @@ -400,5 +379,36 @@ export class Service extends Construct implements IGrantable { }, }), }); + + this.webhookEndpoint = new NodejsFunction(this, "WebhookEndpoint", { + entry: path.join( + require.resolve("@eventual/aws-runtime"), + "../../esm/handlers/webhook-handler.js" + ), + handler: "handle", + runtime: Runtime.NODEJS_16_X, + architecture: Architecture.ARM_64, + bundling: { + mainFields: ["module", "main"], + esbuildArgs: { + "--conditions": "module,import,require", + }, + metafile: true, + }, + environment: { + [ENV_NAMES.TABLE_NAME]: this.table.tableName, + [ENV_NAMES.WORKFLOW_QUEUE_URL]: this.workflowQueue.queueUrl, + [ENV_NAMES.EVENTUAL_WEBHOOK]: "1", + }, + }); + this.webhookEndpoint.addFunctionUrl({ + authType: FunctionUrlAuthType.NONE, + }); + + // the webhook endpoint is allowed to run workflows + this.workflowQueue.grantSendMessages(this.webhookEndpoint); + + // Enable creating history to start a workflow. + this.table.grantReadWriteData(this.webhookEndpoint); } } diff --git a/packages/@eventual/aws-runtime/src/clients/activity-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/activity-runtime-client.ts index 846afa951..a79e60600 100644 --- a/packages/@eventual/aws-runtime/src/clients/activity-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/activity-runtime-client.ts @@ -3,14 +3,14 @@ import { DynamoDBClient, PutItemCommand, } from "@aws-sdk/client-dynamodb"; -import { ScheduleActivityCommand } from "@eventual/core"; +import type eventual from "@eventual/core"; export interface ActivityRuntimeClientProps { dynamo: DynamoDBClient; activityLockTableName: string; } -export class ActivityRuntimeClient { +export class ActivityRuntimeClient implements eventual.ActivityRuntimeClient { constructor(private props: ActivityRuntimeClientProps) {} /** @@ -23,7 +23,7 @@ export class ActivityRuntimeClient { **/ async requestExecutionActivityClaim( executionId: string, - command: ScheduleActivityCommand, + command: eventual.ScheduleActivityCommand, retry: number, claimer?: string ) { @@ -58,7 +58,7 @@ export namespace ActivityLockRecord { export const PARTITION_KEY_PREFIX = `Activity$`; export function key( executionId: string, - command: ScheduleActivityCommand, + command: eventual.ScheduleActivityCommand, retry: number ) { return `${PARTITION_KEY_PREFIX}$${executionId}$${command.seq}${retry}`; diff --git a/packages/@eventual/aws-runtime/src/clients/create.ts b/packages/@eventual/aws-runtime/src/clients/create.ts index 113bd0d8c..ea13c58ba 100644 --- a/packages/@eventual/aws-runtime/src/clients/create.ts +++ b/packages/@eventual/aws-runtime/src/clients/create.ts @@ -47,7 +47,9 @@ export const createWorkflowClient = /*@__PURE__*/ memoize( new WorkflowClient({ sqs: sqs(), workflowQueueUrl: workflowQueueUrl ?? env.workflowQueueUrl(), - executionHistory: createExecutionHistoryClient({ tableName }), + executionHistory: createExecutionHistoryClient({ + tableName: tableName ?? env.tableName(), + }), dynamo: dynamo(), tableName: tableName ?? env.tableName(), }), diff --git a/packages/@eventual/aws-runtime/src/clients/execution-history-client.ts b/packages/@eventual/aws-runtime/src/clients/execution-history-client.ts index 73a56bee7..41fc3aaea 100644 --- a/packages/@eventual/aws-runtime/src/clients/execution-history-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/execution-history-client.ts @@ -11,6 +11,8 @@ import { isHistoryEvent, WorkflowEvent, } from "@eventual/core"; +import type eventual from "@eventual/core"; + import { ulid } from "ulidx"; export interface ExecutionHistoryClientProps { @@ -20,7 +22,7 @@ export interface ExecutionHistoryClientProps { type UnresolvedEvent = Omit; -export class ExecutionHistoryClient { +export class ExecutionHistoryClient implements eventual.ExecutionHistoryClient { constructor(private props: ExecutionHistoryClientProps) {} public async createAndPutEvent( diff --git a/packages/@eventual/aws-runtime/src/clients/timer-client.ts b/packages/@eventual/aws-runtime/src/clients/timer-client.ts index e7a4a6809..721345609 100644 --- a/packages/@eventual/aws-runtime/src/clients/timer-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/timer-client.ts @@ -13,6 +13,7 @@ import { ScheduleForwarderRequest, TimerRequest, } from "../handlers/types.js"; +import type eventual from "@eventual/core"; export interface TimerClientProps { readonly scheduler: SchedulerClient; @@ -29,7 +30,7 @@ export interface TimerClientProps { readonly scheduleForwarderArn: string; } -export class TimerClient { +export class TimerClient implements eventual.TimerClient { constructor(private props: TimerClientProps) {} /** diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts index 73d12b4ec..277fcb896 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts @@ -12,11 +12,12 @@ import { WorkflowStarted, WorkflowTask, } from "@eventual/core"; -import { StartWorkflowRequest } from "src/types.js"; import { ulid } from "ulidx"; import { formatExecutionId } from "../execution-id.js"; import { ExecutionHistoryClient } from "./execution-history-client.js"; +import type eventual from "@eventual/core"; + export interface WorkflowClientProps { readonly dynamo: DynamoDBClient; readonly tableName: string; @@ -25,7 +26,7 @@ export interface WorkflowClientProps { readonly executionHistory: ExecutionHistoryClient; } -export class WorkflowClient { +export class WorkflowClient implements eventual.WorkflowClient { constructor(private props: WorkflowClientProps) {} /** @@ -40,7 +41,7 @@ export class WorkflowClient { input, parentExecutionId, seq, - }: StartWorkflowRequest) { + }: eventual.StartWorkflowRequest) { const executionId = formatExecutionId(workflowName, executionName); console.log("execution input:", input); diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index 820cffff1..a41b48d88 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -39,6 +39,7 @@ import { ActivityWorkerRequest } from "../activity.js"; import { createEvent } from "./execution-history-client.js"; import { TimerRequestType } from "../handlers/types.js"; import { TimerClient } from "./timer-client.js"; +import type eventual from "@eventual/core"; export interface WorkflowRuntimeClientProps { readonly lambda: LambdaClient; @@ -57,7 +58,7 @@ export interface CompleteExecutionRequest { readonly timerClient: TimerClient; } -export class WorkflowRuntimeClient { +export class WorkflowRuntimeClient implements eventual.WorkflowRuntimeClient { constructor(private props: WorkflowRuntimeClientProps) {} async getHistory(executionId: string) { diff --git a/packages/@eventual/aws-runtime/src/functions/start-workflow.ts b/packages/@eventual/aws-runtime/src/functions/start-workflow.ts index b6b3d2145..2e9d99753 100644 --- a/packages/@eventual/aws-runtime/src/functions/start-workflow.ts +++ b/packages/@eventual/aws-runtime/src/functions/start-workflow.ts @@ -1,5 +1,5 @@ import { Handler } from "aws-lambda"; -import type { StartWorkflowRequest } from "../types.js"; +import type { StartWorkflowRequest } from "@eventual/core"; import { createWorkflowClient } from "../clients/index.js"; const workflowClient = createWorkflowClient(); diff --git a/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts index f6b7edb7b..f6c50abac 100644 --- a/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts +++ b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts @@ -1,15 +1,21 @@ +import { getHooks, registerWorkflowClient } from "@eventual/core"; import { APIGatewayProxyEventV2, APIGatewayProxyResultV2 } from "aws-lambda"; -import { getHooks } from "@eventual/core"; import itty from "itty-router"; import { createWorkflowClient } from "src/clients"; -const router = itty.Router({}); +// make the workflow client available to web hooks +registerWorkflowClient(createWorkflowClient()); +// initialize all web hooks onto the central HTTP router +const router = itty.Router({}); getHooks().forEach((hook) => hook(router)); -// TODO: plu -const workflowClient = createWorkflowClient(); - +/** + * Handle inbound webhook API requests. + * + * Each webhook registers routes on the central {@link router} which + * then handles the request. + */ export async function handle( event: APIGatewayProxyEventV2 ): Promise { diff --git a/packages/@eventual/aws-runtime/src/types.ts b/packages/@eventual/aws-runtime/src/types.ts deleted file mode 100644 index 1ea20d38a..000000000 --- a/packages/@eventual/aws-runtime/src/types.ts +++ /dev/null @@ -1,27 +0,0 @@ -export interface StartWorkflowRequest { - /** - * Name of the workflow execution. - * - * Only one workflow can exist for an ID. Requests to start a workflow - * with the name of an existing workflow will fail. - * - * @default - a unique name is generated. - */ - executionName?: string; - /** - * Name of the workflow to execute. - */ - workflowName: string; - /** - * Input payload for the workflow function. - */ - input?: any; - /** - * ID of the parent execution if this is a child workflow - */ - parentExecutionId?: string; - /** - * Sequence ID of this execution if this is a child workflow - */ - seq?: number; -} diff --git a/packages/@eventual/core/package.json b/packages/@eventual/core/package.json index 477749144..3b799e670 100644 --- a/packages/@eventual/core/package.json +++ b/packages/@eventual/core/package.json @@ -16,16 +16,18 @@ "test": "jest" }, "peerDependencies": { - "itty-router": "^2.6.6" + "itty-router": "^2.6.6", + "ulidx": "^0.3.0" }, "devDependencies": { - "itty-router": "2.6.6", "@types/jest": "^29", "@types/node": "^16", + "itty-router": "2.6.6", "jest": "^29", "ts-jest": "^29", "ts-node": "^10.9.1", "typescript": "^4.9.3", + "ulidx": "0.3.0", "zx": "^7.1.1" }, "jest": { diff --git a/packages/@eventual/core/src/index.ts b/packages/@eventual/core/src/index.ts index 45753a1d9..a82be2ee2 100644 --- a/packages/@eventual/core/src/index.ts +++ b/packages/@eventual/core/src/index.ts @@ -10,6 +10,7 @@ export * from "./execution.js"; export * from "./hook.js"; export * from "./interpret.js"; export * from "./result.js"; +export * from "./runtime/index.js"; export * from "./sleep.js"; export * from "./tasks.js"; export * from "./util.js"; diff --git a/packages/@eventual/core/src/runtime/activity-runtime-client.ts b/packages/@eventual/core/src/runtime/activity-runtime-client.ts new file mode 100644 index 000000000..35559d6e5 --- /dev/null +++ b/packages/@eventual/core/src/runtime/activity-runtime-client.ts @@ -0,0 +1,18 @@ +import { ScheduleActivityCommand } from "../command.js"; + +export interface ActivityRuntimeClient { + /** + * Claims a activity for an actor. + * + * Future invocations of the same executionId + future.seq + retry will fail. + * + * @param claimer optional string to correlate the lock to the claimer. + * @return a boolean determining if the claim was granted to the current actor. + **/ + requestExecutionActivityClaim( + executionId: string, + command: ScheduleActivityCommand, + retry: number, + claimer?: string + ): Promise; +} diff --git a/packages/@eventual/core/src/runtime/execution-history-client.ts b/packages/@eventual/core/src/runtime/execution-history-client.ts new file mode 100644 index 000000000..4a916d3b5 --- /dev/null +++ b/packages/@eventual/core/src/runtime/execution-history-client.ts @@ -0,0 +1,38 @@ +import { WorkflowEvent } from "../events.js"; + +export type UnresolvedEvent = Omit< + T, + "id" | "timestamp" +>; + +export interface ExecutionHistoryClient { + createAndPutEvent( + executionId: string, + event: UnresolvedEvent, + time?: Date + ): Promise; + + putEvent( + executionId: string, + event: T + ): Promise; + + /** + * Writes events as a batch into the history table, assigning IDs and timestamp first. + */ + createAndPutEvents( + executionId: string, + events: UnresolvedEvent[], + time?: Date + ): Promise; + + /** + * Writes events as a batch into the execution history table. + */ + putEvents(executionId: string, events: WorkflowEvent[]): Promise; + + /** + * Read an execution's events from the execution history table table + */ + getEvents(executionId: string): Promise; +} diff --git a/packages/@eventual/core/src/runtime/index.ts b/packages/@eventual/core/src/runtime/index.ts new file mode 100644 index 000000000..6c4ea4614 --- /dev/null +++ b/packages/@eventual/core/src/runtime/index.ts @@ -0,0 +1,5 @@ +export * from "./activity-runtime-client.js"; +export * from "./execution-history-client.js"; +export * from "./timer-client.js"; +export * from "./workflow-client.js"; +export * from "./workflow-runtime-client.js"; diff --git a/packages/@eventual/core/src/runtime/timer-client.ts b/packages/@eventual/core/src/runtime/timer-client.ts new file mode 100644 index 000000000..6527752ae --- /dev/null +++ b/packages/@eventual/core/src/runtime/timer-client.ts @@ -0,0 +1,78 @@ +import { HistoryStateEvent } from "../events.js"; + +export interface TimerClient { + /** + * Starts a timer using SQS's message delay. + * + * The timerRequest.untilTime may only be 15 minutes or fewer in the future. + * + * For longer use {@link TimerClient.startTimer}. + * + * The SQS Queue will delay for floor(untilTime - currentTime) seconds until the timer handler can pick up the message. + * + * Finally the timer handler waits the remaining (untilTime - currentTime) milliseconds if necessary and then sends + * the {@link TimerRequest} provided. + */ + startShortTimer(timerRequest: TimerRequest): Promise; + + /** + * Starts a timer of any (positive) length. + * + * If the timer is longer than 15 minutes (configurable via `props.sleepQueueThresholdMillis`), + * the timer will create a EventBridge schedule until the untilTime - props.sleepQueueThresholdMillis + * when the timer will be moved to the SQS queue. + * + * The SQS Queue will delay for floor(untilTime - currentTime) seconds until the timer handler can pick up the message. + * + * Finally the timer handler waits the remaining (untilTime - currentTime) milliseconds if necessary and then sends + * the {@link TimerRequest} provided. + */ + startTimer(timerRequest: TimerRequest): Promise; + + /** + * When startTimer is used, the EventBridge schedule will not self delete. + * + * Use this method to clean the schedule. + * + * The provided schedule-forwarder function will call this method in Eventual when + * the timer is transferred from EventBridge to SQS at `props.sleepQueueThresholdMillis`. + */ + clearSchedule(scheduleName: string): Promise; +} + +export type TimerRequest = TimerForwardEventRequest; + +export enum TimerRequestType { + ForwardEvent = "ForwardEvent", +} + +export interface TimerRequestBase { + type: T; + untilTime: string; +} + +/** + * Forward an event to the Workflow Queue. + */ +export interface TimerForwardEventRequest + extends TimerRequestBase { + executionId: string; + event: HistoryStateEvent; +} + +export function isTimerForwardEventRequest( + timerRequest: TimerRequest +): timerRequest is TimerForwardEventRequest { + return timerRequest && timerRequest.type === TimerRequestType.ForwardEvent; +} + +export interface ScheduleForwarderRequest { + scheduleName: string; + clearSchedule: boolean; + timerRequest: TimerRequest; + forwardTime: string; + /** + * ISO 8601 Timestamp determining when the message should be visible to the timer queue. + */ + untilTime: string; +} diff --git a/packages/@eventual/core/src/runtime/workflow-client.ts b/packages/@eventual/core/src/runtime/workflow-client.ts new file mode 100644 index 000000000..d6d672a2f --- /dev/null +++ b/packages/@eventual/core/src/runtime/workflow-client.ts @@ -0,0 +1,78 @@ +import { HistoryStateEvent } from "../events.js"; + +// a global variable for storing the WorkflowClient +// this is initialized by Eventual's harness lambda functions +let workflowClient: WorkflowClient; + +/** + * Register the global workflow client used by workflow functions + * to start workflows within an eventual-controlled environment. + */ +export function registerWorkflowClient(client: WorkflowClient) { + workflowClient = client; +} + +/** + * Get the global workflow client. + */ +export function getWorkflowClient(): WorkflowClient { + if (workflowClient === undefined) { + throw new Error(`WorkflowClient is not registered`); + } + return workflowClient; +} + +export interface WorkflowClient { + /** + * Start a workflow execution + * @param name Suffix of execution id + * @param input Workflow parameters + * @returns + */ + startWorkflow(request: StartWorkflowRequest): Promise; + /** + * Submit events to be processed by a workflow's orchestrator. + * + * @param executionId ID of the workflow execution + * @param events events to submit for processing + */ + submitWorkflowTask( + executionId: string, + ...events: HistoryStateEvent[] + ): Promise; +} + +export interface StartWorkflowRequest { + /** + * Name of the workflow execution. + * + * Only one workflow can exist for an ID. Requests to start a workflow + * with the name of an existing workflow will fail. + * + * @default - a unique name is generated. + */ + executionName?: string; + /** + * Name of the workflow to execute. + */ + workflowName: string; + /** + * Input payload for the workflow function. + */ + input?: any; + /** + * ID of the parent execution if this is a child workflow + */ + parentExecutionId?: string; + /** + * Sequence ID of this execution if this is a child workflow + */ + seq?: number; +} + +export interface StartWorkflowResponse { + /** + * ID of the started workflow execution. + */ + executionId: string; +} diff --git a/packages/@eventual/core/src/runtime/workflow-runtime-client.ts b/packages/@eventual/core/src/runtime/workflow-runtime-client.ts new file mode 100644 index 000000000..1fe49d87d --- /dev/null +++ b/packages/@eventual/core/src/runtime/workflow-runtime-client.ts @@ -0,0 +1,53 @@ +import { + ScheduleActivityCommand, + SleepForCommand, + SleepUntilCommand, +} from "../command.js"; +import { + ActivityScheduled, + HistoryStateEvent, + SleepScheduled, +} from "../events.js"; +import { CompleteExecution, Execution, FailedExecution } from "../execution.js"; +import { TimerClient } from "./timer-client.js"; + +export interface CompleteExecutionRequest { + executionId: string; + result?: any; + readonly timerClient: TimerClient; +} + +export interface WorkflowRuntimeClient { + getHistory(executionId: string): Promise; + + // TODO: etag + updateHistory( + executionId: string, + events: HistoryStateEvent[] + ): Promise<{ bytes: number }>; + + completeExecution({ + executionId, + result, + }: CompleteExecutionRequest): Promise; + + failExecution( + executionId: string, + error: string, + message: string + ): Promise; + + getExecutions(): Promise; + + scheduleActivity( + workflowName: string, + executionId: string, + command: ScheduleActivityCommand + ): Promise; + + scheduleSleep( + executionId: string, + command: SleepUntilCommand | SleepForCommand, + baseTime: Date + ): Promise; +} diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index a5ada50f2..15e6f9f52 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -21,6 +21,10 @@ import { WorkflowEventType, } from "./events.js"; import { interpret, WorkflowResult } from "./interpret.js"; +import { + getWorkflowClient, + StartWorkflowResponse, +} from "./runtime/workflow-client.js"; export type WorkflowHandler = ( input: any, @@ -46,6 +50,23 @@ export interface Workflow { * To start a workflow from another environment, use {@link start}. */ (input: Parameters[0]): ReturnType; + + /** + * Starts a workflow execution + */ + startExecution(request: { + /** + * Input payload for the workflow. + */ + input: Parameters[0]; + /** + * Optional name of the workflow to start - used to determine the unique ID and enforce idempotency. + * + * @default - a unique ID is generated. + */ + name?: string; + }): Promise; + /** * @internal - this is the internal DSL representation that produces a {@link Program} instead of a Promise. */ @@ -88,12 +109,23 @@ export function workflow( if (workflows.has(name)) { throw new Error(`workflow with name '${name}' already exists`); } - const workflow: Workflow = ((input?: any) => - registerActivity({ + const workflow: Workflow = ((input?: any) => { + return registerActivity({ [EventualSymbol]: EventualKind.WorkflowCall, name, input, - })) as any; + }); + }) as any; + + workflow.startExecution = async function (input) { + return { + executionId: await getWorkflowClient().startWorkflow({ + workflowName: name, + executionName: input.name, + input: input.input, + }), + }; + }; workflow.definition = definition as Workflow["definition"]; // safe to cast because we rely on transformer (it is always the generator API) workflows.set(name, workflow); @@ -162,7 +194,10 @@ export function progressWorkflow( try { return { - ...interpret(program.definition(startEvent.input, context), interpretEvents), + ...interpret( + program.definition(startEvent.input, context), + interpretEvents + ), history: allEvents, }; } catch (err) { @@ -198,11 +233,14 @@ export function generateSyntheticEvents( unresolvedSleep ) .filter((event) => new Date(event.untilTime).getTime() <= now.getTime()) - .map((e) => ({ - type: WorkflowEventType.SleepCompleted, - seq: e.seq, - timestamp: now.toISOString(), - } satisfies SleepCompleted)); + .map( + (e) => + ({ + type: WorkflowEventType.SleepCompleted, + seq: e.seq, + timestamp: now.toISOString(), + } satisfies SleepCompleted) + ); return syntheticSleepComplete; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4614f8c18..0a87fab2f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -277,6 +277,7 @@ importers: ts-jest: ^29 ts-node: ^10.9.1 typescript: ^4.9.3 + ulidx: 0.3.0 zx: ^7.1.1 devDependencies: '@types/jest': 29.2.2 @@ -286,6 +287,7 @@ importers: ts-jest: 29.0.3_6crhf7ajeizammv76u753sn6i4 ts-node: 10.9.1_fvpuwgkpfe3dm3hnpcpbcxmb3y typescript: 4.9.3 + ulidx: 0.3.0 zx: 7.1.1 packages: @@ -6620,7 +6622,6 @@ packages: /layerr/0.1.2: resolution: {integrity: sha512-ob5kTd9H3S4GOG2nVXyQhOu9O8nBgP555XxWPkJI0tR0JeRilfyTp8WtPdIJHLXBmHMSdEq5+KMxiYABeScsIQ==} - dev: false /lerna/5.6.2: resolution: {integrity: sha512-Y0yMPslvnBnTZi7Nrs/gDyYZYauNf61xWNCehISHIORxZmmpoluNkcWTfcyb47is5uJQCv5QJX5xKKubbs+a6w==} @@ -8736,7 +8737,6 @@ packages: resolution: {integrity: sha512-Qvpa2xAzS6fBUpiqHSHWvn6XiSLCAPyNDDz035vsEWmUoXRqC4c9JySLIfdBuK0N1xGBxng6GHDOZgyNQfxAHg==} dependencies: layerr: 0.1.2 - dev: false /unique-filename/2.0.1: resolution: {integrity: sha512-ODWHtkkdx3IAR+veKxFV+VBkUMcN+FaqzUUd7IZzt+0zhDZFPFxhlqwPF3YQvMHx1TD0tdgYl+kuPnJ8E6ql7A==} From de01f0fffce3c8fc05b00f7a45a9c52ffae4814f Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 25 Nov 2022 03:52:32 -0800 Subject: [PATCH 27/39] feat: bundle webhooks --- apps/test-app/package.json | 2 +- packages/@eventual/aws-cdk/src/service.ts | 25 ++-- packages/@eventual/aws-runtime/package.json | 1 + .../entry/{activity-worker.ts => activity.ts} | 2 +- .../actions.ts => entry/injected.ts} | 0 .../aws-runtime/src/entry/orchestrator.ts | 2 +- .../aws-runtime/src/entry/webhook.ts | 4 + .../src/handlers/fetch-polyfill.ts | 12 ++ .../src/handlers/webhook-handler.ts | 23 +++- .../aws-runtime/src/injected/workflow.ts | 3 - .../@eventual/compiler/src/eventual-bundle.ts | 109 ++++++++---------- packages/@eventual/core/src/workflow.ts | 7 +- pnpm-lock.yaml | 2 + 13 files changed, 101 insertions(+), 91 deletions(-) rename packages/@eventual/aws-runtime/src/entry/{activity-worker.ts => activity.ts} (80%) rename packages/@eventual/aws-runtime/src/{injected/actions.ts => entry/injected.ts} (100%) create mode 100644 packages/@eventual/aws-runtime/src/entry/webhook.ts create mode 100644 packages/@eventual/aws-runtime/src/handlers/fetch-polyfill.ts delete mode 100644 packages/@eventual/aws-runtime/src/injected/workflow.ts diff --git a/apps/test-app/package.json b/apps/test-app/package.json index 572c58261..5d6b5d2cd 100644 --- a/apps/test-app/package.json +++ b/apps/test-app/package.json @@ -6,7 +6,7 @@ "build": "cdk synth", "cdk": "cdk", "deploy": "cdk deploy --require-approval=never", - "hotswap": "cdk deploy --hotswap", + "hotswap": "cdk deploy --hotswap --require-approval=never", "eventual": "eventual", "start-my-workflow": "eventual start my-workflow --input '{\"name\": \"world\"}' --tail" }, diff --git a/packages/@eventual/aws-cdk/src/service.ts b/packages/@eventual/aws-cdk/src/service.ts index 1e43704f1..b3651ac88 100644 --- a/packages/@eventual/aws-cdk/src/service.ts +++ b/packages/@eventual/aws-cdk/src/service.ts @@ -188,8 +188,8 @@ export class Service extends Construct implements IGrantable { this.activityWorker = new Function(this, "Worker", { architecture: Architecture.ARM_64, - code: Code.fromAsset(path.join(outDir, "activity-worker")), - // the bundler outputs activity-worker/index.js + code: Code.fromAsset(path.join(outDir, "activity")), + // the bundler outputs activity/index.js handler: "index.default", runtime: Runtime.NODEJS_16_X, memorySize: 512, @@ -380,22 +380,15 @@ export class Service extends Construct implements IGrantable { }), }); - this.webhookEndpoint = new NodejsFunction(this, "WebhookEndpoint", { - entry: path.join( - require.resolve("@eventual/aws-runtime"), - "../../esm/handlers/webhook-handler.js" - ), - handler: "handle", - runtime: Runtime.NODEJS_16_X, + this.webhookEndpoint = new Function(this, "WebhookEndpoint", { architecture: Architecture.ARM_64, - bundling: { - mainFields: ["module", "main"], - esbuildArgs: { - "--conditions": "module,import,require", - }, - metafile: true, - }, + code: Code.fromAsset(path.join(outDir, "webhook")), + // the bundler outputs orchestrator/index.js + handler: "index.default", + runtime: Runtime.NODEJS_16_X, + memorySize: 512, environment: { + NODE_OPTIONS: "--enable-source-maps", [ENV_NAMES.TABLE_NAME]: this.table.tableName, [ENV_NAMES.WORKFLOW_QUEUE_URL]: this.workflowQueue.queueUrl, [ENV_NAMES.EVENTUAL_WEBHOOK]: "1", diff --git a/packages/@eventual/aws-runtime/package.json b/packages/@eventual/aws-runtime/package.json index 04e76f922..dfdf41297 100644 --- a/packages/@eventual/aws-runtime/package.json +++ b/packages/@eventual/aws-runtime/package.json @@ -21,6 +21,7 @@ "aws-lambda": "^1.0.7", "fast-equals": "^4.0.3", "micro-memoize": "^4.0.11", + "node-fetch": "^3.3.0", "ulidx": "^0.3.0" }, "peerDependencies": { diff --git a/packages/@eventual/aws-runtime/src/entry/activity-worker.ts b/packages/@eventual/aws-runtime/src/entry/activity.ts similarity index 80% rename from packages/@eventual/aws-runtime/src/entry/activity-worker.ts rename to packages/@eventual/aws-runtime/src/entry/activity.ts index 560fba442..4e13b5e53 100644 --- a/packages/@eventual/aws-runtime/src/entry/activity-worker.ts +++ b/packages/@eventual/aws-runtime/src/entry/activity.ts @@ -1,5 +1,5 @@ // the user's entry point will register activities as a side effect. -import "@eventual/injected/activities"; +import "@eventual/entry/injected"; import { activityWorker } from "../handlers/activity-worker.js"; export default activityWorker(); diff --git a/packages/@eventual/aws-runtime/src/injected/actions.ts b/packages/@eventual/aws-runtime/src/entry/injected.ts similarity index 100% rename from packages/@eventual/aws-runtime/src/injected/actions.ts rename to packages/@eventual/aws-runtime/src/entry/injected.ts diff --git a/packages/@eventual/aws-runtime/src/entry/orchestrator.ts b/packages/@eventual/aws-runtime/src/entry/orchestrator.ts index cc928918d..19f121f45 100644 --- a/packages/@eventual/aws-runtime/src/entry/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/entry/orchestrator.ts @@ -1,4 +1,4 @@ -import "@eventual/injected/workflow"; +import "@eventual/entry/injected"; import { orchestrator } from "../handlers/orchestrator.js"; export default orchestrator(); diff --git a/packages/@eventual/aws-runtime/src/entry/webhook.ts b/packages/@eventual/aws-runtime/src/entry/webhook.ts new file mode 100644 index 000000000..683ddd030 --- /dev/null +++ b/packages/@eventual/aws-runtime/src/entry/webhook.ts @@ -0,0 +1,4 @@ +import "@eventual/entry/injected"; +import { processWebhook } from "../handlers/webhook-handler"; + +export default processWebhook; diff --git a/packages/@eventual/aws-runtime/src/handlers/fetch-polyfill.ts b/packages/@eventual/aws-runtime/src/handlers/fetch-polyfill.ts new file mode 100644 index 000000000..a94c0b3f7 --- /dev/null +++ b/packages/@eventual/aws-runtime/src/handlers/fetch-polyfill.ts @@ -0,0 +1,12 @@ +// @ts-nocheck + +// TODO: remove once we can upgrade to Node 18 + +import fetch, { Headers, Request, Response } from "node-fetch"; + +if (!globalThis.fetch) { + globalThis.fetch = fetch; + globalThis.Headers = Headers; + globalThis.Request = Request; + globalThis.Response = Response; +} diff --git a/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts index f6c50abac..5f4ecf2ca 100644 --- a/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts +++ b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts @@ -3,12 +3,21 @@ import { APIGatewayProxyEventV2, APIGatewayProxyResultV2 } from "aws-lambda"; import itty from "itty-router"; import { createWorkflowClient } from "src/clients"; +// TODO: remove once we can upgrade to Node 18 +import "./fetch-polyfill"; + // make the workflow client available to web hooks registerWorkflowClient(createWorkflowClient()); // initialize all web hooks onto the central HTTP router const router = itty.Router({}); -getHooks().forEach((hook) => hook(router)); + +getHooks().forEach((hook) => { + console.log("registering hook", hook.toString()); + hook(router); +}); + +router.all("*", () => new Response("Not Found.", { status: 404 })); /** * Handle inbound webhook API requests. @@ -16,16 +25,18 @@ getHooks().forEach((hook) => hook(router)); * Each webhook registers routes on the central {@link router} which * then handles the request. */ -export async function handle( +export async function processWebhook( event: APIGatewayProxyEventV2 ): Promise { - const response: Response = await router.handle({ + const request: itty.Request = { method: event.requestContext.http.method, - // TODO: is this right? - url: event.requestContext.http.path, + url: `http://localhost:3000${event.requestContext.http.path}`, params: event.pathParameters as itty.Obj, query: event.queryStringParameters as itty.Obj, - }); + }; + console.log(request); + const response: Response = await router.handle(request); + console.log(response); const headers: Record = {}; response.headers.forEach((value, key) => (headers[key] = value)); diff --git a/packages/@eventual/aws-runtime/src/injected/workflow.ts b/packages/@eventual/aws-runtime/src/injected/workflow.ts deleted file mode 100644 index 222f77326..000000000 --- a/packages/@eventual/aws-runtime/src/injected/workflow.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { workflow } from "@eventual/core"; - -export default workflow("id", async () => {}); diff --git a/packages/@eventual/compiler/src/eventual-bundle.ts b/packages/@eventual/compiler/src/eventual-bundle.ts index 1b03f315e..1c02057fe 100755 --- a/packages/@eventual/compiler/src/eventual-bundle.ts +++ b/packages/@eventual/compiler/src/eventual-bundle.ts @@ -2,8 +2,8 @@ import fs from "fs/promises"; import { constants } from "fs"; import path from "path"; import esbuild from "esbuild"; -import { eventualESPlugin } from "./esbuild-plugin"; import { esbuildPluginAliasPath } from "esbuild-plugin-alias-path"; +import { eventualESPlugin } from "./esbuild-plugin"; main().catch((err) => { console.error(err); @@ -20,65 +20,56 @@ async function main() { await prepareOutDir(outDir); await Promise.all([ - esbuild - .build({ - mainFields: ["module", "main"], - sourcemap: true, - plugins: [ - esbuildPluginAliasPath({ - alias: { "@eventual/injected/workflow": entry }, - }), - eventualESPlugin, - ], - conditions: ["module", "import", "require"], - // supported with NODE_18.x runtime - // TODO: make this configurable. - // external: ["@aws-sdk"], - platform: "node", - format: "esm", - metafile: true, - bundle: true, - entryPoints: [ - path.join( - require.resolve("@eventual/aws-runtime"), - "../../esm/entry/orchestrator.js" - ), - ], - // // ulid - banner: esmPolyfillRequireBanner(), - outfile: path.join(outDir, "orchestrator/index.mjs"), - }) - .then(writeEsBuildMetafile(path.join(outDir, "orchestrator/meta.json"))), - esbuild - .build({ - mainFields: ["module", "main"], - sourcemap: true, - plugins: [ - esbuildPluginAliasPath({ - alias: { "@eventual/injected/activities": entry }, - }), - ], - conditions: ["module", "import", "require"], - // supported with NODE_18.x runtime - // TODO: make this configurable. - // external: ["@aws-sdk"], - platform: "node", - format: "esm", - metafile: true, - bundle: true, - entryPoints: [ - path.join( - require.resolve("@eventual/aws-runtime"), - "../../esm/entry/activity-worker.js" - ), - ], - banner: esmPolyfillRequireBanner(), - outfile: path.join(outDir, "activity-worker/index.mjs"), - }) - .then( - writeEsBuildMetafile(path.join(outDir, "activity-worker/meta.json")) - ), + build({ + name: "orchestrator", + plugins: [eventualESPlugin], + }), + build({ + name: "activity", + }), + build({ + name: "webhook", + }), ]); + + async function build({ + name, + plugins, + }: { + name: string; + plugins?: esbuild.Plugin[]; + }) { + const bundle = await esbuild.build({ + mainFields: ["module", "main"], + sourcemap: true, + plugins: [ + esbuildPluginAliasPath({ + alias: { + [`@eventual/entry/injected`]: entry!, + }, + }), + ...(plugins ?? []), + ], + conditions: ["module", "import", "require"], + // supported with NODE_18.x runtime + // TODO: make this configurable. + // external: ["@aws-sdk"], + platform: "node", + format: "esm", + metafile: true, + bundle: true, + entryPoints: [ + path.join( + require.resolve("@eventual/aws-runtime"), + `../../esm/entry/${name}.js` + ), + ], + banner: esmPolyfillRequireBanner(), + outfile: path.join(outDir!, `${name}/index.mjs`), + }); + + await writeEsBuildMetafile(path.join(outDir!, `${name}/meta.json`))(bundle); + } } /** diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index 15e6f9f52..dd70a828e 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -109,13 +109,12 @@ export function workflow( if (workflows.has(name)) { throw new Error(`workflow with name '${name}' already exists`); } - const workflow: Workflow = ((input?: any) => { - return registerActivity({ + const workflow: Workflow = ((input?: any) => + registerActivity({ [EventualSymbol]: EventualKind.WorkflowCall, name, input, - }); - }) as any; + })) as any; workflow.startExecution = async function (input) { return { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0a87fab2f..9b05cf8f0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -149,6 +149,7 @@ importers: itty-router: 2.6.6 jest: ^29 micro-memoize: ^4.0.11 + node-fetch: ^3.3.0 ts-jest: ^29 ts-node: ^10.9.1 typescript: ^4.9.3 @@ -162,6 +163,7 @@ importers: aws-lambda: 1.0.7 fast-equals: 4.0.3 micro-memoize: 4.0.11 + node-fetch: 3.3.0 ulidx: 0.3.0 devDependencies: '@aws-sdk/client-dynamodb': 3.214.0 From 37dc16aa01e68e4e3db1852fb3dc6ba41a1d484b Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 25 Nov 2022 04:06:17 -0800 Subject: [PATCH 28/39] fix: parse web hooks --- apps/test-app-runtime/src/open-account.ts | 37 +++++++++++++------ apps/test-app/open-account-input.json | 11 ++++-- .../src/handlers/webhook-handler.ts | 12 ++++-- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/apps/test-app-runtime/src/open-account.ts b/apps/test-app-runtime/src/open-account.ts index 05236ae0c..1302f2c43 100644 --- a/apps/test-app-runtime/src/open-account.ts +++ b/apps/test-app-runtime/src/open-account.ts @@ -33,16 +33,19 @@ interface OpenAccountRequest { type RollbackHandler = () => Promise; -export default workflow("open-account", async (request: OpenAccountRequest) => { - try { - await createAccount(request.accountId); - } catch (err) { - console.error(err); - throw err; - } +export const openAccount = workflow( + "open-account", + async (request: OpenAccountRequest) => { + try { + await createAccount(request.accountId); + } catch (err) { + console.error(err); + throw err; + } - await associateAccountInformation(request); -}); + await associateAccountInformation(request); + } +); // sub-workflow for testing purposes export const associateAccountInformation = workflow( @@ -67,8 +70,20 @@ export const associateAccountInformation = workflow( // register a web hook API route hook((api) => { - api.get("/hello", async () => { - return new Response("hello"); + api.post("/open-account", async (request) => { + console.log(request); + const input = await request.json!(); + + const response = await openAccount.startExecution({ + input, + }); + + return new Response(JSON.stringify(response), { + headers: { + "Content-Type": "application/json", + }, + status: 200, + }); }); }); diff --git a/apps/test-app/open-account-input.json b/apps/test-app/open-account-input.json index 6038e8e61..3c82a9420 100644 --- a/apps/test-app/open-account-input.json +++ b/apps/test-app/open-account-input.json @@ -1,11 +1,16 @@ { - "accountId": "4", - "address": { "address1": "Home" }, + "accountId": "6", + "address": { + "address1": "Home" + }, "email": "test@test.com", "bankDetails": { "accountNumber": "123", "accountType": "savings", - "personOwner": { "firstName": "John", "lastName": "Smith" }, + "personOwner": { + "firstName": "John", + "lastName": "Smith" + }, "routingNumber": "345" } } diff --git a/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts index 5f4ecf2ca..064dc227b 100644 --- a/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts +++ b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts @@ -12,10 +12,7 @@ registerWorkflowClient(createWorkflowClient()); // initialize all web hooks onto the central HTTP router const router = itty.Router({}); -getHooks().forEach((hook) => { - console.log("registering hook", hook.toString()); - hook(router); -}); +getHooks().forEach((hook) => hook(router)); router.all("*", () => new Response("Not Found.", { status: 404 })); @@ -33,6 +30,13 @@ export async function processWebhook( url: `http://localhost:3000${event.requestContext.http.path}`, params: event.pathParameters as itty.Obj, query: event.queryStringParameters as itty.Obj, + async json() { + if (event.body) { + return JSON.parse(event.body!); + } else { + return undefined; + } + }, }; console.log(request); const response: Response = await router.handle(request); From 212d82f7850c8afa47eaf95fb6c1c54111989f59 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 25 Nov 2022 04:07:10 -0800 Subject: [PATCH 29/39] chore: clean --- apps/test-app-runtime/src/open-account.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/test-app-runtime/src/open-account.ts b/apps/test-app-runtime/src/open-account.ts index 1302f2c43..360faa96f 100644 --- a/apps/test-app-runtime/src/open-account.ts +++ b/apps/test-app-runtime/src/open-account.ts @@ -71,7 +71,6 @@ export const associateAccountInformation = workflow( // register a web hook API route hook((api) => { api.post("/open-account", async (request) => { - console.log(request); const input = await request.json!(); const response = await openAccount.startExecution({ From 1996fdbace0c08f1096e41d0e34fe3ecb73d4a65 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 25 Nov 2022 04:08:51 -0800 Subject: [PATCH 30/39] chore: clean --- packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts index 064dc227b..6118b232c 100644 --- a/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts +++ b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts @@ -38,10 +38,7 @@ export async function processWebhook( } }, }; - console.log(request); const response: Response = await router.handle(request); - console.log(response); - const headers: Record = {}; response.headers.forEach((value, key) => (headers[key] = value)); return { From 8b45843e675ff9f62363968112dc9f825079a822 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 25 Nov 2022 04:36:06 -0800 Subject: [PATCH 31/39] chore: move global --- packages/@eventual/core/src/global.ts | 23 +++++++++++++++++++ packages/@eventual/core/src/index.ts | 1 + .../core/src/runtime/workflow-client.ts | 22 ------------------ packages/@eventual/core/src/workflow.ts | 11 +++++---- 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/packages/@eventual/core/src/global.ts b/packages/@eventual/core/src/global.ts index 1982e9f03..2fec31db7 100644 --- a/packages/@eventual/core/src/global.ts +++ b/packages/@eventual/core/src/global.ts @@ -1,4 +1,5 @@ import type { Eventual } from "./eventual.js"; +import type { WorkflowClient } from "./runtime/workflow-client.js"; export function registerActivity(activity: A): A { activityCollector.push(activity); @@ -16,3 +17,25 @@ export function collectActivities() { resetActivityCollector(); return activities; } + +// a global variable for storing the WorkflowClient +// this is initialized by Eventual's harness lambda functions +let workflowClient: WorkflowClient; + +/** + * Register the global workflow client used by workflow functions + * to start workflows within an eventual-controlled environment. + */ +export function registerWorkflowClient(client: WorkflowClient) { + workflowClient = client; +} + +/** + * Get the global workflow client. + */ +export function getWorkflowClient(): WorkflowClient { + if (workflowClient === undefined) { + throw new Error(`WorkflowClient is not registered`); + } + return workflowClient; +} diff --git a/packages/@eventual/core/src/index.ts b/packages/@eventual/core/src/index.ts index a82be2ee2..d636797df 100644 --- a/packages/@eventual/core/src/index.ts +++ b/packages/@eventual/core/src/index.ts @@ -7,6 +7,7 @@ export * from "./error.js"; export * from "./events.js"; export * from "./eventual.js"; export * from "./execution.js"; +export * from "./global.js"; export * from "./hook.js"; export * from "./interpret.js"; export * from "./result.js"; diff --git a/packages/@eventual/core/src/runtime/workflow-client.ts b/packages/@eventual/core/src/runtime/workflow-client.ts index d6d672a2f..e0b022bf2 100644 --- a/packages/@eventual/core/src/runtime/workflow-client.ts +++ b/packages/@eventual/core/src/runtime/workflow-client.ts @@ -1,27 +1,5 @@ import { HistoryStateEvent } from "../events.js"; -// a global variable for storing the WorkflowClient -// this is initialized by Eventual's harness lambda functions -let workflowClient: WorkflowClient; - -/** - * Register the global workflow client used by workflow functions - * to start workflows within an eventual-controlled environment. - */ -export function registerWorkflowClient(client: WorkflowClient) { - workflowClient = client; -} - -/** - * Get the global workflow client. - */ -export function getWorkflowClient(): WorkflowClient { - if (workflowClient === undefined) { - throw new Error(`WorkflowClient is not registered`); - } - return workflowClient; -} - export interface WorkflowClient { /** * Start a workflow execution diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index dd70a828e..9a9921263 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -4,7 +4,11 @@ import { EventualKind, EventualSymbol, } from "./eventual.js"; -import { registerActivity, resetActivityCollector } from "./global.js"; +import { + getWorkflowClient, + registerActivity, + resetActivityCollector, +} from "./global.js"; import type { Program } from "./interpret.js"; import type { Result } from "./result.js"; import { Context, WorkflowContext } from "./context.js"; @@ -21,10 +25,7 @@ import { WorkflowEventType, } from "./events.js"; import { interpret, WorkflowResult } from "./interpret.js"; -import { - getWorkflowClient, - StartWorkflowResponse, -} from "./runtime/workflow-client.js"; +import { StartWorkflowResponse } from "./runtime/workflow-client.js"; export type WorkflowHandler = ( input: any, From 6c979a0aef83375e04cb8c6543e14cc20a4af284 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 26 Nov 2022 01:43:21 -0800 Subject: [PATCH 32/39] fix: standardize request interface for workflow runtime client --- .../src/clients/workflow-runtime-client.ts | 43 +++++++-------- .../aws-runtime/src/handlers/orchestrator.ts | 27 ++++++--- .../src/handlers/webhook-handler.ts | 32 +++++++---- packages/@eventual/core/package.json | 7 ++- packages/@eventual/core/src/hook.ts | 39 +++++++++++-- .../src/runtime/workflow-runtime-client.ts | 55 +++++++++++-------- pnpm-lock.yaml | 7 ++- 7 files changed, 132 insertions(+), 78 deletions(-) diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index a41b48d88..c31cd94af 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -21,12 +21,9 @@ import { CompleteExecution, FailedExecution, Execution, - SleepUntilCommand, - SleepForCommand, SleepScheduled, isSleepUntilCommand, WorkflowEventType, - ScheduleActivityCommand, ActivityScheduled, SleepCompleted, } from "@eventual/core"; @@ -39,7 +36,7 @@ import { ActivityWorkerRequest } from "../activity.js"; import { createEvent } from "./execution-history-client.js"; import { TimerRequestType } from "../handlers/types.js"; import { TimerClient } from "./timer-client.js"; -import type eventual from "@eventual/core"; +import eventual from "@eventual/core"; export interface WorkflowRuntimeClientProps { readonly lambda: LambdaClient; @@ -81,10 +78,10 @@ export class WorkflowRuntimeClient implements eventual.WorkflowRuntimeClient { } // TODO: etag - async updateHistory( - executionId: string, - events: HistoryStateEvent[] - ): Promise<{ bytes: number }> { + async updateHistory({ + executionId, + events, + }: eventual.UpdateHistoryRequest): Promise<{ bytes: number }> { const content = events.map((e) => JSON.stringify(e)).join("\n"); // get current history from s3 await this.props.s3.send( @@ -136,11 +133,11 @@ export class WorkflowRuntimeClient implements eventual.WorkflowRuntimeClient { return createExecutionFromResult(record) as CompleteExecution; } - async failExecution( - executionId: string, - error: string, - message: string - ): Promise { + async failExecution({ + executionId, + error, + message, + }: eventual.FailExecutionRequest): Promise { const executionResult = await this.props.dynamo.send( new UpdateItemCommand({ Key: { @@ -214,11 +211,11 @@ export class WorkflowRuntimeClient implements eventual.WorkflowRuntimeClient { ); } - async scheduleActivity( - workflowName: string, - executionId: string, - command: ScheduleActivityCommand - ) { + async scheduleActivity({ + workflowName, + executionId, + command, + }: eventual.ScheduleActivityRequest) { const request: ActivityWorkerRequest = { scheduledTime: new Date().toISOString(), workflowName, @@ -242,11 +239,11 @@ export class WorkflowRuntimeClient implements eventual.WorkflowRuntimeClient { }); } - async scheduleSleep( - executionId: string, - command: SleepUntilCommand | SleepForCommand, - baseTime: Date - ): Promise { + async scheduleSleep({ + executionId, + command, + baseTime, + }: eventual.ScheduleSleepRequest): Promise { // TODO validate const untilTime = isSleepUntilCommand(command) ? new Date(command.untilTime) diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index bf0d3757b..733b05c20 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -247,7 +247,11 @@ async function orchestrateExecution( const { bytes: historyUpdatedBytes } = await timed( metrics, OrchestratorMetrics.SaveHistoryDuration, - () => workflowRuntimeClient.updateHistory(executionId, newHistoryEvents) + () => + workflowRuntimeClient.updateHistory({ + executionId, + events: newHistoryEvents, + }) ); metrics.setProperty( @@ -285,7 +289,12 @@ async function orchestrateExecution( const execution = await timed( metrics, OrchestratorMetrics.ExecutionStatusUpdateDuration, - () => workflowRuntimeClient.failExecution(executionId, error, message) + () => + workflowRuntimeClient.failExecution({ + executionId, + error, + message, + }) ); logExecutionCompleteMetrics(execution); @@ -364,11 +373,11 @@ async function orchestrateExecution( return await Promise.all( commands.map(async (command) => { if (isScheduleActivityCommand(command)) { - await workflowRuntimeClient.scheduleActivity( - workflow.name, + await workflowRuntimeClient.scheduleActivity({ + workflowName: workflow.name, executionId, - command - ); + command, + }); return createEvent({ type: WorkflowEventType.ActivityScheduled, @@ -394,11 +403,11 @@ async function orchestrateExecution( isSleepUntilCommand(command) ) { // all sleep times are computed using the start time of the WorkflowTaskStarted - return workflowRuntimeClient.scheduleSleep( + return workflowRuntimeClient.scheduleSleep({ executionId, command, - start - ); + baseTime: start, + }); } else { return assertNever(command, `unknown command type`); } diff --git a/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts index 6118b232c..ae71c1a3b 100644 --- a/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts +++ b/packages/@eventual/aws-runtime/src/handlers/webhook-handler.ts @@ -1,7 +1,7 @@ -import { getHooks, registerWorkflowClient } from "@eventual/core"; +import { createRouter, getHooks, registerWorkflowClient } from "@eventual/core"; import { APIGatewayProxyEventV2, APIGatewayProxyResultV2 } from "aws-lambda"; import itty from "itty-router"; -import { createWorkflowClient } from "src/clients"; +import { createWorkflowClient } from "../clients/create"; // TODO: remove once we can upgrade to Node 18 import "./fetch-polyfill"; @@ -10,7 +10,7 @@ import "./fetch-polyfill"; registerWorkflowClient(createWorkflowClient()); // initialize all web hooks onto the central HTTP router -const router = itty.Router({}); +const router = createRouter(); getHooks().forEach((hook) => hook(router)); @@ -38,13 +38,21 @@ export async function processWebhook( } }, }; - const response: Response = await router.handle(request); - const headers: Record = {}; - response.headers.forEach((value, key) => (headers[key] = value)); - return { - headers, - statusCode: response.status, - body: Buffer.from(await response.arrayBuffer()).toString("base64"), - isBase64Encoded: true, - }; + try { + const response = await router.handle(request); + const headers: Record = {}; + response.headers.forEach((value, key) => (headers[key] = value)); + return { + headers, + statusCode: response.status, + body: Buffer.from(await response.arrayBuffer()).toString("base64"), + isBase64Encoded: true, + }; + } catch (err) { + console.error(err); + return { + statusCode: 500, + body: `Internal Server Error`, + }; + } } diff --git a/packages/@eventual/core/package.json b/packages/@eventual/core/package.json index 3b799e670..c38f8eb8b 100644 --- a/packages/@eventual/core/package.json +++ b/packages/@eventual/core/package.json @@ -15,10 +15,12 @@ "scripts": { "test": "jest" }, - "peerDependencies": { - "itty-router": "^2.6.6", + "dependencies": { "ulidx": "^0.3.0" }, + "peerDependencies": { + "itty-router": "^2.6.6" + }, "devDependencies": { "@types/jest": "^29", "@types/node": "^16", @@ -27,7 +29,6 @@ "ts-jest": "^29", "ts-node": "^10.9.1", "typescript": "^4.9.3", - "ulidx": "0.3.0", "zx": "^7.1.1" }, "jest": { diff --git a/packages/@eventual/core/src/hook.ts b/packages/@eventual/core/src/hook.ts index a4e4f7966..94ee3617d 100644 --- a/packages/@eventual/core/src/hook.ts +++ b/packages/@eventual/core/src/hook.ts @@ -1,15 +1,42 @@ import itty from "itty-router"; -const hooks: Hook[] = []; - -export type Hook = ( - router: itty.Router -) => void; +const hooks: Hook[] = ((globalThis as any).hooks = + (globalThis as any).hooks ?? []); export function hook(setup: Hook) { hooks.push(setup); } export function getHooks() { - return [...hooks]; + return hooks.slice(); } + +export type Hook = (router: Router) => void; + +export function createRouter(): Router { + return itty.Router() as any as Router; +} + +export type RouteHandler = ( + request: itty.Request, + ...args: any +) => Response | Promise; + +export type Route = (path: string, ...handlers: RouteHandler[]) => Router; + +export interface Router { + handle: (request: itty.Request, ...extra: any) => Promise; + routes: RouteEntry[]; + all: Route; + get: Route; + head: Route; + post: Route; + put: Route; + delete: Route; + connect: Route; + options: Route; + trace: Route; + patch: Route; +} + +export type RouteEntry = [string, RegExp, RouteHandler]; diff --git a/packages/@eventual/core/src/runtime/workflow-runtime-client.ts b/packages/@eventual/core/src/runtime/workflow-runtime-client.ts index 1fe49d87d..e32a6ce66 100644 --- a/packages/@eventual/core/src/runtime/workflow-runtime-client.ts +++ b/packages/@eventual/core/src/runtime/workflow-runtime-client.ts @@ -17,37 +17,46 @@ export interface CompleteExecutionRequest { readonly timerClient: TimerClient; } +export interface FailExecutionRequest { + executionId: string; + error: string; + message: string; +} + +export interface UpdateHistoryRequest { + executionId: string; + events: HistoryStateEvent[]; +} + +export interface ScheduleActivityRequest { + workflowName: string; + executionId: string; + command: ScheduleActivityCommand; +} + +export interface ScheduleSleepRequest { + executionId: string; + command: SleepUntilCommand | SleepForCommand; + baseTime: Date; +} + export interface WorkflowRuntimeClient { getHistory(executionId: string): Promise; // TODO: etag - updateHistory( - executionId: string, - events: HistoryStateEvent[] - ): Promise<{ bytes: number }>; - - completeExecution({ - executionId, - result, - }: CompleteExecutionRequest): Promise; - - failExecution( - executionId: string, - error: string, - message: string - ): Promise; + updateHistory(request: UpdateHistoryRequest): Promise<{ bytes: number }>; + + completeExecution( + request: CompleteExecutionRequest + ): Promise; + + failExecution(request: FailExecutionRequest): Promise; getExecutions(): Promise; scheduleActivity( - workflowName: string, - executionId: string, - command: ScheduleActivityCommand + request: ScheduleActivityRequest ): Promise; - scheduleSleep( - executionId: string, - command: SleepUntilCommand | SleepForCommand, - baseTime: Date - ): Promise; + scheduleSleep(request: ScheduleSleepRequest): Promise; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9b05cf8f0..b5efa8bfc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -279,8 +279,10 @@ importers: ts-jest: ^29 ts-node: ^10.9.1 typescript: ^4.9.3 - ulidx: 0.3.0 + ulidx: ^0.3.0 zx: ^7.1.1 + dependencies: + ulidx: 0.3.0 devDependencies: '@types/jest': 29.2.2 '@types/node': 16.18.3 @@ -289,7 +291,6 @@ importers: ts-jest: 29.0.3_6crhf7ajeizammv76u753sn6i4 ts-node: 10.9.1_fvpuwgkpfe3dm3hnpcpbcxmb3y typescript: 4.9.3 - ulidx: 0.3.0 zx: 7.1.1 packages: @@ -6624,6 +6625,7 @@ packages: /layerr/0.1.2: resolution: {integrity: sha512-ob5kTd9H3S4GOG2nVXyQhOu9O8nBgP555XxWPkJI0tR0JeRilfyTp8WtPdIJHLXBmHMSdEq5+KMxiYABeScsIQ==} + dev: false /lerna/5.6.2: resolution: {integrity: sha512-Y0yMPslvnBnTZi7Nrs/gDyYZYauNf61xWNCehISHIORxZmmpoluNkcWTfcyb47is5uJQCv5QJX5xKKubbs+a6w==} @@ -8739,6 +8741,7 @@ packages: resolution: {integrity: sha512-Qvpa2xAzS6fBUpiqHSHWvn6XiSLCAPyNDDz035vsEWmUoXRqC4c9JySLIfdBuK0N1xGBxng6GHDOZgyNQfxAHg==} dependencies: layerr: 0.1.2 + dev: false /unique-filename/2.0.1: resolution: {integrity: sha512-ODWHtkkdx3IAR+veKxFV+VBkUMcN+FaqzUUd7IZzt+0zhDZFPFxhlqwPF3YQvMHx1TD0tdgYl+kuPnJ8E6ql7A==} From 105782240fcf4c551191c2271b99f8056e71ca82 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 26 Nov 2022 01:46:43 -0800 Subject: [PATCH 33/39] chore: feedback --- .../src/runtime/workflow-runtime-client.ts | 2 -- packages/@eventual/core/src/workflow.ts | 28 +++++++++++-------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/packages/@eventual/core/src/runtime/workflow-runtime-client.ts b/packages/@eventual/core/src/runtime/workflow-runtime-client.ts index e32a6ce66..70188398b 100644 --- a/packages/@eventual/core/src/runtime/workflow-runtime-client.ts +++ b/packages/@eventual/core/src/runtime/workflow-runtime-client.ts @@ -9,12 +9,10 @@ import { SleepScheduled, } from "../events.js"; import { CompleteExecution, Execution, FailedExecution } from "../execution.js"; -import { TimerClient } from "./timer-client.js"; export interface CompleteExecutionRequest { executionId: string; result?: any; - readonly timerClient: TimerClient; } export interface FailExecutionRequest { diff --git a/packages/@eventual/core/src/workflow.ts b/packages/@eventual/core/src/workflow.ts index 9a9921263..ac455ca34 100644 --- a/packages/@eventual/core/src/workflow.ts +++ b/packages/@eventual/core/src/workflow.ts @@ -32,6 +32,19 @@ export type WorkflowHandler = ( context: Context ) => Promise | Program; +export interface StartExecutionRequest { + /** + * Input payload for the workflow. + */ + input: Input; + /** + * Optional name of the workflow to start - used to determine the unique ID and enforce idempotency. + * + * @default - a unique ID is generated. + */ + name?: string; +} + /** * A {@link Workflow} is a long-running process that orchestrates calls * to other services in a durable and observable way. @@ -55,18 +68,9 @@ export interface Workflow { /** * Starts a workflow execution */ - startExecution(request: { - /** - * Input payload for the workflow. - */ - input: Parameters[0]; - /** - * Optional name of the workflow to start - used to determine the unique ID and enforce idempotency. - * - * @default - a unique ID is generated. - */ - name?: string; - }): Promise; + startExecution( + request: StartExecutionRequest[0]> + ): Promise; /** * @internal - this is the internal DSL representation that produces a {@link Program} instead of a Promise. From a9ea10412db0749c31188bf56dd67660180905b5 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 26 Nov 2022 01:54:30 -0800 Subject: [PATCH 34/39] chore: feedback --- .../src/clients/activity-runtime-client.ts | 8 ++++--- .../aws-runtime/src/clients/create.ts | 22 ++++++++--------- .../src/clients/execution-history-client.ts | 8 ++++--- .../aws-runtime/src/clients/timer-client.ts | 8 +++---- .../src/clients/workflow-client.ts | 10 ++++---- .../src/clients/workflow-runtime-client.ts | 24 ++++++++----------- .../src/handlers/activity-worker.ts | 4 ++++ .../aws-runtime/src/handlers/orchestrator.ts | 3 --- 8 files changed, 44 insertions(+), 43 deletions(-) diff --git a/packages/@eventual/aws-runtime/src/clients/activity-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/activity-runtime-client.ts index a79e60600..ddcb02a68 100644 --- a/packages/@eventual/aws-runtime/src/clients/activity-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/activity-runtime-client.ts @@ -5,13 +5,15 @@ import { } from "@aws-sdk/client-dynamodb"; import type eventual from "@eventual/core"; -export interface ActivityRuntimeClientProps { +export interface AWSActivityRuntimeClientProps { dynamo: DynamoDBClient; activityLockTableName: string; } -export class ActivityRuntimeClient implements eventual.ActivityRuntimeClient { - constructor(private props: ActivityRuntimeClientProps) {} +export class AWSActivityRuntimeClient + implements eventual.ActivityRuntimeClient +{ + constructor(private props: AWSActivityRuntimeClientProps) {} /** * Claims a activity for an actor. diff --git a/packages/@eventual/aws-runtime/src/clients/create.ts b/packages/@eventual/aws-runtime/src/clients/create.ts index ea13c58ba..095fca8c6 100644 --- a/packages/@eventual/aws-runtime/src/clients/create.ts +++ b/packages/@eventual/aws-runtime/src/clients/create.ts @@ -3,14 +3,14 @@ import { LambdaClient } from "@aws-sdk/client-lambda"; import { S3Client } from "@aws-sdk/client-s3"; import { SQSClient } from "@aws-sdk/client-sqs"; import * as env from "../env"; -import { ActivityRuntimeClient } from "./activity-runtime-client"; -import { ExecutionHistoryClient } from "./execution-history-client"; -import { WorkflowClient } from "./workflow-client"; -import { WorkflowRuntimeClient } from "./workflow-runtime-client"; +import { AWSActivityRuntimeClient } from "./activity-runtime-client"; +import { AWSExecutionHistoryClient } from "./execution-history-client"; +import { AWSWorkflowClient } from "./workflow-client"; +import { AWSWorkflowRuntimeClient } from "./workflow-runtime-client"; import memoize from "micro-memoize"; import { deepEqual } from "fast-equals"; import { SchedulerClient } from "@aws-sdk/client-scheduler"; -import { TimerClient, TimerClientProps } from "./timer-client"; +import { AWSTimerClient, AWSTimerClientProps } from "./timer-client"; /** * Client creators to be used by the lambda functions. @@ -29,7 +29,7 @@ export const scheduler = /*@__PURE__*/ memoize(() => new SchedulerClient({})); export const createExecutionHistoryClient = /*@__PURE__*/ memoize( ({ tableName }: { tableName?: string } = {}) => - new ExecutionHistoryClient({ + new AWSExecutionHistoryClient({ dynamo: dynamo(), tableName: tableName ?? env.tableName(), }), @@ -44,7 +44,7 @@ export const createWorkflowClient = /*@__PURE__*/ memoize( tableName?: string; workflowQueueUrl?: string; } = {}) => - new WorkflowClient({ + new AWSWorkflowClient({ sqs: sqs(), workflowQueueUrl: workflowQueueUrl ?? env.workflowQueueUrl(), executionHistory: createExecutionHistoryClient({ @@ -58,15 +58,15 @@ export const createWorkflowClient = /*@__PURE__*/ memoize( export const createActivityRuntimeClient = /*@__PURE__*/ memoize( () => - new ActivityRuntimeClient({ + new AWSActivityRuntimeClient({ activityLockTableName: env.activityLockTableName(), dynamo: dynamo(), }) ); export const createTimerClient = /*@__PURE__*/ memoize( - (props: Partial = {}) => - new TimerClient({ + (props: Partial = {}) => + new AWSTimerClient({ scheduler: props.scheduler ?? scheduler(), schedulerRoleArn: props.schedulerRoleArn ?? env.schedulerRoleArn(), schedulerDlqArn: props.schedulerDlqArn ?? env.schedulerDlqArn(), @@ -90,7 +90,7 @@ export const createWorkflowRuntimeClient = /*@__PURE__*/ memoize( executionHistoryBucket?: string; activityWorkerFunctionName?: string; } = {}) => - new WorkflowRuntimeClient({ + new AWSWorkflowRuntimeClient({ dynamo: dynamo(), s3: s3(), // todo fail when missing diff --git a/packages/@eventual/aws-runtime/src/clients/execution-history-client.ts b/packages/@eventual/aws-runtime/src/clients/execution-history-client.ts index 41fc3aaea..b4c0dffda 100644 --- a/packages/@eventual/aws-runtime/src/clients/execution-history-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/execution-history-client.ts @@ -15,15 +15,17 @@ import type eventual from "@eventual/core"; import { ulid } from "ulidx"; -export interface ExecutionHistoryClientProps { +export interface AWSExecutionHistoryClientProps { readonly dynamo: DynamoDBClient; readonly tableName: string; } type UnresolvedEvent = Omit; -export class ExecutionHistoryClient implements eventual.ExecutionHistoryClient { - constructor(private props: ExecutionHistoryClientProps) {} +export class AWSExecutionHistoryClient + implements eventual.ExecutionHistoryClient +{ + constructor(private props: AWSExecutionHistoryClientProps) {} public async createAndPutEvent( executionId: string, diff --git a/packages/@eventual/aws-runtime/src/clients/timer-client.ts b/packages/@eventual/aws-runtime/src/clients/timer-client.ts index 721345609..5753790f4 100644 --- a/packages/@eventual/aws-runtime/src/clients/timer-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/timer-client.ts @@ -15,7 +15,7 @@ import { } from "../handlers/types.js"; import type eventual from "@eventual/core"; -export interface TimerClientProps { +export interface AWSTimerClientProps { readonly scheduler: SchedulerClient; readonly schedulerRoleArn: string; readonly schedulerDlqArn: string; @@ -30,15 +30,15 @@ export interface TimerClientProps { readonly scheduleForwarderArn: string; } -export class TimerClient implements eventual.TimerClient { - constructor(private props: TimerClientProps) {} +export class AWSTimerClient implements eventual.TimerClient { + constructor(private props: AWSTimerClientProps) {} /** * Starts a timer using SQS's message delay. * * The timerRequest.untilTime may only be 15 minutes or fewer in the future. * - * For longer use {@link TimerClient.startTimer}. + * For longer use {@link AWSTimerClient.startTimer}. * * The SQS Queue will delay for floor(untilTime - currentTime) seconds until the timer handler can pick up the message. * diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts index 277fcb896..f6599c4ad 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts @@ -14,20 +14,20 @@ import { } from "@eventual/core"; import { ulid } from "ulidx"; import { formatExecutionId } from "../execution-id.js"; -import { ExecutionHistoryClient } from "./execution-history-client.js"; +import { AWSExecutionHistoryClient } from "./execution-history-client.js"; import type eventual from "@eventual/core"; -export interface WorkflowClientProps { +export interface AWSWorkflowClientProps { readonly dynamo: DynamoDBClient; readonly tableName: string; readonly sqs: SQSClient; readonly workflowQueueUrl: string; - readonly executionHistory: ExecutionHistoryClient; + readonly executionHistory: AWSExecutionHistoryClient; } -export class WorkflowClient implements eventual.WorkflowClient { - constructor(private props: WorkflowClientProps) {} +export class AWSWorkflowClient implements eventual.WorkflowClient { + constructor(private props: AWSWorkflowClientProps) {} /** * Start a workflow execution diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts index c31cd94af..defdf3a58 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-runtime-client.ts @@ -30,33 +30,29 @@ import { import { createExecutionFromResult, ExecutionRecord, - WorkflowClient, + AWSWorkflowClient, } from "./workflow-client.js"; import { ActivityWorkerRequest } from "../activity.js"; import { createEvent } from "./execution-history-client.js"; import { TimerRequestType } from "../handlers/types.js"; -import { TimerClient } from "./timer-client.js"; +import { AWSTimerClient } from "./timer-client.js"; import eventual from "@eventual/core"; -export interface WorkflowRuntimeClientProps { +export interface AWSWorkflowRuntimeClientProps { readonly lambda: LambdaClient; readonly activityWorkerFunctionName: string; readonly dynamo: DynamoDBClient; readonly s3: S3Client; readonly executionHistoryBucket: string; readonly tableName: string; - readonly workflowClient: WorkflowClient; - readonly timerClient: TimerClient; + readonly workflowClient: AWSWorkflowClient; + readonly timerClient: AWSTimerClient; } -export interface CompleteExecutionRequest { - executionId: string; - result?: any; - readonly timerClient: TimerClient; -} - -export class WorkflowRuntimeClient implements eventual.WorkflowRuntimeClient { - constructor(private props: WorkflowRuntimeClientProps) {} +export class AWSWorkflowRuntimeClient + implements eventual.WorkflowRuntimeClient +{ + constructor(private props: AWSWorkflowRuntimeClientProps) {} async getHistory(executionId: string) { try { @@ -97,7 +93,7 @@ export class WorkflowRuntimeClient implements eventual.WorkflowRuntimeClient { async completeExecution({ executionId, result, - }: CompleteExecutionRequest): Promise { + }: eventual.CompleteExecutionRequest): Promise { const executionResult = await this.props.dynamo.send( new UpdateItemCommand({ Key: { diff --git a/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts b/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts index 874f4b9ef..b8ff90791 100644 --- a/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/activity-worker.ts @@ -4,6 +4,7 @@ import { getCallableActivity, getCallableActivityNames, isWorkflowFailed, + registerWorkflowClient, WorkflowEventType, } from "@eventual/core"; import { Handler } from "aws-lambda"; @@ -24,6 +25,9 @@ const activityRuntimeClient = createActivityRuntimeClient(); const executionHistoryClient = createExecutionHistoryClient(); const workflowClient = createWorkflowClient(); +// make the workflow client available to all activity code +registerWorkflowClient(workflowClient); + export const activityWorker = (): Handler => { return middy( metricScope((metrics) => async (request: ActivityWorkerRequest) => { diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index 733b05c20..e334f9b0a 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -34,7 +34,6 @@ import { inspect } from "util"; import { createEvent } from "../clients/execution-history-client.js"; import { createExecutionHistoryClient, - createTimerClient, createWorkflowClient, createWorkflowRuntimeClient, } from "../clients/index.js"; @@ -48,7 +47,6 @@ import { promiseAllSettledPartitioned } from "../utils.js"; const executionHistoryClient = createExecutionHistoryClient(); const workflowRuntimeClient = createWorkflowRuntimeClient(); const workflowClient = createWorkflowClient(); -const timerClient = createTimerClient(); /** * Creates an entrypoint function for orchestrating a workflow. @@ -313,7 +311,6 @@ async function orchestrateExecution( workflowRuntimeClient.completeExecution({ executionId, result: result.value, - timerClient, }) ); logExecutionCompleteMetrics(execution); From c0eae280ce0ded905340c07914e020de0544e199 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 26 Nov 2022 01:58:21 -0800 Subject: [PATCH 35/39] chore: add webhookEndpointUrl --- packages/@eventual/aws-cdk/src/service.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/@eventual/aws-cdk/src/service.ts b/packages/@eventual/aws-cdk/src/service.ts index b3651ac88..afed8a683 100644 --- a/packages/@eventual/aws-cdk/src/service.ts +++ b/packages/@eventual/aws-cdk/src/service.ts @@ -6,6 +6,7 @@ import { IFunction, Runtime, FunctionUrlAuthType, + FunctionUrl, } from "aws-cdk-lib/aws-lambda"; import { Construct } from "constructs"; import { Bucket, IBucket } from "aws-cdk-lib/aws-s3"; @@ -112,6 +113,11 @@ export class Service extends Construct implements IGrantable { */ public readonly webhookEndpoint: IFunction; + /** + * The URL of the webhook endpoint. + */ + public readonly webhookEndpointUrl: FunctionUrl; + readonly grantPrincipal: IPrincipal; constructor(scope: Construct, id: string, props: WorkflowProps) { @@ -394,7 +400,7 @@ export class Service extends Construct implements IGrantable { [ENV_NAMES.EVENTUAL_WEBHOOK]: "1", }, }); - this.webhookEndpoint.addFunctionUrl({ + this.webhookEndpointUrl = this.webhookEndpoint.addFunctionUrl({ authType: FunctionUrlAuthType.NONE, }); From 6500fdc482d3539ecd7be75cd5c3defc24daf9e7 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 26 Nov 2022 02:00:46 -0800 Subject: [PATCH 36/39] chore: feedback --- packages/@eventual/aws-cdk/src/service.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/@eventual/aws-cdk/src/service.ts b/packages/@eventual/aws-cdk/src/service.ts index afed8a683..f0ffef304 100644 --- a/packages/@eventual/aws-cdk/src/service.ts +++ b/packages/@eventual/aws-cdk/src/service.ts @@ -45,6 +45,9 @@ export interface WorkflowProps { } export class Service extends Construct implements IGrantable { + /** + * Name of this Service. + */ public readonly serviceName: string; /** * S3 bucket that contains events necessary to replay a workflow execution. @@ -107,12 +110,10 @@ export class Service extends Construct implements IGrantable { * Timers - When the EventBridge scheduler fails to invoke the Schedule Forwarder Lambda. */ public readonly dlq: Queue; - /** - * A Lambda Function URL endpoint for accepting inbound webhook requests. + * A Lambda Function for processing inbound webhook requests. */ public readonly webhookEndpoint: IFunction; - /** * The URL of the webhook endpoint. */ From 41a2434b5b8018c6e83916abb409e8d86ecccdac Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 26 Nov 2022 02:02:33 -0800 Subject: [PATCH 37/39] chore: feedback --- .../@eventual/aws-runtime/src/clients/create.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/@eventual/aws-runtime/src/clients/create.ts b/packages/@eventual/aws-runtime/src/clients/create.ts index 095fca8c6..3ee1fecd2 100644 --- a/packages/@eventual/aws-runtime/src/clients/create.ts +++ b/packages/@eventual/aws-runtime/src/clients/create.ts @@ -82,9 +82,9 @@ export const createTimerClient = /*@__PURE__*/ memoize( export const createWorkflowRuntimeClient = /*@__PURE__*/ memoize( ({ - tableName, - executionHistoryBucket, - activityWorkerFunctionName, + tableName = env.tableName(), + executionHistoryBucket = env.executionHistoryBucket(), + activityWorkerFunctionName = env.activityWorkerFunctionName(), }: { tableName?: string; executionHistoryBucket?: string; @@ -94,12 +94,10 @@ export const createWorkflowRuntimeClient = /*@__PURE__*/ memoize( dynamo: dynamo(), s3: s3(), // todo fail when missing - executionHistoryBucket: - executionHistoryBucket ?? env.executionHistoryBucket(), - tableName: tableName ?? env.tableName(), + executionHistoryBucket, + tableName, lambda: lambda(), - activityWorkerFunctionName: - activityWorkerFunctionName ?? env.activityWorkerFunctionName(), + activityWorkerFunctionName, workflowClient: createWorkflowClient(), timerClient: createTimerClient(), }) From 0d924e85880a023447380f59b580b2768628b430 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 26 Nov 2022 02:25:50 -0800 Subject: [PATCH 38/39] feat: CFWorkflowClient --- .../src/clients/workflow-client.ts | 2 + .../@eventual/aws-runtime/src/execution-id.ts | 19 ---- .../aws-runtime/src/handlers/orchestrator.ts | 3 +- .../cloudflare-runtime/src/workflow-client.ts | 96 +++++++++++++++++++ packages/@eventual/core/src/execution.ts | 2 + 5 files changed, 102 insertions(+), 20 deletions(-) delete mode 100644 packages/@eventual/aws-runtime/src/execution-id.ts create mode 100644 packages/@eventual/cloudflare-runtime/src/workflow-client.ts diff --git a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts index fd597c662..b27e583c7 100644 --- a/packages/@eventual/aws-runtime/src/clients/workflow-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/workflow-client.ts @@ -156,5 +156,7 @@ export function createExecutionFromResult( result: execution.result ? JSON.parse(execution.result.S) : undefined, startTime: execution.startTime.S, status: execution.status.S, + name: execution.name.S, + workflowName: execution.workflowName.S, } as Execution; } diff --git a/packages/@eventual/aws-runtime/src/execution-id.ts b/packages/@eventual/aws-runtime/src/execution-id.ts deleted file mode 100644 index 43fa1ec64..000000000 --- a/packages/@eventual/aws-runtime/src/execution-id.ts +++ /dev/null @@ -1,19 +0,0 @@ -export type ExecutionID< - WorkflowName extends string = string, - ID extends string = string -> = `${WorkflowName}/${ID}`; - -export function isExecutionId(a: any): a is ExecutionID { - return typeof a === "string" && a.split("/").length === 2; -} - -export function parseWorkflowName(executionId: ExecutionID): string { - return executionId.split("/")[0]!; -} - -export function formatExecutionId< - WorkflowName extends string, - ID extends string ->(workflowName: string, id: string): ExecutionID { - return `${workflowName}/${id}` as ExecutionID; -} diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index bbe0de8d7..bfccc115b 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -26,6 +26,8 @@ import { WorkflowFailed, WorkflowTaskCompleted, WorkflowTaskStarted, + isExecutionId, + parseWorkflowName, } from "@eventual/core"; import middy from "@middy/core"; import { createMetricsLogger, MetricsLogger, Unit } from "aws-embedded-metrics"; @@ -38,7 +40,6 @@ import { createWorkflowRuntimeClient, } from "../clients/index.js"; import { SQSWorkflowTaskMessage } from "../clients/workflow-client.js"; -import { isExecutionId, parseWorkflowName } from "../execution-id.js"; import { logger, loggerMiddlewares } from "../logger.js"; import { MetricsCommon, OrchestratorMetrics } from "../metrics/constants.js"; import { timed, timedSync } from "../metrics/utils.js"; diff --git a/packages/@eventual/cloudflare-runtime/src/workflow-client.ts b/packages/@eventual/cloudflare-runtime/src/workflow-client.ts new file mode 100644 index 000000000..8f3581299 --- /dev/null +++ b/packages/@eventual/cloudflare-runtime/src/workflow-client.ts @@ -0,0 +1,96 @@ +import { ulid } from "ulidx"; +import { + StartWorkflowRequest, + formatExecutionId, + ExecutionHistoryClient, + WorkflowClient, + ExecutionStatus, + WorkflowStarted, + WorkflowEventType, + HistoryStateEvent, + WorkflowTask, + Execution, +} from "@eventual/core"; +import { KVNamespace, Queue } from "@cloudflare/workers-types"; + +export interface CFWorkflowClientProps { + readonly kv: KVNamespace; + readonly tableName: string; + readonly queue: Queue; + readonly workflowQueueUrl: string; + readonly executionHistory: ExecutionHistoryClient; +} + +export class CFWorkflowClient implements WorkflowClient { + constructor(private props: CFWorkflowClientProps) {} + + /** + * Start a workflow execution + * @param name Suffix of execution id + * @param input Workflow parameters + * @returns + */ + public async startWorkflow({ + executionName = ulid(), + workflowName, + input, + parentExecutionId, + seq, + }: StartWorkflowRequest) { + const executionId = formatExecutionId(workflowName, executionName); + console.log("execution input:", input); + + await this.props.kv.put( + executionId, + JSON.stringify({ + id: executionId, + name: executionName, + workflowName: workflowName, + status: ExecutionStatus.IN_PROGRESS, + startTime: new Date().toISOString(), + ...(parentExecutionId + ? { + parentExecutionId, + seq, + } + : {}), + } satisfies Execution) + ); + + const workflowStartedEvent = + await this.props.executionHistory.createAndPutEvent( + executionId, + { + type: WorkflowEventType.WorkflowStarted, + input, + workflowName, + context: { + name: executionName, + parentId: parentExecutionId, + }, + } + ); + + await this.submitWorkflowTask(executionId, workflowStartedEvent); + + return executionId; + } + + public async submitWorkflowTask( + executionId: string, + ...events: HistoryStateEvent[] + ) { + // send workflow task to workflow queue + + await this.props.queue.send({ + task: { + executionId, + events, + }, + } satisfies QueueWorkflowTaskMessage); + } +} + +export interface QueueWorkflowTaskMessage { + task: WorkflowTask; +} diff --git a/packages/@eventual/core/src/execution.ts b/packages/@eventual/core/src/execution.ts index 2ade8779f..c470d1f9f 100644 --- a/packages/@eventual/core/src/execution.ts +++ b/packages/@eventual/core/src/execution.ts @@ -8,6 +8,8 @@ interface ExecutionBase { id: string; status: ExecutionStatus; startTime: string; + name: string; + workflowName: string; } export type Execution = From c79a6a1976f832692ffc094917f733f7f6f89dd4 Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 27 Nov 2022 08:56:41 -0800 Subject: [PATCH 39/39] fet: add parentExecutionId and seq --- .../src/workflow-runtime-client.ts | 262 ++++++++++++++++++ packages/@eventual/core/src/execution.ts | 2 + 2 files changed, 264 insertions(+) create mode 100644 packages/@eventual/cloudflare-runtime/src/workflow-runtime-client.ts diff --git a/packages/@eventual/cloudflare-runtime/src/workflow-runtime-client.ts b/packages/@eventual/cloudflare-runtime/src/workflow-runtime-client.ts new file mode 100644 index 000000000..f45d08f6c --- /dev/null +++ b/packages/@eventual/cloudflare-runtime/src/workflow-runtime-client.ts @@ -0,0 +1,262 @@ +import { + ExecutionStatus, + HistoryStateEvent, + CompleteExecution, + FailedExecution, + Execution, + SleepScheduled, + isSleepUntilCommand, + WorkflowEventType, + ActivityScheduled, + SleepCompleted, + WorkflowClient, + TimerClient, +} from "@eventual/core"; + +import eventual from "@eventual/core"; + +import { KVNamespace, R2Bucket, R2ObjectBody } from "@cloudflare/workers-types"; + +export interface AWSWorkflowRuntimeClientProps { + readonly lambda: LambdaClient; + readonly activityWorkerFunctionName: string; + readonly kv: KVNamespace; + readonly executionHistoryBucket: R2Bucket; + readonly tableName: string; + readonly workflowClient: WorkflowClient; + readonly timerClient: TimerClient; +} + +export class AWSWorkflowRuntimeClient + implements eventual.WorkflowRuntimeClient +{ + constructor(private props: AWSWorkflowRuntimeClientProps) {} + + async getHistory(executionId: string) { + try { + // get current history from s3 + const historyObject = await this.props.executionHistoryBucket.get( + formatExecutionHistoryKey(executionId) + ); + if (historyObject) { + return historyEntryToEvents(historyObject); + } else { + return []; + } + } catch (err) { + if (err instanceof NoSuchKey) { + return []; + } + throw err; + } + } + + // TODO: etag + async updateHistory({ + executionId, + events, + }: eventual.UpdateHistoryRequest): Promise<{ bytes: number }> { + const content = events.map((e) => JSON.stringify(e)).join("\n"); + + await this.props.executionHistoryBucket.put( + formatExecutionHistoryKey(executionId), + content + ); + + return { bytes: content.length }; + } + + async completeExecution({ + executionId, + result, + }: eventual.CompleteExecutionRequest): Promise { + const executionSer = await this.props.kv.get(executionId); + if (executionSer === null) { + throw new Error(`execution does not exist: ${executionId}`); + } + const execution: Execution = JSON.parse(executionSer); + + const complete: CompleteExecution = { + ...execution, + endTime: new Date().toISOString(), + result, + status: ExecutionStatus.COMPLETE, + }; + + if (execution.status === ExecutionStatus.IN_PROGRESS) { + await this.props.kv.put(executionId, JSON.stringify(complete)); + } else { + console.warn( + `skipping update of execution '${executionId}' because it has already closed` + ); + } + + if (execution.parentExecutionId) { + await this.reportCompletionToParent( + execution.parentExecutionId, + execution.seq!, + result + ); + } + + return complete; + } + + async failExecution({ + executionId, + error, + message, + }: eventual.FailExecutionRequest): Promise { + const executionSer = await this.props.kv.get(executionId); + if (executionSer === null) { + throw new Error(`execution does not exist: ${executionId}`); + } + const execution: Execution = JSON.parse(executionSer); + + const failed: FailedExecution = { + ...execution, + endTime: new Date().toISOString(), + status: ExecutionStatus.FAILED, + error, + message, + }; + + if (execution.status === ExecutionStatus.FAILED) { + await this.props.kv.put(executionId, JSON.stringify(failed)); + } else { + console.warn( + `skipping update of execution '${executionId}' because it has already closed` + ); + } + + if (execution.parentExecutionId) { + await this.reportCompletionToParent( + execution.parentExecutionId, + execution.seq!, + error, + message + ); + } + + if (failed.parentExecutionId) { + await this.reportCompletionToParent( + failed.parentExecutionId, + failed.seq!, + error, + message + ); + } + + return failed; + } + + private async reportCompletionToParent( + parentExecutionId: string, + seq: number, + ...args: [result: any] | [error: string, message: string] + ) { + await this.props.workflowClient.submitWorkflowTask(parentExecutionId, { + seq, + timestamp: new Date().toISOString(), + ...(args.length === 1 + ? { + type: WorkflowEventType.ChildWorkflowCompleted, + result: args[0], + } + : { + type: WorkflowEventType.ChildWorkflowFailed, + error: args[0], + message: args[1], + }), + }); + } + + async getExecutions(): Promise { + const executions = await this.props.dynamo.send( + new QueryCommand({ + TableName: this.props.tableName, + KeyConditionExpression: "pk = :pk", + ExpressionAttributeValues: { + ":pk": { S: ExecutionRecord.PRIMARY_KEY }, + }, + }) + ); + return executions.Items!.map((execution) => + createExecutionFromResult(execution as ExecutionRecord) + ); + } + + async scheduleActivity({ + workflowName, + executionId, + command, + }: eventual.ScheduleActivityRequest) { + const request: ActivityWorkerRequest = { + scheduledTime: new Date().toISOString(), + workflowName, + executionId, + command, + retry: 0, + }; + + await this.props.lambda.send( + new InvokeCommand({ + FunctionName: this.props.activityWorkerFunctionName, + Payload: Buffer.from(JSON.stringify(request)), + InvocationType: InvocationType.Event, + }) + ); + + return createEvent({ + type: WorkflowEventType.ActivityScheduled, + seq: command.seq, + name: command.name, + }); + } + + async scheduleSleep({ + executionId, + command, + baseTime, + }: eventual.ScheduleSleepRequest): Promise { + // TODO validate + const untilTime = isSleepUntilCommand(command) + ? new Date(command.untilTime) + : new Date(baseTime.getTime() + command.durationSeconds * 1000); + const untilTimeIso = untilTime.toISOString(); + + const sleepCompletedEvent: SleepCompleted = { + type: WorkflowEventType.SleepCompleted, + seq: command.seq, + timestamp: untilTimeIso, + }; + + await this.props.timerClient.startTimer({ + type: TimerRequestType.ForwardEvent, + event: sleepCompletedEvent, + untilTime: untilTimeIso, + executionId, + }); + + return createEvent({ + type: WorkflowEventType.SleepScheduled, + seq: command.seq, + untilTime: untilTime.toISOString(), + }); + } +} + +async function historyEntryToEvents( + objectOutput: R2ObjectBody +): Promise { + if (objectOutput.body) { + return (await objectOutput.text()) + .split("\n") + .map((l) => JSON.parse(l)) as HistoryStateEvent[]; + } + return []; +} + +function formatExecutionHistoryKey(executionId: string) { + return `executionHistory/${executionId}`; +} diff --git a/packages/@eventual/core/src/execution.ts b/packages/@eventual/core/src/execution.ts index c470d1f9f..5ebf3c638 100644 --- a/packages/@eventual/core/src/execution.ts +++ b/packages/@eventual/core/src/execution.ts @@ -10,6 +10,8 @@ interface ExecutionBase { startTime: string; name: string; workflowName: string; + parentExecutionId?: string; + seq?: number; } export type Execution =