diff --git a/packages/kn-plugin-workflow/pkg/specs/openapi_minifier.go b/packages/kn-plugin-workflow/pkg/specs/openapi_minifier.go index 09071c81263..7e86a83af89 100644 --- a/packages/kn-plugin-workflow/pkg/specs/openapi_minifier.go +++ b/packages/kn-plugin-workflow/pkg/specs/openapi_minifier.go @@ -99,10 +99,7 @@ func (m *OpenApiMinifier) processFunction(workflowFile string) error { return err } - relativePath, err := filepath.Rel(filepath.Dir(workflowFile), m.params.SpecsDir) - if err != nil { - return err - } + relativePath := filepath.Base(m.params.SpecsDir) if workflow.Functions == nil { return nil @@ -110,7 +107,7 @@ func (m *OpenApiMinifier) processFunction(workflowFile string) error { for _, function := range workflow.Functions { if strings.HasPrefix(function.Operation, relativePath) { - trimmedPrefix := strings.TrimPrefix(function.Operation, relativePath+"/") + trimmedPrefix := strings.TrimPrefix(function.Operation, relativePath+string(os.PathSeparator)) if !strings.Contains(trimmedPrefix, "#") { return fmt.Errorf("Invalid operation format in function: %s", function.Operation) } diff --git a/packages/kn-plugin-workflow/pkg/specs/openapi_minifier_test.go b/packages/kn-plugin-workflow/pkg/specs/openapi_minifier_test.go index d9b7c445b75..6a015800877 100644 --- a/packages/kn-plugin-workflow/pkg/specs/openapi_minifier_test.go +++ b/packages/kn-plugin-workflow/pkg/specs/openapi_minifier_test.go @@ -34,9 +34,15 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" ) +type spec struct { + file string + initial int + minified int +} + type minifyTest struct { workflowFile string - openapiSpecFiles []string + openapiSpecFiles []spec specsDir string subflowsDir string subflows []string @@ -45,28 +51,83 @@ type minifyTest struct { func TestOpenAPIMinify(t *testing.T) { tests := []minifyTest{ { - workflowFile: "testdata/workflow.sw.yaml", // 4 functions, 2 of them are ref to the same openapi spec - openapiSpecFiles: []string{"testdata/flink-openapi.yaml"}, // 5 operations, 3 must left + workflowFile: "testdata/workflow.sw.yaml", // 4 functions, 2 of them are ref to the same openapi spec + openapiSpecFiles: []spec{{"testdata/flink-openapi.yaml", 5, 3}}, // 5 operations, 3 must left specsDir: "specs", subflowsDir: "subflows", }, { - workflowFile: "testdata/workflow2.sw.yaml", // 4 functions, 1 per openapi spec file - openapiSpecFiles: []string{"testdata/flink1-openapi.yaml", "testdata/flink2-openapi.yaml", "testdata/flink3-openapi.yaml", "testdata/flink4-openapi.yaml"}, + workflowFile: "testdata/workflow.sw.json", // 4 functions, 2 of them are ref to the same openapi spec + openapiSpecFiles: []spec{{"testdata/flink-openapi.yaml", 5, 3}}, // 5 operations, 3 must left specsDir: "specs", subflowsDir: "subflows", }, { - workflowFile: "testdata/workflow-empty.sw.yaml", - openapiSpecFiles: []string{}, + workflowFile: "testdata/workflow-json-openapi.sw.json", // 4 functions, 2 of them are ref to the same openapi spec + openapiSpecFiles: []spec{{"testdata/flink-openapi-json.json", 5, 3}}, // 5 operations, 3 must left + specsDir: "specs", + subflowsDir: "subflows", + }, + { + workflowFile: "testdata/workflow2.sw.yaml", // 4 functions, 1 per openapi spec file + openapiSpecFiles: []spec{ + {"testdata/flink1-openapi.yaml", 3, 1}, + {"testdata/flink2-openapi.yaml", 3, 1}, + {"testdata/flink3-openapi.yaml", 3, 1}, + {"testdata/flink4-openapi.yaml", 3, 1}}, + specsDir: "specs", + subflowsDir: "subflows", + }, + { + workflowFile: "testdata/workflow-empty.sw.yaml", // check don't fail with empty workflow + openapiSpecFiles: []spec{}, + specsDir: "specs", + subflowsDir: "subflows", + }, + { + workflowFile: "testdata/workflow-empty.sw.yaml", // check all operations are removed + openapiSpecFiles: []spec{{"testdata/flink-openapi.yaml", 5, 0}}, + specsDir: "specs", + subflowsDir: "subflows", + }, + { + workflowFile: "testdata/workflow-mySpecsDir.sw.yaml", // check all operations are removed, with different specs dir + openapiSpecFiles: []spec{{"testdata/flink-openapi.yaml", 5, 3}}, + specsDir: "mySpecsDir", + subflowsDir: "subflows", + }, + { + workflowFile: "testdata/workflow-mySpecsDir-one-finction.sw.yaml", // check all operations are removed, with different specs dir + openapiSpecFiles: []spec{{"testdata/flink-openapi.yaml", 5, 2}}, + specsDir: "mySpecsDir", + subflowsDir: "subflows", + subflows: []string{"testdata/subflow-mySpecsDir.sw.yaml"}, + }, + { + workflowFile: "testdata/workflow-empty.sw.yaml", // check all operations are removed, with different subflow dir + openapiSpecFiles: []spec{{"testdata/flink-openapi.yaml", 5, 0}}, + specsDir: "mySpecsDir", + subflowsDir: "subflows", + }, + { + workflowFile: "testdata/workflow-empty2.sw.yaml", // check don't fail with workflow with non openapi functions + openapiSpecFiles: []spec{}, specsDir: "specs", subflowsDir: "subflows", }, { - workflowFile: "testdata/workflow-empty2.sw.yaml", - openapiSpecFiles: []string{}, + workflowFile: "testdata/workflow-empty2.sw.yaml", // check functions is on subflow + openapiSpecFiles: []spec{{"testdata/flink-openapi.yaml", 5, 2}}, specsDir: "specs", subflowsDir: "subflows", + subflows: []string{"testdata/subflow.sw.yaml"}, + }, + { + workflowFile: "testdata/workflow-empty2.sw.yaml", // check functions is on subflow, with different subflow and specs dirs + openapiSpecFiles: []spec{{"testdata/flink-openapi.yaml", 5, 2}}, + specsDir: "mySpecsDir", + subflowsDir: "mySubFlowDir", + subflows: []string{"testdata/subflow-mySpecsDir.sw.yaml"}, }, } @@ -85,8 +146,19 @@ func TestOpenAPIMinify(t *testing.T) { t.Fatalf("Error copying workflow file: %v", err) } defer os.Remove(path.Base(test.workflowFile)) + if len(test.subflows) > 0 { + if err := os.Mkdir(test.subflowsDir, 0755); err != nil { + t.Fatalf("Error creating subflows directory: %v", err) + } + defer os.RemoveAll(test.subflowsDir) + for _, subflow := range test.subflows { + if err := copyFile(subflow, path.Join(test.subflowsDir, path.Base(subflow))); err != nil { + t.Fatalf("Error copying subflow file: %v", err) + } + } + } for _, openapiSpecFile := range test.openapiSpecFiles { - if err := copyFile(openapiSpecFile, path.Join(test.specsDir, path.Base(openapiSpecFile))); err != nil { + if err := copyFile(openapiSpecFile.file, path.Join(test.specsDir, path.Base(openapiSpecFile.file))); err != nil { t.Fatalf("Error copying openapi spec file: %v", err) } } @@ -98,22 +170,47 @@ func TestOpenAPIMinify(t *testing.T) { if err != nil { t.Fatalf("Error minifying openapi specs: %v", err) } - if len(test.openapiSpecFiles) > 0 { - assert.NotEmpty(t, minifiedfiles) - } + checkInitial(t, test) checkResult(t, test, minifiedfiles) }) } } +// checkInitial checks the initial number of operations in the openapi specs +func checkInitial(t *testing.T, test minifyTest) { + for _, spec := range test.openapiSpecFiles { + data, err := os.ReadFile(spec.file) + if err != nil { + t.Fatalf("Error reading openapi spec file: %v", err) + } + doc, err := openapi3.NewLoader().LoadFromData(data) + if err != nil { + t.Fatalf("Error loading openapi spec file: %v", err) + } + assert.Equalf(t, spec.initial, len(doc.Paths.Map()), "Initial number of operations in %s is not correct", spec.file) + } +} + +// checkResult checks the number of operations in the minified openapi specs func checkResult(t *testing.T, test minifyTest, minifiedFiles map[string]string) { workflow, err := parseWorkflow(path.Base(test.workflowFile)) if err != nil { t.Fatalf("Error parsing workflow file: %v", err) } - functions := parseFunctions(t, workflow, test) - for file, function := range functions { + functions := map[string]sets.Set[string]{} + parseFunctions(t, functions, workflow, test) + for _, subflow := range test.subflows { + subflowWorkflow, err := parseWorkflow(subflow) + if err != nil { + t.Fatalf("Error parsing subflow file: %v", err) + } + parseFunctions(t, functions, subflowWorkflow, test) + } + + countOfOperationInSpecs := map[string]int{} + + for file, operationSet := range functions { minified := minifiedFiles[file] data, err := os.ReadFile(minified) if err != nil { @@ -122,19 +219,22 @@ func checkResult(t *testing.T, test minifyTest, minifiedFiles map[string]string) doc, err := openapi3.NewLoader().LoadFromData(data) for _, value := range doc.Paths.Map() { for _, operation := range value.Operations() { - assert.True(t, function.Has(operation.OperationID), "Operation %s not found in workflow", operation.OperationID) + assert.True(t, operationSet.Has(operation.OperationID), "Operation %s not found in workflow", operation.OperationID) } } - assert.Equal(t, len(function), len(doc.Paths.Map())) + countOfOperationInSpecs[file] = len(doc.Paths.Map()) + assert.Equal(t, len(operationSet), len(doc.Paths.Map())) + } + for _, spec := range test.openapiSpecFiles { + assert.Equalf(t, spec.minified, countOfOperationInSpecs[path.Base(spec.file)], "Minified number of operations in %s is not correct", spec.file) } -} -func parseFunctions(t *testing.T, workflow *v1alpha08.Flow, test minifyTest) map[string]sets.Set[string] { - functions := map[string]sets.Set[string]{} +} +func parseFunctions(t *testing.T, functions map[string]sets.Set[string], workflow *v1alpha08.Flow, test minifyTest) { for _, function := range workflow.Functions { if strings.HasPrefix(function.Operation, test.specsDir) { - trimmedPrefix := strings.TrimPrefix(function.Operation, test.specsDir+"/") + trimmedPrefix := strings.TrimPrefix(function.Operation, test.specsDir+string(os.PathSeparator)) if !strings.Contains(trimmedPrefix, "#") { t.Fatalf("Invalid operation format in function: %s", function.Operation) } @@ -151,7 +251,6 @@ func parseFunctions(t *testing.T, workflow *v1alpha08.Flow, test minifyTest) map functions[apiFileName].Insert(operation) } } - return functions } func parseWorkflow(workflowFile string) (*v1alpha08.Flow, error) { diff --git a/packages/kn-plugin-workflow/pkg/specs/testdata/flink-openapi-json.json b/packages/kn-plugin-workflow/pkg/specs/testdata/flink-openapi-json.json new file mode 100644 index 00000000000..7049ac6b90d --- /dev/null +++ b/packages/kn-plugin-workflow/pkg/specs/testdata/flink-openapi-json.json @@ -0,0 +1,123 @@ +{ + "openapi": "3.0.1", + "info": { + "title": "Flink JobManager REST API", + "contact": { + "email": "user@flink.apache.org" + }, + "license": { + "name": "Apache 2.0", + "url": "https://www.apache.org/licenses/LICENSE-2.0.html" + }, + "version": "v1/1.20-SNAPSHOT" + }, + "paths": { + "/jars": { + "get": { + "description": "Returns a list of all jars previously uploaded via '/jars/upload'.", + "operationId": "getJarList", + "responses": { + "200": { + "description": "The request was successful." + } + } + } + }, + "/jars/{jarid}/run": { + "post": { + "description": "Submits a job for execution.", + "operationId": "submitJobFromJar", + "parameters": [ + { + "name": "jarid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The request was successful." + } + } + } + }, + "/jobs/{jobid}": { + "get": { + "description": "Returns details of a job.", + "operationId": "getJobDetails", + "parameters": [ + { + "name": "jobid", + "in": "path", + "description": "32-character hexadecimal string value that identifies a job.", + "required": true + } + ], + "responses": { + "200": { + "description": "The request was successful." + } + } + }, + "patch": { + "description": "Terminates a job.", + "operationId": "cancelJob", + "parameters": [ + { + "name": "jobid", + "in": "path", + "description": "32-character hexadecimal string value that identifies a job.", + "required": true + }, + { + "name": "mode", + "in": "query", + "description": "String value that specifies the termination mode. The only supported value is: \"cancel\".", + "required": false, + "style": "form" + } + ], + "responses": { + "202": { + "description": "The request was successful." + } + } + } + }, + "/jars/upload": { + "post": { + "description": "Uploads a jar.", + "operationId": "uploadJar", + "requestBody": { + "content": { + "application/x-java-archive": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "responses": { + "200": { + "description": "The request was successful." + } + } + } + }, + "/test": { + "get": { + "description": "Test endpoint", + "operationId": "test", + "responses": { + "200": { + "description": "The request was successful." + } + } + } + } + } +} diff --git a/packages/kn-plugin-workflow/pkg/specs/testdata/subflow-mySpecsDir.sw.yaml b/packages/kn-plugin-workflow/pkg/specs/testdata/subflow-mySpecsDir.sw.yaml new file mode 100644 index 00000000000..d56c3e798d2 --- /dev/null +++ b/packages/kn-plugin-workflow/pkg/specs/testdata/subflow-mySpecsDir.sw.yaml @@ -0,0 +1,30 @@ +id: fraudhandling +name: Fraud Handling +expressionLang: jsonpath +start: FraudHandling +version: "1.0" +events: + - kind: produced + name: FraudEvaluation + type: fraudEvaluation + source: fraudEvaluation +functions: + - name: getFlinkJobs + operation: mySpecsDir/flink-openapi.yaml#getJarList + - name: runFlinkJob + operation: mySpecsDir/flink-openapi.yaml#submitJobFromJar +states: + - name: FraudHandling + type: switch + dataConditions: + - condition: "{{ $.[?(@.total > 1000)] }}" + transition: FraudVerificationNeeded + - condition: "{{ $.[?(@.total <= 1000)] }}" + end: true + - name: FraudVerificationNeeded + type: inject + data: + fraudEvaluation: true + end: + produceEvents: + - eventRef: FraudEvaluation diff --git a/packages/kn-plugin-workflow/pkg/specs/testdata/subflow.sw.yaml b/packages/kn-plugin-workflow/pkg/specs/testdata/subflow.sw.yaml new file mode 100644 index 00000000000..776924c8455 --- /dev/null +++ b/packages/kn-plugin-workflow/pkg/specs/testdata/subflow.sw.yaml @@ -0,0 +1,30 @@ +id: fraudhandling +name: Fraud Handling +expressionLang: jsonpath +start: FraudHandling +version: "1.0" +events: + - kind: produced + name: FraudEvaluation + type: fraudEvaluation + source: fraudEvaluation +functions: + - name: getFlinkJobs + operation: specs/flink-openapi.yaml#getJarList + - name: runFlinkJob + operation: specs/flink-openapi.yaml#submitJobFromJar +states: + - name: FraudHandling + type: switch + dataConditions: + - condition: "{{ $.[?(@.total > 1000)] }}" + transition: FraudVerificationNeeded + - condition: "{{ $.[?(@.total <= 1000)] }}" + end: true + - name: FraudVerificationNeeded + type: inject + data: + fraudEvaluation: true + end: + produceEvents: + - eventRef: FraudEvaluation diff --git a/packages/kn-plugin-workflow/pkg/specs/testdata/workflow-json-openapi.sw.json b/packages/kn-plugin-workflow/pkg/specs/testdata/workflow-json-openapi.sw.json new file mode 100644 index 00000000000..600c518e94b --- /dev/null +++ b/packages/kn-plugin-workflow/pkg/specs/testdata/workflow-json-openapi.sw.json @@ -0,0 +1,70 @@ +{ + "id": "flink-workflow", + "version": "1.0", + "specVersion": "0.8", + "name": "flink workflow", + "description": "Create a starter flink job management", + "functions": [ + { + "name": "getFlinkJobs", + "operation": "specs/flink-openapi-json.json#getJarList" + }, + { + "name": "runFlinkJob", + "operation": "specs/flink-openapi-json.json#submitJobFromJar" + }, + { + "name": "stopFlinkJob", + "operation": "specs/flink-openapi-json.json#cancelJob" + }, + { + "name": "getJars", + "operation": "specs/flink-openapi-json.json#getJarList" + }, + { + "name": "sysout", + "type": "custom", + "operation": "sysout" + } + ], + "start": "Get Flink Jars", + "states": [ + { + "name": "Get Flink Jars", + "type": "operation", + "actionMode": "sequential", + "actions": [ + { + "name": "Get Flink Jars", + "functionRef": { + "refName": "getJars" + } + } + ], + "transition": "Run Flink Job" + }, + { + "name": "Run Flink Job", + "type": "operation", + "actionMode": "sequential", + "actions": [ + { + "actionDataFilter": { + "useResults": true + }, + "name": "Run Flink Job", + "functionRef": { + "refName": "runFlinkJob", + "arguments": { + "jarid": "72ecfc25-43ca-4f53-a4ee-1aaf93ac709a_flink-streaming-1.0.jar", + "entry-class": "com.demo.flink.streaming.StreamingJob" + } + } + } + ], + "end": { + "terminate": true + } + } + ] +} diff --git a/packages/kn-plugin-workflow/pkg/specs/testdata/workflow-mySpecsDir-one-finction.sw.yaml b/packages/kn-plugin-workflow/pkg/specs/testdata/workflow-mySpecsDir-one-finction.sw.yaml new file mode 100644 index 00000000000..5fdb58a6b5e --- /dev/null +++ b/packages/kn-plugin-workflow/pkg/specs/testdata/workflow-mySpecsDir-one-finction.sw.yaml @@ -0,0 +1,35 @@ +id: flink-workflow +version: "1.0" +specVersion: "0.8" +name: flink workflow +description: Create a starter flink job management +functions: + - name: getFlinkJobs + operation: mySpecsDir/flink-openapi.yaml#getJarList + - name: sysout + type: custom + operation: sysout +start: Get Flink Jars +states: + - name: Get Flink Jars + type: operation + actionMode: sequential + actions: + - name: Get Flink Jars + functionRef: + refName: getJars + transition: Run Flink Job + - name: Run Flink Job + type: operation + actionMode: sequential + actions: + - actionDataFilter: + useResults: true + name: Run Flink Job + functionRef: + refName: runFlinkJob + arguments: + jarid: 72ecfc25-43ca-4f53-a4ee-1aaf93ac709a_flink-streaming-1.0.jar + entry-class: com.demo.flink.streaming.StreamingJob + end: + terminate: true diff --git a/packages/kn-plugin-workflow/pkg/specs/testdata/workflow-mySpecsDir.sw.yaml b/packages/kn-plugin-workflow/pkg/specs/testdata/workflow-mySpecsDir.sw.yaml new file mode 100644 index 00000000000..4aa6d2cafa4 --- /dev/null +++ b/packages/kn-plugin-workflow/pkg/specs/testdata/workflow-mySpecsDir.sw.yaml @@ -0,0 +1,41 @@ +id: flink-workflow +version: "1.0" +specVersion: "0.8" +name: flink workflow +description: Create a starter flink job management +functions: + - name: getFlinkJobs + operation: mySpecsDir/flink-openapi.yaml#getJarList + - name: runFlinkJob + operation: mySpecsDir/flink-openapi.yaml#submitJobFromJar + - name: stopFlinkJob + operation: mySpecsDir/flink-openapi.yaml#cancelJob + - name: getJars + operation: mySpecsDir/flink-openapi.yaml#getJarList + - name: sysout + type: custom + operation: sysout +start: Get Flink Jars +states: + - name: Get Flink Jars + type: operation + actionMode: sequential + actions: + - name: Get Flink Jars + functionRef: + refName: getJars + transition: Run Flink Job + - name: Run Flink Job + type: operation + actionMode: sequential + actions: + - actionDataFilter: + useResults: true + name: Run Flink Job + functionRef: + refName: runFlinkJob + arguments: + jarid: 72ecfc25-43ca-4f53-a4ee-1aaf93ac709a_flink-streaming-1.0.jar + entry-class: com.demo.flink.streaming.StreamingJob + end: + terminate: true diff --git a/packages/kn-plugin-workflow/pkg/specs/testdata/workflow.sw.json b/packages/kn-plugin-workflow/pkg/specs/testdata/workflow.sw.json new file mode 100644 index 00000000000..54a0ed68408 --- /dev/null +++ b/packages/kn-plugin-workflow/pkg/specs/testdata/workflow.sw.json @@ -0,0 +1,70 @@ +{ + "id": "flink-workflow", + "version": "1.0", + "specVersion": "0.8", + "name": "flink workflow", + "description": "Create a starter flink job management", + "functions": [ + { + "name": "getFlinkJobs", + "operation": "specs/flink-openapi.yaml#getJarList" + }, + { + "name": "runFlinkJob", + "operation": "specs/flink-openapi.yaml#submitJobFromJar" + }, + { + "name": "stopFlinkJob", + "operation": "specs/flink-openapi.yaml#cancelJob" + }, + { + "name": "getJars", + "operation": "specs/flink-openapi.yaml#getJarList" + }, + { + "name": "sysout", + "type": "custom", + "operation": "sysout" + } + ], + "start": "Get Flink Jars", + "states": [ + { + "name": "Get Flink Jars", + "type": "operation", + "actionMode": "sequential", + "actions": [ + { + "name": "Get Flink Jars", + "functionRef": { + "refName": "getJars" + } + } + ], + "transition": "Run Flink Job" + }, + { + "name": "Run Flink Job", + "type": "operation", + "actionMode": "sequential", + "actions": [ + { + "actionDataFilter": { + "useResults": true + }, + "name": "Run Flink Job", + "functionRef": { + "refName": "runFlinkJob", + "arguments": { + "jarid": "72ecfc25-43ca-4f53-a4ee-1aaf93ac709a_flink-streaming-1.0.jar", + "entry-class": "com.demo.flink.streaming.StreamingJob" + } + } + } + ], + "end": { + "terminate": true + } + } + ] +}