Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow for pipeline step parameters to be fully JSON #161

Merged
merged 4 commits into from
May 27, 2024
+63 −24
Merged

Allow for pipeline step parameters to be fully JSON #161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0c088e5
refactor: Parameters type to map[string]any
joshtyf May 26, 2024
1f156dd
chore: update seed data
joshtyf May 26, 2024
482acc2
refactor: placeholder replacement methods to work with json parameter…
joshtyf May 26, 2024
11af609
refactor: add type assertion in api step executor
joshtyf May 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 16 additions & 13 deletions backend/database/mongo_seed/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,11 @@ func main() {
StepType: models.APIStep,
NextStepName: "step2",
PrevStepName: "",
Parameters: map[string]string{
"method": "${method}",
"url": "https://httpbin.org/${method}?param=${param}",
"data": "{\"key\": \"${value}\", \"${key}\": \"hardcoded_value\"}",
Parameters: map[string]any{
"method": "${method}",
"url": "https://httpbin.org/${method}?param=${param}",
"data": "{\"key\": \"${value}\", \"${key}\": \"hardcoded_value\"}",
Copy link
Owner Author

@joshtyf joshtyf May 26, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

data is currently a json encoded string. this will eventually be changed to be a proper JSON object like headers

"headers": map[string]any{"Authorization": "Bearer ${token}"},
},
IsTerminalStep: false,
},
Expand All @@ -50,7 +51,7 @@ func main() {
StepType: models.WaitForApprovalStep,
NextStepName: "",
PrevStepName: "step1",
Parameters: map[string]string{},
Parameters: map[string]any{},
IsTerminalStep: true,
},
},
Expand Down Expand Up @@ -99,7 +100,7 @@ func main() {
StepType: models.APIStep,
NextStepName: "step2",
PrevStepName: "",
Parameters: map[string]string{
Parameters: map[string]any{
"method": "GET",
"url": "https://example.com?param=${param}",
},
Expand All @@ -110,7 +111,7 @@ func main() {
StepType: models.APIStep,
NextStepName: "step3",
PrevStepName: "step1",
Parameters: map[string]string{
Parameters: map[string]any{
"method": "GET",
"url": "https://example.com?param=${param}",
},
Expand All @@ -121,7 +122,7 @@ func main() {
StepType: models.APIStep,
NextStepName: "",
PrevStepName: "step2",
Parameters: map[string]string{
Parameters: map[string]any{
"method": "GET",
"url": "https://example.com?param=${param}",
},
Expand Down Expand Up @@ -173,7 +174,7 @@ func main() {
StepType: models.APIStep,
NextStepName: "step2",
PrevStepName: "",
Parameters: map[string]string{
Parameters: map[string]any{
"method": "GET",
"url": "https://example.com?param=${param}",
},
Expand All @@ -184,15 +185,15 @@ func main() {
StepType: models.WaitForApprovalStep,
NextStepName: "step3",
PrevStepName: "step1",
Parameters: map[string]string{},
Parameters: map[string]any{},
IsTerminalStep: false,
},
{
StepName: "step3",
StepType: models.APIStep,
NextStepName: "",
PrevStepName: "step2",
Parameters: map[string]string{
Parameters: map[string]any{
"method": "GET",
"url": "https://example.com?param=${param}",
},
Expand Down Expand Up @@ -244,15 +245,15 @@ func main() {
StepType: models.WaitForApprovalStep,
NextStepName: "step2",
PrevStepName: "",
Parameters: map[string]string{},
Parameters: map[string]any{},
IsTerminalStep: false,
},
{
StepName: "step2",
StepType: models.APIStep,
NextStepName: "",
PrevStepName: "step1",
Parameters: map[string]string{
Parameters: map[string]any{
"method": "GET",
"url": "https://example.com?param=${param}",
},
Expand Down Expand Up @@ -321,6 +322,7 @@ func main() {
"method": "get",
"key": "test_key",
"value": "test_value",
"token": "test_token",
},
}

Expand All @@ -345,6 +347,7 @@ func main() {
"method": "post",
"key": "test_key",
"value": "test_value",
"token": "test_token",
},
}

Expand Down
12 changes: 6 additions & 6 deletions backend/src/database/models/pipeline.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,12 @@ func IsValidPipelineStepType(stepType PipelineStepType) bool {
}

type PipelineStepModel struct {
StepName string `bson:"step_name" json:"step_name"`
StepType PipelineStepType `bson:"step_type" json:"step_type"`
NextStepName string `bson:"next_step_name" json:"next_step_name"`
PrevStepName string `bson:"prev_step_name" json:"prev_step_name"`
Parameters map[string]string `bson:"parameters" json:"parameters"`
IsTerminalStep bool `bson:"is_terminal_step" json:"is_terminal_step"`
StepName string `bson:"step_name" json:"step_name"`
StepType PipelineStepType `bson:"step_type" json:"step_type"`
NextStepName string `bson:"next_step_name" json:"next_step_name"`
PrevStepName string `bson:"prev_step_name" json:"prev_step_name"`
Parameters map[string]any `bson:"parameters" json:"parameters"`
IsTerminalStep bool `bson:"is_terminal_step" json:"is_terminal_step"`
}

type PipelineModel struct {
Expand Down
6 changes: 3 additions & 3 deletions backend/src/execute/steps.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ func (e *apiStepExecutor) execute(ctx context.Context, l *logger.ExecutorLogger)
l.Error("error getting service request from context")
return nil, errors.New("error getting service request from context")
}
requestMethod := step.Parameters["method"]
url := step.Parameters["url"]
requestBody := step.Parameters["data"]
requestMethod := step.Parameters["method"].(string) // TODO: add documentation for parameters, specifically the type for safe type assertion
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when creating the pipeline in the frontend, the user doesn't know what parameters are required for the API step type. he also doesn't know if the parameter value should be of what type. we can add both documentation and validation for this

url := step.Parameters["url"].(string)
requestBody := step.Parameters["data"].(string)
req, err := http.NewRequest(strings.ToUpper(requestMethod), url, bytes.NewBuffer([]byte(requestBody)))
req.Header.Set("Content-Type", "application/json")
l.Info(fmt.Sprintf("method=%s url=%s data=%s", requestMethod, url, requestBody))
Expand Down
40 changes: 38 additions & 2 deletions backend/src/helper/strings.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ import (
)

var (
ErrPlaceholderNotReplaced = errors.New("some placeholders were not replaced")
ErrPlaceholderNotReplaced = errors.New("some placeholders were not replaced")
ErrInvalidTypeForPlaceholderReplacement = errors.New("placeholder replacement is only supported for strings, slices, and maps")
)

func ReplacePlaceholders(input string, values map[string]any) (string, error) {
func ReplacePlaceholdersInString(input string, values map[string]any) (string, error) {
// Regular expression to find placeholders
re := regexp.MustCompile(`\$\{(.*?)\}`)

Expand Down Expand Up @@ -54,6 +55,41 @@ func ReplacePlaceholders(input string, values map[string]any) (string, error) {
return replaced, nil
}

func ReplacePlaceholders(input any, values map[string]any) (any, error) {
switch reflect.TypeOf(input).Kind() {
case reflect.String:
return ReplacePlaceholdersInString(input.(string), values)
case reflect.Slice:
// If the input is a slice, iterate over each element and replace placeholders
output := make([]any, 0)
for _, elem := range input.([]any) {
replaced, err := ReplacePlaceholders(elem, values)
if err != nil {
return nil, err
}
output = append(output, replaced)
}
return output, nil
case reflect.Map:
// If the input is a map, iterate over each key and value and replace placeholders
output := make(map[string]any)
for key, value := range input.(map[string]any) {
replacedKey, err := ReplacePlaceholdersInString(key, values)
if err != nil {
return nil, err
}
replacedValue, err := ReplacePlaceholders(value, values)
if err != nil {
return nil, err
}
output[replacedKey] = replacedValue
}
return output, nil
default:
return nil, ErrInvalidTypeForPlaceholderReplacement
}
}

func StringSliceEqual(a, b []string) bool {
if len(a) != len(b) {
return false
Expand Down
Loading