forked from raystack/compass
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: create delete assets API (#77)
* feat(proto): generate proto regarding delete assets * feat(asset): add refreshed_at field * feat: create translator from query expr to SQL query and ES query * feat(module): add expr-lang/expr module * feat(docs): add comments in QueryExprTranslator * feat(asset): change migration update * feat(asset): add down migration * feat(translator): mark as private func for func that only used in translator * feat: create delete assets API by query expr * refactor: make interface for query expr and implement to postgresql and elasticsearch * refactor: resolve all lint issues * fix: add refreshed_at in insert asset query * feat: add refreshed_at field in get all assets * fix: update return error in delete assets * fix: update asynchronous process in deletion assets * refactor: fix lint issues * feat: fix ConditionalNode logic and toString format * refactor: remove redundant code, and make return error as soon as possible * feat: update mock using mockery * feat: equalize current time at created_at and updated_at based on refreshed_at both in postgresql and elasticsearch * refactor: rename var * refactor: remove unused go module * test: fix existing unit tests * test: fix existing unit tests * refactor: makes codes to more readable * test: create unit test for deletion API * test: create unit test for converter * feat: add validation in deletion query and remove redundant code * feat: make can equals (==) or IN for type and service in delete asset expr * test: create unit test for delete asset expr * feat: change flow of deletion assets and refactor codes based on feedbacks * test: fix unit test * test: fix unit test * refactor: remove unused comment * feat: make maxAttemptRetry to be configurable and improve code performance * feat: make cancel for context, and remove pointer in ESExpr and SQLExpr * refactor: make clean as linter suggestion * test: fix unit test due to cancel func when create new asset service * refactor: change argument of DeleteByQueryExpr in discovery repository and in situ worker * feat: add condition when the complex query result is time and refactor codes * refactor: revise some codes as feedbacks and add test case in es expr test * refactor: clean code as feedbacks * refactor: add comment so more clear * refactor: remove redundant code * feat: add case for MemberNode which handle nested query --------- Co-authored-by: Muhammad Luthfi Fahlevi <[email protected]>
- Loading branch information
1 parent
c0bf977
commit 071df6f
Showing
52 changed files
with
4,798 additions
and
1,779 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| package asset | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| "github.com/goto/compass/pkg/generichelper" | ||
| "github.com/goto/compass/pkg/queryexpr" | ||
| ) | ||
|
|
||
| var ( | ||
| assetJSONTagsSchema = generichelper.GetJSONTags(Asset{}) | ||
| errTypeOrServiceHasWrongOperator = errors.New("identifier type and service must be equals (==) or IN operator") | ||
| errMissRequiredIdentifier = errors.New("must exists these identifiers: refreshed_at, type, and service") | ||
| ) | ||
|
|
||
| type DeleteAssetExpr struct { | ||
| queryexpr.ExprStr | ||
| } | ||
|
|
||
| func (d DeleteAssetExpr) ToQuery() (string, error) { | ||
| return d.ExprStr.ToQuery() | ||
| } | ||
|
|
||
| func (d DeleteAssetExpr) Validate() error { | ||
| identifiersWithOperator, err := queryexpr.GetIdentifiersMap(d.ExprStr.String()) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if err := d.isRequiredIdentifiersExist(identifiersWithOperator); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if err := d.isUsingRightOperator(identifiersWithOperator); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return d.isAllIdentifiersExistInStruct(identifiersWithOperator) | ||
| } | ||
|
|
||
| func (DeleteAssetExpr) isRequiredIdentifiersExist(identifiersWithOperator map[string]string) error { | ||
| isExist := func(jsonTag string) bool { | ||
| return identifiersWithOperator[jsonTag] != "" | ||
| } | ||
| mustExist := isExist("refreshed_at") && isExist("type") && isExist("service") | ||
| if !mustExist { | ||
| return errMissRequiredIdentifier | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (DeleteAssetExpr) isUsingRightOperator(identifiersWithOperator map[string]string) error { | ||
| isOperatorEqualsOrIn := func(jsonTag string) bool { | ||
| return identifiersWithOperator[jsonTag] == "==" || strings.ToUpper(identifiersWithOperator[jsonTag]) == "IN" | ||
| } | ||
| if !isOperatorEqualsOrIn("type") || !isOperatorEqualsOrIn("service") { | ||
| return errTypeOrServiceHasWrongOperator | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (DeleteAssetExpr) isAllIdentifiersExistInStruct(identifiersWithOperator map[string]string) error { | ||
| identifiers := generichelper.GetMapKeys(identifiersWithOperator) | ||
| for _, identifier := range identifiers { | ||
| isFieldValid := generichelper.Contains(assetJSONTagsSchema, identifier) | ||
| if !isFieldValid { | ||
| return fmt.Errorf("%s is not a valid identifier", identifier) | ||
| } | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| package asset_test | ||
|
|
||
| import ( | ||
| "errors" | ||
| "testing" | ||
|
|
||
| "github.com/goto/compass/core/asset" | ||
| "github.com/goto/compass/pkg/queryexpr" | ||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestDeleteAssetExpr_ToQuery(t *testing.T) { | ||
| queryExp := `name == "John" || service not in ["test1","test2","test3"]` | ||
| sqlExpr := queryexpr.SQLExpr(queryExp) | ||
| esExpr := queryexpr.ESExpr(queryExp) | ||
| wrongExpr := queryexpr.SQLExpr("findLast(") | ||
| tests := []struct { | ||
| name string | ||
| exprStr queryexpr.ExprStr | ||
| want string | ||
| wantErr bool | ||
| }{ | ||
| { | ||
| name: "convert to SQL query", | ||
| exprStr: asset.DeleteAssetExpr{ | ||
| ExprStr: sqlExpr, | ||
| }, | ||
| want: "((name = 'John') OR (service NOT IN ('test1', 'test2', 'test3')))", | ||
| wantErr: false, | ||
| }, | ||
| { | ||
| name: "convert to ES query", | ||
| exprStr: asset.DeleteAssetExpr{ | ||
| ExprStr: esExpr, | ||
| }, | ||
| want: `{"query":{"bool":{"should":[{"term":{"name":"John"}},{"bool":{"must_not":[{"terms":{"service.keyword":["test1","test2","test3"]}}]}}]}}}`, | ||
| wantErr: false, | ||
| }, | ||
| { | ||
| name: "got error due to wrong syntax", | ||
| exprStr: asset.DeleteAssetExpr{ | ||
| ExprStr: wrongExpr, | ||
| }, | ||
| want: "", | ||
| wantErr: true, | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| d := asset.DeleteAssetExpr{ | ||
| ExprStr: tt.exprStr, | ||
| } | ||
| got, err := d.ToQuery() | ||
| if (err != nil) != tt.wantErr { | ||
| t.Errorf("ToQuery() error = %v, wantErr %v", err, tt.wantErr) | ||
| return | ||
| } | ||
| if got != tt.want { | ||
| t.Errorf("ToQuery() got = %v, want %v", got, tt.want) | ||
| } | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestDeleteAssetExpr_Validate(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| exprStrFn func() queryexpr.ExprStr | ||
| expectErr error | ||
| wantErr bool | ||
| }{ | ||
| { | ||
| name: "error get identifiers map", | ||
| exprStrFn: func() queryexpr.ExprStr { | ||
| wrongExpr := queryexpr.SQLExpr("findLast(") | ||
| return asset.DeleteAssetExpr{ | ||
| ExprStr: wrongExpr, | ||
| } | ||
| }, | ||
| expectErr: errors.New("error parsing expression"), | ||
| wantErr: true, | ||
| }, | ||
| { | ||
| name: "error miss refreshed_at not exist", | ||
| exprStrFn: func() queryexpr.ExprStr { | ||
| missRefreshedAt := queryexpr.SQLExpr(`updated_at < "2023-12-12 23:59:59" && type == "table" && service in ["test1","test2","test3"]`) | ||
| return asset.DeleteAssetExpr{ | ||
| ExprStr: missRefreshedAt, | ||
| } | ||
| }, | ||
| expectErr: errors.New("must exists these identifiers: refreshed_at, type, and service"), | ||
| wantErr: true, | ||
| }, | ||
| { | ||
| name: "error miss type not exist", | ||
| exprStrFn: func() queryexpr.ExprStr { | ||
| missType := queryexpr.SQLExpr(`refreshed_at < "2023-12-12 23:59:59" && service in ["test1","test2","test3"]`) | ||
| return asset.DeleteAssetExpr{ | ||
| ExprStr: missType, | ||
| } | ||
| }, | ||
| expectErr: errors.New("must exists these identifiers: refreshed_at, type, and service"), | ||
| wantErr: true, | ||
| }, | ||
| { | ||
| name: "error miss service not exist", | ||
| exprStrFn: func() queryexpr.ExprStr { | ||
| missService := queryexpr.SQLExpr(`refreshed_at < "2023-12-12 23:59:59" && type == "table"`) | ||
| return asset.DeleteAssetExpr{ | ||
| ExprStr: missService, | ||
| } | ||
| }, | ||
| expectErr: errors.New("must exists these identifiers: refreshed_at, type, and service"), | ||
| wantErr: true, | ||
| }, | ||
| { | ||
| name: "error wrong operator for type identifier", | ||
| exprStrFn: func() queryexpr.ExprStr { | ||
| wrongTypeOperator := queryexpr.SQLExpr(`refreshed_at < "2023-12-12 23:59:59" && type != "table" && service in ["test1","test2","test3"]`) | ||
| return asset.DeleteAssetExpr{ | ||
| ExprStr: wrongTypeOperator, | ||
| } | ||
| }, | ||
| expectErr: errors.New("identifier type and service must be equals (==) or IN operator"), | ||
| wantErr: true, | ||
| }, | ||
| { | ||
| name: "error wrong operator for service identifier", | ||
| exprStrFn: func() queryexpr.ExprStr { | ||
| wrongServiceOperator := queryexpr.SQLExpr(`refreshed_at < "2023-12-12 23:59:59" && type != "table" && service not in ["test1","test2","test3"]`) | ||
| return asset.DeleteAssetExpr{ | ||
| ExprStr: wrongServiceOperator, | ||
| } | ||
| }, | ||
| expectErr: errors.New("identifier type and service must be equals (==) or IN operator"), | ||
| wantErr: true, | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| err := tt.exprStrFn().Validate() | ||
| if (err != nil) != tt.wantErr { | ||
| t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) | ||
| } | ||
| if err != nil { | ||
| assert.ErrorContains(t, err, tt.expectErr.Error()) | ||
| } | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| package asset | ||
|
|
||
| type DeleteAssetsRequest struct { | ||
| QueryExpr string | ||
| DryRun bool | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.