Doc hub proposal (#452)

* Doc Hub Proposal Signed-off-by: Kevin Su <[email protected]> * wip Signed-off-by: Kevin Su <[email protected]> * wip Signed-off-by: Kevin Su <[email protected]> * wip Signed-off-by: Kevin Su <[email protected]> * wip Signed-off-by: Kevin Su <[email protected]> * wip Signed-off-by: Kevin Su <[email protected]> * wip Signed-off-by: Kevin Su <[email protected]> * wip Signed-off-by: Kevin Su <[email protected]> * wip Signed-off-by: Kevin Su <[email protected]> * lint Signed-off-by: Kevin Su <[email protected]> * fix test error Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * more tests Signed-off-by: Kevin Su <[email protected]> * lint Signed-off-by: Kevin Su <[email protected]> * update database schema Signed-off-by: Kevin Su <[email protected]> * update Signed-off-by: Kevin Su <[email protected]> * Fixed tests Signed-off-by: Kevin Su <[email protected]> * Fixed tests Signed-off-by: Kevin Su <[email protected]> * More tests Signed-off-by: Kevin Su <[email protected]> * update idl Signed-off-by: Kevin Su <[email protected]> * list description entity Signed-off-by: Kevin Su <[email protected]> * more tests Signed-off-by: Kevin Su <[email protected]> * register docs when creating task Signed-off-by: Kevin Su <[email protected]> * update go.sum Signed-off-by: Kevin Su <[email protected]> * update idl Signed-off-by: Kevin Su <[email protected]> * Fix tests Signed-off-by: Kevin Su <[email protected]> * Fix tests Signed-off-by: Kevin Su <[email protected]> * Fix tests Signed-off-by: Kevin Su <[email protected]> * more tests Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * wip Signed-off-by: Kevin Su <[email protected]> * Add short description to workflow Signed-off-by: Kevin Su <[email protected]> * fix test Signed-off-by: Kevin Su <[email protected]> * fix test Signed-off-by: Kevin Su <[email protected]> * fix test Signed-off-by: Kevin Su <[email protected]> * fix test Signed-off-by: Kevin Su <[email protected]> * more test Signed-off-by: Kevin Su <[email protected]> * lint Signed-off-by: Kevin Su <[email protected]> * lint Signed-off-by: Kevin Su <[email protected]> * Lint Signed-off-by: Kevin Su <[email protected]> * update to one transation Signed-off-by: Kevin Su <[email protected]> * Fix tests Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * update comment Signed-off-by: Kevin Su <[email protected]> * lint Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * nit Signed-off-by: Kevin Su <[email protected]> * Bump idl Signed-off-by: Kevin Su <[email protected]> * update tests Signed-off-by: Kevin Su <[email protected]> * update tests Signed-off-by: Kevin Su <[email protected]> * update tests Signed-off-by: Kevin Su <[email protected]> * Add identifier Signed-off-by: Kevin Su <[email protected]> * update idl Signed-off-by: Kevin Su <[email protected]> * wip Signed-off-by: Kevin Su <[email protected]> * merged master Signed-off-by: Kevin Su <[email protected]> * Address comment Signed-off-by: Kevin Su <[email protected]> * Address comment Signed-off-by: Kevin Su <[email protected]> * update migrations.go Signed-off-by: Kevin Su <[email protected]> * nit Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * fix tests Signed-off-by: Kevin Su <[email protected]> * lint Signed-off-by: Kevin Su <[email protected]> * Merged master Signed-off-by: Kevin Su <[email protected]> * more tests Signed-off-by: Kevin Su <[email protected]> Signed-off-by: Kevin Su <[email protected]>
flyteorg · Dec 28, 2022 · d2215ed · d2215ed
1 parent ec5528d
commit d2215ed
Show file tree

Hide file tree

Showing 43 changed files with 1,300 additions and 57 deletions.
diff --git a/pkg/manager/impl/description_entity_manager.go b/pkg/manager/impl/description_entity_manager.go
@@ -0,0 +1,120 @@
+package impl
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
+
+	"github.com/flyteorg/flyteadmin/pkg/common"
+
+	"github.com/flyteorg/flyteadmin/pkg/errors"
+	"github.com/flyteorg/flyteadmin/pkg/manager/impl/util"
+	"github.com/flyteorg/flyteadmin/pkg/manager/impl/validation"
+	"github.com/flyteorg/flyteadmin/pkg/manager/interfaces"
+	repoInterfaces "github.com/flyteorg/flyteadmin/pkg/repositories/interfaces"
+	"github.com/flyteorg/flyteadmin/pkg/repositories/transformers"
+	runtimeInterfaces "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces"
+	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin"
+	"github.com/flyteorg/flytestdlib/contextutils"
+	"github.com/flyteorg/flytestdlib/logger"
+	"github.com/flyteorg/flytestdlib/promutils"
+	"google.golang.org/grpc/codes"
+)
+
+type DescriptionEntityMetrics struct {
+	Scope promutils.Scope
+}
+
+type DescriptionEntityManager struct {
+	db      repoInterfaces.Repository
+	config  runtimeInterfaces.Configuration
+	metrics DescriptionEntityMetrics
+}
+
+func (d *DescriptionEntityManager) GetDescriptionEntity(ctx context.Context, request admin.ObjectGetRequest) (
+	*admin.DescriptionEntity, error) {
+	if err := validation.ValidateDescriptionEntityGetRequest(request); err != nil {
+		logger.Errorf(ctx, "invalid request [%+v]: %v", request, err)
+		return nil, err
+	}
+	ctx = contextutils.WithProjectDomain(ctx, request.Id.Project, request.Id.Domain)
+	return util.GetDescriptionEntity(ctx, d.db, *request.Id)
+}
+
+func (d *DescriptionEntityManager) ListDescriptionEntity(ctx context.Context, request admin.DescriptionEntityListRequest) (*admin.DescriptionEntityList, error) {
+	// Check required fields
+	if err := validation.ValidateDescriptionEntityListRequest(request); err != nil {
+		return nil, err
+	}
+	ctx = contextutils.WithProjectDomain(ctx, request.Id.Project, request.Id.Domain)
+
+	if request.ResourceType == core.ResourceType_WORKFLOW {
+		ctx = contextutils.WithWorkflowID(ctx, request.Id.Name)
+	} else {
+		ctx = contextutils.WithTaskID(ctx, request.Id.Name)
+	}
+
+	filters, err := util.GetDbFilters(util.FilterSpec{
+		Project:        request.Id.Project,
+		Domain:         request.Id.Domain,
+		Name:           request.Id.Name,
+		RequestFilters: request.Filters,
+	}, common.ResourceTypeToEntity[request.ResourceType])
+	if err != nil {
+		logger.Error(ctx, "failed to get database filter")
+		return nil, err
+	}
+	var sortParameter common.SortParameter
+	if request.SortBy != nil {
+		sortParameter, err = common.NewSortParameter(*request.SortBy)
+		if err != nil {
+			return nil, err
+		}
+	}
+	offset, err := validation.ValidateToken(request.Token)
+	if err != nil {
+		return nil, errors.NewFlyteAdminErrorf(codes.InvalidArgument,
+			"invalid pagination token %s for ListWorkflows", request.Token)
+	}
+	listDescriptionEntitiesInput := repoInterfaces.ListResourceInput{
+		Limit:         int(request.Limit),
+		Offset:        offset,
+		InlineFilters: filters,
+		SortParameter: sortParameter,
+	}
+	output, err := d.db.DescriptionEntityRepo().List(ctx, listDescriptionEntitiesInput)
+	if err != nil {
+		logger.Debugf(ctx, "Failed to list workflows with [%+v] with err %v", request.Id, err)
+		return nil, err
+	}
+	descriptionEntityList, err := transformers.FromDescriptionEntityModels(output.Entities)
+	if err != nil {
+		logger.Errorf(ctx,
+			"Failed to transform workflow models [%+v] with err: %v", output.Entities, err)
+		return nil, err
+	}
+	var token string
+	if len(output.Entities) == int(request.Limit) {
+		token = strconv.Itoa(offset + len(output.Entities))
+	}
+	return &admin.DescriptionEntityList{
+		DescriptionEntities: descriptionEntityList,
+		Token:               token,
+	}, nil
+}
+
+func NewDescriptionEntityManager(
+	db repoInterfaces.Repository,
+	config runtimeInterfaces.Configuration,
+	scope promutils.Scope) interfaces.DescriptionEntityInterface {
+
+	metrics := DescriptionEntityMetrics{
+		Scope: scope,
+	}
+	return &DescriptionEntityManager{
+		db:      db,
+		config:  config,
+		metrics: metrics,
+	}
+}
diff --git a/pkg/manager/impl/description_entity_manager_test.go b/pkg/manager/impl/description_entity_manager_test.go
@@ -0,0 +1,146 @@
+package impl
+
+import (
+	"context"
+	"testing"
+
+	"github.com/flyteorg/flyteadmin/pkg/manager/impl/testutils"
+	"github.com/flyteorg/flyteadmin/pkg/repositories/interfaces"
+	repositoryMocks "github.com/flyteorg/flyteadmin/pkg/repositories/mocks"
+	runtimeInterfaces "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces"
+	runtimeMocks "github.com/flyteorg/flyteadmin/pkg/runtime/mocks"
+	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin"
+	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
+	mockScope "github.com/flyteorg/flytestdlib/promutils"
+	"github.com/stretchr/testify/assert"
+)
+
+var descriptionEntityIdentifier = core.Identifier{
+	ResourceType: core.ResourceType_WORKFLOW,
+	Project:      project,
+	Domain:       domain,
+	Name:         name,
+	Version:      version,
+}
+
+var badDescriptionEntityIdentifier = core.Identifier{
+	ResourceType: core.ResourceType_WORKFLOW,
+	Project:      project,
+	Domain:       domain,
+	Name:         "",
+	Version:      version,
+}
+
+func getMockRepositoryForDETest() interfaces.Repository {
+	return repositoryMocks.NewMockRepository()
+}
+
+func getMockConfigForDETest() runtimeInterfaces.Configuration {
+	mockConfig := runtimeMocks.NewMockConfigurationProvider(
+		testutils.GetApplicationConfigWithDefaultDomains(), nil, nil, nil, nil, nil)
+	return mockConfig
+}
+
+func TestDescriptionEntityManager_Get(t *testing.T) {
+	repository := getMockRepositoryForDETest()
+	manager := NewDescriptionEntityManager(repository, getMockConfigForDETest(), mockScope.NewTestScope())
+
+	response, err := manager.GetDescriptionEntity(context.Background(), admin.ObjectGetRequest{
+		Id: &descriptionEntityIdentifier,
+	})
+	assert.NoError(t, err)
+	assert.NotNil(t, response)
+
+	response, err = manager.GetDescriptionEntity(context.Background(), admin.ObjectGetRequest{
+		Id: &badDescriptionEntityIdentifier,
+	})
+	assert.Error(t, err)
+	assert.Nil(t, response)
+}
+
+func TestDescriptionEntityManager_List(t *testing.T) {
+	repository := getMockRepositoryForDETest()
+	manager := NewDescriptionEntityManager(repository, getMockConfigForDETest(), mockScope.NewTestScope())
+
+	t.Run("failed to validate a request", func(t *testing.T) {
+		response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{
+			Id: &admin.NamedEntityIdentifier{
+				Name: "flyte",
+			},
+		})
+		assert.Error(t, err)
+		assert.Nil(t, response)
+	})
+
+	t.Run("failed to sort description entity", func(t *testing.T) {
+		response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{
+			ResourceType: core.ResourceType_TASK,
+			Id: &admin.NamedEntityIdentifier{
+				Name:    "flyte",
+				Project: "project",
+				Domain:  "domain",
+			},
+			Limit:  1,
+			SortBy: &admin.Sort{Direction: 3},
+		})
+		assert.Error(t, err)
+		assert.Nil(t, response)
+	})
+
+	t.Run("failed to validate token", func(t *testing.T) {
+		response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{
+			ResourceType: core.ResourceType_TASK,
+			Id: &admin.NamedEntityIdentifier{
+				Name:    "flyte",
+				Project: "project",
+				Domain:  "domain",
+			},
+			Limit: 1,
+			Token: "hello",
+		})
+		assert.Error(t, err)
+		assert.Nil(t, response)
+	})
+
+	t.Run("list description entities in the task", func(t *testing.T) {
+		response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{
+			ResourceType: core.ResourceType_TASK,
+			Id: &admin.NamedEntityIdentifier{
+				Name:    "flyte",
+				Project: "project",
+				Domain:  "domain",
+			},
+			Limit: 1,
+		})
+		assert.NoError(t, err)
+		assert.NotNil(t, response)
+	})
+
+	t.Run("list description entities in the workflow", func(t *testing.T) {
+		response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{
+			ResourceType: core.ResourceType_WORKFLOW,
+			Id: &admin.NamedEntityIdentifier{
+				Name:    "flyte",
+				Project: "project",
+				Domain:  "domain",
+			},
+			Limit: 1,
+		})
+		assert.NoError(t, err)
+		assert.NotNil(t, response)
+	})
+
+	t.Run("failed to get filter", func(t *testing.T) {
+		response, err := manager.ListDescriptionEntity(context.Background(), admin.DescriptionEntityListRequest{
+			ResourceType: core.ResourceType_WORKFLOW,
+			Id: &admin.NamedEntityIdentifier{
+				Name:    "flyte",
+				Project: "project",
+				Domain:  "domain",
+			},
+			Filters: "wrong",
+		})
+		assert.Error(t, err)
+		assert.Nil(t, response)
+	})
+}
diff --git a/pkg/manager/impl/execution_manager_test.go b/pkg/manager/impl/execution_manager_test.go
@@ -4085,11 +4085,11 @@ func TestCreateSingleTaskExecution(t *testing.T) {
 	repository := getMockRepositoryForExecTest()
 	var getCalledCount = 0
 	var newlyCreatedWorkflow models.Workflow
-	workflowcreateFunc := func(input models.Workflow) error {
+	workflowCreateFunc := func(input models.Workflow, descriptionEntity *models.DescriptionEntity) error {
 		newlyCreatedWorkflow = input
 		return nil
 	}
-	repository.WorkflowRepo().(*repositoryMocks.MockWorkflowRepo).SetCreateCallback(workflowcreateFunc)
+	repository.WorkflowRepo().(*repositoryMocks.MockWorkflowRepo).SetCreateCallback(workflowCreateFunc)
 
 	workflowGetFunc := func(input interfaces.Identifier) (models.Workflow, error) {
 		if getCalledCount <= 1 {

diff --git a/pkg/manager/impl/task_execution_manager_test.go b/pkg/manager/impl/task_execution_manager_test.go
@@ -181,6 +181,15 @@ func TestCreateTaskEvent(t *testing.T) {
 	assert.True(t, createTaskCalled)
 	assert.Nil(t, err)
 	assert.NotNil(t, resp)
+
+	repository.TaskExecutionRepo().(*repositoryMocks.MockTaskExecutionRepo).SetCreateCallback(
+		func(ctx context.Context, input models.TaskExecution) error {
+			return errors.New("failed to insert record into task table")
+		})
+	taskExecManager = NewTaskExecutionManager(repository, getMockExecutionsConfigProvider(), getMockStorageForExecTest(context.Background()), mockScope.NewTestScope(), mockTaskExecutionRemoteURL, nil, nil)
+	resp, err = taskExecManager.CreateTaskExecutionEvent(context.Background(), taskEventRequest)
+	assert.NotNil(t, err)
+	assert.Nil(t, resp)
 }
 
 func TestCreateTaskEvent_Update(t *testing.T) {

diff --git a/pkg/manager/impl/task_manager.go b/pkg/manager/impl/task_manager.go
@@ -107,7 +107,17 @@ func (t *TaskManager) CreateTask(
 			"Failed to transform task model [%+v] with err: %v", finalizedRequest, err)
 		return nil, err
 	}
-	err = t.db.TaskRepo().Create(ctx, taskModel)
+
+	descriptionModel, err := transformers.CreateDescriptionEntityModel(request.Spec.Description, *request.Id)
+	if err != nil {
+		logger.Errorf(ctx,
+			"Failed to transform description model [%+v] with err: %v", request.Spec.Description, err)
+		return nil, err
+	}
+	if descriptionModel != nil {
+		taskModel.ShortDescription = descriptionModel.ShortDescription
+	}
+	err = t.db.TaskRepo().Create(ctx, taskModel, descriptionModel)
 	if err != nil {
 		logger.Debugf(ctx, "Failed to create task model with id [%+v] with err %v", request.Id, err)
 		return nil, err
@@ -120,6 +130,7 @@ func (t *TaskManager) CreateTask(
 			contextWithRuntimeMeta, common.RuntimeVersionKey, finalizedRequest.Spec.Template.Metadata.Runtime.Version)
 		t.metrics.Registered.Inc(contextWithRuntimeMeta)
 	}
+
 	return &admin.TaskCreateResponse{}, nil
 }
 

diff --git a/pkg/manager/impl/task_manager_test.go b/pkg/manager/impl/task_manager_test.go
@@ -73,22 +73,30 @@ func TestCreateTask(t *testing.T) {
 			return models.Task{}, errors.New("foo")
 		})
 	var createCalled bool
-	mockRepository.TaskRepo().(*repositoryMocks.MockTaskRepo).SetCreateCallback(func(input models.Task) error {
+	mockRepository.TaskRepo().(*repositoryMocks.MockTaskRepo).SetCreateCallback(func(input models.Task, descriptionEntity *models.DescriptionEntity) error {
 		assert.Equal(t, []byte{
 			0xbf, 0x79, 0x61, 0x1c, 0xf5, 0xc1, 0xfb, 0x4c, 0xf8, 0xf4, 0xc4, 0x53, 0x5f, 0x8f, 0x73, 0xe2, 0x26, 0x5a,
 			0x18, 0x4a, 0xb7, 0x66, 0x98, 0x3c, 0xab, 0x2, 0x6c, 0x9, 0x9b, 0x90, 0xec, 0x8f}, input.Digest)
 		createCalled = true
 		return nil
 	})
+	mockRepository.DescriptionEntityRepo().(*repositoryMocks.MockDescriptionEntityRepo).SetGetCallback(
+		func(input interfaces.GetDescriptionEntityInput) (models.DescriptionEntity, error) {
+			return models.DescriptionEntity{}, adminErrors.NewFlyteAdminErrorf(codes.NotFound, "NotFound")
+		})
 	taskManager := NewTaskManager(mockRepository, getMockConfigForTaskTest(), getMockTaskCompiler(),
 		mockScope.NewTestScope())
 	request := testutils.GetValidTaskRequest()
 	response, err := taskManager.CreateTask(context.Background(), request)
 	assert.NoError(t, err)
 
-	expectedResponse := &admin.TaskCreateResponse{}
-	assert.Equal(t, expectedResponse, response)
+	assert.Equal(t, &admin.TaskCreateResponse{}, response)
 	assert.True(t, createCalled)
+
+	request.Spec.Description = nil
+	response, err = taskManager.CreateTask(context.Background(), request)
+	assert.NoError(t, err)
+	assert.NotNil(t, response)
 }
 
 func TestCreateTask_ValidationError(t *testing.T) {
@@ -125,7 +133,7 @@ func TestCreateTask_DatabaseError(t *testing.T) {
 			return models.Task{}, errors.New("foo")
 		})
 	expectedErr := errors.New("expected error")
-	taskCreateFunc := func(input models.Task) error {
+	taskCreateFunc := func(input models.Task, descriptionEntity *models.DescriptionEntity) error {
 		return expectedErr
 	}
 

diff --git a/pkg/manager/impl/testutils/mock_requests.go b/pkg/manager/impl/testutils/mock_requests.go
@@ -41,6 +41,7 @@ func GetValidTaskRequest() admin.TaskCreateRequest {
 					},
 				},
 			},
+			Description: &admin.DescriptionEntity{ShortDescription: "hello"},
 		},
 	}
 }
@@ -119,6 +120,7 @@ func GetWorkflowRequest() admin.WorkflowCreateRequest {
 					},
 				},
 			},
+			Description: &admin.DescriptionEntity{ShortDescription: "hello"},
 		},
 	}
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -41,6 +41,7 @@ func GetValidTaskRequest() admin.TaskCreateRequest { @@
     					},
     				},
     			},
+    			Description: &admin.DescriptionEntity{ShortDescription: "hello"},
     		},
     	}
     }
@@ Expand Down Expand Up / @@ -119,6 +120,7 @@ func GetWorkflowRequest() admin.WorkflowCreateRequest { @@
     					},
     				},
     			},
+    			Description: &admin.DescriptionEntity{ShortDescription: "hello"},
     		},
     	}
     }
@@ Expand Down @@