diff --git a/builder/store/database/dataset.go b/builder/store/database/dataset.go index b84bf034..e4a28386 100644 --- a/builder/store/database/dataset.go +++ b/builder/store/database/dataset.go @@ -149,6 +149,7 @@ func (s *DatasetStore) FindByPath(ctx context.Context, namespace string, repoPat NewSelect(). Model(resDataset). Relation("Repository.User"). + Relation("Repository.Mirror"). Where("repository.path =?", fmt.Sprintf("%s/%s", namespace, repoPath)). Scan(ctx) if err != nil { diff --git a/builder/store/database/mirror.go b/builder/store/database/mirror.go index c1fd93b8..a58a88a0 100644 --- a/builder/store/database/mirror.go +++ b/builder/store/database/mirror.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/uptrace/bun" "opencsg.com/csghub-server/common/types" ) @@ -283,3 +284,18 @@ func (s *MirrorStore) IndexWithPagination(ctx context.Context, per, page int) (m return } + +func (s *MirrorStore) UpdateMirrorAndRepository(ctx context.Context, mirror *Mirror, repo *Repository) error { + err := s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + _, err := tx.NewUpdate().Model(mirror).WherePK().Exec(ctx) + if err != nil { + return fmt.Errorf("failed to update mirror: %v", err) + } + _, err = tx.NewUpdate().Model(repo).WherePK().Exec(ctx) + if err != nil { + return fmt.Errorf("failed to update repository: %v", err) + } + return nil + }) + return err +} diff --git a/builder/store/database/model.go b/builder/store/database/model.go index a0f9c301..b115bbba 100644 --- a/builder/store/database/model.go +++ b/builder/store/database/model.go @@ -174,6 +174,7 @@ func (s *ModelStore) FindByPath(ctx context.Context, namespace string, name stri NewSelect(). Model(resModel). Relation("Repository.User"). + Relation("Repository.Mirror"). Where("repository.path =?", fmt.Sprintf("%s/%s", namespace, name)). Limit(1). Scan(ctx) diff --git a/common/types/dataset.go b/common/types/dataset.go index 08e5aa30..f81915f1 100644 --- a/common/types/dataset.go +++ b/common/types/dataset.go @@ -18,27 +18,28 @@ type UpdateDatasetReq struct { } type Dataset struct { - ID int64 `json:"id,omitempty"` - Name string `json:"name"` - Nickname string `json:"nickname"` - Description string `json:"description"` - Likes int64 `json:"likes"` - Downloads int64 `json:"downloads"` - Path string `json:"path"` - RepositoryID int64 `json:"repository_id"` - Repository Repository `json:"repository"` - Private bool `json:"private"` - User User `json:"user"` - Tags []RepoTag `json:"tags"` - Readme string `json:"readme"` - DefaultBranch string `json:"default_branch"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - UserLikes bool `json:"user_likes"` - Source RepositorySource `json:"source"` - SyncStatus RepositorySyncStatus `json:"sync_status"` - License string `json:"license"` - CanWrite bool `json:"can_write"` - CanManage bool `json:"can_manage"` - Namespace *Namespace `json:"namespace"` + ID int64 `json:"id,omitempty"` + Name string `json:"name"` + Nickname string `json:"nickname"` + Description string `json:"description"` + Likes int64 `json:"likes"` + Downloads int64 `json:"downloads"` + Path string `json:"path"` + RepositoryID int64 `json:"repository_id"` + Repository Repository `json:"repository"` + Private bool `json:"private"` + User User `json:"user"` + Tags []RepoTag `json:"tags"` + Readme string `json:"readme"` + DefaultBranch string `json:"default_branch"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + UserLikes bool `json:"user_likes"` + Source RepositorySource `json:"source"` + SyncStatus RepositorySyncStatus `json:"sync_status"` + License string `json:"license"` + CanWrite bool `json:"can_write"` + CanManage bool `json:"can_manage"` + Namespace *Namespace `json:"namespace"` + MirrorLastUpdatedAt time.Time `json:"mirror_last_updated_at"` } diff --git a/common/types/model.go b/common/types/model.go index edbb1d8e..263394a5 100644 --- a/common/types/model.go +++ b/common/types/model.go @@ -154,17 +154,18 @@ type Model struct { // widget UI style: generation,chat WidgetType ModelWidgetType `json:"widget_type" example:"generation"` // url to interact with the model - Status string `json:"status" example:"RUNNING"` - UserLikes bool `json:"user_likes"` - Source RepositorySource `json:"source"` - SyncStatus RepositorySyncStatus `json:"sync_status"` - EnableInference bool `json:"enable_inference"` - EnableFinetune bool `json:"enable_finetune"` - BaseModel string `json:"base_model"` - License string `json:"license"` - CanWrite bool `json:"can_write"` - CanManage bool `json:"can_manage"` - Namespace *Namespace `json:"namespace"` + Status string `json:"status" example:"RUNNING"` + UserLikes bool `json:"user_likes"` + Source RepositorySource `json:"source"` + SyncStatus RepositorySyncStatus `json:"sync_status"` + EnableInference bool `json:"enable_inference"` + EnableFinetune bool `json:"enable_finetune"` + BaseModel string `json:"base_model"` + License string `json:"license"` + CanWrite bool `json:"can_write"` + CanManage bool `json:"can_manage"` + Namespace *Namespace `json:"namespace"` + MirrorLastUpdatedAt time.Time `json:"mirror_last_updated_at"` } type SDKModelInfo struct { diff --git a/component/dataset.go b/component/dataset.go index b6db72f3..9bdb5086 100644 --- a/component/dataset.go +++ b/component/dataset.go @@ -430,16 +430,17 @@ func (c *DatasetComponent) Show(ctx context.Context, namespace, name, currentUse Email: dataset.Repository.User.Email, Avatar: dataset.Repository.User.Avatar, }, - Private: dataset.Repository.Private, - CreatedAt: dataset.CreatedAt, - UpdatedAt: dataset.Repository.UpdatedAt, - UserLikes: likeExists, - Source: dataset.Repository.Source, - SyncStatus: dataset.Repository.SyncStatus, - License: dataset.Repository.License, - CanWrite: permission.CanWrite, - CanManage: permission.CanAdmin, - Namespace: ns, + Private: dataset.Repository.Private, + CreatedAt: dataset.CreatedAt, + UpdatedAt: dataset.Repository.UpdatedAt, + UserLikes: likeExists, + Source: dataset.Repository.Source, + SyncStatus: dataset.Repository.SyncStatus, + License: dataset.Repository.License, + MirrorLastUpdatedAt: dataset.Repository.Mirror.LastUpdatedAt, + CanWrite: permission.CanWrite, + CanManage: permission.CanAdmin, + Namespace: ns, } return resDataset, nil diff --git a/component/model.go b/component/model.go index 294981f6..ebc8a6c4 100644 --- a/component/model.go +++ b/component/model.go @@ -411,12 +411,13 @@ func (c *ModelComponent) Show(ctx context.Context, namespace, name, currentUser CreatedAt: model.CreatedAt, UpdatedAt: model.Repository.UpdatedAt, // TODO:default to ModelWidgetTypeGeneration, need to config later - WidgetType: types.ModelWidgetTypeGeneration, - UserLikes: likeExists, - Source: model.Repository.Source, - SyncStatus: model.Repository.SyncStatus, - BaseModel: model.BaseModel, - License: model.Repository.License, + WidgetType: types.ModelWidgetTypeGeneration, + UserLikes: likeExists, + Source: model.Repository.Source, + SyncStatus: model.Repository.SyncStatus, + BaseModel: model.BaseModel, + License: model.Repository.License, + MirrorLastUpdatedAt: model.Repository.Mirror.LastUpdatedAt, CanWrite: permission.CanWrite, CanManage: permission.CanAdmin, diff --git a/docs/docs.go b/docs/docs.go index 59774656..70076513 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -16643,6 +16643,9 @@ const docTemplate = `{ "likes": { "type": "integer" }, + "mirror_last_updated_at": { + "type": "string" + }, "name": { "type": "string" }, @@ -17087,6 +17090,9 @@ const docTemplate = `{ "likes": { "type": "integer" }, + "mirror_last_updated_at": { + "type": "string" + }, "name": { "type": "string" }, diff --git a/docs/swagger.json b/docs/swagger.json index bce21161..056d8705 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -16632,6 +16632,9 @@ "likes": { "type": "integer" }, + "mirror_last_updated_at": { + "type": "string" + }, "name": { "type": "string" }, @@ -17076,6 +17079,9 @@ "likes": { "type": "integer" }, + "mirror_last_updated_at": { + "type": "string" + }, "name": { "type": "string" }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 2de9481a..1fc8c78b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1268,6 +1268,8 @@ definitions: type: string likes: type: integer + mirror_last_updated_at: + type: string name: type: string namespace: @@ -1570,6 +1572,8 @@ definitions: type: string likes: type: integer + mirror_last_updated_at: + type: string name: type: string namespace: diff --git a/mirror/lfssyncer/minio.go b/mirror/lfssyncer/minio.go index 0687da58..568198cf 100644 --- a/mirror/lfssyncer/minio.go +++ b/mirror/lfssyncer/minio.go @@ -10,6 +10,7 @@ import ( "net/url" "path/filepath" "sync" + "time" "github.com/minio/minio-go/v7" "opencsg.com/csghub-server/builder/store/database" @@ -98,19 +99,21 @@ func (w *MinioLFSSyncWorker) worker(id int) { err = w.SyncLfs(ctx, id, mirror) if err != nil { repo.SyncStatus = types.SyncStatusFailed - _, repoErr := w.repoStore.UpdateRepo(ctx, *repo) - if repoErr != nil { - slog.Error("fail to update repo sync status to failed: %w", slog.Any("error", err)) + mirror.LastMessage = err.Error() + err = w.mirrorStore.UpdateMirrorAndRepository(ctx, mirror, repo) + if err != nil { + slog.Error("fail to update mirror and repository", slog.Int("workerId", id), slog.Any("error", err)) } slog.Error("fail to sync lfs", slog.Int("workerId", id), slog.String("error", err.Error())) continue } - repo.SyncStatus = types.SyncStatusCompleted - _, err = w.repoStore.UpdateRepo(ctx, *repo) + mirror.LastUpdatedAt = time.Now() + err = w.mirrorStore.UpdateMirrorAndRepository(ctx, mirror, repo) if err != nil { - slog.Error("fail to update repo sync status to complete: %w", slog.Any("error", err)) + slog.Error("fail to update mirror and repository", slog.Int("workerId", id), slog.Any("error", err)) } + slog.Info("sync lfs completed", slog.Int("workerId", id)) } } diff --git a/mirror/reposyncer/local_woker.go b/mirror/reposyncer/local_woker.go index 121e9343..1983b845 100644 --- a/mirror/reposyncer/local_woker.go +++ b/mirror/reposyncer/local_woker.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "sync" + "time" "opencsg.com/csghub-server/builder/git" "opencsg.com/csghub-server/builder/git/gitserver" @@ -125,6 +126,12 @@ func (w *LocalMirrorWoker) SyncRepo(ctx context.Context, task queue.MirrorTask) err = w.git.MirrorSync(ctx, req) if err != nil { + mirror.Status = types.MirrorFailed + mirror.LastMessage = fmt.Sprintf("failed mirror remote repo in git server:%s", err.Error()) + updateErr := w.mirrorStore.Update(ctx, mirror) + if updateErr != nil { + return fmt.Errorf("failed to update mirror: %w", updateErr) + } return fmt.Errorf("failed mirror remote repo in git server: %v", err) } slog.Info("Mirror remote repo in git server successfully", "repo_type", mirror.Repository.RepositoryType, "namespace", namespace, "name", name) @@ -158,15 +165,10 @@ func (w *LocalMirrorWoker) SyncRepo(ctx context.Context, task queue.MirrorTask) if err != nil { mirror.Status = types.MirrorIncomplete mirror.LastMessage = err.Error() - err = w.mirrorStore.Update(ctx, mirror) - if err != nil { - return fmt.Errorf("failed to update mirror: %w", err) - } - mirror.Repository.SyncStatus = types.SyncStatusFailed - _, err = w.repoStore.UpdateRepo(ctx, *mirror.Repository) + err = w.mirrorStore.UpdateMirrorAndRepository(ctx, mirror, mirror.Repository) if err != nil { - return fmt.Errorf("failed to update repo sync status to failed: %w", err) + return fmt.Errorf("failed to update mirror and repository: %w", err) } return fmt.Errorf("failed to sync lfs files: %v", err) } @@ -181,7 +183,8 @@ func (w *LocalMirrorWoker) SyncRepo(ctx context.Context, task queue.MirrorTask) } else { mirror.Status = types.MirrorFinished } - + // Update mirror last updated at + mirror.LastUpdatedAt = time.Now() err = w.mirrorStore.Update(ctx, mirror) if err != nil { return fmt.Errorf("failed to update mirror: %w", err)