Skip to content

Commit

Permalink
Merge pull request TencentBlueKing#305 from tbs60/dev_tming
Browse files Browse the repository at this point in the history 
Dev tming
  • Loading branch information
@tming
tming authored Oct 15, 2024
2 parents c746e5c + 578ee98 commit 4e26792
Show file tree
Hide file tree
Showing 9 changed files with 175 additions and 71 deletions.
61 changes: 61 additions & 0 deletions src/backend/booster/bk_dist/common/file/file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,16 +82,33 @@ func GetFileInfoByEnumDir(fp string) *Info {
}
}

type FileType int

const (
Unknown FileType = 0

RealFile FileType = 10
RealDir FileType = 11
LinkFile FileType = 12
LinkDir FileType = 13
)

// Info describe the os.FileInfo and handle some actions
type Info struct {
filePath string
LinkTarget string
FileType FileType

// info and err are return from os.Stat
info os.FileInfo
err error
}

// Key return the uniq key of this file info
func (i *Info) Key() string {
return i.filePath
}

// Path return the file path
func (i *Info) Path() string {
return i.filePath
Expand Down Expand Up @@ -224,13 +241,15 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo
tempf := notf
try := 0
maxtry := 10
var oldi *Info
for {
var i *Info
if statbysearchdir {
i = GetFileInfoByEnumDir(tempf)
} else {
i = Lstat(tempf)
}
i.FileType = RealFile
tempis[tempf] = i
try++

Expand Down Expand Up @@ -279,6 +298,30 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo
continue
}

// 根据当前文件属性，给上一次的文件属性赋值
if loopagain { // 需要等链接的属性
i.FileType = LinkFile // 先假设是指向普通文件的链接
if oldi != nil {
oldi.FileType = LinkFile
blog.Infof("common util: set %s to LinkFile by assume", oldi.filePath)
}
} else {
if i.Basic().IsDir() {
i.FileType = RealDir
if oldi != nil {
oldi.FileType = LinkDir
blog.Infof("common util: set %s to LinkDir", oldi.filePath)
}
} else {
i.FileType = RealFile
if oldi != nil {
oldi.FileType = LinkFile
blog.Infof("common util: set %s to LinkFile", oldi.filePath)
}
}
}
oldi = i

is = append(is, i)

if !loopagain {
Expand All @@ -298,3 +341,21 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo

return is, nil
}

func Uniq(input []*Info) []*Info {
if input == nil {
return input
}

newarr := make([]*Info, 0, len(input)/2)
tempMap := make(map[string]struct{}, len(newarr))
for _, v := range input {
if _, ok := tempMap[v.Key()]; !ok {
tempMap[v.Key()] = struct{}{}
newarr = append(newarr, v)
}
}

return newarr

}
12 changes: 12 additions & 0 deletions src/backend/booster/bk_dist/common/sdk/worker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"net"
"os"

"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file"
dcFile "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file"
"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/protocol"
dcProtocol "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/protocol"
Expand Down Expand Up @@ -186,6 +187,17 @@ type BKSlotRspAck struct {
}

func GetPriority(i *dcFile.Info) FileDescPriority {
switch i.FileType {
case file.RealDir:
return RealDirPriority
case file.LinkDir:
return LinkDirPriority
case file.RealFile:
return RealFilePriority
case file.LinkFile:
return LinkFilePriority
}

isLink := i.Basic().Mode()&os.ModeSymlink != 0
if !isLink {
if i.Basic().IsDir() {
Expand Down
2 changes: 1 addition & 1 deletion src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -665,7 +665,7 @@ func (m *Mgr) ensureFilesWithPriority(
fileDetails []*types.FilesDetails) ([]string, error) {

// 刷新优先级，windows的先不实现
if runtime.GOOS != "windows" {
if runtime.GOOS != "windows" && runtime.GOOS != "darwin" {
freshPriority(fileDetails)
for _, v := range fileDetails {
blog.Debugf("remote: after fresh Priority, file:%+v", *v)
Expand Down
72 changes: 46 additions & 26 deletions src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,44 +204,62 @@ func calculateDependencies(fileDetails []*types.FilesDetails) [][]int {
return dependencies
}

// 判断s2是否是s1的子串
func isSubString(s1, s2 string) bool {
return len(s1) > len(s2) &&
strings.HasPrefix(s1, s2) &&
s2 != "/"
}

// dirDepend 检查 s1的目录 是否依赖 s2的目录
// 优化规则：只有s2是指向目录的链接时，才需要判断依赖关系；
// 因为普通目录（非链接）不影响远端路径的生成；而指向目录的链接，会在保存依赖时提炼出来
func dirDepend(s1, s2 *types.FilesDetails) bool {
is1File := s1.File.Priority == sdk.RealFilePriority || s1.File.Priority == sdk.LinkFilePriority
is2File := s2.File.Priority == sdk.RealFilePriority || s2.File.Priority == sdk.LinkFilePriority
// 如果s1是文件，s2是目录
if is1File {
if is2File { // 如果s1是文件，s2是文件
if isSubString(s1.File.Targetrelativepath, s2.File.Targetrelativepath) {
return true
}
} else { // 如果s1是文件，s2是目录
if isSubString(s1.File.FilePath, s2.File.FilePath) {
return true
}
}
} else {
if is2File { // 如果s1是目录，s2是文件
if isSubString(s1.File.FilePath, s2.File.Targetrelativepath) {
return true
}
} else { // 如果s1是目录，s2是目录
if isSubString(s1.File.FilePath, s2.File.FilePath) {
return true
}
}
if s2.File.Priority != sdk.LinkDirPriority {
return false
}

if isSubString(s1.File.FilePath, s2.File.FilePath) {
return true
}

return false

// is1File := s1.File.Priority == sdk.RealFilePriority || s1.File.Priority == sdk.LinkFilePriority
// is2File := s2.File.Priority == sdk.RealFilePriority || s2.File.Priority == sdk.LinkFilePriority

// // 如果s1是文件，s2是目录
// if is1File {
// if is2File { // 如果s1是文件，s2是文件
// if isSubString(s1.File.Targetrelativepath, s2.File.Targetrelativepath) {
// return true
// }
// } else { // 如果s1是文件，s2是目录
// if isSubString(s1.File.FilePath, s2.File.FilePath) {
// return true
// }
// }
// } else {
// if is2File { // 如果s1是目录，s2是文件
// if isSubString(s1.File.FilePath, s2.File.Targetrelativepath) {
// return true
// }
// } else { // 如果s1是目录，s2是目录
// if isSubString(s1.File.FilePath, s2.File.FilePath) {
// return true
// }
// }
// }

// return false
}

// linkDepend 检查 s1 是否链接到了 s2
func linkDepend(s1, s2 *types.FilesDetails) bool {
if s1.File.Priority != sdk.LinkDirPriority && s1.File.Priority != sdk.LinkFilePriority {
return false
}

if s1.File.LinkTarget != "" && s1.File.LinkTarget == s2.File.FilePath {
return true
}
Expand Down Expand Up @@ -334,9 +352,11 @@ func printLeftDepend(fileDetails []*types.FilesDetails, dependencies [][]int) {
if fileDetails[i].File.Priority < 0 {
dependfiles := []string{}
for _, v := range dependencies[i] {
dependPriority := int(fileDetails[v].File.Priority)
if dependPriority < 0 {
dependfiles = append(dependfiles, fileDetails[v].File.FilePath)
if v >= 0 {
dependPriority := int(fileDetails[v].File.Priority)
if dependPriority < 0 {
dependfiles = append(dependfiles, fileDetails[v].File.FilePath)
}
}
}
blog.Warnf("remote util: after max try, %s wait %s",
Expand Down
23 changes: 13 additions & 10 deletions src/backend/booster/bk_dist/handler/cc/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,14 +223,15 @@ func (cc *TaskCC) FinalExecute(args []string) {

// GetFilterRules add file send filter
func (cc *TaskCC) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
return []dcSDK.FilterRuleItem{
{
Rule: dcSDK.FilterRuleFileSuffix,
Operator: dcSDK.FilterRuleOperatorEqual,
Standard: ".gch",
HandleType: dcSDK.FilterRuleHandleAllDistribution,
},
}, nil
// return []dcSDK.FilterRuleItem{
// {
// Rule: dcSDK.FilterRuleFileSuffix,
// Operator: dcSDK.FilterRuleOperatorEqual,
// Standard: ".gch",
// HandleType: dcSDK.FilterRuleHandleAllDistribution,
// },
// }, nil
return nil, nil
}

func (cc *TaskCC) analyzeIncludes(dependf string) ([]*dcFile.Info, error) {
Expand Down Expand Up @@ -679,8 +680,6 @@ func (cc *TaskCC) trypumpwithcache(command []string) (*dcSDK.BKDistCommand, erro
tstart = tend

if err == nil {
blog.Infof("cc: parse command,got total %d includes files", len(includes))

// add pch file as input
if pchfile != "" {
// includes = append(includes, pchfile)
Expand All @@ -699,6 +698,10 @@ func (cc *TaskCC) trypumpwithcache(command []string) (*dcSDK.BKDistCommand, erro
}
}

oldlen := len(includes)
includes = dcFile.Uniq(includes)
blog.Infof("cc: parse command,got total %d uniq %d includes files", oldlen, len(includes))

inputFiles := []dcSDK.FileDesc{}
// priority := dcSDK.MaxFileDescPriority
for _, f := range includes {
Expand Down
23 changes: 13 additions & 10 deletions src/backend/booster/bk_dist/handler/ue4/cc/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,14 +225,15 @@ func (cc *TaskCC) FinalExecute(args []string) {

// GetFilterRules add file send filter
func (cc *TaskCC) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
return []dcSDK.FilterRuleItem{
{
Rule: dcSDK.FilterRuleFileSuffix,
Operator: dcSDK.FilterRuleOperatorEqual,
Standard: ".gch",
HandleType: dcSDK.FilterRuleHandleAllDistribution,
},
}, nil
// return []dcSDK.FilterRuleItem{
// {
// Rule: dcSDK.FilterRuleFileSuffix,
// Operator: dcSDK.FilterRuleOperatorEqual,
// Standard: ".gch",
// HandleType: dcSDK.FilterRuleHandleAllDistribution,
// },
// }, nil
return nil, nil
}

func (cc *TaskCC) getIncludeExe() (string, error) {
Expand Down Expand Up @@ -816,8 +817,6 @@ func (cc *TaskCC) trypump(command []string) (*dcSDK.BKDistCommand, error, error)
// }

if err == nil {
blog.Infof("cc: parse command,got total %d includes files", len(includes))

// add pch file as input
if pchfile != "" {
// includes = append(includes, pchfile)
Expand All @@ -836,6 +835,10 @@ func (cc *TaskCC) trypump(command []string) (*dcSDK.BKDistCommand, error, error)
}
}

oldlen := len(includes)
includes = dcFile.Uniq(includes)
blog.Infof("cc: parse command,got total %d uniq %d includes files", oldlen, len(includes))

inputFiles := []dcSDK.FileDesc{}
// priority := dcSDK.MaxFileDescPriority
for _, f := range includes {
Expand Down
21 changes: 12 additions & 9 deletions src/backend/booster/bk_dist/handler/ue4/cl/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,13 +339,14 @@ func (cl *TaskCL) FinalExecute(args []string) {

// GetFilterRules add file send filter
func (cl *TaskCL) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
return []dcSDK.FilterRuleItem{
{
Rule: dcSDK.FilterRuleFileSuffix,
Operator: dcSDK.FilterRuleOperatorEqual,
Standard: ".pch",
},
}, nil
// return []dcSDK.FilterRuleItem{
// {
// Rule: dcSDK.FilterRuleFileSuffix,
// Operator: dcSDK.FilterRuleOperatorEqual,
// Standard: ".pch",
// },
// }, nil
return nil, nil
}

func (cl *TaskCL) getIncludeExe() (string, error) {
Expand Down Expand Up @@ -719,8 +720,6 @@ func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error, error)
tstart = tend

if err == nil {
blog.Infof("cl: parse command,got total %d includes files", len(includes))

// add pch file as input
if pchfile != "" {
// includes = append(includes, pchfile)
Expand All @@ -739,6 +738,10 @@ func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error, error)
}
}

oldlen := len(includes)
includes = dcFile.Uniq(includes)
blog.Infof("cc: parse command,got total %d uniq %d includes files", oldlen, len(includes))

inputFiles := []dcSDK.FileDesc{}
// priority := dcSDK.MaxFileDescPriority
for _, f := range includes {
Expand Down
15 changes: 8 additions & 7 deletions src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,13 +187,14 @@ func (cf *TaskCLFilter) FinalExecute(args []string) {

// GetFilterRules add file send filter
func (cf *TaskCLFilter) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
return []dcSDK.FilterRuleItem{
{
Rule: dcSDK.FilterRuleFileSuffix,
Operator: dcSDK.FilterRuleOperatorEqual,
Standard: ".pch",
},
}, nil
// return []dcSDK.FilterRuleItem{
// {
// Rule: dcSDK.FilterRuleFileSuffix,
// Operator: dcSDK.FilterRuleOperatorEqual,
// Standard: ".pch",
// },
// }, nil
return nil, nil
}

func (cf *TaskCLFilter) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) {
Expand Down
Loading

0 comments on commit 4e26792

Please sign in to comment.