From 23e3a691ae5aed1b28592c9d452ffed5163ee834 Mon Sep 17 00:00:00 2001 From: LTLA Date: Wed, 11 Sep 2024 16:38:01 -0700 Subject: [PATCH 1/5] Begin adding support for self-links within an upload. --- transfer.go | 194 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 119 insertions(+), 75 deletions(-) diff --git a/transfer.go b/transfer.go index 0f04540..f83eac7 100644 --- a/transfer.go +++ b/transfer.go @@ -63,7 +63,51 @@ func computeChecksum(path string) (string, error) { return hex.EncodeToString(h.Sum(nil)), nil } -func resolveSymlink( +func createDedupManifest(registry, project, asset string) (map[string]linkMetadata, error) { + // Loading the latest version's metadata into a deduplication index. + // There's no need to check for probational versions here as only + // non-probational versions ever make it into '..latest'. + last_dedup := map[string]linkMetadata{} + asset_dir := filepath.Join(registry, project, asset) + latest_path := filepath.Join(asset_dir, latestFileName) + + _, err := os.Stat(latest_path) + if err == nil { + latest, err := readLatest(asset_dir) + if err != nil { + return nil, fmt.Errorf("failed to identify the latest version; %w", err) + } + + manifest, err := readManifest(filepath.Join(asset_dir, latest.Version)) + if err != nil { + return nil, fmt.Errorf("failed to read the latest version's manifest; %w", err) + } + + for k, v := range manifest { + self := linkMetadata{ + Project: project, + Asset: asset, + Version: latest.Version, + Path: k, + } + if v.Link != nil { + if v.Link.Ancestor != nil { + self.Ancestor = v.Link.Ancestor + } else { + self.Ancestor = v.Link + } + } + last_dedup[strconv.FormatInt(v.Size, 10) + "-" + v.Md5sum] = self + } + + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("failed to stat '" + latest_path + "; %w", err) + } + + return last_dedup, nil +} + +func resolveRegistrySymlink( registry string, project string, asset string, @@ -180,51 +224,9 @@ func createRelativeSymlink(relative_target, relative_link, full_link string) err } func Transfer(source, registry, project, asset, version string) error { - destination := filepath.Join(registry, project, asset, version) - manifest := map[string]interface{}{} - manifest_cache := map[string]map[string]manifestEntry{} - summary_cache := map[string]bool{} - - // Loading the latest version's metadata into a deduplication index. - // There's no need to check for probational versions here as only - // non-probational versions ever make it into '..latest'. - last_dedup := map[string]linkMetadata{} - { - asset_dir := filepath.Join(registry, project, asset) - latest_path := filepath.Join(asset_dir, latestFileName) - - _, err := os.Stat(latest_path) - if err == nil { - latest, err := readLatest(asset_dir) - if err != nil { - return fmt.Errorf("failed to identify the latest version; %w", err) - } - - manifest, err := readManifest(filepath.Join(asset_dir, latest.Version)) - if err != nil { - return fmt.Errorf("failed to read the latest version's manifest; %w", err) - } - - for k, v := range manifest { - self := linkMetadata{ - Project: project, - Asset: asset, - Version: latest.Version, - Path: k, - } - if v.Link != nil { - if v.Link.Ancestor != nil { - self.Ancestor = v.Link.Ancestor - } else { - self.Ancestor = v.Link - } - } - last_dedup[strconv.FormatInt(v.Size, 10) + "-" + v.Md5sum] = self - } - - } else if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("failed to stat '" + latest_path + "; %w", err) - } + last_dedup, err := createDedupManifest(registry, project, asset) + if err != nil { + return err } // Creating a function to add the links. @@ -239,6 +241,16 @@ func Transfer(source, registry, project, asset, version string) error { sublinks[base] = link_info } + type BasicSymLink struct { + Path string + Rel string + } + more_links := []BasicSymLink{} + + // First pass fills the manifest with non-symlink files. + destination := filepath.Join(registry, project, asset, version) + manifest := map[string]interface{}{} + err := filepath.WalkDir(source, func(path string, info fs.DirEntry, err error) error { if err != nil { return fmt.Errorf("failed to walk into '" + path + "'; %w", err) @@ -275,35 +287,7 @@ func Transfer(source, registry, project, asset, version string) error { // Symlinks to files inside the registry are preserved. if restat.Mode() & os.ModeSymlink == os.ModeSymlink { - target, err := os.Readlink(path) - if err != nil { - return fmt.Errorf("failed to read the symlink at '" + path + "'; %w", err) - } - - tstat, err := os.Stat(target) - if err != nil { - return fmt.Errorf("failed to stat link target %q; %w", target, err) - } - - inside, err := filepath.Rel(registry, target) - if err != nil || !filepath.IsLocal(inside) { - return fmt.Errorf("symbolic links to files outside the registry (%q) are not supported", target) - } - if tstat.IsDir() { - return fmt.Errorf("symbolic links to directories (%q) are not supported", target) - } - - obj, err := resolveSymlink(registry, project, asset, version, inside, manifest_cache, summary_cache) - if err != nil { - return fmt.Errorf("failed to resolve the symlink at '" + path + "'; %w", err) - } - manifest[rel] = *obj - - err = createRelativeSymlink(inside, rel, final) - if err != nil { - return fmt.Errorf("failed to create a symlink for '" + rel + "'; %w", err) - } - addLink(rel, obj.Link) + more_links = append(more_links, BasicSymLink{ Path: path, Rel: rel }) return nil } @@ -350,6 +334,66 @@ func Transfer(source, registry, project, asset, version string) error { return err } + // Second pass goes through all the symlinks to existing files in the registry. + manifest_cache := map[string]map[string]manifestEntry{} + summary_cache := map[string]bool{} + + type BasicSymLink struct { + Path string + Rel string + } + more_links := []BasicSymLink{} + + + for entry := range more_links { + path := entry.Path + rel := entry.Rel + + target, err := os.Readlink(path) + if err != nil { + return fmt.Errorf("failed to read the symlink at '" + path + "'; %w", err) + } + + if (!filepath.IsAbs(target)) { + target = filepath.Clean(filepath.Join(filepath.Dir(path), target)) + } + + tstat, err := os.Stat(target) + if err != nil { + return fmt.Errorf("failed to stat link target %q; %w", target, err) + } + if tstat.IsDir() { + return fmt.Errorf("symbolic links to directories (%q) are not supported", target) + } + + inside, err := filepath.Rel(registry, target) + if err != nil || !filepath.IsLocal(inside) { + local_inside, err := filepath.Rel(source, target) + if err == nil && filepath.IsLocal(inside) { + local_links = append(local_links, entry) + continue + } + return fmt.Errorf("symbolic links to files outside the source or registry directories (%q) are not supported", target) + } + + obj, err := resolveRegistrySymlink(registry, project, asset, version, inside, manifest_cache, summary_cache) + if err != nil { + return fmt.Errorf("failed to resolve the symlink at '" + path + "'; %w", err) + } + manifest[rel] = *obj + + err = createRelativeSymlink(inside, rel, final) + if err != nil { + return fmt.Errorf("failed to create a symlink for '" + rel + "'; %w", err) + } + addLink(rel, obj.Link) + } + + // Third pass + + + + // Dumping the JSON metadata. manifest_path := filepath.Join(destination, manifestFileName) err = dumpJson(manifest_path, &manifest) From 2f6bd81b6a28fdf0c1135f0e8598faca6ba151d6 Mon Sep 17 00:00:00 2001 From: LTLA Date: Thu, 12 Sep 2024 09:37:34 -0700 Subject: [PATCH 2/5] Got the damn thing to compile. --- transfer.go | 140 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 115 insertions(+), 25 deletions(-) diff --git a/transfer.go b/transfer.go index f83eac7..b0d860f 100644 --- a/transfer.go +++ b/transfer.go @@ -201,7 +201,7 @@ func resolveRegistrySymlink( return &output, nil } -func createRelativeSymlink(relative_target, relative_link, full_link string) error { +func createRegistrySymlink(relative_target, relative_link, full_link string) error { // Actually creating the link. We convert it to a relative path // within the registry so that the registry is relocatable. working := relative_link @@ -218,7 +218,91 @@ func createRelativeSymlink(relative_target, relative_link, full_link string) err err := os.Symlink(relative_target, full_link) if err != nil { - return fmt.Errorf("failed to create a symlink at '" + full_link + "'; %w", err) + return fmt.Errorf("failed to create a registry symlink at '" + full_link + "'; %w", err) + } + return nil +} + +type localLinkInfo struct { + Target string + Final string +} + +func resolveLocalSymlink( + project string, + asset string, + version string, + rel string, + details *localLinkInfo, + local_links map[string]localLinkInfo, + manifest map[string]manifestEntry, + traversed map[string]bool, + source string, +) (*manifestEntry, error) { + var target_deets *manifestEntry + man_deets, man_ok := manifest[details.Target] + if man_ok { + target_deets = &man_deets + + } else { + _, trav_ok := traversed[rel] + if trav_ok { + return nil, fmt.Errorf("cyclic symlinks detected at '%s'", filepath.Join(source, rel)) + } + traversed[rel] = false + + rel_deets, rel_ok := local_links[details.Target] + if !rel_ok { + // This should never be reached as the input is already screened for symlinks with valid targets. + return nil, fmt.Errorf("symlink at '%s' should point to a file or another symlink", filepath.Join(source, rel)) + } + + ancestor, err := resolveLocalSymlink(project, asset, version, details.Target, &rel_deets, local_links, manifest, traversed, source) + if err != nil { + return nil, err + } + + target_deets = ancestor + } + + output := manifestEntry{ + Size: target_deets.Size, + Md5sum: target_deets.Md5sum, + Link: &linkMetadata{ + Project: project, + Asset: asset, + Version: version, + Path: details.Target, + }, + } + + if target_deets.Link != nil { + if target_deets.Link.Ancestor != nil { + output.Link.Ancestor = target_deets.Link.Ancestor + } else { + output.Link.Ancestor = target_deets.Link + } + } + + manifest[rel] = output + return &output, nil +} + +func createLocalSymlink(relative_target, relative_link, full_link string) error { + // Actually creating the link. We convert it to a relative path + // within the registry so that the registry is relocatable. + working := relative_link + for { + working = filepath.Dir(working) + if working == "." { + break + } + relative_target = filepath.Join("..", relative_target) + } + + err := os.Symlink(relative_target, full_link) + if err != nil { + return fmt.Errorf("failed to create a local symlink at '" + full_link + "'; %w", err) } return nil } @@ -241,17 +325,18 @@ func Transfer(source, registry, project, asset, version string) error { sublinks[base] = link_info } - type BasicSymLink struct { + type basicSymLink struct { Path string Rel string + Final string } - more_links := []BasicSymLink{} + more_links := []basicSymLink{} // First pass fills the manifest with non-symlink files. destination := filepath.Join(registry, project, asset, version) - manifest := map[string]interface{}{} + manifest := map[string]manifestEntry{} - err := filepath.WalkDir(source, func(path string, info fs.DirEntry, err error) error { + err = filepath.WalkDir(source, func(path string, info fs.DirEntry, err error) error { if err != nil { return fmt.Errorf("failed to walk into '" + path + "'; %w", err) } @@ -287,7 +372,7 @@ func Transfer(source, registry, project, asset, version string) error { // Symlinks to files inside the registry are preserved. if restat.Mode() & os.ModeSymlink == os.ModeSymlink { - more_links = append(more_links, BasicSymLink{ Path: path, Rel: rel }) + more_links = append(more_links, basicSymLink{ Path: path, Rel: rel, Final: final }) return nil } @@ -306,7 +391,7 @@ func Transfer(source, registry, project, asset, version string) error { if ok { man_entry.Link = &last_entry manifest[rel] = man_entry - err = createRelativeSymlink(filepath.Join(last_entry.Project, last_entry.Asset, last_entry.Version, last_entry.Path), rel, final) + err = createRegistrySymlink(filepath.Join(last_entry.Project, last_entry.Asset, last_entry.Version, last_entry.Path), rel, final) if err != nil { return fmt.Errorf("failed to create a symlink for '" + rel + "'; %w", err) } @@ -337,17 +422,12 @@ func Transfer(source, registry, project, asset, version string) error { // Second pass goes through all the symlinks to existing files in the registry. manifest_cache := map[string]map[string]manifestEntry{} summary_cache := map[string]bool{} + local_links := map[string]localLinkInfo{} - type BasicSymLink struct { - Path string - Rel string - } - more_links := []BasicSymLink{} - - - for entry := range more_links { + for _, entry := range more_links { path := entry.Path rel := entry.Rel + final := entry.Final target, err := os.Readlink(path) if err != nil { @@ -366,33 +446,43 @@ func Transfer(source, registry, project, asset, version string) error { return fmt.Errorf("symbolic links to directories (%q) are not supported", target) } - inside, err := filepath.Rel(registry, target) - if err != nil || !filepath.IsLocal(inside) { + registry_inside, err := filepath.Rel(registry, target) + if err != nil || !filepath.IsLocal(registry_inside) { local_inside, err := filepath.Rel(source, target) - if err == nil && filepath.IsLocal(inside) { - local_links = append(local_links, entry) + if err == nil && filepath.IsLocal(local_inside) { + local_links[rel] = localLinkInfo{ Target: local_inside, Final: final } continue } return fmt.Errorf("symbolic links to files outside the source or registry directories (%q) are not supported", target) } - obj, err := resolveRegistrySymlink(registry, project, asset, version, inside, manifest_cache, summary_cache) + obj, err := resolveRegistrySymlink(registry, project, asset, version, registry_inside, manifest_cache, summary_cache) if err != nil { return fmt.Errorf("failed to resolve the symlink at '" + path + "'; %w", err) } manifest[rel] = *obj - err = createRelativeSymlink(inside, rel, final) + err = createRegistrySymlink(registry_inside, rel, final) if err != nil { return fmt.Errorf("failed to create a symlink for '" + rel + "'; %w", err) } addLink(rel, obj.Link) } - // Third pass - - + // Third pass to recursively resolve local symlinks. + traversed := map[string]bool{} + for rel, info := range local_links { + man, err := resolveLocalSymlink(project, asset, version, rel, &info, local_links, manifest, traversed, source) + if err != nil { + return err + } + err = createLocalSymlink(info.Target, rel, info.Final) + if err != nil { + return fmt.Errorf("failed to create a symlink for '" + rel + "'; %w", err) + } + addLink(rel, man.Link) + } // Dumping the JSON metadata. manifest_path := filepath.Join(destination, manifestFileName) From ff45e29286d59e3f6463feb46ba63368f1993b49 Mon Sep 17 00:00:00 2001 From: LTLA Date: Thu, 12 Sep 2024 10:52:47 -0700 Subject: [PATCH 3/5] Added tests. --- transfer.go | 29 +++-- transfer_test.go | 271 ++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 244 insertions(+), 56 deletions(-) diff --git a/transfer.go b/transfer.go index b0d860f..4cab610 100644 --- a/transfer.go +++ b/transfer.go @@ -202,8 +202,8 @@ func resolveRegistrySymlink( } func createRegistrySymlink(relative_target, relative_link, full_link string) error { - // Actually creating the link. We convert it to a relative path - // within the registry so that the registry is relocatable. + // We convert the link target to a relative path within the registry so + // that the registry is easily relocatable. working := relative_link for { working = filepath.Dir(working) @@ -253,8 +253,7 @@ func resolveLocalSymlink( rel_deets, rel_ok := local_links[details.Target] if !rel_ok { - // This should never be reached as the input is already screened for symlinks with valid targets. - return nil, fmt.Errorf("symlink at '%s' should point to a file or another symlink", filepath.Join(source, rel)) + return nil, fmt.Errorf("symlink at '%s' should point to a manifest file or another symlink", filepath.Join(source, rel)) } ancestor, err := resolveLocalSymlink(project, asset, version, details.Target, &rel_deets, local_links, manifest, traversed, source) @@ -289,8 +288,6 @@ func resolveLocalSymlink( } func createLocalSymlink(relative_target, relative_link, full_link string) error { - // Actually creating the link. We convert it to a relative path - // within the registry so that the registry is relocatable. working := relative_link for { working = filepath.Dir(working) @@ -438,6 +435,16 @@ func Transfer(source, registry, project, asset, version string) error { target = filepath.Clean(filepath.Join(filepath.Dir(path), target)) } + registry_inside, err := filepath.Rel(registry, target) + if err != nil || !filepath.IsLocal(registry_inside) { + local_inside, err := filepath.Rel(source, target) + if err != nil || !filepath.IsLocal(local_inside) { + return fmt.Errorf("symbolic links to files outside the source or registry directories (%q) are not supported", target) + } + local_links[rel] = localLinkInfo{ Target: local_inside, Final: final } + continue + } + tstat, err := os.Stat(target) if err != nil { return fmt.Errorf("failed to stat link target %q; %w", target, err) @@ -446,16 +453,6 @@ func Transfer(source, registry, project, asset, version string) error { return fmt.Errorf("symbolic links to directories (%q) are not supported", target) } - registry_inside, err := filepath.Rel(registry, target) - if err != nil || !filepath.IsLocal(registry_inside) { - local_inside, err := filepath.Rel(source, target) - if err == nil && filepath.IsLocal(local_inside) { - local_links[rel] = localLinkInfo{ Target: local_inside, Final: final } - continue - } - return fmt.Errorf("symbolic links to files outside the source or registry directories (%q) are not supported", target) - } - obj, err := resolveRegistrySymlink(registry, project, asset, version, registry_inside, manifest_cache, summary_cache) if err != nil { return fmt.Errorf("failed to resolve the symlink at '" + path + "'; %w", err) diff --git a/transfer_test.go b/transfer_test.go index 5a01973..f4dbc3a 100644 --- a/transfer_test.go +++ b/transfer_test.go @@ -186,7 +186,18 @@ func extractSymlinkTarget(path string) (string, error) { return target, nil } -func verifySymlink(manifest map[string]manifestEntry, version_dir, path, contents, target_project, target_asset, target_version, target_path string, has_ancestor bool) error { +func verifySymlink( + manifest map[string]manifestEntry, + version_dir, + path, + contents, + target_project, + target_asset, + target_version, + target_path string, + in_registry bool, + has_ancestor bool, +) error { info, ok := manifest[path] if !ok || int(info.Size) != len(contents) || @@ -209,41 +220,83 @@ func verifySymlink(manifest map[string]manifestEntry, version_dir, path, content if err != nil { return err } - if !strings.HasPrefix(target, "../") || !strings.HasSuffix(target, "/" + target_project + "/" + target_asset + "/" + target_version + "/" + target_path) { - return fmt.Errorf("unexpected symlink target for %q (got %q)", path, target) - } - { - dir, base := filepath.Split(path) - linkmeta_path := filepath.Join(version_dir, dir, linksFileName) - linkmeta_raw, err := os.ReadFile(linkmeta_path) - if err != nil { - return fmt.Errorf("failed to read the link metadata; %w", err) + okay := true + if in_registry { + if !strings.HasPrefix(target, "../") || !strings.HasSuffix(target, "/" + target_project + "/" + target_asset + "/" + target_version + "/" + target_path) { + okay = false } - - var linkmeta map[string]linkMetadata - err = json.Unmarshal(linkmeta_raw, &linkmeta) - if err != nil { - return fmt.Errorf("failed to parse the link metadata; %w", err) + } else { + if filepath.IsAbs(target) { + okay = false + } else { + candidate := filepath.Clean(filepath.Join(filepath.Dir(path), target)) + if !filepath.IsLocal(candidate) { + okay = false + } } + } + if !okay { + return fmt.Errorf("unexpected symlink format for %q (got %q)", path, target) + } - found, ok := linkmeta[base] - if !ok { - return fmt.Errorf("failed to find %q in the link metadata of %q", base, dir) - } + dir, base := filepath.Split(path) + linkmeta_path := filepath.Join(version_dir, dir, linksFileName) + linkmeta_raw, err := os.ReadFile(linkmeta_path) + if err != nil { + return fmt.Errorf("failed to read the link metadata; %w", err) + } - if found.Project != target_project || - found.Asset != target_asset || - found.Version != target_version || - found.Path != target_path || - has_ancestor != (found.Ancestor != nil) { - return fmt.Errorf("unexpected link metadata entry for %q", path) - } + var linkmeta map[string]linkMetadata + err = json.Unmarshal(linkmeta_raw, &linkmeta) + if err != nil { + return fmt.Errorf("failed to parse the link metadata; %w", err) + } + + found, ok := linkmeta[base] + if !ok { + return fmt.Errorf("failed to find %q in the link metadata of %q", base, dir) + } + + if found.Project != target_project || + found.Asset != target_asset || + found.Version != target_version || + found.Path != target_path || + has_ancestor != (found.Ancestor != nil) { + return fmt.Errorf("unexpected link metadata entry for %q", path) } return nil } +func verifyRegistrySymlink( + manifest map[string]manifestEntry, + version_dir, + path, + contents, + target_project, + target_asset, + target_version, + target_path string, + has_ancestor bool, +) error { + return verifySymlink(manifest, version_dir, path, contents, target_project, target_asset, target_version, target_path, true, has_ancestor) +} + +func verifyLocalSymlink( + manifest map[string]manifestEntry, + version_dir, + path, + contents, + target_project, + target_asset, + target_version, + target_path string, + has_ancestor bool, +) error { + return verifySymlink(manifest, version_dir, path, contents, target_project, target_asset, target_version, target_path, false, has_ancestor) +} + func verifyNotSymlink(manifest map[string]manifestEntry, version_dir, path, contents string) error { info, ok := manifest[path] if !ok || int(info.Size) != len(contents) || info.Link != nil { @@ -369,13 +422,13 @@ func TestTransferDeduplication(t *testing.T) { } // Different file name. - err = verifySymlink(man, destination, "evolution/next", "raichu", project, asset, version, "evolution/up", false) + err = verifyRegistrySymlink(man, destination, "evolution/next", "raichu", project, asset, version, "evolution/up", false) if err != nil { t.Fatal(err) } // Same file name. - err = verifySymlink(man, destination, "moves/electric/thunder", "110", project, asset, version, "moves/electric/thunder", false) + err = verifyRegistrySymlink(man, destination, "moves/electric/thunder", "110", project, asset, version, "moves/electric/thunder", false) if err != nil { t.Fatal(err) } @@ -427,7 +480,7 @@ func TestTransferDeduplication(t *testing.T) { t.Fatalf("failed to read the manifest; %v", err) } - err = verifySymlink(man, destination, "evolution/final", "raichu", project, asset, "blue", "evolution/next", true) + err = verifyRegistrySymlink(man, destination, "evolution/final", "raichu", project, asset, "blue", "evolution/next", true) if err != nil { t.Fatal(err) } @@ -436,7 +489,7 @@ func TestTransferDeduplication(t *testing.T) { t.Fatal(err) } - err = verifySymlink(man, destination, "moves/electric/thunderbolt", "90", project, asset, "blue", "moves/electric/thunderbolt", true) + err = verifyRegistrySymlink(man, destination, "moves/electric/thunderbolt", "90", project, asset, "blue", "moves/electric/thunderbolt", true) if err != nil { t.Fatal(err) } @@ -455,7 +508,7 @@ func TestTransferDeduplication(t *testing.T) { t.Fatal(err) } - err = verifySymlink(man, destination, "moves/steel/iron_tail", "100", project, asset, "blue", "moves/steel/iron_tail", false) + err = verifyRegistrySymlink(man, destination, "moves/steel/iron_tail", "100", project, asset, "blue", "moves/steel/iron_tail", false) if err != nil { t.Fatal(err) } @@ -480,7 +533,7 @@ func TestTransferDeduplication(t *testing.T) { t.Fatalf("failed to read the manifest; %v", err) } - err = verifySymlink(man, destination, "evolution/final", "raichu", project, asset, "green", "evolution/final", true) + err = verifyRegistrySymlink(man, destination, "evolution/final", "raichu", project, asset, "green", "evolution/final", true) if err != nil { t.Fatal(err) } @@ -489,13 +542,13 @@ func TestTransferDeduplication(t *testing.T) { t.Fatal(err) } - err = verifySymlink(man, destination, "moves/electric/thunder_shock", "9999", project, asset, "green", "moves/electric/thunder_shock", false) + err = verifyRegistrySymlink(man, destination, "moves/electric/thunder_shock", "9999", project, asset, "green", "moves/electric/thunder_shock", false) if err != nil { t.Fatal(err) } // We can also form new ancestral links. - err = verifySymlink(man, destination, "moves/steel/iron_tail", "100", project, asset, "green", "moves/steel/iron_tail", true) + err = verifyRegistrySymlink(man, destination, "moves/steel/iron_tail", "100", project, asset, "green", "moves/steel/iron_tail", true) if err != nil { t.Fatal(err) } @@ -506,7 +559,7 @@ func TestTransferDeduplication(t *testing.T) { } } -func TestTransferLinks(t *testing.T) { +func TestTransferRegistryLinks(t *testing.T) { reg, err := os.MkdirTemp("", "") if err != nil { t.Fatalf("failed to create the registry; %v", err) @@ -610,12 +663,12 @@ func TestTransferLinks(t *testing.T) { t.Fatalf("failed to read the manifest; %v", err) } - err = verifySymlink(man, destination, "types/first", "electric", "pokemon", "pikachu", "red", "type", false) + err = verifyRegistrySymlink(man, destination, "types/first", "electric", "pokemon", "pikachu", "red", "type", false) if err != nil { t.Fatal(err) } - err = verifySymlink(man, destination, "moves/electric/THUNDERBOLT", "90", "pokemon", "pikachu", "blue", "moves/electric/thunderbolt", true) + err = verifyRegistrySymlink(man, destination, "moves/electric/THUNDERBOLT", "90", "pokemon", "pikachu", "blue", "moves/electric/thunderbolt", true) if err != nil { t.Fatal(err) } @@ -624,7 +677,7 @@ func TestTransferLinks(t *testing.T) { t.Fatal(err) } - err = verifySymlink(man, destination, "best_friend", "pichu", "pokemon", "pikachu", "green", "evolution/down", true) + err = verifyRegistrySymlink(man, destination, "best_friend", "pichu", "pokemon", "pikachu", "green", "evolution/down", true) if err != nil { t.Fatal(err) } @@ -640,7 +693,7 @@ func TestTransferLinks(t *testing.T) { } } -func TestTransferLinkFailures(t *testing.T) { +func TestTransferRegistryLinkFailures(t *testing.T) { reg, err := os.MkdirTemp("", "") if err != nil { t.Fatalf("failed to create the registry; %v", err) @@ -674,7 +727,7 @@ func TestTransferLinkFailures(t *testing.T) { asset := "PIKAPIKA" version := "SILVER" err = Transfer(src, reg, project, asset, version) - if err == nil || !strings.Contains(err.Error(), "outside the registry") { + if err == nil || !strings.Contains(err.Error(), "outside the source or registry") { t.Fatal("expected transfer failure for files outside the registry") } } @@ -820,3 +873,141 @@ func TestTransferLinkFailures(t *testing.T) { } } } + +func TestTransferLocalLinks(t *testing.T) { + reg, err := os.MkdirTemp("", "") + if err != nil { + t.Fatalf("failed to create the registry; %v", err) + } + + src, err := setupSourceForTransferTest() + if err != nil { + t.Fatalf("failed to set up test directories; %v", err) + } + + err = os.Symlink(filepath.Join(src, "type"), filepath.Join(src, "type2")) + if err != nil { + t.Fatalf("failed to create a symlink for 'types2'; %v", err) + } + + err = os.Symlink(filepath.Join(src, "type2"), filepath.Join(src, "evolution", "foo")) // symlink to another symlink. + if err != nil { + t.Fatalf("failed to create a symlink for 'evolution/foo'; %v", err) + } + + err = os.Symlink(filepath.Join("..", "type2"), filepath.Join(src, "evolution", "bar")) // same, but as a relative link. + if err != nil { + t.Fatalf("failed to create a symlink for 'evolution/bar'; %v", err) + } + + err = os.Symlink(filepath.Join("evolution", "up"), filepath.Join(src, "WHEE")) // relative symlink to subdirectory. + if err != nil { + t.Fatalf("failed to create a symlink for 'WHEE'; %v", err) + } + + project := "POKEMON" + asset := "PIKAPIKA" + version := "GOLD" + + err = Transfer(src, reg, project, asset, version) + if err != nil { + t.Fatalf("failed to perform the transfer; %v", err) + } + + destination := filepath.Join(reg, project, asset, version) + man, err := readManifest(destination) + if err != nil { + t.Fatalf("failed to read the manifest; %v", err) + } + + err = verifyLocalSymlink(man, destination, "type2", "electric", project, asset, version, "type", false) + if err != nil { + t.Fatal(err) + } + + err = verifyLocalSymlink(man, destination, "evolution/foo", "electric", project, asset, version, "type2", true) + if err != nil { + t.Fatal(err) + } + err = verifyAncestralSymlink(man, destination, "evolution/foo", reg, project, asset, version, "type") + if err != nil { + t.Fatal(err) + } + + err = verifyLocalSymlink(man, destination, "evolution/bar", "electric", project, asset, version, "type2", true) + if err != nil { + t.Fatal(err) + } + err = verifyAncestralSymlink(man, destination, "evolution/bar", reg, project, asset, version, "type") + if err != nil { + t.Fatal(err) + } + + err = verifyLocalSymlink(man, destination, "WHEE", "raichu", project, asset, version, "evolution/up", false) + if err != nil { + t.Fatal(err) + } +} + +func TestTransferLocalLinkFailures(t *testing.T) { + reg, err := os.MkdirTemp("", "") + if err != nil { + t.Fatalf("failed to create the registry; %v", err) + } + + // Cyclic symlinks. + { + src, err := setupSourceForTransferTest() + if err != nil { + t.Fatalf("failed to set up test directories; %v", err) + } + + err = os.Symlink(filepath.Join(src, "foo"), filepath.Join(src, "bar")) + if err != nil { + t.Fatalf("failed to create a symlink for 'bar'; %v", err) + } + + err = os.Symlink(filepath.Join(src, "bar"), filepath.Join(src, "evolution/whee")) + if err != nil { + t.Fatalf("failed to create a symlink for 'evolution/whee'; %v", err) + } + + err = os.Symlink(filepath.Join(src, "evolution/whee"), filepath.Join(src, "foo")) + if err != nil { + t.Fatalf("failed to create a symlink for 'foo'; %v", err) + } + + project := "POKEMON" + asset := "PIKAPIKA" + version := "GOLD" + + err = Transfer(src, reg, project, asset, version) + if err == nil || !strings.Contains(err.Error(), "cyclic") { + t.Fatalf("failed to detect cyclic local links") + } + } + + // Symlink to a directory. + { + src, err := setupSourceForTransferTest() + if err != nil { + t.Fatalf("failed to set up test directories; %v", err) + } + + err = os.Symlink(filepath.Join(src, "evolution"), filepath.Join(src, "FOOBAR")) + if err != nil { + t.Fatalf("failed to create a symlink for 'FOOBAR'; %v", err) + } + + project := "POKEMON" + asset := "PIKAPIKA" + version := "SILVER" + + err = Transfer(src, reg, project, asset, version) + if err == nil || !strings.Contains(err.Error(), "should point") { + t.Fatalf("failed to detect links to a directory") + } + } +} + + From 52573e1f87f16df604ece6c13df7c0aaa323887e Mon Sep 17 00:00:00 2001 From: LTLA Date: Thu, 12 Sep 2024 10:56:02 -0700 Subject: [PATCH 4/5] Mention local links in the docs. --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c131666..eee3609 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,8 @@ This contains a JSON object with the following properties: When creating a new version of a project's assets, the Gobbler will attempt deduplication based on the file size and MD5 checksum. Specifically, it will inspect the immediate previous version of the asset to see if any other files have a matching size/checksum. If so, it will create a symbolic link to the file in the previous version rather than wasting disk space with a redundant copy. -Users can also directly instruct the Gobbler to create links by supplying symlinks to existing files in the registry. +Users can also directly instruct the Gobbler to create links by supplying symlinks during upload, +either to existing files in the registry or to other files in the same to-be-uploaded version of the asset. Any "linked-from" files (i.e., those identified as copies of other existing files) will be present as symbolic links in the registry. The existence of linked-from files can also be determined from the `..manifest` file for each project-asset-version; @@ -188,12 +189,14 @@ On success, a new project is created with the designated permissions and a JSON ### Uploads and updates To upload a new version of an asset of a project, users should create a temporary directory within the staging directory. -The directory may have any name but should avoid starting with `request-`. +The temporary directory may have any name but should avoid starting with `request-`. Files within this temporary directory will be transferred to the appropriate subdirectory within the registry, subject to the following rules: - Hidden files (i.e., prefixed with `.`) are ignored. - Symbolic links to directories are not allowed. -- Symbolic links to files only allowed if the symlink target is an existing file within a project-asset-version subdirectory of the registry. +- Symbolic links to files are allowed if: + - The symlink target is an existing file within a project-asset-version subdirectory of the registry. + - The symlink target is a file in the same temporary directory. Once this directory is constructed and populated, the user should create a file with the `request-upload-` prefix. This file should be JSON-formatted with the following properties: From 413ae794c5a7799bd51cfaca052c2121e69693f0 Mon Sep 17 00:00:00 2001 From: LTLA Date: Thu, 12 Sep 2024 11:10:59 -0700 Subject: [PATCH 5/5] Remove unnecessary newlines. --- transfer_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/transfer_test.go b/transfer_test.go index f4dbc3a..c460bbf 100644 --- a/transfer_test.go +++ b/transfer_test.go @@ -1009,5 +1009,3 @@ func TestTransferLocalLinkFailures(t *testing.T) { } } } - -