diff --git a/src/OrasProject.Oras/AsyncInvocationExtensions.cs b/src/OrasProject.Oras/AsyncInvocationExtensions.cs new file mode 100644 index 0000000..371a0d9 --- /dev/null +++ b/src/OrasProject.Oras/AsyncInvocationExtensions.cs @@ -0,0 +1,39 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Linq; +using System.Threading.Tasks; + +namespace OrasProject.Oras; + +internal static class AsyncInvocationExtensions +{ + /// + /// Asynchronously invokes all handlers from an event that returns a . + /// All handlers are executed in parallel + /// + /// + /// + /// + internal static Task InvokeAsync( + this Func? eventDelegate, TEventArgs args) + { + if (eventDelegate == null) return Task.CompletedTask; + + var tasks = eventDelegate.GetInvocationList() + .Select(d => (Task?)d.DynamicInvoke(args) ?? Task.CompletedTask); + + return Task.WhenAll(tasks); + } +} diff --git a/src/OrasProject.Oras/CopyGraphOptions.cs b/src/OrasProject.Oras/CopyGraphOptions.cs new file mode 100644 index 0000000..8aeac87 --- /dev/null +++ b/src/OrasProject.Oras/CopyGraphOptions.cs @@ -0,0 +1,74 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Threading.Tasks; +using OrasProject.Oras.Oci; + +namespace OrasProject.Oras; + +/// +/// CopyGraphOptions contains parameters for +/// +public struct CopyGraphOptions +{ + /// + /// PreCopyAsync handles the current descriptor before it is copied. + /// + public event Func? PreCopyAsync; + + /// + /// PreCopy handles the current descriptor before it is copied. + /// + public event Action? PreCopy; + + /// + /// PostCopyAsync handles the current descriptor after it is copied. + /// + public event Func? PostCopyAsync; + + /// + /// PostCopy handles the current descriptor after it is copied. + /// + public event Action? PostCopy; + + /// + /// CopySkippedAsync will be called when the sub-DAG rooted by the current node + /// is skipped. + /// + public event Func? CopySkippedAsync; + + /// + /// CopySkipped will be called when the sub-DAG rooted by the current node + /// is skipped. + /// + public event Action? CopySkipped; + + internal Task OnPreCopyAsync(Descriptor descriptor) + { + PreCopy?.Invoke(descriptor); + return PreCopyAsync?.InvokeAsync(descriptor) ?? Task.CompletedTask; + } + + internal Task OnPostCopyAsync(Descriptor descriptor) + { + PostCopy?.Invoke(descriptor); + return PostCopyAsync?.InvokeAsync(descriptor) ?? Task.CompletedTask; + } + + internal Task OnCopySkippedAsync(Descriptor descriptor) + { + CopySkipped?.Invoke(descriptor); + return CopySkippedAsync?.Invoke(descriptor) ?? Task.CompletedTask; + } +} diff --git a/src/OrasProject.Oras/CopyOptions.cs b/src/OrasProject.Oras/CopyOptions.cs new file mode 100644 index 0000000..a7b7b2c --- /dev/null +++ b/src/OrasProject.Oras/CopyOptions.cs @@ -0,0 +1,22 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace OrasProject.Oras; + +/// +/// CopyOptions contains parameters for +/// +public struct CopyOptions +{ + public CopyGraphOptions CopyGraphOptions; +} diff --git a/src/OrasProject.Oras/Extensions.cs b/src/OrasProject.Oras/Extensions.cs index 6b048ac..c3256a1 100644 --- a/src/OrasProject.Oras/Extensions.cs +++ b/src/OrasProject.Oras/Extensions.cs @@ -11,16 +11,35 @@ // See the License for the specific language governing permissions and // limitations under the License. -using OrasProject.Oras.Oci; using System; using System.Threading; using System.Threading.Tasks; +using OrasProject.Oras.Oci; using static OrasProject.Oras.Content.Extensions; namespace OrasProject.Oras; public static class Extensions { + /// + /// Copy copies a rooted directed acyclic graph (DAG) with the tagged root node + /// in the source Target to the destination Target. + /// The destination reference will be the same as the source reference if the + /// destination reference is left blank. + /// Returns the descriptor of the root node on successful copy. + /// + /// + /// + /// + /// + /// + /// + /// + public static Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, + CancellationToken cancellationToken = default) + { + return src.CopyAsync(srcRef, dst, dstRef, new CopyOptions(), cancellationToken); + } /// /// Copy copies a rooted directed acyclic graph (DAG) with the tagged root node @@ -33,41 +52,51 @@ public static class Extensions /// /// /// + /// /// /// /// - public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CancellationToken cancellationToken = default) + public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CopyOptions copyOptions = default, CancellationToken cancellationToken = default) { if (string.IsNullOrEmpty(dstRef)) { dstRef = srcRef; } var root = await src.ResolveAsync(srcRef, cancellationToken).ConfigureAwait(false); - await src.CopyGraphAsync(dst, root, cancellationToken).ConfigureAwait(false); + await src.CopyGraphAsync(dst, root, copyOptions.CopyGraphOptions, cancellationToken).ConfigureAwait(false); await dst.TagAsync(root, dstRef, cancellationToken).ConfigureAwait(false); return root; } - public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CancellationToken cancellationToken) + public static Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CancellationToken cancellationToken = default) + { + return src.CopyGraphAsync(dst, node, new CopyGraphOptions(), cancellationToken); + } + + public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CopyGraphOptions copyGraphOptions = default, CancellationToken cancellationToken = default) { // check if node exists in target if (await dst.ExistsAsync(node, cancellationToken).ConfigureAwait(false)) { + await copyGraphOptions.OnCopySkippedAsync(node).ConfigureAwait(false); return; } // retrieve successors var successors = await src.GetSuccessorsAsync(node, cancellationToken).ConfigureAwait(false); - // obtain data stream - var dataStream = await src.FetchAsync(node, cancellationToken).ConfigureAwait(false); + // check if the node has successors - if (successors != null) + foreach (var childNode in successors) { - foreach (var childNode in successors) - { - await src.CopyGraphAsync(dst, childNode, cancellationToken).ConfigureAwait(false); - } + await src.CopyGraphAsync(dst, childNode, copyGraphOptions, cancellationToken).ConfigureAwait(false); } + + // perform the copy + await copyGraphOptions.OnPreCopyAsync(node).ConfigureAwait(false); + var dataStream = await src.FetchAsync(node, cancellationToken).ConfigureAwait(false); await dst.PushAsync(node, dataStream, cancellationToken).ConfigureAwait(false); + + // we copied it + await copyGraphOptions.OnPostCopyAsync(node).ConfigureAwait(false); } } diff --git a/tests/OrasProject.Oras.Tests/CopyTest.cs b/tests/OrasProject.Oras.Tests/CopyTest.cs index 4f26873..c27f45b 100644 --- a/tests/OrasProject.Oras.Tests/CopyTest.cs +++ b/tests/OrasProject.Oras.Tests/CopyTest.cs @@ -16,6 +16,7 @@ using System.Text; using System.Text.Json; using Xunit; +using Index = OrasProject.Oras.Oci.Index; namespace OrasProject.Oras.Tests; @@ -142,4 +143,367 @@ public async Task CanCopyBetweenMemoryTargets() } } + + [Fact] + public async Task TestCopyExistedRoot() + { + var src = new MemoryStore(); + var dst = new MemoryStore(); + + // Generate test content + var blobs = new List(); + var descs = new List(); + + void AppendBlob(string mediaType, byte[] blob) + { + blobs.Add(blob); + var desc = new Descriptor + { + MediaType = mediaType, + Digest = Digest.ComputeSHA256(blob), + Size = blob.Length + }; + descs.Add(desc); + } + + void GenerateManifest(Descriptor config, params Descriptor[] layers) + { + var manifest = new Manifest + { + Config = config, + Layers = layers.ToList() + }; + var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); + AppendBlob(MediaType.ImageManifest, manifestBytes); + } + + // Add blobs and generate manifest + AppendBlob(MediaType.ImageConfig, Encoding.UTF8.GetBytes("config")); // Blob 0 + AppendBlob(MediaType.ImageLayer, Encoding.UTF8.GetBytes("foo")); // Blob 1 + AppendBlob(MediaType.ImageLayer, Encoding.UTF8.GetBytes("bar")); // Blob 2 + GenerateManifest(descs[0], descs[1], descs[2]); // Blob 3 + + // Push blobs to the source store + foreach (var (blob, desc) in blobs.Zip(descs)) + { + await src.PushAsync(desc, new MemoryStream(blob), CancellationToken.None); + } + + var root = descs[3]; + var refTag = "foobar"; + var newTag = "newtag"; + + // Tag root node in source + await src.TagAsync(root, refTag, CancellationToken.None); + + // Prepare copy options with OnCopySkippedAsync + var skippedCount = 0; + var skippedAsyncCount = 0; + var copyOptions = new CopyOptions + { + CopyGraphOptions = new CopyGraphOptions() + { + } + }; + + copyOptions.CopyGraphOptions.CopySkipped += _ => skippedCount++; + + copyOptions.CopyGraphOptions.CopySkippedAsync += _ => + { + skippedAsyncCount++; + return Task.CompletedTask; + }; + + // Copy with the source tag + var gotDesc = await src.CopyAsync(refTag, dst, "", copyOptions, CancellationToken.None); + Assert.Equal(root, gotDesc); + + // Copy with a new tag + gotDesc = await src.CopyAsync(refTag, dst, newTag, copyOptions, CancellationToken.None); + Assert.Equal(root, gotDesc); + + // Verify contents in the destination + foreach (var desc in descs) + { + var exists = await dst.ExistsAsync(desc, CancellationToken.None); + Assert.True(exists, $"Destination should contain descriptor {desc.Digest}"); + } + + // Verify the source tag in destination + gotDesc = await dst.ResolveAsync(refTag, CancellationToken.None); + Assert.Equal(root, gotDesc); + + // Verify the new tag in destination + gotDesc = await dst.ResolveAsync(newTag, CancellationToken.None); + Assert.Equal(root, gotDesc); + + // Verify the OnCopySkippedAsync invocation count + Assert.Equal(1, skippedCount); + Assert.Equal(1, skippedAsyncCount); + } + + [Fact] + public async Task TestCopyGraph_FullCopy() + { + var src = new MemoryStore(); + var dst = new MemoryStore(); + + // Generate test content + var blobs = new List(); + var descs = new List(); + + void AppendBlob(string mediaType, byte[] blob) + { + blobs.Add(blob); + var desc = new Descriptor + { + MediaType = mediaType, + Digest = Digest.ComputeSHA256(blob), + Size = blob.Length + }; + descs.Add(desc); + } + + void GenerateManifest(Descriptor config, params Descriptor[] layers) + { + var manifest = new Manifest + { + Config = config, + Layers = layers.ToList() + }; + var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); + AppendBlob(MediaType.ImageManifest, manifestBytes); + } + + void GenerateIndex(params Descriptor[] manifests) + { + var index = new Index + { + Manifests = manifests.ToList() + }; + var indexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(index)); + AppendBlob(MediaType.ImageIndex, indexBytes); + } + + // Append blobs and generate manifests and indices + var getBytes = (string data) => Encoding.UTF8.GetBytes(data); + AppendBlob(MediaType.ImageConfig, getBytes("config")); // Blob 0 + AppendBlob(MediaType.ImageLayer, getBytes("foo")); // Blob 1 + AppendBlob(MediaType.ImageLayer, getBytes("bar")); // Blob 2 + AppendBlob(MediaType.ImageLayer, getBytes("hello")); // Blob 3 + GenerateManifest(descs[0], descs[1], descs[2]); // Blob 4 + GenerateManifest(descs[0], descs[3]); // Blob 5 + GenerateManifest(descs[0], descs[1], descs[2], descs[3]); // Blob 6 + GenerateIndex(descs[4], descs[5]); // Blob 7 + GenerateIndex(descs[6]); // Blob 8 + GenerateIndex(); // Blob 9 + GenerateIndex(descs[7], descs[8], descs[9]); // Blob 10 + + var root = descs[^1]; // The last descriptor as the root + + // Push blobs to the source memory store + for (int i = 0; i < blobs.Count; i++) + { + await src.PushAsync(descs[i], new MemoryStream(blobs[i]), CancellationToken.None); + } + + // Set up tracking storage wrappers for verification + var srcTracker = new StorageTracker(src); + var dstTracker = new StorageTracker(dst); + + // Perform the copy graph operation + var copyOptions = new CopyGraphOptions(); + await srcTracker.CopyGraphAsync(dstTracker, root, copyOptions, CancellationToken.None); + + // Verify contents in the destination + foreach (var (desc, blob) in descs.Zip(blobs, Tuple.Create)) + { + Assert.True(await dst.ExistsAsync(desc, CancellationToken.None), $"Blob {desc.Digest} should exist in destination."); + var fetchedContent = await dst.FetchAsync(desc, CancellationToken.None); + using var memoryStream = new MemoryStream(); + await fetchedContent.CopyToAsync(memoryStream); + Assert.Equal(blob, memoryStream.ToArray()); + } + + // Verify API counts + // REMARKS: FetchCount should equal to blobs.Count + // but since there's no caching implemented, it is not + Assert.Equal(18, srcTracker.FetchCount); + Assert.Equal(0, srcTracker.PushCount); + Assert.Equal(0, srcTracker.ExistsCount); + Assert.Equal(0, dstTracker.FetchCount); + Assert.Equal(blobs.Count, dstTracker.PushCount); + + // REMARKS: ExistsCount should equal to blobs.Count + // but since there's no caching implemented, it is not + Assert.Equal(16, dstTracker.ExistsCount); + } + + [Fact] + public async Task TestCopyWithOptions() + { + var src = new MemoryStore(); + + // Generate test content + var blobs = new List(); + var descs = new List(); + + void AppendBlob(string mediaType, byte[] blob) + { + blobs.Add(blob); + descs.Add(new Descriptor + { + MediaType = mediaType, + Digest = Digest.ComputeSHA256(blob), + Size = blob.Length + }); + } + + void AppendManifest(string arc, string os, string mediaType, byte[] blob) + { + blobs.Add(blob); + descs.Add(new Descriptor + { + MediaType = mediaType, + Digest = Digest.ComputeSHA256(blob), + Size = blob.Length, + Platform = new Platform + { + Architecture = arc, + OS = os + } + }); + } + + void GenerateManifest(string arc, string os, Descriptor config, params Descriptor[] layers) + { + var manifest = new Manifest + { + Config = config, + Layers = layers.ToList() + }; + var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); + AppendManifest(arc, os, MediaType.ImageManifest, manifestBytes); + } + + void GenerateIndex(params Descriptor[] manifests) + { + var index = new Index + { + Manifests = manifests.ToList() + }; + var indexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(index)); + AppendBlob(MediaType.ImageIndex, indexBytes); + } + + // Create blobs and manifests + AppendBlob(MediaType.ImageConfig, Encoding.UTF8.GetBytes("config")); // Blob 0 + AppendBlob(MediaType.ImageLayer, Encoding.UTF8.GetBytes("foo")); // Blob 1 + AppendBlob(MediaType.ImageLayer, Encoding.UTF8.GetBytes("bar")); // Blob 2 + GenerateManifest("test-arc-1", "test-os-1", descs[0], descs[1], descs[2]); // Blob 3 + AppendBlob(MediaType.ImageLayer, Encoding.UTF8.GetBytes("hello")); // Blob 4 + GenerateManifest("test-arc-2", "test-os-2", descs[0], descs[4]); // Blob 5 + GenerateIndex(descs[3], descs[5]); // Blob 6 + + var cancellationToken = CancellationToken.None; + + // Push blobs to source store + for (int i = 0; i < blobs.Count; i++) + { + await src.PushAsync(descs[i], new MemoryStream(blobs[i]), cancellationToken); + } + + var root = descs[6]; + var refTag = "foobar"; + await src.TagAsync(root, refTag, cancellationToken); + + // Test copy with platform filter and hooks + var dst = new MemoryStore(); + var preCopyCount = 0; + var postCopyCount = 0; + var preCopyAsyncCount = 0; + var postCopyAsyncCount = 0; + var copyOptions = new CopyOptions + { + CopyGraphOptions = new CopyGraphOptions + { + } + }; + + copyOptions.CopyGraphOptions.PreCopy += _ => preCopyCount++; + copyOptions.CopyGraphOptions.PreCopyAsync += d => + { + preCopyAsyncCount++; + return Task.CompletedTask; + }; + + copyOptions.CopyGraphOptions.PostCopy += _ => postCopyCount++; + copyOptions.CopyGraphOptions.PostCopyAsync += d => + { + postCopyAsyncCount++; + return Task.CompletedTask; + }; + + var expectedDesc = descs[6]; + var gotDesc = await src.CopyAsync(refTag, dst, "", copyOptions, cancellationToken); + Assert.Equal(expectedDesc, gotDesc); + + // Verify platform-specific contents + var expectedDescs = new[] { descs[0], descs[4], descs[5] }; + foreach (var desc in expectedDescs) + { + Assert.True(await dst.ExistsAsync(desc, cancellationToken)); + } + + // Verify API counts + Assert.Equal(7, preCopyCount); + Assert.Equal(7, preCopyAsyncCount); + Assert.Equal(7, postCopyCount); + Assert.Equal(7, postCopyAsyncCount); + } + + private class StorageTracker : ITarget + { + private readonly ITarget _storage; + + public int FetchCount { get; private set; } + public int PushCount { get; private set; } + public int ExistsCount { get; private set; } + + public IList Fetched { get; } = []; + + public StorageTracker(ITarget storage) + { + _storage = storage; + } + + public async Task ExistsAsync(Descriptor desc, CancellationToken cancellationToken) + { + ExistsCount++; + return await _storage.ExistsAsync(desc, cancellationToken); + } + + public async Task FetchAsync(Descriptor desc, CancellationToken cancellationToken) + { + FetchCount++; + Fetched.Add(desc.Digest); + return await _storage.FetchAsync(desc, cancellationToken); + } + + public async Task PushAsync(Descriptor desc, Stream content, CancellationToken cancellationToken) + { + PushCount++; + await _storage.PushAsync(desc, content, cancellationToken); + } + + public async Task TagAsync(Descriptor desc, string reference, CancellationToken cancellationToken) + { + await _storage.TagAsync(desc, reference, cancellationToken); + } + + public Task ResolveAsync(string reference, CancellationToken cancellationToken) + { + return _storage.ResolveAsync(reference, cancellationToken); + } + } }