From e3e12176bfacf0bfe4dd6fb7096c98e7954f12e4 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 18 Nov 2024 18:16:21 +1100 Subject: [PATCH 01/24] push manifest with subject Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Content/Digest.cs | 14 +- .../Exceptions/NoReferrerUpdateException.cs | 20 +++ src/OrasProject.Oras/Oci/Artifact.cs | 26 ++++ src/OrasProject.Oras/Oci/Descriptor.cs | 13 ++ src/OrasProject.Oras/Oci/Index.cs | 22 ++++ .../Registry/Remote/Auth/Client.cs | 13 ++ .../Remote/HttpResponseMessageExtensions.cs | 12 +- .../Registry/Remote/ManifestStore.cs | 124 +++++++++++++++++- .../Registry/Remote/Referrers.cs | 124 ++++++++++++++++++ .../Registry/Remote/Repository.cs | 1 + .../Registry/Remote/RepositoryOptions.cs | 7 + 11 files changed, 369 insertions(+), 7 deletions(-) create mode 100644 src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs create mode 100644 src/OrasProject.Oras/Oci/Artifact.cs create mode 100644 src/OrasProject.Oras/Registry/Remote/Auth/Client.cs create mode 100644 src/OrasProject.Oras/Registry/Remote/Referrers.cs diff --git a/src/OrasProject.Oras/Content/Digest.cs b/src/OrasProject.Oras/Content/Digest.cs index 2e8c035..e87ed5c 100644 --- a/src/OrasProject.Oras/Content/Digest.cs +++ b/src/OrasProject.Oras/Content/Digest.cs @@ -47,6 +47,18 @@ internal static string Validate(string? digest) return digest; } + internal static string GetAlgorithm(string digest) + { + var validatedDigest = Validate(digest); + return validatedDigest.Split(':')[0]; + } + + internal static string GetRef(string digest) + { + var validatedDigest = Validate(digest); + return validatedDigest.Split(':')[1]; + } + /// /// Generates a SHA-256 digest from a byte array. /// @@ -59,4 +71,4 @@ internal static string ComputeSHA256(byte[] content) var output = $"sha256:{BitConverter.ToString(hash).Replace("-", "")}"; return output.ToLower(); } -} \ No newline at end of file +} diff --git a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs new file mode 100644 index 0000000..a7eaf79 --- /dev/null +++ b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs @@ -0,0 +1,20 @@ +using System; + +namespace OrasProject.Oras.Exceptions; + +public class NoReferrerUpdateException : Exception +{ + public NoReferrerUpdateException() + { + } + + public NoReferrerUpdateException(string message) + : base(message) + { + } + + public NoReferrerUpdateException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/src/OrasProject.Oras/Oci/Artifact.cs b/src/OrasProject.Oras/Oci/Artifact.cs new file mode 100644 index 0000000..4edaa82 --- /dev/null +++ b/src/OrasProject.Oras/Oci/Artifact.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace OrasProject.Oras.Oci; + +public class Artifact +{ + [JsonPropertyName("mediaType")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? MediaType { get; set; } + + [JsonPropertyName("artifactType")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? ArtifactType { get; set; } + + [JsonPropertyName("blobs")] + public required IList Blobs { get; set; } + + [JsonPropertyName("subject")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public Descriptor? Subject { get; set; } + + [JsonPropertyName("annotations")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public IDictionary? Annotations { get; set; } +} diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 9720e15..f06dd30 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -13,6 +13,7 @@ using System.Collections.Generic; using System.Text.Json.Serialization; +using OrasProject.Oras.Content; namespace OrasProject.Oras.Oci; @@ -48,4 +49,16 @@ public class Descriptor public string? ArtifactType { get; set; } internal BasicDescriptor BasicDescriptor => new BasicDescriptor(MediaType, Digest, Size); + + internal static bool IsEmptyOrNull(Descriptor? descriptor) + { + return descriptor == null || descriptor.Size == 0 || string.IsNullOrEmpty(descriptor.Digest) || string.IsNullOrEmpty(descriptor.MediaType); + } + + internal static Descriptor EmptyDescriptor() => new Descriptor + { + MediaType = "", + Digest = "", + Size = 0 + }; } diff --git a/src/OrasProject.Oras/Oci/Index.cs b/src/OrasProject.Oras/Oci/Index.cs index e11eff2..e632923 100644 --- a/src/OrasProject.Oras/Oci/Index.cs +++ b/src/OrasProject.Oras/Oci/Index.cs @@ -12,7 +12,10 @@ // limitations under the License. using System.Collections.Generic; +using System.Text; +using System.Text.Json; using System.Text.Json.Serialization; +using OrasProject.Oras.Content; namespace OrasProject.Oras.Oci; @@ -39,4 +42,23 @@ public class Index : Versioned [JsonPropertyName("annotations")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public IDictionary? Annotations { get; set; } + + internal static (Descriptor, byte[]) GenerateIndex(IList manifests) + { + var index = new Index() + { + Manifests = manifests, + MediaType = Oci.MediaType.ImageIndex, + SchemaVersion = 2 + }; + var indexContent = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(index)); + var indexDesc = new Descriptor() + { + Digest = Digest.ComputeSHA256(indexContent), + MediaType = Oci.MediaType.ImageIndex, + Size = indexContent.Length + }; + + return (indexDesc, indexContent); + } } diff --git a/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs b/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs new file mode 100644 index 0000000..0decf9a --- /dev/null +++ b/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace OrasProject.Oras.Registry.Remote.Auth +{ + internal class Client + { + + } +} diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index eb08dca..0076c06 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -26,7 +26,7 @@ namespace OrasProject.Oras.Registry.Remote; internal static class HttpResponseMessageExtensions { private const string _dockerContentDigestHeader = "Docker-Content-Digest"; - + /// /// Parses the error returned by the remote registry. /// @@ -101,6 +101,14 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string throw new HttpIOException(HttpRequestError.InvalidResponse, $"{response.RequestMessage!.Method} {response.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {digestStr}"); } } + + public static void CheckOciSubjectHeader(this HttpResponseMessage response, Repository repository) + { + if (response.Headers.TryGetValues("OCI-Subject", out var values)) + { + repository.ReferrerState = Referrers.ReferrerState.ReferrerSupported; + } + } /// /// Returns a descriptor generated from the response. @@ -160,7 +168,7 @@ public static async Task GenerateDescriptorAsync(this HttpResponseMe { serverDigest = serverHeaderDigest.FirstOrDefault(); if (!string.IsNullOrEmpty(serverDigest)) - { + { response.VerifyContentDigest(serverDigest); } } diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 549a02b..881a55e 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -14,11 +14,15 @@ using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; using System; +using System.Collections.Generic; using System.IO; using System.Net; using System.Net.Http; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using OrasProject.Oras.Content; +using Index = OrasProject.Oras.Oci.Index; namespace OrasProject.Oras.Registry.Remote; @@ -140,7 +144,9 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell /// /// public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) - => await InternalPushAsync(expected, content, expected.Digest, cancellationToken).ConfigureAwait(false); + { + await PushWithIndexingAsync(expected, content, expected.Digest, cancellationToken).ConfigureAwait(false); + } /// /// PushReferenceASync pushes the manifest with a reference tag. @@ -153,9 +159,116 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok public async Task PushAsync(Descriptor expected, Stream content, string reference, CancellationToken cancellationToken = default) { var contentReference = Repository.ParseReference(reference).ContentReference!; - await InternalPushAsync(expected, content, contentReference, cancellationToken).ConfigureAwait(false); + await PushWithIndexingAsync(expected, content, contentReference, cancellationToken).ConfigureAwait(false); + } + + private async Task PushWithIndexingAsync(Descriptor expected, Stream content, string reference, + CancellationToken cancellationToken = default) + { + switch (expected.MediaType) + { + case MediaType.ImageManifest: + case MediaType.ImageIndex: + if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) + { + await InternalPushAsync(expected, content, reference, cancellationToken).ConfigureAwait(false); + return; + } + + var contentBytes = await content.ReadAllAsync(expected, cancellationToken); + // var initPosition = content.Position; + await InternalPushAsync(expected, new MemoryStream(contentBytes), reference, cancellationToken).ConfigureAwait(false); + if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) + { + return; + } + // content.Seek(initPosition, SeekOrigin.Begin); + await IndexReferrersToPush(expected, new MemoryStream(contentBytes)); + break; + default: + await InternalPushAsync(expected, content, reference, cancellationToken); + break; + } + } + + private async Task IndexReferrersToPush(Descriptor desc, Stream content, CancellationToken cancellationToken = default) + { + Descriptor? subject = null; + switch (desc.MediaType) + { + case MediaType.ImageIndex: + var indexManifest = JsonSerializer.Deserialize(content); + if (indexManifest?.Subject == null) return; + subject = indexManifest.Subject; + desc.ArtifactType = indexManifest.ArtifactType; + desc.Annotations = indexManifest.Annotations; + break; + case MediaType.ImageManifest: + var imageManifest = JsonSerializer.Deserialize(content); + if (imageManifest?.Subject == null) return; + desc.ArtifactType = string.IsNullOrEmpty(imageManifest.ArtifactType) ? imageManifest.Config.MediaType : imageManifest.ArtifactType; + desc.Annotations = imageManifest.Annotations; + break; + default: + return; + } + + Repository.ReferrerState = Referrers.ReferrerState.ReferrerNotSupported; + if (subject == null) + { + throw new InvalidOperationException("Subject was not initialized"); + } + await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd)); } + private async Task UpdateReferrersIndex(Descriptor subject, + Referrers.ReferrerChange referrerChange, CancellationToken cancellationToken = default) + { + try + { + var referrersTag = Referrers.BuildReferrersTag(subject); + var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag); + var updatedReferrers = + Referrers.ApplyReferrerChanges(oldReferrers, new List { referrerChange }); + + if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGc) + { + var (indexDesc, indexContent) = Index.GenerateIndex(updatedReferrers); + await InternalPushAsync(indexDesc, new MemoryStream(indexContent), referrersTag, cancellationToken).ConfigureAwait(false); + } + + if (repository.Options.SkipReferrersGc || Descriptor.IsEmptyOrNull(oldDesc)) + { + return; + } + // delete oldIndexDesc + await DeleteAsync(oldDesc, cancellationToken).ConfigureAwait(false); + } + catch (NoReferrerUpdateException) + { + return; + } + } + + internal async Task<(Descriptor, IList)> PullReferrersIndexList(string referrersTag, CancellationToken cancellationToken = default) + { + try + { + var (desc, content) = await FetchAsync(referrersTag); + var index = JsonSerializer.Deserialize(content); + if (index == null) + { + throw new JsonException("null index manifests list"); + } + return (desc, index.Manifests); + } + catch (NotFoundException) + { + return (Descriptor.EmptyDescriptor(), new List()); + } + } + + /// /// Pushes the manifest content, matching the expected descriptor. /// @@ -163,20 +276,23 @@ public async Task PushAsync(Descriptor expected, Stream content, string referenc /// /// /// - private async Task InternalPushAsync(Descriptor expected, Stream stream, string contentReference, CancellationToken cancellationToken) + private async Task InternalPushAsync(Descriptor expected, Stream stream, string contentReference, + CancellationToken cancellationToken) { - var remoteReference = Repository.ParseReference(contentReference); + var remoteReference = Repository.ParseReference(contentReference); // duplicate? var url = new UriFactory(remoteReference, Repository.Options.PlainHttp).BuildRepositoryManifest(); var request = new HttpRequestMessage(HttpMethod.Put, url); request.Content = new StreamContent(stream); request.Content.Headers.ContentLength = expected.Size; request.Content.Headers.Add("Content-Type", expected.MediaType); + var client = Repository.Options.HttpClient; using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); if (response.StatusCode != HttpStatusCode.Created) { throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); } + response.CheckOciSubjectHeader(Repository); response.VerifyContentDigest(expected.Digest); } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs new file mode 100644 index 0000000..ac25d8e --- /dev/null +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -0,0 +1,124 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; +using OrasProject.Oras.Content; +using OrasProject.Oras.Exceptions; +using OrasProject.Oras.Oci; + +namespace OrasProject.Oras.Registry.Remote; + +public class Referrers +{ + internal enum ReferrerState + { + ReferrerUnknown, + ReferrerSupported, + ReferrerNotSupported + } + + internal record ReferrerChange(Descriptor Referrer, ReferrerOperation ReferrerOperation); + + internal enum ReferrerOperation + { + ReferrerAdd, + ReferrerDelete, + } + + public static string BuildReferrersTag(Descriptor descriptor) + { + return Digest.GetAlgorithm(descriptor.Digest) + "-" + Digest.GetRef(descriptor.Digest); + } + + internal static IList ApplyReferrerChanges(IList oldReferrers, IList referrerChanges) + { + if (oldReferrers == null || referrerChanges == null) + { + throw new NoReferrerUpdateException("referrerChanges or oldReferrers is null in this request"); + } + var updatedReferrers = new List(); + var referrerToIndex = new Dictionary(); + + var updateRequired = false; + foreach (var oldReferrer in oldReferrers) + { + if (Descriptor.IsEmptyOrNull(oldReferrer)) + { + updateRequired = true; + continue; + } + var basicDesc = oldReferrer.BasicDescriptor; + if (referrerToIndex.ContainsKey(basicDesc)) + { + updateRequired = true; + continue; + } + updatedReferrers.Add(oldReferrer); + referrerToIndex[basicDesc] = updatedReferrers.Count - 1; + } + + foreach (var change in referrerChanges) + { + if (Descriptor.IsEmptyOrNull(change.Referrer)) continue; + var basicDesc = change.Referrer.BasicDescriptor; + switch (change.ReferrerOperation) + { + case ReferrerOperation.ReferrerAdd: + if (!referrerToIndex.ContainsKey(basicDesc)) + { + updatedReferrers.Add(change.Referrer); + referrerToIndex[basicDesc] = updatedReferrers.Count - 1; + } + break; + + case ReferrerOperation.ReferrerDelete: + if (referrerToIndex.TryGetValue(basicDesc, out var index)) + { + updatedReferrers[index] = Descriptor.EmptyDescriptor(); + referrerToIndex.Remove(basicDesc); + } + break; + default: + break; + } + } + + if (!updateRequired && referrerToIndex.Count == oldReferrers.Count) + { + foreach (var oldReferrer in oldReferrers) + { + var basicDesc = oldReferrer.BasicDescriptor; + if (!referrerToIndex.ContainsKey(basicDesc)) updateRequired = true; + } + + if (!updateRequired) throw new NoReferrerUpdateException("no referrer update in this request"); + } + + RemoveEmptyDescriptors(updatedReferrers, referrerToIndex.Count); + return updatedReferrers; + } + + internal static void RemoveEmptyDescriptors(List updatedReferrers, int numNonEmptyReferrers) + { + var lastEmptyIndex = 0; + for (var i = 0; i < updatedReferrers.Count; ++i) + { + if (Descriptor.IsEmptyOrNull(updatedReferrers[i])) continue; + + if (i > lastEmptyIndex) updatedReferrers[lastEmptyIndex] = updatedReferrers[i]; + ++lastEmptyIndex; + if (lastEmptyIndex == numNonEmptyReferrers) break; + } + updatedReferrers.RemoveRange(lastEmptyIndex, updatedReferrers.Count - lastEmptyIndex); + } +} diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 62d73bc..33c7720 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -46,6 +46,7 @@ public class Repository : IRepository public IManifestStore Manifests => new ManifestStore(this); public RepositoryOptions Options => _opts; + internal Referrers.ReferrerState ReferrerState { get; set; } = Referrers.ReferrerState.ReferrerUnknown; internal static readonly string[] DefaultManifestMediaTypes = [ diff --git a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs index 0302ea6..1d7b45a 100644 --- a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs +++ b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs @@ -50,4 +50,11 @@ public struct RepositoryOptions /// Reference: https://docs.docker.com/registry/spec/api/#tags /// public int TagListPageSize { get; set; } + + // SkipReferrersGc specifies whether to delete the dangling referrers + // index when referrers tag schema is utilized. + // - If false, the old referrers index will be deleted after the new one is successfully uploaded. + // - If true, the old referrers index is kept. + // By default, it is disabled (set to false). See also: + public bool SkipReferrersGc { get; set; } } From 846e1739728a30d24abb5a37c4b089e5f4f59770 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 19 Nov 2024 17:45:31 +1100 Subject: [PATCH 02/24] add unit tests Signed-off-by: Patrick Pan --- .../Exceptions/NoReferrerUpdateException.cs | 19 +- .../Registry/Remote/ManifestStore.cs | 69 +- .../Registry/Remote/Referrers.cs | 39 +- .../Content/MemoryStoreTest.cs | 3 +- .../Exceptions/ExceptionTest.cs | 8 + tests/OrasProject.Oras.Tests/Oci/IndexTest.cs | 66 + .../Remote/ManifestStoreTest.cs | 359 +++ .../Remote/ReferrersTest.cs | 227 ++ .../Remote/RepositoryTest.cs | 2285 +---------------- .../Remote/Util/RandomDataGenerator.cs | 76 + .../Remote/Util/Util.cs | 41 + 11 files changed, 882 insertions(+), 2310 deletions(-) create mode 100644 tests/OrasProject.Oras.Tests/Oci/IndexTest.cs create mode 100644 tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs create mode 100644 tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs create mode 100644 tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs create mode 100644 tests/OrasProject.Oras.Tests/Remote/Util/Util.cs diff --git a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs index a7eaf79..87ab2b5 100644 --- a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs +++ b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs @@ -1,7 +1,24 @@ -using System; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; namespace OrasProject.Oras.Exceptions; + +/// +/// NoReferrerUpdateException is thrown when no referrer update is needed. +/// public class NoReferrerUpdateException : Exception { public NoReferrerUpdateException() diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 881a55e..5a6a090 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -162,6 +162,16 @@ public async Task PushAsync(Descriptor expected, Stream content, string referenc await PushWithIndexingAsync(expected, content, contentReference, cancellationToken).ConfigureAwait(false); } + /// + /// PushWithIndexingAsync pushes the given manifest to the repository with indexing support. + /// If referrer support is not enabled, the function will first push the content, then process and update + /// the referrers index before pushing the content again. It handles both image manifests and index manifests. + /// + /// + /// + /// + /// + /// private async Task PushWithIndexingAsync(Descriptor expected, Stream content, string reference, CancellationToken cancellationToken = default) { @@ -176,14 +186,19 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, st } var contentBytes = await content.ReadAllAsync(expected, cancellationToken); - // var initPosition = content.Position; - await InternalPushAsync(expected, new MemoryStream(contentBytes), reference, cancellationToken).ConfigureAwait(false); + using (var contentDuplicate = new MemoryStream(contentBytes)) + { + await InternalPushAsync(expected, contentDuplicate, reference, cancellationToken).ConfigureAwait(false); + } if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) { return; } - // content.Seek(initPosition, SeekOrigin.Begin); - await IndexReferrersToPush(expected, new MemoryStream(contentBytes)); + + using (var contentDuplicate = new MemoryStream(contentBytes)) + { + await ProcessReferrersAndPushIndex(expected, contentDuplicate); + } break; default: await InternalPushAsync(expected, content, reference, cancellationToken); @@ -191,7 +206,17 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, st } } - private async Task IndexReferrersToPush(Descriptor desc, Stream content, CancellationToken cancellationToken = default) + /// + /// ProcessReferrersAndPushIndex processes the referrers for the given descriptor by deserializing its content + /// (either as an image manifest or image index), extracting relevant metadata + /// such as the subject, artifact type, and annotations, and then updates the + /// referrers index if applicable. + /// + /// + /// + /// + /// + private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, CancellationToken cancellationToken = default) { Descriptor? subject = null; switch (desc.MediaType) @@ -206,6 +231,7 @@ private async Task IndexReferrersToPush(Descriptor desc, Stream content, Cancell case MediaType.ImageManifest: var imageManifest = JsonSerializer.Deserialize(content); if (imageManifest?.Subject == null) return; + subject = imageManifest.Subject; desc.ArtifactType = string.IsNullOrEmpty(imageManifest.ArtifactType) ? imageManifest.Config.MediaType : imageManifest.ArtifactType; desc.Annotations = imageManifest.Annotations; break; @@ -214,13 +240,18 @@ private async Task IndexReferrersToPush(Descriptor desc, Stream content, Cancell } Repository.ReferrerState = Referrers.ReferrerState.ReferrerNotSupported; - if (subject == null) - { - throw new InvalidOperationException("Subject was not initialized"); - } await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd)); } + /// + /// UpdateReferrersIndex updates the referrers index for a given subject by applying the specified referrer changes. + /// If the referrers index is updated, the new index is pushed to the repository. If referrers + /// garbage collection is not skipped, the old index is deleted. + /// + /// + /// + /// + /// private async Task UpdateReferrersIndex(Descriptor subject, Referrers.ReferrerChange referrerChange, CancellationToken cancellationToken = default) { @@ -229,19 +260,22 @@ private async Task UpdateReferrersIndex(Descriptor subject, var referrersTag = Referrers.BuildReferrersTag(subject); var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag); var updatedReferrers = - Referrers.ApplyReferrerChanges(oldReferrers, new List { referrerChange }); + Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGc) { var (indexDesc, indexContent) = Index.GenerateIndex(updatedReferrers); - await InternalPushAsync(indexDesc, new MemoryStream(indexContent), referrersTag, cancellationToken).ConfigureAwait(false); + using (var content = new MemoryStream(indexContent)) + { + await InternalPushAsync(indexDesc, content, referrersTag, cancellationToken).ConfigureAwait(false); + } } if (repository.Options.SkipReferrersGc || Descriptor.IsEmptyOrNull(oldDesc)) { return; } - // delete oldIndexDesc + await DeleteAsync(oldDesc, cancellationToken).ConfigureAwait(false); } catch (NoReferrerUpdateException) @@ -250,6 +284,15 @@ private async Task UpdateReferrersIndex(Descriptor subject, } } + /// + /// PullReferrersIndexList retrieves the referrers index list associated with the given referrers tag. + /// It fetches the index manifest from the repository, deserializes it into an `Index` object, + /// and returns the descriptor along with the list of manifests (referrers). If the referrers index is not found, + /// an empty descriptor and an empty list are returned. + /// + /// + /// + /// internal async Task<(Descriptor, IList)> PullReferrersIndexList(string referrersTag, CancellationToken cancellationToken = default) { try @@ -279,7 +322,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, private async Task InternalPushAsync(Descriptor expected, Stream stream, string contentReference, CancellationToken cancellationToken) { - var remoteReference = Repository.ParseReference(contentReference); // duplicate? + var remoteReference = Repository.ParseReference(contentReference); var url = new UriFactory(remoteReference, Repository.Options.PlainHttp).BuildRepositoryManifest(); var request = new HttpRequestMessage(HttpMethod.Put, url); request.Content = new StreamContent(stream); diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index ac25d8e..bbc1a7c 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -35,16 +35,24 @@ internal enum ReferrerOperation ReferrerDelete, } - public static string BuildReferrersTag(Descriptor descriptor) + internal static string BuildReferrersTag(Descriptor descriptor) { return Digest.GetAlgorithm(descriptor.Digest) + "-" + Digest.GetRef(descriptor.Digest); } - internal static IList ApplyReferrerChanges(IList oldReferrers, IList referrerChanges) + /// + /// ApplyReferrerChanges applies the specified referrer change (either add or delete) to the existing list of referrers. + /// It updates the list based on the operation defined in the provided `referrerChange`. + /// If the referrer to be added or deleted already exists in the list, it is handled accordingly. + /// + /// + /// + /// The updated referrers list + internal static IList ApplyReferrerChanges(IList oldReferrers, ReferrerChange referrerChange) { - if (oldReferrers == null || referrerChanges == null) + if (oldReferrers == null || referrerChange == null) { - throw new NoReferrerUpdateException("referrerChanges or oldReferrers is null in this request"); + throw new NoReferrerUpdateException("referrerChange or oldReferrers is null in this request"); } var updatedReferrers = new List(); var referrerToIndex = new Dictionary(); @@ -67,32 +75,34 @@ internal static IList ApplyReferrerChanges(IList oldRefe referrerToIndex[basicDesc] = updatedReferrers.Count - 1; } - foreach (var change in referrerChanges) + + if (!Descriptor.IsEmptyOrNull(referrerChange.Referrer)) { - if (Descriptor.IsEmptyOrNull(change.Referrer)) continue; - var basicDesc = change.Referrer.BasicDescriptor; - switch (change.ReferrerOperation) + var basicDesc = referrerChange.Referrer.BasicDescriptor; + switch (referrerChange.ReferrerOperation) { case ReferrerOperation.ReferrerAdd: if (!referrerToIndex.ContainsKey(basicDesc)) { - updatedReferrers.Add(change.Referrer); + updatedReferrers.Add(referrerChange.Referrer); referrerToIndex[basicDesc] = updatedReferrers.Count - 1; } + break; - + case ReferrerOperation.ReferrerDelete: if (referrerToIndex.TryGetValue(basicDesc, out var index)) { updatedReferrers[index] = Descriptor.EmptyDescriptor(); referrerToIndex.Remove(basicDesc); } + break; default: break; } } - + if (!updateRequired && referrerToIndex.Count == oldReferrers.Count) { foreach (var oldReferrer in oldReferrers) @@ -108,6 +118,13 @@ internal static IList ApplyReferrerChanges(IList oldRefe return updatedReferrers; } + /// + /// RemoveEmptyDescriptors removes any empty or null descriptors from the provided list of referrers, ensuring that only non-empty + /// descriptors remain in the list. It optimizes the list by shifting valid descriptors forward and trimming + /// the remaining elements at the end. The list is truncated to only contain non-empty descriptors up to the specified count. + /// + /// + /// internal static void RemoveEmptyDescriptors(List updatedReferrers, int numNonEmptyReferrers) { var lastEmptyIndex = 0; diff --git a/tests/OrasProject.Oras.Tests/Content/MemoryStoreTest.cs b/tests/OrasProject.Oras.Tests/Content/MemoryStoreTest.cs index 42c7676..d7a28ea 100644 --- a/tests/OrasProject.Oras.Tests/Content/MemoryStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Content/MemoryStoreTest.cs @@ -14,6 +14,7 @@ using OrasProject.Oras.Content; using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; +using Index = OrasProject.Oras.Oci.Index; using System.Text; using System.Text.Json; using Xunit; @@ -195,7 +196,7 @@ public async Task ShouldReturnPredecessorsOfNodes() var generateIndex = (List manifests) => { - var index = new Oci.Index + var index = new Index { Manifests = manifests }; diff --git a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs index 0a426e7..84ee0e6 100644 --- a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs +++ b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs @@ -49,4 +49,12 @@ public async Task NotFoundException() await Assert.ThrowsAsync(() => throw new NotFoundException("Not found")); await Assert.ThrowsAsync(() => throw new NotFoundException("Not found", null)); } + + [Fact] + public async Task NoReferrerUpdateException() + { + await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException()); + await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update")); + await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update", null)); + } } diff --git a/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs b/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs new file mode 100644 index 0000000..dacf8b2 --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs @@ -0,0 +1,66 @@ +using System.Text.Json; +using OrasProject.Oras.Content; +using OrasProject.Oras.Oci; +using static OrasProject.Oras.Tests.Remote.Util.Util; +using Xunit; +using Index = OrasProject.Oras.Oci.Index; + +namespace OrasProject.Oras.Tests.Oci; + +public class IndexTest +{ + [Fact] + public void GenerateIndex_CorrectlyGeneratesIndexDescriptor() + { + var expectedManifests = new List + { + new Descriptor + { + Digest = "digest1", + MediaType = MediaType.ImageManifest, + Size = 100 + }, + new Descriptor + { + Digest = "digest2", + MediaType = MediaType.ImageManifest, + Size = 200 + } + }; + + var (generatedIndexDesc, generatedIndexContent) = Index.GenerateIndex(expectedManifests); + Assert.NotNull(generatedIndexDesc); + Assert.Equal(MediaType.ImageIndex, generatedIndexDesc.MediaType); + Assert.Equal(generatedIndexContent.Length, generatedIndexDesc.Size); + Assert.Equal(Digest.ComputeSHA256(generatedIndexContent), generatedIndexDesc.Digest); + + var generatedIndex = JsonSerializer.Deserialize(generatedIndexContent); + Assert.NotNull(generatedIndex); + Assert.Equal(2, generatedIndex.Manifests.Count); + for (var i = 0; i < generatedIndex.Manifests.Count; ++i) + { + Assert.True(AreDescriptorsEqual(generatedIndex.Manifests[i], expectedManifests[i])); + } + Assert.Equal(MediaType.ImageIndex, generatedIndex.MediaType); + Assert.Equal(2, generatedIndex.SchemaVersion); + } + + [Fact] + public void GenerateIndex_CorrectlyGeneratesIndexDescriptorWithEmptyManifests() + { + var expectedManifests = new List(); + var (generatedIndexDesc, generatedIndexContent) = Index.GenerateIndex(expectedManifests); + + Assert.NotNull(generatedIndexDesc); + Assert.Equal(MediaType.ImageIndex, generatedIndexDesc.MediaType); + Assert.Equal(generatedIndexContent.Length, generatedIndexDesc.Size); + Assert.Equal(Digest.ComputeSHA256(generatedIndexContent), generatedIndexDesc.Digest); + + var generatedIndex = JsonSerializer.Deserialize(generatedIndexContent); + Assert.NotNull(generatedIndex); + Assert.Empty(generatedIndex.Manifests); + Assert.Equal(expectedManifests, generatedIndex.Manifests); + Assert.Equal(MediaType.ImageIndex, generatedIndex.MediaType); + Assert.Equal(2, generatedIndex.SchemaVersion); + } +} diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs new file mode 100644 index 0000000..83fd80f --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -0,0 +1,359 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using OrasProject.Oras.Oci; +using OrasProject.Oras.Registry; +using OrasProject.Oras.Registry.Remote; +using static OrasProject.Oras.Tests.Remote.Util.RandomDataGenerator; +using static OrasProject.Oras.Tests.Remote.Util.Util; +using static OrasProject.Oras.Content.Digest; +using Index = OrasProject.Oras.Oci.Index; + + +using Xunit; +using Xunit.Abstractions; + +namespace OrasProject.Oras.Tests.Remote; + +public class ManifestStoreTest +{ + private const string _dockerContentDigestHeader = "Docker-Content-Digest"; + + private ITestOutputHelper _output; + + public ManifestStoreTest(ITestOutputHelper output) + { + _output = output; + } + + /// + /// ManifestStore_PushAsyncWithSubjectAndReferrerSupported tests PushAsync method for pushing manifest with subject when registry supports referrers API + /// + [Fact] + public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() + { + var (_, manifestBytes) = RandomManifestWithSubject(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length + }; + byte[]? receivedManifest = null; + + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); + receivedManifest = buf; + } + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + res.Headers.Add("OCI-Subject", "test"); + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(manifestDesc, new MemoryStream(manifestBytes), cancellationToken); + Assert.Equal(manifestBytes, receivedManifest); + Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); + } + + [Fact] + public async Task ManifestStore_PullReferrersIndexListSuccessfully() + { + var expectedIndex = RandomIndex(); + var expectedIndexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(expectedIndex)); + var expectedIndexDesc = new Descriptor() + { + Digest = ComputeSHA256(expectedIndexBytes), + MediaType = MediaType.ImageIndex, + Size = expectedIndexBytes.Length + }; + + var mockedHttpHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedIndexDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(expectedIndexBytes); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedIndexDesc.Digest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockedHttpHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var (receivedDesc, receivedManifests) = await store.PullReferrersIndexList(expectedIndexDesc.Digest, cancellationToken); + Assert.True(AreDescriptorsEqual(expectedIndexDesc, receivedDesc)); + for (var i = 0; i < receivedManifests.Count; ++i) + { + Assert.True(AreDescriptorsEqual(expectedIndex.Manifests[i], receivedManifests[i])); + } + } + + [Fact] + public async Task ManifestStore_PullReferrersIndexListNotFound() + { + var mockedHttpHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockedHttpHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var (receivedDesc, receivedManifests) = await store.PullReferrersIndexList("test", cancellationToken); + Assert.True(Descriptor.IsEmptyOrNull(receivedDesc)); + Assert.Empty(receivedManifests); + } + + + [Fact] + public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() + { + var oldIndex = RandomIndex(); + var oldIndexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(oldIndex)); + var oldIndexDesc = new Descriptor() + { + Digest = ComputeSHA256(oldIndexBytes), + MediaType = MediaType.ImageIndex, + Size = oldIndexBytes.Length + }; + + // first push + var (firstExpectedManifest, firstExpectedManifestBytes) = RandomManifestWithSubject(); + var firstExpectedManifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(firstExpectedManifestBytes), + Size = firstExpectedManifestBytes.Length, + ArtifactType = MediaType.ImageConfig, + }; + var firstExpectedReferrersList = new List(oldIndex.Manifests); + firstExpectedReferrersList.Add(firstExpectedManifestDesc); + var (firstExpectedIndexReferrersDesc, firstExpectedIndexReferrersBytes) = Index.GenerateIndex(firstExpectedReferrersList); + + // second push + var (_, secondExpectedManifestBytes) = RandomManifestWithSubject(firstExpectedManifest.Subject); + var secondExpectedManifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(secondExpectedManifestBytes), + Size = secondExpectedManifestBytes.Length, + ArtifactType = MediaType.ImageConfig, + }; + var secondExpectedReferrersList = new List(oldIndex.Manifests); + secondExpectedReferrersList.Add(secondExpectedManifestDesc); + var (secondExpectedIndexReferrersDesc, secondExpectedIndexReferrersBytes) = Index.GenerateIndex(secondExpectedReferrersList); + + byte[]? receivedManifestContent = null; + byte[]? receivedIndexContent = null; + var referrersTag = Referrers.BuildReferrersTag(firstExpectedManifest.Subject); + var oldIndexDeleted = false; + var firstIndexDeleted = false; + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var response = new HttpResponseMessage(); + response.RequestMessage = req; + + if (req.Method == HttpMethod.Put && ( + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{firstExpectedManifestDesc.Digest}" || + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{secondExpectedManifestDesc.Digest}" || + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}")) + { + if (req.Content?.Headers?.ContentLength != null) + { + var buffer = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buffer, 0); + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{firstExpectedManifestDesc.Digest}" || + req.RequestUri.AbsolutePath == $"/v2/test/manifests/{secondExpectedManifestDesc.Digest}") receivedManifestContent = buffer; + else receivedIndexContent = buffer; + } + + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{firstExpectedManifestDesc.Digest}") + response.Headers.Add(_dockerContentDigestHeader, new[] { firstExpectedManifestDesc.Digest }); + else if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{secondExpectedManifestDesc.Digest}") + response.Headers.Add(_dockerContentDigestHeader, new[] { secondExpectedManifestDesc.Digest }); + else if (!oldIndexDeleted) response.Headers.Add(_dockerContentDigestHeader, new[] { firstExpectedIndexReferrersDesc.Digest }); + else response.Headers.Add(_dockerContentDigestHeader, new[] { secondExpectedIndexReferrersDesc.Digest }); + + response.StatusCode = HttpStatusCode.Created; + return response; + } else if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(oldIndexBytes); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + if (oldIndexDeleted) response.Headers.Add(_dockerContentDigestHeader, new string[] { firstExpectedIndexReferrersDesc.Digest }); + else response.Headers.Add(_dockerContentDigestHeader, new string[] { oldIndexDesc.Digest }); + response.StatusCode = HttpStatusCode.OK; + return response; + } else if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{oldIndexDesc.Digest}") + { + response.Headers.Add(_dockerContentDigestHeader, new[] { oldIndexDesc.Digest }); + response.StatusCode = HttpStatusCode.Accepted; + oldIndexDeleted = true; + return response; + } else if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{firstExpectedIndexReferrersDesc.Digest}") + { + response.Headers.Add(_dockerContentDigestHeader, new[] { firstExpectedIndexReferrersDesc.Digest }); + response.StatusCode = HttpStatusCode.Accepted; + firstIndexDeleted = true; + return response; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + // First push with referrer tag schema + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(firstExpectedManifestDesc, new MemoryStream(firstExpectedManifestBytes), cancellationToken); + Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(firstExpectedManifestBytes, receivedManifestContent); + Assert.True(oldIndexDeleted); + Assert.Equal(firstExpectedIndexReferrersBytes, receivedIndexContent); + + + // Second push with referrer tag schema + Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + await store.PushAsync(secondExpectedManifestDesc, new MemoryStream(secondExpectedManifestBytes), cancellationToken); + Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(secondExpectedManifestBytes, receivedManifestContent); + Assert.True(firstIndexDeleted); + Assert.Equal(secondExpectedIndexReferrersBytes, receivedIndexContent); + } + + [Fact] + public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupportedWithoutOldIndex() + { + var expectedIndexManifest = new Index() + { + Subject = RandomDescriptor(), + Manifests = new List{ RandomDescriptor(), RandomDescriptor() }, + MediaType = MediaType.ImageIndex, + ArtifactType = MediaType.ImageIndex, + }; + + var expectedIndexManifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(expectedIndexManifest)); + var expectedIndexManifestDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(expectedIndexManifestBytes), + Size = expectedIndexManifestBytes.Length, + ArtifactType = MediaType.ImageIndex, + }; + var expectedReferrers = new List + { + expectedIndexManifestDesc, + }; + + var (expectedIndexReferrersDesc, expectedIndexReferrersBytes) = Index.GenerateIndex(expectedReferrers); + + byte[]? receivedIndexManifestContent = null; + byte[]? receivedIndexReferrersContent = null; + var referrersTag = Referrers.BuildReferrersTag(expectedIndexManifest.Subject); + + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var response = new HttpResponseMessage(); + response.RequestMessage = req; + + if (req.Method == HttpMethod.Put && ( + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedIndexManifestDesc.Digest}" || + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}")) + { + if (req.Content?.Headers?.ContentLength != null) + { + var buffer = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buffer, 0); + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedIndexManifestDesc.Digest}") receivedIndexManifestContent = buffer; + else receivedIndexReferrersContent = buffer; + } + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedIndexManifestDesc.Digest}") + { + response.Headers.Add(_dockerContentDigestHeader, new[] { expectedIndexManifestDesc.Digest }); + } else response.Headers.Add(_dockerContentDigestHeader, new[] { expectedIndexReferrersDesc.Digest }); + response.StatusCode = HttpStatusCode.Created; + return response; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); + Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(expectedIndexManifestBytes, receivedIndexManifestContent); + Assert.Equal(expectedIndexReferrersBytes, receivedIndexReferrersContent); + } +} diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs new file mode 100644 index 0000000..a3615ff --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -0,0 +1,227 @@ +using OrasProject.Oras.Exceptions; +using OrasProject.Oras.Oci; +using OrasProject.Oras.Registry.Remote; +using static OrasProject.Oras.Tests.Remote.Util.Util; +using static OrasProject.Oras.Tests.Remote.Util.RandomDataGenerator; +using Xunit; + +namespace OrasProject.Oras.Tests.Remote; + +public class ReferrersTest +{ + [Fact] + public void ApplyReferrerChanges_ShouldAddNewReferrers() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var newDescriptor = RandomDescriptor(); + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + }; + + var expectedReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + newDescriptor, + }; + var referrerChange = new Referrers.ReferrerChange( + newDescriptor, + Referrers.ReferrerOperation.ReferrerAdd + ); + + var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(3, updatedReferrers.Count); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + } + + [Fact] + public void ApplyReferrerChanges_ShouldDiscardDuplicateReferrers() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var newDescriptor1 = RandomDescriptor(); + + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + oldDescriptor2, + oldDescriptor1, + }; + + var expectedReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + newDescriptor1, + }; + var referrerChange = new Referrers.ReferrerChange( + newDescriptor1, + Referrers.ReferrerOperation.ReferrerAdd + ); + + var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(3, updatedReferrers.Count); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + } + + [Fact] + public void ApplyReferrerChanges_ShouldNotAddNewDuplicateReferrers() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + }; + var referrerChange = new Referrers.ReferrerChange( + oldDescriptor1, + Referrers.ReferrerOperation.ReferrerAdd + ); + + + var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); + Assert.Equal("no referrer update in this request", exception.Message); + } + + [Fact] + public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() + { + var emptyDesc1 = Descriptor.EmptyDescriptor(); + Descriptor? emptyDesc2 = null; + var newDescriptor = RandomDescriptor(); + + var oldReferrers = new List + { + emptyDesc1, + emptyDesc2, + }; + var expectedReferrers = new List + { + newDescriptor, + }; + var referrerChange = new Referrers.ReferrerChange( + newDescriptor, + Referrers.ReferrerOperation.ReferrerAdd + ); + + var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + + Assert.Single(updatedReferrers); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + } + + [Fact] + public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreNull() + { + IList oldReferrers = null; + Referrers.ReferrerChange referrerChange = null; + + var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); + Assert.Equal("referrerChange or oldReferrers is null in this request", exception.Message); + } + + [Fact] + public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreEmpty() + { + var oldReferrers = new List(); + var referrerChange = new Referrers.ReferrerChange(Descriptor.EmptyDescriptor(), Referrers.ReferrerOperation.ReferrerAdd); + + var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); + Assert.Equal("no referrer update in this request", exception.Message); + } + + [Fact] + public void RemoveEmptyDescriptors_ShouldRemoveEmptyDescriptors() + { + var randomDescriptor1 = RandomDescriptor(); + var randomDescriptor2 = RandomDescriptor(); + var randomDescriptor3 = RandomDescriptor(); + var randomDescriptor4 = RandomDescriptor(); + var descriptors = new List + { + Descriptor.EmptyDescriptor(), + randomDescriptor1, + Descriptor.EmptyDescriptor(), + randomDescriptor2, + Descriptor.EmptyDescriptor(), + Descriptor.EmptyDescriptor(), + randomDescriptor3, + randomDescriptor4, + }; + + var expectedDescriptors = new List + { + randomDescriptor1, + randomDescriptor2, + randomDescriptor3, + randomDescriptor4 + }; + Referrers.RemoveEmptyDescriptors(descriptors, 4); + + Assert.Equal(4, descriptors.Count); + Assert.DoesNotContain(Descriptor.EmptyDescriptor(), descriptors); + for (var i = 0; i < descriptors.Count; ++i) + { + Assert.True(AreDescriptorsEqual(descriptors[i], expectedDescriptors[i])); + } + } + + [Fact] + public void RemoveEmptyDescriptors_ShouldReturnAllNonEmptyDescriptors() + { + var randomDescriptor1 = RandomDescriptor(); + var randomDescriptor2 = RandomDescriptor(); + var randomDescriptor3 = RandomDescriptor(); + var randomDescriptor4 = RandomDescriptor(); + var descriptors = new List + { + randomDescriptor1, + randomDescriptor2, + randomDescriptor3, + randomDescriptor4, + }; + + var expectedDescriptors = new List + { + randomDescriptor1, + randomDescriptor2, + randomDescriptor3, + randomDescriptor4 + }; + Referrers.RemoveEmptyDescriptors(descriptors, 4); + Assert.Equal(4, descriptors.Count); + for (var i = 0; i < descriptors.Count; ++i) + { + Assert.True(AreDescriptorsEqual(descriptors[i], expectedDescriptors[i])); + } + } + + [Fact] + public void RemoveEmptyDescriptors_ShouldRemoveAllEmptyDescriptors() + { + var descriptors = new List + { + Descriptor.EmptyDescriptor(), + Descriptor.EmptyDescriptor(), + Descriptor.EmptyDescriptor(), + Descriptor.EmptyDescriptor(), + }; + + Referrers.RemoveEmptyDescriptors(descriptors, 0); + Assert.Empty(descriptors); + } +} diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 84df80a..acecad7 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -1,2284 +1 @@ -// Copyright The ORAS Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using Moq; -using Moq.Protected; -using OrasProject.Oras.Content; -using OrasProject.Oras.Exceptions; -using OrasProject.Oras.Oci; -using OrasProject.Oras.Registry; -using OrasProject.Oras.Registry.Remote; -using System.Collections.Immutable; -using System.Diagnostics; -using System.Net; -using System.Net.Http.Headers; -using System.Text; -using System.Text.Json; -using System.Text.RegularExpressions; -using System.Web; -using Xunit; -using static OrasProject.Oras.Content.Digest; - -namespace OrasProject.Oras.Tests.Remote; - -public class RepositoryTest -{ - public struct TestIOStruct - { - public bool IsTag; - public bool ErrExpectedOnHEAD; - public string ServerCalculatedDigest; - public string ClientSuppliedReference; - public bool ErrExpectedOnGET; - } - - private byte[] _theAmazingBanClan = "Ban Gu, Ban Chao, Ban Zhao"u8.ToArray(); - private const string _theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; - - private const string _dockerContentDigestHeader = "Docker-Content-Digest"; - - // The following truth table aims to cover the expected GET/HEAD request outcome - // for all possible permutations of the client/server "containing a digest", for - // both Manifests and Blobs. Where the results between the two differ, the index - // of the first column has an exclamation mark. - // - // The client is said to "contain a digest" if the user-supplied reference string - // is of the form that contains a digest rather than a tag. The server, on the - // other hand, is said to "contain a digest" if the server responded with the - // special header `Docker-Content-Digest`. - // - // In this table, anything denoted with an asterisk indicates that the true - // response should actually be the opposite of what's expected; for example, - // `*PASS` means we will get a `PASS`, even though the true answer would be its - // diametric opposite--a `FAIL`. This may seem odd, and deserves an explanation. - // This function has blind-spots, and while it can expend power to gain sight, - // i.e., perform the expensive validation, we chose not to. The reason is two- - // fold: a) we "know" that even if we say "!PASS", it will eventually fail later - // when checks are performed, and with that assumption, we have the luxury for - // the second point, which is b) performance. - // - // _______________________________________________________________________________________________________________ - // | ID | CLIENT | SERVER | Manifest.GET | Blob.GET | Manifest.HEAD | Blob.HEAD | - // |----+-----------------+------------------+-----------------------+-----------+---------------------+-----------+ - // | 1 | tag | missing | CALCULATE,PASS | n/a | FAIL | n/a | - // | 2 | tag | presentCorrect | TRUST,PASS | n/a | TRUST,PASS | n/a | - // | 3 | tag | presentIncorrect | TRUST,*PASS | n/a | TRUST,*PASS | n/a | - // | 4 | correctDigest | missing | TRUST,PASS | PASS | TRUST,PASS | PASS | - // | 5 | correctDigest | presentCorrect | TRUST,COMPARE,PASS | PASS | TRUST,COMPARE,PASS | PASS | - // | 6 | correctDigest | presentIncorrect | TRUST,COMPARE,FAIL | FAIL | TRUST,COMPARE,FAIL | FAIL | - // --------------------------------------------------------------------------------------------------------------- - - /// - /// GetTestIOStructMapForGetDescriptorClass returns a map of test cases for different - /// GET/HEAD request outcome for all possible permutations of the client/server "containing a digest", for - /// both Manifests and Blobs. - /// - /// - public static Dictionary GetTestIOStructMapForGetDescriptorClass() - { - string correctDigest = $"sha256:{_theAmazingBanDigest}"; - string incorrectDigest = $"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; - - return new Dictionary - { - ["1. Client:Tag & Server:DigestMissing"] = new TestIOStruct - { - IsTag = true, - ErrExpectedOnHEAD = true - }, - ["2. Client:Tag & Server:DigestValid"] = new TestIOStruct - { - IsTag = true, - ServerCalculatedDigest = correctDigest - }, - ["3. Client:Tag & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct - { - IsTag = true, - ServerCalculatedDigest = incorrectDigest - }, - ["4. Client:DigestValid & Server:DigestMissing"] = new TestIOStruct - { - ClientSuppliedReference = correctDigest - }, - ["5. Client:DigestValid & Server:DigestValid"] = new TestIOStruct - { - ClientSuppliedReference = correctDigest, - ServerCalculatedDigest = correctDigest - }, - ["6. Client:DigestValid & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct - { - ClientSuppliedReference = correctDigest, - ServerCalculatedDigest = incorrectDigest, - ErrExpectedOnHEAD = true, - ErrExpectedOnGET = true - } - }; - } - - /// - /// AreDescriptorsEqual compares two descriptors and returns true if they are equal. - /// - /// - /// - /// - public bool AreDescriptorsEqual(Descriptor a, Descriptor b) - { - return a.MediaType == b.MediaType && a.Digest == b.Digest && a.Size == b.Size; - } - - public static HttpClient CustomClient(Func func) - { - var moqHandler = new Mock(); - moqHandler.Protected().Setup>( - "SendAsync", - ItExpr.IsAny(), - ItExpr.IsAny() - ).ReturnsAsync(func); - return new HttpClient(moqHandler.Object); - } - - private HttpClient CustomClient(Func> func) - { - var moqHandler = new Mock(); - moqHandler.Protected().Setup>( - "SendAsync", - ItExpr.IsAny(), - ItExpr.IsAny() - ).Returns(func); - return new HttpClient(moqHandler.Object); - } - - /// - /// Repository_FetchAsync tests the FetchAsync method of the Repository. - /// - /// - [Fact] - public async Task Repository_FetchAsync() - { - var blob = Encoding.UTF8.GetBytes("hello world"); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = """{"manifests":[]}"""u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var resp = new HttpResponseMessage(); - resp.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - Debug.WriteLine("Expected GET request"); - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - } - - var path = req.RequestUri!.AbsolutePath; - if (path == "/v2/test/blobs/" + blobDesc.Digest) - { - resp.Content = new ByteArrayContent(blob); - resp.Content.Headers.Add("Content-Type", "application/octet-stream"); - resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return resp; - } - - if (path == "/v2/test/manifests/" + indexDesc.Digest) - { - if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(MediaType.ImageIndex))) - { - resp.StatusCode = HttpStatusCode.BadRequest; - Debug.WriteLine("manifest not convertable: " + req.Headers.Accept); - return resp; - } - - resp.Content = new ByteArrayContent(index); - resp.Content.Headers.Add("Content-Type", indexDesc.MediaType); - resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - return resp; - } - - resp.StatusCode = HttpStatusCode.NotFound; - return resp; - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var stream = await repo.FetchAsync(blobDesc, cancellationToken); - var buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - stream = await repo.FetchAsync(indexDesc, cancellationToken); - buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(index, buf); - - } - - /// - /// Repository_PushAsync tests the PushAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_PushAsync() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var uuid = Guid.NewGuid().ToString(); - var gotBlob = new byte[blobDesc.Size]; - var gotIndex = new byte[indexDesc.Size]; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var resp = new HttpResponseMessage(); - resp.RequestMessage = req; - if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") - { - resp.Headers.Location = new Uri("http://localhost:5000/v2/test/blobs/uploads/" + uuid); - resp.StatusCode = HttpStatusCode.Accepted; - return resp; - } - - if (req.Method == HttpMethod.Put && - req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) - { - if (req.Headers.TryGetValues("Content-Type", out var values) && - !values.Contains("application/octet-stream")) - { - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - - } - - var queries = HttpUtility.ParseQueryString(req.RequestUri.Query); - if (queries["digest"] != blobDesc.Digest) - { - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - } - - var stream = req.Content!.ReadAsStream(cancellationToken); - stream.Read(gotBlob); - resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - resp.StatusCode = HttpStatusCode.Created; - return resp; - - } - - if (req.Method == HttpMethod.Put && - req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) - { - if (req.Headers.TryGetValues("Content-Type", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - } - - var stream = req.Content!.ReadAsStream(cancellationToken); - stream.Read(gotIndex); - resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - resp.StatusCode = HttpStatusCode.Created; - return resp; - } - - resp.StatusCode = HttpStatusCode.Forbidden; - return resp; - - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - await repo.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); - Assert.Equal(blob, gotBlob); - await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); - Assert.Equal(index, gotIndex); - } - - /// - /// Repository_ExistsAsync tests the ExistsAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_ExistsAsync() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) - { - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) - { - if (req.Headers.TryGetValues("Accept", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.NotAcceptable); - } - - res.Content.Headers.Add("Content-Type", indexDesc.MediaType); - res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var exists = await repo.ExistsAsync(blobDesc, cancellationToken); - Assert.True(exists); - exists = await repo.ExistsAsync(indexDesc, cancellationToken); - Assert.True(exists); - } - - /// - /// Repository_DeleteAsync tests the DeleteAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_DeleteAsync() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var blobDeleted = false; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var indexDeleted = false; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Delete) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) - { - blobDeleted = true; - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.StatusCode = HttpStatusCode.Accepted; - return res; - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) - { - indexDeleted = true; - // no dockerContentDigestHeader header for manifest deletion - res.StatusCode = HttpStatusCode.Accepted; - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - await repo.DeleteAsync(blobDesc, cancellationToken); - Assert.True(blobDeleted); - await repo.DeleteAsync(indexDesc, cancellationToken); - Assert.True(indexDeleted); - } - - /// - /// Repository_ResolveAsync tests the ResolveAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_ResolveAsync() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var reference = "foobar"; - - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) - { - return new HttpResponseMessage(HttpStatusCode.NotFound); - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest - || req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + reference) - - { - if (req.Headers.TryGetValues("Accept", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - res.Content.Headers.Add("Content-Type", indexDesc.MediaType); - res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - await Assert.ThrowsAsync(async () => - await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); - // await repo.ResolveAsync(blobDesc.Digest, cancellationToken); - var got = await repo.ResolveAsync(indexDesc.Digest, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, got)); - - got = await repo.ResolveAsync(reference, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, got)); - var tagDigestRef = "whatever" + "@" + indexDesc.Digest; - got = await repo.ResolveAsync(tagDigestRef, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, got)); - var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; - got = await repo.ResolveAsync(fqdnRef, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, got)); - } - - /// - /// Repository_ResolveAsync tests the ResolveAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_TagAsync() - { - var blob = "hello"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - byte[]? gotIndex = null; - var reference = "foobar"; - - var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Get && - req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) - { - return new HttpResponseMessage(HttpStatusCode.Found); - } - - if (req.Method == HttpMethod.Get && - req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) - { - if (req.Headers.TryGetValues("Accept", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - res.Content = new ByteArrayContent(index); - res.Content.Headers.Add("Content-Type", indexDesc.MediaType); - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - return res; - } - - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference - || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) - - { - if (req.Headers.TryGetValues("Content-Type", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - if (req.Content != null) - { - gotIndex = await req.Content.ReadAsByteArrayAsync(cancellationToken); - } - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - res.StatusCode = HttpStatusCode.Created; - return res; - } - - return new HttpResponseMessage(HttpStatusCode.Forbidden); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - await Assert.ThrowsAnyAsync( - async () => await repo.TagAsync(blobDesc, reference, cancellationToken)); - await repo.TagAsync(indexDesc, reference, cancellationToken); - Assert.Equal(index, gotIndex); - await repo.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); - Assert.Equal(index, gotIndex); - } - - /// - /// Repository_PushReferenceAsync tests the PushReferenceAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_PushReferenceAsync() - { - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - byte[]? gotIndex = null; - var reference = "foobar"; - - var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) - { - if (req.Headers.TryGetValues("Content-Type", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - if (req.Content != null) - { - gotIndex = await req.Content.ReadAsByteArrayAsync(); - } - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - res.StatusCode = HttpStatusCode.Created; - return res; - } - - return new HttpResponseMessage(HttpStatusCode.Forbidden); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var streamContent = new MemoryStream(index); - await repo.PushAsync(indexDesc, streamContent, reference, cancellationToken); - Assert.Equal(index, gotIndex); - } - - /// - /// Repository_FetchReferenceAsync tests the FetchReferenceAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_FetchReferenceAsyc() - { - var blob = "hello"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var reference = "foobar"; - - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) - { - return new HttpResponseMessage(HttpStatusCode.NotFound); - } - - if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest - || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) - { - if (req.Headers.TryGetValues("Accept", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - res.Content = new ByteArrayContent(index); - res.Content.Headers.Add("Content-Type", indexDesc.MediaType); - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.Found); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - - // test with blob digest - await Assert.ThrowsAsync( - async () => await repo.FetchAsync(blobDesc.Digest, cancellationToken)); - - // test with manifest digest - var data = await repo.FetchAsync(indexDesc.Digest, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); - var buf = new byte[data.Stream.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(index, buf); - - // test with manifest tag - data = await repo.FetchAsync(reference, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); - buf = new byte[data.Stream.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(index, buf); - - // test with manifest tag@digest - var tagDigestRef = "whatever" + "@" + indexDesc.Digest; - data = await repo.FetchAsync(tagDigestRef, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); - buf = new byte[data.Stream.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(index, buf); - - // test with manifest FQDN - var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; - data = await repo.FetchAsync(fqdnRef, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); - - buf = new byte[data.Stream.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(index, buf); - } - - /// - /// Repository_TagsAsync tests the TagsAsync method of the Repository - /// to check if the tags are returned correctly - /// - /// - /// - [Fact] - public async Task Repository_TagsAsync() - { - var tagSet = new List>() - { - new() {"the", "quick", "brown", "fox"}, - new() {"jumps", "over", "the", "lazy"}, - new() {"dog"} - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get || - req.RequestUri?.AbsolutePath != "/v2/test/tags/list" - ) - { - return new HttpResponseMessage(HttpStatusCode.NotFound); - } - - var q = req.RequestUri.Query; - try - { - var n = int.Parse(Regex.Match(q, @"(?<=n=)\d+").Value); - if (n != 4) throw new Exception(); - } - catch - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - var tags = new List(); - var serverUrl = "http://localhost:5000"; - var matched = Regex.Match(q, @"(?<=test=)\w+").Value; - switch (matched) - { - case "foo": - tags = tagSet[1]; - res.Headers.Add("Link", $"<{serverUrl}/v2/test/tags/list?n=4&test=bar>; rel=\"next\""); - break; - case "bar": - tags = tagSet[2]; - break; - default: - tags = tagSet[0]; - res.Headers.Add("Link", $"; rel=\"next\""); - break; - } - - var listOfTags = new Repository.TagList - { - Tags = tags.ToArray() - }; - res.Content = new StringContent(JsonSerializer.Serialize(listOfTags)); - return res; - - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - TagListPageSize = 4, - }); - - var cancellationToken = new CancellationToken(); - - var wantTags = new List(); - foreach (var set in tagSet) - { - wantTags.AddRange(set); - } - var gotTags = new List(); - await foreach (var tag in repo.ListTagsAsync().WithCancellation(cancellationToken)) - { - gotTags.Add(tag); - } - Assert.Equal(wantTags, gotTags); - } - - /// - /// BlobStore_FetchAsync tests the FetchAsync method of the BlobStore - /// - /// - [Fact] - public async Task BlobStore_FetchAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - res.Content = new ByteArrayContent(blob); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - var stream = await store.FetchAsync(blobDesc, cancellationToken); - var buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - - } - - /// - /// BlobStore_FetchAsync_CanSeek tests the FetchAsync method of the BlobStore for a stream that can seek - /// - /// - [Fact] - public async Task BlobStore_FetchAsync_CanSeek() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var seekable = false; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - if (seekable) - { - res.Headers.AcceptRanges.Add("bytes"); - } - - if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) - { - } - - - if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") - { - res.StatusCode = HttpStatusCode.OK; - res.Content = new ByteArrayContent(blob); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - - long start = -1, end = -1; - var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); - if (hv != null && hv.From.HasValue && hv.To.HasValue) - { - start = hv.From.Value; - end = hv.To.Value; - } - - if (start < 0 || start > end || start >= blobDesc.Size) - { - return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); - } - - end++; - if (end > blobDesc.Size) - { - end = blobDesc.Size; - } - - res.StatusCode = HttpStatusCode.PartialContent; - res.Content = new ByteArrayContent(blob[(int)start..(int)end]); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.StatusCode = HttpStatusCode.NotFound; - return res; - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - var stream = await store.FetchAsync(blobDesc, cancellationToken); - var buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - - seekable = true; - stream = await store.FetchAsync(blobDesc, cancellationToken); - buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - - buf = new byte[stream.Length - 3]; - stream.Seek(3, SeekOrigin.Begin); - await stream.ReadAsync(buf, cancellationToken); - var seg = blob[3..]; - Assert.Equal(seg, buf); - } - - /// - /// BlobStore_FetchAsync_ZeroSizedBlob tests the FetchAsync method of the BlobStore for a zero sized blob - /// - /// - [Fact] - public async Task BlobStore_FetchAsync_ZeroSizedBlob() - { - var blob = ""u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - if (req.Headers.TryGetValues("Range", out var rangeHeader)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - var stream = await store.FetchAsync(blobDesc, cancellationToken); - var buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - } - - /// - /// BlobStore_PushAsync tests the PushAsync method of the BlobStore. - /// - /// - [Fact] - public async Task BlobStore_PushAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var gotBlob = new byte[blob.Length]; - var uuid = Guid.NewGuid().ToString(); - var existingQueryParameter = "existingParam=value"; - - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Post && req.RequestUri?.AbsolutePath == $"/v2/test/blobs/uploads/") - { - res.StatusCode = HttpStatusCode.Accepted; - res.Headers.Add("Location", $"/v2/test/blobs/uploads/{uuid}?{existingQueryParameter}"); - return res; - } - - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) - { - // Assert that the existing query parameter is present - var queryParameters = HttpUtility.ParseQueryString(req.RequestUri.Query); - Assert.Equal("value", queryParameters["existingParam"]); - - if (req.Headers.TryGetValues("Content-Type", out var contentType) && - contentType.FirstOrDefault() != "application/octet-stream") - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - // read content into buffer - var stream = req.Content!.ReadAsStream(cancellationToken); - stream.Read(gotBlob); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.StatusCode = HttpStatusCode.Created; - return res; - } - - return new HttpResponseMessage(HttpStatusCode.Forbidden); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - await store.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); - Assert.Equal(blob, gotBlob); - } - - /// - /// BlobStore_ExistsAsync tests the ExistsAsync method of the BlobStore. - /// - /// - [Fact] - public async Task BlobStore_ExistsAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var content = "foobar"u8.ToArray(); - var contentDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(content), - Size = content.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - res.StatusCode = HttpStatusCode.MethodNotAllowed; - return res; - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - var exists = await store.ExistsAsync(blobDesc, cancellationToken); - Assert.True(exists); - exists = await store.ExistsAsync(contentDesc, cancellationToken); - Assert.False(exists); - } - - /// - /// BlobStore_DeleteAsync tests the DeleteAsync method of the BlobStore. - /// - /// - [Fact] - public async Task BlobStore_DeleteAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var blobDeleted = false; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Delete) - { - res.StatusCode = HttpStatusCode.MethodNotAllowed; - return res; - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - blobDeleted = true; - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.StatusCode = HttpStatusCode.Accepted; - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - await store.DeleteAsync(blobDesc, cancellationToken); - Assert.True(blobDeleted); - - var content = "foobar"u8.ToArray(); - var contentDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(content), - Size = content.Length - }; - await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); - } - - /// - /// BlobStore_ResolveAsync tests the ResolveAsync method of the BlobStore. - /// - /// - [Fact] - public async Task BlobStore_ResolveAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - res.StatusCode = HttpStatusCode.MethodNotAllowed; - return res; - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - var got = await store.ResolveAsync(blobDesc.Digest, cancellationToken); - Assert.Equal(blobDesc.Digest, got.Digest); - Assert.Equal(blobDesc.Size, got.Size); - - var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; - got = await store.ResolveAsync(fqdnRef, cancellationToken); - Assert.Equal(blobDesc.Digest, got.Digest); - - var content = "foobar"u8.ToArray(); - var contentDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(content), - Size = content.Length - }; - await Assert.ThrowsAsync(async () => - await store.ResolveAsync(contentDesc.Digest, cancellationToken)); - } - - /// - /// BlobStore_FetchReferenceAsync tests the FetchReferenceAsync method of BlobStore - /// - /// - [Fact] - public async Task BlobStore_FetchReferenceAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - res.StatusCode = HttpStatusCode.MethodNotAllowed; - return res; - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - res.Content = new ByteArrayContent(blob); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - - // test with digest - var gotDesc = await store.FetchAsync(blobDesc.Digest, cancellationToken); - Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); - Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); - - var buf = new byte[gotDesc.Descriptor.Size]; - await gotDesc.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - - // test with FQDN reference - var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; - gotDesc = await store.FetchAsync(fqdnRef, cancellationToken); - Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); - Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); - - var content = "foobar"u8.ToArray(); - var contentDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(content), - Size = content.Length - }; - // test with other digest - await Assert.ThrowsAsync(async () => - await store.FetchAsync(contentDesc.Digest, cancellationToken)); - } - - /// - /// BlobStore_FetchAsyncReferenceAsync_Seek tests the FetchAsync method of BlobStore with seek. - /// - /// - [Fact] - public async Task BlobStore_FetchReferenceAsync_Seek() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var seekable = false; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - if (seekable) - { - res.Headers.AcceptRanges.Add("bytes"); - } - - if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) - { - } - - - if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") - { - res.StatusCode = HttpStatusCode.OK; - res.Content = new ByteArrayContent(blob); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - - var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); - var start = hv != null && hv.To.HasValue ? hv.To.Value : -1; - if (start < 0 || start >= blobDesc.Size) - { - return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); - } - - res.StatusCode = HttpStatusCode.PartialContent; - res.Content = new ByteArrayContent(blob[(int)start..]); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.StatusCode = HttpStatusCode.NotFound; - return res; - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - - var store = new BlobStore(repo); - - // test non-seekable content - - var data = await store.FetchAsync(blobDesc.Digest, cancellationToken); - - Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); - Assert.Equal(data.Descriptor.Size, blobDesc.Size); - - var buf = new byte[data.Descriptor.Size]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - - // test seekable content - seekable = true; - data = await store.FetchAsync(blobDesc.Digest, cancellationToken); - Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); - Assert.Equal(data.Descriptor.Size, blobDesc.Size); - - data.Stream.Seek(3, SeekOrigin.Begin); - buf = new byte[data.Descriptor.Size - 3]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob[3..], buf); - } - - - /// - /// GenerateBlobDescriptor_WithVariusDockerContentDigestHeaders tests the GenerateBlobDescriptor method of BlobStore with various Docker-Content-Digest headers. - /// - /// - /// - [Fact] - public void GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() - { - var reference = new Reference("eastern.haan.com", "from25to220ce"); - var tests = GetTestIOStructMapForGetDescriptorClass(); - foreach ((string testName, TestIOStruct dcdIOStruct) in tests) - { - if (dcdIOStruct.IsTag) - { - continue; - } - HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; - foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) - { - reference.ContentReference = dcdIOStruct.ClientSuppliedReference; - var resp = new HttpResponseMessage(); - if (method == HttpMethod.Get) - { - resp.Content = new ByteArrayContent(_theAmazingBanClan); - resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); - resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); - } - if (!resp.Headers.TryGetValues(_dockerContentDigestHeader, out IEnumerable? values)) - { - resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); - resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); - resp.RequestMessage = new HttpRequestMessage() - { - Method = method - }; - - } - else - { - resp.RequestMessage = new HttpRequestMessage() - { - Method = method - }; - } - - var d = string.Empty; - try - { - d = reference.Digest; - } - catch - { - throw new Exception( - $"[Blob.{method}] {testName}; got digest from a tag reference unexpectedly"); - } - - var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; - if (d.Length == 0) - { - // To avoid an otherwise impossible scenario in the tested code - // path, we set d so that verifyContentDigest does not break. - d = dcdIOStruct.ServerCalculatedDigest; - } - - var err = false; - try - { - resp.GenerateBlobDescriptor(d); - } - catch (Exception e) - { - err = true; - if (!errExpected) - { - throw new Exception( - $"[Blob.{method}] {testName}; expected no error for request, but got err; {e.Message}"); - } - - } - - if (errExpected && !err) - { - throw new Exception($"[Blob.{method}] {testName}; expected error for request, but got none"); - } - } - } - } - - - /// - /// ManifestStore_FetchAsync tests the FetchAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_FetchAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Content = new ByteArrayContent(manifest); - res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - return res; - } - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - var data = await store.FetchAsync(manifestDesc, cancellationToken); - var buf = new byte[data.Length]; - await data.ReadAsync(buf, cancellationToken); - Assert.Equal(manifest, buf); - - var content = """{"manifests":[]}"""u8.ToArray(); - var contentDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(content), - Size = content.Length - }; - await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc, cancellationToken)); - } - - [Fact] - public async Task ManifestStore_FetchAsync_ManifestUnknown() - { - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(HttpStatusCode.Unauthorized); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Content = new StringContent("""{"errors":[{"code":"UNAUTHORIZED","message":"authentication required","detail":[{"Type":"repository","Class":"","Name":"repo","Action":"pull"}]}]}"""); - return res; - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - try - { - var data = await store.FetchAsync("hello", cancellationToken); - Assert.Fail(); - } - catch (ResponseException e) - { - Assert.Equal("UNAUTHORIZED", e.Errors?[0].Code); - } - } - - /// - /// ManifestStore_PushAsync tests the PushAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_PushAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - byte[]? gotManifest = null; - - var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - if (req.Content?.Headers?.ContentLength != null) - { - var buf = new byte[req.Content.Headers.ContentLength.Value]; - (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); - gotManifest = buf; - } - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.StatusCode = HttpStatusCode.Created; - return res; - } - else - { - return new HttpResponseMessage(HttpStatusCode.Forbidden); - } - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - await store.PushAsync(manifestDesc, new MemoryStream(manifest), cancellationToken); - Assert.Equal(manifest, gotManifest); - } - - /// - /// ManifestStore_ExistAsync tests the ExistAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_ExistAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); - res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); - return res; - } - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - var exist = await store.ExistsAsync(manifestDesc, cancellationToken); - Assert.True(exist); - - var content = """{"manifests":[]}"""u8.ToArray(); - var contentDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(content), - Size = content.Length - }; - exist = await store.ExistsAsync(contentDesc, cancellationToken); - Assert.False(exist); - } - - /// - /// ManifestStore_DeleteAsync tests the DeleteAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_DeleteAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - var manifestDeleted = false; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - manifestDeleted = true; - res.StatusCode = HttpStatusCode.Accepted; - return res; - } - if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Content = new ByteArrayContent(manifest); - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); - return res; - } - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - await store.DeleteAsync(manifestDesc, cancellationToken); - Assert.True(manifestDeleted); - - var content = """{"manifests":[]}"""u8.ToArray(); - var contentDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(content), - Size = content.Length - }; - await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); - } - - /// - /// ManifestStore_ResolveAsync tests the ResolveAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_ResolveAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - var reference = "foobar"; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); - res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); - return res; - } - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - var got = await store.ResolveAsync(manifestDesc.Digest, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, got)); - got = await store.ResolveAsync(reference, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, got)); - - var tagDigestRef = "whatever" + "@" + manifestDesc.Digest; - got = await store.ResolveAsync(tagDigestRef, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, got)); - - var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; - got = await store.ResolveAsync(fqdnRef, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, got)); - - var content = """{"manifests":[]}"""u8.ToArray(); - var contentDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(content), - Size = content.Length - }; - - await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); - - } - - /// - /// ManifestStore_FetchReferenceAsync tests the FetchReferenceAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_FetchReferenceAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - var reference = "foobar"; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Content = new ByteArrayContent(manifest); - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); - return res; - } - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - - // test with tag - var data = await store.FetchAsync(reference, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); - var buf = new byte[manifest.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(manifest, buf); - - // test with other tag - var randomRef = "whatever"; - await Assert.ThrowsAsync(async () => await store.FetchAsync(randomRef, cancellationToken)); - - // test with digest - data = await store.FetchAsync(manifestDesc.Digest, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); - - buf = new byte[manifest.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(manifest, buf); - - // test with tag@digest - var tagDigestRef = randomRef + "@" + manifestDesc.Digest; - data = await store.FetchAsync(tagDigestRef, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); - buf = new byte[manifest.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(manifest, buf); - - // test with FQDN - var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; - data = await store.FetchAsync(fqdnRef, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); - buf = new byte[manifest.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(manifest, buf); - } - - /// - /// ManifestStore_TagAsync tests the TagAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_TagAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var index = """{"manifests":[]}"""u8.ToArray(); - var indexDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(index), - Size = index.Length - }; - var gotIndex = new byte[index.Length]; - var reference = "foobar"; - - var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{blobDesc.Digest}") - { - res.StatusCode = HttpStatusCode.NotFound; - return res; - } - if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Content = new ByteArrayContent(index); - res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); - res.Content.Headers.Add("Content-Type", new string[] { indexDesc.MediaType }); - return res; - } - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") - { - if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) - { - res.StatusCode = HttpStatusCode.BadRequest; - return res; - } - if (req.Content?.Headers?.ContentLength != null) - { - var buf = new byte[req.Content.Headers.ContentLength.Value]; - (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); - gotIndex = buf; - } - - res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); - res.StatusCode = HttpStatusCode.Created; - return res; - } - - res.StatusCode = HttpStatusCode.Forbidden; - return res; - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - - await Assert.ThrowsAnyAsync(async () => await store.TagAsync(blobDesc, reference, cancellationToken)); - - await store.TagAsync(indexDesc, reference, cancellationToken); - Assert.Equal(index, gotIndex); - - gotIndex = null; - await store.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); - Assert.Equal(index, gotIndex); - } - - /// - /// ManifestStore_PushReferenceAsync tests the PushReferenceAsync of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_PushReferenceAsync() - { - var index = """{"manifests":[]}"""u8.ToArray(); - var indexDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(index), - Size = index.Length - }; - var gotIndex = new byte[index.Length]; - var reference = "foobar"; - - var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") - { - if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) - { - res.StatusCode = HttpStatusCode.BadRequest; - return res; - } - - if (req.Content?.Headers?.ContentLength != null) - { - var buf = new byte[req.Content.Headers.ContentLength.Value]; - (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); - gotIndex = buf; - } - - res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); - res.StatusCode = HttpStatusCode.Created; - return res; - } - res.StatusCode = HttpStatusCode.Forbidden; - return res; - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - await store.PushAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); - Assert.Equal(index, gotIndex); - } - - /// - /// This test tries copying artifacts from the remote target to the memory target - /// - /// - [Fact] - public async Task CopyFromRepositoryToMemory() - { - var exampleManifest = @"hello world"u8.ToArray(); - - var exampleManifestDescriptor = new Descriptor - { - MediaType = MediaType.Descriptor, - Digest = ComputeSHA256(exampleManifest), - Size = exampleManifest.Length - }; - var exampleUploadUUid = new Guid().ToString(); - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - var path = req.RequestUri != null ? req.RequestUri.AbsolutePath : string.Empty; - var method = req.Method; - if (path.Contains("/blobs/uploads/") && method == HttpMethod.Post) - { - res.StatusCode = HttpStatusCode.Accepted; - res.Headers.Location = new Uri($"{path}/{exampleUploadUUid}"); - res.Headers.Add("Content-Type", MediaType.ImageManifest); - return res; - } - if (path.Contains("/blobs/uploads/" + exampleUploadUUid) && method == HttpMethod.Get) - { - res.StatusCode = HttpStatusCode.Created; - return res; - } - - if (path.Contains("/manifests/latest") && method == HttpMethod.Put) - { - res.StatusCode = HttpStatusCode.Created; - return res; - } - - if (path.Contains("/manifests/" + exampleManifestDescriptor.Digest) || path.Contains("/manifests/latest") && method == HttpMethod.Head) - { - if (method == HttpMethod.Get) - { - res.Content = new ByteArrayContent(exampleManifest); - res.Content.Headers.Add("Content-Type", MediaType.Descriptor); - res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); - res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); - return res; - } - res.Content.Headers.Add("Content-Type", MediaType.Descriptor); - res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); - res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); - return res; - } - - - if (path.Contains("/blobs/") && (method == HttpMethod.Get || method == HttpMethod.Head)) - { - var arr = path.Split("/"); - var digest = arr[arr.Length - 1]; - - - if (digest == exampleManifestDescriptor.Digest) - { - byte[] content = exampleManifest; - res.Content = new ByteArrayContent(content); - res.Content.Headers.Add("Content-Type", exampleManifestDescriptor.MediaType); - res.Content.Headers.Add("Content-Length", content.Length.ToString()); - } - - res.Headers.Add(_dockerContentDigestHeader, digest); - - return res; - } - - if (path.Contains("/manifests/") && method == HttpMethod.Put) - { - res.StatusCode = HttpStatusCode.Created; - return res; - } - - return res; - }; - - var reg = new Registry.Remote.Registry(new RepositoryOptions() - { - Reference = new Reference("localhost:5000"), - HttpClient = CustomClient(func), - }); - var src = await reg.GetRepositoryAsync("source", CancellationToken.None); - - var dst = new MemoryStore(); - var tagName = "latest"; - var desc = await src.CopyAsync(tagName, dst, tagName, CancellationToken.None); - } - - [Fact] - public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() - { - var reference = new Reference("eastern.haan.com", "from25to220ce"); - var tests = GetTestIOStructMapForGetDescriptorClass(); - foreach ((string testName, TestIOStruct dcdIOStruct) in tests) - { - var repo = new Repository(reference.Repository + "/" + reference.Repository); - HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; - var s = new ManifestStore(repo); - foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) - { - reference.ContentReference = dcdIOStruct.ClientSuppliedReference; - var resp = new HttpResponseMessage(); - if (method == HttpMethod.Get) - { - resp.Content = new ByteArrayContent(_theAmazingBanClan); - resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); - resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); - } - else - { - resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); - resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); - } - resp.RequestMessage = new HttpRequestMessage() - { - Method = method - }; - - var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; - - var err = false; - try - { - await resp.GenerateDescriptorAsync(reference, CancellationToken.None); - } - catch (Exception e) - { - err = true; - if (!errExpected) - { - throw new Exception( - $"[Manifest.{method}] {testName}; expected no error for request, but got err; {e.Message}"); - } - - } - if (errExpected && !err) - { - throw new Exception($"[Manifest.{method}] {testName}; expected error for request, but got none"); - } - } - } - - } -} +// Copyright The ORAS Authors. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. using OrasProject.Oras.Content; using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; using OrasProject.Oras.Registry; using OrasProject.Oras.Registry.Remote; using static OrasProject.Oras.Tests.Remote.Util.Util; using System.Diagnostics; using System.Net; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using System.Web; using Xunit; using static OrasProject.Oras.Content.Digest; namespace OrasProject.Oras.Tests.Remote; public class RepositoryTest { public struct TestIOStruct { public bool IsTag; public bool ErrExpectedOnHEAD; public string ServerCalculatedDigest; public string ClientSuppliedReference; public bool ErrExpectedOnGET; } private byte[] _theAmazingBanClan = "Ban Gu, Ban Chao, Ban Zhao"u8.ToArray(); private const string _theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; private const string _dockerContentDigestHeader = "Docker-Content-Digest"; // The following truth table aims to cover the expected GET/HEAD request outcome // for all possible permutations of the client/server "containing a digest", for // both Manifests and Blobs. Where the results between the two differ, the index // of the first column has an exclamation mark. // // The client is said to "contain a digest" if the user-supplied reference string // is of the form that contains a digest rather than a tag. The server, on the // other hand, is said to "contain a digest" if the server responded with the // special header `Docker-Content-Digest`. // // In this table, anything denoted with an asterisk indicates that the true // response should actually be the opposite of what's expected; for example, // `*PASS` means we will get a `PASS`, even though the true answer would be its // diametric opposite--a `FAIL`. This may seem odd, and deserves an explanation. // This function has blind-spots, and while it can expend power to gain sight, // i.e., perform the expensive validation, we chose not to. The reason is two- // fold: a) we "know" that even if we say "!PASS", it will eventually fail later // when checks are performed, and with that assumption, we have the luxury for // the second point, which is b) performance. // // _______________________________________________________________________________________________________________ // | ID | CLIENT | SERVER | Manifest.GET | Blob.GET | Manifest.HEAD | Blob.HEAD | // |----+-----------------+------------------+-----------------------+-----------+---------------------+-----------+ // | 1 | tag | missing | CALCULATE,PASS | n/a | FAIL | n/a | // | 2 | tag | presentCorrect | TRUST,PASS | n/a | TRUST,PASS | n/a | // | 3 | tag | presentIncorrect | TRUST,*PASS | n/a | TRUST,*PASS | n/a | // | 4 | correctDigest | missing | TRUST,PASS | PASS | TRUST,PASS | PASS | // | 5 | correctDigest | presentCorrect | TRUST,COMPARE,PASS | PASS | TRUST,COMPARE,PASS | PASS | // | 6 | correctDigest | presentIncorrect | TRUST,COMPARE,FAIL | FAIL | TRUST,COMPARE,FAIL | FAIL | // --------------------------------------------------------------------------------------------------------------- /// /// GetTestIOStructMapForGetDescriptorClass returns a map of test cases for different /// GET/HEAD request outcome for all possible permutations of the client/server "containing a digest", for /// both Manifests and Blobs. /// /// public static Dictionary GetTestIOStructMapForGetDescriptorClass() { string correctDigest = $"sha256:{_theAmazingBanDigest}"; string incorrectDigest = $"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; return new Dictionary { ["1. Client:Tag & Server:DigestMissing"] = new TestIOStruct { IsTag = true, ErrExpectedOnHEAD = true }, ["2. Client:Tag & Server:DigestValid"] = new TestIOStruct { IsTag = true, ServerCalculatedDigest = correctDigest }, ["3. Client:Tag & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct { IsTag = true, ServerCalculatedDigest = incorrectDigest }, ["4. Client:DigestValid & Server:DigestMissing"] = new TestIOStruct { ClientSuppliedReference = correctDigest }, ["5. Client:DigestValid & Server:DigestValid"] = new TestIOStruct { ClientSuppliedReference = correctDigest, ServerCalculatedDigest = correctDigest }, ["6. Client:DigestValid & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct { ClientSuppliedReference = correctDigest, ServerCalculatedDigest = incorrectDigest, ErrExpectedOnHEAD = true, ErrExpectedOnGET = true } }; } /// /// Repository_FetchAsync tests the FetchAsync method of the Repository. /// /// [Fact] public async Task Repository_FetchAsync() { var blob = Encoding.UTF8.GetBytes("hello world"); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = """{"manifests":[]}"""u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var resp = new HttpResponseMessage(); resp.RequestMessage = req; if (req.Method != HttpMethod.Get) { Debug.WriteLine("Expected GET request"); resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var path = req.RequestUri!.AbsolutePath; if (path == "/v2/test/blobs/" + blobDesc.Digest) { resp.Content = new ByteArrayContent(blob); resp.Content.Headers.Add("Content-Type", "application/octet-stream"); resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return resp; } if (path == "/v2/test/manifests/" + indexDesc.Digest) { if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(MediaType.ImageIndex))) { resp.StatusCode = HttpStatusCode.BadRequest; Debug.WriteLine("manifest not convertable: " + req.Headers.Accept); return resp; } resp.Content = new ByteArrayContent(index); resp.Content.Headers.Add("Content-Type", indexDesc.MediaType); resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return resp; } resp.StatusCode = HttpStatusCode.NotFound; return resp; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var stream = await repo.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); stream = await repo.FetchAsync(indexDesc, cancellationToken); buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); } /// /// Repository_PushAsync tests the PushAsync method of the Repository /// /// [Fact] public async Task Repository_PushAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var uuid = Guid.NewGuid().ToString(); var gotBlob = new byte[blobDesc.Size]; var gotIndex = new byte[indexDesc.Size]; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var resp = new HttpResponseMessage(); resp.RequestMessage = req; if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") { resp.Headers.Location = new Uri("http://localhost:5000/v2/test/blobs/uploads/" + uuid); resp.StatusCode = HttpStatusCode.Accepted; return resp; } if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains("application/octet-stream")) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var queries = HttpUtility.ParseQueryString(req.RequestUri.Query); if (queries["digest"] != blobDesc.Digest) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotBlob); resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); resp.StatusCode = HttpStatusCode.Created; return resp; } if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotIndex); resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); resp.StatusCode = HttpStatusCode.Created; return resp; } resp.StatusCode = HttpStatusCode.Forbidden; return resp; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await repo.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); Assert.Equal(blob, gotBlob); await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_ExistsAsync tests the ExistsAsync method of the Repository /// /// [Fact] public async Task Repository_ExistsAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.NotAcceptable); } res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var exists = await repo.ExistsAsync(blobDesc, cancellationToken); Assert.True(exists); exists = await repo.ExistsAsync(indexDesc, cancellationToken); Assert.True(exists); } /// /// Repository_DeleteAsync tests the DeleteAsync method of the Repository /// /// [Fact] public async Task Repository_DeleteAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var blobDeleted = false; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var indexDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { blobDeleted = true; res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Accepted; return res; } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { indexDeleted = true; // no dockerContentDigestHeader header for manifest deletion res.StatusCode = HttpStatusCode.Accepted; return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await repo.DeleteAsync(blobDesc, cancellationToken); Assert.True(blobDeleted); await repo.DeleteAsync(indexDesc, cancellationToken); Assert.True(indexDeleted); } /// /// Repository_ResolveAsync tests the ResolveAsync method of the Repository /// /// [Fact] public async Task Repository_ResolveAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.NotFound); } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest || req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await Assert.ThrowsAsync(async () => await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); // await repo.ResolveAsync(blobDesc.Digest, cancellationToken); var got = await repo.ResolveAsync(indexDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); got = await repo.ResolveAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); var tagDigestRef = "whatever" + "@" + indexDesc.Digest; got = await repo.ResolveAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; got = await repo.ResolveAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); } /// /// Repository_ResolveAsync tests the ResolveAsync method of the Repository /// /// [Fact] public async Task Repository_TagAsync() { var blob = "hello"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; byte[]? gotIndex = null; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.Found); } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content != null) { gotIndex = await req.Content.ReadAsByteArrayAsync(cancellationToken); } res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await Assert.ThrowsAnyAsync( async () => await repo.TagAsync(blobDesc, reference, cancellationToken)); await repo.TagAsync(indexDesc, reference, cancellationToken); Assert.Equal(index, gotIndex); await repo.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_PushReferenceAsync tests the PushReferenceAsync method of the Repository /// /// [Fact] public async Task Repository_PushReferenceAsync() { var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; byte[]? gotIndex = null; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content != null) { gotIndex = await req.Content.ReadAsByteArrayAsync(); } res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var streamContent = new MemoryStream(index); await repo.PushAsync(indexDesc, streamContent, reference, cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_FetchReferenceAsync tests the FetchReferenceAsync method of the Repository /// /// [Fact] public async Task Repository_FetchReferenceAsyc() { var blob = "hello"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.NotFound); } if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.Found); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); // test with blob digest await Assert.ThrowsAsync( async () => await repo.FetchAsync(blobDesc.Digest, cancellationToken)); // test with manifest digest var data = await repo.FetchAsync(indexDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); var buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest tag data = await repo.FetchAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest tag@digest var tagDigestRef = "whatever" + "@" + indexDesc.Digest; data = await repo.FetchAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; data = await repo.FetchAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); } /// /// Repository_TagsAsync tests the TagsAsync method of the Repository /// to check if the tags are returned correctly /// /// /// [Fact] public async Task Repository_TagsAsync() { var tagSet = new List>() { new() {"the", "quick", "brown", "fox"}, new() {"jumps", "over", "the", "lazy"}, new() {"dog"} }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get || req.RequestUri?.AbsolutePath != "/v2/test/tags/list" ) { return new HttpResponseMessage(HttpStatusCode.NotFound); } var q = req.RequestUri.Query; try { var n = int.Parse(Regex.Match(q, @"(?<=n=)\d+").Value); if (n != 4) throw new Exception(); } catch { return new HttpResponseMessage(HttpStatusCode.BadRequest); } var tags = new List(); var serverUrl = "http://localhost:5000"; var matched = Regex.Match(q, @"(?<=test=)\w+").Value; switch (matched) { case "foo": tags = tagSet[1]; res.Headers.Add("Link", $"<{serverUrl}/v2/test/tags/list?n=4&test=bar>; rel=\"next\""); break; case "bar": tags = tagSet[2]; break; default: tags = tagSet[0]; res.Headers.Add("Link", $"; rel=\"next\""); break; } var listOfTags = new Repository.TagList { Tags = tags.ToArray() }; res.Content = new StringContent(JsonSerializer.Serialize(listOfTags)); return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, TagListPageSize = 4, }); var cancellationToken = new CancellationToken(); var wantTags = new List(); foreach (var set in tagSet) { wantTags.AddRange(set); } var gotTags = new List(); await foreach (var tag in repo.ListTagsAsync().WithCancellation(cancellationToken)) { gotTags.Add(tag); } Assert.Equal(wantTags, gotTags); } /// /// BlobStore_FetchAsync tests the FetchAsync method of the BlobStore /// /// [Fact] public async Task BlobStore_FetchAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); } /// /// BlobStore_FetchAsync_CanSeek tests the FetchAsync method of the BlobStore for a stream that can seek /// /// [Fact] public async Task BlobStore_FetchAsync_CanSeek() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var seekable = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (seekable) { res.Headers.AcceptRanges.Add("bytes"); } if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) { } if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") { res.StatusCode = HttpStatusCode.OK; res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } long start = -1, end = -1; var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); if (hv != null && hv.From.HasValue && hv.To.HasValue) { start = hv.From.Value; end = hv.To.Value; } if (start < 0 || start > end || start >= blobDesc.Size) { return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); } end++; if (end > blobDesc.Size) { end = blobDesc.Size; } res.StatusCode = HttpStatusCode.PartialContent; res.Content = new ByteArrayContent(blob[(int)start..(int)end]); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.NotFound; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); seekable = true; stream = await store.FetchAsync(blobDesc, cancellationToken); buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); buf = new byte[stream.Length - 3]; stream.Seek(3, SeekOrigin.Begin); await stream.ReadAsync(buf, cancellationToken); var seg = blob[3..]; Assert.Equal(seg, buf); } /// /// BlobStore_FetchAsync_ZeroSizedBlob tests the FetchAsync method of the BlobStore for a zero sized blob /// /// [Fact] public async Task BlobStore_FetchAsync_ZeroSizedBlob() { var blob = ""u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (req.Headers.TryGetValues("Range", out var rangeHeader)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); } /// /// BlobStore_PushAsync tests the PushAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_PushAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var gotBlob = new byte[blob.Length]; var uuid = Guid.NewGuid().ToString(); var existingQueryParameter = "existingParam=value"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Post && req.RequestUri?.AbsolutePath == $"/v2/test/blobs/uploads/") { res.StatusCode = HttpStatusCode.Accepted; res.Headers.Add("Location", $"/v2/test/blobs/uploads/{uuid}?{existingQueryParameter}"); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) { // Assert that the existing query parameter is present var queryParameters = HttpUtility.ParseQueryString(req.RequestUri.Query); Assert.Equal("value", queryParameters["existingParam"]); if (req.Headers.TryGetValues("Content-Type", out var contentType) && contentType.FirstOrDefault() != "application/octet-stream") { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } // read content into buffer var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotBlob); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); await store.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); Assert.Equal(blob, gotBlob); } /// /// BlobStore_ExistsAsync tests the ExistsAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_ExistsAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var exists = await store.ExistsAsync(blobDesc, cancellationToken); Assert.True(exists); exists = await store.ExistsAsync(contentDesc, cancellationToken); Assert.False(exists); } /// /// BlobStore_DeleteAsync tests the DeleteAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_DeleteAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var blobDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { blobDeleted = true; res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Accepted; return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); await store.DeleteAsync(blobDesc, cancellationToken); Assert.True(blobDeleted); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); } /// /// BlobStore_ResolveAsync tests the ResolveAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_ResolveAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var got = await store.ResolveAsync(blobDesc.Digest, cancellationToken); Assert.Equal(blobDesc.Digest, got.Digest); Assert.Equal(blobDesc.Size, got.Size); var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; got = await store.ResolveAsync(fqdnRef, cancellationToken); Assert.Equal(blobDesc.Digest, got.Digest); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); } /// /// BlobStore_FetchReferenceAsync tests the FetchReferenceAsync method of BlobStore /// /// [Fact] public async Task BlobStore_FetchReferenceAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); // test with digest var gotDesc = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); var buf = new byte[gotDesc.Descriptor.Size]; await gotDesc.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); // test with FQDN reference var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; gotDesc = await store.FetchAsync(fqdnRef, cancellationToken); Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; // test with other digest await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc.Digest, cancellationToken)); } /// /// BlobStore_FetchAsyncReferenceAsync_Seek tests the FetchAsync method of BlobStore with seek. /// /// [Fact] public async Task BlobStore_FetchReferenceAsync_Seek() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var seekable = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (seekable) { res.Headers.AcceptRanges.Add("bytes"); } if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) { } if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") { res.StatusCode = HttpStatusCode.OK; res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); var start = hv != null && hv.To.HasValue ? hv.To.Value : -1; if (start < 0 || start >= blobDesc.Size) { return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); } res.StatusCode = HttpStatusCode.PartialContent; res.Content = new ByteArrayContent(blob[(int)start..]); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.NotFound; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); // test non-seekable content var data = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); Assert.Equal(data.Descriptor.Size, blobDesc.Size); var buf = new byte[data.Descriptor.Size]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); // test seekable content seekable = true; data = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); Assert.Equal(data.Descriptor.Size, blobDesc.Size); data.Stream.Seek(3, SeekOrigin.Begin); buf = new byte[data.Descriptor.Size - 3]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob[3..], buf); } /// /// GenerateBlobDescriptor_WithVariusDockerContentDigestHeaders tests the GenerateBlobDescriptor method of BlobStore with various Docker-Content-Digest headers. /// /// /// [Fact] public void GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() { var reference = new Reference("eastern.haan.com", "from25to220ce"); var tests = GetTestIOStructMapForGetDescriptorClass(); foreach ((string testName, TestIOStruct dcdIOStruct) in tests) { if (dcdIOStruct.IsTag) { continue; } HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) { reference.ContentReference = dcdIOStruct.ClientSuppliedReference; var resp = new HttpResponseMessage(); if (method == HttpMethod.Get) { resp.Content = new ByteArrayContent(_theAmazingBanClan); resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } if (!resp.Headers.TryGetValues(_dockerContentDigestHeader, out IEnumerable? values)) { resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); resp.RequestMessage = new HttpRequestMessage() { Method = method }; } else { resp.RequestMessage = new HttpRequestMessage() { Method = method }; } var d = string.Empty; try { d = reference.Digest; } catch { throw new Exception( $"[Blob.{method}] {testName}; got digest from a tag reference unexpectedly"); } var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; if (d.Length == 0) { // To avoid an otherwise impossible scenario in the tested code // path, we set d so that verifyContentDigest does not break. d = dcdIOStruct.ServerCalculatedDigest; } var err = false; try { resp.GenerateBlobDescriptor(d); } catch (Exception e) { err = true; if (!errExpected) { throw new Exception( $"[Blob.{method}] {testName}; expected no error for request, but got err; {e.Message}"); } } if (errExpected && !err) { throw new Exception($"[Blob.{method}] {testName}; expected error for request, but got none"); } } } } /// /// ManifestStore_FetchAsync tests the FetchAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_FetchAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var data = await store.FetchAsync(manifestDesc, cancellationToken); var buf = new byte[data.Length]; await data.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc, cancellationToken)); } [Fact] public async Task ManifestStore_FetchAsync_ManifestUnknown() { var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(HttpStatusCode.Unauthorized); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new StringContent( """{"errors":[{"code":"UNAUTHORIZED","message":"authentication required","detail":[{"Type":"repository","Class":"","Name":"repo","Action":"pull"}]}]}"""); return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); try { var data = await store.FetchAsync("hello", cancellationToken); Assert.Fail(); } catch (ResponseException e) { Assert.Equal("UNAUTHORIZED", e.Errors?[0].Code); } } /// /// ManifestStore_PushAsync tests the PushAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_PushAsync() { var configBlob = """config"""u8.ToArray(); var manifestStr = $@"{{""layers"": [], ""size"": 0, ""config"": {{""mediaType"": ""{MediaType.ImageConfig}"", ""digest"": ""{ComputeSHA256(configBlob)}"", ""size"": {configBlob.Length}}}}}"; var manifest = Encoding.UTF8.GetBytes(manifestStr); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; byte[]? gotManifest = null; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotManifest = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } else { return new HttpResponseMessage(HttpStatusCode.Forbidden); } }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.PushAsync(manifestDesc, new MemoryStream(manifest), cancellationToken); Assert.Equal(manifest, gotManifest); } /// /// ManifestStore_ExistAsync tests the ExistAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_ExistAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var exist = await store.ExistsAsync(manifestDesc, cancellationToken); Assert.True(exist); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; exist = await store.ExistsAsync(contentDesc, cancellationToken); Assert.False(exist); } /// /// ManifestStore_DeleteAsync tests the DeleteAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_DeleteAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var manifestDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { manifestDeleted = true; res.StatusCode = HttpStatusCode.Accepted; return res; } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.DeleteAsync(manifestDesc, cancellationToken); Assert.True(manifestDeleted); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); } /// /// ManifestStore_ResolveAsync tests the ResolveAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_ResolveAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var got = await store.ResolveAsync(manifestDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); got = await store.ResolveAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var tagDigestRef = "whatever" + "@" + manifestDesc.Digest; got = await store.ResolveAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; got = await store.ResolveAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); } /// /// ManifestStore_FetchReferenceAsync tests the FetchReferenceAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_FetchReferenceAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); // test with tag var data = await store.FetchAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); var buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with other tag var randomRef = "whatever"; await Assert.ThrowsAsync(async () => await store.FetchAsync(randomRef, cancellationToken)); // test with digest data = await store.FetchAsync(manifestDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with tag@digest var tagDigestRef = randomRef + "@" + manifestDesc.Digest; data = await store.FetchAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; data = await store.FetchAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); } /// /// ManifestStore_TagAsync tests the TagAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_TagAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var index = """{"manifests":[]}"""u8.ToArray(); var indexDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(index), Size = index.Length }; var gotIndex = new byte[index.Length]; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{blobDesc.Digest}") { res.StatusCode = HttpStatusCode.NotFound; return res; } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { indexDesc.MediaType }); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { res.StatusCode = HttpStatusCode.BadRequest; return res; } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotIndex = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } res.StatusCode = HttpStatusCode.Forbidden; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await Assert.ThrowsAnyAsync(async () => await store.TagAsync(blobDesc, reference, cancellationToken)); await store.TagAsync(indexDesc, reference, cancellationToken); Assert.Equal(index, gotIndex); gotIndex = null; await store.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); Assert.Equal(index, gotIndex); } /// /// ManifestStore_PushReferenceAsync tests the PushReferenceAsync of ManifestStore. /// /// [Fact] public async Task ManifestStore_PushReferenceAsync() { var manifest = Encoding.UTF8.GetBytes($@"{{""layers"": []}}"); var indexStr = $@"{{""manifests"":[{{""mediaType"": ""{MediaType.ImageManifest}"", ""digest"": ""{ComputeSHA256(manifest)}"", ""size"": {manifest.Length}}}]}}"; var index = Encoding.UTF8.GetBytes(indexStr); var indexDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(index), Size = index.Length }; var gotIndex = new byte[index.Length]; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { res.StatusCode = HttpStatusCode.BadRequest; return res; } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotIndex = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } res.StatusCode = HttpStatusCode.Forbidden; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.PushAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); Assert.Equal(index, gotIndex); } /// /// This test tries copying artifacts from the remote target to the memory target /// /// [Fact] public async Task CopyFromRepositoryToMemory() { var exampleManifest = @"hello world"u8.ToArray(); var exampleManifestDescriptor = new Descriptor { MediaType = MediaType.Descriptor, Digest = ComputeSHA256(exampleManifest), Size = exampleManifest.Length }; var exampleUploadUUid = new Guid().ToString(); var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; var path = req.RequestUri != null ? req.RequestUri.AbsolutePath : string.Empty; var method = req.Method; if (path.Contains("/blobs/uploads/") && method == HttpMethod.Post) { res.StatusCode = HttpStatusCode.Accepted; res.Headers.Location = new Uri($"{path}/{exampleUploadUUid}"); res.Headers.Add("Content-Type", MediaType.ImageManifest); return res; } if (path.Contains("/blobs/uploads/" + exampleUploadUUid) && method == HttpMethod.Get) { res.StatusCode = HttpStatusCode.Created; return res; } if (path.Contains("/manifests/latest") && method == HttpMethod.Put) { res.StatusCode = HttpStatusCode.Created; return res; } if (path.Contains("/manifests/" + exampleManifestDescriptor.Digest) || path.Contains("/manifests/latest") && method == HttpMethod.Head) { if (method == HttpMethod.Get) { res.Content = new ByteArrayContent(exampleManifest); res.Content.Headers.Add("Content-Type", MediaType.Descriptor); res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); return res; } res.Content.Headers.Add("Content-Type", MediaType.Descriptor); res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); return res; } if (path.Contains("/blobs/") && (method == HttpMethod.Get || method == HttpMethod.Head)) { var arr = path.Split("/"); var digest = arr[arr.Length - 1]; if (digest == exampleManifestDescriptor.Digest) { byte[] content = exampleManifest; res.Content = new ByteArrayContent(content); res.Content.Headers.Add("Content-Type", exampleManifestDescriptor.MediaType); res.Content.Headers.Add("Content-Length", content.Length.ToString()); } res.Headers.Add(_dockerContentDigestHeader, digest); return res; } if (path.Contains("/manifests/") && method == HttpMethod.Put) { res.StatusCode = HttpStatusCode.Created; return res; } return res; }; var reg = new Registry.Remote.Registry(new RepositoryOptions() { Reference = new Reference("localhost:5000"), HttpClient = CustomClient(func), }); var src = await reg.GetRepositoryAsync("source", CancellationToken.None); var dst = new MemoryStore(); var tagName = "latest"; var desc = await src.CopyAsync(tagName, dst, tagName, CancellationToken.None); } [Fact] public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() { var reference = new Reference("eastern.haan.com", "from25to220ce"); var tests = GetTestIOStructMapForGetDescriptorClass(); foreach ((string testName, TestIOStruct dcdIOStruct) in tests) { var repo = new Repository(reference.Repository + "/" + reference.Repository); HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; var s = new ManifestStore(repo); foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) { reference.ContentReference = dcdIOStruct.ClientSuppliedReference; var resp = new HttpResponseMessage(); if (method == HttpMethod.Get) { resp.Content = new ByteArrayContent(_theAmazingBanClan); resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } else { resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } resp.RequestMessage = new HttpRequestMessage() { Method = method }; var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; var err = false; try { await resp.GenerateDescriptorAsync(reference, CancellationToken.None); } catch (Exception e) { err = true; if (!errExpected) { throw new Exception( $"[Manifest.{method}] {testName}; expected no error for request, but got err; {e.Message}"); } } if (errExpected && !err) { throw new Exception($"[Manifest.{method}] {testName}; expected error for request, but got none"); } } } } } \ No newline at end of file diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs new file mode 100644 index 0000000..70803a8 --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs @@ -0,0 +1,76 @@ +using System.Text; +using System.Text.Json; +using OrasProject.Oras.Content; +using OrasProject.Oras.Oci; +using Index = OrasProject.Oras.Oci.Index; + +namespace OrasProject.Oras.Tests.Remote.Util; + +public class RandomDataGenerator +{ + public static int RandomInt(int min, int max) + { + return new Random().Next(min, max); + } + + public static string RandomString() + { + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + var length = RandomInt(1, chars.Length); + char[] stringChars = new char[length]; + for (int i = 0; i < length; ++i) + { + stringChars[i] = chars[RandomInt(0, chars.Length)]; + } + return new string(stringChars); + } + + public static Descriptor RandomDescriptor(string mediaType = MediaType.ImageManifest) + { + var randomBytes = RandomBytes(); + return new Descriptor + { MediaType = mediaType, Digest = Digest.ComputeSHA256(randomBytes), Size = randomBytes.Length }; + } + + public static (Manifest, byte[]) RandomManifest() + { + var manifest = new Manifest + { + Layers = new List(), + Config = new Descriptor{MediaType = MediaType.ImageConfig, Digest = Guid.NewGuid().ToString("N")}, + }; + return (manifest, Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest))); + } + + public static (Manifest, byte[]) RandomManifestWithSubject(Descriptor? subject = null) + { + var manifest = new Manifest + { + Layers = new List(), + Config = new Descriptor{MediaType = MediaType.ImageConfig, Digest = Guid.NewGuid().ToString("N")}, + }; + if (subject == null) manifest.Subject = RandomDescriptor(); + else manifest.Subject = subject; + return (manifest, Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest))); + } + + public static byte[] RandomBytes() + { + return Encoding.UTF8.GetBytes(RandomString()); + } + + public static Index RandomIndex() + { + return new Index() + { + Manifests = new List + { + RandomDescriptor(), + RandomDescriptor(), + }, + MediaType = MediaType.ImageIndex, + }; + } + + +} diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs new file mode 100644 index 0000000..5239247 --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs @@ -0,0 +1,41 @@ +using Moq; +using Moq.Protected; +using OrasProject.Oras.Oci; + +namespace OrasProject.Oras.Tests.Remote.Util; + +public class Util +{ + /// + /// AreDescriptorsEqual compares two descriptors and returns true if they are equal. + /// + /// + /// + /// + public static bool AreDescriptorsEqual(Descriptor a, Descriptor b) + { + return a.MediaType == b.MediaType && a.Digest == b.Digest && a.Size == b.Size; + } + + public static HttpClient CustomClient(Func func) + { + var moqHandler = new Mock(); + moqHandler.Protected().Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny() + ).ReturnsAsync(func); + return new HttpClient(moqHandler.Object); + } + + public static HttpClient CustomClient(Func> func) + { + var moqHandler = new Mock(); + moqHandler.Protected().Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny() + ).Returns(func); + return new HttpClient(moqHandler.Object); + } +} From 7fca0cbef92b2e8b89852ad52772e2dd51317f5c Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 19 Nov 2024 17:56:34 +1100 Subject: [PATCH 03/24] add unit tests Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Oci/Artifact.cs | 26 - .../Registry/Remote/Auth/Client.cs | 13 - .../Remote/HttpResponseMessageExtensions.cs | 1 - .../Registry/Remote/ManifestStore.cs | 6 +- tests/OrasProject.Oras.Tests/Oci/IndexTest.cs | 15 +- .../Remote/ManifestStoreTest.cs | 133 +- .../Remote/ReferrersTest.cs | 15 +- .../Remote/RepositoryTest.cs | 2253 ++++++++++++++++- .../Remote/Util/RandomDataGenerator.cs | 15 +- .../Remote/Util/Util.cs | 15 +- 10 files changed, 2378 insertions(+), 114 deletions(-) delete mode 100644 src/OrasProject.Oras/Oci/Artifact.cs delete mode 100644 src/OrasProject.Oras/Registry/Remote/Auth/Client.cs diff --git a/src/OrasProject.Oras/Oci/Artifact.cs b/src/OrasProject.Oras/Oci/Artifact.cs deleted file mode 100644 index 4edaa82..0000000 --- a/src/OrasProject.Oras/Oci/Artifact.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace OrasProject.Oras.Oci; - -public class Artifact -{ - [JsonPropertyName("mediaType")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public string? MediaType { get; set; } - - [JsonPropertyName("artifactType")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public string? ArtifactType { get; set; } - - [JsonPropertyName("blobs")] - public required IList Blobs { get; set; } - - [JsonPropertyName("subject")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public Descriptor? Subject { get; set; } - - [JsonPropertyName("annotations")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public IDictionary? Annotations { get; set; } -} diff --git a/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs b/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs deleted file mode 100644 index 0decf9a..0000000 --- a/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace OrasProject.Oras.Registry.Remote.Auth -{ - internal class Client - { - - } -} diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 0076c06..5fff8f3 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -26,7 +26,6 @@ namespace OrasProject.Oras.Registry.Remote; internal static class HttpResponseMessageExtensions { private const string _dockerContentDigestHeader = "Docker-Content-Digest"; - /// /// Parses the error returned by the remote registry. /// diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 5a6a090..da02467 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -144,9 +144,8 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell /// /// public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) - { - await PushWithIndexingAsync(expected, content, expected.Digest, cancellationToken).ConfigureAwait(false); - } + => await PushWithIndexingAsync(expected, content, expected.Digest, cancellationToken).ConfigureAwait(false); + /// /// PushReferenceASync pushes the manifest with a reference tag. @@ -328,7 +327,6 @@ private async Task InternalPushAsync(Descriptor expected, Stream stream, string request.Content = new StreamContent(stream); request.Content.Headers.ContentLength = expected.Size; request.Content.Headers.Add("Content-Type", expected.MediaType); - var client = Repository.Options.HttpClient; using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); if (response.StatusCode != HttpStatusCode.Created) diff --git a/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs b/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs index dacf8b2..ca1e066 100644 --- a/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs +++ b/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs @@ -1,4 +1,17 @@ -using System.Text.Json; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.Json; using OrasProject.Oras.Content; using OrasProject.Oras.Oci; using static OrasProject.Oras.Tests.Remote.Util.Util; diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index 83fd80f..3540f3b 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -1,4 +1,17 @@ -using System.Net; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Net; using System.Text; using System.Text.Json; using OrasProject.Oras.Oci; @@ -8,77 +21,13 @@ using static OrasProject.Oras.Tests.Remote.Util.Util; using static OrasProject.Oras.Content.Digest; using Index = OrasProject.Oras.Oci.Index; - - using Xunit; -using Xunit.Abstractions; namespace OrasProject.Oras.Tests.Remote; public class ManifestStoreTest { private const string _dockerContentDigestHeader = "Docker-Content-Digest"; - - private ITestOutputHelper _output; - - public ManifestStoreTest(ITestOutputHelper output) - { - _output = output; - } - - /// - /// ManifestStore_PushAsyncWithSubjectAndReferrerSupported tests PushAsync method for pushing manifest with subject when registry supports referrers API - /// - [Fact] - public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() - { - var (_, manifestBytes) = RandomManifestWithSubject(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifestBytes), - Size = manifestBytes.Length - }; - byte[]? receivedManifest = null; - - var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - if (req.Content?.Headers?.ContentLength != null) - { - var buf = new byte[req.Content.Headers.ContentLength.Value]; - (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); - receivedManifest = buf; - } - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.StatusCode = HttpStatusCode.Created; - res.Headers.Add("OCI-Subject", "test"); - return res; - } - return new HttpResponseMessage(HttpStatusCode.Forbidden); - }; - - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(mockHttpRequestHandler), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); - await store.PushAsync(manifestDesc, new MemoryStream(manifestBytes), cancellationToken); - Assert.Equal(manifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); - } [Fact] public async Task ManifestStore_PullReferrersIndexListSuccessfully() @@ -155,6 +104,60 @@ public async Task ManifestStore_PullReferrersIndexListNotFound() Assert.Empty(receivedManifests); } + /// + /// ManifestStore_PushAsyncWithSubjectAndReferrerSupported tests PushAsync method for pushing manifest with subject when registry supports referrers API + /// + [Fact] + public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() + { + var (_, manifestBytes) = RandomManifestWithSubject(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length + }; + byte[]? receivedManifest = null; + + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); + receivedManifest = buf; + } + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + res.Headers.Add("OCI-Subject", "test"); + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(manifestDesc, new MemoryStream(manifestBytes), cancellationToken); + Assert.Equal(manifestBytes, receivedManifest); + Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); + } + [Fact] public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index a3615ff..3f12ba1 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -1,4 +1,17 @@ -using OrasProject.Oras.Exceptions; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; using OrasProject.Oras.Registry.Remote; using static OrasProject.Oras.Tests.Remote.Util.Util; diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index acecad7..23ed423 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -1 +1,2252 @@ -// Copyright The ORAS Authors. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. using OrasProject.Oras.Content; using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; using OrasProject.Oras.Registry; using OrasProject.Oras.Registry.Remote; using static OrasProject.Oras.Tests.Remote.Util.Util; using System.Diagnostics; using System.Net; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using System.Web; using Xunit; using static OrasProject.Oras.Content.Digest; namespace OrasProject.Oras.Tests.Remote; public class RepositoryTest { public struct TestIOStruct { public bool IsTag; public bool ErrExpectedOnHEAD; public string ServerCalculatedDigest; public string ClientSuppliedReference; public bool ErrExpectedOnGET; } private byte[] _theAmazingBanClan = "Ban Gu, Ban Chao, Ban Zhao"u8.ToArray(); private const string _theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; private const string _dockerContentDigestHeader = "Docker-Content-Digest"; // The following truth table aims to cover the expected GET/HEAD request outcome // for all possible permutations of the client/server "containing a digest", for // both Manifests and Blobs. Where the results between the two differ, the index // of the first column has an exclamation mark. // // The client is said to "contain a digest" if the user-supplied reference string // is of the form that contains a digest rather than a tag. The server, on the // other hand, is said to "contain a digest" if the server responded with the // special header `Docker-Content-Digest`. // // In this table, anything denoted with an asterisk indicates that the true // response should actually be the opposite of what's expected; for example, // `*PASS` means we will get a `PASS`, even though the true answer would be its // diametric opposite--a `FAIL`. This may seem odd, and deserves an explanation. // This function has blind-spots, and while it can expend power to gain sight, // i.e., perform the expensive validation, we chose not to. The reason is two- // fold: a) we "know" that even if we say "!PASS", it will eventually fail later // when checks are performed, and with that assumption, we have the luxury for // the second point, which is b) performance. // // _______________________________________________________________________________________________________________ // | ID | CLIENT | SERVER | Manifest.GET | Blob.GET | Manifest.HEAD | Blob.HEAD | // |----+-----------------+------------------+-----------------------+-----------+---------------------+-----------+ // | 1 | tag | missing | CALCULATE,PASS | n/a | FAIL | n/a | // | 2 | tag | presentCorrect | TRUST,PASS | n/a | TRUST,PASS | n/a | // | 3 | tag | presentIncorrect | TRUST,*PASS | n/a | TRUST,*PASS | n/a | // | 4 | correctDigest | missing | TRUST,PASS | PASS | TRUST,PASS | PASS | // | 5 | correctDigest | presentCorrect | TRUST,COMPARE,PASS | PASS | TRUST,COMPARE,PASS | PASS | // | 6 | correctDigest | presentIncorrect | TRUST,COMPARE,FAIL | FAIL | TRUST,COMPARE,FAIL | FAIL | // --------------------------------------------------------------------------------------------------------------- /// /// GetTestIOStructMapForGetDescriptorClass returns a map of test cases for different /// GET/HEAD request outcome for all possible permutations of the client/server "containing a digest", for /// both Manifests and Blobs. /// /// public static Dictionary GetTestIOStructMapForGetDescriptorClass() { string correctDigest = $"sha256:{_theAmazingBanDigest}"; string incorrectDigest = $"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; return new Dictionary { ["1. Client:Tag & Server:DigestMissing"] = new TestIOStruct { IsTag = true, ErrExpectedOnHEAD = true }, ["2. Client:Tag & Server:DigestValid"] = new TestIOStruct { IsTag = true, ServerCalculatedDigest = correctDigest }, ["3. Client:Tag & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct { IsTag = true, ServerCalculatedDigest = incorrectDigest }, ["4. Client:DigestValid & Server:DigestMissing"] = new TestIOStruct { ClientSuppliedReference = correctDigest }, ["5. Client:DigestValid & Server:DigestValid"] = new TestIOStruct { ClientSuppliedReference = correctDigest, ServerCalculatedDigest = correctDigest }, ["6. Client:DigestValid & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct { ClientSuppliedReference = correctDigest, ServerCalculatedDigest = incorrectDigest, ErrExpectedOnHEAD = true, ErrExpectedOnGET = true } }; } /// /// Repository_FetchAsync tests the FetchAsync method of the Repository. /// /// [Fact] public async Task Repository_FetchAsync() { var blob = Encoding.UTF8.GetBytes("hello world"); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = """{"manifests":[]}"""u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var resp = new HttpResponseMessage(); resp.RequestMessage = req; if (req.Method != HttpMethod.Get) { Debug.WriteLine("Expected GET request"); resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var path = req.RequestUri!.AbsolutePath; if (path == "/v2/test/blobs/" + blobDesc.Digest) { resp.Content = new ByteArrayContent(blob); resp.Content.Headers.Add("Content-Type", "application/octet-stream"); resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return resp; } if (path == "/v2/test/manifests/" + indexDesc.Digest) { if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(MediaType.ImageIndex))) { resp.StatusCode = HttpStatusCode.BadRequest; Debug.WriteLine("manifest not convertable: " + req.Headers.Accept); return resp; } resp.Content = new ByteArrayContent(index); resp.Content.Headers.Add("Content-Type", indexDesc.MediaType); resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return resp; } resp.StatusCode = HttpStatusCode.NotFound; return resp; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var stream = await repo.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); stream = await repo.FetchAsync(indexDesc, cancellationToken); buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); } /// /// Repository_PushAsync tests the PushAsync method of the Repository /// /// [Fact] public async Task Repository_PushAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var uuid = Guid.NewGuid().ToString(); var gotBlob = new byte[blobDesc.Size]; var gotIndex = new byte[indexDesc.Size]; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var resp = new HttpResponseMessage(); resp.RequestMessage = req; if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") { resp.Headers.Location = new Uri("http://localhost:5000/v2/test/blobs/uploads/" + uuid); resp.StatusCode = HttpStatusCode.Accepted; return resp; } if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains("application/octet-stream")) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var queries = HttpUtility.ParseQueryString(req.RequestUri.Query); if (queries["digest"] != blobDesc.Digest) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotBlob); resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); resp.StatusCode = HttpStatusCode.Created; return resp; } if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotIndex); resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); resp.StatusCode = HttpStatusCode.Created; return resp; } resp.StatusCode = HttpStatusCode.Forbidden; return resp; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await repo.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); Assert.Equal(blob, gotBlob); await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_ExistsAsync tests the ExistsAsync method of the Repository /// /// [Fact] public async Task Repository_ExistsAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.NotAcceptable); } res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var exists = await repo.ExistsAsync(blobDesc, cancellationToken); Assert.True(exists); exists = await repo.ExistsAsync(indexDesc, cancellationToken); Assert.True(exists); } /// /// Repository_DeleteAsync tests the DeleteAsync method of the Repository /// /// [Fact] public async Task Repository_DeleteAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var blobDeleted = false; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var indexDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { blobDeleted = true; res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Accepted; return res; } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { indexDeleted = true; // no dockerContentDigestHeader header for manifest deletion res.StatusCode = HttpStatusCode.Accepted; return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await repo.DeleteAsync(blobDesc, cancellationToken); Assert.True(blobDeleted); await repo.DeleteAsync(indexDesc, cancellationToken); Assert.True(indexDeleted); } /// /// Repository_ResolveAsync tests the ResolveAsync method of the Repository /// /// [Fact] public async Task Repository_ResolveAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.NotFound); } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest || req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await Assert.ThrowsAsync(async () => await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); // await repo.ResolveAsync(blobDesc.Digest, cancellationToken); var got = await repo.ResolveAsync(indexDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); got = await repo.ResolveAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); var tagDigestRef = "whatever" + "@" + indexDesc.Digest; got = await repo.ResolveAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; got = await repo.ResolveAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); } /// /// Repository_ResolveAsync tests the ResolveAsync method of the Repository /// /// [Fact] public async Task Repository_TagAsync() { var blob = "hello"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; byte[]? gotIndex = null; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.Found); } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content != null) { gotIndex = await req.Content.ReadAsByteArrayAsync(cancellationToken); } res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await Assert.ThrowsAnyAsync( async () => await repo.TagAsync(blobDesc, reference, cancellationToken)); await repo.TagAsync(indexDesc, reference, cancellationToken); Assert.Equal(index, gotIndex); await repo.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_PushReferenceAsync tests the PushReferenceAsync method of the Repository /// /// [Fact] public async Task Repository_PushReferenceAsync() { var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; byte[]? gotIndex = null; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content != null) { gotIndex = await req.Content.ReadAsByteArrayAsync(); } res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var streamContent = new MemoryStream(index); await repo.PushAsync(indexDesc, streamContent, reference, cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_FetchReferenceAsync tests the FetchReferenceAsync method of the Repository /// /// [Fact] public async Task Repository_FetchReferenceAsyc() { var blob = "hello"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.NotFound); } if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.Found); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); // test with blob digest await Assert.ThrowsAsync( async () => await repo.FetchAsync(blobDesc.Digest, cancellationToken)); // test with manifest digest var data = await repo.FetchAsync(indexDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); var buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest tag data = await repo.FetchAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest tag@digest var tagDigestRef = "whatever" + "@" + indexDesc.Digest; data = await repo.FetchAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; data = await repo.FetchAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); } /// /// Repository_TagsAsync tests the TagsAsync method of the Repository /// to check if the tags are returned correctly /// /// /// [Fact] public async Task Repository_TagsAsync() { var tagSet = new List>() { new() {"the", "quick", "brown", "fox"}, new() {"jumps", "over", "the", "lazy"}, new() {"dog"} }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get || req.RequestUri?.AbsolutePath != "/v2/test/tags/list" ) { return new HttpResponseMessage(HttpStatusCode.NotFound); } var q = req.RequestUri.Query; try { var n = int.Parse(Regex.Match(q, @"(?<=n=)\d+").Value); if (n != 4) throw new Exception(); } catch { return new HttpResponseMessage(HttpStatusCode.BadRequest); } var tags = new List(); var serverUrl = "http://localhost:5000"; var matched = Regex.Match(q, @"(?<=test=)\w+").Value; switch (matched) { case "foo": tags = tagSet[1]; res.Headers.Add("Link", $"<{serverUrl}/v2/test/tags/list?n=4&test=bar>; rel=\"next\""); break; case "bar": tags = tagSet[2]; break; default: tags = tagSet[0]; res.Headers.Add("Link", $"; rel=\"next\""); break; } var listOfTags = new Repository.TagList { Tags = tags.ToArray() }; res.Content = new StringContent(JsonSerializer.Serialize(listOfTags)); return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, TagListPageSize = 4, }); var cancellationToken = new CancellationToken(); var wantTags = new List(); foreach (var set in tagSet) { wantTags.AddRange(set); } var gotTags = new List(); await foreach (var tag in repo.ListTagsAsync().WithCancellation(cancellationToken)) { gotTags.Add(tag); } Assert.Equal(wantTags, gotTags); } /// /// BlobStore_FetchAsync tests the FetchAsync method of the BlobStore /// /// [Fact] public async Task BlobStore_FetchAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); } /// /// BlobStore_FetchAsync_CanSeek tests the FetchAsync method of the BlobStore for a stream that can seek /// /// [Fact] public async Task BlobStore_FetchAsync_CanSeek() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var seekable = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (seekable) { res.Headers.AcceptRanges.Add("bytes"); } if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) { } if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") { res.StatusCode = HttpStatusCode.OK; res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } long start = -1, end = -1; var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); if (hv != null && hv.From.HasValue && hv.To.HasValue) { start = hv.From.Value; end = hv.To.Value; } if (start < 0 || start > end || start >= blobDesc.Size) { return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); } end++; if (end > blobDesc.Size) { end = blobDesc.Size; } res.StatusCode = HttpStatusCode.PartialContent; res.Content = new ByteArrayContent(blob[(int)start..(int)end]); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.NotFound; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); seekable = true; stream = await store.FetchAsync(blobDesc, cancellationToken); buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); buf = new byte[stream.Length - 3]; stream.Seek(3, SeekOrigin.Begin); await stream.ReadAsync(buf, cancellationToken); var seg = blob[3..]; Assert.Equal(seg, buf); } /// /// BlobStore_FetchAsync_ZeroSizedBlob tests the FetchAsync method of the BlobStore for a zero sized blob /// /// [Fact] public async Task BlobStore_FetchAsync_ZeroSizedBlob() { var blob = ""u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (req.Headers.TryGetValues("Range", out var rangeHeader)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); } /// /// BlobStore_PushAsync tests the PushAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_PushAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var gotBlob = new byte[blob.Length]; var uuid = Guid.NewGuid().ToString(); var existingQueryParameter = "existingParam=value"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Post && req.RequestUri?.AbsolutePath == $"/v2/test/blobs/uploads/") { res.StatusCode = HttpStatusCode.Accepted; res.Headers.Add("Location", $"/v2/test/blobs/uploads/{uuid}?{existingQueryParameter}"); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) { // Assert that the existing query parameter is present var queryParameters = HttpUtility.ParseQueryString(req.RequestUri.Query); Assert.Equal("value", queryParameters["existingParam"]); if (req.Headers.TryGetValues("Content-Type", out var contentType) && contentType.FirstOrDefault() != "application/octet-stream") { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } // read content into buffer var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotBlob); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); await store.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); Assert.Equal(blob, gotBlob); } /// /// BlobStore_ExistsAsync tests the ExistsAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_ExistsAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var exists = await store.ExistsAsync(blobDesc, cancellationToken); Assert.True(exists); exists = await store.ExistsAsync(contentDesc, cancellationToken); Assert.False(exists); } /// /// BlobStore_DeleteAsync tests the DeleteAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_DeleteAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var blobDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { blobDeleted = true; res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Accepted; return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); await store.DeleteAsync(blobDesc, cancellationToken); Assert.True(blobDeleted); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); } /// /// BlobStore_ResolveAsync tests the ResolveAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_ResolveAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var got = await store.ResolveAsync(blobDesc.Digest, cancellationToken); Assert.Equal(blobDesc.Digest, got.Digest); Assert.Equal(blobDesc.Size, got.Size); var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; got = await store.ResolveAsync(fqdnRef, cancellationToken); Assert.Equal(blobDesc.Digest, got.Digest); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); } /// /// BlobStore_FetchReferenceAsync tests the FetchReferenceAsync method of BlobStore /// /// [Fact] public async Task BlobStore_FetchReferenceAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); // test with digest var gotDesc = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); var buf = new byte[gotDesc.Descriptor.Size]; await gotDesc.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); // test with FQDN reference var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; gotDesc = await store.FetchAsync(fqdnRef, cancellationToken); Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; // test with other digest await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc.Digest, cancellationToken)); } /// /// BlobStore_FetchAsyncReferenceAsync_Seek tests the FetchAsync method of BlobStore with seek. /// /// [Fact] public async Task BlobStore_FetchReferenceAsync_Seek() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var seekable = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (seekable) { res.Headers.AcceptRanges.Add("bytes"); } if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) { } if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") { res.StatusCode = HttpStatusCode.OK; res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); var start = hv != null && hv.To.HasValue ? hv.To.Value : -1; if (start < 0 || start >= blobDesc.Size) { return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); } res.StatusCode = HttpStatusCode.PartialContent; res.Content = new ByteArrayContent(blob[(int)start..]); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.NotFound; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); // test non-seekable content var data = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); Assert.Equal(data.Descriptor.Size, blobDesc.Size); var buf = new byte[data.Descriptor.Size]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); // test seekable content seekable = true; data = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); Assert.Equal(data.Descriptor.Size, blobDesc.Size); data.Stream.Seek(3, SeekOrigin.Begin); buf = new byte[data.Descriptor.Size - 3]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob[3..], buf); } /// /// GenerateBlobDescriptor_WithVariusDockerContentDigestHeaders tests the GenerateBlobDescriptor method of BlobStore with various Docker-Content-Digest headers. /// /// /// [Fact] public void GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() { var reference = new Reference("eastern.haan.com", "from25to220ce"); var tests = GetTestIOStructMapForGetDescriptorClass(); foreach ((string testName, TestIOStruct dcdIOStruct) in tests) { if (dcdIOStruct.IsTag) { continue; } HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) { reference.ContentReference = dcdIOStruct.ClientSuppliedReference; var resp = new HttpResponseMessage(); if (method == HttpMethod.Get) { resp.Content = new ByteArrayContent(_theAmazingBanClan); resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } if (!resp.Headers.TryGetValues(_dockerContentDigestHeader, out IEnumerable? values)) { resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); resp.RequestMessage = new HttpRequestMessage() { Method = method }; } else { resp.RequestMessage = new HttpRequestMessage() { Method = method }; } var d = string.Empty; try { d = reference.Digest; } catch { throw new Exception( $"[Blob.{method}] {testName}; got digest from a tag reference unexpectedly"); } var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; if (d.Length == 0) { // To avoid an otherwise impossible scenario in the tested code // path, we set d so that verifyContentDigest does not break. d = dcdIOStruct.ServerCalculatedDigest; } var err = false; try { resp.GenerateBlobDescriptor(d); } catch (Exception e) { err = true; if (!errExpected) { throw new Exception( $"[Blob.{method}] {testName}; expected no error for request, but got err; {e.Message}"); } } if (errExpected && !err) { throw new Exception($"[Blob.{method}] {testName}; expected error for request, but got none"); } } } } /// /// ManifestStore_FetchAsync tests the FetchAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_FetchAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var data = await store.FetchAsync(manifestDesc, cancellationToken); var buf = new byte[data.Length]; await data.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc, cancellationToken)); } [Fact] public async Task ManifestStore_FetchAsync_ManifestUnknown() { var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(HttpStatusCode.Unauthorized); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new StringContent( """{"errors":[{"code":"UNAUTHORIZED","message":"authentication required","detail":[{"Type":"repository","Class":"","Name":"repo","Action":"pull"}]}]}"""); return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); try { var data = await store.FetchAsync("hello", cancellationToken); Assert.Fail(); } catch (ResponseException e) { Assert.Equal("UNAUTHORIZED", e.Errors?[0].Code); } } /// /// ManifestStore_PushAsync tests the PushAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_PushAsync() { var configBlob = """config"""u8.ToArray(); var manifestStr = $@"{{""layers"": [], ""size"": 0, ""config"": {{""mediaType"": ""{MediaType.ImageConfig}"", ""digest"": ""{ComputeSHA256(configBlob)}"", ""size"": {configBlob.Length}}}}}"; var manifest = Encoding.UTF8.GetBytes(manifestStr); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; byte[]? gotManifest = null; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotManifest = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } else { return new HttpResponseMessage(HttpStatusCode.Forbidden); } }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.PushAsync(manifestDesc, new MemoryStream(manifest), cancellationToken); Assert.Equal(manifest, gotManifest); } /// /// ManifestStore_ExistAsync tests the ExistAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_ExistAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var exist = await store.ExistsAsync(manifestDesc, cancellationToken); Assert.True(exist); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; exist = await store.ExistsAsync(contentDesc, cancellationToken); Assert.False(exist); } /// /// ManifestStore_DeleteAsync tests the DeleteAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_DeleteAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var manifestDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { manifestDeleted = true; res.StatusCode = HttpStatusCode.Accepted; return res; } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.DeleteAsync(manifestDesc, cancellationToken); Assert.True(manifestDeleted); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); } /// /// ManifestStore_ResolveAsync tests the ResolveAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_ResolveAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var got = await store.ResolveAsync(manifestDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); got = await store.ResolveAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var tagDigestRef = "whatever" + "@" + manifestDesc.Digest; got = await store.ResolveAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; got = await store.ResolveAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); } /// /// ManifestStore_FetchReferenceAsync tests the FetchReferenceAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_FetchReferenceAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); // test with tag var data = await store.FetchAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); var buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with other tag var randomRef = "whatever"; await Assert.ThrowsAsync(async () => await store.FetchAsync(randomRef, cancellationToken)); // test with digest data = await store.FetchAsync(manifestDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with tag@digest var tagDigestRef = randomRef + "@" + manifestDesc.Digest; data = await store.FetchAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; data = await store.FetchAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); } /// /// ManifestStore_TagAsync tests the TagAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_TagAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var index = """{"manifests":[]}"""u8.ToArray(); var indexDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(index), Size = index.Length }; var gotIndex = new byte[index.Length]; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{blobDesc.Digest}") { res.StatusCode = HttpStatusCode.NotFound; return res; } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { indexDesc.MediaType }); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { res.StatusCode = HttpStatusCode.BadRequest; return res; } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotIndex = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } res.StatusCode = HttpStatusCode.Forbidden; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await Assert.ThrowsAnyAsync(async () => await store.TagAsync(blobDesc, reference, cancellationToken)); await store.TagAsync(indexDesc, reference, cancellationToken); Assert.Equal(index, gotIndex); gotIndex = null; await store.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); Assert.Equal(index, gotIndex); } /// /// ManifestStore_PushReferenceAsync tests the PushReferenceAsync of ManifestStore. /// /// [Fact] public async Task ManifestStore_PushReferenceAsync() { var manifest = Encoding.UTF8.GetBytes($@"{{""layers"": []}}"); var indexStr = $@"{{""manifests"":[{{""mediaType"": ""{MediaType.ImageManifest}"", ""digest"": ""{ComputeSHA256(manifest)}"", ""size"": {manifest.Length}}}]}}"; var index = Encoding.UTF8.GetBytes(indexStr); var indexDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(index), Size = index.Length }; var gotIndex = new byte[index.Length]; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { res.StatusCode = HttpStatusCode.BadRequest; return res; } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotIndex = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } res.StatusCode = HttpStatusCode.Forbidden; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.PushAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); Assert.Equal(index, gotIndex); } /// /// This test tries copying artifacts from the remote target to the memory target /// /// [Fact] public async Task CopyFromRepositoryToMemory() { var exampleManifest = @"hello world"u8.ToArray(); var exampleManifestDescriptor = new Descriptor { MediaType = MediaType.Descriptor, Digest = ComputeSHA256(exampleManifest), Size = exampleManifest.Length }; var exampleUploadUUid = new Guid().ToString(); var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; var path = req.RequestUri != null ? req.RequestUri.AbsolutePath : string.Empty; var method = req.Method; if (path.Contains("/blobs/uploads/") && method == HttpMethod.Post) { res.StatusCode = HttpStatusCode.Accepted; res.Headers.Location = new Uri($"{path}/{exampleUploadUUid}"); res.Headers.Add("Content-Type", MediaType.ImageManifest); return res; } if (path.Contains("/blobs/uploads/" + exampleUploadUUid) && method == HttpMethod.Get) { res.StatusCode = HttpStatusCode.Created; return res; } if (path.Contains("/manifests/latest") && method == HttpMethod.Put) { res.StatusCode = HttpStatusCode.Created; return res; } if (path.Contains("/manifests/" + exampleManifestDescriptor.Digest) || path.Contains("/manifests/latest") && method == HttpMethod.Head) { if (method == HttpMethod.Get) { res.Content = new ByteArrayContent(exampleManifest); res.Content.Headers.Add("Content-Type", MediaType.Descriptor); res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); return res; } res.Content.Headers.Add("Content-Type", MediaType.Descriptor); res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); return res; } if (path.Contains("/blobs/") && (method == HttpMethod.Get || method == HttpMethod.Head)) { var arr = path.Split("/"); var digest = arr[arr.Length - 1]; if (digest == exampleManifestDescriptor.Digest) { byte[] content = exampleManifest; res.Content = new ByteArrayContent(content); res.Content.Headers.Add("Content-Type", exampleManifestDescriptor.MediaType); res.Content.Headers.Add("Content-Length", content.Length.ToString()); } res.Headers.Add(_dockerContentDigestHeader, digest); return res; } if (path.Contains("/manifests/") && method == HttpMethod.Put) { res.StatusCode = HttpStatusCode.Created; return res; } return res; }; var reg = new Registry.Remote.Registry(new RepositoryOptions() { Reference = new Reference("localhost:5000"), HttpClient = CustomClient(func), }); var src = await reg.GetRepositoryAsync("source", CancellationToken.None); var dst = new MemoryStore(); var tagName = "latest"; var desc = await src.CopyAsync(tagName, dst, tagName, CancellationToken.None); } [Fact] public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() { var reference = new Reference("eastern.haan.com", "from25to220ce"); var tests = GetTestIOStructMapForGetDescriptorClass(); foreach ((string testName, TestIOStruct dcdIOStruct) in tests) { var repo = new Repository(reference.Repository + "/" + reference.Repository); HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; var s = new ManifestStore(repo); foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) { reference.ContentReference = dcdIOStruct.ClientSuppliedReference; var resp = new HttpResponseMessage(); if (method == HttpMethod.Get) { resp.Content = new ByteArrayContent(_theAmazingBanClan); resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } else { resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } resp.RequestMessage = new HttpRequestMessage() { Method = method }; var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; var err = false; try { await resp.GenerateDescriptorAsync(reference, CancellationToken.None); } catch (Exception e) { err = true; if (!errExpected) { throw new Exception( $"[Manifest.{method}] {testName}; expected no error for request, but got err; {e.Message}"); } } if (errExpected && !err) { throw new Exception($"[Manifest.{method}] {testName}; expected error for request, but got none"); } } } } } \ No newline at end of file +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Content; +using OrasProject.Oras.Exceptions; +using OrasProject.Oras.Oci; +using OrasProject.Oras.Registry; +using OrasProject.Oras.Registry.Remote; +using System.Diagnostics; +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Web; +using Xunit; +using static OrasProject.Oras.Content.Digest; +using static OrasProject.Oras.Tests.Remote.Util.Util; +using static OrasProject.Oras.Tests.Remote.Util.RandomDataGenerator; + +namespace OrasProject.Oras.Tests.Remote; + +public class RepositoryTest +{ + public struct TestIOStruct + { + public bool IsTag; + public bool ErrExpectedOnHEAD; + public string ServerCalculatedDigest; + public string ClientSuppliedReference; + public bool ErrExpectedOnGET; + } + + private byte[] _theAmazingBanClan = "Ban Gu, Ban Chao, Ban Zhao"u8.ToArray(); + private const string _theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; + + private const string _dockerContentDigestHeader = "Docker-Content-Digest"; + + // The following truth table aims to cover the expected GET/HEAD request outcome + // for all possible permutations of the client/server "containing a digest", for + // both Manifests and Blobs. Where the results between the two differ, the index + // of the first column has an exclamation mark. + // + // The client is said to "contain a digest" if the user-supplied reference string + // is of the form that contains a digest rather than a tag. The server, on the + // other hand, is said to "contain a digest" if the server responded with the + // special header `Docker-Content-Digest`. + // + // In this table, anything denoted with an asterisk indicates that the true + // response should actually be the opposite of what's expected; for example, + // `*PASS` means we will get a `PASS`, even though the true answer would be its + // diametric opposite--a `FAIL`. This may seem odd, and deserves an explanation. + // This function has blind-spots, and while it can expend power to gain sight, + // i.e., perform the expensive validation, we chose not to. The reason is two- + // fold: a) we "know" that even if we say "!PASS", it will eventually fail later + // when checks are performed, and with that assumption, we have the luxury for + // the second point, which is b) performance. + // + // _______________________________________________________________________________________________________________ + // | ID | CLIENT | SERVER | Manifest.GET | Blob.GET | Manifest.HEAD | Blob.HEAD | + // |----+-----------------+------------------+-----------------------+-----------+---------------------+-----------+ + // | 1 | tag | missing | CALCULATE,PASS | n/a | FAIL | n/a | + // | 2 | tag | presentCorrect | TRUST,PASS | n/a | TRUST,PASS | n/a | + // | 3 | tag | presentIncorrect | TRUST,*PASS | n/a | TRUST,*PASS | n/a | + // | 4 | correctDigest | missing | TRUST,PASS | PASS | TRUST,PASS | PASS | + // | 5 | correctDigest | presentCorrect | TRUST,COMPARE,PASS | PASS | TRUST,COMPARE,PASS | PASS | + // | 6 | correctDigest | presentIncorrect | TRUST,COMPARE,FAIL | FAIL | TRUST,COMPARE,FAIL | FAIL | + // --------------------------------------------------------------------------------------------------------------- + + /// + /// GetTestIOStructMapForGetDescriptorClass returns a map of test cases for different + /// GET/HEAD request outcome for all possible permutations of the client/server "containing a digest", for + /// both Manifests and Blobs. + /// + /// + public static Dictionary GetTestIOStructMapForGetDescriptorClass() + { + string correctDigest = $"sha256:{_theAmazingBanDigest}"; + string incorrectDigest = $"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; + + return new Dictionary + { + ["1. Client:Tag & Server:DigestMissing"] = new TestIOStruct + { + IsTag = true, + ErrExpectedOnHEAD = true + }, + ["2. Client:Tag & Server:DigestValid"] = new TestIOStruct + { + IsTag = true, + ServerCalculatedDigest = correctDigest + }, + ["3. Client:Tag & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct + { + IsTag = true, + ServerCalculatedDigest = incorrectDigest + }, + ["4. Client:DigestValid & Server:DigestMissing"] = new TestIOStruct + { + ClientSuppliedReference = correctDigest + }, + ["5. Client:DigestValid & Server:DigestValid"] = new TestIOStruct + { + ClientSuppliedReference = correctDigest, + ServerCalculatedDigest = correctDigest + }, + ["6. Client:DigestValid & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct + { + ClientSuppliedReference = correctDigest, + ServerCalculatedDigest = incorrectDigest, + ErrExpectedOnHEAD = true, + ErrExpectedOnGET = true + } + }; + } + + /// + /// Repository_FetchAsync tests the FetchAsync method of the Repository. + /// + /// + [Fact] + public async Task Repository_FetchAsync() + { + var blob = Encoding.UTF8.GetBytes("hello world"); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = """{"manifests":[]}"""u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + Debug.WriteLine("Expected GET request"); + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + + var path = req.RequestUri!.AbsolutePath; + if (path == "/v2/test/blobs/" + blobDesc.Digest) + { + resp.Content = new ByteArrayContent(blob); + resp.Content.Headers.Add("Content-Type", "application/octet-stream"); + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return resp; + } + + if (path == "/v2/test/manifests/" + indexDesc.Digest) + { + if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(MediaType.ImageIndex))) + { + resp.StatusCode = HttpStatusCode.BadRequest; + Debug.WriteLine("manifest not convertable: " + req.Headers.Accept); + return resp; + } + + resp.Content = new ByteArrayContent(index); + resp.Content.Headers.Add("Content-Type", indexDesc.MediaType); + resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + return resp; + } + + resp.StatusCode = HttpStatusCode.NotFound; + return resp; + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var stream = await repo.FetchAsync(blobDesc, cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + stream = await repo.FetchAsync(indexDesc, cancellationToken); + buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + + } + + /// + /// Repository_PushAsync tests the PushAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_PushAsync() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var uuid = Guid.NewGuid().ToString(); + var gotBlob = new byte[blobDesc.Size]; + var gotIndex = new byte[indexDesc.Size]; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") + { + resp.Headers.Location = new Uri("http://localhost:5000/v2/test/blobs/uploads/" + uuid); + resp.StatusCode = HttpStatusCode.Accepted; + return resp; + } + + if (req.Method == HttpMethod.Put && + req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) + { + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains("application/octet-stream")) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + + } + + var queries = HttpUtility.ParseQueryString(req.RequestUri.Query); + if (queries["digest"] != blobDesc.Digest) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + + var stream = req.Content!.ReadAsStream(cancellationToken); + stream.Read(gotBlob); + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + return resp; + + } + + if (req.Method == HttpMethod.Put && + req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + + var stream = req.Content!.ReadAsStream(cancellationToken); + stream.Read(gotIndex); + resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + return resp; + } + + resp.StatusCode = HttpStatusCode.Forbidden; + return resp; + + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + await repo.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); + Assert.Equal(blob, gotBlob); + await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// Repository_ExistsAsync tests the ExistsAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_ExistsAsync() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) + { + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.NotAcceptable); + } + + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var exists = await repo.ExistsAsync(blobDesc, cancellationToken); + Assert.True(exists); + exists = await repo.ExistsAsync(indexDesc, cancellationToken); + Assert.True(exists); + } + + /// + /// Repository_DeleteAsync tests the DeleteAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_DeleteAsync() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var blobDeleted = false; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var indexDeleted = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) + { + blobDeleted = true; + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + indexDeleted = true; + // no dockerContentDigestHeader header for manifest deletion + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + await repo.DeleteAsync(blobDesc, cancellationToken); + Assert.True(blobDeleted); + await repo.DeleteAsync(indexDesc, cancellationToken); + Assert.True(indexDeleted); + } + + /// + /// Repository_ResolveAsync tests the ResolveAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_ResolveAsync() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var reference = "foobar"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest + || req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + reference) + + { + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + await Assert.ThrowsAsync(async () => + await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); + // await repo.ResolveAsync(blobDesc.Digest, cancellationToken); + var got = await repo.ResolveAsync(indexDesc.Digest, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, got)); + + got = await repo.ResolveAsync(reference, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, got)); + var tagDigestRef = "whatever" + "@" + indexDesc.Digest; + got = await repo.ResolveAsync(tagDigestRef, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, got)); + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + got = await repo.ResolveAsync(fqdnRef, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, got)); + } + + /// + /// Repository_ResolveAsync tests the ResolveAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_TagAsync() + { + var blob = "hello"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + byte[]? gotIndex = null; + var reference = "foobar"; + + var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && + req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.Found); + } + + if (req.Method == HttpMethod.Get && + req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + res.Content = new ByteArrayContent(index); + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + return res; + } + + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference + || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + + { + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + if (req.Content != null) + { + gotIndex = await req.Content.ReadAsByteArrayAsync(cancellationToken); + } + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + res.StatusCode = HttpStatusCode.Created; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + await Assert.ThrowsAnyAsync( + async () => await repo.TagAsync(blobDesc, reference, cancellationToken)); + await repo.TagAsync(indexDesc, reference, cancellationToken); + Assert.Equal(index, gotIndex); + await repo.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// Repository_PushReferenceAsync tests the PushReferenceAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_PushReferenceAsync() + { + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + byte[]? gotIndex = null; + var reference = "foobar"; + + var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) + { + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + if (req.Content != null) + { + gotIndex = await req.Content.ReadAsByteArrayAsync(); + } + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + res.StatusCode = HttpStatusCode.Created; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var streamContent = new MemoryStream(index); + await repo.PushAsync(indexDesc, streamContent, reference, cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// Repository_FetchReferenceAsync tests the FetchReferenceAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_FetchReferenceAsyc() + { + var blob = "hello"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var reference = "foobar"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest + || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) + { + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + res.Content = new ByteArrayContent(index); + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.Found); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + // test with blob digest + await Assert.ThrowsAsync( + async () => await repo.FetchAsync(blobDesc.Digest, cancellationToken)); + + // test with manifest digest + var data = await repo.FetchAsync(indexDesc.Digest, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); + var buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + + // test with manifest tag + data = await repo.FetchAsync(reference, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); + buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + + // test with manifest tag@digest + var tagDigestRef = "whatever" + "@" + indexDesc.Digest; + data = await repo.FetchAsync(tagDigestRef, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); + buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + + // test with manifest FQDN + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + data = await repo.FetchAsync(fqdnRef, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); + + buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + } + + /// + /// Repository_TagsAsync tests the TagsAsync method of the Repository + /// to check if the tags are returned correctly + /// + /// + /// + [Fact] + public async Task Repository_TagsAsync() + { + var tagSet = new List>() + { + new() {"the", "quick", "brown", "fox"}, + new() {"jumps", "over", "the", "lazy"}, + new() {"dog"} + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get || + req.RequestUri?.AbsolutePath != "/v2/test/tags/list" + ) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + var q = req.RequestUri.Query; + try + { + var n = int.Parse(Regex.Match(q, @"(?<=n=)\d+").Value); + if (n != 4) throw new Exception(); + } + catch + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + var tags = new List(); + var serverUrl = "http://localhost:5000"; + var matched = Regex.Match(q, @"(?<=test=)\w+").Value; + switch (matched) + { + case "foo": + tags = tagSet[1]; + res.Headers.Add("Link", $"<{serverUrl}/v2/test/tags/list?n=4&test=bar>; rel=\"next\""); + break; + case "bar": + tags = tagSet[2]; + break; + default: + tags = tagSet[0]; + res.Headers.Add("Link", $"; rel=\"next\""); + break; + } + + var listOfTags = new Repository.TagList + { + Tags = tags.ToArray() + }; + res.Content = new StringContent(JsonSerializer.Serialize(listOfTags)); + return res; + + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + TagListPageSize = 4, + }); + + var cancellationToken = new CancellationToken(); + + var wantTags = new List(); + foreach (var set in tagSet) + { + wantTags.AddRange(set); + } + var gotTags = new List(); + await foreach (var tag in repo.ListTagsAsync().WithCancellation(cancellationToken)) + { + gotTags.Add(tag); + } + Assert.Equal(wantTags, gotTags); + } + + /// + /// BlobStore_FetchAsync tests the FetchAsync method of the BlobStore + /// + /// + [Fact] + public async Task BlobStore_FetchAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var stream = await store.FetchAsync(blobDesc, cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + } + + /// + /// BlobStore_FetchAsync_CanSeek tests the FetchAsync method of the BlobStore for a stream that can seek + /// + /// + [Fact] + public async Task BlobStore_FetchAsync_CanSeek() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var seekable = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + if (seekable) + { + res.Headers.AcceptRanges.Add("bytes"); + } + + if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) + { + } + + + if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") + { + res.StatusCode = HttpStatusCode.OK; + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + + long start = -1, end = -1; + var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); + if (hv != null && hv.From.HasValue && hv.To.HasValue) + { + start = hv.From.Value; + end = hv.To.Value; + } + + if (start < 0 || start > end || start >= blobDesc.Size) + { + return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); + } + + end++; + if (end > blobDesc.Size) + { + end = blobDesc.Size; + } + + res.StatusCode = HttpStatusCode.PartialContent; + res.Content = new ByteArrayContent(blob[(int)start..(int)end]); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.StatusCode = HttpStatusCode.NotFound; + return res; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var stream = await store.FetchAsync(blobDesc, cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + seekable = true; + stream = await store.FetchAsync(blobDesc, cancellationToken); + buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + buf = new byte[stream.Length - 3]; + stream.Seek(3, SeekOrigin.Begin); + await stream.ReadAsync(buf, cancellationToken); + var seg = blob[3..]; + Assert.Equal(seg, buf); + } + + /// + /// BlobStore_FetchAsync_ZeroSizedBlob tests the FetchAsync method of the BlobStore for a zero sized blob + /// + /// + [Fact] + public async Task BlobStore_FetchAsync_ZeroSizedBlob() + { + var blob = ""u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + if (req.Headers.TryGetValues("Range", out var rangeHeader)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var stream = await store.FetchAsync(blobDesc, cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + } + + /// + /// BlobStore_PushAsync tests the PushAsync method of the BlobStore. + /// + /// + [Fact] + public async Task BlobStore_PushAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var gotBlob = new byte[blob.Length]; + var uuid = Guid.NewGuid().ToString(); + var existingQueryParameter = "existingParam=value"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri?.AbsolutePath == $"/v2/test/blobs/uploads/") + { + res.StatusCode = HttpStatusCode.Accepted; + res.Headers.Add("Location", $"/v2/test/blobs/uploads/{uuid}?{existingQueryParameter}"); + return res; + } + + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) + { + // Assert that the existing query parameter is present + var queryParameters = HttpUtility.ParseQueryString(req.RequestUri.Query); + Assert.Equal("value", queryParameters["existingParam"]); + + if (req.Headers.TryGetValues("Content-Type", out var contentType) && + contentType.FirstOrDefault() != "application/octet-stream") + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + // read content into buffer + var stream = req.Content!.ReadAsStream(cancellationToken); + stream.Read(gotBlob); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.StatusCode = HttpStatusCode.Created; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + await store.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); + Assert.Equal(blob, gotBlob); + } + + /// + /// BlobStore_ExistsAsync tests the ExistsAsync method of the BlobStore. + /// + /// + [Fact] + public async Task BlobStore_ExistsAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(content), + Size = content.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var exists = await store.ExistsAsync(blobDesc, cancellationToken); + Assert.True(exists); + exists = await store.ExistsAsync(contentDesc, cancellationToken); + Assert.False(exists); + } + + /// + /// BlobStore_DeleteAsync tests the DeleteAsync method of the BlobStore. + /// + /// + [Fact] + public async Task BlobStore_DeleteAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var blobDeleted = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + blobDeleted = true; + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + await store.DeleteAsync(blobDesc, cancellationToken); + Assert.True(blobDeleted); + + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(content), + Size = content.Length + }; + await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); + } + + /// + /// BlobStore_ResolveAsync tests the ResolveAsync method of the BlobStore. + /// + /// + [Fact] + public async Task BlobStore_ResolveAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var got = await store.ResolveAsync(blobDesc.Digest, cancellationToken); + Assert.Equal(blobDesc.Digest, got.Digest); + Assert.Equal(blobDesc.Size, got.Size); + + var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; + got = await store.ResolveAsync(fqdnRef, cancellationToken); + Assert.Equal(blobDesc.Digest, got.Digest); + + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(content), + Size = content.Length + }; + await Assert.ThrowsAsync(async () => + await store.ResolveAsync(contentDesc.Digest, cancellationToken)); + } + + /// + /// BlobStore_FetchReferenceAsync tests the FetchReferenceAsync method of BlobStore + /// + /// + [Fact] + public async Task BlobStore_FetchReferenceAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + + // test with digest + var gotDesc = await store.FetchAsync(blobDesc.Digest, cancellationToken); + Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); + Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); + + var buf = new byte[gotDesc.Descriptor.Size]; + await gotDesc.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + // test with FQDN reference + var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; + gotDesc = await store.FetchAsync(fqdnRef, cancellationToken); + Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); + Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); + + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(content), + Size = content.Length + }; + // test with other digest + await Assert.ThrowsAsync(async () => + await store.FetchAsync(contentDesc.Digest, cancellationToken)); + } + + /// + /// BlobStore_FetchAsyncReferenceAsync_Seek tests the FetchAsync method of BlobStore with seek. + /// + /// + [Fact] + public async Task BlobStore_FetchReferenceAsync_Seek() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var seekable = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + if (seekable) + { + res.Headers.AcceptRanges.Add("bytes"); + } + + if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) + { + } + + + if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") + { + res.StatusCode = HttpStatusCode.OK; + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + + var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); + var start = hv != null && hv.To.HasValue ? hv.To.Value : -1; + if (start < 0 || start >= blobDesc.Size) + { + return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); + } + + res.StatusCode = HttpStatusCode.PartialContent; + res.Content = new ByteArrayContent(blob[(int)start..]); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.StatusCode = HttpStatusCode.NotFound; + return res; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + var store = new BlobStore(repo); + + // test non-seekable content + + var data = await store.FetchAsync(blobDesc.Digest, cancellationToken); + + Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); + Assert.Equal(data.Descriptor.Size, blobDesc.Size); + + var buf = new byte[data.Descriptor.Size]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + // test seekable content + seekable = true; + data = await store.FetchAsync(blobDesc.Digest, cancellationToken); + Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); + Assert.Equal(data.Descriptor.Size, blobDesc.Size); + + data.Stream.Seek(3, SeekOrigin.Begin); + buf = new byte[data.Descriptor.Size - 3]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob[3..], buf); + } + + + /// + /// GenerateBlobDescriptor_WithVariusDockerContentDigestHeaders tests the GenerateBlobDescriptor method of BlobStore with various Docker-Content-Digest headers. + /// + /// + /// + [Fact] + public void GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() + { + var reference = new Reference("eastern.haan.com", "from25to220ce"); + var tests = GetTestIOStructMapForGetDescriptorClass(); + foreach ((string testName, TestIOStruct dcdIOStruct) in tests) + { + if (dcdIOStruct.IsTag) + { + continue; + } + HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; + foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) + { + reference.ContentReference = dcdIOStruct.ClientSuppliedReference; + var resp = new HttpResponseMessage(); + if (method == HttpMethod.Get) + { + resp.Content = new ByteArrayContent(_theAmazingBanClan); + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); + } + if (!resp.Headers.TryGetValues(_dockerContentDigestHeader, out IEnumerable? values)) + { + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); + resp.RequestMessage = new HttpRequestMessage() + { + Method = method + }; + + } + else + { + resp.RequestMessage = new HttpRequestMessage() + { + Method = method + }; + } + + var d = string.Empty; + try + { + d = reference.Digest; + } + catch + { + throw new Exception( + $"[Blob.{method}] {testName}; got digest from a tag reference unexpectedly"); + } + + var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; + if (d.Length == 0) + { + // To avoid an otherwise impossible scenario in the tested code + // path, we set d so that verifyContentDigest does not break. + d = dcdIOStruct.ServerCalculatedDigest; + } + + var err = false; + try + { + resp.GenerateBlobDescriptor(d); + } + catch (Exception e) + { + err = true; + if (!errExpected) + { + throw new Exception( + $"[Blob.{method}] {testName}; expected no error for request, but got err; {e.Message}"); + } + + } + + if (errExpected && !err) + { + throw new Exception($"[Blob.{method}] {testName}; expected error for request, but got none"); + } + } + } + } + + + /// + /// ManifestStore_FetchAsync tests the FetchAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_FetchAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifest), + Size = manifest.Length + }; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(manifest); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var data = await store.FetchAsync(manifestDesc, cancellationToken); + var buf = new byte[data.Length]; + await data.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(content), + Size = content.Length + }; + await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc, cancellationToken)); + } + + [Fact] + public async Task ManifestStore_FetchAsync_ManifestUnknown() + { + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(HttpStatusCode.Unauthorized); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && + !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new StringContent("""{"errors":[{"code":"UNAUTHORIZED","message":"authentication required","detail":[{"Type":"repository","Class":"","Name":"repo","Action":"pull"}]}]}"""); + return res; + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + try + { + var data = await store.FetchAsync("hello", cancellationToken); + Assert.Fail(); + } + catch (ResponseException e) + { + Assert.Equal("UNAUTHORIZED", e.Errors?[0].Code); + } + } + + /// + /// ManifestStore_PushAsync tests the PushAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_PushAsync() + { + var (_, manifestBytes) = RandomManifest(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length + }; + byte[]? gotManifest = null; + + var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); + gotManifest = buf; + } + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + return res; + } + else + { + return new HttpResponseMessage(HttpStatusCode.Forbidden); + } + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.PushAsync(manifestDesc, new MemoryStream(manifestBytes), cancellationToken); + Assert.Equal(manifestBytes, gotManifest); + } + + /// + /// ManifestStore_ExistAsync tests the ExistAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_ExistAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifest), + Size = manifest.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var exist = await store.ExistsAsync(manifestDesc, cancellationToken); + Assert.True(exist); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(content), + Size = content.Length + }; + exist = await store.ExistsAsync(contentDesc, cancellationToken); + Assert.False(exist); + } + + /// + /// ManifestStore_DeleteAsync tests the DeleteAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_DeleteAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifest), + Size = manifest.Length + }; + var manifestDeleted = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + manifestDeleted = true; + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(manifest); + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.DeleteAsync(manifestDesc, cancellationToken); + Assert.True(manifestDeleted); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(content), + Size = content.Length + }; + await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); + } + + /// + /// ManifestStore_ResolveAsync tests the ResolveAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_ResolveAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifest), + Size = manifest.Length + }; + var reference = "foobar"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var got = await store.ResolveAsync(manifestDesc.Digest, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); + got = await store.ResolveAsync(reference, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); + + var tagDigestRef = "whatever" + "@" + manifestDesc.Digest; + got = await store.ResolveAsync(tagDigestRef, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); + + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + got = await store.ResolveAsync(fqdnRef, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(content), + Size = content.Length + }; + + await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); + + } + + /// + /// ManifestStore_FetchReferenceAsync tests the FetchReferenceAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_FetchReferenceAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifest), + Size = manifest.Length + }; + var reference = "foobar"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(manifest); + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + // test with tag + var data = await store.FetchAsync(reference, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); + var buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + // test with other tag + var randomRef = "whatever"; + await Assert.ThrowsAsync(async () => await store.FetchAsync(randomRef, cancellationToken)); + + // test with digest + data = await store.FetchAsync(manifestDesc.Digest, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); + + buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + // test with tag@digest + var tagDigestRef = randomRef + "@" + manifestDesc.Digest; + data = await store.FetchAsync(tagDigestRef, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); + buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + // test with FQDN + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + data = await store.FetchAsync(fqdnRef, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); + buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + } + + /// + /// ManifestStore_TagAsync tests the TagAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_TagAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var index = """{"manifests":[]}"""u8.ToArray(); + var indexDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(index), + Size = index.Length + }; + var gotIndex = new byte[index.Length]; + var reference = "foobar"; + + var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{blobDesc.Digest}") + { + res.StatusCode = HttpStatusCode.NotFound; + return res; + } + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(index); + res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { indexDesc.MediaType }); + return res; + } + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) + { + res.StatusCode = HttpStatusCode.BadRequest; + return res; + } + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); + gotIndex = buf; + } + + res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + return res; + } + + res.StatusCode = HttpStatusCode.Forbidden; + return res; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + await Assert.ThrowsAnyAsync(async () => await store.TagAsync(blobDesc, reference, cancellationToken)); + + await store.TagAsync(indexDesc, reference, cancellationToken); + Assert.Equal(index, gotIndex); + + gotIndex = null; + await store.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// ManifestStore_PushReferenceAsync tests the PushReferenceAsync of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_PushReferenceAsync() + { + var manifest = Encoding.UTF8.GetBytes($@"{{""layers"": []}}"); + var indexStr = $@"{{""manifests"":[{{""mediaType"": ""{MediaType.ImageManifest}"", ""digest"": ""{ComputeSHA256(manifest)}"", ""size"": {manifest.Length}}}]}}"; + var index = Encoding.UTF8.GetBytes(indexStr); + var indexDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(index), + Size = index.Length + }; + var gotIndex = new byte[index.Length]; + var reference = "foobar"; + + var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) + { + res.StatusCode = HttpStatusCode.BadRequest; + return res; + } + + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); + gotIndex = buf; + } + + res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + return res; + } + res.StatusCode = HttpStatusCode.Forbidden; + return res; + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.PushAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// This test tries copying artifacts from the remote target to the memory target + /// + /// + [Fact] + public async Task CopyFromRepositoryToMemory() + { + var exampleManifest = @"hello world"u8.ToArray(); + + var exampleManifestDescriptor = new Descriptor + { + MediaType = MediaType.Descriptor, + Digest = ComputeSHA256(exampleManifest), + Size = exampleManifest.Length + }; + var exampleUploadUUid = new Guid().ToString(); + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + var path = req.RequestUri != null ? req.RequestUri.AbsolutePath : string.Empty; + var method = req.Method; + if (path.Contains("/blobs/uploads/") && method == HttpMethod.Post) + { + res.StatusCode = HttpStatusCode.Accepted; + res.Headers.Location = new Uri($"{path}/{exampleUploadUUid}"); + res.Headers.Add("Content-Type", MediaType.ImageManifest); + return res; + } + if (path.Contains("/blobs/uploads/" + exampleUploadUUid) && method == HttpMethod.Get) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + + if (path.Contains("/manifests/latest") && method == HttpMethod.Put) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + + if (path.Contains("/manifests/" + exampleManifestDescriptor.Digest) || path.Contains("/manifests/latest") && method == HttpMethod.Head) + { + if (method == HttpMethod.Get) + { + res.Content = new ByteArrayContent(exampleManifest); + res.Content.Headers.Add("Content-Type", MediaType.Descriptor); + res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); + res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); + return res; + } + res.Content.Headers.Add("Content-Type", MediaType.Descriptor); + res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); + res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); + return res; + } + + + if (path.Contains("/blobs/") && (method == HttpMethod.Get || method == HttpMethod.Head)) + { + var arr = path.Split("/"); + var digest = arr[arr.Length - 1]; + + + if (digest == exampleManifestDescriptor.Digest) + { + byte[] content = exampleManifest; + res.Content = new ByteArrayContent(content); + res.Content.Headers.Add("Content-Type", exampleManifestDescriptor.MediaType); + res.Content.Headers.Add("Content-Length", content.Length.ToString()); + } + + res.Headers.Add(_dockerContentDigestHeader, digest); + + return res; + } + + if (path.Contains("/manifests/") && method == HttpMethod.Put) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + + return res; + }; + + var reg = new Registry.Remote.Registry(new RepositoryOptions() + { + Reference = new Reference("localhost:5000"), + HttpClient = CustomClient(func), + }); + var src = await reg.GetRepositoryAsync("source", CancellationToken.None); + + var dst = new MemoryStore(); + var tagName = "latest"; + var desc = await src.CopyAsync(tagName, dst, tagName, CancellationToken.None); + } + + [Fact] + public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() + { + var reference = new Reference("eastern.haan.com", "from25to220ce"); + var tests = GetTestIOStructMapForGetDescriptorClass(); + foreach ((string testName, TestIOStruct dcdIOStruct) in tests) + { + var repo = new Repository(reference.Repository + "/" + reference.Repository); + HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; + var s = new ManifestStore(repo); + foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) + { + reference.ContentReference = dcdIOStruct.ClientSuppliedReference; + var resp = new HttpResponseMessage(); + if (method == HttpMethod.Get) + { + resp.Content = new ByteArrayContent(_theAmazingBanClan); + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); + } + else + { + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); + } + resp.RequestMessage = new HttpRequestMessage() + { + Method = method + }; + + var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; + + var err = false; + try + { + await resp.GenerateDescriptorAsync(reference, CancellationToken.None); + } + catch (Exception e) + { + err = true; + if (!errExpected) + { + throw new Exception( + $"[Manifest.{method}] {testName}; expected no error for request, but got err; {e.Message}"); + } + + } + if (errExpected && !err) + { + throw new Exception($"[Manifest.{method}] {testName}; expected error for request, but got none"); + } + } + } + + } +} diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs index 70803a8..f34df44 100644 --- a/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs +++ b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs @@ -1,4 +1,17 @@ -using System.Text; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text; using System.Text.Json; using OrasProject.Oras.Content; using OrasProject.Oras.Oci; diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs index 5239247..6da68ca 100644 --- a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs +++ b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs @@ -1,4 +1,17 @@ -using Moq; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Moq; using Moq.Protected; using OrasProject.Oras.Oci; From 7809549243e9dbd7f2820c1b4a69d065cd7cc18b Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 19 Nov 2024 18:24:51 +1100 Subject: [PATCH 04/24] resolve merge conflicts Signed-off-by: Patrick Pan --- .../Exceptions/NoReferrerUpdateException.cs | 2 +- tests/OrasProject.Oras.Tests/PackerTest.cs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs index 87ab2b5..54ff5d9 100644 --- a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs +++ b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs @@ -30,7 +30,7 @@ public NoReferrerUpdateException(string message) { } - public NoReferrerUpdateException(string message, Exception innerException) + public NoReferrerUpdateException(string message, Exception? innerException) : base(message, innerException) { } diff --git a/tests/OrasProject.Oras.Tests/PackerTest.cs b/tests/OrasProject.Oras.Tests/PackerTest.cs index c656d87..129d214 100644 --- a/tests/OrasProject.Oras.Tests/PackerTest.cs +++ b/tests/OrasProject.Oras.Tests/PackerTest.cs @@ -146,12 +146,12 @@ public async Task TestPackManifestImageV1_0_WithOptions() Layers = layers }; var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); - appendBlob(Oci.MediaType.ImageManifest, manifestBytes); + appendBlob(MediaType.ImageManifest, manifestBytes); }; var getBytes = (string data) => Encoding.UTF8.GetBytes(data); - appendBlob(Oci.MediaType.ImageConfig, getBytes("config")); // blob 0 - appendBlob(Oci.MediaType.ImageLayer, getBytes("hello world")); // blob 1 - appendBlob(Oci.MediaType.ImageLayer, getBytes("goodbye world")); // blob 2 + appendBlob(MediaType.ImageConfig, getBytes("config")); // blob 0 + appendBlob(MediaType.ImageLayer, getBytes("hello world")); // blob 1 + appendBlob(MediaType.ImageLayer, getBytes("goodbye world")); // blob 2 var layers = descs.GetRange(1, 2); var configBytes = Encoding.UTF8.GetBytes("{}"); var configDesc = new Descriptor From c7e441830a1dc9ebbee2e1ed63432805f3023d73 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 19 Nov 2024 18:31:59 +1100 Subject: [PATCH 05/24] add tests Signed-off-by: Patrick Pan --- .../Remote/RepositoryTest.cs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index ded9fca..11dc730 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -1602,8 +1602,7 @@ public async Task ManifestStore_FetchAsync_ManifestUnknown() { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && - !values.Contains(MediaType.ImageManifest)) + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } @@ -2045,16 +2044,15 @@ public async Task ManifestStore_TagAsync() [Fact] public async Task ManifestStore_PushReferenceAsync() { - var manifest = Encoding.UTF8.GetBytes($@"{{""layers"": []}}"); - var indexStr = $@"{{""manifests"":[{{""mediaType"": ""{MediaType.ImageManifest}"", ""digest"": ""{ComputeSHA256(manifest)}"", ""size"": {manifest.Length}}}]}}"; - var index = Encoding.UTF8.GetBytes(indexStr); + var index = RandomIndex(); + var indexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(index)); var indexDesc = new Descriptor { MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(index), - Size = index.Length + Digest = ComputeSHA256(indexBytes), + Size = indexBytes.Length }; - var gotIndex = new byte[index.Length]; + var gotIndex = new byte[indexBytes.Length]; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => @@ -2092,8 +2090,8 @@ public async Task ManifestStore_PushReferenceAsync() }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - await store.PushAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); - Assert.Equal(index, gotIndex); + await store.PushAsync(indexDesc, new MemoryStream(indexBytes), reference, cancellationToken); + Assert.Equal(indexBytes, gotIndex); } /// From a6d552f73edef36b2ed494362b997a68ff43d2b9 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Wed, 20 Nov 2024 09:21:24 +1100 Subject: [PATCH 06/24] add comments Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Oci/Descriptor.cs | 2 +- .../Registry/Remote/HttpResponseMessageExtensions.cs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 8b00626..8e552c6 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -77,7 +77,7 @@ internal static bool IsEmptyOrNull(Descriptor? descriptor) return descriptor == null || descriptor.Size == 0 || string.IsNullOrEmpty(descriptor.Digest) || string.IsNullOrEmpty(descriptor.MediaType); } - internal static Descriptor EmptyDescriptor() => new Descriptor + internal static Descriptor EmptyDescriptor() => new () { MediaType = "", Digest = "", diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 5fff8f3..908ab97 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -101,6 +101,12 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string } } + /// + /// CheckOciSubjectHeader checks if the response header contains "OCI-Subject", + /// repository ReferrerState is set to supported if it is present + /// + /// + /// public static void CheckOciSubjectHeader(this HttpResponseMessage response, Repository repository) { if (response.Headers.TryGetValues("OCI-Subject", out var values)) From 685ab702f5663b8a5a3b141a172f1f86095e171c Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Wed, 20 Nov 2024 11:16:31 +1100 Subject: [PATCH 07/24] add unit test Signed-off-by: Patrick Pan --- .../Remote/ManifestStoreTest.cs | 54 +++++++++++++++---- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index 3540f3b..d13a765 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -110,12 +110,30 @@ public async Task ManifestStore_PullReferrersIndexListNotFound() [Fact] public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() { - var (_, manifestBytes) = RandomManifestWithSubject(); - var manifestDesc = new Descriptor + // first push with image manifest + var (_, expectedManifestBytes) = RandomManifestWithSubject(); + var expectedManifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifestBytes), - Size = manifestBytes.Length + Digest = ComputeSHA256(expectedManifestBytes), + Size = expectedManifestBytes.Length + }; + + // second push with index manifest + var expectedIndexManifest = new Index() + { + Subject = RandomDescriptor(), + Manifests = new List{ RandomDescriptor(), RandomDescriptor() }, + MediaType = MediaType.ImageIndex, + ArtifactType = MediaType.ImageIndex, + }; + var expectedIndexManifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(expectedIndexManifest)); + var expectedIndexManifestDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(expectedIndexManifestBytes), + Size = expectedIndexManifestBytes.Length, + ArtifactType = MediaType.ImageIndex, }; byte[]? receivedManifest = null; @@ -123,11 +141,18 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() { var res = new HttpResponseMessage(); res.RequestMessage = req; - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + if (req.Method == HttpMethod.Put && (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}" || + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedIndexManifestDesc.Digest}" )) { - if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values)) { - return new HttpResponseMessage(HttpStatusCode.BadRequest); + if ((req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}" && + !values.Contains(MediaType.ImageManifest)) || + (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedIndexManifestDesc.Digest}" && + !values.Contains(MediaType.ImageIndex))) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } } if (req.Content?.Headers?.ContentLength != null) { @@ -135,7 +160,8 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); receivedManifest = buf; } - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}") res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedManifestDesc.Digest }); + else res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedIndexManifestDesc.Digest }); res.StatusCode = HttpStatusCode.Created; res.Headers.Add("OCI-Subject", "test"); return res; @@ -143,7 +169,6 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() return new HttpResponseMessage(HttpStatusCode.Forbidden); }; - var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), @@ -152,9 +177,16 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); + + // first push with image manifest Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); - await store.PushAsync(manifestDesc, new MemoryStream(manifestBytes), cancellationToken); - Assert.Equal(manifestBytes, receivedManifest); + await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); + Assert.Equal(expectedManifestBytes, receivedManifest); + Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); + + // second push with index manifest + await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); + Assert.Equal(expectedIndexManifestBytes, receivedManifest); Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); } From 1b9ade38b4fb447aacfb044c67b96ffdbc2b3c40 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Wed, 20 Nov 2024 15:25:07 +1100 Subject: [PATCH 08/24] add unit tests Signed-off-by: Patrick Pan --- .../Remote/ManifestStoreTest.cs | 77 ++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index d13a765..2e21eef 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -104,6 +104,80 @@ public async Task ManifestStore_PullReferrersIndexListNotFound() Assert.Empty(receivedManifests); } + [Fact] + public async Task ManifestStore_PushAsyncWithoutSubject() + { + // first push with image manifest + var (_, expectedManifestBytes) = RandomManifest(); + var expectedManifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(expectedManifestBytes), + Size = expectedManifestBytes.Length + }; + + // second push with image config + var expectedConfigBytes = """config"""u8.ToArray(); + var expectedConfigDesc = new Descriptor + { + MediaType = MediaType.ImageConfig, + Digest = ComputeSHA256(expectedConfigBytes), + Size = expectedConfigBytes.Length + }; + + byte[]? receivedManifest = null; + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedConfigDesc.Digest}" || + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}")) + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values)) + { + if ((req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}" && + !values.Contains(MediaType.ImageManifest)) || + (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedConfigDesc.Digest}" && + !values.Contains(MediaType.ImageConfig))) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + } + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); + receivedManifest = buf; + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}") + res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedManifestDesc.Digest }); + else res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedConfigDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); + Assert.Equal(expectedManifestBytes, receivedManifest); + + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(expectedConfigDesc, new MemoryStream(expectedConfigBytes), cancellationToken); + Assert.Equal(expectedConfigBytes, receivedManifest); + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + } + + /// /// ManifestStore_PushAsyncWithSubjectAndReferrerSupported tests PushAsync method for pushing manifest with subject when registry supports referrers API /// @@ -160,7 +234,8 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); receivedManifest = buf; } - if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}") res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedManifestDesc.Digest }); + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}") + res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedManifestDesc.Digest }); else res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedIndexManifestDesc.Digest }); res.StatusCode = HttpStatusCode.Created; res.Headers.Add("OCI-Subject", "test"); From 504159214ab551a24a61627790564d99ce64490b Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Thu, 21 Nov 2024 15:44:35 +1100 Subject: [PATCH 09/24] resolve comments Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Content/Digest.cs | 13 ------ src/OrasProject.Oras/Oci/Index.cs | 11 +---- .../Remote/HttpResponseMessageExtensions.cs | 9 ++++- .../Registry/Remote/ManifestStore.cs | 40 ++++++++++++++----- .../Registry/Remote/Referrers.cs | 3 +- .../Remote/ReferrersTest.cs | 17 ++++++++ 6 files changed, 59 insertions(+), 34 deletions(-) diff --git a/src/OrasProject.Oras/Content/Digest.cs b/src/OrasProject.Oras/Content/Digest.cs index e87ed5c..15febfb 100644 --- a/src/OrasProject.Oras/Content/Digest.cs +++ b/src/OrasProject.Oras/Content/Digest.cs @@ -46,19 +46,6 @@ internal static string Validate(string? digest) return digest; } - - internal static string GetAlgorithm(string digest) - { - var validatedDigest = Validate(digest); - return validatedDigest.Split(':')[0]; - } - - internal static string GetRef(string digest) - { - var validatedDigest = Validate(digest); - return validatedDigest.Split(':')[1]; - } - /// /// Generates a SHA-256 digest from a byte array. /// diff --git a/src/OrasProject.Oras/Oci/Index.cs b/src/OrasProject.Oras/Oci/Index.cs index e632923..dc81f8d 100644 --- a/src/OrasProject.Oras/Oci/Index.cs +++ b/src/OrasProject.Oras/Oci/Index.cs @@ -51,14 +51,7 @@ internal static (Descriptor, byte[]) GenerateIndex(IList manifests) MediaType = Oci.MediaType.ImageIndex, SchemaVersion = 2 }; - var indexContent = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(index)); - var indexDesc = new Descriptor() - { - Digest = Digest.ComputeSHA256(indexContent), - MediaType = Oci.MediaType.ImageIndex, - Size = indexContent.Length - }; - - return (indexDesc, indexContent); + var indexContent = JsonSerializer.SerializeToUtf8Bytes(index); + return (Descriptor.Create(indexContent, Oci.MediaType.ImageIndex), indexContent); } } diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 908ab97..00473d9 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -107,12 +107,19 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string /// /// /// - public static void CheckOciSubjectHeader(this HttpResponseMessage response, Repository repository) + public static void CheckOCISubjectHeader(this HttpResponseMessage response, Repository repository) { if (response.Headers.TryGetValues("OCI-Subject", out var values)) { + // Set it to ReferrerSupported when the response header contains OCI-Subject repository.ReferrerState = Referrers.ReferrerState.ReferrerSupported; } + + // If the "OCI-Subject" header is NOT set, it means that either the manifest + // has no subject OR the referrers API is NOT supported by the registry. + // + // Since we don't know whether the pushed manifest has a subject or not, + // we do not set the ReferrerState to ReferrerNotSupported here. } /// diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index da02467..7dbb810 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -180,23 +180,30 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, st case MediaType.ImageIndex: if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) { + // Push the manifest straightaway when the registry supports referrers API await InternalPushAsync(expected, content, reference, cancellationToken).ConfigureAwait(false); return; } - + var contentBytes = await content.ReadAllAsync(expected, cancellationToken); using (var contentDuplicate = new MemoryStream(contentBytes)) { + // Push the manifest when ReferrerState is Unknown or NotSupported await InternalPushAsync(expected, contentDuplicate, reference, cancellationToken).ConfigureAwait(false); } if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) { + // Early exit when the registry supports Referrers API + // No need to index referrers list return; } using (var contentDuplicate = new MemoryStream(contentBytes)) { - await ProcessReferrersAndPushIndex(expected, contentDuplicate); + // 1. Index the referrers list using referrers tag schema when manifest contains a subject field + // And the ReferrerState is not supported + // 2. Or do nothing when the manifest does not contain a subject field when ReferrerState is not supported/unknown + await ProcessReferrersAndPushIndex(expected, contentDuplicate, cancellationToken); } break; default: @@ -239,13 +246,16 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, } Repository.ReferrerState = Referrers.ReferrerState.ReferrerNotSupported; - await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd)); + await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken); } /// /// UpdateReferrersIndex updates the referrers index for a given subject by applying the specified referrer changes. /// If the referrers index is updated, the new index is pushed to the repository. If referrers /// garbage collection is not skipped, the old index is deleted. + /// References: + /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#pushing-manifests-with-subject + /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests /// /// /// @@ -256,25 +266,36 @@ private async Task UpdateReferrersIndex(Descriptor subject, { try { + // 1. pull the original referrers index list using referrers tag schema var referrersTag = Referrers.BuildReferrersTag(subject); - var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag); + var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag, cancellationToken); + + // 2. apply the referrer change to referrers list var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + // 3. push the updated referrers list using referrers tag schema if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGc) { + // push a new index in either case: + // 1. the referrers list has been updated with a non-zero size + // 2. OR the updated referrers list is empty but referrers GC + // is skipped, in this case an empty index should still be pushed + // as the old index won't get deleted var (indexDesc, indexContent) = Index.GenerateIndex(updatedReferrers); using (var content = new MemoryStream(indexContent)) { await InternalPushAsync(indexDesc, content, referrersTag, cancellationToken).ConfigureAwait(false); } } - + if (repository.Options.SkipReferrersGc || Descriptor.IsEmptyOrNull(oldDesc)) { + // Skip the delete process if SkipReferrersGc is set to true or the old Descriptor is empty or null return; } - + + // 4. delete the dangling original referrers index, if applicable await DeleteAsync(oldDesc, cancellationToken).ConfigureAwait(false); } catch (NoReferrerUpdateException) @@ -296,7 +317,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, { try { - var (desc, content) = await FetchAsync(referrersTag); + var (desc, content) = await FetchAsync(referrersTag, cancellationToken); var index = JsonSerializer.Deserialize(content); if (index == null) { @@ -318,8 +339,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, /// /// /// - private async Task InternalPushAsync(Descriptor expected, Stream stream, string contentReference, - CancellationToken cancellationToken) + private async Task InternalPushAsync(Descriptor expected, Stream stream, string contentReference, CancellationToken cancellationToken) { var remoteReference = Repository.ParseReference(contentReference); var url = new UriFactory(remoteReference, Repository.Options.PlainHttp).BuildRepositoryManifest(); @@ -333,7 +353,7 @@ private async Task InternalPushAsync(Descriptor expected, Stream stream, string { throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); } - response.CheckOciSubjectHeader(Repository); + response.CheckOCISubjectHeader(Repository); response.VerifyContentDigest(expected.Digest); } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index bbc1a7c..1e851ea 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -37,7 +37,8 @@ internal enum ReferrerOperation internal static string BuildReferrersTag(Descriptor descriptor) { - return Digest.GetAlgorithm(descriptor.Digest) + "-" + Digest.GetRef(descriptor.Digest); + var validatedDigest = Digest.Validate(descriptor.Digest); + return validatedDigest.Substring(0, validatedDigest.IndexOf(':')) + "-" + validatedDigest.Substring(validatedDigest.IndexOf(':') + 1); } /// diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index 3f12ba1..f992b3a 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -22,6 +22,23 @@ namespace OrasProject.Oras.Tests.Remote; public class ReferrersTest { + [Fact] + public void BuildReferrersTag_ShouldReturnReferrersTagSuccessfully() + { + var desc = RandomDescriptor(); + var index = desc.Digest.IndexOf(':'); + var expected = desc.Digest.Substring(0, index) + "-" + desc.Digest.Substring(index + 1); + Assert.Equal(expected, Referrers.BuildReferrersTag(desc)); + } + + [Fact] + public void BuildReferrersTag_ShouldThrowInvalidDigestException() + { + var desc = RandomDescriptor(); + desc.Digest = "sha123321"; + Assert.Throws(() => Referrers.BuildReferrersTag(desc)); + } + [Fact] public void ApplyReferrerChanges_ShouldAddNewReferrers() { From c26ccb6dbdd0b6546aa25b795abe41518856ffd2 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 22 Nov 2024 00:40:40 +1100 Subject: [PATCH 10/24] add SetReferrersSupportLevel func and unit tests Signed-off-by: Patrick Pan --- ...eferrersSupportLevelAlreadySetException.cs | 20 ++++++++++ .../Remote/HttpResponseMessageExtensions.cs | 2 +- .../Registry/Remote/ManifestStore.cs | 8 ++-- .../Registry/Remote/Referrers.cs | 8 ++-- .../Registry/Remote/Repository.cs | 31 ++++++++++++++- .../Exceptions/ExceptionTest.cs | 8 ++++ .../Remote/ManifestStoreTest.cs | 24 ++++++------ .../Remote/RepositoryTest.cs | 38 +++++++++++++++++++ 8 files changed, 116 insertions(+), 23 deletions(-) create mode 100644 src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs diff --git a/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs b/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs new file mode 100644 index 0000000..5a1662d --- /dev/null +++ b/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs @@ -0,0 +1,20 @@ +using System; + +namespace OrasProject.Oras.Exceptions; + +public class ReferrersSupportLevelAlreadySetException : Exception +{ + public ReferrersSupportLevelAlreadySetException() + { + } + + public ReferrersSupportLevelAlreadySetException(string? message) + : base(message) + { + } + + public ReferrersSupportLevelAlreadySetException(string? message, Exception? inner) + : base(message, inner) + { + } +} diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 00473d9..354d3ac 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -112,7 +112,7 @@ public static void CheckOCISubjectHeader(this HttpResponseMessage response, Repo if (response.Headers.TryGetValues("OCI-Subject", out var values)) { // Set it to ReferrerSupported when the response header contains OCI-Subject - repository.ReferrerState = Referrers.ReferrerState.ReferrerSupported; + repository.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); } // If the "OCI-Subject" header is NOT set, it means that either the manifest diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 7dbb810..460244a 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -178,7 +178,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, st { case MediaType.ImageManifest: case MediaType.ImageIndex: - if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) + if (Repository.ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersSupported) { // Push the manifest straightaway when the registry supports referrers API await InternalPushAsync(expected, content, reference, cancellationToken).ConfigureAwait(false); @@ -191,7 +191,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, st // Push the manifest when ReferrerState is Unknown or NotSupported await InternalPushAsync(expected, contentDuplicate, reference, cancellationToken).ConfigureAwait(false); } - if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) + if (Repository.ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersSupported) { // Early exit when the registry supports Referrers API // No need to index referrers list @@ -244,8 +244,8 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, default: return; } - - Repository.ReferrerState = Referrers.ReferrerState.ReferrerNotSupported; + + Repository.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersNotSupported); await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken); } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 1e851ea..9e0aff2 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -20,11 +20,11 @@ namespace OrasProject.Oras.Registry.Remote; public class Referrers { - internal enum ReferrerState + internal enum ReferrersSupportLevel { - ReferrerUnknown, - ReferrerSupported, - ReferrerNotSupported + ReferrersUnknown, + ReferrersSupported, + ReferrersNotSupported } internal record ReferrerChange(Descriptor Referrer, ReferrerOperation ReferrerOperation); diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 5d7104e..44f6f20 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -46,8 +46,9 @@ public class Repository : IRepository public IManifestStore Manifests => new ManifestStore(this); public RepositoryOptions Options => _opts; - internal Referrers.ReferrerState ReferrerState { get; set; } = Referrers.ReferrerState.ReferrerUnknown; - + + internal Referrers.ReferrersSupportLevel ReferrersSupportLevel { get; set; } = Referrers.ReferrersSupportLevel.ReferrersUnknown; + internal static readonly string[] DefaultManifestMediaTypes = [ Docker.MediaType.Manifest, @@ -86,6 +87,32 @@ public Repository(RepositoryOptions options) _opts = options; } + /// + /// SetReferrerSupportLevel indicates the Referrers API support level of the remote repository. + /// + /// SetReferrerSupportLevel is valid only when it is called for the first time. + /// SetReferrerSupportLevel returns ReferrersSupportLevelAlreadySetException if the + /// Referrers API support level has been already set. + /// - When the level is set to ReferrersSupported, the Referrers() function will always + /// request the Referrers API. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers + /// - When the level is set to ReferrersNotSupported, the Referrers() function will always + /// request the Referrers Tag. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema + /// - When the capability is not set, the Referrers() function will automatically + /// determine which API to use. + /// + /// + /// + internal void SetReferrerSupportLevel(Referrers.ReferrersSupportLevel level) + { + if (ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersUnknown) + { + ReferrersSupportLevel = level; + } else if (ReferrersSupportLevel != level) + { + throw new ReferrersSupportLevelAlreadySetException($"current support level: {ReferrersSupportLevel}, latest support level: {level}"); + } + } + /// /// FetchAsync fetches the content identified by the descriptor. /// diff --git a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs index 84ee0e6..ef92d28 100644 --- a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs +++ b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs @@ -57,4 +57,12 @@ public async Task NoReferrerUpdateException() await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update")); await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update", null)); } + + [Fact] + public async Task ReferrersSupportLevelAlreadySetException() + { + await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException()); + await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException("Referrers support level has already been set")); + await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException("Referrers support level has already been set", null)); + } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index 2e21eef..df2bf4b 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -167,14 +167,14 @@ public async Task ManifestStore_PushAsyncWithoutSubject() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); Assert.Equal(expectedManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); await store.PushAsync(expectedConfigDesc, new MemoryStream(expectedConfigBytes), cancellationToken); Assert.Equal(expectedConfigBytes, receivedManifest); - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); } @@ -254,15 +254,15 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() var store = new ManifestStore(repo); // first push with image manifest - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); Assert.Equal(expectedManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); // second push with index manifest await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); Assert.Equal(expectedIndexManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); } @@ -377,18 +377,18 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() var store = new ManifestStore(repo); // First push with referrer tag schema - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); await store.PushAsync(firstExpectedManifestDesc, new MemoryStream(firstExpectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); Assert.Equal(firstExpectedManifestBytes, receivedManifestContent); Assert.True(oldIndexDeleted); Assert.Equal(firstExpectedIndexReferrersBytes, receivedIndexContent); // Second push with referrer tag schema - Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); await store.PushAsync(secondExpectedManifestDesc, new MemoryStream(secondExpectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); Assert.Equal(secondExpectedManifestBytes, receivedManifestContent); Assert.True(firstIndexDeleted); Assert.Equal(secondExpectedIndexReferrersBytes, receivedIndexContent); @@ -460,9 +460,9 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupportedWitho var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); Assert.Equal(expectedIndexManifestBytes, receivedIndexManifestContent); Assert.Equal(expectedIndexReferrersBytes, receivedIndexReferrersContent); } diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 11dc730..5efe889 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -2544,4 +2544,42 @@ public async Task Repository_MountAsync_Fallback_GetContentError() Assert.Equal(testErr, ex); Assert.Equal("post ", sequence); } + + [Fact] + public void SetReferrersSupportLevel_ShouldSet_WhenInitiallyUnknown() + { + var repo = new Repository("localhost:5000/test2"); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + } + + [Fact] + public void SetReferrersSupportLevel_ShouldThrowException_WhenChangingAfterSet() + { + var repo = new Repository("localhost:5000/test2"); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + + var exception = Assert.Throws(() => + repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersNotSupported) + ); + + Assert.Equal("current support level: ReferrersSupported, latest support level: ReferrersNotSupported", exception.Message); + } + + [Fact] + public void SetReferrersSupportLevel_ShouldNotThrowException_WhenSettingSameValue() + { + var repo = new Repository("localhost:5000/test2"); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + + var exception = Record.Exception(() => repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported)); + Assert.Null(exception); + } + + } From ddbf048ac8b72712755840e70a053d798bb6181d Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 22 Nov 2024 16:06:20 +1100 Subject: [PATCH 11/24] remove NoReferrerUpdateException and update tests accordingly Signed-off-by: Patrick Pan --- .../Exceptions/NoReferrerUpdateException.cs | 37 -------- .../Registry/Remote/ManifestStore.cs | 56 ++++++------ .../Registry/Remote/Referrers.cs | 87 ++++++++++++------- .../Registry/Remote/RepositoryOptions.cs | 7 +- .../Exceptions/ExceptionTest.cs | 8 -- .../Remote/ManifestStoreTest.cs | 86 ++++++++++++++++++ .../Remote/ReferrersTest.cs | 27 +++--- 7 files changed, 186 insertions(+), 122 deletions(-) delete mode 100644 src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs diff --git a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs deleted file mode 100644 index 54ff5d9..0000000 --- a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright The ORAS Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; - -namespace OrasProject.Oras.Exceptions; - - -/// -/// NoReferrerUpdateException is thrown when no referrer update is needed. -/// -public class NoReferrerUpdateException : Exception -{ - public NoReferrerUpdateException() - { - } - - public NoReferrerUpdateException(string message) - : base(message) - { - } - - public NoReferrerUpdateException(string message, Exception? innerException) - : base(message, innerException) - { - } -} diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 460244a..c90799c 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -264,44 +264,38 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, private async Task UpdateReferrersIndex(Descriptor subject, Referrers.ReferrerChange referrerChange, CancellationToken cancellationToken = default) { - try - { - // 1. pull the original referrers index list using referrers tag schema - var referrersTag = Referrers.BuildReferrersTag(subject); - var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag, cancellationToken); - - // 2. apply the referrer change to referrers list - var updatedReferrers = - Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + // 1. pull the original referrers index list using referrers tag schema + var referrersTag = Referrers.BuildReferrersTag(subject); + var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag, cancellationToken); + + // 2. apply the referrer change to referrers list + var (updatedReferrers, updateRequired) = + Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + if (!updateRequired) return; - // 3. push the updated referrers list using referrers tag schema - if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGc) - { - // push a new index in either case: - // 1. the referrers list has been updated with a non-zero size - // 2. OR the updated referrers list is empty but referrers GC - // is skipped, in this case an empty index should still be pushed - // as the old index won't get deleted - var (indexDesc, indexContent) = Index.GenerateIndex(updatedReferrers); - using (var content = new MemoryStream(indexContent)) - { - await InternalPushAsync(indexDesc, content, referrersTag, cancellationToken).ConfigureAwait(false); - } - } - - if (repository.Options.SkipReferrersGc || Descriptor.IsEmptyOrNull(oldDesc)) + // 3. push the updated referrers list using referrers tag schema + if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGC) + { + // push a new index in either case: + // 1. the referrers list has been updated with a non-zero size + // 2. OR the updated referrers list is empty but referrers GC + // is skipped, in this case an empty index should still be pushed + // as the old index won't get deleted + var (indexDesc, indexContent) = Index.GenerateIndex(updatedReferrers); + using (var content = new MemoryStream(indexContent)) { - // Skip the delete process if SkipReferrersGc is set to true or the old Descriptor is empty or null - return; + await InternalPushAsync(indexDesc, content, referrersTag, cancellationToken).ConfigureAwait(false); } - - // 4. delete the dangling original referrers index, if applicable - await DeleteAsync(oldDesc, cancellationToken).ConfigureAwait(false); } - catch (NoReferrerUpdateException) + + if (repository.Options.SkipReferrersGC || Descriptor.IsEmptyOrNull(oldDesc)) { + // Skip the delete process if SkipReferrersGC is set to true or the old Descriptor is empty or null return; } + + // 4. delete the dangling original referrers index, if applicable + await DeleteAsync(oldDesc, cancellationToken).ConfigureAwait(false); } /// diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 9e0aff2..061127c 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -48,95 +48,118 @@ internal static string BuildReferrersTag(Descriptor descriptor) /// /// /// - /// The updated referrers list - internal static IList ApplyReferrerChanges(IList oldReferrers, ReferrerChange referrerChange) + /// The updated referrers list, updateRequired + internal static (IList, bool) ApplyReferrerChanges(IList oldReferrers, ReferrerChange referrerChange) { if (oldReferrers == null || referrerChange == null) { - throw new NoReferrerUpdateException("referrerChange or oldReferrers is null in this request"); + return (new List(), false); } + // updatedReferrers is a list to store the updated referrers var updatedReferrers = new List(); - var referrerToIndex = new Dictionary(); + // referrerIndexMap is a Dictionary to store referrer as the key + // and index(int) in the updatedReferrers list as the value + var referrerIndexMap = new Dictionary(); var updateRequired = false; foreach (var oldReferrer in oldReferrers) { if (Descriptor.IsEmptyOrNull(oldReferrer)) { + // Skip any empty or null referrers updateRequired = true; continue; } var basicDesc = oldReferrer.BasicDescriptor; - if (referrerToIndex.ContainsKey(basicDesc)) + if (referrerIndexMap.ContainsKey(basicDesc)) { + // Skip any duplicate referrers updateRequired = true; continue; } + // Update the updatedReferrers list + // Add referrer index in the referrerIndexMap updatedReferrers.Add(oldReferrer); - referrerToIndex[basicDesc] = updatedReferrers.Count - 1; + referrerIndexMap[basicDesc] = updatedReferrers.Count - 1; } - - + if (!Descriptor.IsEmptyOrNull(referrerChange.Referrer)) { var basicDesc = referrerChange.Referrer.BasicDescriptor; switch (referrerChange.ReferrerOperation) { case ReferrerOperation.ReferrerAdd: - if (!referrerToIndex.ContainsKey(basicDesc)) + if (!referrerIndexMap.ContainsKey(basicDesc)) { + // Add the new referrer only when it has not already existed in the referrerIndexMap updatedReferrers.Add(referrerChange.Referrer); - referrerToIndex[basicDesc] = updatedReferrers.Count - 1; + referrerIndexMap[basicDesc] = updatedReferrers.Count - 1; } - + break; - case ReferrerOperation.ReferrerDelete: - if (referrerToIndex.TryGetValue(basicDesc, out var index)) + if (referrerIndexMap.TryGetValue(basicDesc, out var index)) { + // Delete the referrer only when it existed in the referrerIndexMap updatedReferrers[index] = Descriptor.EmptyDescriptor(); - referrerToIndex.Remove(basicDesc); + referrerIndexMap.Remove(basicDesc); } - break; default: break; } } - if (!updateRequired && referrerToIndex.Count == oldReferrers.Count) + // Skip unnecessary update + if (!updateRequired && referrerIndexMap.Count == oldReferrers.Count) { + // Check for any new referrers in the referrerIndexMap that are not present in the oldReferrers list foreach (var oldReferrer in oldReferrers) { var basicDesc = oldReferrer.BasicDescriptor; - if (!referrerToIndex.ContainsKey(basicDesc)) updateRequired = true; + if (!referrerIndexMap.ContainsKey(basicDesc)) + { + updateRequired = true; + break; + } } - if (!updateRequired) throw new NoReferrerUpdateException("no referrer update in this request"); + if (!updateRequired) + { + return (updatedReferrers, false); + } } - RemoveEmptyDescriptors(updatedReferrers, referrerToIndex.Count); - return updatedReferrers; + RemoveEmptyDescriptors(updatedReferrers, referrerIndexMap.Count); + return (updatedReferrers, true); } /// - /// RemoveEmptyDescriptors removes any empty or null descriptors from the provided list of referrers, ensuring that only non-empty - /// descriptors remain in the list. It optimizes the list by shifting valid descriptors forward and trimming - /// the remaining elements at the end. The list is truncated to only contain non-empty descriptors up to the specified count. + /// RemoveEmptyDescriptors removes any empty or null descriptors from the provided list of descriptors, + /// ensuring that only non-empty descriptors remain in the list. + /// It optimizes the list by shifting valid descriptors forward and trimming the remaining elements at the end. + /// The list is truncated to only contain non-empty descriptors up to the specified count. /// - /// - /// - internal static void RemoveEmptyDescriptors(List updatedReferrers, int numNonEmptyReferrers) + /// + /// + internal static void RemoveEmptyDescriptors(List descriptors, int numNonEmptyDescriptors) { var lastEmptyIndex = 0; - for (var i = 0; i < updatedReferrers.Count; ++i) + for (var i = 0; i < descriptors.Count; ++i) { - if (Descriptor.IsEmptyOrNull(updatedReferrers[i])) continue; - - if (i > lastEmptyIndex) updatedReferrers[lastEmptyIndex] = updatedReferrers[i]; + if (Descriptor.IsEmptyOrNull(descriptors[i])) continue; + if (i > lastEmptyIndex) + { + // Move the descriptor at index i to lastEmptyIndex + descriptors[lastEmptyIndex] = descriptors[i]; + } ++lastEmptyIndex; - if (lastEmptyIndex == numNonEmptyReferrers) break; + if (lastEmptyIndex == numNonEmptyDescriptors) + { + // Break the loop when lastEmptyIndex reaches the number of Non-Empty descriptors + break; + } } - updatedReferrers.RemoveRange(lastEmptyIndex, updatedReferrers.Count - lastEmptyIndex); + descriptors.RemoveRange(lastEmptyIndex, descriptors.Count - lastEmptyIndex); } } diff --git a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs index 1d7b45a..4742f9d 100644 --- a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs +++ b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs @@ -51,10 +51,13 @@ public struct RepositoryOptions /// public int TagListPageSize { get; set; } - // SkipReferrersGc specifies whether to delete the dangling referrers + // SkipReferrersGC specifies whether to delete the dangling referrers // index when referrers tag schema is utilized. // - If false, the old referrers index will be deleted after the new one is successfully uploaded. // - If true, the old referrers index is kept. // By default, it is disabled (set to false). See also: - public bool SkipReferrersGc { get; set; } + // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema + // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#pushing-manifests-with-subject + // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests + public bool SkipReferrersGC { get; set; } } diff --git a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs index ef92d28..13888f3 100644 --- a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs +++ b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs @@ -50,14 +50,6 @@ public async Task NotFoundException() await Assert.ThrowsAsync(() => throw new NotFoundException("Not found", null)); } - [Fact] - public async Task NoReferrerUpdateException() - { - await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException()); - await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update")); - await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update", null)); - } - [Fact] public async Task ReferrersSupportLevelAlreadySetException() { diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index df2bf4b..350ae65 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -466,4 +466,90 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupportedWitho Assert.Equal(expectedIndexManifestBytes, receivedIndexManifestContent); Assert.Equal(expectedIndexReferrersBytes, receivedIndexReferrersContent); } + + [Fact] + public async Task ManifestStore_PushAsyncWithSubjectAndNoUpdateRequired() + { + var (oldManifest, oldManifestBytes) = RandomManifestWithSubject(); + var oldIndex = new Index() + { + Manifests = new List + { + new () + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(oldManifestBytes), + Size = oldManifestBytes.Length, + ArtifactType = MediaType.ImageManifest, + } + }, + MediaType = MediaType.ImageIndex, + }; + var oldIndexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(oldIndex)); + var oldIndexDesc = new Descriptor() + { + Digest = ComputeSHA256(oldIndexBytes), + MediaType = MediaType.ImageIndex, + Size = oldIndexBytes.Length + }; + + var expectedManifest = oldManifest; + var expectedManifestBytes = oldManifestBytes; + var expectedManifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(oldManifestBytes), + Size = oldManifestBytes.Length, + ArtifactType = MediaType.ImageManifest, + }; + + byte[]? receivedManifestContent = null; + var referrersTag = Referrers.BuildReferrersTag(expectedManifest.Subject); + + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var response = new HttpResponseMessage(); + response.RequestMessage = req; + + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}") + { + if (req.Content?.Headers?.ContentLength != null) + { + var buffer = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buffer, 0); + receivedManifestContent = buffer; + } + response.Headers.Add(_dockerContentDigestHeader, new[] { expectedManifestDesc.Digest }); + response.StatusCode = HttpStatusCode.Created; + return response; + } else if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(oldIndexBytes); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + response.Headers.Add(_dockerContentDigestHeader, new string[] { oldIndexDesc.Digest }); + response.StatusCode = HttpStatusCode.OK; + return response; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(expectedManifestBytes, receivedManifestContent); + } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index f992b3a..5f39fd5 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -62,12 +62,13 @@ public void ApplyReferrerChanges_ShouldAddNewReferrers() Referrers.ReferrerOperation.ReferrerAdd ); - var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Equal(3, updatedReferrers.Count); for (var i = 0; i < updatedReferrers.Count; ++i) { Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); } + Assert.True(updateRequired); } [Fact] @@ -96,12 +97,13 @@ public void ApplyReferrerChanges_ShouldDiscardDuplicateReferrers() Referrers.ReferrerOperation.ReferrerAdd ); - var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Equal(3, updatedReferrers.Count); for (var i = 0; i < updatedReferrers.Count; ++i) { Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); } + Assert.True(updateRequired); } [Fact] @@ -118,10 +120,9 @@ public void ApplyReferrerChanges_ShouldNotAddNewDuplicateReferrers() oldDescriptor1, Referrers.ReferrerOperation.ReferrerAdd ); - - - var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); - Assert.Equal("no referrer update in this request", exception.Message); + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(2, updatedReferrers.Count); + Assert.False(updateRequired); } [Fact] @@ -145,13 +146,13 @@ public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() Referrers.ReferrerOperation.ReferrerAdd ); - var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); - + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Single(updatedReferrers); for (var i = 0; i < updatedReferrers.Count; ++i) { Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); } + Assert.True(updateRequired); } [Fact] @@ -160,8 +161,9 @@ public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreNull() IList oldReferrers = null; Referrers.ReferrerChange referrerChange = null; - var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); - Assert.Equal("referrerChange or oldReferrers is null in this request", exception.Message); + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Empty(updatedReferrers); + Assert.False(updateRequired); } [Fact] @@ -170,8 +172,9 @@ public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreEmpty() var oldReferrers = new List(); var referrerChange = new Referrers.ReferrerChange(Descriptor.EmptyDescriptor(), Referrers.ReferrerOperation.ReferrerAdd); - var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); - Assert.Equal("no referrer update in this request", exception.Message); + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Empty(updatedReferrers); + Assert.False(updateRequired); } [Fact] From d6e849945d6e48c9364f6edfe4a3f1842e2a8d7f Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 25 Nov 2024 10:32:51 +1100 Subject: [PATCH 12/24] add Index constructor Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Content/Digest.cs | 3 ++- src/OrasProject.Oras/Oci/Index.cs | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/OrasProject.Oras/Content/Digest.cs b/src/OrasProject.Oras/Content/Digest.cs index 15febfb..2e8c035 100644 --- a/src/OrasProject.Oras/Content/Digest.cs +++ b/src/OrasProject.Oras/Content/Digest.cs @@ -46,6 +46,7 @@ internal static string Validate(string? digest) return digest; } + /// /// Generates a SHA-256 digest from a byte array. /// @@ -58,4 +59,4 @@ internal static string ComputeSHA256(byte[] content) var output = $"sha256:{BitConverter.ToString(hash).Replace("-", "")}"; return output.ToLower(); } -} +} \ No newline at end of file diff --git a/src/OrasProject.Oras/Oci/Index.cs b/src/OrasProject.Oras/Oci/Index.cs index dc81f8d..918588d 100644 --- a/src/OrasProject.Oras/Oci/Index.cs +++ b/src/OrasProject.Oras/Oci/Index.cs @@ -12,6 +12,7 @@ // limitations under the License. using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -43,14 +44,19 @@ public class Index : Versioned [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public IDictionary? Annotations { get; set; } + public Index() {} + + [SetsRequiredMembers] + public Index(IList manifests) + { + Manifests = manifests; + MediaType = Oci.MediaType.ImageIndex; + SchemaVersion = 2; + } + internal static (Descriptor, byte[]) GenerateIndex(IList manifests) { - var index = new Index() - { - Manifests = manifests, - MediaType = Oci.MediaType.ImageIndex, - SchemaVersion = 2 - }; + var index = new Index(manifests); var indexContent = JsonSerializer.SerializeToUtf8Bytes(index); return (Descriptor.Create(indexContent, Oci.MediaType.ImageIndex), indexContent); } From cf431f077f91b2eb1744be0f42d072f48de31c8b Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 26 Nov 2024 13:55:50 +1100 Subject: [PATCH 13/24] add license header Signed-off-by: Patrick Pan --- .../ReferrersSupportLevelAlreadySetException.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs b/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs index 5a1662d..87ffd00 100644 --- a/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs +++ b/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs @@ -1,4 +1,17 @@ -using System; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; namespace OrasProject.Oras.Exceptions; From 856c0efef880356750cda114a6e8bad71e511500 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 29 Nov 2024 15:34:14 +1100 Subject: [PATCH 14/24] add lock on SetReferrerState Signed-off-by: Patrick Pan --- ...s => ReferrersStateAlreadySetException.cs} | 8 ++-- .../Remote/HttpResponseMessageExtensions.cs | 2 +- .../Registry/Remote/ManifestStore.cs | 6 +-- .../Registry/Remote/Referrers.cs | 28 ++++++++++---- .../Registry/Remote/Repository.cs | 38 ++++++++++--------- .../Exceptions/ExceptionTest.cs | 6 +-- .../Remote/ManifestStoreTest.cs | 28 +++++++------- .../Remote/ReferrersTest.cs | 11 ------ .../Remote/RepositoryTest.cs | 34 ++++++++--------- 9 files changed, 82 insertions(+), 79 deletions(-) rename src/OrasProject.Oras/Exceptions/{ReferrersSupportLevelAlreadySetException.cs => ReferrersStateAlreadySetException.cs} (72%) diff --git a/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs b/src/OrasProject.Oras/Exceptions/ReferrersStateAlreadySetException.cs similarity index 72% rename from src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs rename to src/OrasProject.Oras/Exceptions/ReferrersStateAlreadySetException.cs index 87ffd00..4c0b7ad 100644 --- a/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs +++ b/src/OrasProject.Oras/Exceptions/ReferrersStateAlreadySetException.cs @@ -15,18 +15,18 @@ namespace OrasProject.Oras.Exceptions; -public class ReferrersSupportLevelAlreadySetException : Exception +public class ReferrersStateAlreadySetException : Exception { - public ReferrersSupportLevelAlreadySetException() + public ReferrersStateAlreadySetException() { } - public ReferrersSupportLevelAlreadySetException(string? message) + public ReferrersStateAlreadySetException(string? message) : base(message) { } - public ReferrersSupportLevelAlreadySetException(string? message, Exception? inner) + public ReferrersStateAlreadySetException(string? message, Exception? inner) : base(message, inner) { } diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 354d3ac..0e4585c 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -112,7 +112,7 @@ public static void CheckOCISubjectHeader(this HttpResponseMessage response, Repo if (response.Headers.TryGetValues("OCI-Subject", out var values)) { // Set it to ReferrerSupported when the response header contains OCI-Subject - repository.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); + repository.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); } // If the "OCI-Subject" header is NOT set, it means that either the manifest diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 5f55f99..c4675c1 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -177,7 +177,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re { case MediaType.ImageManifest: case MediaType.ImageIndex: - if (Repository.ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersSupported) + if (Repository.ReferrersState == Referrers.ReferrersState.ReferrersSupported) { // Push the manifest straightaway when the registry supports referrers API await DoPushAsync(expected, content, reference, cancellationToken).ConfigureAwait(false); @@ -190,7 +190,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re // Push the manifest when ReferrerState is Unknown or NotSupported await DoPushAsync(expected, contentDuplicate, reference, cancellationToken).ConfigureAwait(false); } - if (Repository.ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersSupported) + if (Repository.ReferrersState == Referrers.ReferrersState.ReferrersSupported) { // Early exit when the registry supports Referrers API // No need to index referrers list @@ -244,7 +244,7 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, return; } - Repository.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersNotSupported); + Repository.SetReferrersState(Referrers.ReferrersState.ReferrersNotSupported); await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken); } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 061127c..369b537 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -20,11 +20,11 @@ namespace OrasProject.Oras.Registry.Remote; public class Referrers { - internal enum ReferrersSupportLevel + internal enum ReferrersState { - ReferrersUnknown, - ReferrersSupported, - ReferrersNotSupported + ReferrersUnknown = 0, + ReferrersSupported = 1, + ReferrersNotSupported = 2 } internal record ReferrerChange(Descriptor Referrer, ReferrerOperation ReferrerOperation); @@ -51,10 +51,6 @@ internal static string BuildReferrersTag(Descriptor descriptor) /// The updated referrers list, updateRequired internal static (IList, bool) ApplyReferrerChanges(IList oldReferrers, ReferrerChange referrerChange) { - if (oldReferrers == null || referrerChange == null) - { - return (new List(), false); - } // updatedReferrers is a list to store the updated referrers var updatedReferrers = new List(); // referrerIndexMap is a Dictionary to store referrer as the key @@ -79,10 +75,24 @@ internal static (IList, bool) ApplyReferrerChanges(IList } // Update the updatedReferrers list // Add referrer index in the referrerIndexMap + + // delete + // ...... + if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerDelete) + { + var toBeDeletedBasicDesc = referrerChange.Referrer.BasicDescriptor; + if (basicDesc == toBeDeletedBasicDesc) + { + updateRequired = true; + continue; + } + } updatedReferrers.Add(oldReferrer); referrerIndexMap[basicDesc] = updatedReferrers.Count - 1; } + // old => 1, 1 + // new => nil if (!Descriptor.IsEmptyOrNull(referrerChange.Referrer)) { var basicDesc = referrerChange.Referrer.BasicDescriptor; @@ -101,6 +111,8 @@ internal static (IList, bool) ApplyReferrerChanges(IList if (referrerIndexMap.TryGetValue(basicDesc, out var index)) { // Delete the referrer only when it existed in the referrerIndexMap + // updatedReferrers.Remove(basicDesc); + updatedReferrers[index] = Descriptor.EmptyDescriptor(); referrerIndexMap.Remove(basicDesc); } diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 44f6f20..613400f 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -47,8 +47,14 @@ public class Repository : IRepository public RepositoryOptions Options => _opts; - internal Referrers.ReferrersSupportLevel ReferrersSupportLevel { get; set; } = Referrers.ReferrersSupportLevel.ReferrersUnknown; - + private int _referrersState = (int) Referrers.ReferrersState.ReferrersUnknown; + + internal Referrers.ReferrersState ReferrersState + { + get => (Referrers.ReferrersState) _referrersState; + private set => _referrersState = (int) value; + } + internal static readonly string[] DefaultManifestMediaTypes = [ Docker.MediaType.Manifest, @@ -88,28 +94,26 @@ public Repository(RepositoryOptions options) } /// - /// SetReferrerSupportLevel indicates the Referrers API support level of the remote repository. + /// SetReferrersState indicates the Referrers API state of the remote repository. /// - /// SetReferrerSupportLevel is valid only when it is called for the first time. - /// SetReferrerSupportLevel returns ReferrersSupportLevelAlreadySetException if the - /// Referrers API support level has been already set. - /// - When the level is set to ReferrersSupported, the Referrers() function will always + /// SetReferrersState is valid only when it is called for the first time. + /// SetReferrersState returns ReferrersStateAlreadySetException if the + /// Referrers API state has been already set. + /// - When the state is set to ReferrersSupported, the Referrers() function will always /// request the Referrers API. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers - /// - When the level is set to ReferrersNotSupported, the Referrers() function will always + /// - When the state is set to ReferrersNotSupported, the Referrers() function will always /// request the Referrers Tag. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema - /// - When the capability is not set, the Referrers() function will automatically + /// - When the state is not set, the Referrers() function will automatically /// determine which API to use. /// - /// - /// - internal void SetReferrerSupportLevel(Referrers.ReferrersSupportLevel level) + /// + /// + internal void SetReferrersState(Referrers.ReferrersState state) { - if (ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersUnknown) - { - ReferrersSupportLevel = level; - } else if (ReferrersSupportLevel != level) + var originalReferrersState = (Referrers.ReferrersState) Interlocked.CompareExchange(ref _referrersState, (int)state, (int)Referrers.ReferrersState.ReferrersUnknown); + if (originalReferrersState != Referrers.ReferrersState.ReferrersUnknown && _referrersState != (int) state) { - throw new ReferrersSupportLevelAlreadySetException($"current support level: {ReferrersSupportLevel}, latest support level: {level}"); + throw new ReferrersStateAlreadySetException($"current referrers state: {ReferrersState}, latest referrers state: {state}"); } } diff --git a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs index 13888f3..d9fb8a5 100644 --- a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs +++ b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs @@ -53,8 +53,8 @@ public async Task NotFoundException() [Fact] public async Task ReferrersSupportLevelAlreadySetException() { - await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException()); - await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException("Referrers support level has already been set")); - await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException("Referrers support level has already been set", null)); + await Assert.ThrowsAsync(() => throw new ReferrersStateAlreadySetException()); + await Assert.ThrowsAsync(() => throw new ReferrersStateAlreadySetException("Referrers state has already been set")); + await Assert.ThrowsAsync(() => throw new ReferrersStateAlreadySetException("Referrers state has already been set", null)); } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index 350ae65..e0721e9 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -167,14 +167,14 @@ public async Task ManifestStore_PushAsyncWithoutSubject() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); Assert.Equal(expectedManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(expectedConfigDesc, new MemoryStream(expectedConfigBytes), cancellationToken); Assert.Equal(expectedConfigBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); } @@ -254,15 +254,15 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() var store = new ManifestStore(repo); // first push with image manifest - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); Assert.Equal(expectedManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); // second push with index manifest await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); Assert.Equal(expectedIndexManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); } @@ -377,18 +377,18 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() var store = new ManifestStore(repo); // First push with referrer tag schema - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(firstExpectedManifestDesc, new MemoryStream(firstExpectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); Assert.Equal(firstExpectedManifestBytes, receivedManifestContent); Assert.True(oldIndexDeleted); Assert.Equal(firstExpectedIndexReferrersBytes, receivedIndexContent); // Second push with referrer tag schema - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); await store.PushAsync(secondExpectedManifestDesc, new MemoryStream(secondExpectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); Assert.Equal(secondExpectedManifestBytes, receivedManifestContent); Assert.True(firstIndexDeleted); Assert.Equal(secondExpectedIndexReferrersBytes, receivedIndexContent); @@ -460,9 +460,9 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupportedWitho var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); Assert.Equal(expectedIndexManifestBytes, receivedIndexManifestContent); Assert.Equal(expectedIndexReferrersBytes, receivedIndexReferrersContent); } @@ -547,9 +547,9 @@ public async Task ManifestStore_PushAsyncWithSubjectAndNoUpdateRequired() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); Assert.Equal(expectedManifestBytes, receivedManifestContent); } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index 5f39fd5..215d5a8 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -155,17 +155,6 @@ public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() Assert.True(updateRequired); } - [Fact] - public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreNull() - { - IList oldReferrers = null; - Referrers.ReferrerChange referrerChange = null; - - var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); - Assert.Empty(updatedReferrers); - Assert.False(updateRequired); - } - [Fact] public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreEmpty() { diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 5efe889..6aae7e1 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -2546,40 +2546,38 @@ public async Task Repository_MountAsync_Fallback_GetContentError() } [Fact] - public void SetReferrersSupportLevel_ShouldSet_WhenInitiallyUnknown() + public void SetReferrersState_ShouldSet_WhenInitiallyUnknown() { var repo = new Repository("localhost:5000/test2"); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); - repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); } [Fact] - public void SetReferrersSupportLevel_ShouldThrowException_WhenChangingAfterSet() + public void SetReferrersState_ShouldThrowException_WhenChangingAfterSet() { var repo = new Repository("localhost:5000/test2"); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); - repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); - var exception = Assert.Throws(() => - repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersNotSupported) + var exception = Assert.Throws(() => + repo.SetReferrersState(Referrers.ReferrersState.ReferrersNotSupported) ); - Assert.Equal("current support level: ReferrersSupported, latest support level: ReferrersNotSupported", exception.Message); + Assert.Equal("current referrers state: ReferrersSupported, latest referrers state: ReferrersNotSupported", exception.Message); } [Fact] - public void SetReferrersSupportLevel_ShouldNotThrowException_WhenSettingSameValue() + public void SetReferrersState_ShouldNotThrowException_WhenSettingSameValue() { var repo = new Repository("localhost:5000/test2"); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); - repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); - var exception = Record.Exception(() => repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported)); + var exception = Record.Exception(() => repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported)); Assert.Null(exception); } - - } From fcb121e207e7fc7802e79c7d495b3d9bba99fbad Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 29 Nov 2024 16:50:05 +1100 Subject: [PATCH 15/24] simplify ApplyReferrerChanges Signed-off-by: Patrick Pan --- .../Registry/Remote/Referrers.cs | 93 ++------- .../Remote/ReferrersTest.cs | 185 ++++++++++-------- 2 files changed, 122 insertions(+), 156 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 369b537..6d15a26 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -12,6 +12,7 @@ // limitations under the License. using System.Collections.Generic; +using System.Linq; using OrasProject.Oras.Content; using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; @@ -53,9 +54,8 @@ internal static (IList, bool) ApplyReferrerChanges(IList { // updatedReferrers is a list to store the updated referrers var updatedReferrers = new List(); - // referrerIndexMap is a Dictionary to store referrer as the key - // and index(int) in the updatedReferrers list as the value - var referrerIndexMap = new Dictionary(); + // updatedReferrersSet is a HashSet to store unique referrers + var updatedReferrersSet = new HashSet(); var updateRequired = false; foreach (var oldReferrer in oldReferrers) @@ -67,69 +67,45 @@ internal static (IList, bool) ApplyReferrerChanges(IList continue; } var basicDesc = oldReferrer.BasicDescriptor; - if (referrerIndexMap.ContainsKey(basicDesc)) + if (updatedReferrersSet.Contains(basicDesc)) { // Skip any duplicate referrers updateRequired = true; continue; } // Update the updatedReferrers list - // Add referrer index in the referrerIndexMap - - // delete - // ...... - if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerDelete) + // Add referrer index in the updatedReferrersSet + if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerDelete && Descriptor.Equals(basicDesc, referrerChange.Referrer.BasicDescriptor)) { - var toBeDeletedBasicDesc = referrerChange.Referrer.BasicDescriptor; - if (basicDesc == toBeDeletedBasicDesc) - { - updateRequired = true; - continue; - } + updateRequired = true; + continue; } updatedReferrers.Add(oldReferrer); - referrerIndexMap[basicDesc] = updatedReferrers.Count - 1; + updatedReferrersSet.Add(basicDesc); } - // old => 1, 1 - // new => nil if (!Descriptor.IsEmptyOrNull(referrerChange.Referrer)) { var basicDesc = referrerChange.Referrer.BasicDescriptor; - switch (referrerChange.ReferrerOperation) + if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerAdd) { - case ReferrerOperation.ReferrerAdd: - if (!referrerIndexMap.ContainsKey(basicDesc)) - { - // Add the new referrer only when it has not already existed in the referrerIndexMap - updatedReferrers.Add(referrerChange.Referrer); - referrerIndexMap[basicDesc] = updatedReferrers.Count - 1; - } - - break; - case ReferrerOperation.ReferrerDelete: - if (referrerIndexMap.TryGetValue(basicDesc, out var index)) - { - // Delete the referrer only when it existed in the referrerIndexMap - // updatedReferrers.Remove(basicDesc); - - updatedReferrers[index] = Descriptor.EmptyDescriptor(); - referrerIndexMap.Remove(basicDesc); - } - break; - default: - break; + if (!updatedReferrersSet.Contains(basicDesc)) + { + // Add the new referrer only when it has not already existed in the updatedReferrersSet + updatedReferrers.Add(referrerChange.Referrer); + updatedReferrersSet.Add(basicDesc); + } } } // Skip unnecessary update - if (!updateRequired && referrerIndexMap.Count == oldReferrers.Count) + if (!updateRequired && updatedReferrersSet.Count == oldReferrers.Count) { - // Check for any new referrers in the referrerIndexMap that are not present in the oldReferrers list + // Check for any new referrers in the updatedReferrersSet that are not present in the oldReferrers list foreach (var oldReferrer in oldReferrers) { var basicDesc = oldReferrer.BasicDescriptor; - if (!referrerIndexMap.ContainsKey(basicDesc)) + if (!updatedReferrersSet.Contains(basicDesc)) { updateRequired = true; break; @@ -141,37 +117,6 @@ internal static (IList, bool) ApplyReferrerChanges(IList return (updatedReferrers, false); } } - - RemoveEmptyDescriptors(updatedReferrers, referrerIndexMap.Count); return (updatedReferrers, true); } - - /// - /// RemoveEmptyDescriptors removes any empty or null descriptors from the provided list of descriptors, - /// ensuring that only non-empty descriptors remain in the list. - /// It optimizes the list by shifting valid descriptors forward and trimming the remaining elements at the end. - /// The list is truncated to only contain non-empty descriptors up to the specified count. - /// - /// - /// - internal static void RemoveEmptyDescriptors(List descriptors, int numNonEmptyDescriptors) - { - var lastEmptyIndex = 0; - for (var i = 0; i < descriptors.Count; ++i) - { - if (Descriptor.IsEmptyOrNull(descriptors[i])) continue; - if (i > lastEmptyIndex) - { - // Move the descriptor at index i to lastEmptyIndex - descriptors[lastEmptyIndex] = descriptors[i]; - } - ++lastEmptyIndex; - if (lastEmptyIndex == numNonEmptyDescriptors) - { - // Break the loop when lastEmptyIndex reaches the number of Non-Empty descriptors - break; - } - } - descriptors.RemoveRange(lastEmptyIndex, descriptors.Count - lastEmptyIndex); - } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index 215d5a8..5db44fd 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -71,6 +71,108 @@ public void ApplyReferrerChanges_ShouldAddNewReferrers() Assert.True(updateRequired); } + [Fact] + public void ApplyReferrerChanges_ShouldDeleteReferrers() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var oldDescriptor3 = RandomDescriptor(); + + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + oldDescriptor3 + }; + + var expectedReferrers = new List + { + oldDescriptor1, + oldDescriptor3 + }; + var referrerChange = new Referrers.ReferrerChange( + oldDescriptor2, + Referrers.ReferrerOperation.ReferrerDelete + ); + + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(2, updatedReferrers.Count); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + Assert.True(updateRequired); + } + + + [Fact] + public void ApplyReferrerChanges_ShouldDeleteReferrersWithDuplicates() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var oldDescriptor3 = RandomDescriptor(); + + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + oldDescriptor3, + oldDescriptor2, + oldDescriptor2, + oldDescriptor3, + }; + + var expectedReferrers = new List + { + oldDescriptor1, + oldDescriptor2 + }; + var referrerChange = new Referrers.ReferrerChange( + oldDescriptor3, + Referrers.ReferrerOperation.ReferrerDelete + ); + + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(2, updatedReferrers.Count); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + Assert.True(updateRequired); + } + + [Fact] + public void ApplyReferrerChanges_ShouldNotDeleteReferrersWhenNoUpdateRequired() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var oldDescriptor3 = RandomDescriptor(); + + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + }; + + var expectedReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + }; + var referrerChange = new Referrers.ReferrerChange( + oldDescriptor3, + Referrers.ReferrerOperation.ReferrerDelete + ); + + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(2, updatedReferrers.Count); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + Assert.False(updateRequired); + } + [Fact] public void ApplyReferrerChanges_ShouldDiscardDuplicateReferrers() { @@ -156,7 +258,7 @@ public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() } [Fact] - public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreEmpty() + public void ApplyReferrerChanges_NoUpdateWhenOldAndNewReferrersAreEmpty() { var oldReferrers = new List(); var referrerChange = new Referrers.ReferrerChange(Descriptor.EmptyDescriptor(), Referrers.ReferrerOperation.ReferrerAdd); @@ -165,85 +267,4 @@ public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreEmpty() Assert.Empty(updatedReferrers); Assert.False(updateRequired); } - - [Fact] - public void RemoveEmptyDescriptors_ShouldRemoveEmptyDescriptors() - { - var randomDescriptor1 = RandomDescriptor(); - var randomDescriptor2 = RandomDescriptor(); - var randomDescriptor3 = RandomDescriptor(); - var randomDescriptor4 = RandomDescriptor(); - var descriptors = new List - { - Descriptor.EmptyDescriptor(), - randomDescriptor1, - Descriptor.EmptyDescriptor(), - randomDescriptor2, - Descriptor.EmptyDescriptor(), - Descriptor.EmptyDescriptor(), - randomDescriptor3, - randomDescriptor4, - }; - - var expectedDescriptors = new List - { - randomDescriptor1, - randomDescriptor2, - randomDescriptor3, - randomDescriptor4 - }; - Referrers.RemoveEmptyDescriptors(descriptors, 4); - - Assert.Equal(4, descriptors.Count); - Assert.DoesNotContain(Descriptor.EmptyDescriptor(), descriptors); - for (var i = 0; i < descriptors.Count; ++i) - { - Assert.True(AreDescriptorsEqual(descriptors[i], expectedDescriptors[i])); - } - } - - [Fact] - public void RemoveEmptyDescriptors_ShouldReturnAllNonEmptyDescriptors() - { - var randomDescriptor1 = RandomDescriptor(); - var randomDescriptor2 = RandomDescriptor(); - var randomDescriptor3 = RandomDescriptor(); - var randomDescriptor4 = RandomDescriptor(); - var descriptors = new List - { - randomDescriptor1, - randomDescriptor2, - randomDescriptor3, - randomDescriptor4, - }; - - var expectedDescriptors = new List - { - randomDescriptor1, - randomDescriptor2, - randomDescriptor3, - randomDescriptor4 - }; - Referrers.RemoveEmptyDescriptors(descriptors, 4); - Assert.Equal(4, descriptors.Count); - for (var i = 0; i < descriptors.Count; ++i) - { - Assert.True(AreDescriptorsEqual(descriptors[i], expectedDescriptors[i])); - } - } - - [Fact] - public void RemoveEmptyDescriptors_ShouldRemoveAllEmptyDescriptors() - { - var descriptors = new List - { - Descriptor.EmptyDescriptor(), - Descriptor.EmptyDescriptor(), - Descriptor.EmptyDescriptor(), - Descriptor.EmptyDescriptor(), - }; - - Referrers.RemoveEmptyDescriptors(descriptors, 0); - Assert.Empty(descriptors); - } } From 2b24e071f95ff3c0f741dce50cb51b517d949f7a Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 3 Dec 2024 13:26:53 +1100 Subject: [PATCH 16/24] resolve comments Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Oci/Descriptor.cs | 2 +- .../Remote/HttpResponseMessageExtensions.cs | 2 +- .../Registry/Remote/ManifestStore.cs | 14 +++++------ .../Registry/Remote/Referrers.cs | 24 +++++++++++-------- .../Remote/ManifestStoreTest.cs | 2 +- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 8e552c6..8370aa5 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -72,7 +72,7 @@ public static Descriptor Create(Span data, string mediaType) internal BasicDescriptor BasicDescriptor => new BasicDescriptor(MediaType, Digest, Size); - internal static bool IsEmptyOrNull(Descriptor? descriptor) + internal static bool IsEmptyOrInvalid(Descriptor? descriptor) { return descriptor == null || descriptor.Size == 0 || string.IsNullOrEmpty(descriptor.Digest) || string.IsNullOrEmpty(descriptor.MediaType); } diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 0e4585c..af00c48 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -107,7 +107,7 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string /// /// /// - public static void CheckOCISubjectHeader(this HttpResponseMessage response, Repository repository) + internal static void CheckOCISubjectHeader(this HttpResponseMessage response, Repository repository) { if (response.Headers.TryGetValues("OCI-Subject", out var values)) { diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index c4675c1..d74f30f 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -184,7 +184,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re return; } - var contentBytes = await content.ReadAllAsync(expected, cancellationToken); + var contentBytes = await content.ReadAllAsync(expected, cancellationToken).ConfigureAwait(false); using (var contentDuplicate = new MemoryStream(contentBytes)) { // Push the manifest when ReferrerState is Unknown or NotSupported @@ -202,11 +202,11 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re // 1. Index the referrers list using referrers tag schema when manifest contains a subject field // And the ReferrerState is not supported // 2. Or do nothing when the manifest does not contain a subject field when ReferrerState is not supported/unknown - await ProcessReferrersAndPushIndex(expected, contentDuplicate, cancellationToken); + await ProcessReferrersAndPushIndex(expected, contentDuplicate, cancellationToken).ConfigureAwait(false); } break; default: - await DoPushAsync(expected, content, reference, cancellationToken); + await DoPushAsync(expected, content, reference, cancellationToken).ConfigureAwait(false); break; } } @@ -245,7 +245,7 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, } Repository.SetReferrersState(Referrers.ReferrersState.ReferrersNotSupported); - await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken); + await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken).ConfigureAwait(false); } /// @@ -265,7 +265,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, { // 1. pull the original referrers index list using referrers tag schema var referrersTag = Referrers.BuildReferrersTag(subject); - var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag, cancellationToken); + var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag, cancellationToken).ConfigureAwait(false); // 2. apply the referrer change to referrers list var (updatedReferrers, updateRequired) = @@ -287,7 +287,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, } } - if (repository.Options.SkipReferrersGC || Descriptor.IsEmptyOrNull(oldDesc)) + if (repository.Options.SkipReferrersGC || Descriptor.IsEmptyOrInvalid(oldDesc)) { // Skip the delete process if SkipReferrersGC is set to true or the old Descriptor is empty or null return; @@ -310,7 +310,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, { try { - var (desc, content) = await FetchAsync(referrersTag, cancellationToken); + var (desc, content) = await FetchAsync(referrersTag, cancellationToken).ConfigureAwait(false); var index = JsonSerializer.Deserialize(content); if (index == null) { diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 6d15a26..0e1fbe6 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -52,6 +52,11 @@ internal static string BuildReferrersTag(Descriptor descriptor) /// The updated referrers list, updateRequired internal static (IList, bool) ApplyReferrerChanges(IList oldReferrers, ReferrerChange referrerChange) { + if (Descriptor.IsEmptyOrInvalid(referrerChange.Referrer)) + { + return (oldReferrers, false); + } + // updatedReferrers is a list to store the updated referrers var updatedReferrers = new List(); // updatedReferrersSet is a HashSet to store unique referrers @@ -60,7 +65,7 @@ internal static (IList, bool) ApplyReferrerChanges(IList var updateRequired = false; foreach (var oldReferrer in oldReferrers) { - if (Descriptor.IsEmptyOrNull(oldReferrer)) + if (Descriptor.IsEmptyOrInvalid(oldReferrer)) { // Skip any empty or null referrers updateRequired = true; @@ -84,19 +89,18 @@ internal static (IList, bool) ApplyReferrerChanges(IList updatedReferrersSet.Add(basicDesc); } - if (!Descriptor.IsEmptyOrNull(referrerChange.Referrer)) + + var basicReferrerDesc = referrerChange.Referrer.BasicDescriptor; + if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerAdd) { - var basicDesc = referrerChange.Referrer.BasicDescriptor; - if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerAdd) + if (!updatedReferrersSet.Contains(basicReferrerDesc)) { - if (!updatedReferrersSet.Contains(basicDesc)) - { - // Add the new referrer only when it has not already existed in the updatedReferrersSet - updatedReferrers.Add(referrerChange.Referrer); - updatedReferrersSet.Add(basicDesc); - } + // Add the new referrer only when it has not already existed in the updatedReferrersSet + updatedReferrers.Add(referrerChange.Referrer); + updatedReferrersSet.Add(basicReferrerDesc); } } + // Skip unnecessary update if (!updateRequired && updatedReferrersSet.Count == oldReferrers.Count) diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index e0721e9..e8fd70b 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -100,7 +100,7 @@ public async Task ManifestStore_PullReferrersIndexListNotFound() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var (receivedDesc, receivedManifests) = await store.PullReferrersIndexList("test", cancellationToken); - Assert.True(Descriptor.IsEmptyOrNull(receivedDesc)); + Assert.True(Descriptor.IsEmptyOrInvalid(receivedDesc)); Assert.Empty(receivedManifests); } From c30d6b23bf2540c427ebb20d00b4ac19665f24e7 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 13 Dec 2024 15:00:04 +1100 Subject: [PATCH 17/24] resolve comments Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Oci/Descriptor.cs | 4 +- .../Remote/HttpResponseMessageExtensions.cs | 6 +-- .../Registry/Remote/ManifestStore.cs | 22 +++++++---- .../Registry/Remote/Referrers.cs | 11 +++--- .../Registry/Remote/Repository.cs | 39 +++++++------------ .../Remote/ManifestStoreTest.cs | 30 +++++++------- .../Remote/ReferrersTest.cs | 4 +- .../Remote/RepositoryTest.cs | 24 ++++++------ 8 files changed, 66 insertions(+), 74 deletions(-) diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 8370aa5..3f35c27 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -74,10 +74,10 @@ public static Descriptor Create(Span data, string mediaType) internal static bool IsEmptyOrInvalid(Descriptor? descriptor) { - return descriptor == null || descriptor.Size == 0 || string.IsNullOrEmpty(descriptor.Digest) || string.IsNullOrEmpty(descriptor.MediaType); + return descriptor == null || string.IsNullOrEmpty(descriptor.Digest) || string.IsNullOrEmpty(descriptor.MediaType); } - internal static Descriptor EmptyDescriptor() => new () + internal static Descriptor ZeroDescriptor() => new () { MediaType = "", Digest = "", diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index af00c48..5e672a5 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -111,15 +111,15 @@ internal static void CheckOCISubjectHeader(this HttpResponseMessage response, Re { if (response.Headers.TryGetValues("OCI-Subject", out var values)) { - // Set it to ReferrerSupported when the response header contains OCI-Subject - repository.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); + // Set it to Supported when the response header contains OCI-Subject + repository.ReferrersState = Referrers.ReferrersState.Supported; } // If the "OCI-Subject" header is NOT set, it means that either the manifest // has no subject OR the referrers API is NOT supported by the registry. // // Since we don't know whether the pushed manifest has a subject or not, - // we do not set the ReferrerState to ReferrerNotSupported here. + // we do not set the ReferrerState to NotSupported here. } /// diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index d74f30f..724354c 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -177,7 +177,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re { case MediaType.ImageManifest: case MediaType.ImageIndex: - if (Repository.ReferrersState == Referrers.ReferrersState.ReferrersSupported) + if (Repository.ReferrersState == Referrers.ReferrersState.Supported) { // Push the manifest straightaway when the registry supports referrers API await DoPushAsync(expected, content, reference, cancellationToken).ConfigureAwait(false); @@ -190,7 +190,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re // Push the manifest when ReferrerState is Unknown or NotSupported await DoPushAsync(expected, contentDuplicate, reference, cancellationToken).ConfigureAwait(false); } - if (Repository.ReferrersState == Referrers.ReferrersState.ReferrersSupported) + if (Repository.ReferrersState == Referrers.ReferrersState.Supported) { // Early exit when the registry supports Referrers API // No need to index referrers list @@ -228,14 +228,20 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, { case MediaType.ImageIndex: var indexManifest = JsonSerializer.Deserialize(content); - if (indexManifest?.Subject == null) return; + if (indexManifest?.Subject == null) + { + return; + } subject = indexManifest.Subject; desc.ArtifactType = indexManifest.ArtifactType; desc.Annotations = indexManifest.Annotations; break; case MediaType.ImageManifest: - var imageManifest = JsonSerializer.Deserialize(content); - if (imageManifest?.Subject == null) return; + var imageManifest = JsonSerializer.Deserialize(content); + if (imageManifest?.Subject == null) + { + return; + } subject = imageManifest.Subject; desc.ArtifactType = string.IsNullOrEmpty(imageManifest.ArtifactType) ? imageManifest.Config.MediaType : imageManifest.ArtifactType; desc.Annotations = imageManifest.Annotations; @@ -244,7 +250,7 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, return; } - Repository.SetReferrersState(Referrers.ReferrersState.ReferrersNotSupported); + Repository.ReferrersState = Referrers.ReferrersState.NotSupported; await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken).ConfigureAwait(false); } @@ -306,7 +312,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, /// /// /// - internal async Task<(Descriptor, IList)> PullReferrersIndexList(String referrersTag, CancellationToken cancellationToken = default) + internal async Task<(Descriptor?, IList)> PullReferrersIndexList(String referrersTag, CancellationToken cancellationToken = default) { try { @@ -320,7 +326,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, } catch (NotFoundException) { - return (Descriptor.EmptyDescriptor(), new List()); + return (null, new List()); } } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 0e1fbe6..de495c2 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -19,13 +19,13 @@ namespace OrasProject.Oras.Registry.Remote; -public class Referrers +internal static class Referrers { internal enum ReferrersState { - ReferrersUnknown = 0, - ReferrersSupported = 1, - ReferrersNotSupported = 2 + Unknown = 0, + Supported = 1, + NotSupported = 2 } internal record ReferrerChange(Descriptor Referrer, ReferrerOperation ReferrerOperation); @@ -38,8 +38,7 @@ internal enum ReferrerOperation internal static string BuildReferrersTag(Descriptor descriptor) { - var validatedDigest = Digest.Validate(descriptor.Digest); - return validatedDigest.Substring(0, validatedDigest.IndexOf(':')) + "-" + validatedDigest.Substring(validatedDigest.IndexOf(':') + 1); + return Digest.Validate(descriptor.Digest).Replace(':', '-'); } /// diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 613400f..742dcc6 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -47,12 +47,23 @@ public class Repository : IRepository public RepositoryOptions Options => _opts; - private int _referrersState = (int) Referrers.ReferrersState.ReferrersUnknown; + private int _referrersState = (int) Referrers.ReferrersState.Unknown; + /// + /// ReferrersState indicates the Referrers API state of the remote repository. + /// ReferrersState can be set only once, otherwise it throws ReferrersStateAlreadySetException. + /// internal Referrers.ReferrersState ReferrersState { get => (Referrers.ReferrersState) _referrersState; - private set => _referrersState = (int) value; + set + { + var originalReferrersState = (Referrers.ReferrersState) Interlocked.CompareExchange(ref _referrersState, (int)value, (int)Referrers.ReferrersState.Unknown); + if (originalReferrersState != Referrers.ReferrersState.Unknown && _referrersState != (int)value) + { + throw new ReferrersStateAlreadySetException($"current referrers state: {ReferrersState}, latest referrers state: {value}"); + } + } } internal static readonly string[] DefaultManifestMediaTypes = @@ -93,30 +104,6 @@ public Repository(RepositoryOptions options) _opts = options; } - /// - /// SetReferrersState indicates the Referrers API state of the remote repository. - /// - /// SetReferrersState is valid only when it is called for the first time. - /// SetReferrersState returns ReferrersStateAlreadySetException if the - /// Referrers API state has been already set. - /// - When the state is set to ReferrersSupported, the Referrers() function will always - /// request the Referrers API. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers - /// - When the state is set to ReferrersNotSupported, the Referrers() function will always - /// request the Referrers Tag. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema - /// - When the state is not set, the Referrers() function will automatically - /// determine which API to use. - /// - /// - /// - internal void SetReferrersState(Referrers.ReferrersState state) - { - var originalReferrersState = (Referrers.ReferrersState) Interlocked.CompareExchange(ref _referrersState, (int)state, (int)Referrers.ReferrersState.ReferrersUnknown); - if (originalReferrersState != Referrers.ReferrersState.ReferrersUnknown && _referrersState != (int) state) - { - throw new ReferrersStateAlreadySetException($"current referrers state: {ReferrersState}, latest referrers state: {state}"); - } - } - /// /// FetchAsync fetches the content identified by the descriptor. /// diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index e8fd70b..0b1ac8c 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -100,7 +100,7 @@ public async Task ManifestStore_PullReferrersIndexListNotFound() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var (receivedDesc, receivedManifests) = await store.PullReferrersIndexList("test", cancellationToken); - Assert.True(Descriptor.IsEmptyOrInvalid(receivedDesc)); + Assert.Null(receivedDesc); Assert.Empty(receivedManifests); } @@ -167,14 +167,14 @@ public async Task ManifestStore_PushAsyncWithoutSubject() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); Assert.Equal(expectedManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); await store.PushAsync(expectedConfigDesc, new MemoryStream(expectedConfigBytes), cancellationToken); Assert.Equal(expectedConfigBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); } @@ -254,15 +254,15 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() var store = new ManifestStore(repo); // first push with image manifest - Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); Assert.Equal(expectedManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); // second push with index manifest await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); Assert.Equal(expectedIndexManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); } @@ -377,18 +377,18 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() var store = new ManifestStore(repo); // First push with referrer tag schema - Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); await store.PushAsync(firstExpectedManifestDesc, new MemoryStream(firstExpectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); Assert.Equal(firstExpectedManifestBytes, receivedManifestContent); Assert.True(oldIndexDeleted); Assert.Equal(firstExpectedIndexReferrersBytes, receivedIndexContent); // Second push with referrer tag schema - Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); await store.PushAsync(secondExpectedManifestDesc, new MemoryStream(secondExpectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); Assert.Equal(secondExpectedManifestBytes, receivedManifestContent); Assert.True(firstIndexDeleted); Assert.Equal(secondExpectedIndexReferrersBytes, receivedIndexContent); @@ -460,9 +460,9 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupportedWitho var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); Assert.Equal(expectedIndexManifestBytes, receivedIndexManifestContent); Assert.Equal(expectedIndexReferrersBytes, receivedIndexReferrersContent); } @@ -547,9 +547,9 @@ public async Task ManifestStore_PushAsyncWithSubjectAndNoUpdateRequired() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); Assert.Equal(expectedManifestBytes, receivedManifestContent); } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index 5db44fd..394f272 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -230,7 +230,7 @@ public void ApplyReferrerChanges_ShouldNotAddNewDuplicateReferrers() [Fact] public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() { - var emptyDesc1 = Descriptor.EmptyDescriptor(); + var emptyDesc1 = Descriptor.ZeroDescriptor(); Descriptor? emptyDesc2 = null; var newDescriptor = RandomDescriptor(); @@ -261,7 +261,7 @@ public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() public void ApplyReferrerChanges_NoUpdateWhenOldAndNewReferrersAreEmpty() { var oldReferrers = new List(); - var referrerChange = new Referrers.ReferrerChange(Descriptor.EmptyDescriptor(), Referrers.ReferrerOperation.ReferrerAdd); + var referrerChange = new Referrers.ReferrerChange(Descriptor.ZeroDescriptor(), Referrers.ReferrerOperation.ReferrerAdd); var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Empty(updatedReferrers); diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 6aae7e1..b893c34 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -2549,35 +2549,35 @@ public async Task Repository_MountAsync_Fallback_GetContentError() public void SetReferrersState_ShouldSet_WhenInitiallyUnknown() { var repo = new Repository("localhost:5000/test2"); - Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); - repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); - Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + repo.ReferrersState = Referrers.ReferrersState.Supported; + Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); } [Fact] public void SetReferrersState_ShouldThrowException_WhenChangingAfterSet() { var repo = new Repository("localhost:5000/test2"); - Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); - repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); - Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + repo.ReferrersState = Referrers.ReferrersState.Supported; + Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); var exception = Assert.Throws(() => - repo.SetReferrersState(Referrers.ReferrersState.ReferrersNotSupported) + repo.ReferrersState = Referrers.ReferrersState.NotSupported ); - Assert.Equal("current referrers state: ReferrersSupported, latest referrers state: ReferrersNotSupported", exception.Message); + Assert.Equal("current referrers state: Supported, latest referrers state: NotSupported", exception.Message); } [Fact] public void SetReferrersState_ShouldNotThrowException_WhenSettingSameValue() { var repo = new Repository("localhost:5000/test2"); - Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); - repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); - Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + repo.ReferrersState = Referrers.ReferrersState.Supported; + Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); - var exception = Record.Exception(() => repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported)); + var exception = Record.Exception(() => repo.ReferrersState = Referrers.ReferrersState.Supported); Assert.Null(exception); } } From b20a20280f68f0d4efc9172eee49d0eea178fea9 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 20 Dec 2024 11:13:33 +1100 Subject: [PATCH 18/24] resolve comments Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Oci/Descriptor.cs | 2 +- .../Remote/HttpResponseMessageExtensions.cs | 4 ++-- .../Registry/Remote/ManifestStore.cs | 19 ++++++++++++------- .../Registry/Remote/Referrers.cs | 14 ++++++-------- .../Registry/Remote/RepositoryOptions.cs | 4 ++-- .../Remote/ReferrersTest.cs | 16 ++++++++-------- 6 files changed, 31 insertions(+), 28 deletions(-) diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 3f35c27..99ac2cf 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -72,7 +72,7 @@ public static Descriptor Create(Span data, string mediaType) internal BasicDescriptor BasicDescriptor => new BasicDescriptor(MediaType, Digest, Size); - internal static bool IsEmptyOrInvalid(Descriptor? descriptor) + internal static bool IsNullOrInvalid(Descriptor? descriptor) { return descriptor == null || string.IsNullOrEmpty(descriptor.Digest) || string.IsNullOrEmpty(descriptor.MediaType); } diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 5e672a5..c45552c 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -107,9 +107,9 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string /// /// /// - internal static void CheckOCISubjectHeader(this HttpResponseMessage response, Repository repository) + internal static void CheckOciSubjectHeader(this HttpResponseMessage response, Repository repository) { - if (response.Headers.TryGetValues("OCI-Subject", out var values)) + if (response.Headers.Contains("OCI-Subject")) { // Set it to Supported when the response header contains OCI-Subject repository.ReferrersState = Referrers.ReferrersState.Supported; diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 724354c..ccecf84 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -15,9 +15,11 @@ using OrasProject.Oras.Oci; using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; using System.Net; using System.Net.Http; +using System.Runtime.InteropServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -251,7 +253,7 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, } Repository.ReferrersState = Referrers.ReferrersState.NotSupported; - await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken).ConfigureAwait(false); + await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.Add), cancellationToken).ConfigureAwait(false); } /// @@ -276,10 +278,13 @@ private async Task UpdateReferrersIndex(Descriptor subject, // 2. apply the referrer change to referrers list var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); - if (!updateRequired) return; + if (!updateRequired) + { + return; + } // 3. push the updated referrers list using referrers tag schema - if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGC) + if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGc) { // push a new index in either case: // 1. the referrers list has been updated with a non-zero size @@ -293,9 +298,9 @@ private async Task UpdateReferrersIndex(Descriptor subject, } } - if (repository.Options.SkipReferrersGC || Descriptor.IsEmptyOrInvalid(oldDesc)) + if (repository.Options.SkipReferrersGc || Descriptor.IsNullOrInvalid(oldDesc)) { - // Skip the delete process if SkipReferrersGC is set to true or the old Descriptor is empty or null + // Skip the delete process if SkipReferrersGc is set to true or the old Descriptor is empty or null return; } @@ -326,7 +331,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, } catch (NotFoundException) { - return (null, new List()); + return (null, ImmutableArray.Empty); } } @@ -351,7 +356,7 @@ private async Task DoPushAsync(Descriptor expected, Stream stream, Reference rem { throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); } - response.CheckOCISubjectHeader(Repository); + response.CheckOciSubjectHeader(Repository); response.VerifyContentDigest(expected.Digest); } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index de495c2..724105b 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -32,8 +32,8 @@ internal record ReferrerChange(Descriptor Referrer, ReferrerOperation ReferrerOp internal enum ReferrerOperation { - ReferrerAdd, - ReferrerDelete, + Add, + Delete, } internal static string BuildReferrersTag(Descriptor descriptor) @@ -51,7 +51,7 @@ internal static string BuildReferrersTag(Descriptor descriptor) /// The updated referrers list, updateRequired internal static (IList, bool) ApplyReferrerChanges(IList oldReferrers, ReferrerChange referrerChange) { - if (Descriptor.IsEmptyOrInvalid(referrerChange.Referrer)) + if (Descriptor.IsNullOrInvalid(referrerChange.Referrer)) { return (oldReferrers, false); } @@ -64,7 +64,7 @@ internal static (IList, bool) ApplyReferrerChanges(IList var updateRequired = false; foreach (var oldReferrer in oldReferrers) { - if (Descriptor.IsEmptyOrInvalid(oldReferrer)) + if (Descriptor.IsNullOrInvalid(oldReferrer)) { // Skip any empty or null referrers updateRequired = true; @@ -79,7 +79,7 @@ internal static (IList, bool) ApplyReferrerChanges(IList } // Update the updatedReferrers list // Add referrer index in the updatedReferrersSet - if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerDelete && Descriptor.Equals(basicDesc, referrerChange.Referrer.BasicDescriptor)) + if (referrerChange.ReferrerOperation == ReferrerOperation.Delete && Descriptor.Equals(basicDesc, referrerChange.Referrer.BasicDescriptor)) { updateRequired = true; continue; @@ -88,9 +88,8 @@ internal static (IList, bool) ApplyReferrerChanges(IList updatedReferrersSet.Add(basicDesc); } - var basicReferrerDesc = referrerChange.Referrer.BasicDescriptor; - if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerAdd) + if (referrerChange.ReferrerOperation == ReferrerOperation.Add) { if (!updatedReferrersSet.Contains(basicReferrerDesc)) { @@ -99,7 +98,6 @@ internal static (IList, bool) ApplyReferrerChanges(IList updatedReferrersSet.Add(basicReferrerDesc); } } - // Skip unnecessary update if (!updateRequired && updatedReferrersSet.Count == oldReferrers.Count) diff --git a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs index 4742f9d..77f8259 100644 --- a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs +++ b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs @@ -51,7 +51,7 @@ public struct RepositoryOptions /// public int TagListPageSize { get; set; } - // SkipReferrersGC specifies whether to delete the dangling referrers + // SkipReferrersGc specifies whether to delete the dangling referrers // index when referrers tag schema is utilized. // - If false, the old referrers index will be deleted after the new one is successfully uploaded. // - If true, the old referrers index is kept. @@ -59,5 +59,5 @@ public struct RepositoryOptions // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#pushing-manifests-with-subject // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests - public bool SkipReferrersGC { get; set; } + public bool SkipReferrersGc { get; set; } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index 394f272..226023a 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -59,7 +59,7 @@ public void ApplyReferrerChanges_ShouldAddNewReferrers() }; var referrerChange = new Referrers.ReferrerChange( newDescriptor, - Referrers.ReferrerOperation.ReferrerAdd + Referrers.ReferrerOperation.Add ); var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); @@ -92,7 +92,7 @@ public void ApplyReferrerChanges_ShouldDeleteReferrers() }; var referrerChange = new Referrers.ReferrerChange( oldDescriptor2, - Referrers.ReferrerOperation.ReferrerDelete + Referrers.ReferrerOperation.Delete ); var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); @@ -129,7 +129,7 @@ public void ApplyReferrerChanges_ShouldDeleteReferrersWithDuplicates() }; var referrerChange = new Referrers.ReferrerChange( oldDescriptor3, - Referrers.ReferrerOperation.ReferrerDelete + Referrers.ReferrerOperation.Delete ); var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); @@ -161,7 +161,7 @@ public void ApplyReferrerChanges_ShouldNotDeleteReferrersWhenNoUpdateRequired() }; var referrerChange = new Referrers.ReferrerChange( oldDescriptor3, - Referrers.ReferrerOperation.ReferrerDelete + Referrers.ReferrerOperation.Delete ); var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); @@ -196,7 +196,7 @@ public void ApplyReferrerChanges_ShouldDiscardDuplicateReferrers() }; var referrerChange = new Referrers.ReferrerChange( newDescriptor1, - Referrers.ReferrerOperation.ReferrerAdd + Referrers.ReferrerOperation.Add ); var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); @@ -220,7 +220,7 @@ public void ApplyReferrerChanges_ShouldNotAddNewDuplicateReferrers() }; var referrerChange = new Referrers.ReferrerChange( oldDescriptor1, - Referrers.ReferrerOperation.ReferrerAdd + Referrers.ReferrerOperation.Add ); var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Equal(2, updatedReferrers.Count); @@ -245,7 +245,7 @@ public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() }; var referrerChange = new Referrers.ReferrerChange( newDescriptor, - Referrers.ReferrerOperation.ReferrerAdd + Referrers.ReferrerOperation.Add ); var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); @@ -261,7 +261,7 @@ public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() public void ApplyReferrerChanges_NoUpdateWhenOldAndNewReferrersAreEmpty() { var oldReferrers = new List(); - var referrerChange = new Referrers.ReferrerChange(Descriptor.ZeroDescriptor(), Referrers.ReferrerOperation.ReferrerAdd); + var referrerChange = new Referrers.ReferrerChange(Descriptor.ZeroDescriptor(), Referrers.ReferrerOperation.Add); var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Empty(updatedReferrers); From 07d8e061066aa711e893f066c4011cc1e0f9aff8 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 24 Dec 2024 09:33:04 +1100 Subject: [PATCH 19/24] resolve comments Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Oci/Descriptor.cs | 7 ------- .../Registry/Remote/HttpResponseMessageExtensions.cs | 3 +-- tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs | 4 ++-- tests/OrasProject.Oras.Tests/Remote/Util/Util.cs | 7 +++++++ 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 99ac2cf..9fba38c 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -76,11 +76,4 @@ internal static bool IsNullOrInvalid(Descriptor? descriptor) { return descriptor == null || string.IsNullOrEmpty(descriptor.Digest) || string.IsNullOrEmpty(descriptor.MediaType); } - - internal static Descriptor ZeroDescriptor() => new () - { - MediaType = "", - Digest = "", - Size = 0 - }; } diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index c45552c..47e5cd7 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -109,7 +109,7 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string /// internal static void CheckOciSubjectHeader(this HttpResponseMessage response, Repository repository) { - if (response.Headers.Contains("OCI-Subject")) + if (repository.ReferrersState == Referrers.ReferrersState.Unknown && response.Headers.Contains("OCI-Subject")) { // Set it to Supported when the response header contains OCI-Subject repository.ReferrersState = Referrers.ReferrersState.Supported; @@ -117,7 +117,6 @@ internal static void CheckOciSubjectHeader(this HttpResponseMessage response, Re // If the "OCI-Subject" header is NOT set, it means that either the manifest // has no subject OR the referrers API is NOT supported by the registry. - // // Since we don't know whether the pushed manifest has a subject or not, // we do not set the ReferrerState to NotSupported here. } diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index 226023a..ed01505 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -230,7 +230,7 @@ public void ApplyReferrerChanges_ShouldNotAddNewDuplicateReferrers() [Fact] public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() { - var emptyDesc1 = Descriptor.ZeroDescriptor(); + var emptyDesc1 = ZeroDescriptor(); Descriptor? emptyDesc2 = null; var newDescriptor = RandomDescriptor(); @@ -261,7 +261,7 @@ public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() public void ApplyReferrerChanges_NoUpdateWhenOldAndNewReferrersAreEmpty() { var oldReferrers = new List(); - var referrerChange = new Referrers.ReferrerChange(Descriptor.ZeroDescriptor(), Referrers.ReferrerOperation.Add); + var referrerChange = new Referrers.ReferrerChange(ZeroDescriptor(), Referrers.ReferrerOperation.Add); var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Empty(updatedReferrers); diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs index 6da68ca..4d24bf8 100644 --- a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs +++ b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs @@ -51,4 +51,11 @@ public static HttpClient CustomClient(Func new() + { + MediaType = "", + Digest = "", + Size = 0 + }; } From 27382ae5b39cc22d22a896636f2866d84c29fcd2 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 24 Dec 2024 09:38:48 +1100 Subject: [PATCH 20/24] resolve comments Signed-off-by: Patrick Pan --- .../Registry/Remote/HttpResponseMessageExtensions.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 47e5cd7..b7a98f6 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -26,6 +26,7 @@ namespace OrasProject.Oras.Registry.Remote; internal static class HttpResponseMessageExtensions { private const string _dockerContentDigestHeader = "Docker-Content-Digest"; + /// /// Parses the error returned by the remote registry. /// @@ -100,7 +101,7 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string throw new HttpIOException(HttpRequestError.InvalidResponse, $"{response.RequestMessage!.Method} {response.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {digestStr}"); } } - + /// /// CheckOciSubjectHeader checks if the response header contains "OCI-Subject", /// repository ReferrerState is set to supported if it is present @@ -114,7 +115,7 @@ internal static void CheckOciSubjectHeader(this HttpResponseMessage response, Re // Set it to Supported when the response header contains OCI-Subject repository.ReferrersState = Referrers.ReferrersState.Supported; } - + // If the "OCI-Subject" header is NOT set, it means that either the manifest // has no subject OR the referrers API is NOT supported by the registry. // Since we don't know whether the pushed manifest has a subject or not, From 404337ecba764494fecbe35116cc4f53b9cc445d Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 24 Dec 2024 09:50:53 +1100 Subject: [PATCH 21/24] resolve comments Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Oci/Descriptor.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 9fba38c..f7d738b 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -13,10 +13,7 @@ using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Text; using System.Text.Json.Serialization; -using OrasProject.Oras.Content; namespace OrasProject.Oras.Oci; From 1041bc07a62fa49f5ac3ee5b1f99a0fc04217da9 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 24 Dec 2024 09:52:47 +1100 Subject: [PATCH 22/24] resolve comments Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Registry/Remote/ManifestStore.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index ccecf84..9847827 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -19,7 +19,6 @@ using System.IO; using System.Net; using System.Net.Http; -using System.Runtime.InteropServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; From a3e6289cffefbfa9d7212370580f45d9071e790c Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 3 Jan 2025 18:08:50 +1100 Subject: [PATCH 23/24] resolve comments Signed-off-by: Patrick Pan --- .../Remote/HttpResponseMessageExtensions.cs | 2 +- .../Registry/Remote/ManifestStore.cs | 2 +- .../Registry/Remote/Referrers.cs | 18 +++++++++++------- .../Registry/Remote/Repository.cs | 5 +++++ .../Registry/Remote/RepositoryOptions.cs | 18 ++++++++++-------- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index b7a98f6..58239d9 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -113,7 +113,7 @@ internal static void CheckOciSubjectHeader(this HttpResponseMessage response, Re if (repository.ReferrersState == Referrers.ReferrersState.Unknown && response.Headers.Contains("OCI-Subject")) { // Set it to Supported when the response header contains OCI-Subject - repository.ReferrersState = Referrers.ReferrersState.Supported; + repository.SetReferrersState(true); } // If the "OCI-Subject" header is NOT set, it means that either the manifest diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 9847827..5c77374 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -251,7 +251,7 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, return; } - Repository.ReferrersState = Referrers.ReferrersState.NotSupported; + Repository.SetReferrersState(false); await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.Add), cancellationToken).ConfigureAwait(false); } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 724105b..71639a1 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -70,37 +70,41 @@ internal static (IList, bool) ApplyReferrerChanges(IList updateRequired = true; continue; } + var basicDesc = oldReferrer.BasicDescriptor; - if (updatedReferrersSet.Contains(basicDesc)) + if (referrerChange.ReferrerOperation == ReferrerOperation.Delete && Equals(basicDesc, referrerChange.Referrer.BasicDescriptor)) { - // Skip any duplicate referrers updateRequired = true; continue; } - // Update the updatedReferrers list - // Add referrer index in the updatedReferrersSet - if (referrerChange.ReferrerOperation == ReferrerOperation.Delete && Descriptor.Equals(basicDesc, referrerChange.Referrer.BasicDescriptor)) + + if (updatedReferrersSet.Contains(basicDesc)) { + // Skip any duplicate referrers updateRequired = true; continue; } + + // Update the updatedReferrers list + // Add referrer into the updatedReferrersSet updatedReferrers.Add(oldReferrer); updatedReferrersSet.Add(basicDesc); } - var basicReferrerDesc = referrerChange.Referrer.BasicDescriptor; if (referrerChange.ReferrerOperation == ReferrerOperation.Add) { + var basicReferrerDesc = referrerChange.Referrer.BasicDescriptor; if (!updatedReferrersSet.Contains(basicReferrerDesc)) { // Add the new referrer only when it has not already existed in the updatedReferrersSet updatedReferrers.Add(referrerChange.Referrer); updatedReferrersSet.Add(basicReferrerDesc); + updateRequired = true; } } // Skip unnecessary update - if (!updateRequired && updatedReferrersSet.Count == oldReferrers.Count) + if (!updateRequired) { // Check for any new referrers in the updatedReferrersSet that are not present in the oldReferrers list foreach (var oldReferrer in oldReferrers) diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 742dcc6..fd312f8 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -368,4 +368,9 @@ internal Reference ParseReferenceFromContentReference(string reference) /// public async Task MountAsync(Descriptor descriptor, string fromRepository, Func>? getContent = null, CancellationToken cancellationToken = default) => await ((IMounter)Blobs).MountAsync(descriptor, fromRepository, getContent, cancellationToken).ConfigureAwait(false); + + public void SetReferrersState(bool isSupported) + { + ReferrersState = isSupported ? Referrers.ReferrersState.Supported : Referrers.ReferrersState.NotSupported; + } } diff --git a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs index 77f8259..63e9a5b 100644 --- a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs +++ b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs @@ -51,13 +51,15 @@ public struct RepositoryOptions /// public int TagListPageSize { get; set; } - // SkipReferrersGc specifies whether to delete the dangling referrers - // index when referrers tag schema is utilized. - // - If false, the old referrers index will be deleted after the new one is successfully uploaded. - // - If true, the old referrers index is kept. - // By default, it is disabled (set to false). See also: - // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema - // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#pushing-manifests-with-subject - // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests + /// + /// SkipReferrersGc specifies whether to delete the dangling referrers + /// index when referrers tag schema is utilized. + /// - If false, the old referrers index will be deleted after the new one is successfully uploaded. + /// - If true, the old referrers index is kept. + /// By default, it is disabled (set to false). See also: + /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema + /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#pushing-manifests-with-subject + /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests + /// public bool SkipReferrersGc { get; set; } } From aa8f4a01d0bda70cea1d2ef0ea46fa446a502bc1 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 3 Jan 2025 18:32:52 +1100 Subject: [PATCH 24/24] resolve comments Signed-off-by: Patrick Pan --- .../Registry/Remote/ManifestStore.cs | 4 +++- .../Registry/Remote/Referrers.cs | 21 +------------------ .../Registry/Remote/Repository.cs | 12 +++++++++++ 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 5c77374..b0e35c5 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -251,6 +251,8 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, return; } + // In this case, the manifest contains a subject field and OCI-Subject Header is not set after pushing the manifest to the registry, + // which indicates that the registry does not support referrers API Repository.SetReferrersState(false); await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.Add), cancellationToken).ConfigureAwait(false); } @@ -324,7 +326,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, var index = JsonSerializer.Deserialize(content); if (index == null) { - throw new JsonException("null index manifests list"); + throw new JsonException($"null index manifests list when pulling referrers index list for referrers tag {referrersTag}"); } return (desc, index.Manifests); } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 71639a1..29a34a3 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -103,25 +103,6 @@ internal static (IList, bool) ApplyReferrerChanges(IList } } - // Skip unnecessary update - if (!updateRequired) - { - // Check for any new referrers in the updatedReferrersSet that are not present in the oldReferrers list - foreach (var oldReferrer in oldReferrers) - { - var basicDesc = oldReferrer.BasicDescriptor; - if (!updatedReferrersSet.Contains(basicDesc)) - { - updateRequired = true; - break; - } - } - - if (!updateRequired) - { - return (updatedReferrers, false); - } - } - return (updatedReferrers, true); + return (updatedReferrers, updateRequired); } } diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index fd312f8..186cdb6 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -369,6 +369,18 @@ internal Reference ParseReferenceFromContentReference(string reference) public async Task MountAsync(Descriptor descriptor, string fromRepository, Func>? getContent = null, CancellationToken cancellationToken = default) => await ((IMounter)Blobs).MountAsync(descriptor, fromRepository, getContent, cancellationToken).ConfigureAwait(false); + /// + /// SetReferrersState indicates the Referrers API state of the remote repository. true: supported; false: not supported. + /// SetReferrersState is valid only when it is called for the first time. + /// SetReferrersState returns ReferrersStateAlreadySetException if the Referrers API state has been already set. + /// - When the state is set to true, the relevant functions will always + /// request the Referrers API. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers + /// - When the state is set to false, the relevant functions will always + /// request the Referrers Tag. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema + /// - When the state is not set, the relevant functions will automatically + /// determine which API to use. + /// + /// public void SetReferrersState(bool isSupported) { ReferrersState = isSupported ? Referrers.ReferrersState.Supported : Referrers.ReferrersState.NotSupported;