diff --git a/src/OrasProject.Oras/Content/IPredecessorFinder.cs b/src/OrasProject.Oras/Content/IPredecessorFindable.cs similarity index 91% rename from src/OrasProject.Oras/Content/IPredecessorFinder.cs rename to src/OrasProject.Oras/Content/IPredecessorFindable.cs index ac6192e..8df5ba1 100644 --- a/src/OrasProject.Oras/Content/IPredecessorFinder.cs +++ b/src/OrasProject.Oras/Content/IPredecessorFindable.cs @@ -19,12 +19,12 @@ namespace OrasProject.Oras.Content; /// -/// IPredecessorFinder finds out the nodes directly pointing to a given node of a +/// Finds out the nodes directly pointing to a given node of a /// directed acyclic graph. /// In other words, returns the "parents" of the current descriptor. /// IPredecessorFinder is an extension of Storage. /// -public interface IPredecessorFinder +public interface IPredecessorFindable { /// /// returns the nodes directly pointing to the current node. diff --git a/src/OrasProject.Oras/Content/MemoryGraph.cs b/src/OrasProject.Oras/Content/MemoryGraph.cs index ee1f5c2..3eadcb1 100644 --- a/src/OrasProject.Oras/Content/MemoryGraph.cs +++ b/src/OrasProject.Oras/Content/MemoryGraph.cs @@ -20,7 +20,7 @@ namespace OrasProject.Oras.Content; -internal class MemoryGraph : IPredecessorFinder +internal class MemoryGraph : IPredecessorFindable { private readonly ConcurrentDictionary> _predecessors = new(); diff --git a/src/OrasProject.Oras/Content/MemoryStore.cs b/src/OrasProject.Oras/Content/MemoryStore.cs index 37a5b63..9a9f0f0 100644 --- a/src/OrasProject.Oras/Content/MemoryStore.cs +++ b/src/OrasProject.Oras/Content/MemoryStore.cs @@ -20,7 +20,7 @@ namespace OrasProject.Oras.Content; -public class MemoryStore : ITarget, IPredecessorFinder +public class MemoryStore : ITarget, IPredecessorFindable { private readonly MemoryStorage _storage = new(); private readonly MemoryTagStore _tagResolver = new(); diff --git a/src/OrasProject.Oras/Interfaces/Registry/IRegistry.cs b/src/OrasProject.Oras/Interfaces/Registry/IRegistry.cs deleted file mode 100644 index 7151dfd..0000000 --- a/src/OrasProject.Oras/Interfaces/Registry/IRegistry.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright The ORAS Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Threading; -using System.Threading.Tasks; - -namespace OrasProject.Oras.Interfaces.Registry -{ - public interface IRegistry - { - /// - /// Repository returns a repository reference by the given name. - /// - /// - /// - /// - Task Repository(string name, CancellationToken cancellationToken); - - /// - /// Repositories lists the name of repositories available in the registry. - /// Since the returned repositories may be paginated by the underlying - /// implementation, a function should be passed in to process the paginated - /// repository list. - /// `last` argument is the `last` parameter when invoking the catalog API. - /// If `last` is NOT empty, the entries in the response start after the - /// repo specified by `last`. Otherwise, the response starts from the top - /// of the Repositories list. - /// Note: When implemented by a remote registry, the catalog API is called. - /// However, not all registries supports pagination or conforms the - /// specification. - /// Reference: https://docs.docker.com/registry/spec/api/#catalog - /// See also `Repositories()` in this package. - /// - /// - /// - /// - /// - Task Repositories(string last, Action fn, CancellationToken cancellationToken); - } -} diff --git a/src/OrasProject.Oras/Interfaces/Registry/IRepository.cs b/src/OrasProject.Oras/Interfaces/Registry/IRepository.cs deleted file mode 100644 index 3a6aa84..0000000 --- a/src/OrasProject.Oras/Interfaces/Registry/IRepository.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright The ORAS Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using OrasProject.Oras.Content; - -namespace OrasProject.Oras.Interfaces.Registry -{ - /// - /// Repository is an ORAS target and an union of the blob and the manifest CASs. - /// As specified by https://docs.docker.com/registry/spec/api/, it is natural to - /// assume that IResolver interface only works for manifests. Tagging a - /// blob may be resulted in an `UnsupportedException` error. However, this interface - /// does not restrict tagging blobs. - /// Since a repository is an union of the blob and the manifest CASs, all - /// operations defined in the `IBlobStore` are executed depending on the media - /// type of the given descriptor accordingly. - /// Furthermore, this interface also provides the ability to enforce the - /// separation of the blob and the manifests CASs. - /// - public interface IRepository : ITarget, IReferenceFetcher, IReferencePusher, IDeletable, ITagLister - { - /// - /// Blobs provides access to the blob CAS only, which contains config blobs,layers, and other generic blobs. - /// - /// - IBlobStore Blobs(); - /// - /// Manifests provides access to the manifest CAS only. - /// - /// - IManifestStore Manifests(); - } -} diff --git a/src/OrasProject.Oras/Interfaces/Registry/IRepositoryOption.cs b/src/OrasProject.Oras/Interfaces/Registry/IRepositoryOption.cs deleted file mode 100644 index 6fbbbc2..0000000 --- a/src/OrasProject.Oras/Interfaces/Registry/IRepositoryOption.cs +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright The ORAS Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using OrasProject.Oras.Remote; -using System.Net.Http; - -namespace OrasProject.Oras.Interfaces.Registry -{ - /// - /// IRepositoryOption is used to configure a remote repository. - /// - public interface IRepositoryOption - { - /// - /// Client is the underlying HTTP client used to access the remote registry. - /// - public HttpClient HttpClient { get; set; } - - /// - /// Reference references the remote repository. - /// - public RemoteReference RemoteReference { get; set; } - - /// - /// PlainHTTP signals the transport to access the remote repository via HTTP - /// instead of HTTPS. - /// - public bool PlainHTTP { get; set; } - - - /// - /// ManifestMediaTypes is used in `Accept` header for resolving manifests - /// from references. It is also used in identifying manifests and blobs from - /// descriptors. If an empty list is present, default manifest media types - /// are used. - /// - public string[] ManifestMediaTypes { get; set; } - - /// - /// TagListPageSize specifies the page size when invoking the tag list API. - /// If zero, the page size is determined by the remote registry. - /// Reference: https://docs.docker.com/registry/spec/api/#tags - /// - public int TagListPageSize { get; set; } - - } -} diff --git a/src/OrasProject.Oras/Interfaces/Registry/ITagLister.cs b/src/OrasProject.Oras/Interfaces/Registry/ITagLister.cs deleted file mode 100644 index 22c1c8d..0000000 --- a/src/OrasProject.Oras/Interfaces/Registry/ITagLister.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright The ORAS Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Threading; -using System.Threading.Tasks; - -namespace OrasProject.Oras.Interfaces.Registry -{ - /// - /// ITagLister lists tags by the tag service. - /// - public interface ITagLister - { - /// - /// TagsAsync lists the tags available in the repository. - /// Since the returned tag list may be paginated by the underlying - /// implementation, a function should be passed in to process the paginated - /// tag list. - /// Note: When implemented by a remote registry, the tags API is called. - /// However, not all registries supports pagination or conforms the - /// specification. - /// References: - /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md - /// - https://docs.docker.com/registry/spec/api/#tags - /// - /// The `last` parameter when invoking the tags API. If `last` is NOT empty, the entries in the response start after the tag specified by `last`. Otherwise, the response starts from the top of the Tags list. - /// The function to process the paginated tag list - /// - /// - Task TagsAsync(string last, Action fn, CancellationToken cancellationToken = default); - } -} diff --git a/src/OrasProject.Oras/Interfaces/Registry/IManifestStore.cs b/src/OrasProject.Oras/Registry/IBlobStore.cs similarity index 66% rename from src/OrasProject.Oras/Interfaces/Registry/IManifestStore.cs rename to src/OrasProject.Oras/Registry/IBlobStore.cs index 35dabcb..21709d4 100644 --- a/src/OrasProject.Oras/Interfaces/Registry/IManifestStore.cs +++ b/src/OrasProject.Oras/Registry/IBlobStore.cs @@ -13,13 +13,11 @@ using OrasProject.Oras.Content; -namespace OrasProject.Oras.Interfaces.Registry +namespace OrasProject.Oras.Registry; + +/// +/// IBlobStore is a CAS with the ability to stat and delete its content. +/// +public interface IBlobStore : IStorage, IResolvable, IDeletable, IReferenceFetchable { - /// - /// IManifestStore is a CAS with the ability to stat and delete its content. - /// Besides, IManifestStore provides reference tagging. - /// - public interface IManifestStore : IBlobStore, IReferencePusher, ITaggable - { - } } diff --git a/src/OrasProject.Oras/Interfaces/Registry/IBlobStore.cs b/src/OrasProject.Oras/Registry/IManifestStore.cs similarity index 70% rename from src/OrasProject.Oras/Interfaces/Registry/IBlobStore.cs rename to src/OrasProject.Oras/Registry/IManifestStore.cs index 0658121..4be6cfb 100644 --- a/src/OrasProject.Oras/Interfaces/Registry/IBlobStore.cs +++ b/src/OrasProject.Oras/Registry/IManifestStore.cs @@ -13,12 +13,12 @@ using OrasProject.Oras.Content; -namespace OrasProject.Oras.Interfaces.Registry +namespace OrasProject.Oras.Registry; + +/// +/// IManifestStore is a CAS with the ability to stat and delete its content. +/// Besides, IManifestStore provides reference tagging. +/// +public interface IManifestStore : IBlobStore, IReferencePushable, ITaggable { - /// - /// IBlobStore is a CAS with the ability to stat and delete its content. - /// - public interface IBlobStore : IStorage, IResolvable, IDeletable, IReferenceFetcher - { - } } diff --git a/src/OrasProject.Oras/Interfaces/Registry/IReferenceFetcher.cs b/src/OrasProject.Oras/Registry/IReferenceFetchable.cs similarity index 56% rename from src/OrasProject.Oras/Interfaces/Registry/IReferenceFetcher.cs rename to src/OrasProject.Oras/Registry/IReferenceFetchable.cs index 26d2cc6..46a3d17 100644 --- a/src/OrasProject.Oras/Interfaces/Registry/IReferenceFetcher.cs +++ b/src/OrasProject.Oras/Registry/IReferenceFetchable.cs @@ -16,19 +16,18 @@ using System.Threading; using System.Threading.Tasks; -namespace OrasProject.Oras.Interfaces.Registry +namespace OrasProject.Oras.Registry; + +/// +/// Provides advanced fetch with the tag service. +/// +public interface IReferenceFetchable { /// - /// IReferenceFetcher provides advanced fetch with the tag service. + /// Fetches the content identified by the reference. /// - public interface IReferenceFetcher - { - /// - /// FetchReferenceAsync fetches the content identified by the reference. - /// - /// - /// - /// - Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default); - } + /// + /// + /// + Task<(Descriptor Descriptor, Stream Stream)> FetchAsync(string reference, CancellationToken cancellationToken = default); } diff --git a/src/OrasProject.Oras/Interfaces/Registry/IReferencePusher.cs b/src/OrasProject.Oras/Registry/IReferencePushable.cs similarity index 53% rename from src/OrasProject.Oras/Interfaces/Registry/IReferencePusher.cs rename to src/OrasProject.Oras/Registry/IReferencePushable.cs index 1d6fd3c..7a4e55e 100644 --- a/src/OrasProject.Oras/Interfaces/Registry/IReferencePusher.cs +++ b/src/OrasProject.Oras/Registry/IReferencePushable.cs @@ -16,21 +16,20 @@ using System.Threading; using System.Threading.Tasks; -namespace OrasProject.Oras.Interfaces.Registry +namespace OrasProject.Oras.Registry; + +/// +/// Provides advanced push with the tag service. +/// +public interface IReferencePushable { /// - /// IReferencePusher provides advanced push with the tag service. + /// PushReferenceAsync pushes the manifest with a reference tag. /// - public interface IReferencePusher - { - /// - /// PushReferenceAsync pushes the manifest with a reference tag. - /// - /// - /// - /// - /// - /// - Task PushReferenceAsync(Descriptor descriptor, Stream content, string reference, CancellationToken cancellationToken = default); - } + /// + /// + /// + /// + /// + Task PushAsync(Descriptor descriptor, Stream content, string reference, CancellationToken cancellationToken = default); } diff --git a/src/OrasProject.Oras/Registry/IRegistry.cs b/src/OrasProject.Oras/Registry/IRegistry.cs new file mode 100644 index 0000000..c4db55a --- /dev/null +++ b/src/OrasProject.Oras/Registry/IRegistry.cs @@ -0,0 +1,49 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace OrasProject.Oras.Registry; + +public interface IRegistry +{ + /// + /// Returns a repository reference by the given name. + /// + /// + /// + /// + Task GetRepository(string name, CancellationToken cancellationToken = default); + + /// + /// Repositories lists the name of repositories available in the registry. + /// Since the returned repositories may be paginated by the underlying + /// implementation, a function should be passed in to process the paginated + /// repository list. + /// `last` argument is the `last` parameter when invoking the catalog API. + /// If `last` is NOT empty, the entries in the response start after the + /// repo specified by `last`. Otherwise, the response starts from the top + /// of the Repositories list. + /// Note: When implemented by a remote registry, the catalog API is called. + /// However, not all registries supports pagination or conforms the + /// specification. + /// Reference: https://docs.docker.com/registry/spec/api/#catalog + /// See also `Repositories()` in this package. + /// + /// + /// + /// + IAsyncEnumerable ListRepositoriesAsync(string? last = default, CancellationToken cancellationToken = default); +} diff --git a/src/OrasProject.Oras/Registry/IRepository.cs b/src/OrasProject.Oras/Registry/IRepository.cs new file mode 100644 index 0000000..b163e2f --- /dev/null +++ b/src/OrasProject.Oras/Registry/IRepository.cs @@ -0,0 +1,41 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Content; + +namespace OrasProject.Oras.Registry; + +/// +/// Repository is an ORAS target and an union of the blob and the manifest CASs. +/// As specified by https://docs.docker.com/registry/spec/api/, it is natural to +/// assume that IResolver interface only works for manifests. Tagging a +/// blob may be resulted in an `UnsupportedException` error. However, this interface +/// does not restrict tagging blobs. +/// Since a repository is an union of the blob and the manifest CASs, all +/// operations defined in the `IBlobStore` are executed depending on the media +/// type of the given descriptor accordingly. +/// Furthermore, this interface also provides the ability to enforce the +/// separation of the blob and the manifests CASs. +/// +public interface IRepository : ITarget, IReferenceFetchable, IReferencePushable, IDeletable, ITagListable +{ + /// + /// Blobs provides access to the blob CAS only, which contains config blobs,layers, and other generic blobs. + /// + IBlobStore Blobs { get; } + + /// + /// Manifests provides access to the manifest CAS only. + /// + IManifestStore Manifests { get; } +} diff --git a/src/OrasProject.Oras/Registry/ITagListable.cs b/src/OrasProject.Oras/Registry/ITagListable.cs new file mode 100644 index 0000000..510e64f --- /dev/null +++ b/src/OrasProject.Oras/Registry/ITagListable.cs @@ -0,0 +1,40 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; +using System.Threading; + +namespace OrasProject.Oras.Registry; + +/// +/// Lists tags by the tag service. +/// +public interface ITagListable +{ + /// + /// Lists the tags available in the repository. + /// Since the returned tag list may be paginated by the underlying + /// implementation, a function should be passed in to process the paginated + /// tag list. + /// Note: When implemented by a remote registry, the tags API is called. + /// However, not all registries supports pagination or conforms the + /// specification. + /// References: + /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md + /// - https://docs.docker.com/registry/spec/api/#tags + /// + /// The `last` parameter when invoking the tags API. If `last` is NOT empty, the entries in the response start after the tag specified by `last`. Otherwise, the response starts from the top of the Tags list. + /// + /// + IAsyncEnumerable ListTagsAsync(string? last = default, CancellationToken cancellationToken = default); +} diff --git a/src/OrasProject.Oras/Remote/Auth/HttpClientWithBasicAuth.cs b/src/OrasProject.Oras/Registry/Remote/Auth/HttpClientWithBasicAuth.cs similarity index 100% rename from src/OrasProject.Oras/Remote/Auth/HttpClientWithBasicAuth.cs rename to src/OrasProject.Oras/Registry/Remote/Auth/HttpClientWithBasicAuth.cs diff --git a/src/OrasProject.Oras/Registry/Remote/BlobStore.cs b/src/OrasProject.Oras/Registry/Remote/BlobStore.cs new file mode 100644 index 0000000..dd74572 --- /dev/null +++ b/src/OrasProject.Oras/Registry/Remote/BlobStore.cs @@ -0,0 +1,227 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Content; +using OrasProject.Oras.Exceptions; +using OrasProject.Oras.Oci; +using OrasProject.Oras.Remote; +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace OrasProject.Oras.Registry.Remote; + +internal class BlobStore : IBlobStore +{ + + public Repository Repository { get; set; } + + public BlobStore(Repository repository) + { + Repository = repository; + + } + + + public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) + { + var remoteReference = Repository.RemoteReference; + Digest.Validate(target.Digest); + remoteReference.Reference = target.Digest; + var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); + var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); + switch (resp.StatusCode) + { + case HttpStatusCode.OK: + // server does not support seek as `Range` was ignored. + if (resp.Content.Headers.ContentLength is var size && size != -1 && size != target.Size) + { + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); + } + return await resp.Content.ReadAsStreamAsync(); + case HttpStatusCode.NotFound: + throw new NotFoundException($"{target.Digest}: not found"); + default: + throw await ErrorUtility.ParseErrorResponse(resp); + } + } + + /// + /// ExistsAsync returns true if the described content exists. + /// + /// + /// + /// + public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) + { + try + { + await ResolveAsync(target.Digest, cancellationToken); + return true; + } + catch (NotFoundException) + { + return false; + } + + } + + /// + /// PushAsync pushes the content, matching the expected descriptor. + /// Existing content is not checked by PushAsync() to minimize the number of out-going + /// requests. + /// Push is done by conventional 2-step monolithic upload instead of a single + /// `POST` request for better overall performance. It also allows early fail on + /// authentication errors. + /// References: + /// - https://docs.docker.com/registry/spec/api/#pushing-an-image + /// - https://docs.docker.com/registry/spec/api/#initiate-blob-upload + /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pushing-a-blob-monolithically + /// + /// + /// + /// + /// + public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) + { + var url = URLUtiliity.BuildRepositoryBlobUploadURL(Repository.PlainHTTP, Repository.RemoteReference); + using var resp = await Repository.HttpClient.PostAsync(url, null, cancellationToken); + var reqHostname = resp.RequestMessage.RequestUri.Host; + var reqPort = resp.RequestMessage.RequestUri.Port; + if (resp.StatusCode != HttpStatusCode.Accepted) + { + throw await ErrorUtility.ParseErrorResponse(resp); + } + + string location; + // monolithic upload + if (!resp.Headers.Location.IsAbsoluteUri) + { + location = resp.RequestMessage.RequestUri.Scheme + "://" + resp.RequestMessage.RequestUri.Authority + resp.Headers.Location; + } + else + { + location = resp.Headers.Location.ToString(); + } + // work-around solution for https://github.com/oras-project/oras-go/issues/177 + // For some registries, if the port 443 is explicitly set to the hostname plicitly set to the hostname + // like registry.wabbit-networks.io:443/myrepo, blob push will fail since + // the hostname of the Location header in the response is set to + // registry.wabbit-networks.io instead of registry.wabbit-networks.io:443. + var uri = new UriBuilder(location); + var locationHostname = uri.Host; + var locationPort = uri.Port; + // if location port 443 is missing, add it back + if (reqPort == 443 && locationHostname == reqHostname && locationPort != reqPort) + { + location = new UriBuilder($"{locationHostname}:{reqPort}").ToString(); + } + + url = location; + + var req = new HttpRequestMessage(HttpMethod.Put, url); + req.Content = new StreamContent(content); + req.Content.Headers.ContentLength = expected.Size; + + // the expected media type is ignored as in the API doc. + req.Content.Headers.Add("Content-Type", "application/octet-stream"); + + // add digest key to query string with expected digest value + req.RequestUri = new UriBuilder($"{req.RequestUri}?digest={expected.Digest}").Uri; + + //reuse credential from previous POST request + resp.Headers.TryGetValues("Authorization", out var auth); + if (auth != null) + { + req.Headers.Add("Authorization", auth.FirstOrDefault()); + } + using var resp2 = await Repository.HttpClient.SendAsync(req, cancellationToken); + if (resp2.StatusCode != HttpStatusCode.Created) + { + throw await ErrorUtility.ParseErrorResponse(resp2); + } + + return; + } + + /// + /// ResolveAsync resolves a reference to a descriptor. + /// + /// + /// + /// + public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) + { + var remoteReference = Repository.ParseReference(reference); + var refDigest = remoteReference.Digest(); + var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); + var requestMessage = new HttpRequestMessage(HttpMethod.Head, url); + using var resp = await Repository.HttpClient.SendAsync(requestMessage, cancellationToken); + return resp.StatusCode switch + { + HttpStatusCode.OK => Repository.GenerateBlobDescriptor(resp, refDigest), + HttpStatusCode.NotFound => throw new NotFoundException($"{remoteReference.Reference}: not found"), + _ => throw await ErrorUtility.ParseErrorResponse(resp) + }; + } + + /// + /// DeleteAsync deletes the content identified by the given descriptor. + /// + /// + /// + /// + public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) + { + await Repository.DeleteAsync(target, false, cancellationToken); + } + + /// + /// FetchReferenceAsync fetches the blob identified by the reference. + /// The reference must be a digest. + /// + /// + /// + /// + public async Task<(Descriptor Descriptor, Stream Stream)> FetchAsync(string reference, CancellationToken cancellationToken = default) + { + var remoteReference = Repository.ParseReference(reference); + var refDigest = remoteReference.Digest(); + var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); + var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); + switch (resp.StatusCode) + { + case HttpStatusCode.OK: + // server does not support seek as `Range` was ignored. + Descriptor desc; + if (resp.Content.Headers.ContentLength == -1) + { + desc = await ResolveAsync(refDigest, cancellationToken); + } + else + { + desc = Repository.GenerateBlobDescriptor(resp, refDigest); + } + + return (desc, await resp.Content.ReadAsStreamAsync()); + case HttpStatusCode.NotFound: + throw new NotFoundException(); + default: + throw await ErrorUtility.ParseErrorResponse(resp); + } + } +} diff --git a/src/OrasProject.Oras/Remote/ErrorUtility.cs b/src/OrasProject.Oras/Registry/Remote/ErrorUtility.cs similarity index 100% rename from src/OrasProject.Oras/Remote/ErrorUtility.cs rename to src/OrasProject.Oras/Registry/Remote/ErrorUtility.cs diff --git a/src/OrasProject.Oras/Remote/HttpClientExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpClientExtensions.cs similarity index 100% rename from src/OrasProject.Oras/Remote/HttpClientExtensions.cs rename to src/OrasProject.Oras/Registry/Remote/HttpClientExtensions.cs diff --git a/src/OrasProject.Oras/Registry/Remote/IRepositoryOption.cs b/src/OrasProject.Oras/Registry/Remote/IRepositoryOption.cs new file mode 100644 index 0000000..5db0845 --- /dev/null +++ b/src/OrasProject.Oras/Registry/Remote/IRepositoryOption.cs @@ -0,0 +1,56 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Remote; +using System.Net.Http; + +namespace OrasProject.Oras.Registry.Remote; + +/// +/// IRepositoryOption is used to configure a remote repository. +/// +public interface IRepositoryOption +{ + /// + /// Client is the underlying HTTP client used to access the remote registry. + /// + public HttpClient HttpClient { get; set; } + + /// + /// Reference references the remote repository. + /// + public RemoteReference RemoteReference { get; set; } + + /// + /// PlainHTTP signals the transport to access the remote repository via HTTP + /// instead of HTTPS. + /// + public bool PlainHTTP { get; set; } + + + /// + /// ManifestMediaTypes is used in `Accept` header for resolving manifests + /// from references. It is also used in identifying manifests and blobs from + /// descriptors. If an empty list is present, default manifest media types + /// are used. + /// + public string[] ManifestMediaTypes { get; set; } + + /// + /// TagListPageSize specifies the page size when invoking the tag list API. + /// If zero, the page size is determined by the remote registry. + /// Reference: https://docs.docker.com/registry/spec/api/#tags + /// + public int TagListPageSize { get; set; } + +} diff --git a/src/OrasProject.Oras/Remote/LinkUtility.cs b/src/OrasProject.Oras/Registry/Remote/LinkUtility.cs similarity index 100% rename from src/OrasProject.Oras/Remote/LinkUtility.cs rename to src/OrasProject.Oras/Registry/Remote/LinkUtility.cs diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs new file mode 100644 index 0000000..2baef97 --- /dev/null +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -0,0 +1,344 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Content; +using OrasProject.Oras.Exceptions; +using OrasProject.Oras.Oci; +using OrasProject.Oras.Remote; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; + +namespace OrasProject.Oras.Registry.Remote; + +public class ManifestStore : IManifestStore +{ + public Repository Repository { get; set; } + + public ManifestStore(Repository repository) + { + Repository = repository; + } + + /// + /// FetchASync fetches the content identified by the descriptor. + /// + /// + /// + /// + /// + /// + public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) + { + var remoteReference = Repository.RemoteReference; + remoteReference.Reference = target.Digest; + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); + var req = new HttpRequestMessage(HttpMethod.Get, url); + req.Headers.Add("Accept", target.MediaType); + var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); + + switch (resp.StatusCode) + { + case HttpStatusCode.OK: + break; + case HttpStatusCode.NotFound: + throw new NotFoundException($"digest {target.Digest} not found"); + default: + throw await ErrorUtility.ParseErrorResponse(resp); + } + var mediaType = resp.Content.Headers?.ContentType.MediaType; + if (mediaType != target.MediaType) + { + throw new Exception( + $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch response Content-Type {mediaType}: expect {target.MediaType}"); + } + if (resp.Content.Headers.ContentLength is var size && size != -1 && size != target.Size) + { + throw new Exception( + $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); + } + Repository.VerifyContentDigest(resp, target.Digest); + return await resp.Content.ReadAsStreamAsync(); + } + + /// + /// ExistsAsync returns true if the described content exists. + /// + /// + /// + /// + public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) + { + try + { + await ResolveAsync(target.Digest, cancellationToken); + return true; + } + catch (NotFoundException) + { + return false; + } + + } + + /// + /// PushAsync pushes the content, matching the expected descriptor. + /// + /// + /// + /// + /// + public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) + { + await InternalPushAsync(expected, content, expected.Digest, cancellationToken); + } + + + /// + /// PushAsync pushes the manifest content, matching the expected descriptor. + /// + /// + /// + /// + /// + private async Task InternalPushAsync(Descriptor expected, Stream stream, string reference, CancellationToken cancellationToken) + { + var remoteReference = Repository.RemoteReference; + remoteReference.Reference = reference; + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); + var req = new HttpRequestMessage(HttpMethod.Put, url); + req.Content = new StreamContent(stream); + req.Content.Headers.ContentLength = expected.Size; + req.Content.Headers.Add("Content-Type", expected.MediaType); + var client = Repository.HttpClient; + using var resp = await client.SendAsync(req, cancellationToken); + if (resp.StatusCode != HttpStatusCode.Created) + { + throw await ErrorUtility.ParseErrorResponse(resp); + } + Repository.VerifyContentDigest(resp, expected.Digest); + } + + public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) + { + var remoteReference = Repository.ParseReference(reference); + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); + var req = new HttpRequestMessage(HttpMethod.Head, url); + req.Headers.Add("Accept", ManifestUtility.ManifestAcceptHeader(Repository.ManifestMediaTypes)); + using var res = await Repository.HttpClient.SendAsync(req, cancellationToken); + + return res.StatusCode switch + { + HttpStatusCode.OK => await GenerateDescriptorAsync(res, remoteReference, req.Method), + HttpStatusCode.NotFound => throw new NotFoundException($"reference {reference} not found"), + _ => throw await ErrorUtility.ParseErrorResponse(res) + }; + } + + /// + /// GenerateDescriptor returns a descriptor generated from the response. + /// + /// + /// + /// + /// + /// + public async Task GenerateDescriptorAsync(HttpResponseMessage res, RemoteReference reference, HttpMethod httpMethod) + { + string mediaType; + try + { + // 1. Validate Content-Type + mediaType = res.Content.Headers.ContentType.MediaType; + MediaTypeHeaderValue.Parse(mediaType); + } + catch (Exception e) + { + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: invalid response `Content-Type` header; {e.Message}"); + } + + // 2. Validate Size + if (!res.Content.Headers.ContentLength.HasValue || res.Content.Headers.ContentLength == -1) + { + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: unknown response Content-Length"); + } + + // 3. Validate Client Reference + string refDigest = string.Empty; + try + { + refDigest = reference.Digest(); + } + catch (Exception) + { + } + + + // 4. Validate Server Digest (if present) + res.Content.Headers.TryGetValues("Docker-Content-Digest", out IEnumerable serverHeaderDigest); + var serverDigest = serverHeaderDigest?.First(); + if (!string.IsNullOrEmpty(serverDigest)) + { + try + { + Repository.VerifyContentDigest(res, serverDigest); + } + catch (Exception) + { + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: invalid response header value: `Docker-Content-Digest: {serverHeaderDigest}`"); + } + } + + // 5. Now, look for specific error conditions; + string contentDigest; + + if (string.IsNullOrEmpty(serverDigest)) + { + if (httpMethod == HttpMethod.Head) + { + if (string.IsNullOrEmpty(refDigest)) + { + // HEAD without server `Docker-Content-Digest` + // immediate fail + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: HTTP {httpMethod} request missing required header {serverHeaderDigest}"); + } + // Otherwise, just trust the client-supplied digest + contentDigest = refDigest; + } + else + { + // GET without server `Docker-Content-Digest header forces the + // expensive calculation + string calculatedDigest; + try + { + calculatedDigest = await CalculateDigestFromResponse(res); + } + catch (Exception e) + { + throw new Exception($"failed to calculate digest on response body; {e.Message}"); + } + contentDigest = calculatedDigest; + } + } + else + { + contentDigest = serverDigest; + } + if (!string.IsNullOrEmpty(refDigest) && refDigest != contentDigest) + { + throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: invalid response; digest mismatch in {serverHeaderDigest}: received {contentDigest} when expecting {refDigest}"); + } + + // 6. Finally, if we made it this far, then all is good; return the descriptor + return new Descriptor + { + MediaType = mediaType, + Digest = contentDigest, + Size = res.Content.Headers.ContentLength.Value + }; + } + + /// + /// CalculateDigestFromResponse calculates the actual digest of the response body + /// taking care not to destroy it in the process + /// + /// + static async Task CalculateDigestFromResponse(HttpResponseMessage res) + { + var bytes = await res.Content.ReadAsByteArrayAsync(); + return Digest.ComputeSHA256(bytes); + } + + /// + /// DeleteAsync removes the manifest content identified by the descriptor. + /// + /// + /// + /// + public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) + { + await Repository.DeleteAsync(target, true, cancellationToken); + } + + + /// + /// FetchReferenceAsync fetches the manifest identified by the reference. + /// + /// + /// + /// + public async Task<(Descriptor Descriptor, Stream Stream)> FetchAsync(string reference, CancellationToken cancellationToken = default) + { + var remoteReference = Repository.ParseReference(reference); + var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); + var req = new HttpRequestMessage(HttpMethod.Get, url); + req.Headers.Add("Accept", ManifestUtility.ManifestAcceptHeader(Repository.ManifestMediaTypes)); + var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); + switch (resp.StatusCode) + { + case HttpStatusCode.OK: + Descriptor desc; + if (resp.Content.Headers.ContentLength == -1) + { + desc = await ResolveAsync(reference, cancellationToken); + } + else + { + desc = await GenerateDescriptorAsync(resp, remoteReference, HttpMethod.Get); + } + + return (desc, await resp.Content.ReadAsStreamAsync()); + case HttpStatusCode.NotFound: + throw new NotFoundException($"{req.Method} {req.RequestUri}: manifest unknown"); + default: + throw await ErrorUtility.ParseErrorResponse(resp); + + } + } + + /// + /// PushReferenceASync pushes the manifest with a reference tag. + /// + /// + /// + /// + /// + /// + public async Task PushAsync(Descriptor expected, Stream content, string reference, + CancellationToken cancellationToken = default) + { + var remoteReference = Repository.ParseReference(reference); + await InternalPushAsync(expected, content, remoteReference.Reference, cancellationToken); + } + + /// + /// TagAsync tags a manifest descriptor with a reference string. + /// + /// + /// + /// + /// + public async Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default) + { + var remoteReference = Repository.ParseReference(reference); + var rc = await FetchAsync(descriptor, cancellationToken); + await InternalPushAsync(descriptor, rc, remoteReference.Reference, cancellationToken); + } +} diff --git a/src/OrasProject.Oras/Remote/ManifestUtility.cs b/src/OrasProject.Oras/Registry/Remote/ManifestUtility.cs similarity index 100% rename from src/OrasProject.Oras/Remote/ManifestUtility.cs rename to src/OrasProject.Oras/Registry/Remote/ManifestUtility.cs diff --git a/src/OrasProject.Oras/Remote/Registry.cs b/src/OrasProject.Oras/Registry/Remote/Registry.cs similarity index 80% rename from src/OrasProject.Oras/Remote/Registry.cs rename to src/OrasProject.Oras/Registry/Remote/Registry.cs index 4ed4715..608d5ca 100644 --- a/src/OrasProject.Oras/Remote/Registry.cs +++ b/src/OrasProject.Oras/Registry/Remote/Registry.cs @@ -12,10 +12,13 @@ // limitations under the License. using OrasProject.Oras.Exceptions; -using OrasProject.Oras.Interfaces.Registry; +using OrasProject.Oras.Registry; +using OrasProject.Oras.Registry.Remote; using System; +using System.Collections.Generic; using System.Net; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -86,7 +89,7 @@ public async Task PingAsync(CancellationToken cancellationToken) /// /// /// - public Task Repository(string name, CancellationToken cancellationToken) + public Task GetRepository(string name, CancellationToken cancellationToken) { var reference = new RemoteReference { @@ -95,30 +98,35 @@ public Task Repository(string name, CancellationToken cancellationT }; return Task.FromResult(new Repository(reference, this)); - } - - + } + + /// /// Repositories returns a list of repositories from the remote registry. /// /// - /// /// /// - public async Task Repositories(string last, Action fn, CancellationToken cancellationToken) - { - try - { - var url = URLUtiliity.BuildRegistryCatalogURL(PlainHTTP, RemoteReference); - while (true) + public async IAsyncEnumerable ListRepositoriesAsync(string? last = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var url = URLUtiliity.BuildRegistryCatalogURL(PlainHTTP, RemoteReference); + var done = false; + while (!done) + { + IEnumerable repositories = Array.Empty(); + try { - url = await RepositoryPageAsync(last, fn, url, cancellationToken); + url = await RepositoryPageAsync(last, values => repositories = values, url, cancellationToken); last = ""; } - } - catch (LinkUtility.NoLinkHeaderException) - { - return; + catch (LinkUtility.NoLinkHeaderException) + { + done = true; + } + foreach (var repository in repositories) + { + yield return repository; + } } } @@ -130,11 +138,11 @@ public async Task Repositories(string last, Action fn, CancellationTok /// /// /// - private async Task RepositoryPageAsync(string last, Action fn, string url, CancellationToken cancellationToken) + private async Task RepositoryPageAsync(string? last, Action fn, string url, CancellationToken cancellationToken) { var uriBuilder = new UriBuilder(url); var query = ParseQueryString(uriBuilder.Query); - if (TagListPageSize > 0 || last != "") + if (TagListPageSize > 0 || !string.IsNullOrEmpty(last)) { if (TagListPageSize > 0) { @@ -159,6 +167,6 @@ private async Task RepositoryPageAsync(string last, Action fn, var repositories = JsonSerializer.Deserialize(data); fn(repositories.Repositories); return LinkUtility.ParseLink(response); - } + } } } diff --git a/src/OrasProject.Oras/Remote/RemoteReference.cs b/src/OrasProject.Oras/Registry/Remote/RemoteReference.cs similarity index 100% rename from src/OrasProject.Oras/Remote/RemoteReference.cs rename to src/OrasProject.Oras/Registry/Remote/RemoteReference.cs diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs new file mode 100644 index 0000000..c284e42 --- /dev/null +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -0,0 +1,483 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Content; +using OrasProject.Oras.Exceptions; +using OrasProject.Oras.Oci; +using OrasProject.Oras.Remote; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using static System.Web.HttpUtility; + +namespace OrasProject.Oras.Registry.Remote; + +/// +/// Repository is an HTTP client to a remote repository +/// +public class Repository : IRepository, IRepositoryOption +{ + /// + /// HttpClient is the underlying HTTP client used to access the remote registry. + /// + public HttpClient HttpClient { get; set; } + + /// + /// ReferenceObj references the remote repository. + /// + public RemoteReference RemoteReference { get; set; } + + /// + /// PlainHTTP signals the transport to access the remote repository via HTTP + /// instead of HTTPS. + /// + public bool PlainHTTP { get; set; } + + + /// + /// ManifestMediaTypes is used in `Accept` header for resolving manifests + /// from references. It is also used in identifying manifests and blobs from + /// descriptors. If an empty list is present, default manifest media types + /// are used. + /// + public string[] ManifestMediaTypes { get; set; } + + /// + /// TagListPageSize specifies the page size when invoking the tag list API. + /// If zero, the page size is determined by the remote registry. + /// Reference: https://docs.docker.com/registry/spec/api/#tags + /// + public int TagListPageSize { get; set; } + + /// + /// Creates a client to the remote repository identified by a reference + /// Example: localhost:5000/hello-world + /// + /// + public Repository(string reference) + { + RemoteReference = RemoteReference.ParseReference(reference); + HttpClient = new HttpClient(); + HttpClient.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); + } + + /// + /// Creates a client to the remote repository using a reference and a HttpClient + /// + /// + /// + public Repository(string reference, HttpClient httpClient) + { + RemoteReference = RemoteReference.ParseReference(reference); + HttpClient = httpClient; + } + + /// + /// This constructor customizes the HttpClient and sets the properties + /// using values from the parameter. + /// + /// + /// + internal Repository(RemoteReference reference, IRepositoryOption option) + { + reference.ValidateRepository(); + HttpClient = option.HttpClient; + RemoteReference = reference; + ManifestMediaTypes = option.ManifestMediaTypes; + PlainHTTP = option.PlainHTTP; + TagListPageSize = option.TagListPageSize; + } + + /// + /// BlobStore detects the blob store for the given descriptor. + /// + /// + /// + private IBlobStore BlobStore(Descriptor desc) + { + if (ManifestUtility.IsManifest(ManifestMediaTypes, desc)) + { + return Manifests; + } + + return Blobs; + } + + + + /// + /// FetchAsync fetches the content identified by the descriptor. + /// + /// + /// + /// + public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) + { + return await BlobStore(target).FetchAsync(target, cancellationToken); + } + + /// + /// ExistsAsync returns true if the described content exists. + /// + /// + /// + /// + public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) + { + return await BlobStore(target).ExistsAsync(target, cancellationToken); + } + + /// + /// PushAsync pushes the content, matching the expected descriptor. + /// + /// + /// + /// + /// + public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) + { + await BlobStore(expected).PushAsync(expected, content, cancellationToken); + } + + /// + /// ResolveAsync resolves a reference to a manifest descriptor + /// See all ManifestMediaTypes + /// + /// + /// + /// + public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) + { + return await Manifests.ResolveAsync(reference, cancellationToken); + } + + /// + /// TagAsync tags a manifest descriptor with a reference string. + /// + /// + /// + /// + /// + public async Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default) + { + await Manifests.TagAsync(descriptor, reference, cancellationToken); + } + + /// + /// FetchReference fetches the manifest identified by the reference. + /// The reference can be a tag or digest. + /// + /// + /// + /// + public async Task<(Descriptor Descriptor, Stream Stream)> FetchAsync(string reference, CancellationToken cancellationToken = default) + { + return await Manifests.FetchAsync(reference, cancellationToken); + } + + /// + /// PushReference pushes the manifest with a reference tag. + /// + /// + /// + /// + /// + /// + public async Task PushAsync(Descriptor descriptor, Stream content, string reference, + CancellationToken cancellationToken = default) + { + await Manifests.PushAsync(descriptor, content, reference, cancellationToken); + } + + /// + /// DeleteAsync removes the content identified by the descriptor. + /// + /// + /// + /// + public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) + { + await BlobStore(target).DeleteAsync(target, cancellationToken); + } + + /// + /// TagsAsync returns a list of tags in a repository + /// + /// + /// + /// + public async Task> TagsAsync(ITagListable repo, CancellationToken cancellationToken) + { + var tags = new List(); + await foreach (var tag in repo.ListTagsAsync().WithCancellation(cancellationToken)) + { + tags.Add(tag); + } + return tags; + } + + /// + /// TagsAsync lists the tags available in the repository. + /// See also `TagListPageSize`. + /// If `last` is NOT empty, the entries in the response start after the + /// tag specified by `last`. Otherwise, the response starts from the top + /// of the Tags list. + /// References: + /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-discovery + /// - https://docs.docker.com/registry/spec/api/#tags + /// + /// + /// + /// + /// + public async IAsyncEnumerable ListTagsAsync(string? last = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var url = URLUtiliity.BuildRepositoryTagListURL(PlainHTTP, RemoteReference); + var done = false; + while (!done) + { + IEnumerable tags = Array.Empty(); + try + { + url = await TagsPageAsync(last, values => tags = values, url, cancellationToken); + last = ""; + } + catch (LinkUtility.NoLinkHeaderException) + { + done = true; + } + foreach (var tag in tags) + { + yield return tag; + } + } + } + + /// + /// TagsPageAsync returns a single page of tag list with the next link. + /// + /// + /// + /// + /// + private async Task TagsPageAsync(string? last, Action fn, string url, CancellationToken cancellationToken) + { + var uriBuilder = new UriBuilder(url); + var query = ParseQueryString(uriBuilder.Query); + if (TagListPageSize > 0 || !string.IsNullOrEmpty(last)) + { + if (TagListPageSize > 0) + { + query["n"] = TagListPageSize.ToString(); + } + if (!string.IsNullOrEmpty(last)) + { + query["last"] = last; + } + } + + uriBuilder.Query = query.ToString(); + using var resp = await HttpClient.GetAsync(uriBuilder.ToString(), cancellationToken); + if (resp.StatusCode != HttpStatusCode.OK) + { + throw await ErrorUtility.ParseErrorResponse(resp); + + } + var data = await resp.Content.ReadAsStringAsync(); + var tagList = JsonSerializer.Deserialize(data); + fn(tagList.Tags); + return LinkUtility.ParseLink(resp); + } + + /// + /// DeleteAsync removes the content identified by the descriptor in the + /// entity blobs or manifests. + /// + /// + /// + /// + /// + /// + /// + internal async Task DeleteAsync(Descriptor target, bool isManifest, CancellationToken cancellationToken) + { + var remoteReference = RemoteReference; + remoteReference.Reference = target.Digest; + string url; + if (isManifest) + { + url = URLUtiliity.BuildRepositoryManifestURL(PlainHTTP, remoteReference); + } + else + { + url = URLUtiliity.BuildRepositoryBlobURL(PlainHTTP, remoteReference); + } + + using var resp = await HttpClient.DeleteAsync(url, cancellationToken); + + switch (resp.StatusCode) + { + case HttpStatusCode.Accepted: + VerifyContentDigest(resp, target.Digest); + break; + case HttpStatusCode.NotFound: + throw new NotFoundException($"digest {target.Digest} not found"); + default: + throw await ErrorUtility.ParseErrorResponse(resp); + } + } + + + /// + /// VerifyContentDigest verifies "Docker-Content-Digest" header if present. + /// OCI distribution-spec states the Docker-Content-Digest header is optional. + /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#legacy-docker-support-http-headers + /// + /// + /// + /// + internal static void VerifyContentDigest(HttpResponseMessage resp, string expected) + { + if (!resp.Content.Headers.TryGetValues("Docker-Content-Digest", out var digestValues)) return; + var digestStr = digestValues.FirstOrDefault(); + if (string.IsNullOrEmpty(digestStr)) + { + return; + } + + string contentDigest; + try + { + contentDigest = Digest.Validate(digestStr); + } + catch (Exception) + { + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response header: `Docker-Content-Digest: {digestStr}`"); + } + if (contentDigest != expected) + { + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {digestStr}"); + } + } + + + /// + /// Blobs provides access to the blob CAS only, which contains + /// layers, and other generic blobs. + /// + public IBlobStore Blobs => new BlobStore(this); + + + /// + /// Manifests provides access to the manifest CAS only. + /// + /// + public IManifestStore Manifests => new ManifestStore(this); + + /// + /// ParseReference resolves a tag or a digest reference to a fully qualified + /// reference from a base reference Reference. + /// Tag, digest, or fully qualified references are accepted as input. + /// If reference is a fully qualified reference, then ParseReference parses it + /// and returns the parsed reference. If the parsed reference does not share + /// the same base reference with the Repository, ParseReference throws an + /// error, InvalidReferenceException. + /// + /// + /// + public RemoteReference ParseReference(string reference) + { + RemoteReference remoteReference; + var hasError = false; + try + { + remoteReference = RemoteReference.ParseReference(reference); + } + catch (Exception) + { + hasError = true; + remoteReference = new RemoteReference + { + Registry = RemoteReference.Registry, + Repository = RemoteReference.Repository, + Reference = reference + }; + //reference is not a FQDN + if (reference.IndexOf("@") is var index && index != -1) + { + // `@` implies *digest*, so drop the *tag* (irrespective of what it is). + remoteReference.Reference = reference[(index + 1)..]; + remoteReference.ValidateReferenceAsDigest(); + } + else + { + remoteReference.ValidateReference(); + } + + } + + if (!hasError) + { + if (remoteReference.Registry != RemoteReference.Registry || + remoteReference.Repository != RemoteReference.Repository) + { + throw new InvalidReferenceException( + $"mismatch between received {JsonSerializer.Serialize(remoteReference)} and expected {JsonSerializer.Serialize(RemoteReference)}"); + } + } + if (string.IsNullOrEmpty(remoteReference.Reference)) + { + throw new InvalidReferenceException(); + } + return remoteReference; + + } + + + /// + /// GenerateBlobDescriptor returns a descriptor generated from the response. + /// + /// + /// + /// + /// + public static Descriptor GenerateBlobDescriptor(HttpResponseMessage resp, string refDigest) + { + var mediaType = resp.Content.Headers.ContentType.MediaType; + if (string.IsNullOrEmpty(mediaType)) + { + mediaType = "application/octet-stream"; + } + var size = resp.Content.Headers.ContentLength.Value; + if (size == -1) + { + throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: unknown response Content-Length"); + } + + VerifyContentDigest(resp, refDigest); + + return new Descriptor + { + MediaType = mediaType, + Digest = refDigest, + Size = size + }; + } +} diff --git a/src/OrasProject.Oras/Remote/ResponseTypes.cs b/src/OrasProject.Oras/Registry/Remote/ResponseTypes.cs similarity index 100% rename from src/OrasProject.Oras/Remote/ResponseTypes.cs rename to src/OrasProject.Oras/Registry/Remote/ResponseTypes.cs diff --git a/src/OrasProject.Oras/Remote/URLUtiliity.cs b/src/OrasProject.Oras/Registry/Remote/URLUtiliity.cs similarity index 100% rename from src/OrasProject.Oras/Remote/URLUtiliity.cs rename to src/OrasProject.Oras/Registry/Remote/URLUtiliity.cs diff --git a/src/OrasProject.Oras/Remote/Repository.cs b/src/OrasProject.Oras/Remote/Repository.cs deleted file mode 100644 index 1640d14..0000000 --- a/src/OrasProject.Oras/Remote/Repository.cs +++ /dev/null @@ -1,1007 +0,0 @@ -// Copyright The ORAS Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using OrasProject.Oras.Content; -using OrasProject.Oras.Exceptions; -using OrasProject.Oras.Interfaces.Registry; -using OrasProject.Oras.Oci; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using static System.Web.HttpUtility; -namespace OrasProject.Oras.Remote -{ - - /// - /// Repository is an HTTP client to a remote repository - /// - public class Repository : IRepository, IRepositoryOption - { - /// - /// HttpClient is the underlying HTTP client used to access the remote registry. - /// - public HttpClient HttpClient { get; set; } - - /// - /// ReferenceObj references the remote repository. - /// - public RemoteReference RemoteReference { get; set; } - - /// - /// PlainHTTP signals the transport to access the remote repository via HTTP - /// instead of HTTPS. - /// - public bool PlainHTTP { get; set; } - - - /// - /// ManifestMediaTypes is used in `Accept` header for resolving manifests - /// from references. It is also used in identifying manifests and blobs from - /// descriptors. If an empty list is present, default manifest media types - /// are used. - /// - public string[] ManifestMediaTypes { get; set; } - - /// - /// TagListPageSize specifies the page size when invoking the tag list API. - /// If zero, the page size is determined by the remote registry. - /// Reference: https://docs.docker.com/registry/spec/api/#tags - /// - public int TagListPageSize { get; set; } - - /// - /// Creates a client to the remote repository identified by a reference - /// Example: localhost:5000/hello-world - /// - /// - public Repository(string reference) - { - RemoteReference = RemoteReference.ParseReference(reference); - HttpClient = new HttpClient(); - HttpClient.DefaultRequestHeaders.Add("User-Agent", new string[] { "oras-dotnet" }); - } - - /// - /// Creates a client to the remote repository using a reference and a HttpClient - /// - /// - /// - public Repository(string reference, HttpClient httpClient) - { - RemoteReference = RemoteReference.ParseReference(reference); - HttpClient = httpClient; - } - - /// - /// This constructor customizes the HttpClient and sets the properties - /// using values from the parameter. - /// - /// - /// - internal Repository(RemoteReference reference, IRepositoryOption option) - { - reference.ValidateRepository(); - HttpClient = option.HttpClient; - RemoteReference = reference; - ManifestMediaTypes = option.ManifestMediaTypes; - PlainHTTP = option.PlainHTTP; - TagListPageSize = option.TagListPageSize; - } - - /// - /// BlobStore detects the blob store for the given descriptor. - /// - /// - /// - private IBlobStore BlobStore(Descriptor desc) - { - if (ManifestUtility.IsManifest(ManifestMediaTypes, desc)) - { - return Manifests(); - } - - return Blobs(); - } - - - - /// - /// FetchAsync fetches the content identified by the descriptor. - /// - /// - /// - /// - public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) - { - return await BlobStore(target).FetchAsync(target, cancellationToken); - } - - /// - /// ExistsAsync returns true if the described content exists. - /// - /// - /// - /// - public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) - { - return await BlobStore(target).ExistsAsync(target, cancellationToken); - } - - /// - /// PushAsync pushes the content, matching the expected descriptor. - /// - /// - /// - /// - /// - public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) - { - await BlobStore(expected).PushAsync(expected, content, cancellationToken); - } - - /// - /// ResolveAsync resolves a reference to a manifest descriptor - /// See all ManifestMediaTypes - /// - /// - /// - /// - public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) - { - return await Manifests().ResolveAsync(reference, cancellationToken); - } - - /// - /// TagAsync tags a manifest descriptor with a reference string. - /// - /// - /// - /// - /// - public async Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default) - { - await Manifests().TagAsync(descriptor, reference, cancellationToken); - } - - /// - /// FetchReference fetches the manifest identified by the reference. - /// The reference can be a tag or digest. - /// - /// - /// - /// - public async Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) - { - return await Manifests().FetchReferenceAsync(reference, cancellationToken); - } - - /// - /// PushReference pushes the manifest with a reference tag. - /// - /// - /// - /// - /// - /// - public async Task PushReferenceAsync(Descriptor descriptor, Stream content, string reference, - CancellationToken cancellationToken = default) - { - await Manifests().PushReferenceAsync(descriptor, content, reference, cancellationToken); - } - - /// - /// DeleteAsync removes the content identified by the descriptor. - /// - /// - /// - /// - public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) - { - await BlobStore(target).DeleteAsync(target, cancellationToken); - } - - /// - /// TagsAsync returns a list of tags in a repository - /// - /// - /// - /// - public async Task> TagsAsync(ITagLister repo, CancellationToken cancellationToken) - { - var res = new List(); - await repo.TagsAsync( - string.Empty, - (tags) => - { - res.AddRange(tags); - - }, cancellationToken); - return res; - } - /// - /// TagsAsync lists the tags available in the repository. - /// See also `TagListPageSize`. - /// If `last` is NOT empty, the entries in the response start after the - /// tag specified by `last`. Otherwise, the response starts from the top - /// of the Tags list. - /// References: - /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-discovery - /// - https://docs.docker.com/registry/spec/api/#tags - /// - /// - /// - /// - /// - public async Task TagsAsync(string last, Action fn, CancellationToken cancellationToken = default) - { - try - { - var url = URLUtiliity.BuildRepositoryTagListURL(PlainHTTP, RemoteReference); - while (true) - { - url = await TagsPageAsync(last, fn, url, cancellationToken); - last = ""; - } - } - catch (LinkUtility.NoLinkHeaderException) - { - return; - } - - } - - /// - /// TagsPageAsync returns a single page of tag list with the next link. - /// - /// - /// - /// - /// - /// - private async Task TagsPageAsync(string last, Action fn, string url, CancellationToken cancellationToken) - { - var uriBuilder = new UriBuilder(url); - var query = ParseQueryString(uriBuilder.Query); - if (TagListPageSize > 0 || last != "") - { - if (TagListPageSize > 0) - { - query["n"] = TagListPageSize.ToString(); - } - if (last != "") - { - query["last"] = last; - } - } - - uriBuilder.Query = query.ToString(); - using var resp = await HttpClient.GetAsync(uriBuilder.ToString(), cancellationToken); - if (resp.StatusCode != HttpStatusCode.OK) - { - throw await ErrorUtility.ParseErrorResponse(resp); - - } - var data = await resp.Content.ReadAsStringAsync(); - var tagList = JsonSerializer.Deserialize(data); - fn(tagList.Tags); - return LinkUtility.ParseLink(resp); - } - - /// - /// DeleteAsync removes the content identified by the descriptor in the - /// entity blobs or manifests. - /// - /// - /// - /// - /// - /// - /// - internal async Task DeleteAsync(Descriptor target, bool isManifest, CancellationToken cancellationToken) - { - var remoteReference = RemoteReference; - remoteReference.Reference = target.Digest; - string url; - if (isManifest) - { - url = URLUtiliity.BuildRepositoryManifestURL(PlainHTTP, remoteReference); - } - else - { - url = URLUtiliity.BuildRepositoryBlobURL(PlainHTTP, remoteReference); - } - - using var resp = await HttpClient.DeleteAsync(url, cancellationToken); - - switch (resp.StatusCode) - { - case HttpStatusCode.Accepted: - VerifyContentDigest(resp, target.Digest); - break; - case HttpStatusCode.NotFound: - throw new NotFoundException($"digest {target.Digest} not found"); - default: - throw await ErrorUtility.ParseErrorResponse(resp); - } - } - - - /// - /// VerifyContentDigest verifies "Docker-Content-Digest" header if present. - /// OCI distribution-spec states the Docker-Content-Digest header is optional. - /// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#legacy-docker-support-http-headers - /// - /// - /// - /// - internal static void VerifyContentDigest(HttpResponseMessage resp, string expected) - { - if (!resp.Content.Headers.TryGetValues("Docker-Content-Digest", out var digestValues)) return; - var digestStr = digestValues.FirstOrDefault(); - if (string.IsNullOrEmpty(digestStr)) - { - return; - } - - string contentDigest; - try - { - contentDigest = Digest.Validate(digestStr); - } - catch (Exception) - { - throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response header: `Docker-Content-Digest: {digestStr}`"); - } - if (contentDigest != expected) - { - throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {digestStr}"); - } - } - - - /// - /// Blobs provides access to the blob CAS only, which contains - /// layers, and other generic blobs. - /// - /// - public IBlobStore Blobs() - { - return new BlobStore(this); - } - - - /// - /// Manifests provides access to the manifest CAS only. - /// - /// - public IManifestStore Manifests() - { - return new ManifestStore(this); - } - - /// - /// ParseReference resolves a tag or a digest reference to a fully qualified - /// reference from a base reference Reference. - /// Tag, digest, or fully qualified references are accepted as input. - /// If reference is a fully qualified reference, then ParseReference parses it - /// and returns the parsed reference. If the parsed reference does not share - /// the same base reference with the Repository, ParseReference throws an - /// error, InvalidReferenceException. - /// - /// - /// - public RemoteReference ParseReference(string reference) - { - RemoteReference remoteReference; - var hasError = false; - try - { - remoteReference = RemoteReference.ParseReference(reference); - } - catch (Exception) - { - hasError = true; - remoteReference = new RemoteReference - { - Registry = RemoteReference.Registry, - Repository = RemoteReference.Repository, - Reference = reference - }; - //reference is not a FQDN - if (reference.IndexOf("@") is var index && index != -1) - { - // `@` implies *digest*, so drop the *tag* (irrespective of what it is). - remoteReference.Reference = reference[(index + 1)..]; - remoteReference.ValidateReferenceAsDigest(); - } - else - { - remoteReference.ValidateReference(); - } - - } - - if (!hasError) - { - if (remoteReference.Registry != RemoteReference.Registry || - remoteReference.Repository != RemoteReference.Repository) - { - throw new InvalidReferenceException( - $"mismatch between received {JsonSerializer.Serialize(remoteReference)} and expected {JsonSerializer.Serialize(RemoteReference)}"); - } - } - if (string.IsNullOrEmpty(remoteReference.Reference)) - { - throw new InvalidReferenceException(); - } - return remoteReference; - - } - - - /// - /// GenerateBlobDescriptor returns a descriptor generated from the response. - /// - /// - /// - /// - /// - public static Descriptor GenerateBlobDescriptor(HttpResponseMessage resp, string refDigest) - { - var mediaType = resp.Content.Headers.ContentType.MediaType; - if (string.IsNullOrEmpty(mediaType)) - { - mediaType = "application/octet-stream"; - } - var size = resp.Content.Headers.ContentLength.Value; - if (size == -1) - { - throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: unknown response Content-Length"); - } - - VerifyContentDigest(resp, refDigest); - - return new Descriptor - { - MediaType = mediaType, - Digest = refDigest, - Size = size - }; - } - - } - - public class ManifestStore : IManifestStore - { - public Repository Repository { get; set; } - public ManifestStore(Repository repository) - { - Repository = repository; - - } - - /// - /// FetchASync fetches the content identified by the descriptor. - /// - /// - /// - /// - /// - /// - public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) - { - var remoteReference = Repository.RemoteReference; - remoteReference.Reference = target.Digest; - var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); - var req = new HttpRequestMessage(HttpMethod.Get, url); - req.Headers.Add("Accept", target.MediaType); - var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); - - switch (resp.StatusCode) - { - case HttpStatusCode.OK: - break; - case HttpStatusCode.NotFound: - throw new NotFoundException($"digest {target.Digest} not found"); - default: - throw await ErrorUtility.ParseErrorResponse(resp); - } - var mediaType = resp.Content.Headers?.ContentType.MediaType; - if (mediaType != target.MediaType) - { - throw new Exception( - $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch response Content-Type {mediaType}: expect {target.MediaType}"); - } - if (resp.Content.Headers.ContentLength is var size && size != -1 && size != target.Size) - { - throw new Exception( - $"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); - } - Repository.VerifyContentDigest(resp, target.Digest); - return await resp.Content.ReadAsStreamAsync(); - } - - /// - /// ExistsAsync returns true if the described content exists. - /// - /// - /// - /// - public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) - { - try - { - await ResolveAsync(target.Digest, cancellationToken); - return true; - } - catch (NotFoundException) - { - return false; - } - - } - - /// - /// PushAsync pushes the content, matching the expected descriptor. - /// - /// - /// - /// - /// - public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) - { - await PushAsync(expected, content, expected.Digest, cancellationToken); - } - - - /// - /// PushAsync pushes the manifest content, matching the expected descriptor. - /// - /// - /// - /// - /// - private async Task PushAsync(Descriptor expected, Stream stream, string reference, CancellationToken cancellationToken) - { - var remoteReference = Repository.RemoteReference; - remoteReference.Reference = reference; - var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); - var req = new HttpRequestMessage(HttpMethod.Put, url); - req.Content = new StreamContent(stream); - req.Content.Headers.ContentLength = expected.Size; - req.Content.Headers.Add("Content-Type", expected.MediaType); - var client = Repository.HttpClient; - using var resp = await client.SendAsync(req, cancellationToken); - if (resp.StatusCode != HttpStatusCode.Created) - { - throw await ErrorUtility.ParseErrorResponse(resp); - } - Repository.VerifyContentDigest(resp, expected.Digest); - } - - public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) - { - var remoteReference = Repository.ParseReference(reference); - var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); - var req = new HttpRequestMessage(HttpMethod.Head, url); - req.Headers.Add("Accept", ManifestUtility.ManifestAcceptHeader(Repository.ManifestMediaTypes)); - using var res = await Repository.HttpClient.SendAsync(req, cancellationToken); - - return res.StatusCode switch - { - HttpStatusCode.OK => await GenerateDescriptorAsync(res, remoteReference, req.Method), - HttpStatusCode.NotFound => throw new NotFoundException($"reference {reference} not found"), - _ => throw await ErrorUtility.ParseErrorResponse(res) - }; - } - - /// - /// GenerateDescriptor returns a descriptor generated from the response. - /// - /// - /// - /// - /// - /// - public async Task GenerateDescriptorAsync(HttpResponseMessage res, RemoteReference reference, HttpMethod httpMethod) - { - string mediaType; - try - { - // 1. Validate Content-Type - mediaType = res.Content.Headers.ContentType.MediaType; - MediaTypeHeaderValue.Parse(mediaType); - } - catch (Exception e) - { - throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: invalid response `Content-Type` header; {e.Message}"); - } - - // 2. Validate Size - if (!res.Content.Headers.ContentLength.HasValue || res.Content.Headers.ContentLength == -1) - { - throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: unknown response Content-Length"); - } - - // 3. Validate Client Reference - string refDigest = string.Empty; - try - { - refDigest = reference.Digest(); - } - catch (Exception) - { - } - - - // 4. Validate Server Digest (if present) - res.Content.Headers.TryGetValues("Docker-Content-Digest", out IEnumerable serverHeaderDigest); - var serverDigest = serverHeaderDigest?.First(); - if (!string.IsNullOrEmpty(serverDigest)) - { - try - { - Repository.VerifyContentDigest(res, serverDigest); - } - catch (Exception) - { - throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: invalid response header value: `Docker-Content-Digest: {serverHeaderDigest}`"); - } - } - - // 5. Now, look for specific error conditions; - string contentDigest; - - if (string.IsNullOrEmpty(serverDigest)) - { - if (httpMethod == HttpMethod.Head) - { - if (string.IsNullOrEmpty(refDigest)) - { - // HEAD without server `Docker-Content-Digest` - // immediate fail - throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: HTTP {httpMethod} request missing required header {serverHeaderDigest}"); - } - // Otherwise, just trust the client-supplied digest - contentDigest = refDigest; - } - else - { - // GET without server `Docker-Content-Digest header forces the - // expensive calculation - string calculatedDigest; - try - { - calculatedDigest = await CalculateDigestFromResponse(res); - } - catch (Exception e) - { - throw new Exception($"failed to calculate digest on response body; {e.Message}"); - } - contentDigest = calculatedDigest; - } - } - else - { - contentDigest = serverDigest; - } - if (!string.IsNullOrEmpty(refDigest) && refDigest != contentDigest) - { - throw new Exception($"{res.RequestMessage.Method} {res.RequestMessage.RequestUri}: invalid response; digest mismatch in {serverHeaderDigest}: received {contentDigest} when expecting {refDigest}"); - } - - // 6. Finally, if we made it this far, then all is good; return the descriptor - return new Descriptor - { - MediaType = mediaType, - Digest = contentDigest, - Size = res.Content.Headers.ContentLength.Value - }; - } - - /// - /// CalculateDigestFromResponse calculates the actual digest of the response body - /// taking care not to destroy it in the process - /// - /// - static async Task CalculateDigestFromResponse(HttpResponseMessage res) - { - var bytes = await res.Content.ReadAsByteArrayAsync(); - return Digest.ComputeSHA256(bytes); - } - - /// - /// DeleteAsync removes the manifest content identified by the descriptor. - /// - /// - /// - /// - public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) - { - await Repository.DeleteAsync(target, true, cancellationToken); - } - - - /// - /// FetchReferenceAsync fetches the manifest identified by the reference. - /// - /// - /// - /// - public async Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) - { - var remoteReference = Repository.ParseReference(reference); - var url = URLUtiliity.BuildRepositoryManifestURL(Repository.PlainHTTP, remoteReference); - var req = new HttpRequestMessage(HttpMethod.Get, url); - req.Headers.Add("Accept", ManifestUtility.ManifestAcceptHeader(Repository.ManifestMediaTypes)); - var resp = await Repository.HttpClient.SendAsync(req, cancellationToken); - switch (resp.StatusCode) - { - case HttpStatusCode.OK: - Descriptor desc; - if (resp.Content.Headers.ContentLength == -1) - { - desc = await ResolveAsync(reference, cancellationToken); - } - else - { - desc = await GenerateDescriptorAsync(resp, remoteReference, HttpMethod.Get); - } - - return (desc, await resp.Content.ReadAsStreamAsync()); - case HttpStatusCode.NotFound: - throw new NotFoundException($"{req.Method} {req.RequestUri}: manifest unknown"); - default: - throw await ErrorUtility.ParseErrorResponse(resp); - - } - } - - /// - /// PushReferenceASync pushes the manifest with a reference tag. - /// - /// - /// - /// - /// - /// - public async Task PushReferenceAsync(Descriptor expected, Stream content, string reference, - CancellationToken cancellationToken = default) - { - var remoteReference = Repository.ParseReference(reference); - await PushAsync(expected, content, remoteReference.Reference, cancellationToken); - } - - /// - /// TagAsync tags a manifest descriptor with a reference string. - /// - /// - /// - /// - /// - public async Task TagAsync(Descriptor descriptor, string reference, CancellationToken cancellationToken = default) - { - var remoteReference = Repository.ParseReference(reference); - var rc = await FetchAsync(descriptor, cancellationToken); - await PushAsync(descriptor, rc, remoteReference.Reference, cancellationToken); - } - } - - internal class BlobStore : IBlobStore - { - - public Repository Repository { get; set; } - - public BlobStore(Repository repository) - { - Repository = repository; - - } - - - public async Task FetchAsync(Descriptor target, CancellationToken cancellationToken = default) - { - var remoteReference = Repository.RemoteReference; - Digest.Validate(target.Digest); - remoteReference.Reference = target.Digest; - var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); - var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); - switch (resp.StatusCode) - { - case HttpStatusCode.OK: - // server does not support seek as `Range` was ignored. - if (resp.Content.Headers.ContentLength is var size && size != -1 && size != target.Size) - { - throw new Exception($"{resp.RequestMessage.Method} {resp.RequestMessage.RequestUri}: mismatch Content-Length"); - } - return await resp.Content.ReadAsStreamAsync(); - case HttpStatusCode.NotFound: - throw new NotFoundException($"{target.Digest}: not found"); - default: - throw await ErrorUtility.ParseErrorResponse(resp); - } - } - - /// - /// ExistsAsync returns true if the described content exists. - /// - /// - /// - /// - public async Task ExistsAsync(Descriptor target, CancellationToken cancellationToken = default) - { - try - { - await ResolveAsync(target.Digest, cancellationToken); - return true; - } - catch (NotFoundException) - { - return false; - } - - } - - /// - /// PushAsync pushes the content, matching the expected descriptor. - /// Existing content is not checked by PushAsync() to minimize the number of out-going - /// requests. - /// Push is done by conventional 2-step monolithic upload instead of a single - /// `POST` request for better overall performance. It also allows early fail on - /// authentication errors. - /// References: - /// - https://docs.docker.com/registry/spec/api/#pushing-an-image - /// - https://docs.docker.com/registry/spec/api/#initiate-blob-upload - /// - https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pushing-a-blob-monolithically - /// - /// - /// - /// - /// - public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) - { - var url = URLUtiliity.BuildRepositoryBlobUploadURL(Repository.PlainHTTP, Repository.RemoteReference); - using var resp = await Repository.HttpClient.PostAsync(url, null, cancellationToken); - var reqHostname = resp.RequestMessage.RequestUri.Host; - var reqPort = resp.RequestMessage.RequestUri.Port; - if (resp.StatusCode != HttpStatusCode.Accepted) - { - throw await ErrorUtility.ParseErrorResponse(resp); - } - - string location; - // monolithic upload - if (!resp.Headers.Location.IsAbsoluteUri) - { - location = resp.RequestMessage.RequestUri.Scheme + "://" + resp.RequestMessage.RequestUri.Authority + resp.Headers.Location; - } - else - { - location = resp.Headers.Location.ToString(); - } - // work-around solution for https://github.com/oras-project/oras-go/issues/177 - // For some registries, if the port 443 is explicitly set to the hostname plicitly set to the hostname - // like registry.wabbit-networks.io:443/myrepo, blob push will fail since - // the hostname of the Location header in the response is set to - // registry.wabbit-networks.io instead of registry.wabbit-networks.io:443. - var uri = new UriBuilder(location); - var locationHostname = uri.Host; - var locationPort = uri.Port; - // if location port 443 is missing, add it back - if (reqPort == 443 && locationHostname == reqHostname && locationPort != reqPort) - { - location = new UriBuilder($"{locationHostname}:{reqPort}").ToString(); - } - - url = location; - - var req = new HttpRequestMessage(HttpMethod.Put, url); - req.Content = new StreamContent(content); - req.Content.Headers.ContentLength = expected.Size; - - // the expected media type is ignored as in the API doc. - req.Content.Headers.Add("Content-Type", "application/octet-stream"); - - // add digest key to query string with expected digest value - req.RequestUri = new UriBuilder($"{req.RequestUri}?digest={expected.Digest}").Uri; - - //reuse credential from previous POST request - resp.Headers.TryGetValues("Authorization", out var auth); - if (auth != null) - { - req.Headers.Add("Authorization", auth.FirstOrDefault()); - } - using var resp2 = await Repository.HttpClient.SendAsync(req, cancellationToken); - if (resp2.StatusCode != HttpStatusCode.Created) - { - throw await ErrorUtility.ParseErrorResponse(resp2); - } - - return; - } - - /// - /// ResolveAsync resolves a reference to a descriptor. - /// - /// - /// - /// - public async Task ResolveAsync(string reference, CancellationToken cancellationToken = default) - { - var remoteReference = Repository.ParseReference(reference); - var refDigest = remoteReference.Digest(); - var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); - var requestMessage = new HttpRequestMessage(HttpMethod.Head, url); - using var resp = await Repository.HttpClient.SendAsync(requestMessage, cancellationToken); - return resp.StatusCode switch - { - HttpStatusCode.OK => Repository.GenerateBlobDescriptor(resp, refDigest), - HttpStatusCode.NotFound => throw new NotFoundException($"{remoteReference.Reference}: not found"), - _ => throw await ErrorUtility.ParseErrorResponse(resp) - }; - } - - /// - /// DeleteAsync deletes the content identified by the given descriptor. - /// - /// - /// - /// - public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) - { - await Repository.DeleteAsync(target, false, cancellationToken); - } - - /// - /// FetchReferenceAsync fetches the blob identified by the reference. - /// The reference must be a digest. - /// - /// - /// - /// - public async Task<(Descriptor Descriptor, Stream Stream)> FetchReferenceAsync(string reference, CancellationToken cancellationToken = default) - { - var remoteReference = Repository.ParseReference(reference); - var refDigest = remoteReference.Digest(); - var url = URLUtiliity.BuildRepositoryBlobURL(Repository.PlainHTTP, remoteReference); - var resp = await Repository.HttpClient.GetAsync(url, cancellationToken); - switch (resp.StatusCode) - { - case HttpStatusCode.OK: - // server does not support seek as `Range` was ignored. - Descriptor desc; - if (resp.Content.Headers.ContentLength == -1) - { - desc = await ResolveAsync(refDigest, cancellationToken); - } - else - { - desc = Repository.GenerateBlobDescriptor(resp, refDigest); - } - - return (desc, await resp.Content.ReadAsStreamAsync()); - case HttpStatusCode.NotFound: - throw new NotFoundException(); - default: - throw await ErrorUtility.ParseErrorResponse(resp); - } - } - } -} diff --git a/tests/OrasProject.Oras.Tests/RemoteTest/RegistryTest.cs b/tests/OrasProject.Oras.Tests/RemoteTest/RegistryTest.cs index 5d21dda..75a3c05 100644 --- a/tests/OrasProject.Oras.Tests/RemoteTest/RegistryTest.cs +++ b/tests/OrasProject.Oras.Tests/RemoteTest/RegistryTest.cs @@ -144,20 +144,17 @@ public async Task Repositories() var cancellationToken = new CancellationToken(); registry.TagListPageSize = 4; - - - var index = 0; - await registry.Repositories("", (string[] got) => - { - if (index > 2) - { - throw new Exception($"Error out of range: {index}"); - } - - var repos = repoSet[index]; - index++; - Assert.Equal(got, repos); - }, cancellationToken); + var wantRepositories = new List(); + foreach (var set in repoSet) + { + wantRepositories.AddRange(set); + } + var gotRepositories = new List(); + await foreach (var repo in registry.ListRepositoriesAsync().WithCancellation(cancellationToken)) + { + gotRepositories.Add(repo); + } + Assert.Equal(wantRepositories, gotRepositories); } } } diff --git a/tests/OrasProject.Oras.Tests/RemoteTest/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/RemoteTest/RepositoryTest.cs index 472ef3e..8f9381d 100644 --- a/tests/OrasProject.Oras.Tests/RemoteTest/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/RemoteTest/RepositoryTest.cs @@ -16,6 +16,7 @@ using OrasProject.Oras.Content; using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; +using OrasProject.Oras.Registry.Remote; using OrasProject.Oras.Remote; using System.Collections.Immutable; using System.Diagnostics; @@ -648,7 +649,7 @@ public async Task Repository_PushReferenceAsync() repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var streamContent = new MemoryStream(index); - await repo.PushReferenceAsync(indexDesc, streamContent, reference, cancellationToken); + await repo.PushAsync(indexDesc, streamContent, reference, cancellationToken); Assert.Equal(index, gotIndex); } @@ -713,17 +714,17 @@ public async Task Repository_FetchReferenceAsyc() // test with blob digest await Assert.ThrowsAsync( - async () => await repo.FetchReferenceAsync(blobDesc.Digest, cancellationToken)); + async () => await repo.FetchAsync(blobDesc.Digest, cancellationToken)); // test with manifest digest - var data = await repo.FetchReferenceAsync(indexDesc.Digest, cancellationToken); + var data = await repo.FetchAsync(indexDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); var buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest tag - data = await repo.FetchReferenceAsync(reference, cancellationToken); + data = await repo.FetchAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); @@ -731,7 +732,7 @@ await Assert.ThrowsAsync( // test with manifest tag@digest var tagDigestRef = "whatever" + "@" + indexDesc.Digest; - data = await repo.FetchReferenceAsync(tagDigestRef, cancellationToken); + data = await repo.FetchAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); @@ -739,7 +740,7 @@ await Assert.ThrowsAsync( // test with manifest FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; - data = await repo.FetchReferenceAsync(fqdnRef, cancellationToken); + data = await repo.FetchAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; @@ -818,18 +819,17 @@ public async Task Repository_TagsAsync() var cancellationToken = new CancellationToken(); - var index = 0; - await repo.TagsAsync("", (string[] got) => + var wantTags = new List(); + foreach (var set in tagSet) { - if (index > 2) - { - throw new Exception($"Error out of range: {index}"); - } - - var tags = tagSet[index]; - index++; - Assert.Equal(got, tags); - }, cancellationToken); + wantTags.AddRange(set); + } + var gotTags = new List(); + await foreach (var tag in repo.ListTagsAsync().WithCancellation(cancellationToken)) + { + gotTags.Add(tag); + } + Assert.Equal(wantTags, gotTags); } /// @@ -1292,7 +1292,7 @@ public async Task BlobStore_FetchReferenceAsync() var store = new BlobStore(repo); // test with digest - var gotDesc = await store.FetchReferenceAsync(blobDesc.Digest, cancellationToken); + var gotDesc = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); @@ -1302,7 +1302,7 @@ public async Task BlobStore_FetchReferenceAsync() // test with FQDN reference var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; - gotDesc = await store.FetchReferenceAsync(fqdnRef, cancellationToken); + gotDesc = await store.FetchAsync(fqdnRef, cancellationToken); Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); @@ -1315,7 +1315,7 @@ public async Task BlobStore_FetchReferenceAsync() }; // test with other digest await Assert.ThrowsAsync(async () => - await store.FetchReferenceAsync(contentDesc.Digest, cancellationToken)); + await store.FetchAsync(contentDesc.Digest, cancellationToken)); } /// @@ -1393,7 +1393,7 @@ public async Task BlobStore_FetchReferenceAsync_Seek() // test non-seekable content - var data = await store.FetchReferenceAsync(blobDesc.Digest, cancellationToken); + var data = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); Assert.Equal(data.Descriptor.Size, blobDesc.Size); @@ -1404,7 +1404,7 @@ public async Task BlobStore_FetchReferenceAsync_Seek() // test seekable content seekable = true; - data = await store.FetchReferenceAsync(blobDesc.Digest, cancellationToken); + data = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); Assert.Equal(data.Descriptor.Size, blobDesc.Size); @@ -1840,7 +1840,7 @@ public async Task ManifestStore_FetchReferenceAsync() var store = new ManifestStore(repo); // test with tag - var data = await store.FetchReferenceAsync(reference, cancellationToken); + var data = await store.FetchAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); var buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); @@ -1848,10 +1848,10 @@ public async Task ManifestStore_FetchReferenceAsync() // test with other tag var randomRef = "whatever"; - await Assert.ThrowsAsync(async () => await store.FetchReferenceAsync(randomRef, cancellationToken)); + await Assert.ThrowsAsync(async () => await store.FetchAsync(randomRef, cancellationToken)); // test with digest - data = await store.FetchReferenceAsync(manifestDesc.Digest, cancellationToken); + data = await store.FetchAsync(manifestDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; @@ -1860,7 +1860,7 @@ public async Task ManifestStore_FetchReferenceAsync() // test with tag@digest var tagDigestRef = randomRef + "@" + manifestDesc.Digest; - data = await store.FetchReferenceAsync(tagDigestRef, cancellationToken); + data = await store.FetchAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); @@ -1868,7 +1868,7 @@ public async Task ManifestStore_FetchReferenceAsync() // test with FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; - data = await store.FetchReferenceAsync(fqdnRef, cancellationToken); + data = await store.FetchAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); @@ -2007,7 +2007,7 @@ public async Task ManifestStore_PushReferenceAsync() repo.PlainHTTP = true; var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - await store.PushReferenceAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); + await store.PushAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); Assert.Equal(index, gotIndex); } @@ -2097,9 +2097,9 @@ public async Task CopyFromRepositoryToMemory() return res; }; - var reg = new Registry("localhost:5000"); + var reg = new Remote.Registry("localhost:5000"); reg.HttpClient = CustomClient(func); - var src = await reg.Repository("source", CancellationToken.None); + var src = await reg.GetRepository("source", CancellationToken.None); var dst = new MemoryStore(); var tagName = "latest";