From 4ef85ac1d628a1a52ef51deee6d8bdf5bd584d1c Mon Sep 17 00:00:00 2001 From: John Turpish Date: Mon, 6 Nov 2023 08:57:25 -0500 Subject: [PATCH] Fixes #39 Made some requestors. PoolRequestor now retries. Tweaks necessary. --- .gitignore | 1 + cmake/conanfile.txt | 1 + cmake/patch.py | 30 +- cmake/setup.cmake | 7 + component/block_http_request.cc | 83 + component/block_http_request.h | 38 + component/cache_requestor.cc | 2 +- ...y_requests.cc => chromium_ipfs_context.cc} | 71 +- ...way_requests.h => chromium_ipfs_context.h} | 27 +- component/dns_txt_request.cc | 39 + component/dns_txt_request.h | 36 + component/inter_request_state.cc | 32 +- component/inter_request_state.h | 9 +- component/interceptor.cc | 8 +- component/ipfs_url_loader.cc | 97 +- component/ipfs_url_loader.h | 4 +- component/ipns_url_loader.cc | 2 +- component/ipns_url_loader.h | 2 +- component/network_requestor.cc | 2 +- component/patches/118.0.5993.136.patch | 483 + component/patches/119.0.6045.21.patch | 5 +- ...8.0.5993.118.patch => 120.0.6099.18.patch} | 132 +- component/patches/121.0.6100.0.patch | 12631 +++++++++++++++- ...{120.0.6099.0.patch => 121.0.6103.3.patch} | 37 +- ...{120.0.6090.0.patch => 121.0.6104.0.patch} | 43 +- ...{120.0.6078.0.patch => 121.0.6110.0.patch} | 43 +- component/url_loader_factory.cc | 20 +- doc/design_notes.md | 2 +- doc/original_design.md | 6 +- library/BUILD.gn.in | 4 + library/CMakeLists.txt | 23 +- library/conanfile.py | 1 + library/include/ipfs_client/context_api.h | 46 +- .../ipfs_client/gw/block_request_splitter.h | 17 + .../ipfs_client/gw/default_requestor.h | 13 + .../ipfs_client/gw/dnslink_requestor.h | 20 + .../include/ipfs_client/gw/gateway_request.h | 20 +- .../ipfs_client/gw/inline_request_handler.h | 13 + library/include/ipfs_client/gw/requestor.h | 59 + .../ipfs_client/gw/terminating_requestor.h | 13 + library/include/ipfs_client/ipfs_request.h | 3 + library/include/ipfs_client/ipld/dag_node.h | 4 + library/include/ipfs_client/ipns_cbor_entry.h | 21 + library/include/ipfs_client/ipns_record.h | 30 +- library/include/ipfs_client/orchestrator.h | 28 +- library/include/ipfs_client/response.h | 3 + .../include/ipfs_client/signing_key_type.h | 14 + .../libp2p/multi/content_identifier.hpp | 6 - library/src/ipfs_client/context_api.cc | 4 + .../ipfs_client/gw/block_request_splitter.cc | 33 + .../gw/block_request_splitter_unittest.cc | 50 + .../src/ipfs_client/gw/default_requestor.cc | 24 + .../src/ipfs_client/gw/dnslink_requestor.cc | 70 + .../ipfs_client/gw/gateway_http_requestor.cc | 170 + .../ipfs_client/gw/gateway_http_requestor.h | 32 + .../gw/gateway_http_requestor_unittest.cc | 152 + library/src/ipfs_client/gw/gateway_request.cc | 109 +- .../ipfs_client/gw/inline_request_handler.cc | 20 + library/src/ipfs_client/gw/requestor.cc | 132 + library/src/ipfs_client/gw/requestor_pool.cc | 66 + library/src/ipfs_client/gw/requestor_pool.h | 26 + .../src/ipfs_client/gw/requestor_unittest.cc | 54 + .../ipfs_client/gw/terminating_requestor.cc | 29 + library/src/ipfs_client/ipfs_request.cc | 14 +- library/src/ipfs_client/ipld/chunk.cc | 4 + library/src/ipfs_client/ipld/dag_node.cc | 8 +- .../src/ipfs_client/ipld/directory_shard.cc | 16 +- .../src/ipfs_client/ipld/small_directory.cc | 10 +- library/src/ipfs_client/ipld/unixfs_file.cc | 9 +- library/src/ipfs_client/ipns_record.cc | 82 +- .../src/ipfs_client/ipns_record_unittest.cc | 270 +- library/src/ipfs_client/orchestrator.cc | 84 +- .../src/ipfs_client/orchestrator_unittest.cc | 104 +- library/src/ipfs_client/response.cc | 5 + library/src/ipfs_client/scheduler.cc | 3 +- library/src/ipfs_client/scoring.md | 2 +- library/src/ipfs_client/signing_key_type.cc | 15 + .../unixfs_path_resolver_unittest.cc | 12 + library/src/libp2p/common/hexutil.cc | 95 - .../src/libp2p/multi/content_identifier.cc | 15 - library/src/libp2p/multi/multihash.cc | 14 - library/src/libp2p/multi/uvarint.cc | 4 - library/src/libp2p/peer/peer_id.cc | 4 - ...7wt7odvi3waeab7xbxyebjhbzgguj2uv63t4hsvdri | 260 + ...r7xknfpzp7wch53owkr6dxv45cspxn7hfqccdel4fe | Bin 0 -> 843 bytes 85 files changed, 15676 insertions(+), 556 deletions(-) create mode 100644 component/block_http_request.cc create mode 100644 component/block_http_request.h rename component/{gateway_requests.cc => chromium_ipfs_context.cc} (83%) rename component/{gateway_requests.h => chromium_ipfs_context.h} (65%) create mode 100644 component/dns_txt_request.cc create mode 100644 component/dns_txt_request.h create mode 100644 component/patches/118.0.5993.136.patch rename component/patches/{118.0.5993.118.patch => 120.0.6099.18.patch} (82%) rename component/patches/{120.0.6099.0.patch => 121.0.6103.3.patch} (92%) rename component/patches/{120.0.6090.0.patch => 121.0.6104.0.patch} (92%) rename component/patches/{120.0.6078.0.patch => 121.0.6110.0.patch} (92%) create mode 100644 library/include/ipfs_client/gw/block_request_splitter.h create mode 100644 library/include/ipfs_client/gw/default_requestor.h create mode 100644 library/include/ipfs_client/gw/dnslink_requestor.h create mode 100644 library/include/ipfs_client/gw/inline_request_handler.h create mode 100644 library/include/ipfs_client/gw/requestor.h create mode 100644 library/include/ipfs_client/gw/terminating_requestor.h create mode 100644 library/include/ipfs_client/ipns_cbor_entry.h create mode 100644 library/include/ipfs_client/signing_key_type.h create mode 100644 library/src/ipfs_client/gw/block_request_splitter.cc create mode 100644 library/src/ipfs_client/gw/block_request_splitter_unittest.cc create mode 100644 library/src/ipfs_client/gw/default_requestor.cc create mode 100644 library/src/ipfs_client/gw/dnslink_requestor.cc create mode 100644 library/src/ipfs_client/gw/gateway_http_requestor.cc create mode 100644 library/src/ipfs_client/gw/gateway_http_requestor.h create mode 100644 library/src/ipfs_client/gw/gateway_http_requestor_unittest.cc create mode 100644 library/src/ipfs_client/gw/inline_request_handler.cc create mode 100644 library/src/ipfs_client/gw/requestor.cc create mode 100644 library/src/ipfs_client/gw/requestor_pool.cc create mode 100644 library/src/ipfs_client/gw/requestor_pool.h create mode 100644 library/src/ipfs_client/gw/requestor_unittest.cc create mode 100644 library/src/ipfs_client/gw/terminating_requestor.cc create mode 100644 library/src/ipfs_client/signing_key_type.cc delete mode 100644 library/src/libp2p/common/hexutil.cc create mode 100644 test_data/blocks/bafkreiagoa7j73kp7wt7odvi3waeab7xbxyebjhbzgguj2uv63t4hsvdri create mode 100644 test_data/blocks/bafybeic7wwiyffvkr7xknfpzp7wch53owkr6dxv45cspxn7hfqccdel4fe diff --git a/.gitignore b/.gitignore index ed25966b..8cc30443 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__ CMakeLists.txt.user CMakeUserPresets.json *.orig +.*.swp diff --git a/cmake/conanfile.txt b/cmake/conanfile.txt index d5467d41..05c58e72 100644 --- a/cmake/conanfile.txt +++ b/cmake/conanfile.txt @@ -2,6 +2,7 @@ abseil/20220623.1 boost/1.81.0 gtest/1.13.0 + nlohmann_json/3.11.2 openssl/1.1.1t protobuf/3.21.4 [options] diff --git a/cmake/patch.py b/cmake/patch.py index d9e1072b..ec71fda4 100755 --- a/cmake/patch.py +++ b/cmake/patch.py @@ -147,11 +147,10 @@ def electron_version(self, branch='main'): self.up_rels['electron-main'] = toks[i] return toks[i] - def unavailable(self): avail = list(map(as_int, self.available())) version_set = {} - fuzz = 114 + fuzz = 59877 def check(version, version_set, s): i = as_int(version) by = (fuzz,0) @@ -182,6 +181,25 @@ def check(version, version_set, s): result.sort(reverse=True) return map(lambda x: x[1:], result) + def out_of_date(self, p): + with open(f'{self.pdir}/{p}.patch') as f: + lines = f.readlines() + fl = Patcher.file_lines(lines, 'chrome/browser/flag-metadata.json') + return '+ "name": "enable-ipfs",\n' in fl + + @staticmethod + def file_lines(lines: list[str], path): + start = f'diff --git a/{path} b/{path}\n' + if not start in lines: + # print('Did not find',start,'in',lines) + return [] + i = lines.index(start) + 1 + #print(start,'found at',i) + for j in range(i, i + 9999): + if len(lines) == j or lines[j].startswith('diff --git'): + return lines[i:j] + return [] + if __name__ == '__main__': if argv[1] == 'apply': @@ -200,5 +218,13 @@ def check(version, version_set, s): rels = p.release_versions(chan, os) for rel in rels: print(chan, os, rel) + elif argv[1] == 'old': + pr = Patcher('/mnt/big/lbl/code/chromium/src', 'git', 'Debug') + if len(argv) > 2: + for p in argv[2:]: + print(p, pr.out_of_date(p)) + else: + for p in pr.available(): + print(p, pr.out_of_date(p)) else: Patcher(*argv[1:]).create_patch_file() diff --git a/cmake/setup.cmake b/cmake/setup.cmake index 251ca563..427a620a 100644 --- a/cmake/setup.cmake +++ b/cmake/setup.cmake @@ -103,4 +103,11 @@ function(with_vocab target) OpenSSL::Crypto ) endif() + find_package(nlohmann_json) + if(nlohmann_json_FOUND) + target_link_libraries(${target} + PUBLIC + nlohmann_json::nlohmann_json + ) + endif() endfunction() diff --git a/component/block_http_request.cc b/component/block_http_request.cc new file mode 100644 index 00000000..fbde9289 --- /dev/null +++ b/component/block_http_request.cc @@ -0,0 +1,83 @@ +#include "block_http_request.h" + +#include +#include +#include + +using Self = ipfs::BlockHttpRequest; + +namespace { +constexpr net::NetworkTrafficAnnotationTag kTrafficAnnotation = + net::DefineNetworkTrafficAnnotation("ipfs_gateway_request", R"( + semantics { + sender: "IPFS component" + description: + "Sends a request to an IPFS gateway." + trigger: + "Processing of an ipfs:// or ipns:// URL." + data: "None" + destination: WEBSITE + } + policy { + cookies_allowed: NO + setting: "EnableIpfs" + } + )"); +} + +Self::BlockHttpRequest(ipfs::HttpRequestDescription req_inf, + HttpCompleteCallback cb) + : inf_{req_inf}, callback_{cb} {} +Self::~BlockHttpRequest() noexcept {} + +void Self::send(raw_ptr loader_factory) { + auto req = std::make_unique(); + req->url = GURL{inf_.url}; + req->priority = net::HIGHEST; // TODO + if (!inf_.accept.empty()) { + VLOG(2) << inf_.url << " has Accept header " << inf_.accept; + req->headers.SetHeader("Accept", inf_.accept); + } + using L = network::SimpleURLLoader; + loader_ = L::Create(std::move(req), kTrafficAnnotation, FROM_HERE); + loader_->SetTimeoutDuration(base::Seconds(inf_.timeout_seconds)); + loader_->SetAllowHttpErrorResults(true); + auto bound = base::BindOnce(&Self::OnResponse, base::Unretained(this), + shared_from_this()); + DCHECK(loader_factory); + loader_->DownloadToString(loader_factory, std::move(bound), + gw::BLOCK_RESPONSE_BUFFER_SIZE); +} +void Self::OnResponse(std::shared_ptr, + std::unique_ptr body) { + DCHECK(loader_); + int status = loader_->NetError() ? 500 : 200; + ContextApi::HeaderAccess header_accessor = [](auto) { return std::string{}; }; + auto sz = body ? body->size() : 0UL; + VLOG(1) << "Handling HTTP response body of size " << sz << " with NetErr " + << loader_->NetError() << " for " << inf_.url; + auto const* head = loader_->ResponseInfo(); + if (head) { + DCHECK(head->headers); + auto status_line = head->headers->GetStatusLine(); + auto sp = status_line.find(' '); + if (sp < status_line.size()) { + VLOG(1) << "HTTP response status='" << status_line << "'."; + status = std::atoi(status_line.c_str() + sp + 1); + } else { + LOG(WARNING) << "Status line malformed."; + } + header_accessor = [head](std::string_view k) { + std::string val; + head->headers->EnumerateHeader(nullptr, k, &val); + return val; + }; + } else { + LOG(WARNING) << "No response header info?"; + } + if (body) { + callback_(status, *body, header_accessor); + } else { + callback_(status, "", header_accessor); + } +} diff --git a/component/block_http_request.h b/component/block_http_request.h new file mode 100644 index 00000000..781a7ab6 --- /dev/null +++ b/component/block_http_request.h @@ -0,0 +1,38 @@ +#ifndef IPFS_BLOCK_HTTP_REQUEST_H_ +#define IPFS_BLOCK_HTTP_REQUEST_H_ + +#include + +#include + +namespace network { +struct ResourceRequest; +class SimpleURLLoader; +} // namespace network +namespace network::mojom { +class URLLoaderFactory; +} + +namespace ipfs { +class BlockHttpRequest : public std::enable_shared_from_this { + // TODO ween oneself off of SimpleURLLoader + // std::array buffer_; + std::unique_ptr loader_; + + public: + using HttpCompleteCallback = ipfs::ContextApi::HttpCompleteCallback; + BlockHttpRequest(ipfs::HttpRequestDescription, HttpCompleteCallback); + ~BlockHttpRequest() noexcept; + + void send(raw_ptr); + + private: + ipfs::HttpRequestDescription const inf_; + HttpCompleteCallback callback_; + + void OnResponse(std::shared_ptr, + std::unique_ptr body); +}; +} // namespace ipfs + +#endif // IPFS_BLOCK_HTTP_REQUEST_H_ diff --git a/component/cache_requestor.cc b/component/cache_requestor.cc index 19426a97..ed0d27ba 100644 --- a/component/cache_requestor.cc +++ b/component/cache_requestor.cc @@ -1,6 +1,6 @@ #include "cache_requestor.h" -#include "gateway_requests.h" +#include "chromium_ipfs_context.h" #include "inter_request_state.h" #include diff --git a/component/gateway_requests.cc b/component/chromium_ipfs_context.cc similarity index 83% rename from component/gateway_requests.cc rename to component/chromium_ipfs_context.cc index 7e609028..5ae0a1a3 100644 --- a/component/gateway_requests.cc +++ b/component/chromium_ipfs_context.cc @@ -1,18 +1,20 @@ -#include "gateway_requests.h" +#include "chromium_ipfs_context.h" +#include "block_http_request.h" #include "crypto_api.h" #include "inter_request_state.h" #include "ipns_cbor.h" +#include #include "base/strings/escape.h" #include "net/base/mime_sniffer.h" #include "net/base/mime_util.h" #include "services/network/public/cpp/resource_request.h" -#include "services/network/public/cpp/simple_url_loader.h" #include "services/network/public/mojom/url_response_head.mojom.h" #include "url/gurl.h" #include +#include #include #include @@ -20,7 +22,7 @@ #include -using Self = ipfs::GatewayRequests; +using Self = ipfs::ChromiumIpfsContext; constexpr net::NetworkTrafficAnnotationTag kTrafficAnnotation = net::DefineNetworkTrafficAnnotation("ipfs_gateway_request", R"( @@ -207,17 +209,10 @@ bool Self::ProcessResponse(BusyGateway& gw, auto duration = (end_time - start_time).InMillisecondsRoundedUp(); if (cid.value().content_type == libp2p::multi::MulticodecType::Code::LIBP2P_KEY) { - auto as_peer = libp2p::peer::PeerId::fromHash(cid.value().content_address); - if (!as_peer.has_value()) { - LOG(INFO) << cid_str - << " has the codec of being a libp2p key, like one would " - "expect of a Peer ID, but it does not parse as a Peer ID."; - return false; - } auto* bytes = reinterpret_cast(body->data()); - auto record = - ValidateIpnsRecord({bytes, body->size()}, as_peer.value(), - crypto_api::VerifySignature, ParseCborIpns); + // auto record = ValidateIpnsRecord({bytes, body->size()}, + // as_peer.value(), crypto_api::VerifySignature, ParseCborIpns); + auto record = ValidateIpnsRecord({bytes, body->size()}, cid.value(), *this); if (!record.has_value()) { LOG(ERROR) << "IPNS record failed to validate! From: " << gw.url(); return false; @@ -247,7 +242,9 @@ bool Self::ProcessResponse(BusyGateway& gw, auto& orc = state_->orchestrator(); orc.add_node(cid_str, ipld::DagNode::fromBlock(block)); if (gw.srcreq) { - orc.build_response(gw.srcreq->dependent); + if (gw.srcreq->dependent->ready_after()) { + orc.build_response(gw.srcreq->dependent); + } } else { LOG(ERROR) << "This BusyGateway with response has no top-level " "IpfsRequest associated with it " @@ -291,13 +288,14 @@ std::string Self::MimeType(std::string extension, } else { result.clear(); } - if (net::SniffMimeType({content.data(), content.size()}, GURL{url}, result, + auto head_size = std::min(content.size(), 999'999UL); + if (net::SniffMimeType({content.data(), head_size}, GURL{url}, result, net::ForceSniffFileUrlsForHtml::kDisabled, &result)) { VLOG(1) << "Got " << result << " from content of " << url; } if (result.empty() || result == "application/octet-stream") { // C'mon, man - net::SniffMimeTypeFromLocalData({content.data(), content.size()}, &result); + net::SniffMimeTypeFromLocalData({content.data(), head_size}, &result); LOG(INFO) << "Falling all the way back to content type " << result; } return result; @@ -319,14 +317,49 @@ void Self::RequestByCid(std::string cid, prio, {}); sched_.IssueRequests(me); } +void Self::SendDnsTextRequest(std::string host, + DnsTextResultsCallback res, + DnsTextCompleteCallback don) { + if (dns_reqs_.find(host) != dns_reqs_.end()) { + LOG(ERROR) << "Requested resolution of DNSLink host " << host + << " multiple times."; + } + auto don_wrap = [don, this, host]() { + don(); + LOG(INFO) << "Finished resolving " << host << " via DNSLink"; + dns_reqs_.erase(host); + }; + dns_reqs_[host] = std::make_unique(host, res, don_wrap, + network_context_.get()); +} +void Self::SendHttpRequest(HttpRequestDescription req_inf, + HttpCompleteCallback cb) const { + DCHECK(loader_factory_); + auto ptr = std::make_shared(req_inf, cb); + ptr->send(loader_factory_); +} +auto Self::deserialize_cbor(ByteView bytes) const -> IpnsCborEntry { + return ParseCborIpns(bytes); +} +bool Self::verify_key_signature(SigningKeyType t, + ByteView signature, + ByteView data, + ByteView key_bytes) const { + return crypto_api::VerifySignature(static_cast(t), signature, + data, key_bytes); +} -Self::GatewayRequests(InterRequestState& state) - : state_{state}, +Self::ChromiumIpfsContext( + InterRequestState& state, + raw_ptr network_context) + : network_context_{network_context}, + state_{state}, sched_([this]() { return state_->gateways().GenerateList(); }) {} -Self::~GatewayRequests() { +Self::~ChromiumIpfsContext() { LOG(WARNING) << "API dtor - are all URIs loaded?"; } Self::GatewayUrlLoader::GatewayUrlLoader(BusyGateway&& bg) : GatewayRequest(std::move(bg)) {} Self::GatewayUrlLoader::~GatewayUrlLoader() noexcept {} + diff --git a/component/gateway_requests.h b/component/chromium_ipfs_context.h similarity index 65% rename from component/gateway_requests.h rename to component/chromium_ipfs_context.h index 44f0588e..8d9e062d 100644 --- a/component/gateway_requests.h +++ b/component/chromium_ipfs_context.h @@ -1,5 +1,7 @@ -#ifndef IPFS_GATEWAY_REQUESTS_H_ -#define IPFS_GATEWAY_REQUESTS_H_ +#ifndef IPFS_CHROMIUM_IPFS_CONTEXT_H_ +#define IPFS_CHROMIUM_IPFS_CONTEXT_H_ + +#include "dns_txt_request.h" #include #include @@ -24,7 +26,7 @@ class InterRequestState; class IpfsRequest; class NetworkRequestor; -class GatewayRequests final : public ContextApi { +class ChromiumIpfsContext final : public ContextApi { struct GatewayUrlLoader : public ipfs::GatewayRequest { GatewayUrlLoader(BusyGateway&&); GatewayUrlLoader(GatewayRequest&&); @@ -33,9 +35,11 @@ class GatewayRequests final : public ContextApi { }; raw_ptr loader_factory_ = nullptr; + raw_ptr network_context_; raw_ref state_; Scheduler sched_; std::function)> disc_cb_; + std::map> dns_reqs_; void Request(std::string task, std::shared_ptr, Priority); std::shared_ptr InitiateGatewayRequest(BusyGateway) override; @@ -43,6 +47,16 @@ class GatewayRequests final : public ContextApi { std::string_view content, std::string const& url) const override; std::string UnescapeUrlComponent(std::string_view) const override; + void SendDnsTextRequest(std::string, + DnsTextResultsCallback, + DnsTextCompleteCallback) override; + void SendHttpRequest(HttpRequestDescription req_inf, + HttpCompleteCallback cb) const override; + IpnsCborEntry deserialize_cbor(ByteView) const override; + bool verify_key_signature(SigningKeyType, + ByteView signature, + ByteView data, + ByteView key_bytes) const override; void OnResponse(std::shared_ptr, std::shared_ptr, @@ -59,8 +73,9 @@ class GatewayRequests final : public ContextApi { Priority); public: - GatewayRequests(InterRequestState&); - ~GatewayRequests(); + ChromiumIpfsContext(InterRequestState&, + raw_ptr network_context); + ~ChromiumIpfsContext(); void SetLoaderFactory(network::mojom::URLLoaderFactory&); Scheduler& scheduler(); void Discover(std::function)>) override; @@ -68,4 +83,4 @@ class GatewayRequests final : public ContextApi { } // namespace ipfs -#endif // IPFS_GATEWAY_REQUESTS_H_ +#endif // IPFS_CHROMIUM_IPFS_CONTEXT_H_ diff --git a/component/dns_txt_request.cc b/component/dns_txt_request.cc new file mode 100644 index 00000000..c7e8e667 --- /dev/null +++ b/component/dns_txt_request.cc @@ -0,0 +1,39 @@ +#include "dns_txt_request.h" + +#include +#include + +namespace moj = network::mojom; +using Self = ipfs::DnsTxtRequest; + +Self::DnsTxtRequest(std::string host, + ipfs::ContextApi::DnsTextResultsCallback res, + ipfs::ContextApi::DnsTextCompleteCallback don, + moj::NetworkContext* network_context) + : results_callback_{res}, completion_callback_{don} { + auto params = moj::ResolveHostParameters::New(); + params->dns_query_type = net::DnsQueryType::TXT; + params->initial_priority = net::RequestPriority::HIGHEST; + params->source = net::HostResolverSource::ANY; + params->cache_usage = moj::ResolveHostParameters_CacheUsage::STALE_ALLOWED; + params->secure_dns_policy = moj::SecureDnsPolicy::ALLOW; + params->purpose = moj::ResolveHostParameters::Purpose::kUnspecified; + LOG(INFO) << "Querying DNS for TXT records on " << host; + auto hrh = moj::HostResolverHost::NewHostPortPair({host, 0}); + auto nak = net::NetworkAnonymizationKey::CreateTransient(); + network_context->ResolveHost(std::move(hrh), nak, std::move(params), + recv_.BindNewPipeAndPassRemote()); +} +Self::~DnsTxtRequest() {} + +void Self::OnTextResults(std::vector const& results) { + LOG(INFO) << "Hit " << results.size() << " DNS TXT results."; + results_callback_(results); +} +void Self::OnComplete(int32_t result, + const ::net::ResolveErrorInfo&, + const absl::optional<::net::AddressList>&, + const absl::optional&) { + LOG(INFO) << "DNS Results done with code: " << result; + completion_callback_(); +} diff --git a/component/dns_txt_request.h b/component/dns_txt_request.h new file mode 100644 index 00000000..a6c72e46 --- /dev/null +++ b/component/dns_txt_request.h @@ -0,0 +1,36 @@ +#ifndef IPFS_DNS_TXT_REQUEST_H_ +#define IPFS_DNS_TXT_REQUEST_H_ + +#include + +#include +#include + +namespace network::mojom { +class NetworkContext; +} + +namespace ipfs { +class DnsTxtRequest final : public network::ResolveHostClientBase { + ipfs::ContextApi::DnsTextResultsCallback results_callback_; + ipfs::ContextApi::DnsTextCompleteCallback completion_callback_; + mojo::Receiver recv_{this}; + + using Endpoints = std::vector<::net::HostResolverEndpointResult>; + void OnTextResults(std::vector const&) override; + void OnComplete(int32_t result, + ::net::ResolveErrorInfo const&, + absl::optional<::net::AddressList> const&, + absl::optional const&) override; + + public: + DnsTxtRequest(std::string, + ipfs::ContextApi::DnsTextResultsCallback, + ipfs::ContextApi::DnsTextCompleteCallback, + network::mojom::NetworkContext*); + DnsTxtRequest(DnsTxtRequest&&) = delete; + ~DnsTxtRequest() noexcept override; +}; +} // namespace ipfs + +#endif // IPFS_DNS_TXT_REQUEST_H_ diff --git a/component/inter_request_state.cc b/component/inter_request_state.cc index ee1e2041..3406c8b4 100644 --- a/component/inter_request_state.cc +++ b/component/inter_request_state.cc @@ -1,11 +1,13 @@ #include "inter_request_state.h" -#include "gateway_requests.h" +#include "chromium_ipfs_context.h" #include "network_requestor.h" #include "base/logging.h" #include "content/public/browser/browser_context.h" +#include + #include #include #include @@ -26,10 +28,10 @@ auto Self::FromBrowserContext(content::BrowserContext* context) } base::SupportsUserData::Data* existing = context->GetUserData(user_data_key); if (existing) { - VLOG(1) << "Re-using existing IPFS state."; + VLOG(2) << "Re-using existing IPFS state."; return *static_cast(existing); } - LOG(INFO) << "Creating new IPFS state for this browser context."; + VLOG(1) << "Creating new IPFS state for this browser context."; auto owned = std::make_unique(context->GetPath()); ipfs::InterRequestState* raw = owned.get(); context->SetUserData(user_data_key, std::move(owned)); @@ -59,12 +61,13 @@ auto Self::requestor() -> BlockRequestor& { } return requestor_; } -std::shared_ptr Self::api() { +std::shared_ptr Self::api() { auto existing = api_.lock(); if (existing) { return existing; } - auto created = std::make_shared(*this); + auto created = + std::make_shared(*this, network_context_); api_ = created; auto t = std::time(nullptr); if (t - last_discovery_ > 300) { @@ -90,7 +93,7 @@ void send_gateway_request(Self* me, struct DagListenerAdapter final : public ipfs::DagListener { std::shared_ptr gw_req; std::string bytes; - std::shared_ptr api; + std::shared_ptr api; void ReceiveBlockBytes(std::string_view b) override { LOG(INFO) << "DagListenerAdapter::ReceiveBlockBytes(" << b.size() << "B)"; bytes.assign(b); @@ -119,25 +122,18 @@ void send_gateway_request(Self* me, 9, req); sched.IssueRequests(me->api()); } -std::string detect_mime(Self* me, - std::string a, - std::string_view b, - std::string const& c) { - auto api = me->api(); - return static_cast(api.get())->MimeType(a, b, c); -} } // namespace auto Self::orchestrator() -> Orchestrator& { if (!orc_) { auto gwreq = [this](auto p) { send_gateway_request(this, p); }; - auto mimer = [this](auto a, auto b, auto& c) { - return detect_mime(this, a, b, c); - }; - orc_ = std::make_shared(gwreq, mimer); + auto rtor = gw::default_requestor(gateways().GenerateList(), api()); + orc_ = std::make_shared(gwreq, rtor, api()); } return *orc_; } - +void Self::set_network_context(raw_ptr val) { + network_context_ = val; +} Self::InterRequestState(base::FilePath p) : disk_path_{p} {} Self::~InterRequestState() noexcept {} diff --git a/component/inter_request_state.h b/component/inter_request_state.h index 6b5f1354..910c4f42 100644 --- a/component/inter_request_state.h +++ b/component/inter_request_state.h @@ -10,6 +10,7 @@ #include "ipfs_client/orchestrator.h" #include "base/supports_user_data.h" +#include "services/network/network_context.h" namespace content { class BrowserContext; @@ -17,17 +18,18 @@ class BrowserContext; namespace ipfs { class Scheduler; -class GatewayRequests; +class ChromiumIpfsContext; class InterRequestState : public base::SupportsUserData::Data { Gateways gws_; BlockStorage storage_; ChainedRequestors requestor_; IpnsNames names_; - std::weak_ptr api_; + std::weak_ptr api_; std::time_t last_discovery_ = 0; std::shared_ptr mem_, dsk_; base::FilePath const disk_path_; std::shared_ptr orc_; // TODO - map of origin to Orchestrator + raw_ptr network_context_; public: InterRequestState(base::FilePath); @@ -38,9 +40,10 @@ class InterRequestState : public base::SupportsUserData::Data { BlockRequestor& requestor(); IpnsNames& names() { return names_; } Scheduler& scheduler(); - std::shared_ptr api(); + std::shared_ptr api(); std::array,2> serialized_caches(); Orchestrator& orchestrator(); + void set_network_context(raw_ptr); static InterRequestState& FromBrowserContext(content::BrowserContext*); }; diff --git a/component/interceptor.cc b/component/interceptor.cc index f304d682..0001e69d 100644 --- a/component/interceptor.cc +++ b/component/interceptor.cc @@ -20,12 +20,15 @@ void Interceptor::MaybeCreateLoader(network::ResourceRequest const& req, content::BrowserContext* context, LoaderCallback loader_callback) { auto& state = InterRequestState::FromBrowserContext(context); + state.set_network_context(network_context_); + /* if (req.url.SchemeIs("ipns")) { auto ipns_loader = std::make_shared( state, req.url.host(), network_context_, *loader_factory_); std::move(loader_callback) .Run(base::BindOnce(&ipfs::IpnsUrlLoader::StartHandling, ipns_loader)); - } else if (req.url.SchemeIs("ipfs")) { + } else */ + if (req.url.SchemeIs("ipfs") || req.url.SchemeIs("ipns")) { auto hdr_str = req.headers.ToString(); std::replace(hdr_str.begin(), hdr_str.end(), '\r', ' '); LOG(INFO) << req.url.spec() << " getting intercepted! Headers: \n" @@ -35,8 +38,9 @@ void Interceptor::MaybeCreateLoader(network::ResourceRequest const& req, std::make_shared(*loader_factory_, state); std::move(loader_callback) .Run(base::BindOnce(&ipfs::IpfsUrlLoader::StartRequest, loader)); + } else { - VLOG(1) << req.url.spec() << " has host '" << req.url.host() + VLOG(2) << req.url.spec() << " has host '" << req.url.host() << "' and is not being intercepted."; std::move(loader_callback).Run({}); // SEP } diff --git a/component/ipfs_url_loader.cc b/component/ipfs_url_loader.cc index a433b362..cadda7ba 100644 --- a/component/ipfs_url_loader.cc +++ b/component/ipfs_url_loader.cc @@ -1,6 +1,6 @@ #include "ipfs_url_loader.h" -#include "gateway_requests.h" +#include "chromium_ipfs_context.h" #include "inter_request_state.h" #include "summarize_headers.h" @@ -71,76 +71,39 @@ void ipfs::IpfsUrlLoader::StartRequest( if (me->original_url_.empty()) { me->original_url_ = resource_request.url.spec(); } - if (resource_request.url.SchemeIs("ipfs")) { - auto ref = resource_request.url.spec(); - DCHECK_EQ(ref.substr(0, 7), "ipfs://"); - // TODO these kinds of shenanigans should have their own special utils file - ref.erase(4, 2); - auto e = ref.find_first_of("#?"); - if (e < ref.size()) { - LOG(INFO) << "Dropping params/frags from '" << ref << "'"; - ref.resize(e); - LOG(INFO) << "Now have '" << ref << "'"; - } - me->StartUnixFsProc(me, ref); + if (resource_request.url.SchemeIs("ipfs") || + resource_request.url.SchemeIs("ipns")) { + auto ns = resource_request.url.scheme(); + auto cid_str = resource_request.url.host(); + auto path = resource_request.url.path(); + auto abs_path = "/" + ns + "/" + cid_str + path; + LOG(INFO) << resource_request.url.spec() << " -> " << abs_path; + me->root_ = cid_str; + me->api_->SetLoaderFactory(*(me->lower_loader_factory_)); + auto whendone = [me](IpfsRequest const& req, ipfs::Response const& res) { + LOG(INFO) << "whendone(" << req.path().to_string() << ',' << res.status_ + << ',' << res.body_.size() << "B mime=" << res.mime_ << ')'; + if (!res.body_.empty()) { + me->ReceiveBlockBytes(res.body_); + } + me->status_ = res.status_; + if (res.status_ / 100 == 4) { + auto p = req.path(); + p.pop(); + std::string cid{p.pop()}; + me->DoesNotExist(cid, p.to_string()); + } else { + me->BlocksComplete(res.mime_); + } + DCHECK(me->complete_); + }; + auto req = std::make_shared(abs_path, whendone); + me->state_->orchestrator().build_response(req); } else { LOG(ERROR) << "Wrong scheme: " << resource_request.url.scheme(); } } -void ipfs::IpfsUrlLoader::StartUnixFsProc(ptr me, std::string_view ipfs_ref) { - VLOG(1) << "Requesting " << ipfs_ref << " by blocks."; - DCHECK_EQ(ipfs_ref.substr(0, 5), "ipfs/"); - auto second_slash = ipfs_ref.find_first_of("/?", 5); - auto cid = ipfs_ref.substr(5, second_slash - 5); - second_slash = ipfs_ref.find('/', 5); - auto qmark = ipfs_ref.find_first_of("?#"); - std::string remainder{"/"}; - if (second_slash < ipfs_ref.size()) { - remainder.assign(ipfs_ref.substr(second_slash + 1)); - } else if (qmark && qmark < ipfs_ref.size()) { - remainder.append(ipfs_ref.substr(qmark)); - } - VLOG(1) << "cid=" << cid << " remainder=" << remainder; - me->root_ = cid; - me->api_->SetLoaderFactory(*lower_loader_factory_); - /* - me->resolver_ = std::make_shared( - me->state_->storage(), me->state_->requestor(), std::string{cid}, - remainder, me->api_); - me->stepper_ = std::make_unique(); - me->stepper_->Start(FROM_HERE, base::Milliseconds(500), - base::BindRepeating(&IpfsUrlLoader::TakeStep, me)); - me->TakeStep(); - */ - auto whendone = [me](IpfsRequest const& req, ipfs::Response const& res) { - LOG(INFO) << "whendone(" << req.path().to_string() << ',' << res.status_ - << ',' << res.body_.size() << "B)"; - if (!res.body_.empty()) { - me->ReceiveBlockBytes(res.body_); - } - me->status_ = res.status_; - if (res.status_ / 100 == 4) { - auto p = req.path(); - p.pop(); - std::string cid{p.pop()}; - me->DoesNotExist(cid, p.to_string()); - } else { - me->BlocksComplete(res.mime_); - } - }; - auto abs_path = std::string{"/ipfs/"}; - abs_path.append(cid); - if (!remainder.empty()) { - if (abs_path.back() != '/' && remainder[0] != '/') { - abs_path.push_back('/'); - } - abs_path.append(remainder); - } - auto req = std::make_shared(abs_path, whendone); - me->state_->orchestrator().build_response(req); -} - void ipfs::IpfsUrlLoader::TakeStep() { if (complete_) { LOG(INFO) << "Timed step(" << original_url_ << "): done."; @@ -197,8 +160,6 @@ void ipfs::IpfsUrlLoader::BlocksComplete(std::string mime_type) { head->was_fetched_via_spdy = false; if (resolver_) { AppendGatewayHeaders(resolver_->involved_cids(), *head->headers); - } else { - LOG(INFO) << "TODO"; } for (auto& [n, v] : additional_outgoing_headers_) { VLOG(1) << "Appending 'additional' header:" << n << '=' << v << '.'; diff --git a/component/ipfs_url_loader.h b/component/ipfs_url_loader.h index f0634b3c..796822f8 100644 --- a/component/ipfs_url_loader.h +++ b/component/ipfs_url_loader.h @@ -15,7 +15,7 @@ #include namespace ipfs { -class GatewayRequests; +class ChromiumIpfsContext; class UnixFsPathResolver; } // namespace ipfs @@ -72,7 +72,7 @@ class IpfsUrlLoader final : public network::mojom::URLLoader, mojo::ScopedDataPipeProducerHandle pipe_prod_ = {}; mojo::ScopedDataPipeConsumerHandle pipe_cons_ = {}; bool complete_ = false; - std::shared_ptr api_; + std::shared_ptr api_; std::string original_url_; std::shared_ptr resolver_; std::string partial_block_; diff --git a/component/ipns_url_loader.cc b/component/ipns_url_loader.cc index 24dfe322..f56c56d6 100644 --- a/component/ipns_url_loader.cc +++ b/component/ipns_url_loader.cc @@ -1,6 +1,6 @@ #include "ipns_url_loader.h" -#include "gateway_requests.h" +#include "chromium_ipfs_context.h" #include "inter_request_state.h" #include "net/base/net_errors.h" diff --git a/component/ipns_url_loader.h b/component/ipns_url_loader.h index f1b9c101..6531ce38 100644 --- a/component/ipns_url_loader.h +++ b/component/ipns_url_loader.h @@ -39,7 +39,7 @@ class IpnsUrlLoader : public network::ResolveHostClientBase, mojo::PendingRemote client_remote_; std::shared_ptr ipfs_loader_; raw_ptr network_context_; - std::shared_ptr api_; + std::shared_ptr api_; raw_ref http_loader_; public: diff --git a/component/network_requestor.cc b/component/network_requestor.cc index a05ac6a1..1e306b12 100644 --- a/component/network_requestor.cc +++ b/component/network_requestor.cc @@ -1,6 +1,6 @@ #include "network_requestor.h" -#include "gateway_requests.h" +#include "chromium_ipfs_context.h" #include "inter_request_state.h" using Self = ipfs::NetworkRequestor; diff --git a/component/patches/118.0.5993.136.patch b/component/patches/118.0.5993.136.patch new file mode 100644 index 00000000..20fcf1c3 --- /dev/null +++ b/component/patches/118.0.5993.136.patch @@ -0,0 +1,483 @@ +diff --git a/chrome/browser/BUILD.gn b/chrome/browser/BUILD.gn +index 31bf3d854d9a6..9aaffa8aab34f 100644 +--- a/chrome/browser/BUILD.gn ++++ b/chrome/browser/BUILD.gn +@@ -39,6 +39,7 @@ import("//rlz/buildflags/buildflags.gni") + import("//sandbox/features.gni") + import("//testing/libfuzzer/fuzzer_test.gni") + import("//third_party/blink/public/public_features.gni") ++import("//third_party/ipfs_client/args.gni") + import("//third_party/protobuf/proto_library.gni") + import("//third_party/webrtc/webrtc.gni") + import("//third_party/widevine/cdm/widevine.gni") +@@ -2525,7 +2526,9 @@ static_library("browser") { + "//ui/web_dialogs", + "//ui/webui", + ] +- ++ if (enable_ipfs) { ++ deps += [ "//components/ipfs" ] ++ } + if (!is_android) { + deps += [ + "//components/user_education/common", +diff --git a/chrome/browser/about_flags.cc b/chrome/browser/about_flags.cc +index 859416382602f..e82d2468dca8f 100644 +--- a/chrome/browser/about_flags.cc ++++ b/chrome/browser/about_flags.cc +@@ -208,6 +208,7 @@ + #include "third_party/blink/public/common/features_generated.h" + #include "third_party/blink/public/common/forcedark/forcedark_switches.h" + #include "third_party/blink/public/common/switches.h" ++#include "third_party/ipfs_client/ipfs_buildflags.h" + #include "ui/accessibility/accessibility_features.h" + #include "ui/accessibility/accessibility_switches.h" + #include "ui/base/ui_base_features.h" +@@ -308,6 +309,10 @@ + #include "extensions/common/switches.h" + #endif // BUILDFLAG(ENABLE_EXTENSIONS) + ++#if BUILDFLAG(ENABLE_IPFS) ++#include "components/ipfs/ipfs_features.h" ++#endif ++ + #if BUILDFLAG(ENABLE_PDF) + #include "pdf/pdf_features.h" + #endif +@@ -9688,6 +9693,14 @@ const FeatureEntry kFeatureEntries[] = { + flag_descriptions::kOmitCorsClientCertDescription, kOsAll, + FEATURE_VALUE_TYPE(network::features::kOmitCorsClientCert)}, + ++#if BUILDFLAG(ENABLE_IPFS) ++ {"enable-ipfs", ++ flag_descriptions::kEnableIpfsName, ++ flag_descriptions::kEnableIpfsDescription, ++ kOsMac | kOsWin | kOsLinux,//TODO: These are the only variants currently getting built, but that is not likely to remain the case ++ FEATURE_VALUE_TYPE(ipfs::kEnableIpfs)}, ++#endif ++ + {"use-idna2008-non-transitional", + flag_descriptions::kUseIDNA2008NonTransitionalName, + flag_descriptions::kUseIDNA2008NonTransitionalDescription, kOsAll, +diff --git a/chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.cc b/chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.cc +index 4c88614c68c25..f8bb12a3b0c2e 100644 +--- a/chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.cc ++++ b/chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.cc +@@ -10,6 +10,8 @@ + #include "chrome/browser/custom_handlers/protocol_handler_registry_factory.h" + #include "chrome/browser/external_protocol/external_protocol_handler.h" + #include "chrome/browser/profiles/profile.h" ++#include "third_party/ipfs_client/ipfs_buildflags.h" ++ + #if BUILDFLAG(IS_ANDROID) + #include "chrome/browser/profiles/profile_android.h" + #endif +@@ -18,6 +20,9 @@ + #include "chrome/browser/ui/android/omnibox/jni_headers/ChromeAutocompleteSchemeClassifier_jni.h" + #endif + #include "components/custom_handlers/protocol_handler_registry.h" ++#if BUILDFLAG(ENABLE_IPFS) ++#include "components/ipfs/ipfs_features.h" ++#endif + #include "content/public/common/url_constants.h" + #include "url/url_util.h" + +@@ -55,12 +60,20 @@ ChromeAutocompleteSchemeClassifier::GetInputTypeForScheme( + if (scheme.empty()) { + return metrics::OmniboxInputType::EMPTY; + } +- if (base::IsStringASCII(scheme) && +- (ProfileIOData::IsHandledProtocol(scheme) || +- base::EqualsCaseInsensitiveASCII(scheme, content::kViewSourceScheme) || +- base::EqualsCaseInsensitiveASCII(scheme, url::kJavaScriptScheme) || +- base::EqualsCaseInsensitiveASCII(scheme, url::kDataScheme))) { +- return metrics::OmniboxInputType::URL; ++ if (base::IsStringASCII(scheme)) { ++ if (ProfileIOData::IsHandledProtocol(scheme) || ++ base::EqualsCaseInsensitiveASCII(scheme, content::kViewSourceScheme) || ++ base::EqualsCaseInsensitiveASCII(scheme, url::kJavaScriptScheme) || ++ base::EqualsCaseInsensitiveASCII(scheme, url::kDataScheme)) { ++ return metrics::OmniboxInputType::URL; ++ } ++#if BUILDFLAG(ENABLE_IPFS) ++ if (base::FeatureList::IsEnabled(ipfs::kEnableIpfs) && ++ (base::EqualsCaseInsensitiveASCII(scheme, "ipfs") || base::EqualsCaseInsensitiveASCII(scheme, "ipns")) ++ ) { ++ return metrics::OmniboxInputType::URL; ++ } ++#endif + } + + // Also check for schemes registered via registerProtocolHandler(), which +diff --git a/chrome/browser/chrome_content_browser_client.cc b/chrome/browser/chrome_content_browser_client.cc +index 6f989cff781d7..c01f0588089cf 100644 +--- a/chrome/browser/chrome_content_browser_client.cc ++++ b/chrome/browser/chrome_content_browser_client.cc +@@ -359,6 +359,7 @@ + #include "third_party/blink/public/common/switches.h" + #include "third_party/blink/public/mojom/browsing_topics/browsing_topics.mojom.h" + #include "third_party/blink/public/public_buildflags.h" ++#include "third_party/ipfs_client/ipfs_buildflags.h" + #include "third_party/widevine/cdm/buildflags.h" + #include "ui/base/clipboard/clipboard_format_type.h" + #include "ui/base/l10n/l10n_util.h" +@@ -476,6 +477,12 @@ + #include "chrome/browser/fuchsia/chrome_browser_main_parts_fuchsia.h" + #endif + ++#if BUILDFLAG(ENABLE_IPFS) ++#include "components/ipfs/interceptor.h" ++#include "components/ipfs/ipfs_features.h" ++#include "components/ipfs/url_loader_factory.h" ++#endif ++ + #if BUILDFLAG(IS_CHROMEOS) + #include "base/debug/leak_annotations.h" + #include "chrome/browser/chromeos/enterprise/incognito_navigation_throttle.h" +@@ -6112,12 +6119,23 @@ void ChromeContentBrowserClient:: + const absl::optional& request_initiator_origin, + NonNetworkURLLoaderFactoryMap* factories) { + #if BUILDFLAG(IS_CHROMEOS_ASH) || BUILDFLAG(ENABLE_EXTENSIONS) || \ +- !BUILDFLAG(IS_ANDROID) ++ !BUILDFLAG(IS_ANDROID) || BUILDFLAG(ENABLE_IPFS) + content::RenderFrameHost* frame_host = + RenderFrameHost::FromID(render_process_id, render_frame_id); + WebContents* web_contents = WebContents::FromRenderFrameHost(frame_host); + #endif // BUILDFLAG(IS_CHROMEOS_ASH) || BUILDFLAG(ENABLE_EXTENSIONS) || \ +- // !BUILDFLAG(IS_ANDROID) ++ // !BUILDFLAG(IS_ANDROID) || BUILDFLAG(ENABLE_IPFS) ++#if BUILDFLAG(ENABLE_IPFS) ++ if (base::FeatureList::IsEnabled(ipfs::kEnableIpfs)) { ++ network::mojom::URLLoaderFactory* default_factory = g_browser_process->system_network_context_manager()->GetURLLoaderFactory(); ++ ipfs::IpfsURLLoaderFactory::Create( ++ factories, ++ web_contents->GetBrowserContext(), ++ default_factory, ++ GetSystemNetworkContext() ++ ); ++ } ++#endif + + #if BUILDFLAG(IS_CHROMEOS_ASH) + if (web_contents) { +@@ -6259,6 +6277,11 @@ ChromeContentBrowserClient::WillCreateURLLoaderRequestInterceptors( + scoped_refptr navigation_response_task_runner) { + std::vector> + interceptors; ++#if BUILDFLAG(ENABLE_IPFS) ++ if (base::FeatureList::IsEnabled(ipfs::kEnableIpfs)) { ++ interceptors.push_back(std::make_unique(g_browser_process->system_network_context_manager()->GetURLLoaderFactory(), GetSystemNetworkContext())); ++ } ++#endif + #if BUILDFLAG(ENABLE_OFFLINE_PAGES) + interceptors.push_back( + std::make_unique( +diff --git a/chrome/browser/flag-metadata.json b/chrome/browser/flag-metadata.json +index 39537fb78bd7e..21b64b887947e 100644 +--- a/chrome/browser/flag-metadata.json ++++ b/chrome/browser/flag-metadata.json +@@ -2997,6 +2997,11 @@ + "owners": [ "hanxi", "wychen" ], + "expiry_milestone": 130 + }, ++ { ++ "name": "enable-ipfs", ++ "owners": [ "//components/ipfs/OWNERS" ], ++ "expiry_milestone": 150 ++ }, + { + "name": "enable-isolated-sandboxed-iframes", + "owners": [ "wjmaclean@chromium.org", "alexmos@chromium.org", "creis@chromium.org" ], +diff --git a/chrome/browser/flag_descriptions.cc b/chrome/browser/flag_descriptions.cc +index 5829b6a0b3401..cb19beb1603bb 100644 +--- a/chrome/browser/flag_descriptions.cc ++++ b/chrome/browser/flag_descriptions.cc +@@ -256,6 +256,11 @@ const char kEnableBenchmarkingDescription[] = + "after 3 restarts. On the third restart, the flag will appear to be off " + "but the effect is still active."; + ++#if BUILDFLAG(ENABLE_IPFS) ++extern const char kEnableIpfsName[] = "Enable IPFS"; ++extern const char kEnableIpfsDescription[] = "Enable ipfs:// and ipns:// URLs"; ++#endif ++ + const char kPreloadingOnPerformancePageName[] = + "Preloading Settings on Performance Page"; + const char kPreloadingOnPerformancePageDescription[] = +diff --git a/chrome/browser/flag_descriptions.h b/chrome/browser/flag_descriptions.h +index ff72a308d37e7..4422b2b41ca54 100644 +--- a/chrome/browser/flag_descriptions.h ++++ b/chrome/browser/flag_descriptions.h +@@ -21,6 +21,7 @@ + #include "pdf/buildflags.h" + #include "printing/buildflags/buildflags.h" + #include "third_party/blink/public/common/buildflags.h" ++#include "third_party/ipfs_client/ipfs_buildflags.h" + + // This file declares strings used in chrome://flags. These messages are not + // translated, because instead of end-users they target Chromium developers and +@@ -157,6 +158,11 @@ extern const char kDownloadWarningImprovementsDescription[]; + extern const char kEnableBenchmarkingName[]; + extern const char kEnableBenchmarkingDescription[]; + ++#if BUILDFLAG(ENABLE_IPFS) ++extern const char kEnableIpfsName[]; ++extern const char kEnableIpfsDescription[]; ++#endif ++ + #if BUILDFLAG(USE_FONTATIONS_BACKEND) + extern const char kFontationsFontBackendName[]; + extern const char kFontationsFontBackendDescription[]; +diff --git a/chrome/common/chrome_content_client.cc b/chrome/common/chrome_content_client.cc +index 246ec9c5c911f..4b3a9619cc652 100644 +--- a/chrome/common/chrome_content_client.cc ++++ b/chrome/common/chrome_content_client.cc +@@ -53,6 +53,7 @@ + #include "net/http/http_util.h" + #include "pdf/buildflags.h" + #include "ppapi/buildflags/buildflags.h" ++#include "third_party/ipfs_client/ipfs_buildflags.h" + #include "third_party/widevine/cdm/buildflags.h" + #include "ui/base/l10n/l10n_util.h" + #include "ui/base/resource/resource_bundle.h" +@@ -296,6 +297,14 @@ void ChromeContentClient::AddAdditionalSchemes(Schemes* schemes) { + #if BUILDFLAG(IS_ANDROID) + schemes->local_schemes.push_back(url::kContentScheme); + #endif ++ for ( const char* ip_s : {"ipfs", "ipns"} ) { ++ schemes->standard_schemes.push_back(ip_s); ++#if BUILDFLAG(ENABLE_IPFS) ++ schemes->cors_enabled_schemes.push_back(ip_s); ++ schemes->secure_schemes.push_back(ip_s); ++ schemes->csp_bypassing_schemes.push_back(ip_s); ++#endif ++ } + } + + std::u16string ChromeContentClient::GetLocalizedString(int message_id) { +diff --git a/components/open_from_clipboard/clipboard_recent_content_generic.cc b/components/open_from_clipboard/clipboard_recent_content_generic.cc +index 4dcafecbc66c6..5f4fde8808e4f 100644 +--- a/components/open_from_clipboard/clipboard_recent_content_generic.cc ++++ b/components/open_from_clipboard/clipboard_recent_content_generic.cc +@@ -12,6 +12,7 @@ + #include "ui/base/clipboard/clipboard.h" + #include "ui/base/data_transfer_policy/data_transfer_endpoint.h" + #include "url/url_util.h" ++#include "third_party/ipfs_client/ipfs_buildflags.h" + + #if BUILDFLAG(IS_ANDROID) + #include "ui/base/clipboard/clipboard_android.h" +@@ -21,6 +22,9 @@ namespace { + // Schemes appropriate for suggestion by ClipboardRecentContent. + const char* kAuthorizedSchemes[] = { + url::kAboutScheme, url::kDataScheme, url::kHttpScheme, url::kHttpsScheme, ++#if BUILDFLAG(ENABLE_IPFS) ++ "ipfs", "ipns" ++#endif + // TODO(mpearson): add support for chrome:// URLs. Right now the scheme + // for that lives in content and is accessible via + // GetEmbedderRepresentationOfAboutScheme() or content::kChromeUIScheme +diff --git a/net/dns/dns_config_service_linux.cc b/net/dns/dns_config_service_linux.cc +index 5273da5190277..12b28b86a4c00 100644 +--- a/net/dns/dns_config_service_linux.cc ++++ b/net/dns/dns_config_service_linux.cc +@@ -272,11 +272,11 @@ bool IsNsswitchConfigCompatible( + // Ignore any entries after `kDns` because Chrome will fallback to the + // system resolver if a result was not found in DNS. + return true; +- ++ case NsswitchReader::Service::kResolve: ++ break; + case NsswitchReader::Service::kMdns: + case NsswitchReader::Service::kMdns4: + case NsswitchReader::Service::kMdns6: +- case NsswitchReader::Service::kResolve: + case NsswitchReader::Service::kNis: + RecordIncompatibleNsswitchReason( + IncompatibleNsswitchReason::kIncompatibleService, +diff --git a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +index 4eadf46ea0c24..d62fc7fb14e01 100644 +--- a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc ++++ b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +@@ -67,7 +67,7 @@ class URLSchemesRegistry final { + // is considered secure. Additional checks are performed to ensure that + // other http pages are filtered out. + service_worker_schemes({"http", "https"}), +- fetch_api_schemes({"http", "https"}), ++ fetch_api_schemes({"http", "https", "ipfs", "ipns"}), + allowed_in_referrer_schemes({"http", "https"}) { + for (auto& scheme : url::GetCorsEnabledSchemes()) + cors_enabled_schemes.insert(scheme.c_str()); +diff --git a/url/BUILD.gn b/url/BUILD.gn +index 7a10d5d28206f..bb117200c692a 100644 +--- a/url/BUILD.gn ++++ b/url/BUILD.gn +@@ -5,6 +5,7 @@ + import("//build/buildflag_header.gni") + import("//testing/libfuzzer/fuzzer_test.gni") + import("//testing/test.gni") ++import("//third_party/ipfs_client/args.gni") + import("features.gni") + + import("//build/config/android/jni.gni") +@@ -62,10 +63,18 @@ component("url") { + + defines = [ "IS_URL_IMPL" ] + +- public_deps = [ "//base" ] ++ public_deps = [ ++ "//base", ++ "//third_party/ipfs_client:ipfs_buildflags", ++ ] + + deps = [ "//base/third_party/dynamic_annotations" ] + ++ if (enable_ipfs) { ++ sources += [ "url_canon_ipfs.cc" ] ++ deps += [ "//third_party/ipfs_client:ipfs_client" ] ++ } ++ + if (is_win) { + # Don't conflict with Windows' "url.dll". + output_name = "url_lib" +diff --git a/url/url_canon.h b/url/url_canon.h +index 94b44426fa33a..d481e73a0ee76 100644 +--- a/url/url_canon.h ++++ b/url/url_canon.h +@@ -13,6 +13,7 @@ + #include "base/memory/raw_ptr_exclusion.h" + #include "base/numerics/clamped_math.h" + #include "url/third_party/mozilla/url_parse.h" ++#include "third_party/ipfs_client/ipfs_buildflags.h" + + namespace url { + +@@ -688,6 +689,25 @@ bool CanonicalizeMailtoURL(const char16_t* spec, + CanonOutput* output, + Parsed* new_parsed); + ++#if BUILDFLAG(ENABLE_IPFS) ++COMPONENT_EXPORT(URL) ++bool CanonicalizeIpfsURL(const char* spec, ++ int spec_len, ++ const Parsed& parsed, ++ SchemeType scheme_type, ++ CharsetConverter* query_converter, ++ CanonOutput* output, ++ Parsed* new_parsed); ++COMPONENT_EXPORT(URL) ++bool CanonicalizeIpfsURL(const char16_t* spec, ++ int spec_len, ++ const Parsed& parsed, ++ SchemeType scheme_type, ++ CharsetConverter* query_converter, ++ CanonOutput* output, ++ Parsed* new_parsed); ++#endif ++ + // Part replacer -------------------------------------------------------------- + + // Internal structure used for storing separate strings for each component. +diff --git a/url/url_canon_ipfs.cc b/url/url_canon_ipfs.cc +new file mode 100644 +index 0000000000000..da3a5f032b5e8 +--- /dev/null ++++ b/url/url_canon_ipfs.cc +@@ -0,0 +1,72 @@ ++#include "url_canon_internal.h" ++ ++#include ++#include ++ ++#include ++ ++namespace m = libp2p::multi; ++using Cid = m::ContentIdentifier; ++using CidCodec = m::ContentIdentifierCodec; ++ ++bool url::CanonicalizeIpfsURL(const char* spec, ++ int spec_len, ++ const Parsed& parsed, ++ SchemeType scheme_type, ++ CharsetConverter* charset_converter, ++ CanonOutput* output, ++ Parsed* output_parsed) { ++ if ( spec_len < 1 || !spec ) { ++ return false; ++ } ++ if ( parsed.host.len < 1 ) { ++ return false; ++ } ++ std::string cid_str{ spec + parsed.host.begin, static_cast(parsed.host.len) }; ++ auto maybe_cid = CidCodec::fromString(cid_str); ++ if ( !maybe_cid.has_value() ) { ++ auto e = libp2p::multi::Stringify(maybe_cid.error()); ++ std::ostringstream err; ++ err << e << ' ' ++ << std::string_view{spec,static_cast(spec_len)}; ++ maybe_cid = ipfs::id_cid::forText( err.str() ); ++ } ++ auto cid = maybe_cid.value(); ++ if ( cid.version == Cid::Version::V0 ) { ++ //TODO dcheck content_type == DAG_PB && content_address.getType() == sha256 ++ cid = Cid{ ++ Cid::Version::V1, ++ cid.content_type, ++ cid.content_address ++ }; ++ } ++ auto as_str = CidCodec::toString(cid); ++ if ( !as_str.has_value() ) { ++ return false; ++ } ++ std::string stdurl{ spec, static_cast(parsed.host.begin) }; ++ stdurl.append( as_str.value() ); ++ stdurl.append( spec + parsed.host.end(), spec_len - parsed.host.end() ); ++ spec = stdurl.data(); ++ spec_len = static_cast(stdurl.size()); ++ Parsed parsed_input; ++ ParseStandardURL(spec, spec_len, &parsed_input); ++ return CanonicalizeStandardURL( ++ spec, spec_len, ++ parsed_input, ++ scheme_type, ++ charset_converter, ++ output, output_parsed ++ ); ++} ++bool url::CanonicalizeIpfsURL(const char16_t* spec, ++ int spec_len, ++ const Parsed& parsed, ++ SchemeType scheme_type, ++ CharsetConverter* query_converter, ++ CanonOutput* output, ++ Parsed* new_parsed) { ++ RawCanonOutput<2048> as8; ++ ConvertUTF16ToUTF8(spec, spec_len, &as8); ++ return CanonicalizeIpfsURL(as8.data(), as8.length(), parsed, scheme_type, query_converter, output, new_parsed); ++} +diff --git a/url/url_util.cc b/url/url_util.cc +index 001c50e72fd24..3f42bc4fb521c 100644 +--- a/url/url_util.cc ++++ b/url/url_util.cc +@@ -19,6 +19,7 @@ + #include "url/url_constants.h" + #include "url/url_file.h" + #include "url/url_util_internal.h" ++#include "third_party/ipfs_client/ipfs_buildflags.h" + + namespace url { + +@@ -290,7 +291,11 @@ bool DoCanonicalize(const CHAR* spec, + success = CanonicalizeFileSystemURL(spec, spec_len, parsed_input, + charset_converter, output, + output_parsed); +- ++ } else if (DoCompareSchemeComponent(spec, scheme, "ipfs")) { ++ // Switch multibase away from case-sensitive ones before continuing canonicalization. ++ ParseStandardURL(spec, spec_len, &parsed_input); ++ success = CanonicalizeIpfsURL(spec, spec_len, parsed_input, scheme_type, ++ charset_converter, output, output_parsed); + } else if (DoIsStandard(spec, scheme, &scheme_type)) { + // All "normal" URLs. + ParseStandardURL(spec, spec_len, &parsed_input); diff --git a/component/patches/119.0.6045.21.patch b/component/patches/119.0.6045.21.patch index 37333877..4b7bbcad 100644 --- a/component/patches/119.0.6045.21.patch +++ b/component/patches/119.0.6045.21.patch @@ -466,14 +466,13 @@ index 81b546212d93d..43638079a7818 100644 success = CanonicalizeFileSystemURL(spec, spec_len, parsed_input, charset_converter, output, output_parsed); -- -+#if BUILDFLAG(ENABLE_IPFS) + + } else if (DoCompareSchemeComponent(spec, scheme, "ipfs")) { + // Switch multibase away from case-sensitive ones before continuing canonicalization. + ParseStandardURL(spec, spec_len, &parsed_input); + success = CanonicalizeIpfsURL(spec, spec_len, parsed_input, scheme_type, + charset_converter, output, output_parsed); -+#endif ++ } else if (DoIsStandard(spec, scheme, &scheme_type)) { // All "normal" URLs. ParseStandardURL(spec, spec_len, &parsed_input); diff --git a/component/patches/118.0.5993.118.patch b/component/patches/120.0.6099.18.patch similarity index 82% rename from component/patches/118.0.5993.118.patch rename to component/patches/120.0.6099.18.patch index b758d8d0..4b6f8514 100644 --- a/component/patches/118.0.5993.118.patch +++ b/component/patches/120.0.6099.18.patch @@ -1,22 +1,8 @@ -diff --git a/base/threading/thread_local_internal.h b/base/threading/thread_local_internal.h -index ed99410ea8a31..d89d48ba981ab 100644 ---- a/base/threading/thread_local_internal.h -+++ b/base/threading/thread_local_internal.h -@@ -30,8 +30,7 @@ class CheckedThreadLocalOwnedPointer { - public: - CheckedThreadLocalOwnedPointer() = default; - -- CheckedThreadLocalOwnedPointer(const CheckedThreadLocalOwnedPointer&) = -- delete; -+ CheckedThreadLocalOwnedPointer(const CheckedThreadLocalOwnedPointer&) = delete; - CheckedThreadLocalOwnedPointer& operator=( - const CheckedThreadLocalOwnedPointer&) = delete; - diff --git a/chrome/browser/BUILD.gn b/chrome/browser/BUILD.gn -index 31bf3d854d9a6..4dd2760b90ae1 100644 +index bce36d7ffb408..6899bd840ebbf 100644 --- a/chrome/browser/BUILD.gn +++ b/chrome/browser/BUILD.gn -@@ -39,6 +39,7 @@ import("//rlz/buildflags/buildflags.gni") +@@ -40,6 +40,7 @@ import("//rlz/buildflags/buildflags.gni") import("//sandbox/features.gni") import("//testing/libfuzzer/fuzzer_test.gni") import("//third_party/blink/public/public_features.gni") @@ -24,7 +10,7 @@ index 31bf3d854d9a6..4dd2760b90ae1 100644 import("//third_party/protobuf/proto_library.gni") import("//third_party/webrtc/webrtc.gni") import("//third_party/widevine/cdm/widevine.gni") -@@ -2568,6 +2569,10 @@ static_library("browser") { +@@ -2658,6 +2659,10 @@ static_library("browser") { ] } @@ -36,10 +22,10 @@ index 31bf3d854d9a6..4dd2760b90ae1 100644 deps += [ "//chrome/browser/screen_ai:screen_ai_dlc_installer" ] } diff --git a/chrome/browser/about_flags.cc b/chrome/browser/about_flags.cc -index 859416382602f..e82d2468dca8f 100644 +index 9ab4bb62b9f7e..fdbe2476803a5 100644 --- a/chrome/browser/about_flags.cc +++ b/chrome/browser/about_flags.cc -@@ -208,6 +208,7 @@ +@@ -213,6 +213,7 @@ #include "third_party/blink/public/common/features_generated.h" #include "third_party/blink/public/common/forcedark/forcedark_switches.h" #include "third_party/blink/public/common/switches.h" @@ -47,7 +33,7 @@ index 859416382602f..e82d2468dca8f 100644 #include "ui/accessibility/accessibility_features.h" #include "ui/accessibility/accessibility_switches.h" #include "ui/base/ui_base_features.h" -@@ -308,6 +309,10 @@ +@@ -314,6 +315,10 @@ #include "extensions/common/switches.h" #endif // BUILDFLAG(ENABLE_EXTENSIONS) @@ -58,7 +44,7 @@ index 859416382602f..e82d2468dca8f 100644 #if BUILDFLAG(ENABLE_PDF) #include "pdf/pdf_features.h" #endif -@@ -9688,6 +9693,14 @@ const FeatureEntry kFeatureEntries[] = { +@@ -9914,6 +9919,14 @@ const FeatureEntry kFeatureEntries[] = { flag_descriptions::kOmitCorsClientCertDescription, kOsAll, FEATURE_VALUE_TYPE(network::features::kOmitCorsClientCert)}, @@ -124,10 +110,19 @@ index 4c88614c68c25..f8bb12a3b0c2e 100644 // Also check for schemes registered via registerProtocolHandler(), which diff --git a/chrome/browser/chrome_content_browser_client.cc b/chrome/browser/chrome_content_browser_client.cc -index 6f989cff781d7..e5e0dd4c59d61 100644 +index eb7b5332f5e4c..3e232b3af9357 100644 --- a/chrome/browser/chrome_content_browser_client.cc +++ b/chrome/browser/chrome_content_browser_client.cc -@@ -359,6 +359,7 @@ +@@ -228,6 +228,8 @@ + #include "components/error_page/common/localized_error.h" + #include "components/error_page/content/browser/net_error_auto_reloader.h" + #include "components/google/core/common/google_switches.h" ++#include "components/ipfs/interceptor.h" ++#include "components/ipfs/url_loader_factory.h" + #include "components/keep_alive_registry/keep_alive_types.h" + #include "components/keep_alive_registry/scoped_keep_alive.h" + #include "components/language/core/browser/pref_names.h" +@@ -364,6 +366,7 @@ #include "third_party/blink/public/common/switches.h" #include "third_party/blink/public/mojom/browsing_topics/browsing_topics.mojom.h" #include "third_party/blink/public/public_buildflags.h" @@ -135,7 +130,7 @@ index 6f989cff781d7..e5e0dd4c59d61 100644 #include "third_party/widevine/cdm/buildflags.h" #include "ui/base/clipboard/clipboard_format_type.h" #include "ui/base/l10n/l10n_util.h" -@@ -476,6 +477,12 @@ +@@ -487,6 +490,12 @@ #include "chrome/browser/fuchsia/chrome_browser_main_parts_fuchsia.h" #endif @@ -147,8 +142,8 @@ index 6f989cff781d7..e5e0dd4c59d61 100644 + #if BUILDFLAG(IS_CHROMEOS) #include "base/debug/leak_annotations.h" - #include "chrome/browser/chromeos/enterprise/incognito_navigation_throttle.h" -@@ -6112,12 +6119,23 @@ void ChromeContentBrowserClient:: + #include "chrome/browser/apps/intent_helper/chromeos_disabled_apps_throttle.h" +@@ -6177,12 +6186,23 @@ void ChromeContentBrowserClient:: const absl::optional& request_initiator_origin, NonNetworkURLLoaderFactoryMap* factories) { #if BUILDFLAG(IS_CHROMEOS_ASH) || BUILDFLAG(ENABLE_EXTENSIONS) || \ @@ -174,7 +169,7 @@ index 6f989cff781d7..e5e0dd4c59d61 100644 #if BUILDFLAG(IS_CHROMEOS_ASH) if (web_contents) { -@@ -6259,6 +6277,11 @@ ChromeContentBrowserClient::WillCreateURLLoaderRequestInterceptors( +@@ -6324,6 +6344,11 @@ ChromeContentBrowserClient::WillCreateURLLoaderRequestInterceptors( scoped_refptr navigation_response_task_runner) { std::vector> interceptors; @@ -186,11 +181,27 @@ index 6f989cff781d7..e5e0dd4c59d61 100644 #if BUILDFLAG(ENABLE_OFFLINE_PAGES) interceptors.push_back( std::make_unique( +diff --git a/chrome/browser/flag-metadata.json b/chrome/browser/flag-metadata.json +index 895098c0bd462..b9bcd90fd481c 100644 +--- a/chrome/browser/flag-metadata.json ++++ b/chrome/browser/flag-metadata.json +@@ -2863,6 +2863,11 @@ + "owners": [ "hanxi@chromium.org", "wychen@chromium.org" ], + "expiry_milestone": 130 + }, ++ { ++ "name": "enable-ipfs", ++ "owners": [ "//components/ipfs/OWNERS" ], ++ "expiry_milestone": 150 ++ }, + { + "name": "enable-isolated-sandboxed-iframes", + "owners": [ "wjmaclean@chromium.org", "alexmos@chromium.org", "creis@chromium.org" ], diff --git a/chrome/browser/flag_descriptions.cc b/chrome/browser/flag_descriptions.cc -index 5829b6a0b3401..cb19beb1603bb 100644 +index 0d17970001d18..e4141fc8aa12b 100644 --- a/chrome/browser/flag_descriptions.cc +++ b/chrome/browser/flag_descriptions.cc -@@ -256,6 +256,11 @@ const char kEnableBenchmarkingDescription[] = +@@ -248,6 +248,11 @@ const char kEnableBenchmarkingDescription[] = "after 3 restarts. On the third restart, the flag will appear to be off " "but the effect is still active."; @@ -203,10 +214,10 @@ index 5829b6a0b3401..cb19beb1603bb 100644 "Preloading Settings on Performance Page"; const char kPreloadingOnPerformancePageDescription[] = diff --git a/chrome/browser/flag_descriptions.h b/chrome/browser/flag_descriptions.h -index ff72a308d37e7..4422b2b41ca54 100644 +index 64e4e09d02a1d..8718b40facba9 100644 --- a/chrome/browser/flag_descriptions.h +++ b/chrome/browser/flag_descriptions.h -@@ -21,6 +21,7 @@ +@@ -22,6 +22,7 @@ #include "pdf/buildflags.h" #include "printing/buildflags/buildflags.h" #include "third_party/blink/public/common/buildflags.h" @@ -214,7 +225,7 @@ index ff72a308d37e7..4422b2b41ca54 100644 // This file declares strings used in chrome://flags. These messages are not // translated, because instead of end-users they target Chromium developers and -@@ -157,6 +158,11 @@ extern const char kDownloadWarningImprovementsDescription[]; +@@ -165,6 +166,11 @@ extern const char kDownloadWarningImprovementsDescription[]; extern const char kEnableBenchmarkingName[]; extern const char kEnableBenchmarkingDescription[]; @@ -274,8 +285,21 @@ index 5273da5190277..12b28b86a4c00 100644 case NsswitchReader::Service::kNis: RecordIncompatibleNsswitchReason( IncompatibleNsswitchReason::kIncompatibleService, +diff --git a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +index 4eadf46ea0c24..d62fc7fb14e01 100644 +--- a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc ++++ b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +@@ -67,7 +67,7 @@ class URLSchemesRegistry final { + // is considered secure. Additional checks are performed to ensure that + // other http pages are filtered out. + service_worker_schemes({"http", "https"}), +- fetch_api_schemes({"http", "https"}), ++ fetch_api_schemes({"http", "https", "ipfs", "ipns"}), + allowed_in_referrer_schemes({"http", "https"}) { + for (auto& scheme : url::GetCorsEnabledSchemes()) + cors_enabled_schemes.insert(scheme.c_str()); diff --git a/url/BUILD.gn b/url/BUILD.gn -index 7a10d5d28206f..6a700e3e4a690 100644 +index c525c166979d6..ce2b1ae43c0a7 100644 --- a/url/BUILD.gn +++ b/url/BUILD.gn @@ -5,6 +5,7 @@ @@ -285,25 +309,17 @@ index 7a10d5d28206f..6a700e3e4a690 100644 +import("//third_party/ipfs_client/args.gni") import("features.gni") - import("//build/config/android/jni.gni") -@@ -62,15 +63,25 @@ component("url") { - - defines = [ "IS_URL_IMPL" ] - -- public_deps = [ "//base" ] -+ public_deps = [ -+ "//base", -+ "//third_party/ipfs_client", -+ ] - -- deps = [ "//base/third_party/dynamic_annotations" ] -+ deps = [ -+ "//base/third_party/dynamic_annotations", -+ ] + import("//build/config/cronet/config.gni") +@@ -67,6 +68,7 @@ component("url") { + public_deps = [ + "//base", + "//build:robolectric_buildflags", ++ "//third_party/ipfs_client:ipfs_buildflags", + ] - if (is_win) { - # Don't conflict with Windows' "url.dll". - output_name = "url_lib" + configs += [ "//build/config/compiler:wexit_time_destructors" ] +@@ -89,6 +91,11 @@ component("url") { + public_configs = [ "//third_party/jdk" ] } + if (enable_ipfs) { @@ -311,14 +327,14 @@ index 7a10d5d28206f..6a700e3e4a690 100644 + deps += [ "//third_party/ipfs_client:ipfs_client" ] + } + - # ICU support. - if (use_platform_icu_alternatives) { - if (is_android) { + if (is_win) { + # Don't conflict with Windows' "url.dll". + output_name = "url_lib" diff --git a/url/url_canon.h b/url/url_canon.h -index 94b44426fa33a..19f1f0789164d 100644 +index d3a7fabf09fa8..06db17242248f 100644 --- a/url/url_canon.h +++ b/url/url_canon.h -@@ -688,6 +688,23 @@ bool CanonicalizeMailtoURL(const char16_t* spec, +@@ -697,6 +697,23 @@ bool CanonicalizeMailtoURL(const char16_t* spec, CanonOutput* output, Parsed* new_parsed); @@ -421,10 +437,10 @@ index 0000000000000..da3a5f032b5e8 + return CanonicalizeIpfsURL(as8.data(), as8.length(), parsed, scheme_type, query_converter, output, new_parsed); +} diff --git a/url/url_util.cc b/url/url_util.cc -index 001c50e72fd24..97476afbf56e3 100644 +index 9258cfcfada47..daf10e4c3b741 100644 --- a/url/url_util.cc +++ b/url/url_util.cc -@@ -291,6 +291,12 @@ bool DoCanonicalize(const CHAR* spec, +@@ -277,6 +277,12 @@ bool DoCanonicalize(const CHAR* spec, charset_converter, output, output_parsed); diff --git a/component/patches/121.0.6100.0.patch b/component/patches/121.0.6100.0.patch index ec0d91ec..e5080b2b 100644 --- a/component/patches/121.0.6100.0.patch +++ b/component/patches/121.0.6100.0.patch @@ -238,6 +238,2479 @@ index 246ec9c5c911f..5d66d133a7907 100644 } std::u16string ChromeContentClient::GetLocalizedString(int message_id) { +diff --git a/components/ipfs/BUILD.gn b/components/ipfs/BUILD.gn +new file mode 100644 +index 0000000000000..4442217db7375 +--- /dev/null ++++ b/components/ipfs/BUILD.gn +@@ -0,0 +1,61 @@ ++import("//testing/test.gni") ++import("//third_party/ipfs_client/args.gni") ++ ++if (enable_ipfs) { ++ ++ component("ipfs") { ++ sources = [ ++ "cache_requestor.cc", ++ "cache_requestor.h", ++ "crypto_api.cc", ++ "crypto_api.h", ++ "gateway_requests.cc", ++ "gateway_requests.h", ++ "inter_request_state.cc", ++ "inter_request_state.h", ++ "interceptor.cc", ++ "interceptor.h", ++ "ipfs_features.cc", ++ "ipfs_features.h", ++ "ipfs_url_loader.cc", ++ "ipfs_url_loader.h", ++ "ipns_cbor.cc", ++ "ipns_cbor.h", ++ "ipns_url_loader.cc", ++ "ipns_url_loader.h", ++ "network_requestor.cc", ++ "network_requestor.h", ++ "profile_prefs.cc", ++ "profile_prefs.h", ++ "summarize_headers.cc", ++ "summarize_headers.h", ++ "url_loader_factory.cc", ++ "url_loader_factory.h", ++ ] ++ defines = [ ] ++ include_dirs = [ ++ ".", ++ "ipfs_client", ++ "ipfs_client/unix_fs", ++ ] ++ deps = [ ++ "//content", ++ "//crypto", ++ "//base", ++ "//components/cbor", ++ "//components/prefs", ++ "//components/webcrypto:webcrypto", ++ "//mojo/public/cpp/bindings", ++ "//services/network:network_service", ++ "//services/network/public/cpp:cpp", ++ "//services/network/public/mojom:url_loader_base", ++ "//url", ++ "//third_party/blink/public:blink", ++ ] ++ public_deps = [ ++ "//third_party/ipfs_client", ++ ] ++ defines = [ "IS_IPFS_IMPL" ] ++ } ++ ++} +diff --git a/components/ipfs/README.md b/components/ipfs/README.md +new file mode 100644 +index 0000000000000..1333ed77b7e1e +--- /dev/null ++++ b/components/ipfs/README.md +@@ -0,0 +1 @@ ++TODO +diff --git a/components/ipfs/cache_requestor.cc b/components/ipfs/cache_requestor.cc +new file mode 100644 +index 0000000000000..19426a979bb0e +--- /dev/null ++++ b/components/ipfs/cache_requestor.cc +@@ -0,0 +1,242 @@ ++#include "cache_requestor.h" ++ ++#include "gateway_requests.h" ++#include "inter_request_state.h" ++ ++#include ++#include ++ ++#include ++ ++using Self = ipfs::CacheRequestor; ++namespace dc = disk_cache; ++ ++Self::CacheRequestor(net::CacheType typ, ++ InterRequestState& state, ++ base::FilePath base) ++ : type_{typ}, state_{state} { ++ if (!base.empty()) { ++ path_ = base.AppendASCII("IpfsBlockCache"); ++ } ++ Start(); ++} ++void Self::Start() { ++ if (pending_) { ++ return; ++ } ++ auto result = dc::CreateCacheBackend( ++ type_, net::CACHE_BACKEND_DEFAULT, {}, path_, 0, ++ dc::ResetHandling::kNeverReset, ++ // dc::ResetHandling::kResetOnError, ++ nullptr, base::BindOnce(&Self::Assign, base::Unretained(this))); ++ LOG(INFO) << "Start(" << result.net_error << ')' << result.net_error; ++ pending_ = result.net_error == net::ERR_IO_PENDING; ++ if (!pending_) { ++ Assign(std::move(result)); ++ } ++} ++Self::~CacheRequestor() noexcept = default; ++ ++void Self::Assign(dc::BackendResult res) { ++ pending_ = false; ++ if (res.net_error == net::OK) { ++ LOG(INFO) << "Initialized a cache of type " << name(); ++ cache_.swap(res.backend); ++ } else { ++ LOG(ERROR) << "Trouble opening " << name() << ": " << res.net_error; ++ Start(); ++ } ++} ++void Self::FetchEntry( ++ std::string key, ++ net::RequestPriority priority, ++ std::function hit, ++ std::function miss) { ++ Task task; ++ task.key = key; ++ task.hit = hit; ++ task.miss = miss; ++ StartFetch(task, priority); ++} ++void Self::RequestByCid(std::string cid, ++ std::shared_ptr listen, ++ Priority prio) { ++ DCHECK(listen); ++ Task task; ++ task.key = cid; ++ task.listener = listen; ++ auto p = std::min(prio + 0, net::MAXIMUM_PRIORITY + 0); ++ StartFetch(task, static_cast(p)); ++} ++void Self::StartFetch(Task& task, net::RequestPriority priority) { ++ if (pending_) { ++ Start(); ++ task.Fail(); ++ return; ++ } ++ auto bound = base::BindOnce(&Self::OnOpen, base::Unretained(this), task); ++ auto res = cache_->OpenEntry(task.key, priority, std::move(bound)); ++ if (res.net_error() != net::ERR_IO_PENDING) { ++ OnOpen(task, std::move(res)); ++ } ++} ++ ++namespace { ++std::shared_ptr GetEntry(dc::EntryResult& result) { ++ auto* e = result.ReleaseEntry(); ++ auto deleter = [](auto e) { ++ if (e) { ++ e->Close(); ++ } ++ }; ++ return {e, deleter}; ++} ++} // namespace ++ ++void Self::OnOpen(Task task, dc::EntryResult res) { ++ VLOG(2) << "OnOpen(" << res.net_error() << ")"; ++ if (res.net_error() != net::OK) { ++ VLOG(2) << "Failed to find " << task.key << " in " << name(); ++ task.Fail(); ++ return; ++ } ++ task.entry = GetEntry(res); ++ DCHECK(task.entry); ++ task.buf = base::MakeRefCounted(2 * 1024 * 1024); ++ DCHECK(task.buf); ++ auto bound = ++ base::BindOnce(&Self::OnHeaderRead, base::Unretained(this), task); ++ auto code = task.entry->ReadData(0, 0, task.buf.get(), task.buf->size(), ++ std::move(bound)); ++ if (code != net::ERR_IO_PENDING) { ++ OnHeaderRead(task, code); ++ } ++} ++ ++void Self::OnHeaderRead(Task task, int code) { ++ if (code <= 0) { ++ LOG(ERROR) << "Failed to read headers for entry " << task.key << " in " ++ << name(); ++ task.Fail(); ++ return; ++ } ++ task.header.assign(task.buf->data(), static_cast(code)); ++ auto bound = base::BindOnce(&Self::OnBodyRead, base::Unretained(this), task); ++ code = task.entry->ReadData(1, 0, task.buf.get(), task.buf->size(), ++ std::move(bound)); ++ if (code != net::ERR_IO_PENDING) { ++ OnBodyRead(task, code); ++ } ++} ++void Self::OnBodyRead(Task task, int code) { ++ if (code <= 0) { ++ LOG(ERROR) << "Failed to read body for entry " << task.key << " in " ++ << name(); ++ task.Fail(); ++ return; ++ } ++ task.body.assign(task.buf->data(), static_cast(code)); ++ if (task.listener) { ++ VLOG(2) << "Cache hit on " << task.key; ++ task.SetHeaders(name()); ++ auto& stor = state_->storage(); ++ stor.Store(task.key, std::move(task.header), std::move(task.body)); ++ state_->scheduler().IssueRequests(state_->api()); ++ } else { ++ task.hit(task.body, task.header); ++ } ++} ++ ++void Self::Store(std::string cid, std::string headers, std::string body) { ++ VLOG(2) << "Store(" << name() << ',' << cid << ',' << headers.size() << ',' ++ << body.size() << ')'; ++ auto bound = base::BindOnce(&Self::OnEntryCreated, base::Unretained(this), ++ cid, headers, body); ++ auto res = cache_->OpenOrCreateEntry(cid, net::LOW, std::move(bound)); ++ if (res.net_error() != net::ERR_IO_PENDING) { ++ OnEntryCreated(cid, headers, body, std::move(res)); ++ } ++} ++void Self::OnEntryCreated(std::string cid, ++ std::string headers, ++ std::string body, ++ disk_cache::EntryResult result) { ++ if (result.opened()) { ++ VLOG(1) << "No need to write an entry for " << cid << " in " << name() ++ << " as it is already there and immutable."; ++ } else if (result.net_error() == net::OK) { ++ auto entry = GetEntry(result); ++ auto buf = base::MakeRefCounted(headers); ++ DCHECK(buf); ++ auto bound = base::BindOnce(&Self::OnHeaderWritten, base::Unretained(this), ++ buf, body, entry); ++ auto code = ++ entry->WriteData(0, 0, buf.get(), buf->size(), std::move(bound), true); ++ if (code != net::ERR_IO_PENDING) { ++ OnHeaderWritten(buf, body, entry, code); ++ } ++ } else { ++ LOG(ERROR) << "Failed to create an entry for " << cid << " in " << name(); ++ } ++} ++void Self::OnHeaderWritten(scoped_refptr buf, ++ std::string body, ++ std::shared_ptr entry, ++ int code) { ++ if (code < 0) { ++ LOG(ERROR) << "Failed to write header info for " << entry->GetKey() ++ << " in " << name(); ++ return; ++ } ++ buf = base::MakeRefCounted(body); ++ DCHECK(buf); ++ auto f = [](scoped_refptr, int c) { ++ VLOG(1) << "body write " << c; ++ }; ++ auto bound = base::BindOnce(f, buf); ++ entry->WriteData(1, 0, buf.get(), buf->size(), std::move(bound), true); ++} ++ ++std::string_view Self::name() const { ++ switch (type_) { ++ case net::CacheType::DISK_CACHE: ++ return "disk"; ++ case net::CacheType::MEMORY_CACHE: ++ return "memory"; ++ default: ++ return "other"; ++ } ++} ++ ++void Self::Task::Fail() { ++ VLOG(2) << "TaskFail for key: " << key; ++ if (listener) { ++ listener->NotHere(key, ""); ++ } ++ if (miss) { ++ miss(); ++ } ++} ++void Self::Task::SetHeaders(std::string_view source) { ++ auto heads = base::MakeRefCounted(header); ++ DCHECK(heads); ++ std::string value{"blockcache-"}; ++ value.append(key); ++ value.append(";desc=\"Load from local browser block cache\";dur="); ++ auto dur = base::TimeTicks::Now() - start; ++ value.append(std::to_string(dur.InMillisecondsRoundedUp())); ++ heads->SetHeader("Server-Timing", value); ++ VLOG(2) << "From cache: Server-Timing: " << value << "; Block-Cache-" << key ++ << ": " << source; ++ heads->SetHeader("Block-Cache-" + key, {source.data(), source.size()}); ++ header = heads->raw_headers(); ++} ++void Self::Expire(std::string const& key) { ++ if (cache_ && !pending_) { ++ cache_->DoomEntry(key, net::RequestPriority::LOWEST, base::DoNothing()); ++ } ++} ++ ++Self::Task::Task() = default; ++Self::Task::Task(Task const&) = default; ++Self::Task::~Task() noexcept = default; +diff --git a/components/ipfs/cache_requestor.h b/components/ipfs/cache_requestor.h +new file mode 100644 +index 0000000000000..969a04e29dce3 +--- /dev/null ++++ b/components/ipfs/cache_requestor.h +@@ -0,0 +1,81 @@ ++#ifndef CACHE_REQUESTOR_H_ ++#define CACHE_REQUESTOR_H_ ++ ++#include ++#include ++#include ++ ++#include ++#include ++#include ++ ++#include ++ ++#include ++ ++namespace ipfs { ++ ++class BlockStorage; ++class InterRequestState; ++ ++class CacheRequestor : public BlockRequestor { ++ public: ++ CacheRequestor(net::CacheType, InterRequestState&, base::FilePath); ++ virtual ~CacheRequestor() noexcept; ++ void Store(std::string cid, std::string headers, std::string body); ++ void FetchEntry(std::string key, ++ net::RequestPriority priority, ++ std::function hit, ++ std::function miss); ++ void Expire(std::string const& key); ++ ++ std::string_view name() const; ++ ++ private: ++ struct Task { ++ Task(); ++ Task(Task const&); ++ ~Task() noexcept; ++ std::string key; ++ std::shared_ptr listener; ++ base::TimeTicks start = base::TimeTicks::Now(); ++ std::string header; ++ std::string body; ++ scoped_refptr buf; ++ std::shared_ptr entry; ++ std::function hit; ++ std::function miss; ++ ++ void SetHeaders(std::string_view); ++ void Fail(); ++ }; ++ net::CacheType const type_; ++ raw_ref state_; ++ std::unique_ptr cache_; ++ bool pending_ = false; ++ base::FilePath path_; ++ ++ void RequestByCid(std::string cid, ++ std::shared_ptr, ++ Priority) override; ++ ++ void Start(); ++ ++ void StartFetch(Task& t, net::RequestPriority priority); ++ void Assign(disk_cache::BackendResult); ++ void OnOpen(Task, disk_cache::EntryResult); ++ void OnHeaderRead(Task, int); ++ void OnBodyRead(Task, int); ++ ++ void OnEntryCreated(std::string c, ++ std::string h, ++ std::string b, ++ disk_cache::EntryResult); ++ void OnHeaderWritten(scoped_refptr buf, ++ std::string body, ++ std::shared_ptr entry, ++ int); ++}; ++} // namespace ipfs ++ ++#endif // CACHE_REQUESTOR_H_ +diff --git a/components/ipfs/crypto_api.cc b/components/ipfs/crypto_api.cc +new file mode 100644 +index 0000000000000..bd4550d160935 +--- /dev/null ++++ b/components/ipfs/crypto_api.cc +@@ -0,0 +1,87 @@ ++#include "crypto_api.h" ++ ++#include "base/logging.h" ++#include "components/webcrypto/algorithm_implementations.h" ++#include "components/webcrypto/status.h" ++#include "third_party/blink/public/platform/web_crypto_key.h" ++#include "third_party/boringssl/src/include/openssl/evp.h" ++ ++namespace { ++int ToEvpKeyType(ipfs::ipns::KeyType t) { ++ using ipfs::ipns::KeyType; ++ switch (t) { ++ case KeyType::ECDSA: ++ LOG(ERROR) << "Check on ECDSA key type translation."; ++ return EVP_PKEY_EC; ++ case KeyType::Ed25519: ++ return EVP_PKEY_ED25519; ++ case KeyType::RSA: ++ return EVP_PKEY_RSA; ++ case KeyType::Secp256k1: ++ LOG(ERROR) << "Check on Secp256k1 key type translation."; ++ return EVP_PKEY_DSA; ++ default: ++ LOG(ERROR) << "Invalid key type: " << static_cast(t); ++ return EVP_PKEY_NONE; ++ } ++} ++} // namespace ++ ++namespace cpto = ipfs::crypto_api; ++/* ++auto crypt::GetAlgo(ipfs::ipns::KeyType t) -> Algo { ++ switch (t) { ++ case ipfs::ipns::KeyType::Ed25519: ++ return {blink::kWebCryptoAlgorithmIdEcdsa, ++ webcrypto::CreateEcdhImplementation()}; ++ default: ++ LOG(ERROR) << "Unimplemented key type: " << static_cast(t); ++ return {}; ++ } ++} ++ */ ++bool cpto::VerifySignature(ipfs::ipns::KeyType key_type, ++ ipfs::ByteView signature, ++ ipfs::ByteView data, ++ ipfs::ByteView key_bytes) { ++ auto* key_p = reinterpret_cast(key_bytes.data()); ++ auto* data_p = reinterpret_cast(data.data()); ++ auto* sig_p = reinterpret_cast(signature.data()); ++ auto kt = ToEvpKeyType(key_type); ++ std::clog << "data:"; ++ for (auto b : data) { ++ std::clog << ' ' << std::hex << static_cast(b); ++ } ++ std::clog << ' ' << data.size() << " bytes.\n"; ++ bssl::UniquePtr pkey(EVP_PKEY_new_raw_public_key( ++ kt, /*engine*/ nullptr, key_p, key_bytes.size())); ++ bssl::ScopedEVP_MD_CTX ctx; ++ if (!EVP_DigestVerifyInit(ctx.get(), /*pctx=*/nullptr, /*type=*/nullptr, ++ /*e=*/nullptr, pkey.get())) { ++ LOG(ERROR) << "EVP_DigestVerifyInit failed"; ++ return false; ++ } ++ // auto* prefix = reinterpret_cast( ++ // "\x69\x70\x6e\x73\x2d\x73\x69\x67\x6e\x61\x74\x75\x72\x65\x3a"); ++ // std::basic_string to_verify = prefix; ++ // to_verify.append(data_p, data.size()); ++ auto result = ++ EVP_DigestVerify(ctx.get(), sig_p, signature.size(), data_p, data.size()); ++ // to_verify.data(), to_verify.size()); ++ LOG(INFO) << "EVP_DigestVerify returned " << result; ++ return result == 1; ++ /* ++ auto* pre_p = reinterpret_cast(prefix.data()); ++ if (!EVP_DigestVerifyUpdate(ctx.get(), pre_p, prefix.size())) { ++ LOG(ERROR) << "EVP_DigestVerifyUpdate failed on prefix."; ++ return false; ++ } ++ if (!EVP_DigestVerifyUpdate(ctx.get(), data_p, data.size())) { ++ LOG(ERROR) << "EVP_DigestVerifyUpdate failed on actual data."; ++ return false; ++ } ++ auto result = EVP_DigestVerifyFinal(ctx.get(), sig_p, signature.size()); ++ LOG(INFO) << "EVP_DigestVerifyFinal returned " << result; ++ return result == 1; ++ */ ++} +diff --git a/components/ipfs/crypto_api.h b/components/ipfs/crypto_api.h +new file mode 100644 +index 0000000000000..1363bb1fec6df +--- /dev/null ++++ b/components/ipfs/crypto_api.h +@@ -0,0 +1,22 @@ ++#ifndef IPFS_VALIDATE_SIGNATURE_H_ ++#define IPFS_VALIDATE_SIGNATURE_H_ ++ ++#include "components/webcrypto/algorithm_implementation.h" ++ ++#include "third_party/ipfs_client/keys.pb.h" ++ ++#include ++ ++namespace ipfs::crypto_api { ++/* ++using Algo = std::pair>; ++Algo GetAlgo(ipfs::ipns::KeyType); ++*/ ++bool VerifySignature(ipfs::ipns::KeyType, ++ ByteView signature, ++ ByteView data, ++ ByteView key); ++} // namespace ipfs::crypto_api ++ ++#endif // IPFS_VALIDATE_SIGNATURE_H_ +diff --git a/components/ipfs/gateway_requests.cc b/components/ipfs/gateway_requests.cc +new file mode 100644 +index 0000000000000..7e60902892508 +--- /dev/null ++++ b/components/ipfs/gateway_requests.cc +@@ -0,0 +1,332 @@ ++#include "gateway_requests.h" ++ ++#include "crypto_api.h" ++#include "inter_request_state.h" ++#include "ipns_cbor.h" ++ ++#include "base/strings/escape.h" ++#include "net/base/mime_sniffer.h" ++#include "net/base/mime_util.h" ++#include "services/network/public/cpp/resource_request.h" ++#include "services/network/public/cpp/simple_url_loader.h" ++#include "services/network/public/mojom/url_response_head.mojom.h" ++#include "url/gurl.h" ++ ++#include ++#include ++ ++#include ++#include ++ ++#include ++ ++using Self = ipfs::GatewayRequests; ++ ++constexpr net::NetworkTrafficAnnotationTag kTrafficAnnotation = ++ net::DefineNetworkTrafficAnnotation("ipfs_gateway_request", R"( ++ semantics { ++ sender: "IPFS component" ++ description: ++ "Sends a request to an IPFS gateway for a block." ++ trigger: ++ "Processing of an ipfs:// URL (possibly from an ipns:// URL)." ++ data: "None" ++ destination: WEBSITE ++ } ++ policy { ++ cookies_allowed: NO ++ setting: "Currently, this feature cannot be disabled by settings. TODO" ++ } ++ )"); ++namespace { ++constexpr std::size_t MB = 1'000'000UL; ++std::unique_ptr discovery_loader; ++void parse_discover_response(std::function)> cb, ++ std::unique_ptr body) { ++ std::vector discovered; ++ if (!body) { ++ LOG(ERROR) << "Failed to discover gateways."; ++ cb(discovered); ++ return; ++ } ++ std::string_view r{*body}; ++ while (r.size()) { ++ VLOG(1) << "Discovered gateways, body remaining to parse: " << r; ++ auto i = r.find('"'); ++ if (i == std::string_view::npos) { ++ break; ++ } ++ r.remove_prefix(++i); ++ i = r.find('"'); ++ discovered.emplace_back(r.substr(0, i)); ++ r.remove_prefix(++i); ++ } ++ cb(discovered); ++ discovery_loader.reset(); ++} ++} // namespace ++void Self::Discover(std::function)> cb) { ++ if (!loader_factory_) { ++ LOG(INFO) << "Can't discover as I have no loader factory."; ++ disc_cb_ = cb; ++ return; ++ } ++ if (discovery_loader) { ++ LOG(INFO) ++ << "Not issuing a discovery request, as one has already been sent."; ++ return; ++ } ++ auto req = std::make_unique(); ++ req->url = GURL{"https://orchestrator.strn.pl/nodes/nearby"}; ++ discovery_loader = network::SimpleURLLoader::Create( ++ std::move(req), kTrafficAnnotation, FROM_HERE); ++ auto bound = base::BindOnce(&parse_discover_response, cb); ++ LOG(INFO) << "Issuing discovery request to: " ++ "https://orchestrator.strn.pl/nodes/nearby"; ++ discovery_loader->DownloadToString(loader_factory_, std::move(bound), MB); ++} ++ ++auto Self::InitiateGatewayRequest(BusyGateway assigned) ++ -> std::shared_ptr { ++ auto url = assigned.url(); ++ ++ GOOGLE_DCHECK_GT(url.size(), 0U); ++ auto url_suffix = assigned.task(); ++ auto req = std::make_unique(); ++ req->url = GURL{url}; ++ req->priority = net::HIGHEST; // TODO ++ if (!assigned.accept().empty()) { ++ req->headers.SetHeader("Accept", assigned.accept()); ++ } ++ auto out = std::make_shared(std::move(assigned)); ++ GOOGLE_DCHECK_GT(out->gateway->url_prefix().size(), 0U); ++ out->loader = network::SimpleURLLoader::Create(std::move(req), ++ kTrafficAnnotation, FROM_HERE); ++ if (url.find("format=ipns-record") == std::string::npos) { ++ out->loader->SetTimeoutDuration(base::Seconds(32)); ++ } else { ++ VLOG(1) << "Doing an IPNS record query, so giving it a long timeout."; ++ out->loader->SetTimeoutDuration(base::Seconds(256)); ++ } ++ auto start_time = base::TimeTicks::Now(); ++ // out->listener = listener; ++ auto cb = base::BindOnce(&Self::OnResponse, base::Unretained(this), ++ shared_from_this(), out, start_time); ++ DCHECK(loader_factory_); ++ // TODO - proper requesting with full features (SetPriority, etc.). ++ out->loader->DownloadToString(loader_factory_, std::move(cb), 2UL * MB); ++ return out; ++} ++void Self::OnResponse(std::shared_ptr api, ++ std::shared_ptr req, ++ base::TimeTicks start_time, ++ std::unique_ptr body) { ++ auto sz = body ? body->size() : 0UL; ++ LOG(INFO) << "OnResponse(...," << start_time << ", " << sz << "B )"; ++ DCHECK(req); ++ auto task = req->task(); ++ if (task.empty()) { ++ LOG(ERROR) << "Got a response for an empty task!"; ++ return; ++ } ++ auto url = req->url(); ++ // LOG(INFO) << "Got a response for " << task << " via " << url; ++ auto& bg = req->gateway; ++ auto& ldr = req->loader; ++ // auto listener = req->listener; ++ if (ProcessResponse(bg, ldr.get(), body.get(), start_time)) { ++ LOG(INFO) << url << " success."; ++ bg.Success(state_->gateways(), shared_from_this()); ++ } else { ++ LOG(INFO) << url << " failure."; ++ bg.Failure(state_->gateways(), shared_from_this()); ++ } ++ VLOG(1) << "Recheck for more activity."; ++ state_->storage().CheckListening(); ++ state_->scheduler().IssueRequests(api); ++} ++ ++bool Self::ProcessResponse(BusyGateway& gw, ++ network::SimpleURLLoader* ldr, ++ std::string* body, ++ base::TimeTicks start_time) { ++ auto end_time = base::TimeTicks::Now(); ++ if (!gw) { ++ LOG(ERROR) << "No gateway."; ++ return false; ++ } ++ VLOG(2) << "ProcessResponse(" << gw.url() << ')'; ++ if (!ldr) { ++ LOG(ERROR) << "No loader for processing " << gw.url(); ++ return false; ++ } ++ if (!body) { ++ LOG(INFO) << "ProcessResponse(" << gw.url() ++ << ") Null body - presumably http error.\n"; ++ return false; ++ } ++ network::mojom::URLResponseHead const* head = ldr->ResponseInfo(); ++ if (!head) { ++ LOG(INFO) << "ProcessResponse(" << gw.url() << ") Null head.\n"; ++ return false; ++ } ++ DCHECK(gw.url().find("?format=") < gw.url().size() || gw.accept().size() > 0); ++ std::string reported_content_type; ++ head->headers->EnumerateHeader(nullptr, "Content-Type", ++ &reported_content_type); ++ // application/vnd.ipld.raw ++ // -- OR -- ++ // application/vnd.ipfs.ipns-record ++ constexpr std::string_view content_type_prefix{"application/vnd.ip"}; ++ if (reported_content_type.compare(0, content_type_prefix.size(), ++ content_type_prefix)) { ++ VLOG(1) << '\n' ++ << gw.url() << " reported a content type of " ++ << reported_content_type ++ << " strongly implying that it's a full request, not a single " ++ "block. TODO: Remove " ++ << gw->url_prefix() << " from list of gateways?\n"; ++ state_->gateways().demote(gw->url_prefix()); ++ return false; ++ } ++ auto cid_str = gw.task(); ++ cid_str.erase(0, 5); // ipfs/ ++ auto qmark = cid_str.find('?'); ++ if (qmark < cid_str.size()) { ++ cid_str.erase(qmark); ++ } ++ if (state_->storage().Get(cid_str)) { ++ // LOG(INFO) << "Got multiple successful responses for " << cid_str; ++ return true; ++ } ++ auto cid = libp2p::multi::ContentIdentifierCodec::fromString(cid_str); ++ if (!cid.has_value()) { ++ LOG(ERROR) << "Invalid CID '" << cid_str << "'."; ++ return false; ++ } ++ auto duration = (end_time - start_time).InMillisecondsRoundedUp(); ++ if (cid.value().content_type == ++ libp2p::multi::MulticodecType::Code::LIBP2P_KEY) { ++ auto as_peer = libp2p::peer::PeerId::fromHash(cid.value().content_address); ++ if (!as_peer.has_value()) { ++ LOG(INFO) << cid_str ++ << " has the codec of being a libp2p key, like one would " ++ "expect of a Peer ID, but it does not parse as a Peer ID."; ++ return false; ++ } ++ auto* bytes = reinterpret_cast(body->data()); ++ auto record = ++ ValidateIpnsRecord({bytes, body->size()}, as_peer.value(), ++ crypto_api::VerifySignature, ParseCborIpns); ++ if (!record.has_value()) { ++ LOG(ERROR) << "IPNS record failed to validate! From: " << gw.url(); ++ return false; ++ } ++ LOG(INFO) << "IPNS record from " << gw.url() << " points " << cid_str ++ << " to " << record.value().value; ++ ValidatedIpns entry{record.value()}; ++ entry.resolution_ms = duration; ++ entry.gateway_source = gw->url_prefix(); ++ state_->names().AssignName(cid_str, std::move(entry)); ++ scheduler().IssueRequests(shared_from_this()); ++ return true; ++ } else { ++ Block block{cid.value(), *body}; ++ if (block.cid_matches_data()) { ++ head->headers->SetHeader("Block-Source", ++ cid_str + ", " + gw->url_prefix() + " @" + ++ std::to_string(std::time(nullptr))); ++ VLOG(1) << "L3: Storing CID=" << cid_str; ++ // Note this header is added _after_ storing in long-term cache ++ head->headers->SetHeader( ++ "Server-Timing", ++ "gateway-fetch-" + cid_str + ";desc=\"" + gw->url_prefix() + ++ " : load over http(s)\";dur=" + std::to_string(duration)); ++ state_->storage().Store(cid_str, cid.value(), ++ head->headers->raw_headers(), *body); ++ auto& orc = state_->orchestrator(); ++ orc.add_node(cid_str, ipld::DagNode::fromBlock(block)); ++ if (gw.srcreq) { ++ orc.build_response(gw.srcreq->dependent); ++ } else { ++ LOG(ERROR) << "This BusyGateway with response has no top-level " ++ "IpfsRequest associated with it " ++ << gw.url() << " " << gw.accept(); ++ } ++ scheduler().IssueRequests(shared_from_this()); ++ return true; ++ } else { ++ LOG(ERROR) << "You tried to store some bytes as a block for a CID (" ++ << cid_str << ") that does not correspond to those bytes!"; ++ // TODO ban the gateway outright ++ return false; ++ } ++ } ++} ++ ++auto Self::scheduler() -> Scheduler& { ++ return sched_; ++} ++void Self::SetLoaderFactory(network::mojom::URLLoaderFactory& lf) { ++ loader_factory_ = &lf; ++ if (disc_cb_) { ++ LOG(INFO) << "Have loader factory, calling Discover"; ++ Discover(disc_cb_); ++ disc_cb_ = {}; ++ } ++} ++ ++std::string Self::MimeType(std::string extension, ++ std::string_view content, ++ std::string const& url) const { ++ std::string result; ++ auto fp_ext = base::FilePath::FromUTF8Unsafe(extension).value(); ++ VLOG(1) << "extension=" << extension << "content.size()=" << content.size() ++ << "(as-if) url for mime type:" << url; ++ if (extension.empty()) { ++ result.clear(); ++ } else if (net::GetWellKnownMimeTypeFromExtension(fp_ext, &result)) { ++ VLOG(1) << "Got " << result << " from extension " << extension << " for " ++ << url; ++ } else { ++ result.clear(); ++ } ++ if (net::SniffMimeType({content.data(), content.size()}, GURL{url}, result, ++ net::ForceSniffFileUrlsForHtml::kDisabled, &result)) { ++ VLOG(1) << "Got " << result << " from content of " << url; ++ } ++ if (result.empty() || result == "application/octet-stream") { ++ // C'mon, man ++ net::SniffMimeTypeFromLocalData({content.data(), content.size()}, &result); ++ LOG(INFO) << "Falling all the way back to content type " << result; ++ } ++ return result; ++} ++std::string Self::UnescapeUrlComponent(std::string_view comp) const { ++ using Rule = base::UnescapeRule; ++ auto rules = Rule::PATH_SEPARATORS | ++ Rule::URL_SPECIAL_CHARS_EXCEPT_PATH_SEPARATORS | Rule::SPACES; ++ auto result = base::UnescapeURLComponent({comp.data(), comp.size()}, rules); ++ VLOG(1) << "UnescapeUrlComponent(" << comp << ")->'" << result << "'"; ++ return result; ++} ++void Self::RequestByCid(std::string cid, ++ std::shared_ptr listener, ++ Priority prio) { ++ auto me = shared_from_this(); ++ LOG(ERROR) << "Look out! RequestByCid(" << cid << ",...," << prio << ')'; ++ sched_.Enqueue(me, listener, {}, "ipfs/" + cid, "application/vnd.ipld.raw", ++ prio, {}); ++ sched_.IssueRequests(me); ++} ++ ++Self::GatewayRequests(InterRequestState& state) ++ : state_{state}, ++ sched_([this]() { return state_->gateways().GenerateList(); }) {} ++Self::~GatewayRequests() { ++ LOG(WARNING) << "API dtor - are all URIs loaded?"; ++} ++ ++Self::GatewayUrlLoader::GatewayUrlLoader(BusyGateway&& bg) ++ : GatewayRequest(std::move(bg)) {} ++Self::GatewayUrlLoader::~GatewayUrlLoader() noexcept {} +diff --git a/components/ipfs/gateway_requests.h b/components/ipfs/gateway_requests.h +new file mode 100644 +index 0000000000000..44f0588ee6acf +--- /dev/null ++++ b/components/ipfs/gateway_requests.h +@@ -0,0 +1,71 @@ ++#ifndef IPFS_GATEWAY_REQUESTS_H_ ++#define IPFS_GATEWAY_REQUESTS_H_ ++ ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include ++ ++#include ++ ++namespace network { ++class SimpleURLLoader; ++namespace mojom { ++class URLLoaderFactory; ++} ++} // namespace network ++ ++namespace ipfs { ++class InterRequestState; ++class IpfsRequest; ++class NetworkRequestor; ++ ++class GatewayRequests final : public ContextApi { ++ struct GatewayUrlLoader : public ipfs::GatewayRequest { ++ GatewayUrlLoader(BusyGateway&&); ++ GatewayUrlLoader(GatewayRequest&&); ++ ~GatewayUrlLoader() noexcept override; ++ std::unique_ptr loader; ++ }; ++ ++ raw_ptr loader_factory_ = nullptr; ++ raw_ref state_; ++ Scheduler sched_; ++ std::function)> disc_cb_; ++ ++ void Request(std::string task, std::shared_ptr, Priority); ++ std::shared_ptr InitiateGatewayRequest(BusyGateway) override; ++ std::string MimeType(std::string extension, ++ std::string_view content, ++ std::string const& url) const override; ++ std::string UnescapeUrlComponent(std::string_view) const override; ++ ++ void OnResponse(std::shared_ptr, ++ std::shared_ptr, ++ base::TimeTicks, ++ std::unique_ptr); ++ bool ProcessResponse(BusyGateway&, ++ network::SimpleURLLoader*, ++ std::string*, ++ base::TimeTicks); ++ friend class NetworkRequestor; ++ ++ void RequestByCid(std::string cid, ++ std::shared_ptr, ++ Priority); ++ ++ public: ++ GatewayRequests(InterRequestState&); ++ ~GatewayRequests(); ++ void SetLoaderFactory(network::mojom::URLLoaderFactory&); ++ Scheduler& scheduler(); ++ void Discover(std::function)>) override; ++}; ++ ++} // namespace ipfs ++ ++#endif // IPFS_GATEWAY_REQUESTS_H_ +diff --git a/components/ipfs/inter_request_state.cc b/components/ipfs/inter_request_state.cc +new file mode 100644 +index 0000000000000..ee1e2041b29dc +--- /dev/null ++++ b/components/ipfs/inter_request_state.cc +@@ -0,0 +1,143 @@ ++#include "inter_request_state.h" ++ ++#include "gateway_requests.h" ++#include "network_requestor.h" ++ ++#include "base/logging.h" ++#include "content/public/browser/browser_context.h" ++ ++#include ++#include ++#include ++#include ++ ++using Self = ipfs::InterRequestState; ++ ++namespace { ++constexpr char user_data_key[] = "ipfs_request_userdata"; ++} ++ ++auto Self::FromBrowserContext(content::BrowserContext* context) ++ -> InterRequestState& { ++ if (!context) { ++ LOG(WARNING) << "No browser context! Using a default IPFS state."; ++ static ipfs::InterRequestState static_state({}); ++ return static_state; ++ } ++ base::SupportsUserData::Data* existing = context->GetUserData(user_data_key); ++ if (existing) { ++ VLOG(1) << "Re-using existing IPFS state."; ++ return *static_cast(existing); ++ } ++ LOG(INFO) << "Creating new IPFS state for this browser context."; ++ auto owned = std::make_unique(context->GetPath()); ++ ipfs::InterRequestState* raw = owned.get(); ++ context->SetUserData(user_data_key, std::move(owned)); ++ return *raw; ++} ++auto Self::serialized_caches() -> std::array { ++ if (!mem_) { ++ auto p = mem_ = std::make_shared( ++ net::CacheType::MEMORY_CACHE, *this, base::FilePath{}); ++ storage().AddStorageHook( ++ [p](auto c, auto h, auto b) { p->Store(c, h, b); }); ++ } ++ if (!dsk_) { ++ auto p = dsk_ = std::make_shared(net::CacheType::DISK_CACHE, ++ *this, disk_path_); ++ storage().AddStorageHook( ++ [p](auto c, auto h, auto b) { p->Store(c, h, b); }); ++ } ++ return {mem_, dsk_}; ++} ++auto Self::requestor() -> BlockRequestor& { ++ if (!requestor_.Valid()) { ++ serialized_caches(); ++ requestor_.Add(mem_); ++ requestor_.Add(dsk_); ++ requestor_.Add(std::make_shared(*this)); ++ } ++ return requestor_; ++} ++std::shared_ptr Self::api() { ++ auto existing = api_.lock(); ++ if (existing) { ++ return existing; ++ } ++ auto created = std::make_shared(*this); ++ api_ = created; ++ auto t = std::time(nullptr); ++ if (t - last_discovery_ > 300) { ++ created->Discover([this](auto v) { gws_.AddGateways(v); }); ++ last_discovery_ = t; ++ } ++ return created; ++} ++auto Self::scheduler() -> Scheduler& { ++ auto api = api_.lock(); ++ DCHECK(api); ++ return api->scheduler(); ++} ++ ++namespace { ++ ++void send_gateway_request(Self* me, ++ std::shared_ptr req) { ++ if (!req->dependent) { ++ LOG(FATAL) << "This makes no sense whatsoever - why do you want to request " ++ "things if nothing awaits."; ++ } ++ struct DagListenerAdapter final : public ipfs::DagListener { ++ std::shared_ptr gw_req; ++ std::string bytes; ++ std::shared_ptr api; ++ void ReceiveBlockBytes(std::string_view b) override { ++ LOG(INFO) << "DagListenerAdapter::ReceiveBlockBytes(" << b.size() << "B)"; ++ bytes.assign(b); ++ } ++ void BlocksComplete(std::string mime_type) override { ++ LOG(INFO) << "DagListenerAdapter::BlocksComplete(" << mime_type << ")"; ++ ipfs::Response r{mime_type, 200, std::move(bytes), ""}; ++ gw_req->dependent->finish(r); ++ } ++ void NotHere(std::string_view cid, std::string_view path) override { ++ LOG(INFO) << "DagListenerAdapter::NotHere(" << cid << ',' << path << ")"; ++ api->scheduler().IssueRequests(api); ++ } ++ void DoesNotExist(std::string_view cid, std::string_view path) override { ++ LOG(INFO) << "DagListenerAdapter::DoesNotExist(" << cid << ',' << path ++ << ")"; ++ ipfs::Response r{"", 404, "", ""}; ++ gw_req->dependent->finish(r); ++ } ++ }; ++ auto dl = std::make_shared(); ++ dl->api = me->api(); ++ dl->gw_req = req; ++ auto& sched = me->scheduler(); ++ sched.Enqueue(me->api(), dl, {}, req->url_suffix().substr(1), req->accept(), ++ 9, req); ++ sched.IssueRequests(me->api()); ++} ++std::string detect_mime(Self* me, ++ std::string a, ++ std::string_view b, ++ std::string const& c) { ++ auto api = me->api(); ++ return static_cast(api.get())->MimeType(a, b, c); ++} ++} // namespace ++ ++auto Self::orchestrator() -> Orchestrator& { ++ if (!orc_) { ++ auto gwreq = [this](auto p) { send_gateway_request(this, p); }; ++ auto mimer = [this](auto a, auto b, auto& c) { ++ return detect_mime(this, a, b, c); ++ }; ++ orc_ = std::make_shared(gwreq, mimer); ++ } ++ return *orc_; ++} ++ ++Self::InterRequestState(base::FilePath p) : disk_path_{p} {} ++Self::~InterRequestState() noexcept {} +diff --git a/components/ipfs/inter_request_state.h b/components/ipfs/inter_request_state.h +new file mode 100644 +index 0000000000000..6b5f135407bdd +--- /dev/null ++++ b/components/ipfs/inter_request_state.h +@@ -0,0 +1,49 @@ ++#ifndef IPFS_INTER_REQUEST_STATE_H_ ++#define IPFS_INTER_REQUEST_STATE_H_ ++ ++#include "cache_requestor.h" ++ ++#include "ipfs_client/block_storage.h" ++#include "ipfs_client/chained_requestors.h" ++#include "ipfs_client/gateways.h" ++#include "ipfs_client/ipns_names.h" ++#include "ipfs_client/orchestrator.h" ++ ++#include "base/supports_user_data.h" ++ ++namespace content { ++class BrowserContext; ++} ++ ++namespace ipfs { ++class Scheduler; ++class GatewayRequests; ++class InterRequestState : public base::SupportsUserData::Data { ++ Gateways gws_; ++ BlockStorage storage_; ++ ChainedRequestors requestor_; ++ IpnsNames names_; ++ std::weak_ptr api_; ++ std::time_t last_discovery_ = 0; ++ std::shared_ptr mem_, dsk_; ++ base::FilePath const disk_path_; ++ std::shared_ptr orc_; // TODO - map of origin to Orchestrator ++ ++ public: ++ InterRequestState(base::FilePath); ++ ~InterRequestState() noexcept override; ++ ++ Gateways& gateways() { return gws_; } ++ BlockStorage& storage() { return storage_; } ++ BlockRequestor& requestor(); ++ IpnsNames& names() { return names_; } ++ Scheduler& scheduler(); ++ std::shared_ptr api(); ++ std::array,2> serialized_caches(); ++ Orchestrator& orchestrator(); ++ ++ static InterRequestState& FromBrowserContext(content::BrowserContext*); ++}; ++} // namespace ipfs ++ ++#endif // IPFS_INTER_REQUEST_STATE_H_ +diff --git a/components/ipfs/interceptor.cc b/components/ipfs/interceptor.cc +new file mode 100644 +index 0000000000000..f304d68210bfc +--- /dev/null ++++ b/components/ipfs/interceptor.cc +@@ -0,0 +1,43 @@ ++#include "interceptor.h" ++ ++#include "inter_request_state.h" ++#include "ipfs_url_loader.h" ++#include "ipns_url_loader.h" ++ ++#include "base/logging.h" ++#include "services/network/public/cpp/resource_request.h" ++#include "services/network/public/mojom/url_response_head.mojom.h" ++#include "services/network/url_loader_factory.h" ++#include "url/url_util.h" ++ ++using Interceptor = ipfs::Interceptor; ++ ++Interceptor::Interceptor(network::mojom::URLLoaderFactory* handles_http, ++ network::mojom::NetworkContext* network_context) ++ : loader_factory_{handles_http}, network_context_{network_context} {} ++ ++void Interceptor::MaybeCreateLoader(network::ResourceRequest const& req, ++ content::BrowserContext* context, ++ LoaderCallback loader_callback) { ++ auto& state = InterRequestState::FromBrowserContext(context); ++ if (req.url.SchemeIs("ipns")) { ++ auto ipns_loader = std::make_shared( ++ state, req.url.host(), network_context_, *loader_factory_); ++ std::move(loader_callback) ++ .Run(base::BindOnce(&ipfs::IpnsUrlLoader::StartHandling, ipns_loader)); ++ } else if (req.url.SchemeIs("ipfs")) { ++ auto hdr_str = req.headers.ToString(); ++ std::replace(hdr_str.begin(), hdr_str.end(), '\r', ' '); ++ LOG(INFO) << req.url.spec() << " getting intercepted! Headers: \n" ++ << hdr_str; ++ DCHECK(context); ++ auto loader = ++ std::make_shared(*loader_factory_, state); ++ std::move(loader_callback) ++ .Run(base::BindOnce(&ipfs::IpfsUrlLoader::StartRequest, loader)); ++ } else { ++ VLOG(1) << req.url.spec() << " has host '" << req.url.host() ++ << "' and is not being intercepted."; ++ std::move(loader_callback).Run({}); // SEP ++ } ++} +diff --git a/components/ipfs/interceptor.h b/components/ipfs/interceptor.h +new file mode 100644 +index 0000000000000..38d5ff39967bb +--- /dev/null ++++ b/components/ipfs/interceptor.h +@@ -0,0 +1,28 @@ ++#ifndef IPFS_INTERCEPTOR_H_ ++#define IPFS_INTERCEPTOR_H_ ++ ++#include "content/public/browser/url_loader_request_interceptor.h" ++ ++namespace network::mojom { ++class URLLoaderFactory; ++class NetworkContext; ++} // namespace network::mojom ++ ++namespace ipfs { ++ ++class COMPONENT_EXPORT(IPFS) Interceptor final ++ : public content::URLLoaderRequestInterceptor { ++ raw_ptr loader_factory_; ++ raw_ptr network_context_; ++ ++ void MaybeCreateLoader(network::ResourceRequest const&, ++ content::BrowserContext*, ++ LoaderCallback) override; ++ ++ public: ++ Interceptor(network::mojom::URLLoaderFactory* handles_http, ++ network::mojom::NetworkContext*); ++}; ++} // namespace ipfs ++ ++#endif // IPFS_INTERCEPTOR_H_ +diff --git a/components/ipfs/ipfs_features.cc b/components/ipfs/ipfs_features.cc +new file mode 100644 +index 0000000000000..a0a729d5aa8e6 +--- /dev/null ++++ b/components/ipfs/ipfs_features.cc +@@ -0,0 +1,7 @@ ++#include "ipfs_features.h" ++ ++namespace ipfs { ++ ++BASE_FEATURE(kEnableIpfs, "EnableIpfs", base::FEATURE_DISABLED_BY_DEFAULT); ++ ++} +diff --git a/components/ipfs/ipfs_features.h b/components/ipfs/ipfs_features.h +new file mode 100644 +index 0000000000000..2e54462b135a9 +--- /dev/null ++++ b/components/ipfs/ipfs_features.h +@@ -0,0 +1,13 @@ ++#ifndef IPFS_IPFS_FEATURES_H_ ++#define IPFS_IPFS_FEATURES_H_ ++ ++#include "base/component_export.h" ++#include "base/feature_list.h" ++ ++namespace ipfs { ++ ++COMPONENT_EXPORT(IPFS) BASE_DECLARE_FEATURE(kEnableIpfs); ++ ++} // namespace ipfs ++ ++#endif // IPFS_IPFS_FEATURES_H_ +diff --git a/components/ipfs/ipfs_url_loader.cc b/components/ipfs/ipfs_url_loader.cc +new file mode 100644 +index 0000000000000..a433b362f41e5 +--- /dev/null ++++ b/components/ipfs/ipfs_url_loader.cc +@@ -0,0 +1,241 @@ ++#include "ipfs_url_loader.h" ++ ++#include "gateway_requests.h" ++#include "inter_request_state.h" ++#include "summarize_headers.h" ++ ++#include "ipfs_client/gateways.h" ++#include "ipfs_client/ipfs_request.h" ++#include "ipfs_client/unixfs_path_resolver.h" ++ ++#include "base/debug/stack_trace.h" ++#include "base/notreached.h" ++#include "base/strings/stringprintf.h" ++#include "base/threading/platform_thread.h" ++#include "net/http/http_status_code.h" ++#include "services/network/public/cpp/parsed_headers.h" ++#include "services/network/public/cpp/simple_url_loader.h" ++#include "services/network/public/mojom/url_loader_factory.mojom.h" ++#include "services/network/public/mojom/url_response_head.mojom.h" ++#include "services/network/url_loader_factory.h" ++ ++#include ++ ++#include ++ ++ipfs::IpfsUrlLoader::IpfsUrlLoader( ++ network::mojom::URLLoaderFactory& handles_http, ++ InterRequestState& state) ++ : state_{state}, lower_loader_factory_{handles_http}, api_{state_->api()} {} ++ipfs::IpfsUrlLoader::~IpfsUrlLoader() noexcept { ++ if (!complete_) { ++ LOG(ERROR) << "Premature IPFS URLLoader dtor, uri was '" << original_url_ ++ << "' " << base::debug::StackTrace(); ++ } ++} ++ ++void ipfs::IpfsUrlLoader::FollowRedirect( ++ std::vector const& // removed_headers ++ , ++ net::HttpRequestHeaders const& // modified_headers ++ , ++ net::HttpRequestHeaders const& // modified_cors_exempt_headers ++ , ++ absl::optional<::GURL> const& // new_url ++) { ++ NOTIMPLEMENTED(); ++} ++ ++void ipfs::IpfsUrlLoader::SetPriority(net::RequestPriority priority, ++ int32_t intra_prio_val) { ++ VLOG(1) << "TODO SetPriority(" << priority << ',' << intra_prio_val << ')'; ++} ++ ++void ipfs::IpfsUrlLoader::PauseReadingBodyFromNet() { ++ NOTIMPLEMENTED(); ++} ++ ++void ipfs::IpfsUrlLoader::ResumeReadingBodyFromNet() { ++ NOTIMPLEMENTED(); ++} ++ ++void ipfs::IpfsUrlLoader::StartRequest( ++ std::shared_ptr me, ++ network::ResourceRequest const& resource_request, ++ mojo::PendingReceiver receiver, ++ mojo::PendingRemote client) { ++ DCHECK(!me->receiver_.is_bound()); ++ DCHECK(!me->client_.is_bound()); ++ me->receiver_.Bind(std::move(receiver)); ++ me->client_.Bind(std::move(client)); ++ if (me->original_url_.empty()) { ++ me->original_url_ = resource_request.url.spec(); ++ } ++ if (resource_request.url.SchemeIs("ipfs")) { ++ auto ref = resource_request.url.spec(); ++ DCHECK_EQ(ref.substr(0, 7), "ipfs://"); ++ // TODO these kinds of shenanigans should have their own special utils file ++ ref.erase(4, 2); ++ auto e = ref.find_first_of("#?"); ++ if (e < ref.size()) { ++ LOG(INFO) << "Dropping params/frags from '" << ref << "'"; ++ ref.resize(e); ++ LOG(INFO) << "Now have '" << ref << "'"; ++ } ++ me->StartUnixFsProc(me, ref); ++ } else { ++ LOG(ERROR) << "Wrong scheme: " << resource_request.url.scheme(); ++ } ++} ++ ++void ipfs::IpfsUrlLoader::StartUnixFsProc(ptr me, std::string_view ipfs_ref) { ++ VLOG(1) << "Requesting " << ipfs_ref << " by blocks."; ++ DCHECK_EQ(ipfs_ref.substr(0, 5), "ipfs/"); ++ auto second_slash = ipfs_ref.find_first_of("/?", 5); ++ auto cid = ipfs_ref.substr(5, second_slash - 5); ++ second_slash = ipfs_ref.find('/', 5); ++ auto qmark = ipfs_ref.find_first_of("?#"); ++ std::string remainder{"/"}; ++ if (second_slash < ipfs_ref.size()) { ++ remainder.assign(ipfs_ref.substr(second_slash + 1)); ++ } else if (qmark && qmark < ipfs_ref.size()) { ++ remainder.append(ipfs_ref.substr(qmark)); ++ } ++ VLOG(1) << "cid=" << cid << " remainder=" << remainder; ++ me->root_ = cid; ++ me->api_->SetLoaderFactory(*lower_loader_factory_); ++ /* ++ me->resolver_ = std::make_shared( ++ me->state_->storage(), me->state_->requestor(), std::string{cid}, ++ remainder, me->api_); ++ me->stepper_ = std::make_unique(); ++ me->stepper_->Start(FROM_HERE, base::Milliseconds(500), ++ base::BindRepeating(&IpfsUrlLoader::TakeStep, me)); ++ me->TakeStep(); ++ */ ++ auto whendone = [me](IpfsRequest const& req, ipfs::Response const& res) { ++ LOG(INFO) << "whendone(" << req.path().to_string() << ',' << res.status_ ++ << ',' << res.body_.size() << "B)"; ++ if (!res.body_.empty()) { ++ me->ReceiveBlockBytes(res.body_); ++ } ++ me->status_ = res.status_; ++ if (res.status_ / 100 == 4) { ++ auto p = req.path(); ++ p.pop(); ++ std::string cid{p.pop()}; ++ me->DoesNotExist(cid, p.to_string()); ++ } else { ++ me->BlocksComplete(res.mime_); ++ } ++ }; ++ auto abs_path = std::string{"/ipfs/"}; ++ abs_path.append(cid); ++ if (!remainder.empty()) { ++ if (abs_path.back() != '/' && remainder[0] != '/') { ++ abs_path.push_back('/'); ++ } ++ abs_path.append(remainder); ++ } ++ auto req = std::make_shared(abs_path, whendone); ++ me->state_->orchestrator().build_response(req); ++} ++ ++void ipfs::IpfsUrlLoader::TakeStep() { ++ if (complete_) { ++ LOG(INFO) << "Timed step(" << original_url_ << "): done."; ++ stepper_->Stop(); ++ stepper_.reset(); ++ } else { ++ VLOG(2) << "Timed step(" << original_url_ << "): still going."; ++ resolver_->Step(shared_from_this()); ++ } ++} ++ ++void ipfs::IpfsUrlLoader::OverrideUrl(GURL u) { ++ original_url_ = u.spec(); ++} ++void ipfs::IpfsUrlLoader::AddHeader(std::string_view a, std::string_view b) { ++ VLOG(1) << "AddHeader(" << a << ',' << b << ')'; ++ additional_outgoing_headers_.emplace_back(a, b); ++} ++ ++void ipfs::IpfsUrlLoader::BlocksComplete(std::string mime_type) { ++ VLOG(1) << "Resolved from unix-fs dag a file of type: " << mime_type ++ << " will report it as " << original_url_; ++ if (complete_) { ++ return; ++ } ++ auto result = ++ mojo::CreateDataPipe(partial_block_.size(), pipe_prod_, pipe_cons_); ++ if (result) { ++ LOG(ERROR) << " ERROR: TaskFailed to create data pipe: " << result; ++ return; ++ } ++ complete_ = true; ++ auto head = network::mojom::URLResponseHead::New(); ++ head->mime_type = mime_type; ++ std::uint32_t byte_count = partial_block_.size(); ++ VLOG(1) << "Calling WriteData(" << byte_count << ")"; ++ pipe_prod_->WriteData(partial_block_.data(), &byte_count, ++ MOJO_BEGIN_WRITE_DATA_FLAG_ALL_OR_NONE); ++ VLOG(1) << "Called WriteData(" << byte_count << ")"; ++ head->content_length = byte_count; ++ head->headers = ++ net::HttpResponseHeaders::TryToCreate("access-control-allow-origin: *"); ++ if (!head->headers) { ++ LOG(ERROR) << "\n\tFailed to create headers!\n"; ++ return; ++ } ++ auto* reason = ++ net::GetHttpReasonPhrase(static_cast(status_)); ++ auto status_line = base::StringPrintf("HTTP/1.1 %d %s", status_, reason); ++ LOG(INFO) << "Returning with status line '" << status_line << "'.\n"; ++ head->headers->ReplaceStatusLine(status_line); ++ head->headers->SetHeader("Content-Type", mime_type); ++ head->headers->SetHeader("Access-Control-Allow-Origin", "*"); ++ head->was_fetched_via_spdy = false; ++ if (resolver_) { ++ AppendGatewayHeaders(resolver_->involved_cids(), *head->headers); ++ } else { ++ LOG(INFO) << "TODO"; ++ } ++ for (auto& [n, v] : additional_outgoing_headers_) { ++ VLOG(1) << "Appending 'additional' header:" << n << '=' << v << '.'; ++ head->headers->AddHeader(n, v); ++ } ++ VLOG(1) << "Calling PopulateParsedHeaders"; ++ head->parsed_headers = ++ network::PopulateParsedHeaders(head->headers.get(), GURL{original_url_}); ++ VLOG(1) << "Sending response for " << original_url_ << " with mime type " ++ << head->mime_type << " @" << (void*)(this) ++ ; ++ client_->OnReceiveResponse(std::move(head), std::move(pipe_cons_), ++ absl::nullopt); ++ client_->OnComplete(network::URLLoaderCompletionStatus{}); ++ stepper_.reset(); ++} ++ ++void ipfs::IpfsUrlLoader::DoesNotExist(std::string_view cid, ++ std::string_view path) { ++ LOG(ERROR) << "Immutable data 404 for " << cid << '/' << path; ++ complete_ = true; ++ client_->OnComplete( ++ network::URLLoaderCompletionStatus{net::ERR_FILE_NOT_FOUND}); ++ stepper_.reset(); ++} ++void ipfs::IpfsUrlLoader::NotHere(std::string_view cid, std::string_view path) { ++ LOG(INFO) << "TODO " << __func__ << '(' << cid << ',' << path << ')'; ++} ++ ++void ipfs::IpfsUrlLoader::ReceiveBlockBytes(std::string_view content) { ++ partial_block_.append(content); ++ VLOG(2) << "Recived a block of size " << content.size() << " now have " ++ << partial_block_.size() << " bytes."; ++} ++ ++void ipfs::IpfsUrlLoader::AppendGatewayHeaders( ++ std::vector const& cids, ++ net::HttpResponseHeaders& out) { ++ summarize_headers(cids, root_, out, state_->storage()); ++} +diff --git a/components/ipfs/ipfs_url_loader.h b/components/ipfs/ipfs_url_loader.h +new file mode 100644 +index 0000000000000..f0634b3c823e0 +--- /dev/null ++++ b/components/ipfs/ipfs_url_loader.h +@@ -0,0 +1,100 @@ ++#ifndef COMPONENTS_IPFS_URL_LOADER_H_ ++#define COMPONENTS_IPFS_URL_LOADER_H_ 1 ++ ++#include "ipfs_client/dag_listener.h" ++#include "ipfs_client/scheduler.h" ++ ++#include "base/debug/debugging_buildflags.h" ++#include "base/timer/timer.h" ++#include "mojo/public/cpp/bindings/receiver_set.h" ++#include "mojo/public/cpp/system/data_pipe.h" ++#include "services/network/public/cpp/resolve_host_client_base.h" ++#include "services/network/public/cpp/resource_request.h" ++#include "services/network/public/mojom/url_loader.mojom.h" ++ ++#include ++ ++namespace ipfs { ++class GatewayRequests; ++class UnixFsPathResolver; ++} // namespace ipfs ++ ++namespace network::mojom { ++class URLLoaderFactory; ++class HostResolver; ++class NetworkContext; ++} // namespace network::mojom ++namespace network { ++class SimpleURLLoader; ++} ++ ++namespace ipfs { ++class InterRequestState; ++ ++class IpfsUrlLoader final : public network::mojom::URLLoader, ++ public DagListener { ++ void FollowRedirect( ++ std::vector const& removed_headers, ++ net::HttpRequestHeaders const& modified_headers, ++ net::HttpRequestHeaders const& modified_cors_exempt_headers, ++ absl::optional<::GURL> const& new_url) override; ++ void SetPriority(net::RequestPriority priority, ++ int32_t intra_priority_value) override; ++ void PauseReadingBodyFromNet() override; ++ void ResumeReadingBodyFromNet() override; ++ ++ public: ++ explicit IpfsUrlLoader(network::mojom::URLLoaderFactory& handles_http, ++ InterRequestState& state); ++ ~IpfsUrlLoader() noexcept override; ++ ++ using ptr = std::shared_ptr; ++ ++ // Passed as the RequestHandler for ++ // Interceptor::MaybeCreateLoader. ++ static void StartRequest( ++ ptr, ++ network::ResourceRequest const& resource_request, ++ mojo::PendingReceiver receiver, ++ mojo::PendingRemote client); ++ ++ void OverrideUrl(GURL); ++ void AddHeader(std::string_view,std::string_view); ++ void extra(std::shared_ptr xtra) { extra_ = xtra; } ++ ++ private: ++ using RequestHandle = std::unique_ptr; ++ ++ raw_ref state_; ++ mojo::Receiver receiver_{this}; ++ mojo::Remote client_; ++ raw_ref lower_loader_factory_; ++ mojo::ScopedDataPipeProducerHandle pipe_prod_ = {}; ++ mojo::ScopedDataPipeConsumerHandle pipe_cons_ = {}; ++ bool complete_ = false; ++ std::shared_ptr api_; ++ std::string original_url_; ++ std::shared_ptr resolver_; ++ std::string partial_block_; ++ std::vector> additional_outgoing_headers_; ++ std::shared_ptr extra_; ++ std::unique_ptr stepper_; ++ std::string root_; ++ int status_ = 200; ++ ++ void CreateBlockRequest(std::string cid); ++ ++ void ReceiveBlockBytes(std::string_view) override; ++ void BlocksComplete(std::string mime_type) override; ++ void DoesNotExist(std::string_view cid, std::string_view path) override; ++ void NotHere(std::string_view cid, std::string_view path) override; ++ ++ void StartUnixFsProc(ptr, std::string_view); ++ void AppendGatewayHeaders(std::vector const& cids, net::HttpResponseHeaders&); ++ void AppendGatewayInfoHeader(std::string const&, net::HttpResponseHeaders&); ++ void TakeStep(); ++}; ++ ++} // namespace ipfs ++ ++#endif +diff --git a/components/ipfs/ipns_cbor.cc b/components/ipfs/ipns_cbor.cc +new file mode 100644 +index 0000000000000..5aada976d9580 +--- /dev/null ++++ b/components/ipfs/ipns_cbor.cc +@@ -0,0 +1,85 @@ ++#include "ipns_cbor.h" ++ ++#include "base/logging.h" ++#include "components/cbor/reader.h" ++ ++namespace { ++bool Assign(cbor::Value const& v, std::string& o) { ++ if (v.is_string()) { ++ o = v.GetString(); ++ return true; ++ } else if (v.is_bytestring()) { ++ auto& b = v.GetBytestring(); ++ o.assign(reinterpret_cast(b.data()), b.size()); ++ return true; ++ } ++ return false; ++} ++bool Assign(cbor::Value const& v, std::uint64_t& o) { ++ if (v.is_unsigned()) { ++ o = v.GetUnsigned(); ++ return true; ++ } ++ return false; ++} ++template ++void Assign(cbor::Value::MapValue const& m, char const* n, V& out) { ++ auto i = m.find(cbor::Value{n}); ++ if (i == m.end()) { ++ LOG(ERROR) << "CBOR contains no key '" << n << "'!"; ++ return; ++ } ++ if (Assign(i->second, out)) { ++ LOG(INFO) << "Assigned successfully '" << n << "'='" << out << "'."; ++ } else { ++ LOG(ERROR) << "Type mismatch on " << n ++ << " type received: " << static_cast(i->second.type()); ++ } ++} ++ ++void PrettyPrint(cbor::Value const& v) { ++ std::clog << "t=" << static_cast(v.type()) << ' '; ++ if (v.is_map()) { ++ auto& m = v.GetMap(); ++ std::clog << "{\n"; ++ for (auto& e : m) { ++ std::clog << " "; ++ PrettyPrint(e.first); ++ std::clog << " ==> "; ++ PrettyPrint(e.second); ++ std::clog << '\n'; ++ } ++ std::clog << '}'; ++ } else if (v.is_string()) { ++ std::clog << '"' << v.GetString() << '"'; ++ } else if (v.is_integer()) { ++ std::clog << v.GetInteger(); ++ } else { ++ std::clog << ""; ++ } ++} ++ ++} // namespace ++ ++auto ipfs::ParseCborIpns(ipfs::ByteView bytes) -> IpnsCborEntry { ++ auto val = cbor::Reader::Read(as_octets(bytes)); ++ if (!val) { ++ LOG(ERROR) << "Buffer failed to parse as CBOR!"; ++ return {}; ++ } ++ if (!val->is_map()) { ++ LOG(ERROR) << "The CBOR buffer parsed as value of type " ++ << static_cast(val->type()) << " rather than a map!"; ++ return {}; ++ } ++ PrettyPrint(*val); ++ IpnsCborEntry result; ++ auto& m = val->GetMap(); ++ Assign(m, "Value", result.value); ++ Assign(m, "Validity", result.validity); ++ Assign(m, "ValidityType", result.validityType); ++ Assign(m, "TTL", result.ttl); ++ Assign(m, "Sequence", result.sequence); ++ ++ return result; ++} +\ No newline at end of file +diff --git a/components/ipfs/ipns_cbor.h b/components/ipfs/ipns_cbor.h +new file mode 100644 +index 0000000000000..ba4f4fed4c2d3 +--- /dev/null ++++ b/components/ipfs/ipns_cbor.h +@@ -0,0 +1,10 @@ ++#ifndef IPFS_IPNS_CBOR_H_ ++#define IPFS_IPNS_CBOR_H_ ++ ++#include ++ ++namespace ipfs { ++IpnsCborEntry ParseCborIpns(ByteView); ++} ++ ++#endif // IPFS_IPNS_CBOR_H_ +diff --git a/components/ipfs/ipns_url_loader.cc b/components/ipfs/ipns_url_loader.cc +new file mode 100644 +index 0000000000000..24dfe3224a71f +--- /dev/null ++++ b/components/ipfs/ipns_url_loader.cc +@@ -0,0 +1,278 @@ ++#include "ipns_url_loader.h" ++ ++#include "gateway_requests.h" ++#include "inter_request_state.h" ++ ++#include "net/base/net_errors.h" ++#include "services/network/public/cpp/resource_request.h" ++#include "services/network/public/mojom/network_context.mojom.h" ++#include "services/network/public/mojom/url_loader.mojom.h" ++#include "services/network/public/mojom/url_response_head.mojom.h" ++ ++#include ++ ++#include ++#include ++ ++namespace moj = network::mojom; ++ ++ipfs::IpnsUrlLoader::IpnsUrlLoader( ++ InterRequestState& state, ++ std::string host, ++ moj::NetworkContext* network_context, ++ network::mojom::URLLoaderFactory& handles_http) ++ : state_{state}, ++ host_(host), ++ ipfs_loader_(std::make_shared(handles_http, state)), ++ network_context_{network_context}, ++ http_loader_{handles_http} { ++ DCHECK(network_context); ++} ++ipfs::IpnsUrlLoader::~IpnsUrlLoader() noexcept {} ++ ++void ipfs::IpnsUrlLoader::StartHandling( ++ std::shared_ptr me, ++ network::ResourceRequest const& resource_request, ++ mojo::PendingReceiver receiver, ++ mojo::PendingRemote client) { ++ me->ipfs_loader_->extra(me); ++ me->request_ = resource_request; ++ me->client_remote_ = std::move(client); ++ me->loader_receiver_ = std::move(receiver); ++ me->Next(); ++} ++ ++void ipfs::IpnsUrlLoader::OnTextResults( ++ std::vector const& text_results) { ++ LOG(INFO) << "There are " << text_results.size() << " text records."; ++ constexpr std::string_view prefix{"dnslink=/"}; ++ // https://dnslink.io/#multiple-records ++ std::string result; ++ for (auto& text_result : text_results) { ++ if (text_result.compare(0, prefix.size(), prefix)) { ++ LOG(INFO) << "Irrelevant text result: " << text_result; ++ } else { ++ auto replacement = std::string_view{text_result}.substr(prefix.size()); ++ LOG(INFO) << "Text result '" << text_result << "' -> replacment string '" ++ << replacement << "'."; ++ if (replacement.substr(0, 2) != "ip") { ++ LOG(ERROR) << "Unsupported DNSLink redirect '" << replacement << "'."; ++ } else if (replacement.substr(3, 2) != "s/") { ++ } else if (result.empty() || replacement < result) { ++ // Note that "ipfs/..." < "ipns/..." ++ result.assign(replacement); ++ } ++ } ++ } ++ if (result.empty()) { ++ state_->names().NoSuchName(host_); ++ LOG(ERROR) ++ << "_dnslink. domain exists, but contains no /ipfs or /ipns entry"; ++ } else { ++ state_->names().AssignDnsLink(host_, result); ++ } ++} ++void ipfs::IpnsUrlLoader::OnComplete( ++ int32_t result, ++ ::net::ResolveErrorInfo const&, ++ absl::optional<::net::AddressList> const&, ++ absl::optional> const&) { ++ VLOG(1) << "Done with a DNS request."; ++ auto _ = recv_.Unbind(); ++ if (result == net::Error::OK) { ++ Next(); ++ } else { ++ LOG(ERROR) << "Error resolving _dnslink." << host_ << " : " << result; ++ state_->names().NoSuchName(host_); ++ FailNameResolution(); ++ } ++} ++void ipfs::IpnsUrlLoader::Next() { ++ auto resolved = state_->names().NameResolvedTo(host_); ++ if (resolved.empty()) { ++ if (!RequestIpnsRecord()) { ++ VLOG(1) << "Treating '" << host_ << "' as a DNSLink host."; ++ QueryDns(host_); ++ } ++ } else if (resolved == IpnsNames::kNoSuchName) { ++ VLOG(1) << "We have given up on resolving DNSLink " << host_; ++ FailNameResolution(); ++ } else if (request_ && resolved.substr(0, 5) == "ipfs/") { ++ DoIpfs(); ++ } else if (resolved.substr(0, 5) == "ipns/") { ++ LOG(INFO) << "Moving an indirection down from " << host_ << " to " ++ << resolved; ++ host_.assign(resolved, 5); ++ Next(); ++ } ++} ++void ipfs::IpnsUrlLoader::DoIpfs() { ++ auto resolved = state_->names().NameResolvedTo(host_); ++ auto from_url = request_.value().url; ++ std::string to{resolved}; ++ DCHECK_GT(to.size(), 5U); ++ DCHECK_EQ(to[0], 'i'); ++ DCHECK_EQ(to[1], 'p'); ++ DCHECK(to[2] == 'f' || to[2] == 'n'); ++ DCHECK_EQ(to[3], 's'); ++ DCHECK_EQ(to[4], '/'); ++ to.insert(4, ":/"); ++ if (from_url.has_path()) { ++ to.append(from_url.path()); ++ } ++ if (from_url.has_query()) { ++ // Stuff following '?' up to the ref. The getters will not include the '?'. ++ to.push_back('?'); ++ to.append(from_url.query()); ++ } ++ if (from_url.has_ref()) { ++ // Stuff following '#' to the end of the string. This will be %-escaped ++ // UTF-8. The getters will not include the '#'. ++ to.push_back('#'); ++ to.append(from_url.ref()); ++ } ++ GURL to_url{to}; ++ VLOG(1) << "Treating " << from_url << " as " << to_url; ++ ipfs_loader_->OverrideUrl(from_url); ++ auto* entry = state_->names().Entry(host_); ++ if (entry) { ++ auto duration = entry->resolution_ms; ++ ipfs_loader_->AddHeader( ++ "Server-Timing", ++ "ipns;desc=\"IPNS record request\";dur=" + std::to_string(duration)); ++ ipfs_loader_->AddHeader( ++ "IPNS-Name-Source", ++ entry->gateway_source + " @" + std::to_string(entry->fetch_time)); ++ } ++ request_.value().url = to_url; ++ ipfs::IpfsUrlLoader::StartRequest(ipfs_loader_, *request_, ++ std::move(loader_receiver_), ++ std::move(client_remote_)); ++ ipfs_loader_->extra({}); ++} ++ ++network::mojom::URLLoader& ipfs::IpnsUrlLoader::under() { ++ return *ipfs_loader_; ++} ++ ++void ipfs::IpnsUrlLoader::FollowRedirect( ++ std::vector const& removed_headers, ++ ::net::HttpRequestHeaders const& modified_headers, ++ ::net::HttpRequestHeaders const& modified_cors_exempt_headers, ++ absl::optional<::GURL> const& new_url) { ++ under().FollowRedirect(removed_headers, modified_headers, ++ modified_cors_exempt_headers, new_url); ++} ++void ipfs::IpnsUrlLoader::SetPriority(::net::RequestPriority priority, ++ int32_t intra_priority_value) { ++ under().SetPriority(priority, intra_priority_value); ++} ++void ipfs::IpnsUrlLoader::PauseReadingBodyFromNet() { ++ under().PauseReadingBodyFromNet(); ++} ++void ipfs::IpnsUrlLoader::ResumeReadingBodyFromNet() { ++ under().ResumeReadingBodyFromNet(); ++} ++void ipfs::IpnsUrlLoader::QueryDns(std::string_view host) { ++ if (recv_.is_bound()) { ++ LOG(INFO) << "Awaiting DNS response."; ++ return; ++ } ++ auto params = moj::ResolveHostParameters::New(); ++ params->dns_query_type = net::DnsQueryType::TXT; ++ params->initial_priority = net::RequestPriority::HIGHEST; ++ params->source = net::HostResolverSource::ANY; ++ params->cache_usage = moj::ResolveHostParameters_CacheUsage::STALE_ALLOWED; ++ params->secure_dns_policy = moj::SecureDnsPolicy::ALLOW; ++ params->purpose = moj::ResolveHostParameters::Purpose::kUnspecified; ++ std::string dnslink_host{"_dnslink."}; ++ dnslink_host.append(host); ++ LOG(INFO) << "Querying DNS for TXT records on '" << dnslink_host ++ << "' in service of resolving " ++ << (request_ ? request_->url.spec() : host_); ++ auto hrh = moj::HostResolverHost::NewHostPortPair({dnslink_host, 0}); ++ auto nak = net::NetworkAnonymizationKey::CreateTransient(); ++ network_context_->ResolveHost(std::move(hrh), nak, std::move(params), ++ recv_.BindNewPipeAndPassRemote()); ++} ++void ipfs::IpnsUrlLoader::FailNameResolution() { ++ if (client_remote_.is_valid()) { ++ mojo::Remote client; ++ client.Bind(std::move(client_remote_)); ++ client->OnComplete( ++ network::URLLoaderCompletionStatus(net::ERR_NAME_NOT_RESOLVED)); ++ } ++} ++bool ipfs::IpnsUrlLoader::RequestIpnsRecord() { ++ auto cid = libp2p::multi::ContentIdentifierCodec::fromString(host_); ++ if (!cid.has_value()) { ++ return false; ++ } ++ if (cid.value().content_type != ++ libp2p::multi::MulticodecType::Code::LIBP2P_KEY) { ++ return false; ++ } ++ if (api_) { ++ // true because this is true IPNS ++ // ... but return early because we have already requested it ++ state_->scheduler().IssueRequests(api_); ++ return true; ++ } ++ auto key = "ipns/" + host_; ++ auto caches = state_->serialized_caches(); ++ auto check = ++ [this, key]( ++ std::function fail, ++ std::shared_ptr cache) -> std::function { ++ return [this, key, fail, cache]() { ++ LOG(INFO) << "Attempting to fetch IPNS record from " << cache->name(); ++ auto hit = [this, cache](auto b, auto) { CacheHit(cache, b); }; ++ cache->FetchEntry(key, net::HIGHEST, hit, fail); ++ }; ++ }; ++ std::function last_resort = [this]() { ++ LOG(INFO) << "All caches missed for " << this->host_; ++ this->RequestFromGateway(); ++ }; ++ auto chain = ++ std::accumulate(caches.rbegin(), caches.rend(), last_resort, check); ++ chain(); ++ return true; ++} ++void ipfs::IpnsUrlLoader::CacheHit(std::shared_ptr cache, ++ std::string_view body) { ++ LOG(INFO) << "IPNS cache hit! " << host_ << "=" << body; ++ auto result = ValidatedIpns::Deserialize(std::string{body}); ++ if (result.use_until < std::time(nullptr)) { ++ LOG(WARNING) << "Unfortunately, the cache entry for " << host_ ++ << " is too old - it expired at " << result.use_until; ++ cache->Expire("ipns/" + host_); ++ RequestFromGateway(); ++ } else { ++ state_->names().AssignName(host_, result); ++ Complete(); ++ } ++} ++void ipfs::IpnsUrlLoader::RequestFromGateway() { ++ api_ = state_->api(); ++ api_->SetLoaderFactory(*http_loader_); ++ state_->scheduler().Enqueue(api_, {}, shared_from_this(), "ipns/" + host_, ++ "application/vnd.ipfs.ipns-record", 3, {}); ++ state_->scheduler().IssueRequests(api_); ++} ++void ipfs::IpnsUrlLoader::Complete() { ++ LOG(INFO) << "NameListener's Complete called for an IpnsLoader."; ++ auto* entry = state_->names().Entry(host_); ++ if (entry) { ++ auto caches = state_->serialized_caches(); ++ LOG(INFO) << "Storing the resolution of ipns://" << host_ << " in " ++ << caches.size() << " cache backends."; ++ for (auto cache : caches) { ++ cache->Store("ipns/" + host_, "IPNS", entry->Serialize()); ++ } ++ Next(); ++ } else { ++ LOG(ERROR) << "Failed to resolve IPNS " << host_; ++ FailNameResolution(); ++ } ++} +diff --git a/components/ipfs/ipns_url_loader.h b/components/ipfs/ipns_url_loader.h +new file mode 100644 +index 0000000000000..f1b9c1013aba7 +--- /dev/null ++++ b/components/ipfs/ipns_url_loader.h +@@ -0,0 +1,89 @@ ++#ifndef IPFS_IPNS_REDIRECT_H_ ++#define IPFS_IPNS_REDIRECT_H_ ++ ++#include "ipfs_url_loader.h" ++ ++#include ++#include ++ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++namespace network { ++struct ResourceRequest; ++namespace mojom { ++class NetworkContext; ++class URLLoader; ++class URLLoaderClient; ++} // namespace mojom ++} // namespace network ++ ++namespace ipfs { ++class CacheRequestor; ++class ContextApi; ++class InterRequestState; ++ ++class IpnsUrlLoader : public network::ResolveHostClientBase, ++ public network::mojom::URLLoader, ++ public NameListener { ++ raw_ref state_; ++ std::string host_; ++ std::optional request_; ++ mojo::Receiver recv_{this}; ++ mojo::PendingReceiver loader_receiver_; ++ mojo::PendingRemote client_remote_; ++ std::shared_ptr ipfs_loader_; ++ raw_ptr network_context_; ++ std::shared_ptr api_; ++ raw_ref http_loader_; ++ ++ public: ++ explicit IpnsUrlLoader(InterRequestState& state, ++ std::string host, ++ network::mojom::NetworkContext* network_context, ++ network::mojom::URLLoaderFactory& handles_http); ++ ~IpnsUrlLoader() noexcept override; ++ ++ static void StartHandling( ++ std::shared_ptr, ++ network::ResourceRequest const&, ++ mojo::PendingReceiver, ++ mojo::PendingRemote); ++ ++ private: ++ using Endpoints = std::vector<::net::HostResolverEndpointResult>; ++ void OnTextResults(std::vector const&) override; ++ void OnComplete(int32_t result, ++ ::net::ResolveErrorInfo const&, ++ absl::optional<::net::AddressList> const&, ++ absl::optional const&) override; ++ ++ void Next(); ++ void QueryDns(std::string_view); ++ void DoIpfs(); ++ void FailNameResolution(); ++ network::mojom::URLLoader& under(); ++ bool RequestIpnsRecord(); ++ void RequestFromGateway(); ++ void CacheHit(std::shared_ptr,std::string_view); ++ ++ void FollowRedirect( ++ std::vector const& removed_headers, ++ ::net::HttpRequestHeaders const& modified_headers, ++ ::net::HttpRequestHeaders const& modified_cors_exempt_headers, ++ absl::optional<::GURL> const& new_url) override; ++ void SetPriority(::net::RequestPriority priority, ++ int32_t intra_priority_value) override; ++ void PauseReadingBodyFromNet() override; ++ void ResumeReadingBodyFromNet() override; ++ ++ void Complete() override; // From NameListener ++}; ++} // namespace ipfs ++ ++#endif // IPFS_IPNS_REDIRECT_H_ +diff --git a/components/ipfs/network_requestor.cc b/components/ipfs/network_requestor.cc +new file mode 100644 +index 0000000000000..a05ac6a14aede +--- /dev/null ++++ b/components/ipfs/network_requestor.cc +@@ -0,0 +1,16 @@ ++#include "network_requestor.h" ++ ++#include "gateway_requests.h" ++#include "inter_request_state.h" ++ ++using Self = ipfs::NetworkRequestor; ++ ++Self::NetworkRequestor(InterRequestState& state) : state_{state} {} ++Self::~NetworkRequestor() noexcept = default; ++ ++void Self::RequestByCid(std::string cid, ++ std::shared_ptr listen, ++ Priority prio) { ++ auto api = state_->api(); ++ api->RequestByCid(cid, listen, prio); ++} +diff --git a/components/ipfs/network_requestor.h b/components/ipfs/network_requestor.h +new file mode 100644 +index 0000000000000..aa812633d12f2 +--- /dev/null ++++ b/components/ipfs/network_requestor.h +@@ -0,0 +1,27 @@ ++#ifndef NETWORKREQUESTOR_H ++#define NETWORKREQUESTOR_H ++ ++#include ++ ++#include ++ ++namespace ipfs { ++ ++class InterRequestState; ++ ++class NetworkRequestor : public BlockRequestor { ++ raw_ref state_; ++ ++ public: ++ NetworkRequestor(InterRequestState& state); ++ virtual ~NetworkRequestor() noexcept; ++ ++ private: ++ void RequestByCid(std::string cid, ++ std::shared_ptr, ++ Priority) override; ++ ++}; ++} // namespace ipfs ++ ++#endif // NETWORKREQUESTOR_H +diff --git a/components/ipfs/profile_prefs.cc b/components/ipfs/profile_prefs.cc +new file mode 100644 +index 0000000000000..c981df549308e +--- /dev/null ++++ b/components/ipfs/profile_prefs.cc +@@ -0,0 +1,25 @@ ++#include "profile_prefs.h" ++ ++#include "base/values.h" ++#include "components/prefs/pref_registry_simple.h" ++ ++#include ++ ++#include //TODO rm ++ ++void ipfs::RegisterProfilePrefs(PrefRegistrySimple* registry, ++ std::string const& // locale ++) { ++ DCHECK(registry); ++ if (!registry) { ++ return; ++ } ++ auto dflt = Gateways::DefaultGateways(); ++ base::Value::Dict pref_val; ++ for (auto [k, v] : dflt) { ++ // std::clog << k << '=' << v << '\n'; ++ pref_val.Set(k, v); ++ } ++ // std::clog << "\n\n\t registering \"net.ipfs.gateways\"\n\n"; ++ registry->RegisterDictionaryPref("net.ipfs.gateways", std::move(pref_val)); ++} +diff --git a/components/ipfs/profile_prefs.h b/components/ipfs/profile_prefs.h +new file mode 100644 +index 0000000000000..883742f699c4c +--- /dev/null ++++ b/components/ipfs/profile_prefs.h +@@ -0,0 +1,16 @@ ++#ifndef IPFS_PROFILE_PREFS_H_ ++#define IPFS_PROFILE_PREFS_H_ ++ ++#include "base/component_export.h" ++ ++#include ++ ++class PrefRegistrySimple; ++ ++namespace ipfs { ++COMPONENT_EXPORT(IPFS) ++void RegisterProfilePrefs(PrefRegistrySimple* registry, ++ std::string const& locale = ""); ++} // namespace ipfs ++ ++#endif // IPFS_PROFILE_PREFS_H_ +diff --git a/components/ipfs/summarize_headers.cc b/components/ipfs/summarize_headers.cc +new file mode 100644 +index 0000000000000..294651bf45355 +--- /dev/null ++++ b/components/ipfs/summarize_headers.cc +@@ -0,0 +1,105 @@ ++#include "summarize_headers.h" ++ ++#include "base/logging.h" ++#include "services/network/public/mojom/url_response_head.mojom.h" ++ ++#include ++ ++void ipfs::summarize_headers(std::vector const& cids, ++ std::string const& root, ++ net::HttpResponseHeaders& out, ++ BlockStorage& storage) { ++ VLOG(1) << std::time(nullptr); ++ std::map bytes_per; ++ std::map blocks_per; ++ std::map millis_per; ++ for (auto& cid : cids) { ++ auto* raw = storage.GetHeaders(cid); ++ if (!raw || raw->empty()) { ++ LOG(ERROR) << "Trouble fetching headers from gateway response " << cid; ++ return; ++ } ++ auto gw_heads = base::MakeRefCounted(*raw); ++ if (!gw_heads) { ++ std::ostringstream escaped; ++ for (auto c : *raw) { ++ if (std::isgraph(c)) { ++ escaped << c; ++ } else { ++ escaped << '<' << std::hex << std::setw(2) << std::setfill('0') ++ << static_cast(c) << '>'; ++ } ++ } ++ LOG(ERROR) << "Failed to parse raw string as headers for " << cid << " : " ++ << escaped.str(); ++ return; ++ } ++ std::size_t i = 0UL; ++ std::string name, value; ++ while (gw_heads->EnumerateHeaderLines(&i, &name, &value)) { ++ if (name == "Server-Timing" && cid == root) { ++ auto dur = value.find("dur="); ++ if (dur < value.size()) { ++ out.AddHeader(name, "ipfs-ttfb;desc=\"til first block(root)\";" + ++ value.substr(dur)); ++ } else { ++ LOG(ERROR) << "Server-Timing with no 'dur=': '" << value << "'."; ++ } ++ } else if (name == "Server-Timing") { ++ auto desc = value.find(";desc=\""); ++ if (desc >= value.size()) { ++ LOG(WARNING) << "Server-Timing with no desc"; ++ continue; ++ } ++ auto dur = value.find(";dur="); ++ if (dur >= value.size()) { ++ LOG(WARNING) << "Server-Timing header with no dur on CID " << cid ++ << " (from HTTP) : " << value; ++ continue; ++ } ++ std::string gw = "cache"; ++ auto load = value.find(" : load over http(s)", desc); ++ if (load < value.size()) { ++ auto start = desc + 7; ++ auto len = load - start; ++ gw = value.substr(start, len); ++ } ++ auto ms = std::atol(value.c_str() + dur + 5); ++ millis_per[gw] += ms; ++ } else if (name == "Block-Source") { ++ auto comma = value.find(", "); ++ if (comma >= value.size()) { ++ continue; ++ } ++ auto at = value.find(" @", comma); ++ if (at >= value.size()) { ++ continue; ++ } ++ auto start = comma + 2; ++ auto len = at - start; ++ auto gw = value.substr(start, len); ++ blocks_per[gw]++; ++ bytes_per[gw] += storage.Get(cid)->unparsed().size(); ++ } else if (!name.find("Block-Cache-")) { ++ blocks_per["cache"]++; ++ bytes_per["cache"] += storage.Get(cid)->unparsed().size(); ++ } else { ++ // LOG(INFO) << "Dropping header '" << name << "' for " << cid; ++ } ++ } ++ } ++ for (auto [gateway, block_count] : blocks_per) { ++ std::ostringstream val; ++ val << gateway << ";blocks=" << block_count; ++ auto it = bytes_per.find(gateway); ++ if (it != bytes_per.end()) { ++ val << ";bytes=" << it->second; ++ } ++ it = millis_per.find(gateway); ++ if (it != millis_per.end()) { ++ val << ";cdur=" << it->second; ++ } ++ VLOG(1) << "Add header Ipfs-Block-Source: " << val.str(); ++ out.AddHeader("Ipfs-Block-Source", val.str()); ++ } ++} +diff --git a/components/ipfs/summarize_headers.h b/components/ipfs/summarize_headers.h +new file mode 100644 +index 0000000000000..d2dddfbb76044 +--- /dev/null ++++ b/components/ipfs/summarize_headers.h +@@ -0,0 +1,20 @@ ++#ifndef IPFS_SUMMARIZE_HEADERS_H_ ++#define IPFS_SUMMARIZE_HEADERS_H_ ++ ++#include ++#include ++ ++namespace net { ++class HttpResponseHeaders; ++} ++ ++namespace ipfs { ++class BlockStorage; ++ ++void summarize_headers(std::vector const& cids, ++ std::string const& root, ++ net::HttpResponseHeaders& out, ++ BlockStorage& storage); ++} // namespace ipfs ++ ++#endif // IPFS_SUMMARIZE_HEADERS_H_ +diff --git a/components/ipfs/url_loader_factory.cc b/components/ipfs/url_loader_factory.cc +new file mode 100644 +index 0000000000000..4010387d0b813 +--- /dev/null ++++ b/components/ipfs/url_loader_factory.cc +@@ -0,0 +1,58 @@ ++#include "url_loader_factory.h" ++ ++#include "inter_request_state.h" ++#include "ipfs_url_loader.h" ++#include "ipns_url_loader.h" ++ ++void ipfs::IpfsURLLoaderFactory::Create( ++ NonNetworkURLLoaderFactoryMap* in_out, ++ content::BrowserContext* context, ++ URLLoaderFactory* default_factory, ++ network::mojom::NetworkContext* net_ctxt) { ++ for (char const* scheme : {"ipfs", "ipns"}) { ++ mojo::PendingRemote pending; ++ new IpfsURLLoaderFactory(scheme, pending.InitWithNewPipeAndPassReceiver(), ++ context, default_factory, net_ctxt); ++ in_out->emplace(scheme, std::move(pending)); ++ } ++} ++ ++ipfs::IpfsURLLoaderFactory::IpfsURLLoaderFactory( ++ std::string scheme, ++ mojo::PendingReceiver factory_receiver, ++ content::BrowserContext* context, ++ URLLoaderFactory* default_factory, ++ network::mojom::NetworkContext* net_ctxt) ++ : network::SelfDeletingURLLoaderFactory(std::move(factory_receiver)), ++ scheme_{scheme}, ++ context_{context}, ++ default_factory_{default_factory}, ++ network_context_{net_ctxt} {} ++ ++ipfs::IpfsURLLoaderFactory::~IpfsURLLoaderFactory() noexcept {} ++ ++void ipfs::IpfsURLLoaderFactory::CreateLoaderAndStart( ++ mojo::PendingReceiver loader, ++ int32_t /*request_id*/, ++ uint32_t /*options*/, ++ network::ResourceRequest const& request, ++ mojo::PendingRemote client, ++ net::MutableNetworkTrafficAnnotationTag const& // traffic_annotation ++) { ++ VLOG(1) << "IPFS subresource: case=" << scheme_ ++ << " url=" << request.url.spec(); ++ DCHECK(default_factory_); ++ if (scheme_ == "ipfs") { ++ auto ptr = std::make_shared( ++ *default_factory_, InterRequestState::FromBrowserContext(context_)); ++ ptr->StartRequest(ptr, request, std::move(loader), std::move(client)); ++ } else if (scheme_ == "ipns") { ++ auto ptr = std::make_shared( ++ InterRequestState::FromBrowserContext(context_), request.url.host(), ++ network_context_, *default_factory_); ++ ptr->StartHandling(ptr, request, std::move(loader), std::move(client)); ++ ++ } else { ++ NOTREACHED(); ++ } ++} +diff --git a/components/ipfs/url_loader_factory.h b/components/ipfs/url_loader_factory.h +new file mode 100644 +index 0000000000000..2ae56afa33cae +--- /dev/null ++++ b/components/ipfs/url_loader_factory.h +@@ -0,0 +1,54 @@ ++#ifndef IPFS_URL_LOADER_FACTORY_H_ ++#define IPFS_URL_LOADER_FACTORY_H_ ++ ++#include "services/network/public/cpp/self_deleting_url_loader_factory.h" ++#include "services/network/public/mojom/url_loader_factory.mojom.h" ++ ++#include ++ ++namespace content { ++class BrowserContext; ++} ++namespace network { ++namespace mojom { ++class NetworkContext; ++} ++} // namespace network ++ ++namespace ipfs { ++using NonNetworkURLLoaderFactoryMap = ++ std::map>; ++ ++class COMPONENT_EXPORT(IPFS) IpfsURLLoaderFactory ++ : public network::SelfDeletingURLLoaderFactory { ++ public: ++ static void Create(NonNetworkURLLoaderFactoryMap* in_out, ++ content::BrowserContext*, ++ URLLoaderFactory*, ++ network::mojom::NetworkContext*); ++ ++ private: ++ IpfsURLLoaderFactory(std::string, ++ mojo::PendingReceiver, ++ content::BrowserContext*, ++ network::mojom::URLLoaderFactory*, ++ network::mojom::NetworkContext*); ++ ~IpfsURLLoaderFactory() noexcept override; ++ void CreateLoaderAndStart( ++ mojo::PendingReceiver loader, ++ int32_t request_id, ++ uint32_t options, ++ network::ResourceRequest const& request, ++ mojo::PendingRemote client, ++ net::MutableNetworkTrafficAnnotationTag const& traffic_annotation) ++ override; ++ ++ std::string scheme_; ++ raw_ptr context_; ++ raw_ptr default_factory_; ++ raw_ptr network_context_; ++}; ++} // namespace ipfs ++ ++#endif // IPFS_URL_LOADER_FACTORY_H_ diff --git a/components/open_from_clipboard/clipboard_recent_content_generic.cc b/components/open_from_clipboard/clipboard_recent_content_generic.cc index 4dcafecbc66c6..d205209c08162 100644 --- a/components/open_from_clipboard/clipboard_recent_content_generic.cc @@ -262,26 +2735,10144 @@ index 5273da5190277..12b28b86a4c00 100644 - + case NsswitchReader::Service::kResolve: + break; - case NsswitchReader::Service::kMdns: - case NsswitchReader::Service::kMdns4: - case NsswitchReader::Service::kMdns6: -- case NsswitchReader::Service::kResolve: - case NsswitchReader::Service::kNis: - RecordIncompatibleNsswitchReason( - IncompatibleNsswitchReason::kIncompatibleService, -diff --git a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc -index 4eadf46ea0c24..d62fc7fb14e01 100644 ---- a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc -+++ b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc -@@ -67,7 +67,7 @@ class URLSchemesRegistry final { - // is considered secure. Additional checks are performed to ensure that - // other http pages are filtered out. - service_worker_schemes({"http", "https"}), -- fetch_api_schemes({"http", "https"}), -+ fetch_api_schemes({"http", "https", "ipfs", "ipns"}), - allowed_in_referrer_schemes({"http", "https"}) { - for (auto& scheme : url::GetCorsEnabledSchemes()) - cors_enabled_schemes.insert(scheme.c_str()); + case NsswitchReader::Service::kMdns: + case NsswitchReader::Service::kMdns4: + case NsswitchReader::Service::kMdns6: +- case NsswitchReader::Service::kResolve: + case NsswitchReader::Service::kNis: + RecordIncompatibleNsswitchReason( + IncompatibleNsswitchReason::kIncompatibleService, +diff --git a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +index 4eadf46ea0c24..d62fc7fb14e01 100644 +--- a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc ++++ b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +@@ -67,7 +67,7 @@ class URLSchemesRegistry final { + // is considered secure. Additional checks are performed to ensure that + // other http pages are filtered out. + service_worker_schemes({"http", "https"}), +- fetch_api_schemes({"http", "https"}), ++ fetch_api_schemes({"http", "https", "ipfs", "ipns"}), + allowed_in_referrer_schemes({"http", "https"}) { + for (auto& scheme : url::GetCorsEnabledSchemes()) + cors_enabled_schemes.insert(scheme.c_str()); +diff --git a/third_party/ipfs_client/BUILD.gn b/third_party/ipfs_client/BUILD.gn +new file mode 100644 +index 0000000000000..1f2e755cd31cd +--- /dev/null ++++ b/third_party/ipfs_client/BUILD.gn +@@ -0,0 +1,200 @@ ++import("args.gni") ++import("//build/buildflag_header.gni") ++ ++buildflag_header("ipfs_buildflags") { ++ header = "ipfs_buildflags.h" ++ flags = [ "ENABLE_IPFS=$enable_ipfs" ] ++} ++ ++config("external_config") { ++ include_dirs = [ ++ "include", ++ ] ++} ++ ++if (enable_ipfs) { ++ cxx_sources = [ ++ "include/ipfs_client/block_requestor.h", ++ "include/ipfs_client/block_storage.h", ++ "include/ipfs_client/busy_gateway.h", ++ "include/ipfs_client/chained_requestors.h", ++ "include/ipfs_client/context_api.h", ++ "include/ipfs_client/dag_block.h", ++ "include/ipfs_client/dag_listener.h", ++ "include/ipfs_client/export.h", ++ "include/ipfs_client/gateway.h", ++ "include/ipfs_client/gateways.h", ++ "include/ipfs_client/gw/gateway_request.h", ++ "include/ipfs_client/identity_cid.h", ++ "include/ipfs_client/ipfs_request.h", ++ "include/ipfs_client/ipld/dag_node.h", ++ "include/ipfs_client/ipld/link.h", ++ "include/ipfs_client/ipns_names.h", ++ "include/ipfs_client/ipns_record.h", ++ "include/ipfs_client/logger.h", ++ "include/ipfs_client/name_listener.h", ++ "include/ipfs_client/orchestrator.h", ++ "include/ipfs_client/response.h", ++ "include/ipfs_client/scheduler.h", ++ "include/ipfs_client/unixfs_path_resolver.h", ++ "include/ipfs_client/url_spec.h", ++ "include/libp2p/basic/varint_prefix_reader.hpp", ++ "include/libp2p/common/hexutil.hpp", ++ "include/libp2p/common/types.hpp", ++ "include/libp2p/crypto/common.hpp", ++ "include/libp2p/crypto/error.hpp", ++ "include/libp2p/crypto/hasher.hpp", ++ "include/libp2p/crypto/key.h", ++ "include/libp2p/crypto/protobuf/protobuf_key.hpp", ++ "include/libp2p/crypto/sha/sha256.hpp", ++ "include/libp2p/multi/content_identifier.hpp", ++ "include/libp2p/multi/content_identifier_codec.hpp", ++ "include/libp2p/multi/hash_type.hpp", ++ "include/libp2p/multi/multibase_codec.hpp", ++ "include/libp2p/multi/multibase_codec/codecs/base16.h", ++ "include/libp2p/multi/multibase_codec/codecs/base32.hpp", ++ "include/libp2p/multi/multibase_codec/codecs/base36.hpp", ++ "include/libp2p/multi/multibase_codec/codecs/base58.hpp", ++ "include/libp2p/multi/multibase_codec/codecs/base_error.hpp", ++ "include/libp2p/multi/multibase_codec/multibase_codec_impl.hpp", ++ "include/libp2p/multi/multicodec_type.hpp", ++ "include/libp2p/multi/multihash.hpp", ++ "include/libp2p/multi/uvarint.hpp", ++ "include/libp2p/peer/peer_id.hpp", ++ "include/smhasher/MurmurHash3.h", ++ "include/vocab/byte.h", ++ "include/vocab/byte_view.h", ++ "include/vocab/endian.h", ++ "include/vocab/expected.h", ++ "include/vocab/flat_mapset.h", ++ "include/vocab/i128.h", ++ "include/vocab/raw_ptr.h", ++ "include/vocab/slash_delimited.h", ++ "include/vocab/span.h", ++ "include/vocab/stringify.h", ++ "src/ipfs_client/block_requestor.cc", ++ "src/ipfs_client/block_storage.cc", ++ "src/ipfs_client/busy_gateway.cc", ++ "src/ipfs_client/chained_requestors.cc", ++ "src/ipfs_client/context_api.cc", ++ "src/ipfs_client/dag_block.cc", ++ "src/ipfs_client/gateway.cc", ++ "src/ipfs_client/gateways.cc", ++ "src/ipfs_client/generated_directory_listing.cc", ++ "src/ipfs_client/generated_directory_listing.h", ++ "src/ipfs_client/gw/gateway_request.cc", ++ "src/ipfs_client/identity_cid.cc", ++ "src/ipfs_client/ipfs_request.cc", ++ "src/ipfs_client/ipld/chunk.cc", ++ "src/ipfs_client/ipld/chunk.h", ++ "src/ipfs_client/ipld/dag_node.cc", ++ "src/ipfs_client/ipld/directory_shard.cc", ++ "src/ipfs_client/ipld/directory_shard.h", ++ "src/ipfs_client/ipld/ipns_name.cc", ++ "src/ipfs_client/ipld/ipns_name.h", ++ "src/ipfs_client/ipld/link.cc", ++ "src/ipfs_client/ipld/root.cc", ++ "src/ipfs_client/ipld/root.h", ++ "src/ipfs_client/ipld/small_directory.cc", ++ "src/ipfs_client/ipld/small_directory.h", ++ "src/ipfs_client/ipld/unixfs_file.cc", ++ "src/ipfs_client/ipld/unixfs_file.h", ++ "src/ipfs_client/ipns_names.cc", ++ "src/ipfs_client/ipns_record.cc", ++ "src/ipfs_client/logger.cc", ++ "src/ipfs_client/orchestrator.cc", ++ "src/ipfs_client/path2url.cc", ++ "src/ipfs_client/path2url.h", ++ "src/ipfs_client/redirects.cc", ++ "src/ipfs_client/redirects.h", ++ "src/ipfs_client/response.cc", ++ "src/ipfs_client/scheduler.cc", ++ "src/ipfs_client/unix_fs/dir_shard.cc", ++ "src/ipfs_client/unix_fs/dir_shard.h", ++ "src/ipfs_client/unix_fs/guess_content_type.cc", ++ "src/ipfs_client/unix_fs/guess_content_type.h", ++ "src/ipfs_client/unix_fs/multi_node_file.cc", ++ "src/ipfs_client/unix_fs/multi_node_file.h", ++ "src/ipfs_client/unix_fs/node_helper.cc", ++ "src/ipfs_client/unix_fs/node_helper.h", ++ "src/ipfs_client/unix_fs/plain_directory.cc", ++ "src/ipfs_client/unix_fs/plain_directory.h", ++ "src/ipfs_client/unix_fs/plain_file.cc", ++ "src/ipfs_client/unix_fs/plain_file.h", ++ "src/ipfs_client/unixfs_path_resolver.cc", ++ "src/libp2p/basic/varint_prefix_reader.cc", ++ "src/libp2p/common/hexutil.cc", ++ "src/libp2p/crypto/hasher.cc", ++ "src/libp2p/crypto/protobuf_key.hpp", ++ "src/libp2p/crypto/provider.h", ++ "src/libp2p/crypto/sha/sha256.cc", ++ "src/libp2p/multi/content_identifier.cc", ++ "src/libp2p/multi/content_identifier_codec.cc", ++ "src/libp2p/multi/multibase_codec/codecs/base16.cc", ++ "src/libp2p/multi/multibase_codec/codecs/base32.cc", ++ "src/libp2p/multi/multibase_codec/codecs/base36.cc", ++ "src/libp2p/multi/multibase_codec/codecs/base58.cc", ++ "src/libp2p/multi/multibase_codec/multibase_codec_impl.cc", ++ "src/libp2p/multi/multihash.cc", ++ "src/libp2p/multi/uvarint.cc", ++ "src/libp2p/peer/peer_id.cc", ++ "src/log_macros.h", ++ "src/smhasher/MurmurHash3.cc", ++ "src/vocab/byte_view.cc", ++ "src/vocab/slash_delimited.cc", ++ ] ++ static_library("ipfs_client") { ++ if (is_nacl) { ++ sources = cxx_sources - [ ++ "src/ipfs_client/block_storage.cc", ++ "src/ipfs_client/dag_block.cc", ++ "src/ipfs_client/ipld/dag_node.cc", ++ "src/ipfs_client/ipns_names.cc", ++ "src/ipfs_client/ipns_record.cc", ++ "src/ipfs_client/logger.cc", ++ "src/ipfs_client/unix_fs/dir_shard.cc", ++ "src/ipfs_client/unix_fs/multi_node_file.cc", ++ "src/ipfs_client/unix_fs/node_helper.cc", ++ "src/ipfs_client/unix_fs/plain_directory.cc", ++ "src/ipfs_client/unix_fs/plain_file.cc", ++ "src/ipfs_client/unixfs_path_resolver.cc", ++ ] ++ } else { ++ sources = cxx_sources ++ } ++ include_dirs = [ ++ "include", ++ "src", ++ "..", ++ "../boringssl/src/include" ++ ] ++ public_configs = [ ++ ":external_config" ++ ] ++ public_deps = [ ++ "//third_party/abseil-cpp:absl", ++ "//base", ++ ] ++ deps = [ ++ "//third_party/abseil-cpp:absl", ++ "//base", ++ ] ++ if (!is_nacl) { ++ public_deps += [ ++ ":protos", ++ "//third_party/protobuf:protobuf_lite", ++ ] ++ } ++ } ++} ++ ++import("//third_party/protobuf/proto_library.gni") ++ ++proto_library("protos") { ++ sources = [ ++ "ipns_record.proto", ++ "keys.proto", ++ "pb_dag.proto", ++ "unix_fs.proto", ++ ] ++} +diff --git a/third_party/ipfs_client/README.chromium b/third_party/ipfs_client/README.chromium +new file mode 100644 +index 0000000000000..e69de29bb2d1d +diff --git a/third_party/ipfs_client/README.md b/third_party/ipfs_client/README.md +new file mode 100644 +index 0000000000000..0e6ffadd2ebbc +--- /dev/null ++++ b/third_party/ipfs_client/README.md +@@ -0,0 +1,6 @@ ++# ipfs-client ++ ++## TODO ++ ++Need to fill out this README to explain how to use ipfs-client in other contexts. ++ +diff --git a/third_party/ipfs_client/args.gni b/third_party/ipfs_client/args.gni +new file mode 100644 +index 0000000000000..bb13519b23e89 +--- /dev/null ++++ b/third_party/ipfs_client/args.gni +@@ -0,0 +1,3 @@ ++declare_args() { ++ enable_ipfs = false ++} +diff --git a/third_party/ipfs_client/conanfile.py b/third_party/ipfs_client/conanfile.py +new file mode 100644 +index 0000000000000..8e5af42365c5b +--- /dev/null ++++ b/third_party/ipfs_client/conanfile.py +@@ -0,0 +1,48 @@ ++from conan import ConanFile ++from conan.tools.cmake import CMake, CMakeToolchain, CMakeDeps ++from shutil import which ++import sys ++from os.path import dirname, join, realpath ++sys.path.append(realpath(join(dirname(__file__), '..', 'cmake'))) ++import version ++ ++ ++class IpfsChromium(ConanFile): ++ name = "ipfs-chromium" ++ version = version.deduce() ++ settings = "os", "compiler", "build_type", "arch" ++ generators = "CMakeDeps" ++ _PB = 'protobuf/3.21.9' ++ requires = [ ++ 'abseil/20230125.3', ++ 'boost/1.81.0', ++ 'gtest/1.13.0', ++ 'openssl/1.1.1t', ++ _PB, ++ ] ++ default_options = {"boost/*:header_only": True} ++ tool_requires = [ ++ 'cmake/3.22.6', ++ 'ninja/1.11.1', ++ _PB, ++ ] ++ ++ ++ def generate(self): ++ tc = CMakeToolchain(self, 'Ninja') ++ #tc.variables["FOO"] = True ++ tc.generate() ++ ++ def build(self): ++ cmake = CMake(self) ++ cmake.configure(variables={ ++ "CXX_VERSION": 17, #TODO ++ "INSIDE_CONAN": True ++ }) ++ cmake.build() ++ ++ ++ def build_requirements(self): ++ if not which("doxygen"): ++ self.tool_requires("doxygen/1.9.4") ++# gperf doxygen ccache lcov +diff --git a/third_party/ipfs_client/include/ipfs_client/block_requestor.h b/third_party/ipfs_client/include/ipfs_client/block_requestor.h +new file mode 100644 +index 0000000000000..42ae26e519760 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/block_requestor.h +@@ -0,0 +1,48 @@ ++#ifndef BLOCK_REQUESTOR_H_ ++#define BLOCK_REQUESTOR_H_ ++ ++#include ++ ++#include ++#include ++#include ++ ++namespace ipfs { ++ ++/*! ++ * \brief The urgency of a gateway request ++ * \details Determines how many gateways should be involved, and how burdened a ++ * gateway should be before not also taking this one on concurrently. Zero is ++ * a special value that indicates the block isn't actually required now, but ++ * rather might be required soonish (prefetch). There are some cases of ++ * special handling for that. ++ */ ++using Priority = std::uint_least16_t; ++ ++class DagListener; ++ ++/*! ++ * \brief Interface for classes that can asynchronously fetch a block for a CID ++ * \details This is one of the interfaces using code is meant to implement. ++ * Common usages: ++ * * A class that requests blocks from gateways ++ * * A cache that must act asynchronously (perhaps on-disk) ++ * * ChainedRequestors : a chain-of-responsibility combining multiple ++ */ ++class BlockRequestor { ++ public: ++ /** ++ * \brief Request a single block from gateway(s). ++ * \param cid - MB-MH string representation of the Content IDentifier ++ * \param dl - Someone who may be interested ++ * \param priority - Urgency of the request ++ * \note The DagListener is mostly about lifetime extension, since it's ++ * waiting on something which is waiting on this ++ */ ++ virtual void RequestByCid(std::string cid, ++ std::shared_ptr dl, ++ Priority priority) = 0; ++}; ++} // namespace ipfs ++ ++#endif // BLOCK_REQUESTOR_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/block_storage.h b/third_party/ipfs_client/include/ipfs_client/block_storage.h +new file mode 100644 +index 0000000000000..26f48ed8c5fb5 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/block_storage.h +@@ -0,0 +1,144 @@ ++#ifndef IPFS_BLOCKS_H_ ++#define IPFS_BLOCKS_H_ ++ ++#include "dag_block.h" ++#include "vocab/flat_mapset.h" ++ ++#include ++#include ++#include ++ ++namespace libp2p::multi { ++struct ContentIdentifier; ++} ++ ++namespace ipfs { ++class DagListener; ++class ContextApi; ++ ++class UnixFsPathResolver; ++ ++/*! ++ * \brief Immediate access to recently-accessed blocks ++ * \details Blocks are held in-memory, using pretty standard containers, as ++ * already-parsed ipfs::Block objects. ++ */ ++class BlockStorage { ++ public: ++ BlockStorage(); ++ ++ BlockStorage(BlockStorage const&) = delete; ++ ++ ~BlockStorage() noexcept; ++ ++ /*! ++ * \brief Store a Block for later access. ++ * \param cid_str - The string representation of cid ++ * \param cid - The Content IDentifier ++ * \param headers - Associated HTTP headers ++ * \param body - The raw bytes of the block ++ * \param block - The block being stored ++ * \return Whether this block is now stored in *this ++ */ ++ bool Store(std::string cid_str, ++ Cid const& cid, ++ std::string headers, ++ std::string const& body, ++ Block&& block); ++ ++ /*! ++ * \name Store (Convenience) ++ * Convenience functions for ++ * ipfs::BlockStorage::Store(std::string,Cid const&,std::string,std::string ++ * const&,Block&&) ++ */ ++ ///@{ ++ bool Store(std::string headers, std::string const& body, Block&& block); ++ bool Store(std::string const& cid, std::string headers, std::string body); ++ bool Store(std::string cid_str, ++ Cid const& cid, ++ std::string headers, ++ std::string body); ++ bool Store(Cid const& cid, ++ std::string headers, ++ std::string const& body, ++ Block&&); ++ ///@} ++ ++ /*! ++ * \brief Get a block! ++ * \details cid must match string-wise exactly: same multibase & all. ++ * For identity codecs, returns the data even if not stored. ++ * \param cid - String representation of the CID for the block. ++ * \return Non-owning pointer if found, nullptr ++ * otherwise ++ */ ++ Block const* Get(std::string const& cid); ++ ++ /*! ++ * \brief Get HTTP headers associated with the block ++ * \param cid - String representation of the CID for the block. ++ * \return nullptr iff ! Get(cid) ; ++ * Empty string if the headers have never been set ; ++ * Otherwise, application-specific std::string (as-stored) ++ */ ++ std::string const* GetHeaders(std::string const& cid); ++ ++ /*! ++ * \brief Indicate that a particular path resolver is waiting on a CID to ++ * become available ++ */ ++ void AddListening(UnixFsPathResolver*); ++ ++ /*! ++ * \brief Indicate that a particular path resolver is no longer waiting ++ */ ++ void StopListening(UnixFsPathResolver*); ++ ++ /*! ++ * \brief Normally called internally ++ * \details Checks to see if any listening path resolver appears to be waiting ++ * on a CID which is now available. ++ */ ++ void CheckListening(); ++ ++ /*! ++ * \brief Type for callbacks about new blocks ++ * \details The parameters to the hook are ++ * * CID string ++ * * HTTP headers ++ * * raw bytes of the block ++ */ ++ using SerializedStorageHook = ++ std::function; ++ ++ /*! ++ * \brief Register a callback that will be called when any new block goes into ++ * storage ++ */ ++ void AddStorageHook(SerializedStorageHook); ++ ++ private: ++ struct Record { ++ Record(); ++ ~Record() noexcept; ++ std::time_t last_access = 0L; ++ std::string cid_str = {}; ++ Block block = {}; ++ std::string headers = {}; ++ }; ++ std::list records_ = std::list(0xFFUL); ++ using Iter = decltype(records_)::iterator; ++ flat_map cid2record_; ++ flat_set listening_; ++ bool checking_ = false; ++ std::vector hooks_; ++ ++ Record const* GetInternal(std::string const&); ++ Record* FindFree(std::time_t); ++ Record* Allocate(); ++ Record* StoreIdentity(std::string const&, Cid const&); ++}; ++} // namespace ipfs ++ ++#endif // IPFS_BLOCKS_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/busy_gateway.h b/third_party/ipfs_client/include/ipfs_client/busy_gateway.h +new file mode 100644 +index 0000000000000..e5dccc050367f +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/busy_gateway.h +@@ -0,0 +1,94 @@ ++#ifndef IPFS_BUSY_GATEWAY_H_ ++#define IPFS_BUSY_GATEWAY_H_ ++ ++#include ++ ++#include ++ ++#include ++#include ++ ++namespace { ++struct TestParams; ++} ++ ++namespace ipfs { ++class ContextApi; ++class DagListener; ++class Gateway; ++class Gateways; ++class IpfsRequest; ++class Scheduler; ++ ++namespace gw { ++struct GatewayRequest; ++} ++ ++/*! ++ * \brief RAII class embodying the assignment of a given task to a given gateway ++ */ ++class BusyGateway { ++ public: ++ BusyGateway(BusyGateway const&) = delete; ++ BusyGateway(BusyGateway&&); ++ ~BusyGateway(); ++ ++ /*! ++ * \name Gateway Access ++ * Pointer semantics to access the underlying gateway. ++ */ ++ ///@{ ++ Gateway& operator*(); ++ Gateway* operator->(); ++ Gateway const* operator->() const; ++ Gateway* get(); ++ explicit operator bool() const; ++ void reset(); ++ ///@} ++ ++ // bool operator==(BusyGateway const&) const; ++ ++ /*! ++ * \brief Indicate that the task has successfully completed ++ * \param gws - a list that may be modified as a result ++ * \param api - usable access to the context ++ */ ++ void Success(Gateways& gws, std::shared_ptr api); ++ ++ /*! ++ * \brief Indicate that the task has unsuccessfully completed ++ * \param gws - a list that may be modified as a result ++ * \param api - usable access to the context ++ */ ++ void Failure(Gateways&, std::shared_ptr); ++ ++ /*! ++ * \brief What task does this refer to? ++ * \return Suffix to the URL being fetched ++ */ ++ std::string const& task() const { return spec_.suffix; } ++ ++ /*! ++ * \brief The full url intended to be fetched here ++ * \return get()->url_prefix() + task() ++ */ ++ std::string url() const { return prefix_ + task(); } ++ ++ std::string_view accept() const { return spec_.accept; } ++ ++ static void TestAccess(TestParams*); ++ std::shared_ptr srcreq; // TODO no ++ ++ private: ++ friend class Scheduler; ++ BusyGateway(std::string, UrlSpec, Scheduler*); ++ ++ std::string prefix_; ++ UrlSpec spec_; ++ raw_ptr scheduler_; ++ std::size_t maybe_offset_; ++}; ++ ++} // namespace ipfs ++ ++#endif +diff --git a/third_party/ipfs_client/include/ipfs_client/chained_requestors.h b/third_party/ipfs_client/include/ipfs_client/chained_requestors.h +new file mode 100644 +index 0000000000000..c9411fb52697e +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/chained_requestors.h +@@ -0,0 +1,45 @@ ++#ifndef CHAINED_REQUESTORS_H ++#define CHAINED_REQUESTORS_H ++ ++#include "block_requestor.h" ++ ++#include ++ ++namespace ipfs { ++ ++/*! ++ * \brief A BlockRequestor that chains together BlockRequestors ++ * \details It iterates through the chain returning the first successful hit. ++ */ ++class ChainedRequestors : public BlockRequestor { ++ public: ++ using Ptr = std::shared_ptr; ///< Owning pointer to the iface ++ ++ /*! ++ * \brief Append a requestor to the end of the chain ++ * \param p shared-ownership pointer to the requestor ++ */ ++ void Add(Ptr); ++ ++ /*! ++ * \brief Check if the chain appears valid ++ * \return non-zero number of requestors in the chain ++ */ ++ bool Valid() const; ++ ++ ChainedRequestors(); ++ ~ChainedRequestors() noexcept; ++ ++ private: ++ std::vector chain_; ++ ++ /*! ++ * \brief Implement BlockRequestor interface ++ */ ++ void RequestByCid(std::string, ++ std::shared_ptr, ++ Priority) override; ++}; ++} // namespace ipfs ++ ++#endif // CHAINED_REQUESTORS_H +diff --git a/third_party/ipfs_client/include/ipfs_client/context_api.h b/third_party/ipfs_client/include/ipfs_client/context_api.h +new file mode 100644 +index 0000000000000..139cf14cff59f +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/context_api.h +@@ -0,0 +1,69 @@ ++#ifndef IPFS_CONTEXT_API_H_ ++#define IPFS_CONTEXT_API_H_ ++ ++#include "busy_gateway.h" ++ ++#include ++#include ++ ++namespace ipfs { ++class IpfsRequest; ++ ++/*! ++ * \brief Represents an issued request ++ */ ++class GatewayRequest { ++ public: ++ BusyGateway gateway; ///< The assignment ++ ++ GatewayRequest(BusyGateway&&); ///< Move-only ++ virtual ~GatewayRequest() noexcept; ++ std::string task() const; ///< Same as gateway.task() ++ std::string url() const; ///< Same as gateway.url() ++}; ++ ++/** ++ * \brief Interface that provides functionality from whatever ++ * environment you're using this library in. ++ * \note A user of this library must implement this, but will probably do so ++ * only once. ++ */ ++class ContextApi : public std::enable_shared_from_this { ++ public: ++ ///< Send a single http request to its gateway as scheduled ++ virtual std::shared_ptr InitiateGatewayRequest( ++ BusyGateway) = 0; ++ ++ /*! ++ * \brief Determine a mime type for a given file. ++ * \param extension - "File extension" not including ., e.g. "html" ++ * \param content - The content of the resource or a large prefix thereof ++ * \param url - A URL it was fetched from (of any sort, ipfs:// is fine) ++ */ ++ virtual std::string MimeType(std::string extension, ++ std::string_view content, ++ std::string const& url) const = 0; ++ ++ /*! ++ * \brief Remove URL escaping, e.g. %20 ++ * \param url_comp - a single component of the URL, e.g. a element of the path ++ * not including / ++ * \return The unescaped string ++ */ ++ virtual std::string UnescapeUrlComponent(std::string_view url_comp) const = 0; ++ ++ /*! ++ * \brief Discover more gateways. ++ * \details The in-chromium implementation of this hits ++ * https://orchestrator.strn.pl/nodes/nearby ++ * Another implementation might kick off an mDNS probe? ++ * \param cb - A callback, called with a list of gateway ++ * prefixes, e.g. https://gateway.ipfs.io/ ++ * \note TRAILING SLASH (/) EXPECTED! ++ */ ++ virtual void Discover(std::function)> cb) = 0; ++}; ++ ++} // namespace ipfs ++ ++#endif +diff --git a/third_party/ipfs_client/include/ipfs_client/dag_block.h b/third_party/ipfs_client/include/ipfs_client/dag_block.h +new file mode 100644 +index 0000000000000..df29fe2f8e69f +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/dag_block.h +@@ -0,0 +1,146 @@ ++#ifndef IPFS_DAG_BLOCK_H_ ++#define IPFS_DAG_BLOCK_H_ ++ ++#if __has_include() ++#include ++#include ++#else ++#include "ipfs_client/pb_dag.pb.h" ++#include "ipfs_client/unix_fs.pb.h" ++#endif ++ ++#include ++#include ++#include ++ ++#include ++ ++#include ++#include ++ ++namespace ipfs { ++ ++using Cid = libp2p::multi::ContentIdentifier; ++ ++/*! ++ * \brief Something to which a CID may refer directly. ++ * \details A block may be "raw" - just a bunch of bytes. ++ * Or it could be an UnixFS-encoded node in a DAG ++ * Or it could be something else, like DAG-CBOR ++ * But this class really only handles the first 2 so far. ++ */ ++class Block { ++ public: ++ using Multicodec = libp2p::multi::MulticodecType::Code; ++ ++ /*! ++ * \brief Initialize from stream ++ * \param cid - The Content IDentifier ++ * \param stream - Stream from which one can read the bytes of the block ++ */ ++ Block(Cid const& cid, std::istream& stream); ++ ++ /*! ++ * \brief Initialize from block of bytes ++ * \param cid - The Content IDentifier ++ * \param bytes - The bytes to be interpreted as a maybe-node ++ * \note It's not really a string - certainly not text in any way. ++ * It's just a container of arbitrary bytes. ++ */ ++ Block(Cid const& cid, std::string const& bytes); ++ ++ Block(Block const&); ++ Block& operator=(Block const&) = default; ++ ++ Block(); ///< Construct an invalid block ++ ++ ~Block() noexcept; ++ ++ bool valid() const; ///< Check if the block appears valid ++ ++ /*! ++ * \brief The kinds of things a block may be representing ++ */ ++ enum class Type { ++ Raw, ++ Directory, ++ File, ++ Metadata, ++ Symlink, ++ HAMTShard, ++ FileChunk, ++ NonFs, ++ Invalid, ++ }; ++ ++ Type type() const; ///< Accessor for this block's type ++ ++ bool is_file() const; ///< type() == File || type() == FileChunk ++ ++ bool is_directory() const; ///< type() == Directory ++ ++ /*! ++ * \brief Get file size from the UnixFS node data ++ * \return zero if such data is not available, e.g. for raw or directory ++ */ ++ std::uint64_t file_size() const; ++ ++ std::string const& chunk_data() const; ///< data field from a UnixFS node ++ ++ std::string const& unparsed() const; ///< Original bytes (with protobuf bits) ++ ++ /*! ++ * \brief Accessor for all UnixFS data as a protobuf object ++ * \deprecated ++ */ ++ unix_fs::Data const& fsdata() const { return fsdata_; } ++ ++ Cid const& cid() const; ///< Getter for Content IDentifier ++ ++ void mime_type(std::string_view); ///< mime type setter - not managed ++ ++ std::string const& mime_type() const; ///< Getter for mime type ++ ++ bool cid_matches_data() const; ///< Basic validation ++ ++ std::vector binary_hash(libp2p::multi::HashType) const; ++ ++ /*! ++ * \brief Iterate through the links of this UnixFS node ++ * \param foo - Called for each link with (name, cid) ++ * should return converts-to-bool ++ * name is convertable from std::string const& ++ * cid is convertable from std::string&& ++ */ ++ template ++ void List(Functor foo) const { ++ for (auto& link : node_.links()) { ++ // protobuf uses string for binary data, too ++ auto hash = ipfs::ByteView{ ++ reinterpret_cast(link.hash().data()), ++ link.hash().size()}; ++ if (!foo(link.name(), LinkCid(hash))) { ++ break; ++ } ++ } ++ } ++ ++ private: ++ pb_dag::PBNode node_; ++ unix_fs::Data fsdata_; ++ bool valid_ = false; ++ bool fs_node_ = false; ++ std::string mime_ = {}; ++ std::optional cid_ = std::nullopt; ++ std::string original_bytes_; ++ ++ std::string LinkCid(ipfs::ByteView) const; ++ ++ void InitFromRaw(std::string const& content_bytes); ++}; ++ ++} // namespace ipfs ++ ++std::ostream& operator<<(std::ostream&, ipfs::Block::Type); ++ ++#endif // IPFS_DAG_BLOCK_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/dag_listener.h b/third_party/ipfs_client/include/ipfs_client/dag_listener.h +new file mode 100644 +index 0000000000000..6287abeaed535 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/dag_listener.h +@@ -0,0 +1,53 @@ ++#ifndef IPFS_DAG_LISTENER_H_ ++#define IPFS_DAG_LISTENER_H_ ++ ++#include ++#include ++#include ++ ++namespace ipfs { ++ ++/*! ++ * \brief Implemented by things waiting on a particular path through a DAG ++ */ ++class DagListener : public std::enable_shared_from_this { ++ public: ++ /*! ++ * \brief You're receiving the next N bytes of a requested file ++ * \details This is called once per block, which may be the entire file. ++ * \param bytes - the bytes you are receiving ++ */ ++ virtual void ReceiveBlockBytes(std::string_view bytes) = 0; ++ ++ /*! ++ * \brief The file you were receiving in ReceiveBlockBytes is done. ++ * \param mime_type - a guess at what the mime type of the file may be ++ */ ++ virtual void BlocksComplete(std::string mime_type) = 0; ++ ++ /*! ++ * \brief There was a general failure getting the blocks. ++ * \details Though used internally, when viewed externally this usually means ++ * you're requesting a CID which was simply not found on your gateways. ++ * \param cid - String representation of the root of the DAG ++ * \param path - The path that could not be resolved. ++ * \todo Unfortunately right now path is always the full requested path. ++ * If an intermediary directory failed, it should probably just be the ++ * path up to that point. And may in the future be so. ++ */ ++ virtual void NotHere(std::string_view cid, std::string_view path) = 0; ++ ++ /*! ++ * \brief The requested path DOES NOT EXIST, conclusively, regardless of gateway. ++ * \details Cannot occur without a path, as it indicates a directory (sharded or ++ * otherwise) was successfully retrieved but lacked the link necessary to ++ * descend farther down the path. ++ * \param cid - String representation of the root of the DAG ++ * \param path - The path that could not be resolved. ++ * \sa NotHere - the parameters carry the same meaning. ++ */ ++ virtual void DoesNotExist(std::string_view cid, std::string_view path) = 0; ++}; ++} // namespace ipfs ++ ++#endif +diff --git a/third_party/ipfs_client/include/ipfs_client/export.h b/third_party/ipfs_client/include/ipfs_client/export.h +new file mode 100644 +index 0000000000000..8161da33aca16 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/export.h +@@ -0,0 +1,16 @@ ++#ifndef IPFS_EXPORT_H_ ++#define IPFS_EXPORT_H_ ++ ++#if __has_include() ++#include ++#else ++ ++#ifndef IS_IPFS_IMPL ++#if !defined(COMPONENT_EXPORT) ++#define COMPONENT_EXPORT(IPFS) ++#endif ++#endif ++ ++#endif ++ ++#endif // IPFS_EXPORT_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/gateway.h b/third_party/ipfs_client/include/ipfs_client/gateway.h +new file mode 100644 +index 0000000000000..2693395365b0a +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/gateway.h +@@ -0,0 +1,82 @@ ++#ifndef CHROMIUM_IPFS_GATEWAY_H_ ++#define CHROMIUM_IPFS_GATEWAY_H_ ++ ++#include "url_spec.h" ++ ++#include "vocab/flat_mapset.h" ++ ++#include ++#include ++ ++namespace ipfs { ++ ++/*! ++ * \brief A known http gateway serving up blocks & IPNS records ++ */ ++class Gateway { ++ std::string prefix_; ++ flat_map failed_requests_; ++ unsigned priority_; /// This is not request priority. This is how eager we ++ /// are to use this particular gateway. ++ flat_set tasks_; ++ ++ public: ++ /*! ++ * \brief Construct a new gateway ++ * \param url_prefix - including at least full origin ++ * not including path parts like ipfs/ or ipns/ as those will be added ++ * It must end in a slash (/) ++ * \param priority - Performs 2 duties: ++ * * Relative to other gateways, how preferred it is for new requests. ++ * * How many concurrent requests before it's considered overloaded and ++ * completely ineligible. ++ * \sa [scoring](md_library_src_ipfs_client_scoring.html) ++ */ ++ Gateway(std::string url_prefix, unsigned priority); ++ Gateway(Gateway const&); ++ ~Gateway(); ++ ++ std::string const& url_prefix() const; ///< Get this gateway's identity ++ long load() const; ///< How busy is it currently with outstanding requests ++ ++ /*! ++ * \brief accept a new task ++ * \param suffix - Appended to url_prefix() provides the URL of the request ++ * \param need - How important is this task ++ * \return whether the task was accepted ++ */ ++ bool accept(UrlSpec const& req, long need); ++ ++ /*! ++ * \brief Indicate a task completed successfully ++ * \param ts - The suffix of the task ++ */ ++ void TaskSuccess(UrlSpec const& ts); ++ ++ /*! ++ * \brief Indicate a task completed unsuccessfully ++ * \param ts - The suffix of the task ++ */ ++ void TaskFailed(UrlSpec const&); ++ ++ /*! ++ * \brief Indicate a task was cancelled ++ * \param ts - The suffix of the task ++ */ ++ void TaskCancelled(UrlSpec const&); ++ ++ /*! ++ * \brief Whether this gateway has previously failed to serve that task ++ * \param suffix - The suffix of the task ++ */ ++ bool PreviouslyFailed(UrlSpec const&) const; ++ ++ /*! ++ * \return Whether this gateway should be generally preferred over another ++ */ ++ bool operator<(Gateway const&) const; ++}; ++ ++} // namespace ipfs ++ ++#endif // CHROMIUM_IPFS_GATEWAY_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/gateways.h b/third_party/ipfs_client/include/ipfs_client/gateways.h +new file mode 100644 +index 0000000000000..0b00f87fc67be +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/gateways.h +@@ -0,0 +1,58 @@ ++#ifndef CHROMIUM_IPFS_GATEWAYS_H_ ++#define CHROMIUM_IPFS_GATEWAYS_H_ ++ ++#include "export.h" ++#include "gateway.h" ++#include "vocab/flat_mapset.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace ipfs { ++using GatewayList = std::vector; ++class ContextApi; ++ ++/*! ++ * \brief All known IPFS gateways ++ */ ++class Gateways { ++ flat_map known_gateways_; ++ std::default_random_engine random_engine_; ++ std::geometric_distribution dist_; ++ int up_log_ = 1; ++ ++ public: ++ /*! ++ * \brief The hard-coded list of gateways at startup ++ */ ++ static std::vector> DefaultGateways(); ++ ++ Gateways(); ++ ~Gateways(); ++ GatewayList GenerateList(); ///< Get a sorted list of gateways for requesting ++ ++ /*! ++ * \brief Good gateway, handle more! ++ * \param prefix - identify the gateway by its URL prefix ++ */ ++ void promote(std::string const& prefix); ++ ++ /*! ++ * \brief Bad gateway, move toward the back of the line. ++ * \param prefix - identify the gateway by its URL prefix ++ */ ++ void demote(std::string const& prefix); ++ ++ /*! ++ * \brief Bulk load a bunch of new gateways ++ * \param prefices - list of URL gateways by prefix ++ */ ++ void AddGateways(std::vector prefices); ++}; ++} // namespace ipfs ++ ++#endif // CHROMIUM_IPFS_GATEWAYS_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/gw/gateway_request.h b/third_party/ipfs_client/include/ipfs_client/gw/gateway_request.h +new file mode 100644 +index 0000000000000..cf727d8c5c898 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/gw/gateway_request.h +@@ -0,0 +1,36 @@ ++#ifndef IPFS_TRUSTLESS_REQUEST_H_ ++#define IPFS_TRUSTLESS_REQUEST_H_ ++ ++#include ++ ++#include ++ ++#include ++#include ++#include ++ ++namespace ipfs { ++class IpfsRequest; ++class Orchestrator; ++} // namespace ipfs ++ ++namespace ipfs::gw { ++enum class Type { Block, Car, Ipns, DnsLink, Providers, Identity }; ++ ++struct GatewayRequest { ++ Type type; ++ std::string main_param; ///< CID, IPNS name, hostname ++ std::string path; ///< For CAR requests ++ std::shared_ptr dependent; ++ std::shared_ptr orchestrator; ++ std::optional cid; ++ ++ std::string url_suffix() const; ++ std::string_view accept() const; ++ std::string_view identity_data() const; ++ static std::shared_ptr fromIpfsPath(SlashDelimited); ++}; ++ ++} // namespace ipfs::gw ++ ++#endif // IPFS_TRUSTLESS_REQUEST_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/identity_cid.h b/third_party/ipfs_client/include/ipfs_client/identity_cid.h +new file mode 100644 +index 0000000000000..e142c56cc621d +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/identity_cid.h +@@ -0,0 +1,16 @@ ++#ifndef IPFS_IDENTITY_CID_H_ ++#define IPFS_IDENTITY_CID_H_ 1 ++ ++#include ++ ++#include ++ ++namespace ipfs { ++namespace id_cid { ++using Cid = libp2p::multi::ContentIdentifier; ++ ++Cid forText(std::string_view); ++} // namespace id_cid ++} // namespace ipfs ++ ++#endif +diff --git a/third_party/ipfs_client/include/ipfs_client/ipfs_request.h b/third_party/ipfs_client/include/ipfs_client/ipfs_request.h +new file mode 100644 +index 0000000000000..2ca22934e6078 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/ipfs_request.h +@@ -0,0 +1,26 @@ ++#ifndef IPFS_IPFS_REQUEST_H_ ++#define IPFS_IPFS_REQUEST_H_ ++ ++#include ++ ++#include ++#include ++ ++namespace ipfs { ++struct Response; ++class IpfsRequest { ++ public: ++ using Finisher = std::function; ++ ++ private: ++ std::string path_; ++ Finisher callback_; ++ ++ public: ++ IpfsRequest(std::string path, Finisher); ++ SlashDelimited path() const { return std::string_view{path_}; } ++ void finish(Response& r); ++}; ++} // namespace ipfs ++ ++#endif // IPFS_IPFS_REQUEST_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/ipld/dag_node.h b/third_party/ipfs_client/include/ipfs_client/ipld/dag_node.h +new file mode 100644 +index 0000000000000..10b41af3f925c +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/ipld/dag_node.h +@@ -0,0 +1,59 @@ ++#ifndef IPFS_DAG_NODE_H_ ++#define IPFS_DAG_NODE_H_ ++ ++#include "link.h" ++ ++#include ++#include "ipfs_client/response.h" ++ ++#include ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace ipfs { ++class Block; ++struct ValidatedIpns; ++} ++ ++namespace ipfs::ipld { ++ ++class DagNode; ++using NodePtr = std::shared_ptr; ++class DirShard; ++ ++struct MoreDataNeeded { ++ std::vector ipfs_abs_paths_; ++}; ++enum class ProvenAbsent {}; ++ ++using ResolveResult = std::variant; ++/** ++ * @brief A block, an IPNS record, etc. ++ */ ++class DagNode : public std::enable_shared_from_this { ++ protected: ++ std::vector> links_; ++ ++ public: ++ using BlockLookup = std::function; ++ virtual ResolveResult resolve(SlashDelimited path, ++ BlockLookup, ++ std::string& up_to_here) = 0; ++ ++ virtual NodePtr rooted(); ++ virtual NodePtr deroot(); ++ virtual DirShard* as_hamt(); // Wish I had access to dynamic_cast ++ ++ static NodePtr fromBlock(Block const&); ++ static NodePtr fromIpnsRecord(ValidatedIpns const&); ++}; ++} // namespace ipfs::ipld ++ ++#endif // IPFS_DAG_NODE_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/ipld/link.h b/third_party/ipfs_client/include/ipfs_client/ipld/link.h +new file mode 100644 +index 0000000000000..a0d290b25dd3d +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/ipld/link.h +@@ -0,0 +1,22 @@ ++#ifndef IPFS_LINK_H_ ++#define IPFS_LINK_H_ ++ ++#include ++#include ++ ++namespace ipfs::ipld { ++ ++class DagNode; ++using Ptr = std::shared_ptr; ++ ++class Link { ++ public: ++ std::string cid; ++ Ptr node; ++ ++ Link(std::string); ++ explicit Link(std::string, std::shared_ptr); ++}; ++} // namespace ipfs::ipld ++ ++#endif // IPFS_LINK_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/ipns_names.h b/third_party/ipfs_client/include/ipfs_client/ipns_names.h +new file mode 100644 +index 0000000000000..b611365b87874 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/ipns_names.h +@@ -0,0 +1,69 @@ ++#ifndef IPNS_NAME_RESOLVER_H_ ++#define IPNS_NAME_RESOLVER_H_ ++ ++#include ++#include ++ ++#include ++ ++namespace ipfs { ++ ++/*! ++ * \brief Fast synchronous access to IPNS & DNSLink name resolution ++ */ ++class IpnsNames { ++ flat_map names_; ++ ++ public: ++ IpnsNames(); ++ ~IpnsNames(); ++ ++ /*! ++ * \brief Get the already-known "value"/target of a given name ++ * \param name - either a mb-mf IPNS (key) name, or a host with DNSLink ++ * \return ++ * * if resolution is incomplete: "" ++ * * if it is known not to resolve: kNoSuchName ++ * * otherwise an IPFS path witout leading /, e.g.: ++ * - ipfs/bafybeicfqz46dj67nkhxaylqd5sknnidsr4oaw4hhsjrgdmcwt73sow2d4/ ++ * - ipns/k51qzi5uqu5dlvj2baxnqndepeb86cbk3ng7n3i46uzyxzyqj2xjonzllnv0v8 ++ */ ++ std::string_view NameResolvedTo(std::string_view name) const; ++ ++ /*! ++ * \brief Store an IPNS record that already validated for this name ++ * \param name - The name that resolves with this ++ * \param rec - The record modulo validation bits ++ */ ++ void AssignName(std::string const& name, ValidatedIpns rec); ++ ++ /*! ++ * \brief Assign a target path to a DNSLink host ++ * \param host - The original host NOT including a "_dnslink." prefix ++ * \param target - an IPFS path witout leading / ++ */ ++ void AssignDnsLink(std::string const& host, std::string_view target); ++ ++ /*! ++ * \brief Store the definitive absence of a resolution ++ * \details This is useful because code will check resolution here before ++ * trying to resolve it fresh again, and you can stop that if you know ++ * it will never work. ++ */ ++ void NoSuchName(std::string const& name); ++ ++ /*! ++ * \brief Fetch the all the stored IPNS record data ++ * \param name - the IPNS name it was stored with ++ * \return nullptr if missing, otherwise non-owning pointer to record ++ */ ++ ValidatedIpns const* Entry(std::string const& name); ++ ++ /*! ++ * \brief A special value constant ++ */ ++ static constexpr std::string_view kNoSuchName{"NO_SUCH_NAME"}; ++}; ++} // namespace ipfs ++ ++#endif // IPNS_NAME_RESOLVER_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/ipns_record.h b/third_party/ipfs_client/include/ipfs_client/ipns_record.h +new file mode 100644 +index 0000000000000..607c41d676fe5 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/ipns_record.h +@@ -0,0 +1,85 @@ ++#ifndef IPFS_IPNS_RECORD_H_ ++#define IPFS_IPNS_RECORD_H_ ++ ++#include ++ ++#if __has_include() ++#include ++#else ++#include "ipfs_client/keys.pb.h" ++#endif ++ ++#include ++#include ++ ++namespace libp2p::peer { ++class PeerId; ++} ++ ++namespace ipfs { ++ ++/*! ++ * \brief Parsed out data contained in the CBOR data of an IPNS record. ++ */ ++struct IpnsCborEntry { ++ std::string value; ///< The "value" (target) the name points at ++ std::string validity; ///< Value to compare for validity (i.e. expiration) ++ std::uint64_t validityType; ///< Way to deterimine current validity ++ std::uint64_t sequence; ///< Distinguish other IPNS records for the same name ++ std::uint64_t ttl; ///< Recommended caching time ++}; ++ ++using CborDeserializer = IpnsCborEntry(ByteView); ++ ++using CryptoSignatureVerifier = bool(ipns::KeyType, ++ ByteView, ++ ByteView, ++ ByteView); ++ ++std::optional ValidateIpnsRecord( ++ ByteView top_level_bytes, ++ libp2p::peer::PeerId const& name, ++ CryptoSignatureVerifier, ++ CborDeserializer); ++ ++/*! ++ * \brief Data from IPNS record modulo the verification parts ++ */ ++struct ValidatedIpns { ++ std::string value; ///< The path the record claims the IPNS name points to ++ std::time_t use_until; ///< An expiration timestamp ++ std::time_t cache_until; ///< Inspired by TTL ++ ++ /*! ++ * \brief The version of the record ++ * \details Higher sequence numbers obsolete lower ones ++ */ ++ std::uint64_t sequence; ++ std::int64_t resolution_ms; ///< How long it took to fetch the record ++ ++ /*! ++ * \brief When the record was fetched ++ */ ++ std::time_t fetch_time = std::time(nullptr); ++ std::string gateway_source; ///< Who gave us this record? ++ ++ ValidatedIpns(); ///< Create an invalid default object ++ ValidatedIpns(IpnsCborEntry const&); ++ ValidatedIpns(ValidatedIpns&&); ++ ValidatedIpns(ValidatedIpns const&); ++ ValidatedIpns& operator=(ValidatedIpns const&); ++ ++ std::string Serialize() const; ///< Turn into a well-defined list of bytes ++ ++ /*! ++ * \brief Create a ValidatedIpns from untyped bytes ++ * \param bytes - Output from a former call to Serialize() ++ * \note Is used by disk cache ++ * \return Recreation of the old object ++ */ ++ static ValidatedIpns Deserialize(std::string bytes); ++}; ++ ++} // namespace ipfs ++ ++#endif // IPFS_IPNS_RECORD_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/logger.h b/third_party/ipfs_client/include/ipfs_client/logger.h +new file mode 100644 +index 0000000000000..35191ac5f832c +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/logger.h +@@ -0,0 +1,34 @@ ++#ifndef IPFS_LOGGER_H_ ++#define IPFS_LOGGER_H_ ++ ++#include ++ ++namespace ipfs::log { ++ ++enum class Level { ++ TRACE = -2, ++ DEBUG = -1, ++ INFO = 0, ++ WARN = 1, ++ ERROR = 2, ++ FATAL = 3, ++ OFF ++}; ++ ++void SetLevel(Level); ++ ++using Handler = void (*)(std::string const&, char const*, int, Level); ++void SetHandler(Handler); ++ ++void DefaultHandler(std::string const& message, ++ char const* source_file, ++ int source_line, ++ Level for_prefix); ++ ++std::string_view LevelDescriptor(Level); ++ ++bool IsInitialized(); ++ ++} // namespace ipfs::log ++ ++#endif // LOGGER_H +diff --git a/third_party/ipfs_client/include/ipfs_client/name_listener.h b/third_party/ipfs_client/include/ipfs_client/name_listener.h +new file mode 100644 +index 0000000000000..fb307b8e5fd71 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/name_listener.h +@@ -0,0 +1,19 @@ ++#ifndef IPFS_NAME_LISTENER_H_ ++#define IPFS_NAME_LISTENER_H_ ++ ++#include ++ ++namespace ipfs { ++ ++/*! ++ * \brief Implemented by classes that await IPNS and/or DNSLink resolution ++ */ ++class NameListener : public std::enable_shared_from_this { ++ public: ++ ++ /// Called when available in IpnsNames ++ virtual void Complete() = 0; ++}; ++} // namespace ipfs ++ ++#endif // IPFS_NAME_LISTENER_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/orchestrator.h b/third_party/ipfs_client/include/ipfs_client/orchestrator.h +new file mode 100644 +index 0000000000000..716d8b6e22796 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/orchestrator.h +@@ -0,0 +1,34 @@ ++#ifndef IPFS_ORCHESTRATOR_H_ ++#define IPFS_ORCHESTRATOR_H_ ++ ++#include "ipfs_client/ipld/dag_node.h" ++ ++#include "vocab/flat_mapset.h" ++ ++#include ++#include ++ ++namespace ipfs { ++ ++class Orchestrator : public std::enable_shared_from_this { ++ flat_map dags_; ++ ++ public: ++ using GatewayAccess = ++ std::function)>; ++ using MimeDetection = std::function< ++ std::string(std::string, std::string_view, std::string const&)>; ++ explicit Orchestrator(GatewayAccess, MimeDetection); ++ void build_response(std::shared_ptr); ++ void add_node(std::string key, ipld::NodePtr); ++ ++ private: ++ GatewayAccess gw_requestor_; ++ MimeDetection mimer_; ++ void from_tree(std::shared_ptr, ipld::NodePtr&, SlashDelimited); ++ bool gw_request(std::shared_ptr, SlashDelimited path); ++ std::string sniff(SlashDelimited, std::string const&) const; ++}; ++} // namespace ipfs::ipld ++ ++#endif // IPFS_ORCHESTRATOR_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/response.h b/third_party/ipfs_client/include/ipfs_client/response.h +new file mode 100644 +index 0000000000000..cf5e146b6cb34 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/response.h +@@ -0,0 +1,21 @@ ++#ifndef IPFS_RESPONSE_H_ ++#define IPFS_RESPONSE_H_ ++ ++#include ++ ++#include ++ ++namespace ipfs { ++ ++struct Response { ++ std::string mime_; ++ std::uint16_t status_; ++ std::string body_; ++ std::string location_; ++ ++ static Response PLAIN_NOT_FOUND; ++}; ++ ++} // namespace ipfs ++ ++#endif // IPFS_RESPONSE_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/scheduler.h b/third_party/ipfs_client/include/ipfs_client/scheduler.h +new file mode 100644 +index 0000000000000..a85c165cf2e2d +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/scheduler.h +@@ -0,0 +1,96 @@ ++#ifndef IPFS_SCHEDULER_H_ ++#define IPFS_SCHEDULER_H_ ++ ++#include "block_requestor.h" ++#include "busy_gateway.h" ++#include "gateways.h" ++#include "url_spec.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace ipfs { ++class DagListener; ++class NameListener; ++class ContextApi; ++class Gateway; ++class GatewayRequest; ++class Scheduler; ++ ++/*! ++ * \brief Matchmaker for Gateway & Task ++ */ ++class Scheduler { ++ public: ++ /*! ++ * \brief Gateways to use. Sort order relevant. ++ */ ++ explicit Scheduler(std::function list_gen); ++ ~Scheduler(); ++ ++ /*! ++ * \brief Request a task be eventually handled ++ * \param api - General APIs that may be used during processing ++ * \param dl - Who's waiting on some bytes. Typically null for ipns requests. ++ * \param nl - Who's waiting on resolution. Typically null for ipfs requests. ++ * \param sfx - The end to the URL, mixed and matched with gateway prefixes. ++ * \param pio - How urgent is this request? ++ */ ++ void Enqueue(std::shared_ptr api, ++ std::shared_ptr dl, ++ std::shared_ptr nl, ++ std::string const& sfx, ++ std::string_view accept, ++ Priority pio, ++ std::shared_ptr top); ++ ++ /*! ++ * \brief Check enqueued requests to see if a gateway request should be made ++ * \param api - General APIs that may be used during processing ++ * \return Whether all enqueued requests are pending (false=saturated) ++ */ ++ bool IssueRequests(std::shared_ptr api); ++ ++ /*! ++ * \brief Check whether the set of gateways have ALL failed this task ++ * \param task - The task one might consider not enqueuing again. ++ * \return Whether the candidate gateway list is exhausted ++ */ ++ bool DetectCompleteFailure(UrlSpec const& task) const; ++ ++ /*! ++ * \brief Indicate this task has completed ++ * \param task - URL suffix of the task in question ++ */ ++ void TaskComplete(UrlSpec const& task); ++ ++ private: ++ std::function list_gen_; ++ friend class BusyGateway; ++ GatewayList gateways_; ++ struct Todo { ++ Todo(); ++ ~Todo(); ++ std::set> requests; ++ std::set> dag_listeners; ++ std::set> name_listeners; ++ std::set> source_reqs; ++ Priority priority; ++ long under_target() const; ++ }; ++ std::map task2todo_; ++ ++ void Issue(std::shared_ptr, ++ std::shared_ptr&, ++ std::vector todos, ++ unsigned up_to); ++ void CheckSwap(std::size_t); ++}; ++ ++} // namespace ipfs ++ ++#endif // IPFS_SCHEDULER_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/unixfs_path_resolver.h b/third_party/ipfs_client/include/ipfs_client/unixfs_path_resolver.h +new file mode 100644 +index 0000000000000..ac77811527e96 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/unixfs_path_resolver.h +@@ -0,0 +1,109 @@ ++#ifndef IPFS_UNIXFS_PATH_RESOLVER_H_ ++#define IPFS_UNIXFS_PATH_RESOLVER_H_ ++ ++#include "dag_block.h" ++#include "dag_listener.h" ++#include "scheduler.h" ++ ++#include ++#include ++#include ++ ++namespace ipfs { ++ ++class BlockRequestor; ++class BlockStorage; ++class ContextApi; ++class GeneratedDirectoryListing; ++ ++namespace unix_fs { ++class NodeHelper; ++} ++ ++/*! ++ * \brief Asynchronously descend a UnixFS dag given a root CID and a path. ++ * Get the file the combo refers to. ++ */ ++class UnixFsPathResolver { ++ public: ++ /*! ++ * \brief Create a new UnixFsPathResolver ++ * \param stor - block storage to fetch existing blocks from and into which ++ * new blocks are stored ++ * \param req - The object used to request blocks we don't have ++ * \param cid - The root CID of the dag in question ++ * \param path - The path down the dag being resolved ++ */ ++ UnixFsPathResolver(BlockStorage& stor, ++ BlockRequestor& req, ++ std::string cid, ++ std::string path, ++ std::shared_ptr api); ++ ~UnixFsPathResolver() noexcept; ++ ++ /*! ++ * \brief Attempt to move closer to resolution ++ * \param dl - The listener waiting for resolution ++ * \pre dl is not null ++ */ ++ void Step(std::shared_ptr dl); ++ ++ /*! ++ * \brief Fetch the listener previously passed to UnixFsPathResolver::Step ++ * \note This class does not manage or extend the lifetime. ++ * \return most recent listener, or null if not available ++ */ ++ std::shared_ptr MaybeGetPreviousListener(); ++ ++ /*! ++ * \brief The CID of the block currently being waited upon ++ * \return MB-MF string representation of the CID ++ */ ++ std::string const& current_cid() const; ++ ++ /*! ++ * \brief Simple getter ++ * \return the `path` argument to the constructor ++ */ ++ std::string const& original_path() const; ++ ++ /*! ++ * \brief Simple getter ++ * \return the current priority for generated requests ++ */ ++ Priority priority() const { return prio_; } ++ ++ /*! ++ * \brief All CIDs that were parsed to resolve the path ++ * \return list of MB-MF string representations of the CIDs ++ * \note The root CID may be from the constructor, but others are from links ++ * in the nodes traversed. They may be of various forms, including CIDv0. ++ */ ++ std::vector const& involved_cids() { return involved_cids_; } ++ ++ private: ++ BlockStorage& storage_; ++ BlockRequestor& requestor_; ++ std::string cid_; ++ std::string path_; ++ std::string original_path_; ++ std::shared_ptr api_; ++ std::unique_ptr helper_; ++ std::weak_ptr old_listener_; ++ Priority prio_ = 9; ++ struct PriorReq { ++ Priority prio; ++ std::time_t when; ++ }; ++ std::unordered_map already_requested_; ++ std::vector involved_cids_; ++ ++ void Request(std::shared_ptr&, std::string const& cid, Priority); ++ void GetHelper(Block::Type); ++ std::string pop_path(); ++ void AddInvolvedCid(std::string const&); ++}; ++ ++} // namespace ipfs ++ ++#endif // IPFS_UNIXFS_PATH_RESOLVER_H_ +diff --git a/third_party/ipfs_client/include/ipfs_client/url_spec.h b/third_party/ipfs_client/include/ipfs_client/url_spec.h +new file mode 100644 +index 0000000000000..a61aec25d5968 +--- /dev/null ++++ b/third_party/ipfs_client/include/ipfs_client/url_spec.h +@@ -0,0 +1,24 @@ ++#ifndef IPFS_URL_SPEC_H_ ++#define IPFS_URL_SPEC_H_ ++ ++// TODO - Give more thought to how this interplays with gw::Request ++ ++#include ++#include ++ ++namespace ipfs { ++struct UrlSpec { ++ std::string suffix; ++ std::string_view accept; ++ ++ bool operator<(UrlSpec const& rhs) const { ++ if (suffix != rhs.suffix) { ++ return suffix < rhs.suffix; ++ } ++ return accept < rhs.accept; ++ } ++ bool none() const { return suffix.empty(); } ++}; ++} // namespace ipfs ++ ++#endif // IPFS_URL_SPEC_H_ +diff --git a/third_party/ipfs_client/include/libp2p/basic/varint_prefix_reader.hpp b/third_party/ipfs_client/include/libp2p/basic/varint_prefix_reader.hpp +new file mode 100644 +index 0000000000000..70f484887725c +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/basic/varint_prefix_reader.hpp +@@ -0,0 +1,63 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_VARINT_PREFIX_READER_HPP ++#define LIBP2P_VARINT_PREFIX_READER_HPP ++ ++#include ++ ++namespace libp2p::basic { ++ ++/// Collects and interprets varint from incoming data, ++/// Reader is stateful, see varint_prefix_reader_test.cpp for usage examples ++class VarintPrefixReader { ++ public: ++ /// Current state ++ enum State { ++ /// Needs more bytes ++ kUnderflow, ++ ++ /// Varint is ready, value() is ultimate ++ kReady, ++ ++ /// Overflow of uint64_t, too many bytes with high bit set ++ kOverflow, ++ ++ /// consume() called when state is kReady ++ kError ++ }; ++ ++ /// Returns state ++ State state() const { return state_; } ++ ++ /// Returns current value, called when state() == kReady ++ uint64_t value() const { return value_; } ++ ++ /// Resets reader's state ++ void reset(); ++ ++ /// Consumes one byte from wire, returns reader's state ++ /// (or kError if called when state() == kReady) ++ State consume(uint8_t byte); ++ ++ /// Consumes bytes from buffer. ++ /// On success, modifies buffer (cuts off first bytes which were consumed), ++ /// returns reader's state ++ /// (or kError if called when state() == kReady) ++ State consume(ipfs::ByteView& buffer); ++ ++ private: ++ /// Current value accumulated ++ uint64_t value_ = 0; ++ ++ /// Current reader's state ++ State state_ = kUnderflow; ++ ++ /// Bytes got at the moment, this controls overflow of value_ ++ uint8_t got_bytes_ = 0; ++}; ++} // namespace libp2p::basic ++ ++#endif // LIBP2P_VARINT_PREFIX_READER_HPP +diff --git a/third_party/ipfs_client/include/libp2p/common/hexutil.hpp b/third_party/ipfs_client/include/libp2p/common/hexutil.hpp +new file mode 100644 +index 0000000000000..6dc63ecbb3e0c +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/common/hexutil.hpp +@@ -0,0 +1,106 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_HEXUTIL_HPP ++#define LIBP2P_HEXUTIL_HPP ++ ++#include "vocab/byte_view.h" ++#include "vocab/expected.h" ++ ++#include ++#include ++ ++namespace libp2p::common { ++ ++/** ++ * @brief error codes for exceptions that may occur during unhexing ++ */ ++enum class UnhexError { NOT_ENOUGH_INPUT = 1, NON_HEX_INPUT, UNKNOWN }; ++ ++/** ++ * @brief Converts an integer to an uppercase hex representation ++ */ ++std::string int_to_hex(uint64_t n, size_t fixed_width = 2) noexcept; ++ ++/** ++ * @brief Converts bytes to uppercase hex representation ++ * @param array bytes ++ * @param len length of bytes ++ * @return hexstring ++ */ ++std::string hex_upper(ipfs::ByteView bytes) noexcept; ++ ++/** ++ * @brief Converts bytes to hex representation ++ * @param array bytes ++ * @param len length of bytes ++ * @return hexstring ++ */ ++std::string hex_lower(ipfs::ByteView bytes) noexcept; ++ ++/** ++ * @brief Converts hex representation to bytes ++ * @param array individual chars ++ * @param len length of chars ++ * @return result containing array of bytes if input string is hex encoded and ++ * has even length ++ * ++ * @note reads both uppercase and lowercase hexstrings ++ * ++ * @see ++ * https://www.boost.org/doc/libs/1_51_0/libs/algorithm/doc/html/the_boost_algorithm_library/Misc/hex.html ++ */ ++ipfs::expected, UnhexError> unhex(std::string_view hex); ++ ++/** ++ * Creates unsigned bytes span out of string, debug purpose helper ++ * @param str String ++ * @return Span ++ */ ++inline ipfs::ByteView sv2span(const std::string_view& str) { ++ return ipfs::ByteView((ipfs::Byte*)str.data(), // NOLINT ++ str.size()); // NOLINT ++} ++ ++/** ++ * sv2span() identity overload, for uniformity ++ * @param s Span ++ * @return s ++ */ ++inline ipfs::ByteView sv2span(const ipfs::ByteView& s) { ++ return s; ++} ++ ++/** ++ * Makes printable string out of bytes, for diagnostic purposes ++ * @tparam Bytes Bytes or char sequence ++ * @param str Input ++ * @return String ++ */ ++template ++inline std::string dumpBin(const Bytes& str) { ++ std::string ret; ++ ret.reserve(str.size() + 2); ++ bool non_printable_detected = false; ++ for (auto c : str) { ++ if (std::isprint(c) != 0) { ++ ret.push_back((char)c); // NOLINT ++ } else { ++ ret.push_back('?'); ++ non_printable_detected = true; ++ } ++ } ++ if (non_printable_detected) { ++ ret.reserve(ret.size() * 3); ++ ret += " ("; ++ ret += hex_lower(sv2span(str)); ++ ret += ')'; ++ } ++ return ret; ++} ++ ++} // namespace libp2p::common ++ ++#endif // LIBP2P_HEXUTIL_HPP +diff --git a/third_party/ipfs_client/include/libp2p/common/types.hpp b/third_party/ipfs_client/include/libp2p/common/types.hpp +new file mode 100644 +index 0000000000000..a112d1bf5d3db +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/common/types.hpp +@@ -0,0 +1,39 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_P2P_COMMON_TYPES_HPP ++#define LIBP2P_P2P_COMMON_TYPES_HPP ++ ++#include "vocab/byte_view.h" ++ ++#include ++#include ++#include ++#include ++ ++namespace libp2p::common { ++/** ++ * Sequence of bytes ++ */ ++using ByteArray = std::vector; ++// using ByteArray = std::string; ++ ++template ++void append(Collection& c, Item&& g) { ++ c.insert(c.end(), g.begin(), g.end()); ++} ++ ++template ++void append(Collection& c, char g) { ++ c.push_back(g); ++} ++ ++/// Hash256 as a sequence of 32 bytes ++using Hash256 = std::array; ++/// Hash512 as a sequence of 64 bytes ++using Hash512 = std::array; ++} // namespace libp2p::common ++ ++#endif // LIBP2P_P2P_COMMON_TYPES_HPP +diff --git a/third_party/ipfs_client/include/libp2p/crypto/common.hpp b/third_party/ipfs_client/include/libp2p/crypto/common.hpp +new file mode 100644 +index 0000000000000..fd4158b8c82d2 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/crypto/common.hpp +@@ -0,0 +1,56 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_CRYPTO_COMMON_HPP ++#define LIBP2P_CRYPTO_COMMON_HPP ++ ++#include ++#include ++#include ++ ++namespace libp2p::crypto::common { ++ /** ++ * @struct AesSecret provides key and iv for AES cipher ++ * @tparam KeySize key size ++ * @tparam IvSize iv size ++ */ ++ template ++ struct AesSecret { ++ std::array key; ++ std::array iv; ++ }; ++ ++ /** ++ * Values for AES-128 ++ */ ++ using Aes128Secret = AesSecret<16, 16>; ++ ++ /** ++ * Values for AES-256 ++ */ ++ using Aes256Secret = AesSecret<32, 16>; ++ ++ /** ++ * Supported hash types ++ */ ++ enum class HashType { SHA1, SHA256, SHA512 }; ++ ++ /** ++ * Supported types of RSA keys ++ */ ++ enum class RSAKeyType { RSA1024, RSA2048, RSA4096 }; ++ ++ /** ++ * Supported ECDH curves ++ */ ++ enum class CurveType { P256, P384, P521 }; ++ ++ /** ++ * Supported cipher types ++ */ ++ enum class CipherType { AES128, AES256 }; ++} // namespace libp2p::crypto::common ++ ++#endif // LIBP2P_CRYPTO_COMMON_HPP +diff --git a/third_party/ipfs_client/include/libp2p/crypto/error.hpp b/third_party/ipfs_client/include/libp2p/crypto/error.hpp +new file mode 100644 +index 0000000000000..5067ba85d9a61 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/crypto/error.hpp +@@ -0,0 +1,92 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_CRYPTO_ERROR_HPP ++#define LIBP2P_CRYPTO_ERROR_HPP ++ ++#include ++ ++namespace libp2p::crypto { ++enum class CryptoProviderError { ++ INVALID_KEY_TYPE = 1, ///< failed to unmarshal key type, wrong value ++ UNKNOWN_KEY_TYPE, ///< failed to unmarshal key ++ FAILED_UNMARSHAL_DATA, ///< protobuf error, failed to unmarshal data ++ SIGNATURE_GENERATION_FAILED, ///< general signature creation failure ++ SIGNATURE_VERIFICATION_FAILED, ///< general signature verification failure ++}; ++ ++enum class OpenSslError { ++ FAILED_INITIALIZE_CONTEXT = 1, ///< failed to initialize context ++ FAILED_INITIALIZE_OPERATION, ///< failed to initialize operation ++ FAILED_ENCRYPT_UPDATE, ///< failed to update encryption ++ FAILED_DECRYPT_UPDATE, ///< failed to update decryption ++ FAILED_ENCRYPT_FINALIZE, ///< failed to finalize encryption ++ FAILED_DECRYPT_FINALIZE, ///< failed to finalize decryption ++ WRONG_IV_SIZE, ///< wrong iv size ++ WRONG_KEY_SIZE, ///< wrong key size ++ STREAM_FINALIZED, ///< crypt update operations cannot be performed after ++ ///< stream finalization ++}; ++ ++enum class HmacProviderError { ++ UNSUPPORTED_HASH_METHOD = 1, ///< hash method id provided is not supported ++ FAILED_CREATE_CONTEXT, ///< failed to create context ++ FAILED_INITIALIZE_CONTEXT, ///< failed to initialize context ++ FAILED_UPDATE_DIGEST, ///< failed to update digest ++ FAILED_FINALIZE_DIGEST, ///< failed to finalize digest ++ WRONG_DIGEST_SIZE, ///< wrong digest size ++}; ++ ++enum class RandomProviderError { ++ TOKEN_NOT_EXISTS = 1, ///< /dev/urandom path not present in system ++ FAILED_OPEN_FILE, ///< failed to open random provider file ++ FAILED_FETCH_BYTES, ///< failed to fetch bytes from source ++ INVALID_PROVIDER_TYPE, ///< invalid or unsupported random provider type ++}; ++ ++enum class KeyGeneratorError { ++ CANNOT_GENERATE_UNSPECIFIED = 1, ///< you need to specify valid key type ++ UNKNOWN_KEY_TYPE, ///< unknown key type ++ GENERATOR_NOT_INITIALIZED, ///< generator not initialized ++ KEY_GENERATION_FAILED, ///< key generation failed ++ KEY_DERIVATION_FAILED, ///< failed to derive key ++ FILE_NOT_FOUND, ///< file not found ++ FAILED_TO_READ_FILE, ///< failed to read file ++ INCORRECT_BITS_COUNT, ///< incorrect bits option ++ UNSUPPORTED_KEY_TYPE, ///< key type is not supported ++ WRONG_KEY_TYPE, ///< incorrect key type ++ CANNOT_LOAD_UNSPECIFIED, ///< cannot load unspecified key ++ GET_KEY_BYTES_FAILED, ///< failed to get key bytes from PKEY ++ INTERNAL_ERROR, ///< internal error happened ++ UNSUPPORTED_CURVE_TYPE, ///< generator for curve type is not implemented ++}; ++ ++enum class KeyValidatorError { ++ WRONG_PUBLIC_KEY_SIZE = 1, ///< public key has wrong size ++ WRONG_PRIVATE_KEY_SIZE, ///< private key has wrong size ++ INVALID_PUBLIC_KEY, ///< invalid public key ++ INVALID_PRIVATE_KEY, ///< private key cannot be decoded ++ KEYS_DONT_MATCH, ///< public key is not derived from private ++ DIFFERENT_KEY_TYPES, ///< key types in pair don't match ++ UNSUPPORTED_CRYPTO_ALGORYTHM, ///< crypto algorythm is not implemented ++ UNKNOWN_CRYPTO_ALGORYTHM, ///< unknown crypto algorythm ++}; ++ ++using Error = std::variant; ++} // namespace libp2p::crypto ++ ++// OUTCOME_HPP_DECLARE_ERROR(libp2p::crypto, CryptoProviderError) ++// OUTCOME_HPP_DECLARE_ERROR(libp2p::crypto, OpenSslError) ++// OUTCOME_HPP_DECLARE_ERROR(libp2p::crypto, HmacProviderError) ++// OUTCOME_HPP_DECLARE_ERROR(libp2p::crypto, RandomProviderError) ++// OUTCOME_HPP_DECLARE_ERROR(libp2p::crypto, KeyGeneratorError) ++// OUTCOME_HPP_DECLARE_ERROR(libp2p::crypto, KeyValidatorError) ++ ++#endif // LIBP2P_CRYPTO_ERROR_HPP +diff --git a/third_party/ipfs_client/include/libp2p/crypto/hasher.hpp b/third_party/ipfs_client/include/libp2p/crypto/hasher.hpp +new file mode 100644 +index 0000000000000..a194296b27a03 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/crypto/hasher.hpp +@@ -0,0 +1,55 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_SRC_CRYPTO_HASHER_HPP ++#define LIBP2P_SRC_CRYPTO_HASHER_HPP ++ ++#include ++#include ++#include ++ ++#include "vocab/byte_view.h" ++#include "vocab/expected.h" ++ ++namespace libp2p::crypto { ++ ++ using libp2p::crypto::common::HashType; ++ ++ class Hasher { ++ public: ++ virtual ~Hasher() = default; ++ ++ using Result = ipfs::expected; ++ ++ /// appends a new chunk of data ++ virtual Result write(ipfs::ByteView data) = 0; ++ ++ /** ++ * Calculates the current digest. ++ * Does not affect the internal state. ++ * New data still could be fed via write method. ++ */ ++ virtual Result digestOut(ipfs::span out) const = 0; ++ ++ /// resets the internal state ++ virtual Result reset() = 0; ++ ++ /// hash size in bytes ++ virtual size_t digestSize() const = 0; ++ ++ /// block size in bytes for the most optimal hash update via write method ++ virtual size_t blockSize() const = 0; ++ ++ /// runtime identifiable hasher type ++ virtual HashType hashType() const = 0; ++ ++ ipfs::expected, Error> digest() const; ++ }; ++ ++ std::unique_ptr CreateHasher(multi::HashType); ++ ++} // namespace libp2p::crypto ++ ++#endif // LIBP2P_SRC_CRYPTO_HASHER_HPP +diff --git a/third_party/ipfs_client/include/libp2p/crypto/key.h b/third_party/ipfs_client/include/libp2p/crypto/key.h +new file mode 100644 +index 0000000000000..8198e41122fdd +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/crypto/key.h +@@ -0,0 +1,100 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_LIBP2P_CRYPTO_KEY_HPP ++#define LIBP2P_LIBP2P_CRYPTO_KEY_HPP ++ ++#include ++ ++#include "libp2p/common/types.hpp" ++ ++namespace libp2p::crypto { ++ ++using Buffer = libp2p::common::ByteArray; ++ ++struct Key { ++ /** ++ * Supported types of all keys ++ */ ++ enum class Type { ++ UNSPECIFIED = 100, ++ RSA = 0, ++ Ed25519 = 1, ++ Secp256k1 = 2, ++ ECDSA = 3 ++ }; ++ ++ Key(Type, std::vector); ++ ~Key() noexcept; ++ Type type = Type::UNSPECIFIED; ///< key type ++ std::vector data{}; ///< key content ++}; ++ ++inline bool operator==(const Key& lhs, const Key& rhs) { ++ return lhs.type == rhs.type && lhs.data == rhs.data; ++} ++ ++inline bool operator!=(const Key& lhs, const Key& rhs) { ++ return !(lhs == rhs); ++} ++ ++struct PublicKey : public Key {}; ++ ++struct PrivateKey : public Key {}; ++ ++struct KeyPair { ++ PublicKey publicKey; ++ PrivateKey privateKey; ++}; ++ ++using Signature = std::vector; ++ ++inline bool operator==(const KeyPair& a, const KeyPair& b) { ++ return a.publicKey == b.publicKey && a.privateKey == b.privateKey; ++} ++ ++/** ++ * Result of ephemeral key generation ++ * ++struct EphemeralKeyPair { ++ Buffer ephemeral_public_key; ++ std::function(Buffer)> shared_secret_generator; ++}; ++*/ ++ ++/** ++ * Type of the stretched key ++ * ++struct StretchedKey { ++ Buffer iv; ++ Buffer cipher_key; ++ Buffer mac_key; ++}; ++*/ ++} // namespace libp2p::crypto ++ ++namespace std { ++template <> ++struct hash { ++ size_t operator()(const libp2p::crypto::Key& x) const; ++}; ++ ++template <> ++struct hash { ++ size_t operator()(const libp2p::crypto::PrivateKey& x) const; ++}; ++ ++template <> ++struct hash { ++ size_t operator()(const libp2p::crypto::PublicKey& x) const; ++}; ++ ++template <> ++struct hash { ++ size_t operator()(const libp2p::crypto::KeyPair& x) const; ++}; ++} // namespace std ++ ++#endif // LIBP2P_LIBP2P_CRYPTO_KEY_HPP +diff --git a/third_party/ipfs_client/include/libp2p/crypto/protobuf/protobuf_key.hpp b/third_party/ipfs_client/include/libp2p/crypto/protobuf/protobuf_key.hpp +new file mode 100644 +index 0000000000000..1a0d7ae7a2d4e +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/crypto/protobuf/protobuf_key.hpp +@@ -0,0 +1,29 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef KAGOME_PROTOBUF_KEY_HPP ++#define KAGOME_PROTOBUF_KEY_HPP ++ ++// #include ++ ++#include ++ ++#include ++ ++namespace libp2p::crypto { ++/** ++ * Strict type for key, which is encoded into Protobuf format ++ */ ++struct ProtobufKey { //: public boost::equality_comparable { ++ explicit ProtobufKey(std::vector key); ++ ~ProtobufKey() noexcept; ++ ++ std::vector key; ++ ++ bool operator==(const ProtobufKey& other) const { return key == other.key; } ++}; ++} // namespace libp2p::crypto ++ ++#endif // KAGOME_PROTOBUF_KEY_HPP +diff --git a/third_party/ipfs_client/include/libp2p/crypto/sha/sha256.hpp b/third_party/ipfs_client/include/libp2p/crypto/sha/sha256.hpp +new file mode 100644 +index 0000000000000..edff50eeac24b +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/crypto/sha/sha256.hpp +@@ -0,0 +1,48 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_SHA256_HPP ++#define LIBP2P_SHA256_HPP ++ ++#include ++#include ++#include "libp2p/common/types.hpp" ++ ++namespace libp2p::crypto { ++ ++class Sha256 : public Hasher { ++ public: ++ Sha256(); ++ ++ ~Sha256() override; ++ ++ Result write(ipfs::ByteView data) override; ++ ++ Result digestOut(ipfs::span out) const override; ++ ++ Result reset() override; ++ ++ size_t digestSize() const override; ++ ++ size_t blockSize() const override; ++ ++ HashType hashType() const override; ++ ++ private: ++ void sinkCtx(); ++ ++ SHA256_CTX ctx_; ++ bool initialized_; ++}; ++ ++/** ++ * Take a SHA-256 hash from bytes ++ * @param input to be hashed ++ * @return hashed bytes ++ */ ++ipfs::expected sha256(ipfs::ByteView input); ++} // namespace libp2p::crypto ++ ++#endif // LIBP2P_SHA256_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/content_identifier.hpp b/third_party/ipfs_client/include/libp2p/multi/content_identifier.hpp +new file mode 100644 +index 0000000000000..c29eb7c54b090 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/content_identifier.hpp +@@ -0,0 +1,52 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_CONTENT_IDENTIFIER_HPP ++#define LIBP2P_CONTENT_IDENTIFIER_HPP ++ ++#include ++ ++#include ++#include ++#include ++ ++namespace libp2p::multi { ++ ++/** ++ * A CID is a self-describing content-addressed identifier. It uses ++ * cryptographic hashes to achieve content addressing. It uses several ++ * multiformats to achieve flexible self-description, namely multihash for ++ * hashes, multicodec for data content types, and multibase to encode the CID ++ * itself into strings. Concretely, it's a typed content address: a tuple of ++ * (content-type, content-address). ++ * ++ * @note multibase may be omitted in non text-based protocols and is generally ++ * needed only for CIDs serialized to a string, so it is not present in this ++ * structure ++ */ ++struct ContentIdentifier { ++ enum class Version { V0 = 0, V1 = 1 }; ++ ++ ContentIdentifier(Version version, ++ MulticodecType::Code content_type, ++ Multihash content_address); ++ ++ /** ++ * @param base is a human-readable multibase prefix ++ * @returns human readable representation of the CID ++ */ ++ std::string toPrettyString(const std::string& base) const; ++ ++ bool operator==(const ContentIdentifier& c) const; ++ bool operator<(const ContentIdentifier& c) const; ++ ++ Version version; ++ MulticodecType::Code content_type; ++ Multihash content_address; ++}; ++ ++} // namespace libp2p::multi ++ ++#endif // LIBP2P_CONTENT_IDENTIFIER_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/content_identifier_codec.hpp b/third_party/ipfs_client/include/libp2p/multi/content_identifier_codec.hpp +new file mode 100644 +index 0000000000000..623b363ba3de6 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/content_identifier_codec.hpp +@@ -0,0 +1,99 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_CONTENT_IDENTIFIER_CODEC_HPP ++#define LIBP2P_CONTENT_IDENTIFIER_CODEC_HPP ++ ++#include ++#include ++ ++namespace libp2p::multi { ++ ++/** ++ * Serializes and deserializes CID to byte representation. ++ * To serialize it to a multibase encoded string, use MultibaseCodec ++ * @see MultibaseCodec ++ */ ++class ContentIdentifierCodec { ++ public: ++ enum class EncodeError { ++ INVALID_CONTENT_TYPE = 1, ++ INVALID_HASH_TYPE, ++ INVALID_HASH_LENGTH, ++ VERSION_UNSUPPORTED, ++ INVALID_BASE_ENCODING, ++ }; ++ ++ enum class DecodeError { ++ EMPTY_VERSION = 1, ++ EMPTY_MULTICODEC, ++ MALFORMED_VERSION, ++ RESERVED_VERSION, ++ CID_TOO_SHORT, ++ BAD_MULTIHASH, ++ BAD_MULTIBASE, ++ UNSUPPORTED_MULTIBASE ++ }; ++ ++ static ipfs::expected, EncodeError> encode( ++ const ContentIdentifier& cid); ++ ++ /// Encodes arbitrary byte buffer into CID v.0 wire format ++ static std::vector encodeCIDV0(const void* byte_buffer, ++ size_t sz); ++ ++ /// Encodes arbitrary byte buffer into CID v.1 wire format ++ static std::vector encodeCIDV1(MulticodecType::Code content_type, ++ const Multihash& mhash); ++ ++ static ipfs::expected decode( ++ ipfs::ByteView bytes); ++ ++ /** ++ * @brief Encode CID to string representation ++ * @param cid - input CID for encode ++ * Encoding: ++ * Base58 for CID v0 ++ * Base32 for CID v1 ++ * @return CID string ++ */ ++ static ipfs::expected toString( ++ const ContentIdentifier& cid); ++ ++ /** ++ * @brief Encode CID to string representation ++ * @param cid - input CID for encode ++ * @param base - type of the encoding ++ * @return CID string ++ */ ++ static ipfs::expected toStringOfBase( ++ const ContentIdentifier& cid, ++ MultibaseCodec::Encoding base); ++ ++ /** ++ * @brief Decode string representation of CID to CID ++ * @param str - CID string representation ++ * @return CID ++ */ ++ static ipfs::expected fromString( ++ const std::string& str); ++}; ++ ++std::string_view Stringify(libp2p::multi::ContentIdentifierCodec::DecodeError); ++ ++} // namespace libp2p::multi ++ ++std::ostream& operator<<(std::ostream&, ++ libp2p::multi::ContentIdentifierCodec::EncodeError); ++std::ostream& operator<<(std::ostream&, ++ libp2p::multi::ContentIdentifierCodec::DecodeError); ++ ++#include ++// OUTCOME_HPP_DECLARE_ERROR(libp2p::multi, ++// ContentIdentifierCodec::EncodeError); ++// OUTCOME_HPP_DECLARE_ERROR(libp2p::multi, ++// ContentIdentifierCodec::DecodeError); ++ ++#endif // LIBP2P_CONTENT_IDENTIFIER_CODEC_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/hash_type.hpp b/third_party/ipfs_client/include/libp2p/multi/hash_type.hpp +new file mode 100644 +index 0000000000000..fdb863c23c73e +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/hash_type.hpp +@@ -0,0 +1,30 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_HASH_TYPE_HPP ++#define LIBP2P_HASH_TYPE_HPP ++ ++#include ++ ++namespace libp2p::multi { ++ /// TODO(Harrm) FIL-14: Hash types are a part of multicodec table, it would be ++ /// good to move them there to avoid duplication and allow for extraction of ++ /// human-friendly name of a type from its code ++ /// @see MulticodecType ++ /// https://github.com/multiformats/js-multihash/blob/master/src/constants.js ++ enum class HashType : std::uint64_t { ++ identity = 0x0, ++ sha1 = 0x11, ++ sha256 = 0x12, ++ sha512 = 0x13, ++ blake2b_256 = 0xb220, ++ blake2s128 = 0xb250, ++ blake2s256 = 0xb260, ++ sha2_256_trunc254_padded = 0x1012, ++ poseidon_bls12_381_a1_fc1 = 0xb401, ++ }; ++} // namespace libp2p::multi ++ ++#endif // LIBP2P_HASH_TYPE_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/multibase_codec.hpp b/third_party/ipfs_client/include/libp2p/multi/multibase_codec.hpp +new file mode 100644 +index 0000000000000..c7b9cbd1f7d40 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/multibase_codec.hpp +@@ -0,0 +1,65 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_MULTIBASE_HPP ++#define LIBP2P_MULTIBASE_HPP ++ ++#include "vocab/expected.h" ++ ++#include ++#include ++#include ++ ++#include ++ ++namespace libp2p::multi { ++/** ++ * Allows to distinguish between different base-encoded binaries ++ * See more: https://github.com/multiformats/multibase ++ */ ++class MultibaseCodec { ++ public: ++ enum class Error { UNSUPPORTED_BASE = 1, INPUT_TOO_SHORT, BASE_CODEC_ERROR }; ++ ++ using ByteBuffer = common::ByteArray; ++ using FactoryResult = ipfs::expected; ++ ++ virtual ~MultibaseCodec() = default; ++ /** ++ * Encodings, supported by this Multibase ++ * @sa https://github.com/multiformats/multibase#multibase-table ++ */ ++ enum class Encoding : char { ++ BASE16_LOWER = 'f', ++ BASE16_UPPER = 'F', ++ BASE32_LOWER = 'b', ++ BASE32_UPPER = 'B', ++ BASE36 = 'k', ++ BASE58 = 'z', ++ BASE64 = 'm' ++ }; ++ ++ /** ++ * Encode the incoming bytes ++ * @param bytes to be encoded ++ * @param encoding - base of the desired encoding ++ * @return encoded string WITH an encoding prefix ++ */ ++ virtual std::string encode(const ByteBuffer& bytes, ++ Encoding encoding) const = 0; ++ ++ /** ++ * Decode the incoming string ++ * @param string to be decoded ++ * @return bytes, if decoding was successful, error otherwise ++ */ ++ virtual FactoryResult decode(std::string_view string) const = 0; ++}; ++ ++bool case_critical(MultibaseCodec::Encoding); ++ ++} // namespace libp2p::multi ++ ++#endif // LIBP2P_MULTIBASE_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base16.h b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base16.h +new file mode 100644 +index 0000000000000..72a74237eb2ee +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base16.h +@@ -0,0 +1,24 @@ ++#ifndef IPFS_BASE32_H_ ++#define IPFS_BASE32_H_ ++ ++#include "base_error.hpp" ++ ++#include ++#include ++ ++#include ++ ++#include ++ ++namespace ipfs::base16 { ++std::string encodeLower(ByteView bytes); ++std::string encodeUpper(ByteView bytes); ++ ++using libp2p::common::ByteArray; ++using libp2p::multi::detail::BaseError; ++using Decoded = ipfs::expected; ++Decoded decode(std::string_view string); ++ ++} // namespace ipfs::base16 ++ ++#endif // IPFS_BASE32_H_ +diff --git a/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base32.hpp b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base32.hpp +new file mode 100644 +index 0000000000000..c24dc59d54121 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base32.hpp +@@ -0,0 +1,52 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_BASE32_HPP ++#define LIBP2P_BASE32_HPP ++ ++#include "base_error.hpp" ++ ++#include ++#include ++ ++/** ++ * Encode/decode to/from base32 format ++ * Implementation is taken from ++ * https://github.com/mjg59/tpmtotp/blob/master/base32.c ++ */ ++namespace libp2p::multi::detail { ++ ++/** ++ * Encode bytes to base32 uppercase string ++ * @param bytes to be encoded ++ * @return encoded string ++ */ ++std::string encodeBase32Upper(ipfs::ByteView bytes); ++/** ++ * Encode bytes to base32 lowercase string ++ * @param bytes to be encoded ++ * @return encoded string ++ */ ++std::string encodeBase32Lower(ipfs::ByteView bytes); ++ ++/** ++ * Decode base32 uppercase to bytes ++ * @param string to be decoded ++ * @return decoded bytes in case of success ++ */ ++ipfs::expected decodeBase32Upper( ++ std::string_view string); ++ ++/** ++ * Decode base32 lowercase string to bytes ++ * @param string to be decoded ++ * @return decoded bytes in case of success ++ */ ++ipfs::expected decodeBase32Lower( ++ std::string_view string); ++ ++} // namespace libp2p::multi::detail ++ ++#endif // LIBP2P_BASE32_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base36.hpp b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base36.hpp +new file mode 100644 +index 0000000000000..20006df216d51 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base36.hpp +@@ -0,0 +1,42 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_BASE36_HPP ++#define LIBP2P_BASE36_HPP ++ ++#include "base_error.hpp" ++ ++#include ++#include ++ ++/** ++ * Encode/decode to/from base36 format ++ */ ++namespace libp2p::multi::detail { ++ ++/** ++ * Encode bytes to base36 uppercase string ++ * @param bytes to be encoded ++ * @return encoded string ++ */ ++std::string encodeBase36Upper(ipfs::ByteView bytes); ++/** ++ * Encode bytes to base36 lowercase string ++ * @param bytes to be encoded ++ * @return encoded string ++ */ ++std::string encodeBase36Lower(ipfs::ByteView bytes); ++ ++/** ++ * Decode base36 (case-insensitively) to bytes ++ * @param string to be decoded ++ * @return decoded bytes in case of success ++ */ ++ipfs::expected decodeBase36( ++ std::string_view string); ++ ++} // namespace libp2p::multi::detail ++ ++#endif // LIBP2P_BASE36_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base58.hpp b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base58.hpp +new file mode 100644 +index 0000000000000..2cfb4576ca104 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base58.hpp +@@ -0,0 +1,42 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_BASE58_HPP ++#define LIBP2P_BASE58_HPP ++ ++#include "base_error.hpp" ++ ++#include ++ ++#include ++#include ++ ++#include ++#include ++ ++/** ++ * Encode/decode to/from base58 format ++ * Implementation is taken from ++ * https://github.com/bitcoin/bitcoin/blob/master/src/base58.h ++ */ ++namespace libp2p::multi::detail { ++ ++/** ++ * Encode bytes to base58 string ++ * @param bytes to be encoded ++ * @return encoded string ++ */ ++std::string encodeBase58(ipfs::ByteView bytes); ++ ++/** ++ * Decode base58 string to bytes ++ * @param string to be decoded ++ * @return decoded bytes in case of success ++ */ ++ipfs::expected decodeBase58( ++ std::string_view string); ++} // namespace libp2p::multi::detail ++ ++#endif // LIBP2P_BASE58_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base_error.hpp b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base_error.hpp +new file mode 100644 +index 0000000000000..a0ab1b6c54be5 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/codecs/base_error.hpp +@@ -0,0 +1,24 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_BASE_ERROR_HPP ++#define LIBP2P_BASE_ERROR_HPP ++ ++namespace libp2p::multi::detail { ++ ++enum class BaseError { ++ INVALID_BASE58_INPUT = 1, ++ INVALID_BASE64_INPUT, ++ INVALID_BASE32_INPUT, ++ INVALID_BASE36_INPUT, ++ NON_UPPERCASE_INPUT, ++ NON_LOWERCASE_INPUT, ++ UNIMPLEMENTED_MULTIBASE, ++ INVALID_BASE16_INPUT ++}; ++ ++} ++ ++#endif // LIBP2P_BASE_ERROR_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/multibase_codec/multibase_codec_impl.hpp b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/multibase_codec_impl.hpp +new file mode 100644 +index 0000000000000..5fe1cac5a3c16 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/multibase_codec/multibase_codec_impl.hpp +@@ -0,0 +1,29 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_MULTIBASE_IMPL_HPP ++#define LIBP2P_MULTIBASE_IMPL_HPP ++ ++#include ++ ++#include ++ ++namespace libp2p::multi { ++/** ++ * Implementation of the MultibaseCodec with fair codecs ++ */ ++class MultibaseCodecImpl : public MultibaseCodec { ++ public: ++ ~MultibaseCodecImpl() override = default; ++ ++ std::string encode(const ByteBuffer& bytes, Encoding encoding) const override; ++ ++ FactoryResult decode(std::string_view string) const override; ++}; ++} // namespace libp2p::multi ++ ++// OUTCOME_HPP_DECLARE_ERROR(libp2p::multi, MultibaseCodecImpl::Error) ++ ++#endif // LIBP2P_MULTIBASE_IMPL_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/multicodec_type.hpp b/third_party/ipfs_client/include/libp2p/multi/multicodec_type.hpp +new file mode 100644 +index 0000000000000..3bab21d67c2c5 +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/multicodec_type.hpp +@@ -0,0 +1,75 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_MULTICODECTYPE_HPP ++#define LIBP2P_MULTICODECTYPE_HPP ++ ++#include ++ ++namespace libp2p::multi { ++ ++/** ++ * LibP2P uses "protocol tables" to agree upon the mapping from one multicodec ++ * code. These tables can be application specific, though, like with other ++ * multiformats, there is a globally agreed upon table with common protocols ++ * and formats. ++ */ ++class MulticodecType { ++ public: ++ enum class Code { ++ IDENTITY = 0x00, ++ SHA1 = 0x11, ++ SHA2_256 = 0x12, ++ SHA2_512 = 0x13, ++ SHA3_512 = 0x14, ++ SHA3_384 = 0x15, ++ SHA3_256 = 0x16, ++ SHA3_224 = 0x17, ++ RAW = 0x55, ++ DAG_PB = 0x70, ++ DAG_CBOR = 0x71, ++ LIBP2P_KEY = 0x72, ++ FILECOIN_COMMITMENT_UNSEALED = 0xf101, ++ FILECOIN_COMMITMENT_SEALED = 0xf102, ++ }; ++ ++ constexpr static std::string_view getName(Code code) { ++ switch (code) { ++ case Code::IDENTITY: ++ return "identity"; ++ case Code::SHA1: ++ return "sha1"; ++ case Code::SHA2_256: ++ return "sha2-256"; ++ case Code::SHA2_512: ++ return "sha2-512"; ++ case Code::SHA3_224: ++ return "sha3-224"; ++ case Code::SHA3_256: ++ return "sha3-256"; ++ case Code::SHA3_384: ++ return "sha3-384"; ++ case Code::SHA3_512: ++ return "sha3-512"; ++ case Code::RAW: ++ return "raw"; ++ case Code::DAG_PB: ++ return "dag-pb"; ++ case Code::DAG_CBOR: ++ return "dag-cbor"; ++ case Code::LIBP2P_KEY: ++ return "libp2p-key"; ++ case Code::FILECOIN_COMMITMENT_UNSEALED: ++ return "fil-commitment-unsealed"; ++ case Code::FILECOIN_COMMITMENT_SEALED: ++ return "fil-commitment-sealed"; ++ } ++ return "unknown"; ++ } ++}; ++ ++} // namespace libp2p::multi ++ ++#endif // LIBP2P_MULTICODECTYPE_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/multihash.hpp b/third_party/ipfs_client/include/libp2p/multi/multihash.hpp +new file mode 100644 +index 0000000000000..43acb6857fc6a +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/multihash.hpp +@@ -0,0 +1,160 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_MULTIHASH_HPP ++#define LIBP2P_MULTIHASH_HPP ++ ++#include "vocab/byte_view.h" ++#include "vocab/expected.h" ++ ++#include ++#include ++#include ++ ++#include ++#include ++ ++namespace libp2p::multi { ++ ++/** ++ * Special format of hash used in Libp2p. Allows to differentiate between ++ * outputs of different hash functions. More ++ * https://github.com/multiformats/multihash ++ */ ++class Multihash { ++ public: ++ Multihash(const Multihash& other); ++ ++ Multihash& operator=(const Multihash& other) = default; ++ ++ Multihash(Multihash&& other) noexcept; ++ ++ Multihash& operator=(Multihash&& other) noexcept = default; ++ ++ ~Multihash() noexcept; ++ ++ static constexpr uint8_t kMaxHashLength = 127; ++ ++ enum class Error { ++ ZERO_INPUT_LENGTH = 1, ++ INPUT_TOO_LONG, ++ INPUT_TOO_SHORT, ++ INCONSISTENT_LENGTH, ++ INVALID_HEXADECIMAL_INPUT ++ }; ++ ++ /** ++ * @brief Creates a multihash from hash type and the hash itself. Note that ++ * the max hash length is 127 ++ * @param type - numerical code of the hash type. ++ * @param hash - binary buffer with the hash ++ * @return result with the multihash in case of success ++ */ ++ static ipfs::expected create(HashType type, ++ ipfs::ByteView hash); ++ ++ /** ++ * @brief Creates a multihash from a string, which represents a binary ++ * buffer in hexadecimal form. The first byte denotes the hash type, the ++ * second one contains the hash length, and the following are the hash ++ * itself ++ * @param hex - the string with hexadecimal representation of the multihash ++ * @return result with the multihash in case of success ++ */ ++ static ipfs::expected createFromHex(std::string_view hex); ++ ++ /** ++ * @brief Creates a multihash from a binary ++ * buffer. The first byte denotes the hash type, the ++ * second one contains the hash length, and the following are the hash ++ * itself ++ * @param b - the buffer with the multihash ++ * @return result with the multihash in case of success ++ */ ++ static ipfs::expected createFromBytes(ipfs::ByteView b); ++ ++ /** ++ * @return the info about hash type ++ */ ++ const HashType& getType() const; ++ ++ /** ++ * @return the hash stored in this multihash ++ */ ++ ipfs::ByteView getHash() const; ++ ++ /** ++ * @return a string with hexadecimal representation of the multihash ++ */ ++ std::string toHex() const; ++ ++ /** ++ * @return a buffer with the multihash, including its type, length and hash ++ */ ++ std::vector const& toBuffer() const; ++ ++ /** ++ * @return Pre-calculated hash for std containers ++ */ ++ size_t stdHash() const; ++ ++ bool operator==(const Multihash& other) const; ++ ++ bool operator!=(const Multihash& other) const; ++ ++ bool operator<(const Multihash& other) const; ++ ++ private: ++ /** ++ * Header consists of hash type and hash size, both 1 byte or 2 hex digits ++ * long, thus 4 hex digits long in total ++ */ ++ static constexpr uint8_t kHeaderSize = 4; ++ ++ /** ++ * Constructs a multihash from a type and a hash. Inits length_ with the ++ * size of the hash. Does no checks on the validity of provided data, in ++ * contrary to public fabric methods ++ * @param type - the info about the hash type ++ * @param hash - a binary buffer with the hash ++ */ ++ Multihash(HashType type, ipfs::ByteView hash); ++ ++ /** ++ * Contains a one byte hash type, a one byte hash length, and the stored ++ * hash itself ++ */ ++ struct Data { ++ // TODO(artem): move to small_vector ++ // as soon as toBuffer() -> span is acceptable ++ std::vector bytes; ++ uint8_t hash_offset{}; ///< size of non-hash data from the beginning ++ HashType type; ++ size_t std_hash; ///< Hash for unordered containers ++ ++ Data(HashType t, ipfs::ByteView h); ++ ++ ~Data() noexcept; ++ }; ++ ++ const Data& data() const; ++ ++ std::shared_ptr data_; ++}; ++ ++std::string to_string(Multihash::Error); ++ ++} // namespace libp2p::multi ++ ++namespace std { ++template <> ++struct hash { ++ size_t operator()(const libp2p::multi::Multihash& x) const { ++ return x.stdHash(); ++ } ++}; ++} // namespace std ++ ++#endif // LIBP2P_MULTIHASH_HPP +diff --git a/third_party/ipfs_client/include/libp2p/multi/uvarint.hpp b/third_party/ipfs_client/include/libp2p/multi/uvarint.hpp +new file mode 100644 +index 0000000000000..65426daaf2d9a +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/multi/uvarint.hpp +@@ -0,0 +1,101 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_VARINT_HPP ++#define LIBP2P_VARINT_HPP ++ ++#include "vocab/byte_view.h" ++ ++#include ++ ++#include ++#include ++#include ++ ++// #include ++// #include ++ ++namespace libp2p::multi { ++ ++/** ++ * @class Encodes and decodes unsigned integers into and from ++ * variable-length byte arrays using LEB128 algorithm. ++ */ ++class UVarint { ++ public: ++ /** ++ * Constructs a varint from an unsigned integer 'number' ++ * @param number ++ */ ++ explicit UVarint(uint64_t number); ++ ++ /** ++ * Constructs a varint from an array of raw bytes, which are ++ * meant to be an already encoded unsigned varint ++ * @param varint_bytes an array of bytes representing an unsigned varint ++ */ ++ explicit UVarint(ipfs::ByteView varint_bytes); ++ ++ /** ++ * Constructs a varint from an array of raw bytes, which beginning may or ++ * may not be an encoded varint ++ * @param varint_bytes an array of bytes, possibly representing an unsigned ++ * varint ++ */ ++ static std::optional create(ipfs::ByteView varint_bytes); ++ ++ /** ++ * Converts a varint back to a usual unsigned integer. ++ * @return an integer previously encoded to the varint ++ */ ++ uint64_t toUInt64() const; ++ ++ /** ++ * @return an array view to raw bytes of the stored varint ++ */ ++ ipfs::ByteView toBytes() const; ++ ++ std::vector const& toVector() const; ++ ++ std::string toHex() const; ++ ++ /** ++ * Assigns the varint to an unsigned integer, encoding the latter ++ * @param n the integer to encode and store ++ * @return this varint ++ */ ++ UVarint& operator=(uint64_t n); ++ ++ bool operator==(const UVarint& r) const; ++ bool operator!=(const UVarint& r) const; ++ bool operator<(const UVarint& r) const; ++ ++ /** ++ * @return the number of bytes currently stored in a varint ++ */ ++ size_t size() const; ++ ++ /** ++ * @param varint_bytes an array with a raw byte representation of a varint ++ * @return the size of the varint stored in the array, if its content is a ++ * valid varint. Otherwise, the result is undefined ++ */ ++ static size_t calculateSize(ipfs::ByteView varint_bytes); ++ ++ UVarint() = delete; ++ UVarint(UVarint const&); ++ UVarint& operator=(UVarint const&); ++ ~UVarint() noexcept; ++ ++ private: ++ /// private ctor for unsafe creation ++ UVarint(ipfs::ByteView varint_bytes, size_t varint_size); ++ ++ std::vector bytes_{}; ++}; ++ ++} // namespace libp2p::multi ++ ++#endif // LIBP2P_VARINT_HPP +diff --git a/third_party/ipfs_client/include/libp2p/peer/peer_id.hpp b/third_party/ipfs_client/include/libp2p/peer/peer_id.hpp +new file mode 100644 +index 0000000000000..eadc40c80ce8a +--- /dev/null ++++ b/third_party/ipfs_client/include/libp2p/peer/peer_id.hpp +@@ -0,0 +1,123 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef LIBP2P_PEER_ID_HPP ++#define LIBP2P_PEER_ID_HPP ++ ++#include ++#include ++#include ++#include ++#include ++#include "libp2p/crypto/key.h" ++ ++#include ++#include ++ ++namespace libp2p::peer { ++ ++/** ++ * Unique identifier of the peer - SHA256 Multihash, in most cases, of its ++ * public key ++ */ ++class PeerId { ++ enum class FactoryError { SUCCESS = 0, SHA256_EXPECTED = 1 }; ++ using Error = std::variant; ++ using FactoryResult = ipfs::expected; ++ ++ public: ++ PeerId(const PeerId& other) = default; ++ PeerId& operator=(const PeerId& other) = default; ++ PeerId(PeerId&& other) noexcept = default; ++ PeerId& operator=(PeerId&& other) noexcept = default; ++ ~PeerId() = default; ++ ++ /** ++ * Create a PeerId from the public key ++ * @param key, from which PeerId is to be created ++ * @return instance of PeerId ++ */ ++ static FactoryResult fromPublicKey(const crypto::ProtobufKey& key); ++ ++ /** ++ * Create a PeerId from the byte array (serialized multihash). ++ * @param v buffer ++ * @return instance of PeerId ++ */ ++ static FactoryResult fromBytes(ipfs::ByteView v); ++ ++ /** ++ * Create a PeerId from base58-encoded string (not Multibase58!) with its ++ * SHA256-hashed ID ++ * @param id, with which PeerId is to be created ++ * @return instance of PeerId in case of success, error otherwise ++ */ ++ static FactoryResult fromBase58(std::string_view id); ++ ++ static FactoryResult FromMultibase(std::string_view id); ++ ++ /** ++ * Create a PeerId from SHA256 hash of its ID ++ * @param hash, with which PeerId is to be created; MUST be SHA256 ++ * @return instance of PeerId in case of success, error otherwise ++ */ ++ static FactoryResult fromHash(const multi::Multihash& hash); ++ ++ /** ++ * Get a base58 (not Multibase58!) representation of this PeerId ++ * @return base58-encoded SHA256 multihash of the peer's ID ++ */ ++ std::string toBase58() const; ++ ++ /** ++ * @brief Get a hex representation of this PeerId. ++ */ ++ std::string toHex() const; ++ ++ /** ++ * Creates a vector representation of PeerId. ++ */ ++ const std::vector& toVector() const; ++ ++ /** ++ * Get a SHA256 multihash of the peer's ID ++ * @return multihash ++ */ ++ const multi::Multihash& toMultihash() const; ++ ++ bool operator<(const PeerId& other) const; ++ bool operator==(const PeerId& other) const; ++ bool operator!=(const PeerId& other) const; ++ ++ private: ++ /// if key, from which a PeerId is created, does not exceed this size, it's ++ /// put as a PeerId as-is, without SHA-256 hashing ++ static constexpr size_t kMaxInlineKeyLength = 42; ++ ++ /** ++ * Create an instance of PeerId ++ * @param hash, with which PeerId is to be created ++ */ ++ explicit PeerId(multi::Multihash hash); ++ ++ multi::Multihash hash_; ++}; ++ ++} // namespace libp2p::peer ++ ++namespace std { ++template <> ++struct hash { ++ size_t operator()(const libp2p::peer::PeerId& peer_id) const; ++}; ++} // namespace std ++ ++// OUTCOME_HPP_DECLARE_ERROR(libp2p::peer, PeerId::FactoryError) ++ ++#endif // LIBP2P_PEER_ID_HPP +diff --git a/third_party/ipfs_client/include/smhasher/MurmurHash3.h b/third_party/ipfs_client/include/smhasher/MurmurHash3.h +new file mode 100644 +index 0000000000000..e1c6d34976c6a +--- /dev/null ++++ b/third_party/ipfs_client/include/smhasher/MurmurHash3.h +@@ -0,0 +1,37 @@ ++//----------------------------------------------------------------------------- ++// MurmurHash3 was written by Austin Appleby, and is placed in the public ++// domain. The author hereby disclaims copyright to this source code. ++ ++#ifndef _MURMURHASH3_H_ ++#define _MURMURHASH3_H_ ++ ++//----------------------------------------------------------------------------- ++// Platform-specific functions and macros ++ ++// Microsoft Visual Studio ++ ++#if defined(_MSC_VER) && (_MSC_VER < 1600) ++ ++typedef unsigned char uint8_t; ++typedef unsigned int uint32_t; ++typedef unsigned __int64 uint64_t; ++ ++// Other compilers ++ ++#else // defined(_MSC_VER) ++ ++#include ++ ++#endif // !defined(_MSC_VER) ++ ++//----------------------------------------------------------------------------- ++ ++void MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed, void * out ); ++ ++void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out ); ++ ++void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out ); ++ ++//----------------------------------------------------------------------------- ++ ++#endif // _MURMURHASH3_H_ +diff --git a/third_party/ipfs_client/include/vocab/byte.h b/third_party/ipfs_client/include/vocab/byte.h +new file mode 100644 +index 0000000000000..392124245e08a +--- /dev/null ++++ b/third_party/ipfs_client/include/vocab/byte.h +@@ -0,0 +1,37 @@ ++#ifndef IPFS_BYTE_H_ ++#define IPFS_BYTE_H_ ++ ++#include ++#include ++ ++#include ++#include ++#include ++ ++#ifdef __cpp_lib_byte ++ ++namespace ipfs { ++using Byte = std::byte; ++} // namespace ipfs ++ ++#else ++namespace ipfs { ++enum class Byte : std::uint_least8_t {}; ++} // namespace ipfs ++#endif ++ ++namespace { ++[[maybe_unused]] std::ostream& operator<<(std::ostream& str, ipfs::Byte b) { ++ return str << std::hex << std::setw(2) << std::setfill('0') ++ << static_cast(b); ++} ++} // namespace ++ ++namespace { ++// libc++ provides this, but for some reason libstdc++ does not ++[[maybe_unused]] std::uint8_t to_integer(ipfs::Byte b) { ++ return static_cast(b); ++} ++} // namespace ++ ++#endif // IPFS_BYTE_H_ +diff --git a/third_party/ipfs_client/include/vocab/byte_view.h b/third_party/ipfs_client/include/vocab/byte_view.h +new file mode 100644 +index 0000000000000..92bbe2ad12efd +--- /dev/null ++++ b/third_party/ipfs_client/include/vocab/byte_view.h +@@ -0,0 +1,17 @@ ++#ifndef CHROMIUM_IPFS_BYTE_VIEW_H ++#define CHROMIUM_IPFS_BYTE_VIEW_H ++ ++#include "byte.h" ++#include "span.h" ++ ++#include ++ ++namespace ipfs { ++using ByteView = span; ++ ++inline span as_octets(ByteView bytes) { ++ return {reinterpret_cast(bytes.data()), bytes.size()}; ++} ++} // namespace ipfs ++ ++#endif // CHROMIUM_IPFS_BYTE_VIEW_H +diff --git a/third_party/ipfs_client/include/vocab/endian.h b/third_party/ipfs_client/include/vocab/endian.h +new file mode 100644 +index 0000000000000..2423006c7c02b +--- /dev/null ++++ b/third_party/ipfs_client/include/vocab/endian.h +@@ -0,0 +1,21 @@ ++#ifndef IPFS_ENDIAN_H_ ++#define IPFS_ENDIAN_H_ ++ ++#if __has_include() ++#include ++#endif ++#if __has_include() ++#include ++#endif ++ ++#ifdef htobe64 ++// Good ++#elif __has_include() ++#include ++#define htobe64 absl::ghtonll ++#elif __has_include() ++#include ++#define htobe64 native_to_big ++#endif ++ ++#endif // IPFS_ENDIAN_H_ +diff --git a/third_party/ipfs_client/include/vocab/expected.h b/third_party/ipfs_client/include/vocab/expected.h +new file mode 100644 +index 0000000000000..2006f2bf01397 +--- /dev/null ++++ b/third_party/ipfs_client/include/vocab/expected.h +@@ -0,0 +1,44 @@ ++#ifndef IPFS_EXPECTED_H_ ++#define IPFS_EXPECTED_H_ ++ ++// std::expected isn't available until C++23 and we need to support C++17 ++// boost::outcome isn't available inside the Chromium tree ++// absl::StatusOr doesn't allow templating or extending the error type, and ++// translating the specific error codes into generic ones isn't great. ++ ++#if __has_include("base/types/expected.h") ++#include "base/types/expected.h" ++namespace ipfs { ++template ++using expected = base::expected; ++template ++using unexpected = base::unexpected; ++} // namespace ipfs ++#elif __has_cpp_attribute(__cpp_lib_expected) ++ ++#include ++namespace ipfs { ++template ++using expected = std::expected; ++template ++using unexpected = std::unexpected; ++} // namespace ipfs ++ ++#elif __has_include() ++ ++// If the API differences between std::expected and boost::outcome::checked ++// become a problem, consider wrapping as proposed in the FAQ: ++// https://www.boost.org/doc/libs/master/libs/outcome/doc/html/faq.html#how-far-away-from-the-proposed-std-expected-t-e-is-outcome-s-checked-t-e ++#include ++namespace ipfs { ++template ++using expected = boost::outcome_v2::checked; ++template ++using unexpected = Error; ++} // namespace ipfs ++ ++#else ++#error Get an expected implementation ++#endif ++ ++#endif // IPFS_EXPECTED_H_ +diff --git a/third_party/ipfs_client/include/vocab/flat_mapset.h b/third_party/ipfs_client/include/vocab/flat_mapset.h +new file mode 100644 +index 0000000000000..1630e3f9ca358 +--- /dev/null ++++ b/third_party/ipfs_client/include/vocab/flat_mapset.h +@@ -0,0 +1,39 @@ ++#ifndef CHROMIUM_IPFS_VOCAB_MAP_SET_H_ ++#define CHROMIUM_IPFS_VOCAB_MAP_SET_H_ ++ ++#if __has_include("base/containers/flat_map.h") // Chromium ++ ++#include "base/containers/flat_map.h" ++#include "base/containers/flat_set.h" ++#include "base/debug/debugging_buildflags.h" ++namespace ipfs { ++using base::flat_map; ++using base::flat_set; ++} // namespace ipfs ++ ++#elif __has_cpp_attribute(__cpp_lib_flat_map) && \ ++ __has_cpp_attribute(__cpp_lib_flat_set) ++ ++#include ++#include ++namespace ipfs { ++using std::flat_map; ++using std::flat_set; ++} // namespace ipfs ++ ++#elif __has_include() //Boost ++#include ++#include ++namespace ipfs { ++using boost::container::flat_map; ++using boost::container::flat_set; ++} // namespace ipfs ++ ++#else ++ ++#error \ ++ "Provide an implementation for flat_map and flat_set, or install boost or have a Chromium tree or use a newer C++ version." ++ ++#endif ++ ++#endif // CHROMIUM_IPFS_VOCAB_MAP_SET_H_ +diff --git a/third_party/ipfs_client/include/vocab/i128.h b/third_party/ipfs_client/include/vocab/i128.h +new file mode 100644 +index 0000000000000..4aa36cc09877f +--- /dev/null ++++ b/third_party/ipfs_client/include/vocab/i128.h +@@ -0,0 +1,16 @@ ++#ifndef IPFS_I128_H_ ++#define IPFS_I128_H_ ++ ++#if __has_include() ++#include ++namespace ipfs { ++using Int_128 = absl::int128; ++} ++#else ++namespace ipfs { ++// TODO Check if available, if not use boost multiprecision ++using Int_128 = __int128; ++} // namespace ipfs ++#endif ++ ++#endif // IPFS_I128_H_ +diff --git a/third_party/ipfs_client/include/vocab/raw_ptr.h b/third_party/ipfs_client/include/vocab/raw_ptr.h +new file mode 100644 +index 0000000000000..5bfd3f70ac74f +--- /dev/null ++++ b/third_party/ipfs_client/include/vocab/raw_ptr.h +@@ -0,0 +1,64 @@ ++#ifndef IPFS_OBSERVER_PTR_H_ ++#define IPFS_OBSERVER_PTR_H_ ++ ++#if __has_include("base/memory/raw_ptr.h") ++#include "base/memory/raw_ptr.h" ++ ++namespace ipfs { ++template ++using raw_ptr = base::raw_ptr; ++} ++ ++#elif defined(__has_cpp_attribute) && \ ++ __has_cpp_attribute(__cpp_lib_experimental_observer_ptr) ++#include ++ ++namespace ipfs { ++template ++using raw_ptr = std::experimental::observer_ptr; ++} ++ ++#else ++ ++#include ++ ++namespace ipfs { ++ ++/*! ++ * \brief Just an observing (non-owning) pointer. ++ */ ++template ++class raw_ptr { ++ T* ptr_; ++ ++ public: ++ // Chromium's raw_ptr has a default ctor whose semantics depend on build ++ // config. For components/ipfs purposes, there is no reason to ever default ++ // construct. Set it to nullptr. We have time needed to assign a word. ++ raw_ptr() = delete; ++ ++ raw_ptr(T* p) : ptr_{p} {} ++ raw_ptr(raw_ptr&&) = default; ++ raw_ptr(raw_ptr const&) = default; ++ ++ raw_ptr& operator=(raw_ptr const&) = default; ++ ++ T* get() { return ptr_; } ++ T const* get() const { return ptr_; } ++ explicit operator bool() const { return !!ptr_; } ++ T* operator->() { return ptr_; } ++ T const* operator->() const { return ptr_; } ++ raw_ptr& operator=(T* p) { ++ ptr_ = p; ++ return *this; ++ } ++ T& operator*() { ++ assert(ptr_); ++ return *ptr_; ++ } ++}; ++} // namespace ipfs ++ ++#endif ++ ++#endif // IPFS_OBSERVER_PTR_H_ +diff --git a/third_party/ipfs_client/include/vocab/slash_delimited.h b/third_party/ipfs_client/include/vocab/slash_delimited.h +new file mode 100644 +index 0000000000000..554656c5db44c +--- /dev/null ++++ b/third_party/ipfs_client/include/vocab/slash_delimited.h +@@ -0,0 +1,22 @@ ++#ifndef IPFS_SLASH_DELIMITED_H_ ++#define IPFS_SLASH_DELIMITED_H_ ++ ++#include ++#include ++ ++namespace ipfs { ++struct SlashDelimited { ++ std::string_view remainder_; ++ ++ public: ++ SlashDelimited(std::string_view unowned); ++ explicit operator bool() const; ++ std::string_view pop(); ++ std::string_view pop_all(); ++ std::string_view pop_n(std::size_t); ++ std::string_view peek_back() const; ++ std::string to_string() const { return std::string{remainder_}; } ++}; ++} // namespace ipfs ++ ++#endif // IPFS_SLASH_DELIMITED_H_ +diff --git a/third_party/ipfs_client/include/vocab/span.h b/third_party/ipfs_client/include/vocab/span.h +new file mode 100644 +index 0000000000000..ad73c33b2c624 +--- /dev/null ++++ b/third_party/ipfs_client/include/vocab/span.h +@@ -0,0 +1,65 @@ ++#ifndef IPFS_SPAN_H_ ++#define IPFS_SPAN_H_ ++ ++#if __has_include("base/containers/span.h") ++ ++#include "base/containers/span.h" ++namespace ipfs { ++template ++using span = base::span; ++} // namespace ipfs ++ ++#elif __has_cpp_attribute(__cpp_lib_span) ++ ++#include ++namespace ipfs { ++template ++using span = std::span; ++} // namespace ipfs ++ ++#elif __has_include() ++ ++#include ++namespace ipfs { ++template ++using span = absl::Span; ++} // namespace ipfs ++ ++#elif __has_include() ++ ++#include ++namespace ipfs { ++template ++using span = boost::span; ++} // namespace ipfs ++ ++#elif __has_include() ++ ++// Prior to Boost 1.78, span did not exist in core yet ++#include ++#include ++namespace ipfs { ++template ++class span : public boost::beast::span { ++ public: ++ span(Value* d, std::size_t n) : boost::beast::span{d, n} {} ++ ++ template ++ span(std::vector const& v) ++ : boost::beast::span{v.data(), v.size()} {} ++ ++ span subspan(std::size_t off) const { ++ return span{this->data() + off, this->size() - off}; ++ } ++ Value& operator[](std::size_t i) { return this->data()[i]; } ++}; ++} // namespace ipfs ++ ++#else ++ ++#error \ ++ "No good implementation of span available. Implement one, move to a newer C++, or provide Boost or Abseil." ++ ++#endif ++ ++#endif // IPFS_SPAN_H_ +diff --git a/third_party/ipfs_client/include/vocab/stringify.h b/third_party/ipfs_client/include/vocab/stringify.h +new file mode 100644 +index 0000000000000..8572ebfef7165 +--- /dev/null ++++ b/third_party/ipfs_client/include/vocab/stringify.h +@@ -0,0 +1,17 @@ ++#ifndef IPFS_STRINGIFY_H_ ++#define IPFS_STRINGIFY_H_ ++ ++#include ++ ++namespace ipfs { ++namespace { ++template ++std::string Stringify(T const& t) { ++ std::ostringstream oss; ++ oss << t; ++ return oss.str(); ++} ++} // namespace ++} // namespace ipfs ++ ++#endif // IPFS_STRINGIFY_H_ +diff --git a/third_party/ipfs_client/ipns_record.proto b/third_party/ipfs_client/ipns_record.proto +new file mode 100644 +index 0000000000000..6018931b7466f +--- /dev/null ++++ b/third_party/ipfs_client/ipns_record.proto +@@ -0,0 +1,38 @@ ++syntax = "proto2"; ++option optimize_for = LITE_RUNTIME; ++package ipfs.ipns; ++ ++message IpnsEntry { ++ enum ValidityType { ++ // setting an EOL says "this record is valid until..." ++ EOL = 0; ++ } ++ ++ // deserialized copy of data[value] ++ optional bytes value = 1; ++ ++ // legacy field, verify 'signatureV2' instead ++ optional bytes signatureV1 = 2; ++ ++ // deserialized copies of data[validityType] and data[validity] ++ optional ValidityType validityType = 3; ++ optional bytes validity = 4; ++ ++ // deserialized copy of data[sequence] ++ optional uint64 sequence = 5; ++ ++ // record TTL in nanoseconds, a deserialized copy of data[ttl] ++ optional uint64 ttl = 6; ++ ++ // in order for nodes to properly validate a record upon receipt, they need the public ++ // key associated with it. For old RSA keys, its easiest if we just send this as part of ++ // the record itself. For newer Ed25519 keys, the public key can be embedded in the ++ // IPNS Name itself, making this field unnecessary. ++ optional bytes pubKey = 7; ++ ++ // the signature of the IPNS record ++ optional bytes signatureV2 = 8; ++ ++ // extensible record data in DAG-CBOR format ++ optional bytes data = 9; ++} +diff --git a/third_party/ipfs_client/keys.proto b/third_party/ipfs_client/keys.proto +new file mode 100644 +index 0000000000000..a6f4f75ddba93 +--- /dev/null ++++ b/third_party/ipfs_client/keys.proto +@@ -0,0 +1,22 @@ ++syntax = "proto2"; ++option optimize_for = LITE_RUNTIME; ++package ipfs.ipns; ++ ++enum KeyType { ++ RSA = 0; ++ Ed25519 = 1; ++ Secp256k1 = 2; ++ ECDSA = 3; ++} ++ ++// PublicKey ++message PublicKey { ++ required KeyType Type = 1; ++ required bytes Data = 2; ++} ++ ++// PrivateKey ++message PrivateKey { ++ required KeyType Type = 1; ++ required bytes Data = 2; ++} +diff --git a/third_party/ipfs_client/pb_dag.proto b/third_party/ipfs_client/pb_dag.proto +new file mode 100644 +index 0000000000000..5cd027631c6de +--- /dev/null ++++ b/third_party/ipfs_client/pb_dag.proto +@@ -0,0 +1,23 @@ ++syntax = "proto2"; ++option optimize_for = LITE_RUNTIME; ++package ipfs.pb_dag; ++ ++message PBLink { ++ // binary CID (with no multibase prefix) of the target object ++ optional bytes Hash = 1; ++ ++ // UTF-8 string name ++ optional string Name = 2; ++ ++ // cumulative size of target object ++ optional uint64 Tsize = 3; ++} ++ ++message PBNode { ++ // refs to other objects ++ repeated PBLink Links = 2; ++ ++ // opaque user data ++ optional bytes Data = 1; ++} ++ +diff --git a/third_party/ipfs_client/src/ipfs_client/block_requestor.cc b/third_party/ipfs_client/src/ipfs_client/block_requestor.cc +new file mode 100644 +index 0000000000000..8a63e6f7ae0cc +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/block_requestor.cc +@@ -0,0 +1 @@ ++#include +diff --git a/third_party/ipfs_client/src/ipfs_client/block_storage.cc b/third_party/ipfs_client/src/ipfs_client/block_storage.cc +new file mode 100644 +index 0000000000000..e8bb7c9e07a8c +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/block_storage.cc +@@ -0,0 +1,218 @@ ++#include "ipfs_client/block_storage.h" ++#include "ipfs_client/unixfs_path_resolver.h" ++ ++#include ++ ++#include "log_macros.h" ++#include "vocab/stringify.h" ++ ++using Codec = libp2p::multi::ContentIdentifierCodec; ++using MultiCodec = libp2p::multi::MulticodecType::Code; ++ ++bool ipfs::BlockStorage::Store(std::string cid_str, ++ ipfs::Cid const&, ++ std::string headers, ++ std::string const& body, ++ Block&& block) { ++ VLOG(2) << "Store(" << cid_str << ')'; ++ auto t = std::time(nullptr); ++ auto it = cid2record_.find(cid_str); ++ if (it != cid2record_.end()) { ++ if (it->second) { ++ VLOG(1) << cid_str << " already stored."; ++ it->second->last_access = t; ++ CheckListening(); ++ return false; ++ } else { ++ LOG(INFO) << "Filling in a null placeholder."; ++ } ++ } ++ auto* into = FindFree(t); ++ cid2record_[cid_str] = into; ++ if (into->cid_str.empty()) { ++ VLOG(1) << "Storing " << cid_str << " in fresh node @" << (void*)into; ++ } else { ++ VLOG(1) << "Evicting " << into->cid_str << " to make room for " << cid_str; ++ cid2record_.erase(into->cid_str); ++ } ++ into->cid_str = cid_str; ++ into->last_access = t; ++ into->block = std::move(block); ++ into->headers = std::move(headers); ++ if (into->headers.size() && body.size()) { ++ for (auto& hook : hooks_) { ++ hook(cid_str, into->headers, body); ++ } ++ } ++ CheckListening(); ++ return true; ++} ++bool ipfs::BlockStorage::Store(const ipfs::Cid& cid, ++ std::string headers, ++ std::string const& body, ++ ipfs::Block&& block) { ++ auto cid_val = Codec::toString(cid); ++ if (cid_val.has_value()) { ++ return Store(cid_val.value(), cid, headers, body, std::move(block)); ++ } else { ++ return false; ++ } ++} ++bool ipfs::BlockStorage::Store(std::string headers, ++ std::string const& body, ++ ipfs::Block&& block) { ++ return Store(block.cid(), headers, body, std::move(block)); ++} ++bool ipfs::BlockStorage::Store(std::string const& cid, ++ std::string headers, ++ std::string body) { ++ VLOG(2) << "Store(cid=" << cid << " headers.size()=" << headers.size() ++ << " body.size()=" << body.size() << ')'; ++ DCHECK(headers != body); ++ auto cid_res = Codec::fromString(cid); ++ DCHECK(cid_res.has_value()); ++ return Store(cid, cid_res.value(), headers, body); ++} ++bool ipfs::BlockStorage::Store(std::string cid_str, ++ const ipfs::Cid& cid, ++ std::string headers, ++ std::string body) { ++ VLOG(2) << "Store(cid=" << cid_str ++ << ", , headers.size()=" << headers.size() ++ << " body.size()=" << body.size() << ')'; ++ return Store(cid_str, cid, headers, body, {cid, body}); ++} ++auto ipfs::BlockStorage::GetInternal(std::string const& cid) -> Record const* { ++ auto it = cid2record_.find(cid); ++ if (it == cid2record_.end()) { ++ auto parsed = Codec::fromString(cid); ++ if (parsed.has_value()) { ++ if (parsed.value().content_type == MultiCodec::LIBP2P_KEY) { ++ return nullptr; ++ } ++ auto hash_type = parsed.value().content_address.getType(); ++ if (hash_type == libp2p::multi::HashType::identity) { ++ LOG(INFO) << "Handling identity CID: " << cid; ++ return StoreIdentity(cid, parsed.value()); ++ } ++ } else { ++ LOG(ERROR) << " '" << cid << "' is not even a valid CID"; ++ return nullptr; ++ } ++ return nullptr; ++ } ++ auto* rec = it->second; ++ if (!rec) { ++ LOG(INFO) << "Placeholder null had been stored for " << cid; ++ return nullptr; ++ } ++ if (cid != rec->cid_str) { ++ LOG(FATAL) << cid << " mapped into entry with CID " << rec->cid_str; ++ } ++ rec->last_access = std::time(nullptr); ++ return rec; ++} ++ipfs::Block const* ipfs::BlockStorage::Get(std::string const& cid) { ++ auto* result = GetInternal(cid); ++ if (result) { ++ return &(result->block); ++ } ++ return nullptr; ++} ++std::string const* ipfs::BlockStorage::GetHeaders(const std::string& cid) { ++ auto* result = GetInternal(cid); ++ if (result) { ++ VLOG(2) << "GetHeaders(" << cid << ")->size()=" << result->headers.size(); ++ return &(result->headers); ++ } ++ LOG(ERROR) << "Found no headers for " << cid << "!"; ++ return nullptr; ++} ++void ipfs::BlockStorage::AddListening(UnixFsPathResolver* p) { ++ // LOG(INFO) << "AddListening(" << p->current_cid() << ')'; ++ listening_.insert(p); ++} ++ ++void ipfs::BlockStorage::StopListening(UnixFsPathResolver* p) { ++ VLOG(1) << "StopListening(" << p->current_cid() << ',' << p->original_path() ++ << ')'; ++ auto e = std::remove(listening_.begin(), listening_.end(), p); ++ listening_.erase(e, listening_.end()); ++} ++ ++void ipfs::BlockStorage::CheckListening() { ++ if (checking_) { ++ return; ++ } ++ checking_ = true; ++ auto looking = true; ++ while (looking) { ++ looking = false; ++ for (UnixFsPathResolver* ptr : listening_) { ++ auto cid = ptr->current_cid(); ++ if (cid.empty()) { ++ VLOG(1) << "Kicking a listener out."; ++ looking = true; ++ auto prev = ptr->MaybeGetPreviousListener(); ++ listening_.erase(ptr); ++ if (prev) { ++ ptr->Step(prev); ++ } ++ break; ++ } ++ auto it = cid2record_.find(cid); ++ if (it != cid2record_.end() && it->second && it->second->cid_str == cid) { ++ auto prev = ptr->MaybeGetPreviousListener(); ++ if (prev) { ++ VLOG(1) << "Stepping listener of (" << cid << ")"; ++ ptr->Step(prev); ++ looking = true; ++ } ++ } ++ } ++ } ++ checking_ = false; ++} ++auto ipfs::BlockStorage::FindFree(std::time_t now) -> Record* { ++ DCHECK(records_.size() > 99); ++ std::list l; ++ l.splice(l.end(), records_, records_.begin()); ++ records_.splice(records_.end(), l); ++ if (now - records_.back().last_access > 300) { ++ VLOG(1) << records_.back().last_access << " is too old."; ++ return &records_.back(); ++ } ++ VLOG(1) << "Not ready to kick out @ " << (void*)&records_.back() ++ << " yet: " << records_.back().last_access; ++ if (now - records_.front().last_access > 300) { ++ return FindFree(now); ++ } ++ return Allocate(); ++} ++auto ipfs::BlockStorage::Allocate() -> Record* { ++ VLOG(1) << "Expanding store size to " << (records_.size() + 1UL); ++ records_.emplace_back(); ++ return &records_.back(); ++} ++void ipfs::BlockStorage::AddStorageHook(SerializedStorageHook h) { ++ hooks_.push_back(h); ++} ++auto ipfs::BlockStorage::StoreIdentity(std::string const& cid_str, ++ Cid const& cid) -> Record* { ++ auto now = std::time(nullptr); ++ auto* record = FindFree(now); ++ record->cid_str = cid_str; ++ record->last_access = now + 9999; ++ auto data = cid.content_address.getHash(); ++ auto* p = reinterpret_cast(data.data()); ++ record->block = Block{cid, std::string{p, data.size()}}; ++ return record; ++} ++ ++ipfs::BlockStorage::BlockStorage() {} ++ipfs::BlockStorage::~BlockStorage() noexcept { ++ LOG(INFO) << "BlockStorage dtor!"; ++} ++ ++ipfs::BlockStorage::Record::Record() = default; ++ipfs::BlockStorage::Record::~Record() noexcept = default; +diff --git a/third_party/ipfs_client/src/ipfs_client/busy_gateway.cc b/third_party/ipfs_client/src/ipfs_client/busy_gateway.cc +new file mode 100644 +index 0000000000000..4d96b166cdec2 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/busy_gateway.cc +@@ -0,0 +1,116 @@ ++#include ++#include ++#include ++#include ++#include ++ ++#include "log_macros.h" ++ ++#include ++ ++ipfs::BusyGateway::BusyGateway(std::string pre, UrlSpec spec, Scheduler* sched) ++ : prefix_(pre), spec_(spec), scheduler_{sched}, maybe_offset_{0UL} {} ++ipfs::BusyGateway::BusyGateway(BusyGateway&& rhs) ++ : prefix_(rhs.prefix_), ++ spec_(rhs.spec_), ++ scheduler_(rhs.scheduler_), ++ maybe_offset_(0UL) { ++ // LOG(INFO) << "BusyGateway(" << prefix_ << ',' << suffix_ << ')'; ++ rhs.prefix_.clear(); ++ rhs.spec_ = {}; ++ rhs.scheduler_ = nullptr; ++ rhs.maybe_offset_ = 0UL; ++ srcreq = rhs.srcreq; ++} ++ipfs::BusyGateway::~BusyGateway() { ++ if (*this && get()) { ++ (*this)->TaskCancelled(spec_); ++ } ++} ++ipfs::Gateway* ipfs::BusyGateway::get() { ++ if (!scheduler_ || prefix_.empty() || spec_.none()) { ++ return nullptr; ++ } ++ auto& gws = scheduler_->gateways_; ++ std::string pre = prefix_; ++ auto match = [pre](auto& gw) { return gw.url_prefix() == pre; }; ++ auto found = std::find_if(gws.begin() + maybe_offset_, gws.end(), match); ++ if (gws.end() != found) { ++ maybe_offset_ = std::distance(gws.begin(), gws.end()); ++ return &*found; ++ } else if (maybe_offset_) { ++ maybe_offset_ = 0UL; ++ return get(); ++ } else { ++ return nullptr; ++ } ++} ++ipfs::Gateway const* ipfs::BusyGateway::operator->() const { ++ return const_cast(this)->get(); ++} ++ipfs::Gateway* ipfs::BusyGateway::operator->() { ++ return get(); ++} ++ipfs::Gateway& ipfs::BusyGateway::operator*() { ++ DCHECK(get()); ++ return *get(); ++} ++ipfs::BusyGateway::operator bool() const { ++ return scheduler_ && prefix_.size() && !spec_.none(); ++} ++void ipfs::BusyGateway::reset() { ++ // LOG(INFO) << "BusyGateway::reset()"; ++ if (scheduler_) { ++ auto todo_it = scheduler_->task2todo_.find(spec_); ++ if (todo_it != scheduler_->task2todo_.end()) { ++ std::set>& reqs = ++ todo_it->second.requests; ++ auto e = std::find_if(reqs.begin(), reqs.end(), [this](auto& r) { ++ return r->gateway.get() == get(); ++ }); ++ if (reqs.end() != e) { ++ reqs.erase(e); ++ } ++ // scheduler_->UpdateDevPage(); ++ } ++ scheduler_ = nullptr; ++ } ++ prefix_.clear(); ++ spec_ = {}; ++} ++void ipfs::BusyGateway::Success(Gateways& g, std::shared_ptr api) { ++ if (prefix_.empty()) { ++ return; ++ } ++ g.promote(prefix_); ++ DCHECK(get()); ++ get()->TaskSuccess(spec_); ++ auto sched = scheduler_; ++ sched->TaskComplete(spec_); ++ if (maybe_offset_) { ++ sched->CheckSwap(--maybe_offset_); ++ } ++ reset(); ++ sched->IssueRequests(api); ++} ++void ipfs::BusyGateway::Failure(Gateways& g, std::shared_ptr api) { ++ DCHECK(prefix_.size() > 0U); ++ g.demote(prefix_); ++ get()->TaskFailed(spec_); ++ auto sched = scheduler_; ++ sched->CheckSwap(maybe_offset_); ++ if (sched->DetectCompleteFailure(spec_)) { ++ LOG(WARNING) << "Giving up on task " << spec_.suffix ++ << " due to complete failure."; ++ if (srcreq) { ++ srcreq->dependent->finish(Response::PLAIN_NOT_FOUND); ++ } ++ sched->TaskComplete(spec_); ++ } else { ++ LOG(INFO) << prefix_ << " gave up on " << spec_.suffix ++ << " , but maybe others may continue."; ++ sched->IssueRequests(api); ++ } ++ reset(); ++ sched->IssueRequests(api); ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/chained_requestors.cc b/third_party/ipfs_client/src/ipfs_client/chained_requestors.cc +new file mode 100644 +index 0000000000000..53bfa681cc902 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/chained_requestors.cc +@@ -0,0 +1,64 @@ ++#include ++ ++#include ++ ++#include ++ ++#include "log_macros.h" ++ ++using Self = ipfs::ChainedRequestors; ++ ++void Self::Add(Ptr p) { ++ chain_.push_back(p); ++} ++bool Self::Valid() const { ++ return !chain_.empty(); ++} ++void Self::RequestByCid(std::string cid, ++ std::shared_ptr listen, ++ Priority priority) { ++ DCHECK(Valid()); ++ struct NextChainLinkListener : public ipfs::DagListener { ++ Priority prio; ++ std::shared_ptr real; ++ std::weak_ptr me; ++ raw_ptr chain; ++ std::size_t index = 0UL; ++ std::string cid; ++ void ReceiveBlockBytes(std::string_view b) override { ++ LOG(INFO) << "L" << (index + 1) << ": bytes " << b.size(); ++ real->ReceiveBlockBytes(b); ++ } ++ void BlocksComplete(std::string mime_type) override { ++ LOG(INFO) << "L" << (index + 1) << ": hit" << cid; ++ real->BlocksComplete(mime_type); ++ } ++ void DoesNotExist(std::string_view c, std::string_view path) override { ++ LOG(INFO) << "L" << (index + 1) << ": " << c << '/' << path ++ << " DOES NOT EXIST"; ++ real->DoesNotExist(c, path); ++ } ++ void NotHere(std::string_view c, std::string_view path) override { ++ DCHECK_EQ(c, cid); ++ if (++index < chain->chain_.size()) { ++ VLOG(2) << cid << " / " << path << " missed on approach #" << index; ++ auto next = chain->chain_.at(index); ++ next->RequestByCid(std::string{cid}, me.lock(), prio); ++ } else { ++ LOG(ERROR) << cid << " / " << path << " has failed completely."; ++ real->NotHere(cid, path); ++ } ++ } ++ NextChainLinkListener(Self* c) : chain{c} {} ++ virtual ~NextChainLinkListener() noexcept = default; ++ }; ++ auto chainer = std::make_shared(this); ++ chainer->prio = priority; ++ chainer->real = listen; ++ chainer->cid = cid; ++ chainer->me = chainer; ++ chain_.at(0UL)->RequestByCid(cid, chainer, priority); ++} ++ ++Self::ChainedRequestors() = default; ++Self::~ChainedRequestors() noexcept = default; +diff --git a/third_party/ipfs_client/src/ipfs_client/context_api.cc b/third_party/ipfs_client/src/ipfs_client/context_api.cc +new file mode 100644 +index 0000000000000..f5edcff6a818c +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/context_api.cc +@@ -0,0 +1,15 @@ ++#include ++ ++#include ++ ++ipfs::GatewayRequest::GatewayRequest(BusyGateway&& bg) ++ : gateway(std::move(bg)) {} ++ ++ipfs::GatewayRequest::~GatewayRequest() noexcept {} ++ ++std::string ipfs::GatewayRequest::url() const { ++ return gateway.url(); ++} ++std::string ipfs::GatewayRequest::task() const { ++ return gateway.task(); ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/dag_block.cc b/third_party/ipfs_client/src/ipfs_client/dag_block.cc +new file mode 100644 +index 0000000000000..e757bab5e1698 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/dag_block.cc +@@ -0,0 +1,224 @@ ++#include "ipfs_client/dag_block.h" ++ ++#include ++#include ++#include ++ ++#include "log_macros.h" ++ ++#include ++ ++#include ++ ++using MC = libp2p::multi::MulticodecType; ++ ++namespace { ++std::string get_bytes(std::string const& s) { ++ return s; ++} ++ ++std::string get_bytes(std::istream& is) { ++ return std::string(std::istreambuf_iterator(is), {}); ++} ++ ++bool parse(ipfs::pb_dag::PBNode& n, std::istream& s) { ++ return n.ParseFromIstream(&s); ++} ++ ++bool parse(ipfs::pb_dag::PBNode& n, std::string const& s) { ++ return n.ParseFromString(s); ++} ++ ++using Multicodec = libp2p::multi::MulticodecType::Code; ++ ++template ++std::pair InitBlock(Multicodec c, ++ From& from, ++ ipfs::pb_dag::PBNode& n, ++ ipfs::unix_fs::Data& d) { ++ switch (c) { ++ case Multicodec::DAG_PB: ++ if (parse(n, from)) { ++ return {true, d.ParseFromString(n.data())}; ++ } ++ break; ++ case Multicodec::RAW: ++ case Multicodec::IDENTITY: ++ d.set_type(ipfs::unix_fs::Data_DataType_File); ++ d.set_data(get_bytes(from)); ++ d.set_filesize(d.data().size()); ++ n.set_data(d.SerializeAsString()); ++ return {true, true}; ++ default: ++ LOG(FATAL) << "Block-initialization unsupported for multicodec: " ++ << static_cast(c) << '(' ++ << std::string{MC::getName(c)} << ')'; ++ } ++ return {false, false}; ++} ++} // namespace ++ ++ipfs::Block::Block(Cid const& c, std::istream& s) : cid_(c) { ++ std::tie(valid_, fs_node_) = InitBlock(c.content_type, s, node_, fsdata_); ++} ++ ++ipfs::Block::Block(Cid const& c, std::string const& s) ++ : cid_(c), original_bytes_(s) { ++ std::tie(valid_, fs_node_) = InitBlock(c.content_type, s, node_, fsdata_); ++} ++ ++ipfs::Block::Block(Block const& rhs) ++ : node_(rhs.node_), ++ fsdata_(rhs.fsdata_), ++ valid_(rhs.valid_), ++ fs_node_(rhs.fs_node_), ++ mime_(rhs.mime_), ++ cid_(rhs.cid_), ++ original_bytes_(rhs.original_bytes_) {} ++ ++ipfs::Block::Block() = default; ++ ++ipfs::Block::~Block() noexcept {} ++ ++void ipfs::Block::InitFromRaw(std::string const& content_bytes) { ++ fsdata_.set_type(unix_fs::Data_DataType_File); ++ fsdata_.set_data(content_bytes); ++ fsdata_.set_filesize(content_bytes.size()); ++ node_.set_data(fsdata_.SerializeAsString()); ++ valid_ = true; ++ fs_node_ = true; ++} ++ ++bool ipfs::Block::valid() const { ++ return valid_; ++} ++ ++auto ipfs::Block::type() const -> Type { ++ if (!valid()) { ++ return Type::Invalid; ++ } ++ if (!fs_node_) { ++ return Type::NonFs; ++ } ++ if (is_file()) { ++ if (fsdata_.blocksizes_size() > 0) { ++ return Type::File; ++ } else { ++ return Type::FileChunk; ++ } ++ } ++ if (fsdata_.type()) { ++ return static_cast(fsdata_.type()); ++ } ++ return Type::Invalid; ++} ++ ++bool ipfs::Block::is_file() const { ++ return valid() && fs_node_ && fsdata_.type() == unix_fs::Data_DataType_File; ++} ++ ++bool ipfs::Block::is_directory() const { ++ return valid() && fs_node_ && ++ fsdata_.type() == unix_fs::Data_DataType_Directory; ++} ++ ++std::uint64_t ipfs::Block::file_size() const { ++ if (fs_node_ && fsdata_.has_filesize()) { ++ return fsdata_.filesize(); ++ } else { ++ return 0UL; ++ } ++} ++ ++std::string const& ipfs::Block::chunk_data() const { ++ return fsdata_.data(); ++} ++ ++std::string const& ipfs::Block::unparsed() const { ++ return node_.data(); ++} ++ ++// std::string const& ipfs::Block::mime_type() const { ++// return mime_; ++// } ++auto ipfs::Block::cid() const -> Cid const& { ++ DCHECK(cid_.has_value()); ++ return cid_.value(); ++} ++// void ipfs::Block::mime_type(std::string_view val) { ++// mime_.assign(val); ++// } ++ ++std::string ipfs::Block::LinkCid(ipfs::ByteView binary_link_hash) const { ++ using Codec = libp2p::multi::ContentIdentifierCodec; ++ auto result = Codec::decode(binary_link_hash); ++ if (!result.has_value()) { ++ LOG(FATAL) << "Failed to decode link CID as binary ( link from " ++ << Codec::toString(cid()).value() ++ << "): " << Stringify(result.error()); ++ } ++ auto str_res = Codec::toString(result.value()); ++ if (!str_res.has_value()) { ++ LOG(FATAL) << "Failed to decode binary link CID as string (link from " ++ << Codec::toString(cid()).value() ++ << "): " << Stringify(result.error()); ++ } ++ return str_res.value(); ++} ++ ++bool ipfs::Block::cid_matches_data() const { ++ if (!cid_) { ++ // TODO - probably remove those constructors and make cid_ not optional ++ return true; ++ } ++ auto cid_hash = cid_->content_address.getHash(); ++ auto hash_type = cid_->content_address.getType(); ++ if (hash_type == libp2p::multi::HashType::identity) { ++ return true; ++ } ++ auto hashed = this->binary_hash(hash_type); ++ return std::equal(cid_hash.begin(), cid_hash.end(), hashed.begin(), ++ hashed.end()); ++} ++ ++std::vector ipfs::Block::binary_hash( ++ libp2p::multi::HashType algo) const { ++ ipfs::ByteView bytes{reinterpret_cast(original_bytes_.data()), ++ original_bytes_.size()}; ++ auto hasher = libp2p::crypto::CreateHasher(algo); ++ std::vector result(hasher->digestSize(), Byte{}); ++ if (hasher->write(bytes).value()) { ++ if (!hasher->digestOut({result.data(), result.size()}).has_value()) { ++ LOG(ERROR) << "Error getting digest."; ++ } ++ } else { ++ LOG(ERROR) << "Attempt to hash bytes returned false"; ++ } ++ return result; ++} ++ ++std::ostream& operator<<(std::ostream& s, ipfs::Block::Type t) { ++ switch (t) { ++ case ipfs::Block::Type::Raw: ++ return s << "Raw"; ++ case ipfs::Block::Type::Directory: ++ return s << "Directory"; ++ case ipfs::Block::Type::File: ++ return s << "File"; ++ case ipfs::Block::Type::Metadata: ++ return s << "Metadata"; ++ case ipfs::Block::Type::Symlink: ++ return s << "Symlink"; ++ case ipfs::Block::Type::HAMTShard: ++ return s << "HAMTShard"; ++ case ipfs::Block::Type::FileChunk: ++ return s << "FileChunk"; ++ case ipfs::Block::Type::NonFs: ++ return s << "NonFs"; ++ case ipfs::Block::Type::Invalid: ++ return s << "Invalid"; ++ default: ++ return s << "Invalid value for Block::Type enum (" << static_cast(t) ++ << ')'; ++ } ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/gateway.cc b/third_party/ipfs_client/src/ipfs_client/gateway.cc +new file mode 100644 +index 0000000000000..1d0ebeab03699 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/gateway.cc +@@ -0,0 +1,63 @@ ++#include "ipfs_client/gateway.h" ++#include "log_macros.h" ++ ++ipfs::Gateway::Gateway(std::string url_prefix, unsigned int priority) ++ : prefix_{std::move(url_prefix)}, priority_{priority} {} ++ipfs::Gateway::Gateway(Gateway const& other) ++ : prefix_{other.prefix_}, priority_{other.priority_} {} ++ipfs::Gateway::~Gateway() {} ++ ++// Less means should-be-preferred ++bool ipfs::Gateway::operator<(Gateway const& rhs) const { ++ if (failed_requests_.size() != rhs.failed_requests_.size()) { ++ return failed_requests_.size() < rhs.failed_requests_.size(); ++ } ++ if (priority_ != rhs.priority_) { ++ return priority_ > rhs.priority_; ++ } ++ if (tasks_.size() != rhs.tasks_.size()) { ++ return tasks_.size() < rhs.tasks_.size(); ++ } ++ return prefix_ < rhs.prefix_; ++} ++bool ipfs::Gateway::accept(UrlSpec const& spec, long need) { ++ if (need < 0) { ++ return false; ++ } ++ if (static_cast(need) < tasks_.size() / 2) { ++ return false; ++ } ++ if (priority_ < tasks_.size() * tasks_.size()) { ++ return false; ++ } ++ if (PreviouslyFailed(spec)) { ++ return false; ++ } ++ return tasks_.insert(spec).second; ++} ++std::string const& ipfs::Gateway::url_prefix() const { ++ return prefix_; ++} ++long ipfs::Gateway::load() const { ++ return static_cast(tasks_.size()); ++} ++void ipfs::Gateway::TaskSuccess(UrlSpec const& task) { ++ tasks_.erase(task); ++ ++priority_; ++} ++void ipfs::Gateway::TaskFailed(UrlSpec const& task) { ++ // LOG(INFO) << prefix_ << task << " TaskFailed"; ++ failed_requests_[task] = std::time(nullptr); ++ priority_ /= 2; ++ tasks_.erase(task); ++} ++void ipfs::Gateway::TaskCancelled(UrlSpec const& task) { ++ tasks_.erase(task); ++} ++bool ipfs::Gateway::PreviouslyFailed(UrlSpec const& spec) const { ++ auto it = failed_requests_.find(spec); ++ if (it == failed_requests_.end()) { ++ return false; ++ } ++ return std::time(nullptr) - it->second < 3; ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/gateways.cc b/third_party/ipfs_client/src/ipfs_client/gateways.cc +new file mode 100644 +index 0000000000000..0e0b2c0846220 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/gateways.cc +@@ -0,0 +1,124 @@ ++#include ++ ++#include ++ ++#include "log_macros.h" ++ ++#include ++#include ++#include ++ ++using namespace std::string_literals; ++ ++ipfs::Gateways::Gateways() ++ : random_engine_{std::random_device{}()}, dist_{0.01} { ++ auto gws = DefaultGateways(); ++ for (auto [k, v] : gws) { ++ known_gateways_[k] = v; ++ } ++} ++ipfs::Gateways::~Gateways() {} ++ ++auto ipfs::Gateways::GenerateList() -> GatewayList { ++ GatewayList result; ++ for (auto [k, v] : known_gateways_) { ++ result.push_back({k, v + dist_(random_engine_)}); ++ } ++ std::sort(result.begin(), result.end()); ++ return result; ++} ++ ++void ipfs::Gateways::promote(std::string const& key) { ++ auto it = known_gateways_.find(key); ++ if (known_gateways_.end() == it) { ++ LOG(ERROR) << "Can't promote (" << key ++ << ") because I don't know that one."; ++ } else { ++ auto l = known_gateways_.at(key)++; ++ if (l % (++up_log_ / 2) <= 9) { ++ LOG(INFO) << "Promote(" << key << ")"; ++ } ++ } ++} ++void ipfs::Gateways::demote(std::string const& key) { ++ auto it = known_gateways_.find(key); ++ if (known_gateways_.end() == it) { ++ VLOG(2) << "Can't demote " << key << " as I don't have that gateway."; ++ } else if (it->second) { ++ if (it->second-- % 3 == 0) { ++ LOG(INFO) << "Demote(" << key << ") to " << it->second; ++ } ++ } else { ++ LOG(INFO) << "Demoted(" << key << ") for the last time - dropping."; ++ known_gateways_.erase(it); ++ } ++} ++ ++void ipfs::Gateways::AddGateways(std::vector v) { ++ LOG(INFO) << "AddGateways(" << v.size() << ')'; ++ for (auto& ip : v) { ++ if (ip.empty()) { ++ LOG(ERROR) << "ERROR: Attempted to add empty string as gateway!"; ++ continue; ++ } ++ std::string prefix; ++ if (ip.find("://") == std::string::npos) { ++ prefix = "http://"; ++ prefix.append(ip); ++ } else { ++ prefix = ip; ++ } ++ if (prefix.back() != '/') { ++ prefix.push_back('/'); ++ } ++ if (known_gateways_.insert({prefix, 99}).second) { ++ VLOG(1) << "Adding discovered gateway " << prefix; ++ } ++ } ++} ++ ++std::vector> ipfs::Gateways::DefaultGateways() { ++ auto* ovr = std::getenv("IPFS_GATEWAY"); ++ if (ovr && *ovr) { ++ std::istringstream user_override{ovr}; ++ std::vector> result; ++ std::string gw; ++ while (user_override >> gw) { ++ if ( gw.empty() ) { ++ continue; ++ } ++ if ( gw.back() != '/' ) { ++ gw.push_back('/'); ++ } ++ result.emplace_back( gw, 0 ); ++ } ++ auto N = static_cast(result.size()); ++ for (auto i = 0; i < N; ++i) { ++ auto& r = result[i]; ++ r.second = N - i; ++ LOG(INFO) << "User-specified gateway: " << r.first << '=' << r.second; ++ } ++ return result; ++ } ++ return {{"http://localhost:8080/"s, 923}, ++ {"https://ipfs.io/"s, 920}, ++ {"https://gateway.ipfs.io/"s, 916}, ++ {"https://jcsl.hopto.org/"s, 914}, ++ {"https://dweb.link/"s, 910}, ++ {"https://ipfs.joaoleitao.org/"s, 894}, ++ {"https://gateway.pinata.cloud/"s, 886}, ++ {"https://ipfs.runfission.com/"s, 797}, ++ {"https://nftstorage.link/"s, 313}, ++ {"https://ipfs.jpu.jp/"s, 312}, ++ {"https://w3s.link/"s, 311}, ++ {"https://jorropo.net/"s, 294}, ++ {"https://ipfs.fleek.co/"s, 281}, ++ {"https://permaweb.eu.org/"s, 226}, ++ {"https://hardbin.com/"s, 164}, ++ {"https://ipfs.scalaproject.io/"s, 51}, ++ {"https://ipfs.soul-network.com/"s, 49}, ++ {"https://storry.tv/"s, 25}, ++ {"https://ipfs-gateway.cloud/"s, 2}, ++ {"https://ipfs.storry.tv/"s, 1}, ++ {"https://ipfs.anonymize.com/"s, 0}}; ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/generated_directory_listing.cc b/third_party/ipfs_client/src/ipfs_client/generated_directory_listing.cc +new file mode 100644 +index 0000000000000..a3ffdc849163b +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/generated_directory_listing.cc +@@ -0,0 +1,47 @@ ++#include "generated_directory_listing.h" ++ ++#include "log_macros.h" ++ ++ipfs::GeneratedDirectoryListing::GeneratedDirectoryListing( ++ std::string_view base_path) ++ : html_("\n "), base_path_(base_path) { ++ if (base_path.empty() || base_path[0] != '/') { ++ base_path_.insert(0UL, 1UL, '/'); ++ } ++ if (base_path_.back() != '/') { ++ base_path_.push_back('/'); ++ } ++ html_.append(base_path_) ++ .append(" (directory listing)\n") ++ .append(" \n") ++ .append("
    \n"); ++ if (base_path.find_first_not_of("/") < base_path.size()) { ++ std::string_view dotdotpath{base_path_}; ++ dotdotpath.remove_suffix(1); // Remove that trailing / ++ auto last_slash = dotdotpath.find_last_of("/"); ++ dotdotpath = dotdotpath.substr(0, last_slash + 1UL); ++ AddLink("..", dotdotpath); ++ } ++} ++ ++void ipfs::GeneratedDirectoryListing::AddEntry(std::string_view name) { ++ // auto path = base_path_; ++ // path.append(name); ++ // AddLink(name, path); ++ AddLink(name, name); ++} ++void ipfs::GeneratedDirectoryListing::AddLink(std::string_view name, ++ std::string_view path) { ++ LOG(INFO) << "Adding link to generated index.html " << name << '=' << path; ++ html_.append("
  • \n") ++ .append(" ") ++ .append(name) ++ .append("\n") ++ .append("
    • \n"); ++} ++ ++std::string const& ipfs::GeneratedDirectoryListing::Finish() { ++ return html_.append("
\n").append(" \n").append("\n"); ++} +\ No newline at end of file +diff --git a/third_party/ipfs_client/src/ipfs_client/generated_directory_listing.h b/third_party/ipfs_client/src/ipfs_client/generated_directory_listing.h +new file mode 100644 +index 0000000000000..8daa0ec01cb9e +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/generated_directory_listing.h +@@ -0,0 +1,41 @@ ++#ifndef IPFS_GENERATED_DIRECTORY_LISTING_H_ ++#define IPFS_GENERATED_DIRECTORY_LISTING_H_ ++ ++#include ++#include ++ ++namespace ipfs { ++ ++/*! ++ * \brief An index.html listing out a directory node's content ++ */ ++class GeneratedDirectoryListing { ++ public: ++ ++ /*! ++ * \brief Get the HTML preamble going ++ * \param base_path - The path _to_ this directory ++ */ ++ GeneratedDirectoryListing(std::string_view base_path); ++ ++ /*! ++ * \brief Add an entry to the list ++ * \param name - The directory's way of referring to that CID ++ */ ++ void AddEntry(std::string_view name); ++ ++ /*! ++ * \brief Finish up all the HTML stuff at the end. ++ * \return The generated HTML ++ */ ++ std::string const& Finish(); ++ ++ private: ++ std::string html_; ++ std::string base_path_; ++ ++ void AddLink(std::string_view name, std::string_view path); ++}; ++} // namespace ipfs ++ ++#endif // IPFS_GENERATED_DIRECTORY_LISTING_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/gw/gateway_request.cc b/third_party/ipfs_client/src/ipfs_client/gw/gateway_request.cc +new file mode 100644 +index 0000000000000..69de795e7ba34 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/gw/gateway_request.cc +@@ -0,0 +1,101 @@ ++#include "ipfs_client/gw/gateway_request.h" ++ ++#include "ipfs_client/response.h" ++ ++#include "log_macros.h" ++ ++#include ++ ++using namespace std::literals; ++ ++using Self = ipfs::gw::GatewayRequest; ++using CidCodec = libp2p::multi::ContentIdentifierCodec; ++ ++std::shared_ptr Self::fromIpfsPath(ipfs::SlashDelimited p) { ++ auto name_space = p.pop(); ++ auto result = std::make_shared(); ++ result->main_param = p.pop(); ++ auto maybe_cid = CidCodec::fromString(result->main_param); ++ if (maybe_cid.has_value()) { ++ result->cid = maybe_cid.value(); ++ } else { ++ result->cid = std::nullopt; ++ } ++ if (name_space == "ipfs") { ++ if (!result->cid.has_value()) { ++ LOG(ERROR) << "IPFS request with invalid/unsupported CID " ++ << result->main_param; ++ } ++ if (result->cid.value().content_address.getType() == ++ libp2p::multi::HashType::identity) { ++ result->type = Type::Identity; ++ } else { ++ result->path = p.pop_all(); ++ result->type = result->path.empty() ? Type::Block : Type::Car; ++ } ++ } else if (name_space == "ipns") { ++ result->path = p.pop_all(); ++ if (CidCodec::fromString(result->main_param).has_value()) { ++ result->type = Type::Ipns; ++ } else { ++ result->type = Type::DnsLink; ++ } ++ } else { ++ LOG(FATAL) << "Unsupported namespace in ipfs path: /" << name_space << '/' ++ << p.pop_all(); ++ } ++ return result; ++} ++ ++std::string Self::url_suffix() const { ++ switch (type) { ++ case Type::Block: ++ return "/ipfs/" + main_param; ++ case Type::Car: ++ return "/ipfs/" + main_param + "/" + path + "?dag-scope=entity"; ++ case Type::Ipns: ++ return "/ipns/" + main_param; ++ case Type::Providers: ++ return "/routing/v1/providers/" + main_param; ++ case Type::DnsLink: ++ LOG(FATAL) << "Don't try to use HTTP(s) for DNS TXT records."; ++ return {}; ++ default: ++ LOG(FATAL) << "Invalid gateway request type: " << static_cast(type); ++ return {}; ++ } ++} ++std::string_view Self::accept() const { ++ switch (type) { ++ case Type::Block: ++ return "application/vnd.ipld.raw"sv; ++ case Type::Ipns: ++ return "application/vnd.ipfs.ipns-record"sv; ++ case Type::Car: ++ return "application/vnd.ipld.car"sv; ++ case Type::Providers: ++ return "application/json"sv; ++ case Type::DnsLink: ++ // TODO : not sure this advice is 100% good, actually. ++ // If the user's system setup allows for text records to actually work, ++ // it would be good to respect their autonomy and try to follow the ++ // system's DNS setup. However, it's extremely easy to get yourself in a ++ // situation where Chromium _cannot_ access text records. If you're in ++ // that scenario, it might be better to try to use an IPFS gateway with ++ // DNSLink capability. ++ LOG(FATAL) << "Don't try to use HTTP(s) for DNS TXT records."; ++ return {}; ++ default: ++ LOG(FATAL) << "Invalid gateway request type: " << static_cast(type); ++ return {}; ++ } ++} ++ ++auto Self::identity_data() const -> std::string_view { ++ if (type != Type::Identity) { ++ return ""; ++ } ++ auto hash = cid.value().content_address.getHash(); ++ auto d = reinterpret_cast(hash.data()); ++ return std::string_view{d, hash.size()}; ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/identity_cid.cc b/third_party/ipfs_client/src/ipfs_client/identity_cid.cc +new file mode 100644 +index 0000000000000..913132966a203 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/identity_cid.cc +@@ -0,0 +1,23 @@ ++#include ++ ++#include ++ ++#include ++ ++namespace Self = ipfs::id_cid; ++namespace m = libp2p::multi; ++ ++auto Self::forText(std::string_view txt) -> Cid { ++ txt = txt.substr(0UL, m::Multihash::kMaxHashLength); ++ auto p = reinterpret_cast(txt.data()); ++ auto b = ByteView{p, txt.size()}; ++ auto mh = m::Multihash::create(m::HashType::identity, b); ++ if (mh.has_value()) { ++ return Cid{Cid::Version::V1, m::MulticodecType::Code::RAW, mh.value()}; ++ } else { ++ LOG(FATAL) ++ << "We really shouldn't be able to fail to 'hash' using identity " ++ << static_cast(mh.error()); ++ return forText("Unreachable"); ++ } ++} +\ No newline at end of file +diff --git a/third_party/ipfs_client/src/ipfs_client/ipfs_request.cc b/third_party/ipfs_client/src/ipfs_client/ipfs_request.cc +new file mode 100644 +index 0000000000000..5e25e6873b37d +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipfs_request.cc +@@ -0,0 +1,19 @@ ++#include ++ ++#include "log_macros.h" ++ ++using Self = ipfs::IpfsRequest; ++ ++// Self::IpfsRequest(std::string path_p) ++// : path_{path_p}, callback_([](auto&, auto&) {}) {} ++Self::IpfsRequest(std::string path_p, Finisher f) ++ : path_{path_p}, callback_{f} {} ++ ++void Self::finish(ipfs::Response& r) { ++ LOG(INFO) << "IpfsRequest::finish(...);"; ++ callback_(*this, r); ++ // TODO - cancel other gw req pointing into this ++ callback_ = [](auto& q, auto&) { ++ LOG(INFO) << "IPFS request " << q.path().pop_all() << " satisfied multiply"; ++ }; ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/chunk.cc b/third_party/ipfs_client/src/ipfs_client/ipld/chunk.cc +new file mode 100644 +index 0000000000000..2eb452f68e445 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/chunk.cc +@@ -0,0 +1,16 @@ ++#include "chunk.h" ++ ++using Chunk = ipfs::ipld::Chunk; ++ ++Chunk::Chunk(std::string data) : data_{data} {} ++Chunk::~Chunk() {} ++ ++auto Chunk::resolve(ipfs::SlashDelimited path, ++ ipfs::ipld::DagNode::BlockLookup, ++ std::string&) -> ResolveResult { ++ if (path) { ++ return ProvenAbsent{}; ++ } ++ return Response{"", 200, data_, {}}; ++} ++ +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/chunk.h b/third_party/ipfs_client/src/ipfs_client/ipld/chunk.h +new file mode 100644 +index 0000000000000..03f41bca8fa1e +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/chunk.h +@@ -0,0 +1,18 @@ ++#ifndef IPFS_CHUNK_H_ ++#define IPFS_CHUNK_H_ ++ ++#include ++ ++namespace ipfs::ipld { ++class Chunk : public DagNode { ++ std::string const data_; ++ ++ ResolveResult resolve(SlashDelimited, BlockLookup, std::string&) override; ++ ++ public: ++ explicit Chunk(std::string); ++ virtual ~Chunk() noexcept; ++}; ++} // namespace ipfs::ipld ++ ++#endif // IPFS_CHUNK_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/dag_node.cc b/third_party/ipfs_client/src/ipfs_client/ipld/dag_node.cc +new file mode 100644 +index 0000000000000..0a92fb5e8cd0b +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/dag_node.cc +@@ -0,0 +1,62 @@ ++#include ++ ++#include "chunk.h" ++#include "directory_shard.h" ++#include "ipns_name.h" ++#include "root.h" ++#include "small_directory.h" ++#include "unixfs_file.h" ++ ++#include ++#include ++ ++#include "log_macros.h" ++ ++using Node = ipfs::ipld::DagNode; ++ ++std::shared_ptr Node::fromBlock(ipfs::Block const& block) { ++ std::shared_ptr result; ++ switch (block.type()) { ++ case Block::Type::FileChunk: ++ result = std::make_shared(block.chunk_data()); ++ break; ++ case Block::Type::Directory: ++ result = std::make_shared(); ++ break; ++ case Block::Type::File: ++ result = std::make_shared(); ++ break; ++ case Block::Type::HAMTShard: ++ if (block.fsdata().has_fanout()) { ++ result = std::make_shared(block.fsdata().fanout()); ++ } else { ++ result = std::make_shared(); ++ } ++ break; ++ case Block::Type::Invalid: ++ LOG(ERROR) << "Invalid block."; ++ return result; ++ default: ++ LOG(FATAL) << "TODO " << static_cast(block.type()); ++ } ++ auto add_link = [&result](auto& n, auto c) { ++ result->links_.emplace_back(n, c); ++ return true; ++ }; ++ block.List(add_link); ++ return result; ++} ++ ++auto Node::fromIpnsRecord(ipfs::ValidatedIpns const& v) -> NodePtr { ++ return std::make_shared(v.value); ++} ++ ++std::shared_ptr Node::deroot() { ++ return shared_from_this(); ++} ++std::shared_ptr Node::rooted() { ++ return std::make_shared(shared_from_this()); ++} ++auto Node::as_hamt() -> DirShard* { ++ return nullptr; ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/directory_shard.cc b/third_party/ipfs_client/src/ipfs_client/ipld/directory_shard.cc +new file mode 100644 +index 0000000000000..4e0032fe4f1e7 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/directory_shard.cc +@@ -0,0 +1,102 @@ ++#include "directory_shard.h" ++ ++#include "log_macros.h" ++ ++#include ++ ++#include ++ ++#define ABSL_USES_STD_STRING_VIEW 1 ++#include ++ ++#include ++#include ++#include ++ ++using namespace std::literals; ++ ++using Self = ipfs::ipld::DirShard; ++ ++auto Self::resolve(ipfs::SlashDelimited relpath, ++ ipfs::ipld::DagNode::BlockLookup blu, ++ std::string& to_here) -> ResolveResult { ++ if (!relpath) { ++ // TODO check if index.html is present and if not implement indexing ++ auto result = resolve("index.html"sv, blu, to_here); ++ auto resp = std::get_if(&result); ++ if (resp) { ++ resp->mime_ = "text/html"; ++ } ++ return result; ++ } ++ auto name = relpath.pop(); ++ auto hash = hexhash(name); ++ return resolve_internal(hash.begin(), hash.end(), name, relpath, blu, ++ to_here); ++} ++auto Self::resolve_internal(ipfs::ipld::DirShard::HashIter hash_b, ++ ipfs::ipld::DirShard::HashIter hash_e, ++ std::string_view element_name, ++ ipfs::SlashDelimited path_after_dir, ++ ipfs::ipld::DagNode::BlockLookup blu, ++ std::string& path_to_dir) -> ResolveResult { ++ for (auto& [name, link] : links_) { ++ if (hash_b != hash_e && !absl::StartsWith(name, *hash_b)) { ++ continue; ++ } ++ if (!link.node) { ++ link.node = blu(link.cid); ++ } ++ if (!link.node) { ++ return MoreDataNeeded{std::vector{"/ipfs/" + link.cid}}; ++ } ++ if (absl::EndsWith(name, element_name)) { ++ return link.node->resolve(path_after_dir, blu, path_to_dir); ++ } ++ auto downcast = link.node->as_hamt(); ++ if (downcast) { ++ if (hash_b == hash_e) { ++ LOG(ERROR) << "Ran out of hash bits."; ++ return ProvenAbsent{}; ++ } ++ return downcast->resolve_internal(std::next(hash_b), hash_e, element_name, ++ path_after_dir, blu, path_to_dir); ++ } else { ++ return ProvenAbsent{}; ++ } ++ } ++ return ProvenAbsent{}; ++} ++std::vector Self::hexhash(std::string_view path_element) const { ++ auto hex_width = 0U; ++ for (auto x = fanout_; (x >>= 4); ++hex_width) ++ ; ++ std::array digest = {0U, 0U}; ++ MurmurHash3_x64_128(path_element.data(), path_element.size(), 0, ++ digest.data()); ++ auto corrected_digest = htobe64(digest[0]); ++ VLOG(1) << "Hash: " << digest[0] << ' ' << digest[1] << " -> " ++ << corrected_digest; ++ std::vector result; ++ for (auto d : digest) { ++ auto hash_bits = htobe64(d); ++ while (hash_bits) { ++ // 2. Pop the log2(fanout_) lowest bits from the path component hash ++ // digest,... ++ auto popped = hash_bits % fanout_; ++ hash_bits /= fanout_; ++ std::ostringstream oss; ++ // ... then hex encode (using 0-F) using little endian those bits ... ++ oss << std::setfill('0') << std::setw(hex_width) << std::uppercase ++ << std::hex << popped; ++ result.push_back(oss.str()); ++ } ++ } ++ return result; ++} ++ ++Self::DirShard(std::uint64_t fanout) : fanout_{fanout} {} ++Self::~DirShard() {} ++Self* Self::as_hamt() { ++ return this; ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/directory_shard.h b/third_party/ipfs_client/src/ipfs_client/ipld/directory_shard.h +new file mode 100644 +index 0000000000000..0a4a6ccaa9562 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/directory_shard.h +@@ -0,0 +1,28 @@ ++#ifndef IPFS_DIRECTORY_SHARD_H_ ++#define IPFS_DIRECTORY_SHARD_H_ 1 ++ ++#include ++ ++namespace ipfs::ipld { ++class DirShard : public DagNode { ++ std::uint64_t const fanout_; ++ ++ ResolveResult resolve(SlashDelimited, BlockLookup, std::string&) override; ++ DirShard* as_hamt() override; ++ ++ std::vector hexhash(std::string_view path_element) const; ++ using HashIter = std::vector::const_iterator; ++ ResolveResult resolve_internal(HashIter, ++ HashIter, ++ std::string_view, ++ SlashDelimited, ++ BlockLookup, ++ std::string&); ++ ++ public: ++ explicit DirShard(std::uint64_t fanout = 256UL); ++ virtual ~DirShard() noexcept; ++}; ++} // namespace ipfs::ipld ++ ++#endif // IPFS_DIRECTORY_SHARD_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/ipns_name.cc b/third_party/ipfs_client/src/ipfs_client/ipld/ipns_name.cc +new file mode 100644 +index 0000000000000..9b77e55c697c0 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/ipns_name.cc +@@ -0,0 +1,22 @@ ++#include "ipns_name.h" ++ ++#include "log_macros.h" ++ ++using Self = ipfs::ipld::IpnsName; ++ ++Self::IpnsName(std::string target_abs_path) : target_path_{target_abs_path} {} ++ ++auto Self::resolve(ipfs::SlashDelimited path, ++ ipfs::ipld::DagNode::BlockLookup blu, ++ std::string& up_to_here) -> ResolveResult { ++ if (!target_) { ++ SlashDelimited t{target_path_}; ++ t.pop(); // Discard namespace, though realistically it's going to be ipfs ++ // basically all the time ++ target_ = blu(std::string{t.pop()}); ++ } ++ if (target_) { ++ return target_->resolve(path, blu, up_to_here); ++ } ++ return MoreDataNeeded{std::vector{target_path_}}; ++} +\ No newline at end of file +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/ipns_name.h b/third_party/ipfs_client/src/ipfs_client/ipld/ipns_name.h +new file mode 100644 +index 0000000000000..33648483d7ed5 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/ipns_name.h +@@ -0,0 +1,21 @@ ++#ifndef IPFS_IPLD_IPNS_NAME_H_ ++#define IPFS_IPLD_IPNS_NAME_H_ ++ ++#include "ipfs_client/ipld/dag_node.h" ++ ++namespace ipfs::ipld { ++class IpnsName : public DagNode { ++ std::string const target_path_; ++ NodePtr target_; ++ ++ ResolveResult resolve(SlashDelimited path, ++ BlockLookup, ++ std::string& up_to_here) override; ++ ++ public: ++ IpnsName(std::string target_abs_path); ++ virtual ~IpnsName() noexcept {} ++}; ++} // namespace ipfs::ipld ++ ++#endif // IPFS_IPLD_IPNS_NAME_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/link.cc b/third_party/ipfs_client/src/ipfs_client/ipld/link.cc +new file mode 100644 +index 0000000000000..f9dad98e58840 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/link.cc +@@ -0,0 +1,6 @@ ++#include "ipfs_client/ipld/link.h" ++ ++using Self = ipfs::ipld::Link; ++ ++Self::Link(std::string cid_s) : cid{cid_s} {} ++Self::Link(std::string s, std::shared_ptr n) : cid{s}, node{n} {} +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/root.cc b/third_party/ipfs_client/src/ipfs_client/ipld/root.cc +new file mode 100644 +index 0000000000000..799c4ee31c6c9 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/root.cc +@@ -0,0 +1,78 @@ ++#include "root.h" ++ ++#include "log_macros.h" ++ ++using namespace std::literals; ++ ++using Self = ipfs::ipld::Root; ++using Ptr = std::shared_ptr; ++ ++Self::Root(std::shared_ptr under) { ++ links_.push_back({{}, Link{{}, under}}); ++} ++Self::~Root() {} ++ ++Ptr Self::deroot() { ++ return links_.at(0).second.node; ++} ++Ptr Self::rooted() { ++ return shared_from_this(); ++} ++ ++auto Self::resolve(SlashDelimited path, BlockLookup blu, std::string& to_here) ++ -> ResolveResult { ++ auto result = deroot()->resolve(path, blu, to_here); ++ if (std::get_if(&result)) { ++ auto missing_path = path.to_string(); ++ if (path.pop() == "_redirects") { ++ return result; ++ } ++ if (!redirects_.has_value()) { ++ result = deroot()->resolve("_redirects"sv, blu, to_here); ++ auto redirect_resp = std::get_if(&result); ++ if (redirect_resp && redirect_resp->status_ == 200) { ++ redirects_ = redirects::File(redirect_resp->body_); ++ } else { ++ // Either this is ProvenAbsent, in which case this will be interpreted ++ // as the original ProvenAbsent Or it's MoreDataNeeded but for ++ // _redirects, which is what we need now ++ return result; ++ } ++ } ++ if (redirects_.has_value() && redirects_.value().valid()) { ++ Response* resp = nullptr; ++ auto status = redirects_.value().rewrite(missing_path); ++ switch (status / 100) { ++ case 0: // no rewrites available ++ return result; ++ case 2: ++ return resolve(std::string_view{missing_path}, blu, to_here); ++ case 3: ++ // Let the redirect happen ++ return Response{"", static_cast(status), "", ++ missing_path}; ++ case 4: ++ result = ++ deroot()->resolve(std::string_view{missing_path}, blu, to_here); ++ if (std::get_if(&result)) { ++ return Response{"", 500, "", missing_path}; ++ } ++ resp = std::get_if(&result); ++ if (resp) { ++ resp->status_ = status; ++ return *resp; ++ } ++ return result; // MoreDataNeeded to fetch e.g. custom 404 page ++ default: ++ LOG(ERROR) << "Unsupported status came back from _redirects file: " ++ << status; ++ } ++ } ++ return ProvenAbsent{}; ++ } ++ auto resp = std::get_if(&result); ++ if (resp && resp->location_.empty()) { ++ resp->location_ = path.to_string(); ++ } ++ return result; ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/root.h b/third_party/ipfs_client/src/ipfs_client/ipld/root.h +new file mode 100644 +index 0000000000000..458ab4d34855e +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/root.h +@@ -0,0 +1,25 @@ ++#ifndef IPFS_ROOT_H_ ++#define IPFS_ROOT_H_ ++ ++#include ++#include ++ ++#include ++ ++namespace ipfs::ipld { ++class Root : public DagNode { ++ std::optional redirects_; ++ ++ ResolveResult resolve(SlashDelimited path, ++ BlockLookup, ++ std::string&) override; ++ std::shared_ptr rooted() override; ++ std::shared_ptr deroot() override; ++ ++ public: ++ Root(std::shared_ptr); ++ virtual ~Root() noexcept; ++}; ++} // namespace ipfs::ipld ++ ++#endif // IPFS_ROOT_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/small_directory.cc b/third_party/ipfs_client/src/ipfs_client/ipld/small_directory.cc +new file mode 100644 +index 0000000000000..fbf6c93869744 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/small_directory.cc +@@ -0,0 +1,67 @@ ++#include "small_directory.h" ++ ++#include "ipfs_client/generated_directory_listing.h" ++#include "ipfs_client/path2url.h" ++ ++#include "log_macros.h" ++ ++using namespace std::literals; ++ ++using Self = ipfs::ipld::SmallDirectory; ++ ++namespace { ++ipfs::ipld::NodePtr& node(ipfs::ipld::Link& l, ++ ipfs::ipld::DagNode::BlockLookup f) { ++ if (!l.node) { ++ l.node = f(l.cid); ++ } ++ return l.node; ++} ++} // namespace ++ ++auto Self::resolve(SlashDelimited path, BlockLookup blu, std::string& to_here) ++ -> ResolveResult { ++ if (!path) { ++ GeneratedDirectoryListing index_html{path2url(to_here)}; ++ for (auto& [name, link] : links_) { ++ LOG(INFO) << "Listing " << to_here << " encountered " << name << '=' ++ << link.cid; ++ if (name == "index.html") { ++ auto& n = node(link, blu); ++ if (n) { ++ auto result = n->resolve(""sv, blu, to_here.append("/index.html")); ++ auto resp = std::get_if(&result); ++ if (resp) { ++ resp->mime_ = "text/html"; ++ } ++ return result; ++ } else { ++ return MoreDataNeeded{std::vector{"/ipfs/" + link.cid}}; ++ } ++ } else { ++ index_html.AddEntry(name); ++ } ++ } ++ return Response{"text/html", 200, index_html.Finish(), ""}; ++ } ++ auto name = path.pop(); ++ // TODO binary search ++ auto match = [&name](auto& l) { return l.first == name; }; ++ auto it = std::find_if(links_.begin(), links_.end(), match); ++ if (links_.end() == it) { ++ return ProvenAbsent{}; ++ } ++ auto& link = it->second; ++ auto& nod = node(link, blu); ++ if (nod) { ++ if (to_here.back() != '/') { ++ to_here.push_back('/'); ++ } ++ to_here.append(name); ++ return nod->resolve(path, blu, to_here); ++ } else { ++ return MoreDataNeeded{std::vector{"/ipfs/" + link.cid}}; ++ } ++} ++ ++Self::~SmallDirectory() {} +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/small_directory.h b/third_party/ipfs_client/src/ipfs_client/ipld/small_directory.h +new file mode 100644 +index 0000000000000..b78729f8f8fd3 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/small_directory.h +@@ -0,0 +1,17 @@ ++#ifndef IPFS_UNIXFS_DIRECTORY_H_ ++#define IPFS_UNIXFS_DIRECTORY_H_ ++ ++#include "ipfs_client/ipld/link.h" ++ ++#include ++ ++namespace ipfs::ipld { ++class SmallDirectory : public DagNode { ++ ResolveResult resolve(SlashDelimited, BlockLookup, std::string&) override; ++ ++ public: ++ virtual ~SmallDirectory() noexcept; ++}; ++} // namespace ipfs::ipld ++ ++#endif // IPFS_UNIXFS_DIRECTORY_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/unixfs_file.cc b/third_party/ipfs_client/src/ipfs_client/ipld/unixfs_file.cc +new file mode 100644 +index 0000000000000..4faa064664b3b +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/unixfs_file.cc +@@ -0,0 +1,47 @@ ++#include "unixfs_file.h" ++ ++using namespace std::literals; ++ ++using Self = ipfs::ipld::UnixfsFile; ++ ++auto Self::resolve(ipfs::SlashDelimited path, ++ ipfs::ipld::DagNode::BlockLookup blu, ++ std::string& to_here) -> ResolveResult { ++ if (path) { ++ // You can't path through a file. ++ return ProvenAbsent{}; ++ } ++ std::vector missing; ++ std::string body; ++ for (auto& child : links_) { ++ auto& link = child.second; ++ if (!link.node) { ++ link.node = blu(link.cid); ++ } ++ if (link.node) { ++ auto recurse = link.node->resolve(""sv, blu, to_here); ++ auto mdn = std::get_if(&recurse); ++ if (mdn) { ++ missing.insert(missing.end(), mdn->ipfs_abs_paths_.begin(), ++ mdn->ipfs_abs_paths_.end()); ++ continue; ++ } ++ if (missing.empty()) { ++ body.append(std::get(recurse).body_); ++ } ++ } else { ++ missing.push_back("/ipfs/" + link.cid); ++ } ++ } ++ if (missing.empty()) { ++ return Response{ ++ "", ++ 200, ++ body, ++ ""s, ++ }; ++ } ++ return MoreDataNeeded{missing}; ++} ++ ++Self::~UnixfsFile() {} +diff --git a/third_party/ipfs_client/src/ipfs_client/ipld/unixfs_file.h b/third_party/ipfs_client/src/ipfs_client/ipld/unixfs_file.h +new file mode 100644 +index 0000000000000..bedd5c1dd2862 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipld/unixfs_file.h +@@ -0,0 +1,15 @@ ++#ifndef IPFS_UNIXFS_FILE_H_ ++#define IPFS_UNIXFS_FILE_H_ ++ ++#include ++ ++namespace ipfs::ipld { ++class UnixfsFile : public DagNode { ++ ResolveResult resolve(SlashDelimited, BlockLookup, std::string&) override; ++ ++ public: ++ virtual ~UnixfsFile() noexcept; ++}; ++} // namespace ipfs::ipld ++ ++#endif // IPFS_UNIXFS_FILE_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/ipns_names.cc b/third_party/ipfs_client/src/ipfs_client/ipns_names.cc +new file mode 100644 +index 0000000000000..0347c88c2fa2e +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipns_names.cc +@@ -0,0 +1,112 @@ ++#include ++ ++#include ++#include "log_macros.h" ++ ++using Self = ipfs::IpnsNames; ++ ++void Self::NoSuchName(std::string const& name) { ++ names_[name]; // If it already exists, leave it. ++} ++void Self::AssignName(std::string const& name, ValidatedIpns entry) { ++ auto& res = entry.value; ++ if (res.size() && res.front() == '/') { ++ res.erase(0, 1); ++ } ++ auto endofcid = res.find_first_of("/?#", 6); ++ using namespace libp2p::multi; ++ auto cid_str = res.substr(5, endofcid); ++ LOG(INFO) << "IPNS points to CID " << cid_str; ++ auto dec_res = ContentIdentifierCodec::fromString(cid_str); ++ if (dec_res.has_value()) { ++ auto cid = dec_res.value(); ++ if (dec_res.value().version == ContentIdentifier::Version::V0) { ++ // TODO - implement ipns properly. A peer ID could actually fail this ++ // check, I believe. ++ DCHECK_EQ(res.substr(0, 5), "ipfs/"); ++ cid = ++ ContentIdentifier(ContentIdentifier::Version::V1, ++ MulticodecType::Code::DAG_PB, cid.content_address); ++ } ++ auto enc_res = ContentIdentifierCodec::toStringOfBase( ++ cid, MultibaseCodec::Encoding::BASE32_LOWER); ++ DCHECK(enc_res.has_value()); ++ auto desensitized = res.substr(0, 5); ++ desensitized.append(enc_res.value()); ++ if (endofcid < res.size()) { ++ auto extra = res.substr(endofcid); ++ LOG(INFO) << name << " resolution contains oddity '" << extra; ++ desensitized.append(extra); ++ } ++ LOG(INFO) << name << " now resolves to (desensitized)" << desensitized; ++ entry.value = desensitized; ++ } else { ++ LOG(INFO) << name << " now resolves to (extra level)" << res; ++ } ++ auto it = names_.find(name); ++ if (it == names_.end()) { ++ names_.emplace(name, std::move(entry)); ++ } else if (it->second.sequence < entry.sequence) { ++ LOG(INFO) << "Updating IPNS record for " << name << " from sequence " ++ << it->second.sequence << " where it pointed to " ++ << it->second.value << " to sequence " << entry.sequence ++ << " where it points to " << entry.value; ++ it->second = std::move(entry); ++ } else { ++ LOG(INFO) << "Discarding redundant IPNS record for " << name; ++ } ++} ++void Self::AssignDnsLink(std::string const& name, std::string_view target) { ++ ValidatedIpns v; ++ v.value.assign(target); ++ auto t = std::time(nullptr); ++ v.use_until = v.cache_until = t + 300; ++ AssignName(name, std::move(v)); ++} ++ ++std::string_view Self::NameResolvedTo(std::string_view original_name) const { ++ std::string name{original_name}; ++ std::string_view prev = ""; ++ auto trailer = names_.end(); ++ auto trail_step = false; ++ auto now = std::time(nullptr); ++ while (true) { ++ auto it = names_.find(name); ++ if (names_.end() == it) { ++ LOG(INFO) << "Host not in immediate access map: " << name << " (" ++ << std::string{original_name} << ')'; ++ return prev; ++ } else if (it == trailer) { ++ LOG(ERROR) << "Host cycle found in IPNS: " << std::string{original_name} ++ << ' ' << name; ++ return ""; ++ } ++ auto& target = it->second.value; ++ if (target.empty()) { ++ return kNoSuchName; ++ } ++ if (target.at(2) == 'f') { ++ return target; ++ } ++ if (it->second.use_until < now) { ++ return prev; ++ } ++ if (trail_step) { ++ if (trailer == names_.end()) { ++ trailer = names_.find(name); ++ } else { ++ trailer = names_.find(trailer->second.value.substr(5)); ++ } ++ } ++ trail_step = !trail_step; ++ prev = it->second.value; ++ name.assign(prev, 5); ++ } ++} ++auto Self::Entry(std::string const& name) -> ValidatedIpns const* { ++ auto it = names_.find(name); ++ return it == names_.end() ? nullptr : &(it->second); ++} ++ ++Self::IpnsNames() {} ++Self::~IpnsNames() {} +diff --git a/third_party/ipfs_client/src/ipfs_client/ipns_record.cc b/third_party/ipfs_client/src/ipfs_client/ipns_record.cc +new file mode 100644 +index 0000000000000..73598b6b92838 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/ipns_record.cc +@@ -0,0 +1,157 @@ ++#include ++ ++#include "log_macros.h" ++ ++#if __has_include() ++#include ++#else ++#include "ipfs_client/ipns_record.pb.h" ++#endif ++ ++// #include ++#include ++#include ++ ++namespace { ++bool matches(libp2p::multi::Multihash const& hash, ++ ipfs::ByteView pubkey_bytes) { ++ auto hasher = libp2p::crypto::CreateHasher(hash.getType()); ++ std::vector result(hasher->digestSize(), ipfs::Byte{}); ++ if (hasher->write(pubkey_bytes).value()) { ++ if (!hasher->digestOut({result.data(), result.size()}).has_value()) { ++ LOG(ERROR) << "Error getting digest."; ++ } ++ } else { ++ LOG(ERROR) << "Attempt to hash bytes returned false"; ++ } ++ return std::equal(result.begin(), result.end(), hash.getHash().begin(), ++ hash.getHash().end()); ++} ++} // namespace ++auto ipfs::ValidateIpnsRecord(ByteView top_level_bytes, ++ libp2p::peer::PeerId const& name, ++ CryptoSignatureVerifier verify, ++ CborDeserializer dser) ++ -> std::optional { ++ // https://github.com/ipfs/specs/blob/main/ipns/IPNS.md#record-verification ++ ++ // Before parsing the protobuf, confirm that the serialized IpnsEntry bytes ++ // sum to less than or equal to the size limit. ++ if (top_level_bytes.size() > 10240UL) { ++ LOG(ERROR) << "IPNS record too large: " << top_level_bytes.size(); ++ return {}; ++ } ++ ++ ipfs::ipns::IpnsEntry entry; ++ if (!entry.ParseFromArray(top_level_bytes.data(), top_level_bytes.size())) { ++ LOG(ERROR) << "Failed to parse top-level bytes as a protobuf"; ++ return {}; ++ } ++ ++ // Confirm IpnsEntry.signatureV2 and IpnsEntry.data are present and are not ++ // empty ++ if (!entry.has_signaturev2()) { ++ LOG(ERROR) << "IPNS record contains no .signatureV2!"; ++ return {}; ++ } ++ if (!entry.has_data() || entry.data().empty()) { ++ LOG(ERROR) << "IPNS record has no .data"; ++ return {}; ++ } ++ ++ // The only supported value is 0, which indicates the validity field contains ++ // the expiration date after which the IPNS record becomes invalid. ++ DCHECK_EQ(entry.validitytype(), 0); ++ ++ auto parsed = dser({reinterpret_cast(entry.data().data()), ++ entry.data().size()}); ++ if (parsed.value != entry.value()) { ++ LOG(ERROR) << "CBOR contains value '" << parsed.value ++ << "' but PB contains value '" << entry.value() << "'!"; ++ return {}; ++ } ++ ipfs::ByteView public_key; ++ if (entry.has_pubkey()) { ++ public_key = ipfs::ByteView{ ++ reinterpret_cast(entry.pubkey().data()), ++ entry.pubkey().size()}; ++ if (!matches(name.toMultihash(), public_key)) { ++ LOG(ERROR) << "Given IPNS record contains a pubkey that does not match " ++ "the hash from the IPNS name that fetched it!"; ++ return {}; ++ } ++ } else if (name.toMultihash().getType() == ++ libp2p::multi::HashType::identity) { ++ public_key = name.toMultihash().getHash(); ++ } else { ++ LOG(ERROR) << "IPNS record contains no public key, and the IPNS name " ++ << name.toMultihash().toHex() ++ << " is a true hash, not identity. Validation impossible."; ++ return {}; ++ } ++ ipfs::ipns::PublicKey pk; ++ auto* pkbp = reinterpret_cast(public_key.data()); ++ if (!pk.ParseFromArray(pkbp, public_key.size())) { ++ LOG(ERROR) << "Failed to parse public key bytes"; ++ return {}; ++ } ++ LOG(INFO) << "Record contains a public key of type " << pk.type() ++ << " and points to " << entry.value(); ++ auto& signature_str = entry.signaturev2(); ++ ByteView signature{reinterpret_cast(signature_str.data()), ++ signature_str.size()}; ++ // https://specs.ipfs.tech/ipns/ipns-record/#record-verification ++ // Create bytes for signature verification by concatenating ipns-signature: ++ // prefix (bytes in hex: 69706e732d7369676e61747572653a) with raw CBOR bytes ++ // from IpnsEntry.data ++ auto bytes_str = entry.data(); ++ bytes_str.insert( ++ 0, "\x69\x70\x6e\x73\x2d\x73\x69\x67\x6e\x61\x74\x75\x72\x65\x3a"); ++ ByteView bytes{reinterpret_cast(bytes_str.data()), ++ bytes_str.size()}; ++ ByteView key_bytes{reinterpret_cast(pk.data().data()), ++ pk.data().size()}; ++ if (verify(pk.type(), signature, bytes, key_bytes)) { ++ LOG(INFO) << "Verification passed."; ++ return parsed; ++ } else { ++ LOG(ERROR) << "Verification failed!!"; ++ return {}; ++ } ++} ++ ++ipfs::ValidatedIpns::ValidatedIpns() = default; ++ipfs::ValidatedIpns::ValidatedIpns(ValidatedIpns&&) = default; ++ipfs::ValidatedIpns::ValidatedIpns(ValidatedIpns const&) = default; ++auto ipfs::ValidatedIpns::operator=(ValidatedIpns const&) ++ -> ValidatedIpns& = default; ++ipfs::ValidatedIpns::ValidatedIpns(IpnsCborEntry const& e) ++ : value{e.value}, sequence{e.sequence} { ++ std::istringstream ss{e.validity}; ++ std::tm t; ++ ss >> std::get_time(&t, "%Y-%m-%dT%H:%M:%S"); ++ long ttl = (e.ttl / 1'000'000'000UL) + 1; ++ use_until = std::mktime(&t); ++ cache_until = std::time(nullptr) + ttl; ++ if (use_until < cache_until) { ++ LOG(WARNING) << "IPNS record expiring!"; ++ use_until = cache_until; ++ } ++} ++ ++std::string ipfs::ValidatedIpns::Serialize() const { ++ DCHECK_EQ(value.find(' '), std::string::npos); ++ DCHECK_EQ(gateway_source.find(' '), std::string::npos); ++ std::ostringstream ss; ++ ss << std::hex << sequence << ' ' << use_until << ' ' << cache_until << ' ' ++ << fetch_time << ' ' << resolution_ms << ' ' << value << ' ' ++ << gateway_source; ++ return ss.str(); ++} ++auto ipfs::ValidatedIpns::Deserialize(std::string s) -> ValidatedIpns { ++ std::istringstream ss(s); ++ ValidatedIpns e; ++ ss >> std::hex >> e.sequence >> e.use_until >> e.cache_until >> ++ e.fetch_time >> e.resolution_ms >> e.value >> e.gateway_source; ++ return e; ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/logger.cc b/third_party/ipfs_client/src/ipfs_client/logger.cc +new file mode 100644 +index 0000000000000..ce3db73816c0b +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/logger.cc +@@ -0,0 +1,74 @@ ++#include ++ ++#include ++ ++#include ++ ++namespace lg = ipfs::log; ++ ++namespace { ++lg::Level current_level = lg::Level::WARN; ++lg::Handler current_handler = nullptr; ++ ++void CheckLevel(google::protobuf::LogLevel lv, ++ char const* f, ++ int l, ++ std::string const& m) { ++ auto lev = static_cast(lv); ++ if (lev < static_cast(current_level)) { ++ return; ++ } ++ if (!current_handler) { ++ return; ++ } ++ current_handler(m, f, l, static_cast(lev)); ++} ++ ++} // namespace ++ ++void lg::SetLevel(Level lev) { ++ current_level = lev; ++} ++ ++void lg::SetHandler(Handler h) { ++ current_handler = h; ++ google::protobuf::SetLogHandler(&CheckLevel); ++} ++ ++void lg::DefaultHandler(std::string const& message, ++ char const* source_file, ++ int source_line, ++ Level lev) { ++ std::clog << source_file << ':' << source_line << ": " << LevelDescriptor(lev) ++ << ": " << message << '\n'; ++ if (lev == Level::FATAL) { ++ std::abort(); ++ } ++} ++ ++std::string_view lg::LevelDescriptor(Level l) { ++ switch (l) { ++ case Level::TRACE: ++ return "trace"; ++ case Level::DEBUG: ++ return "debug"; ++ case Level::INFO: ++ return "note"; // The next 3 are gcc- & clang-inspired ++ case Level::WARN: ++ return "warning"; ++ case Level::ERROR: ++ return "error"; ++ case Level::FATAL: ++ return " ### FATAL ERROR ### "; ++ default: ++ return "Unknown log level used: possible corruption?"; ++ } ++} ++ ++bool lg::IsInitialized() { ++ if (current_handler) { ++ return true; ++ } ++ SetHandler(&DefaultHandler); ++ return false; ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/orchestrator.cc b/third_party/ipfs_client/src/ipfs_client/orchestrator.cc +new file mode 100644 +index 0000000000000..de6d2e6a8aa56 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/orchestrator.cc +@@ -0,0 +1,108 @@ ++#include "ipfs_client/orchestrator.h" ++ ++#include ++ ++#include "ipld/chunk.h" ++ ++#include "log_macros.h" ++#include "path2url.h" ++ ++using namespace std::literals; ++ ++using Self = ipfs::Orchestrator; ++ ++Self::Orchestrator(GatewayAccess ga, MimeDetection mimer) ++ : gw_requestor_{ga}, mimer_{mimer} {} ++ ++void Self::build_response(std::shared_ptr req) { ++ auto req_path = req->path(); ++ LOG(INFO) << "build_response(" << req_path.to_string() << ')'; ++ req_path.pop(); // namespace ++ std::string origin{req_path.pop()}; ++ auto it = dags_.find(origin); ++ if (dags_.end() == it) { ++ if (gw_request(req, req->path())) { ++ build_response(req); ++ } ++ } else { ++ from_tree(req, it->second, req_path); ++ } ++} ++void Self::from_tree(std::shared_ptr req, ++ ipfs::ipld::NodePtr& node, ++ SlashDelimited relative_path) { ++ auto root = node->rooted(); ++ auto block_look_up = [this](auto& k) { ++ auto i = dags_.find(k); ++ return i == dags_.end() ? ipld::NodePtr{} : i->second; ++ }; ++ auto start = std::string{req->path().pop_n(2)}; ++ auto result = root->resolve(relative_path, block_look_up, start); ++ auto response = std::get_if(&result); ++ if (response) { ++ if (response->mime_.empty() && !response->body_.empty()) { ++ if (response->location_.empty()) { ++ response->mime_ = sniff(req->path(), response->body_); ++ } else { ++ std::string hit_path{req->path().pop_n(2)}; ++ if (hit_path.back() != '/' && response->location_.front() != '/') { ++ hit_path.push_back('/'); ++ } ++ hit_path.append(response->location_); ++ response->mime_ = sniff(SlashDelimited{hit_path}, response->body_); ++ } ++ } ++ // TODO is this necessary? It might be convenient to have this info ++ // available, even if it's usually redundant. ++ // if (response->status_ / 100 != 3) { ++ // response->location_.clear(); ++ // } ++ req->finish(*response); ++ } else if (std::get_if(&result)) { ++ req->finish(Response::PLAIN_NOT_FOUND); ++ } else { ++ for (auto& path : std::get(result).ipfs_abs_paths_) { ++ if (gw_request(req, std::string_view{path})) { ++ from_tree(req, node, relative_path); ++ return; ++ } ++ } ++ } ++} ++bool Self::gw_request(std::shared_ptr ir, ++ ipfs::SlashDelimited path) { ++ auto req = gw::GatewayRequest::fromIpfsPath(path); ++ if (req->type == gw::Type::Identity) { ++ auto node = ++ std::make_shared(std::string{req->identity_data()}); ++ add_node(req->main_param, node); ++ return true; ++ } ++ req->dependent = ir; ++ req->orchestrator = shared_from_this(); ++ gw_requestor_(req); ++ if (req->type == gw::Type::Car) { ++ gw_request(ir, path.pop_n(2)); ++ } ++ return false; ++} ++ ++void Self::add_node(std::string key, ipfs::ipld::NodePtr p) { ++ if (p) { ++ LOG(INFO) << "add_node(" << key << ')'; ++ dags_[key] = p; ++ } else { ++ LOG(ERROR) << "NULL block attempted to be added for " << key; ++ } ++} ++ ++std::string Self::sniff(ipfs::SlashDelimited p, std::string const& body) const { ++ auto fake_url = path2url(p.to_string()); ++ auto file_name = p.peek_back(); ++ auto dot = file_name.find_last_of('.'); ++ std::string_view ext = ""; ++ if (dot < file_name.size()) { ++ ext = file_name.substr(dot + 1); ++ } ++ return mimer_(std::string{ext}, body, fake_url); ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/path2url.cc b/third_party/ipfs_client/src/ipfs_client/path2url.cc +new file mode 100644 +index 0000000000000..0d7cf305a47b4 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/path2url.cc +@@ -0,0 +1,16 @@ ++#include "path2url.h" ++ ++#include "log_macros.h" ++ ++std::string ipfs::path2url(std::string p) { ++ while (!p.empty() && p[0] == '/') { ++ p.erase(0UL, 1UL); ++ } ++ DCHECK_EQ(p.at(0), 'i'); ++ DCHECK_EQ(p.at(1), 'p'); ++ DCHECK(p.at(2) == 'f' || p.at(2) == 'n'); ++ DCHECK_EQ(p.at(3), 's'); ++ DCHECK_EQ(p.at(4), '/'); ++ p.insert(4, ":/"); ++ return p; ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/path2url.h b/third_party/ipfs_client/src/ipfs_client/path2url.h +new file mode 100644 +index 0000000000000..683e92d759b4e +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/path2url.h +@@ -0,0 +1,10 @@ ++#ifndef IPFS_PATH2URL_H_ ++#define IPFS_PATH2URL_H_ ++ ++#include ++ ++namespace ipfs { ++std::string path2url(std::string path_as_string); ++} ++ ++#endif // IPFS_PATH2URL_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/redirects.cc b/third_party/ipfs_client/src/ipfs_client/redirects.cc +new file mode 100644 +index 0000000000000..86f78ad432cc2 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/redirects.cc +@@ -0,0 +1,249 @@ ++#include "redirects.h" ++ ++#include "log_macros.h" ++ ++#include ++ ++#include ++ ++namespace r = ipfs::redirects; ++using namespace std::literals; ++ ++namespace { ++// 2.4.4 Max File Size ++// The file size must not exceed 64 KiB. ++constexpr std::size_t MAX_SIZE = 64UL * 1024UL * 1024UL; ++ ++// Not including \n which terminates lines ++constexpr std::string_view WHITESPACE = " \t\f\r\v\n"; ++ ++// https://specs.ipfs.tech/http-gateways/web-redirects-file/#status ++constexpr int DEFAULT_STATUS = 301; ++// https://specs.ipfs.tech/http-gateways/web-redirects-file/#error-handling ++constexpr int PARSE_ERROR_STATUS = 500; ++} // namespace ++ ++r::Directive::Directive(std::string_view from, std::string_view to, int status) ++ : to_{to}, status_{status} { ++ SlashDelimited comp_str_s{from}; ++ while (comp_str_s) { ++ auto comp_str = comp_str_s.pop(); ++ if (comp_str.empty()) { ++ LOG(ERROR) << "Got empty slash-delimited component. Should not have."; ++ return; ++ } else if (comp_str == "*") { ++ components_.emplace_back(ComponentType::SPLAT, comp_str); ++ } else if (comp_str[0] == ':') { ++ components_.emplace_back(ComponentType::PLACEHOLDER, comp_str); ++ } else { ++ components_.emplace_back(ComponentType::LITERAL, comp_str); ++ } ++ } ++} ++std::uint16_t r::Directive::rewrite(std::string& path) const { ++ auto input = SlashDelimited{path}; ++ auto result = to_; ++ auto replace = [&result](std::string_view ph, std::string_view val) { ++ std::size_t pos; ++ while ((pos = result.find(ph)) < result.size()) { ++ result.replace(pos, ph.size(), val); ++ } ++ }; ++ for (auto [type, comp_str] : components_) { ++ if (!input) { ++ VLOG(2) << "Ran out of input in [" << path ++ << "] before running out of pattern components to match against " ++ "(was looking for [" ++ << comp_str << "]. Not a match."; ++ return 0; ++ } ++ if (type == ComponentType::LITERAL) { ++ if (comp_str != input.pop()) { ++ return 0; ++ } ++ } else if (type == ComponentType::PLACEHOLDER) { ++ replace(comp_str, input.pop()); ++ } else { ++ replace(":splat"sv, input.pop_all()); ++ } ++ } ++ if (input) { ++ return 0; ++ } else { ++ path = result; ++ return status_; ++ } ++} ++std::string r::Directive::error() const { ++ if (status_ < 200 || status_ > 451) { ++ return "UNSUPPORTED STATUS " + std::to_string(status_); ++ } ++ if (components_.empty()) { ++ return "Empty directive pattern"; ++ } ++ if (to_.empty()) { ++ return "Empty redirect target location"; ++ } ++ if (to_.at(0) != '/') { ++ return "Location must begin with /"; ++ } ++ return {}; ++} ++ ++std::uint16_t r::File::rewrite(std::string& missing_path) const { ++ for (auto& directive : directives_) { ++ auto status = directive.rewrite(missing_path); ++ if (status) { ++ return status; ++ } ++ } ++ return 0; ++} ++r::File::File(std::string_view to_parse) { ++ if (to_parse.size() > MAX_SIZE) { ++ error_ = "INPUT FILE TOO LARGE " + std::to_string(to_parse.size()); ++ return; ++ } ++ for (auto line_number = 1; valid() && to_parse.size(); ++line_number) { ++ auto line_end = to_parse.find('\n'); ++ auto line = to_parse.substr(0UL, line_end); ++ if (!parse_line(line, line_number)) { ++ LOG(INFO) << "Line #" << line_number << " ignored: [" << line << ']'; ++ } else if (directives_.empty()) { ++ LOG(ERROR) << "Expected to have a directive after parsing line #" ++ << line_number << ": " << line; ++ } else if (directives_.back().valid()) { ++ VLOG(1) << "Line #" << line_number << " parsed. " << line; ++ } else { ++ error_ = "FAILURE PARSING LINE # " + std::to_string(line_number); ++ error_.append(": ") ++ .append(directives_.back().error()) ++ .append(" [") ++ .append(line) ++ .push_back(']'); ++ LOG(ERROR) << error_; ++ return; ++ } ++ if (line_end < to_parse.size()) { ++ to_parse.remove_prefix(line_end + 1); ++ } else { ++ break; ++ } ++ } ++ if (directives_.empty()) { ++ error_ = "No redirection directives in _redirects"; ++ LOG(ERROR) << error_; ++ } ++} ++ ++namespace { ++std::pair parse_status(std::string_view line, ++ std::size_t col); ++} ++bool r::File::parse_line(std::string_view line, int line_number) { ++ if (line.empty()) { ++ // empty line is not a directive ++ return false; ++ } ++ auto bpos = line.find_first_not_of(WHITESPACE); ++ if (bpos == std::string_view::npos) { ++ // effectively empty line ++ return false; ++ } else if (line[bpos] == '#') { ++ // https://specs.ipfs.tech/http-gateways/web-redirects-file/#comments ++ return false; ++ } ++ auto epos = line.find_first_of(WHITESPACE, bpos); ++ if (epos == std::string_view::npos) { ++ error_ = "Parsing _redirects file: line # " + std::to_string(line_number); ++ error_ ++ .append(" , expected at least 2 tokens (from and to) for directive: [") ++ .append(line) ++ .append("], but didn't even get whitespace to end from"); ++ return false; ++ } ++ auto from = line.substr(bpos, epos - bpos); ++ bpos = line.find_first_not_of(WHITESPACE, epos); ++ if (bpos == std::string_view::npos) { ++ error_ = "Parsing _redirects file: line # " + std::to_string(line_number); ++ error_ ++ .append(" , expected at least 2 tokens (from and to) for directive: [") ++ .append(line) ++ .append("], but didn't get a to"); ++ return false; ++ } ++ epos = line.find_first_of(WHITESPACE, bpos); ++ auto to = line.substr(bpos, epos - bpos); ++ auto [status, err] = parse_status(line, epos); ++ if (err.empty()) { ++ directives_.emplace_back(from, to, status); ++ return true; ++ } else { ++ error_ = err; ++ LOG(ERROR) << "Error parsing status on line #" << line_number << " [" ++ << line << "]."; ++ return false; ++ } ++} ++ ++namespace { ++ ++std::pair parse_status(std::string_view line, ++ std::size_t col) { ++ if (col >= line.size()) { ++ VLOG(2) << " No status specified, using default."; ++ return {DEFAULT_STATUS, ""}; ++ } ++ auto b = line.find_first_not_of(WHITESPACE, col); ++ if (b >= line.size()) { ++ VLOG(2) ++ << " No status specified (line ended in whitespace), using default."; ++ return {DEFAULT_STATUS, ""}; ++ } ++ auto status_str = line.substr(b); ++ if (status_str.size() < 3) { ++ return {PARSE_ERROR_STATUS, ++ " Not enough characters for a valid status string: [" + ++ std::string{status_str} + "]."}; ++ } ++ auto good = [](int i) { return std::make_pair(i, ""s); }; ++ auto unsupported = [status_str]() { ++ return std::make_pair( ++ PARSE_ERROR_STATUS, ++ "Unsupported status specified in directive:" + std::string{status_str}); ++ }; ++ /* ++ * 200 - OK treated as a rewrite, without changing the URL in the browser. ++ * 301 - Permanent Redirect (default) ++ * 302 - Found (commonly used for Temporary Redirect) ++ * 303 - See Other (replacing PUT and POST with GET) ++ * 307 - Temporary Redirect (explicitly preserving body and HTTP method) ++ * 308 - Permanent Redirect (preserving body & method of original request) ++ * 404 - Not Found (Useful for a pretty 404 page) ++ * 410 - Gone ++ * 451 - Unavailable For Legal Reasons ++ */ ++ switch (status_str[0]) { ++ case '2': ++ return status_str == "200" ? good(200) : unsupported(); ++ case '3': ++ if (status_str[1] != '0') { ++ return unsupported(); ++ } ++ return good(300 + status_str[2] - '0'); ++ case '4': ++ switch (status_str[1]) { ++ case '0': ++ return status_str[2] == '4' ? good(404) : unsupported(); ++ case '1': ++ return status_str[2] == '0' ? good(410) : unsupported(); ++ case '5': ++ return status_str[2] == '1' ? good(451) : unsupported(); ++ default: ++ return unsupported(); ++ } ++ default: ++ return unsupported(); ++ } ++} ++} // namespace +diff --git a/third_party/ipfs_client/src/ipfs_client/redirects.h b/third_party/ipfs_client/src/ipfs_client/redirects.h +new file mode 100644 +index 0000000000000..8b6fdb121a8b6 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/redirects.h +@@ -0,0 +1,41 @@ ++#ifndef IPFS_REDIRECTS_H_ ++#define IPFS_REDIRECTS_H_ ++ ++#include ++ ++#include ++#include ++#include ++ ++namespace ipfs { ++namespace redirects { ++class Directive { ++ enum class ComponentType { LITERAL, PLACEHOLDER, SPLAT }; ++ std::vector> components_; ++ std::string const to_; ++ int const status_; ++ ++ public: ++ Directive(std::string_view, std::string_view, int); ++ std::uint16_t rewrite(std::string&) const; ++ std::string error() const; ++ bool valid() const { return error().empty(); } ++}; ++class File { ++ std::vector directives_; ++ std::string error_; ++ ++ public: ++ File(std::string_view to_parse); ++ ++ bool valid() const { return error().empty(); } ++ std::string const& error() const { return error_; } ++ std::uint16_t rewrite(std::string& missing_path) const; ++ ++ private: ++ bool parse_line(std::string_view, int); ++}; ++} // namespace redirects ++} // namespace ipfs ++ ++#endif // IPFS_REDIRECTS_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/response.cc b/third_party/ipfs_client/src/ipfs_client/response.cc +new file mode 100644 +index 0000000000000..062042d4abfb1 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/response.cc +@@ -0,0 +1,6 @@ ++#include "ipfs_client/response.h" ++ ++using Self = ipfs::Response; ++ ++Self Self::PLAIN_NOT_FOUND{std::string{}, static_cast(404), ++ std::string{}, std::string{}}; +diff --git a/third_party/ipfs_client/src/ipfs_client/scheduler.cc b/third_party/ipfs_client/src/ipfs_client/scheduler.cc +new file mode 100644 +index 0000000000000..71a7296310927 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/scheduler.cc +@@ -0,0 +1,141 @@ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "log_macros.h" ++ ++#include ++#include ++#include ++ ++ipfs::Scheduler::Scheduler(std::function list_gen) ++ : list_gen_(list_gen), gateways_{list_gen()} { ++ VLOG(1) << "Scheduler ctor"; ++} ++ ++ipfs::Scheduler::~Scheduler() { ++ LOG(INFO) << "Scheduler dtor"; ++} ++ ++void ipfs::Scheduler::Enqueue(std::shared_ptr api, ++ std::shared_ptr dag_listener, ++ std::shared_ptr name_listener, ++ std::string const& suffix, ++ std::string_view accept, ++ Priority p, ++ std::shared_ptr top) { ++ LOG(INFO) << "Scheduler::Enqueue(...," << suffix << ',' << accept << ',' << p ++ << ')'; ++ if (!top) { ++ LOG(ERROR) << "No IpfsRequest?"; ++ } ++ if (suffix.empty()) { ++ LOG(ERROR) << "Do not issue a request with no task!"; ++ } else { ++ auto key = UrlSpec{suffix, accept}; ++ auto& todo = task2todo_[key]; ++ todo.priority = std::max(todo.priority, p); ++ if (dag_listener) { ++ todo.dag_listeners.insert(dag_listener); ++ } ++ if (name_listener) { ++ todo.name_listeners.insert(name_listener); ++ } ++ if (top) { ++ todo.source_reqs.insert(top); ++ } else { ++ LOG(WARNING) << "Enqueue with no top: " << suffix; ++ } ++ } ++ IssueRequests(api); ++} ++bool ipfs::Scheduler::IssueRequests(std::shared_ptr api) { ++ // LOG(INFO) << "Scheduler::IssueRequests"; ++ decltype(task2todo_)::value_type* unmet = nullptr; ++ auto assign = [this, api](auto& gw, auto& task, auto& todo, auto need) { ++ if (gw.accept(task, need)) { ++ BusyGateway bgw{gw.url_prefix(), task, this}; ++ std::shared_ptr top; ++ if (todo.source_reqs.empty()) { ++ LOG(ERROR) << "IssueRequests with no top-level requests awaiting!"; ++ } else { ++ top = *todo.source_reqs.begin(); // This is wrong, but so is ++ // everything about this ++ } ++ bgw.srcreq = top; ++ auto req = api->InitiateGatewayRequest(std::move(bgw)); ++ todo.requests.insert(req); ++ VLOG(2) << "Initiated request " << req->url() << " (" << need << ')' ++ << todo.source_reqs.size() << " await."; ++ return true; ++ } ++ return false; ++ }; ++ for (auto& e : task2todo_) { ++ auto& task = e.first; ++ auto& todo = e.second; ++ auto need = todo.under_target(); ++ for (auto& gw : gateways_) { ++ if (assign(gw, task, todo, need)) { ++ need = todo.under_target(); ++ } else if (!unmet || unmet->second.under_target() < need) { ++ unmet = &e; ++ } ++ } ++ } ++ if (unmet) { ++ for (auto& gw : gateways_) { ++ if (gw.load() < 2 && assign(gw, unmet->first, unmet->second, 987)) { ++ unmet = nullptr; ++ break; ++ } ++ } ++ } ++ return !unmet; ++} ++ ++bool ipfs::Scheduler::DetectCompleteFailure(UrlSpec const& task) const { ++ auto fail_count = ++ std::count_if(gateways_.begin(), gateways_.end(), ++ [&task](auto& g) { return g.PreviouslyFailed(task); }); ++ // LOG(INFO) << task << " has already failed on " << fail_count ++ // << "gateways."; ++ return fail_count >= static_cast(gateways_.size()); ++} ++void ipfs::Scheduler::CheckSwap(std::size_t index) { ++ if (index + 1UL < gateways_.size() && ++ gateways_[index + 1UL] < gateways_[index]) { ++ std::swap(gateways_[index], gateways_[index + 1UL]); ++ } ++} ++void ipfs::Scheduler::TaskComplete(UrlSpec const& task) { ++ auto todo = task2todo_.find(task); ++ if (task2todo_.end() == todo) { ++ VLOG(2) << "An unknown TODO " << task.suffix << " finished."; ++ return; ++ } ++ LOG(INFO) << "Task " << task.suffix << " completed with " ++ << todo->second.name_listeners.size() << " name listeners and " ++ << todo->second.source_reqs.size() << " IpfsRequest s"; ++ // Don't need to call back on dag listeners because storage covered that ++ for (auto& nl : todo->second.name_listeners) { ++ LOG(INFO) << "Notifying a name listener that its listener is ready."; ++ nl->Complete(); ++ } ++ for (auto& r : todo->second.source_reqs) { ++ r->orchestrator->build_response(r->dependent); ++ } ++ task2todo_.erase(todo); ++} ++ ++ipfs::Scheduler::Todo::Todo() {} ++ipfs::Scheduler::Todo::~Todo() {} ++long ipfs::Scheduler::Todo::under_target() const { ++ long target = priority; ++ return target - static_cast(requests.size()); ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/scoring.md b/third_party/ipfs_client/src/ipfs_client/scoring.md +new file mode 100644 +index 0000000000000..d6c79916e574f +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/scoring.md +@@ -0,0 +1,35 @@ ++This is meant to document how gateway scoring is done in ipfs-chromium _today_. ++ ++This is *not* meant to imply that it's the best approach, or that we will maintain it this way long-term. ++ ++Each gateway has an associated score, used to determine preference for sending requests there and also how many requests should be sent concurrently. ++ ++It's an unsigned integer in two forms: ++ ++* A canonical score. ++ - It begins as a hand-written, hard-coded constant. ++ - When a gateway successfully returns a useful result before cancellation, its score increases by 1. ++ - If a gateway fails to return a useful response for any reason (timeout, doesn't support fetching IPNS records, etc.): ++ - If the score is already zero, the gateway is removed from the list entirely ++ - Otherwise the score is decreased by 1 ++ - Having a request cancelled because some other gateway successfully returned a result for an identical request first does _not_ alter the score. ++ - These changes are intended to be persisted in the future ++* A temporary score ++ * When a top-level ipfs:// or ipns:// request begins, it fetches a (gateway request) scheduler ++ * If there is already a busy scheduler, it is re-used ++ * If ipfs traffic is starting up from nil, instead a new scheduler is instantiated ++ * When a new scheduler is instantiate for this, it requests the list of gateways, with their scores. ++ * The scores retrieved are the canonical score _plus_ a small non-negative random integer (geometric distribution tending toward zero) ++ * This helps to ensure you won't always be sending requests to the same gateways in the same order, or in a deterministic way. ++ ++When issuing a request to another gateway, it checks the gateways in descending-score order: ++* If the gateway already has a request for that data pending, or has already failed to return successfully, it's skipped. ++* If the gateways score is less than the number of pending requests currently sent to that gateway, it's skipped as it's considered already-too-busy ++* The "need" is generally calculated as the target number of gateways desired to be involved (based on request priority), subtracting the number of gateways currently processing a request for this data. ++* If the "need" is less than half the count of pending requests on this gateway, it's skipped as it's simply not urgent enough to justify further overloading this gateway. ++ ++ ++On a side-note, the hard-coded starting points for the scoring effectively encodes known information about those gateways. ++For example: http://localhost:8080/ is scored extremely highly. There's a good chance it has the resource you're looking for, and if it doesn't you may want to send a request that way anyhow so that it will in the future. ++Conversely, https://ipfs.anonymize.com/ is rarely helpful and is barely hanging on at the bottom. ++https://jcsl.hopto.org/ is scored higher than one might imagine, given that it's not even commercially-hosted. But ipfs-chromium today is disproportionately used on the same set of test links, and jcsl.hopto will generally have those because it's John's home and his node. +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/dir_shard.cc b/third_party/ipfs_client/src/ipfs_client/unix_fs/dir_shard.cc +new file mode 100644 +index 0000000000000..ef644de11313d +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/dir_shard.cc +@@ -0,0 +1,152 @@ ++#include "dir_shard.h" ++ ++#include ++#include ++ ++#include ++ ++#include ++ ++#include "log_macros.h" ++ ++#include ++ ++using Self = ipfs::unix_fs::DirShard; ++ ++bool Self::Process(std::unique_ptr& next_helper, ++ std::shared_ptr listener, ++ std::function requestor, ++ std::string& target_cid) { ++ if (!block()) { ++ requestor(cid_, resolver_->priority()); ++ return false; ++ } ++ auto& block = *(this->block()); ++ if (!(block.valid())) { ++ LOG(ERROR) << "DirShard dealing with an invalid block: " << cid_ << " / " ++ << resolver_->original_path(); ++ listener->NotHere(cid_, resolver_->original_path()); ++ return false; ++ } ++ SomePrefetch(requestor); ++ ++ // node.Data.hashType indicates a multihash function to use to digest path ++ // components used for sharding. ++ // It MUST be murmur3-x64-64 (multihash 0x22). ++ GOOGLE_DCHECK_EQ(block.fsdata().hashtype(), 0x22); ++ if (next_path_element_.empty()) { ++ // TODO - there could one day be a HAMT directory with no index.html ++ next_path_element_ = "index.html"; ++ hamt_hexs_.clear(); ++ } ++ /* stolen from spec ++ * ####### Path resolution on HAMTs ++ * Steps: ++ * 1. Take the current path component... ++ */ ++ bool missing_descendents = false; ++ if (missing_descendents) { ++ LOG(WARNING) ++ << "Waiting on more blocks before dealing with this HAMT node."; ++ return false; ++ } ++ auto fanout = block.fsdata().has_fanout() ? block.fsdata().fanout() : 256; ++ if (hamt_hexs_.empty()) { ++ VLOG(1) << "Had no hexes, hash path element: " << next_path_element_; ++ HashPathElement(fanout); ++ } ++ if (hamt_hexs_.empty()) { ++ LOG(ERROR) << "Somehow failed to hash out the path element?"; ++ return false; ++ } ++ bool found = false; ++ block.List([&](auto& name, auto cid) { ++ // Fun fact: there is a spec-defined sort order to these children. ++ // We *could* do a binary search. ++ if (!absl::StartsWith(name, hamt_hexs_.front())) { ++ // LOG(INFO) << name << " isn't the right child for " << ++ // hamt_hexs_.front() ; ++ return true; ++ } ++ found = true; ++ VLOG(1) << "As we move down a Hamt, " << cid_ << " -> " << name << " / " ++ << cid << " fanout=" << block.fsdata().fanout(); ++ target_cid = cid; ++ if (absl::EndsWith(name, next_path_element_)) { ++ VLOG(1) << "Found our calling! Leaving the Hamt in favor of " << name; ++ next_helper.reset(); // Let higher detect what the next level down is ++ } else if (hamt_hexs_.front() == name) { ++ VLOG(1) << "One more level down the Hamt, following " << name; ++ auto help = std::make_unique(next_path_element_); ++ Delegate(*help); ++ help->cid(cid); ++ help->hamt_hexs_.assign(std::next(hamt_hexs_.begin()), hamt_hexs_.end()); ++ next_helper = std::move(help); ++ } else { ++ LOG(ERROR) << "Was looking for " << cid_ << '/' ++ << resolver_->original_path() ++ << " and got as far as HAMT hash bits " << hamt_hexs_.front() ++ << " but the corresponding link is " << name ++ << " and does not end with " << next_path_element_; ++ found = false; ++ } ++ return false; ++ }); ++ if (found) { ++ return true; ++ } ++ target_cid.clear(); ++ listener->DoesNotExist(cid_, resolver_->original_path()); ++ return false; ++} ++ ++void Self::HashPathElement(std::uint64_t fanout) { ++ // L_VAR(fanout); ++ GOOGLE_DCHECK_GT(fanout, 0); ++ auto hex_width = 0U; ++ for (auto x = fanout; (x >>= 4); ++hex_width) ++ ; ++ // ... then hash it using the multihash id provided in Data.hashType. ++ // absl::uint128 digest{}; ++ std::array digest = {0U, 0U}; ++ // Rust's fastmurmur3 also passes 0 for seed, and iroh uses that. ++ MurmurHash3_x64_128(next_path_element_.data(), next_path_element_.size(), 0, ++ digest.data()); ++ auto bug_compatible_digest = htobe64(digest[0]); ++ VLOG(1) << "Hash: " << digest[0] << ' ' << digest[1] << " -> " ++ << bug_compatible_digest; ++ for (auto d : digest) { ++ auto hash_bits = htobe64(d); ++ while (hash_bits) { ++ // 2. Pop the log2(fanout) lowest bits from the path component hash ++ // digest,... ++ auto popped = hash_bits % fanout; ++ hash_bits /= fanout; ++ // L_VAR(hash_bits); ++ std::ostringstream oss; ++ // ... then hex encode (using 0-F) using little endian those bits ... ++ oss << std::setfill('0') << std::setw(hex_width) << std::uppercase ++ << std::hex << popped; ++ hamt_hexs_.push_back(oss.str()); ++ } ++ } ++} ++void Self::SomePrefetch(std::function requestor) { ++ auto i = 0; ++ block()->List([this, requestor, &i](auto&, auto cid) { ++ // if (storage_->Get(cid, false)) { ++ if (storage_->Get(cid)) { ++ return true; ++ } ++ if (++i > skips_per_pass_) { ++ ++skips_per_pass_; ++ requestor(cid, 0); ++ return false; ++ } ++ return true; ++ }); ++} ++ ++Self::DirShard(std::string next_path_element) ++ : next_path_element_{next_path_element} {} ++Self::~DirShard() noexcept {} +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/dir_shard.h b/third_party/ipfs_client/src/ipfs_client/unix_fs/dir_shard.h +new file mode 100644 +index 0000000000000..3a1a02eaad34d +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/dir_shard.h +@@ -0,0 +1,42 @@ ++#ifndef IPFS_DIR_SHARD_H_ ++#define IPFS_DIR_SHARD_H_ ++ ++#include "node_helper.h" ++ ++namespace ipfs { ++class ContextApi; ++class UnixFsPathResolver; ++namespace unix_fs { ++ ++/*! ++ * \brief Process a sharded directory (likely over multiple blocks) ++ */ ++class DirShard : public NodeHelper { ++ public: ++ /*! ++ * \brief Create the object to descend past the HAMT ++ * \param Element meaning the thing delimited by /, this is the name of the ++ * directory entry we're looking for. ++ */ ++ DirShard(std::string next_path_element); ++ ~DirShard() noexcept override; ++ ++ private: ++ std::string next_path_element_; ++ std::vector hamt_hexs_; ++ int skips_per_pass_ = 0; ++ ++ bool Process(std::unique_ptr&, ++ std::shared_ptr, ++ std::function, ++ std::string&) override; ++ ++ void HashPathElement(std::uint64_t fanout); ++ void SomePrefetch(std::function); ++}; ++ ++} // namespace unix_fs ++ ++} // namespace ipfs ++ ++#endif // IPFS_DIR_SHARD_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/guess_content_type.cc b/third_party/ipfs_client/src/ipfs_client/unix_fs/guess_content_type.cc +new file mode 100644 +index 0000000000000..6b1b627cdd406 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/guess_content_type.cc +@@ -0,0 +1,31 @@ ++#include "guess_content_type.h" ++ ++#include ++ ++#include "log_macros.h" ++ ++std::string ipfs::unix_fs::GuessContentType(ContextApi& api, ++ std::string_view path, ++ std::string_view content) { ++ auto dot = path.rfind('.'); ++ auto slash = path.rfind('/'); ++ std::string ext; ++ if (dot < path.size() && (slash < dot || slash == std::string::npos)) { ++ ext = path.substr(dot + 1); ++ } ++ std::string url{ ++ "ipfs://bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354/"}; ++ url.append(path); ++ auto mime = api.MimeType(ext, content, url); ++ if (mime.size()) { ++ // TODO, store mime in block ++ VLOG(1) << "Detected mime " << mime << " for " << path ++ ++ << " from URL " << url; ++ return mime; ++ } ++ // TODO fetch the mime from block if available ++ LOG(ERROR) << "\n\t###\tTODO:\tCould not determine mime type for '" << path ++ << "'.\t###\n\n"; ++ return "TODO"; ++} +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/guess_content_type.h b/third_party/ipfs_client/src/ipfs_client/unix_fs/guess_content_type.h +new file mode 100644 +index 0000000000000..090be0a5e26c4 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/guess_content_type.h +@@ -0,0 +1,16 @@ ++#ifndef IPFS_GUESS_CONTENT_TYPE_H_ ++#define IPFS_GUESS_CONTENT_TYPE_H_ ++ ++#include ++#include ++ ++namespace ipfs { ++class ContextApi; ++namespace unix_fs { ++std::string GuessContentType(ContextApi& api, ++ std::string_view path, ++ std::string_view content); ++} ++} // namespace ipfs ++ ++#endif // IPFS_GUESS_CONTENT_TYPE_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/multi_node_file.cc b/third_party/ipfs_client/src/ipfs_client/unix_fs/multi_node_file.cc +new file mode 100644 +index 0000000000000..4492661310467 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/multi_node_file.cc +@@ -0,0 +1,138 @@ ++#include "multi_node_file.h" ++ ++#include "guess_content_type.h" ++ ++#include ++#include ++#include ++ ++#include ++ ++#include "log_macros.h" ++ ++using Self = ipfs::unix_fs::MultiNodeFile; ++ ++bool Self::Process(std::unique_ptr&, ++ std::shared_ptr listener, ++ std::function requestor, ++ std::string& target_cid) { ++ if (top_cid_.empty()) { ++ top_cid_ = cid_; ++ } ++ if (!block()) { ++ requestor(cid_, resolver_->priority()); ++ return false; ++ } ++ if (!storage_->Get(top_cid_)) { ++ requestor(top_cid_, resolver_->priority()); ++ return false; ++ } ++ Fetch(requestor, target_cid); ++ if (Write(listener)) { ++ listener->BlocksComplete( ++ GuessContentType(*api_, resolver_->original_path(), FirstChunk())); ++ target_cid.clear(); ++ return true; ++ } ++ return false; ++} ++ ++void Self::Fetch(std::function requestor, ++ std::string& target) { ++ auto idx = -1L; ++ auto* top = storage_->Get(top_cid_); ++ if (!top) { ++ requestor(top_cid_, resolver_->priority()); ++ return; ++ } ++ top->List([this, requestor, &idx, &target](auto&, auto cid) { ++ ++idx; ++ if (idx >= static_cast(children_.size())) { ++ children_.emplace_back(cid, std::nullopt); ++ } ++ auto child_block = storage_->Get(cid); ++ if (!child_block) { ++ requestor(cid, resolver_->priority()); ++ target = cid; ++ return true; ++ } ++ if (child_block->type() != Block::Type::File) { ++ return true; ++ } ++ auto& child = children_[idx].second; ++ if (!child.has_value()) { ++ child = MultiNodeFile(); ++ Delegate(*child); ++ child->cid(cid); ++ child->top_cid_ = cid; ++ } ++ child->Fetch(requestor, target); ++ return true; ++ }); ++} ++bool Self::Write(std::shared_ptr listener) { ++ VLOG(1) << "Write:" << cid_ << " have children: " << children_.size(); ++ for (; written_until_ < children_.size(); ++written_until_) { ++ auto& child = children_[written_until_]; ++ VLOG(2) << "child[" << written_until_ << "]:" << child.first; ++ if (child.second.has_value()) { ++ if (child.second->Write(listener)) { ++ VLOG(1) << "Successfully recursed."; ++ continue; ++ } else { ++ return false; ++ } ++ } ++ auto* child_block = storage_->Get(child.first); ++ if (!child_block) { ++ return false; ++ } ++ switch (child_block->type()) { ++ case Block::Type::FileChunk: ++ listener->ReceiveBlockBytes(child_block->chunk_data()); ++ break; ++ case Block::Type::NonFs: ++ listener->ReceiveBlockBytes(child_block->unparsed()); ++ break; ++ case Block::Type::File: ++ LOG(ERROR) << "Child helper should already be populated."; ++ return false; ++ default: ++ LOG(ERROR) << "Member of a multi-node file of unhandled type: " ++ << Stringify(child_block->type()); ++ return false; ++ } ++ } ++ DCHECK_EQ(written_until_, children_.size()); ++ return true; ++} ++std::string Self::FirstChunk() { ++ if (children_.empty()) { ++ return {}; ++ } ++ auto& child = children_.front(); ++ if (child.second.has_value()) { ++ return child.second->FirstChunk(); ++ } ++ auto* first = storage_->Get(child.first); ++ if (!first) { ++ return {}; ++ } ++ switch (first->type()) { ++ case Block::Type::FileChunk: ++ return first->chunk_data(); ++ case Block::Type::NonFs: ++ return first->unparsed(); ++ case Block::Type::File: ++ LOG(ERROR) << "Child helper should already be populated."; ++ return {}; ++ default: ++ LOG(ERROR) << "Member of a multi-node file of unhandled type: " ++ << Stringify(first->type()); ++ return {}; ++ } ++} ++ ++Self::MultiNodeFile() {} ++Self::MultiNodeFile(MultiNodeFile const& rhs) = default; ++Self::~MultiNodeFile() noexcept {} +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/multi_node_file.h b/third_party/ipfs_client/src/ipfs_client/unix_fs/multi_node_file.h +new file mode 100644 +index 0000000000000..fdaa7bb6fb9b3 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/multi_node_file.h +@@ -0,0 +1,39 @@ ++#ifndef IPFS_MULTI_NODE_FILE_H_ ++#define IPFS_MULTI_NODE_FILE_H_ ++ ++#include "node_helper.h" ++ ++#include ++#include ++ ++namespace ipfs { ++namespace unix_fs { ++ ++/*! ++ * \brief Process a file that's not a block but an almagation of blocks ++ */ ++class MultiNodeFile : public NodeHelper { ++ public: ++ MultiNodeFile(); ++ MultiNodeFile(MultiNodeFile const&); ++ ~MultiNodeFile() noexcept override; ++ ++ private: ++ std::vector>> children_; ++ std::size_t written_until_ = 0UL; ++ std::string top_cid_; ++ ++ bool Process(std::unique_ptr&, ++ std::shared_ptr, ++ std::function, ++ std::string&) override; ++ ++ void Fetch(std::function, std::string&); ++ bool Write(std::shared_ptr); ++ std::string FirstChunk() ; ++}; ++ ++} // namespace unix_fs ++} // namespace ipfs ++ ++#endif // IPFS_MULTI_NODE_FILE_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/node_helper.cc b/third_party/ipfs_client/src/ipfs_client/unix_fs/node_helper.cc +new file mode 100644 +index 0000000000000..799322ae51725 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/node_helper.cc +@@ -0,0 +1,49 @@ ++#include "node_helper.h" ++ ++#include "dir_shard.h" ++#include "multi_node_file.h" ++#include "plain_directory.h" ++#include "plain_file.h" ++ ++#include ++ ++#include ++ ++#include "log_macros.h" ++ ++using Self = ipfs::unix_fs::NodeHelper; ++ ++auto Self::block() -> Block const* { ++ return storage_->Get(cid_); ++} ++ ++void Self::Delegate(ipfs::unix_fs::NodeHelper& other) const { ++ other.cid_ = cid_; ++ VLOG(1) << "NodeHelper::Delegate(storage @" << (void*)(storage_.get()) << ')'; ++ other.storage_ = storage_; ++ other.resolver_ = resolver_; ++ other.api_ = api_; ++} ++void Self::storage(ipfs::BlockStorage& val) { ++ VLOG(1) << "NodeHelper::storage(@" << (void*)(&val) << ')'; ++ storage_ = &val; ++} ++auto Self::FromBlockType(Block::Type typ, std::string path_element) ++ -> std::unique_ptr { ++ switch (typ) { ++ case Block::Type::Directory: ++ return std::make_unique(path_element); ++ case Block::Type::FileChunk: ++ return std::make_unique(); ++ case Block::Type::File: ++ return std::make_unique(); ++ case Block::Type::HAMTShard: ++ return std::make_unique(path_element); ++ default: ++ LOG(FATAL) << "TODO : Unhandled UnixFS node " << path_element ++ << " of type " << Stringify(typ); ++ return {}; ++ } ++} ++ ++Self::~NodeHelper() noexcept {} +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/node_helper.h b/third_party/ipfs_client/src/ipfs_client/unix_fs/node_helper.h +new file mode 100644 +index 0000000000000..7082c8cf016f4 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/node_helper.h +@@ -0,0 +1,77 @@ ++#ifndef IPFS_NODE_HELPER_H_ ++#define IPFS_NODE_HELPER_H_ ++ ++#include ++#include ++ ++#include ++ ++#include ++#include ++ ++namespace ipfs { ++ ++class BlockStorage; ++class ContextApi; ++class DagListener; ++class UnixFsPathResolver; ++ ++namespace unix_fs { ++ ++/*! ++ * \brief Deal with a block that is a UnixFS node of some sort. ++ */ ++class NodeHelper { ++ public: ++ /*! ++ * \brief Create the appropriate subclass ++ * \param typ - ++ * \param path_element - if it's a directory-like, ++ * tell it what entry you're trying to resolve ++ * \return A subclass instance that might need some more info ++ */ ++ static std::unique_ptr FromBlockType(Block::Type typ, ++ std::string path_element); ++ ++ /*! ++ * \brief Get as far in the processing of this block that we can right now ++ * \param[out] nh - the next helper if it is knowable ++ * \param requestor - a way to request missing blocks ++ * \param target_cid[in,out] - The CID of the block currently under ++ * investigation. If being directed to a different block should change. ++ * \return Whether target_cid and nh are populated and should be swapped ++ * \todo This interface is just bad ++ */ ++ virtual bool Process(std::unique_ptr& nh, ++ std::shared_ptr dl, ++ std::function requestor, ++ std::string& target_cid) = 0; ++ ++ /* ++ * \name Setters ++ * call these on a fresh NodeHelper so it knows what it's doing. ++ */ ++ ///@{ ++ void cid(std::string const& val) { cid_ = val; } ///< Block/node in question ++ void storage(BlockStorage& val); ///< Storage to interact with ++ ++ /// sad that it needs this ++ void resolver(UnixFsPathResolver& val) { resolver_ = &val; } ++ void api(ContextApi& val) { api_ = &val; } ///< API for the environment ++ ///@{ ++ ++ virtual ~NodeHelper() noexcept; ++ ++ protected: ++ std::string cid_; ++ raw_ptr storage_ = nullptr; ++ raw_ptr resolver_ = nullptr; ++ raw_ptr api_ = nullptr; ++ ++ Block const* block(); ++ void Delegate(NodeHelper&) const; ++}; ++} // namespace unix_fs ++} // namespace ipfs ++ ++#endif // IPFS_NODE_HELPER_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_directory.cc b/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_directory.cc +new file mode 100644 +index 0000000000000..c2af264678de0 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_directory.cc +@@ -0,0 +1,65 @@ ++#include "plain_directory.h" ++ ++#include ++#include ++#include ++ ++#include "ipfs_client/generated_directory_listing.h" ++#include "log_macros.h" ++ ++using Self = ipfs::unix_fs::PlainDirectory; ++ ++Self::PlainDirectory(std::string next_path_element) ++ : next_path_element_{next_path_element} {} ++ ++bool Self::Process(std::unique_ptr&, ++ std::shared_ptr listener, ++ std::function requestor, ++ std::string& target_cid) { ++ block()->List([requestor](auto&, auto cid) { ++ // Mild prefetch ++ requestor(cid, 0); ++ return true; ++ }); ++ auto const old_target = target_cid; ++ if (next_path_element_.empty()) { ++ // The user's URL actually ends in the directory itself ++ block()->List([&target_cid](auto& name, auto cid) { ++ if (name == "index.html") { ++ target_cid = cid; ++ return false; ++ } ++ return true; ++ }); ++ if (target_cid != old_target) { ++ // Found an index.html - will descend to that ++ return true; ++ } ++ ipfs::GeneratedDirectoryListing list{resolver_->original_path()}; ++ block()->List([&list](auto& name, auto) { ++ list.AddEntry(name); ++ return true; ++ }); ++ listener->ReceiveBlockBytes(list.Finish()); ++ listener->BlocksComplete("text/html"); ++ target_cid.clear(); ++ return true; ++ } ++ block()->List([this, &target_cid](auto& name, auto cid) { ++ if (name == next_path_element_) { ++ target_cid = cid; ++ return false; ++ } ++ return true; ++ }); ++ if (target_cid != old_target) { ++ // Found the element in the directory, descending the path. ++ } else { ++ // There was a specific element requested and it does not exist. 404 ++ listener->DoesNotExist(target_cid, resolver_->original_path()); ++ target_cid.clear(); ++ } ++ return true; ++} ++ ++Self::~PlainDirectory() noexcept {} +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_directory.h b/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_directory.h +new file mode 100644 +index 0000000000000..ad9300ec23bb6 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_directory.h +@@ -0,0 +1,35 @@ ++#ifndef IPFS_PLAIN_DIRECTORY_H_ ++#define IPFS_PLAIN_DIRECTORY_H_ ++ ++#include "node_helper.h" ++ ++namespace ipfs { ++class ContextApi; ++class UnixFsPathResolver; ++namespace unix_fs { ++ ++/*! ++ * \brief Helper for a normal UnixFS directory ++ */ ++class PlainDirectory : public NodeHelper { ++ public: ++ /*! ++ * \brief Construct with the entry desired ++ * \param The name of the next file/directory after this. ++ */ ++ PlainDirectory(std::string next_path_element); ++ ~PlainDirectory() noexcept override; ++ ++ private: ++ std::string const next_path_element_; ++ ++ bool Process(std::unique_ptr&, ++ std::shared_ptr, ++ std::function, ++ std::string&) override; ++}; ++ ++} // namespace unix_fs ++} // namespace ipfs ++ ++#endif // IPFS_PLAIN_DIRECTORY_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_file.cc b/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_file.cc +new file mode 100644 +index 0000000000000..4783464894373 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_file.cc +@@ -0,0 +1,26 @@ ++#include "plain_file.h" ++ ++#include ++#include ++#include ++ ++#include "guess_content_type.h" ++ ++#include "log_macros.h" ++ ++using Self = ipfs::unix_fs::PlainFile; ++ ++Self::PlainFile() {} ++ ++bool Self::Process(std::unique_ptr&, ++ std::shared_ptr listener, ++ std::function, ++ std::string& target_cid) { ++ listener->ReceiveBlockBytes(block()->chunk_data()); ++ listener->BlocksComplete(GuessContentType(*api_, resolver_->original_path(), ++ block()->chunk_data())); ++ target_cid.clear(); ++ return true; ++} ++ ++Self::~PlainFile() noexcept {} +diff --git a/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_file.h b/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_file.h +new file mode 100644 +index 0000000000000..923b0786f6194 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unix_fs/plain_file.h +@@ -0,0 +1,29 @@ ++#ifndef IPFS_PLAIN_FILE_H_ ++#define IPFS_PLAIN_FILE_H_ ++ ++#include "node_helper.h" ++ ++namespace ipfs { ++class ContextApi; ++class UnixFsPathResolver; ++namespace unix_fs { ++ ++/*! ++ * \brief Helper for a single-block file ++ */ ++class PlainFile : public NodeHelper { ++ public: ++ PlainFile(); ++ ~PlainFile() noexcept override; ++ ++ private: ++ bool Process(std::unique_ptr&, ++ std::shared_ptr, ++ std::function, ++ std::string&) override; ++}; ++ ++} // namespace unix_fs ++} // namespace ipfs ++ ++#endif // IPFS_PLAIN_FILE_H_ +diff --git a/third_party/ipfs_client/src/ipfs_client/unixfs_path_resolver.cc b/third_party/ipfs_client/src/ipfs_client/unixfs_path_resolver.cc +new file mode 100644 +index 0000000000000..b04272a70b988 +--- /dev/null ++++ b/third_party/ipfs_client/src/ipfs_client/unixfs_path_resolver.cc +@@ -0,0 +1,153 @@ ++#include "ipfs_client/unixfs_path_resolver.h" ++ ++#include "unix_fs/node_helper.h" ++ ++#include ++#include ++#include ++#include ++ ++#include "vocab/stringify.h" ++ ++#include "log_macros.h" ++ ++using Self = ipfs::UnixFsPathResolver; ++ ++void Self::Step(std::shared_ptr listener) { ++ old_listener_ = listener; ++ if (cid_.empty()) { ++ return; ++ } ++ VLOG(2) << "Stepping... " << cid_ << " / " << original_path_ << " / " ++ << path_; ++ AddInvolvedCid(cid_); ++ Block const* block = storage_.Get(cid_); ++ if (!block) { ++ auto it = already_requested_.find(cid_); ++ if (it == already_requested_.end()) { ++ VLOG(1) << "Current block " << cid_ << " not found. Requesting."; ++ Request(listener, cid_, prio_); ++ } else { ++ already_requested_.erase(it); ++ } ++ return; ++ } ++ if (!block->valid() || block->type() == Block::Type::Invalid) { ++ LOG(ERROR) << "Encountered broken block!! " << cid_; ++ listener->NotHere(cid_, original_path_); ++ return; ++ } ++ if (!helper_) { ++ GetHelper(block->type()); ++ } ++ if (!helper_) { ++ listener->NotHere(cid_, original_path_); ++ return; ++ } ++ auto requestor = [this, &listener](std::string cid, Priority prio) { ++ this->Request(listener, cid, prio); ++ }; ++ using Codec = libp2p::multi::ContentIdentifierCodec; ++ DCHECK_EQ(cid_, Codec::toString(block->cid()).value()); ++ std::unique_ptr helper; ++ if (helper_->Process(helper, listener, requestor, cid_)) { ++ helper_.swap(helper); ++ VLOG(2) << "Taking another step for " << cid_; ++ Step(listener); ++ } ++} ++ ++void Self::GetHelper(Block::Type typ) { ++ VLOG(1) << "Encountered " << cid_ << " of type " << ipfs::Stringify(typ); ++ helper_ = unix_fs::NodeHelper::FromBlockType(typ, pop_path()); ++ if (helper_) { ++ helper_->cid(cid_); ++ helper_->resolver(*this); ++ helper_->storage(storage_); ++ helper_->api(*api_); ++ } ++} ++ ++void Self::Request(std::shared_ptr& listener, ++ std::string const& cid, ++ Priority prio) { ++ if (storage_.Get(cid)) { ++ return; ++ } ++ auto it = already_requested_.find(cid); ++ auto t = std::time(nullptr); ++ if (it == already_requested_.end()) { ++ VLOG(2) << "Request(" << cid << ',' << static_cast(prio) << ')'; ++ already_requested_[cid] = {prio, t}; ++ requestor_.RequestByCid(cid, listener, prio); ++ if (prio) { ++ storage_.AddListening(this); ++ AddInvolvedCid(cid); ++ } ++ } else if (prio > it->second.prio) { ++ LOG(INFO) << "Increase Request priority(" << cid << ',' ++ << static_cast(prio) << ')'; ++ it->second.prio = prio; ++ it->second.when = t; ++ requestor_.RequestByCid(cid, listener, prio); ++ } ++} ++ ++void Self::AddInvolvedCid(std::string const& cid) { ++ if (involved_cids_.end() == ++ std::find(involved_cids_.begin(), involved_cids_.end(), cid)) { ++ involved_cids_.push_back(cid); ++ } ++} ++ ++Self::UnixFsPathResolver(BlockStorage& store, ++ BlockRequestor& requestor, ++ std::string cid, ++ std::string path, ++ std::shared_ptr api) ++ : storage_{store}, ++ requestor_{requestor}, ++ cid_{cid}, ++ path_{path}, ++ original_path_(path), ++ api_(api) { ++ VLOG(1) << "UnixFsPathResolver(storage@" << (void*)(&store) << ')'; ++ constexpr std::string_view filename_param{"filename="}; ++ auto fn = original_path_.find(filename_param); ++ if (fn != std::string::npos) { ++ original_path_.erase(0, fn + filename_param.size()); ++ auto amp = original_path_.find('&'); ++ if (amp != std::string::npos) { ++ original_path_.erase(amp); ++ } ++ } ++} ++Self::~UnixFsPathResolver() noexcept { ++ VLOG(1) << "~UnixFsPathResolver: " << involved_cids_.size() << ' ' << cid_ ++ << '/' << original_path(); ++ storage_.StopListening(this); ++} ++std::shared_ptr Self::MaybeGetPreviousListener() { ++ return old_listener_.lock(); ++} ++std::string const& Self::current_cid() const { ++ return cid_; ++} ++std::string const& Self::original_path() const { ++ return original_path_; ++} ++std::string Self::pop_path() { ++ while (path_.size() && path_.front() == '/') { ++ path_.erase(0, 1); ++ } ++ auto slash = path_.find('/'); ++ std::string rv; ++ if (slash > path_.size()) { ++ rv = path_; ++ path_.clear(); ++ } else { ++ rv = path_.substr(0, slash); ++ path_.erase(0, slash + 1); ++ } ++ return api_->UnescapeUrlComponent(rv); ++} +diff --git a/third_party/ipfs_client/src/libp2p/basic/varint_prefix_reader.cc b/third_party/ipfs_client/src/libp2p/basic/varint_prefix_reader.cc +new file mode 100644 +index 0000000000000..e57ca6d295f71 +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/basic/varint_prefix_reader.cc +@@ -0,0 +1,71 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#include "libp2p/basic/varint_prefix_reader.hpp" ++ ++namespace libp2p::basic { ++ ++namespace { ++constexpr uint8_t kHighBitMask = 0x80; ++ ++// just because 64 == 9*7 + 1 ++constexpr uint8_t kMaxBytes = 10; ++ ++} // namespace ++ ++void VarintPrefixReader::reset() { ++ value_ = 0; ++ state_ = kUnderflow; ++ got_bytes_ = 0; ++} ++ ++VarintPrefixReader::State VarintPrefixReader::consume(uint8_t byte) { ++ if (state_ == kUnderflow) { ++ bool next_byte_needed = (byte & kHighBitMask) != 0; ++ uint64_t tmp = byte & ~kHighBitMask; ++ ++ switch (++got_bytes_) { ++ case 1: ++ break; ++ case kMaxBytes: ++ if (tmp > 1 || next_byte_needed) { ++ state_ = kOverflow; ++ return state_; ++ } ++ [[fallthrough]]; ++ default: ++ tmp <<= 7 * (got_bytes_ - 1); ++ break; ++ } ++ ++ value_ += tmp; ++ if (!next_byte_needed) { ++ state_ = kReady; ++ } ++ ++ } else if (state_ == kReady) { ++ return kError; ++ } ++ ++ return state_; ++} ++ ++VarintPrefixReader::State VarintPrefixReader::consume(ipfs::ByteView& buffer) { ++ size_t consumed = 0; ++ State s(state_); ++ for (auto byte : buffer) { ++ ++consumed; ++ s = consume(to_integer(byte)); ++ if (s != kUnderflow) { ++ break; ++ } ++ } ++ if (consumed > 0 && (s == kReady || s == kUnderflow)) { ++ buffer = buffer.subspan(consumed); ++ } ++ return s; ++} ++ ++} // namespace libp2p::basic +diff --git a/third_party/ipfs_client/src/libp2p/common/hexutil.cc b/third_party/ipfs_client/src/libp2p/common/hexutil.cc +new file mode 100644 +index 0000000000000..82416ccfbdf94 +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/common/hexutil.cc +@@ -0,0 +1,95 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#include "vocab/byte_view.h" ++ ++#include "libp2p/common/hexutil.hpp" ++ ++#include ++ ++/* ++OUTCOME_CPP_DEFINE_CATEGORY(libp2p::common, UnhexError, e) { ++ using libp2p::common::UnhexError; ++ switch (e) { ++ case UnhexError::NON_HEX_INPUT: ++ return "Input contains non-hex characters"; ++ case UnhexError::NOT_ENOUGH_INPUT: ++ return "Input contains odd number of characters"; ++ default: ++ return "Unknown error"; ++ } ++} ++*/ ++namespace libp2p::common { ++ ++std::string int_to_hex(uint64_t n, size_t fixed_width) noexcept { ++ std::stringstream result; ++ result.width(static_cast(fixed_width)); ++ result.fill('0'); ++ result << std::hex << std::uppercase << n; ++ auto str = result.str(); ++ if (str.length() % 2 != 0) { ++ str.push_back('\0'); ++ for (int64_t i = static_cast(str.length()) - 2; i >= 0; --i) { ++ str[i + 1] = str[i]; ++ } ++ str[0] = '0'; ++ } ++ return str; ++} ++ ++namespace { ++std::string to_hex(ipfs::ByteView bytes, std::string_view alpha) { ++ std::string res(bytes.size() * 2, '\x00'); ++ auto o = res.begin(); ++ for (auto b : bytes) { ++ auto i = to_integer(b); ++ *(o++) = alpha[i / 256]; ++ *(o++) = alpha[i % 256]; ++ } ++ return res; ++} ++} // namespace ++ ++std::string hex_upper(ipfs::ByteView bytes) noexcept { ++ return to_hex(bytes, "0123456789ABCDEF"); ++} ++ ++std::string hex_lower(ipfs::ByteView bytes) noexcept { ++ return to_hex(bytes, "0123456789abcdef"); ++} ++ ++ipfs::expected, UnhexError> unhex( ++ std::string_view hex) { ++ if (hex.size() % 2) { ++ return ipfs::unexpected(UnhexError::NOT_ENOUGH_INPUT); ++ } ++ std::vector blob; ++ blob.reserve(hex.size() / 2); ++ bool bad_input = false; ++ auto toi = [&bad_input](auto c) { ++ if (c >= '0' && c <= '9') { ++ return c - '0'; ++ } else if (c >= 'a' && c <= 'f') { ++ return c - 'a' + 10; ++ } else if (c >= 'A' && c <= 'F') { ++ return c - 'F' + 10; ++ } else { ++ bad_input = true; ++ return 0; ++ } ++ }; ++ for (auto i = 0U; i < hex.size(); i += 2) { ++ auto hi = toi(hex[i]); ++ auto lo = toi(hex[i + 1]); ++ blob.push_back(static_cast((hi << 4) | lo)); ++ } ++ if (bad_input) { ++ return ipfs::unexpected(UnhexError::NOT_ENOUGH_INPUT); ++ } else { ++ return blob; ++ } ++} ++} // namespace libp2p::common +diff --git a/third_party/ipfs_client/src/libp2p/crypto/hasher.cc b/third_party/ipfs_client/src/libp2p/crypto/hasher.cc +new file mode 100644 +index 0000000000000..f57c7a6607d02 +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/crypto/hasher.cc +@@ -0,0 +1,16 @@ ++#include ++#include ++ ++#include "log_macros.h" ++ ++std::unique_ptr libp2p::crypto::CreateHasher( ++ multi::HashType algo) { ++ switch (algo) { ++ case multi::HashType::sha256: ++ return std::make_unique(); ++ default: ++ LOG(FATAL) << "Unimplemented hash type: " ++ << static_cast(algo); ++ } ++ return {}; ++} +diff --git a/third_party/ipfs_client/src/libp2p/crypto/protobuf_key.hpp b/third_party/ipfs_client/src/libp2p/crypto/protobuf_key.hpp +new file mode 100644 +index 0000000000000..459426f8c58a2 +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/crypto/protobuf_key.hpp +@@ -0,0 +1,29 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#ifndef KAGOME_PROTOBUF_KEY_HPP ++#define KAGOME_PROTOBUF_KEY_HPP ++ ++#include ++#include ++ ++#include ++ ++namespace libp2p::crypto { ++ /** ++ * Strict type for key, which is encoded into Protobuf format ++ */ ++ struct ProtobufKey : public boost::equality_comparable { ++ explicit ProtobufKey(std::vector key) : key{std::move(key)} {} ++ ++ std::vector key; ++ ++ bool operator==(const ProtobufKey &other) const { ++ return key == other.key; ++ } ++ }; ++} // namespace libp2p::crypto ++ ++#endif // KAGOME_PROTOBUF_KEY_HPP +diff --git a/third_party/ipfs_client/src/libp2p/crypto/provider.h b/third_party/ipfs_client/src/libp2p/crypto/provider.h +new file mode 100644 +index 0000000000000..0ffb79f649643 +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/crypto/provider.h +@@ -0,0 +1,28 @@ ++#ifndef IPFS_PROVIDER_H_ ++#define IPFS_PROVIDER_H_ ++ ++#include "libp2p/crypto/key.h" ++ ++#include ++ ++#include ++#include ++ ++namespace libp2p::crypto { ++class Provider { ++ public: ++ template ++ using Result = ipfs::expected; ++ ++ virtual Result derive(const PrivateKey& private_key) const = 0; ++ ++ virtual Result sign(ipfs::ByteView message, ++ const PrivateKey& private_key) const = 0; ++ ++ virtual Result verify(ipfs::ByteView message, ++ const Signature& signature, ++ const PublicKey& key) const = 0; ++}; ++} // namespace libp2p::crypto ++ ++#endif // IPFS_PROVIDER_H_ +diff --git a/third_party/ipfs_client/src/libp2p/crypto/sha/sha256.cc b/third_party/ipfs_client/src/libp2p/crypto/sha/sha256.cc +new file mode 100644 +index 0000000000000..3354a41d162c3 +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/crypto/sha/sha256.cc +@@ -0,0 +1,99 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#include ++ ++#include ++ ++#include ++#include ++ ++#if __GNUG__ ++// OpenSSL wants us to use 3.0 ++#pragma GCC diagnostic ignored "-Wdeprecated-declarations" ++#endif ++ ++namespace libp2p::crypto { ++Sha256::Sha256() { // NOLINT ++ initialized_ = 1 == SHA256_Init(&ctx_); ++} ++ ++Sha256::~Sha256() { ++ sinkCtx(); ++} ++ ++auto Sha256::write(ipfs::ByteView data) -> Result { ++ if (not initialized_) { ++ return ipfs::unexpected{ ++ HmacProviderError::FAILED_INITIALIZE_CONTEXT}; ++ } ++ if (1 != SHA256_Update(&ctx_, data.data(), data.size())) { ++ return ipfs::unexpected{HmacProviderError::FAILED_UPDATE_DIGEST}; ++ } ++ return true; ++} ++ ++auto Sha256::digestOut(ipfs::span out) const -> Result { ++ if (not initialized_) { ++ return ipfs::unexpected{ ++ HmacProviderError::FAILED_INITIALIZE_CONTEXT}; ++ } ++ if (out.size() != static_cast(digestSize())) { ++ return ipfs::unexpected{HmacProviderError::WRONG_DIGEST_SIZE}; ++ } ++ SHA256_CTX ctx = ctx_; ++ if (1 != SHA256_Final(reinterpret_cast(out.data()), &ctx)) { ++ return ipfs::unexpected{HmacProviderError::FAILED_FINALIZE_DIGEST}; ++ } ++ return true; ++} ++ ++auto Sha256::reset() -> Result { ++ sinkCtx(); ++ if (1 != SHA256_Init(&ctx_)) { ++ return ipfs::unexpected{ ++ HmacProviderError::FAILED_INITIALIZE_CONTEXT}; ++ } ++ initialized_ = true; ++ return true; ++} ++ ++size_t Sha256::digestSize() const { ++ return SHA256_DIGEST_LENGTH; ++} ++ ++size_t Sha256::blockSize() const { ++ return SHA256_CBLOCK; ++} ++ ++void Sha256::sinkCtx() { ++ if (initialized_) { ++ libp2p::common::Hash256 digest; ++ SHA256_Final(reinterpret_cast(digest.data()), &ctx_); ++ memset(digest.data(), 0, digest.size()); ++ initialized_ = false; ++ } ++} ++ ++HashType Sha256::hashType() const { ++ return HashType::SHA256; ++} ++ ++ipfs::expected sha256(ipfs::ByteView input) { ++ Sha256 sha; ++ auto write_res = sha.write(input); ++ if (!write_res.has_value()) { ++ return ipfs::unexpected{write_res.error()}; ++ } ++ libp2p::common::Hash256 result; ++ auto dig_res = sha.digestOut( ++ {reinterpret_cast(result.data()), result.size()}); ++ if (dig_res.has_value()) { ++ return result; ++ } else { ++ return ipfs::unexpected{dig_res.error()}; ++ } ++} ++} // namespace libp2p::crypto +diff --git a/third_party/ipfs_client/src/libp2p/multi/content_identifier.cc b/third_party/ipfs_client/src/libp2p/multi/content_identifier.cc +new file mode 100644 +index 0000000000000..8f84eb8a5619b +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/multi/content_identifier.cc +@@ -0,0 +1,47 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#include "libp2p/multi/content_identifier.hpp" ++#include "libp2p/common/hexutil.hpp" ++ ++#include ++ ++namespace libp2p::multi { ++ ++ContentIdentifier::ContentIdentifier(Version version, ++ MulticodecType::Code content_type, ++ Multihash content_address) ++ : version{version}, ++ content_type{content_type}, ++ content_address{std::move(content_address)} {} ++ ++std::string ContentIdentifier::toPrettyString(const std::string& base) const { ++ /// TODO(Harrm) FIL-14: hash type is a subset of multicodec type, better ++ /// move them to one place ++ auto hash_type = MulticodecType::getName( ++ static_cast(content_address.getType())); ++ std::string hash_hex = common::hex_lower(content_address.getHash()); ++ std::string hash_length = ++ std::to_string(content_address.getHash().size() * 8); ++ std::string v = "cidv" + std::to_string(static_cast(version)); ++ std::ostringstream oss; ++ oss << base << " - " << v << " - " << MulticodecType::getName(content_type) ++ << " - " << hash_type << '-' << hash_length << '-' << hash_hex; ++ return oss.str(); ++} ++ ++bool ContentIdentifier::operator==(const ContentIdentifier& c) const { ++ return version == c.version and content_type == c.content_type and ++ content_address == c.content_address; ++} ++ ++bool ContentIdentifier::operator<(const ContentIdentifier& c) const { ++ return version < c.version || ++ (version == c.version && (content_type < c.content_type || ++ (content_type == c.content_type && ++ content_address < c.content_address))); ++} ++ ++} // namespace libp2p::multi +diff --git a/third_party/ipfs_client/src/libp2p/multi/content_identifier_codec.cc b/third_party/ipfs_client/src/libp2p/multi/content_identifier_codec.cc +new file mode 100644 +index 0000000000000..e4e14fbb78c37 +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/multi/content_identifier_codec.cc +@@ -0,0 +1,239 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#include "libp2p/multi/content_identifier_codec.hpp" ++#include ++#include ++#include ++#include "libp2p/multi/multicodec_type.hpp" ++ ++#include "log_macros.h" ++ ++#include ++ ++std::ostream& operator<<(std::ostream& str, ++ libp2p::multi::ContentIdentifierCodec::EncodeError e) { ++ using E = libp2p::multi::ContentIdentifierCodec::EncodeError; ++ switch (e) { ++ case E::INVALID_CONTENT_TYPE: ++ return str << "Content type does not conform the version"; ++ case E::INVALID_HASH_LENGTH: ++ return str << "Hash length is invalid; Must be 32 bytes for sha256 in " ++ "version 0"; ++ case E::INVALID_HASH_TYPE: ++ return str << "Hash type is invalid; Must be sha256 in version 0"; ++ case E::INVALID_BASE_ENCODING: ++ return str << "Invalid base encoding"; ++ case E::VERSION_UNSUPPORTED: ++ return str << "Content identifier version unsupported"; ++ default: ++ LOG(FATAL) << "Invalid EncodeError: " << static_cast(e); ++ return str << "invalid error"; ++ } ++} ++std::ostream& operator<<(std::ostream& str, ++ libp2p::multi::ContentIdentifierCodec::DecodeError e) { ++ return str << libp2p::multi::Stringify(e); ++} ++namespace libp2p::multi { ++std::string_view Stringify(ContentIdentifierCodec::DecodeError e) { ++ using E = ContentIdentifierCodec::DecodeError; ++ switch (e) { ++ case E::EMPTY_MULTICODEC: ++ return "Multicodec prefix is absent"; ++ case E::EMPTY_VERSION: ++ return "Version is absent"; ++ case E::MALFORMED_VERSION: ++ return "Version is malformed; Must be a non-negative integer"; ++ case E::RESERVED_VERSION: ++ return "Version is greater than the latest version"; ++ case E::CID_TOO_SHORT: ++ return "CID too short"; ++ case E::BAD_MULTIHASH: ++ return "Bad multihash input"; ++ case E::BAD_MULTIBASE: ++ return "Bad multibase input"; ++ case E::UNSUPPORTED_MULTIBASE: ++ return "Unsupported multibase"; ++ default: ++ LOG(FATAL) << "Invalid decode error " << static_cast(e); ++ return "invalid error"; ++ } ++} ++ ++ipfs::expected, ContentIdentifierCodec::EncodeError> ++ContentIdentifierCodec::encode(const ContentIdentifier& cid) { ++ std::vector bytes; ++ if (cid.version == ContentIdentifier::Version::V1) { ++ UVarint version(static_cast(cid.version)); ++ common::append(bytes, version.toBytes()); ++ UVarint type(static_cast(cid.content_type)); ++ common::append(bytes, type.toBytes()); ++ auto const& hash = cid.content_address.toBuffer(); ++ common::append(bytes, hash); ++ ++ } else if (cid.version == ContentIdentifier::Version::V0) { ++ if (cid.content_type != MulticodecType::Code::DAG_PB) { ++ return ipfs::unexpected{EncodeError::INVALID_CONTENT_TYPE}; ++ } ++ if (cid.content_address.getType() != HashType::sha256) { ++ return ipfs::unexpected{EncodeError::INVALID_HASH_TYPE}; ++ } ++ if (cid.content_address.getHash().size() != 32) { ++ return ipfs::unexpected{EncodeError::INVALID_HASH_LENGTH}; ++ } ++ auto const& hash = cid.content_address.toBuffer(); ++ common::append(bytes, hash); ++ } ++ return bytes; ++} ++ ++std::vector ContentIdentifierCodec::encodeCIDV0( ++ const void* byte_buffer, ++ size_t sz) { ++ auto digest_res = crypto::sha256(ipfs::ByteView{ ++ reinterpret_cast(byte_buffer), sz}); // NOLINT ++ // BOOST_ASSERT(digest_res.has_value()); ++ // TODO DCHECK ++ auto& hash = digest_res.value(); ++ ++ std::vector bytes; ++ bytes.reserve(hash.size()); ++ bytes.push_back(static_cast(HashType::sha256)); ++ bytes.push_back(static_cast(hash.size())); ++ std::transform(hash.begin(), hash.end(), std::back_inserter(bytes), ++ [](auto b) { return ipfs::Byte{b}; }); ++ return bytes; ++} ++ ++std::vector ContentIdentifierCodec::encodeCIDV1( ++ MulticodecType::Code content_type, ++ const Multihash& mhash) { ++ std::vector bytes; ++ // Reserve space for CID version size + content-type size + multihash size ++ bytes.reserve(1 + 1 + mhash.toBuffer().size()); ++ bytes.push_back(ipfs::Byte{1}); // CID version ++ bytes.push_back(static_cast(content_type)); // Content-Type ++ std::copy(mhash.toBuffer().begin(), mhash.toBuffer().end(), ++ std::back_inserter(bytes)); // multihash data ++ return bytes; ++} ++ ++auto ContentIdentifierCodec::decode(ipfs::ByteView bytes) ++ -> ipfs::expected { ++ if (bytes.size() == 34 and bytes[0] == ipfs::Byte{0x12} and ++ bytes[1] == ipfs::Byte{0x20}) { ++ auto hash = Multihash::createFromBytes(bytes); ++ if (hash.has_value()) { ++ return ContentIdentifier(ContentIdentifier::Version::V0, ++ MulticodecType::Code::DAG_PB, ++ std::move(hash.value())); ++ } else { ++ return ipfs::unexpected{DecodeError::BAD_MULTIHASH}; ++ } ++ } ++ auto version_opt = UVarint::create(bytes); ++ if (!version_opt) { ++ return ipfs::unexpected{DecodeError::EMPTY_VERSION}; ++ } ++ auto version = version_opt.value().toUInt64(); ++ if (version == 1) { ++ auto version_length = UVarint::calculateSize(bytes); ++ auto multicodec_opt = ++ UVarint::create(bytes.subspan(static_cast(version_length))); ++ if (!multicodec_opt) { ++ return ipfs::unexpected{DecodeError::EMPTY_MULTICODEC}; ++ } ++ auto multicodec_length = UVarint::calculateSize( ++ bytes.subspan(static_cast(version_length))); ++ auto hash = Multihash::createFromBytes( ++ bytes.subspan(version_length + multicodec_length)); ++ if (hash.has_value()) { ++ return ContentIdentifier( ++ ContentIdentifier::Version::V1, ++ MulticodecType::Code(multicodec_opt.value().toUInt64()), ++ std::move(hash.value())); ++ } else { ++ return ipfs::unexpected{DecodeError::BAD_MULTIHASH}; ++ } ++ } ++ if (version <= 0) { ++ return ipfs::unexpected{DecodeError::MALFORMED_VERSION}; ++ } ++ return ipfs::unexpected{DecodeError::RESERVED_VERSION}; ++} ++ ++ipfs::expected ++ContentIdentifierCodec::toString(const ContentIdentifier& cid) { ++ std::string result; ++ auto encode_result = encode(cid); ++ if (!encode_result.has_value()) { ++ return ipfs::unexpected{encode_result.error()}; ++ } ++ auto& cid_bytes = encode_result.value(); ++ switch (cid.version) { ++ case ContentIdentifier::Version::V0: ++ result = detail::encodeBase58(cid_bytes); ++ break; ++ case ContentIdentifier::Version::V1: ++ result = MultibaseCodecImpl().encode( ++ cid_bytes, MultibaseCodec::Encoding::BASE32_LOWER); ++ break; ++ default: ++ return ipfs::unexpected{EncodeError::VERSION_UNSUPPORTED}; ++ } ++ return result; ++} ++ ++auto ContentIdentifierCodec::toStringOfBase(const ContentIdentifier& cid, ++ MultibaseCodec::Encoding base) ++ -> ipfs::expected { ++ std::string result; ++ auto enc_res = encode(cid); ++ if (!enc_res.has_value()) { ++ return ipfs::unexpected{enc_res.error()}; ++ } ++ auto& cid_bytes = enc_res.value(); ++ switch (cid.version) { ++ case ContentIdentifier::Version::V0: ++ if (base != MultibaseCodec::Encoding::BASE58) { ++ return ipfs::unexpected{ ++ EncodeError::INVALID_BASE_ENCODING}; ++ } ++ result = detail::encodeBase58(cid_bytes); ++ break; ++ case ContentIdentifier::Version::V1: ++ result = MultibaseCodecImpl().encode(cid_bytes, base); ++ break; ++ default: ++ return ipfs::unexpected{EncodeError::VERSION_UNSUPPORTED}; ++ } ++ return result; ++} ++ ++auto ContentIdentifierCodec::fromString(const std::string& str) ++ -> ipfs::expected { ++ if (str.size() < 2) { ++ return ipfs::unexpected{DecodeError::CID_TOO_SHORT}; ++ } ++ ++ if (str.size() == 46 && str.substr(0, 2) == "Qm") { ++ auto hash = detail::decodeBase58(str); ++ if (hash.has_value()) { ++ return decode(hash.value()); ++ } else { ++ return ipfs::unexpected{DecodeError::BAD_MULTIHASH}; ++ } ++ } ++ auto bytes = MultibaseCodecImpl().decode(str); ++ if (bytes.has_value()) { ++ return decode(bytes.value()); ++ } else if (bytes.error() == MultibaseCodec::Error::UNSUPPORTED_BASE) { ++ return ipfs::unexpected{DecodeError::UNSUPPORTED_MULTIBASE}; ++ } else { ++ return ipfs::unexpected{DecodeError::BAD_MULTIBASE}; ++ } ++} ++} // namespace libp2p::multi +diff --git a/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base16.cc b/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base16.cc +new file mode 100644 +index 0000000000000..b032cccbbdc1c +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base16.cc +@@ -0,0 +1,104 @@ ++#include ++ ++namespace b16 = ipfs::base16; ++ ++namespace { ++std::uint8_t to_i(char c); ++template ++char to_c(std::uint8_t n) { ++ if (n < 10) { ++ return n + '0'; ++ } else { ++ return n - 10 + a; ++ } ++} ++template ++std::string encode(ipfs::ByteView bytes) { ++ std::string result; ++ result.reserve(bytes.size() * 2); ++ for (auto b : bytes) { ++ auto i = to_integer(b); ++ result.push_back(to_c(i >> 4)); ++ result.push_back(to_c(i & 0xF)); ++ } ++ return result; ++} ++} // namespace ++ ++std::string b16::encodeLower(ByteView bytes) { ++ return encode<'a'>(bytes); ++} ++std::string b16::encodeUpper(ByteView bytes) { ++ return encode<'A'>(bytes); ++} ++auto b16::decode(std::string_view s) -> Decoded { ++ ByteArray result(s.size() / 2, ipfs::Byte{}); ++ for (auto i = 0U; i + 1U < s.size(); i += 2U) { ++ auto a = to_i(s[i]); ++ auto b = to_i(s[i + 1]); ++ if (a > 0xF || b > 0xF) { ++ return ipfs::unexpected{BaseError::INVALID_BASE16_INPUT}; ++ } ++ result[i / 2] = ipfs::Byte{static_cast((a << 4) | b)}; ++ } ++ if (s.size() % 2) { ++ auto a = to_i(s.back()); ++ if (a <= 0xF) { ++ result.push_back(ipfs::Byte{a}); ++ } ++ } ++ return result; ++} ++ ++namespace { ++std::uint8_t to_i(char c) { ++ switch (c) { ++ case '0': ++ return 0; ++ case '1': ++ return 1; ++ case '2': ++ return 2; ++ case '3': ++ return 3; ++ case '4': ++ return 4; ++ case '5': ++ return 5; ++ case '6': ++ return 6; ++ case '7': ++ return 7; ++ case '8': ++ return 8; ++ case '9': ++ return 9; ++ case 'a': ++ return 10; ++ case 'b': ++ return 11; ++ case 'c': ++ return 12; ++ case 'd': ++ return 13; ++ case 'e': ++ return 14; ++ case 'f': ++ return 15; ++ case 'A': ++ return 10; ++ case 'B': ++ return 11; ++ case 'C': ++ return 12; ++ case 'D': ++ return 13; ++ case 'E': ++ return 14; ++ case 'F': ++ return 15; ++ default: ++ return 0xFF; ++ } ++} ++} // namespace +diff --git a/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base32.cc b/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base32.cc +new file mode 100644 +index 0000000000000..36030d0b445fb +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base32.cc +@@ -0,0 +1,200 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++/** ++ * base32 (de)coder implementation as specified by RFC4648. ++ * ++ * Copyright (c) 2010 Adrien Kunysz ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a copy ++ * of this software and associated documentation files (the "Software"), to deal ++ * in the Software without restriction, including without limitation the rights ++ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell ++ * copies of the Software, and to permit persons to whom the Software is ++ * furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice shall be included in ++ * all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ++ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, ++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN ++ * THE SOFTWARE. ++ **/ ++ ++#include "libp2p/multi/multibase_codec/codecs/base32.hpp" ++#include "libp2p/multi/multibase_codec/codecs/base_error.hpp" ++ ++#include ++ ++namespace libp2p::multi::detail { ++const std::string kUpperBase32Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; ++const std::string kLowerBase32Alphabet = "abcdefghijklmnopqrstuvwxyz234567"; ++ ++enum Base32Mode { ++ LOWER, ++ UPPER, ++}; ++ ++int get_byte(int block) { ++ return block * 5 / 8; ++} ++ ++int get_bit(int block) { ++ return 8 - 5 - block * 5 % 8; ++} ++ ++char encode_char(unsigned char c, Base32Mode mode) { ++ if (mode == Base32Mode::UPPER) { ++ return kUpperBase32Alphabet[c & 0x1F]; // 0001 1111 ++ } ++ return kLowerBase32Alphabet[c & 0x1F]; ++} ++ ++unsigned char shift_right(uint8_t byte, int8_t offset) { ++ if (offset > 0) { ++ return byte >> offset; ++ } ++ ++ return byte << -offset; ++} ++ ++unsigned char shift_left(uint8_t byte, int8_t offset) { ++ return shift_right(byte, -offset); ++} ++ ++int encode_sequence(ipfs::span plain, ++ ipfs::span coded, ++ Base32Mode mode) { ++ for (int block = 0; block < 8; block++) { ++ int byte = get_byte(block); ++ int bit = get_bit(block); ++ ++ if (byte >= static_cast(plain.size())) { ++ return block; ++ } ++ ++ unsigned char c = shift_right(plain[byte], bit); ++ ++ if (bit < 0 && byte < static_cast(plain.size()) - 1L) { ++ c |= shift_right(plain[byte + 1], 8 + bit); ++ } ++ coded[block] = encode_char(c, mode); ++ } ++ return 8; ++} ++ ++std::string encodeBase32(ipfs::ByteView bytes, Base32Mode mode) { ++ std::string result; ++ if (bytes.size() % 5 == 0) { ++ result = std::string(bytes.size() / 5 * 8, ' '); ++ } else { ++ result = std::string((bytes.size() / 5 + 1) * 8, ' '); ++ } ++ ++ for (size_t i = 0, j = 0; i < bytes.size(); i += 5, j += 8) { ++ int n = encode_sequence( ++ ipfs::span(reinterpret_cast(&bytes[i]), ++ std::min(bytes.size() - i, 5)), ++ ipfs::span(&result[j], 8U), mode); ++ if (n < 8) { ++ result.erase(result.end() - (8 - n), result.end()); ++ } ++ } ++ ++ return result; ++} ++ ++std::string encodeBase32Upper(ipfs::ByteView bytes) { ++ return encodeBase32(bytes, Base32Mode::UPPER); ++} ++ ++std::string encodeBase32Lower(ipfs::ByteView bytes) { ++ return encodeBase32(bytes, Base32Mode::LOWER); ++} ++ ++int decode_char(unsigned char c, Base32Mode mode) { ++ char decoded_ch = -1; ++ ++ if (mode == Base32Mode::UPPER) { ++ if (c >= 'A' && c <= 'Z') { ++ decoded_ch = c - 'A'; // NOLINT ++ } ++ } else { ++ if (c >= 'a' && c <= 'z') { ++ decoded_ch = c - 'a'; // NOLINT ++ } ++ } ++ if (c >= '2' && c <= '7') { ++ decoded_ch = c - '2' + 26; // NOLINT ++ } ++ ++ return decoded_ch; ++} ++ ++ipfs::expected decode_sequence(ipfs::span coded, ++ ipfs::span plain, ++ Base32Mode mode) { ++ plain[0] = 0; ++ for (int block = 0; block < 8; block++) { ++ int bit = get_bit(block); ++ int byte = get_byte(block); ++ ++ if (block >= static_cast(coded.size())) { ++ return byte; ++ } ++ int c = decode_char(coded[block], mode); ++ if (c < 0) { ++ // return absl::InvalidArgumentError("INVALID_BASE32_INPUT"); ++ return ipfs::unexpected{BaseError::INVALID_BASE32_INPUT}; ++ } ++ ++ plain[byte] |= shift_left(c, bit); ++ if (bit < 0) { ++ plain[byte + 1] = shift_left(c, 8 + bit); ++ } ++ } ++ return 5; ++} ++ ++ipfs::expected decodeBase32( ++ std::string_view string, ++ Base32Mode mode) { ++ common::ByteArray result; ++ if (string.size() % 8 == 0) { ++ result = common::ByteArray(string.size() / 8 * 5, ipfs::Byte{0}); ++ } else { ++ result = common::ByteArray((string.size() / 8 + 1) * 5, ipfs::Byte{0}); ++ } ++ ++ for (size_t i = 0, j = 0; i < string.size(); i += 8, j += 5) { ++ auto n = decode_sequence( ++ ipfs::span(&string[i], ++ std::min(string.size() - i, 8)), ++ ipfs::span(reinterpret_cast(&result[j]), 5U), mode); ++ if (!n.has_value()) { ++ return ipfs::unexpected{n.error()}; ++ } ++ if (n.value() < 5) { ++ result.erase(result.end() - (5 - n.value()), result.end()); ++ } ++ } ++ return result; ++} ++ ++ipfs::expected decodeBase32Upper( ++ std::string_view string) { ++ return decodeBase32(string, Base32Mode::UPPER); ++} ++ ++ipfs::expected decodeBase32Lower( ++ std::string_view string) { ++ return decodeBase32(string, Base32Mode::LOWER); ++} ++ ++} // namespace libp2p::multi::detail +diff --git a/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base36.cc b/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base36.cc +new file mode 100644 +index 0000000000000..7fe33abf44b3f +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base36.cc +@@ -0,0 +1,56 @@ ++#include ++ ++#include ++ ++#include ++ ++namespace det = libp2p::multi::detail; ++ ++namespace { ++constexpr double kLengthRatio = 0.646240625; // log(36)/log(256) ++ ++std::int_least16_t digit_value(char digit) { ++ if (digit < '0') { ++ return -1; ++ } else if (digit <= '9') { ++ return digit - '0'; ++ } else if (digit < 'A') { ++ return -2; ++ } else if (digit <= 'Z') { ++ return (digit - 'A') + 10; ++ } else if (digit < 'a') { ++ return -3; ++ } else if (digit <= 'z') { ++ return (digit - 'a') + 10; ++ } else { ++ return -4; ++ } ++} ++int operator*(int a, ipfs::Byte b) { ++ return a * static_cast(b); ++} ++} // namespace ++ ++std::string det::encodeBase36Lower(ipfs::ByteView) { ++ std::abort(); ++} ++ ++auto det::decodeBase36(std::string_view str_b36) ++ -> ipfs::expected { ++ common::ByteArray out; ++ out.resize(std::ceil(static_cast(str_b36.size()) * kLengthRatio), ++ ipfs::Byte{}); ++ for (auto digit : str_b36) { // chunk) { ++ int val = digit_value(digit); ++ if (val < 0) { ++ return ipfs::unexpected{BaseError::INVALID_BASE36_INPUT}; ++ } ++ auto mod_byte = [&val](auto& b) { ++ val += 36 * b; ++ b = static_cast(val & 0xFF); ++ val >>= 8; ++ }; ++ std::for_each(out.rbegin(), out.rend(), mod_byte); ++ } ++ return out; ++} +diff --git a/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base58.cc b/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base58.cc +new file mode 100644 +index 0000000000000..c3af0a2a317bb +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/multi/multibase_codec/codecs/base58.cc +@@ -0,0 +1,187 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#include "libp2p/multi/multibase_codec/codecs/base58.hpp" ++ ++#include ++#include ++ ++// #include ++#include "libp2p/multi/multibase_codec/codecs/base_error.hpp" ++ ++namespace { ++// All alphanumeric characters except for "0", "I", "O", and "l" ++constexpr std::string_view pszBase58 = ++ "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; ++ ++// clang-format off ++ constexpr std::array mapBase58 = { ++ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, ++ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, ++ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, ++ -1, 0, 1, 2, 3, 4, 5, 6, 7, 8,-1,-1,-1,-1,-1,-1, ++ -1, 9,10,11,12,13,14,15, 16,-1,17,18,19,20,21,-1, ++ 22,23,24,25,26,27,28,29, 30,31,32,-1,-1,-1,-1,-1, ++ -1,33,34,35,36,37,38,39, 40,41,42,43,-1,44,45,46, ++ 47,48,49,50,51,52,53,54, 55,56,57,-1,-1,-1,-1,-1, ++ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, ++ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, ++ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, ++ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, ++ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, ++ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, ++ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, ++ -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, ++ }; ++// clang-format on ++ ++/** ++ * Tests if the given character is a whitespace character. The whitespace ++ * characters are: space, form-feed ('\f'), newline ('\n'), carriage return ++ * ('\r'), horizontal tab ('\t'), and vertical tab ('\v') ++ * @param c - char to be tested ++ * @return true, if char is space, false otherwise ++ */ ++constexpr bool isSpace(char c) noexcept { ++ return c == ' ' || c == '\f' || c == '\n' || c == '\r' || c == '\t' || ++ c == '\v'; ++} ++} // namespace ++ ++namespace libp2p::multi::detail { ++ ++/** ++ * Actual implementation of the encoding ++ * @param pbegin - pointer to the beginning of bytes collection ++ * @param pend - pointer to the end of bytes collection ++ * @return encoded string ++ */ ++std::string encodeImpl(const unsigned char* pbegin, const unsigned char* pend) { ++ // Skip & count leading zeroes. ++ int zeroes = 0; ++ int length = 0; ++ while (pbegin != pend && *pbegin == 0) { ++ std::advance(pbegin, 1); ++ zeroes++; ++ } ++ // Allocate enough space in big-endian base58 representation. ++ // log(256) / log(58), rounded up. ++ // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions) ++ int size = (pend - pbegin) * 138 / 100 + 1; ++ std::vector b58(size); ++ // Process the bytes. ++ while (pbegin != pend) { ++ int carry = *pbegin; ++ int i = 0; ++ // Apply "b58 = b58 * 256 + ch". ++ for (auto it = b58.rbegin(); ++ (carry != 0 || i < length) && (it != b58.rend()); it++, i++) { ++ carry += 256 * (*it); ++ *it = carry % 58; ++ carry /= 58; ++ } ++ ++ length = i; ++ std::advance(pbegin, 1); ++ } ++ // Skip leading zeroes in base58 result. ++ auto it = b58.begin() + (size - length); ++ while (it != b58.end() && *it == 0) { ++ it++; ++ } ++ // Translate the result into a string. ++ std::string str; ++ str.reserve(zeroes + (b58.end() - it)); ++ str.assign(zeroes, '1'); ++ while (it != b58.end()) { ++ str += pszBase58[*it]; ++ std::advance(it, 1); ++ } ++ return str; ++} ++ ++/** ++ * Actual implementation of the decoding ++ * @param psz - pointer to the string to be decoded ++ * @return decoded bytes, if the process went successfully, none otherwise ++ */ ++std::optional> decodeImpl(const char* psz) { ++ // Skip leading spaces. ++ while ((*psz != '0') && isSpace(*psz)) { ++ std::advance(psz, 1); ++ } ++ // Skip and count leading '1's. ++ int zeroes = 0; ++ int length = 0; ++ while (*psz == '1') { ++ zeroes++; ++ std::advance(psz, 1); ++ } ++ // Allocate enough space in big-endian base256 representation. ++ // log(58) / log(256), rounded up. ++ // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions) ++ int size = strlen(psz) * 733 / 1000 + 1; ++ std::vector b256(size); ++ // Process the characters. ++ while (*psz && !isSpace(*psz)) { // NOLINT ++ // Decode base58 character ++ int carry = mapBase58.at(static_cast(*psz)); ++ if (carry == -1) { // Invalid b58 character ++ return {}; ++ } ++ int i = 0; ++ for (auto it = b256.rbegin(); ++ (carry != 0 || i < length) && (it != b256.rend()); ++it, ++i) { ++ carry += 58 * (*it); ++ *it = carry % 256; ++ carry /= 256; ++ } ++ if (carry != 0) { ++ return {}; ++ } ++ length = i; ++ std::advance(psz, 1); ++ } ++ // Skip trailing spaces. ++ while (isSpace(*psz)) { ++ std::advance(psz, 1); ++ } ++ if (*psz != 0) { ++ return {}; ++ } ++ // Skip leading zeroes in b256. ++ auto it = b256.begin() + (size - length); ++ while (it != b256.end() && *it == 0) { ++ it++; ++ } ++ // Copy result into output vector. ++ std::vector vch(zeroes + (b256.end() - it)); ++ vch.assign(zeroes, 0x00); ++ while (it != b256.end()) { ++ vch.push_back(*(it++)); ++ } ++ return vch; ++} ++ ++std::string encodeBase58(ipfs::ByteView bytes) { ++ unsigned char const* ptr = ++ reinterpret_cast(bytes.data()); ++ return encodeImpl(ptr, ptr + bytes.size()); ++} ++ ++ipfs::expected decodeBase58( ++ std::string_view string) { ++ auto decoded_bytes = decodeImpl(string.data()); ++ if (decoded_bytes) { ++ ipfs::Byte const* b = ++ reinterpret_cast(decoded_bytes->data()); ++ ipfs::Byte const* e = std::next(b, decoded_bytes->size()); ++ return common::ByteArray(b, e); ++ } ++ // return absl::InvalidArgumentError("INVALID_BASE58_INPUT"); ++ return ipfs::unexpected(BaseError::INVALID_BASE58_INPUT); ++} ++ ++} // namespace libp2p::multi::detail +diff --git a/third_party/ipfs_client/src/libp2p/multi/multibase_codec/multibase_codec_impl.cc b/third_party/ipfs_client/src/libp2p/multi/multibase_codec/multibase_codec_impl.cc +new file mode 100644 +index 0000000000000..c550146c56568 +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/multi/multibase_codec/multibase_codec_impl.cc +@@ -0,0 +1,127 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#include ++ ++#include "log_macros.h" ++ ++#include ++ ++#include ++#include ++#include ++#include ++ ++namespace { ++using namespace libp2p::multi; // NOLINT ++using namespace libp2p::multi::detail; // NOLINT ++ ++/** ++ * Get the encoding by a character ++ * @param ch of encoding ++ * @return related encoding, if character stands for one of them, none ++ * otherwise ++ */ ++std::optional encodingByChar(char ch) { ++ switch (ch) { ++ case 'f': ++ return MultibaseCodec::Encoding::BASE16_LOWER; ++ case 'F': ++ return MultibaseCodec::Encoding::BASE16_UPPER; ++ case 'b': ++ return MultibaseCodec::Encoding::BASE32_LOWER; ++ case 'B': ++ return MultibaseCodec::Encoding::BASE32_UPPER; ++ case 'k': ++ return MultibaseCodec::Encoding::BASE36; ++ case 'z': ++ return MultibaseCodec::Encoding::BASE58; ++ case 'm': ++ return MultibaseCodec::Encoding::BASE64; ++ default: ++ return std::nullopt; ++ } ++} ++ ++struct CodecFunctions { ++ using EncodeFuncType = decltype(encodeBase58); ++ using DecodeFuncType = decltype(decodeBase58); ++ ++ EncodeFuncType* encode; ++ DecodeFuncType* decode; ++}; ++ ++ ++/// all available codec functions ++const std::unordered_map codecs{ ++ {MultibaseCodec::Encoding::BASE16_UPPER, ++ CodecFunctions{&ipfs::base16::encodeUpper, &ipfs::base16::decode}}, ++ {MultibaseCodec::Encoding::BASE16_LOWER, ++ CodecFunctions{&ipfs::base16::encodeLower, &ipfs::base16::decode}}, ++ {MultibaseCodec::Encoding::BASE32_UPPER, ++ CodecFunctions{&encodeBase32Upper, &decodeBase32Upper}}, ++ {MultibaseCodec::Encoding::BASE32_LOWER, ++ CodecFunctions{&encodeBase32Lower, &decodeBase32Lower}}, ++ {MultibaseCodec::Encoding::BASE36, ++ CodecFunctions{&encodeBase36Lower, &decodeBase36}}, ++ {MultibaseCodec::Encoding::BASE58, ++ CodecFunctions{&encodeBase58, &decodeBase58}} ++ //, {MultibaseCodec::Encoding::BASE64,CodecFunctions{&todo_encode, ++ // &todo_decode}} ++}; ++} // namespace ++ ++namespace libp2p::multi { ++using common::ByteArray; ++ ++std::string MultibaseCodecImpl::encode(const ByteArray& bytes, ++ Encoding encoding) const { ++ if (bytes.empty()) { ++ return ""; ++ } ++ ++ return static_cast(encoding) + codecs.at(encoding).encode(bytes); ++} ++ ++auto MultibaseCodecImpl::decode(std::string_view string) const ++ -> FactoryResult { ++ if (string.length() < 2) { ++ return ipfs::unexpected{Error::INPUT_TOO_SHORT}; ++ } ++ ++ auto encoding_base = encodingByChar(string.front()); ++ if (!encoding_base) { ++ return ipfs::unexpected{Error::UNSUPPORTED_BASE}; ++ } ++ auto codec = codecs.find(*encoding_base); ++ if (codecs.end() == codec) { ++ return ipfs::unexpected{Error::UNSUPPORTED_BASE}; ++ } ++ auto result = codec->second.decode(string.substr(1)); ++ if (result.has_value()) { ++ return result.value(); ++ } else { ++ return ipfs::unexpected{Error::BASE_CODEC_ERROR}; ++ } ++} ++} // namespace libp2p::multi ++ ++bool libp2p::multi::case_critical(MultibaseCodec::Encoding e) { ++ using E = MultibaseCodec::Encoding; ++ switch (e) { ++ case E::BASE16_LOWER: ++ case E::BASE16_UPPER: ++ case E::BASE32_LOWER: ++ case E::BASE32_UPPER: ++ case E::BASE36: ++ return false; ++ case E::BASE58: ++ case E::BASE64: ++ return true; ++ } ++ LOG(FATAL) << "TODO implement encode for this multibase encoding " ++ << static_cast(e); ++ return false; ++} +diff --git a/third_party/ipfs_client/src/libp2p/multi/multihash.cc b/third_party/ipfs_client/src/libp2p/multi/multihash.cc +new file mode 100644 +index 0000000000000..a7652c34b382b +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/multi/multihash.cc +@@ -0,0 +1,170 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#include "libp2p/multi/multihash.hpp" ++ ++#include "libp2p/basic/varint_prefix_reader.hpp" ++#include "libp2p/common/hexutil.hpp" ++#include "libp2p/common/types.hpp" ++ ++using libp2p::common::ByteArray; ++using libp2p::common::hex_upper; ++using libp2p::common::unhex; ++ ++std::string libp2p::multi::to_string(Multihash::Error e) { ++ using E = libp2p::multi::Multihash::Error; ++ switch (e) { ++ case E::ZERO_INPUT_LENGTH: ++ return "The length encoded in the header is zero"; ++ case E::INCONSISTENT_LENGTH: ++ return "The length encoded in the input data header doesn't match the " ++ "actual length of the input data"; ++ case E::INPUT_TOO_LONG: ++ return "The length of the input exceeds the maximum length of " + ++ std::to_string(libp2p::multi::Multihash::kMaxHashLength); ++ case E::INPUT_TOO_SHORT: ++ return "The length of the input is less than the required minimum of two " ++ "bytes for the multihash header"; ++ default: ++ return "Unknown error"; ++ } ++} ++ ++namespace libp2p::multi { ++ ++Multihash::Multihash(HashType type, ipfs::ByteView hash) ++ : data_(std::make_shared(type, hash)) {} ++ ++Multihash::Multihash(const Multihash& other) : data_{other.data_} {} ++ ++Multihash::Multihash(Multihash&& other) noexcept : data_{other.data_} {} ++ ++Multihash::~Multihash() noexcept {} ++ ++namespace { ++template ++inline void appendVarint(Buffer& buffer, ipfs::Byte t) { ++ do { ++ auto byte = t & ipfs::Byte{0x7F}; ++ t >>= 7; ++ if (t != ipfs::Byte{0}) { ++ byte |= ipfs::Byte{0x80}; ++ } ++ buffer.push_back(byte); ++ } while (t != ipfs::Byte{0}); ++} ++} // namespace ++ ++Multihash::Data::Data(HashType t, ipfs::ByteView h) : type(t) { ++ bytes.reserve(h.size() + 4); ++ appendVarint(bytes, static_cast(type)); ++ // BOOST_ASSERT(h.size() <= std::numeric_limits::max()); ++ bytes.push_back(static_cast(h.size())); ++ hash_offset = bytes.size(); ++ bytes.insert(bytes.end(), h.begin(), h.end()); ++ std_hash = std::hash{}(std::string_view{ ++ reinterpret_cast(bytes.data()), bytes.size()}); ++} ++ ++Multihash::Data::~Data() noexcept {} ++ ++const Multihash::Data& Multihash::data() const { ++ return *data_; ++} ++ ++size_t Multihash::stdHash() const { ++ return data().std_hash; ++} ++ ++using Result = ipfs::expected; ++ ++Result Multihash::create(HashType type, ipfs::ByteView hash) { ++ if (hash.size() > kMaxHashLength) { ++ return ipfs::unexpected(Error::INPUT_TOO_LONG); ++ } ++ ++ return Multihash{type, hash}; ++} ++ ++Result Multihash::createFromHex(std::string_view hex) { ++ auto result = unhex(hex); ++ if (result.has_value()) { ++ auto view = ipfs::ByteView{result.value().data(), result.value().size()}; ++ return Multihash::createFromBytes(view); ++ } else { ++ return ipfs::unexpected(Error::INVALID_HEXADECIMAL_INPUT); ++ } ++} ++ ++Result Multihash::createFromBytes(ipfs::ByteView b) { ++ if (b.size() < kHeaderSize) { ++ return ipfs::unexpected(Error::INPUT_TOO_SHORT); ++ } ++ ++ basic::VarintPrefixReader vr; ++ if (vr.consume(b) != basic::VarintPrefixReader::kReady) { ++ return ipfs::unexpected(Error::INPUT_TOO_SHORT); ++ } ++ ++ const auto type = static_cast(vr.value()); ++ if (b.empty()) { ++ return ipfs::unexpected(Error::INPUT_TOO_SHORT); ++ } ++ ++ auto length = to_integer(b[0]); ++ ipfs::ByteView hash = b.subspan(1); ++ ++ if (length == 0) { ++ return ipfs::unexpected(Error::ZERO_INPUT_LENGTH); ++ } ++ ++ if (hash.size() != length) { ++ return ipfs::unexpected(Error::INCONSISTENT_LENGTH); ++ } ++ ++ return Multihash::create(type, hash); ++} ++ ++const HashType& Multihash::getType() const { ++ return data().type; ++} ++ ++ipfs::ByteView Multihash::getHash() const { ++ const auto& d = data(); ++ return ipfs::ByteView(d.bytes.data(), d.bytes.size()).subspan(d.hash_offset); ++} ++ ++std::string Multihash::toHex() const { ++ return hex_upper(data().bytes); ++} ++ ++std::vector const& Multihash::toBuffer() const { ++ auto& d = data(); ++ return d.bytes; ++} ++ ++bool Multihash::operator==(const Multihash& other) const { ++ const auto& a = data(); ++ const auto& b = other.data(); ++ if (data_ == other.data_) { ++ return true; ++ } ++ return a.bytes == b.bytes && a.type == b.type; ++} ++ ++bool Multihash::operator!=(const Multihash& other) const { ++ return !(this->operator==(other)); ++} ++ ++bool Multihash::operator<(const class libp2p::multi::Multihash& other) const { ++ const auto& a = data(); ++ const auto& b = other.data(); ++ if (a.type == b.type) { ++ return a.bytes < b.bytes; ++ } ++ return a.type < b.type; ++} ++ ++} // namespace libp2p::multi +diff --git a/third_party/ipfs_client/src/libp2p/multi/uvarint.cc b/third_party/ipfs_client/src/libp2p/multi/uvarint.cc +new file mode 100644 +index 0000000000000..b8b2712c91a0a +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/multi/uvarint.cc +@@ -0,0 +1,111 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#include ++ ++#include ++ ++namespace libp2p::multi { ++using common::hex_upper; ++ ++UVarint::UVarint(UVarint const& rhs) : bytes_(rhs.bytes_) {} ++UVarint::UVarint(uint64_t number) { ++ do { ++ auto byte = static_cast(number) & ipfs::Byte{0x7f}; ++ number >>= 7; ++ if (number != 0) { ++ byte |= ipfs::Byte{0x80}; ++ } ++ bytes_.push_back(byte); ++ } while (number != 0); ++} ++ ++UVarint::UVarint(ipfs::ByteView varint_bytes) ++ : bytes_(varint_bytes.begin(), ++ varint_bytes.begin() + calculateSize(varint_bytes)) {} ++ ++UVarint::UVarint(ipfs::ByteView varint_bytes, size_t varint_size) ++ : bytes_(varint_bytes.begin(), varint_bytes.begin() + varint_size) {} ++ ++std::optional UVarint::create(ipfs::ByteView varint_bytes) { ++ size_t size = calculateSize(varint_bytes); ++ if (size > 0) { ++ return UVarint{varint_bytes, size}; ++ } ++ return {}; ++} ++ ++uint64_t UVarint::toUInt64() const { ++ uint64_t res = 0; ++ size_t index = 0; ++ for (const auto& byte : bytes_) { ++ res += static_cast((byte & ipfs::Byte{0x7f})) << index; ++ index += 7; ++ } ++ return res; ++} ++ ++ipfs::ByteView UVarint::toBytes() const { ++ return ipfs::ByteView{bytes_.data(), bytes_.size()}; ++} ++ ++std::vector const& UVarint::toVector() const { ++ return bytes_; ++} ++ ++std::string UVarint::toHex() const { ++ return hex_upper(bytes_); ++} ++ ++size_t UVarint::size() const { ++ return bytes_.size(); ++} ++ ++UVarint& UVarint::operator=(UVarint const& rhs) { ++ bytes_ = rhs.bytes_; // actually OK even if &rhs == this ++ return *this; ++} ++UVarint& UVarint::operator=(uint64_t n) { ++ *this = UVarint(n); ++ return *this; ++} ++ ++bool UVarint::operator==(const UVarint& r) const { ++ return std::equal(bytes_.begin(), bytes_.end(), r.bytes_.begin(), ++ r.bytes_.end()); ++} ++ ++bool UVarint::operator!=(const UVarint& r) const { ++ return !(*this == r); ++} ++ ++bool UVarint::operator<(const UVarint& r) const { ++ return toUInt64() < r.toUInt64(); ++} ++ ++size_t UVarint::calculateSize(ipfs::ByteView varint_bytes) { ++ size_t size = 0; ++ size_t shift = 0; ++ constexpr size_t capacity = sizeof(uint64_t) * 8; ++ bool last_byte_found = false; ++ for (const auto& byte : varint_bytes) { ++ ++size; ++ std::uint_least64_t slice = to_integer(byte) & 0x7f; ++ if (shift >= capacity || ((slice << shift) >> shift) != slice) { ++ size = 0; ++ break; ++ } ++ if ((byte & ipfs::Byte{0x80}) == ipfs::Byte{0}) { ++ last_byte_found = true; ++ break; ++ } ++ shift += 7; ++ } ++ return last_byte_found ? size : 0; ++} ++ ++UVarint::~UVarint() noexcept {} ++ ++} // namespace libp2p::multi +diff --git a/third_party/ipfs_client/src/libp2p/peer/peer_id.cc b/third_party/ipfs_client/src/libp2p/peer/peer_id.cc +new file mode 100644 +index 0000000000000..4fd03bf7caedd +--- /dev/null ++++ b/third_party/ipfs_client/src/libp2p/peer/peer_id.cc +@@ -0,0 +1,140 @@ ++/** ++ * Copyright Soramitsu Co., Ltd. All Rights Reserved. ++ * SPDX-License-Identifier: Apache-2.0 ++ */ ++ ++#include ++ ++// #include ++#include ++#include ++#include ++#include ++ ++/* ++OUTCOME_CPP_DEFINE_CATEGORY(libp2p::peer, PeerId::FactoryError, e) { ++ using E = libp2p::peer::PeerId::FactoryError; ++ switch (e) { ++ case E::SUCCESS: ++ return "success"; ++ case E::SHA256_EXPECTED: ++ return "expected a sha-256 multihash"; ++ } ++ return "unknown error"; ++} ++*/ ++namespace libp2p::peer { ++using common::ByteArray; ++using multi::Multihash; ++using multi::detail::decodeBase58; ++using multi::detail::encodeBase58; ++ ++PeerId::PeerId(multi::Multihash hash) : hash_{std::move(hash)} {} ++ ++PeerId::FactoryResult PeerId::fromPublicKey(const crypto::ProtobufKey& key) { ++ std::vector hash; ++ ++ auto algo = multi::HashType::sha256; ++ if (key.key.size() <= kMaxInlineKeyLength) { ++ algo = multi::HashType::identity; ++ hash = key.key; ++ } else { ++ // OUTCOME_TRY(shash, crypto::sha256(key.key)); ++ auto shash = crypto::sha256(key.key); ++ if (shash.has_value()) { ++ hash.assign(shash.value().begin(), shash.value().end()); ++ } else { ++ return ipfs::unexpected{shash.error()}; ++ } ++ } ++ ++ // OUTCOME_TRY(multihash, Multihash::create(algo, hash)); ++ auto multihash = Multihash::create(algo, hash); ++ if (multihash.has_value()) { ++ return PeerId{std::move(multihash.value())}; ++ } else { ++ return ipfs::unexpected(multihash.error()); ++ } ++} ++ ++PeerId::FactoryResult PeerId::fromBase58(std::string_view id) { ++ // OUTCOME_TRY(decoded_id, decodeBase58(id)); ++ auto decoded_id = decodeBase58(id); ++ if (!decoded_id.has_value()) { ++ return ipfs::unexpected{decoded_id.error()}; ++ } ++ // OUTCOME_TRY(hash, Multihash::createFromBytes(decoded_id)); ++ auto hash_res = Multihash::createFromBytes(decoded_id.value()); ++ if (!hash_res.has_value()) { ++ return ipfs::unexpected{hash_res.error()}; ++ } ++ auto& hash = hash_res.value(); ++ if (hash.getType() != multi::HashType::sha256 && ++ hash.toBuffer().size() > kMaxInlineKeyLength) { ++ return ipfs::unexpected{FactoryError::SHA256_EXPECTED}; ++ } ++ ++ return PeerId{std::move(hash)}; ++} ++ ++auto PeerId::FromMultibase(std::string_view id) -> FactoryResult { ++ multi::MultibaseCodecImpl c; ++ auto res = c.decode(id); ++ if (!res.has_value()) { ++ return ipfs::unexpected{res.error()}; ++ } ++ return fromBytes(res.value()); ++} ++ ++PeerId::FactoryResult PeerId::fromHash(const Multihash& hash) { ++ if (hash.getType() != multi::HashType::sha256 && ++ hash.toBuffer().size() > kMaxInlineKeyLength) { ++ return ipfs::unexpected{FactoryError::SHA256_EXPECTED}; ++ } ++ ++ return PeerId{hash}; ++} ++ ++bool PeerId::operator<(const PeerId& other) const { ++ return this->hash_ < other.hash_; ++} ++ ++bool PeerId::operator==(const PeerId& other) const { ++ return this->hash_ == other.hash_; ++} ++ ++bool PeerId::operator!=(const PeerId& other) const { ++ return !(*this == other); ++} ++ ++std::string PeerId::toBase58() const { ++ return encodeBase58(hash_.toBuffer()); ++} ++ ++const std::vector& PeerId::toVector() const { ++ return hash_.toBuffer(); ++} ++ ++std::string PeerId::toHex() const { ++ return hash_.toHex(); ++} ++ ++const multi::Multihash& PeerId::toMultihash() const { ++ return hash_; ++} ++ ++PeerId::FactoryResult PeerId::fromBytes(ipfs::ByteView v) { ++ // OUTCOME_TRY(mh, Multihash::createFromBytes(v)); ++ auto mh = Multihash::createFromBytes(v); ++ if (mh.has_value()) { ++ return fromHash(mh.value()); ++ } else { ++ return ipfs::unexpected{mh.error()}; ++ } ++} ++} // namespace libp2p::peer ++ ++size_t std::hash::operator()( ++ const libp2p::peer::PeerId& peer_id) const { ++ return std::hash()(peer_id.toMultihash()); ++} +diff --git a/third_party/ipfs_client/src/log_macros.h b/third_party/ipfs_client/src/log_macros.h +new file mode 100644 +index 0000000000000..9a2f2b9b9e180 +--- /dev/null ++++ b/third_party/ipfs_client/src/log_macros.h +@@ -0,0 +1,39 @@ ++#ifndef IPFS_LOG_MACROS_H_ ++#define IPFS_LOG_MACROS_H_ ++ ++ ++#if __has_include("base/logging.h") //In Chromium ++ ++#include "base/logging.h" ++#include "base/check_op.h" ++ ++#else // Not in Chromium ++ ++#include ++ ++#include ++ ++#define DCHECK_EQ GOOGLE_DCHECK_EQ ++#define DCHECK_GT GOOGLE_DCHECK_GT ++#define DCHECK GOOGLE_DCHECK ++#define LOG GOOGLE_LOG ++// TODO ++#define VLOG(X) \ ++ ::google::protobuf::internal::LogFinisher() = \ ++ ::google::protobuf::internal::LogMessage( \ ++ static_cast<::google::protobuf::LogLevel>( \ ++ ::google::protobuf::LOGLEVEL_INFO - X), \ ++ __FILE__, __LINE__) ++ ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wunused-variable" ++namespace { ++static bool is_logging_initialized = ::ipfs::log::IsInitialized(); ++} ++#pragma GCC diagnostic pop ++ ++#endif //Chromium in-tree check ++ ++#define L_VAR(X) LOG(INFO) << "VAR " << #X << "='" << (X) << '\''; ++ ++#endif // IPFS_LOG_MACROS_H_ +diff --git a/third_party/ipfs_client/src/smhasher/MurmurHash3.cc b/third_party/ipfs_client/src/smhasher/MurmurHash3.cc +new file mode 100644 +index 0000000000000..677aedf1d7a55 +--- /dev/null ++++ b/third_party/ipfs_client/src/smhasher/MurmurHash3.cc +@@ -0,0 +1,424 @@ ++//----------------------------------------------------------------------------- ++// MurmurHash3 was written by Austin Appleby, and is placed in the public ++// domain. The author hereby disclaims copyright to this source code. ++ ++// Note - The x86 and x64 versions do _not_ produce the same results, as the ++// algorithms are optimized for their respective platforms. You can still ++// compile and run any of them on any platform, but your performance with the ++// non-native version will be less than optimal. ++ ++#include "smhasher/MurmurHash3.h" ++#ifdef __GNUG__ ++#pragma GCC diagnostic ignored "-Wimplicit-fallthrough" ++#endif ++ ++#ifdef __clang__ ++#pragma clang diagnostic ignored "-Wimplicit-fallthrough" ++#endif ++//----------------------------------------------------------------------------- ++// Platform-specific functions and macros ++ ++// Microsoft Visual Studio ++ ++#if defined(_MSC_VER) ++ ++#define FORCE_INLINE __forceinline ++ ++#include ++ ++#define ROTL32(x, y) _rotl(x, y) ++#define ROTL64(x, y) _rotl64(x, y) ++ ++#define BIG_CONSTANT(x) (x) ++ ++// Other compilers ++ ++#else // defined(_MSC_VER) ++ ++#define FORCE_INLINE inline __attribute__((always_inline)) ++ ++inline uint32_t rotl32(uint32_t x, int8_t r) { ++ return (x << r) | (x >> (32 - r)); ++} ++ ++inline uint64_t rotl64(uint64_t x, int8_t r) { ++ return (x << r) | (x >> (64 - r)); ++} ++ ++#define ROTL32(x, y) rotl32(x, y) ++#define ROTL64(x, y) rotl64(x, y) ++ ++#define BIG_CONSTANT(x) (x##LLU) ++ ++#endif // !defined(_MSC_VER) ++ ++//----------------------------------------------------------------------------- ++// Block read - if your platform needs to do endian-swapping or can only ++// handle aligned reads, do the conversion here ++ ++FORCE_INLINE uint32_t getblock32(const uint32_t* p, int i) { ++ return p[i]; ++} ++ ++FORCE_INLINE uint64_t getblock64(const uint64_t* p, int i) { ++ return p[i]; ++} ++ ++//----------------------------------------------------------------------------- ++// Finalization mix - force all bits of a hash block to avalanche ++ ++FORCE_INLINE uint32_t fmix32(uint32_t h) { ++ h ^= h >> 16; ++ h *= 0x85ebca6b; ++ h ^= h >> 13; ++ h *= 0xc2b2ae35; ++ h ^= h >> 16; ++ ++ return h; ++} ++ ++//---------- ++ ++FORCE_INLINE uint64_t fmix64(uint64_t k) { ++ k ^= k >> 33; ++ k *= BIG_CONSTANT(0xff51afd7ed558ccd); ++ k ^= k >> 33; ++ k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); ++ k ^= k >> 33; ++ ++ return k; ++} ++ ++//----------------------------------------------------------------------------- ++ ++void MurmurHash3_x86_32(const void* key, int len, uint32_t seed, void* out) { ++ const uint8_t* data = (const uint8_t*)key; ++ const int nblocks = len / 4; ++ ++ uint32_t h1 = seed; ++ ++ const uint32_t c1 = 0xcc9e2d51; ++ const uint32_t c2 = 0x1b873593; ++ ++ //---------- ++ // body ++ ++ const uint32_t* blocks = (const uint32_t*)(data + nblocks * 4); ++ ++ for (int i = -nblocks; i; i++) { ++ uint32_t k1 = getblock32(blocks, i); ++ ++ k1 *= c1; ++ k1 = ROTL32(k1, 15); ++ k1 *= c2; ++ ++ h1 ^= k1; ++ h1 = ROTL32(h1, 13); ++ h1 = h1 * 5 + 0xe6546b64; ++ } ++ ++ //---------- ++ // tail ++ ++ const uint8_t* tail = (const uint8_t*)(data + nblocks * 4); ++ ++ uint32_t k1 = 0; ++ ++ switch (len & 3) { ++ case 3: ++ k1 ^= tail[2] << 16; ++ case 2: ++ k1 ^= tail[1] << 8; ++ case 1: ++ k1 ^= tail[0]; ++ k1 *= c1; ++ k1 = ROTL32(k1, 15); ++ k1 *= c2; ++ h1 ^= k1; ++ }; ++ ++ //---------- ++ // finalization ++ ++ h1 ^= len; ++ ++ h1 = fmix32(h1); ++ ++ *(uint32_t*)out = h1; ++} ++ ++//----------------------------------------------------------------------------- ++ ++void MurmurHash3_x86_128(const void* key, ++ const int len, ++ uint32_t seed, ++ void* out) { ++ const uint8_t* data = (const uint8_t*)key; ++ const int nblocks = len / 16; ++ ++ uint32_t h1 = seed; ++ uint32_t h2 = seed; ++ uint32_t h3 = seed; ++ uint32_t h4 = seed; ++ ++ const uint32_t c1 = 0x239b961b; ++ const uint32_t c2 = 0xab0e9789; ++ const uint32_t c3 = 0x38b34ae5; ++ const uint32_t c4 = 0xa1e38b93; ++ ++ //---------- ++ // body ++ ++ const uint32_t* blocks = (const uint32_t*)(data + nblocks * 16); ++ ++ for (int i = -nblocks; i; i++) { ++ uint32_t k1 = getblock32(blocks, i * 4 + 0); ++ uint32_t k2 = getblock32(blocks, i * 4 + 1); ++ uint32_t k3 = getblock32(blocks, i * 4 + 2); ++ uint32_t k4 = getblock32(blocks, i * 4 + 3); ++ ++ k1 *= c1; ++ k1 = ROTL32(k1, 15); ++ k1 *= c2; ++ h1 ^= k1; ++ ++ h1 = ROTL32(h1, 19); ++ h1 += h2; ++ h1 = h1 * 5 + 0x561ccd1b; ++ ++ k2 *= c2; ++ k2 = ROTL32(k2, 16); ++ k2 *= c3; ++ h2 ^= k2; ++ ++ h2 = ROTL32(h2, 17); ++ h2 += h3; ++ h2 = h2 * 5 + 0x0bcaa747; ++ ++ k3 *= c3; ++ k3 = ROTL32(k3, 17); ++ k3 *= c4; ++ h3 ^= k3; ++ ++ h3 = ROTL32(h3, 15); ++ h3 += h4; ++ h3 = h3 * 5 + 0x96cd1c35; ++ ++ k4 *= c4; ++ k4 = ROTL32(k4, 18); ++ k4 *= c1; ++ h4 ^= k4; ++ ++ h4 = ROTL32(h4, 13); ++ h4 += h1; ++ h4 = h4 * 5 + 0x32ac3b17; ++ } ++ ++ //---------- ++ // tail ++ ++ const uint8_t* tail = (const uint8_t*)(data + nblocks * 16); ++ ++ uint32_t k1 = 0; ++ uint32_t k2 = 0; ++ uint32_t k3 = 0; ++ uint32_t k4 = 0; ++ ++ switch (len & 15) { ++ case 15: ++ k4 ^= tail[14] << 16; ++ case 14: ++ k4 ^= tail[13] << 8; ++ case 13: ++ k4 ^= tail[12] << 0; ++ k4 *= c4; ++ k4 = ROTL32(k4, 18); ++ k4 *= c1; ++ h4 ^= k4; ++ ++ case 12: ++ k3 ^= tail[11] << 24; ++ case 11: ++ k3 ^= tail[10] << 16; ++ case 10: ++ k3 ^= tail[9] << 8; ++ case 9: ++ k3 ^= tail[8] << 0; ++ k3 *= c3; ++ k3 = ROTL32(k3, 17); ++ k3 *= c4; ++ h3 ^= k3; ++ ++ case 8: ++ k2 ^= tail[7] << 24; ++ case 7: ++ k2 ^= tail[6] << 16; ++ case 6: ++ k2 ^= tail[5] << 8; ++ case 5: ++ k2 ^= tail[4] << 0; ++ k2 *= c2; ++ k2 = ROTL32(k2, 16); ++ k2 *= c3; ++ h2 ^= k2; ++ ++ case 4: ++ k1 ^= tail[3] << 24; ++ case 3: ++ k1 ^= tail[2] << 16; ++ case 2: ++ k1 ^= tail[1] << 8; ++ case 1: ++ k1 ^= tail[0] << 0; ++ k1 *= c1; ++ k1 = ROTL32(k1, 15); ++ k1 *= c2; ++ h1 ^= k1; ++ }; ++ ++ //---------- ++ // finalization ++ ++ h1 ^= len; ++ h2 ^= len; ++ h3 ^= len; ++ h4 ^= len; ++ ++ h1 += h2; ++ h1 += h3; ++ h1 += h4; ++ h2 += h1; ++ h3 += h1; ++ h4 += h1; ++ ++ h1 = fmix32(h1); ++ h2 = fmix32(h2); ++ h3 = fmix32(h3); ++ h4 = fmix32(h4); ++ ++ h1 += h2; ++ h1 += h3; ++ h1 += h4; ++ h2 += h1; ++ h3 += h1; ++ h4 += h1; ++ ++ ((uint32_t*)out)[0] = h1; ++ ((uint32_t*)out)[1] = h2; ++ ((uint32_t*)out)[2] = h3; ++ ((uint32_t*)out)[3] = h4; ++} ++ ++//----------------------------------------------------------------------------- ++ ++void MurmurHash3_x64_128(const void* key, ++ const int len, ++ const uint32_t seed, ++ void* out) { ++ const uint8_t* data = (const uint8_t*)key; ++ const int nblocks = len / 16; ++ ++ uint64_t h1 = seed; ++ uint64_t h2 = seed; ++ ++ const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5); ++ const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f); ++ ++ //---------- ++ // body ++ ++ const uint64_t* blocks = (const uint64_t*)(data); ++ ++ for (int i = 0; i < nblocks; i++) { ++ uint64_t k1 = getblock64(blocks, i * 2 + 0); ++ uint64_t k2 = getblock64(blocks, i * 2 + 1); ++ ++ k1 *= c1; ++ k1 = ROTL64(k1, 31); ++ k1 *= c2; ++ h1 ^= k1; ++ ++ h1 = ROTL64(h1, 27); ++ h1 += h2; ++ h1 = h1 * 5 + 0x52dce729; ++ ++ k2 *= c2; ++ k2 = ROTL64(k2, 33); ++ k2 *= c1; ++ h2 ^= k2; ++ ++ h2 = ROTL64(h2, 31); ++ h2 += h1; ++ h2 = h2 * 5 + 0x38495ab5; ++ } ++ ++ //---------- ++ // tail ++ ++ const uint8_t* tail = (const uint8_t*)(data + nblocks * 16); ++ ++ uint64_t k1 = 0; ++ uint64_t k2 = 0; ++ ++ switch (len & 15) { ++ case 15: ++ k2 ^= ((uint64_t)tail[14]) << 48; ++ case 14: ++ k2 ^= ((uint64_t)tail[13]) << 40; ++ case 13: ++ k2 ^= ((uint64_t)tail[12]) << 32; ++ case 12: ++ k2 ^= ((uint64_t)tail[11]) << 24; ++ case 11: ++ k2 ^= ((uint64_t)tail[10]) << 16; ++ case 10: ++ k2 ^= ((uint64_t)tail[9]) << 8; ++ case 9: ++ k2 ^= ((uint64_t)tail[8]) << 0; ++ k2 *= c2; ++ k2 = ROTL64(k2, 33); ++ k2 *= c1; ++ h2 ^= k2; ++ ++ case 8: ++ k1 ^= ((uint64_t)tail[7]) << 56; ++ case 7: ++ k1 ^= ((uint64_t)tail[6]) << 48; ++ case 6: ++ k1 ^= ((uint64_t)tail[5]) << 40; ++ case 5: ++ k1 ^= ((uint64_t)tail[4]) << 32; ++ case 4: ++ k1 ^= ((uint64_t)tail[3]) << 24; ++ case 3: ++ k1 ^= ((uint64_t)tail[2]) << 16; ++ case 2: ++ k1 ^= ((uint64_t)tail[1]) << 8; ++ case 1: ++ k1 ^= ((uint64_t)tail[0]) << 0; ++ k1 *= c1; ++ k1 = ROTL64(k1, 31); ++ k1 *= c2; ++ h1 ^= k1; ++ }; ++ ++ //---------- ++ // finalization ++ ++ h1 ^= len; ++ h2 ^= len; ++ ++ h1 += h2; ++ h2 += h1; ++ ++ h1 = fmix64(h1); ++ h2 = fmix64(h2); ++ ++ h1 += h2; ++ h2 += h1; ++ ++ ((uint64_t*)out)[0] = h1; ++ ((uint64_t*)out)[1] = h2; ++} ++ ++//----------------------------------------------------------------------------- +diff --git a/third_party/ipfs_client/src/vocab/byte_view.cc b/third_party/ipfs_client/src/vocab/byte_view.cc +new file mode 100644 +index 0000000000000..66d35686d56ef +--- /dev/null ++++ b/third_party/ipfs_client/src/vocab/byte_view.cc +@@ -0,0 +1 @@ ++#include "vocab/byte_view.h" +diff --git a/third_party/ipfs_client/src/vocab/slash_delimited.cc b/third_party/ipfs_client/src/vocab/slash_delimited.cc +new file mode 100644 +index 0000000000000..0ef33662a6954 +--- /dev/null ++++ b/third_party/ipfs_client/src/vocab/slash_delimited.cc +@@ -0,0 +1,64 @@ ++#include ++ ++using Self = ipfs::SlashDelimited; ++ ++Self::SlashDelimited(std::string_view unowned) : remainder_{unowned} {} ++ ++Self::operator bool() const { ++ return remainder_.find_first_not_of("/") < remainder_.size(); ++} ++std::string_view Self::pop() { ++ if (remainder_.empty()) { ++ return remainder_; ++ } ++ auto slash = remainder_.find('/'); ++ if (slash == std::string_view::npos) { ++ return pop_all(); ++ } ++ auto result = remainder_.substr(0UL, slash); ++ remainder_.remove_prefix(slash + 1); ++ if (result.empty()) { ++ return pop(); ++ } else { ++ return result; ++ } ++} ++std::string_view Self::pop_all() { ++ auto result = remainder_; ++ remainder_ = ""; ++ return result; ++} ++std::string_view Self::pop_n(std::size_t n) { ++ std::size_t a = 0UL; ++ while (n) { ++ auto slash = remainder_.find('/', a); ++ auto non_slash = remainder_.find_first_not_of("/", a); ++ if (slash == std::string_view::npos) { ++ auto result = remainder_; ++ remainder_ = ""; ++ return result; ++ } ++ if (non_slash < slash) { ++ --n; ++ } ++ a = slash + 1UL; ++ } ++ auto result = remainder_.substr(0UL, a - 1); ++ remainder_.remove_prefix(a); ++ return result; ++} ++std::string_view Self::peek_back() const { ++ auto s = remainder_; ++ while (!s.empty() && s.back() == '/') { ++ s.remove_suffix(1); ++ } ++ if (s.empty()) { ++ return s; ++ } ++ auto last_slash = s.find_last_of('/'); ++ if (last_slash < remainder_.size()) { ++ return remainder_.substr(last_slash + 1); ++ } else { ++ return s; ++ } ++} +diff --git a/third_party/ipfs_client/unix_fs.proto b/third_party/ipfs_client/unix_fs.proto +new file mode 100644 +index 0000000000000..9d117a4d66bdf +--- /dev/null ++++ b/third_party/ipfs_client/unix_fs.proto +@@ -0,0 +1,32 @@ ++syntax = "proto2"; ++option optimize_for = LITE_RUNTIME; ++package ipfs.unix_fs; ++ ++message Data { ++ enum DataType { ++ Raw = 0; ++ Directory = 1; ++ File = 2; ++ Metadata = 3; ++ Symlink = 4; ++ HAMTShard = 5; ++ } ++ ++ required DataType Type = 1; ++ optional bytes Data = 2; ++ optional uint64 filesize = 3; ++ repeated uint64 blocksizes = 4; ++ optional uint64 hashType = 5; ++ optional uint64 fanout = 6; ++ optional uint32 mode = 7; ++ optional UnixTime mtime = 8; ++} ++ ++message Metadata { ++ optional string MimeType = 1; ++} ++ ++message UnixTime { ++ required int64 Seconds = 1; ++ optional fixed32 FractionalNanoseconds = 2; ++} diff --git a/url/BUILD.gn b/url/BUILD.gn index c525c166979d6..ce2b1ae43c0a7 100644 --- a/url/BUILD.gn diff --git a/component/patches/120.0.6099.0.patch b/component/patches/121.0.6103.3.patch similarity index 92% rename from component/patches/120.0.6099.0.patch rename to component/patches/121.0.6103.3.patch index 6f7b560c..398887d3 100644 --- a/component/patches/120.0.6099.0.patch +++ b/component/patches/121.0.6103.3.patch @@ -1,5 +1,5 @@ diff --git a/chrome/browser/BUILD.gn b/chrome/browser/BUILD.gn -index bce36d7ffb408..6899bd840ebbf 100644 +index 9feb694b4e410..fe16faf7a9fa3 100644 --- a/chrome/browser/BUILD.gn +++ b/chrome/browser/BUILD.gn @@ -40,6 +40,7 @@ import("//rlz/buildflags/buildflags.gni") @@ -10,7 +10,7 @@ index bce36d7ffb408..6899bd840ebbf 100644 import("//third_party/protobuf/proto_library.gni") import("//third_party/webrtc/webrtc.gni") import("//third_party/widevine/cdm/widevine.gni") -@@ -2658,6 +2659,10 @@ static_library("browser") { +@@ -2657,6 +2658,10 @@ static_library("browser") { ] } @@ -22,7 +22,7 @@ index bce36d7ffb408..6899bd840ebbf 100644 deps += [ "//chrome/browser/screen_ai:screen_ai_dlc_installer" ] } diff --git a/chrome/browser/about_flags.cc b/chrome/browser/about_flags.cc -index 74e37236658a8..c2795d9e58618 100644 +index aba92a8ce8422..e5b409742ae49 100644 --- a/chrome/browser/about_flags.cc +++ b/chrome/browser/about_flags.cc @@ -213,6 +213,7 @@ @@ -44,7 +44,7 @@ index 74e37236658a8..c2795d9e58618 100644 #if BUILDFLAG(ENABLE_PDF) #include "pdf/pdf_features.h" #endif -@@ -9828,6 +9833,14 @@ const FeatureEntry kFeatureEntries[] = { +@@ -9787,6 +9792,14 @@ const FeatureEntry kFeatureEntries[] = { flag_descriptions::kOmitCorsClientCertDescription, kOsAll, FEATURE_VALUE_TYPE(network::features::kOmitCorsClientCert)}, @@ -110,10 +110,10 @@ index 4c88614c68c25..f8bb12a3b0c2e 100644 // Also check for schemes registered via registerProtocolHandler(), which diff --git a/chrome/browser/chrome_content_browser_client.cc b/chrome/browser/chrome_content_browser_client.cc -index c4d9748b5cc7c..b74e318f8495f 100644 +index cac5aa93883dc..c8beecf246eb2 100644 --- a/chrome/browser/chrome_content_browser_client.cc +++ b/chrome/browser/chrome_content_browser_client.cc -@@ -228,6 +228,8 @@ +@@ -229,6 +229,8 @@ #include "components/error_page/common/localized_error.h" #include "components/error_page/content/browser/net_error_auto_reloader.h" #include "components/google/core/common/google_switches.h" @@ -122,7 +122,7 @@ index c4d9748b5cc7c..b74e318f8495f 100644 #include "components/keep_alive_registry/keep_alive_types.h" #include "components/keep_alive_registry/scoped_keep_alive.h" #include "components/language/core/browser/pref_names.h" -@@ -363,6 +365,7 @@ +@@ -366,6 +368,7 @@ #include "third_party/blink/public/common/switches.h" #include "third_party/blink/public/mojom/browsing_topics/browsing_topics.mojom.h" #include "third_party/blink/public/public_buildflags.h" @@ -130,7 +130,7 @@ index c4d9748b5cc7c..b74e318f8495f 100644 #include "third_party/widevine/cdm/buildflags.h" #include "ui/base/clipboard/clipboard_format_type.h" #include "ui/base/l10n/l10n_util.h" -@@ -486,6 +489,12 @@ +@@ -489,6 +492,12 @@ #include "chrome/browser/fuchsia/chrome_browser_main_parts_fuchsia.h" #endif @@ -143,7 +143,7 @@ index c4d9748b5cc7c..b74e318f8495f 100644 #if BUILDFLAG(IS_CHROMEOS) #include "base/debug/leak_annotations.h" #include "chrome/browser/apps/intent_helper/chromeos_disabled_apps_throttle.h" -@@ -6143,12 +6152,23 @@ void ChromeContentBrowserClient:: +@@ -6170,12 +6179,23 @@ void ChromeContentBrowserClient:: const absl::optional& request_initiator_origin, NonNetworkURLLoaderFactoryMap* factories) { #if BUILDFLAG(IS_CHROMEOS_ASH) || BUILDFLAG(ENABLE_EXTENSIONS) || \ @@ -169,7 +169,7 @@ index c4d9748b5cc7c..b74e318f8495f 100644 #if BUILDFLAG(IS_CHROMEOS_ASH) if (web_contents) { -@@ -6290,6 +6310,11 @@ ChromeContentBrowserClient::WillCreateURLLoaderRequestInterceptors( +@@ -6317,6 +6337,11 @@ ChromeContentBrowserClient::WillCreateURLLoaderRequestInterceptors( scoped_refptr navigation_response_task_runner) { std::vector> interceptors; @@ -182,7 +182,7 @@ index c4d9748b5cc7c..b74e318f8495f 100644 interceptors.push_back( std::make_unique( diff --git a/chrome/browser/flag_descriptions.cc b/chrome/browser/flag_descriptions.cc -index eb7c66097dc8c..4c7694d41a643 100644 +index cc1f3549227e9..a4ab4f1e09ea2 100644 --- a/chrome/browser/flag_descriptions.cc +++ b/chrome/browser/flag_descriptions.cc @@ -248,6 +248,11 @@ const char kEnableBenchmarkingDescription[] = @@ -198,7 +198,7 @@ index eb7c66097dc8c..4c7694d41a643 100644 "Preloading Settings on Performance Page"; const char kPreloadingOnPerformancePageDescription[] = diff --git a/chrome/browser/flag_descriptions.h b/chrome/browser/flag_descriptions.h -index ba3965c94f820..65841ee890daa 100644 +index 5dd5c631c33e2..e8b0deb43e13f 100644 --- a/chrome/browser/flag_descriptions.h +++ b/chrome/browser/flag_descriptions.h @@ -22,6 +22,7 @@ @@ -269,6 +269,19 @@ index 5273da5190277..12b28b86a4c00 100644 case NsswitchReader::Service::kNis: RecordIncompatibleNsswitchReason( IncompatibleNsswitchReason::kIncompatibleService, +diff --git a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +index 4eadf46ea0c24..d62fc7fb14e01 100644 +--- a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc ++++ b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +@@ -67,7 +67,7 @@ class URLSchemesRegistry final { + // is considered secure. Additional checks are performed to ensure that + // other http pages are filtered out. + service_worker_schemes({"http", "https"}), +- fetch_api_schemes({"http", "https"}), ++ fetch_api_schemes({"http", "https", "ipfs", "ipns"}), + allowed_in_referrer_schemes({"http", "https"}) { + for (auto& scheme : url::GetCorsEnabledSchemes()) + cors_enabled_schemes.insert(scheme.c_str()); diff --git a/url/BUILD.gn b/url/BUILD.gn index c525c166979d6..ce2b1ae43c0a7 100644 --- a/url/BUILD.gn diff --git a/component/patches/120.0.6090.0.patch b/component/patches/121.0.6104.0.patch similarity index 92% rename from component/patches/120.0.6090.0.patch rename to component/patches/121.0.6104.0.patch index 90330f0b..ec1159f4 100644 --- a/component/patches/120.0.6090.0.patch +++ b/component/patches/121.0.6104.0.patch @@ -1,5 +1,5 @@ diff --git a/chrome/browser/BUILD.gn b/chrome/browser/BUILD.gn -index bdf2ec39d2c7d..63967792609cb 100644 +index 835e8aa2c4e82..d3e1580791fc2 100644 --- a/chrome/browser/BUILD.gn +++ b/chrome/browser/BUILD.gn @@ -40,6 +40,7 @@ import("//rlz/buildflags/buildflags.gni") @@ -10,7 +10,7 @@ index bdf2ec39d2c7d..63967792609cb 100644 import("//third_party/protobuf/proto_library.gni") import("//third_party/webrtc/webrtc.gni") import("//third_party/widevine/cdm/widevine.gni") -@@ -2645,6 +2646,10 @@ static_library("browser") { +@@ -2655,6 +2656,10 @@ static_library("browser") { ] } @@ -22,7 +22,7 @@ index bdf2ec39d2c7d..63967792609cb 100644 deps += [ "//chrome/browser/screen_ai:screen_ai_dlc_installer" ] } diff --git a/chrome/browser/about_flags.cc b/chrome/browser/about_flags.cc -index d8fd6c19e69c8..99d94dbfd214d 100644 +index f0eddd66d5a44..dab8461e6f4e2 100644 --- a/chrome/browser/about_flags.cc +++ b/chrome/browser/about_flags.cc @@ -211,6 +211,7 @@ @@ -33,7 +33,7 @@ index d8fd6c19e69c8..99d94dbfd214d 100644 #include "ui/accessibility/accessibility_features.h" #include "ui/accessibility/accessibility_switches.h" #include "ui/base/ui_base_features.h" -@@ -311,6 +312,10 @@ +@@ -312,6 +313,10 @@ #include "extensions/common/switches.h" #endif // BUILDFLAG(ENABLE_EXTENSIONS) @@ -44,7 +44,7 @@ index d8fd6c19e69c8..99d94dbfd214d 100644 #if BUILDFLAG(ENABLE_PDF) #include "pdf/pdf_features.h" #endif -@@ -9752,6 +9757,14 @@ const FeatureEntry kFeatureEntries[] = { +@@ -9801,6 +9806,14 @@ const FeatureEntry kFeatureEntries[] = { flag_descriptions::kOmitCorsClientCertDescription, kOsAll, FEATURE_VALUE_TYPE(network::features::kOmitCorsClientCert)}, @@ -110,10 +110,10 @@ index 4c88614c68c25..f8bb12a3b0c2e 100644 // Also check for schemes registered via registerProtocolHandler(), which diff --git a/chrome/browser/chrome_content_browser_client.cc b/chrome/browser/chrome_content_browser_client.cc -index c1ba6f648e9be..2b613a5a62e5b 100644 +index 582913e2eae7f..7f73f358c2409 100644 --- a/chrome/browser/chrome_content_browser_client.cc +++ b/chrome/browser/chrome_content_browser_client.cc -@@ -228,6 +228,8 @@ +@@ -229,6 +229,8 @@ #include "components/error_page/common/localized_error.h" #include "components/error_page/content/browser/net_error_auto_reloader.h" #include "components/google/core/common/google_switches.h" @@ -122,7 +122,7 @@ index c1ba6f648e9be..2b613a5a62e5b 100644 #include "components/keep_alive_registry/keep_alive_types.h" #include "components/keep_alive_registry/scoped_keep_alive.h" #include "components/language/core/browser/pref_names.h" -@@ -363,6 +365,7 @@ +@@ -366,6 +368,7 @@ #include "third_party/blink/public/common/switches.h" #include "third_party/blink/public/mojom/browsing_topics/browsing_topics.mojom.h" #include "third_party/blink/public/public_buildflags.h" @@ -130,7 +130,7 @@ index c1ba6f648e9be..2b613a5a62e5b 100644 #include "third_party/widevine/cdm/buildflags.h" #include "ui/base/clipboard/clipboard_format_type.h" #include "ui/base/l10n/l10n_util.h" -@@ -486,6 +489,12 @@ +@@ -489,6 +492,12 @@ #include "chrome/browser/fuchsia/chrome_browser_main_parts_fuchsia.h" #endif @@ -143,7 +143,7 @@ index c1ba6f648e9be..2b613a5a62e5b 100644 #if BUILDFLAG(IS_CHROMEOS) #include "base/debug/leak_annotations.h" #include "chrome/browser/apps/intent_helper/chromeos_disabled_apps_throttle.h" -@@ -6150,12 +6159,23 @@ void ChromeContentBrowserClient:: +@@ -6170,12 +6179,23 @@ void ChromeContentBrowserClient:: const absl::optional& request_initiator_origin, NonNetworkURLLoaderFactoryMap* factories) { #if BUILDFLAG(IS_CHROMEOS_ASH) || BUILDFLAG(ENABLE_EXTENSIONS) || \ @@ -169,7 +169,7 @@ index c1ba6f648e9be..2b613a5a62e5b 100644 #if BUILDFLAG(IS_CHROMEOS_ASH) if (web_contents) { -@@ -6297,6 +6317,11 @@ ChromeContentBrowserClient::WillCreateURLLoaderRequestInterceptors( +@@ -6317,6 +6337,11 @@ ChromeContentBrowserClient::WillCreateURLLoaderRequestInterceptors( scoped_refptr navigation_response_task_runner) { std::vector> interceptors; @@ -182,10 +182,10 @@ index c1ba6f648e9be..2b613a5a62e5b 100644 interceptors.push_back( std::make_unique( diff --git a/chrome/browser/flag_descriptions.cc b/chrome/browser/flag_descriptions.cc -index 0f9b20a5d4612..6aa0b7f62c9e2 100644 +index a38737d26cd38..7bdfa5238ca3e 100644 --- a/chrome/browser/flag_descriptions.cc +++ b/chrome/browser/flag_descriptions.cc -@@ -242,6 +242,11 @@ const char kEnableBenchmarkingDescription[] = +@@ -248,6 +248,11 @@ const char kEnableBenchmarkingDescription[] = "after 3 restarts. On the third restart, the flag will appear to be off " "but the effect is still active."; @@ -198,7 +198,7 @@ index 0f9b20a5d4612..6aa0b7f62c9e2 100644 "Preloading Settings on Performance Page"; const char kPreloadingOnPerformancePageDescription[] = diff --git a/chrome/browser/flag_descriptions.h b/chrome/browser/flag_descriptions.h -index 2b88b03775c39..2899d99a1a60c 100644 +index a72e2e5a99f1f..aecc299069c01 100644 --- a/chrome/browser/flag_descriptions.h +++ b/chrome/browser/flag_descriptions.h @@ -22,6 +22,7 @@ @@ -209,7 +209,7 @@ index 2b88b03775c39..2899d99a1a60c 100644 // This file declares strings used in chrome://flags. These messages are not // translated, because instead of end-users they target Chromium developers and -@@ -160,6 +161,11 @@ extern const char kDownloadWarningImprovementsDescription[]; +@@ -165,6 +166,11 @@ extern const char kDownloadWarningImprovementsDescription[]; extern const char kEnableBenchmarkingName[]; extern const char kEnableBenchmarkingDescription[]; @@ -269,6 +269,19 @@ index 5273da5190277..12b28b86a4c00 100644 case NsswitchReader::Service::kNis: RecordIncompatibleNsswitchReason( IncompatibleNsswitchReason::kIncompatibleService, +diff --git a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +index 4eadf46ea0c24..d62fc7fb14e01 100644 +--- a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc ++++ b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +@@ -67,7 +67,7 @@ class URLSchemesRegistry final { + // is considered secure. Additional checks are performed to ensure that + // other http pages are filtered out. + service_worker_schemes({"http", "https"}), +- fetch_api_schemes({"http", "https"}), ++ fetch_api_schemes({"http", "https", "ipfs", "ipns"}), + allowed_in_referrer_schemes({"http", "https"}) { + for (auto& scheme : url::GetCorsEnabledSchemes()) + cors_enabled_schemes.insert(scheme.c_str()); diff --git a/url/BUILD.gn b/url/BUILD.gn index c525c166979d6..ce2b1ae43c0a7 100644 --- a/url/BUILD.gn diff --git a/component/patches/120.0.6078.0.patch b/component/patches/121.0.6110.0.patch similarity index 92% rename from component/patches/120.0.6078.0.patch rename to component/patches/121.0.6110.0.patch index 1eead7d7..0784fa80 100644 --- a/component/patches/120.0.6078.0.patch +++ b/component/patches/121.0.6110.0.patch @@ -1,5 +1,5 @@ diff --git a/chrome/browser/BUILD.gn b/chrome/browser/BUILD.gn -index 466789a1b089c..2996aea6842cb 100644 +index 95c288ea91780..9bb47b8c9a609 100644 --- a/chrome/browser/BUILD.gn +++ b/chrome/browser/BUILD.gn @@ -40,6 +40,7 @@ import("//rlz/buildflags/buildflags.gni") @@ -10,7 +10,7 @@ index 466789a1b089c..2996aea6842cb 100644 import("//third_party/protobuf/proto_library.gni") import("//third_party/webrtc/webrtc.gni") import("//third_party/widevine/cdm/widevine.gni") -@@ -2643,6 +2644,10 @@ static_library("browser") { +@@ -2656,6 +2657,10 @@ static_library("browser") { ] } @@ -22,7 +22,7 @@ index 466789a1b089c..2996aea6842cb 100644 deps += [ "//chrome/browser/screen_ai:screen_ai_dlc_installer" ] } diff --git a/chrome/browser/about_flags.cc b/chrome/browser/about_flags.cc -index 3de290ccddb18..454c00da6fb6c 100644 +index bec81b0940bb5..199e5b0f959ba 100644 --- a/chrome/browser/about_flags.cc +++ b/chrome/browser/about_flags.cc @@ -211,6 +211,7 @@ @@ -33,7 +33,7 @@ index 3de290ccddb18..454c00da6fb6c 100644 #include "ui/accessibility/accessibility_features.h" #include "ui/accessibility/accessibility_switches.h" #include "ui/base/ui_base_features.h" -@@ -310,6 +311,10 @@ +@@ -312,6 +313,10 @@ #include "extensions/common/switches.h" #endif // BUILDFLAG(ENABLE_EXTENSIONS) @@ -44,7 +44,7 @@ index 3de290ccddb18..454c00da6fb6c 100644 #if BUILDFLAG(ENABLE_PDF) #include "pdf/pdf_features.h" #endif -@@ -9626,6 +9631,14 @@ const FeatureEntry kFeatureEntries[] = { +@@ -9793,6 +9798,14 @@ const FeatureEntry kFeatureEntries[] = { flag_descriptions::kOmitCorsClientCertDescription, kOsAll, FEATURE_VALUE_TYPE(network::features::kOmitCorsClientCert)}, @@ -110,10 +110,10 @@ index 4c88614c68c25..f8bb12a3b0c2e 100644 // Also check for schemes registered via registerProtocolHandler(), which diff --git a/chrome/browser/chrome_content_browser_client.cc b/chrome/browser/chrome_content_browser_client.cc -index f3af7751185a0..4318308df095f 100644 +index f3a34ded006fb..8a326c428c39a 100644 --- a/chrome/browser/chrome_content_browser_client.cc +++ b/chrome/browser/chrome_content_browser_client.cc -@@ -228,6 +228,8 @@ +@@ -229,6 +229,8 @@ #include "components/error_page/common/localized_error.h" #include "components/error_page/content/browser/net_error_auto_reloader.h" #include "components/google/core/common/google_switches.h" @@ -122,7 +122,7 @@ index f3af7751185a0..4318308df095f 100644 #include "components/keep_alive_registry/keep_alive_types.h" #include "components/keep_alive_registry/scoped_keep_alive.h" #include "components/language/core/browser/pref_names.h" -@@ -363,6 +365,7 @@ +@@ -367,6 +369,7 @@ #include "third_party/blink/public/common/switches.h" #include "third_party/blink/public/mojom/browsing_topics/browsing_topics.mojom.h" #include "third_party/blink/public/public_buildflags.h" @@ -130,7 +130,7 @@ index f3af7751185a0..4318308df095f 100644 #include "third_party/widevine/cdm/buildflags.h" #include "ui/base/clipboard/clipboard_format_type.h" #include "ui/base/l10n/l10n_util.h" -@@ -485,6 +488,12 @@ +@@ -490,6 +493,12 @@ #include "chrome/browser/fuchsia/chrome_browser_main_parts_fuchsia.h" #endif @@ -143,7 +143,7 @@ index f3af7751185a0..4318308df095f 100644 #if BUILDFLAG(IS_CHROMEOS) #include "base/debug/leak_annotations.h" #include "chrome/browser/apps/intent_helper/chromeos_disabled_apps_throttle.h" -@@ -6175,12 +6184,23 @@ void ChromeContentBrowserClient:: +@@ -6183,12 +6192,23 @@ void ChromeContentBrowserClient:: const absl::optional& request_initiator_origin, NonNetworkURLLoaderFactoryMap* factories) { #if BUILDFLAG(IS_CHROMEOS_ASH) || BUILDFLAG(ENABLE_EXTENSIONS) || \ @@ -169,7 +169,7 @@ index f3af7751185a0..4318308df095f 100644 #if BUILDFLAG(IS_CHROMEOS_ASH) if (web_contents) { -@@ -6322,6 +6342,11 @@ ChromeContentBrowserClient::WillCreateURLLoaderRequestInterceptors( +@@ -6330,6 +6350,11 @@ ChromeContentBrowserClient::WillCreateURLLoaderRequestInterceptors( scoped_refptr navigation_response_task_runner) { std::vector> interceptors; @@ -182,10 +182,10 @@ index f3af7751185a0..4318308df095f 100644 interceptors.push_back( std::make_unique( diff --git a/chrome/browser/flag_descriptions.cc b/chrome/browser/flag_descriptions.cc -index efedc3b756301..9cf289d7d12fa 100644 +index a7e9114f878bf..5518adc2ea6fd 100644 --- a/chrome/browser/flag_descriptions.cc +++ b/chrome/browser/flag_descriptions.cc -@@ -247,6 +247,11 @@ const char kEnableBenchmarkingDescription[] = +@@ -248,6 +248,11 @@ const char kEnableBenchmarkingDescription[] = "after 3 restarts. On the third restart, the flag will appear to be off " "but the effect is still active."; @@ -198,7 +198,7 @@ index efedc3b756301..9cf289d7d12fa 100644 "Preloading Settings on Performance Page"; const char kPreloadingOnPerformancePageDescription[] = diff --git a/chrome/browser/flag_descriptions.h b/chrome/browser/flag_descriptions.h -index f83a9159a7ced..b534ea9006d81 100644 +index 0f2c3caba5ec2..4a0fc4d93dbfd 100644 --- a/chrome/browser/flag_descriptions.h +++ b/chrome/browser/flag_descriptions.h @@ -22,6 +22,7 @@ @@ -209,7 +209,7 @@ index f83a9159a7ced..b534ea9006d81 100644 // This file declares strings used in chrome://flags. These messages are not // translated, because instead of end-users they target Chromium developers and -@@ -163,6 +164,11 @@ extern const char kDownloadWarningImprovementsDescription[]; +@@ -165,6 +166,11 @@ extern const char kDownloadWarningImprovementsDescription[]; extern const char kEnableBenchmarkingName[]; extern const char kEnableBenchmarkingDescription[]; @@ -269,6 +269,19 @@ index 5273da5190277..12b28b86a4c00 100644 case NsswitchReader::Service::kNis: RecordIncompatibleNsswitchReason( IncompatibleNsswitchReason::kIncompatibleService, +diff --git a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +index 4eadf46ea0c24..d62fc7fb14e01 100644 +--- a/third_party/blink/renderer/platform/weborigin/scheme_registry.cc ++++ b/third_party/blink/renderer/platform/weborigin/scheme_registry.cc +@@ -67,7 +67,7 @@ class URLSchemesRegistry final { + // is considered secure. Additional checks are performed to ensure that + // other http pages are filtered out. + service_worker_schemes({"http", "https"}), +- fetch_api_schemes({"http", "https"}), ++ fetch_api_schemes({"http", "https", "ipfs", "ipns"}), + allowed_in_referrer_schemes({"http", "https"}) { + for (auto& scheme : url::GetCorsEnabledSchemes()) + cors_enabled_schemes.insert(scheme.c_str()); diff --git a/url/BUILD.gn b/url/BUILD.gn index c525c166979d6..ce2b1ae43c0a7 100644 --- a/url/BUILD.gn diff --git a/component/url_loader_factory.cc b/component/url_loader_factory.cc index 4010387d..950c5548 100644 --- a/component/url_loader_factory.cc +++ b/component/url_loader_factory.cc @@ -42,17 +42,19 @@ void ipfs::IpfsURLLoaderFactory::CreateLoaderAndStart( VLOG(1) << "IPFS subresource: case=" << scheme_ << " url=" << request.url.spec(); DCHECK(default_factory_); - if (scheme_ == "ipfs") { + if (scheme_ == "ipfs" || scheme_ == "ipns") { auto ptr = std::make_shared( *default_factory_, InterRequestState::FromBrowserContext(context_)); ptr->StartRequest(ptr, request, std::move(loader), std::move(client)); - } else if (scheme_ == "ipns") { - auto ptr = std::make_shared( - InterRequestState::FromBrowserContext(context_), request.url.host(), - network_context_, *default_factory_); - ptr->StartHandling(ptr, request, std::move(loader), std::move(client)); - } else { - NOTREACHED(); - } + } /* else if (scheme_ == "ipns") { + auto ptr = std::make_shared( + InterRequestState::FromBrowserContext(context_), request.url.host(), + network_context_, *default_factory_); + ptr->StartHandling(ptr, request, std::move(loader), std::move(client)); + + } else { + NOTREACHED(); + } + */ } diff --git a/doc/design_notes.md b/doc/design_notes.md index 867beb0a..f9be995a 100644 --- a/doc/design_notes.md +++ b/doc/design_notes.md @@ -80,7 +80,7 @@ It's an unsigned integer in two forms: When issuing a request to another gateway, it checks the gateways in descending-score order: * If the gateway already has a request for that data pending, or has already failed to return successfully, it's skipped. * If the gateways score is less than the number of pending requests currently sent to that gateway, it's skipped as it's considered already-too-busy -* The "need" is generally calculated as the target number of gateways desired to be involved (based on request priority), subtracting the number of gateways currently processing a request for this data. +* The "need" is generally calculated as the target number of gateways desired to be involved (based on request parallel), subtracting the number of gateways currently processing a request for this data. * If the "need" is less than half the count of pending requests on this gateway, it's skipped as it's simply not urgent enough to justify further overloading this gateway. On a side-note, the hard-coded starting points for the scoring effectively encodes known information about those gateways. diff --git a/doc/original_design.md b/doc/original_design.md index c0a2b5df..214a7826 100644 --- a/doc/original_design.md +++ b/doc/original_design.md @@ -90,7 +90,7 @@ graph TD; goodfree --NO--> badfree{"Are there any FREE that have not already TaskFailed this request?"} badfree --NO--> anybusy{"Are there any BUSY gateways?"} anybusy --YES--> wait(("Wait for pending requests to finish (return flow control)")) - badfree --YES--> selbf["Select the one with the fewest TaskFailed requests, initial priority tiebreaks"] + badfree --YES--> selbf["Select the one with the fewest TaskFailed requests, initial parallel tiebreaks"] selbf --> mark_busy["Mark the gateway as BUSY"] goodfree --YES--> selbf mark_busy --> send["Create and send an HTTP URLRequest"] @@ -100,11 +100,11 @@ graph TD; resp[/"URL response"/] --> markfree["Mark the gateway as FREE"] --> success{"status=200 + body?"} style resp fill:#FFE,color:black success --YES--> cancel["Cancel identical requests"] --> markgood["Mark the gateway as GOOD"] - markgood --> incprio["Indicate to Gateways that this gateway's score/priority should be a bit higher"] + markgood --> incprio["Indicate to Gateways that this gateway's score/parallel should be a bit higher"] incprio ----> successed(["Return response (see previous diagram)"]) style successed fill:#9CF,color:black success --NO--> addbad["Add the URL suffix to this gateway's set of failures."] - addbad --> decprio[Indicate to Gateways that this gateway's score/priority should be a bit lower] --> selreq + addbad --> decprio[Indicate to Gateways that this gateway's score/parallel should be a bit lower] --> selreq anybusy --NO---> all_failed(["As all gateways have TaskFailed on a given request, report that failure"]) style all_failed fill:#9CF,color:black ``` diff --git a/library/BUILD.gn.in b/library/BUILD.gn.in index 6e89f3e6..91c67183 100644 --- a/library/BUILD.gn.in +++ b/library/BUILD.gn.in @@ -21,10 +21,14 @@ if (enable_ipfs) { sources = cxx_sources - [ "src/ipfs_client/block_storage.cc", "src/ipfs_client/dag_block.cc", + "src/ipfs_client/gw/gateway_request.cc", + "src/ipfs_client/gw/gateway_http_requestor.cc", + "src/ipfs_client/gw/requestor.cc", "src/ipfs_client/ipld/dag_node.cc", "src/ipfs_client/ipns_names.cc", "src/ipfs_client/ipns_record.cc", "src/ipfs_client/logger.cc", + "src/ipfs_client/signing_key_type.cc", "src/ipfs_client/unix_fs/dir_shard.cc", "src/ipfs_client/unix_fs/multi_node_file.cc", "src/ipfs_client/unix_fs/node_helper.cc", diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index e2118443..404b3d54 100644 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -155,25 +155,28 @@ if(GTest_FOUND) ) with_vocab(unit_test_runner) find_program(VALGRIND "valgrind") + add_custom_target(run_tests + DEPENDS unit_test_runner + COMMAND $ + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + ) if(VALGRIND) - add_custom_target(run_tests + add_custom_command( + OUTPUT ut.cert DEPENDS unit_test_runner - COMMAND "${VALGRIND}" --tool=memcheck --leak-check=yes --exit-on-first-error=yes --error-exitcode=99 "--suppressions=${CMAKE_SOURCE_DIR}/cmake/valgrind.suppressions.txt" $ + COMMAND "${VALGRIND}" --tool=memcheck --leak-check=yes --exit-on-first-error=yes --error-exitcode=99 "--suppressions=${CMAKE_SOURCE_DIR}/cmake/valgrind.suppressions.txt" $ + COMMAND "${CMAKE_COMMAND}" -E touch ut.cert + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" ) else() - add_custom_target(run_tests + add_custom_command( + OUTPUT ut.cert DEPENDS unit_test_runner COMMAND $ + COMMAND "${CMAKE_COMMAND}" -E touch ut.cert WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" ) endif() - add_custom_command( - OUTPUT ut.cert - DEPENDS unit_test_runner - COMMAND $ - COMMAND "${CMAKE_COMMAND}" -E touch ut.cert - WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" - ) if(TEST_BY_DEFAULT) add_custom_target(ut ALL diff --git a/library/conanfile.py b/library/conanfile.py index 8e5af423..cacc6036 100644 --- a/library/conanfile.py +++ b/library/conanfile.py @@ -17,6 +17,7 @@ class IpfsChromium(ConanFile): 'abseil/20230125.3', 'boost/1.81.0', 'gtest/1.13.0', + 'nlohmann_json/3.11.2', 'openssl/1.1.1t', _PB, ] diff --git a/library/include/ipfs_client/context_api.h b/library/include/ipfs_client/context_api.h index 139cf14c..97195165 100644 --- a/library/include/ipfs_client/context_api.h +++ b/library/include/ipfs_client/context_api.h @@ -2,9 +2,15 @@ #define IPFS_CONTEXT_API_H_ #include "busy_gateway.h" +#include "ipns_cbor_entry.h" +#include "signing_key_type.h" +#include + +#include #include #include +#include namespace ipfs { class IpfsRequest; @@ -22,6 +28,14 @@ class GatewayRequest { std::string url() const; ///< Same as gateway.url() }; +struct HttpRequestDescription { + std::string url; + int timeout_seconds; + std::string accept; + std::optional max_response_size; + bool operator==(HttpRequestDescription const&) const; +}; + /** * \brief Interface that provides functionality from whatever * environment you're using this library in. @@ -30,9 +44,19 @@ class GatewayRequest { */ class ContextApi : public std::enable_shared_from_this { public: - ///< Send a single http request to its gateway as scheduled - virtual std::shared_ptr InitiateGatewayRequest( - BusyGateway) = 0; + using HttpRequestDescription = ::ipfs::HttpRequestDescription; + using HeaderAccess = std::function; + using HttpCompleteCallback = + std::function; + virtual void SendHttpRequest(HttpRequestDescription, + HttpCompleteCallback cb) const = 0; + + using DnsTextResultsCallback = + std::function const&)>; + using DnsTextCompleteCallback = std::function; + virtual void SendDnsTextRequest(std::string hostname, + DnsTextResultsCallback, + DnsTextCompleteCallback) = 0; /*! * \brief Determine a mime type for a given file. @@ -52,6 +76,18 @@ class ContextApi : public std::enable_shared_from_this { */ virtual std::string UnescapeUrlComponent(std::string_view url_comp) const = 0; + using IpnsCborEntry = ::ipfs::IpnsCborEntry; + // TODO: accept the Type as a parameter as a reference to a base that knows + // about its members + virtual IpnsCborEntry deserialize_cbor(ByteView) const = 0; + + using SigningKeyType = ::ipfs::SigningKeyType; + using ByteView = ::ipfs::ByteView; + virtual bool verify_key_signature(SigningKeyType, + ByteView signature, + ByteView data, + ByteView key_bytes) const = 0; + /*! * \brief Discover more gateways. * \details The in-chromium implementation of this hits @@ -62,6 +98,10 @@ class ContextApi : public std::enable_shared_from_this { * \note TRAILING SLASH (/) EXPECTED! */ virtual void Discover(std::function)> cb) = 0; + + // TODO drop + virtual std::shared_ptr InitiateGatewayRequest( + BusyGateway) = 0; }; } // namespace ipfs diff --git a/library/include/ipfs_client/gw/block_request_splitter.h b/library/include/ipfs_client/gw/block_request_splitter.h new file mode 100644 index 00000000..0f308a99 --- /dev/null +++ b/library/include/ipfs_client/gw/block_request_splitter.h @@ -0,0 +1,17 @@ +#ifndef IPFS_BLOCK_REQUEST_SPLITTER_H_ +#define IPFS_BLOCK_REQUEST_SPLITTER_H_ + +#include "requestor.h" + +namespace ipfs { +class ContextApi; +} + +namespace ipfs::gw { +class BlockRequestSplitter final : public Requestor { + HandleOutcome handle(RequestPtr) override; + std::string_view name() const override; +}; +} // namespace ipfs::gw + +#endif // IPFS_BLOCK_REQUEST_SPLITTER_H_ diff --git a/library/include/ipfs_client/gw/default_requestor.h b/library/include/ipfs_client/gw/default_requestor.h new file mode 100644 index 00000000..18bce267 --- /dev/null +++ b/library/include/ipfs_client/gw/default_requestor.h @@ -0,0 +1,13 @@ +#ifndef IPFS_DEFAULT_REQUESTOR_LIST_H_ +#define IPFS_DEFAULT_REQUESTOR_LIST_H_ + +#include "requestor.h" + +#include + +namespace ipfs::gw { +std::shared_ptr default_requestor(GatewayList, + std::shared_ptr); +} + +#endif // IPFS_DEFAULT_REQUESTOR_LIST_H_ diff --git a/library/include/ipfs_client/gw/dnslink_requestor.h b/library/include/ipfs_client/gw/dnslink_requestor.h new file mode 100644 index 00000000..a175d880 --- /dev/null +++ b/library/include/ipfs_client/gw/dnslink_requestor.h @@ -0,0 +1,20 @@ +#ifndef IPFS_DNSLINK_REQUESTOR_H_ +#define IPFS_DNSLINK_REQUESTOR_H_ + +#include "requestor.h" + +namespace ipfs { +class ContextApi; +} + +namespace ipfs::gw { +class DnsLinkRequestor final : public Requestor { + HandleOutcome handle(RequestPtr) override; + std::string_view name() const override; + + public: + explicit DnsLinkRequestor(std::shared_ptr); +}; +} // namespace ipfs::gw + +#endif // IPFS_DNSLINK_REQUESTOR_H_ diff --git a/library/include/ipfs_client/gw/gateway_request.h b/library/include/ipfs_client/gw/gateway_request.h index cf727d8c..72d23cc7 100644 --- a/library/include/ipfs_client/gw/gateway_request.h +++ b/library/include/ipfs_client/gw/gateway_request.h @@ -1,10 +1,13 @@ #ifndef IPFS_TRUSTLESS_REQUEST_H_ #define IPFS_TRUSTLESS_REQUEST_H_ +#include + #include #include +#include #include #include #include @@ -15,7 +18,11 @@ class Orchestrator; } // namespace ipfs namespace ipfs::gw { -enum class Type { Block, Car, Ipns, DnsLink, Providers, Identity }; +class Requestor; + +enum class Type { Block, Car, Ipns, DnsLink, Providers, Identity, Zombie }; + +constexpr std::size_t BLOCK_RESPONSE_BUFFER_SIZE = 2 * 1024 * 1024; struct GatewayRequest { Type type; @@ -23,14 +30,25 @@ struct GatewayRequest { std::string path; ///< For CAR requests std::shared_ptr dependent; std::shared_ptr orchestrator; + std::shared_ptr retry_at; std::optional cid; + short parallel = 0; + std::string affinity; std::string url_suffix() const; std::string_view accept() const; std::string_view identity_data() const; + short timeout_seconds() const; + bool is_http() const; + std::optional max_response_size() const; + std::optional describe_http() const; + std::string debug_string() const; + static std::shared_ptr fromIpfsPath(SlashDelimited); }; } // namespace ipfs::gw +std::ostream& operator<<(std::ostream&, ipfs::gw::Type); + #endif // IPFS_TRUSTLESS_REQUEST_H_ diff --git a/library/include/ipfs_client/gw/inline_request_handler.h b/library/include/ipfs_client/gw/inline_request_handler.h new file mode 100644 index 00000000..11e2a156 --- /dev/null +++ b/library/include/ipfs_client/gw/inline_request_handler.h @@ -0,0 +1,13 @@ +#ifndef IPFS_INLINE_REQUEST_HANDLER_H_ +#define IPFS_INLINE_REQUEST_HANDLER_H_ + +#include "requestor.h" + +namespace ipfs::gw { +class InlineRequestHandler final : public Requestor { + HandleOutcome handle(RequestPtr) override; + std::string_view name() const override; +}; +} // namespace ipfs::gw + +#endif // IPFS_INLINE_REQUEST_HANDLER_H_ diff --git a/library/include/ipfs_client/gw/requestor.h b/library/include/ipfs_client/gw/requestor.h new file mode 100644 index 00000000..0ee1834e --- /dev/null +++ b/library/include/ipfs_client/gw/requestor.h @@ -0,0 +1,59 @@ +#ifndef IPFS_REQUESTOR_H_ +#define IPFS_REQUESTOR_H_ + +#include +#include +#include + +namespace ipfs::ipld { +class DagNode; +} +namespace ipfs { +class ContextApi; +struct Response; +} // namespace ipfs + +namespace ipfs::gw { +struct GatewayRequest; +class RequestPool; +using RequestPtr = std::shared_ptr; + +class Requestor : public std::enable_shared_from_this { + protected: + Requestor() {} + + friend class RequestorPool; + enum class HandleOutcome : char { + NOT_HANDLED = 'N', + PENDING = 'P', + DONE = 'D', + PARALLEL = 'L', + MAYBE_LATER = 'M' + }; + virtual HandleOutcome handle(RequestPtr) = 0; + virtual void iterate_nodes( + GatewayRequest const& req, + Response const& res, + std::function)>) const; + + void receive_response(RequestPtr, ipfs::Response const&) const; + void success(RequestPtr, std::string_view body) const; + void failure(RequestPtr) const; + void definitive_failure(RequestPtr) const; + void forward(RequestPtr) const; + + std::shared_ptr api_; + + public: + virtual std::string_view name() const = 0; + + virtual ~Requestor() noexcept {} + void request(std::shared_ptr); + Requestor& or_else(std::shared_ptr p); + + private: + std::shared_ptr next_; +}; +} // namespace ipfs::gw + +#endif // IPFS_REQUESTOR_H_ diff --git a/library/include/ipfs_client/gw/terminating_requestor.h b/library/include/ipfs_client/gw/terminating_requestor.h new file mode 100644 index 00000000..f668c1b3 --- /dev/null +++ b/library/include/ipfs_client/gw/terminating_requestor.h @@ -0,0 +1,13 @@ +#ifndef IPFS_TERMINATING_REQUESTOR_H_ +#define IPFS_TERMINATING_REQUESTOR_H_ + +#include "requestor.h" + +namespace ipfs::gw { +class TerminatingRequestor : public Requestor { + std::string_view name() const override; + HandleOutcome handle(RequestPtr) override; +}; +} // namespace ipfs::gw + +#endif // IPFS_TERMINATING_REQUESTOR_H_ diff --git a/library/include/ipfs_client/ipfs_request.h b/library/include/ipfs_client/ipfs_request.h index 2ca22934..9c0cf8d2 100644 --- a/library/include/ipfs_client/ipfs_request.h +++ b/library/include/ipfs_client/ipfs_request.h @@ -15,11 +15,14 @@ class IpfsRequest { private: std::string path_; Finisher callback_; + std::size_t waiting_ = 0UL; public: IpfsRequest(std::string path, Finisher); SlashDelimited path() const { return std::string_view{path_}; } void finish(Response& r); + void till_next(std::size_t); + bool ready_after(); }; } // namespace ipfs diff --git a/library/include/ipfs_client/ipld/dag_node.h b/library/include/ipfs_client/ipld/dag_node.h index 10b41af3..0bca255a 100644 --- a/library/include/ipfs_client/ipld/dag_node.h +++ b/library/include/ipfs_client/ipld/dag_node.h @@ -19,6 +19,7 @@ namespace ipfs { class Block; +class ContextApi; struct ValidatedIpns; } @@ -40,6 +41,7 @@ using ResolveResult = std::variant; class DagNode : public std::enable_shared_from_this { protected: std::vector> links_; + std::shared_ptr api_; public: using BlockLookup = std::function; @@ -51,6 +53,8 @@ class DagNode : public std::enable_shared_from_this { virtual NodePtr deroot(); virtual DirShard* as_hamt(); // Wish I had access to dynamic_cast + void set_api(std::shared_ptr); + static NodePtr fromBlock(Block const&); static NodePtr fromIpnsRecord(ValidatedIpns const&); }; diff --git a/library/include/ipfs_client/ipns_cbor_entry.h b/library/include/ipfs_client/ipns_cbor_entry.h new file mode 100644 index 00000000..23033979 --- /dev/null +++ b/library/include/ipfs_client/ipns_cbor_entry.h @@ -0,0 +1,21 @@ +#ifndef IPFS_IPNS_CBOR_ENTRY_H_ +#define IPFS_IPNS_CBOR_ENTRY_H_ + +#include +#include + +namespace ipfs { +/*! + * \brief Parsed out data contained in the CBOR data of an IPNS record. + */ +struct IpnsCborEntry { + std::string value; ///< The "value" (target) the name points at + std::string validity; ///< Value to compare for validity (i.e. expiration) + std::uint64_t validityType; ///< Way to deterimine current validity + std::uint64_t sequence; ///< Distinguish other IPNS records for the same name + std::uint64_t ttl; ///< Recommended caching time +}; + +} // namespace ipfs + +#endif // IPFS_IPNS_CBOR_ENTRY_H_ diff --git a/library/include/ipfs_client/ipns_record.h b/library/include/ipfs_client/ipns_record.h index 607c41d6..388a0f96 100644 --- a/library/include/ipfs_client/ipns_record.h +++ b/library/include/ipfs_client/ipns_record.h @@ -1,6 +1,8 @@ #ifndef IPFS_IPNS_RECORD_H_ #define IPFS_IPNS_RECORD_H_ +#include + #include #if __has_include() @@ -15,32 +17,24 @@ namespace libp2p::peer { class PeerId; } +namespace libp2p::multi { +struct ContentIdentifier; +} namespace ipfs { -/*! - * \brief Parsed out data contained in the CBOR data of an IPNS record. - */ -struct IpnsCborEntry { - std::string value; ///< The "value" (target) the name points at - std::string validity; ///< Value to compare for validity (i.e. expiration) - std::uint64_t validityType; ///< Way to deterimine current validity - std::uint64_t sequence; ///< Distinguish other IPNS records for the same name - std::uint64_t ttl; ///< Recommended caching time -}; - -using CborDeserializer = IpnsCborEntry(ByteView); +class ContextApi; -using CryptoSignatureVerifier = bool(ipns::KeyType, - ByteView, - ByteView, - ByteView); +constexpr static std::size_t MAX_IPNS_PB_SERIALIZED_SIZE = 10 * 1024; +std::optional ValidateIpnsRecord( + ByteView top_level_bytes, + libp2p::multi::ContentIdentifier const& name, + ContextApi&); std::optional ValidateIpnsRecord( ByteView top_level_bytes, libp2p::peer::PeerId const& name, - CryptoSignatureVerifier, - CborDeserializer); + ContextApi&); /*! * \brief Data from IPNS record modulo the verification parts diff --git a/library/include/ipfs_client/orchestrator.h b/library/include/ipfs_client/orchestrator.h index 716d8b6e..64d5f17f 100644 --- a/library/include/ipfs_client/orchestrator.h +++ b/library/include/ipfs_client/orchestrator.h @@ -3,30 +3,42 @@ #include "ipfs_client/ipld/dag_node.h" -#include "vocab/flat_mapset.h" +#include +#include #include #include namespace ipfs { -class Orchestrator : public std::enable_shared_from_this { - flat_map dags_; +class ContextApi; +class Orchestrator : public std::enable_shared_from_this { public: using GatewayAccess = std::function)>; using MimeDetection = std::function< std::string(std::string, std::string_view, std::string const&)>; - explicit Orchestrator(GatewayAccess, MimeDetection); + explicit Orchestrator(GatewayAccess, + std::shared_ptr requestor, + std::shared_ptr = {}); void build_response(std::shared_ptr); - void add_node(std::string key, ipld::NodePtr); + bool add_node(std::string key, ipld::NodePtr); + bool has_key(std::string const& k) const; private: + flat_map dags_; GatewayAccess gw_requestor_; - MimeDetection mimer_; - void from_tree(std::shared_ptr, ipld::NodePtr&, SlashDelimited); - bool gw_request(std::shared_ptr, SlashDelimited path); + std::shared_ptr api_; + std::shared_ptr requestor_; + + void from_tree(std::shared_ptr, + ipld::NodePtr&, + SlashDelimited, + std::string const&); + bool gw_request(std::shared_ptr, + SlashDelimited path, + std::string const& aff); std::string sniff(SlashDelimited, std::string const&) const; }; } // namespace ipfs::ipld diff --git a/library/include/ipfs_client/response.h b/library/include/ipfs_client/response.h index cf5e146b..6ec392bf 100644 --- a/library/include/ipfs_client/response.h +++ b/library/include/ipfs_client/response.h @@ -14,6 +14,9 @@ struct Response { std::string location_; static Response PLAIN_NOT_FOUND; + static Response HOST_NOT_FOUND; + + constexpr static std::uint16_t HOST_NOT_FOUND_STATUS = 503; }; } // namespace ipfs diff --git a/library/include/ipfs_client/signing_key_type.h b/library/include/ipfs_client/signing_key_type.h new file mode 100644 index 00000000..4a74ad0f --- /dev/null +++ b/library/include/ipfs_client/signing_key_type.h @@ -0,0 +1,14 @@ +#ifndef IPFS_SIGNING_KEY_TYPE_H_ +#define IPFS_SIGNING_KEY_TYPE_H_ + +namespace ipfs { +enum class SigningKeyType : int { + RSA, + Ed25519, + Secp256k1, + ECDSA, + KeyTypeCount +}; +} + +#endif // IPFS_SIGNING_KEY_TYPE_H_ diff --git a/library/include/libp2p/multi/content_identifier.hpp b/library/include/libp2p/multi/content_identifier.hpp index c29eb7c5..d677ddf3 100644 --- a/library/include/libp2p/multi/content_identifier.hpp +++ b/library/include/libp2p/multi/content_identifier.hpp @@ -33,12 +33,6 @@ struct ContentIdentifier { MulticodecType::Code content_type, Multihash content_address); - /** - * @param base is a human-readable multibase prefix - * @returns human readable representation of the CID - */ - std::string toPrettyString(const std::string& base) const; - bool operator==(const ContentIdentifier& c) const; bool operator<(const ContentIdentifier& c) const; diff --git a/library/src/ipfs_client/context_api.cc b/library/src/ipfs_client/context_api.cc index f5edcff6..c335b1dc 100644 --- a/library/src/ipfs_client/context_api.cc +++ b/library/src/ipfs_client/context_api.cc @@ -13,3 +13,7 @@ std::string ipfs::GatewayRequest::url() const { std::string ipfs::GatewayRequest::task() const { return gateway.task(); } +bool ipfs::HttpRequestDescription::operator==( + HttpRequestDescription const& r) const { + return url == r.url && accept == r.accept; +} diff --git a/library/src/ipfs_client/gw/block_request_splitter.cc b/library/src/ipfs_client/gw/block_request_splitter.cc new file mode 100644 index 00000000..7665ab23 --- /dev/null +++ b/library/src/ipfs_client/gw/block_request_splitter.cc @@ -0,0 +1,33 @@ +#include + +#include + +using Self = ipfs::gw::BlockRequestSplitter; + +std::string_view Self::name() const { + return "BlockRequestSplitter"; +} +auto Self::handle(ipfs::gw::RequestPtr r) -> HandleOutcome { + if (r->type != Type::Car) { + return HandleOutcome::NOT_HANDLED; + } + // if (r->path.empty()) { + r->type = Type::Block; + return HandleOutcome::NOT_HANDLED; + /* } + { + auto br = std::make_shared(*r); + br->type = Type::Block; + br->path.clear(); + forward(br); + } + { + auto pr = std::make_shared(*r); + pr->type = Type::Providers; + pr->path.clear(); + pr->affinity.clear(); + forward(pr); + } + return HandleOutcome::NOT_HANDLED; + */ +} \ No newline at end of file diff --git a/library/src/ipfs_client/gw/block_request_splitter_unittest.cc b/library/src/ipfs_client/gw/block_request_splitter_unittest.cc new file mode 100644 index 00000000..f7410d34 --- /dev/null +++ b/library/src/ipfs_client/gw/block_request_splitter_unittest.cc @@ -0,0 +1,50 @@ +#include + +#include + +#include + +namespace g = ipfs::gw; + +using Tested = g::BlockRequestSplitter; +using Req = g::GatewayRequest; + +namespace { +struct Recording : public g::Requestor { + std::string_view name() const { return "recording requestor for test"; } + std::vector requests_received; + HandleOutcome handle(g::RequestPtr r) { + requests_received.push_back(r); + return HandleOutcome::DONE; + } +}; +} // namespace +/* +TEST(BlockRequestSplitterTest, typical) { + Tested tested; + auto rec = std::make_shared(); + tested.or_else(rec); + auto req = std::make_shared(); + req->type = g::Type::Car; + req->main_param = "cid"; + req->path = "path"; + req->parallel = 123; + tested.request(req); + EXPECT_EQ(rec->requests_received.size(), 3); + EXPECT_TRUE(rec->requests_received.at(0)->type == g::Type::Block) << +static_cast(rec->requests_received.at(0)->type); + EXPECT_TRUE(rec->requests_received.at(1)->type == g::Type::Providers) << +static_cast(rec->requests_received.at(0)->type); + EXPECT_TRUE(rec->requests_received.at(2)->type == g::Type::Car) << +static_cast(rec->requests_received.at(0)->type); + EXPECT_EQ(rec->requests_received.at(0)->main_param, "cid"); + EXPECT_EQ(rec->requests_received.at(1)->main_param, "cid"); + EXPECT_EQ(rec->requests_received.at(2)->main_param, "cid"); + EXPECT_EQ(rec->requests_received.at(0)->path, ""); + EXPECT_EQ(rec->requests_received.at(1)->path, ""); + EXPECT_EQ(rec->requests_received.at(2)->path, "path"); + EXPECT_EQ(rec->requests_received.at(0)->parallel, 123); + EXPECT_EQ(rec->requests_received.at(1)->parallel, 123); + EXPECT_EQ(rec->requests_received.at(2)->parallel, 123); +} +*/ diff --git a/library/src/ipfs_client/gw/default_requestor.cc b/library/src/ipfs_client/gw/default_requestor.cc new file mode 100644 index 00000000..88ac9170 --- /dev/null +++ b/library/src/ipfs_client/gw/default_requestor.cc @@ -0,0 +1,24 @@ +#include + +#include +#include +#include +#include +#include +#include + +auto ipfs::gw::default_requestor(ipfs::GatewayList gws, + std::shared_ptr api) + -> std::shared_ptr { + auto result = std::make_shared(); + auto pool = std::make_shared(); + result->or_else(std::make_shared()) + .or_else(std::make_shared(api)) + .or_else(pool) + .or_else(std::make_shared()); + for (auto& gw : gws) { + auto gwr = std::make_shared(gw.url_prefix(), api); + pool->add(gwr); + } + return result; +} diff --git a/library/src/ipfs_client/gw/dnslink_requestor.cc b/library/src/ipfs_client/gw/dnslink_requestor.cc new file mode 100644 index 00000000..03124447 --- /dev/null +++ b/library/src/ipfs_client/gw/dnslink_requestor.cc @@ -0,0 +1,70 @@ +#include + +#include "ipfs_client/ipld/ipns_name.h" + +#include +#include + +#include +#include +#include + +#include "log_macros.h" + +#define ABSL_USES_STD_STRING_VIEW 1 // Thanks, Windows! +#include + +using Self = ipfs::gw::DnsLinkRequestor; +using namespace std::literals; + +Self::DnsLinkRequestor(std::shared_ptr api) { + api_ = api; +} +std::string_view Self::name() const { + return "DNSLink requestor"; +} +namespace { +bool parse_results(ipfs::gw::RequestPtr req, + std::vector const& results); +} +auto Self::handle(ipfs::gw::RequestPtr req) -> HandleOutcome { + if (req->type != Type::DnsLink) { + return HandleOutcome::NOT_HANDLED; + } + // std::function requires target be copy-constructible + auto success = std::make_shared(); + *success = false; + auto res = [req, success](std::vector const& results) { + *success = *success || parse_results(req, results); + }; + auto don = [success, req]() { + LOG(INFO) << "DNSLink request completed for " << req->main_param + << " success=" << *success; + if (!*success) { + req->dependent->finish(ipfs::Response::HOST_NOT_FOUND); + } + }; + api_->SendDnsTextRequest("_dnslink." + req->main_param, res, std::move(don)); + return HandleOutcome::PENDING; +} +namespace { +bool parse_results(ipfs::gw::RequestPtr req, + std::vector const& results) { + constexpr auto prefix = "dnslink="sv; + LOG(INFO) << "Scanning " << results.size() << " DNS TXT records for " + << req->main_param << " looking for dnslink..."; + for (auto& result : results) { + if (absl::StartsWith(result, prefix)) { + LOG(INFO) << "DNSLink result=" << result; + auto target = result.substr(prefix.size()); + auto node = std::make_shared(target); + req->orchestrator->add_node(req->main_param, node); + req->orchestrator->build_response(req->dependent); + return true; + } else { + LOG(INFO) << "Irrelevant TXT result, ignored: " << result; + } + } + return false; +} +} // namespace diff --git a/library/src/ipfs_client/gw/gateway_http_requestor.cc b/library/src/ipfs_client/gw/gateway_http_requestor.cc new file mode 100644 index 00000000..ec95b06e --- /dev/null +++ b/library/src/ipfs_client/gw/gateway_http_requestor.cc @@ -0,0 +1,170 @@ +#include "gateway_http_requestor.h" + +#include +#include + +#include +#include +#include +#include + +#include + +#include "log_macros.h" + +using Self = ipfs::gw::GatewayHttpRequestor; +using ReqTyp = ipfs::gw::Type; +using CidCodec = libp2p::multi::ContentIdentifierCodec; + +std::string_view Self::name() const { + return "simplistic HTTP requestor"; +} +auto Self::handle(ipfs::gw::RequestPtr r) -> HandleOutcome { + DCHECK(r); + DCHECK(r->orchestrator); + DCHECK(r->dependent); + DCHECK_GT(prefix_.size(), 0UL); + if (!r->is_http()) { + LOG(ERROR) << name() << " only handles HTTP requests"; + return HandleOutcome::NOT_HANDLED; + } + auto req_key = r->url_suffix().append(r->accept()); + if (seen_.count(req_key)) { + return HandleOutcome::NOT_HANDLED; + } + if (target(*r) <= r->parallel) { + return HandleOutcome::MAYBE_LATER; + } + auto desc = r->describe_http(); + if (!desc.has_value() || desc.value().url.empty()) { + LOG(ERROR) + << r->debug_string() + << " is HTTP but can't describe the HTTP request that would happen?"; + return HandleOutcome::NOT_HANDLED; + } + if (prefix_.back() == '/' && desc.value().url[0] == '/') { + desc.value().url.insert(0, prefix_, 0UL, prefix_.size() - 1UL); + } else { + desc.value().url.insert(0, prefix_); + } + auto cb = [this, r](std::int16_t status, std::string_view body, + ContextApi::HeaderAccess) { + if (r->parallel) { + r->parallel--; + } + if (pending) { + pending--; + } + if (status / 100 == 2) { + auto nod = node_from_type(r->cid, r->type, body); + if (nod) { + if (r->orchestrator->add_node(r->main_param, nod)) { + r->type = Type::Zombie; + r->orchestrator->build_response(r->dependent); + } + if (typ_good_.insert(r->type).second) { + LOG(INFO) << prefix_ << " OK with requests of type " + << static_cast(r->type); + } else if (typ_bad_.erase(r->type)) { + LOG(INFO) << prefix_ << " truly OK with requests of type " + << static_cast(r->type); + } + if (aff_good_.insert(r->affinity).second) { + LOG(INFO) << prefix_ << " likes requests in the neighborhood of " + << r->affinity; + } else if (aff_bad_.erase(r->affinity)) { + LOG(INFO) << prefix_ << " truly OK with affinity " << r->affinity; + } + return; + } else { + LOG(INFO) << "Got an unuseful response from " << prefix_ + << " forwarding request " << r->debug_string() + << " to next requestor."; + } + } else { + LOG(INFO) << r->debug_string() << " got a failure of status " << status + << " from " << prefix_; + } + aff_bad_.insert(r->affinity); + typ_bad_.insert(r->type); + forward(r); + }; + DCHECK(api_); + api_->SendHttpRequest(desc.value(), cb); + seen_.insert(req_key); + pending++; + return HandleOutcome::PENDING; +} + +Self::GatewayHttpRequestor(std::string gateway_prefix, + std::shared_ptr api) + : prefix_{gateway_prefix} { + api_ = api; +} +Self::~GatewayHttpRequestor() {} + +ipfs::ipld::NodePtr Self::node_from_type(std::optional const& cid, + ReqTyp t, + std::string_view body) const { + switch (t) { + case ReqTyp::Block: { + if (cid.has_value()) { + ipfs::Block blk{cid.value(), std::string{body}}; + if (blk.valid() && blk.cid_matches_data()) { + return ipfs::ipld::DagNode::fromBlock(blk); + } + } + return {}; + } + case ReqTyp::Ipns: { + if (cid.has_value()) { + auto byte_ptr = reinterpret_cast(body.data()); + auto rec = ipfs::ValidateIpnsRecord({byte_ptr, body.size()}, + cid.value(), *api_); + if (rec.has_value()) { + return ipfs::ipld::DagNode::fromIpnsRecord(rec.value()); + } else { + LOG(ERROR) << "IPNS record failed to validate!"; + } + } + return {}; + } + case ReqTyp::Identity: + LOG(ERROR) << "An HTTP response from a gateway received for an identity " + "(inlined) CID"; + return {}; + case ReqTyp::DnsLink: + LOG(WARNING) << "HTTP responses to DnsLink requests not yet implemented, " + "and it's not clear they will be."; + return {}; + case ReqTyp::Car: + LOG(INFO) << "TODO responses to Car requests not yet implemented."; + return {}; + case ReqTyp::Providers: + LOG(INFO) << "TODO responses to Car requests not yet implemented: " + << body; + return {}; + case ReqTyp::Zombie: + return {}; + } + return {}; // TODO +} +int Self::target(GatewayRequest const& r) const { + int result = 0; + if (pending == 0) { + result += 1; + } + if (typ_good_.count(r.type)) { + result += 3; + } + if (!typ_bad_.count(r.type)) { + result += 2; + } + if (aff_good_.count(r.affinity)) { + result += 5; + } + if (aff_bad_.count(r.affinity) == 0UL) { + result += 4; + } + return result; +} diff --git a/library/src/ipfs_client/gw/gateway_http_requestor.h b/library/src/ipfs_client/gw/gateway_http_requestor.h new file mode 100644 index 00000000..15dad14f --- /dev/null +++ b/library/src/ipfs_client/gw/gateway_http_requestor.h @@ -0,0 +1,32 @@ +#ifndef IPFS_GATEWAY_HTTP_REQUESTOR_H_ +#define IPFS_GATEWAY_HTTP_REQUESTOR_H_ + +#include +#include + +#include +#include + +namespace ipfs::gw { +class GatewayHttpRequestor final : public Requestor { + std::string prefix_; + std::set seen_; + std::set aff_good_, aff_bad_; + std::set typ_good_, typ_bad_; + std::size_t pending = 0UL; + + HandleOutcome handle(RequestPtr) override; + std::string_view name() const override; + using Cid = libp2p::multi::ContentIdentifier; + ipfs::ipld::NodePtr node_from_type(std::optional const& cid, + ipfs::gw::Type, + std::string_view body) const; + int target(GatewayRequest const&) const; + + public: + GatewayHttpRequestor(std::string gateway_prefix, std::shared_ptr); + ~GatewayHttpRequestor() noexcept override; +}; +} // namespace ipfs::gw + +#endif // IPFS_GATEWAY_HTTP_REQUESTOR_H_ diff --git a/library/src/ipfs_client/gw/gateway_http_requestor_unittest.cc b/library/src/ipfs_client/gw/gateway_http_requestor_unittest.cc new file mode 100644 index 00000000..d111db5f --- /dev/null +++ b/library/src/ipfs_client/gw/gateway_http_requestor_unittest.cc @@ -0,0 +1,152 @@ +#include + +#include + +#include +#include + +using T = ipfs::gw::GatewayHttpRequestor; +using R = ipfs::gw::GatewayRequest; +using P = ipfs::gw::RequestPtr; + +using namespace std::literals; + +namespace { +struct FakeApi final : public ipfs::ContextApi { + std::vector mutable http_requests_sent; + std::vector mutable cbs; + void SendHttpRequest(HttpRequestDescription d, + HttpCompleteCallback cb) const { + http_requests_sent.push_back(d); + cbs.push_back(cb); + } + + void SendDnsTextRequest(std::string hostname, + DnsTextResultsCallback, + DnsTextCompleteCallback) {} + std::string MimeType(std::string extension, + std::string_view content, + std::string const& url) const { + return ""; + } + std::string UnescapeUrlComponent(std::string_view url_comp) const { + return ""; + } + IpnsCborEntry deserialize_cbor(ByteView) const { return {}; } + bool verify_key_signature(SigningKeyType, + ByteView signature, + ByteView data, + ByteView key_bytes) const { + return true; + } + + void Discover(std::function)> cb) {} + std::shared_ptr InitiateGatewayRequest( + ipfs::BusyGateway) { + return {}; + } +}; +struct FakeRtor : public ipfs::gw::Requestor { + std::string_view name() const { return "fake requestor"; } + std::vector

requests_forwarded; + HandleOutcome handle(P p) { + requests_forwarded.push_back(p); + return HandleOutcome::DONE; + } +}; +struct GatewayHttpRequestorTest : public ::testing::Test { + std::shared_ptr api = std::make_shared(); + std::shared_ptr t = std::make_shared("scheme://host", api); + std::shared_ptr chain = std::make_shared(); + std::shared_ptr orc; + P req() { + t->or_else(chain); + auto noop = [](auto) {}; + auto noop2 = [](auto, auto) {}; + auto req = R::fromIpfsPath( + "/ipfs/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"sv); + req->orchestrator = orc = + std::make_shared(noop, t, api); + req->dependent = std::make_shared( + "/ipfs/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", + noop2); + return req; + } + ipfs::gw::Requestor& sup() { return *t; } +}; +} // namespace +TEST_F(GatewayHttpRequestorTest, name) { + EXPECT_EQ(sup().name(), "simplistic HTTP requestor"); +} +TEST_F(GatewayHttpRequestorTest, without_slash) { + t->request(req()); + EXPECT_EQ(api->http_requests_sent.size(), 1); + EXPECT_EQ(api->http_requests_sent.at(0).url, + "scheme://host/ipfs/" + "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"); +} +TEST_F(GatewayHttpRequestorTest, with_slash) { + t = std::make_shared("scheme://host/", api); + t->request(req()); + EXPECT_EQ(api->http_requests_sent.size(), 1); + EXPECT_EQ(api->http_requests_sent.at(0).url, + "scheme://host/ipfs/" + "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"); +} +TEST_F(GatewayHttpRequestorTest, block_has_accept) { + t->request(req()); + EXPECT_EQ(api->http_requests_sent.size(), 1); + EXPECT_EQ(api->http_requests_sent.at(0).accept, "application/vnd.ipld.raw"); +} +TEST_F(GatewayHttpRequestorTest, callback_failure_forwards) { + t->request(req()); + EXPECT_EQ(api->http_requests_sent.size(), 1); + EXPECT_EQ(api->http_requests_sent.at(0).accept, "application/vnd.ipld.raw"); + EXPECT_EQ(api->cbs.size(), 1); + auto noop = [](auto) { return std::string{}; }; + api->cbs.at(0)(404, "", noop); + EXPECT_EQ(chain->requests_forwarded.size(), 1); +} +TEST_F(GatewayHttpRequestorTest, callback_success_adds_node) { + t->request(req()); + EXPECT_EQ(api->http_requests_sent.size(), 1); + EXPECT_EQ(api->http_requests_sent.at(0).accept, "application/vnd.ipld.raw"); + EXPECT_EQ(api->cbs.size(), 1); + auto noop = [](auto) { return std::string{}; }; + api->cbs.at(0)(200, "", noop); + EXPECT_TRUE(orc->has_key( + "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")); + EXPECT_EQ(chain->requests_forwarded.size(), 0); +} +TEST_F(GatewayHttpRequestorTest, high_parallel_means_maybe_later) { + auto r = req(); + r->parallel = 8; + t->request(r); + EXPECT_EQ(api->http_requests_sent.size(), 0); + EXPECT_EQ(api->cbs.size(), 0); + EXPECT_EQ(chain->requests_forwarded.size(), 1); +} +TEST_F(GatewayHttpRequestorTest, high_parallel_passeswithaffinity) { + auto r = req(); + EXPECT_TRUE(orc); + t->request(r); + EXPECT_EQ(api->http_requests_sent.size(), 1); + EXPECT_EQ(api->cbs.size(), 1); + EXPECT_EQ(chain->requests_forwarded.size(), 0); + EXPECT_EQ(api->http_requests_sent.at(0).url, + "scheme://host/ipfs/" + "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"); + auto noop = [](auto) { return std::string{}; }; + api->cbs.at(0)(200, "", noop); + r->parallel = 8; + r->type = ipfs::gw::Type::Car; + r->path = "index.html"; + t->request(r); + EXPECT_EQ(api->http_requests_sent.size(), 2); + EXPECT_EQ(api->cbs.size(), 2); + EXPECT_EQ(chain->requests_forwarded.size(), 0); + EXPECT_EQ(api->http_requests_sent.at(1).url, + "scheme://host/ipfs/" + "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/" + "index.html?dag-scope=entity"); +} diff --git a/library/src/ipfs_client/gw/gateway_request.cc b/library/src/ipfs_client/gw/gateway_request.cc index 69de795e..5931a314 100644 --- a/library/src/ipfs_client/gw/gateway_request.cc +++ b/library/src/ipfs_client/gw/gateway_request.cc @@ -1,6 +1,7 @@ -#include "ipfs_client/gw/gateway_request.h" +#include -#include "ipfs_client/response.h" +#include +#include #include "log_macros.h" @@ -25,6 +26,7 @@ std::shared_ptr Self::fromIpfsPath(ipfs::SlashDelimited p) { if (!result->cid.has_value()) { LOG(ERROR) << "IPFS request with invalid/unsupported CID " << result->main_param; + return {}; } if (result->cid.value().content_address.getType() == libp2p::multi::HashType::identity) { @@ -60,10 +62,12 @@ std::string Self::url_suffix() const { case Type::DnsLink: LOG(FATAL) << "Don't try to use HTTP(s) for DNS TXT records."; return {}; - default: - LOG(FATAL) << "Invalid gateway request type: " << static_cast(type); + case Type::Identity: + case Type::Zombie: return {}; } + LOG(FATAL) << "Unhandled gateway request type: " << static_cast(type); + return {}; } std::string_view Self::accept() const { switch (type) { @@ -85,10 +89,32 @@ std::string_view Self::accept() const { // DNSLink capability. LOG(FATAL) << "Don't try to use HTTP(s) for DNS TXT records."; return {}; - default: - LOG(FATAL) << "Invalid gateway request type: " << static_cast(type); + case Type::Identity: + case Type::Zombie: return {}; } + LOG(FATAL) << "Invalid gateway request type: " << static_cast(type); + return {}; +} +short Self::timeout_seconds() const { + switch (type) { + case Type::DnsLink: + return 32; + case Type::Block: + return 64; + case Type::Providers: + return 128; + case Type::Car: + return 256; + case Type::Ipns: + return 512; + case Type::Identity: + case Type::Zombie: + return 0; + } + LOG(FATAL) << "timeout_seconds() called for unsupported gateway request type " + << static_cast(type); + return 0; } auto Self::identity_data() const -> std::string_view { @@ -99,3 +125,74 @@ auto Self::identity_data() const -> std::string_view { auto d = reinterpret_cast(hash.data()); return std::string_view{d, hash.size()}; } + +bool Self::is_http() const { + return type != Type::DnsLink && type != Type::Identity; +} +auto Self::describe_http() const -> std::optional { + if (!is_http()) { + return {}; + } + return HttpRequestDescription{url_suffix(), timeout_seconds(), + std::string{accept()}, max_response_size()}; +} +std::optional Self::max_response_size() const { + switch (type) { + case Type::Identity: + return 0; + case Type::DnsLink: + return std::nullopt; + case Type::Ipns: + return MAX_IPNS_PB_SERIALIZED_SIZE; + case Type::Block: + return BLOCK_RESPONSE_BUFFER_SIZE; + case Type::Car: { + auto n = std::count(path.begin(), path.end(), '/'); + // now n >= number of path components, but there is also a block for root + n++; + return n * BLOCK_RESPONSE_BUFFER_SIZE; + } + case Type::Zombie: + return 0; + case Type::Providers: + // This one's tricky. + // One could easily guess a pracitical limit to the size of a Peer, + // and the spec says it SHOULD be limited to 100 peers. + // But there's no guaranteed limits. A peer could have an unlimited + // number of multiaddrs. And they're allowed to throw in arbitrary + // fields I'm supposed to ignore. So in theory it could be infinitely + // large. + return std::nullopt; + } + LOG(ERROR) << "Invalid gateway request type " << static_cast(type); + return std::nullopt; +} +std::ostream& operator<<(std::ostream& s, ipfs::gw::Type t) { + using ipfs::gw::Type; + switch (t) { + case Type::Block: + return s << "Block"; + case Type::Car: + return s << "Car"; + case Type::Ipns: + return s << "Ipns"; + case Type::DnsLink: + return s << "DnsLink"; + case Type::Providers: + return s << "Providers"; + case Type::Identity: + return s << "Identity"; + case Type::Zombie: + return s << "CompletedRequest"; + } + return s << "InvalidType=" << static_cast(t); +} +std::string Self::debug_string() const { + std::ostringstream oss; + oss << "Request{Type=" << type << ' ' << main_param; + if (!path.empty()) { + oss << ' ' << path; + } + oss << " plel=" << parallel << '}'; + return oss.str(); +} diff --git a/library/src/ipfs_client/gw/inline_request_handler.cc b/library/src/ipfs_client/gw/inline_request_handler.cc new file mode 100644 index 00000000..43f49ef7 --- /dev/null +++ b/library/src/ipfs_client/gw/inline_request_handler.cc @@ -0,0 +1,20 @@ +#include + +#include +#include +#include + +using Self = ipfs::gw::InlineRequestHandler; + +std::string_view Self::name() const { + return "InlineRequestHandler"; +} +auto Self::handle(ipfs::gw::RequestPtr req) -> HandleOutcome { + if (req->type != gw::Type::Identity) { + return HandleOutcome::NOT_HANDLED; + } + auto node = std::make_shared(std::string{req->identity_data()}); + req->orchestrator->add_node(req->main_param, node); + req->orchestrator->build_response(req->dependent); + return HandleOutcome::DONE; +} diff --git a/library/src/ipfs_client/gw/requestor.cc b/library/src/ipfs_client/gw/requestor.cc new file mode 100644 index 00000000..6e346173 --- /dev/null +++ b/library/src/ipfs_client/gw/requestor.cc @@ -0,0 +1,132 @@ +#include + +#include +#include + +#include +#include +#include +#include + +#include + +#include "log_macros.h" + +using Self = ipfs::gw::Requestor; +using ReqPtr = std::shared_ptr; +using CidCodec = libp2p::multi::ContentIdentifierCodec; + +Self& Self::or_else(std::shared_ptr p) { + if (next_) { + next_->or_else(p); + } else { + VLOG(2) << name() << " is followed by " << p->name(); + next_ = p; + } + if (api_ && !p->api_) { + LOG(INFO) << name() << " granting context to " << p->name(); + p->api_ = api_; + } + return *this; +} + +void Self::request(ReqPtr req) { + if (!req) { + return; + } + switch (handle(req)) { + case HandleOutcome::MAYBE_LATER: + if (!(req->retry_at)) { + req->retry_at = shared_from_this(); + } + forward(req); + break; + case HandleOutcome::PARALLEL: + case HandleOutcome::NOT_HANDLED: + if (next_) { + VLOG(2) << name() << " not handled request, passing along to " + << next_->name(); + next_->request(req); + } else { + LOG(ERROR) << "Ran out of Requestors in the chain while looking for " + "one that can handle " + << req->debug_string(); + definitive_failure(req); + } + break; + case HandleOutcome::PENDING: + VLOG(1) << req->debug_string() << " sent via requestor " << name(); + break; + case HandleOutcome::DONE: + VLOG(1) << req->debug_string() << " finished synchronously: " << name(); + break; + } +} +void Self::failure(ipfs::gw::RequestPtr r) const { + if (next_) { + LOG(WARNING) << name() << " failed on " << r->debug_string() + << " ... passing along to " << next_->name(); + next_->request(r); + } else { + definitive_failure(r); + } +} +void Self::definitive_failure(ipfs::gw::RequestPtr r) const { + DCHECK(r); + DCHECK(r->dependent); + r->dependent->finish(Response::PLAIN_NOT_FOUND); +} +void Self::success(ipfs::gw::RequestPtr req, std::string_view body) const { + Response res; + res.status_ = 200; + res.body_.assign(body); + receive_response(req, res); +} +void Self::iterate_nodes( + GatewayRequest const& req, + Response const& res, + std::function cb) const { + if (res.body_.empty()) { + return; + } + auto cid = CidCodec::fromString(req.main_param); + if (!cid.has_value()) { + return; + } + Block b{cid.value(), res.body_}; + if (!b.valid()) { + return; + } + auto n = ipld::DagNode::fromBlock(b); + if (n) { + cb(req.main_param, n); + } +} +void Self::receive_response(ipfs::gw::RequestPtr req, + ipfs::Response const& res) const { + if (!req->orchestrator) { + LOG(ERROR) << name() << ": Got response for " << req->debug_string() + << " but it doesn't have its orchestrator pointer set, so can't " + "deliver response."; + return; + } + auto o = req->orchestrator; + iterate_nodes(*req, res, + [o](std::string k, ipld::NodePtr n) { o->add_node(k, n); }); + if (res.status_ / 100 == 2) { + req->orchestrator->build_response(req->dependent); + } else if (req->parallel == 0) { + if (req->retry_at) { + auto p = req->retry_at; + req->retry_at.reset(); + p->request(req); + } else { + LOG(ERROR) << "Finally gave up on " << req->debug_string(); + } + } +} +void Self::forward(ipfs::gw::RequestPtr req) const { + if (next_) { + next_->request(req); + } +} \ No newline at end of file diff --git a/library/src/ipfs_client/gw/requestor_pool.cc b/library/src/ipfs_client/gw/requestor_pool.cc new file mode 100644 index 00000000..9c74eac7 --- /dev/null +++ b/library/src/ipfs_client/gw/requestor_pool.cc @@ -0,0 +1,66 @@ +#include "requestor_pool.h" + +#include + +#include "log_macros.h" + +using Self = ipfs::gw::RequestorPool; + +std::string_view Self::name() const { + return "requestor pool"; +} +Self& Self::add(std::shared_ptr r) { + if (api_ && !(r->api_)) { + r->api_ = api_; + } + pool.push_back(r); + r->or_else(shared_from_this()); + return *this; +} +auto Self::handle(ipfs::gw::RequestPtr req) -> HandleOutcome { + for (auto i = 0UL; i * 2 < waiting.size(); ++i) { + auto popped = waiting.front(); + check(popped.first, popped.second); + } + return check(req, 0UL); +} +auto Self::check(ipfs::gw::RequestPtr req, std::size_t start) -> HandleOutcome { + using O = HandleOutcome; + if (req->type == Type::Zombie) { + return O::DONE; + } + auto next_retry = pool.size(); + for (auto i = start; i < pool.size(); ++i) { + auto& tor = pool[i]; + switch (tor->handle(req)) { + case O::DONE: + LOG(INFO) << "RequestorPool::handle returning DONE because a member of " + "the pool's handle returned DONE."; + return O::DONE; + case O::PENDING: + case O::PARALLEL: + req->parallel++; + break; + case O::MAYBE_LATER: + if (next_retry == pool.size()) { + next_retry = i; + } + break; + case O::NOT_HANDLED: + break; + } + } + if (req->parallel > 0) { + LOG(INFO) << req->parallel << " requestors have picked up the task."; + return O::PENDING; + } + if (next_retry < pool.size()) { + LOG(INFO) << "No requestors are available for " << req->debug_string() + << " right now, will retry at index " << next_retry; + waiting.emplace(req, next_retry); + return O::PENDING; + } + LOG(WARNING) << "Have exhausted all requestors in pool looking for " + << req->debug_string(); + return O::NOT_HANDLED; +} diff --git a/library/src/ipfs_client/gw/requestor_pool.h b/library/src/ipfs_client/gw/requestor_pool.h new file mode 100644 index 00000000..3218da60 --- /dev/null +++ b/library/src/ipfs_client/gw/requestor_pool.h @@ -0,0 +1,26 @@ +#ifndef IPFS_REQUESTOR_POOL_H_ +#define IPFS_REQUESTOR_POOL_H_ + +#include + +#include + +#include +#include + +namespace ipfs::gw { +class RequestorPool : public Requestor { + std::string_view name() const override; + HandleOutcome handle(RequestPtr) override; + + HandleOutcome check(RequestPtr, std::size_t); + + std::vector> pool; + std::queue> waiting; + + public: + RequestorPool& add(std::shared_ptr); +}; +} // namespace ipfs::gw + +#endif // IPFS_REQUESTOR_POOL_H_ diff --git a/library/src/ipfs_client/gw/requestor_unittest.cc b/library/src/ipfs_client/gw/requestor_unittest.cc new file mode 100644 index 00000000..5e534bd1 --- /dev/null +++ b/library/src/ipfs_client/gw/requestor_unittest.cc @@ -0,0 +1,54 @@ +#include + +#include + +#include + +namespace g = ipfs::gw; +using T = g::Requestor; + +namespace { +struct TestRequestor : public g::Requestor { + std::vector requests_sent_to_handle; + std::vector outcomes; + HandleOutcome handle(g::RequestPtr r) { + requests_sent_to_handle.push_back(r); + auto res = outcomes.at(0); + outcomes.erase(outcomes.begin()); + return res; + } + std::string name_ = "tested requestor"; + std::string_view name() const { return name_; } + using Out = HandleOutcome; +}; +using Out = TestRequestor::Out; +struct RequestorTest : public ::testing::Test { + std::shared_ptr a = std::make_shared(); + std::shared_ptr b = std::make_shared(); + std::shared_ptr req_ = + std::make_shared(); +}; +} // namespace + +TEST_F(RequestorTest, can_ctor) {} + +TEST_F(RequestorTest, nothandled_continues) { + a->or_else(b); + a->outcomes.push_back(Out::NOT_HANDLED); + b->outcomes.push_back(Out::PENDING); + a->request(req_); + EXPECT_EQ(a->requests_sent_to_handle.size(), 1); + EXPECT_EQ(b->requests_sent_to_handle.size(), 1); + EXPECT_TRUE(a->requests_sent_to_handle.at(0) == req_); + EXPECT_TRUE(b->requests_sent_to_handle.at(0) == req_); +} + +TEST_F(RequestorTest, done_stops_request) { + a->or_else(b); + a->outcomes.push_back(Out::DONE); + b->outcomes.push_back(Out::PENDING); + a->request(req_); + EXPECT_EQ(a->requests_sent_to_handle.size(), 1); + EXPECT_EQ(b->requests_sent_to_handle.size(), 0); + EXPECT_TRUE(a->requests_sent_to_handle.at(0) == req_); +} diff --git a/library/src/ipfs_client/gw/terminating_requestor.cc b/library/src/ipfs_client/gw/terminating_requestor.cc new file mode 100644 index 00000000..56ed2036 --- /dev/null +++ b/library/src/ipfs_client/gw/terminating_requestor.cc @@ -0,0 +1,29 @@ +#include "ipfs_client/gw/terminating_requestor.h" + +#include + +#include "log_macros.h" + +using Self = ipfs::gw::TerminatingRequestor; + +std::string_view Self::name() const { + return "Terminating requestor"; +} +auto Self::handle(ipfs::gw::RequestPtr r) -> HandleOutcome { + if (r->type == Type::Zombie) { + return HandleOutcome::DONE; + } else if (r->parallel) { + return HandleOutcome::PENDING; + } else if (r->retry_at) { + LOG(INFO) << "Calling through retry..."; + auto p = r->retry_at; + r->retry_at.reset(); + p->request(r); + return HandleOutcome::PENDING; + } else { + LOG(ERROR) << "Out of options, giving up on gateway request " + << r->debug_string(); + definitive_failure(r); + return HandleOutcome::DONE; + } +} diff --git a/library/src/ipfs_client/ipfs_request.cc b/library/src/ipfs_client/ipfs_request.cc index 5e25e687..84102251 100644 --- a/library/src/ipfs_client/ipfs_request.cc +++ b/library/src/ipfs_client/ipfs_request.cc @@ -8,12 +8,22 @@ using Self = ipfs::IpfsRequest; // : path_{path_p}, callback_([](auto&, auto&) {}) {} Self::IpfsRequest(std::string path_p, Finisher f) : path_{path_p}, callback_{f} {} - +void Self::till_next(std::size_t w) { + waiting_ = w; +} void Self::finish(ipfs::Response& r) { - LOG(INFO) << "IpfsRequest::finish(...);"; + LOG(INFO) << "IpfsRequest::finish(" << waiting_ << ");"; + if (waiting_) { + if (--waiting_) { + return; + } + } callback_(*this, r); // TODO - cancel other gw req pointing into this callback_ = [](auto& q, auto&) { LOG(INFO) << "IPFS request " << q.path().pop_all() << " satisfied multiply"; }; } +bool Self::ready_after() { + return waiting_ == 0 || 0 == --waiting_; +} diff --git a/library/src/ipfs_client/ipld/chunk.cc b/library/src/ipfs_client/ipld/chunk.cc index 2eb452f6..caaa866d 100644 --- a/library/src/ipfs_client/ipld/chunk.cc +++ b/library/src/ipfs_client/ipld/chunk.cc @@ -1,5 +1,7 @@ #include "chunk.h" +#include "log_macros.h" + using Chunk = ipfs::ipld::Chunk; Chunk::Chunk(std::string data) : data_{data} {} @@ -9,6 +11,8 @@ auto Chunk::resolve(ipfs::SlashDelimited path, ipfs::ipld::DagNode::BlockLookup, std::string&) -> ResolveResult { if (path) { + LOG(ERROR) << "Can't resolve a path (" << path.to_string() + << ") inside of a file chunk!"; return ProvenAbsent{}; } return Response{"", 200, data_, {}}; diff --git a/library/src/ipfs_client/ipld/dag_node.cc b/library/src/ipfs_client/ipld/dag_node.cc index 0a92fb5e..2ae41e7c 100644 --- a/library/src/ipfs_client/ipld/dag_node.cc +++ b/library/src/ipfs_client/ipld/dag_node.cc @@ -18,8 +18,9 @@ std::shared_ptr Node::fromBlock(ipfs::Block const& block) { std::shared_ptr result; switch (block.type()) { case Block::Type::FileChunk: - result = std::make_shared(block.chunk_data()); - break; + return std::make_shared(block.chunk_data()); + case Block::Type::NonFs: + return std::make_shared(block.unparsed()); case Block::Type::Directory: result = std::make_shared(); break; @@ -60,3 +61,6 @@ std::shared_ptr Node::rooted() { auto Node::as_hamt() -> DirShard* { return nullptr; } +void Node::set_api(std::shared_ptr api) { + api_ = api; +} \ No newline at end of file diff --git a/library/src/ipfs_client/ipld/directory_shard.cc b/library/src/ipfs_client/ipld/directory_shard.cc index 4e0032fe..ade6f5ba 100644 --- a/library/src/ipfs_client/ipld/directory_shard.cc +++ b/library/src/ipfs_client/ipld/directory_shard.cc @@ -2,6 +2,7 @@ #include "log_macros.h" +#include #include #include @@ -20,6 +21,8 @@ using Self = ipfs::ipld::DirShard; auto Self::resolve(ipfs::SlashDelimited relpath, ipfs::ipld::DagNode::BlockLookup blu, std::string& to_here) -> ResolveResult { + LOG(INFO) << "DirShard::resolve(" << to_here << " / " << relpath.to_string() + << ')'; if (!relpath) { // TODO check if index.html is present and if not implement indexing auto result = resolve("index.html"sv, blu, to_here); @@ -29,7 +32,7 @@ auto Self::resolve(ipfs::SlashDelimited relpath, } return result; } - auto name = relpath.pop(); + auto name = api_->UnescapeUrlComponent(relpath.pop()); auto hash = hexhash(name); return resolve_internal(hash.begin(), hash.end(), name, relpath, blu, to_here); @@ -40,8 +43,11 @@ auto Self::resolve_internal(ipfs::ipld::DirShard::HashIter hash_b, ipfs::SlashDelimited path_after_dir, ipfs::ipld::DagNode::BlockLookup blu, std::string& path_to_dir) -> ResolveResult { + auto hash_chunk = hash_b == hash_e ? std::string{} : *hash_b; + VLOG(1) << "Scanning directory shard looking for " << hash_chunk << ',' + << element_name; for (auto& [name, link] : links_) { - if (hash_b != hash_e && !absl::StartsWith(name, *hash_b)) { + if (!absl::StartsWith(name, hash_chunk)) { continue; } if (!link.node) { @@ -51,6 +57,9 @@ auto Self::resolve_internal(ipfs::ipld::DirShard::HashIter hash_b, return MoreDataNeeded{std::vector{"/ipfs/" + link.cid}}; } if (absl::EndsWith(name, element_name)) { + LOG(INFO) << "Found " << element_name + << ", leaving HAMT sharded directory " << name << "->" + << link.cid; return link.node->resolve(path_after_dir, blu, path_to_dir); } auto downcast = link.node->as_hamt(); @@ -59,6 +68,9 @@ auto Self::resolve_internal(ipfs::ipld::DirShard::HashIter hash_b, LOG(ERROR) << "Ran out of hash bits."; return ProvenAbsent{}; } + LOG(INFO) << "Found hash chunk, continuing to next level of HAMT sharded " + "directory " + << name << "->" << link.cid; return downcast->resolve_internal(std::next(hash_b), hash_e, element_name, path_after_dir, blu, path_to_dir); } else { diff --git a/library/src/ipfs_client/ipld/small_directory.cc b/library/src/ipfs_client/ipld/small_directory.cc index ccff62e0..ec73a649 100644 --- a/library/src/ipfs_client/ipld/small_directory.cc +++ b/library/src/ipfs_client/ipld/small_directory.cc @@ -1,5 +1,6 @@ #include "small_directory.h" +#include #include "ipfs_client/generated_directory_listing.h" #include "ipfs_client/path2url.h" @@ -22,7 +23,7 @@ ipfs::ipld::NodePtr& node(ipfs::ipld::Link& l, auto Self::resolve(SlashDelimited path, BlockLookup blu, std::string& to_here) -> ResolveResult { if (!path) { - // GeneratedDirectoryListing index_html{path2url(to_here)}; + LOG(INFO) << "Directory listing requested."; GeneratedDirectoryListing index_html{to_here}; for (auto& [name, link] : links_) { LOG(INFO) << "Listing " << to_here << " encountered " << name << '=' @@ -45,11 +46,13 @@ auto Self::resolve(SlashDelimited path, BlockLookup blu, std::string& to_here) } return Response{"text/html", 200, index_html.Finish(), ""}; } - auto name = path.pop(); + auto name = api_->UnescapeUrlComponent(path.pop()); + LOG(INFO) << "Looking for '" << name << "' in directory."; // TODO binary search auto match = [&name](auto& l) { return l.first == name; }; auto it = std::find_if(links_.begin(), links_.end(), match); if (links_.end() == it) { + LOG(INFO) << name << " does not exist in directory."; return ProvenAbsent{}; } auto& link = it->second; @@ -59,8 +62,11 @@ auto Self::resolve(SlashDelimited path, BlockLookup blu, std::string& to_here) to_here.push_back('/'); } to_here.append(name); + LOG(INFO) << "Descending into " << link.cid << " for " << name; return nod->resolve(path, blu, to_here); } else { + LOG(INFO) << "Should descending into " << link.cid << " for " << name + << " but can't because it's missing. Will request."; return MoreDataNeeded{std::vector{"/ipfs/" + link.cid}}; } } diff --git a/library/src/ipfs_client/ipld/unixfs_file.cc b/library/src/ipfs_client/ipld/unixfs_file.cc index 4faa0646..8e538524 100644 --- a/library/src/ipfs_client/ipld/unixfs_file.cc +++ b/library/src/ipfs_client/ipld/unixfs_file.cc @@ -1,5 +1,7 @@ #include "unixfs_file.h" +#include "log_macros.h" + using namespace std::literals; using Self = ipfs::ipld::UnixfsFile; @@ -8,7 +10,8 @@ auto Self::resolve(ipfs::SlashDelimited path, ipfs::ipld::DagNode::BlockLookup blu, std::string& to_here) -> ResolveResult { if (path) { - // You can't path through a file. + LOG(ERROR) << "Can't path through a file, (at " << to_here + << ") but given the path " << path.to_string(); return ProvenAbsent{}; } std::vector missing; @@ -30,10 +33,14 @@ auto Self::resolve(ipfs::SlashDelimited path, body.append(std::get(recurse).body_); } } else { + LOG(INFO) << "In order to resolve the file at path " << to_here + << " I need CID " << link.cid; missing.push_back("/ipfs/" + link.cid); } } if (missing.empty()) { + LOG(INFO) << "Assembled file (or file part) of " << body.size() + << " bytes from multiple nodes."; return Response{ "", 200, diff --git a/library/src/ipfs_client/ipns_record.cc b/library/src/ipfs_client/ipns_record.cc index 73598b6b..b1ca813d 100644 --- a/library/src/ipfs_client/ipns_record.cc +++ b/library/src/ipfs_client/ipns_record.cc @@ -1,5 +1,7 @@ #include +#include + #include "log_macros.h" #if __has_include() @@ -8,8 +10,8 @@ #include "ipfs_client/ipns_record.pb.h" #endif -// #include #include +#include #include namespace { @@ -28,16 +30,29 @@ bool matches(libp2p::multi::Multihash const& hash, hash.getHash().end()); } } // namespace + +auto ipfs::ValidateIpnsRecord(ipfs::ByteView top_level_bytes, + libp2p::multi::ContentIdentifier const& name, + ContextApi& api) -> std::optional { + DCHECK_EQ(name.content_type, libp2p::multi::MulticodecType::Code::LIBP2P_KEY); + if (name.content_type != libp2p::multi::MulticodecType::Code::LIBP2P_KEY) { + return {}; + } + auto as_peer = libp2p::peer::PeerId::fromHash(name.content_address); + if (as_peer.has_value()) { + return ValidateIpnsRecord(top_level_bytes, as_peer.value(), api); + } else { + return {}; + } +} auto ipfs::ValidateIpnsRecord(ByteView top_level_bytes, libp2p::peer::PeerId const& name, - CryptoSignatureVerifier verify, - CborDeserializer dser) - -> std::optional { + ContextApi& api) -> std::optional { // https://github.com/ipfs/specs/blob/main/ipns/IPNS.md#record-verification // Before parsing the protobuf, confirm that the serialized IpnsEntry bytes // sum to less than or equal to the size limit. - if (top_level_bytes.size() > 10240UL) { + if (top_level_bytes.size() > MAX_IPNS_PB_SERIALIZED_SIZE) { LOG(ERROR) << "IPNS record too large: " << top_level_bytes.size(); return {}; } @@ -63,9 +78,10 @@ auto ipfs::ValidateIpnsRecord(ByteView top_level_bytes, // the expiration date after which the IPNS record becomes invalid. DCHECK_EQ(entry.validitytype(), 0); - auto parsed = dser({reinterpret_cast(entry.data().data()), - entry.data().size()}); - if (parsed.value != entry.value()) { + auto parsed = + api.deserialize_cbor({reinterpret_cast(entry.data().data()), + entry.data().size()}); + if (entry.has_value() && parsed.value != entry.value()) { LOG(ERROR) << "CBOR contains value '" << parsed.value << "' but PB contains value '" << entry.value() << "'!"; return {}; @@ -85,7 +101,7 @@ auto ipfs::ValidateIpnsRecord(ByteView top_level_bytes, public_key = name.toMultihash().getHash(); } else { LOG(ERROR) << "IPNS record contains no public key, and the IPNS name " - << name.toMultihash().toHex() + << name.toBase58() << " is a true hash, not identity. Validation impossible."; return {}; } @@ -101,8 +117,8 @@ auto ipfs::ValidateIpnsRecord(ByteView top_level_bytes, ByteView signature{reinterpret_cast(signature_str.data()), signature_str.size()}; // https://specs.ipfs.tech/ipns/ipns-record/#record-verification - // Create bytes for signature verification by concatenating ipns-signature: - // prefix (bytes in hex: 69706e732d7369676e61747572653a) with raw CBOR bytes + // Create bytes for signature verification by concatenating + // ipto_hex(ns-signature:// prefix (bytes in hex: 69706e732d7369676e61747572653a) with raw CBOR bytes // from IpnsEntry.data auto bytes_str = entry.data(); bytes_str.insert( @@ -111,13 +127,47 @@ auto ipfs::ValidateIpnsRecord(ByteView top_level_bytes, bytes_str.size()}; ByteView key_bytes{reinterpret_cast(pk.data().data()), pk.data().size()}; - if (verify(pk.type(), signature, bytes, key_bytes)) { - LOG(INFO) << "Verification passed."; - return parsed; - } else { + if (!api.verify_key_signature(static_cast(pk.type()), + signature, bytes, key_bytes)) { LOG(ERROR) << "Verification failed!!"; return {}; } + // TODO check expiration date + LOG(INFO) << "V2 verification passed."; + // IpnsEntry.value must match IpnsEntry.data[Value] + if (entry.has_value() && entry.value() != parsed.value) { + LOG(ERROR) << "IPNS " << name.toBase58() << " has different values for V1(" + << entry.value() << ") and V2(" << parsed.value << ')'; + return {}; + } + if (entry.has_validity() && entry.validity() != parsed.validity) { + LOG(ERROR) << "IPNS " << name.toBase58() + << " has different validity for V1(" << entry.validity() + << ") and V2(" << parsed.validity << ')'; + return {}; + } + if (entry.has_validitytype() && + entry.validitytype() != static_cast(parsed.validityType)) { + LOG(ERROR) << "IPNS " << name.toBase58() + << " has different validity types for V1(" + << entry.validitytype() << ") and V2(" << parsed.validityType + << ')'; + return {}; + } + if (entry.has_sequence() && entry.sequence() != parsed.sequence) { + LOG(ERROR) << "IPNS " << name.toBase58() + << " has different validity types for V1(" << entry.sequence() + << ") and V2(" << parsed.sequence << ')'; + return {}; + } + if (entry.has_ttl() && entry.ttl() != parsed.ttl) { + LOG(ERROR) << "IPNS " << name.toBase58() + << " has different validity types for V1(" << entry.ttl() + << ") and V2(" << parsed.ttl << ')'; + return {}; + } + LOG(INFO) << "V1 verification also passes for " << name.toBase58(); + return parsed; } ipfs::ValidatedIpns::ValidatedIpns() = default; @@ -128,7 +178,7 @@ auto ipfs::ValidatedIpns::operator=(ValidatedIpns const&) ipfs::ValidatedIpns::ValidatedIpns(IpnsCborEntry const& e) : value{e.value}, sequence{e.sequence} { std::istringstream ss{e.validity}; - std::tm t; + std::tm t = {}; ss >> std::get_time(&t, "%Y-%m-%dT%H:%M:%S"); long ttl = (e.ttl / 1'000'000'000UL) + 1; use_until = std::mktime(&t); diff --git a/library/src/ipfs_client/ipns_record_unittest.cc b/library/src/ipfs_client/ipns_record_unittest.cc index ae3b54f3..3cec0553 100644 --- a/library/src/ipfs_client/ipns_record_unittest.cc +++ b/library/src/ipfs_client/ipns_record_unittest.cc @@ -1,12 +1,72 @@ #include +#include +#include #include -#include #include #include +namespace { +struct Api final : public ipfs::ContextApi { + void SendHttpRequest(ipfs::HttpRequestDescription, + HttpCompleteCallback cb) const { + throw 1; + } + void SendDnsTextRequest(std::string hostname, + DnsTextResultsCallback, + DnsTextCompleteCallback) { + throw 1; + } + std::string MimeType(std::string extension, + std::string_view content, + std::string const& url) const { + throw 1; + } + bool verify_key_signature(SigningKeyType, + ByteView signature, + ByteView data, + ByteView key_bytes) const { + return true; + } + std::string UnescapeUrlComponent(std::string_view url_comp) const { throw 1; } + ipfs::IpnsCborEntry deserialize_cbor(ipfs::ByteView b) const { + return dser_(b); + } + bool verify_key_signature(ipfs::SigningKeyType, + ipfs::ByteView signature, + ipfs::ByteView data, + ipfs::ByteView key_bytes) { + return true; + } + void Discover(std::function)> cb) { throw 1; } + std::shared_ptr InitiateGatewayRequest( + ipfs::BusyGateway) { + throw 1; + } + std::function dser_; + Api(std::function f = {}) { + if (f) { + dser_ = f; + } else { + dser_ = [](auto) { + ipfs::IpnsCborEntry e; + e.value = + "/ipfs/bafybeig57t2dp435aupttilimd6767kppfebaa3gnunmqden66dgkhugwi"; + e.validity = "2023-03-24T05:10:02.161162321Z"; + e.validityType = 0; + e.sequence = 384; + e.ttl = 60000000000; + return e; + }; + } + } + virtual ~Api() noexcept {} +}; +} // namespace + TEST(IpnsRecordTest, AKnownKuboRecord) { + Api api; std::array xxd{ 0x0a, 0x41, 0x2f, 0x69, 0x70, 0x66, 0x73, 0x2f, 0x62, 0x61, 0x66, 0x79, 0x62, 0x65, 0x69, 0x67, 0x35, 0x37, 0x74, 0x32, 0x64, 0x70, 0x34, 0x33, @@ -54,19 +114,9 @@ TEST(IpnsRecordTest, AKnownKuboRecord) { EXPECT_EQ(ci.content_address.getType(), libp2p::multi::HashType::identity); auto hash = ci.content_address.getHash(); auto my_name_res = libp2p::peer::PeerId::fromHash(ci.content_address); - - // auto my_name_res = - // libp2p::peer::PeerId::fromBase58("12D3KooWGDMwwqrpcYKpKCgxuKT2NfqPqa94QnkoBBpqvCaiCzWd"); ASSERT_TRUE(my_name_res.has_value()); auto& my_name = my_name_res.value(); - auto result = ipfs::ValidateIpnsRecord( - known_record, my_name, [](auto, auto, auto, auto) { return true; }, - [](auto) { - ipfs::IpnsCborEntry e; - e.value = - "/ipfs/bafybeig57t2dp435aupttilimd6767kppfebaa3gnunmqden66dgkhugwi"; - return e; - }); + auto result = ipfs::ValidateIpnsRecord(known_record, my_name, api); std::string_view expected{ "/ipfs/bafybeig57t2dp435aupttilimd6767kppfebaa3gnunmqden66dgkhugwi"}; EXPECT_EQ(result.has_value(), true); @@ -84,11 +134,205 @@ TEST(IpnsRecordTest, SerializeValidatedIpns) { auto actual = v.Serialize(); auto expected = "1 2 3 4 5 value gateway_source"; EXPECT_EQ(actual, expected); + auto w = ipfs::ValidatedIpns::Deserialize(actual); + EXPECT_EQ(w.sequence, v.sequence); + EXPECT_EQ(w.use_until, v.use_until); + EXPECT_EQ(w.cache_until, v.cache_until); + EXPECT_EQ(w.fetch_time, v.fetch_time); + EXPECT_EQ(w.resolution_ms, v.resolution_ms); + EXPECT_EQ(w.gateway_source, v.gateway_source); + EXPECT_EQ(w.value, v.value); } TEST(IpnsRecordTest, TooBig) { ipfs::Byte* p; auto peer = libp2p::peer::PeerId::fromBase58( "12D3KooWGDMwwqrpcYKpKCgxuKT2NfqPqa94QnkoBBpqvCaiCzWd"); - auto actual = ipfs::ValidateIpnsRecord({p, 12345}, peer.value(), {}, {}); + Api api; + auto actual = ipfs::ValidateIpnsRecord({p, 12345}, peer.value(), api); EXPECT_FALSE(actual.has_value()); } +TEST(IpnsRecordTest, copyctorcopiesfields) { + ipfs::ValidatedIpns a; + a.value = "val"; + a.gateway_source = "gw"; + a.sequence = 0; + a.resolution_ms = 1; + a.fetch_time = 2; + a.cache_until = 3; + a.use_until = 4; + auto b = a; + EXPECT_EQ(b.value, "val"); + EXPECT_EQ(b.gateway_source, "gw"); + auto i = 0; + EXPECT_EQ(b.sequence, i++); + EXPECT_EQ(b.resolution_ms, i++); + EXPECT_EQ(b.fetch_time, i++); + EXPECT_EQ(b.cache_until, i++); + EXPECT_EQ(b.use_until, i++); +} +TEST(IpnsRecordTest, fromcborentry) { + auto e = ipfs::IpnsCborEntry{"v", "2023-11-05T15:06:39.790199045Z", 0, 1, 99}; + auto now = std::time(nullptr); + ipfs::ValidatedIpns v = e; + EXPECT_EQ(v.value, "v"); + EXPECT_EQ(v.sequence, 1); + EXPECT_EQ(v.use_until, v.cache_until); + auto time_left = v.use_until - now; + EXPECT_GE(time_left, 1); + EXPECT_LE(time_left, 3); +} + +#if __has_include() +#include +using j = nlohmann::json; +TEST(IpnsRecordTest, V2_Only_Lean) { + std::array from_spec{ + 0x42, 0x40, 0x6a, 0x76, 0x98, 0x58, 0x5e, 0x2b, 0x17, 0x07, 0x09, 0xd4, + 0x94, 0x7e, 0x14, 0x8b, 0x18, 0xb1, 0x20, 0xd7, 0x6d, 0xb4, 0x35, 0x15, + 0xed, 0xac, 0xf2, 0x90, 0xdf, 0x96, 0xb7, 0x1e, 0x29, 0xc0, 0xca, 0x5a, + 0x16, 0x51, 0x02, 0x56, 0x81, 0x4f, 0x82, 0x55, 0x53, 0x2e, 0xc4, 0x54, + 0x9a, 0xde, 0x26, 0xf4, 0xca, 0x83, 0x35, 0xf4, 0xc4, 0x1e, 0x2c, 0xff, + 0xfe, 0xb1, 0xba, 0x0a, 0xa5, 0x01, 0x4a, 0x78, 0xa5, 0x63, 0x54, 0x54, + 0x4c, 0x1b, 0x00, 0x00, 0x01, 0xa3, 0x18, 0x5c, 0x50, 0x00, 0x65, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x58, 0x24, 0x2f, 0x69, 0x70, 0x66, 0x73, 0x2f, + 0x62, 0x61, 0x66, 0x6b, 0x71, 0x61, 0x64, 0x74, 0x77, 0x67, 0x69, 0x77, + 0x77, 0x36, 0x33, 0x74, 0x6d, 0x70, 0x65, 0x71, 0x68, 0x65, 0x7a, 0x6c, + 0x64, 0x6e, 0x35, 0x7a, 0x67, 0x69, 0x68, 0x53, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x65, 0x00, 0x68, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x69, 0x74, + 0x79, 0x58, 0x1b, 0x32, 0x31, 0x32, 0x33, 0x2d, 0x30, 0x38, 0x2d, 0x31, + 0x34, 0x54, 0x31, 0x32, 0x3a, 0x31, 0x37, 0x3a, 0x30, 0x33, 0x2e, 0x36, + 0x39, 0x34, 0x30, 0x35, 0x32, 0x5a, 0x6c, 0x56, 0x61, 0x6c, 0x69, 0x64, + 0x69, 0x74, 0x79, 0x54, 0x79, 0x70, 0x65, 0x00}; + auto* p = reinterpret_cast(from_spec.data()); + ipfs::ByteView as_seen_in_spec{p, from_spec.size()}; + // k51qzi5uqu5dm4tm0wt8srkg9h9suud4wuiwjimndrkydqm81cqtlb5ak6p7ku + auto peer = libp2p::peer::PeerId::fromBase58( + "12D3KooWRtQ4MBxXXzioRZHs7NeGuyNHzJiN8X15wxydH4cnGDYZ"); + ASSERT_TRUE(peer.has_value()); + Api api{[](ipfs::ByteView c) { + auto v = j::from_cbor(c); + // std::cout << std::setw(2) << v << std::endl; + ipfs::IpnsCborEntry e; + e.sequence = v.at("Sequence"); + auto& bin_val = v.at("Value").get_binary(); + e.value.assign(bin_val.begin(), bin_val.end()); + return e; + }}; + auto actual = ipfs::ValidateIpnsRecord(as_seen_in_spec, peer.value(), api); + EXPECT_TRUE(actual); +} +TEST(IpnsRecordTest, has_pubkey_field) { + std::array dump_from_cli{ + 0x0a, 0x34, 0x2f, 0x69, 0x70, 0x66, 0x73, 0x2f, 0x51, 0x6d, 0x56, 0x73, + 0x39, 0x58, 0x63, 0x48, 0x44, 0x7a, 0x58, 0x65, 0x48, 0x6d, 0x7a, 0x73, + 0x6d, 0x48, 0x47, 0x37, 0x33, 0x54, 0x54, 0x67, 0x6f, 0x66, 0x53, 0x71, + 0x42, 0x51, 0x70, 0x53, 0x54, 0x52, 0x52, 0x48, 0x33, 0x36, 0x44, 0x69, + 0x68, 0x67, 0x62, 0x59, 0x66, 0x34, 0x12, 0x80, 0x02, 0x4f, 0x8f, 0xd2, + 0x62, 0x29, 0x25, 0x01, 0x91, 0x04, 0xc9, 0xde, 0xb3, 0x50, 0x40, 0x78, + 0xc2, 0x7e, 0x62, 0xa8, 0xce, 0x4f, 0x21, 0xb7, 0x8d, 0xdf, 0x7e, 0x0c, + 0xda, 0xef, 0x13, 0xb3, 0x64, 0xa6, 0x5c, 0x32, 0xf3, 0x0a, 0x58, 0x44, + 0x48, 0x54, 0xde, 0xd5, 0x0a, 0x6a, 0x3f, 0xf1, 0xc1, 0xbf, 0x2d, 0xe8, + 0x90, 0x37, 0xbe, 0xd3, 0x43, 0xe0, 0x56, 0xcf, 0x99, 0x45, 0x2a, 0xaa, + 0x64, 0x3c, 0x34, 0xed, 0xf5, 0x36, 0x3f, 0x30, 0xee, 0xdb, 0xd7, 0xfd, + 0xc6, 0x31, 0x08, 0x6b, 0x9c, 0x62, 0x18, 0xa3, 0xd2, 0x61, 0x60, 0xe5, + 0x3c, 0xfd, 0xdc, 0x2e, 0xb9, 0x61, 0xe8, 0xa0, 0x0e, 0xa6, 0xc9, 0x73, + 0x3d, 0x68, 0xf6, 0xdf, 0x6b, 0xee, 0x4f, 0xed, 0x4f, 0x69, 0xce, 0x0f, + 0x52, 0x29, 0x97, 0x75, 0x04, 0x27, 0xf1, 0x5d, 0x06, 0x5c, 0x6d, 0x09, + 0xaa, 0xde, 0xea, 0xd9, 0x01, 0x87, 0x31, 0x3f, 0x84, 0xad, 0x7f, 0xb6, + 0x5b, 0xb3, 0x02, 0x5d, 0x49, 0x5a, 0x2e, 0x52, 0x11, 0xd4, 0x51, 0x8c, + 0x66, 0xa5, 0x41, 0xba, 0xc3, 0x71, 0x0d, 0x9e, 0x67, 0xd5, 0xa9, 0x53, + 0x07, 0x72, 0xb9, 0x95, 0xce, 0xe7, 0x72, 0x1a, 0xa6, 0xb5, 0xd4, 0xc9, + 0x23, 0x88, 0xb8, 0x47, 0x08, 0xb4, 0x7b, 0xb5, 0xfb, 0x3d, 0x59, 0xba, + 0x0c, 0x30, 0x06, 0xa8, 0xef, 0xd8, 0x14, 0x2f, 0x34, 0x0f, 0x94, 0x7b, + 0xb0, 0x82, 0x12, 0xa5, 0x1f, 0x7a, 0x3e, 0xdc, 0x58, 0x2c, 0xf4, 0x81, + 0x66, 0x2a, 0x51, 0x62, 0x44, 0x7a, 0xab, 0x5c, 0x8b, 0x91, 0x7e, 0x97, + 0xd6, 0x6a, 0xa4, 0xb8, 0x45, 0xbd, 0xe1, 0x1e, 0xba, 0xec, 0x1f, 0x90, + 0xf0, 0xc2, 0x47, 0xc7, 0xce, 0xd3, 0xfc, 0x5a, 0x55, 0xb1, 0xc9, 0x0f, + 0x7c, 0xd8, 0x86, 0x7d, 0xdb, 0x64, 0x21, 0x28, 0x81, 0x7f, 0x52, 0x09, + 0x40, 0x18, 0x00, 0x22, 0x1e, 0x32, 0x30, 0x32, 0x33, 0x2d, 0x31, 0x31, + 0x2d, 0x30, 0x37, 0x54, 0x31, 0x35, 0x3a, 0x30, 0x36, 0x3a, 0x33, 0x39, + 0x2e, 0x37, 0x39, 0x30, 0x31, 0x39, 0x39, 0x30, 0x34, 0x35, 0x5a, 0x28, + 0x00, 0x30, 0x80, 0xb0, 0x9d, 0xc2, 0xdf, 0x01, 0x3a, 0xab, 0x02, 0x08, + 0x00, 0x12, 0xa6, 0x02, 0x30, 0x82, 0x01, 0x22, 0x30, 0x0d, 0x06, 0x09, + 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, + 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, 0x01, 0x01, + 0x00, 0xb0, 0xe4, 0x02, 0xef, 0x68, 0xd9, 0x27, 0xe6, 0xe8, 0x66, 0xeb, + 0xd8, 0xcd, 0x24, 0xc5, 0x44, 0x58, 0x4c, 0x57, 0xac, 0x59, 0x4c, 0xe6, + 0x7a, 0x8a, 0x86, 0xc5, 0xb6, 0xef, 0x8d, 0x86, 0xbf, 0x8e, 0xa0, 0x49, + 0xd8, 0x20, 0x4c, 0x60, 0xf7, 0x88, 0xed, 0x26, 0xff, 0x01, 0xec, 0x8e, + 0x4b, 0x59, 0xa0, 0x6f, 0xdd, 0x65, 0x2b, 0x03, 0xbf, 0xae, 0x0b, 0xbe, + 0x12, 0xb9, 0xea, 0x83, 0x49, 0x0f, 0x6b, 0xb3, 0x1a, 0x5c, 0x19, 0x02, + 0x7a, 0x16, 0x9d, 0x2d, 0x23, 0x65, 0xb8, 0x01, 0xc1, 0x8b, 0x57, 0x14, + 0x55, 0xbe, 0xa6, 0x74, 0x4c, 0x78, 0xb5, 0xde, 0x92, 0x6c, 0x23, 0x7f, + 0x7e, 0xf0, 0xd8, 0xea, 0xf9, 0x0a, 0x5b, 0xbd, 0xc2, 0xca, 0xb6, 0x1c, + 0x08, 0xe6, 0xc5, 0x00, 0x9c, 0xc6, 0xbd, 0x14, 0x9e, 0x50, 0xd3, 0x43, + 0xb0, 0xb2, 0xf9, 0xb9, 0xd8, 0x45, 0xf9, 0x25, 0xbc, 0x62, 0x54, 0x13, + 0xa2, 0x6e, 0xb2, 0xbb, 0x76, 0xfa, 0x68, 0xcb, 0xe2, 0x2f, 0xb8, 0xd4, + 0x31, 0x99, 0xfc, 0x44, 0xa2, 0xcd, 0xea, 0xca, 0x30, 0xcb, 0xf6, 0x97, + 0xc5, 0x35, 0x16, 0xaa, 0xbc, 0x12, 0xb9, 0x84, 0xc6, 0x5a, 0x37, 0xb7, + 0x5f, 0x0b, 0x8e, 0x48, 0xf9, 0x15, 0xe4, 0xb6, 0xa5, 0x73, 0xce, 0xd9, + 0xa7, 0x9f, 0x7a, 0xff, 0xd8, 0x07, 0x86, 0x1b, 0xa3, 0xb7, 0xf2, 0xbc, + 0xdc, 0xad, 0xf3, 0x21, 0x7d, 0xcb, 0x4c, 0x08, 0xa2, 0xec, 0x68, 0x37, + 0xe1, 0xf9, 0xff, 0x2d, 0x6e, 0x7c, 0x19, 0xbd, 0xc1, 0x96, 0x46, 0xae, + 0xfe, 0x69, 0x00, 0xd1, 0x20, 0x8d, 0xe4, 0xca, 0x7c, 0x36, 0x80, 0xf3, + 0xfb, 0x05, 0xe4, 0x5e, 0xd0, 0xd7, 0x1e, 0x7e, 0x56, 0x69, 0x25, 0x81, + 0xa0, 0x2f, 0x78, 0x71, 0x9c, 0x85, 0x59, 0x42, 0xe2, 0x9b, 0x5a, 0xe2, + 0x3c, 0x48, 0x75, 0x8d, 0x91, 0x02, 0x03, 0x01, 0x00, 0x01, 0x42, 0x80, + 0x02, 0x3e, 0x34, 0x34, 0x9b, 0xa1, 0x0f, 0xb7, 0x7f, 0xbb, 0xbe, 0x88, + 0x81, 0x43, 0x96, 0x25, 0x13, 0x9b, 0x0a, 0x91, 0x56, 0xa9, 0xce, 0x0c, + 0xb6, 0x20, 0x78, 0xb6, 0xf8, 0x5e, 0xb3, 0xd8, 0x03, 0xab, 0x06, 0x2f, + 0xfe, 0x78, 0x07, 0xbf, 0x75, 0x23, 0x2a, 0x2b, 0xaf, 0x23, 0x7a, 0xd4, + 0xec, 0xba, 0x9d, 0xa6, 0x71, 0x7d, 0x90, 0xbb, 0x4c, 0x6e, 0xc0, 0x13, + 0x38, 0xc8, 0x0d, 0x0d, 0x89, 0x71, 0xab, 0xb3, 0x5c, 0x16, 0x94, 0x6a, + 0xea, 0xda, 0xc1, 0x0b, 0xa6, 0x6d, 0x5b, 0x41, 0x7a, 0xf4, 0x7e, 0xaf, + 0x0f, 0xe9, 0xfc, 0x40, 0x7f, 0xff, 0xd6, 0x7d, 0x15, 0xef, 0x8f, 0x9f, + 0xaa, 0xe2, 0x31, 0x3a, 0x49, 0x24, 0x39, 0xc8, 0xdb, 0x8e, 0xfb, 0xcc, + 0x3e, 0xb2, 0x0d, 0x7e, 0xd0, 0x84, 0xdc, 0x98, 0x08, 0xe8, 0xa3, 0x76, + 0x44, 0xb6, 0xe0, 0xee, 0xe2, 0xde, 0xde, 0x07, 0xe6, 0x7a, 0xf0, 0xee, + 0xda, 0x00, 0x77, 0xae, 0x96, 0x7c, 0xfb, 0xdb, 0x0a, 0xa0, 0x8b, 0xdd, + 0xb1, 0x0b, 0x48, 0x56, 0xcf, 0x35, 0x9c, 0xfc, 0x21, 0x7e, 0x73, 0xc3, + 0xa6, 0x41, 0x8d, 0x5f, 0xe7, 0x39, 0xe9, 0x0d, 0x65, 0xe8, 0x06, 0xcf, + 0xb4, 0xce, 0x3c, 0xe1, 0x5e, 0xa0, 0xa1, 0x66, 0xd0, 0xc8, 0xd9, 0x27, + 0x48, 0x46, 0x40, 0x09, 0xbf, 0x06, 0xae, 0x06, 0x74, 0x61, 0x3a, 0x8b, + 0x6e, 0xaf, 0xdf, 0x15, 0xb7, 0x61, 0xc0, 0xb6, 0x7d, 0x04, 0xff, 0x79, + 0x69, 0x2f, 0xf6, 0xe3, 0x1b, 0xde, 0x2c, 0xb8, 0x4b, 0x6c, 0x58, 0x73, + 0x07, 0x65, 0xed, 0x2c, 0xe8, 0x38, 0x50, 0x67, 0x87, 0x2b, 0x2a, 0x36, + 0x01, 0x64, 0x74, 0x55, 0xe8, 0xf4, 0xad, 0x8f, 0x7d, 0x49, 0xcb, 0xf1, + 0x8f, 0x66, 0xd3, 0xc1, 0xf3, 0xad, 0xb9, 0x67, 0xb1, 0x15, 0x89, 0x42, + 0x78, 0x61, 0x4b, 0xfe, 0x85, 0x4a, 0x8b, 0x01, 0xa5, 0x63, 0x54, 0x54, + 0x4c, 0x1b, 0x00, 0x00, 0x00, 0x0d, 0xf8, 0x47, 0x58, 0x00, 0x65, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x58, 0x34, 0x2f, 0x69, 0x70, 0x66, 0x73, 0x2f, + 0x51, 0x6d, 0x56, 0x73, 0x39, 0x58, 0x63, 0x48, 0x44, 0x7a, 0x58, 0x65, + 0x48, 0x6d, 0x7a, 0x73, 0x6d, 0x48, 0x47, 0x37, 0x33, 0x54, 0x54, 0x67, + 0x6f, 0x66, 0x53, 0x71, 0x42, 0x51, 0x70, 0x53, 0x54, 0x52, 0x52, 0x48, + 0x33, 0x36, 0x44, 0x69, 0x68, 0x67, 0x62, 0x59, 0x66, 0x34, 0x68, 0x53, + 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x00, 0x68, 0x56, 0x61, 0x6c, + 0x69, 0x64, 0x69, 0x74, 0x79, 0x58, 0x1e, 0x32, 0x30, 0x32, 0x33, 0x2d, + 0x31, 0x31, 0x2d, 0x30, 0x37, 0x54, 0x31, 0x35, 0x3a, 0x30, 0x36, 0x3a, + 0x33, 0x39, 0x2e, 0x37, 0x39, 0x30, 0x31, 0x39, 0x39, 0x30, 0x34, 0x35, + 0x5a, 0x6c, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x69, 0x74, 0x79, 0x54, 0x79, + 0x70, 0x65, 0x00 + + }; + auto* p = reinterpret_cast(dump_from_cli.data()); + ipfs::ByteView as_seen_in_spec{p, dump_from_cli.size()}; + auto c = libp2p::multi::ContentIdentifierCodec::fromString( + "k2k4r8p8axjle4tulaj8f6423nuwnyjswl3iq7ppmqci5efgn7vg6ah4"); + ASSERT_TRUE(c.has_value()); + Api api{[](ipfs::ByteView c) { + auto v = j::from_cbor(c); + // std::cout << std::setw(2) << v << std::endl; + ipfs::IpnsCborEntry e; + e.sequence = v.at("Sequence"); + auto& bin_val = v.at("Value").get_binary(); + e.value.assign(bin_val.begin(), bin_val.end()); + auto& bin_vdty = v.at("Validity").get_binary(); + e.validity.assign(bin_vdty.begin(), bin_vdty.end()); + e.ttl = v.at("TTL"); + e.validityType = v.at("ValidityType"); + return e; + }}; + auto actual = ipfs::ValidateIpnsRecord(as_seen_in_spec, c.value(), api); + EXPECT_TRUE(actual); +} +#endif diff --git a/library/src/ipfs_client/orchestrator.cc b/library/src/ipfs_client/orchestrator.cc index de6d2e6a..38092603 100644 --- a/library/src/ipfs_client/orchestrator.cc +++ b/library/src/ipfs_client/orchestrator.cc @@ -1,9 +1,9 @@ #include "ipfs_client/orchestrator.h" +#include +#include #include -#include "ipld/chunk.h" - #include "log_macros.h" #include "path2url.h" @@ -11,26 +11,36 @@ using namespace std::literals; using Self = ipfs::Orchestrator; -Self::Orchestrator(GatewayAccess ga, MimeDetection mimer) - : gw_requestor_{ga}, mimer_{mimer} {} +Self::Orchestrator(GatewayAccess ga, + std::shared_ptr requestor, + std::shared_ptr api) + : gw_requestor_{ga}, api_{api}, requestor_{requestor} { + DCHECK(requestor); +} void Self::build_response(std::shared_ptr req) { + if (!req->ready_after()) { + return; + } auto req_path = req->path(); LOG(INFO) << "build_response(" << req_path.to_string() << ')'; req_path.pop(); // namespace - std::string origin{req_path.pop()}; - auto it = dags_.find(origin); + std::string affinity{req_path.pop()}; + auto it = dags_.find(affinity); if (dags_.end() == it) { - if (gw_request(req, req->path())) { + if (gw_request(req, req->path(), affinity)) { build_response(req); } } else { - from_tree(req, it->second, req_path); + LOG(INFO) << "Requesting root " << affinity << " resolve path " + << req_path.to_string(); + from_tree(req, it->second, req_path, affinity); } } void Self::from_tree(std::shared_ptr req, ipfs::ipld::NodePtr& node, - SlashDelimited relative_path) { + SlashDelimited relative_path, + std::string const& affinity) { auto root = node->rooted(); auto block_look_up = [this](auto& k) { auto i = dags_.find(k); @@ -52,57 +62,63 @@ void Self::from_tree(std::shared_ptr req, response->mime_ = sniff(SlashDelimited{hit_path}, response->body_); } } - // TODO is this necessary? It might be convenient to have this info - // available, even if it's usually redundant. - // if (response->status_ / 100 != 3) { - // response->location_.clear(); - // } req->finish(*response); } else if (std::get_if(&result)) { req->finish(Response::PLAIN_NOT_FOUND); } else { - for (auto& path : std::get(result).ipfs_abs_paths_) { - if (gw_request(req, std::string_view{path})) { - from_tree(req, node, relative_path); - return; + auto& missing_paths = + std::get(result).ipfs_abs_paths_; + req->till_next(missing_paths.size()); + bool repeat = false; + for (auto& path : missing_paths) { + if (gw_request(req, std::string_view{path}, affinity)) { + repeat = true; } } + if (repeat) { + from_tree(req, node, relative_path, affinity); + } } } bool Self::gw_request(std::shared_ptr ir, - ipfs::SlashDelimited path) { + ipfs::SlashDelimited path, + std::string const& aff) { auto req = gw::GatewayRequest::fromIpfsPath(path); - if (req->type == gw::Type::Identity) { - auto node = - std::make_shared(std::string{req->identity_data()}); - add_node(req->main_param, node); - return true; - } req->dependent = ir; req->orchestrator = shared_from_this(); - gw_requestor_(req); - if (req->type == gw::Type::Car) { - gw_request(ir, path.pop_n(2)); - } + req->affinity = aff; + // gw_requestor_(req); + requestor_->request(req); return false; } -void Self::add_node(std::string key, ipfs::ipld::NodePtr p) { +bool Self::add_node(std::string key, ipfs::ipld::NodePtr p) { if (p) { LOG(INFO) << "add_node(" << key << ')'; - dags_[key] = p; + if (dags_.insert({key, p}).second) { + p->set_api(api_); + return true; + } } else { LOG(ERROR) << "NULL block attempted to be added for " << key; } + return false; } std::string Self::sniff(ipfs::SlashDelimited p, std::string const& body) const { auto fake_url = path2url(p.to_string()); auto file_name = p.peek_back(); auto dot = file_name.find_last_of('.'); - std::string_view ext = ""; + std::string ext = ""; if (dot < file_name.size()) { - ext = file_name.substr(dot + 1); + ext.assign(file_name, dot + 1); } - return mimer_(std::string{ext}, body, fake_url); + auto result = api_->MimeType(ext, body, fake_url); + LOG(INFO) << "Deduced mime from (ext=" << ext << " body of " << body.size() + << " bytes, 'url'=" << fake_url << ")=" << result; + return result; } + +bool Self::has_key(std::string const& k) const { + return dags_.count(k); +} \ No newline at end of file diff --git a/library/src/ipfs_client/orchestrator_unittest.cc b/library/src/ipfs_client/orchestrator_unittest.cc index 5e023741..0481e8ec 100644 --- a/library/src/ipfs_client/orchestrator_unittest.cc +++ b/library/src/ipfs_client/orchestrator_unittest.cc @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -15,47 +16,59 @@ namespace i = ipfs; namespace ii = i::ipld; +namespace ig = i::gw; using namespace std::literals; using Success = i::Response; using Codec = libp2p::multi::ContentIdentifierCodec; namespace { -struct OrchestratingRealData : public ::testing::Test { - // std::shared_ptr orc_ = std::make_shared( - // [this](auto r) { load_test_data(r->main_param); }); - std::shared_ptr orc_; - OrchestratingRealData() { - auto f = [this](auto r) { load_test_data(r->main_param, *r); }; - auto m = [](auto e, auto c, auto& u) { - auto result = "Mime from extension=" + e + " 'url'=" + u + " content='"; - if (c.size() < 9) { - result.append(c); - } else { - result.append(c.substr(0, 8)).append("..."); - } - return result + "'"; - }; - resp_.body_ = "No response received."; - resp_.status_ = 0; - resp_.mime_ = "uninit - no mime provided"; - resp_.location_ = "uninit"; - orc_ = std::make_shared(f, m); +struct FakeApi final : public ipfs::ContextApi { + bool verify_key_signature(ipfs::SigningKeyType, + ipfs::ByteView signature, + ipfs::ByteView data, + ipfs::ByteView key_bytes) const { + return false; } - i::Response resp_; - void dorequest(std::string_view ipfs_path) { - std::cout << "dorequest(" << ipfs_path << ")\n"; - auto f = [this](auto&, auto& r) { resp_ = r; }; - auto top_req = std::make_shared(std::string{ipfs_path}, f); - orc_->build_response(top_req); + std::string MimeType(std::string e, + std::string_view c, + std::string const& u) const { + auto result = "Mime from extension=" + e + " 'url'=" + u + " content='"; + if (c.size() < 9) { + result.append(c); + } else { + result.append(c.substr(0, 8)).append("..."); + } + return result + "'"; } - - void load_test_data(std::string cid, i::gw::GatewayRequest r) { + std::string UnescapeUrlComponent(std::string_view url_comp) { + return std::string{url_comp}; + } + std::string UnescapeUrlComponent(std::string_view u) const { + return std::string{u}; + } + ipfs::IpnsCborEntry deserialize_cbor(ipfs::ByteView) const { return {}; } + std::shared_ptr InitiateGatewayRequest( + ipfs::BusyGateway) { + return nullptr; + } + void Discover(std::function)> cb) {} + void SendDnsTextRequest(std::string, + DnsTextResultsCallback, + DnsTextCompleteCallback) {} + void SendHttpRequest(ipfs::HttpRequestDescription, + HttpCompleteCallback) const {} +}; +struct TestRequestor final : public ig::Requestor { + std::string_view name() const { return "return test requestor"; } + HandleOutcome handle(ig::RequestPtr r) { + auto cid = r->main_param; + auto orc_ = r->orchestrator; auto base_dir = std::filesystem::path{__FILE__}; while (!is_directory(base_dir / "test_data" / "blocks") && base_dir.generic_string().size() > 2) { base_dir = base_dir.parent_path(); } - switch (r.type) { + switch (r->type) { case i::gw::Type::Ipns: { auto dir = base_dir / "test_data" / "names"; auto f = dir / cid; @@ -75,6 +88,7 @@ struct OrchestratingRealData : public ::testing::Test { auto node = ii::DagNode::fromIpnsRecord(testingnoneed2validate); orc_->add_node(cid, node); } break; + case i::gw::Type::Car: case i::gw::Type::Block: { auto blocs_dir = base_dir / "test_data" / "blocks"; EXPECT_TRUE(is_directory(blocs_dir)); @@ -92,9 +106,7 @@ struct OrchestratingRealData : public ::testing::Test { std::cout << cmd << '\n'; auto ec = std::system(cmd.c_str()); EXPECT_EQ(ec, 0); - resp_.status_ = static_cast(987); - resp_.body_ = cid + " fetched"; - return; + return HandleOutcome::DONE; } } break; case i::gw::Type::DnsLink: { @@ -113,13 +125,31 @@ struct OrchestratingRealData : public ::testing::Test { std::getline(fs, target); orc_->add_node(cid, std::make_shared(target)); } break; - case i::gw::Type::Car: - return; default: - return; + return HandleOutcome::DONE; } - // retry - r.orchestrator->build_response(r.dependent); + orc_->build_response(r->dependent); + return HandleOutcome::DONE; + } +}; +struct OrchestratingRealData : public ::testing::Test { + std::shared_ptr api_ = std::make_shared(); + std::shared_ptr orc_; + OrchestratingRealData() { + auto f = [](auto) {}; + resp_.body_ = "No response received."; + resp_.status_ = 0; + resp_.mime_ = "uninit - no mime provided"; + resp_.location_ = "uninit"; + orc_ = std::make_shared( + f, std::make_shared(), api_); + } + i::Response resp_; + void dorequest(std::string_view ipfs_path) { + std::cout << "dorequest(" << ipfs_path << ")\n"; + auto f = [this](auto&, auto& r) { resp_ = r; }; + auto top_req = std::make_shared(std::string{ipfs_path}, f); + orc_->build_response(top_req); } std::string abs_path(std::string rest) { return "/ipfs/QmYBhLYDwVFvxos9h8CGU2ibaY66QNgv8hpfewxaQrPiZj" + rest; diff --git a/library/src/ipfs_client/response.cc b/library/src/ipfs_client/response.cc index 062042d4..95cb0d6c 100644 --- a/library/src/ipfs_client/response.cc +++ b/library/src/ipfs_client/response.cc @@ -4,3 +4,8 @@ using Self = ipfs::Response; Self Self::PLAIN_NOT_FOUND{std::string{}, static_cast(404), std::string{}, std::string{}}; +Self Self::HOST_NOT_FOUND{ + "text/plain", Self::HOST_NOT_FOUND_STATUS, + "either a hostname didn't resolve a DNS TXT records for dnslink=, or we " + "can't find a gateway with the necessary IPNS record", + std::string{}}; diff --git a/library/src/ipfs_client/scheduler.cc b/library/src/ipfs_client/scheduler.cc index 71a72963..0f639548 100644 --- a/library/src/ipfs_client/scheduler.cc +++ b/library/src/ipfs_client/scheduler.cc @@ -29,8 +29,7 @@ void ipfs::Scheduler::Enqueue(std::shared_ptr api, std::string_view accept, Priority p, std::shared_ptr top) { - LOG(INFO) << "Scheduler::Enqueue(...," << suffix << ',' << accept << ',' << p - << ')'; + VLOG(1) << "Sched::EnQ(...," << suffix << ',' << accept << ',' << p << ')'; if (!top) { LOG(ERROR) << "No IpfsRequest?"; } diff --git a/library/src/ipfs_client/scoring.md b/library/src/ipfs_client/scoring.md index d6c79916..01b1ce67 100644 --- a/library/src/ipfs_client/scoring.md +++ b/library/src/ipfs_client/scoring.md @@ -25,7 +25,7 @@ It's an unsigned integer in two forms: When issuing a request to another gateway, it checks the gateways in descending-score order: * If the gateway already has a request for that data pending, or has already failed to return successfully, it's skipped. * If the gateways score is less than the number of pending requests currently sent to that gateway, it's skipped as it's considered already-too-busy -* The "need" is generally calculated as the target number of gateways desired to be involved (based on request priority), subtracting the number of gateways currently processing a request for this data. +* The "need" is generally calculated as the target number of gateways desired to be involved (based on request parallel), subtracting the number of gateways currently processing a request for this data. * If the "need" is less than half the count of pending requests on this gateway, it's skipped as it's simply not urgent enough to justify further overloading this gateway. diff --git a/library/src/ipfs_client/signing_key_type.cc b/library/src/ipfs_client/signing_key_type.cc new file mode 100644 index 00000000..b6489a47 --- /dev/null +++ b/library/src/ipfs_client/signing_key_type.cc @@ -0,0 +1,15 @@ +#include + +#include + +using T = ipfs::SigningKeyType; +namespace n = ipfs::ipns; + +// It is critically important that these 2 enumerations remain in-synch. +// However, some headers that reference SigningKeyType need to be able to +// compile without access to protobuf. +static_assert(static_cast(T::RSA) == n::RSA); +static_assert(static_cast(T::Ed25519) == n::Ed25519); +static_assert(static_cast(T::Secp256k1) == n::Secp256k1); +static_assert(static_cast(T::ECDSA) == n::ECDSA); +static_assert(static_cast(T::KeyTypeCount) == n::KeyType_ARRAYSIZE); diff --git a/library/src/ipfs_client/unixfs_path_resolver_unittest.cc b/library/src/ipfs_client/unixfs_path_resolver_unittest.cc index b754a7b4..a314ddae 100644 --- a/library/src/ipfs_client/unixfs_path_resolver_unittest.cc +++ b/library/src/ipfs_client/unixfs_path_resolver_unittest.cc @@ -37,6 +37,18 @@ struct Api final : public ipfs::ContextApi { std::string const& url) const { throw 8; } + ipfs::IpnsCborEntry deserialize_cbor(ipfs::ByteView) const { return {}; } + bool verify_key_signature(ipfs::SigningKeyType, + ipfs::ByteView signature, + ipfs::ByteView data, + ipfs::ByteView key_bytes) const { + return false; + } + void SendDnsTextRequest(std::string, + DnsTextResultsCallback, + DnsTextCompleteCallback) {} + void SendHttpRequest(ipfs::HttpRequestDescription, + HttpCompleteCallback cb) const {} std::size_t head_size = 0UL; std::string MimeType(std::string ext, std::string_view cont, diff --git a/library/src/libp2p/common/hexutil.cc b/library/src/libp2p/common/hexutil.cc deleted file mode 100644 index 82416ccf..00000000 --- a/library/src/libp2p/common/hexutil.cc +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "vocab/byte_view.h" - -#include "libp2p/common/hexutil.hpp" - -#include - -/* -OUTCOME_CPP_DEFINE_CATEGORY(libp2p::common, UnhexError, e) { - using libp2p::common::UnhexError; - switch (e) { - case UnhexError::NON_HEX_INPUT: - return "Input contains non-hex characters"; - case UnhexError::NOT_ENOUGH_INPUT: - return "Input contains odd number of characters"; - default: - return "Unknown error"; - } -} -*/ -namespace libp2p::common { - -std::string int_to_hex(uint64_t n, size_t fixed_width) noexcept { - std::stringstream result; - result.width(static_cast(fixed_width)); - result.fill('0'); - result << std::hex << std::uppercase << n; - auto str = result.str(); - if (str.length() % 2 != 0) { - str.push_back('\0'); - for (int64_t i = static_cast(str.length()) - 2; i >= 0; --i) { - str[i + 1] = str[i]; - } - str[0] = '0'; - } - return str; -} - -namespace { -std::string to_hex(ipfs::ByteView bytes, std::string_view alpha) { - std::string res(bytes.size() * 2, '\x00'); - auto o = res.begin(); - for (auto b : bytes) { - auto i = to_integer(b); - *(o++) = alpha[i / 256]; - *(o++) = alpha[i % 256]; - } - return res; -} -} // namespace - -std::string hex_upper(ipfs::ByteView bytes) noexcept { - return to_hex(bytes, "0123456789ABCDEF"); -} - -std::string hex_lower(ipfs::ByteView bytes) noexcept { - return to_hex(bytes, "0123456789abcdef"); -} - -ipfs::expected, UnhexError> unhex( - std::string_view hex) { - if (hex.size() % 2) { - return ipfs::unexpected(UnhexError::NOT_ENOUGH_INPUT); - } - std::vector blob; - blob.reserve(hex.size() / 2); - bool bad_input = false; - auto toi = [&bad_input](auto c) { - if (c >= '0' && c <= '9') { - return c - '0'; - } else if (c >= 'a' && c <= 'f') { - return c - 'a' + 10; - } else if (c >= 'A' && c <= 'F') { - return c - 'F' + 10; - } else { - bad_input = true; - return 0; - } - }; - for (auto i = 0U; i < hex.size(); i += 2) { - auto hi = toi(hex[i]); - auto lo = toi(hex[i + 1]); - blob.push_back(static_cast((hi << 4) | lo)); - } - if (bad_input) { - return ipfs::unexpected(UnhexError::NOT_ENOUGH_INPUT); - } else { - return blob; - } -} -} // namespace libp2p::common diff --git a/library/src/libp2p/multi/content_identifier.cc b/library/src/libp2p/multi/content_identifier.cc index 8f84eb8a..6015fbba 100644 --- a/library/src/libp2p/multi/content_identifier.cc +++ b/library/src/libp2p/multi/content_identifier.cc @@ -17,21 +17,6 @@ ContentIdentifier::ContentIdentifier(Version version, content_type{content_type}, content_address{std::move(content_address)} {} -std::string ContentIdentifier::toPrettyString(const std::string& base) const { - /// TODO(Harrm) FIL-14: hash type is a subset of multicodec type, better - /// move them to one place - auto hash_type = MulticodecType::getName( - static_cast(content_address.getType())); - std::string hash_hex = common::hex_lower(content_address.getHash()); - std::string hash_length = - std::to_string(content_address.getHash().size() * 8); - std::string v = "cidv" + std::to_string(static_cast(version)); - std::ostringstream oss; - oss << base << " - " << v << " - " << MulticodecType::getName(content_type) - << " - " << hash_type << '-' << hash_length << '-' << hash_hex; - return oss.str(); -} - bool ContentIdentifier::operator==(const ContentIdentifier& c) const { return version == c.version and content_type == c.content_type and content_address == c.content_address; diff --git a/library/src/libp2p/multi/multihash.cc b/library/src/libp2p/multi/multihash.cc index a7652c34..3667664d 100644 --- a/library/src/libp2p/multi/multihash.cc +++ b/library/src/libp2p/multi/multihash.cc @@ -88,16 +88,6 @@ Result Multihash::create(HashType type, ipfs::ByteView hash) { return Multihash{type, hash}; } -Result Multihash::createFromHex(std::string_view hex) { - auto result = unhex(hex); - if (result.has_value()) { - auto view = ipfs::ByteView{result.value().data(), result.value().size()}; - return Multihash::createFromBytes(view); - } else { - return ipfs::unexpected(Error::INVALID_HEXADECIMAL_INPUT); - } -} - Result Multihash::createFromBytes(ipfs::ByteView b) { if (b.size() < kHeaderSize) { return ipfs::unexpected(Error::INPUT_TOO_SHORT); @@ -136,10 +126,6 @@ ipfs::ByteView Multihash::getHash() const { return ipfs::ByteView(d.bytes.data(), d.bytes.size()).subspan(d.hash_offset); } -std::string Multihash::toHex() const { - return hex_upper(data().bytes); -} - std::vector const& Multihash::toBuffer() const { auto& d = data(); return d.bytes; diff --git a/library/src/libp2p/multi/uvarint.cc b/library/src/libp2p/multi/uvarint.cc index b8b2712c..ca6c5cfd 100644 --- a/library/src/libp2p/multi/uvarint.cc +++ b/library/src/libp2p/multi/uvarint.cc @@ -55,10 +55,6 @@ std::vector const& UVarint::toVector() const { return bytes_; } -std::string UVarint::toHex() const { - return hex_upper(bytes_); -} - size_t UVarint::size() const { return bytes_.size(); } diff --git a/library/src/libp2p/peer/peer_id.cc b/library/src/libp2p/peer/peer_id.cc index 4fd03bf7..be21e302 100644 --- a/library/src/libp2p/peer/peer_id.cc +++ b/library/src/libp2p/peer/peer_id.cc @@ -115,10 +115,6 @@ const std::vector& PeerId::toVector() const { return hash_.toBuffer(); } -std::string PeerId::toHex() const { - return hash_.toHex(); -} - const multi::Multihash& PeerId::toMultihash() const { return hash_; } diff --git a/test_data/blocks/bafkreiagoa7j73kp7wt7odvi3waeab7xbxyebjhbzgguj2uv63t4hsvdri b/test_data/blocks/bafkreiagoa7j73kp7wt7odvi3waeab7xbxyebjhbzgguj2uv63t4hsvdri new file mode 100644 index 00000000..61c5e36b --- /dev/null +++ b/test_data/blocks/bafkreiagoa7j73kp7wt7odvi3waeab7xbxyebjhbzgguj2uv63t4hsvdri @@ -0,0 +1,260 @@ +html { + box-sizing: border-box; +} + +/* *, +*:before, +*:after { + box-sizing: inherit; +} */ + +body { + margin: 0; + border-width: 0; + padding: 0; + display: flex; + justify-content: center; +} + +h1, +h2 { + color: #333; + text-shadow: #1114 1px 1px 1px; + text-align: center; + border-bottom: none; + padding: 0; +} + +#container { + margin: 0; + border-width: 0; + padding: 0; + display: flex; + justify-content: center; + background:black; + position: absolute; + left:0; + top:0; + height:100%; + width:100%; + overflow: auto; +} + +#content { + width: 100%; + overflow: hidden; + font-size: 1em; + background: black; + max-width: 8096px; +} + +#list { + background: white; +} + +.nojs #content { + column-count: 4; + column-gap: 0; +} + +.item { + display: block; + position: relative; + overflow: hidden; +} + +.nojs #content .item { + display: inline-block; + width: 100%; +} + +.item img { + width: 100%; + height: auto; + transition: all 0.3s; + transform: scale(1); + background: white; +} + +.item:hover img { + transform: scale(1.1); +} + +.item figure { + margin: 0; +} + +.item figcaption { + position: absolute; + bottom: 0; + width: 100%; + vertical-align: bottom; + background: rgba(48, 48, 48, 0.5); + color: white; + padding: 0.6em; + font-weight: bold; + transition: all 0.3s; + transform: scale(1); +} + +/* .item:hover figcaption { + transform: scale(1); + bottom: 50%; +} */ + +#footer { + margin: 0; +} + +#footer ul { + margin: 0; + margin-left: 10%; +} + +@media only screen and (min-width: 8096px) { + h1 { + font-size: 0.75vw; + } + + h2 { + font-size: 0.5vw; + } + + #content .item { + width: 3.125%; + } + + .nojs #content { + column-count: 32; + } + + ul { + column-count: 32; + } +} + +@media only screen and (min-width: 4048px) { + h1 { + font-size: 1.5vw; + } + + h2 { + font-size: 1vw; + } + + #content .item { + width: 6.25%; + } + + .nojs #content { + column-count: 16; + } + + ul { + column-count: 16; + } +} + +@media only screen and (max-width: 4047px) and (min-width: 2024px) { + h1 { + font-size: 3vw; + } + + h2 { + font-size: 2vw; + } + + #content .item { + width: 12.5%; + } + + .nojs #content { + column-count: 8; + } + + ul { + column-count: 8; + } +} + +@media only screen and (max-width: 2023px) and (min-width: 1024px) { + h1 { + font-size: 6vw; + } + + h2 { + font-size: 4vw; + } + + #content .item { + width: 25%; + } + + .nojs #content { + column-count: 4; + } + + ul { + column-count: 4; + } +} + +@media only screen and (max-width: 1023px) and (min-width: 768px) { + h1 { + font-size: 6vw; + } + + h2 { + font-size: 4vw; + } + + #content { + font-size: 1em; + } + + #content .item { + width: 33.3333%; + } + + .nojs #content { + column-count: 3; + } + + ul { + column-count: 2; + } + + #footer ul li { + font-size: 0.9em; + } +} + +@media only screen and (max-width: 767px) { + h1 { + font-size: 7vw; + } + + h2 { + font-size: 5vw; + } + + #content { + font-size: 1.2em; + } + + #content .item { + width: 50% + } + + .nojs #content { + column-count: 2; + } + + ul { + column-count: 1; + } + + #footer ul li { + font-size: 1em; + } +} \ No newline at end of file diff --git a/test_data/blocks/bafybeic7wwiyffvkr7xknfpzp7wch53owkr6dxv45cspxn7hfqccdel4fe b/test_data/blocks/bafybeic7wwiyffvkr7xknfpzp7wch53owkr6dxv45cspxn7hfqccdel4fe new file mode 100644 index 0000000000000000000000000000000000000000..5bad4c76f0165b0086ec049d60daf29eddbb1a70 GIT binary patch literal 843 zcmWe~_a!Cks5Ou~gYM*>zUQ ziKYX8LgJ3Ttqr{-Bw*~En^>HmS5&E&o0+GVRV=Z552KI+M0dyZ@Go1xY@7Sz#;0tC zrC)Z>{{HFIt8R0iNxKT_7sv|S5E3;tOwP|MNzE(KD@ZIV&d*E%yZot=kO9Qy=0Rm< zwJ*2J7%BOyI~VVoHrwTPnY_zei_RbN%n#)4a)nq-O>)a6)=qX4GK1Lp>CbsPCAk%> zH#Q z`@4Iz#Ll0YeLOJr6UVo&0=zdYJt_~VuM`rqFv`pYhHY_tPJUuaY6{H1{|trfA&zBG z_!<1_@}Alc)3%2%jclw*bjv&G^=Z-k^9FYo$;{X>RY=&$v^+Jbz$3LN->Eb&B_|c+ zguCl(VBwV+G)*&m(wo@E@K+Ub>Hjr%_=~YuJd|ftpUAROxR{4mNW|GFGp{(cs3bKd zzPO|kXaFdpkG)`n`9934v%x5d)7<#w+@f#yUcCDM?d9r8Zqji+6DO{JA+$nLh}+o| zp|MFo$OIB<-tX;q<=hpUzR!G}eY)hH&F?$@8yKvwW16lO823>rHC~9*#UL%QEE5=8 z5)ag1VRlY3`1sGB1$!Rm^8KE5eskrHmiun2(rrDyCzj5Q`Au%_{#G;bSWMJCN z$t*63&&baO`}q_r%*!)h7An0l2>^Yo BNLT;> literal 0 HcmV?d00001