Fixes #57

Redirects file parses.
Plugging it in was tricky.
Time for refactor.

.github/workflows/library.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install --yes cmake ninja-build lcov binutils doxygen graphviz libc6{,-dev}
+          sudo apt-get install --yes cmake ninja-build lcov binutils doxygen graphviz libc6{,-dev} valgrind
           npm install -g @marp-team/marp-cli
       - name: Configure
         shell: bash
@@ -27,6 +27,9 @@ jobs:
             -B build \
             -D CMAKE_BUILD_TYPE=Debug
       - name: Run Tests
+        shell: bash
+        run: cmake --build build --config Debug --target run_tests
+      - name: Generate Coverage Report
         shell: bash
         run: cmake --build build --config Debug --target cov
       - name: Upload coverage reports to Codecov.com

cmake/chromium.py

@@ -235,7 +235,7 @@ def sync_dir(source_relative, target_relative, complete=True):
 n = join(out, 'obj', 'components', 'ipfs', 'ipfs.ninja')
 if not isfile(n) or (isfile(UPDATED) and getmtime(UPDATED) > getmtime(n)):
     a = [python, join(depot_tools_dir, 'gn.py'), 'gen', '--args='+gnargs.replace("'", ""), out]
-    print('Running gn gen', a)
+    verbose('Running gn gen', a)
     run(a)
 
 for target in argv[2:]:

cmake/inc_link.py

@@ -14,9 +14,10 @@
 inc_link = join(build_dir,'component','inc_link')
 chromium_src = vars['CHROMIUM_SOURCE_TREE'].rstrip('/')
 profile = vars['CHROMIUM_PROFILE']
+gen_dir = join(chromium_src,'out',profile,'gen')
 source_bases = [
     chromium_src,
-    join(chromium_src,'out',profile,'gen'),
+    gen_dir,
     join(chromium_src,'v8','include'),
     join(chromium_src,'third_party', 'abseil-cpp')
     ]
@@ -119,6 +120,15 @@ def search() -> bool:
     global preempt
     link_count = 0
     unfound_count = 0
+    # ipfs_client_gen = join(gen_dir,'third_party','ipfs_client')
+    # for h in listdir(ipfs_client_gen):
+    #     if h.endswith('.h'):
+    #         source = join(ipfs_client_gen, h)
+    #         target = join(inc_link, 'third_party', 'ipfs_client', h)
+    #         if isfile(source) and not exists(target):
+    #             print("linked our gen file ",source,target)
+    #             symlink(source, target)
+    #             link_count += 1
     compile_commands = json.load(open(join(build_dir,'compile_commands.json')))
     commands = []
     for command_obj in compile_commands:

cmake/patch.py

@@ -151,7 +151,7 @@ def electron_version(self, branch='main'):
     def unavailable(self):
         avail = list(map(as_int, self.available()))
         version_set = {}
-        fuzz = 121
+        fuzz = 114
         def check(version, version_set, s):
             i = as_int(version)
             by = (fuzz,0)
@@ -163,7 +163,6 @@ def check(version, version_set, s):
                     by = ( d, a )
             if version not in version_set:
                 sortable = [int(c) for c in version.split('.')]
-                # print('Adding',version,s)
                 version_set[version] = [sortable, version, s]
             elif s not in version_set[version]:
                 #print('2 Adding',version,s)

cmake/trim.patches.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -ex
+cd `dirname "${0}"`/../component/patches
+#okregex='kEnableIpfs'
+#okregex='deps .= . "//components/ipfs" '
+okregex='flag-metadata.json'
+if ! grep -qE "${okregex}" *.patch
+then
+  echo "Regex broken: ${okregex}"
+  exit 9
+elif grep -LE "${okregex}" *.patch | grep -F .
+then
+  grep -LE "${okregex}" *.patch | sort -V | head -n 2 | xargs git rm
+else
+  echo "Update regex - it currently covers _all_ patches: '${okregex}'"
+  exit 8
+fi

cmake/valgrind.suppressions.txt

@@ -0,0 +1,14 @@
+{
+   IgnoreGcovProblems
+   Memcheck:Addr2
+   ...
+   fun:__gcov_exit
+   ...
+}
+{
+   DontAskMeWhyGetEnvDoesThis
+   Memcheck:Cond
+   ...
+   fun:getenv
+   ...
+}

component/BUILD.gn.in

@@ -26,6 +26,8 @@ if (enable_ipfs) {
         "//services/network/public/mojom:url_loader_base",
         "//url",
         "//third_party/blink/public:blink",
+      ]
+      public_deps = [
         "//third_party/ipfs_client",
       ]
       defines = [ "IS_IPFS_IMPL" ]

component/CMakeLists.txt

@@ -1,3 +1,4 @@
+include(setup)
 
 if(NOT CHROMIUM_PROFILE)
     message(FATAL_ERROR "Your Chromium profile should name a realistic subdir of chromium/src/out, perhaps Debug, Release, or Default")
@@ -150,16 +151,20 @@ target_compile_options(out_of_tree
   PUBLIC
     ${WARNING_FLAGS}
   )
+find_package(Protobuf)
 target_link_libraries(out_of_tree
   PUBLIC
+    protobuf::libprotobuf
     ipfs_client
   )
 target_include_directories(out_of_tree
   SYSTEM
   BEFORE
-  PUBLIC
+  PRIVATE
+    "${protobuf_INCLUDE_DIR}"
     "${CMAKE_CURRENT_BINARY_DIR}/inc_link"
   )
+
 target_include_directories(out_of_tree
   PUBLIC
   ../library/src

component/gateway_requests.cc

@@ -95,6 +95,9 @@ auto Self::InitiateGatewayRequest(BusyGateway assigned)
   auto req = std::make_unique<network::ResourceRequest>();
   req->url = GURL{url};
   req->priority = net::HIGHEST;  // TODO
+  if (!assigned.accept().empty()) {
+    req->headers.SetHeader("Accept", assigned.accept());
+  }
   auto out = std::make_shared<GatewayUrlLoader>(std::move(assigned));
   GOOGLE_DCHECK_GT(out->gateway->url_prefix().size(), 0U);
   out->loader = network::SimpleURLLoader::Create(std::move(req),
@@ -118,6 +121,8 @@ void Self::OnResponse(std::shared_ptr<ContextApi> api,
                       std::shared_ptr<GatewayUrlLoader> req,
                       base::TimeTicks start_time,
                       std::unique_ptr<std::string> body) {
+  auto sz = body ? body->size() : 0UL;
+  LOG(INFO) << "OnResponse(...," << start_time << ", " << sz << "B )";
   DCHECK(req);
   auto task = req->task();
   if (task.empty()) {
@@ -130,10 +135,13 @@ void Self::OnResponse(std::shared_ptr<ContextApi> api,
   auto& ldr = req->loader;
   //  auto listener = req->listener;
   if (ProcessResponse(bg, ldr.get(), body.get(), start_time)) {
+    LOG(INFO) << url << " success.";
     bg.Success(state_->gateways(), shared_from_this());
   } else {
+    LOG(INFO) << url << " failure.";
     bg.Failure(state_->gateways(), shared_from_this());
   }
+  VLOG(1) << "Recheck for more activity.";
   state_->storage().CheckListening();
   state_->scheduler().IssueRequests(api);
 }
@@ -152,18 +160,17 @@ bool Self::ProcessResponse(BusyGateway& gw,
     LOG(ERROR) << "No loader for processing " << gw.url();
     return false;
   }
-  // LOG(INFO) << "Neterror(" << ldr->NetError() << ')';
   if (!body) {
-    //    LOG(INFO) << "ProcessResponse(" << gw.url()
-    //              << ") Null body - presumably http error.\n";
+    LOG(INFO) << "ProcessResponse(" << gw.url()
+              << ") Null body - presumably http error.\n";
     return false;
   }
   network::mojom::URLResponseHead const* head = ldr->ResponseInfo();
   if (!head) {
     LOG(INFO) << "ProcessResponse(" << gw.url() << ") Null head.\n";
     return false;
   }
-  GOOGLE_DCHECK_LT(gw.url().find("?format="), gw.url().size());
+  DCHECK(gw.url().find("?format=") < gw.url().size() || gw.accept().size() > 0);
   std::string reported_content_type;
   head->headers->EnumerateHeader(nullptr, "Content-Type",
                                  &reported_content_type);
@@ -184,7 +191,10 @@ bool Self::ProcessResponse(BusyGateway& gw,
   }
   auto cid_str = gw.task();
   cid_str.erase(0, 5);  // ipfs/
-  cid_str.erase(cid_str.find('?'));
+  auto qmark = cid_str.find('?');
+  if (qmark < cid_str.size()) {
+    cid_str.erase(qmark);
+  }
   if (state_->storage().Get(cid_str)) {
     // LOG(INFO) << "Got multiple successful responses for " << cid_str;
     return true;
@@ -234,6 +244,15 @@ bool Self::ProcessResponse(BusyGateway& gw,
               " : load over http(s)\";dur=" + std::to_string(duration));
       state_->storage().Store(cid_str, cid.value(),
                               head->headers->raw_headers(), *body);
+      auto& orc = state_->orchestrator();
+      orc.add_node(cid_str, ipld::DagNode::fromBlock(block));
+      if (gw.srcreq) {
+        orc.build_response(gw.srcreq->dependent);
+      } else {
+        LOG(ERROR) << "This BusyGateway with response has no top-level "
+                      "IpfsRequest associated with it "
+                   << gw.url() << " " << gw.accept();
+      }
       scheduler().IssueRequests(shared_from_this());
       return true;
     } else {
@@ -295,7 +314,9 @@ void Self::RequestByCid(std::string cid,
                         std::shared_ptr<DagListener> listener,
                         Priority prio) {
   auto me = shared_from_this();
-  sched_.Enqueue(me, listener, {}, "ipfs/" + cid + "?format=raw", prio);
+  LOG(ERROR) << "Look out! RequestByCid(" << cid << ",...," << prio << ')';
+  sched_.Enqueue(me, listener, {}, "ipfs/" + cid, "application/vnd.ipld.raw",
+                 prio, {});
   sched_.IssueRequests(me);
 }
 

component/gateway_requests.h

@@ -21,6 +21,7 @@ class URLLoaderFactory;
 
 namespace ipfs {
 class InterRequestState;
+class IpfsRequest;
 class NetworkRequestor;
 
 class GatewayRequests final : public ContextApi {

component/inter_request_state.cc

@@ -6,14 +6,19 @@
 #include "base/logging.h"
 #include "content/public/browser/browser_context.h"
 
+#include <ipfs_client/dag_listener.h>
+#include <ipfs_client/ipfs_request.h>
+#include <ipfs_client/response.h>
 #include <ipfs_client/scheduler.h>
 
+using Self = ipfs::InterRequestState;
+
 namespace {
 constexpr char user_data_key[] = "ipfs_request_userdata";
 }
 
-auto ipfs::InterRequestState::FromBrowserContext(
-    content::BrowserContext* context) -> InterRequestState& {
+auto Self::FromBrowserContext(content::BrowserContext* context)
+    -> InterRequestState& {
   if (!context) {
     LOG(WARNING) << "No browser context! Using a default IPFS state.";
     static ipfs::InterRequestState static_state({});
@@ -30,8 +35,7 @@ auto ipfs::InterRequestState::FromBrowserContext(
   context->SetUserData(user_data_key, std::move(owned));
   return *raw;
 }
-auto ipfs::InterRequestState::serialized_caches()
-    -> std::array<decltype(mem_), 2> {
+auto Self::serialized_caches() -> std::array<decltype(mem_), 2> {
   if (!mem_) {
     auto p = mem_ = std::make_shared<CacheRequestor>(
         net::CacheType::MEMORY_CACHE, *this, base::FilePath{});
@@ -46,7 +50,7 @@ auto ipfs::InterRequestState::serialized_caches()
   }
   return {mem_, dsk_};
 }
-auto ipfs::InterRequestState::requestor() -> BlockRequestor& {
+auto Self::requestor() -> BlockRequestor& {
   if (!requestor_.Valid()) {
     serialized_caches();
     requestor_.Add(mem_);
@@ -55,7 +59,7 @@ auto ipfs::InterRequestState::requestor() -> BlockRequestor& {
   }
   return requestor_;
 }
-std::shared_ptr<ipfs::GatewayRequests> ipfs::InterRequestState::api() {
+std::shared_ptr<ipfs::GatewayRequests> Self::api() {
   auto existing = api_.lock();
   if (existing) {
     return existing;
@@ -69,11 +73,71 @@ std::shared_ptr<ipfs::GatewayRequests> ipfs::InterRequestState::api() {
   }
   return created;
 }
-auto ipfs::InterRequestState::scheduler() -> Scheduler& {
+auto Self::scheduler() -> Scheduler& {
   auto api = api_.lock();
   DCHECK(api);
   return api->scheduler();
 }
 
-ipfs::InterRequestState::InterRequestState(base::FilePath p) : disk_path_{p} {}
-ipfs::InterRequestState::~InterRequestState() noexcept {}
+namespace {
+
+void send_gateway_request(Self* me,
+                          std::shared_ptr<ipfs::gw::GatewayRequest> req) {
+  if (!req->dependent) {
+    LOG(FATAL) << "This makes no sense whatsoever - why do you want to request "
+                  "things if nothing awaits.";
+  }
+  struct DagListenerAdapter final : public ipfs::DagListener {
+    std::shared_ptr<ipfs::gw::GatewayRequest> gw_req;
+    std::string bytes;
+    std::shared_ptr<ipfs::GatewayRequests> api;
+    void ReceiveBlockBytes(std::string_view b) override {
+      LOG(INFO) << "DagListenerAdapter::ReceiveBlockBytes(" << b.size() << "B)";
+      bytes.assign(b);
+    }
+    void BlocksComplete(std::string mime_type) override {
+      LOG(INFO) << "DagListenerAdapter::BlocksComplete(" << mime_type << ")";
+      ipfs::Response r{mime_type, 200, std::move(bytes), ""};
+      gw_req->dependent->finish(r);
+    }
+    void NotHere(std::string_view cid, std::string_view path) override {
+      LOG(INFO) << "DagListenerAdapter::NotHere(" << cid << ',' << path << ")";
+      api->scheduler().IssueRequests(api);
+    }
+    void DoesNotExist(std::string_view cid, std::string_view path) override {
+      LOG(INFO) << "DagListenerAdapter::DoesNotExist(" << cid << ',' << path
+                << ")";
+      ipfs::Response r{"", 404, "", ""};
+      gw_req->dependent->finish(r);
+    }
+  };
+  auto dl = std::make_shared<DagListenerAdapter>();
+  dl->api = me->api();
+  dl->gw_req = req;
+  auto& sched = me->scheduler();
+  sched.Enqueue(me->api(), dl, {}, req->url_suffix().substr(1), req->accept(),
+                9, req);
+  sched.IssueRequests(me->api());
+}
+std::string detect_mime(Self* me,
+                        std::string a,
+                        std::string_view b,
+                        std::string const& c) {
+  auto api = me->api();
+  return static_cast<ipfs::ContextApi*>(api.get())->MimeType(a, b, c);
+}
+}  // namespace
+
+auto Self::orchestrator() -> Orchestrator& {
+  if (!orc_) {
+    auto gwreq = [this](auto p) { send_gateway_request(this, p); };
+    auto mimer = [this](auto a, auto b, auto& c) {
+      return detect_mime(this, a, b, c);
+    };
+    orc_ = std::make_shared<Orchestrator>(gwreq, mimer);
+  }
+  return *orc_;
+}
+
+Self::InterRequestState(base::FilePath p) : disk_path_{p} {}
+Self::~InterRequestState() noexcept {}

component/inter_request_state.h

@@ -7,6 +7,7 @@
 #include "ipfs_client/chained_requestors.h"
 #include "ipfs_client/gateways.h"
 #include "ipfs_client/ipns_names.h"
+#include "ipfs_client/orchestrator.h"
 
 #include "base/supports_user_data.h"
 
@@ -26,6 +27,7 @@ class InterRequestState : public base::SupportsUserData::Data {
   std::time_t last_discovery_ = 0;
   std::shared_ptr<CacheRequestor> mem_, dsk_;
   base::FilePath const disk_path_;
+  std::shared_ptr<Orchestrator> orc_;  // TODO - map of origin to Orchestrator
 
  public:
   InterRequestState(base::FilePath);
@@ -38,6 +40,7 @@ class InterRequestState : public base::SupportsUserData::Data {
   Scheduler& scheduler();
   std::shared_ptr<GatewayRequests> api();
   std::array<std::shared_ptr<CacheRequestor>,2> serialized_caches();
+  Orchestrator& orchestrator();
 
   static InterRequestState& FromBrowserContext(content::BrowserContext*);
 };