Don't hold forever

an IPNS record.
Expiration date.

component/cache_requestor.cc

@@ -47,7 +47,7 @@ void Self::Assign(dc::BackendResult res) {
   }
 }
 auto Self::handle(RequestPtr req) -> HandleOutcome {
-  if (startup_pending_) {
+  if (startup_pending_ || !(req->cachable())) {
     return HandleOutcome::NOT_HANDLED;
   }
   Task task;
@@ -132,19 +132,20 @@ void Self::OnBodyRead(Task task, int code) {
   task.body.assign(task.buf->data(), static_cast<std::size_t>(code));
   if (task.request) {
     task.SetHeaders(name());
-    if (task.request->RespondSuccessfully(task.body, api_)) {
-      VLOG(2) << "Cache hit on " << task.key << " for "
-              << task.request->debug_string();
+    bool valid = false;
+    task.request->RespondSuccessfully(task.body, api_, &valid);
+    if (valid) {
+      LOG(INFO) << "Cache hit for " << task.key;
     } else {
-      LOG(ERROR) << "Had a BAD cached response for " << task.key;
+      LOG(ERROR) << "Had a bad or expired cached response for " << task.key;
       Expire(task.key);
       Miss(task);
     }
   }
 }
 void Self::Store(std::string key, std::string headers, std::string body) {
-  VLOG(2) << "Store(" << name() << ',' << key << ',' << headers.size() << ','
-          << body.size() << ')';
+  LOG(INFO) << "Store(" << name() << ',' << key << ',' << headers.size() << ','
+            << body.size() << ')';
   auto bound = base::BindOnce(&Self::OnEntryCreated, base::Unretained(this),
                               key, headers, body);
   auto res = cache_->OpenOrCreateEntry(key, net::LOW, std::move(bound));

library/include/ipfs_client/gw/gateway_request.h

@@ -61,9 +61,11 @@ class GatewayRequest {
   std::optional<HttpRequestDescription> describe_http(std::string_view) const;
   std::string debug_string() const;
   void orchestrator(std::shared_ptr<Orchestrator> const&);
+  bool cachable() const;
 
   bool RespondSuccessfully(std::string_view,
-                           std::shared_ptr<ContextApi> const& api);
+                           std::shared_ptr<ContextApi> const& api,
+                           bool* valid = nullptr);
   void Hook(std::function<void(std::string_view)>);
   bool PartiallyRedundant() const;
   std::string Key() const;

library/include/ipfs_client/gw/requestor.h

@@ -18,17 +18,18 @@ class GatewayRequest;
 using RequestPtr = std::shared_ptr<GatewayRequest>;
 
 class Requestor : public std::enable_shared_from_this<Requestor> {
- protected:
-  Requestor() {}
-
-  friend class RequestorPool;
+ public:
   enum class HandleOutcome : char {
     NOT_HANDLED = 'N',
     PENDING = 'P',
     DONE = 'D',
     PARALLEL = 'L',
     MAYBE_LATER = 'M'
   };
+
+ protected:
+  Requestor() {}
+
   virtual HandleOutcome handle(RequestPtr) = 0;
 
   void definitive_failure(RequestPtr) const;

library/include/ipfs_client/ipld/dag_node.h

@@ -30,6 +30,8 @@ namespace ipfs::ipld {
 
 using NodePtr = std::shared_ptr<DagNode>;
 class DirShard;
+class DnsLinkName;
+class IpnsName;
 
 struct MoreDataNeeded {
   MoreDataNeeded(std::string one) : ipfs_abs_paths_{{one}} {}
@@ -90,7 +92,14 @@ class DagNode : public std::enable_shared_from_this<DagNode> {
 
   virtual NodePtr rooted();
   virtual NodePtr deroot();
-  virtual DirShard* as_hamt();  // Wish I had access to dynamic_cast
+
+  // Wish I had access to dynamic_cast
+  virtual DnsLinkName const* as_dnslink() const { return nullptr; }
+  virtual DirShard* as_hamt() { return nullptr; }
+  virtual IpnsName const* as_ipns() const { return nullptr; }
+
+  virtual bool expired() const;
+  virtual bool PreferOver(DagNode const& another) const;
 
   void set_api(std::shared_ptr<ContextApi>);
 };

library/src/ipfs_client/gw/dnslink_requestor.cc

@@ -1,6 +1,6 @@
 #include <ipfs_client/gw/dnslink_requestor.h>
 
-#include "ipfs_client/ipld/ipns_name.h"
+#include "ipfs_client/ipld/dnslink_name.h"
 
 #include <ipfs_client/gw/gateway_request.h>
 #include <ipfs_client/ipld/dag_node.h>

library/src/ipfs_client/gw/gateway_request.cc

@@ -3,6 +3,7 @@
 #include <ipfs_client/gw/providers_response.h>
 #include <ipfs_client/ipld/chunk.h>
 #include <ipfs_client/ipld/dag_node.h>
+#include <ipfs_client/ipld/dnslink_name.h>
 #include <ipfs_client/ipld/ipns_name.h>
 
 #include <ipfs_client/car.h>
@@ -219,6 +220,23 @@ std::string_view ipfs::gw::name(ipfs::gw::Type t) {
   std::sprintf(buf.data(), "InvalidType %d", static_cast<std::int8_t>(t));
   return buf.data();
 }
+bool Self::cachable() const {
+  using ipfs::gw::Type;
+  switch (type) {
+    case Type::Car:
+      return path.find("/ipns/") == std::string::npos;
+    case Type::Block:
+    case Type::Ipns:
+      return true;
+    case Type::DnsLink:
+    case Type::Providers:
+    case Type::Identity:
+    case Type::Zombie:
+      return false;
+  }
+  LOG(ERROR) << "Unhandled request type: " << debug_string();
+  return false;
+}
 std::string Self::debug_string() const {
   std::ostringstream oss;
   oss << "Request{Type=" << type << ' ' << main_param;
@@ -237,9 +255,13 @@ std::string Self::Key() const {
   return rv;
 }
 bool Self::RespondSuccessfully(std::string_view bytes,
-                               std::shared_ptr<ContextApi> const& api) {
+                               std::shared_ptr<ContextApi> const& api,
+                               bool* valid) {
   using namespace ipfs::ipld;
   bool success = false;
+  if (valid) {
+    *valid = false;
+  }
   switch (type) {
     case Type::Block: {
       DCHECK(cid.has_value());
@@ -249,11 +271,19 @@ bool Self::RespondSuccessfully(std::string_view bytes,
       }
       DCHECK(api);
       auto node = DagNode::fromBytes(api, cid.value(), bytes);
+      if (!node) {
+        return false;
+      } else if (valid) {
+        *valid = true;
+      }
       success = orchestrator_->add_node(main_param, node);
     } break;
     case Type::Identity:
       success = orchestrator_->add_node(
           main_param, std::make_shared<Chunk>(std::string{bytes}));
+      if (valid) {
+        *valid = true;
+      }
       break;
     case Type::Ipns:
       if (cid.has_value()) {
@@ -262,43 +292,55 @@ bool Self::RespondSuccessfully(std::string_view bytes,
         auto rec = ipfs::ValidateIpnsRecord({byte_ptr, bytes.size()},
                                             cid.value(), *api);
         if (rec.has_value()) {
-          auto node = DagNode::fromIpnsRecord(rec.value());
+          auto node = std::make_shared<IpnsName>(rec.value());
           success = orchestrator_->add_node(main_param, node);
+          if (valid) {
+            *valid = !node->expired();
+            LOG(INFO) << "IPNS node created " << main_param << ' ' << success
+                      << " vs. " << *valid;
+          }
         } else {
           LOG(ERROR) << "IPNS record failed to validate!";
           return false;
         }
       }
       break;
-    case Type::DnsLink:
+    case Type::DnsLink: {
       LOG(INFO) << "Resolved " << debug_string() << " to " << bytes;
+      auto node = std::make_shared<ipld::DnsLinkName>(bytes);
       if (orchestrator_) {
-        success = orchestrator_->add_node(
-            main_param, std::make_shared<ipld::IpnsName>(bytes));
+        success = orchestrator_->add_node(main_param, node);
       } else {
         LOG(FATAL) << "I have no orchestrator!!";
       }
-      break;
+      if (valid) {
+        *valid = !node->expired();
+      }
+    } break;
     case Type::Car: {
       DCHECK(api);
       Car car(as_bytes(bytes), *api);
-      auto added = 0;
       while (auto block = car.NextBlock()) {
         auto cid_s = block->cid.to_string();
         auto n = DagNode::fromBytes(api, block->cid, block->bytes);
         if (!n) {
           LOG(ERROR) << "Unable to handle block from CAR: " << cid_s;
-        } else if (orchestrator_->add_node(cid_s, n)) {
-          ++added;
-        } else {
-          LOG(INFO) << "Did not add node from CAR: " << cid_s;
+          continue;
+        }
+        if (valid) {
+          *valid = true;
+        }
+        if (orchestrator_->add_node(cid_s, n)) {
+          success = true;
         }
       }
-      success = added > 0;
       break;
     }
     case Type::Providers:
       success = providers::ProcessResponse(bytes, *api);
+      if (valid) {
+        *valid = success;
+      }
       break;
     case Type::Zombie:
       LOG(WARNING) << "Responding to a zombie is ill-advised.";

library/src/ipfs_client/gw/multi_gateway_requestor.cc

@@ -15,6 +15,9 @@ std::string_view Self::name() const {
   return "multi-gateway requestor";
 }
 auto Self::handle(RequestPtr r) -> HandleOutcome {
+  if (!r || !api_) {
+    return HandleOutcome::DONE;
+  }
   if (!r->is_http()) {
     LOG(INFO) << r->debug_string() << " is not an HTTP request.";
     return HandleOutcome::NOT_HANDLED;

library/src/ipfs_client/gw/multi_gateway_requestor_unittest.cc

@@ -0,0 +1,21 @@
+#include "multi_gateway_requestor.h"
+
+#include <mock_api.h>
+
+#include <ipfs_client/gw/gateway_request.h>
+
+namespace {
+ig::RequestPtr block_req() {
+  auto rv = std::make_shared<ig::GatewayRequest>();
+  rv->type = ig::Type::Block;
+  rv->main_param =
+      "bafybeid4dzlxm6h4r6kfvx6jp6vj4nteplmbve224lx2s3lgjubyufsuo4";
+  return rv;
+}
+}  // namespace
+
+TEST(MultiGatewayRequestor, FailsWithoutApi) {
+  ig::MultiGatewayRequestor t;
+  auto o = t.handle(block_req());
+  EXPECT_TRUE(o == ig::Requestor::HandleOutcome::DONE);
+}

library/src/ipfs_client/ipld/dag_node.cc

@@ -4,7 +4,7 @@
 #include "dag_cbor_node.h"
 #include "dag_json_node.h"
 #include "directory_shard.h"
-#include "ipns_name.h"
+#include "dnslink_name.h"
 #include "root.h"
 #include "small_directory.h"
 #include "symlink.h"
@@ -144,7 +144,7 @@ std::shared_ptr<Node> Node::fromBlock(ipfs::PbDag const& block) {
 }
 
 auto Node::fromIpnsRecord(ipfs::ValidatedIpns const& v) -> NodePtr {
-  return std::make_shared<IpnsName>(v.value);
+  return std::make_shared<DnsLinkName>(v.value);
 }
 
 std::shared_ptr<Node> Node::deroot() {
@@ -153,9 +153,6 @@ std::shared_ptr<Node> Node::deroot() {
 std::shared_ptr<Node> Node::rooted() {
   return std::make_shared<Root>(shared_from_this());
 }
-auto Node::as_hamt() -> DirShard* {
-  return nullptr;
-}
 void Node::set_api(std::shared_ptr<ContextApi> api) {
   api_ = api;
 }
@@ -229,6 +226,12 @@ auto Node::FindChild(std::string_view link_key) -> Link* {
   }
   return nullptr;
 }
+bool Node::expired() const {
+  return false;
+}
+bool Node::PreferOver(Node const&) const {
+  return false;
+}
 
 std::ostream& operator<<(std::ostream& s, ipfs::ipld::PathChange const& c) {
   return s << "PathChange{" << c.new_path << '}';

library/src/ipfs_client/ipld/dnslink_name.cc

@@ -0,0 +1,53 @@
+#include "dnslink_name.h"
+
+#include "log_macros.h"
+
+using Self = ipfs::ipld::DnsLinkName;
+namespace ch = std::chrono;
+
+Self::DnsLinkName(std::string_view target_abs_path)
+    : expiration_(ch::system_clock::now() + ch::minutes(5)) {
+  SlashDelimited target{target_abs_path};
+  target_namespace_ = target.pop();
+  target_root_ = target.pop();
+  links_.emplace_back("", Link{target_root_, nullptr});
+  target_path_.assign(target.to_string());
+}
+
+auto Self::resolve(ResolutionState& params) -> ResolveResult {
+  auto& node = links_.at(0).second.node;
+  if (!node) {
+    node = params.GetBlock(target_root_);
+  }
+  if (!node) {
+    return MoreDataNeeded(target_namespace_ + "/" + target_root_);
+  }
+  if (target_path_.empty()) {
+    return node->resolve(params);
+  }
+  auto path = target_path_;
+  path.append("/").append(params.PathToResolve().to_view());
+  auto altered = params.WithPath(path);
+  LOG(WARNING) << "Odd case: name points at /ns/root/MORE/PATH ("
+               << target_namespace_ << '/' << target_root_ << '/'
+               << target_path_ << "): " << params.MyPath()
+               << " will be resolved as " << path;
+  auto lower = node->resolve(params);
+  if (auto* mdn = std::get_if<MoreDataNeeded>(&lower)) {
+    if (expired()) {
+      auto refresh_path = "/ipns/" + params.MyPath().pop_back();
+      mdn->ipfs_abs_paths_.push_back(refresh_path);
+    }
+  }
+  return lower;
+}
+bool Self::expired() const {
+  return expiration_ < ch::system_clock::now();
+}
+bool Self::PreferOver(DagNode const& another) const {
+  auto* other = another.as_dnslink();
+  if (!other) {
+    return true;
+  }
+  return expiration_ > other->expiration_ + ch::seconds(1);
+}

library/src/ipfs_client/ipld/dnslink_name.h

@@ -0,0 +1,27 @@
+#ifndef IPFS_IPLD_DNSLINK_NAME_H_
+#define IPFS_IPLD_DNSLINK_NAME_H_
+
+#include "ipfs_client/ipld/dag_node.h"
+
+#include <chrono>
+
+namespace ipfs::ipld {
+class DnsLinkName : public DagNode {
+  std::string target_namespace_;
+  std::string target_root_;
+  std::string target_path_;
+  std::chrono::system_clock::time_point expiration_;
+
+  ResolveResult resolve(ResolutionState& params) override;
+  bool PreferOver(DagNode const& another) const override;
+  DnsLinkName const* as_dnslink() const override { return this; }
+
+ public:
+  DnsLinkName(std::string_view target_abs_path);
+  virtual ~DnsLinkName() noexcept {}
+
+  bool expired() const override;
+};
+}  // namespace ipfs::ipld
+
+#endif  // IPFS_IPLD_DNSLINK_NAME_H_