From e5ea311f6c915e832752e15dea7c3ae8d57749d1 Mon Sep 17 00:00:00 2001 From: ian Date: Thu, 11 Apr 2024 23:41:08 +0100 Subject: [PATCH 01/11] Add bulk get links call which can be used to efficiently mirror a whole DAG --- .../java/org/peergos/AggregatedMetrics.java | 1 + src/main/java/org/peergos/Want.java | 14 +++++++++- src/main/java/org/peergos/net/APIHandler.java | 27 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/peergos/AggregatedMetrics.java b/src/main/java/org/peergos/AggregatedMetrics.java index 3087c76f..19ae578e 100644 --- a/src/main/java/org/peergos/AggregatedMetrics.java +++ b/src/main/java/org/peergos/AggregatedMetrics.java @@ -25,6 +25,7 @@ private static Counter build(String name, String help) { public static final Counter API_BLOCK_RM = build("api_block_rm", "Total calls to block/rm."); public static final Counter API_BLOCK_RM_BULK = build("api_block_rm_bulk", "Total calls to block/rm/bulk."); public static final Counter API_BLOCK_STAT = build("api_block_stat", "Total calls to block/stat."); + public static final Counter API_BLOCK_STAT_BULK = build("api_block_stat_bulk", "Total calls to block/stat/bulk."); public static final Counter API_REFS_LOCAL = build("api_refs_local", "Total calls to refs/local."); public static final Counter API_BLOCK_HAS = build("api_block_has", "Total calls to block/has."); public static final Counter API_BLOOM_ADD = build("api_bloom_add", "Total calls to bloom/add."); diff --git a/src/main/java/org/peergos/Want.java b/src/main/java/org/peergos/Want.java index 9de2b0fd..07300294 100644 --- a/src/main/java/org/peergos/Want.java +++ b/src/main/java/org/peergos/Want.java @@ -1,10 +1,11 @@ package org.peergos; import io.ipfs.cid.Cid; +import org.peergos.config.*; import java.util.*; -public class Want { +public class Want implements Jsonable { public final Cid cid; public final Optional authHex; public Want(Cid cid, Optional authHex) { @@ -24,6 +25,17 @@ public boolean equals(Object o) { return cid.equals(want.cid) && authHex.equals(want.authHex); } + public Map toJson() { + Map m = new LinkedHashMap<>(); + m.put("c", cid.toString()); + authHex.ifPresent(h -> m.put("a", h)); + return m; + } + + public static Want fromJson(Map m) { + return new Want(Cid.decode(m.get("c")), Optional.ofNullable(m.get("a"))); + } + @Override public int hashCode() { return Objects.hash(cid, authHex); diff --git a/src/main/java/org/peergos/net/APIHandler.java b/src/main/java/org/peergos/net/APIHandler.java index a1e6af40..9de199ee 100755 --- a/src/main/java/org/peergos/net/APIHandler.java +++ b/src/main/java/org/peergos/net/APIHandler.java @@ -5,6 +5,7 @@ import io.libp2p.core.PeerId; import io.libp2p.crypto.keys.*; import org.peergos.*; +import org.peergos.cbor.*; import org.peergos.protocol.ipns.*; import org.peergos.protocol.ipns.pb.*; import org.peergos.util.*; @@ -30,6 +31,7 @@ public class APIHandler extends Handler { public static final String RM = "block/rm"; public static final String RM_BULK = "block/rm/bulk"; public static final String STAT = "block/stat"; + public static final String BULK_STAT = "block/stat/bulk"; public static final String REFS_LOCAL = "refs/local"; public static final String BLOOM_ADD = "bloom/add"; public static final String HAS = "block/has"; @@ -99,6 +101,31 @@ public void handleCallToAPI(HttpExchange httpExchange) { } break; } + case BULK_STAT: { + AggregatedMetrics.API_BLOCK_STAT_BULK.inc(); + Map json = (Map) JSONParser.parse(new String(readFully(httpExchange.getRequestBody()))); + List wants = ((List>)json.get("wants")) + .stream() + .map(Want::fromJson) + .collect(Collectors.toList()); + Set peers = Optional.ofNullable(params.get("peers")) + .map(p -> p.stream().map(PeerId::fromBase58).collect(Collectors.toSet())) + .orElse(Collections.emptySet()); + List blocks = ipfs.getBlocks(wants, peers, true); + if (wants.size() == blocks.size()) { + List> links = blocks.stream() + .map(b -> CborObject.getLinks(b.hash, b.block).stream().map(Cid::toString).collect(Collectors.toList())) + .collect(Collectors.toList()); + replyJson(httpExchange, JSONParser.toString(links)); + } else { + try { + httpExchange.sendResponseHeaders(400, 0); + } catch (IOException ioe) { + HttpUtil.replyError(httpExchange, ioe); + } + } + break; + } case PUT: { // https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-block-put AggregatedMetrics.API_BLOCK_PUT.inc(); List format = params.get("format"); From 4a433158219697f730fc0d5bfcdc8d8cf5bcf383 Mon Sep 17 00:00:00 2001 From: ian Date: Fri, 12 Apr 2024 11:22:07 +0100 Subject: [PATCH 02/11] Fix mirroring raw blocks via bulk stat api --- src/main/java/org/peergos/net/APIHandler.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/peergos/net/APIHandler.java b/src/main/java/org/peergos/net/APIHandler.java index 9de199ee..73f58352 100755 --- a/src/main/java/org/peergos/net/APIHandler.java +++ b/src/main/java/org/peergos/net/APIHandler.java @@ -114,7 +114,9 @@ public void handleCallToAPI(HttpExchange httpExchange) { List blocks = ipfs.getBlocks(wants, peers, true); if (wants.size() == blocks.size()) { List> links = blocks.stream() - .map(b -> CborObject.getLinks(b.hash, b.block).stream().map(Cid::toString).collect(Collectors.toList())) + .map(b -> b.hash.codec.equals(Cid.Codec.Raw) ? + Collections.emptyList() : + CborObject.getLinks(b.hash, b.block).stream().map(Cid::toString).collect(Collectors.toList())) .collect(Collectors.toList()); replyJson(httpExchange, JSONParser.toString(links)); } else { From de9479bf655c1a9a3d46898df5e4f705f1073656 Mon Sep 17 00:00:00 2001 From: ian Date: Fri, 12 Apr 2024 14:43:33 +0100 Subject: [PATCH 03/11] Don't return identity hash links --- src/main/java/org/peergos/net/APIHandler.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/peergos/net/APIHandler.java b/src/main/java/org/peergos/net/APIHandler.java index 73f58352..aeea8a2f 100755 --- a/src/main/java/org/peergos/net/APIHandler.java +++ b/src/main/java/org/peergos/net/APIHandler.java @@ -116,7 +116,11 @@ public void handleCallToAPI(HttpExchange httpExchange) { List> links = blocks.stream() .map(b -> b.hash.codec.equals(Cid.Codec.Raw) ? Collections.emptyList() : - CborObject.getLinks(b.hash, b.block).stream().map(Cid::toString).collect(Collectors.toList())) + CborObject.getLinks(b.hash, b.block) + .stream() + .filter(c -> c.getType() != Multihash.Type.id) + .map(Cid::toString) + .collect(Collectors.toList())) .collect(Collectors.toList()); replyJson(httpExchange, JSONParser.toString(links)); } else { From 24eba3d5c896daa77732f29489811882a585307d Mon Sep 17 00:00:00 2001 From: ian Date: Fri, 12 Apr 2024 17:02:42 +0100 Subject: [PATCH 04/11] Add failing bitswap test --- src/test/java/org/peergos/BitswapTest.java | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/test/java/org/peergos/BitswapTest.java b/src/test/java/org/peergos/BitswapTest.java index eba5dbf6..1f6c95e5 100644 --- a/src/test/java/org/peergos/BitswapTest.java +++ b/src/test/java/org/peergos/BitswapTest.java @@ -46,6 +46,42 @@ public void getBlock() { } } + @Test + public void getTenBlocks() { + HostBuilder builder1 = HostBuilder.create(TestPorts.getPort(), + new RamProviderStore(1000), new RamRecordStore(), new RamBlockstore(), (c, p, a) -> CompletableFuture.completedFuture(true)); + Host node1 = builder1.build(); + RamBlockstore blockstore2 = new RamBlockstore(); + HostBuilder builder2 = HostBuilder.create(TestPorts.getPort(), + new RamProviderStore(1000), new RamRecordStore(), blockstore2, (c, p, a) -> CompletableFuture.completedFuture(true)); + Host node2 = builder2.build(); + node1.start().join(); + node2.start().join(); + try { + Multiaddr address2 = node2.listenAddresses().get(0); + List hashes = new ArrayList<>(); + Random random = new Random(28); + for (int i=0; i < 10; i++) { + byte[] blockData = new byte[1024*1024]; + random.nextBytes(blockData); + Cid hash = blockstore2.put(blockData, Cid.Codec.Raw).join(); + hashes.add(hash); + } + + Bitswap bitswap1 = builder1.getBitswap().get(); + node1.getAddressBook().addAddrs(address2.getPeerId(), 0, address2).join(); + List receivedBlocks = bitswap1.get(hashes.stream().map(Want::new).collect(Collectors.toList()), node1, Set.of(address2.getPeerId()), false) + .stream() + .map(f -> f.join()) + .collect(Collectors.toList()); + if (receivedBlocks.size() != hashes.size()) + throw new IllegalStateException("Incorrect number of blocks returned!"); + } finally { + node1.stop(); + node2.stop(); + } + } + @Test public void blockFlooder() { HostBuilder builder1 = HostBuilder.create(TestPorts.getPort(), From 3df14f4d6d6e296df53436fe7de7636d73b63662 Mon Sep 17 00:00:00 2001 From: ian Date: Wed, 17 Apr 2024 12:30:00 +0100 Subject: [PATCH 05/11] Fix receiving > 2MiB of blocks in a bitswap stream --- src/main/java/org/peergos/blockstore/RamBlockstore.java | 5 +++-- .../java/org/peergos/protocol/bitswap/BitswapEngine.java | 2 +- .../org/peergos/protocol/bitswap/BitswapProtocol.java | 9 +++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/peergos/blockstore/RamBlockstore.java b/src/main/java/org/peergos/blockstore/RamBlockstore.java index 41b69e06..248bbaa2 100644 --- a/src/main/java/org/peergos/blockstore/RamBlockstore.java +++ b/src/main/java/org/peergos/blockstore/RamBlockstore.java @@ -4,6 +4,7 @@ import io.ipfs.multihash.*; import org.peergos.*; import org.peergos.blockstore.metadatadb.BlockMetadata; +import org.peergos.cbor.*; import org.peergos.util.*; import java.util.*; @@ -60,7 +61,7 @@ public CompletableFuture> refs(boolean useBlockstore) { @Override public CompletableFuture getBlockMetadata(Cid h) { - throw new IllegalStateException("Unsupported operation!"); + byte[] block = get(h).join().get(); + return Futures.of(new BlockMetadata(block.length, CborObject.getLinks(h, block))); } - } diff --git a/src/main/java/org/peergos/protocol/bitswap/BitswapEngine.java b/src/main/java/org/peergos/protocol/bitswap/BitswapEngine.java index 381a2f60..3e9927c8 100644 --- a/src/main/java/org/peergos/protocol/bitswap/BitswapEngine.java +++ b/src/main/java/org/peergos/protocol/bitswap/BitswapEngine.java @@ -265,7 +265,7 @@ public void receiveMessage(MessageOuterClass.Message msg, Stream source, Counter waiter.result.complete(new HashedBlock(c, data)); localWants.remove(w); } else - LOG.info("Received block we don't want: z" + c.toBase58() + " from " + sourcePeerId.bareMultihash()); + LOG.info("Received block we don't want: " + c + " from " + sourcePeerId.bareMultihash()); } } catch (IOException e) { e.printStackTrace(); diff --git a/src/main/java/org/peergos/protocol/bitswap/BitswapProtocol.java b/src/main/java/org/peergos/protocol/bitswap/BitswapProtocol.java index bfc9da43..4754133e 100644 --- a/src/main/java/org/peergos/protocol/bitswap/BitswapProtocol.java +++ b/src/main/java/org/peergos/protocol/bitswap/BitswapProtocol.java @@ -5,7 +5,7 @@ import io.prometheus.client.*; import org.jetbrains.annotations.*; import org.peergos.protocol.bitswap.pb.*; -import org.peergos.util.Logging; +import org.peergos.util.*; import java.util.concurrent.*; import java.util.logging.*; @@ -36,7 +36,7 @@ public class BitswapProtocol extends ProtobufProtocolHandler private final BitswapEngine engine; public BitswapProtocol(BitswapEngine engine) { - super(MessageOuterClass.Message.getDefaultInstance(), engine.maxMessageSize(), engine.maxMessageSize()); + super(MessageOuterClass.Message.getDefaultInstance(), engine.maxMessageSize(), Long.MAX_VALUE); this.engine = engine; } @@ -78,5 +78,10 @@ public void onMessage(@NotNull Stream stream, MessageOuterClass.Message msg) { receivedBytes.inc(msg.getSerializedSize()); engine.receiveMessage(msg, stream, sentBytes); } + + @Override + public void onException(@Nullable Throwable cause) { + LOG.log(Level.WARNING, cause.getMessage(), cause); + } } } From bcd28ffcc3e54022439c9c0db0f5bc05e98b9c80 Mon Sep 17 00:00:00 2001 From: ian Date: Wed, 17 Apr 2024 12:30:55 +0100 Subject: [PATCH 06/11] Bump version --- src/main/java/org/peergos/net/APIHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/peergos/net/APIHandler.java b/src/main/java/org/peergos/net/APIHandler.java index aeea8a2f..3faca1c8 100755 --- a/src/main/java/org/peergos/net/APIHandler.java +++ b/src/main/java/org/peergos/net/APIHandler.java @@ -19,7 +19,7 @@ public class APIHandler extends Handler { public static final String API_URL = "/api/v0/"; - public static final Version CURRENT_VERSION = Version.parse("0.7.6"); + public static final Version CURRENT_VERSION = Version.parse("0.7.7"); private static final Logger LOG = Logging.LOG(); private static final boolean LOGGING = true; From 3f197e5bef4168bc8e51d44751c59d86c241e7ba Mon Sep 17 00:00:00 2001 From: ian Date: Wed, 17 Apr 2024 13:46:01 +0100 Subject: [PATCH 07/11] Bump version in pom --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index be66de19..2ca98887 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.github.peergos nabu - v0.7.6 + v0.7.7 UTF-8 From d426bc6bc627d5c31ae06d1ed8cac634e944732a Mon Sep 17 00:00:00 2001 From: ian Date: Fri, 19 Apr 2024 09:21:41 +0100 Subject: [PATCH 08/11] Log up to 2 cids we are requesting on bitswap --- src/main/java/org/peergos/protocol/bitswap/Bitswap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/peergos/protocol/bitswap/Bitswap.java b/src/main/java/org/peergos/protocol/bitswap/Bitswap.java index d3448ab9..d7a02a51 100644 --- a/src/main/java/org/peergos/protocol/bitswap/Bitswap.java +++ b/src/main/java/org/peergos/protocol/bitswap/Bitswap.java @@ -124,7 +124,7 @@ public void sendWants(Host us, Set wants, Set peers) { Map haves = engine.getHaves(); // broadcast to all connected bitswap peers if none are supplied Set audience = peers.isEmpty() ? getBroadcastAudience() : peers; - LOG.info("Send wants: " + wants.size() + " to " + audience); + LOG.info("Send wants: " + wants.size() + " to " + audience + " cids: " + wants.stream().limit(2).map(w -> w.cid).collect(Collectors.toList())); List wantsProto = wants.stream() .map(want -> MessageOuterClass.Message.Wantlist.Entry.newBuilder() .setWantType(audience.size() <= 2 || haves.containsKey(want) ? From 04e5ca5f42e18d120cd634d9a2faff1a5349bc44 Mon Sep 17 00:00:00 2001 From: ian Date: Fri, 19 Apr 2024 09:28:07 +0100 Subject: [PATCH 09/11] Do not try and dial DNS multiaddr in Kademlia which is a privacy leak --- src/main/java/org/peergos/protocol/dht/Kademlia.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/peergos/protocol/dht/Kademlia.java b/src/main/java/org/peergos/protocol/dht/Kademlia.java index 4633d8fa..07cccbb7 100644 --- a/src/main/java/org/peergos/protocol/dht/Kademlia.java +++ b/src/main/java/org/peergos/protocol/dht/Kademlia.java @@ -293,6 +293,7 @@ private Multiaddr[] getPublic(PeerAddresses target) { private CompletableFuture dialPeer(PeerAddresses target, Host us) { Multiaddr[] multiaddrs = target.addresses.stream() .map(a -> Multiaddr.fromString(a.toString())) + .filter(a -> ! a.has(Protocol.DNS) && ! a.has(Protocol.DNS4) && ! a.has(Protocol.DNS6)) .collect(Collectors.toList()).toArray(new Multiaddr[0]); return dial(us, PeerId.fromBase58(target.peerId.toBase58()), multiaddrs).getController(); } From 3f7eb42e869e42335c750faa31ac0e697b43e92e Mon Sep 17 00:00:00 2001 From: ian Date: Mon, 29 Apr 2024 10:30:47 +0100 Subject: [PATCH 10/11] Remove bouncy castle signatures from jar --- pom.xml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pom.xml b/pom.xml index 2ca98887..54802f42 100644 --- a/pom.xml +++ b/pom.xml @@ -47,6 +47,25 @@ ${kotlin.version} true + + org.apache.maven.plugins + maven-dependency-plugin + 2.6 + + + unpack-dependencies + package + + unpack-dependencies + + + system + META-INF/*.SF,META-INF/*.DSA,META-INF/*.RSA + ${project.build.directory}/classes + + + + org.apache.maven.plugins maven-compiler-plugin @@ -101,6 +120,16 @@ true + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + From 1da5bbf13758bd4c4660642276be8c4a0a26782c Mon Sep 17 00:00:00 2001 From: ian Date: Mon, 29 Apr 2024 10:31:58 +0100 Subject: [PATCH 11/11] Bump version v0.7.8 --- pom.xml | 2 +- src/main/java/org/peergos/net/APIHandler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 54802f42..4737fcaf 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.github.peergos nabu - v0.7.7 + v0.7.8 UTF-8 diff --git a/src/main/java/org/peergos/net/APIHandler.java b/src/main/java/org/peergos/net/APIHandler.java index 3faca1c8..caefc788 100755 --- a/src/main/java/org/peergos/net/APIHandler.java +++ b/src/main/java/org/peergos/net/APIHandler.java @@ -19,7 +19,7 @@ public class APIHandler extends Handler { public static final String API_URL = "/api/v0/"; - public static final Version CURRENT_VERSION = Version.parse("0.7.7"); + public static final Version CURRENT_VERSION = Version.parse("0.7.8"); private static final Logger LOG = Logging.LOG(); private static final boolean LOGGING = true;