Added documentation for the DHT(Discovery)Community

doc/further-reading/dht.rst

@@ -0,0 +1,44 @@
+DHT(Discovery)Community
+=======================
+
+This document contains a description of how to use the ``DHTCommunity`` class for distributed hash table (DHT) data storage and the ``DHTDiscoveryCommunity`` extension of this functionality, which provides functionality to connect given public keys.
+
+In particular this document will **not** discuss how distributed hash table work, for this we refer the reader to other resources on the Internet.
+
+
+Storing values and finding keys
+-------------------------------
+
+The ``DHTCommunity`` is the main overlay that allows for decentralized key-value storage.
+There are two main functions in this overlay: the ``store_value()`` function and the ``find_values()`` function.
+
+When you call ``store_value()``, you choose the globally unique ``key`` that your given value ``data`` is stored under.
+You can, but are not required to, sign this new stored value with your public key to provide it with authenticity.
+Note that this function may lead to a ``ipv8.dht.DHTError``: in this case, you will have to try again later.
+An example of a call that stores the signed value ``b"my value"`` under the key ``b"my key"``, is the following.
+
+.. literalinclude:: dht_1.py
+   :lines: 43-47
+
+The value can later be retrieved from the network by calling ``find_values()`` with the key that the information was stored under.
+The following snippet retrieves the value that was stored in the previous snippet, under the ``b"my key"`` key.
+
+.. literalinclude:: dht_1.py
+   :lines: 49-55
+
+Note that multiple peers may respond with answers and if (a) the orginal value is not signed or (b) multiple values are published under the same key, the reported values may be different.
+In this example, only one value is published and it is signed so only a single value is ever returned.
+
+Finding peers
+-------------
+
+The ``DHTDiscoveryCommunity`` allows for peers to be found by their public key.
+You can search for public keys by their SHA-1 hash (conveniently available as ``Peer.mid``).
+To do so, you can call ``connect_peer()`` with the hash/mid as shown in the following example.
+
+
+.. literalinclude:: dht_1.py
+   :lines: 58-65
+
+Note that you may need a few attempts to find the peer you are looking for.
+Of course, if the peer you are looking for is not online, you may be waiting forever.

doc/further-reading/dht_1.py

@@ -0,0 +1,67 @@
+from asyncio import run, sleep
+from itertools import combinations
+from typing import cast
+
+from ipv8.configuration import ConfigBuilder
+from ipv8.dht import DHTError
+from ipv8.dht.community import DHTCommunity
+from ipv8.dht.discovery import DHTDiscoveryCommunity
+from ipv8.peer import Peer
+from ipv8_service import IPv8
+
+
+async def main() -> None:
+    instances = []
+
+    # Put some peers in the network
+    for _ in range(10):
+        config = ConfigBuilder().clear_keys()
+        config.config["overlays"] = [o for o in config.config["overlays"] if o["class"] == "DHTDiscoveryCommunity"]
+        config.add_ephemeral_key("anonymous id")
+        config.set_address("127.0.0.1")  # We don't want this test to connect to the actual network!
+        ipv8 = IPv8(config.finalize())
+        instances.append(ipv8)
+        await ipv8.start()
+
+    # Supercharge introductions, normally this takes longer
+    for id1, id2 in combinations(range(10), 2):
+        overlay1 = instances[id1].get_overlay(DHTCommunity)
+        overlay2 = instances[id2].get_overlay(DHTCommunity)
+        peer1 = Peer(overlay2.my_peer.public_key.key_to_bin(), ("127.0.0.1", overlay2.my_estimated_lan[1]))
+        peer1.address_frozen = True
+        peer2 = Peer(overlay1.my_peer.public_key.key_to_bin(), ("127.0.0.1", overlay1.my_estimated_lan[1]))
+        peer2.address_frozen = True
+        overlay1.network.add_verified_peer(peer2)
+        overlay1.get_requesting_node(peer2)
+        overlay2.network.add_verified_peer(peer1)
+        overlay2.get_requesting_node(peer1)
+    for i in range(10):
+        await instances[i].get_overlay(DHTDiscoveryCommunity).store_peer()
+        instances[i].get_overlay(DHTDiscoveryCommunity).ping_all()
+
+    dht_community = cast(DHTCommunity, instances[0].get_overlay(DHTCommunity))
+    try:
+        await dht_community.store_value(b"my key", b"my value", True)
+        print(dht_community.my_peer.public_key.key_to_bin(), "published b'my value' under b'my key'!")
+    except DHTError as e:
+        print("Failed to store my value under my key!", e)
+
+    try:
+        results = await dht_community.find_values(b"my key")
+        print(f"We got results from {len(results)} peers!")
+        for value, signer_key in results:
+            print(f"The value {value} was found, signed by {signer_key}")
+    except DHTError as e:
+        print("Failed to find key!", e)
+
+    dht_discovery_community = cast(DHTDiscoveryCommunity, instances[7].get_overlay(DHTDiscoveryCommunity))
+    some_peer_mid = instances[2].keys["anonymous id"].mid
+    while True:
+        try:
+            await sleep(0.5)
+            await dht_discovery_community.connect_peer(some_peer_mid)
+            break
+        except DHTError as e:
+            print("Failed to connect to peer!", e)
+
+run(main())

doc/index.rst

@@ -82,13 +82,7 @@ Table of contents
    further-reading/advanced_identity.rst
    further-reading/advanced_peer_discovery.rst
    further-reading/anonymization.rst
-
-.. toctree::
-   :maxdepth: 2
-   :caption: Deprecated/Archive:
-
-   deprecated/attestation_prototype.rst
-   deprecated/attestation_tutorial.rst
+   further-reading/dht.rst
 
 
 Search

ipv8/dht/community.py

@@ -9,6 +9,7 @@
 from binascii import hexlify, unhexlify
 from collections import defaultdict, deque
 from collections.abc import Coroutine, Iterator, Sequence
+from functools import reduce
 from itertools import zip_longest
 from typing import TYPE_CHECKING, Any, Optional, cast
 
@@ -625,9 +626,7 @@ def post_process_values(self, values: list[bytes]) -> list[DHTValue]:
         return [*results, *((data[1], None) for data in unpacked[None])]
 
     async def find(self, target: bytes, force_nodes: bool, offset: int,
-                   debug: bool) -> Sequence[DHTValue] | \
-                                   tuple[Sequence[DHTValue], list[Crawl]] | \
-                                   Sequence[Node]:
+                   debug: bool) -> tuple[list[DHTValue | Node], ...] | tuple[tuple[DHTValue, ...], list[Crawl]]:
         """
         Get the values belonging to the given target key.
         """
@@ -637,14 +636,13 @@ async def find(self, target: bytes, force_nodes: bool, offset: int,
         for routing_table in self.routing_tables.values():
             crawl = Crawl(target, routing_table, force_nodes=force_nodes, offset=offset)
             futures.append(self._find(crawl, debug=debug))
-        results: list[list[Any] |
-                      list[DHTValue] |
+        results: list[list[DHTValue | Node] |
                       tuple[list[DHTValue], Crawl]] = await gather(*futures)
 
         if debug:
             results_debug = cast(list[tuple[list[DHTValue], Crawl]], results)
             return tuple(*[r[0] for r in results]), [r[1] for r in results_debug]
-        return tuple(*results)
+        return tuple(reduce(lambda current, element: current.extend(element) or current, results, []))
 
     async def find_values(self, target: bytes, offset: int = 0,
                           debug: bool = False) -> Sequence[DHTValue] | tuple[Sequence[DHTValue], list[Crawl]]: