Add multiaddr as subpackage (#637)

* Add multiaddr as subpackage

* Remove py-multiaddr dependency

* Move to hivemind.utils, add copyright notices

* Add multiaddr tests

* Add newline in requirements.txt

---------

Co-authored-by: Max Ryabinin <mryabinin0@gmail.com>

Author: dvmazur

hivemind/dht/dht.py

@@ -7,13 +7,12 @@
 from functools import partial
 from typing import Awaitable, Callable, Iterable, List, Optional, Sequence, TypeVar, Union
 
-from multiaddr import Multiaddr
-
 from hivemind.dht.node import DEFAULT_NUM_WORKERS, DHTNode
 from hivemind.dht.routing import DHTKey, DHTValue, Subkey
 from hivemind.dht.validation import CompositeValidator, RecordValidatorBase
 from hivemind.p2p import P2P, PeerID
 from hivemind.utils import MPFuture, get_logger, switch_to_uvloop
+from hivemind.utils.multiaddr import Multiaddr
 from hivemind.utils.timed_storage import DHTExpiration, ValueWithExpiration
 
 logger = get_logger(__name__)

hivemind/dht/node.py

@@ -23,7 +23,6 @@
     Union,
 )
 
-from multiaddr import Multiaddr
 from sortedcontainers import SortedSet
 
 from hivemind.dht.crypto import DHTRecord, RecordValidatorBase
@@ -34,6 +33,7 @@
 from hivemind.p2p import P2P, PeerID
 from hivemind.utils import MSGPackSerializer, SerializerBase, get_logger
 from hivemind.utils.auth import AuthorizerBase
+from hivemind.utils.multiaddr import Multiaddr
 from hivemind.utils.timed_storage import DHTExpiration, TimedStorage, ValueWithExpiration
 
 logger = get_logger(__name__)

hivemind/p2p/p2p_daemon.py

@@ -12,7 +12,6 @@
 from typing import Any, AsyncIterator, Awaitable, Callable, List, Optional, Sequence, Tuple, Type, TypeVar, Union
 
 from google.protobuf.message import Message
-from multiaddr import Multiaddr
 
 import hivemind.hivemind_cli as cli
 import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient
@@ -24,6 +23,7 @@
 from hivemind.utils.asyncio import as_aiter, asingle
 from hivemind.utils.crypto import RSAPrivateKey
 from hivemind.utils.logging import get_logger, golog_level_to_python, loglevel, python_level_to_golog
+from hivemind.utils.multiaddr import Multiaddr
 
 logger = get_logger(__name__)
 

hivemind/p2p/p2p_daemon_bindings/control.py

@@ -9,8 +9,6 @@
 from typing import AsyncIterator, Awaitable, Callable, Dict, Iterable, Optional, Sequence, Tuple
 from uuid import UUID, uuid4
 
-from multiaddr import Multiaddr, protocols
-
 from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, PeerInfo, StreamInfo
 from hivemind.p2p.p2p_daemon_bindings.utils import (
     DispatchFailure,
@@ -22,6 +20,7 @@
 )
 from hivemind.proto import p2pd_pb2 as p2pd_pb
 from hivemind.utils.logging import get_logger
+from hivemind.utils.multiaddr import Multiaddr, protocols
 
 StreamHandler = Callable[[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter], Awaitable[None]]
 

hivemind/p2p/p2p_daemon_bindings/datastructures.py

@@ -10,9 +10,9 @@
 import base58
 import multihash
 from cryptography.hazmat.primitives import serialization
-from multiaddr import Multiaddr
 
 from hivemind.proto import crypto_pb2, p2pd_pb2
+from hivemind.utils.multiaddr import Multiaddr
 
 
 class PeerID:

hivemind/p2p/p2p_daemon_bindings/p2pclient.py

@@ -8,8 +8,6 @@
 from contextlib import asynccontextmanager
 from typing import AsyncIterator, Iterable, Sequence, Tuple
 
-from multiaddr import Multiaddr
-
 from hivemind.p2p.p2p_daemon_bindings.control import (
     DEFAULT_MAX_MSG_SIZE,
     ControlClient,
@@ -18,6 +16,7 @@
     TUnaryHandler,
 )
 from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, PeerInfo, StreamInfo
+from hivemind.utils.multiaddr import Multiaddr
 
 
 class Client:

hivemind/utils/multiaddr/__init__.py

@@ -0,0 +1,29 @@
+# This code is originally taken from https://github.com/multiformats/py-multiaddr
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2014-2015 Steven Buss
+# Copyright (c) 2019-2020 Alexander Schlarb
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+from .multiaddr import Multiaddr  # NOQA
+
+__author__ = "Steven Buss"
+__email__ = "steven.buss@gmail.com"
+__version__ = "0.0.9"

hivemind/utils/multiaddr/codecs/__init__.py

@@ -0,0 +1,45 @@
+# This code is originally taken from https://github.com/multiformats/py-multiaddr
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2014-2015 Steven Buss
+# Copyright (c) 2019-2020 Alexander Schlarb
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import importlib
+
+# These are special sizes
+LENGTH_PREFIXED_VAR_SIZE = -1
+
+
+class NoneCodec:
+    SIZE = 0
+    IS_PATH = False
+
+
+CODEC_CACHE = {}
+
+
+def codec_by_name(name):
+    if name is None:  # Special “do nothing – expect nothing” pseudo-codec
+        return NoneCodec
+    codec = CODEC_CACHE.get(name)
+    if not codec:
+        codec = CODEC_CACHE[name] = importlib.import_module(".{0}".format(name), __name__)
+    return codec

hivemind/utils/multiaddr/codecs/cid.py

@@ -0,0 +1,152 @@
+# This code is originally taken from https://github.com/multiformats/py-multiaddr
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2014-2015 Steven Buss
+# Copyright (c) 2019-2020 Alexander Schlarb
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import base58
+import cid
+
+from . import LENGTH_PREFIXED_VAR_SIZE
+
+SIZE = LENGTH_PREFIXED_VAR_SIZE
+IS_PATH = False
+
+
+# Spec: https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#string-representation
+CIDv0_PREFIX_TO_LENGTH = {
+    # base58btc prefixes for valid lengths 1 – 42 with the identity “hash” function
+    "12": [5, 12, 19, 23, 30, 41, 52, 56],
+    "13": [9, 16, 34, 45],
+    "14": [27, 38, 49, 60],
+    "15": [3, 6, 20],
+    "16": [3, 6, 13, 20, 31, 42, 53],
+    "17": [3, 13, 42],
+    "18": [3],
+    "19": [3, 24, 57],
+    "1A": [24, 35, 46],
+    "1B": [35],
+    "1D": [17],
+    "1E": [10, 17],
+    "1F": [10],
+    "1G": [10, 28, 50],
+    "1H": [28, 39],
+    "1P": [21],
+    "1Q": [21],
+    "1R": [21, 54],
+    "1S": [54],
+    "1T": [7, 32, 43],
+    "1U": [7, 32, 43],
+    "1V": [7],
+    "1W": [7, 14],
+    "1X": [7, 14],
+    "1Y": [7, 14],
+    "1Z": [7, 14],
+    "1f": [4],
+    "1g": [4, 58],
+    "1h": [4, 25, 58],
+    "1i": [4, 25],
+    "1j": [4, 25],
+    "1k": [4, 25, 47],
+    "1m": [4, 36, 47],
+    "1n": [4, 36],
+    "1o": [4, 36],
+    "1p": [4],
+    "1q": [4],
+    "1r": [4],
+    "1s": [4],
+    "1t": [4],
+    "1u": [4],
+    "1v": [4],
+    "1w": [4],
+    "1x": [4],
+    "1y": [4],
+    "1z": [4, 18],
+    # base58btc prefix for length 42 with the sha256 hash function
+    "Qm": [46],
+}
+
+PROTO_NAME_TO_CIDv1_CODEC = {
+    # The “p2p” multiaddr protocol requires all keys to use the “libp2p-key” multicodec
+    "p2p": "libp2p-key",
+}
+
+
+def to_bytes(proto, string):
+    expected_codec = PROTO_NAME_TO_CIDv1_CODEC.get(proto.name)
+
+    if len(string) in CIDv0_PREFIX_TO_LENGTH.get(string[0:2], ()):  # CIDv0
+        # Upgrade the wire (binary) representation of any received CIDv0 string
+        # to CIDv1 if we can determine which multicodec value to use
+        if expected_codec:
+            return cid.make_cid(1, expected_codec, base58.b58decode(string)).buffer
+
+        return base58.b58decode(string)
+    else:  # CIDv1+
+        parsed = cid.from_string(string)
+
+        # Ensure CID has correct codec for protocol
+        if expected_codec and parsed.codec != expected_codec:
+            raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec".format(proto.name, expected_codec))
+
+        return parsed.buffer
+
+
+def _is_binary_cidv0_multihash(buf):
+    if buf.startswith(b"\x12\x20") and len(buf) == 34:  # SHA2-256
+        return True
+
+    if (buf[0] == 0x00 and buf[1] in range(43)) and len(buf) == (buf[1] + 2):  # Identity hash
+        return True
+
+    return False
+
+
+def to_string(proto, buf):
+    expected_codec = PROTO_NAME_TO_CIDv1_CODEC.get(proto.name)
+
+    if _is_binary_cidv0_multihash(buf):  # CIDv0
+        if not expected_codec:
+            # Simply encode as base58btc as there is nothing better to do
+            return base58.b58encode(buf).decode("ascii")
+
+        # “Implementations SHOULD display peer IDs using the first (raw
+        #  base58btc encoded multihash) format until the second format is
+        #  widely supported.”
+        #
+        # In the future the following line should instead convert the multihash
+        # to CIDv1 and with the `expected_codec` and wrap it in base32:
+        #   return cid.make_cid(1, expected_codec, buf).encode("base32").decode("ascii")
+        return base58.b58encode(buf).decode("ascii")
+    else:  # CIDv1+
+        parsed = cid.from_bytes(buf)
+
+        # Ensure CID has correct codec for protocol
+        if expected_codec and parsed.codec != expected_codec:
+            raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec".format(proto.name, expected_codec))
+
+        # “Implementations SHOULD display peer IDs using the first (raw
+        #  base58btc encoded multihash) format until the second format is
+        #  widely supported.”
+        if expected_codec and _is_binary_cidv0_multihash(parsed.multihash):
+            return base58.b58encode(parsed.multihash).decode("ascii")
+
+        return parsed.encode("base32").decode("ascii")

hivemind/utils/multiaddr/codecs/domain.py

@@ -0,0 +1,41 @@
+# This code is originally taken from https://github.com/multiformats/py-multiaddr
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2014-2015 Steven Buss
+# Copyright (c) 2019-2020 Alexander Schlarb
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import idna
+
+from . import LENGTH_PREFIXED_VAR_SIZE
+
+SIZE = LENGTH_PREFIXED_VAR_SIZE
+IS_PATH = False
+
+
+def to_bytes(proto, string):
+    return idna.uts46_remap(string).encode("utf-8")
+
+
+def to_string(proto, buf):
+    string = buf.decode("utf-8")
+    for label in string.split("."):
+        idna.check_label(label)
+    return string