From 3fc61030e423235aa5307d681cdf1611cdc2c185 Mon Sep 17 00:00:00 2001 From: Alecks Gates Date: Mon, 31 Oct 2022 23:39:04 -0500 Subject: [PATCH 1/5] Add lighthive node to circuit breaker cache on bad error responses Re-use the lighthive client as much as possible. --- src/podping_hivewriter/cli/podping.py | 11 +++++++++-- src/podping_hivewriter/hive.py | 7 +++++++ src/podping_hivewriter/podping_hivewriter.py | 13 ++++++++++++- src/podping_hivewriter/podping_settings.py | 12 ++++++++---- src/podping_hivewriter/podping_settings_manager.py | 9 +++++++-- ..._failure_retry_handles_invalid_error_response.py | 2 +- 6 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/podping_hivewriter/cli/podping.py b/src/podping_hivewriter/cli/podping.py index 4aa1319..d413011 100644 --- a/src/podping_hivewriter/cli/podping.py +++ b/src/podping_hivewriter/cli/podping.py @@ -6,6 +6,7 @@ import rfc3987 import typer from lighthive.broadcast.base58 import Base58 +from lighthive.client import Client from podping_hivewriter import __version__ from podping_hivewriter.constants import ( @@ -83,8 +84,8 @@ class Config: testnet: bool testnet_node: str testnet_chainid: str - operation_id: str + lighthive_client: Client def exit_cli(_): @@ -231,6 +232,7 @@ def server( settings_manager = PodpingSettingsManager( ignore_updates=Config.ignore_config_updates, hive_operation_period=Config.hive_operation_period, + client=Config.lighthive_client, ) _podping_hivewriter = PodpingHivewriter( @@ -246,6 +248,7 @@ def server( dry_run=Config.dry_run, daemon=True, status=Config.status, + client=Config.lighthive_client, ) try: @@ -374,7 +377,11 @@ def callback( # Check the account exists posting_keys = [hive_posting_key] - client = get_client(posting_keys=posting_keys) + client = get_client( + posting_keys=posting_keys, + loglevel=logging.ERROR, + ) + Config.lighthive_client = client account_exists = client.get_accounts([hive_account]) if not account_exists: logging.error( diff --git a/src/podping_hivewriter/hive.py b/src/podping_hivewriter/hive.py index 086f0a7..6b8cd83 100644 --- a/src/podping_hivewriter/hive.py +++ b/src/podping_hivewriter/hive.py @@ -71,6 +71,13 @@ def get_allowed_accounts( return set(master_account.following()) except Exception as e: logging.warning(f"Unable to get account followers: {e} - retrying") + client.circuit_breaker_cache[client.current_node] = True + logging.warning( + "Ignoring node %s for %d seconds", + client.current_node, + client.circuit_breaker_ttl, + ) + client.next_node() async def listen_for_custom_json_operations( diff --git a/src/podping_hivewriter/podping_hivewriter.py b/src/podping_hivewriter/podping_hivewriter.py index 9988dae..958f7e0 100644 --- a/src/podping_hivewriter/podping_hivewriter.py +++ b/src/podping_hivewriter/podping_hivewriter.py @@ -10,6 +10,7 @@ from typing import List, Optional, Set, Tuple, Union import rfc3987 +from lighthive.client import Client from lighthive.datastructures import Operation from lighthive.exceptions import RPCNodeException @@ -54,6 +55,7 @@ def __init__( dry_run=False, daemon=True, status=True, + client: Client = None, ): super().__init__() @@ -71,7 +73,7 @@ def __init__( self.daemon: bool = daemon self.status: bool = status - self.lighthive_client = get_client( + self.lighthive_client = client or get_client( posting_keys=posting_keys, loglevel=logging.ERROR, ) @@ -512,6 +514,15 @@ async def failure_retry( sys.exit(EXIT_CODE_INVALID_POSTING_KEY) except (KeyError, AttributeError): logging.warning("Malformed error response") + self.lighthive_client.circuit_breaker_cache[ + self.lighthive_client.current_node + ] = True + logging.warning( + "Ignoring node %s for %d seconds", + self.lighthive_client.current_node, + self.lighthive_client.circuit_breaker_ttl, + ) + self.lighthive_client.next_node() except NotEnoughResourceCredits as ex: logging.warning(ex) # 10s + exponential back off: need time for RC delegation diff --git a/src/podping_hivewriter/podping_settings.py b/src/podping_hivewriter/podping_settings.py index 16b3e65..6018d2f 100644 --- a/src/podping_hivewriter/podping_settings.py +++ b/src/podping_hivewriter/podping_settings.py @@ -8,10 +8,12 @@ from podping_hivewriter.models.podping_settings import PodpingSettings -async def get_settings_from_hive(account_name: str) -> Optional[dict]: +async def get_settings_from_hive( + account_name: str, client: Client = None +) -> Optional[dict]: """Returns podping settings if they exist""" # Must use main chain for settings - client: Client = get_client() + client: Client = client or get_client() account = client.account(account_name) raw_meta = account.raw_data.get("posting_json_metadata") if raw_meta: @@ -23,7 +25,9 @@ async def get_settings_from_hive(account_name: str) -> Optional[dict]: logging.error(f"posting_json_metadata for account {account_name} is empty") -async def get_podping_settings(account_name: str) -> PodpingSettings: +async def get_podping_settings( + account_name: str, client: Client = None +) -> PodpingSettings: """Return PodpingSettings object""" - settings_dict = await get_settings_from_hive(account_name) + settings_dict = await get_settings_from_hive(account_name, client) return PodpingSettings.parse_obj(settings_dict) diff --git a/src/podping_hivewriter/podping_settings_manager.py b/src/podping_hivewriter/podping_settings_manager.py index c05f178..c52924c 100644 --- a/src/podping_hivewriter/podping_settings_manager.py +++ b/src/podping_hivewriter/podping_settings_manager.py @@ -2,6 +2,7 @@ import logging from timeit import default_timer as timer +from lighthive.client import Client from pydantic import ValidationError from podping_hivewriter.async_context import AsyncContext @@ -10,9 +11,13 @@ class PodpingSettingsManager(AsyncContext): - def __init__(self, ignore_updates=False, hive_operation_period=None): + def __init__( + self, ignore_updates=False, hive_operation_period=None, client: Client = None + ): super().__init__() + self.client = client + self.ignore_updates = ignore_updates if hive_operation_period: self.override_hive_operation_period = hive_operation_period @@ -45,7 +50,7 @@ async def _update_podping_settings_loop(self): async def update_podping_settings(self) -> None: try: podping_settings = await get_podping_settings( - self._settings.control_account + self._settings.control_account, self.client ) self.last_update_time = timer() except ValidationError as e: diff --git a/tests/regression/test_#51_failure_retry_handles_invalid_error_response.py b/tests/regression/test_#51_failure_retry_handles_invalid_error_response.py index 902cd39..4c280a0 100644 --- a/tests/regression/test_#51_failure_retry_handles_invalid_error_response.py +++ b/tests/regression/test_#51_failure_retry_handles_invalid_error_response.py @@ -63,7 +63,7 @@ def mock_broadcast(*args, **kwargs): writer.close() - logging_warning_stub.assert_called_once_with("Malformed error response") + assert logging_warning_stub.call_count == 2 assert logging_error_stub.call_count == 4 assert failure_count == 1 From 3f1ca81e1ddd9c1c29629648dd059ebf0b152198 Mon Sep 17 00:00:00 2001 From: Alecks Gates Date: Mon, 31 Oct 2022 23:40:24 -0500 Subject: [PATCH 2/5] Version 1.2.8 --- poetry.lock | 87 +++++++++++++++++++++++++++++--------------------- pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 53 insertions(+), 38 deletions(-) diff --git a/poetry.lock b/poetry.lock index ace11ce..838c107 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,6 +1,6 @@ [[package]] name = "anyio" -version = "3.6.1" +version = "3.6.2" description = "High level compatibility layer for multiple asynchronous event loop implementations" category = "main" optional = false @@ -13,7 +13,7 @@ sniffio = ">=1.1" [package.extras] doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"] -trio = ["trio (>=0.16)"] +trio = ["trio (>=0.16,<0.22)"] [[package]] name = "asgiref" @@ -38,7 +38,7 @@ python-versions = ">=3.5" dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "backoff" @@ -143,7 +143,7 @@ optional = false python-versions = ">=3.6.0" [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "click" @@ -155,11 +155,11 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "colorama" -version = "0.4.5" +version = "0.4.6" description = "Cross-platform colored terminal text." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" [[package]] name = "coverage" @@ -176,7 +176,7 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 toml = ["tomli"] [[package]] -name = "Cython" +name = "cython" version = "0.29.32" description = "The Cython compiler for writing C extensions for the Python language." category = "main" @@ -218,6 +218,17 @@ six = ">=1.9.0" gmpy = ["gmpy"] gmpy2 = ["gmpy2"] +[[package]] +name = "exceptiongroup" +version = "1.0.0" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "flake8" version = "5.0.4" @@ -243,8 +254,8 @@ python-versions = ">=3.6" smmap = ">=3.0.1,<6" [[package]] -name = "GitPython" -version = "3.1.28" +name = "gitpython" +version = "3.1.29" description = "GitPython is a python library used to interact with Git repositories" category = "dev" optional = false @@ -325,9 +336,9 @@ python-versions = ">=3.6.1,<4.0" [package.extras] colors = ["colorama (>=0.4.3,<0.5.0)"] -pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +pipfile-deprecated-finder = ["pipreqs", "requirementslib"] plugins = ["setuptools"] -requirements_deprecated_finder = ["pip-api", "pipreqs"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] [[package]] name = "lighthive" @@ -414,7 +425,7 @@ python-versions = ">=3.7" [[package]] name = "pbr" -version = "5.10.0" +version = "5.11.0" description = "Python Build Reasonableness" category = "dev" optional = false @@ -515,7 +526,7 @@ py = "*" [[package]] name = "pytest" -version = "7.1.3" +version = "7.2.0" description = "pytest: simple powerful testing with Python" category = "dev" optional = false @@ -524,11 +535,11 @@ python-versions = ">=3.7" [package.dependencies] attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" -py = ">=1.8.2" -tomli = ">=1.0.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] @@ -600,14 +611,14 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2022.4" +version = "2022.6" description = "World timezone definitions, modern and historical" category = "main" optional = false python-versions = "*" [[package]] -name = "PyYAML" +name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" category = "dev" @@ -642,7 +653,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "rfc3986" @@ -708,7 +719,7 @@ python-versions = ">=3.7" [[package]] name = "stevedore" -version = "4.0.0" +version = "4.1.0" description = "Manage dynamic plugins for Python applications" category = "dev" optional = false @@ -788,8 +799,8 @@ content-hash = "7ea13c8285a8f95efbbfef8a8475bd73c97d68fbe053ff3326df9283534e8295 [metadata.files] anyio = [ - {file = "anyio-3.6.1-py3-none-any.whl", hash = "sha256:cb29b9c70620506a9a8f87a309591713446953302d7d995344d0d7c6c0c9a7be"}, - {file = "anyio-3.6.1.tar.gz", hash = "sha256:413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b"}, + {file = "anyio-3.6.2-py3-none-any.whl", hash = "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"}, + {file = "anyio-3.6.2.tar.gz", hash = "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421"}, ] asgiref = [ {file = "asgiref-3.5.2-py3-none-any.whl", hash = "sha256:1d2880b792ae8757289136f1db2b7b99100ce959b2aa57fd69dab783d05afac4"}, @@ -902,8 +913,8 @@ click = [ {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] colorama = [ - {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, - {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] coverage = [ {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, @@ -957,7 +968,7 @@ coverage = [ {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, ] -Cython = [ +cython = [ {file = "Cython-0.29.32-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:39afb4679b8c6bf7ccb15b24025568f4f9b4d7f9bf3cbd981021f542acecd75b"}, {file = "Cython-0.29.32-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:dbee03b8d42dca924e6aa057b836a064c769ddfd2a4c2919e65da2c8a362d528"}, {file = "Cython-0.29.32-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ba622326f2862f9c1f99ca8d47ade49871241920a352c917e16861e25b0e5c3"}, @@ -1010,6 +1021,10 @@ ecdsa = [ {file = "ecdsa-0.18.0-py2.py3-none-any.whl", hash = "sha256:80600258e7ed2f16b9aa1d7c295bd70194109ad5a30fdee0eaeefef1d4c559dd"}, {file = "ecdsa-0.18.0.tar.gz", hash = "sha256:190348041559e21b22a1d65cee485282ca11a6f81d503fddb84d5017e9ed1e49"}, ] +exceptiongroup = [ + {file = "exceptiongroup-1.0.0-py3-none-any.whl", hash = "sha256:2ac84b496be68464a2da60da518af3785fff8b7ec0d090a581604bc870bdee41"}, + {file = "exceptiongroup-1.0.0.tar.gz", hash = "sha256:affbabf13fb6e98988c38d9c5650e701569fe3c1de3233cfb61c5f33774690ad"}, +] flake8 = [ {file = "flake8-5.0.4-py2.py3-none-any.whl", hash = "sha256:7a1cf6b73744f5806ab95e526f6f0d8c01c66d7bbe349562d22dfca20610b248"}, {file = "flake8-5.0.4.tar.gz", hash = "sha256:6fbe320aad8d6b95cec8b8e47bc933004678dc63095be98528b7bdd2a9f510db"}, @@ -1018,9 +1033,9 @@ gitdb = [ {file = "gitdb-4.0.9-py3-none-any.whl", hash = "sha256:8033ad4e853066ba6ca92050b9df2f89301b8fc8bf7e9324d412a63f8bf1a8fd"}, {file = "gitdb-4.0.9.tar.gz", hash = "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa"}, ] -GitPython = [ - {file = "GitPython-3.1.28-py3-none-any.whl", hash = "sha256:77bfbd299d8709f6af7e0c70840ef26e7aff7cf0c1ed53b42dd7fc3a310fcb02"}, - {file = "GitPython-3.1.28.tar.gz", hash = "sha256:6bd3451b8271132f099ceeaf581392eaf6c274af74bb06144307870479d0697c"}, +gitpython = [ + {file = "GitPython-3.1.29-py3-none-any.whl", hash = "sha256:41eea0deec2deea139b459ac03656f0dd28fc4a3387240ec1d3c259a2c47850f"}, + {file = "GitPython-3.1.29.tar.gz", hash = "sha256:cc36bfc4a3f913e66805a28e84703e419d9c264c1077e537b54f0e1af85dbefd"}, ] h11 = [ {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, @@ -1096,8 +1111,8 @@ pathspec = [ {file = "pathspec-0.10.1.tar.gz", hash = "sha256:7ace6161b621d31e7902eb6b5ae148d12cfd23f4a249b9ffb6b9fee12084323d"}, ] pbr = [ - {file = "pbr-5.10.0-py2.py3-none-any.whl", hash = "sha256:da3e18aac0a3c003e9eea1a81bd23e5a3a75d745670dcf736317b7d966887fdf"}, - {file = "pbr-5.10.0.tar.gz", hash = "sha256:cfcc4ff8e698256fc17ea3ff796478b050852585aa5bae79ecd05b2ab7b39b9a"}, + {file = "pbr-5.11.0-py2.py3-none-any.whl", hash = "sha256:db2317ff07c84c4c63648c9064a79fe9d9f5c7ce85a9099d4b6258b3db83225a"}, + {file = "pbr-5.11.0.tar.gz", hash = "sha256:b97bc6695b2aff02144133c2e7399d5885223d42b7912ffaec2ca3898e673bfe"}, ] platformdirs = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, @@ -1169,8 +1184,8 @@ pypytools = [ {file = "pypytools-0.6.2.tar.gz", hash = "sha256:a140c053e4d1c0bae835f40662eb0040a75190771acf2b2a8832dfa7beefe529"}, ] pytest = [ - {file = "pytest-7.1.3-py3-none-any.whl", hash = "sha256:1377bda3466d70b55e3f5cecfa55bb7cfcf219c7964629b967c37cf0bda818b7"}, - {file = "pytest-7.1.3.tar.gz", hash = "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39"}, + {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, + {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, ] pytest-asyncio = [ {file = "pytest-asyncio-0.19.0.tar.gz", hash = "sha256:ac4ebf3b6207259750bc32f4c1d8fcd7e79739edbc67ad0c58dd150b1d072fed"}, @@ -1193,10 +1208,10 @@ python-dateutil = [ {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] pytz = [ - {file = "pytz-2022.4-py2.py3-none-any.whl", hash = "sha256:2c0784747071402c6e99f0bafdb7da0fa22645f06554c7ae06bf6358897e9c91"}, - {file = "pytz-2022.4.tar.gz", hash = "sha256:48ce799d83b6f8aab2020e369b627446696619e79645419610b9facd909b3174"}, + {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, + {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, ] -PyYAML = [ +pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, @@ -1347,8 +1362,8 @@ sniffio = [ {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] stevedore = [ - {file = "stevedore-4.0.0-py3-none-any.whl", hash = "sha256:87e4d27fe96d0d7e4fc24f0cbe3463baae4ec51e81d95fbe60d2474636e0c7d8"}, - {file = "stevedore-4.0.0.tar.gz", hash = "sha256:f82cc99a1ff552310d19c379827c2c64dd9f85a38bcd5559db2470161867b786"}, + {file = "stevedore-4.1.0-py3-none-any.whl", hash = "sha256:3b1cbd592a87315f000d05164941ee5e164899f8fc0ce9a00bb0f321f40ef93e"}, + {file = "stevedore-4.1.0.tar.gz", hash = "sha256:02518a8f0d6d29be8a445b7f2ac63753ff29e8f2a2faa01777568d5500d777a6"}, ] tomli = [ {file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"}, diff --git a/pyproject.toml b/pyproject.toml index 8f0f39f..16c3129 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "podping-hivewriter" -version = "1.2.7" +version = "1.2.8" license = "MIT" authors = ["Alecks Gates ", "Brian of London "] maintainers = ["Alecks Gates ", "Brian of London "] diff --git a/setup.py b/setup.py index 172692c..39fef7e 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup_kwargs = { "name": "podping-hivewriter", - "version": "1.2.7", + "version": "1.2.8", "description": "This is a tool used to submit RFC 3987-compliant International Resource Identifiers as a Podping notification on the Hive blockchain.", "long_description": "# podping-hivewriter\nThe Hive writer component of Podping. You will need a Hive account, see section [Hive account and Authorization](#hive-account) below.\n\n## What is Podping?\n\nPodping is a mechanism of using decentralized communication to relay notification of updates of RSS feeds that use The Podcast Namespace. It does so by supplying minimum relevant metadata to consumers to be able to make efficient and actionable decisions, allowing them to decide what to do with given RSS feeds without parsing them ahead of time.\n\n*This* project provides a standardized way of posting a \"podping\" specifically to the Hive blockcahin.\n\n## Running podping-hivewriter\n\nThe project has two modes of running: `write` mode and `server` mode.\n\n`write` mode is primarily useful for people with a very small number of feeds to publish updates for relatively infrequently (i.e. a few times a day or less).\n\n`server` mode is for hosts (or other services like the Podcast Index's [podping.cloud](https://podping.cloud/)) who publish updates for a significant amount of feeds on a regular basis. Not that the average small-time podcast can't run it, but it's overkill. This mode is for efficiency only, as the `server` will batch process feeds as they come in to make the most use of the Hive blockchain.\n\nSee the dedicated [CLI docs](CLI.md) for more information on configuration options, including environment variables.\n\n### Container\n\nThe container images are hosted on [Docker Hub](https://hub.docker.com/r/podcastindexorg/podping-hivewriter). Images are currently based on Debian bullseye-based PyPy 3.8 with the following architectures: `amd64`\n\nThese images can be run in either `write` or `server` mode and is likely the easiest option for users who do not have experience installing Python packages.\n\n#### Command Line\n\nRunning in `write` mode with command line options, like `--dry-run` for example, add them with the full podping command.\nSettings can also be passed with the `-e` option for Docker. Note, we leave out `-p 9999:9999` here because we're not running the server.\n\n```shell\ndocker run --rm \\\n -e PODPING_HIVE_ACCOUNT= \\\n -e PODPING_HIVE_POSTING_KEY= \\\n docker.io/podcastindexorg/podping-hivewriter \\\n --dry-run write https://www.example.com/feed.xml\n```\n\nRun in `server` mode, passing local port 9999 to port 9999 in the container.\nENV variables can be passed to docker with `--env-file` option after modifying the `.env.EXAMPLE` file and renaming it to `.env`\n\n```shell\ndocker run --rm -p 9999:9999 --env-file .env --name podping docker.io/podcastindexorg/podping-hivewriter\n```\n\nAs another example for running in `server` mode, to run in *detached* mode, note the `-d` in the `docker run` options. Also note that `write` or `server` must come *after* the command line options for `podping`:\n```shell\ndocker run --rm -d \\\n -p 9999:9999 --env-file .env \\\n --name podping \\\n docker.io/podcastindexorg/podping-hivewriter \\\n --livetest server\n```\n\nOne running you can view and follow the live output with:\n```shell\ndocker logs podping -f\n```\n\nSee the [CLI docs](https://github.com/Podcastindex-org/podping-hivewriter/blob/main/CLI.md) for default values.\n\n\n#### docker-compose\n\n```yaml\nversion: '2.0'\nservices:\n podping-hivewriter:\n image: docker.io/podcastindexorg/podping-hivewriter\n restart: always\n ports:\n - \"9999:9999\"\n environment:\n - PODPING_HIVE_ACCOUNT=\n - PODPING_HIVE_POSTING_KEY=\n - PODPING_LISTEN_IP=0.0.0.0\n - PODPING_LISTEN_PORT=9999\n - PODPING_LIVETEST=false\n - PODPING_DRY_RUN=false\n - PODPING_STATUS=true\n - PODPING_IGNORE_CONFIG_UPDATES=false\n - PODPING_I_KNOW_WHAT_IM_DOING=false\n - PODPING_DEBUG=false\n```\n\nAssuming you just copy-pasted without reading, the above will fail at first. As noted in the [server command documentation](https://github.com/Podcastindex-org/podping-hivewriter/blob/main/CLI.md#podping-server):\n\n>WARNING: DO NOT run this on a publicly accessible host. There currently is NO authentication required to submit to the server. Set to * or 0.0.0.0 for all interfaces.\n\nAs all Docker installations vary, we set `0.0.0.0` as the listen IP for connectivity. This doesn't affect the IP address docker listens on when we tell it to pass port `9999` through to the container. If you understand the consequences of this, set `PODPING_I_KNOW_WHAT_IM_DOING` to `true`.\n\nThis is a temporary measure to limit potential misconfiguration until we fully bundle the `podping.cloud` HTTP front end. Then again, if you're running this, you're probably Dave.\n\n\n### CLI Install\n\nThe following have been tested on Linux and macOS. However, Windows should work also. If you have issues on Windows we highly recommend the [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/) and/or Docker.\n\n#### Using [pipx](https://pypa.github.io/pipx/) (preferred over pip)\n```shell\npipx install podping-hivewriter\n```\n\n#### Using pip\n```shell\npip install --user podping-hivewriter\n```\n\n#### Installing the server\n\nIf you'd like to install the server component, it's hidden behind the extra flag `server`. This is to make it easier to install only the `write` CLI component `podping-hivewriter` on non-standard systems without a configured development enviornment.\n\n```shell\npipx install podping-hivewriter[server]\n```\n\nMake sure you have `~/.local/bin/` on your `PATH`.\n\nSee the dedicated [CLI docs](CLI.md) for more information.\n\n## Podping reasons\n\nPodping accepts various different \"reasons\" for publishing updates to RSS feeds:\n\n* `update` -- A general indication that an RSS feed has been updated\n* `live` -- An indication that an RSS feed has been updated and a contained [``](https://github.com/Podcastindex-org/podcast-namespace/blob/main/docs/1.0.md#live-item) tag's status attribute has been changed to live.\n* `liveEnd` -- An indication that an RSS feed has been updated and either the status attribute of an existing [``](https://github.com/Podcastindex-org/podcast-namespace/blob/main/docs/1.0.md#live-item) has been changed from live to ended or a [``](https://github.com/Podcastindex-org/podcast-namespace/blob/main/docs/1.0.md#live-item) that previously had a status attribute of live has been removed from the feed entirely.\n\nThe canonical list of reasons within the scope of this project is [maintained in this schema](https://github.com/Podcastindex-org/podping-hivewriter/blob/main/src/podping_hivewriter/schema/reason.capnp).\n\n## Mediums\n\nPodping accepts various different \"mediums\" for identifying types of RSS feeds using the Podcast Namespace. Please check the [``](https://github.com/Podcastindex-org/podcast-namespace/blob/main/docs/1.0.md#medium) specification for the full list.\n\n`podping-hivewriter` *may* lag behind the specification, and if it does, please let us know or submit a pull request.\n\nThe canonical list of mediums within the scope of this project is [maintained in this schema](https://github.com/Podcastindex-org/podping-hivewriter/blob/main/src/podping_hivewriter/schema/medium.capnp).\n\n## Development\n\nYou'll need a few extras:\n\n1. [capnproto](https://capnproto.org/). Linux: `capnproto` package in your package manager. On a Mac: `brew instal capnp`\n2. [Poetry](https://python-poetry.org/docs/)\n\n\nWe use [poetry](https://python-poetry.org/) for dependency management. Once you have it, clone this repo and run:\n\n```shell\npoetry install\n```\n\nThen to switch to the virtual environment, use:\n\n```shell\npoetry shell\n```\nMake sure you have a the environment variables `PODPING_HIVE_ACCOUNT` and `PODPING_HIVE_POSTING_KEY` set.\n\nAfter that you should be able to run the `podping` command or run the tests:\n\n```shell\npytest\n```\n\nTo run all tests, make sure to set the necessary environment variables for your Hive account. This will take many minutes:\n\n```shell\npytest --runslow\n```\n\n### Building the image locally with Docker\n\nLocally build the podping-hivewriter container with a \"develop\" tag\n\n```shell\ndocker build -t podping-hivewriter:develop .\n```\n\nSee above for more details on running the docker CLI.\n\n## Hive account\n\nIf you need a Hive account, please download the [Hive Keychain extension for your browser](https://hive-keychain.com/) then use this link to get your account from [https://HiveOnboard.com?ref=podping](https://hiveonboard.com?ref=podping). You will need at least 20 Hive Power \"powered up\" to get started (worth around $10). Please contact [@brianoflondon](https://peakd.com/@brianoflondon) brian@podping.org if you need assistance getting set up.\n\nIf you use the [Hiveonboard]((https://hiveonboard.com?ref=podping)) link `podping` will **delegate** enough Hive Power to get you started. If, for any reason, Hiveonboard is not giving out free accounts, please contact [@brianoflondon](https://peakd.com/@brianoflondon) either on [PodcastIndex Social](https://podcastindex.social/invite/U2m6FY3T) or [Telegram](https://t.me/brianoflondon).\n\n### Permissions and Authorization\n\nYou don't need permission, but you do need to tell `podping` that you want to send valid `podpings`:\n\n- Hive is a so-called \"permissionless\" blockchain. Once you have a Hive Account and a minimal amount of Hive Power, that account can post to Hive, including sending `podpings`.\n- Nobody can block any valid Hive Account from sending and nobody can help you if you lose your keys.\n- Whilst anyone can post `podpings` to Hive, there is a need to register your Hive Accountname for those `podpings` to be recognized by all clients. This is merely a spam-prevention measure and clients may choose to ignore it.\n- Please contact new@podping.org or send a Hive Transfer to [@podping](https://peakd.com/@podping) to have your account validated.\n- Side note on keys: `podping` uses the `posting-key` which is the lowest value of the four Hive keys (`owner`, `active`, `memo`, `posting` and there is usually a `master password` which can generate all the keys). That is not to say that losing control of it is a good idea, but that key is not authorized to make financially important transfers. It can, however, post public information so should be treated carefully and kept secure.\n\nFor a [comprehensive explanation of Hive and Podping, please see this post](https://peakd.com/podping/@brianoflondon/podping-and-podcasting-20-funding-to-put-hive-at-the-center-of-global-podcasting-infrastructure).", "author": "Alecks Gates", From 9b9580aa72b5358b09471957f966fc2c3be4a96a Mon Sep 17 00:00:00 2001 From: Alecks Gates Date: Mon, 31 Oct 2022 23:58:50 -0500 Subject: [PATCH 3/5] Update pytest deps --- poetry.lock | 8 ++++---- pyproject.toml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 838c107..f155c7b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -546,7 +546,7 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2. [[package]] name = "pytest-asyncio" -version = "0.19.0" +version = "0.20.1" description = "Pytest support for asyncio" category = "dev" optional = false @@ -795,7 +795,7 @@ server = ["pyzmq"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "7ea13c8285a8f95efbbfef8a8475bd73c97d68fbe053ff3326df9283534e8295" +content-hash = "fb0f9e8cb9dc90f6c9ad3e01b4e261e03384d2f58c1fd0cf8a4abe64aeb7211d" [metadata.files] anyio = [ @@ -1188,8 +1188,8 @@ pytest = [ {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, ] pytest-asyncio = [ - {file = "pytest-asyncio-0.19.0.tar.gz", hash = "sha256:ac4ebf3b6207259750bc32f4c1d8fcd7e79739edbc67ad0c58dd150b1d072fed"}, - {file = "pytest_asyncio-0.19.0-py3-none-any.whl", hash = "sha256:7a97e37cfe1ed296e2e84941384bdd37c376453912d397ed39293e0916f521fa"}, + {file = "pytest-asyncio-0.20.1.tar.gz", hash = "sha256:626699de2a747611f3eeb64168b3575f70439b06c3d0206e6ceaeeb956e65519"}, + {file = "pytest_asyncio-0.20.1-py3-none-any.whl", hash = "sha256:2c85a835df33fda40fe3973b451e0c194ca11bc2c007eabff90bb3d156fc172b"}, ] pytest-cov = [ {file = "pytest-cov-4.0.0.tar.gz", hash = "sha256:996b79efde6433cdbd0088872dbc5fb3ed7fe1578b68cdbba634f14bb8dd0470"}, diff --git a/pyproject.toml b/pyproject.toml index 16c3129..2bda592 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,9 +35,9 @@ lighthive = "^0.4.0" [tool.poetry.dev-dependencies] black = "^21.5b2" isort = "^5.8.0" -pytest = "^7.1.2" +pytest = "^7.2.0" pytest-cov = "^4.0.0" -pytest-asyncio = "^0.19.0" +pytest-asyncio = "^0.20.0" bandit = "^1.7.4" pytest-timeout = "^2.0.2" typer-cli = "^0.0.12" From 95004ea674d3e57c28959a59e53e3bfe93f303aa Mon Sep 17 00:00:00 2001 From: Alecks Gates Date: Tue, 1 Nov 2022 00:25:54 -0500 Subject: [PATCH 4/5] Remove hard coded node whitelist --- src/podping_hivewriter/hive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/podping_hivewriter/hive.py b/src/podping_hivewriter/hive.py index 6b8cd83..73679f8 100644 --- a/src/podping_hivewriter/hive.py +++ b/src/podping_hivewriter/hive.py @@ -31,13 +31,13 @@ def get_client( nodes = [os.getenv("PODPING_TESTNET_NODE")] chain = {"chain_id": os.getenv("PODPING_TESTNET_CHAINID")} else: - nodes = [ + """nodes = [ "https://api.hive.blog", "https://api.deathwing.me", "https://hive-api.arcange.eu", "https://api.openhive.network", "https://api.hive.blue", - ] + ]""" client = Client( keys=posting_keys, nodes=nodes, From a8c3ddb18291f84a51fdde5e5248bce69f728203 Mon Sep 17 00:00:00 2001 From: Alecks Gates Date: Tue, 1 Nov 2022 00:42:21 -0500 Subject: [PATCH 5/5] Add whitelist back but shuffle node list --- src/podping_hivewriter/hive.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/podping_hivewriter/hive.py b/src/podping_hivewriter/hive.py index 73679f8..297ba3b 100644 --- a/src/podping_hivewriter/hive.py +++ b/src/podping_hivewriter/hive.py @@ -2,6 +2,7 @@ import itertools import logging import os +from random import shuffle from timeit import default_timer as timer from typing import List, Optional, Set @@ -31,13 +32,14 @@ def get_client( nodes = [os.getenv("PODPING_TESTNET_NODE")] chain = {"chain_id": os.getenv("PODPING_TESTNET_CHAINID")} else: - """nodes = [ + nodes = [ "https://api.hive.blog", "https://api.deathwing.me", "https://hive-api.arcange.eu", "https://api.openhive.network", "https://api.hive.blue", - ]""" + ] + shuffle(nodes) client = Client( keys=posting_keys, nodes=nodes,