Merge pull request #703 from skalenetwork/beta

2.1 Stable

.github/workflows/issue_check.yml

@@ -0,0 +1,19 @@
+name: Get linked issues
+on:
+  pull_request:
+    types: [ edited, synchronize, opened, reopened ]
+
+jobs:
+  check-linked-issues:
+    name: Check if pull request has linked issues
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get issues
+        id: get-issues
+        uses: mondeja/pr-linked-issues-action@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: PR has not linked issues
+        if: join(steps.get-issues.outputs.issues) == ''
+        run: 
+          exit 1

CODEOWNERS

@@ -1,2 +1,2 @@
-* @dmitry-tk @kladkogex @badrogger
+* @dmytrotkk @kladkogex @badrogger
 *.md @skalenetwork/docowners

node_cli/cli/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.1.2'
+__version__ = '2.2.0'
 
 if __name__ == "__main__":
     print(__version__)

node_cli/cli/health.py

@@ -47,7 +47,7 @@ def containers(all):
 @click.option(
     '--json',
     'json_format',
-    help=G_TEXTS['common']['json']['help'],
+    help=G_TEXTS['common']['json'],
     is_flag=True
 )
 def schains(json_format: bool) -> None:

node_cli/cli/node.py

@@ -17,58 +17,40 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-import ipaddress
-from urllib.parse import urlparse
-
 import click
 
 from node_cli.core.node import (
     configure_firewall_rules,
-    get_node_signature, init, restore,
+    get_node_signature,
+    init,
+    restore,
     register_node as register,
-    update, backup,
-    set_maintenance_mode_on, set_maintenance_mode_off,
-    turn_off, turn_on, get_node_info,
-    set_domain_name, run_checks
+    update,
+    backup,
+    set_maintenance_mode_on,
+    set_maintenance_mode_off,
+    turn_off,
+    turn_on,
+    get_node_info,
+    set_domain_name,
+    run_checks
 )
 from node_cli.configs import DEFAULT_NODE_BASE_PORT
 from node_cli.configs.env import ALLOWED_ENV_TYPES
-from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd
+from node_cli.utils.decorators import check_inited
+from node_cli.utils.helper import (
+    abort_if_false,
+    safe_load_texts,
+    streamed_cmd,
+    IP_TYPE
+)
+from node_cli.utils.meta import get_meta_info
+from node_cli.utils.print_formatters import print_meta_info
 
 
 TEXTS = safe_load_texts()
 
 
-class UrlType(click.ParamType):
-    name = 'url'
-
-    def convert(self, value, param, ctx):
-        try:
-            result = urlparse(value)
-        except ValueError:
-            self.fail(f'Some characters are not allowed in {value}',
-                      param, ctx)
-        if not all([result.scheme, result.netloc]):
-            self.fail(f'Expected valid url. Got {value}', param, ctx)
-        return value
-
-
-class IpType(click.ParamType):
-    name = 'ip'
-
-    def convert(self, value, param, ctx):
-        try:
-            ipaddress.ip_address(value)
-        except ValueError:
-            self.fail(f'expected valid ipv4/ipv6 address. Got {value}',
-                      param, ctx)
-        return value
-
-
-URL_TYPE = UrlType()
-IP_TYPE = IpType()
-
-
 @click.group()
 def node_cli():
     pass
@@ -236,3 +218,19 @@ def check(network):
               prompt='Are you sure you want to reconfigure firewall rules?')
 def configure_firewall():
     configure_firewall_rules()
+
+
+@node.command(help='Show node version information')
+@check_inited
+@click.option(
+    '--json',
+    'raw',
+    is_flag=True,
+    help=TEXTS['common']['json']
+)
+def version(raw: bool) -> None:
+    meta_info = get_meta_info(raw=raw)
+    if raw:
+        print(meta_info)
+    else:
+        print_meta_info(meta_info)

node_cli/cli/schains.py

@@ -17,9 +17,11 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+from typing import Optional
+
 import click
 
-from node_cli.utils.helper import abort_if_false
+from node_cli.utils.helper import abort_if_false, IP_TYPE
 from node_cli.core.schains import (
     describe,
     get_schain_firewall_rules,
@@ -72,8 +74,15 @@ def show_rules(schain_name: str) -> None:
 @click.option('--yes', is_flag=True, callback=abort_if_false,
               expose_value=False,
               prompt='Are you sure? Repair mode may corrupt working SKALE chain data.')
-def repair(schain_name: str) -> None:
-    toggle_schain_repair_mode(schain_name)
+@click.option(
+    '--snapshot-from',
+    type=IP_TYPE,
+    default=None,
+    hidden=True,
+    help='Ip of the node from to download snapshot from'
+)
+def repair(schain_name: str, snapshot_from: Optional[str] = None) -> None:
+    toggle_schain_repair_mode(schain_name, snapshot_from=snapshot_from)
 
 
 @schains.command('info', help='Show info about schain')

node_cli/configs/__init__.py

@@ -51,12 +51,7 @@
 SGX_CERTIFICATES_DIR_NAME = 'sgx_certs'
 
 COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml')
-FILESTORAGE_INFO_FILE = os.path.join(
-    CONTAINER_CONFIG_PATH, 'filestorage_info.json')
-FILESTORAGE_ARTIFACTS_FILE = os.path.join(
-    NODE_DATA_PATH, 'filestorage_artifacts.json')
-ENVIRONMENT_PARAMS_FILEPATH = os.path.join(
-    CONTAINER_CONFIG_PATH, 'environment_params.yaml')
+STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'static_params.yaml')
 NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2')
 NGINX_CONFIG_FILEPATH = os.path.join(NODE_DATA_PATH, 'nginx.conf')
 
@@ -81,6 +76,7 @@
 
 IPTABLES_DIR = '/etc/iptables/'
 IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4')
+DEFAULT_SSH_PORT = 22
 
 FLASK_SECRET_KEY_FILENAME = 'flask_db_key.txt'
 FLASK_SECRET_KEY_FILE = os.path.join(NODE_DATA_PATH, FLASK_SECRET_KEY_FILENAME)
@@ -152,3 +148,6 @@ def _get_env():
 DOCKER_SOCKET_PATH = '/var/run/skale/docker.sock'
 
 CHECK_REPORT_PATH = os.path.join(REPORTS_PATH, 'checks.json')
+
+AUTOLOAD_KERNEL_MODULES_PATH = '/etc/modules'
+BTRFS_KERNEL_MODULE = 'btrfs'

node_cli/configs/node_options.py

@@ -0,0 +1,23 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of node-cli
+#
+#   Copyright (C) 2022 SKALE Labs
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import os
+from node_cli.configs import NODE_DATA_PATH
+
+NODE_OPTIONS_FILEPATH = os.path.join(NODE_DATA_PATH, 'node_options.json')

node_cli/core/checks.py

@@ -46,7 +46,8 @@
     CONTAINER_CONFIG_PATH,
     DOCKER_CONFIG_FILEPATH,
     DOCKER_DAEMON_HOSTS,
-    REPORTS_PATH
+    REPORTS_PATH,
+    STATIC_PARAMS_FILEPATH
 )
 from node_cli.core.resources import get_disk_size
 from node_cli.utils.helper import run_cmd, safe_mkdir
@@ -66,15 +67,13 @@
 FuncList = List[Func]
 
 
-def get_env_params(
+def get_static_params(
     env_type: str = 'mainnet',
     config_path: str = CONTAINER_CONFIG_PATH
 ) -> Dict:
-    environment_params_path = os.path.join(
-        config_path,
-        'environment_params.yaml'
-    )
-    with open(environment_params_path) as requirements_file:
+    status_params_filename = os.path.basename(STATIC_PARAMS_FILEPATH)
+    static_params_filepath = os.path.join(config_path, status_params_filename)
+    with open(static_params_filepath) as requirements_file:
         ydata = yaml.load(requirements_file, Loader=yaml.Loader)
         return ydata['envs'][env_type]
 
@@ -208,9 +207,14 @@ def check(self) -> ResultList:
 
 
 class MachineChecker(BaseChecker):
-    def __init__(self, requirements: Dict, disk_device: str) -> None:
+    def __init__(
+            self,
+            requirements: Dict,
+            disk_device: str,
+            network_timeout: Optional[int] = None) -> None:
         self.requirements = requirements
         self.disk_device = disk_device
+        self.network_timeout = network_timeout or NETWORK_CHECK_TIMEOUT
 
     @preinstall
     def cpu_total(self) -> CheckResult:
@@ -281,7 +285,7 @@ def disk(self) -> CheckResult:
     def network(self) -> CheckResult:
         name = 'network'
         try:
-            socket.setdefaulttimeout(NETWORK_CHECK_TIMEOUT)
+            socket.setdefaulttimeout(self.network_timeout)
             socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect(
                 (CLOUDFLARE_DNS_HOST, CLOUDFLARE_DNS_HOST_PORT))
             return self._ok(name=name)
@@ -521,7 +525,7 @@ def run_checks(
     check_type: CheckType = CheckType.ALL
 ) -> ResultList:
     logger.info('Executing checks. Type: %s', check_type)
-    requirements = get_env_params(env_type, config_path)
+    requirements = get_static_params(env_type, config_path)
     checkers = get_all_checkers(disk, requirements)
     checks = get_checks(checkers, check_type)
     results = [check() for check in checks]

node_cli/core/host.py

@@ -26,7 +26,8 @@
 from node_cli.core.resources import update_resource_allocation
 
 from node_cli.configs import (
-    ADMIN_PORT, DEFAULT_URL_SCHEME, NODE_DATA_PATH,
+    ADMIN_PORT, AUTOLOAD_KERNEL_MODULES_PATH,
+    BTRFS_KERNEL_MODULE, DEFAULT_URL_SCHEME, NODE_DATA_PATH,
     SKALE_DIR, CONTAINER_CONFIG_PATH, CONTRACTS_PATH,
     ETH_STATE_PATH, NODE_CERTS_PATH, SGX_CERTS_PATH,
     REPORTS_PATH, REDIS_DATA_PATH,
@@ -124,6 +125,38 @@ def init_data_dir():
     safe_mkdir(NODE_DATA_PATH)
 
 
+def is_btrfs_module_autoloaded(modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH):
+    if not os.path.isfile(modules_filepath):
+        return False
+    with open(modules_filepath) as modules_file:
+        modules = set(
+            map(
+                lambda line: line.strip(),
+                filter(
+                    lambda line: not line.startswith('#'),
+                    modules_file.readlines()
+                )
+            )
+        )
+        return BTRFS_KERNEL_MODULE in modules
+
+
+def add_btrfs_module_to_autoload(modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH):
+    with open(modules_filepath, 'a') as modules_file:
+        modules_file.write(f'{BTRFS_KERNEL_MODULE}\n')
+
+
+def ensure_btrfs_kernel_module_autoloaded(
+    modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH
+):
+    logger.debug('Checking if btrfs is in %s', modules_filepath)
+    if not is_btrfs_module_autoloaded(modules_filepath):
+        logger.info('Adding btrfs module to %s', modules_filepath)
+        add_btrfs_module_to_autoload(modules_filepath)
+    else:
+        logger.debug('btrfs is already in %s', modules_filepath)
+
+
 def validate_abi_files(json_result=False):
     results = [
         validate_abi(abi_filepath)

node_cli/core/iptables.py

@@ -18,9 +18,16 @@
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import logging
+import socket
 import sys
 from pathlib import Path
-from node_cli.configs import IPTABLES_DIR, IPTABLES_RULES_STATE_FILEPATH, ENV
+
+from node_cli.configs import (
+    IPTABLES_DIR,
+    IPTABLES_RULES_STATE_FILEPATH,
+    ENV,
+    DEFAULT_SSH_PORT
+)
 from node_cli.utils.helper import run_cmd
 
 
@@ -38,7 +45,6 @@
 
 ALLOWED_INCOMING_TCP_PORTS = [
     '80',  # filestorage
-    '22',  # ssh
     '311',  # watchdog https
     '8080',  # http
     '443',  # https
@@ -131,15 +137,32 @@ def drop_all_udp(chain: iptc.Chain) -> None:
     ensure_rule(chain, r)
 
 
+def get_ssh_port(ssh_service_name='ssh'):
+    try:
+        return socket.getservbyname(ssh_service_name)
+    except OSError:
+        logger.exception('Cannot get ssh service port')
+        return DEFAULT_SSH_PORT
+
+
+def allow_ssh(chain: iptc.Chain) -> None:
+    ssh_port = get_ssh_port()
+    accept_incoming(chain, str(ssh_port), 'tcp')
+
+
 def allow_base_ports(chain: iptc.Chain) -> None:
-    logger.debug('Allowing base ports...')
+    logger.info('Configuring ssh port')
+    allow_ssh(chain)
+    logger.info('Configuring incoming tcp ports')
     for port in ALLOWED_INCOMING_TCP_PORTS:
         accept_incoming(chain, port, 'tcp')
+    logger.info('Configuring incoming udp ports')
     for port in ALLOWED_INCOMING_UDP_PORTS:
         accept_incoming(chain, port, 'udp')
 
 
-def accept_incoming(chain, port, protocol) -> None:
+def accept_incoming(chain: iptc.Chain, port: str, protocol: str) -> None:
+    logger.debug('Going to allow %s traffic from %s port', protocol, port)
     rule = iptc.Rule()
     rule.protocol = protocol
     match = iptc.Match(rule, protocol)