diff --git a/.github/workflows/issue_check.yml b/.github/workflows/issue_check.yml new file mode 100644 index 00000000..ee5ec6b2 --- /dev/null +++ b/.github/workflows/issue_check.yml @@ -0,0 +1,19 @@ +name: Get linked issues +on: + pull_request: + types: [ edited, synchronize, opened, reopened ] + +jobs: + check-linked-issues: + name: Check if pull request has linked issues + runs-on: ubuntu-latest + steps: + - name: Get issues + id: get-issues + uses: mondeja/pr-linked-issues-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + - name: PR has not linked issues + if: join(steps.get-issues.outputs.issues) == '' + run: + exit 1 diff --git a/CODEOWNERS b/CODEOWNERS index fac00956..f8254e9d 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,2 +1,2 @@ -* @dmitry-tk @kladkogex @badrogger +* @dmytrotkk @kladkogex @badrogger *.md @skalenetwork/docowners diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index b670efde..680cf87f 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.1.2' +__version__ = '2.2.0' if __name__ == "__main__": print(__version__) diff --git a/node_cli/cli/health.py b/node_cli/cli/health.py index 808082c8..fef51a3d 100644 --- a/node_cli/cli/health.py +++ b/node_cli/cli/health.py @@ -47,7 +47,7 @@ def containers(all): @click.option( '--json', 'json_format', - help=G_TEXTS['common']['json']['help'], + help=G_TEXTS['common']['json'], is_flag=True ) def schains(json_format: bool) -> None: diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 6a54a805..4c3618a2 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -17,58 +17,40 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import ipaddress -from urllib.parse import urlparse - import click from node_cli.core.node import ( configure_firewall_rules, - get_node_signature, init, restore, + get_node_signature, + init, + restore, register_node as register, - update, backup, - set_maintenance_mode_on, set_maintenance_mode_off, - turn_off, turn_on, get_node_info, - set_domain_name, run_checks + update, + backup, + set_maintenance_mode_on, + set_maintenance_mode_off, + turn_off, + turn_on, + get_node_info, + set_domain_name, + run_checks ) from node_cli.configs import DEFAULT_NODE_BASE_PORT from node_cli.configs.env import ALLOWED_ENV_TYPES -from node_cli.utils.helper import abort_if_false, safe_load_texts, streamed_cmd +from node_cli.utils.decorators import check_inited +from node_cli.utils.helper import ( + abort_if_false, + safe_load_texts, + streamed_cmd, + IP_TYPE +) +from node_cli.utils.meta import get_meta_info +from node_cli.utils.print_formatters import print_meta_info TEXTS = safe_load_texts() -class UrlType(click.ParamType): - name = 'url' - - def convert(self, value, param, ctx): - try: - result = urlparse(value) - except ValueError: - self.fail(f'Some characters are not allowed in {value}', - param, ctx) - if not all([result.scheme, result.netloc]): - self.fail(f'Expected valid url. Got {value}', param, ctx) - return value - - -class IpType(click.ParamType): - name = 'ip' - - def convert(self, value, param, ctx): - try: - ipaddress.ip_address(value) - except ValueError: - self.fail(f'expected valid ipv4/ipv6 address. Got {value}', - param, ctx) - return value - - -URL_TYPE = UrlType() -IP_TYPE = IpType() - - @click.group() def node_cli(): pass @@ -236,3 +218,19 @@ def check(network): prompt='Are you sure you want to reconfigure firewall rules?') def configure_firewall(): configure_firewall_rules() + + +@node.command(help='Show node version information') +@check_inited +@click.option( + '--json', + 'raw', + is_flag=True, + help=TEXTS['common']['json'] +) +def version(raw: bool) -> None: + meta_info = get_meta_info(raw=raw) + if raw: + print(meta_info) + else: + print_meta_info(meta_info) diff --git a/node_cli/cli/schains.py b/node_cli/cli/schains.py index db70a49e..cd4a7e00 100644 --- a/node_cli/cli/schains.py +++ b/node_cli/cli/schains.py @@ -17,9 +17,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Optional + import click -from node_cli.utils.helper import abort_if_false +from node_cli.utils.helper import abort_if_false, IP_TYPE from node_cli.core.schains import ( describe, get_schain_firewall_rules, @@ -72,8 +74,15 @@ def show_rules(schain_name: str) -> None: @click.option('--yes', is_flag=True, callback=abort_if_false, expose_value=False, prompt='Are you sure? Repair mode may corrupt working SKALE chain data.') -def repair(schain_name: str) -> None: - toggle_schain_repair_mode(schain_name) +@click.option( + '--snapshot-from', + type=IP_TYPE, + default=None, + hidden=True, + help='Ip of the node from to download snapshot from' +) +def repair(schain_name: str, snapshot_from: Optional[str] = None) -> None: + toggle_schain_repair_mode(schain_name, snapshot_from=snapshot_from) @schains.command('info', help='Show info about schain') diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index df789afb..cb7a1b36 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -51,12 +51,7 @@ SGX_CERTIFICATES_DIR_NAME = 'sgx_certs' COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') -FILESTORAGE_INFO_FILE = os.path.join( - CONTAINER_CONFIG_PATH, 'filestorage_info.json') -FILESTORAGE_ARTIFACTS_FILE = os.path.join( - NODE_DATA_PATH, 'filestorage_artifacts.json') -ENVIRONMENT_PARAMS_FILEPATH = os.path.join( - CONTAINER_CONFIG_PATH, 'environment_params.yaml') +STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'static_params.yaml') NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') NGINX_CONFIG_FILEPATH = os.path.join(NODE_DATA_PATH, 'nginx.conf') @@ -81,6 +76,7 @@ IPTABLES_DIR = '/etc/iptables/' IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4') +DEFAULT_SSH_PORT = 22 FLASK_SECRET_KEY_FILENAME = 'flask_db_key.txt' FLASK_SECRET_KEY_FILE = os.path.join(NODE_DATA_PATH, FLASK_SECRET_KEY_FILENAME) @@ -152,3 +148,6 @@ def _get_env(): DOCKER_SOCKET_PATH = '/var/run/skale/docker.sock' CHECK_REPORT_PATH = os.path.join(REPORTS_PATH, 'checks.json') + +AUTOLOAD_KERNEL_MODULES_PATH = '/etc/modules' +BTRFS_KERNEL_MODULE = 'btrfs' diff --git a/node_cli/configs/node_options.py b/node_cli/configs/node_options.py new file mode 100644 index 00000000..6565b3d7 --- /dev/null +++ b/node_cli/configs/node_options.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +from node_cli.configs import NODE_DATA_PATH + +NODE_OPTIONS_FILEPATH = os.path.join(NODE_DATA_PATH, 'node_options.json') diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 8ccd1d64..e094fe99 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -46,7 +46,8 @@ CONTAINER_CONFIG_PATH, DOCKER_CONFIG_FILEPATH, DOCKER_DAEMON_HOSTS, - REPORTS_PATH + REPORTS_PATH, + STATIC_PARAMS_FILEPATH ) from node_cli.core.resources import get_disk_size from node_cli.utils.helper import run_cmd, safe_mkdir @@ -66,15 +67,13 @@ FuncList = List[Func] -def get_env_params( +def get_static_params( env_type: str = 'mainnet', config_path: str = CONTAINER_CONFIG_PATH ) -> Dict: - environment_params_path = os.path.join( - config_path, - 'environment_params.yaml' - ) - with open(environment_params_path) as requirements_file: + status_params_filename = os.path.basename(STATIC_PARAMS_FILEPATH) + static_params_filepath = os.path.join(config_path, status_params_filename) + with open(static_params_filepath) as requirements_file: ydata = yaml.load(requirements_file, Loader=yaml.Loader) return ydata['envs'][env_type] @@ -208,9 +207,14 @@ def check(self) -> ResultList: class MachineChecker(BaseChecker): - def __init__(self, requirements: Dict, disk_device: str) -> None: + def __init__( + self, + requirements: Dict, + disk_device: str, + network_timeout: Optional[int] = None) -> None: self.requirements = requirements self.disk_device = disk_device + self.network_timeout = network_timeout or NETWORK_CHECK_TIMEOUT @preinstall def cpu_total(self) -> CheckResult: @@ -281,7 +285,7 @@ def disk(self) -> CheckResult: def network(self) -> CheckResult: name = 'network' try: - socket.setdefaulttimeout(NETWORK_CHECK_TIMEOUT) + socket.setdefaulttimeout(self.network_timeout) socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect( (CLOUDFLARE_DNS_HOST, CLOUDFLARE_DNS_HOST_PORT)) return self._ok(name=name) @@ -521,7 +525,7 @@ def run_checks( check_type: CheckType = CheckType.ALL ) -> ResultList: logger.info('Executing checks. Type: %s', check_type) - requirements = get_env_params(env_type, config_path) + requirements = get_static_params(env_type, config_path) checkers = get_all_checkers(disk, requirements) checks = get_checks(checkers, check_type) results = [check() for check in checks] diff --git a/node_cli/core/host.py b/node_cli/core/host.py index f6736c4f..2dbdcc14 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -26,7 +26,8 @@ from node_cli.core.resources import update_resource_allocation from node_cli.configs import ( - ADMIN_PORT, DEFAULT_URL_SCHEME, NODE_DATA_PATH, + ADMIN_PORT, AUTOLOAD_KERNEL_MODULES_PATH, + BTRFS_KERNEL_MODULE, DEFAULT_URL_SCHEME, NODE_DATA_PATH, SKALE_DIR, CONTAINER_CONFIG_PATH, CONTRACTS_PATH, ETH_STATE_PATH, NODE_CERTS_PATH, SGX_CERTS_PATH, REPORTS_PATH, REDIS_DATA_PATH, @@ -124,6 +125,38 @@ def init_data_dir(): safe_mkdir(NODE_DATA_PATH) +def is_btrfs_module_autoloaded(modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH): + if not os.path.isfile(modules_filepath): + return False + with open(modules_filepath) as modules_file: + modules = set( + map( + lambda line: line.strip(), + filter( + lambda line: not line.startswith('#'), + modules_file.readlines() + ) + ) + ) + return BTRFS_KERNEL_MODULE in modules + + +def add_btrfs_module_to_autoload(modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH): + with open(modules_filepath, 'a') as modules_file: + modules_file.write(f'{BTRFS_KERNEL_MODULE}\n') + + +def ensure_btrfs_kernel_module_autoloaded( + modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH +): + logger.debug('Checking if btrfs is in %s', modules_filepath) + if not is_btrfs_module_autoloaded(modules_filepath): + logger.info('Adding btrfs module to %s', modules_filepath) + add_btrfs_module_to_autoload(modules_filepath) + else: + logger.debug('btrfs is already in %s', modules_filepath) + + def validate_abi_files(json_result=False): results = [ validate_abi(abi_filepath) diff --git a/node_cli/core/host2/__init__.py b/node_cli/core/host2/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/node_cli/core/iptables.py b/node_cli/core/iptables.py index 4af19695..fe072931 100644 --- a/node_cli/core/iptables.py +++ b/node_cli/core/iptables.py @@ -18,9 +18,16 @@ # along with this program. If not, see . import logging +import socket import sys from pathlib import Path -from node_cli.configs import IPTABLES_DIR, IPTABLES_RULES_STATE_FILEPATH, ENV + +from node_cli.configs import ( + IPTABLES_DIR, + IPTABLES_RULES_STATE_FILEPATH, + ENV, + DEFAULT_SSH_PORT +) from node_cli.utils.helper import run_cmd @@ -38,7 +45,6 @@ ALLOWED_INCOMING_TCP_PORTS = [ '80', # filestorage - '22', # ssh '311', # watchdog https '8080', # http '443', # https @@ -131,15 +137,32 @@ def drop_all_udp(chain: iptc.Chain) -> None: ensure_rule(chain, r) +def get_ssh_port(ssh_service_name='ssh'): + try: + return socket.getservbyname(ssh_service_name) + except OSError: + logger.exception('Cannot get ssh service port') + return DEFAULT_SSH_PORT + + +def allow_ssh(chain: iptc.Chain) -> None: + ssh_port = get_ssh_port() + accept_incoming(chain, str(ssh_port), 'tcp') + + def allow_base_ports(chain: iptc.Chain) -> None: - logger.debug('Allowing base ports...') + logger.info('Configuring ssh port') + allow_ssh(chain) + logger.info('Configuring incoming tcp ports') for port in ALLOWED_INCOMING_TCP_PORTS: accept_incoming(chain, port, 'tcp') + logger.info('Configuring incoming udp ports') for port in ALLOWED_INCOMING_UDP_PORTS: accept_incoming(chain, port, 'udp') -def accept_incoming(chain, port, protocol) -> None: +def accept_incoming(chain: iptc.Chain, port: str, protocol: str) -> None: + logger.debug('Going to allow %s traffic from %s port', protocol, port) rule = iptc.Rule() rule.protocol = protocol match = iptc.Match(rule, protocol) diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py new file mode 100644 index 00000000..0bb74ed0 --- /dev/null +++ b/node_cli/core/node_options.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging + +from node_cli.utils.helper import read_json, write_json, init_file +from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH + +logger = logging.getLogger(__name__) + + +class NodeOptions: + def __init__( + self, + filepath: str = NODE_OPTIONS_FILEPATH + ): + self.filepath = filepath + init_file(filepath, {}) + + def _get(self, field_name: str): + config = read_json(self.filepath) + return config.get(field_name) + + def _set(self, field_name: str, field_value) -> None: + config = read_json(self.filepath) + config[field_name] = field_value + write_json(self.filepath, config) + + def all(self) -> dict: + return read_json(self.filepath) diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index ab27430a..f0e51f06 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -29,10 +29,7 @@ from node_cli.utils.helper import ( write_json, read_json, run_cmd, extract_env_params, safe_load_yml ) -from node_cli.configs import ( - ALLOCATION_FILEPATH, ENVIRONMENT_PARAMS_FILEPATH, - SNAPSHOTS_SHARED_VOLUME -) +from node_cli.configs import ALLOCATION_FILEPATH, STATIC_PARAMS_FILEPATH, SNAPSHOTS_SHARED_VOLUME from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, TIMES, TIMEOUT, TEST_DIVIDER, SMALL_DIVIDER, MEDIUM_DIVIDER, LARGE_DIVIDER, @@ -75,8 +72,7 @@ def compose_resource_allocation_config( env_type: str, params_by_env_type: Dict = None ) -> Dict: - params_by_env_type = params_by_env_type or \ - safe_load_yml(ENVIRONMENT_PARAMS_FILEPATH) + params_by_env_type = params_by_env_type or safe_load_yml(STATIC_PARAMS_FILEPATH) common_config = params_by_env_type['common'] schain_cpu_alloc, ima_cpu_alloc = get_cpu_alloc(common_config) schain_mem_alloc, ima_mem_alloc = get_memory_alloc(common_config) diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index 4904def1..84a558ad 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -1,6 +1,8 @@ import logging import pprint +from typing import Optional + from node_cli.utils.helper import get_request, post_request, error_exit from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.print_formatters import ( @@ -68,11 +70,17 @@ def show_config(name: str) -> None: error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) -def toggle_schain_repair_mode(schain: str) -> None: +def toggle_schain_repair_mode( + schain: str, + snapshot_from: Optional[str] = None +) -> None: + json_params = {'schain_name': schain} + if snapshot_from: + json_params.update({'snapshot_from': snapshot_from}) status, payload = post_request( blueprint=BLUEPRINT_NAME, method='repair', - json={'schain_name': schain} + json=json_params ) if status == 'ok': print('Schain has been set for repair') diff --git a/node_cli/core/ssl.py b/node_cli/core/ssl.py index 38bfc73d..5cd511f2 100644 --- a/node_cli/core/ssl.py +++ b/node_cli/core/ssl.py @@ -90,8 +90,8 @@ def check_ssl_connection(host, port, silent=False): with detached_subprocess(ssl_check_cmd, expose_output=expose_output) as dp: time.sleep(1) code = dp.poll() - if code is not None: - logger.error('Healthcheck connection failed') + if code != 0 and code is not None: + logger.error('Healthcheck connection failed with code %d', code) raise SSLHealthcheckError('OpenSSL connection verification failed') diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 4a6577ae..1d7bd27e 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -19,18 +19,21 @@ import functools import logging -from typing import Dict +from typing import Dict, Optional from node_cli.cli.info import VERSION from node_cli.configs import CONTAINER_CONFIG_PATH, CONTAINER_CONFIG_TMP_PATH -from node_cli.core.host import link_env_file, prepare_host +from node_cli.core.host import ensure_btrfs_kernel_module_autoloaded, link_env_file, prepare_host from node_cli.core.docker_config import configure_docker from node_cli.core.nginx import generate_nginx_config +from node_cli.core.node_options import NodeOptions from node_cli.core.resources import update_resource_allocation, init_shared_space_volume from node_cli.operations.common import ( - backup_old_contracts, download_contracts, download_filestorage_artifacts, configure_filebeat, + backup_old_contracts, + download_contracts, + configure_filebeat, configure_flask, unpack_backup_archive ) from node_cli.operations.docker_lvmpy import docker_lvmpy_update, docker_lvmpy_install @@ -92,13 +95,14 @@ def update(env_filepath: str, env: Dict) -> None: sync_skale_node() + ensure_btrfs_kernel_module_autoloaded() + if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() backup_old_contracts() download_contracts(env) - download_filestorage_artifacts() docker_lvmpy_update(env) generate_nginx_config() @@ -132,9 +136,10 @@ def update(env_filepath: str, env: Dict) -> None: @checked_host -def init(env_filepath: str, env: str) -> bool: +def init(env_filepath: str, env: Dict, snapshot_from: Optional[str] = None) -> bool: sync_skale_node() + ensure_btrfs_kernel_module_autoloaded() if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() @@ -146,7 +151,6 @@ def init(env_filepath: str, env: str) -> bool: ) link_env_file() download_contracts(env) - download_filestorage_artifacts() configure_filebeat() configure_flask() @@ -156,6 +160,9 @@ def init(env_filepath: str, env: str) -> bool: docker_lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) + node_options = NodeOptions() + node_options.snapshot_from = snapshot_from + update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], @@ -194,6 +201,7 @@ def restore(env, backup_path): print_failed_requirements_checks(failed_checks) return False + ensure_btrfs_kernel_module_autoloaded() if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 5fd6bef7..cfe79b42 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -29,12 +29,15 @@ from distutils.dir_util import copy_tree from node_cli.configs import ( - CONTRACTS_PATH, BACKUP_CONTRACTS_PATH, - MANAGER_CONTRACTS_FILEPATH, IMA_CONTRACTS_FILEPATH, SRC_FILEBEAT_CONFIG_PATH, G_CONF_HOME, - FILESTORAGE_INFO_FILE, FILESTORAGE_ARTIFACTS_FILE, FILEBEAT_CONFIG_PATH, FLASK_SECRET_KEY_FILE + CONTRACTS_PATH, + BACKUP_CONTRACTS_PATH, + G_CONF_HOME, + FILEBEAT_CONFIG_PATH, + FLASK_SECRET_KEY_FILE, + IMA_CONTRACTS_FILEPATH, + MANAGER_CONTRACTS_FILEPATH, + SRC_FILEBEAT_CONFIG_PATH ) -from node_cli.utils.helper import read_json - logger = logging.getLogger(__name__) @@ -49,13 +52,6 @@ def download_contracts(env): urllib.request.urlretrieve(env['IMA_CONTRACTS_ABI_URL'], IMA_CONTRACTS_FILEPATH) -def download_filestorage_artifacts(): - logger.info('Updating filestorage artifacts') - fs_artifacts_url = read_json(FILESTORAGE_INFO_FILE)['artifacts_url'] - logger.debug(f'Downloading {fs_artifacts_url} to {FILESTORAGE_ARTIFACTS_FILE}') - urllib.request.urlretrieve(fs_artifacts_url, FILESTORAGE_ARTIFACTS_FILE) - - def configure_filebeat(): logger.info('Configuring filebeat...') copyfile(SRC_FILEBEAT_CONFIG_PATH, FILEBEAT_CONFIG_PATH) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 54255f2d..9a2911c8 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -37,7 +37,7 @@ logger = logging.getLogger(__name__) -SCHAIN_REMOVE_TIMEOUT = 60 +SCHAIN_REMOVE_TIMEOUT = 300 IMA_REMOVE_TIMEOUT = 20 MAIN_COMPOSE_CONTAINERS = ('skale-api', 'bounty', 'skale-admin') @@ -93,28 +93,28 @@ def remove_dynamic_containers(): def rm_all_schain_containers(): schain_containers = get_all_schain_containers() - remove_containers(schain_containers, stop_timeout=SCHAIN_REMOVE_TIMEOUT) + remove_containers(schain_containers, timeout=SCHAIN_REMOVE_TIMEOUT) def rm_all_ima_containers(): ima_containers = get_all_ima_containers() - remove_containers(ima_containers, stop_timeout=IMA_REMOVE_TIMEOUT) + remove_containers(ima_containers, timeout=IMA_REMOVE_TIMEOUT) -def remove_containers(containers, stop_timeout): +def remove_containers(containers, timeout): for container in containers: - safe_rm(container, stop_timeout=stop_timeout) + safe_rm(container, timeout=timeout) -def safe_rm(container: Container, stop_timeout=DOCKER_DEFAULT_STOP_TIMEOUT, **kwargs): +def safe_rm(container: Container, timeout=DOCKER_DEFAULT_STOP_TIMEOUT, **kwargs): """ Saves docker container logs (last N lines) in the .skale/node_data/log/.removed_containers folder. Then stops and removes container with specified params. """ container_name = container.name logger.info( - f'Stopping container: {container_name}, timeout: {stop_timeout}') - container.stop(timeout=stop_timeout) + f'Stopping container: {container_name}, timeout: {timeout}') + container.stop(timeout=timeout) backup_container_logs(container) logger.info(f'Removing container: {container_name}, kwargs: {kwargs}') container.remove(**kwargs) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index f8c3235b..bcfa61b9 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -17,10 +17,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import ipaddress import json import os import re import sys +from urllib.parse import urlparse import yaml import shutil @@ -81,6 +83,11 @@ def write_json(path, content): json.dump(content, outfile, indent=4) +def init_file(path, content=None): + if not os.path.exists(path): + write_json(path, content) + + def run_cmd( cmd, env={}, @@ -350,3 +357,33 @@ def rsync_dirs(src: str, dest: str) -> None: logger.info(f'Syncing {dest} with {src}') run_cmd(['rsync', '-r', f'{src}/', dest]) run_cmd(['rsync', '-r', f'{src}/.git', dest]) + + +class UrlType(click.ParamType): + name = 'url' + + def convert(self, value, param, ctx): + try: + result = urlparse(value) + except ValueError: + self.fail(f'Some characters are not allowed in {value}', + param, ctx) + if not all([result.scheme, result.netloc]): + self.fail(f'Expected valid url. Got {value}', param, ctx) + return value + + +class IpType(click.ParamType): + name = 'ip' + + def convert(self, value, param, ctx): + try: + ipaddress.ip_address(value) + except ValueError: + self.fail(f'expected valid ipv4/ipv6 address. Got {value}', + param, ctx) + return value + + +URL_TYPE = UrlType() +IP_TYPE = IpType() diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index c48c5e35..b49d9f61 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -23,11 +23,13 @@ def __new__(cls, version=DEFAULT_VERSION, config_stream=DEFAULT_CONFIG_STREAM, ) -def get_meta_info() -> CliMeta: +def get_meta_info(raw: bool = False) -> CliMeta: if not os.path.isfile(META_FILEPATH): return None with open(META_FILEPATH) as meta_file: plain_meta = json.load(meta_file) + if raw: + return plain_meta return CliMeta(**plain_meta) diff --git a/node_cli/utils/print_formatters.py b/node_cli/utils/print_formatters.py index 703c595d..4f0fdc7a 100644 --- a/node_cli/utils/print_formatters.py +++ b/node_cli/utils/print_formatters.py @@ -27,6 +27,7 @@ from node_cli.configs import LONG_LINE from node_cli.configs.cli_logger import DEBUG_LOG_FILEPATH +from node_cli.utils.meta import CliMeta from node_cli.utils.texts import Texts TEXTS = Texts() @@ -270,7 +271,7 @@ def print_node_cmd_error(): def print_node_info(node, node_status): - print(inspect.cleandoc(f''' + print(inspect.cleandoc(f""" {LONG_LINE} Node info Name: {node['name']} @@ -281,7 +282,7 @@ def print_node_info(node, node_status): Domain name: {node['domain_name']} Status: {node_status} {LONG_LINE} - ''')) + """)) def print_err_response(error_payload): @@ -307,3 +308,13 @@ def print_failed_requirements_checks(failed_checks: list) -> None: block_len = (len(drawing.split()[0]) - len(main_header)) // 2 print('=' * block_len + main_header + '=' * block_len) print(drawing) + + +def print_meta_info(meta_info: CliMeta) -> None: + print(inspect.cleandoc(f""" + {LONG_LINE} + Version: {meta_info.version} + Config Stream: {meta_info.config_stream} + Lvmpy stream: {meta_info.docker_lvmpy_stream} + {LONG_LINE} + """)) diff --git a/scripts/build.sh b/scripts/build.sh index 432eb7d2..cdde93c0 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -40,7 +40,7 @@ echo "VERSION = '$VERSION'" >> $DIST_INFO_FILEPATH EXECUTABLE_NAME=skale-$VERSION-$OS -pyinstaller --onefile main.spec +pyinstaller main.spec mv $PARENT_DIR/dist/main $PARENT_DIR/dist/$EXECUTABLE_NAME diff --git a/setup.py b/setup.py index 1301bcba..3dfcc3f5 100644 --- a/setup.py +++ b/setup.py @@ -20,16 +20,16 @@ def find_version(*file_paths): extras_require = { 'linter': [ - "flake8==3.7.9", - "isort>=4.2.15,<5.8.1", + "flake8==5.0.4", + "isort>=4.2.15,<5.10.2", ], 'dev': [ "bumpversion==0.6.0", - "pytest==6.2.3", - "pytest-cov==2.9.0", - "twine==2.0.0", - "mock==4.0.2", - "freezegun==0.3.15" + "pytest==7.1.2", + "pytest-cov==3.0.0", + "twine==4.0.1", + "mock==4.0.3", + "freezegun==1.2.2" ] } @@ -50,22 +50,22 @@ def find_version(*file_paths): author_email='support@skalelabs.com', url='https://github.com/skalenetwork/node-cli', install_requires=[ - "click==7.1.2", - "docker==4.2.2", - "PyInstaller==3.6", - "texttable==1.6.2", - "python-dateutil==2.8.1", - "Jinja2==2.11.2", - "psutil==5.7.0", - "python-dotenv==0.13.0", - "terminaltables==3.1.0", - "requests==2.23.0", - "GitPython==3.1.14", - "PyYAML==5.4.1", - "packaging==20.9", - "python-debian==0.1.39", + "click==8.1.3", + "PyInstaller==5.6.2", + "docker==6.0.1", + "texttable==1.6.4", + "python-dateutil==2.8.2", + "Jinja2==3.1.2", + "psutil==5.9.1", + "python-dotenv==0.21.0", + "terminaltables==3.1.10", + "requests==2.28.1", + "GitPython==3.1.27", + "packaging==21.3", + "python-debian==0.1.48", "python-iptables==1.0.0", - "MarkupSafe==2.0.1", + "PyYAML==6.0", + "MarkupSafe==2.1.1", ], python_requires='>=3.6,<4', extras_require=extras_require, diff --git a/tests/.skale/config/environment_params.yaml b/tests/.skale/config/environment_params.yaml deleted file mode 100644 index 2f6fdef1..00000000 --- a/tests/.skale/config/environment_params.yaml +++ /dev/null @@ -1,98 +0,0 @@ -common: - schain: - cpu: - skaled: 0.8 - ima: 0.2 - mem: - skaled: 0.8 - ima: 0.2 - volume_limits: - max_consensus_storage_bytes: 0.3 - max_skaled_leveldb_storage_bytes: 0.3 - max_file_storage_bytes: 0.3 - max_reserved_storage_bytes: 0.1 - leveldb_limits: - contract_storage: 0.6 - db_storage: 0.4 - base_rotate_after_block_divider: 61035.1 - shared_space_coefficient: 1 -envs: - mainnet: - server: - cpu_total: 8 - cpu_physical: 1 - memory: 32000000000 - swap: 16000000000 - disk: 1900000000000 - - package: - iptables-persistent: 1.0.4 - lvm2: 2.02.0 - btrfs-progs: 4.15.1 - lsof: "4.89" - psmisc: 23.1-1 - - docker: - docker-api: 1.41.0 - docker-engine: 20.10.7 - docker-compose: 1.27.4 - - testnet: - server: - cpu_total: 8 - cpu_physical: 1 - memory: 32000000000 - swap: 16000000000 - disk: 200000000000 - - package: - iptables-persistent: 1.0.4 - lvm2: 2.02.0 - btrfs-progs: 4.15.1 - lsof: "4.89" - psmisc: 23.1-1 - - docker: - docker-api: 1.41.0 - docker-engine: 20.10.7 - docker-compose: 1.27.4 - - qanet: - server: - cpu_total: 8 - cpu_physical: 1 - memory: 32000000000 - swap: 16000000000 - disk: 200000000000 - - package: - iptables-persistent: 1.0.4 - lvm2: 2.02.0 - btrfs-progs: 4.15.1 - lsof: "4.89" - psmisc: 23.1-1 - - docker: - docker-api: 1.41.0 - docker-engine: 20.10.7 - docker-compose: 1.27.4 - - devnet: - server: - cpu_total: 1 - cpu_physical: 1 - memory: 2000000000 - swap: 2000000000 - disk: 80000000000 - - package: - iptables-persistent: 1.0.4 - lvm2: 2.02.0 - btrfs-progs: 4.15.1 - lsof: "4.89" - psmisc: 23.1-1 - - docker: - docker-api: 1.41.0 - docker-engine: 20.10.7 - docker-compose: 1.27.4 diff --git a/tests/.skale/config/static_params.yaml b/tests/.skale/config/static_params.yaml new file mode 100644 index 00000000..a0ea143f --- /dev/null +++ b/tests/.skale/config/static_params.yaml @@ -0,0 +1,306 @@ +common: + schain: + cpu: + skaled: 0.8 + ima: 0.2 + mem: + skaled: 0.8 + ima: 0.2 + volume_limits: + max_consensus_storage_bytes: 0.3 + max_skaled_leveldb_storage_bytes: 0.3 + max_file_storage_bytes: 0.3 + max_reserved_storage_bytes: 0.1 + leveldb_limits: + contract_storage: 0.6 + db_storage: 0.2 # leveldb may use x2 storage, so 0.4 divided by 2, actually using 0.4 + shared_space_coefficient: 1 +envs: + mainnet: + server: + cpu_total: 8 + cpu_physical: 1 + memory: 32000000000 + swap: 16000000000 + disk: 1900000000000 + + package: + iptables-persistent: 1.0.4 + lvm2: 2.02.0 + btrfs-progs: 4.15.1 + lsof: "4.89" + psmisc: 23.1-1 + + docker: + docker-api: 1.41.0 + docker-engine: 20.10.7 + docker-compose: 1.27.4 + + schain: + revertableFSPatchTimestamp: 1000000 + contractStoragePatchTimestamp: 1000000 + snapshotIntervalSec: 86400 + emptyBlockIntervalMs: 10000 + snapshotDownloadTimeout: 18000 + snapshotDownloadInactiveTimeout: 120 + + schain_cmd: + ["-v 3", "--web3-trace", "--enable-debug-behavior-apis", "--aa no"] + + node: + common: + bindIP: "0.0.0.0" + logLevel: "info" + logLevelConfig: "info" + small: + minCacheSize: 1000000 + maxCacheSize: 2000000 + collectionQueueSize: 2 + collectionDuration: 10 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 25 + medium: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + large: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + test: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + test4: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + + testnet: + server: + cpu_total: 8 + cpu_physical: 1 + memory: 32000000000 + swap: 16000000000 + disk: 200000000000 + + package: + iptables-persistent: 1.0.4 + lvm2: 2.02.0 + btrfs-progs: 4.15.1 + lsof: "4.89" + psmisc: 23.1-1 + + docker: + docker-api: 1.41.0 + docker-engine: 20.10.7 + docker-compose: 1.27.4 + + schain: + revertableFSPatchTimestamp: 1000000 + contractStoragePatchTimestamp: 1000000 + snapshotIntervalSec: 86400 + emptyBlockIntervalMs: 10000 + snapshotDownloadTimeout: 18000 + snapshotDownloadInactiveTimeout: 120 + + schain_cmd: + ["-v 3", "--web3-trace", "--enable-debug-behavior-apis", "--aa no"] + + node: + common: + bindIP: "0.0.0.0" + logLevel: "info" + logLevelConfig: "info" + small: + minCacheSize: 1000000 + maxCacheSize: 2000000 + collectionQueueSize: 2 + collectionDuration: 10 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 25 + medium: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + large: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + test: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + test4: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + + qanet: + server: + cpu_total: 8 + cpu_physical: 1 + memory: 32000000000 + swap: 16000000000 + disk: 200000000000 + + package: + iptables-persistent: 1.0.4 + lvm2: 2.02.0 + btrfs-progs: 4.15.1 + lsof: "4.89" + psmisc: 23.1-1 + + docker: + docker-api: 1.41.0 + docker-engine: 20.10.7 + docker-compose: 1.27.4 + + schain: + revertableFSPatchTimestamp: 1000000 + contractStoragePatchTimestamp: 1000000 + snapshotIntervalSec: 86400 + emptyBlockIntervalMs: 10000 + snapshotDownloadTimeout: 18000 + snapshotDownloadInactiveTimeout: 120 + + schain_cmd: + ["-v 3", "--web3-trace", "--enable-debug-behavior-apis", "--aa no"] + + node: + common: + bindIP: "0.0.0.0" + logLevel: "info" + logLevelConfig: "info" + small: + minCacheSize: 1000000 + maxCacheSize: 2000000 + collectionQueueSize: 2 + collectionDuration: 10 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 25 + medium: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + large: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + test: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + test4: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + + + devnet: + server: + cpu_total: 1 + cpu_physical: 1 + memory: 2000000000 + swap: 2000000000 + disk: 80000000000 + + package: + iptables-persistent: 1.0.4 + lvm2: 2.02.0 + btrfs-progs: 4.15.1 + lsof: "4.89" + psmisc: 23.1-1 + + docker: + docker-api: 1.41.0 + docker-engine: 20.10.7 + docker-compose: 1.27.4 + + schain: + revertableFSPatchTimestamp: 1000000 + contractStoragePatchTimestamp: 1000000 + snapshotIntervalSec: 86400 + emptyBlockIntervalMs: 10000 + snapshotDownloadTimeout: 18000 + snapshotDownloadInactiveTimeout: 120 + + schain_cmd: + ["-v 3", "--web3-trace", "--enable-debug-behavior-apis", "--aa no"] + + node: + common: + bindIP: "0.0.0.0" + logLevel: "info" + logLevelConfig: "info" + small: + minCacheSize: 1000000 + maxCacheSize: 2000000 + collectionQueueSize: 2 + collectionDuration: 10 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 25 + medium: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + large: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + test: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 + test4: + minCacheSize: 8000000 + maxCacheSize: 16000000 + collectionQueueSize: 20 + collectionDuration: 60 + transactionQueueSize: 1000 + maxOpenLeveldbFiles: 1000 diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index f654e03e..3b9fe1d0 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -24,16 +24,26 @@ import logging from node_cli.configs import SKALE_DIR, G_CONF_HOME -from node_cli.cli.node import (node_info, register_node, signature, - backup_node, restore_node, - set_node_in_maintenance, - remove_node_from_maintenance, - _turn_off, _turn_on, _set_domain_name) +from node_cli.cli.node import ( + node_info, + register_node, + signature, + backup_node, + restore_node, + set_node_in_maintenance, + remove_node_from_maintenance, + version, + _turn_off, + _turn_on, + _set_domain_name +) from node_cli.utils.helper import init_default_logger from tests.helper import ( - response_mock, run_command_mock, - run_command, subprocess_run_mock + response_mock, + run_command, + run_command_mock, + subprocess_run_mock ) from tests.resources_test import BIG_DISK_SIZE @@ -389,3 +399,14 @@ def test_set_domain_name(): _set_domain_name, ['-d', 'skale.test', '--yes']) assert result.exit_code == 0 assert result.output == 'Setting new domain name: skale.test\nDomain name successfully changed\n' # noqa + + +def test_node_version(meta_file_v2): + result = run_command(version) + assert result.exit_code == 0 + assert result.output == '--------------------------------------------------\nVersion: 0.1.1\nConfig Stream: develop\nLvmpy stream: 1.1.2\n--------------------------------------------------\n' # noqa + + result = run_command(version, ['--json']) + print(repr(result.output)) + assert result.exit_code == 0 + assert result.output == "{'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_stream': '1.1.2'}\n" # noqa diff --git a/tests/conftest.py b/tests/conftest.py index e118b10a..70aada8f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,11 +29,17 @@ import yaml from node_cli.configs import ( - ENVIRONMENT_PARAMS_FILEPATH, GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH, - REMOVED_CONTAINERS_FOLDER_PATH + CONTAINER_CONFIG_TMP_PATH, + GLOBAL_SKALE_CONF_FILEPATH, + GLOBAL_SKALE_DIR, + REMOVED_CONTAINERS_FOLDER_PATH, + STATIC_PARAMS_FILEPATH ) -from node_cli.utils.global_config import generate_g_config_file +from node_cli.configs import META_FILEPATH from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH +from node_cli.utils.global_config import generate_g_config_file + +from tests.helper import TEST_META_V1, TEST_META_V2 TEST_ENV_PARAMS = """ @@ -114,14 +120,14 @@ @pytest.fixture def net_params_file(): - with open(ENVIRONMENT_PARAMS_FILEPATH, 'w') as f: + with open(STATIC_PARAMS_FILEPATH, 'w') as f: yaml.dump( yaml.load(TEST_ENV_PARAMS, Loader=yaml.Loader), stream=f, Dumper=yaml.Dumper ) - yield ENVIRONMENT_PARAMS_FILEPATH - os.remove(ENVIRONMENT_PARAMS_FILEPATH) + yield STATIC_PARAMS_FILEPATH + os.remove(STATIC_PARAMS_FILEPATH) @pytest.fixture() @@ -193,3 +199,41 @@ def resource_alloc(): json.dump({}, alloc_file) yield RESOURCE_ALLOCATION_FILEPATH os.remove(RESOURCE_ALLOCATION_FILEPATH) + + +@pytest.fixture +def meta_file_v1(): + with open(META_FILEPATH, 'w') as f: + json.dump(TEST_META_V1, f) + try: + yield META_FILEPATH + finally: + os.remove(META_FILEPATH) + + +@pytest.fixture +def meta_file_v2(): + with open(META_FILEPATH, 'w') as f: + json.dump(TEST_META_V2, f) + try: + yield META_FILEPATH + finally: + os.remove(META_FILEPATH) + + +@pytest.fixture +def ensure_meta_removed(): + try: + yield + finally: + if os.path.isfile(META_FILEPATH): + os.remove(META_FILEPATH) + + +@pytest.fixture +def tmp_config_dir(): + os.mkdir(CONTAINER_CONFIG_TMP_PATH) + try: + yield CONTAINER_CONFIG_TMP_PATH + finally: + shutil.rmtree(CONTAINER_CONFIG_TMP_PATH) diff --git a/tests/core/host/kernel_config_test.py b/tests/core/host/kernel_config_test.py new file mode 100644 index 00000000..92e4e8e1 --- /dev/null +++ b/tests/core/host/kernel_config_test.py @@ -0,0 +1,34 @@ +import os + +import pytest + +from node_cli.core.host import ( + is_btrfs_module_autoloaded, + ensure_btrfs_kernel_module_autoloaded +) + + +@pytest.fixture +def tmp_path(tmp_dir_path): + path = os.path.join(tmp_dir_path, 'modules') + return path + + +def test_btrfs_module_autoload_config(tmp_path): + ensure_btrfs_kernel_module_autoloaded(tmp_path) + assert is_btrfs_module_autoloaded(tmp_path) + with open(tmp_path) as tmp_file: + tmp_file.read() == 'btrfs\n' + + +def test_is_btrfs_module_autoloaded_negative(tmp_path): + assert not is_btrfs_module_autoloaded(tmp_path) + with open(tmp_path, 'w') as tmp_file: + tmp_file.write('') + + assert not is_btrfs_module_autoloaded(tmp_path) + + with open(tmp_path, 'w') as tmp_file: + tmp_file.write('# btrfs') + + assert not is_btrfs_module_autoloaded(tmp_path) diff --git a/tests/core/iptables_test.py b/tests/core/iptables_test.py new file mode 100644 index 00000000..03ae7c31 --- /dev/null +++ b/tests/core/iptables_test.py @@ -0,0 +1,18 @@ +import socket + +import mock + +from node_cli.core.iptables import allow_ssh, get_ssh_port + + +def test_get_ssh_port(): + assert get_ssh_port() == 22 + assert get_ssh_port('http') == 80 + with mock.patch.object(socket, 'getservbyname', side_effect=OSError): + assert get_ssh_port() == 22 + + +def test_allow_ssh(): + chain = mock.Mock() + chain.rules = [] + allow_ssh(chain) diff --git a/tests/core_checks_test.py b/tests/core_checks_test.py index e58c9da9..6e75bbdd 100644 --- a/tests/core_checks_test.py +++ b/tests/core_checks_test.py @@ -1,10 +1,13 @@ import os -from pip._internal import main as pipmain +import shutil import time +from pip._internal import main as pipmain import mock import pytest +from node_cli.configs import STATIC_PARAMS_FILEPATH + from node_cli.core.checks import ( CheckType, DockerChecker, @@ -12,6 +15,7 @@ get_all_checkers, get_checks, get_report, + get_static_params, MachineChecker, merge_reports, PackageChecker, @@ -108,9 +112,9 @@ def test_checks_swap(server_req): def test_checks_network(server_req): - checker = MachineChecker(server_req, 'test-disk') + checker = MachineChecker(server_req, 'test-disk', network_timeout=10) r = checker.network() - assert r.status == 'ok' + assert r.status == 'ok', r.info assert r.name == 'network' @@ -135,7 +139,7 @@ def test_checks_disk(server_req): def test_checks_machine_check(server_req): - checker = MachineChecker(server_req, 'test-disk') + checker = MachineChecker(server_req, 'test-disk', network_timeout=10) result = checker.check() assert not all([r.status == 'ok' for r in result]) @@ -375,3 +379,11 @@ def test_merge_report(): {'name': 'test2', 'status': 'ok', 'info': 'Test1'}, {'name': 'test3', 'status': 'failed', 'info': 'Test1'} ] + + +def test_get_static_params(tmp_config_dir): + params = get_static_params() + shutil.copy(STATIC_PARAMS_FILEPATH, tmp_config_dir) + tmp_params = get_static_params(config_path=tmp_config_dir) + assert params['server']['cpu_total'] == 8 + assert params == tmp_params diff --git a/tests/helper.py b/tests/helper.py index c4b66516..16290b8d 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -22,6 +22,19 @@ from mock import Mock, MagicMock +TEST_META_V1 = { + 'version': '0.1.1', + 'config_stream': 'develop' +} + +TEST_META_V2 = { + 'version': '0.1.1', + 'config_stream': 'develop', + 'docker_lvmpy_stream': '1.1.2' + +} + + def response_mock(status_code=0, json_data=None, headers=None, raw=None): result = MagicMock() diff --git a/tests/resources_test.py b/tests/resources_test.py index 3eb04950..b5613dec 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -4,7 +4,7 @@ import mock import pytest -from node_cli.configs import ALLOCATION_FILEPATH, ENVIRONMENT_PARAMS_FILEPATH +from node_cli.configs import ALLOCATION_FILEPATH, STATIC_PARAMS_FILEPATH from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.core.resources import ( compose_resource_allocation_config, @@ -29,7 +29,7 @@ @pytest.fixture def params_by_env_type(): - return safe_load_yml(ENVIRONMENT_PARAMS_FILEPATH) + return safe_load_yml(STATIC_PARAMS_FILEPATH) @pytest.fixture diff --git a/tests/tools_meta_test.py b/tests/tools_meta_test.py index 184defde..95ab7658 100644 --- a/tests/tools_meta_test.py +++ b/tests/tools_meta_test.py @@ -1,7 +1,4 @@ import json -import os - -import pytest from node_cli.configs import META_FILEPATH from node_cli.utils.meta import ( @@ -10,41 +7,7 @@ ensure_meta, get_meta_info, save_meta, update_meta ) - - -TEST_META_V1 = { - 'version': '0.1.1', - 'config_stream': 'develop' -} - -TEST_META_V2 = { - 'version': '0.1.1', - 'config_stream': 'develop', - 'docker_lvmpy_stream': '1.1.2' -} - - -@pytest.fixture -def meta_file_v1(): - with open(META_FILEPATH, 'w') as f: - json.dump(TEST_META_V1, f) - yield META_FILEPATH - os.remove(META_FILEPATH) - - -@pytest.fixture -def meta_file_v2(): - with open(META_FILEPATH, 'w') as f: - json.dump(TEST_META_V2, f) - yield META_FILEPATH - os.remove(META_FILEPATH) - - -@pytest.fixture -def ensure_meta_removed(): - yield - if os.path.isfile(META_FILEPATH): - os.remove(META_FILEPATH) +from tests.helper import TEST_META_V1, TEST_META_V2 def test_get_meta_info_v1(meta_file_v1): diff --git a/text.yml b/text.yml index 862635c7..d1a660fb 100644 --- a/text.yml +++ b/text.yml @@ -8,8 +8,7 @@ health: help: Info about connected SGX server common: - json: - help: Show data in JSON format + json: Show data in JSON format node: base: "SKALE node commands"