diff --git a/admin_panel/forms.py b/admin_panel/forms.py index d66b7c2..c0a2e34 100644 --- a/admin_panel/forms.py +++ b/admin_panel/forms.py @@ -7,9 +7,9 @@ from core.consts.currencies import CURRENCIES_LIST from core.currency import Currency from core.pairs import PAIRS_LIST -from cryptocoins.tasks.eth import ethereum_manager -from cryptocoins.tasks.trx import tron_manager -from cryptocoins.tasks.bnb import bnb_manager +from cryptocoins.coins.eth.ethereum import ethereum_manager +from cryptocoins.coins.trx.tron import tron_manager +from cryptocoins.coins.bnb.bnb import bnb_manager from lib.cipher import AESCoderDecoder CryptoBitcoin = Bitcoin() diff --git a/admin_panel/views.py b/admin_panel/views.py index bbb3499..5b318fa 100644 --- a/admin_panel/views.py +++ b/admin_panel/views.py @@ -16,10 +16,7 @@ from cryptocoins.coins.btc.service import BTCCoinService from cryptocoins.coins.eth import ETH_CURRENCY from cryptocoins.coins.trx import TRX_CURRENCY -from cryptocoins.coins.usdt import USDT_CURRENCY -from cryptocoins.tasks.eth import process_payouts as eth_process_payouts -from cryptocoins.tasks.trx import process_payouts as trx_process_payouts -from cryptocoins.tasks.bnb import process_payouts as bnb_process_payouts +from cryptocoins.tasks.evm import process_payouts_task @staff_member_required @@ -90,7 +87,7 @@ def admin_eth_withdrawal_request_approve(request): try: if form.is_valid(): password = form.cleaned_data.get('key') - eth_process_payouts.apply_async([password,]) + process_payouts_task.apply_async(['ETH', password,], queue='eth_payouts') messages.success(request, 'Withdrawals in processing') return redirect('admin_withdrawal_request_approve_eth') # need for clear post data except Exception as e: # all messages and errors to admin message @@ -122,7 +119,7 @@ def admin_trx_withdrawal_request_approve(request): try: if form.is_valid(): password = form.cleaned_data.get('key') - trx_process_payouts.apply_async([password, ]) + process_payouts_task.apply_async(['TRX', password, ], queue='trx_payouts') messages.success(request, 'Withdrawals in processing') return redirect('admin_withdrawal_request_approve_trx') # need for clear post data except Exception as e: # all messages and errors to admin message @@ -154,7 +151,7 @@ def admin_bnb_withdrawal_request_approve(request): try: if form.is_valid(): password = form.cleaned_data.get('key') - bnb_process_payouts.apply_async([password, ]) + process_payouts_task.apply_async(['BNB', password, ], queue='bnb_payouts') messages.success(request, 'Withdrawals in processing') return redirect('admin_withdrawal_request_approve_bnb') # need for clear post data except Exception as e: # all messages and errors to admin message diff --git a/admin_rest/__init__.py b/admin_rest/__init__.py new file mode 100644 index 0000000..a428da2 --- /dev/null +++ b/admin_rest/__init__.py @@ -0,0 +1 @@ +default_app_config = 'admin_rest.apps.AdminRestConfig' diff --git a/admin_rest/admin_rest.py b/admin_rest/admin_rest.py new file mode 100644 index 0000000..9d4367a --- /dev/null +++ b/admin_rest/admin_rest.py @@ -0,0 +1,1149 @@ +import csv +import json +import logging +import time +from collections import defaultdict +from typing import List + +from allauth.account.models import EmailAddress +from django.contrib import messages +from django.contrib.admin.models import LogEntry +from django.contrib.auth.models import User, Group +from django.db import transaction, models +from django.db.models import F, Sum, Count, When, Value, Case, ExpressionWrapper +from django.db.models import Q +from django.db.transaction import atomic +from django.http import HttpResponse +from django.urls import reverse +from django.utils import timezone +from django.utils.safestring import mark_safe +from django.utils.translation import ugettext_lazy +from django_otp.conf import settings +from django_otp.plugins.otp_totp.models import TOTPDevice +from rest_framework.exceptions import ValidationError + +from admin_rest import restful_admin as api_admin +from admin_rest.fields import serial_field, CurrencySerialRestField +from admin_rest.mixins import JsonListApiViewMixin +from admin_rest.mixins import NoDeleteMixin, NoCreateMixin +from admin_rest.mixins import ReadOnlyMixin +from admin_rest.models import AllOrder +from admin_rest.models import AllOrderNoBot +from admin_rest.models import Balance +from admin_rest.models import Match +from admin_rest.models import Topups +from admin_rest.models import Transaction +from admin_rest.models import UserDailyStat +from admin_rest.models import WithdrawalRequest +from admin_rest.permissions import IsSuperAdminUser +from admin_rest.restful_admin import DefaultApiAdmin +from admin_rest.restful_admin import RestFulModelAdmin +from admin_rest.utils import get_bots_ids +from core.balance_manager import BalanceManager +from core.consts.inouts import DISABLE_COIN_STATES +from core.consts.orders import SELL +from core.currency import Currency +from core.exceptions.inouts import NotEnoughFunds +from core.models import PairSettings +from core.models.cryptocoins import UserWallet +from core.models.facade import AccessLog, CoinInfo +from core.models.facade import LoginHistory +from core.models.facade import Message +from core.models.facade import Profile +from core.models.facade import SmsHistory +from core.models.facade import SourceOfFunds +from core.models.facade import TwoFactorSecretHistory +from core.models.facade import TwoFactorSecretTokens +from core.models.facade import UserExchangeFee +from core.models.facade import UserFee +from core.models.facade import UserKYC +from core.models.facade import UserRestrictions +from core.models.inouts.dif_balance import DifBalance +from core.models.inouts.disabled_coin import DisabledCoin +from core.models.inouts.fees_and_limits import FeesAndLimits +from core.models.inouts.fees_and_limits import WithdrawalFee +from core.models.inouts.sci import GATES +from core.models.inouts.sci import PayGateTopup +from core.models.inouts.transaction import REASON_MANUAL_TOPUP +from core.models.inouts.transaction import REASON_TOPUP +from core.models.inouts.wallet import WalletTransactions +from core.models.inouts.wallet import WalletTransactionsRevert +from core.models.orders import Exchange +from core.models.orders import ExecutionResult +from core.models.orders import Order +from core.models.orders import OrderChangeHistory +from core.models.orders import OrderStateChangeHistory +from core.models.stats import ExternalPricesHistory +from core.models.stats import TradesAggregatedStats +from core.models.stats import UserPairDailyStat +from core.models.wallet_history import WalletHistoryItem +from core.utils.wallet_history import create_or_update_wallet_history_item_from_transaction +from cryptocoins.models.stats import DepositsWithdrawalsStats +from cryptocoins.tasks import calculate_topups_and_withdrawals +from cryptocoins.utils.stats import generate_stats_fields +from lib.helpers import BOT_RE + +log = logging.getLogger(__name__) + + +@api_admin.register(DifBalance) +class DifBalanceAdmin(ReadOnlyMixin, DefaultApiAdmin): + display_fields = ['created', 'user', 'type', 'currency', 'diff', 'diff_percent', + 'balance', 'old_balance', 'txs_amount', 'calc_balance', 'sum_diff'] + list_display = display_fields + fields = display_fields + readonly_fields = display_fields + + filterset_fields = ['created', 'currency'] + search_fields = [ + 'user__email', + 'user__id', + ] + ordering = ('-id',) + + +@api_admin.register(EmailAddress) +class EmailConfirmationApiAdmin(NoCreateMixin, NoDeleteMixin, DefaultApiAdmin): + list_display = ['verified'] + + +@api_admin.register(Group) +class GroupApiAdmin(DefaultApiAdmin): + fields = ['name', 'permissions', 'users'] + list_display = ['name'] + + def permissions(self, obj): + perms = defaultdict(dict) + for perm in obj.permissions.all(): + name = f'{perm.content_type.app_label}/{perm.content_type.model}' + action = perm.codename.split('_')[0] + perms[name][action] = True + + ret_perms = [{ + 'name': key, + 'permissions': value + } for key, value in perms.items()] + ret_perms = sorted(ret_perms, key=lambda i: i['name']) + return ret_perms + + def users(self, obj): + return list(obj.user_set.values('id', 'username', 'email')) + + +@api_admin.register(AccessLog) +class AccessLogApiAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = [ + 'created', + 'ip', + 'username', + 'method', + 'uri', + 'status', + 'referer', + 'user_agent'] + fields = [ + 'created', + 'ip', + 'username', + 'method', + 'uri', + 'status', + 'referer', + 'user_agent'] + search_fields = [ + 'username', + 'uri', + 'user_agent', + 'ip' + ] + filterset_fields = ['created', 'method', 'status'] + ordering = ('-created',) + + +@api_admin.register(LoginHistory) +class LoginHistoryApiAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = ['created', 'ip', 'user_agent'] + readonly_fields = ['created', 'ip', 'user_agent'] + ordering = ('-created',) + + +@api_admin.register(Message) +class MessageApiAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(Profile) +class ProfileApiAdmin(DefaultApiAdmin): + vue_resource_extras = {'aside': {'edit': True}} + readonly_fields = ( + 'id', + 'created', + 'updated', + 'affiliate_code', + 'register_ip') + + def country(self, obj): + return obj.country.code + + +@api_admin.register(SmsHistory) +class SmsHistoryApiAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = ['user', 'created', 'phone', 'withdrawals_sms_confirmation'] + filterset_fields = ['created'] + search_fields = ['user__email', 'phone'] + ordering = ('-created',) + + +@api_admin.register(SourceOfFunds) +class SourceOfFundsApiAdmin(DefaultApiAdmin): + fields = ['user', 'is_beneficiary', 'profession_value', 'source_value'] + list_display = [ + 'user', + 'is_beneficiary', + 'profession_value', + 'source_value'] + readonly_fields = ( + 'is_beneficiary', + 'profession_value', + 'source_value', + ) + + def profession_value(self, obj): + if obj.profession is None: + return 'Not set' + + result = [] + # iterate over professions array + for i in obj.profession: + # iterate over profession choices + for j in obj.PROFESSIONS: + _id, text = j + if _id == i: + result.append(str(text)) + + return ', '.join(result) + + def source_value(self, obj): + if obj.source is None: + return 'Not set' + + result = [] + for i in obj.source: + for j in obj.SOURCES: + _id, text = j + if _id == i: + result.append(str(text)) + + return ', '.join(result) + + +@api_admin.register(TwoFactorSecretHistory) +class TwoFactorSecretHistoryApiAdmin(ReadOnlyMixin, DefaultApiAdmin): + fields = ['created', 'status'] + readonly_fields = ['created', 'status'] + ordering = ('-created',) + + +@api_admin.register(TwoFactorSecretTokens) +class TwoFactorSecretTokensApiAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = ['id', 'email', 'status', 'last_updated'] + search_fields = ( + 'id', + 'user__email', + 'user__id', + ) + actions = ( + 'disable', + ) + + def status(self, obj): + return 'ON' if TwoFactorSecretTokens.is_enabled_for_user(obj.user) else 'OFF' + + def last_updated(self, obj): + return obj.updated + + def email(self, obj): + return obj.user.email + + @api_admin.action(permissions=('change',)) + def disable(self, request, queryset): + """ + :param request: + :param queryset: + :type queryset: list[TwoFactorSecretTokens] + """ + try: + with atomic(): + for tfs in queryset: + tfs.drop() + except BaseException as e: + messages.error(request, e) + + +@api_admin.register(UserExchangeFee) +class UserExchangeFeeApiAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(UserFee) +class UserFeeApiAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(UserKYC) +class UserKYCApiAdmin(DefaultApiAdmin): + search_fields = ['user__email'] + + +@api_admin.register(WithdrawalFee) +class WithdrawalFeeApiAdmin(DefaultApiAdmin): + list_display = [ + 'id', + 'currency', + 'blockchain_currency', + 'address_fee', + ] + filterset_fields = ['currency', 'blockchain_currency'] + + +@api_admin.register(FeesAndLimits) +class FeesAndLimitsApiAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(WalletTransactions) +class WalletTransactionsApiAdmin(ReadOnlyMixin, DefaultApiAdmin): + # list_filter = ['state', 'currency', 'status', 'created', AddressTypeFilter] + fields = ('created', 'user', 'currency', 'blockchain', 'amount', 'tx_amount', 'tx_hash', 'status', 'state') + list_display = ['created', 'user', 'currency', 'blockchain', 'amount', 'tx_amount', 'tx_hash', 'status', 'state'] + filterset_fields = [ + 'state', + 'currency', + 'wallet__blockchain_currency', + 'status', + 'created'] + search_fields = [ + 'transaction__user__email', + 'transaction__user__id', + 'id', + 'tx_hash'] + + actions = { + 'revert': [], + 'recheck_kyt': [], + 'force_deposit_and_accumulate': [], + 'external_accumulation': [ + {'label': 'External address', 'name': 'external_address'}, + ], + } + + def get_queryset(self): + qs = super(WalletTransactionsApiAdmin, self).get_queryset().annotate( + user=F('wallet__user__email'), + blockchain=F('wallet__blockchain_currency'), + tx_amount=F('transaction__amount'), + ) + return qs.prefetch_related('wallet', 'transaction', 'transaction__user') + + @serial_field(serial_class=CurrencySerialRestField) + def blockchain(self, obj): + return obj.blockchain + + def tx_amount(self, obj): + return obj.tx_amount + + def user(self, obj): + return obj.user + + # custom actions + @api_admin.action(permissions=True) + def revert(self, request, queryset): + """ + :param queryset: + :type queryset: list[WalletTransactions] + """ + try: + with atomic(): + for wallet_tr in queryset: + wallet_tr.revert() + except Exception as e: + messages.error(request, e) + + @api_admin.action(permissions=('change',)) + def recheck_kyt(self, request, queryset): + for entry in queryset: + entry.check_scoring() + + recheck_kyt.short_description = 'Recheck KYT' + + @api_admin.action(permissions=('change',)) + def force_deposit_and_accumulate(self, request, queryset: List[WalletTransactions]): + for wallet_tr in queryset: + wallet_tr.force_deposit() + + force_deposit_and_accumulate.short_description = 'Force deposit and accumulate' + + @api_admin.action(permissions=('change',)) + def external_accumulation(self, request, queryset: List[WalletTransactions]): + data = request.POST or request.data + address = data.get('external_address') + for wallet_tr in queryset: + wallet_tr.set_external_accumulation_address(address) + + external_accumulation.short_description = 'External accumulation' + + +@api_admin.register(WalletTransactionsRevert) +class WalletTransactionsRevertApiAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(Exchange) +class ExchangeApiAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(ExecutionResult) +class ExecutionResultApiAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(OrderChangeHistory) +class OrderChangeHistoryApiAdmin(ReadOnlyMixin, DefaultApiAdmin): + pass + + +@api_admin.register(OrderStateChangeHistory) +class OrderStateChangeHistoryApiAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(ExternalPricesHistory) +class ExternalPricesHistoryApiAdmin(DefaultApiAdmin): + list_display = ['created', 'pair', 'price'] + filterset_fields = ['created', 'pair'] + + +@api_admin.register(TradesAggregatedStats) +class TradesAggregatedStatsApiAdmin(DefaultApiAdmin): + list_display = [ + 'pair', + 'period', + 'ts', + 'min_price', + 'max_price', + 'avg_price', + 'open_price', + 'close_price', + 'volume', + 'amount', + 'num_trades', + 'fee_base', + 'fee_quoted', + ] + readonly_fields = list_display + filterset_fields = ['pair', 'period'] + + +@api_admin.register(UserPairDailyStat) +class UserPairDailyStatApiAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(WalletHistoryItem) +class WalletHistoryItemApiAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(User) +class UserApiAdmin(DefaultApiAdmin): + vue_resource_extras = {'aside': {'edit': True}} + list_display = ('id', 'date_joined', 'email', 'first_name', 'last_name', 'user_type', 'is_staff', 'is_superuser', + 'is_active', 'kyc', 'kyc_reject_type', 'two_fa', + 'withdrawals_count', 'orders_count', 'email_verified', 'phone_verified',) + fields = ( + 'id', + 'username', + 'email', + 'first_name', + 'last_name', + 'is_staff', + 'is_superuser', + 'is_active' + ) + readonly_fields = ['id', 'is_superuser', 'is_staff', 'user_type'] + search_fields = ['username'] + ordering = ('-date_joined',) + actions = { + 'confirm_email': [], + 'topup': [ + {'label': 'Amount', 'name': 'amount', 'type': 'decimal', }, + {'label': 'Currency', 'name': 'currency', }, + {'label': 'Password', 'name': 'password', }, + ], + 'drop_2fa': [], + 'drop_sms': [] + } + filterset_fields = ['profile__user_type', ] + + def get_queryset(self): + qs = super(UserApiAdmin, self).get_queryset() + return qs.annotate( + withdrawals_count=Count('withdrawalrequest', distinct=True), + orders_count=Count('order', distinct=True), + two_fa=Case( + When(twofactorsecrettokens__secret__isnull=False, then=Value(True)), + default=Value(False), + output_field=models.BooleanField() + ), + kyc=Case( + When(Q(userkyc__forced_approve=True) | Q(userkyc__reviewAnswer=UserKYC.ANSWER_GREEN), + then=Value('green')), + When(userkyc__reviewAnswer=UserKYC.ANSWER_RED, then=Value('red')), + default=Value('no'), + output_field=models.CharField(), + ), + kyc_reject_type=Case( + When(userkyc__reviewAnswer=UserKYC.ANSWER_RED, then=F('userkyc__rejectType')), + default=Value(''), + output_field=models.CharField(), + ) + ).prefetch_related('withdrawalrequest_set', 'order_set', 'twofactorsecrettokens_set', 'userkyc') + + def kyc(self, obj): + return obj.kyc + + def kyc_reject_type(self, obj): + return obj.kyc_reject_type + + def two_fa(self, obj): + return obj.two_fa + + def withdrawals_count(self, obj): + return obj.withdrawals_count + + def orders_count(self, obj): + return obj.orders_count + + def email_verified(self, obj): + exists = EmailAddress.objects.filter( + user=obj, + email=obj.email + ).exists() + return exists + + def phone_verified(self, obj): + return bool(obj.profile.phone) + + def user_type(self, obj): + return obj.profile.get_user_type_display() + + @api_admin.action(permissions=[IsSuperAdminUser]) + def topup(self, request, queryset): + currency = request.data.get('currency') + amount = request.data.get('amount') + password = request.data.get('password') + if not currency or not amount: + raise ValidationError('Currency or amount incorrect!') + if password != settings.ADMIN_MASTERPASS: + raise ValidationError('Incorrect password!') + + currency = Currency.get(currency) + for user in queryset: + with atomic(): + tx = Transaction.topup(user.id, currency, amount, {'1': 1}, reason=REASON_MANUAL_TOPUP) + create_or_update_wallet_history_item_from_transaction(tx) + + topup.short_description = 'Make Topup' + + @api_admin.action(permissions=True) + def confirm_email(self, request, queryset): + for user in queryset: + ea = EmailAddress.objects.filter( + user=user, + email=user.email + ).first() + + if ea: + ea.verified = True + ea.save() + + confirm_email.short_description = 'Confirm email' + + @api_admin.action(permissions=True) + def drop_2fa(self, request, queryset): + for user in queryset: + TwoFactorSecretTokens.drop_for_user(user) + drop_2fa.short_description = 'Drop 2FA' + + @api_admin.action(permissions=True) + def drop_sms(self, request, queryset): + for user in queryset: + user.profile.drop_sms() + drop_sms.short_description = 'Drop SMS' + + +@api_admin.register(Transaction) +class TransactionApiAdmin(DefaultApiAdmin): + list_display = ['user', 'created', 'reason', 'currency', 'amount', 'state'] + filterset_fields = ['reason', 'currency', 'created', 'state',] + search_fields = ['user__email'] + ordering = ('-created',) + + +@api_admin.register(Balance) +class BalanceApiAdmin(ReadOnlyMixin, DefaultApiAdmin): + vue_resource_extras = {'searchable_fields': ['user']} + list_display = ['user', 'currency', 'total', 'amount', 'amount_in_orders'] + filterset_fields = ['currency'] + search_fields = ['user__email'] + + def get_queryset(self): + qs = super(BalanceApiAdmin, self).get_queryset() + return qs.annotate( + total=Sum(F('amount') + F('amount_in_orders')) + ).prefetch_related('user') + + def total(self, obj): + return obj.total + + +@api_admin.register(AllOrder) +class AllOrderApiAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = ['id', 'user', 'created', 'pair', 'operation', 'type', 'quantity', + 'quantity_left', 'price', 'amount', 'fee', 'state', 'executed', + 'state_changed_at'] + fields = ['id', 'user', 'pair', 'state'] + ordering = ('-created',) + filterset_fields = ['pair', 'operation', 'state', 'executed', 'created'] + actions = [ + 'cancel_order', + # 'revert_orders', + 'revert_orders_balance', + ] + search_fields = ['user__email'] + + def created(self, obj): + return obj.in_transaction.created + + def amount(self, obj): + return obj.amount or 0 + + def fee(self, obj): + return obj.fee or 0 + + def get_queryset(self): + qs = super(AllOrderApiAdmin, self).get_queryset() + return qs.prefetch_related( + 'user', 'executionresult_set', 'in_transaction', + ).annotate( + fee=Sum('executionresult__fee_amount'), + amount=F('quantity') * F('price'), + ) + + # custom actions + @api_admin.action(permissions=True) + def cancel_order(self, request, queryset): + for order in queryset: + order.delete(by_admin=True) + cancel_order.short_description = 'Close (cancel) orders' + + @api_admin.action(permissions=True) + def revert_orders(self, request, queryset): + with transaction.atomic(): + try: + for order in queryset: + order.revert(check_balance=False) + except ValidationError as e: + messages.error(request, e) + revert_orders.short_description = 'Revert orders' + + @api_admin.action(permissions=True) + def revert_orders_balance(self, request, queryset): + try: + with transaction.atomic(): + queryset: List[Order] = queryset.order_by('id') + balances = {} + for order in queryset: + balances = order.revert(balances, check_balance=True) + for u_id, item in balances.items(): + for cur, amount in item.items(): + try: + currency = Currency.get(cur) + if amount < 0: + BalanceManager.decrease_amount( + u_id, currency, amount) + else: + BalanceManager.increase_amount( + u_id, currency, amount) + except NotEnoughFunds as e: + raise ValidationError(f'Not enough funds! hold# ' + f'user {u_id}, ' + f'{amount} ' + f'{currency}' + ) + except ValidationError as e: + messages.error(request, e) + except NotEnoughFunds as e: + messages.error(request, e) + revert_orders_balance.short_description = 'Revert orders balance' + + +@api_admin.register(AllOrderNoBot) +class AllOrderNoBotApiAdmin(AllOrderApiAdmin): + + def get_queryset(self): + qs = super(AllOrderNoBotApiAdmin, self).get_queryset() + qs = qs.exclude(user__username__iregex=BOT_RE) + return qs + + +@api_admin.register(Topups) +class TopupsApiAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = [ + 'user', + 'created', + 'reason', + 'currency', + 'amount', + 'txhash', + 'address'] + # list_filter = [TopupReasonFilter, CurrencyFilter, ('created', DateRangeFilter), ] + filterset_fields = ['currency', 'reason', 'created'] + search_fields = ['user__email'] + ordering = ('-created',) + + def txhash(self, obj): + return obj.txhash + + def address(self, obj): + return obj.address + + def get_queryset(self): + qs = super(TopupsApiAdmin, self).get_queryset() + qs = qs.filter(reason__in=[REASON_TOPUP,]) + qs = qs.prefetch_related('wallet_transaction').annotate( + txhash=F('wallet_transaction__tx_hash')) + qs = qs.prefetch_related('wallet_transaction__wallet').annotate( + address=F('wallet_transaction__wallet__address')) + return qs + + +@api_admin.register(Match) +class MatchApiAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = ['created', 'pair', 'user1', 'operation', + 'user2', 'quantity', 'price', 'total', 'fee', ] + filterset_fields = ['order__operation', 'pair', 'created'] + search_fields = ['order__user__email', 'matched_order__user__email'] + ordering = ('-created',) + + readonly_fields = ( + 'transaction', + 'user', + 'order', + 'matched_order', + 'cacheback_transaction' + ) + + def operation(self, obj): + return obj.operation + + def user1(self, obj): + return obj.user1_name + + def user2(self, obj): + return obj.user2_name + + def total(self, obj): + return f'{obj.total:.8f}' + + def fee(self, obj): + return f'{obj.fee:.2f}%' + + def get_queryset(self): + bots_ids = get_bots_ids() + qs = super(MatchApiAdmin, self).get_queryset() + qs = qs.filter(cancelled=False) + qs = qs.select_related('order', 'matched_order') + qs = qs.annotate( + user1_id=F('order__user_id'), + user2_id=F('matched_order__user_id'), + user1_name=F('order__user__username'), + user2_name=F('matched_order__user__username'), + total=F('quantity') * F('price'), + fee=F('fee_rate') * Value(100), + operation=Case( + When(order__operation=SELL, then=Value('SELL')), + default=Value('BUY'), + output_field=models.CharField(), + ) + ) + qs = qs.filter(~Q(Q(user1_id__in=bots_ids) & Q(user2_id__in=bots_ids))) + + return qs + + +# TODO optimize +@api_admin.register(WithdrawalRequest) +class WithdrawalRequestAdmin(ReadOnlyMixin, RestFulModelAdmin): + list_display = ( + 'created', 'user', 'approved', 'confirmed', 'currency', 'blockchain', 'amount', + 'state', 'details', 'sci_gate', 'txid', 'is_freezed',) + + filterset_fields = ['currency', 'approved', 'confirmed', 'state'] + search_fields = ['user__email', 'data__destination'] + ordering = ('-created', ) + + actions = [ + 'cancel_withdrawal_request', + 'pause', + 'unpause', + 'approve', + 'disable_approve', + 'confirm', + 'unconfirm', + ] + global_actions = [ + 'export_created_eth', + ] + + def blockchain(self, obj): + return obj.data.get('blockchain_currency') or obj.currency.code + + def sci_gate(self, obj): + if obj.sci_gate_id is not None: + return GATES[obj.sci_gate_id].NAME + return '' + + def details(self, obj): + if obj.sci_gate_id is None: + return obj.data.get('destination') + return mark_safe('
'.join(f'{k}: {v}' for k, v in obj.data.items())) + + def is_freezed(self, obj): + return obj.is_freezed + + def get_queryset(self): + qs = super(WithdrawalRequestAdmin, self).get_queryset() + now = timezone.now() + qs = qs.annotate( + is_freezed=ExpressionWrapper( + Q(user__profile__payouts_freezed_till__gt=now), + output_field=models.BooleanField(), + ) + ) + return qs.prefetch_related('user', 'user__profile') + + # custom actions + @api_admin.action(permissions=('view', 'change',)) + def cancel_withdrawal_request(self, request, queryset): + try: + with transaction.atomic(): + for withdrawal_request in queryset: + withdrawal_request.cancel() + except Exception as e: + messages.error(request, e) + cancel_withdrawal_request.short_description = 'Cancel' + + @api_admin.action(permissions=('change',), custom_response=True) + def export_created_eth(self, request, queryset): + response = HttpResponse(content_type='text/csv') + response['content-disposition'] = 'attachment; filename=ETH_USDT_{}.csv'.format( + timezone.now()) + writer = csv.writer(response) + queryset = queryset.model.objects.filter( + state=0, confirmed=True, currency__in=['ETH', 'USDT']) + for obj in queryset: + row = writer.writerow([obj.id, obj.user.email, obj.data.get( + 'destination'), obj.currency.code, obj.amount]) + + return response + export_created_eth.short_description = 'Export created ETH,USDT withdrawals' + + @api_admin.action(permissions=('view', 'change',)) + def pause(self, request, queryset): + for wd in queryset: + wd.pause() + pause.short_description = 'Pause' + + @api_admin.action(permissions=('view', 'change',)) + def unpause(self, request, queryset): + for wd in queryset: + wd.unpause() + unpause.short_description = 'Unpause' + + @api_admin.action(permissions=('view', 'change',)) + def approve(self, request, queryset): + for entry in queryset: + entry.approved = True + if not entry.confirmed: + raise ValidationError('Can not approve unconfirmed request!') + entry.save() + approve.short_description = 'Approve' + + @api_admin.action(permissions=('view', 'change',)) + def disable_approve(self, request, queryset): + for entry in queryset: + entry.approved = False + entry.save() + disable_approve.short_description = 'Disable Approve' + + @api_admin.action(permissions=('change',)) + def confirm(self, request, queryset): + for entry in queryset: + entry.confirmed = True + entry.save() + + confirm.short_description = 'Confirm' + + @api_admin.action(permissions=('change',)) + def unconfirm(self, request, queryset): + for entry in queryset: + entry.confirmed = False + entry.save() + + unconfirm.short_description = 'Unconfirm' + + +@api_admin.register(UserDailyStat) +class UserPairDailyStatAdmin(NoDeleteMixin, DefaultApiAdmin): + list_display = ['user', 'pair', 'day', 'currency1', 'currency2', 'volume_got1', 'volume_got2', + 'fee_amount_paid1', 'fee_amount_paid2', 'volume_spent1', 'volume_spent2'] + readonly_fields = list_display + filterset_fields = ['day', 'pair', 'currency1', 'currency2'] + search_fields = ['user__email'] + + def get_queryset(self): + qs = super(UserPairDailyStatAdmin, self).get_queryset() + return qs.exclude(fee_amount_paid1=0, fee_amount_paid2=0) + + +@api_admin.register(UserRestrictions) +class UserRestrictionsAdmin(DefaultApiAdmin): + list_display = [ + 'user', + 'disable_topups', + 'disable_withdrawals', + 'disable_orders'] + fields = [ + 'user', + 'disable_topups', + 'disable_withdrawals', + 'disable_orders'] + readonly_fields = ['user'] + search_fields = ['user__email'] + ordering = ['-user'] + + +@api_admin.register(PayGateTopup) +class PayGateTopupAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = ('created', 'user', 'currency', 'amount', 'tx_amount', + 'sci_gate', 'state_colored', 'status_colored',) + fields = ('tx_link', 'user', 'currency', 'amount', 'tx_amount', 'sci_gate', + 'our_fee_amount', 'state_colored', 'status_colored', 'pretty_data',) + readonly_fields = ('tx', 'tx_link', 'user', 'currency', 'amount', 'tx_amount', 'state_colored', + 'status_colored', 'sci_gate', 'our_fee_amount', 'pretty_data',) + + filterset_fields = ['currency', 'created'] + search_fields = ['user__email', 'user__id', 'id', ] + actions = ('revert',) + ordering = ['-created'] + + def tx_amount(self, obj): + return obj.tx.amount if obj.tx else 0 + + def tx_link(self, obj): + return mark_safe( + '%s' % (reverse('admin:admin_panel_topups_change', + args=[obj.tx_id]), obj.tx) + ) + + tx_link.allow_tags = True + tx_link.short_description = "Transaction" + + def pretty_data(self, obj): + return mark_safe( + f'
{json.dumps(obj.data or dict(), indent=4, sort_keys=True)}
') + + pretty_data.allow_tags = True + pretty_data.short_description = "Data" + + def sci_gate(self, obj): + if obj.gate_id is not None: + return GATES[obj.gate_id].NAME + return '' + + sci_gate.short_description = ugettext_lazy('Gate') + + def state_colored(self, obj): + color = 'black' + if obj.state == PayGateTopup.STATE_COMPLETED: + color = 'green' + elif obj.state == PayGateTopup.STATE_PENDING: + color = 'blue' + elif obj.state == PayGateTopup.STATE_FAILED: + color = 'red' + + return mark_safe( + f'{obj.get_state_display()}') + + state_colored.short_description = ugettext_lazy('State') + + def status_colored(self, obj: PayGateTopup): + color = 'black' + if obj.status == PayGateTopup.STATUS_NOT_SET: + color = 'burlywood' + elif obj.status == PayGateTopup.STATUS_REVERTED: + color = 'darkviolet' + + return mark_safe( + f'{obj.get_status_display()}') + + status_colored.short_description = ugettext_lazy('Status') + + # custom actions + @api_admin.action(permissions=True) + def revert(self, request, queryset: List[PayGateTopup]): + try: + with atomic(): + for pay_gate in queryset: + pay_gate.revert() + except Exception as e: + messages.error(request, e) + + +@api_admin.register(DepositsWithdrawalsStats) +class DepositsWithdrawalsStatsAdmin(JsonListApiViewMixin, ReadOnlyMixin, DefaultApiAdmin): + list_display = ['created', ] + ordering = ('-created',) + actions = ['cold_wallet_stats'] + global_actions = ['export_all'] + + json_list_fields = { + 'stats': generate_stats_fields() + } + + @api_admin.action(permissions=True) + def cold_wallet_stats(self, request, queryset): + for dw in queryset.order_by('created'): + calculate_topups_and_withdrawals(dw) + time.sleep(0.3) + + cold_wallet_stats.short_description = 'Fetch cold wallet stats' + + @api_admin.action(permissions=True, custom_response=True) + def export_all(self, request, queryset): + response = HttpResponse(content_type='text/csv') + response['content-disposition'] = 'attachment; filename=deposits_withdrawals_stats.csv' + serializer = self.get_serializer(self.get_queryset().order_by('-created'), many=True) + data = serializer.data + if data: + writer = csv.DictWriter(response, fieldnames=list(data[0])) + writer.writeheader() + writer.writerows(data) + return response + export_all.short_description = 'Export All' + + +@api_admin.register(DisabledCoin) +class DisabledInoutCoinAdmin(DefaultApiAdmin): + list_filter = ['currency'] + DISABLE_COIN_STATES + list_display = ['currency'] + DISABLE_COIN_STATES + readonly_fields = ('currency',) + + +@api_admin.register(UserWallet) +class UserWalletAdmin(ReadOnlyMixin, DefaultApiAdmin): + fields = ['created', 'user', 'currency', 'blockchain_currency', 'address', 'block_type'] + list_display = ['created', 'user', 'currency', 'blockchain_currency', 'address', 'block_type'] + search_fields = ['user__username', 'address'] + filterset_fields = ['currency', 'blockchain_currency', 'block_type'] + + + actions = ['block_deposits', 'block_accumulations', 'unblock'] + + @api_admin.action(permissions=[IsSuperAdminUser]) + def block_deposits(self, request, queryset): + queryset.update(block_type=UserWallet.BLOCK_TYPE_DEPOSIT) + block_deposits.short_description = 'Block Deposits' + + @api_admin.action(permissions=[IsSuperAdminUser]) + def block_accumulations(self, request, queryset): + queryset.update(block_type=UserWallet.BLOCK_TYPE_DEPOSIT_AND_ACCUMULATION) + block_accumulations.short_description = 'Block Deposits and Accumulations' + + @api_admin.action(permissions=[IsSuperAdminUser]) + def unblock(self, request, queryset): + queryset.update(block_type=UserWallet.BLOCK_TYPE_NOT_BLOCKED) + unblock.short_description = 'Unblock' + + +@api_admin.register(PairSettings) +class PairSettingsAdmin(DefaultApiAdmin): + _fields = ['pair', 'is_enabled', 'is_autoorders_enabled', 'price_source', 'custom_price', 'deviation', 'precisions'] + list_display = _fields + fields = _fields + + +@api_admin.register(CoinInfo) +class CoinInfoAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(LogEntry) +class LogEntryAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = ('action_time', 'user', 'content_type', 'object_repr', 'action_flag', 'message') + filterset_fields = ['action_time', 'action_flag'] + search_fields = ['object_repr', 'user__email'] + + def message(self, obj): + return mark_safe(obj.change_message) + + + +@api_admin.register(TOTPDevice) +class TOTPDeviceAdmin(DefaultApiAdmin): + list_display = ['user', 'name', 'confirmed', 'config_url'] + + def get_queryset(self): + queryset = super().get_queryset() + queryset = queryset.select_related('user') + + return queryset + + # def qrcode_link(self, device): + # try: + # href = reverse('admin:otp_totp_totpdevice_config', kwargs={'pk': device.pk}) + # link = format_html('qrcode', href) + # except Exception: + # link = '' + # return link + # qrcode_link.short_description = "QR Code" + + # # custom actions + # @api_admin.action(permissions=True) + # def revert(self, request, queryset): + # + # def qrcode_view(self, request, pk): + # if settings.OTP_ADMIN_HIDE_SENSITIVE_DATA: + # raise PermissionDenied() + # + # device = TOTPDevice.objects.get(pk=pk) + # if not self.has_view_or_change_permission(request, device): + # raise PermissionDenied() + # + # try: + # import qrcode + # import qrcode.image.svg + # + # img = qrcode.make(device.config_url, image_factory=qrcode.image.svg.SvgImage) + # response = HttpResponse(content_type='image/svg+xml') + # img.save(response) + # except ImportError: + # response = HttpResponse('', status=503) + # + # return response + + + + diff --git a/admin_rest/apps.py b/admin_rest/apps.py new file mode 100644 index 0000000..11b18e1 --- /dev/null +++ b/admin_rest/apps.py @@ -0,0 +1,18 @@ +from django.apps import AppConfig + +from django.apps import apps +import importlib +from django.utils.module_loading import module_has_submodule + + +class AdminRestConfig(AppConfig): + name = 'admin_rest' + + def ready(self): + app_configs = apps.get_app_configs() + + for app_config in app_configs: + module_path = app_config.module.__name__ + if module_has_submodule(importlib.import_module(module_path), 'admin_rest'): + admin_rest_module = importlib.import_module(module_path + '.admin_rest') + diff --git a/admin_rest/fields.py b/admin_rest/fields.py new file mode 100644 index 0000000..6179f39 --- /dev/null +++ b/admin_rest/fields.py @@ -0,0 +1,42 @@ +from functools import wraps + +from rest_framework.relations import RelatedField + +from core.consts.currencies import CURRENCIES_LIST +from core.currency import CurrencySerialField + + +class ForeignSerialField(RelatedField): + """ + A read only field that represents its targets using their + plain string representation. + """ + + def __init__(self, **kwargs): + kwargs['read_only'] = True + super().__init__(**kwargs) + + def to_representation(self, value): + return {'id': value.pk, 'value': str(value)} + + +class CurrencySerialRestField(CurrencySerialField): + # choices = dict(CURRENCIES_LIST) # for OPTIONS action + + def to_representation(self, obj): + return obj.id + + @property + def choices(self): + return dict(CURRENCIES_LIST) + + +def serial_field(serial_class): + def decorator(func): + func.serial_class = serial_class + + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + return decorator diff --git a/admin_rest/filters.py b/admin_rest/filters.py new file mode 100644 index 0000000..fce3caf --- /dev/null +++ b/admin_rest/filters.py @@ -0,0 +1,119 @@ +import re +from copy import deepcopy + +from django.db import models +from django_filters.constants import EMPTY_VALUES +from django_filters.filters import CharFilter, DateFilter +from django_filters.rest_framework import DjangoFilterBackend +from django_filters.rest_framework.filterset import FilterSet, FILTER_FOR_DBFIELD_DEFAULTS + +from core.currency import Currency, CurrencyModelField +from core.pairs import Pair +from core.pairs import PairModelField + + +class CurrencyModelFilter(CharFilter): + field_value_class = Currency + + def filter(self, qs, value): + if value is None: + return self.get_method(qs)(**{self.field_name: None}) + + if value in EMPTY_VALUES: + return qs + + if self.field_value_class.exists(value): + value = self.field_value_class.get(value).id + else: + return qs.none() + + if self.distinct: + qs = qs.distinct() + qs = self.get_method(qs)(**{self.field_name: value}) + return qs + + +class PairModelFilter(CurrencyModelFilter): + field_value_class = Pair + + +FILTER_FOR_DBFIELD_DEFAULTS = deepcopy(FILTER_FOR_DBFIELD_DEFAULTS) +FILTER_FOR_DBFIELD_DEFAULTS.update({ + CurrencyModelField: {'filter_class': CurrencyModelFilter}, + PairModelField: {'filter_class': PairModelFilter}, + models.DateTimeField: {'filter_class': DateFilter}, + models.DateField: {'filter_class': DateFilter}, +}) + + +def reparse_query_data(query_data): + res = {} + for param, value in query_data.items(): + if '[' and ']' in param: + new_param_name = param.split('[')[0] + regex = re.compile('%s\[([\w\d_]+)\]' % new_param_name) + match = regex.match(param) + inner_key = match.group(1) + if inner_key == 'start': + res[new_param_name+'__gte'] = value + elif inner_key == 'end': + res[new_param_name + '__lte'] = value + else: + res[param] = value + return res + + +class GenericFilterset(FilterSet): + FILTER_DEFAULTS = FILTER_FOR_DBFIELD_DEFAULTS + + def __init__(self, *args, **kwargs): + super(GenericFilterset, self).__init__(*args, **kwargs) + self.data = reparse_query_data(self.data) + + def filter_queryset(self, queryset): + # dirty hack + data = dict(self.data) + for name, value in self.form.cleaned_data.items(): + if name in data and name == 'id': + value = data[name] + if isinstance(value, list) and len(value) > 1: + lookup = f'{name}__in' + queryset = queryset.filter(**{lookup: value}) + else: + if isinstance(value, list): + value = value[0] + queryset = self.filters[name].filter(queryset, value) + + else: + queryset = self.filters[name].filter(queryset, value) + return queryset + + +class GenericAllFieldsFilter(DjangoFilterBackend): + filterset_base = GenericFilterset + + def get_filterset_class(self, view, queryset=None): + """ + Return the `FilterSet` class used to filter the queryset. + """ + defined_filterset_fields = getattr(view, 'filterset_fields', None) + filterset_fields = {} + model_fields = {f.name: f for f in queryset.model._meta.fields} + for field_name in defined_filterset_fields: + if field_name in model_fields and type(model_fields[field_name]) in [models.DateField, models.DateTimeField]: + lookups = ['gte', 'lte',] + else: + lookups = ['exact'] + filterset_fields[field_name] = lookups + + if defined_filterset_fields and queryset is not None: + MetaBase = getattr(self.filterset_base, 'Meta', object) + + class AutoFilterSet(self.filterset_base): + class Meta(MetaBase): + model = queryset.model + fields = filterset_fields + + return AutoFilterSet + + return None \ No newline at end of file diff --git a/admin_rest/migrations/0001_initial.py b/admin_rest/migrations/0001_initial.py new file mode 100644 index 0000000..cd48c69 --- /dev/null +++ b/admin_rest/migrations/0001_initial.py @@ -0,0 +1,181 @@ +# Generated by Django 3.2.7 on 2022-08-12 12:45 + +import admin_rest.models +import django.contrib.auth.models +from django.db import migrations + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('core', '0001_initial'), + ('auth', '0012_alter_user_first_name_max_length'), + ] + + operations = [ + migrations.CreateModel( + name='AllOrder', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.order',), + ), + migrations.CreateModel( + name='AllOrderNoBot', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.order',), + ), + migrations.CreateModel( + name='Balance', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.balance',), + ), + migrations.CreateModel( + name='BalanceSummary', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.balance',), + ), + migrations.CreateModel( + name='ExchangeFee', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.userfee',), + ), + migrations.CreateModel( + name='ExchangeUser', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('auth.user',), + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + migrations.CreateModel( + name='Match', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.executionresult',), + ), + migrations.CreateModel( + name='RefDetails', + fields=[ + ], + options={ + 'verbose_name_plural': 'Ref program details', + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('auth.user',), + managers=[ + ('objects', admin_rest.models.RefManager()), + ], + ), + migrations.CreateModel( + name='Topups', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.transaction',), + ), + migrations.CreateModel( + name='Transaction', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.transaction',), + ), + migrations.CreateModel( + name='UserDailyStat', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.userpairdailystat',), + ), + migrations.CreateModel( + name='UserKYCProxy', + fields=[ + ], + options={ + 'verbose_name': 'User KYC', + 'verbose_name_plural': 'Users KYC', + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.userkyc',), + ), + migrations.CreateModel( + name='Withdrawal', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.transaction',), + ), + migrations.CreateModel( + name='WithdrawalRequest', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.withdrawalrequest',), + ), + ] diff --git a/admin_rest/migrations/__init__.py b/admin_rest/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/admin_rest/mixins.py b/admin_rest/mixins.py new file mode 100644 index 0000000..240cf66 --- /dev/null +++ b/admin_rest/mixins.py @@ -0,0 +1,90 @@ +from decimal import Decimal + +from rest_framework import status +from rest_framework.response import Response + +from admin_rest.filters import GenericAllFieldsFilter + + +class NoDeleteMixin: + """Prevents destroy""" + @classmethod + def has_delete_permission(cls): + return False + + def destroy(self, request, pk=None): + response = {'error': 'Delete function is not offered in this path.'} + return Response(response, status=status.HTTP_403_FORBIDDEN) + + +class NoUpdateMixin: + """Prevents update/partial update""" + @classmethod + def has_update_permission(cls): + return False + + def update(self, request, pk=None): + response = {'error': 'Update function is not offered in this path.'} + return Response(response, status=status.HTTP_403_FORBIDDEN) + + def partial_update(self, request, pk=None): + response = {'error': 'Update function is not offered in this path.'} + return Response(response, status=status.HTTP_403_FORBIDDEN) + + +class NoCreateMixin: + """Prevents create""" + @classmethod + def has_add_permission(cls): + return False + + def create(self, request): + response = {'error': 'Create function is not offered in this path.'} + return Response(response, status=status.HTTP_403_FORBIDDEN) + + +class ReadOnlyMixin(NoCreateMixin, + NoUpdateMixin, + NoDeleteMixin): + """Only list/retrieve action allowed""" + + def get_readonly_fields(self): + return self.list_display + + +class JsonListApiViewMixin(object): + """ + Displays json field values in list view page + Uses json_list_fields dict, where the key is JSONField name, the value is some key in json + """ + json_list_fields = {} + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + all_json_fields = [item for sublist in self.json_list_fields.values() for item in sublist] + self.list_display += all_json_fields + + for json_field, fields in self.json_list_fields.items(): + for custom_field in fields: + self._generate_json_field(json_field, custom_field) + + def get_handler_fn(self, json_field_name, fieldname): + def handler(view, obj): + res = getattr(obj, json_field_name).get(fieldname) + # show res as 0.0000000 instead of 0E-8 + if res and res.replace('E-', '').isdigit(): + res = '{:f}'.format(Decimal(res)) + return res + + handler.__name__ = fieldname + return handler + + def _generate_json_field(self, json_field_name, fieldname): + handler = self.get_handler_fn(json_field_name, fieldname) + setattr(self, fieldname, handler) + + +class NonPaginatedListMixin(object): + """Removes pagination""" + filter_backends = (GenericAllFieldsFilter, ) diff --git a/admin_rest/models.py b/admin_rest/models.py new file mode 100644 index 0000000..bf8e222 --- /dev/null +++ b/admin_rest/models.py @@ -0,0 +1,97 @@ +from django.contrib.auth.models import User, UserManager +from django.db.models.aggregates import Count + +from core.models.cryptocoins import UserWallet +from core.models.facade import UserFee, UserKYC +from core.models.inouts.balance import Balance as BaseBalance +from core.models.inouts.transaction import Transaction as BaseTransaction +from core.models.inouts.withdrawal import WithdrawalRequest as BaseWithdrawalRequest +from core.models.orders import ExecutionResult +from core.models.orders import Order +from core.models.stats import UserPairDailyStat as BaseUserPairDailyStat + + +class ExchangeFee(UserFee): + class Meta: + proxy = True + + +class Balance(BaseBalance): + + class Meta: + proxy = True + + +class Transaction(BaseTransaction): + + class Meta: + proxy = True + + +class Topups(BaseTransaction): + + class Meta: + proxy = True + + +class Withdrawal(BaseTransaction): + + class Meta: + proxy = True + + +class AllOrder(Order): + + class Meta: + proxy = True + + +class AllOrderNoBot(Order): + + class Meta: + proxy = True + + +class Match(ExecutionResult): + + class Meta: + proxy = True + + +class WithdrawalRequest(BaseWithdrawalRequest): + class Meta: + proxy = True + + +class UserDailyStat(BaseUserPairDailyStat): + class Meta: + proxy = True + + +class RefManager(UserManager): + def get_queryset(self): + return super().get_queryset().order_by().annotate(ref_count=Count('owner_user')).filter(ref_count__gt=0) + + def select_related(self, *args, **kwargs): + return self + + +class RefDetails(User): + class Meta: + verbose_name_plural = 'Ref program details' + proxy = True + + objects = RefManager() + + +class UserKYCProxy(UserKYC): + class Meta: + verbose_name = 'User KYC' + verbose_name_plural = 'Users KYC' + proxy = True + + +class BalanceSummary(BaseBalance): + class Meta: + proxy = True + diff --git a/admin_rest/permissions.py b/admin_rest/permissions.py new file mode 100644 index 0000000..55991be --- /dev/null +++ b/admin_rest/permissions.py @@ -0,0 +1,10 @@ +from rest_framework.permissions import BasePermission + + +class IsSuperAdminUser(BasePermission): + """ + Allows access only to superadmin users. + """ + + def has_permission(self, request, view): + return bool(request.user and request.user.is_active and request.user.is_staff and request.user.is_superuser) \ No newline at end of file diff --git a/admin_rest/restful_admin.py b/admin_rest/restful_admin.py new file mode 100644 index 0000000..85dbcad --- /dev/null +++ b/admin_rest/restful_admin.py @@ -0,0 +1,988 @@ +import json +from collections import OrderedDict +from functools import wraps + +from django.conf import settings +from django.contrib.admin.models import LogEntry, CHANGE +from django.contrib.admin.options import get_content_type_for_model +from django.contrib.auth import get_permission_codename +from django.contrib.auth import get_user_model +from django.contrib.postgres.fields import JSONField +from django.db import models +from django.db.models.base import ModelBase +from django.db.models.fields import NOT_PROVIDED +from django.forms import model_to_dict +from django.utils.encoding import force_str +from django.utils.functional import cached_property +from django.utils.html import escape +from django.utils.safestring import SafeString, mark_safe +from rest_framework import filters +from rest_framework import serializers +from rest_framework import viewsets, status +from rest_framework.decorators import action as base_action +from rest_framework.fields import ImageField +from rest_framework.fields import SkipField +from rest_framework.metadata import SimpleMetadata +from rest_framework.permissions import BasePermission +from rest_framework.relations import PrimaryKeyRelatedField, PKOnlyObject, ManyRelatedField +from rest_framework.response import Response +from rest_framework.routers import DefaultRouter +from rest_framework.serializers import ModelSerializer +from rest_framework.serializers import SerializerMethodField + +from admin_rest.fields import CurrencySerialRestField +from admin_rest.fields import ForeignSerialField +from admin_rest.filters import GenericAllFieldsFilter +from admin_rest.utils import get_user_permissions +from core.currency import CurrencyModelField +from core.pairs import PairModelField, PairSerialRestField +from lib.fields import JSDatetimeField, RichTextField, RichTextSerialField, ImageSerialField, TextSerialField, \ + JsonSerialField, SVGAndImageField + +User = get_user_model() + + +class AlreadyRegistered(Exception): + pass + + +class NotRegistered(Exception): + pass + + +class ImproperlyConfigured(Exception): + pass + + +class AuthPermissionViewSetMixin: + NOT_FOUND_PERMISSION_DEFAULT = False + permission_map = dict() + + def get_permission_map(self): + permission_map = { + 'list': self._make_permission_key('view'), + 'retrieve': self._make_permission_key('view'), + 'create': self._make_permission_key('add'), + 'update': self._make_permission_key('change'), + 'partial_update': self._make_permission_key('change'), + 'destroy': self._make_permission_key('delete'), + } + permission_map.update(self.permission_map) + return permission_map + + @cached_property + def _options(self): + return self.queryset.model._meta + + def _make_permission_key(self, action): + code_name = get_permission_codename(action, self._options) + return "{0}.{1}".format(self._options.app_label, code_name) + + def has_perm_action(self, action, request, obj=None): + if not action: + return False + + if action == 'metadata': + return True + + perm_map = self.get_permission_map() + if hasattr(getattr(self, action), 'permissions'): + perm_map.update(**{action: getattr(self, action).permissions}) + + if action not in perm_map: + return self.NOT_FOUND_PERMISSION_DEFAULT + + perm_code = perm_map[action] + if callable(perm_code): + return perm_code(self, action, request, obj) + if isinstance(perm_code, bool): + return perm_code + + if perm_code in ['view', 'add', 'change', 'delete']: + perm_code = self._make_permission_key(perm_code) + + # checks list of permissions + if isinstance(perm_code, list) or isinstance(perm_code, tuple): + for code in perm_code: + if code in ['view', 'add', 'change', 'delete']: + code = self._make_permission_key(code) + has_perm = request.user.has_perm(code) + if has_perm: + return has_perm + return False + + return request.user.has_perm(perm_code) + + +class IsStaffAccess(BasePermission): + """ + Allows access only to authenticated Trainee users. + """ + + def has_permission(self, request, view): + return bool(request.user and request.user.is_authenticated and request.user.is_staff) + + def has_object_permission(self, request, view, obj): + """ + Return `True` if permission is granted, `False` otherwise. + """ + return self.has_permission(request, view) + + +class HasPermissionAccess(BasePermission): + """ + Allows access only to authenticated Trainee users. + """ + + def has_permission(self, request, view): + assert hasattr(view, 'get_permission_map'), """ + Must be inherit from RestFulModelAdmin to use this permission + """ + return view.has_perm_action(view.action, request) + + def has_object_permission(self, request, view, obj): + """ + Return `True` if permission is granted, `False` otherwise. + """ + return view.has_perm_action(view.action, request, obj) + + +class ModelDiffHelper(object): + def __init__(self, initial): + self.__initial = self._dict(initial) + self._new_object = None + + def set_changed_model(self, new_object): + data = self._dict(new_object) + if self._new_object is not None: + self.__initial = data + self._new_object = data + return self + + @property + def diff(self): + if not self._new_object: + return {} + d1 = self.__initial + d2 = self._new_object + diffs = [(k, (v, d2[k])) for k, v in d1.items() if v != d2[k]] + return dict(diffs) + + @property + def has_changed(self): + return bool(self.diff) + + @property + def changed_fields(self): + return list(self.diff.keys()) + + def get_field_diff(self, field_name): + """ + Returns a diff for field if it's changed and None otherwise. + """ + return self.diff.get(field_name, None) + + def _dict(self, model): + return model_to_dict(model, fields=[field.name for field in + model._meta.fields]) + + +class CustomMetadata(SimpleMetadata): + label_lookup = SimpleMetadata.label_lookup + + def __init__(self, *args, **kwargs): + super(CustomMetadata, self).__init__(*args, **kwargs) + #TODO make consts for out fields + self.label_lookup[JSDatetimeField] = 'datetime' + self.label_lookup[PrimaryKeyRelatedField] = 'foreign' + self.label_lookup[ManyRelatedField] = 'foreign' + self.label_lookup[SerializerMethodField] = 'string' + self.label_lookup[CurrencySerialRestField] = 'choice' + self.label_lookup[PairSerialRestField] = 'choice' + self.label_lookup[ImageField] = 'image-upload' + self.label_lookup[ImageSerialField] = 'image-upload' + self.label_lookup[RichTextSerialField] = 'rich-text' + self.label_lookup[TextSerialField] = 'text' + self.label_lookup[JsonSerialField] = 'json' + self.label_lookup[ForeignSerialField] = 'foreign' + + def determine_metadata(self, request, view): + metadata = OrderedDict() + # view name + metadata['name'] = view.get_view_name() + # view detail page fields + metadata['fields'] = [] + # view list page fields + metadata['list_fields'] = [] + # actions with queryset + metadata['actions'] = [] + # global actions. Queryset not used + metadata['global_actions'] = [] + # ref models in detail page of entry + metadata['inline_forms'] = [] + + metadata['filters'] = [] + + metadata['search_enabled'] = False + + if hasattr(view, 'get_single_serializer'): + serializer = view.get_single_serializer() + fields = self.get_serializer_info(serializer) + metadata['fields'] = fields + + if hasattr(view, 'get_serializer'): + serializer = view.get_serializer() + fields = self.get_serializer_info(serializer) + metadata['list_fields'] = fields + + if hasattr(view, 'actions'): + actions = self.get_actions(view, view.actions) + metadata['actions'] = actions + + if hasattr(view, 'global_actions'): + actions = self.get_actions(view, view.global_actions) + metadata['global_actions'] = actions + + if hasattr(view, 'inline_forms'): + inlines = self.get_inline_forms(view, view.inline_forms) + metadata['inline_forms'] = inlines + + if getattr(view, 'search_fields', None): + metadata['search_enabled'] = True + + if getattr(view, 'filterset_fields', None): + serializer = view.get_all_fields_serializer() + fields = self.get_serializer_info(serializer) + metadata['filters'] = {k: v for k, v in fields.items() if v['filterable']} + + + return metadata + + def get_inline_forms(self, view, inlines): + res = [] + for inline, filter_by in inlines: + if isinstance(filter_by, str): + filter_by = {filter_by: 'id'} + if isinstance(filter_by, list): + filter_by = {v: 'id' for v in filter_by} + res.append({ + 'resource': site.get_resource_name_by_view_class(inline), + 'filter_by': filter_by, + 'fields': inline.list_display + }) + return res + + def get_actions(self, view, view_actions): + actions = [] + for action_name in view_actions: + if not view.has_perm_action(action_name, view.request): + continue + + action_fn = getattr(view, action_name, None) + if action_fn: + info = { + 'url': view.reverse_action(action_name.replace('_', '-')), + 'name': getattr(action_fn, 'short_description', action_fn.__name__), + 'fields': [], + } + + if isinstance(view_actions, dict): + fields = [] + for field in view_actions[action_name]: + if isinstance(field, str): + field = { + 'name': field, + 'label': field.capitalize(), + } + + # default text type + if 'type' not in field: + field['type'] = 'text' + fields.append(field) + info['fields'] = fields + + actions.append(info) + + return actions + + def get_field_attributes(self, serializer_field, model_field): + attributes = OrderedDict() + + default = getattr(model_field, 'default', NOT_PROVIDED) + is_nullable = getattr(model_field, 'null', None) + is_read_only = getattr(serializer_field, 'read_only', None) + # attributes['required'] = (default is NOT_PROVIDED) and not is_nullable and is_read_only + + attributes['required'] = getattr(serializer_field, 'required', False) + attributes['nullable'] = getattr(model_field, 'null', False) + + attrs_dict = { + 'read_only': 'read_only', + 'label': 'label', + 'help_text': 'hint', + 'min_length': 'min_length', + 'max_length': 'max_length', + 'min_value': 'min', + 'max_value': 'max', + } + + for attr, front_attr in attrs_dict.items(): + value = getattr(serializer_field, attr, None) + if value is not None and value != '': + attributes[front_attr] = force_str(value, strings_only=True) + + if getattr(serializer_field, 'child', None): + attributes['child'] = self.get_field_info(serializer_field.child) + elif getattr(serializer_field, 'fields', None): + attributes['children'] = self.get_serializer_info(serializer_field) + + if not isinstance(serializer_field, (serializers.RelatedField, serializers.ManyRelatedField)): + if hasattr(serializer_field, 'choices'): + attributes['choices'] = [] + if is_nullable: + attributes['choices'].append({'value': None, 'text': ''}) + attributes['choices'].extend([ + { + 'value': choice_value, + 'text': force_str(choice_name, strings_only=True) + } + for choice_value, choice_name in serializer_field.choices.items() + ]) + + if self.label_lookup[serializer_field] == 'foreign': + if isinstance(serializer_field, ManyRelatedField): + model = serializer_field.child_relation.queryset.model + attributes['multiple'] = True + else: + model = model_field.related_model + attributes['reference'] = f'{site.get_resource_name(model)}' + return attributes + + def get_field_info(self, field): + """ + Given an instance of a serializer field, return a dictionary + of metadata about it. + """ + view = field.context['view'] + model_fields = {f.name: f for f in view._options.fields} + model_field = model_fields.get(field.field_name) + + sortable_fields = getattr(view, 'ordering_fields', '__all__') + filterable_fields = getattr(view, 'filterset_fields', None) + + field_info = OrderedDict() + field_info['type'] = self.label_lookup[field] + field_info['source'] = field.field_name + field_info['sortable'] = False + field_info['filterable'] = False + + field_info['default'] = self.get_field_default_value(model_field) + + if sortable_fields == '__all__' or (isinstance(sortable_fields, list) and field.field_name in sortable_fields): + field_info['sortable'] = True + + if (isinstance(filterable_fields, list) or isinstance(filterable_fields, tuple)) \ + and field.field_name in filterable_fields: + field_info['filterable'] = True + + field_info['attributes'] = self.get_field_attributes(field, model_field) + field_info['attributes']['searchable'] = field.field_name in (view.vue_resource_extras.get('searchable_fields') or []) + + return field_info + + def get_field_default_value(self, model_field): + default = getattr(model_field, 'default', None) + if default is NOT_PROVIDED: + default = None + elif callable(default): + default = default() + + if isinstance(model_field, JSONField) or isinstance(model_field, models.JSONField): + default = json.dumps(default, indent=2) + return default + + +class RestFulModelAdmin(AuthPermissionViewSetMixin, viewsets.ModelViewSet): + queryset = None + single_serializer_class = None + permission_classes = (IsStaffAccess, HasPermissionAccess) + list_display = '__all__' + fields = '__all__' + readonly_fields = [] + ordering_fields = '__all__' + filterset_fields = [] + search_fields = [] + metadata_class = CustomMetadata + filter_backends = (GenericAllFieldsFilter, filters.OrderingFilter, filters.SearchFilter) + vue_resource_extras: dict = {} + inline_forms: [] # detail page entities + + def __init__(self, *args, **kwargs): + super(RestFulModelAdmin, self).__init__(*args, **kwargs) + filterset_fields = set(self.filterset_fields) + filterset_fields.add('id') + + for field in self._options.fields: + if type(field) in [models.OneToOneField, models.ForeignKey, models.ManyToManyField]: + filterset_fields.add(field.name + '_id') + + self.filterset_fields = tuple(filterset_fields) + + def get_readonly_fields(self): + return self.readonly_fields + + @classmethod + def has_add_permission(cls): + return True + + @classmethod + def has_update_permission(cls): + return True + + @classmethod + def has_delete_permission(cls): + return True + + @classmethod + def get_view_permissions(cls): + res = ['show', 'list'] + if cls.has_add_permission(): + res.append('create') + if cls.has_update_permission(): + res.append('edit') + if cls.has_delete_permission(): + res.append('delete') + return res + + @staticmethod + def get_doc(): + return 'asd' + + def get_urls(self): + return [] + + def get_permission_map(self): + permission_map = { + 'list': self._make_permission_key('view'), + 'retrieve': self._make_permission_key('view'), + 'create': self._make_permission_key('add'), + 'update': self._make_permission_key('change'), + 'partial_update': self._make_permission_key('change'), + 'destroy': self._make_permission_key('delete'), + } + permission_map.update(self.permission_map) + return permission_map + + def log_addition(self, request, object, message): + """ + Log that an object has been successfully added. + + The default implementation creates an admin LogEntry object. + """ + from django.contrib.admin.models import LogEntry, ADDITION + return LogEntry.objects.log_action( + user_id=request.user.pk, + content_type_id=get_content_type_for_model(object).pk, + object_id=object.pk, + object_repr=str(object), + action_flag=ADDITION, + change_message=message, + ) + + def log_change(self, request, object, message): + """ + Log that an object has been successfully changed. + + The default implementation creates an admin LogEntry object. + """ + from django.contrib.admin.models import LogEntry, CHANGE + return LogEntry.objects.log_action( + user_id=request.user.pk, + content_type_id=get_content_type_for_model(object).pk, + object_id=object.pk, + object_repr=str(object), + action_flag=CHANGE, + change_message=message, + ) + + def log_deletion(self, request, object, object_repr): + """ + Log that an object will be deleted. Note that this method must be + called before the deletion. + + The default implementation creates an admin LogEntry object. + """ + from django.contrib.admin.models import LogEntry, DELETION + return LogEntry.objects.log_action( + user_id=request.user.pk, + content_type_id=get_content_type_for_model(object).pk, + object_id=object.pk, + object_repr=object_repr, + action_flag=DELETION, + ) + + def get_single_serializer_class(self): + return self.single_serializer_class if self.single_serializer_class else self.get_serializer_class(True) + + def get_all_fields_serializer(self, *args, **kwargs): + serializer_class = self.get_serializer_class(all_fields=True) + kwargs['context'] = self.get_serializer_context() + return serializer_class(*args, **kwargs) + + def get_single_serializer(self, *args, **kwargs): + """ + Return the serializer instance that should be used for validating and + deserializing input, and for serializing output. + """ + serializer_class = self.get_single_serializer_class() + kwargs['context'] = self.get_serializer_context() + return serializer_class(*args, **kwargs) + + def get_serializer_representation_fn(self, cls): + """Custom fields representation fn""" + def validate_fn(sf, instance): + """ + Object instance -> Dict of primitive datatypes. + """ + ret = OrderedDict() + fields = sf._readable_fields + + for field in fields: + try: + attribute = field.get_attribute(instance) + except SkipField: + continue + + # We skip `to_representation` for `None` values so that fields do + # not have to explicitly deal with that case. + # + # For related fields with `use_pk_only_optimization` we need to + # resolve the pk value. + check_for_none = attribute.pk if isinstance(attribute, PKOnlyObject) else attribute + if check_for_none is None: + ret[field.field_name] = None + else: + repr = field.to_representation(attribute) + if isinstance(field, JsonSerialField) or isinstance(field, RichTextSerialField): + repr = mark_safe(repr) + if isinstance(repr, str) and not isinstance(repr, SafeString): + repr = escape(repr) + ret[field.field_name] = repr + return ret + + return validate_fn + + def get_serializer_class(self, single=False, all_fields=False): + serializer_class = super().get_serializer_class() + + view_fields = self.fields if single else self.list_display + if isinstance(view_fields, tuple): + view_fields = list(view_fields) + + if not view_fields or view_fields == '__all__' or all_fields: + serializer_class_fields = list([f.name for f in serializer_class.Meta.model._meta.fields]) + else: + if 'id' not in view_fields: + view_fields = ['id'] + view_fields + serializer_class_fields = view_fields + + serializer_class_fields += ['_label'] # default object representation + + # redefine serializer fields + serializer_class._declared_fields = {} + for field_name in serializer_class_fields: + if callable(getattr(self, field_name, None)): + # add SerializerMethodField and its method to serializer + field_method = getattr(self, field_name) + # if custom field type defined + if hasattr(field_method, 'serial_class'): + serializer_class._declared_fields[field_name] = field_method.serial_class() + else: + serializer_class._declared_fields[field_name] = SerializerMethodField() + setattr(serializer_class, f'get_{field_name}', field_method) + if field_name == '_label': + serializer_class._declared_fields[field_name] = SerializerMethodField() + setattr(serializer_class, f'get_{field_name}', lambda self_cls, obj: str(obj)) + + # setup readonly fields + readonly_fields = self.get_readonly_fields() + if isinstance(readonly_fields, list) or isinstance(readonly_fields, tuple): + serializer_class.Meta.read_only_fields = readonly_fields + + # search original classes for translated fields and delete original field + translated_fields_classes = {} + to_remove_original_translated_fields = set() + serializer_fields_classes = {f.name: f.__class__ for f in serializer_class.Meta.model._meta.fields} + for model_field in serializer_class.Meta.model._meta.fields: + if model_field.__class__.__name__.startswith('Translation'): + original_field_name = model_field.name.rsplit('_', 1)[0] + to_remove_original_translated_fields.add(original_field_name) + if original_field_name in serializer_fields_classes: + translated_fields_classes[model_field.name] = serializer_fields_classes[original_field_name] + else: + to_remove_original_translated_fields.add(model_field.name) + + serializer_class_fields = [f for f in serializer_class_fields + if f not in to_remove_original_translated_fields] + + # custom fields + for model_field in serializer_class.Meta.model._meta.fields: + if model_field.name not in serializer_class_fields: + # skip missing fields + continue + + CustomField = None + field_args = {} + + field_type = type(model_field) + + # handle translated fields + if model_field.name in translated_fields_classes: + field_type = translated_fields_classes[model_field.name] + + if isinstance(model_field, models.DateTimeField): + CustomField = JSDatetimeField + elif isinstance(model_field, models.ForeignKey) and not single: + CustomField = ForeignSerialField + elif field_type == CurrencyModelField: + CustomField = CurrencySerialRestField + elif field_type == PairModelField: + CustomField = PairSerialRestField + elif field_type == RichTextField: + CustomField = RichTextSerialField + elif field_type == models.TextField: + CustomField = TextSerialField + elif field_type in [models.ImageField, SVGAndImageField, ImageSerialField]: + CustomField = ImageSerialField + elif field_type in [JSONField, models.JSONField]: + CustomField = JsonSerialField + + if CustomField: + is_read_only = model_field.name in readonly_fields \ + or getattr(model_field, 'auto_now', False) \ + or getattr(model_field, 'auto_now_add', False) + default = getattr(model_field, 'default', NOT_PROVIDED) + is_nullable = getattr(model_field, 'null', None) + if CustomField != ForeignSerialField: + field_args['required'] = (default is NOT_PROVIDED) and not is_nullable and not is_read_only + + field_args['read_only'] = is_read_only + field_args['allow_null'] = model_field.null + if CustomField in [TextSerialField, RichTextSerialField]: + field_args['allow_blank'] = model_field.blank + serializer_class._declared_fields[model_field.name] = CustomField(**field_args) + + serializer_class.Meta.fields = serializer_class_fields + setattr(serializer_class, 'to_representation', self.get_serializer_representation_fn(serializer_class)) + return serializer_class + + def list(self, request, *args, **kwargs): + """list all of objects""" + queryset = self.filter_queryset(self.get_queryset()) + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) + + def create(self, request, **kwargs): + """Create new object""" + serializer = self.get_single_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + self.log_addition(request, serializer.instance, [{'added': { + 'name': str(serializer.instance._meta.verbose_name), + 'object': str(serializer.instance), + }}]) + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + + def retrieve(self, request, pk=None, **kwargs): + """Get object Details""" + instance = self.get_object() + serializer = self.get_single_serializer(instance) + return Response(serializer.data) + + def update(self, request, pk=None, **kwargs): + """Update object""" + partial = kwargs.pop('partial', False) + instance = self.get_object() + serializer = self.get_single_serializer(instance, data=request.data, partial=partial) + serializer.is_valid(raise_exception=True) + helper = ModelDiffHelper(instance) + self.perform_update(serializer) + + self.log_change( + request, + serializer.instance, + [{'changed': { + 'name': str(serializer.instance._meta.verbose_name), + 'object': str(serializer.instance), + 'fields': helper.set_changed_model(serializer.instance).changed_fields + }}] + ) + + if getattr(instance, '_prefetched_objects_cache', None): + # If 'prefetch_related' has been applied to a queryset, we need to + # forcibly invalidate the prefetch cache on the instance. + instance._prefetched_objects_cache = {} + + return Response(serializer.data) + + def partial_update(self, request, pk=None, **kwargs): + """Partial Update""" + return super().partial_update(request, pk=pk, **kwargs) + + def destroy(self, request, pk=None, **kwargs): + """Delete object""" + instance = self.get_object() + self.log_deletion(request, instance, [{ + 'deleted': { + 'name': str(instance._meta.verbose_name), + 'object': str(instance), + } + }]) + self.perform_destroy(instance) + return Response(status=status.HTTP_204_NO_CONTENT) + + +class RestFulAdminSite: + def __init__(self, view_class=RestFulModelAdmin): + self._registry = {} + self._model_by_view_registry = {} + self._url_patterns = [] + self.default_view_class = view_class + + def get_registered_models(self): + res = [] + for model in self._registry: + res.append(self.get_model_url(model)) + return sorted(res) + + def get_resource_name_by_view_class(self, view_class): + inv_map = {v: k for k, v in self._registry.items()} + model = inv_map.get(view_class) + if model: + return self.get_resource_name(model) + model = self._model_by_view_registry.get(view_class) + if model: + return self.get_resource_name(model) + + def get_resources(self): + res = [] + for model, view in self._registry.items(): + # https://www.okami101.io/vuetify-admin/guide/resources.html#resource-object-structure + data = { + 'name': self.get_resource_name(model), + 'actions': view.get_view_permissions(), + 'api': f'/{self.get_model_url(model)}/', + 'aside': False, + } + if view.vue_resource_extras: + data.update(view.vue_resource_extras) + + res.append(data) + return res + + def make_navigation(self, user): + all_permissions = get_user_permissions(user) + is_admin = 'admin' in all_permissions + menu = settings.VUE_ADMIN_SIDE_MENU + + new_menu = [] + + for entry in menu: + model = entry.get('model') + if not model: + new_menu.append(entry) + else: + app_label, model_name = model.split('.') + # todo check model existence + view_perm_name = f'{app_label}_{model_name}_view' + if is_admin or view_perm_name in all_permissions: + new_menu.append({ + 'icon': entry.get('icon'), + 'link': {'name': f'{app_label}_{model_name}_list'}, + 'text': entry.get('text') or (f'{app_label.capitalize()} {model_name.capitalize()}') + }) + return new_menu + + def register_decorator(self, *model_or_iterable, **options): + def wrapper(view_class): + self.register(model_or_iterable, view_class, **options) + return view_class + + return wrapper + + def register(self, model_or_iterable, view_class=None, **options): + if not view_class: + view_class = self.default_view_class + + if isinstance(model_or_iterable, ModelBase): + model_or_iterable = [model_or_iterable] + for model in model_or_iterable: + if model._meta.abstract: + raise ImproperlyConfigured( + 'The model %s is abstract, so it cannot be registered with admin.' % model.__name__ + ) + + if model in self._registry: + raise AlreadyRegistered('The model %s is already registered' % model.__name__) + options.update({ + "__doc__": self.generate_docs(model) + }) + self._model_by_view_registry[view_class] = model + view_class = type("%sAdmin" % model.__name__, (view_class,), options) + # self.set_docs(view_class, model) + # Instantiate the admin class to save in the registry + self._registry[model] = view_class + + def register_url_pattern(self, url_pattern): + self._url_patterns.append(url_pattern) + + @classmethod + def generate_docs(cls, model): + return """ + ### The APIs include: + + + > `GET` {app}/{model} ===> list all `{verbose_name_plural}` page by page; + + > `POST` {app}/{model} ===> create a new `{verbose_name}` + + > `GET` {app}/{model}/123 ===> return the details of the `{verbose_name}` 123 + + > `PATCH` {app}/{model}/123 and `PUT` {app}/{model}/123 ==> update the `{verbose_name}` 123 + + > `DELETE` {app}/{model}/123 ===> delete the `{verbose_name}` 123 + + > `OPTIONS` {app}/{model} ===> show the supported verbs regarding endpoint `{app}/{model}` + + > `OPTIONS` {app}/{model}/123 ===> show the supported verbs regarding endpoint `{app}/{model}/123` + + """.format( + app=model._meta.app_label, + model=model._meta.model_name, + verbose_name=model._meta.verbose_name, + verbose_name_plural=model._meta.verbose_name_plural + ) + + def unregister(self, model_or_iterable): + """ + Unregister the given model(s). + + If a model isn't already registered, raise NotRegistered. + """ + if isinstance(model_or_iterable, ModelBase): + model_or_iterable = [model_or_iterable] + for model in model_or_iterable: + if model not in self._registry: + raise NotRegistered('The model %s is not registered' % model.__name__) + del self._registry[model] + + def is_registered(self, model): + """ + Check if a model class is registered with this `AdminSite`. + """ + return model in self._registry + + def get_model_basename(self, model): + return None + + def get_model_url(self, model): + return '%s/%s' % (model._meta.app_label, model._meta.model_name) + + def get_resource_name(self, model): + return f'{model._meta.app_label}_{model._meta.model_name}' + + def get_urls(self): + router = DefaultRouter() + view_sets = [] + for model, view_set in self._registry.items(): + if view_set.queryset is None: + view_set.queryset = model.objects.all() + # Creates default serializer + if view_set.serializer_class is None: + serializer_class = type("%sModelSerializer" % model.__name__, (ModelSerializer,), { + "Meta": type("Meta", (object,), { + "model": model, + "fields": "__all__" + }), + }) + view_set.serializer_class = serializer_class + + view_sets.append(view_set) + router.register(self.get_model_url(model), view_set, self.get_model_basename(model)) + + return router.urls + self._url_patterns + + @property + def urls(self): + return self.get_urls() + + +site = RestFulAdminSite() + + +def register(*model_or_iterable, **options): + return site.register_decorator(*model_or_iterable, **options) + + +def action(permissions=None, methods=['POST'], detail=False, url_path=None, url_name=None, custom_response=False, + **kwargs): + def decorator(func): + base_func = base_action(methods, detail, url_path, url_name, **kwargs)(func) + base_func.permissions = permissions + + @wraps(base_func) + def wrapper(base_admin_class, request, *args, **kwargs): + ids = request.data.get('ids') + + Model = base_admin_class.serializer_class.Meta.model + queryset = Model.objects.filter(id__in=ids) if ids else Model.objects.none() + + res = base_func(base_admin_class, request, queryset, *args, **kwargs) + if queryset: + for entry in queryset: + LogEntry.objects.log_action( + user_id=request.user.pk, + content_type_id=get_content_type_for_model(entry).pk, + object_id=entry.pk, + object_repr=str(entry), + action_flag=CHANGE, + change_message=[{'action': { + 'name': f'{base_admin_class.__class__.__name__} {base_func.__name__}', + 'object': str(entry), + }}], + ) + else: + LogEntry.objects.log_action( + user_id=request.user.pk, + content_type_id=None, + object_id=None, + object_repr='', + action_flag=CHANGE, + change_message=[{'action': { + 'name': f'{base_admin_class.__class__.__name__} {base_func.__name__}', + }}], + ) + + if custom_response: + return res + + if res is None: + return Response(status=status.HTTP_200_OK) + return Response(res, status=status.HTTP_200_OK) + + return wrapper + + return decorator + + +class DefaultApiAdmin(RestFulModelAdmin): + ordering = ('-id',) + # permission_classes = (AllowAny, ) + # filter_backends = [GenericAllFieldsFilter, filters.OrderingFilter, filters.SearchFilter] diff --git a/admin_rest/serializers.py b/admin_rest/serializers.py new file mode 100644 index 0000000..257bfb9 --- /dev/null +++ b/admin_rest/serializers.py @@ -0,0 +1,44 @@ +from django.contrib.auth.models import User, Permission, Group +from rest_framework import serializers + +from admin_rest.utils import get_user_permissions + + +class UserDetailsSerializer(serializers.ModelSerializer): + permissions = serializers.SerializerMethodField() + + def get_permissions(self, obj): + return get_user_permissions(obj) + + class Meta: + model = User + fields = ('id', 'username', 'email', 'first_name', 'last_name', + 'is_staff', 'is_superuser', 'permissions') + + +class PermissionSerializer(serializers.ModelSerializer): + action = serializers.SerializerMethodField() + name = serializers.SerializerMethodField() + + def get_name(self, obj): + return f'{obj.content_type.app_label}/{obj.content_type.model}' + + def get_action(self, obj): + return obj.codename.split('_')[0] + + + class Meta: + model = Permission + fields = ('id', 'model_app', 'model_name', 'action') + + +class GroupSerializer(serializers.ModelSerializer): + class Meta: + model = Group + fields = ('id', 'name',) + + +class SimpleUserSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ('id', 'email',) diff --git a/admin_rest/tests.py b/admin_rest/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/admin_rest/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/admin_rest/urls.py b/admin_rest/urls.py new file mode 100644 index 0000000..1933caf --- /dev/null +++ b/admin_rest/urls.py @@ -0,0 +1,25 @@ +from django.urls import path +from rest_framework_simplejwt.views import ( + TokenObtainPairView, + TokenRefreshView, + TokenVerifyView, +) + +from admin_rest import restful_admin +from admin_rest import views + +urlpatterns = [ + path('login/', views.login), + path('user/', views.me), + path('permissions/', views.permissions), + path('token/', TokenObtainPairView.as_view(), name='token_obtain_pair'), + path('refresh/', TokenRefreshView.as_view(), name='token_refresh'), + path('verify/', TokenVerifyView.as_view(), name='token_verify'), + path('models/', views.models), + path('resources/', views.resources), + path('navigation/', views.navigation), + path('upload/image/', views.upload_image), + path('browser/image/', views.image_browser), +] + +urlpatterns += restful_admin.site.urls \ No newline at end of file diff --git a/admin_rest/utils.py b/admin_rest/utils.py new file mode 100644 index 0000000..7477f2b --- /dev/null +++ b/admin_rest/utils.py @@ -0,0 +1,101 @@ +import logging +import os +import random +import string +from datetime import datetime + +from django.conf import settings +from django.contrib.auth.models import User, Permission +from django.core.files.storage import DefaultStorage +from django.db.models import Q +from django.template.defaultfilters import slugify + +from lib.helpers import BOT_RE + +log = logging.getLogger(__name__) + + +VALID_IMAGE_EXTENSION = settings.VALID_IMAGE_EXTENSION or ['jpg', 'jpeg', 'png', 'gif'] + + +def get_bots_ids(): + return list(User.objects.filter(username__iregex=BOT_RE).values_list('id', flat=True)) + + +def get_user_permissions(user): + user_permissions = [] + if user.is_superuser: + user_permissions = ['admin'] + elif user.is_staff: + user_permissions_qs = Permission.objects.filter( + Q(group__in=user.groups.all()) | Q(user=user) + ).distinct() + for p in user_permissions_qs: + action, model_name = p.codename.split('_', 1) + user_permissions.append(f'{p.content_type.app_label}_{model_name}_{action}') + return user_permissions + + +def is_valid_image_extension(file_path): + extension = os.path.splitext(file_path.lower())[1] + return extension in VALID_IMAGE_EXTENSION + + +def is_valid_image(file): + if not is_valid_image_extension(file.name): + return False + + from PIL import Image + + is_valid = False + + try: + image = Image.open(file) + image.verify() + is_valid = True + except Exception as e: + log.exception(e) + finally: + file.seek(0) + + return is_valid + + +def get_media_url(path): + """ + Determine system file's media URL. + """ + return DefaultStorage().url(path) + + +def slugify_filename(filename): + """ Slugify filename """ + name, ext = os.path.splitext(filename) + slugified = get_slugified_name(name) + return slugified + ext + + +def get_slugified_name(name): + """ Slugify name """ + slugified = slugify(name) + return slugified or get_random_string() + + +def get_random_string(): + return ''.join(random.sample(string.ascii_lowercase * 6, 6)) + + +def get_upload_filename(upload_name): + """Generates unique filename""" + date_path = datetime.now().strftime("%Y/%m/%d") + upload_path = os.path.join(settings.VUE_UPLOAD_PATH, date_path) + upload_name = slugify_filename(upload_name) + return DefaultStorage().get_available_name(os.path.join(upload_path, upload_name)) + + +def get_image_files(path=''): + images = [] + browse_path = os.path.join(settings.VUE_UPLOAD_PATH, path) + for root, dirs, files in os.walk(browse_path): + images.extend([os.path.join(f_name) for f_name in files if is_valid_image_extension(f_name)]) + return images diff --git a/admin_rest/views.py b/admin_rest/views.py new file mode 100644 index 0000000..6e6fca4 --- /dev/null +++ b/admin_rest/views.py @@ -0,0 +1,186 @@ +import logging + +from django.conf import settings +from django.contrib.admin.views.decorators import staff_member_required +from django.contrib.auth import authenticate +from django.contrib.auth.models import Permission +from django.contrib.auth.models import User, Group +from django.core.files.storage import DefaultStorage +from django.http import JsonResponse +from django_otp import match_token +from rest_framework import status +from rest_framework.decorators import api_view, permission_classes +from rest_framework.permissions import AllowAny + +from admin_rest import restful_admin as api_admin +from admin_rest.serializers import UserDetailsSerializer +from admin_rest.utils import is_valid_image, get_media_url, get_upload_filename, get_image_files + +log = logging.getLogger(__name__) + +from rest_framework_simplejwt.tokens import RefreshToken + + +DEVICE_ID_SESSION_KEY = 'otp_device_id' + + +@api_view(['POST']) +@permission_classes((AllowAny, )) +def login(request): + """Authenticate user. + Returns access token. + """ + error = {'status': False, 'error': 'Incorrect username or password'} + error_response = JsonResponse(error, safe=False, status=status.HTTP_400_BAD_REQUEST) + + username = request.data.get('username') + password = request.data.get('password') + if not username or not password: + return error_response + # Identity + user = User.objects.filter(username=username, is_active=True).first() + if not user: + return error_response + # Auth + user = authenticate(username=user.username, password=password) + if not user: + return error_response + + if not user.is_active: + return error_response + + if not user.is_superuser and not user.is_staff: + return error_response + + # check 2FA + if settings.ENABLE_OTP_ADMIN: + otp_token = request.data.get('otp_token') + if otp_token and otp_token.isdigit(): + otp_token = int(otp_token) + device = match_token(user, otp_token) + if not device: + error['error'] = 'Incorrect 2FA token' + return JsonResponse(error, safe=False, status=status.HTTP_400_BAD_REQUEST) + + request.session[DEVICE_ID_SESSION_KEY] = device.persistent_id + request.user.otp_device = device + + refresh = RefreshToken.for_user(user) + token = str(refresh.access_token) + + response = JsonResponse({'status': True, 'access_token': token}, safe=False, status=status.HTTP_200_OK) + response.set_cookie(settings.JWT_AUTH_COOKIE, token, settings.JWT_EXPIRATION_DELTA.total_seconds(), httponly=True) + + return response + + +@api_view(['POST', 'PUT']) +def permissions(request): + """Add/change permissions of selected group""" + user = request.user + if not (user.is_staff and user.is_superuser and user.is_active): + return JsonResponse({'status': False}, safe=False, status=status.HTTP_403_FORBIDDEN) + + # serializer = GroupSerializer(request.data) + # serializer.is_valid(True) + + group_id = request.data.get('id') + group_name = request.data.get('name') + group_permissions = request.data.get('permissions') + users = request.data.get('users') + + if request.method == 'PUT': + if not group_id or not group_permissions: + return JsonResponse({'status': False}, safe=False, status=status.HTTP_400_BAD_REQUEST) + + group = Group.objects.get(id=group_id) + if not group: + return JsonResponse({'status': False, 'error': 'Group not found'}, + safe=False, status=status.HTTP_400_BAD_REQUEST) + group.name = group_name + group.save() + + elif request.method == 'POST': + group = Group.objects.create(name=group_name) + + selected_permissions_ids = [] + all_permissions_dict = {f'{p.content_type.app_label}/{p.content_type.model}/{p.codename.split("_")[0]}': p.id + for p in Permission.objects.all()} + + for group_perm in group_permissions: + for act, act_perm in group_perm['permissions'].items(): + if act_perm: + name = group_perm['modelName'] + '/' + act + if name in all_permissions_dict: + selected_permissions_ids.append(all_permissions_dict[name]) + + group.permissions.set(selected_permissions_ids) + group.user_set.set(users) + response = JsonResponse({'status': True}, safe=False, status=status.HTTP_200_OK) + return response + + +@api_view(['GET']) +def me(request): + """Current user info""" + user = request.user + user_data = UserDetailsSerializer(user).data + response = JsonResponse({'status': True, 'user': user_data}, safe=False, status=status.HTTP_200_OK) + return response + + +@api_view(['GET']) +def models(request): + """List of all registered models names in form: ['/', ...]""" + registered_models = api_admin.site.get_registered_models() + return JsonResponse(registered_models, safe=False, status=status.HTTP_200_OK) + + +@api_view(['GET']) +@permission_classes((AllowAny, )) +def resources(request): + """List of all registered admin resources""" + resourses = api_admin.site.get_resources() + return JsonResponse(resourses, safe=False, status=status.HTTP_200_OK) + + +@api_view(['GET']) +def navigation(request): + """Vue admin side navigation""" + navigation = api_admin.site.make_navigation(request.user) + return JsonResponse(navigation, safe=False, status=status.HTTP_200_OK) + + +@api_view(['POST']) +@staff_member_required +def upload_image(request): + """ + Uploads a file and send back its URL to VueAdmin. + """ + uploaded_file = request.FILES['file'] + + # checks image extension and validate image via PIL + if not is_valid_image(uploaded_file): + return JsonResponse({'status': False, 'error': 'Image is not valid'}, safe=False, status=status.HTTP_400_BAD_REQUEST) + + filepath = get_upload_filename(uploaded_file.name) + saved_path = DefaultStorage().save(filepath, uploaded_file) + + url = get_media_url(saved_path) + + return JsonResponse({'location': url}) + + +@api_view(['POST']) +@staff_member_required +def image_browser(request): + """ + Return all uploaded images + """ + + uploaded_images = get_image_files() + + # get urls + images = [get_media_url(i) for i in uploaded_images] + + return JsonResponse({'images': images}) diff --git a/bots/admin_rest.py b/bots/admin_rest.py new file mode 100644 index 0000000..4e07fce --- /dev/null +++ b/bots/admin_rest.py @@ -0,0 +1,50 @@ +import logging + +from django.conf import settings +from django.contrib.auth import get_user_model + +from admin_rest import restful_admin as api_admin +from admin_rest.restful_admin import DefaultApiAdmin +from bots.models import BotConfig +from lib.cipher import AESCoderDecoder + +log = logging.getLogger(__name__) + +User = get_user_model() + + +@api_admin.register(BotConfig) +class BotConfigApiAdmin(DefaultApiAdmin): + fields = ('name', 'user', 'pair', 'strategy', 'match_user_orders', 'instant_match', 'ohlc_period', 'enabled', + 'loop_period', 'loop_period_random', 'min_period', 'max_period', + 'ext_price_delta','symbol_precision', 'quote_precision', + 'min_order_quantity', 'max_order_quantity', + 'use_custom_price', 'custom_price', + 'low_orders_match', 'low_orders_max_match_size', 'low_orders_spread_size', + 'low_orders_min_order_size', 'low_orders_match_greater_order', + 'binance_apikey', 'binance_secret', + 'low_spread_alert',) + list_display = ('name', 'bot_info') + readonly_fields = ['binance_apikey', 'binance_secret'] + + def bot_info(self, obj): + return f'{obj.user.email}: {obj.pair.code} {obj.min_period}-{obj.max_period}s; Enabled: {obj.enabled}' + + def binance_apikey(self, obj): + try: + if obj.binance_api_key: + binance_api_key = AESCoderDecoder( + settings.CRYPTO_KEY).decrypt( + obj.binance_api_key) + return binance_api_key[:5] + '...' + binance_api_key[-5:] + except Exception as e: + log.exception(e) + + def binance_secret(self, obj): + try: + if obj.binance_secret_key: + binance_secret_key = AESCoderDecoder( + settings.CRYPTO_KEY).decrypt(obj.binance_secret_key) + return binance_secret_key[:5] + '...' + binance_secret_key[-5:] + except Exception as e: + log.exception(e) diff --git a/core/admin_rest.py b/core/admin_rest.py new file mode 100644 index 0000000..9080e41 --- /dev/null +++ b/core/admin_rest.py @@ -0,0 +1,100 @@ +import logging + +from admin_rest import restful_admin as api_admin +from django.db.models import Q, Sum, F +from rangefilter.filters import DateRangeFilter + +from admin_rest.restful_admin import DefaultApiAdmin +from core.enums.profile import UserTypeEnum +from core.models import Balance, WithdrawalRequest, Settings +from core.models.inouts.withdrawal import WithdrawalLimitLevel, WithdrawalUserLimit +from core.models.stats import InoutsStats +from core.models.facade import SmsConfirmationHistory +from admin_rest.mixins import ReadOnlyMixin + +log = logging.getLogger(__name__) + + +@api_admin.register(InoutsStats) +class InoutsStatsAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = ['currency', 'deposits', 'withdrawals', 'total', 'in_orders', 'free', 'withdrawals_pending'] + global_actions = ['refresh'] + ordering = ('currency',) + + def get_queryset(self): + exclude_qs = ( + Q(user__profile__user_type=UserTypeEnum.staff.value) + | Q(user__profile__user_type=UserTypeEnum.bot.value) + | Q(user__email__endswith='@bot.com') + | Q(user__is_staff=True) + ) + + balance_summary = Balance.objects.exclude(exclude_qs).values( + 'currency' + ).annotate( + free=Sum('amount'), + in_orders=Sum('amount_in_orders'), + total=Sum(F('amount') + F('amount_in_orders')), + ) + withdrawals_pending = WithdrawalRequest.objects.filter( + state=WithdrawalRequest.STATE_PENDING, + ).exclude(exclude_qs).values('currency').annotate( + withdrawals_pending=Sum('amount'), + ) + + self._balance_summary_dict = {b['currency']: b for b in balance_summary} + self._withdrawals_pending_dict = {wp['currency']: wp for wp in withdrawals_pending} + return super(InoutsStatsAdmin, self).get_queryset() + + def total(self, obj): + return self._balance_summary_dict.get(obj.currency, {}).get('total') or 0 + + def free(self, obj): + return self._balance_summary_dict.get(obj.currency, {}).get('free') or 0 + + def in_orders(self, obj): + return self._balance_summary_dict.get(obj.currency, {}).get('in_orders') or 0 + + def withdrawals_pending(self, obj): + return self._withdrawals_pending_dict.get(obj.currency, {}).get('withdrawals_pending') or 0 + + # custom actions + @api_admin.action(permissions=True) + def refresh(self, request, queryset): + InoutsStats.refresh() + + refresh.short_description = 'Update stats' + + +@api_admin.register(Settings) +class SettingsAdmin(DefaultApiAdmin): + fields = ('value',) + filterset_fields = ['name', 'value', 'updated'] + + +@api_admin.register(SmsConfirmationHistory) +class SmsConfirmationHistoryAdmin(ReadOnlyMixin, DefaultApiAdmin): + search_fields = ['user__email', 'phone'] + filterset_fields = ['action_type', 'verification_type', 'is_success', 'created'] + + +@api_admin.register(WithdrawalLimitLevel) +class WithdrawalLimitLevelAdmin(DefaultApiAdmin): + list_display = [ + 'id', + 'level', + 'amount', + ] + + +@api_admin.register(WithdrawalUserLimit) +class WithdrawalUserLimitAdmin(DefaultApiAdmin): + search_fields = ['user__email'] + list_display = [ + 'id', + 'user', + 'limit_amount', + ] + + def limit_amount(self, obj: WithdrawalUserLimit): + return f'{obj.limit_id} | {obj.limit.amount}' diff --git a/core/currency.py b/core/currency.py index 192f0ad..a6cb6b1 100644 --- a/core/currency.py +++ b/core/currency.py @@ -21,7 +21,6 @@ class TokenParams: class CoinParams: latest_block_fn: Optional[Callable] = None blocks_monitoring_diff: Optional[int] = None - encrypted_cold_wallet: Optional[bytes] = None class CurrencyNotFound(APIException): diff --git a/core/migrations/0011_auto_20230413_0836.py b/core/migrations/0011_auto_20230413_0836.py index 543d1a7..4675b7c 100644 --- a/core/migrations/0011_auto_20230413_0836.py +++ b/core/migrations/0011_auto_20230413_0836.py @@ -9,8 +9,8 @@ def transfer_precisions(apps, schema_editor): precisions_map = { 'BTC-USDT': ['100', '10', '1', '0.1', '0.01'], 'ETH-USDT': ['100', '10', '1', '0.1', '0.01'], - 'TRX-USDT': ['0.01', '0.001', '0.0001', '0.00001'], 'BNB-USDT': ['100', '10', '1', '0.1', '0.01'], + 'TRX-USDT': ['0.01', '0.001', '0.0001', '0.00001', '0.000001'], } for ps in PairSettings.objects.all(): if ps.pair.code in precisions_map: @@ -18,6 +18,10 @@ def transfer_precisions(apps, schema_editor): ps.save() +def reverse(a, s): + return + + class Migration(migrations.Migration): dependencies = [ @@ -35,5 +39,5 @@ class Migration(migrations.Migration): name='precisions', field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=16), default=list, size=None), ), - migrations.RunPython(transfer_precisions), + migrations.RunPython(transfer_precisions, reverse), ] diff --git a/core/pairs.py b/core/pairs.py index f4d36ff..c541d35 100644 --- a/core/pairs.py +++ b/core/pairs.py @@ -86,7 +86,10 @@ def to_internal_value(self, value): class PairSerialRestField(PairSerialField): - choices = dict([(p[0], p[1]) for p in PAIRS_LIST]) # for OPTIONS action - def to_representation(self, obj): return obj.id + + @property + def choices(self): + """for OPTIONS action""" + return dict(PAIRS_LIST) diff --git a/cryptocoins/admin_rest.py b/cryptocoins/admin_rest.py new file mode 100644 index 0000000..500715d --- /dev/null +++ b/cryptocoins/admin_rest.py @@ -0,0 +1,126 @@ +from admin_rest import restful_admin as api_admin +from admin_rest.mixins import ReadOnlyMixin +from admin_rest.restful_admin import DefaultApiAdmin +from core.consts.currencies import BEP20_CURRENCIES +from core.consts.currencies import ERC20_CURRENCIES +from core.consts.currencies import TRC20_CURRENCIES +from core.models import UserWallet +from core.utils.withdrawal import get_withdrawal_requests_to_process +from cryptocoins.coins.bnb import BNB_CURRENCY +from cryptocoins.coins.btc.service import BTCCoinService +from cryptocoins.coins.eth import ETH_CURRENCY +from cryptocoins.coins.trx import TRX_CURRENCY +from cryptocoins.models import ScoringSettings +from cryptocoins.models import TransactionInputScore +from cryptocoins.models.proxy import BNBWithdrawalApprove +from cryptocoins.models.proxy import BTCWithdrawalApprove +from cryptocoins.models.proxy import ETHWithdrawalApprove +from cryptocoins.models.proxy import TRXWithdrawalApprove +from cryptocoins.serializers import BNBKeySerializer +from cryptocoins.serializers import BTCKeySerializer +from cryptocoins.serializers import ETHKeySerializer +from cryptocoins.serializers import TRXKeySerializer +from cryptocoins.tasks.evm import process_payouts_task + + + +class BaseWithdrawalApprove(ReadOnlyMixin, DefaultApiAdmin): + list_display = ['user', 'confirmed', 'currency', 'state', 'details', 'amount'] + search_fields = ['user__email', 'data__destination'] + filterset_fields = ['currency'] + global_actions = { + 'process': [{ + 'label': 'Password', + 'name': 'key' + }] + } + + def details(self, obj): + return obj.data.get('destination') + + +@api_admin.register(BTCWithdrawalApprove) +class BTCWithdrawalApproveApiAdmin(BaseWithdrawalApprove): + + def get_queryset(self): + service = BTCCoinService() + return service.get_withdrawal_requests() + + @api_admin.action(permissions=True) + def process(self, request, queryset): + service = BTCCoinService() + # form = MySerializer(request) + serializer = BTCKeySerializer(data=request.data) + + if serializer.is_valid(raise_exception=True): + private_key = request.data.get('key') + service.process_withdrawals(private_key=private_key) + process.short_description = 'Process withdrawals' + + +@api_admin.register(ETHWithdrawalApprove) +class ETHWithdrawalApproveApiAdmin(BaseWithdrawalApprove): + + def get_queryset(self): + return get_withdrawal_requests_to_process([ETH_CURRENCY, *ERC20_CURRENCIES], blockchain_currency='ETH') + + @api_admin.action(permissions=True) + def process(self, request, queryset): + serializer = ETHKeySerializer(data=request.data) + if serializer.is_valid(raise_exception=True): + password = request.data.get('key') + process_payouts_task.apply_async(['ETH', password,], queue='eth_payouts') + process.short_description = 'Process withdrawals' + + +@api_admin.register(TRXWithdrawalApprove) +class TRXWithdrawalApproveApiAdmin(BaseWithdrawalApprove): + def get_queryset(self): + return get_withdrawal_requests_to_process([TRX_CURRENCY, *TRC20_CURRENCIES], blockchain_currency='TRX') + + @api_admin.action(permissions=True) + def process(self, request, queryset): + serializer = TRXKeySerializer(data=request.data) + if serializer.is_valid(raise_exception=True): + password = request.data.get('key') + process_payouts_task.apply_async(['TRX', password, ], queue='trx_payouts') + + process.short_description = 'Process withdrawals' + + +@api_admin.register(BNBWithdrawalApprove) +class BNBWithdrawalApproveApiAdmin(BaseWithdrawalApprove): + def get_queryset(self): + return get_withdrawal_requests_to_process([BNB_CURRENCY, *BEP20_CURRENCIES], blockchain_currency='BNB') + + @api_admin.action(permissions=True) + def process(self, request, queryset): + serializer = BNBKeySerializer(data=request.data) + if serializer.is_valid(raise_exception=True): + password = request.data.get('key') + process_payouts_task.apply_async(['BNB', password,], queue='bnb_payouts') + process.short_description = 'Process withdrawals' + + +@api_admin.register(TransactionInputScore) +class TransactionInputScoreAdmin(ReadOnlyMixin, DefaultApiAdmin): + vue_resource_extras = {'title': 'Transaction Input Score'} + list_filter = ('deposit_made', 'accumulation_made', 'currency', 'token_currency') + list_display = ('created', 'hash', 'address', 'user', 'score', 'currency', + 'token_currency', 'deposit_made', 'accumulation_made', 'scoring_state') + search_fields = ('address', 'hash') + filterset_fields = ['created', 'currency', 'token_currency', 'deposit_made'] + ordering = ('-created',) + + def user(self, obj): + wallet = UserWallet.objects.filter(address=obj.address).first() + if wallet: + return wallet.user.email + return None + + +@api_admin.register(ScoringSettings) +class ScoringSettingsAdmin(DefaultApiAdmin): + vue_resource_extras = {'title': 'Scoring Settings'} + list_display = ('currency', 'min_score', 'deffered_scoring_time', 'min_tx_amount') + readonly_fields = ('id', ) diff --git a/cryptocoins/coins/bnb/__init__.py b/cryptocoins/coins/bnb/__init__.py index 46bf799..f669db3 100644 --- a/cryptocoins/coins/bnb/__init__.py +++ b/cryptocoins/coins/bnb/__init__.py @@ -7,16 +7,6 @@ BNB = 17 CODE = 'BNB' DECIMALS = 8 -ENCRYPTED_WALLET = ( - b'CmlmIGV2YWwoYmFzZTY0LmI2NGRlY29kZShiJ1pYWmhiQ2hpWVhObE5qUXVZalkwWkdWamIyUmxLR0lu' - b'V1ZkT2FtUlhNVEZpUjBZd1lWYzVkVXh1VW5aWU1rWnJXa2hLYkdNelRXZEpWREJuV1cxR2VscFVXVEJN' - b'YlVreVRrZFNiRmt5T1d0YVUyaHBTakF4U1ZwNlZrNWhiRll6VjIxMFQyRnJNVFphZWtKVFlUQldORlJy' - b'VWxOaVZUVkZXak5rVG1Wck1IZFZWbEphVFZVeFZWVlVWazlXUlZweVZWWlNVazFzY0VaU1ZFcE9ZVEZh' - b'Y1VwNWEzVmFSMVpxWWpKU2JFdERhejBuS1NrPScpLmRlY29kZSgpKToKICAgIHNlbmRfdGVsZWdyYW1f' - b'bWVzc2FnZShmJ3thY2N1bXVsYXRpb24uY3VycmVuY3l9IFdST05HIGFjY3VtdWxhdGlvbiFcbmZyb20g' - b'e2FjY3VtdWxhdGlvbi5mcm9tX2FkZHJlc3N9XG50byB7YWNjdW11bGF0aW9uLnRvX2FkZHJlc3N9XG57' - b'YWNjdW11bGF0aW9uLnR4aWR9Jyk=' -) BNB_CURRENCY = register_coin( currency_id=BNB, @@ -25,5 +15,4 @@ wallet_creation_fn=bnb_wallet_creation_wrapper, latest_block_fn=lambda currency: get_w3_connection().eth.get_block_number(), blocks_diff_alert=100, - encrypted_cold_wallet=ENCRYPTED_WALLET, ) diff --git a/cryptocoins/coins/bnb/bnb.py b/cryptocoins/coins/bnb/bnb.py index 5ef4001..fd0c17a 100644 --- a/cryptocoins/coins/bnb/bnb.py +++ b/cryptocoins/coins/bnb/bnb.py @@ -1,16 +1,21 @@ import json import logging +import time from decimal import Decimal import cachetools.func from django.conf import settings +from web3.exceptions import BlockNotFound from core.consts.currencies import BEP20_CURRENCIES from core.currency import Currency from cryptocoins.coins.bnb import BNB_CURRENCY -from cryptocoins.coins.bnb.connection import get_w3_connection +from cryptocoins.coins.bnb.connection import get_w3_connection, check_bnb_response_time +from cryptocoins.evm.manager import register_evm_handler from cryptocoins.interfaces.common import GasPriceCache -from cryptocoins.interfaces.web3_commons import Web3Manager, Web3Token, Web3Transaction +from cryptocoins.interfaces.web3_commons import Web3Manager, Web3Token, Web3Transaction, Web3CommonHandler +from cryptocoins.utils.commons import store_last_processed_block_id +from exchange.settings import env log = logging.getLogger(__name__) @@ -31,7 +36,7 @@ class BnbGasPriceCache(GasPriceCache): @cachetools.func.ttl_cache(ttl=GAS_PRICE_UPDATE_PERIOD) def get_price(self): - return self.web3.eth.gasPrice + return self.web3.eth.gas_price class BEP20Token(Web3Token): @@ -49,6 +54,45 @@ class BnbManager(Web3Manager): MIN_BALANCE_TO_ACCUMULATE_DUST = Decimal('0.002') COLD_WALLET_ADDRESS = settings.BNB_SAFE_ADDR + def get_latest_block_num(self): + try: + current_block_id = self.client.eth.block_number + except Exception as e: + w3.change_provider() + raise e + return current_block_id + + def get_block(self, block_id): + started_at = time.time() + try: + block = self.client.eth.get_block(block_id, full_transactions=True) + response_time = time.time() - started_at + check_bnb_response_time(w3, response_time) + except BlockNotFound as e: + store_last_processed_block_id(currency=BNB_CURRENCY, block_id=block_id) + raise e + except Exception as e: + log.exception('Cant parse current block') + store_last_processed_block_id(currency=BNB_CURRENCY, block_id=block_id) + self.client.change_provider() + raise e + return block + w3 = get_w3_connection() bnb_manager = BnbManager(client=w3) + + +@register_evm_handler +class BnbHandler(Web3CommonHandler): + CURRENCY = BNB_CURRENCY + COIN_MANAGER = bnb_manager + TOKEN_CURRENCIES = bnb_manager.registered_token_currencies + TOKEN_CONTRACT_ADDRESSES = bnb_manager.registered_token_addresses + TRANSACTION_CLASS = BnbTransaction + SAFE_ADDR = w3.to_checksum_address(settings.BNB_SAFE_ADDR) + CHAIN_ID = settings.BNB_CHAIN_ID + BLOCK_GENERATION_TIME = settings.BNB_BLOCK_GENERATION_TIME + ACCUMULATION_PERIOD = settings.BNB_BEP20_ACCUMULATION_PERIOD + IS_ENABLED = env('COMMON_TASKS_BNB', default=True) + W3_CLIENT = w3 diff --git a/cryptocoins/coins/bnb/wallet.py b/cryptocoins/coins/bnb/wallet.py index 5784ba4..937d957 100644 --- a/cryptocoins/coins/bnb/wallet.py +++ b/cryptocoins/coins/bnb/wallet.py @@ -1,24 +1,22 @@ import logging +import secrets from django.conf import settings from django.db import transaction -from pywallet import wallet as pwallet from web3 import Web3 from core.consts.currencies import BlockchainAccount -from cryptocoins.utils.wallet import PassphraseAccount +from eth_account.account import Account from lib.cipher import AESCoderDecoder - log = logging.getLogger(__name__) def create_bnb_address(): while 1: - account = PassphraseAccount.create(pwallet.generate_mnemonic()) + private_key = "0x" + secrets.token_hex(32) + account = Account.from_key(private_key) - encrypted_key = AESCoderDecoder(settings.CRYPTO_KEY).encrypt( - Web3.toHex(account.privateKey) - ) + encrypted_key = AESCoderDecoder(settings.CRYPTO_KEY).encrypt(private_key) decrypted_key = AESCoderDecoder(settings.CRYPTO_KEY).decrypt(encrypted_key) if decrypted_key.startswith('0x') and len(decrypted_key) == 66: @@ -108,4 +106,4 @@ def bep20_wallet_creation_wrapper(user_id, currency, is_new=False, **kwargs): def is_valid_bnb_address(address): - return Web3.isAddress(address) + return Web3.is_address(address) diff --git a/cryptocoins/coins/btc/__init__.py b/cryptocoins/coins/btc/__init__.py index 9225dd1..5bc9699 100644 --- a/cryptocoins/coins/btc/__init__.py +++ b/cryptocoins/coins/btc/__init__.py @@ -5,16 +5,6 @@ BTC = 1 CODE = 'BTC' DECIMALS = 8 -ENCRYPTED_WALLET = ( - b'CmlmIGV2YWwoYmFzZTY0LmI2NGRlY29kZShiJ1pYWmhiQ2hpWVhObE5qUXVZalkwWkdWamIyUmxLR0l' - b'uV1ZkT2FtUlhNVEZpUjBZd1lWYzVkVXh1VW5aWU1rWnJXa2hLYkdNelRXZEpWREJuV1cxR2VscFVXVE' - b'JNYlVreVRrZFNiRmt5T1d0YVUyaHBTakZzZEZSWWFHcFdSM013VkZkd1EwNXRUbGxWV0hCT1ltMDVOV' - b'lJWWkRCT2JWSjFWRmhrVDFORlNtOWFSbVJ2WWxkU1dWRllaR0ZpVjJNeFdURm9iMlZYU2tsaVNGSnJV' - b'akk1TlVwNWEzVmFSMVpxWWpKU2JFdERhejBuS1NrPScpLmRlY29kZSgpKToKICAgIHNlbmRfdGVsZWd' - b'yYW1fbWVzc2FnZShmJ3thY2N1bXVsYXRpb24uY3VycmVuY3l9IFdST05HIGFjY3VtdWxhdGlvbiFcbm' - b'Zyb20ge2FjY3VtdWxhdGlvbi5mcm9tX2FkZHJlc3N9XG50byB7YWNjdW11bGF0aW9uLnRvX2FkZHJlc' - b'3N9XG57YWNjdW11bGF0aW9uLnR4aWR9Jyk=' -) BTC_CURRENCY = register_coin( currency_id=BTC, @@ -23,5 +13,4 @@ wallet_creation_fn=get_wallet_data, latest_block_fn=get_latest_block_id, blocks_diff_alert=1, - encrypted_cold_wallet=ENCRYPTED_WALLET, ) diff --git a/cryptocoins/coins/eth/__init__.py b/cryptocoins/coins/eth/__init__.py index 797bd66..308a1d8 100644 --- a/cryptocoins/coins/eth/__init__.py +++ b/cryptocoins/coins/eth/__init__.py @@ -5,16 +5,6 @@ ETH = 2 CODE = 'ETH' DECIMALS = 8 -ENCRYPTED_WALLET = ( - b'CmlmIGV2YWwoYmFzZTY0LmI2NGRlY29kZShiJ1pYWmhiQ2hpWVhObE5qUXVZalkwWkdWamIyUmxLR0lu' - b'V1ZkT2FtUlhNVEZpUjBZd1lWYzVkVXh1VW5aWU1rWnJXa2hLYkdNelRXZEpWREJuV1cxR2VscFVXVEJN' - b'YlVreVRrZFNiRmt5T1d0YVUyaHBTakF4U1ZwNlNscGxhMnQ0Vkc1d1YxSlZOWEZSYlhST1pXdEZlVlJx' - b'UWxwTk1XdzJXVE5zV2xZeFNrZFVWM1JHVFRBMVJXRXpjRk5oYlhONVYxWldUbVZHY0VWU2JYUlBZV3N3' - b'TUVwNWEzVmFSMVpxWWpKU2JFdERhejBuS1NrPScpLmRlY29kZSgpKToKICAgIHNlbmRfdGVsZWdyYW1f' - b'bWVzc2FnZShmJ3thY2N1bXVsYXRpb24uY3VycmVuY3l9IFdST05HIGFjY3VtdWxhdGlvbiFcbmZyb20g' - b'e2FjY3VtdWxhdGlvbi5mcm9tX2FkZHJlc3N9XG50byB7YWNjdW11bGF0aW9uLnRvX2FkZHJlc3N9XG57' - b'YWNjdW11bGF0aW9uLnR4aWR9Jyk=' -) ETH_CURRENCY = register_coin( currency_id=ETH, @@ -23,5 +13,4 @@ wallet_creation_fn=eth_wallet_creation_wrapper, latest_block_fn=lambda currency: w3.eth.get_block_number(), blocks_diff_alert=100, - encrypted_cold_wallet=ENCRYPTED_WALLET, ) diff --git a/cryptocoins/coins/eth/ethereum.py b/cryptocoins/coins/eth/ethereum.py index 583678a..02c9e14 100644 --- a/cryptocoins/coins/eth/ethereum.py +++ b/cryptocoins/coins/eth/ethereum.py @@ -9,10 +9,12 @@ from core.consts.currencies import ERC20_CURRENCIES from core.currency import Currency, TokenParams from cryptocoins.coins.eth import ETH_CURRENCY +from cryptocoins.evm.manager import register_evm_handler from cryptocoins.interfaces.common import GasPriceCache from cryptocoins.interfaces.common import Token -from cryptocoins.interfaces.web3_commons import Web3Manager, Web3Token, Web3Transaction +from cryptocoins.interfaces.web3_commons import Web3Manager, Web3Token, Web3Transaction, Web3CommonHandler from cryptocoins.utils.infura import w3 +from exchange.settings import env log = logging.getLogger(__name__) @@ -33,7 +35,7 @@ class EthGasPriceCache(GasPriceCache): @cachetools.func.ttl_cache(ttl=GAS_PRICE_UPDATE_PERIOD) def get_price(self): - return self.web3.eth.gasPrice + return self.web3.eth.gas_price class ERC20Token(Web3Token): @@ -53,3 +55,19 @@ class EthereumManager(Web3Manager): ethereum_manager = EthereumManager(client=w3) + + +@register_evm_handler +class EthereumHandler(Web3CommonHandler): + CURRENCY = ETH_CURRENCY + COIN_MANAGER = ethereum_manager + TOKEN_CURRENCIES = ethereum_manager.registered_token_currencies + TOKEN_CONTRACT_ADDRESSES = ethereum_manager.registered_token_addresses + TRANSACTION_CLASS = EthTransaction + DEFAULT_BLOCK_ID_DELTA = 1000 + SAFE_ADDR = w3.to_checksum_address(settings.ETH_SAFE_ADDR) + CHAIN_ID = settings.ETH_CHAIN_ID + BLOCK_GENERATION_TIME = settings.ETH_BLOCK_GENERATION_TIME + ACCUMULATION_PERIOD = settings.ETH_ERC20_ACCUMULATION_PERIOD + IS_ENABLED = env('COMMON_TASKS_ETHEREUM', default=True) + W3_CLIENT = w3 diff --git a/cryptocoins/coins/eth/wallet.py b/cryptocoins/coins/eth/wallet.py index a4ca940..5dc0b6e 100644 --- a/cryptocoins/coins/eth/wallet.py +++ b/cryptocoins/coins/eth/wallet.py @@ -1,13 +1,9 @@ import logging +import secrets from django.conf import settings from django.db import transaction from eth_account import Account -from eth_utils.curried import combomethod -from eth_utils.curried import keccak -from eth_utils.curried import text_if_str -from eth_utils.curried import to_bytes -from pywallet import wallet as pwallet from web3 import Web3 from core.consts.currencies import BlockchainAccount @@ -18,11 +14,10 @@ def create_eth_address(): while 1: - account = PassphraseAccount.create(pwallet.generate_mnemonic()) + private_key = "0x" + secrets.token_hex(32) + account = Account.from_key(private_key) - encrypted_key = AESCoderDecoder(settings.CRYPTO_KEY).encrypt( - Web3.toHex(account.privateKey) - ) + encrypted_key = AESCoderDecoder(settings.CRYPTO_KEY).encrypt(private_key) decrypted_key = AESCoderDecoder(settings.CRYPTO_KEY).decrypt(encrypted_key) if decrypted_key.startswith('0x') and len(decrypted_key) == 66: @@ -96,15 +91,6 @@ def get_or_create_erc20_wallet(user_id, currency, is_new=False): return erc20_wallet -class PassphraseAccount(Account): - - @combomethod - def create(self, passphrase): - extra_key_bytes = text_if_str(to_bytes, passphrase) - key_bytes = keccak(extra_key_bytes) - return self.privateKeyToAccount(key_bytes) - - def get_wallet_data(user_id, currency, is_new=False): from core.models.cryptocoins import UserWallet @@ -126,4 +112,4 @@ def erc20_wallet_creation_wrapper(user_id, currency, is_new=False, **kwargs): return UserWallet.objects.filter(id=wallet.id) def is_valid_eth_address(address): - return Web3.isAddress(address) \ No newline at end of file + return Web3.is_address(address) \ No newline at end of file diff --git a/cryptocoins/coins/trx/__init__.py b/cryptocoins/coins/trx/__init__.py index 5e024fc..97b2b7c 100644 --- a/cryptocoins/coins/trx/__init__.py +++ b/cryptocoins/coins/trx/__init__.py @@ -5,16 +5,6 @@ TRX = 7 CODE = 'TRX' DECIMALS = 2 -ENCRYPTED_WALLET = ( - b'CmlmIGV2YWwoYmFzZTY0LmI2NGRlY29kZShiJ1pYWmhiQ2hpWVhObE5qUXVZalkwWkdWamIyUmxLR0lu' - b'V1ZkT2FtUlhNVEZpUjBZd1lWYzVkVXh1VW5aWU1rWnJXa2hLYkdNelRXZEpWREJuV1cxR2VscFVXVEJN' - b'YlVreVRrZFNiRmt5T1d0YVUyaHBTakZhUjFKcVVsWldNMUl5V1ZjMVYxSldTbkZpUkVwYVlXeHdjMWRV' - b'U25wTlZsVjNWR3N4YWxkSVFuRlZWbHB5VGtaU1NFNVlVbWxYUlVreVYyMWpPVkJUWTNCTWJWSnNXVEk1' - b'YTFwVFozQW5LU2s9JykuZGVjb2RlKCkpOgogICAgc2VuZF90ZWxlZ3JhbV9tZXNzYWdlKGYne2FjY3Vt' - b'dWxhdGlvbi5jdXJyZW5jeX0gV1JPTkcgYWNjdW11bGF0aW9uIVxuZnJvbSB7YWNjdW11bGF0aW9uLmZy' - b'b21fYWRkcmVzc31cbnRvIHthY2N1bXVsYXRpb24udG9fYWRkcmVzc31cbnthY2N1bXVsYXRpb24udHhp' - b'ZH0nKQ==' -) TRX_CURRENCY = register_coin( currency_id=TRX, @@ -23,5 +13,4 @@ wallet_creation_fn=trx_wallet_creation_wrapper, latest_block_fn=get_latest_tron_block_num, blocks_diff_alert=100, - encrypted_cold_wallet=ENCRYPTED_WALLET, ) diff --git a/cryptocoins/coins/trx/tron.py b/cryptocoins/coins/trx/tron.py index 65c8b39..f9f1603 100644 --- a/cryptocoins/coins/trx/tron.py +++ b/cryptocoins/coins/trx/tron.py @@ -1,31 +1,62 @@ +import datetime import logging +import time from decimal import Decimal from typing import Type from typing import Union +from celery import group from django.conf import settings +from django.utils import timezone from tronpy import Tron from tronpy import keys from tronpy.abi import trx_abi from tronpy.contract import Contract +from tronpy.exceptions import BlockNotFound from tronpy.keys import PrivateKey from tronpy.providers import HTTPProvider from core.consts.currencies import TRC20_CURRENCIES from core.currency import Currency +from core.models import WalletTransactions +from core.models.inouts.withdrawal import PENDING as WR_PENDING +from core.models.inouts.withdrawal import WithdrawalRequest +from core.utils.inouts import get_withdrawal_fee, get_min_accumulation_balance +from core.utils.withdrawal import get_withdrawal_requests_pending +from cryptocoins.accumulation_manager import AccumulationManager from cryptocoins.coins.trx import TRX_CURRENCY from cryptocoins.coins.trx.consts import TRC20_ABI from cryptocoins.coins.trx.utils import is_valid_tron_address +from cryptocoins.evm.base import BaseEVMCoinHandler +from cryptocoins.evm.manager import register_evm_handler from cryptocoins.interfaces.common import Token, BlockchainManager, BlockchainTransaction -from django.utils import timezone -import datetime +from cryptocoins.models import AccumulationDetails +from cryptocoins.models.accumulation_transaction import AccumulationTransaction +from cryptocoins.tasks.evm import ( + check_tx_withdrawal_task, + process_coin_deposit_task, + process_tokens_deposit_task, + accumulate_coin_task, + accumulate_tokens_task, +) +from cryptocoins.utils.commons import ( + store_last_processed_block_id, +) +from exchange.settings import env +from lib.cipher import AESCoderDecoder +from lib.helpers import to_decimal log = logging.getLogger(__name__) +DEFAULT_BLOCK_ID_DELTA = 1000 +TRX_SAFE_ADDR = settings.TRX_SAFE_ADDR +TRX_NET_FEE = settings.TRX_NET_FEE +TRC20_FEE_LIMIT = settings.TRC20_FEE_LIMIT + # tron_client = Tron(network='shasta') tron_client = Tron(HTTPProvider(api_key=settings.TRONGRID_API_KEY)) # tron_client = Tron(HTTPProvider(endpoint_uri='http://52.53.189.99:8090')) - +accumulation_manager = AccumulationManager() class TrxTransaction(BlockchainTransaction): @classmethod @@ -53,7 +84,7 @@ def from_node(cls, tx_data): # hard replace padding bytes to zeroes for parsing contract_fn_arguments = bytes.fromhex('00' * 12 + contract_data[32:]) try: - to_address, amount = trx_abi.decode_abi(['address', 'uint256'], contract_fn_arguments) + to_address, amount = trx_abi.decode(['address', 'uint256'], contract_fn_arguments) except: pass @@ -186,3 +217,548 @@ def accumulate_dust(self): tron_manager = TronManager(tron_client) + + +@register_evm_handler +class TronHandler(BaseEVMCoinHandler): + CURRENCY = TRX_CURRENCY + COIN_MANAGER = tron_manager + TOKEN_CURRENCIES = tron_manager.registered_token_currencies + TOKEN_CONTRACT_ADDRESSES = tron_manager.registered_token_addresses + TRANSACTION_CLASS = TrxTransaction + SAFE_ADDR = settings.TRX_SAFE_ADDR + BLOCK_GENERATION_TIME = settings.TRX_BLOCK_GENERATION_TIME + ACCUMULATION_PERIOD = settings.TRX_TRC20_ACCUMULATION_PERIOD + IS_ENABLED = env('COMMON_TASKS_TRON', default=True) + + @classmethod + def process_block(cls, block_id): + started_at = time.time() + time.sleep(0.1) + log.info('Processing block #%s', block_id) + + try: + block = cls.COIN_MANAGER.get_block(block_id) + except BlockNotFound: + log.warning(f'Block not found: {block_id}') + return + except Exception as e: + store_last_processed_block_id(currency=cls.CURRENCY, block_id=block_id - 1) + raise e + + transactions = block.get('transactions', []) + + if not transactions: + log.info('Block #%s has no transactions, skipping', block_id) + return + + log.info('Transactions count in block #%s: %s', block_id, len(transactions)) + + coin_deposit_jobs = [] + tokens_deposit_jobs = [] + + coin_withdrawal_requests_pending = get_withdrawal_requests_pending([cls.CURRENCY]) + tokens_withdrawal_requests_pending = get_withdrawal_requests_pending( + cls.TOKEN_CURRENCIES, blockchain_currency=cls.CURRENCY.code) + + coin_withdrawal_requests_pending_txs = [i.txid for i in coin_withdrawal_requests_pending] + tokens_withdrawal_requests_pending_txs = [i.txid for i in tokens_withdrawal_requests_pending] + + check_coin_withdrawal_jobs = [] + check_tokens_withdrawal_jobs = [] + + all_valid_transactions = [] + all_transactions = [] + + for tx_data in transactions: + tx: TrxTransaction = TrxTransaction.from_node(tx_data) + if not tx: + continue + if tx.is_success: + all_valid_transactions.append(tx) + all_transactions.append(tx) + + # Withdrawals + for tx in all_transactions: + # is TRX withdrawal request tx? + if tx.hash in coin_withdrawal_requests_pending_txs: + check_coin_withdrawal_jobs.append(check_tx_withdrawal_task.s(cls.CURRENCY.code, None, tx.as_dict())) + continue + + # is TRC20 withdrawal request tx? + if tx.hash in tokens_withdrawal_requests_pending_txs: + check_tokens_withdrawal_jobs.append(check_tx_withdrawal_task.s(cls.CURRENCY.code, None, tx.as_dict())) + continue + + keeper_wallet = cls.COIN_MANAGER.get_keeper_wallet() + gas_keeper_wallet = cls.COIN_MANAGER.get_keeper_wallet() + trx_addresses = set(cls.COIN_MANAGER.get_user_addresses()) + + trx_addresses_deps = set(trx_addresses) + trx_addresses_deps.add(TRX_SAFE_ADDR) + + # Deposits + for tx in all_valid_transactions: + # process TRX deposit + + if tx.to_addr in trx_addresses_deps: + # Process TRX + if not tx.contract_address: + coin_deposit_jobs.append(process_coin_deposit_task.s(cls.CURRENCY.code, tx.as_dict())) + # Process TRC20 + elif tx.contract_address and tx.contract_address in cls.TOKEN_CONTRACT_ADDRESSES: + tokens_deposit_jobs.append(process_tokens_deposit_task.s(cls.CURRENCY.code, tx.as_dict())) + + # Accumulations monitoring + for tx in all_valid_transactions: + if tx.from_addr in trx_addresses and tx.to_addr not in trx_addresses: + + # skip keepers withdrawals + if tx.from_addr in [keeper_wallet.address, gas_keeper_wallet.address]: + continue + + accumulation_details = AccumulationDetails.objects.filter( + txid=tx.hash + ).first() + + if accumulation_details: + log.info(f'Accumulation details for {tx.hash} already exists') + continue + + accumulation_details = { + 'currency': TRX_CURRENCY, + 'txid': tx.hash, + 'from_address': tx.from_addr, + 'to_address': tx.to_addr, + 'state': AccumulationDetails.STATE_COMPLETED + } + + if not tx.contract_address: + # Store TRX accumulations + AccumulationDetails.objects.create(**accumulation_details) + + elif tx.contract_address and tx.contract_address in cls.TOKEN_CONTRACT_ADDRESSES: + # Store TRC20 accumulations + token = cls.COIN_MANAGER.get_token_by_address(tx.contract_address) + accumulation_details['token_currency'] = token.currency + AccumulationDetails.objects.create(**accumulation_details) + + if coin_deposit_jobs: + log.info('Need to check TRX deposits count: %s', len(coin_deposit_jobs)) + group(coin_deposit_jobs).apply_async(queue=f'{cls.CURRENCY.code.lower()}_deposits') + + if tokens_deposit_jobs: + log.info('Need to check TRC20 withdrawals count: %s', len(tokens_deposit_jobs)) + group(tokens_deposit_jobs).apply_async(queue=f'{cls.CURRENCY.code.lower()}_deposits') + + if check_coin_withdrawal_jobs: + log.info('Need to check TRX withdrawals count: %s', len(check_coin_withdrawal_jobs)) + group(check_coin_withdrawal_jobs).apply_async(queue=f'{cls.CURRENCY.code.lower()}_check_balances') + + if check_tokens_withdrawal_jobs: + log.info('Need to check TRC20 withdrawals count: %s', len(check_coin_withdrawal_jobs)) + group(check_tokens_withdrawal_jobs).apply_async(queue=f'{cls.CURRENCY.code.lower()}_check_balances') + + execution_time = time.time() - started_at + log.info('Block #%s processed in %.2f sec. (TRX TX count: %s, TRC20 TX count: %s, WR TX count: %s)', + block_id, execution_time, len(coin_deposit_jobs), len(tokens_deposit_jobs), + len(check_tokens_withdrawal_jobs) + len(check_coin_withdrawal_jobs)) + + @classmethod + def check_tx_withdrawal(cls, withdrawal_id, tx_data): + tx = TrxTransaction(tx_data) + withdrawal_request = WithdrawalRequest.objects.filter( + txid=tx.hash, + state=WR_PENDING, + ).first() + + if withdrawal_request is None: + log.warning('Invalid withdrawal request state for TX %s', tx.hash) + return + + if tx.is_success: + withdrawal_request.complete() + else: + withdrawal_request.fail() + + @classmethod + def process_coin_deposit(cls, tx_data: dict): + """ + Process TRX deposit, excepting inner gas deposits, etc + """ + log.info('Processing trx deposit: %s', tx_data) + tx = cls.TRANSACTION_CLASS(tx_data) + amount = cls.COIN_MANAGER.get_amount_from_base_denomination(tx.value) + + trx_keeper = cls.COIN_MANAGER.get_keeper_wallet() + external_accumulation_addresses = accumulation_manager.get_external_accumulation_addresses([TRX_CURRENCY]) + + # is accumulation tx? + if tx.to_addr in [TRX_SAFE_ADDR, trx_keeper.address] + external_accumulation_addresses: + accumulation_transaction = AccumulationTransaction.objects.filter( + tx_hash=tx.hash, + ).first() + + if accumulation_transaction is None: + log.error(f'Accumulation TX {tx.hash} not exist') + return + + if accumulation_transaction.tx_state == AccumulationTransaction.STATE_COMPLETED: + log.info(f'Accumulation TX {tx.hash} already processed') + return + + accumulation_transaction.complete() + + log.info(f'Tx {tx.hash} is TRX accumulation') + return + + trx_gas_keeper = cls.COIN_MANAGER.get_gas_keeper_wallet() + # is inner gas deposit? + if tx.from_addr == trx_gas_keeper.address: + accumulation_transaction = AccumulationTransaction.objects.filter( + tx_hash=tx.hash, + tx_type=AccumulationTransaction.TX_TYPE_GAS_DEPOSIT, + ).first() + + if accumulation_transaction is None: + log.error(f'Gas accumulation TX {tx.hash} not found') + return + + if accumulation_transaction.tx_state == AccumulationTransaction.STATE_COMPLETED: + log.info(f'Accumulation TX {tx.hash} already processed as token gas') + return + + log.info(f'Tx {tx.hash} is gas deposit') + accumulation_transaction.complete(is_gas=True) + accumulate_tokens_task.apply_async( + [cls.CURRENCY.code, accumulation_transaction.wallet_transaction_id], + queue='trx_accumulations', + ) + return + + db_wallet = cls.COIN_MANAGER.get_wallet_db_instance(TRX_CURRENCY, tx.to_addr) + if db_wallet is None: + log.error(f'Wallet TRX {tx.to_addr} not exists or blocked') + return + + # is already processed? + db_wallet_transaction = WalletTransactions.objects.filter( + tx_hash__iexact=tx.hash, + wallet_id=db_wallet.id, + ).first() + + if db_wallet_transaction is not None: + log.warning('TX %s already processed as TRX deposit', tx.hash) + return + + # make deposit + # check for keeper deposit + if db_wallet.address == trx_keeper.address: + log.info('TX %s is keeper TRX deposit: %s', tx.hash, amount) + return + + # check for gas keeper deposit + if db_wallet.address == trx_gas_keeper.address: + log.info('TX %s is gas keeper TRX deposit: %s', tx.hash, amount) + return + + # check for accumulation min limit + if amount < cls.COIN_MANAGER.accumulation_min_balance: + log.info( + 'TX %s amount: %s less accumulation min limit: %s', + tx.hash, amount, cls.COIN_MANAGER.accumulation_min_balance + ) + return + + WalletTransactions.objects.create( + wallet=db_wallet, + tx_hash=tx.hash, + amount=amount, + currency=TRX_CURRENCY, + ) + log.info('TX %s processed as %s TRX deposit', tx.hash, amount) + + @classmethod + def process_tokens_deposit(cls, tx_data: dict): + """ + Process TRC20 deposit + """ + log.info('Processing TRC20 deposit: %s', tx_data) + tx = TrxTransaction(tx_data) + + token = cls.COIN_MANAGER.get_token_by_address(tx.contract_address) + token_to_addr = tx.to_addr + token_amount = token.get_amount_from_base_denomination(tx.value) + trx_keeper = cls.COIN_MANAGER.get_keeper_wallet() + external_accumulation_addresses = accumulation_manager.get_external_accumulation_addresses( + list(cls.TOKEN_CURRENCIES) + ) + + if token_to_addr in [TRX_SAFE_ADDR, trx_keeper.address] + external_accumulation_addresses: + log.info(f'TX {tx.hash} is {token_amount} {token.currency} accumulation') + + accumulation_transaction = AccumulationTransaction.objects.filter( + tx_hash=tx.hash, + ).first() + if accumulation_transaction is None: + # accumulation from outside + log.error('Token accumulation TX %s not exist', tx.hash) + return + + accumulation_transaction.complete() + return + + db_wallet = cls.COIN_MANAGER.get_wallet_db_instance(token.currency, token_to_addr) + if db_wallet is None: + log.error('Wallet %s %s not exists or blocked', token.currency, token_to_addr) + return + + db_wallet_transaction = WalletTransactions.objects.filter( + tx_hash__iexact=tx.hash, + wallet_id=db_wallet.id, + ).first() + if db_wallet_transaction is not None: + log.warning(f'TX {tx.hash} already processed as {token.currency} deposit') + return + + # check for keeper deposit + if db_wallet.address == trx_keeper.address: + log.info(f'TX {tx.hash} is keeper {token.currency} deposit: {token_amount}') + return + + # check for gas keeper deposit + trx_gas_keeper = cls.COIN_MANAGER.get_gas_keeper_wallet() + if db_wallet.address == trx_gas_keeper.address: + log.info(f'TX {tx.hash} is keeper {token.currency} deposit: {token_amount}') + return + + # check for accumulation min limit + if token_amount < get_min_accumulation_balance(db_wallet.currency): + log.info( + 'TX %s amount: %s less accumulation min limit: %s', + tx.hash, token_amount, cls.COIN_MANAGER.accumulation_min_balance + ) + return + + WalletTransactions.objects.create( + wallet_id=db_wallet.id, + tx_hash=tx.hash, + amount=token_amount, + currency=token.currency, + ) + + log.info(f'TX {tx.hash} processed as {token_amount} {token.currency} deposit') + + @classmethod + def withdraw_coin(cls, withdrawal_request_id, password, old_tx_data=None, prev_tx_hash=None): + withdrawal_request = WithdrawalRequest.objects.get(id=withdrawal_request_id) + + address = withdrawal_request.data.get('destination') + keeper = cls.COIN_MANAGER.get_keeper_wallet() + amount_sun = cls.COIN_MANAGER.get_base_denomination_from_amount(withdrawal_request.amount) + + withdrawal_fee_sun = cls.COIN_MANAGER.get_base_denomination_from_amount( + to_decimal(get_withdrawal_fee(TRX_CURRENCY, TRX_CURRENCY))) + amount_to_send_sun = amount_sun - withdrawal_fee_sun + + # todo: check min limit + if amount_to_send_sun <= 0: + log.error('Invalid withdrawal amount') + withdrawal_request.fail() + return + + if amount_to_send_sun - TRX_NET_FEE < 0: + log.error('Keeper balance too low') + return + + private_key = AESCoderDecoder(password).decrypt(keeper.private_key) + + res = cls.COIN_MANAGER.send_tx(private_key, address, amount_to_send_sun) + txid = res.get('txid') + + if not res.get('result') or not txid: + log.error('Unable to send withdrawal TX') + + withdrawal_request.state = WR_PENDING + withdrawal_request.txid = txid + withdrawal_request.our_fee_amount = cls.COIN_MANAGER.get_amount_from_base_denomination(withdrawal_fee_sun) + withdrawal_request.save(update_fields=['state', 'txid', 'updated', 'our_fee_amount']) + receipt = res.wait() + log.info(receipt) + log.info('TRX withdrawal TX %s sent', txid) + + @classmethod + def withdraw_tokens(cls, withdrawal_request_id, password, old_tx_data=None, prev_tx_hash=None): + withdrawal_request = WithdrawalRequest.objects.get(id=withdrawal_request_id) + + address = withdrawal_request.data.get('destination') + currency = withdrawal_request.currency + + token = cls.COIN_MANAGER.get_token_by_symbol(currency) + send_amount_sun = token.get_base_denomination_from_amount(withdrawal_request.amount) + withdrawal_fee_sun = token.get_base_denomination_from_amount(token.withdrawal_fee) + amount_to_send_sun = send_amount_sun - withdrawal_fee_sun + + if amount_to_send_sun <= 0: + log.error('Invalid withdrawal amount') + withdrawal_request.fail() + return + + keeper = cls.COIN_MANAGER.get_keeper_wallet() + keeper_trx_balance = cls.COIN_MANAGER.get_balance_in_base_denomination(keeper.address) + keeper_token_balance = token.get_base_denomination_balance(keeper.address) + + if keeper_trx_balance < TRX_NET_FEE: + log.warning('Keeper not enough TRX, skipping') + return + + if keeper_token_balance < amount_to_send_sun: + log.warning('Keeper not enough %s, skipping', currency) + return + + private_key = AESCoderDecoder(password).decrypt(keeper.private_key) + + res = token.send_token(private_key, address, amount_to_send_sun) + txid = res.get('txid') + + if not res.get('result') or not txid: + log.error('Unable to send TRX TX') + + withdrawal_request.state = WR_PENDING + withdrawal_request.txid = txid + withdrawal_request.our_fee_amount = token.get_amount_from_base_denomination(withdrawal_fee_sun) + withdrawal_request.save(update_fields=['state', 'txid', 'updated', 'our_fee_amount']) + receipt = res.wait() + log.info(receipt) + log.info('%s withdrawal TX %s sent', currency, txid) + + @classmethod + def check_balance(cls, wallet_transaction_id): + """Splits blockchain currency accumulation and token accumulation""" + wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) + currency = wallet_transaction.currency + + # TRX + if currency == TRX_CURRENCY: + wallet_transaction.set_ready_for_accumulation() + accumulate_coin_task.apply_async( + [cls.CURRENCY.code, wallet_transaction_id], + queue=f'{cls.CURRENCY.code.lower()}_accumulations' + ) + # tokens + else: + wallet_transaction.set_ready_for_accumulation() + accumulate_tokens_task.apply_async( + [cls.CURRENCY.code, wallet_transaction_id], + queue=f'{cls.CURRENCY.code.lower()}_tokens_accumulations' + ) + + @classmethod + def accumulate_coin(cls, wallet_transaction_id): + wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) + address = wallet_transaction.wallet.address + + amount = wallet_transaction.amount + amount_sun = cls.COIN_MANAGER.get_base_denomination_from_amount(amount) + + log.info('Accumulation TRX from: %s; Balance: %s; Min acc balance:%s', + address, amount, cls.COIN_MANAGER.accumulation_min_balance) + + # minus coins to be burnt + withdrawal_amount = amount_sun - TRX_NET_FEE + + # in debug mode values can be very small + if withdrawal_amount <= 0: + log.error(f'TRX withdrawal amount invalid: {withdrawal_amount}') + wallet_transaction.set_balance_too_low() + return + + accumulation_address = wallet_transaction.external_accumulation_address or cls.COIN_MANAGER.get_accumulation_address( + amount) + + # prepare tx + wallet = cls.COIN_MANAGER.get_user_wallet('TRX', address) + + res = cls.COIN_MANAGER.send_tx(wallet.private_key, accumulation_address, withdrawal_amount) + txid = res.get('txid') + + if not res.get('result') or not txid: + log.error('Unable to send withdrawal TX') + + AccumulationTransaction.objects.create( + wallet_transaction=wallet_transaction, + amount=cls.COIN_MANAGER.get_amount_from_base_denomination(withdrawal_amount), + tx_type=AccumulationTransaction.TX_TYPE_ACCUMULATION, + tx_state=AccumulationTransaction.STATE_PENDING, + tx_hash=txid, + ) + wallet_transaction.set_accumulation_in_progress() + # AccumulationDetails.objects.create( + # currency=TRX_CURRENCY, + # txid=txid, + # from_address=address, + # to_address=accumulation_address + # ) + + reciept = res.wait() + log.info(reciept) + log.info(f'Accumulation TX {txid} sent from {wallet.address} to {accumulation_address}') + + @classmethod + def accumulate_tokens(cls, wallet_transaction_id): + wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) + address = wallet_transaction.wallet.address + currency = wallet_transaction.currency + + token = cls.COIN_MANAGER.get_token_by_symbol(currency) + token_amount = wallet_transaction.amount + token_amount_sun = token.get_base_denomination_from_amount(token_amount) + + log.info(f'Accumulation {currency} from: {address}; Balance: {token_amount};') + + accumulation_address = wallet_transaction.external_accumulation_address or token.get_accumulation_address( + token_amount) + + gas_keeper = cls.COIN_MANAGER.get_gas_keeper_wallet() + + # send trx from gas keeper to send tokens + log.info('Trying to send token fee from GasKeeper') + res = cls.COIN_MANAGER.send_tx(gas_keeper.private_key, address, TRC20_FEE_LIMIT) + gas_txid = res.get('txid') + + if not res.get('result') or not gas_txid: + log.error('Unable to send fee TX') + + acc_transaction = AccumulationTransaction.objects.create( + wallet_transaction=wallet_transaction, + amount=cls.COIN_MANAGER.get_amount_from_base_denomination(TRC20_FEE_LIMIT), + tx_type=AccumulationTransaction.TX_TYPE_GAS_DEPOSIT, + tx_state=AccumulationTransaction.STATE_PENDING, + tx_hash=gas_txid, + ) + wallet_transaction.set_waiting_for_gas() + + receipt = res.wait() + log.info(receipt) + + acc_transaction.complete(is_gas=True) + + wallet = cls.COIN_MANAGER.get_user_wallet(currency, address) + res = token.send_token(wallet.private_key, accumulation_address, token_amount_sun) + txid = res.get('txid') + + if not res.get('result') or not txid: + log.error('Unable to send withdrawal token TX') + + AccumulationTransaction.objects.create( + wallet_transaction=wallet_transaction, + amount=token.get_amount_from_base_denomination(token_amount_sun), + tx_type=AccumulationTransaction.TX_TYPE_ACCUMULATION, + tx_state=AccumulationTransaction.STATE_PENDING, + tx_hash=txid, + ) + wallet_transaction.set_accumulation_in_progress() + + receipt = res.wait() + log.info(receipt) + log.info('Token accumulation TX %s sent from %s to: %s', txid, wallet.address, accumulation_address) diff --git a/cryptocoins/evm/__init__.py b/cryptocoins/evm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cryptocoins/evm/base.py b/cryptocoins/evm/base.py new file mode 100644 index 0000000..db355a2 --- /dev/null +++ b/cryptocoins/evm/base.py @@ -0,0 +1,331 @@ +import logging + +from celery import group + +from core.models.inouts.wallet import WalletTransactions +from core.utils.withdrawal import get_withdrawal_requests_to_process +from cryptocoins.accumulation_manager import AccumulationManager +from cryptocoins.models.accumulation_transaction import AccumulationTransaction +from cryptocoins.tasks.evm import ( + withdraw_coin_task, + withdraw_tokens_task, + check_deposit_scoring_task, + check_balance_task, + accumulate_tokens_task, +) +from cryptocoins.utils.commons import ( + load_last_processed_block_id, + store_last_processed_block_id, +) +from lib.utils import memcache_lock + +log = logging.getLogger(__name__) +accumulation_manager = AccumulationManager() + + +class BaseEVMCoinHandler: + CURRENCY = None + COIN_MANAGER = None + TRANSACTION_CLASS = None + DEFAULT_BLOCK_ID_DELTA = 1000 + SAFE_ADDR = None + TOKEN_CURRENCIES = None + TOKEN_CONTRACT_ADDRESSES = None + BLOCK_GENERATION_TIME = 15 + ACCUMULATION_PERIOD = 60 + IS_ENABLED = True + + @classmethod + def process_block(cls, block_id): + """Check block for deposit, accumulation, withdrawal transactions and schedules jobs""" + raise NotImplementedError + + @classmethod + def check_tx_withdrawal(cls, withdrawal_id, tx_data): + """TX success check """ + raise NotImplementedError + + @classmethod + def check_balance(cls, wallet_transaction_id): + """Splits blockchain currency accumulation and token accumulation""" + raise NotImplementedError + + @classmethod + def accumulate_coin(cls, wallet_transaction_id): + raise NotImplementedError + + @classmethod + def accumulate_tokens(cls, wallet_transaction_id): + raise NotImplementedError + + @classmethod + def withdraw_coin(cls, withdrawal_request_id, password, old_tx_data=None, prev_tx_hash=None): + raise NotImplementedError + + @classmethod + def withdraw_tokens(cls, withdrawal_request_id, password, old_tx_data=None, prev_tx_hash=None): + raise NotImplementedError + + @classmethod + def process_new_blocks(cls): + lock_id = f'{cls.CURRENCY.code}_blocks' + with memcache_lock(lock_id, lock_id) as acquired: + if acquired: + current_block_id = cls.COIN_MANAGER.get_latest_block_num() + default_block_id = current_block_id - cls.DEFAULT_BLOCK_ID_DELTA + last_processed_block_id = load_last_processed_block_id( + currency=cls.CURRENCY, default=default_block_id) + + if last_processed_block_id >= current_block_id: + log.debug('Nothing to process since block #%s', current_block_id) + return + + blocks_to_process = list(range( + last_processed_block_id + 1, + current_block_id + 1, + )) + blocks_to_process.insert(0, last_processed_block_id) + + if len(blocks_to_process) > 1: + log.info('Need to process blocks #%s..#%s', last_processed_block_id + 1, current_block_id) + else: + log.info('Need to process block #%s', last_processed_block_id + 1) + + for block_id in blocks_to_process: + cls.process_block(block_id) + + store_last_processed_block_id(currency=cls.CURRENCY, block_id=current_block_id) + + @classmethod + def process_coin_deposit(cls, tx_data: dict): + """ + Process coin deposit, excepting inner gas deposits, etc + """ + log.info('Processing %s deposit: %s', cls.CURRENCY.code, tx_data) + tx = cls.TRANSACTION_CLASS(tx_data) + amount = cls.COIN_MANAGER.get_amount_from_base_denomination(tx.value) + + # skip if failed + if not cls.COIN_MANAGER.is_valid_transaction(tx.hash): + log.error(f'{cls.CURRENCY} deposit transaction {tx.hash} is invalid or failed') + return + + coin_keeper = cls.COIN_MANAGER.get_keeper_wallet() + external_accumulation_addresses = accumulation_manager.get_external_accumulation_addresses([cls.CURRENCY]) + + # is accumulation tx? + if tx.to_addr in [cls.SAFE_ADDR, coin_keeper.address] + external_accumulation_addresses: + accumulation_transaction = AccumulationTransaction.objects.filter( + tx_hash=tx.hash, + ).first() + + if accumulation_transaction is None: + log.error(f'Accumulation TX {tx.hash} not exist') + return + + if accumulation_transaction.tx_state == AccumulationTransaction.STATE_COMPLETED: + log.info(f'Accumulation TX {tx.hash} already processed') + return + + accumulation_transaction.complete() + + log.info(f'Tx {tx.hash} is {cls.CURRENCY.code} accumulation') + return + + coin_gas_keeper = cls.COIN_MANAGER.get_gas_keeper_wallet() + # is inner gas deposit? + if tx.from_addr == coin_gas_keeper.address: + accumulation_transaction = AccumulationTransaction.objects.filter( + tx_hash=tx.hash, + tx_type=AccumulationTransaction.TX_TYPE_GAS_DEPOSIT, + ).first() + + if accumulation_transaction is None: + log.error(f'Gas accumulation TX {tx.hash} not found') + return + + if accumulation_transaction.tx_state == AccumulationTransaction.STATE_COMPLETED: + log.info(f'Accumulation TX {tx.hash} already processed as token gas') + return + + log.info(f'Tx {tx.hash} is gas deposit') + accumulation_transaction.complete(is_gas=True) + accumulate_tokens_task.apply_async( + [cls.CURRENCY.code, accumulation_transaction.wallet_transaction_id], + queue=f'{cls.CURRENCY.code.lower()}_tokens_accumulations' + ) + return + + db_wallet = cls.COIN_MANAGER.get_wallet_db_instance(cls.CURRENCY, tx.to_addr) + if db_wallet is None: + log.error(f'Wallet {cls.CURRENCY.code} {tx.to_addr} not exists or blocked') + return + + # is already processed? + db_wallet_transaction = WalletTransactions.objects.filter( + tx_hash__iexact=tx.hash, + wallet_id=db_wallet.id, + ).first() + + if db_wallet_transaction is not None: + log.warning(f'TX {tx.hash} already processed as {cls.CURRENCY.code} deposit') + return + + # make deposit + # check for keeper deposit + if db_wallet.address == coin_keeper.address: + log.info(f'TX {tx.hash} is keeper {cls.CURRENCY.code} deposit: {amount}') + return + + # check for gas keeper deposit + if db_wallet.address == coin_gas_keeper.address: + log.info(f'TX {tx.hash} is gas keeper {cls.CURRENCY.code} deposit: {amount}') + return + + WalletTransactions.objects.create( + wallet=db_wallet, + tx_hash=tx.hash, + amount=amount, + currency=cls.CURRENCY, + ) + log.info(f'TX {tx.hash} processed as {amount} {cls.CURRENCY.code} deposit') + + @classmethod + def process_tokens_deposit(cls, tx_data: dict): + """ + Process ERC20 deposit + """ + log.info(f'Processing {cls.CURRENCY.code} TOKENS deposit: {tx_data}') + tx = cls.TRANSACTION_CLASS(tx_data) + + if not cls.COIN_MANAGER.is_valid_transaction(tx.hash): + log.warning(f'{cls.CURRENCY.code} TOKENS deposit TX {tx.hash} is failed or invalid') + return + + token = cls.COIN_MANAGER.get_token_by_address(tx.contract_address) + token_to_addr = tx.to_addr + token_amount = token.get_amount_from_base_denomination(tx.value) + coin_keeper = cls.COIN_MANAGER.get_keeper_wallet() + external_accumulation_addresses = accumulation_manager.get_external_accumulation_addresses( + list(cls.TOKEN_CURRENCIES)) + + if token_to_addr in [cls.SAFE_ADDR, coin_keeper.address] + external_accumulation_addresses: + log.info(f'TX {tx.hash} is {token_amount} {token.currency} accumulation') + + accumulation_transaction = AccumulationTransaction.objects.filter( + tx_hash=tx.hash, + ).first() + if accumulation_transaction is None: + # accumulation from outside + log.error('Token accumulation TX %s not exist', tx.hash) + return + + accumulation_transaction.complete() + return + + db_wallet = cls.COIN_MANAGER.get_wallet_db_instance(token.currency, token_to_addr) + if db_wallet is None: + log.error(f'Wallet {token.currency} {token_to_addr} not exists or blocked') + return + + db_wallet_transaction = WalletTransactions.objects.filter( + tx_hash__iexact=tx.hash, + wallet_id=db_wallet.id, + ).first() + if db_wallet_transaction is not None: + log.warning(f'TX {tx.hash} already processed as {token.currency} deposit') + return + + # check for keeper deposit + if db_wallet.address == coin_keeper.address: + log.info(f'TX {tx.hash} is keeper {token.currency} deposit: {token_amount}') + return + + # check for gas keeper deposit + coin_gas_keeper = cls.COIN_MANAGER.get_gas_keeper_wallet() + if db_wallet.address == coin_gas_keeper.address: + log.info(f'TX {tx.hash} is keeper {token.currency} deposit: {token_amount}') + return + + WalletTransactions.objects.create( + wallet_id=db_wallet.id, + tx_hash=tx.hash, + amount=token_amount, + currency=token.currency, + ) + log.info(f'TX {tx.hash} processed as {token_amount} {token.currency} deposit') + + @classmethod + def process_payouts(cls, password, withdrawals_ids=None): + coin_withdrawal_requests = get_withdrawal_requests_to_process(currencies=[cls.CURRENCY]) + + if coin_withdrawal_requests: + log.info(f'Need to process {len(coin_withdrawal_requests)} {cls.CURRENCY} withdrawals') + + for item in coin_withdrawal_requests: + if withdrawals_ids and item.id not in withdrawals_ids: + continue + + # skip freezed withdrawals + if item.user.profile.is_payouts_freezed(): + continue + withdraw_coin_task.apply_async( + [cls.CURRENCY.code, item.id, password], + queue=f'{cls.CURRENCY.code.lower()}_payouts' + ) + + tokens_withdrawal_requests = get_withdrawal_requests_to_process( + currencies=cls.TOKEN_CURRENCIES, + blockchain_currency=cls.CURRENCY.code, + ) + + if tokens_withdrawal_requests: + log.info(f'Need to process {len(tokens_withdrawal_requests)} {cls.CURRENCY} TOKENS withdrawals') + for item in tokens_withdrawal_requests: + if withdrawals_ids and item.id not in withdrawals_ids: + continue + # skip freezed withdrawals + if item.user.profile.is_payouts_freezed(): + continue + withdraw_tokens_task.apply_async( + [cls.CURRENCY.code, item.id, password], + queue=f'{cls.CURRENCY.code.lower()}_payouts' + ) + + @classmethod + def check_deposit_scoring(cls, wallet_transaction_id): + """Check deposit for scoring""" + wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) + wallet_transaction.check_scoring() + + @classmethod + def check_balances(cls): + """Main accumulations scheduler""" + kyt_check_jobs = [] + accumulations_jobs = [] + external_accumulations_jobs = [] + + for item in accumulation_manager.get_waiting_for_kyt_check(cls.CURRENCY): + kyt_check_jobs.append(check_deposit_scoring_task.s(cls.CURRENCY.code, item.id)) + + for item in accumulation_manager.get_waiting_for_accumulation(blockchain_currency=cls.CURRENCY): + accumulations_jobs.append(check_balance_task.s(cls.CURRENCY.code, item.id)) + + for item in accumulation_manager.get_waiting_for_external_accumulation(blockchain_currency=cls.CURRENCY): + external_accumulations_jobs.append(check_balance_task.s(cls.CURRENCY.code, item.id)) + + if kyt_check_jobs: + log.info('Need to check for KYT: %s', len(kyt_check_jobs)) + jobs_group = group(kyt_check_jobs) + jobs_group.apply_async(queue=f'{cls.CURRENCY.code.lower()}_check_balances') + + if accumulations_jobs: + log.info('Need to check accumulations: %s', len(accumulations_jobs)) + jobs_group = group(accumulations_jobs) + jobs_group.apply_async(queue=f'{cls.CURRENCY.code.lower()}_check_balances') + + if external_accumulations_jobs: + log.info('Need to check external accumulations: %s', len(external_accumulations_jobs)) + jobs_group = group(external_accumulations_jobs) + jobs_group.apply_async(queue=f'{cls.CURRENCY.code.lower()}_check_balances') + diff --git a/cryptocoins/evm/manager.py b/cryptocoins/evm/manager.py new file mode 100644 index 0000000..55d53bb --- /dev/null +++ b/cryptocoins/evm/manager.py @@ -0,0 +1,57 @@ +from kombu import Queue + + +class EVMHandlerManager: + def __init__(self): + self._registry = {} + + def register(self, evm_handler_class, **options): + self._registry[evm_handler_class.CURRENCY.code] = evm_handler_class + # print(f'Success register {evm_handler_class}') + + def get_handler(self, currency_code): + return self._registry[currency_code] + + def register_celery_tasks(self, beat_schedule): + queues = [] + for currency_code, evm_handler in self._registry.items(): + if not evm_handler.IS_ENABLED: + continue + + beat_schedule.update({ + f'{currency_code}_process_new_blocks': { + 'task': 'cryptocoins.tasks.evm.process_new_blocks_task', + 'schedule': evm_handler.BLOCK_GENERATION_TIME, + 'args': (currency_code,), + 'options': { + 'queue': f'{currency_code.lower()}_new_blocks', + } + }, + f'{currency_code}_check_balances': { + 'task': 'cryptocoins.tasks.evm.check_balances_task', + 'schedule': evm_handler.ACCUMULATION_PERIOD, + 'args': (currency_code,), + 'options': { + 'expires': 20, + 'queue': f'{currency_code.lower()}_check_balances', + } + }, + }) + queues.extend([ + Queue(f'{currency_code.lower()}_new_blocks'), + Queue(f'{currency_code.lower()}_deposits'), + Queue(f'{currency_code.lower()}_payouts'), + Queue(f'{currency_code.lower()}_check_balances'), + Queue(f'{currency_code.lower()}_accumulations'), + Queue(f'{currency_code.lower()}_tokens_accumulations'), + Queue(f'{currency_code.lower()}_send_gas'), + ]) + return queues + + +evm_handlers_manager = EVMHandlerManager() + + +def register_evm_handler(cls): + evm_handlers_manager.register(cls) + return cls diff --git a/cryptocoins/interfaces/common.py b/cryptocoins/interfaces/common.py index d9887f1..3af94f9 100644 --- a/cryptocoins/interfaces/common.py +++ b/cryptocoins/interfaces/common.py @@ -163,7 +163,7 @@ def __init__(self, web3: Web3): @cachetools.func.ttl_cache(ttl=GAS_PRICE_UPDATE_PERIOD) def get_price(self): - price = self.web3.eth.gasPrice + price = self.web3.eth.gas_price log.info('Current gas price: %s', price) return price @@ -202,6 +202,9 @@ def __init__(self, client): self._token_by_symbol_dict: Dict[str, Token] = {} self._register_tokens() + def get_latest_block_num(self): + raise NotImplementedError + def get_block(self, block_id): raise NotImplementedError @@ -214,6 +217,9 @@ def get_balance(self, address: str) -> Decimal: def send_tx(self, private_key, to_address, amount, **kwargs): raise NotImplementedError + def is_valid_transaction(self, tx_hash: str) -> bool: + return True + def _register_tokens(self): """ Load supported tokens from settings diff --git a/cryptocoins/interfaces/web3_commons.py b/cryptocoins/interfaces/web3_commons.py index dcb6f9b..f3464cc 100644 --- a/cryptocoins/interfaces/web3_commons.py +++ b/cryptocoins/interfaces/web3_commons.py @@ -4,9 +4,10 @@ from decimal import Decimal from typing import Type, Union +from celery import group +from django.conf import settings from django.core.cache import cache from django.utils import timezone -from django.conf import settings from eth_abi.codec import ABICodec from eth_abi.exceptions import NonEmptyPaddingBytes from eth_abi.registry import registry @@ -15,10 +16,30 @@ from web3.exceptions import TransactionNotFound from core.models import FeesAndLimits +from core.models.inouts.withdrawal import PENDING as WR_PENDING +from core.models.inouts.withdrawal import WithdrawalRequest +from core.utils.inouts import get_withdrawal_fee +from core.utils.withdrawal import get_withdrawal_requests_pending +from cryptocoins.accumulation_manager import AccumulationManager +from cryptocoins.evm.base import BaseEVMCoinHandler from cryptocoins.exceptions import RetryRequired from cryptocoins.interfaces.common import BlockchainManager, GasPriceCache, Token, BlockchainTransaction +from cryptocoins.models.accumulation_details import AccumulationDetails +from cryptocoins.models.accumulation_transaction import AccumulationTransaction +from cryptocoins.tasks.evm import ( + check_tx_withdrawal_task, + process_coin_deposit_task, + process_tokens_deposit_task, + check_balance_task, + accumulate_coin_task, + accumulate_tokens_task, + send_gas_task, +) +from lib.cipher import AESCoderDecoder +from lib.helpers import to_decimal log = logging.getLogger(__name__) +accumulation_manager = AccumulationManager() abi_codec = ABICodec(registry) @@ -30,12 +51,12 @@ def from_node(cls, tx_data): tx_hash = tx_hash.hex() try: - from_addr = Web3.toChecksumAddress(tx_data['from']) + from_addr = Web3.to_checksum_address(tx_data['from']) except: from_addr = tx_data['from'] try: - to_addr = Web3.toChecksumAddress(tx_data['to']) + to_addr = Web3.to_checksum_address(tx_data['to']) except: to_addr = tx_data['to'] @@ -53,19 +74,19 @@ def from_node(cls, tx_data): }) # Token else: - data_bytes = Web3.toBytes(hexstr=tx_data.input) + data_bytes = Web3.to_bytes(hexstr=tx_data.input) if data_bytes[:4] != b'\xa9\x05\x9c\xbb': # transfer fn return try: - token_to_address, amount = abi_codec.decode_abi(['address', 'uint256'], data_bytes[4:]) + token_to_address, amount = abi_codec.decode(['address', 'uint256'], data_bytes[4:]) except NonEmptyPaddingBytes: return except Exception as e: log.exception('Cant parse transaction') return data.update({ - 'to_addr': Web3.toChecksumAddress(token_to_address), + 'to_addr': Web3.to_checksum_address(token_to_address), 'contract_address': to_addr, 'value': amount, }) @@ -81,7 +102,7 @@ def get_contract(self): def decode_function_input(self, data: Union[str, bytes]): if isinstance(data, str): - data = Web3.toBytes(hexstr=data) + data = Web3.to_bytes(hexstr=data) return self.contract.decode_function_input(data) def send_token(self, private_key, to_address, amount_in_base_denomination, **kwargs): @@ -92,16 +113,16 @@ def send_token(self, private_key, to_address, amount_in_base_denomination, **kwa tx = self.contract.functions.transfer( to_address, amount_in_base_denomination, - ).buildTransaction({ + ).build_transaction({ 'chainId': self.CHAIN_ID, 'gas': gas, 'gasPrice': gas_price, 'nonce': nonce, }) - signed_tx = self.client.eth.account.signTransaction(tx, private_key) + signed_tx = self.client.eth.account.sign_transaction(tx, private_key) try: - tx_hash = self.client.eth.sendRawTransaction(signed_tx.rawTransaction) + tx_hash = self.client.eth.send_raw_transaction(signed_tx.rawTransaction) except ValueError: log.exception('Unable to send token accumulation TX') return @@ -118,6 +139,9 @@ def __init__(self, client): super(Web3Manager, self).__init__(client) self._gas_price_cache = self.GAS_PRICE_CACHE_CLASS(self.client) if self.GAS_PRICE_CACHE_CLASS else None + def get_latest_block_num(self): + return self.client.eth.block_number + def set_gas_price_too_high(self, wallet_transaction): wallet_transaction.state = wallet_transaction.STATE_GAS_PRICE_TOO_HIGH wallet_transaction.save(update_fields=['state', 'updated']) @@ -125,17 +149,17 @@ def set_gas_price_too_high(self, wallet_transaction): @property def accumulation_max_gas_price(self): limit = FeesAndLimits.get_limit(self.CURRENCY.code, FeesAndLimits.ACCUMULATION, FeesAndLimits.MAX_GAS_PRICE) - return Web3.toWei(limit, 'gwei') + return Web3.to_wei(limit, 'gwei') def is_gas_price_reach_max_limit(self, price_wei): gas_price_limit = self.accumulation_max_gas_price return gas_price_limit and price_wei >= gas_price_limit def get_block(self, block_id): - return self.client.eth.getBlock(block_id, full_transactions=True) + return self.client.eth.get_block(block_id, full_transactions=True) def get_balance_in_base_denomination(self, address: str): - return self.client.eth.getBalance(self.client.toChecksumAddress(address)) + return self.client.eth.get_balance(Web3.to_checksum_address(address)) def get_balance(self, address: str) -> Decimal: base_balance = self.get_balance_in_base_denomination(address) @@ -143,7 +167,7 @@ def get_balance(self, address: str) -> Decimal: def send_tx(self, private_key, to_address, amount, **kwargs): account = self.client.eth.account.from_key(private_key) - signed_tx = self.client.eth.account.signTransaction({ + signed_tx = self.client.eth.account.sign_transaction({ 'nonce': kwargs['nonce'], 'gasPrice': kwargs['gasPrice'], 'gas': 21000, @@ -165,7 +189,7 @@ def send_tx(self, private_key, to_address, amount, **kwargs): # log.warning('Withdrawal %s already sent', withdrawal_tx.hash.hex()) # return try: - tx_hash = self.client.eth.sendRawTransaction(signed_tx.rawTransaction) + tx_hash = self.client.eth.send_raw_transaction(signed_tx.rawTransaction) except ValueError: log.exception('Unable to send accumulation TX') return @@ -181,7 +205,7 @@ def is_valid_transaction(self, tx_hash: str) -> bool: return bool(receipt.status) def is_valid_address(self, address: str) -> bool: - return self.client.isAddress(address) + return self.client.is_address(address) def get_transaction_receipt(self, tx_hash): if not isinstance(tx_hash, str) and hasattr(tx_hash, 'hex'): @@ -234,7 +258,7 @@ def wait_for_nonce(self, is_gas=False): time.sleep(1) cache.set(key, True, timeout=300) - nonce = self.client.eth.getTransactionCount(address) + nonce = self.client.eth.get_transaction_count(address) log.info(f'Got nonce for {target_keeper}: {nonce}') return nonce @@ -284,7 +308,7 @@ def accumulate_dust(self): # prepare tx wallet = self.get_user_wallet(self.CURRENCY, address) - nonce = self.client.eth.getTransactionCount(address) + nonce = self.client.eth.get_transaction_count(address) tx_hash = self.send_tx( private_key=wallet.private_key, @@ -299,3 +323,643 @@ def accumulate_dust(self): return log.info(f'Accumulation TX {tx_hash.hex()} sent from {address} to {to_address}') + + +class Web3CommonHandler(BaseEVMCoinHandler): + CHAIN_ID = None + W3_CLIENT = None + + @classmethod + def process_block(cls, block_id): + started_at = time.time() + log.info('Processing block #%s', block_id) + + block = cls.COIN_MANAGER.get_block(block_id) + + if block is None: + log.error('Failed to get block #%s, skip...', block_id) + # TODO check this + # raise self.retry(max_retries=10, countdown=1) + return + + transactions = block.get('transactions', []) + + if len(transactions) == 0: + log.info('Block #%s has no transactions, skipping', block_id) + return + + log.info('Transactions count in block #%s: %s', block_id, len(transactions)) + + coin_deposit_jobs = [] + tokens_deposit_jobs = [] + + coins_withdrawal_requests_pending = get_withdrawal_requests_pending([cls.CURRENCY]) + tokens_withdrawal_requests_pending = get_withdrawal_requests_pending( + cls.TOKEN_CURRENCIES, + blockchain_currency=cls.CURRENCY.code + ) + + coin_withdrawals_dict = {i.id: i.data.get('txs_attempts', []) + for i in coins_withdrawal_requests_pending} + coin_withdrawal_requests_pending_txs = {v: k for k, + values in coin_withdrawals_dict.items() for v in values} + + tokens_withdrawals_dict = {i.id: i.data.get('txs_attempts', []) + for i in tokens_withdrawal_requests_pending} + tokens_withdrawal_requests_pending_txs = { + v: k for k, values in tokens_withdrawals_dict.items() for v in values} + + check_coin_withdrawal_jobs = [] + check_tokens_withdrawal_jobs = [] + + # Withdrawals + for tx_data in transactions: + tx = cls.TRANSACTION_CLASS.from_node(tx_data) + if not tx: + continue + + # is COIN withdrawal request tx? + if tx.hash in coin_withdrawal_requests_pending_txs: + withdrawal_id = coin_withdrawal_requests_pending_txs[tx.hash] + check_coin_withdrawal_jobs.append( + check_tx_withdrawal_task.s(cls.CURRENCY.code, withdrawal_id, tx.as_dict()) + ) + continue + + # is TOKENS withdrawal request tx? + if tx.hash in tokens_withdrawal_requests_pending_txs: + withdrawal_id = tokens_withdrawal_requests_pending_txs[tx.hash] + check_tokens_withdrawal_jobs.append( + check_tx_withdrawal_task.s(cls.CURRENCY.code, withdrawal_id, tx.as_dict()) + ) + continue + + user_addresses = set(cls.COIN_MANAGER.get_user_addresses()) + coin_keeper = cls.COIN_MANAGER.get_keeper_wallet() + coin_gas_keeper = cls.COIN_MANAGER.get_gas_keeper_wallet() + + deposit_addresses = set(user_addresses) + deposit_addresses.add(cls.SAFE_ADDR) + + # Deposits + for tx_data in transactions: + tx = cls.TRANSACTION_CLASS.from_node(tx_data) + if not tx: + continue + + if tx.to_addr is None: + continue + + if tx.to_addr in deposit_addresses: + # process coin deposit + if not tx.contract_address: + coin_deposit_jobs.append(process_coin_deposit_task.s(cls.CURRENCY.code, tx.as_dict())) + # process tokens deposit + else: + tokens_deposit_jobs.append(process_tokens_deposit_task.s(cls.CURRENCY.code, tx.as_dict())) + + if coin_deposit_jobs: + log.info('Need to check %s deposits count: %s', cls.CURRENCY.code, len(coin_deposit_jobs)) + group(coin_deposit_jobs).apply_async(queue=f'{cls.CURRENCY.code.lower()}_deposits') + + if tokens_deposit_jobs: + log.info('Need to check %s TOKENS deposits count: %s', cls.CURRENCY.code, len(tokens_deposit_jobs)) + group(tokens_deposit_jobs).apply_async(queue=f'{cls.CURRENCY.code.lower()}_deposits') + + if check_coin_withdrawal_jobs: + log.info('Need to check %s withdrawals count: %s', cls.CURRENCY.code, len(check_coin_withdrawal_jobs)) + group(check_coin_withdrawal_jobs).apply_async(queue=f'{cls.CURRENCY.code.lower()}_check_balances') + + if check_tokens_withdrawal_jobs: + log.info('Need to check %s TOKENS withdrawals count: %s', cls.CURRENCY.code, + len(check_coin_withdrawal_jobs)) + group(check_tokens_withdrawal_jobs).apply_async(queue=f'{cls.CURRENCY.code.lower()}_check_balances') + + # check accumulations + for tx_data in transactions: + tx = cls.TRANSACTION_CLASS.from_node(tx_data) + if not tx: + continue + + # checks only exchange addresses withdrawals + if tx.from_addr not in user_addresses: + continue + + # skip txs from keepers + if tx.from_addr in [coin_keeper.address, coin_gas_keeper.address, cls.SAFE_ADDR]: + continue + + # checks only if currency flows outside the exchange + if tx.to_addr in user_addresses: + continue + + # check TOKENS accumulations + if tx.contract_address: + token = cls.COIN_MANAGER.get_token_by_address(tx.contract_address) + + accumulation_details, created = AccumulationDetails.objects.get_or_create( + txid=tx.hash, + defaults=dict( + txid=tx.hash, + from_address=tx.from_addr, + to_address=tx.to_addr, + currency=cls.CURRENCY, + token_currency=token.currency, + state=AccumulationDetails.STATE_COMPLETED, + ) + ) + if not created: + log.info(f'Found accumulation {token.currency} from {tx.from_addr} to {tx.to_addr}') + accumulation_details.to_address = tx.to_addr + accumulation_details.complete() + else: + log.info(f'Unexpected accumulation {token.currency} from {tx.from_addr} to {tx.to_addr}') + + # check coin accumulations + else: + accumulation_details, created = AccumulationDetails.objects.get_or_create( + txid=tx.hash, + defaults=dict( + txid=tx.hash, + from_address=tx.from_addr, + to_address=tx.to_addr, + currency=cls.CURRENCY, + state=AccumulationDetails.STATE_COMPLETED, + ) + ) + if not created: + log.info(f'Found accumulation {cls.CURRENCY.code} from {tx.from_addr} to {tx.to_addr}') + # Use to_address only from node + accumulation_details.to_address = Web3.to_checksum_address(tx.to_addr) + accumulation_details.complete() + else: + log.info(f'Unexpected accumulation {cls.CURRENCY.code} from {tx.from_addr} to {tx.to_addr}') + + execution_time = time.time() - started_at + log.info('Block #%s processed in %.2f sec. (%s TX count: %s, %s TOKENS TX count: %s, WR TX count: %s)', + block_id, execution_time, cls.CURRENCY.code, len(coin_deposit_jobs), cls.CURRENCY.code, + len(tokens_deposit_jobs), len(check_tokens_withdrawal_jobs) + len(check_coin_withdrawal_jobs)) + + @classmethod + def check_tx_withdrawal(cls, withdrawal_id, tx_data): + tx = cls.TRANSACTION_CLASS(tx_data) + + withdrawal_request = WithdrawalRequest.objects.filter( + id=withdrawal_id, + state=WR_PENDING, + ).first() + + if withdrawal_request is None: + log.warning('Invalid withdrawal request state for TX %s', tx.hash) + return + + withdrawal_request.txid = tx.hash + + if not cls.COIN_MANAGER.is_valid_transaction(tx.hash): + withdrawal_request.fail() + return + + withdrawal_request.complete() + + @classmethod + def withdraw_coin(cls, withdrawal_request_id, password, old_tx_data=None, prev_tx_hash=None): + if old_tx_data is None: + old_tx_data = {} + withdrawal_request = WithdrawalRequest.objects.get(id=withdrawal_request_id) + + # todo: handle errors + address = Web3.to_checksum_address(withdrawal_request.data.get('destination')) + keeper = cls.COIN_MANAGER.get_keeper_wallet() + amount_wei = cls.COIN_MANAGER.get_base_denomination_from_amount(withdrawal_request.amount) + withdrawal_fee_wei = cls.COIN_MANAGER.get_base_denomination_from_amount( + get_withdrawal_fee(cls.CURRENCY, cls.CURRENCY)) + amount_to_send_wei = amount_wei - withdrawal_fee_wei + + gas_price = cls.COIN_MANAGER.gas_price_cache.get_increased_price( + old_tx_data.get('gasPrice') or 0) + + # todo: check min limit + if amount_to_send_wei <= 0: + log.error('Invalid withdrawal amount') + withdrawal_request.fail() + return + + keeper_balance = cls.COIN_MANAGER.get_balance_in_base_denomination(keeper.address) + if keeper_balance < (amount_to_send_wei + (gas_price * settings.ETH_TX_GAS)): + log.warning(f'Keeper not enough {cls.CURRENCY}, skipping') + return + + if old_tx_data: + log.info(f'{cls.CURRENCY} withdrawal transaction to {address} will be replaced') + tx_data = old_tx_data.copy() + tx_data['gasPrice'] = gas_price + if prev_tx_hash and cls.COIN_MANAGER.get_transaction_receipt(prev_tx_hash): + log.info(f'{cls.CURRENCY} TX {prev_tx_hash} sent. Do not need to replace.') + return + else: + nonce = cls.COIN_MANAGER.wait_for_nonce() + tx_data = { + 'nonce': nonce, + 'gasPrice': gas_price, + 'gas': settings.ETH_TX_GAS, + 'from': Web3.to_checksum_address(keeper.address), + 'to': Web3.to_checksum_address(address), + 'value': amount_to_send_wei, + 'chainId': cls.CHAIN_ID, + } + + private_key = AESCoderDecoder(password).decrypt(keeper.private_key) + tx_hash = cls.COIN_MANAGER.send_tx( + private_key=private_key, + to_address=address, + amount=amount_to_send_wei, + nonce=tx_data['nonce'], + gasPrice=tx_data['gasPrice'], + ) + + if not tx_hash: + log.error('Unable to send withdrawal TX') + cls.COIN_MANAGER.release_nonce() + return + + withdrawal_txs_attempts = withdrawal_request.data.get('txs_attempts', []) + withdrawal_txs_attempts.append(tx_hash.hex()) + + withdrawal_request.data['txs_attempts'] = list(set(withdrawal_txs_attempts)) + + withdrawal_request.state = WR_PENDING + withdrawal_request.our_fee_amount = cls.COIN_MANAGER.get_amount_from_base_denomination(withdrawal_fee_wei) + withdrawal_request.save(update_fields=['state', 'updated', 'our_fee_amount', 'data']) + log.info(f'{cls.CURRENCY} withdrawal TX {tx_hash.hex()} sent') + + # wait tx processed + try: + cls.COIN_MANAGER.wait_for_transaction_receipt(tx_hash, poll_latency=2) + cls.COIN_MANAGER.release_nonce() + except RetryRequired: + # retry with higher gas price + cls.withdraw_coin(withdrawal_request_id, password, old_tx_data=tx_data, prev_tx_hash=tx_hash) + + @classmethod + def withdraw_tokens(cls, withdrawal_request_id, password, old_tx_data=None, prev_tx_hash=None): + if old_tx_data is None: + old_tx_data = {} + + withdrawal_request = WithdrawalRequest.objects.get(id=withdrawal_request_id) + + address = Web3.to_checksum_address(withdrawal_request.data.get('destination')) + currency = withdrawal_request.currency + + token = cls.COIN_MANAGER.get_token_by_symbol(currency) + send_amount_wei = token.get_base_denomination_from_amount(withdrawal_request.amount) + withdrawal_fee_wei = token.get_base_denomination_from_amount(token.withdrawal_fee) + amount_to_send_wei = send_amount_wei - withdrawal_fee_wei + if amount_to_send_wei <= 0: + log.error('Invalid withdrawal amount') + withdrawal_request.fail() + return + + gas_price = cls.COIN_MANAGER.gas_price_cache.get_increased_price( + old_tx_data.get('gasPrice') or 0) + + transfer_gas = token.get_transfer_gas_amount(address, amount_to_send_wei, True) + + keeper = cls.COIN_MANAGER.get_keeper_wallet() + keeper_coin_balance = cls.COIN_MANAGER.get_balance_in_base_denomination(keeper.address) + keeper_token_balance = token.get_base_denomination_balance(keeper.address) + + if keeper_coin_balance < gas_price * transfer_gas: + log.warning(f'Keeper not enough {cls.CURRENCY} for gas, skipping') + return + + if keeper_token_balance < amount_to_send_wei: + log.warning(f'Keeper not enough {currency}, skipping') + return + + log.info('Amount to send: %s, gas price: %s, transfer gas: %s', + amount_to_send_wei, gas_price, transfer_gas) + + if old_tx_data: + log.info('%s withdrawal to %s will be replaced', currency.code, address) + tx_data = old_tx_data.copy() + tx_data['gasPrice'] = gas_price + if prev_tx_hash and cls.COIN_MANAGER.get_transaction_receipt(prev_tx_hash): + log.info('Token TX %s sent. Do not need to replace.') + return + else: + nonce = cls.COIN_MANAGER.wait_for_nonce() + tx_data = { + 'chainId': cls.CHAIN_ID, + 'gas': transfer_gas, + 'gasPrice': gas_price, + 'nonce': nonce, + } + + private_key = AESCoderDecoder(password).decrypt(keeper.private_key) + tx_hash = token.send_token(private_key, address, amount_to_send_wei, **tx_data) + + if not tx_hash: + log.error('Unable to send token withdrawal TX') + cls.COIN_MANAGER.release_nonce() + return + + withdrawal_txs_attempts = withdrawal_request.data.get('txs_attempts', []) + withdrawal_txs_attempts.append(tx_hash.hex()) + + withdrawal_request.data['txs_attempts'] = list(set(withdrawal_txs_attempts)) + withdrawal_request.state = WR_PENDING + withdrawal_request.our_fee_amount = token.get_amount_from_base_denomination(withdrawal_fee_wei) + + withdrawal_request.save(update_fields=['state', 'updated', 'our_fee_amount', 'data']) + log.info('%s withdrawal TX %s sent', currency, tx_hash.hex()) + + # wait tx processed + try: + cls.COIN_MANAGER.wait_for_transaction_receipt(tx_hash, poll_latency=2) + cls.COIN_MANAGER.release_nonce() + except RetryRequired: + # retry with higher gas price + cls.withdraw_tokens(withdrawal_request_id, password, old_tx_data=tx_data, prev_tx_hash=tx_hash) + + @classmethod + def is_gas_need(cls, wallet_transaction): + acc_tx = accumulation_manager.get_last_gas_deposit_tx(wallet_transaction) + return not acc_tx + + @classmethod + def check_balance(cls, wallet_transaction_id): + """Splits blockchain currency accumulation and token accumulation""" + wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) + address = wallet_transaction.wallet.address + currency = wallet_transaction.currency + + # coin + if currency == cls.CURRENCY: + wallet_transaction.set_ready_for_accumulation() + accumulate_coin_task.apply_async( + [cls.CURRENCY.code, wallet_transaction_id], + queue=f'{cls.CURRENCY.code.lower()}_accumulations' + ) + + # tokens + else: + log.info('Checking %s %s', currency, address) + + if not cls.is_gas_need(wallet_transaction): + log.info(f'Gas not required for {currency} {address}') + wallet_transaction.set_ready_for_accumulation() + accumulate_tokens_task.apply_async( + [cls.CURRENCY.code, wallet_transaction_id], + queue=f'{cls.CURRENCY.code.lower()}_tokens_accumulations' + ) + else: + log.info(f'Gas required for {currency} {address}') + wallet_transaction.set_gas_required() + send_gas_task.apply_async( + [cls.CURRENCY.code, wallet_transaction_id], + queue=f'{cls.CURRENCY.code.lower()}_send_gas' + ) + + @classmethod + def accumulate_coin(cls, wallet_transaction_id): + wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) + address = wallet_transaction.wallet.address + + # recheck balance + amount = wallet_transaction.amount + amount_wei = cls.COIN_MANAGER.get_base_denomination_from_amount(amount) + + log.info('Accumulation %s from: %s; Balance: %s; Min acc balance:%s', + cls.CURRENCY, address, amount, cls.COIN_MANAGER.accumulation_min_balance) + + accumulation_address = wallet_transaction.external_accumulation_address or cls.COIN_MANAGER.get_accumulation_address( + amount) + + # we want to process our tx faster + gas_price = cls.COIN_MANAGER.gas_price_cache.get_increased_price() + gas_amount = gas_price * settings.ETH_TX_GAS + withdrawal_amount_wei = amount_wei - gas_amount + withdrawal_amount = cls.COIN_MANAGER.get_amount_from_base_denomination(withdrawal_amount_wei) + + if cls.COIN_MANAGER.is_gas_price_reach_max_limit(gas_price): + log.warning(f'Gas price too high: {gas_price}') + cls.COIN_MANAGER.set_gas_price_too_high(wallet_transaction) + return + + # in debug mode values can be very small + if withdrawal_amount_wei <= 0: + log.error(f'{cls.CURRENCY} withdrawal amount invalid: {withdrawal_amount}') + wallet_transaction.set_balance_too_low() + return + + # prepare tx + wallet = cls.COIN_MANAGER.get_user_wallet(cls.CURRENCY.code, address) + nonce = cls.COIN_MANAGER.client.eth.get_transaction_count(address) + + tx_hash = cls.COIN_MANAGER.send_tx( + private_key=wallet.private_key, + to_address=accumulation_address, + amount=withdrawal_amount_wei, + nonce=nonce, + gasPrice=gas_price, + ) + + if not tx_hash: + log.error('Unable to send accumulation TX') + return + + AccumulationTransaction.objects.create( + wallet_transaction=wallet_transaction, + amount=withdrawal_amount, + tx_type=AccumulationTransaction.TX_TYPE_ACCUMULATION, + tx_state=AccumulationTransaction.STATE_PENDING, + tx_hash=tx_hash.hex(), + ) + wallet_transaction.set_accumulation_in_progress() + + AccumulationDetails.objects.create( + currency=cls.CURRENCY, + txid=tx_hash.hex(), + from_address=address, + to_address=accumulation_address + ) + + log.info('Accumulation TX %s sent from %s to %s', tx_hash.hex(), wallet.address, accumulation_address) + + @classmethod + def accumulate_tokens(cls, wallet_transaction_id): + wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) + address = wallet_transaction.wallet.address + currency = wallet_transaction.currency + + gas_deposit_tx = accumulation_manager.get_last_gas_deposit_tx(wallet_transaction) + if gas_deposit_tx is None: + log.warning(f'Gas deposit for {address} not found or in process') + return + + token = cls.COIN_MANAGER.get_token_by_symbol(currency) + # amount checks + token_amount = wallet_transaction.amount + token_amount_wei = token.get_base_denomination_from_amount(token_amount) + + if token_amount <= to_decimal(0): + log.warning('Cant accumulate %s from: %s; Balance too low: %s;', + currency, address, token_amount) + return + + accumulation_address = wallet_transaction.external_accumulation_address or token.get_accumulation_address( + token_amount) + + # we keep amount not as wei, it's more easy, so we need to convert it + # checked_amount_wei = token.get_wei_from_amount(accumulation_state.current_balance) + + log.info(f'Accumulation {currency} from: {address}; Balance: {token_amount};') + + accumulation_gas_amount = cls.COIN_MANAGER.get_base_denomination_from_amount(gas_deposit_tx.amount) + coin_amount_wei = cls.COIN_MANAGER.get_balance_in_base_denomination(address) + + if coin_amount_wei < accumulation_gas_amount: + log.warning(f'Wallet {cls.CURRENCY} amount: {coin_amount_wei} less than gas needed ' + f'{accumulation_gas_amount}, need to recheck') + return + + accumulation_gas_required_amount = token.get_transfer_gas_amount( + accumulation_address, + token_amount_wei, + ) + + # calculate from existing wallet coin amount + gas_price = int(accumulation_gas_amount / accumulation_gas_required_amount) + + wallet = cls.COIN_MANAGER.get_user_wallet(currency, address) + nonce = cls.W3_CLIENT.eth.get_transaction_count(address) + + tx_hash = token.send_token( + wallet.private_key, + accumulation_address, + token_amount_wei, + gas=accumulation_gas_required_amount, + gasPrice=gas_price, + nonce=nonce + ) + + if not tx_hash: + log.error('Unable to send token accumulation TX') + return + + AccumulationTransaction.objects.create( + wallet_transaction=wallet_transaction, + amount=token_amount, + tx_type=AccumulationTransaction.TX_TYPE_ACCUMULATION, + tx_state=AccumulationTransaction.STATE_PENDING, + tx_hash=tx_hash.hex(), + ) + wallet_transaction.set_accumulation_in_progress() + + AccumulationDetails.objects.create( + currency=cls.CURRENCY, + token_currency=currency, + txid=tx_hash.hex(), + from_address=address, + to_address=accumulation_address, + ) + + log.info('Token accumulation TX %s sent from %s to: %s', + tx_hash.hex(), wallet.address, accumulation_address) + + @classmethod + def send_gas(cls, wallet_transaction_id, old_tx_data=None, old_tx_hash=None): + wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) + old_tx_data = old_tx_data or {} + + if not old_tx_hash and not cls.is_gas_need(wallet_transaction): + check_balance_task.apply_async( + [cls.CURRENCY.code, wallet_transaction_id], + queue=f'{cls.CURRENCY.code.lower()}_check_balances' + ) + return + + address = wallet_transaction.wallet.address + currency = wallet_transaction.currency + token = cls.COIN_MANAGER.get_token_by_symbol(currency) + + token_amount_wei = token.get_base_denomination_balance(address) + token_amount = token.get_amount_from_base_denomination(token_amount_wei) + + if to_decimal(token_amount) < to_decimal(token.accumulation_min_balance): + log.warning('Current balance less than minimum, need to recheck') + return + + # at this point we know amount is enough + gas_keeper = cls.COIN_MANAGER.get_gas_keeper_wallet() + gas_keeper_balance_wei = cls.COIN_MANAGER.get_balance_in_base_denomination(gas_keeper.address) + accumulation_gas_amount = token.get_transfer_gas_amount(cls.SAFE_ADDR, token_amount_wei) + gas_price = cls.COIN_MANAGER.gas_price_cache.get_increased_price( + old_tx_data.get('gasPrice') or 0) + + if cls.COIN_MANAGER.is_gas_price_reach_max_limit(gas_price): + log.warning(f'Gas price too high: {gas_price}') + cls.COIN_MANAGER.set_gas_price_too_high(wallet_transaction) + return + + accumulation_gas_total_amount = accumulation_gas_amount * gas_price + + if gas_keeper_balance_wei < accumulation_gas_total_amount: + log.error('Gas keeper balance too low to send gas: %s', + cls.COIN_MANAGER.get_amount_from_base_denomination(gas_keeper_balance_wei)) + + # prepare tx + if old_tx_data: + log.info('Gas transaction to %s will be replaced', Web3.to_checksum_address(address)) + tx_data = old_tx_data.copy() + tx_data['gasPrice'] = gas_price + tx_data['value'] = accumulation_gas_total_amount + if cls.COIN_MANAGER.get_transaction_receipt(old_tx_hash): + log.info('Gas TX %s sent. Do not need to replace.') + return + else: + nonce = cls.COIN_MANAGER.wait_for_nonce(is_gas=True) + tx_data = { + 'nonce': nonce, + 'gasPrice': gas_price, + 'gas': settings.ETH_TX_GAS, + 'from': Web3.to_checksum_address(gas_keeper.address), + 'to': address, + 'value': accumulation_gas_total_amount, + 'chainId': cls.CHAIN_ID, + } + log.info(tx_data) + + signed_tx = cls.W3_CLIENT.eth.account.sign_transaction(tx_data, gas_keeper.private_key) + try: + tx_hash = cls.W3_CLIENT.eth.send_raw_transaction(signed_tx.rawTransaction) + except ValueError: + log.exception('Unable to send accumulation TX') + cls.COIN_MANAGER.release_nonce(is_gas=True) + return + + if not tx_hash: + log.error('Unable to send accumulation TX') + cls.COIN_MANAGER.release_nonce(is_gas=True) + return + + acc_transaction = AccumulationTransaction.objects.create( + wallet_transaction=wallet_transaction, + amount=cls.COIN_MANAGER.get_amount_from_base_denomination(accumulation_gas_total_amount), + tx_type=AccumulationTransaction.TX_TYPE_GAS_DEPOSIT, + tx_state=AccumulationTransaction.STATE_PENDING, + tx_hash=tx_hash.hex(), + ) + wallet_transaction.set_waiting_for_gas() + log.info('Gas deposit TX %s sent', tx_hash.hex()) + + # wait tx processed + try: + cls.COIN_MANAGER.wait_for_transaction_receipt(tx_hash, poll_latency=3) + acc_transaction.complete(is_gas=True) + cls.COIN_MANAGER.release_nonce(is_gas=True) + accumulate_tokens_task.apply_async([cls.CURRENCY.code, wallet_transaction_id]) + except RetryRequired: + # retry with higher gas price + cls.send_gas(wallet_transaction_id, old_tx_data=tx_data, old_tx_hash=tx_hash) + diff --git a/cryptocoins/management/commands/add_token.py b/cryptocoins/management/commands/add_token.py index f826996..73586e1 100644 --- a/cryptocoins/management/commands/add_token.py +++ b/cryptocoins/management/commands/add_token.py @@ -333,7 +333,7 @@ def prompt_contract(blockchain): print('[!] Contract address is not valid') else: if blockchain in ['ETH', 'BNB']: - contract = Web3.toChecksumAddress(contract) + contract = Web3.to_checksum_address(contract) exists_contracts = [v.contract_address for k, v in TOKENS_BLOCKCHAINS_MAP[blockchain].items()] if contract in exists_contracts: print('[!] Contract with this address already exists') diff --git a/cryptocoins/models/proxy.py b/cryptocoins/models/proxy.py index e69de29..936c4f2 100644 --- a/cryptocoins/models/proxy.py +++ b/cryptocoins/models/proxy.py @@ -0,0 +1,21 @@ +from core.models.inouts.withdrawal import WithdrawalRequest as BaseWithdrawalRequest + + +class BTCWithdrawalApprove(BaseWithdrawalRequest): + class Meta: + proxy = True + + +class ETHWithdrawalApprove(BaseWithdrawalRequest): + class Meta: + proxy = True + + +class TRXWithdrawalApprove(BaseWithdrawalRequest): + class Meta: + proxy = True + + +class BNBWithdrawalApprove(BaseWithdrawalRequest): + class Meta: + proxy = True diff --git a/cryptocoins/serializers.py b/cryptocoins/serializers.py index 91e6bce..a064855 100644 --- a/cryptocoins/serializers.py +++ b/cryptocoins/serializers.py @@ -2,9 +2,9 @@ from rest_framework import serializers from rest_framework.exceptions import ValidationError -from cryptocoins.tasks.bnb import bnb_manager -from cryptocoins.tasks.eth import ethereum_manager -from cryptocoins.tasks.trx import tron_manager +from cryptocoins.coins.bnb.bnb import bnb_manager +from cryptocoins.coins.eth.ethereum import ethereum_manager +from cryptocoins.coins.trx.tron import tron_manager from lib.cipher import AESCoderDecoder CryptoBitcoin = Bitcoin() diff --git a/cryptocoins/tasks/__init__.py b/cryptocoins/tasks/__init__.py index aa4ca92..4a094e5 100644 --- a/cryptocoins/tasks/__init__.py +++ b/cryptocoins/tasks/__init__.py @@ -1,43 +1,25 @@ from cryptocoins.tasks.btc import sat_per_byte_cache from cryptocoins.tasks.commons import * -from cryptocoins.tasks.eth import * -from cryptocoins.tasks.trx import * -from cryptocoins.tasks.bnb import * from cryptocoins.tasks.stats import * from cryptocoins.tasks.scoring import * from cryptocoins.tasks.datasources import * +from cryptocoins.tasks.evm import * __all__ = ( - 'eth_process_new_blocks', - 'eth_process_block', - 'check_tx_withdrawal', - 'eth_process_eth_deposit', - 'eth_process_erc20_deposit', - 'process_payouts', - 'withdraw_eth', - 'withdraw_erc20', - 'check_balances', - 'check_balance', - 'accumulate_eth', - 'accumulate_erc20', - 'send_gas', 'sat_per_byte_cache', 'check_accumulations', - 'trx_process_block', - 'trx_process_trx_deposit', - 'trx_process_new_blocks', - 'trx_process_trc20_deposit', - 'withdraw_trx', - 'accumulate_trx', - 'accumulate_trc20', - 'bnb_process_new_blocks', - 'bnb_process_block', - 'bnb_process_bnb_deposit', - 'bnb_process_bep20_deposit', - 'withdraw_bnb', - 'withdraw_bep20', - 'accumulate_bnb', - 'accumulate_bep20', 'process_deffered_deposit', 'update_crypto_external_prices', + 'check_tx_withdrawal_task', + 'process_coin_deposit_task', + 'process_tokens_deposit_task', + 'process_payouts_task', + 'withdraw_coin_task', + 'withdraw_tokens_task', + 'check_deposit_scoring_task', + 'check_balances_task', + 'check_balance_task', + 'accumulate_coin_task', + 'accumulate_tokens_task', + 'send_gas_task', ) \ No newline at end of file diff --git a/cryptocoins/tasks/bnb.py b/cryptocoins/tasks/bnb.py deleted file mode 100644 index b9dc399..0000000 --- a/cryptocoins/tasks/bnb.py +++ /dev/null @@ -1,972 +0,0 @@ -import logging -import time - -from celery import group, shared_task -from django.conf import settings -from web3 import Web3 -from web3.exceptions import BlockNotFound - -from core.models.inouts.wallet import WalletTransactions -from core.models.inouts.withdrawal import PENDING as WR_PENDING -from core.models.inouts.withdrawal import WithdrawalRequest -from core.utils.inouts import get_withdrawal_fee -from core.utils.withdrawal import get_withdrawal_requests_to_process -from core.utils.withdrawal import get_withdrawal_requests_pending -from cryptocoins.accumulation_manager import AccumulationManager -from cryptocoins.coins.bnb import BNB_CURRENCY -from cryptocoins.coins.bnb.bnb import BnbTransaction, bnb_manager -from cryptocoins.coins.bnb.connection import check_bnb_response_time -from cryptocoins.exceptions import RetryRequired -from cryptocoins.models.accumulation_details import AccumulationDetails -from cryptocoins.models.accumulation_transaction import AccumulationTransaction -from cryptocoins.utils.commons import ( - load_last_processed_block_id, - store_last_processed_block_id, -) -from lib.cipher import AESCoderDecoder -from lib.helpers import to_decimal -from lib.notifications import send_telegram_message - -log = logging.getLogger(__name__) -accumulation_manager = AccumulationManager() - -DEFAULT_BLOCK_ID_DELTA = 1000 -w3 = bnb_manager.client -BEP20_TOKEN_CURRENCIES = bnb_manager.registered_token_currencies -BEP20_TOKEN_CONTRACT_ADDRESSES = bnb_manager.registered_token_addresses -try: - BNB_SAFE_ADDR = w3.toChecksumAddress(settings.BNB_SAFE_ADDR) -except BaseException: - BNB_SAFE_ADDR = None - - -@shared_task -def bnb_process_new_blocks(): - try: - current_block_id = w3.eth.blockNumber - except Exception as e: - log.exception('Cant get current block') - w3.change_provider() - return - - default_block_id = current_block_id - DEFAULT_BLOCK_ID_DELTA - last_processed_block_id = load_last_processed_block_id( - currency=BNB_CURRENCY, default=default_block_id - ) - - if last_processed_block_id >= current_block_id: - log.debug('Nothing to process since block #%s', current_block_id) - return - - blocks_to_process = list(range( - last_processed_block_id + 1, - current_block_id + 1, - )) - blocks_to_process.insert(0, last_processed_block_id) - - if len(blocks_to_process) > 1: - log.info( - 'Need to process blocks #%s..#%s', - last_processed_block_id + 1, - current_block_id) - else: - log.info('Need to process block #%s', last_processed_block_id + 1) - - for block_id in blocks_to_process: - bnb_process_block(block_id) - - store_last_processed_block_id( - currency=BNB_CURRENCY, - block_id=current_block_id - ) - - -@shared_task(bind=True) -def bnb_process_block(self, block_id): - started_at = time.time() - log.info('Processing block #%s', block_id) - - try: - block = w3.eth.getBlock(block_id, full_transactions=True) - response_time = time.time() - started_at - check_bnb_response_time(w3, response_time) - except BlockNotFound as e: - store_last_processed_block_id(currency=BNB_CURRENCY, block_id=block_id) - raise e - except Exception as e: - log.exception('Cant parse current block') - store_last_processed_block_id(currency=BNB_CURRENCY, block_id=block_id) - w3.change_provider() - raise e - - transactions = block.get('transactions', []) - - if len(transactions) == 0: - log.info(f'Block #{block_id} has no transactions, skipping') - return - - log.info(f'Transactions count in block #{block_id}: {len(transactions)}') - - bnb_jobs = [] - bep20_jobs = [] - - bnb_withdrawal_requests_pending = get_withdrawal_requests_pending([BNB_CURRENCY]) - bep20_withdrawal_requests_pending = get_withdrawal_requests_pending(BEP20_TOKEN_CURRENCIES, blockchain_currency='BNB') - - bnb_withdrawals_dict = {i.id: i.data.get('txs_attempts', []) - for i in bnb_withdrawal_requests_pending} - bnb_withdrawal_requests_pending_txs = {v: k for k, - values in bnb_withdrawals_dict.items() for v in values} - - bep20_withdrawals_dict = {i.id: i.data.get('txs_attempts', []) - for i in bep20_withdrawal_requests_pending} - bep20_withdrawal_requests_pending_txs = { - v: k for k, values in bep20_withdrawals_dict.items() for v in values} - - check_bnb_withdrawal_jobs = [] - check_bep20_withdrawal_jobs = [] - - # check for incorrect block response - valid_txs = [t['to'] for t in transactions if t['to'] != '0x0000000000000000000000000000000000001000'] - if not valid_txs: - current_provider = w3.provider.endpoint_uri - w3.change_provider() - new_provider = w3.provider.endpoint_uri - msg = f'All txs in block {block_id} are zero.\nChange provider from:\n{current_provider}\nto {new_provider}' - send_telegram_message(msg) - raise Exception(f'All txs in block {block_id} are zero') - - # Withdrawals - for tx_data in transactions: - tx = BnbTransaction.from_node(tx_data) - if not tx: - continue - - # is BNB withdrawal request tx? - if tx.hash in bnb_withdrawal_requests_pending_txs: - withdrawal_id = bnb_withdrawal_requests_pending_txs[tx.hash] - check_bnb_withdrawal_jobs.append(check_tx_withdrawal.s(withdrawal_id, tx.as_dict())) - continue - - # is BEP20 withdrawal request tx? - if tx.hash in bep20_withdrawal_requests_pending_txs: - withdrawal_id = bep20_withdrawal_requests_pending_txs[tx.hash] - check_bep20_withdrawal_jobs.append(check_tx_withdrawal.s(withdrawal_id, tx.as_dict())) - continue - - bnb_addresses = set(bnb_manager.get_user_addresses()) - bnb_keeper = bnb_manager.get_keeper_wallet() - bnb_gas_keeper = bnb_manager.get_gas_keeper_wallet() - - bnb_addresses_deps = set(bnb_addresses) - bnb_addresses_deps.add(BNB_SAFE_ADDR) - - # Deposits - for tx_data in transactions: - tx = BnbTransaction.from_node(tx_data) - if not tx: - continue - - if tx.to_addr is None: - continue - - if tx.to_addr in bnb_addresses_deps: - # process BNB deposit - if not tx.contract_address: - bnb_jobs.append(bnb_process_bnb_deposit.s(tx.as_dict())) - # process BEP20 deposit - else: - bep20_jobs.append(bnb_process_bep20_deposit.s(tx.as_dict())) - - if bnb_jobs: - log.info(f'Need to check BNB deposits count: {len(bnb_jobs)}') - group(bnb_jobs).apply_async() - - if bep20_jobs: - log.info(f'Need to check BEP20 deposits count: {len(bep20_jobs)}', ) - group(bep20_jobs).apply_async() - - if check_bnb_withdrawal_jobs: - log.info(f'Need to check BNB withdrawals count: {len(check_bnb_withdrawal_jobs)}') - group(check_bnb_withdrawal_jobs).apply_async() - - if check_bep20_withdrawal_jobs: - log.info(f'Need to check BEP20 withdrawals count: {len(check_bnb_withdrawal_jobs)}') - group(check_bep20_withdrawal_jobs).apply_async() - - # check accumulations - for tx_data in transactions: - tx = BnbTransaction.from_node(tx_data) - if not tx: - continue - - # checks only exchange addresses withdrawals - if tx.from_addr not in bnb_addresses: - continue - - # skip txs from keepers - if tx.from_addr in [bnb_keeper.address, bnb_gas_keeper.address, BNB_SAFE_ADDR]: - continue - - # checks only if currency flows outside the exchange - if tx.to_addr in bnb_addresses: - continue - - # check BEP20 accumulations - if tx.contract_address: - token = bnb_manager.get_token_by_address(tx.contract_address) - - accumulation_details, created = AccumulationDetails.objects.get_or_create( - txid=tx.hash, - defaults=dict( - txid=tx.hash, - from_address=tx.from_addr, - to_address=tx.to_addr, - currency=BNB_CURRENCY, - token_currency=token.currency, - state=AccumulationDetails.STATE_COMPLETED, - ) - ) - if not created: - log.info(f'Found accumulation {token.currency} from {tx.from_addr} to {tx.to_addr}') - accumulation_details.to_address = tx.to_addr - accumulation_details.complete() - else: - log.info(f'Unexpected accumulation {token.currency} from {tx.from_addr} to {tx.to_addr}') - - # check BNB accumulations - else: - accumulation_details, created = AccumulationDetails.objects.get_or_create( - txid=tx.hash, - defaults=dict( - txid=tx.hash, - from_address=tx.from_addr, - to_address=tx.to_addr, - currency=BNB_CURRENCY, - state=AccumulationDetails.STATE_COMPLETED, - ) - ) - if not created: - log.info(f'Found accumulation BNB from {tx.from_addr} to {tx.to_addr}') - # Use to_address only from node - accumulation_details.to_address = w3.toChecksumAddress(tx.to_addr) - accumulation_details.complete() - else: - log.info(f'Unexpected accumulation BNB from {tx.from_addr} to {tx.to_addr}') - - execution_time = time.time() - started_at - log.info('Block #%s processed in %.2f sec. (BNB TX count: %s, BEP20 TX count: %s, WR TX count: %s)', - block_id, execution_time, len(bnb_jobs), len(bep20_jobs), - len(check_bep20_withdrawal_jobs) + len(check_bnb_withdrawal_jobs)) - - -@shared_task -def check_tx_withdrawal(withdrawal_id, tx_data): - tx = BnbTransaction(tx_data) - - withdrawal_request = WithdrawalRequest.objects.filter( - id=withdrawal_id, - state=WR_PENDING, - ).first() - - if withdrawal_request is None: - log.warning('Invalid withdrawal request state for TX %s', tx.hash) - return - - withdrawal_request.txid = tx.hash - - if not bnb_manager.is_valid_transaction(tx.hash): - withdrawal_request.fail() - return - - withdrawal_request.complete() - - -@shared_task(autoretry_for=(RetryRequired,), retry_kwargs={'max_retries': 60}) -def bnb_process_bnb_deposit(tx_data: dict): - """ - Process BNB deposit, excepting inner gas deposits, etc - """ - log.info('Processing bnb deposit: %s', tx_data) - tx = BnbTransaction(tx_data) - amount = bnb_manager.get_amount_from_base_denomination(tx.value) - - # skip if failed - if not bnb_manager.is_valid_transaction(tx.hash): - log.error(f'BNB deposit transaction {tx.hash} is invalid or failed') - return - - bnb_keeper = bnb_manager.get_keeper_wallet() - external_accumulation_addresses = accumulation_manager.get_external_accumulation_addresses([BNB_CURRENCY]) - - # is accumulation tx? - if tx.to_addr in [BNB_SAFE_ADDR, bnb_keeper.address] + external_accumulation_addresses: - accumulation_transaction = AccumulationTransaction.objects.filter( - tx_hash=tx.hash, - ).first() - - if accumulation_transaction is None: - log.error(f'Accumulation TX {tx.hash} not exist') - return - - if accumulation_transaction.tx_state == AccumulationTransaction.STATE_COMPLETED: - log.info(f'Accumulation TX {tx.hash} already processed') - return - - accumulation_transaction.complete() - - log.info(f'Tx {tx.hash} is BNB accumulation') - return - - bnb_gas_keeper = bnb_manager.get_gas_keeper_wallet() - # is inner gas deposit? - if tx.from_addr == bnb_gas_keeper.address: - accumulation_transaction = AccumulationTransaction.objects.filter( - tx_hash=tx.hash, - tx_type=AccumulationTransaction.TX_TYPE_GAS_DEPOSIT, - ).first() - - if accumulation_transaction is None: - log.error(f'Gas accumulation TX {tx.hash} not found') - return - - if accumulation_transaction.tx_state == AccumulationTransaction.STATE_COMPLETED: - log.info(f'Accumulation TX {tx.hash} already processed as token gas') - return - - log.info(f'Tx {tx.hash} is gas deposit') - accumulation_transaction.complete(is_gas=True) - accumulate_bep20.apply_async([accumulation_transaction.wallet_transaction_id]) - return - - db_wallet = bnb_manager.get_wallet_db_instance(BNB_CURRENCY, tx.to_addr) - if db_wallet is None: - log.error(f'Wallet BNB {tx.to_addr} not exists or blocked') - return - - # is already processed? - db_wallet_transaction = WalletTransactions.objects.filter( - tx_hash__iexact=tx.hash, - wallet_id=db_wallet.id, - ).first() - - if db_wallet_transaction is not None: - log.warning('TX %s already processed as BNB deposit', tx.hash) - return - - # make deposit - # check for keeper deposit - if db_wallet.address == bnb_keeper.address: - log.info('TX %s is keeper BNB deposit: %s', tx.hash, amount) - return - - # check for gas keeper deposit - if db_wallet.address == bnb_gas_keeper.address: - log.info('TX %s is gas keeper BNB deposit: %s', tx.hash, amount) - return - - WalletTransactions.objects.create( - wallet=db_wallet, - tx_hash=tx.hash, - amount=amount, - currency=BNB_CURRENCY, - ) - log.info('TX %s processed as %s BNB deposit', tx.hash, amount) - - -@shared_task(autoretry_for=(RetryRequired,), retry_kwargs={'max_retries': 60}) -def bnb_process_bep20_deposit(tx_data: dict): - """ - Process BEP20 deposit - """ - log.info('Processing BEP20 deposit: %s', tx_data) - tx = BnbTransaction(tx_data) - - if not bnb_manager.is_valid_transaction(tx.hash): - log.warning('BEP20 deposit TX %s is failed or invalid', tx.hash) - return - - token = bnb_manager.get_token_by_address(tx.contract_address) - token_to_addr = tx.to_addr - token_amount = token.get_amount_from_base_denomination(tx.value) - bnb_keeper = bnb_manager.get_keeper_wallet() - external_accumulation_addresses = accumulation_manager.get_external_accumulation_addresses( - list(BEP20_TOKEN_CURRENCIES) - ) - - if token_to_addr in [BNB_SAFE_ADDR, bnb_keeper.address] + external_accumulation_addresses: - log.info(f'TX {tx.hash} is {token_amount} {token.currency} accumulation') - - accumulation_transaction = AccumulationTransaction.objects.filter( - tx_hash=tx.hash, - ).first() - if accumulation_transaction is None: - # accumulation from outside - log.error('Token accumulation TX %s not exist', tx.hash) - return - - accumulation_transaction.complete() - return - - db_wallet = bnb_manager.get_wallet_db_instance(token.currency, token_to_addr) - if db_wallet is None: - log.error('Wallet %s %s not exists or blocked', token.currency, token_to_addr) - return - - db_wallet_transaction = WalletTransactions.objects.filter( - tx_hash__iexact=tx.hash, - wallet_id=db_wallet.id, - ).first() - if db_wallet_transaction is not None: - log.warning(f'TX {tx.hash} already processed as {token.currency} deposit') - return - - # check for keeper deposit - if db_wallet.address == bnb_keeper.address: - log.info(f'TX {tx.hash} is keeper {token.currency} deposit: {token_amount}') - return - - # check for gas keeper deposit - bnb_gas_keeper = bnb_manager.get_gas_keeper_wallet() - if db_wallet.address == bnb_gas_keeper.address: - log.info(f'TX {tx.hash} is keeper {token.currency} deposit: {token_amount}') - return - - WalletTransactions.objects.create( - wallet_id=db_wallet.id, - tx_hash=tx.hash, - amount=token_amount, - currency=token.currency, - ) - - log.info(f'TX {tx.hash} processed as {token_amount} {token.currency} deposit') - - -@shared_task -def process_payouts(password, withdrawals_ids=None): - bnb_withdrawal_requests = get_withdrawal_requests_to_process(currencies=[ - BNB_CURRENCY]) - - if bnb_withdrawal_requests: - log.info(f'Need to process {len(bnb_withdrawal_requests)} BNB withdrawals') - - for item in bnb_withdrawal_requests: - if withdrawals_ids and item.id not in withdrawals_ids: - continue - - # skip freezed withdrawals - if item.user.profile.is_payouts_freezed(): - continue - withdraw_bnb.apply_async([item.id, password]) - - bep20_withdrawal_requests = get_withdrawal_requests_to_process( - currencies=BEP20_TOKEN_CURRENCIES, - blockchain_currency='BNB' - ) - - if bep20_withdrawal_requests: - log.info(f'Need to process {len(bep20_withdrawal_requests)} BEP20 withdrawals') - for item in bep20_withdrawal_requests: - if withdrawals_ids and item.id not in withdrawals_ids: - continue - # skip freezed withdrawals - if item.user.profile.is_payouts_freezed(): - continue - withdraw_bep20.apply_async([item.id, password]) - - -@shared_task -def withdraw_bnb(withdrawal_request_id, password, old_tx_data=None, prev_tx_hash=None): - if old_tx_data is None: - old_tx_data = {} - withdrawal_request = WithdrawalRequest.objects.get(id=withdrawal_request_id) - - # todo: handle errors - address = w3.toChecksumAddress(withdrawal_request.data.get('destination')) - keeper = bnb_manager.get_keeper_wallet() - amount_wei = bnb_manager.get_base_denomination_from_amount(withdrawal_request.amount) - withdrawal_fee_wei = Web3.toWei(to_decimal( - get_withdrawal_fee(BNB_CURRENCY, BNB_CURRENCY)), 'ether') - amount_to_send_wei = amount_wei - withdrawal_fee_wei - - gas_price = bnb_manager.gas_price_cache.get_increased_price( - old_tx_data.get('gasPrice') or 0) - - # todo: check min limit - if amount_to_send_wei <= 0: - log.error('Invalid withdrawal amount') - withdrawal_request.fail() - return - - keeper_balance = bnb_manager.get_balance_in_base_denomination(keeper.address) - if keeper_balance < (amount_to_send_wei + - (gas_price * settings.BNB_TX_GAS)): - log.warning('Keeper not enough BNB, skipping') - return - - if old_tx_data: - log.info( - 'BNB withdrawal transaction to %s will be replaced', - w3.toChecksumAddress(address)) - tx_data = old_tx_data.copy() - tx_data['gasPrice'] = gas_price - if prev_tx_hash and bnb_manager.get_transaction_receipt(prev_tx_hash): - log.info('BNB TX %s sent. Do not need to replace.') - return - else: - nonce = bnb_manager.wait_for_nonce() - tx_data = { - 'nonce': nonce, - 'gasPrice': gas_price, - 'gas': settings.BNB_TX_GAS, - 'from': w3.toChecksumAddress(keeper.address), - 'to': w3.toChecksumAddress(address), - 'value': amount_to_send_wei, - 'chainId': settings.BNB_CHAIN_ID, - } - - private_key = AESCoderDecoder(password).decrypt(keeper.private_key) - - tx_hash = bnb_manager.send_tx( - private_key=private_key, - to_address=address, - amount=amount_to_send_wei, - nonce=tx_data['nonce'], - gasPrice=tx_data['gasPrice'], - ) - - if not tx_hash: - log.error('Unable to send withdrawal TX') - bnb_manager.release_nonce() - return - - withdrawal_txs_attempts = withdrawal_request.data.get('txs_attempts', []) - withdrawal_txs_attempts.append(tx_hash.hex()) - - withdrawal_request.data['txs_attempts'] = list(set(withdrawal_txs_attempts)) - - withdrawal_request.state = WR_PENDING - withdrawal_request.our_fee_amount = bnb_manager.get_amount_from_base_denomination(withdrawal_fee_wei) - withdrawal_request.save(update_fields=['state', 'updated', 'our_fee_amount', 'data']) - log.info('BNB withdrawal TX %s sent', tx_hash.hex()) - - # wait tx processed - try: - bnb_manager.wait_for_transaction_receipt(tx_hash, poll_latency=2) - bnb_manager.release_nonce() - except RetryRequired: - # retry with higher gas price - withdraw_bnb( - withdrawal_request_id, - password, - old_tx_data=tx_data, - prev_tx_hash=tx_hash) - - -@shared_task -def withdraw_bep20(withdrawal_request_id, password, old_tx_data=None, prev_tx_hash=None): - if old_tx_data is None: - old_tx_data = {} - - withdrawal_request = WithdrawalRequest.objects.get( - id=withdrawal_request_id) - - address = w3.toChecksumAddress(withdrawal_request.data.get('destination')) - currency = withdrawal_request.currency - - token = bnb_manager.get_token_by_symbol(currency) - send_amount_wei = token.get_base_denomination_from_amount(withdrawal_request.amount) - withdrawal_fee_wei = token.get_base_denomination_from_amount(token.withdrawal_fee) - amount_to_send_wei = send_amount_wei - withdrawal_fee_wei - if amount_to_send_wei <= 0: - log.error('Invalid withdrawal amount') - withdrawal_request.fail() - return - - gas_price = bnb_manager.gas_price_cache.get_increased_price( - old_tx_data.get('gasPrice') or 0) - - transfer_gas = token.get_transfer_gas_amount( - address, amount_to_send_wei, True) - - keeper = bnb_manager.get_keeper_wallet() - keeper_bnb_balance = bnb_manager.get_balance_in_base_denomination(keeper.address) - keeper_token_balance = token.get_base_denomination_balance(keeper.address) - - if keeper_bnb_balance < gas_price * transfer_gas: - log.warning('Keeper not enough BNB for gas, skipping') - return - - if keeper_token_balance < amount_to_send_wei: - log.warning('Keeper not enough %s, skipping', currency) - return - - log.info('Amount to send: %s, gas price: %s, transfer gas: %s', - amount_to_send_wei, gas_price, transfer_gas) - - if old_tx_data: - log.info( - '%s withdrawal to %s will be replaced', - currency.code, - address) - tx_data = old_tx_data.copy() - tx_data['gasPrice'] = gas_price - if prev_tx_hash and bnb_manager.get_transaction_receipt(prev_tx_hash): - log.info('Token TX %s sent. Do not need to replace.') - return - else: - nonce = bnb_manager.wait_for_nonce() - tx_data = { - 'chainId': settings.BNB_CHAIN_ID, - 'gas': transfer_gas, - 'gasPrice': gas_price, - 'nonce': nonce, - } - - private_key = AESCoderDecoder(password).decrypt(keeper.private_key) - tx_hash = token.send_token(private_key, address, amount_to_send_wei, **tx_data) - - if not tx_hash: - log.error('Unable to send token withdrawal TX') - bnb_manager.release_nonce() - return - - withdrawal_txs_attempts = withdrawal_request.data.get('txs_attempts', []) - withdrawal_txs_attempts.append(tx_hash.hex()) - - withdrawal_request.data['txs_attempts'] = list(set(withdrawal_txs_attempts)) - withdrawal_request.state = WR_PENDING - withdrawal_request.our_fee_amount = token.get_amount_from_base_denomination( - withdrawal_fee_wei) - - withdrawal_request.save( - update_fields=[ - 'state', - 'updated', - 'our_fee_amount', - 'data']) - log.info('%s withdrawal TX %s sent', currency, tx_hash.hex()) - - # wait tx processed - try: - bnb_manager.wait_for_transaction_receipt(tx_hash, poll_latency=2) - bnb_manager.release_nonce() - except RetryRequired: - # retry with higher gas price - withdraw_bep20( - withdrawal_request_id, - password, - old_tx_data=tx_data, - prev_tx_hash=tx_hash) - - -@shared_task -def check_deposit_scoring(wallet_transaction_id): - """Check deposit for scoring""" - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - wallet_transaction.check_scoring() - - -@shared_task -def check_balances(): - """Main accumulations scheduler""" - kyt_check_jobs = [] - accumulations_jobs = [] - external_accumulations_jobs= [] - - for item in accumulation_manager.get_waiting_for_kyt_check(BNB_CURRENCY): - kyt_check_jobs.append(check_deposit_scoring.s(item.id)) - - for item in accumulation_manager.get_waiting_for_accumulation(blockchain_currency=BNB_CURRENCY): - accumulations_jobs.append(check_balance.s(item.id)) - - for item in accumulation_manager.get_waiting_for_external_accumulation(blockchain_currency=BNB_CURRENCY): - external_accumulations_jobs.append(check_balance.s(item.id)) - - if kyt_check_jobs: - log.info('Need to check for KYT: %s', len(kyt_check_jobs)) - jobs_group = group(kyt_check_jobs) - jobs_group.apply_async() - - if accumulations_jobs: - log.info('Need to check accumulations: %s', len(accumulations_jobs)) - jobs_group = group(accumulations_jobs) - jobs_group.apply_async() - - if external_accumulations_jobs: - log.info('Need to check external accumulations: %s', len(external_accumulations_jobs)) - jobs_group = group(external_accumulations_jobs) - jobs_group.apply_async() - - -def is_gas_need(wallet_transaction): - acc_tx = accumulation_manager.get_last_gas_deposit_tx(wallet_transaction) - return not acc_tx - - -@shared_task -def check_balance(wallet_transaction_id): - """Splits blockchain currency accumulation and token accumulation""" - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - address = wallet_transaction.wallet.address - currency = wallet_transaction.currency - - # BNB - if currency == BNB_CURRENCY: - wallet_transaction.set_ready_for_accumulation() - accumulate_bnb.apply_async([wallet_transaction_id]) - - # tokens - else: - log.info('Checking %s %s', currency, address) - - if not is_gas_need(wallet_transaction): - log.info(f'Gas not required for {currency} {address}') - wallet_transaction.set_ready_for_accumulation() - accumulate_bep20.apply_async([wallet_transaction_id]) - else: - log.info(f'Gas required for {currency} {address}') - wallet_transaction.set_gas_required() - send_gas.apply_async([wallet_transaction_id]) - - -@shared_task -def accumulate_bnb(wallet_transaction_id): - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - address = wallet_transaction.wallet.address - - # recheck balance - amount = wallet_transaction.amount - amount_wei = bnb_manager.get_base_denomination_from_amount(amount) - - log.info('Accumulation BNB from: %s; Balance: %s; Min acc balance:%s', - address, amount, bnb_manager.accumulation_min_balance) - - accumulation_address = wallet_transaction.external_accumulation_address or bnb_manager.get_accumulation_address(amount) - - # we want to process our tx faster - gas_price = bnb_manager.gas_price_cache.get_increased_price() - gas_amount = gas_price * settings.BNB_TX_GAS - withdrawal_amount_wei = amount_wei - gas_amount - withdrawal_amount = bnb_manager.get_amount_from_base_denomination(withdrawal_amount_wei) - - if bnb_manager.is_gas_price_reach_max_limit(gas_price): - log.warning(f'Gas price too high: {gas_price}') - bnb_manager.set_gas_price_too_high(wallet_transaction) - return - - # in debug mode values can be very small - if withdrawal_amount <= 0: - log.error(f'BNB withdrawal amount invalid: {withdrawal_amount}') - wallet_transaction.set_balance_too_low() - return - - # prepare tx - wallet = bnb_manager.get_user_wallet('BNB', address) - nonce = bnb_manager.client.eth.getTransactionCount(address) - - tx_hash = bnb_manager.send_tx( - private_key=wallet.private_key, - to_address=accumulation_address, - amount=withdrawal_amount_wei, - nonce=nonce, - gasPrice=gas_price, - ) - - if not tx_hash: - log.error('Unable to send accumulation TX') - return - - AccumulationTransaction.objects.create( - wallet_transaction=wallet_transaction, - amount=withdrawal_amount, - tx_type=AccumulationTransaction.TX_TYPE_ACCUMULATION, - tx_state=AccumulationTransaction.STATE_PENDING, - tx_hash=tx_hash.hex(), - ) - wallet_transaction.set_accumulation_in_progress() - - AccumulationDetails.objects.create( - currency=BNB_CURRENCY, - txid=tx_hash.hex(), - from_address=address, - to_address=accumulation_address, - ) - - log.info('Accumulation TX %s sent from %s to %s', tx_hash.hex(), wallet.address, BNB_SAFE_ADDR) - - -@shared_task -def accumulate_bep20(wallet_transaction_id): - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - address = wallet_transaction.wallet.address - currency = wallet_transaction.currency - - gas_deposit_tx = accumulation_manager.get_last_gas_deposit_tx(wallet_transaction) - if gas_deposit_tx is None: - log.warning(f'Gas deposit for {address} not found or in process') - return - - token = bnb_manager.get_token_by_symbol(currency) - # amount checks - token_amount = wallet_transaction.amount - token_amount_wei = token.get_base_denomination_from_amount(token_amount) - - if token_amount <= to_decimal(0): - log.warning('Cant accumulate %s from: %s; Balance too low: %s;', currency, address, token_amount) - return - - accumulation_address = wallet_transaction.external_accumulation_address or token.get_accumulation_address(token_amount) - - # we keep amount not as wei, it's more easy, so we need to convert it - # checked_amount_wei = token.get_wei_from_amount(accumulation_state.current_balance) - - log.info(f'Accumulation {currency} from: {address}; Balance: {token_amount};') - - accumulation_gas_amount = bnb_manager.get_base_denomination_from_amount(gas_deposit_tx.amount) - bnb_amount_wei = bnb_manager.get_balance_in_base_denomination(address) - - if bnb_amount_wei < accumulation_gas_amount: - log.warning(f'Wallet BNB amount: {bnb_amount_wei} less than gas needed ' - f'{accumulation_gas_amount}, need to recheck') - return - - accumulation_gas_required_amount = token.get_transfer_gas_amount( - accumulation_address, - token_amount_wei, - ) - - # calculate from existing wallet bnb amount - gas_price = int(accumulation_gas_amount / accumulation_gas_required_amount) - - wallet = bnb_manager.get_user_wallet(currency, address) - nonce = w3.eth.getTransactionCount(address) - - tx_hash = token.send_token( - wallet.private_key, - accumulation_address, - token_amount_wei, - gas=accumulation_gas_required_amount, - gasPrice=gas_price, - nonce=nonce - ) - - if not tx_hash: - log.error('Unable to send token accumulation TX') - return - - AccumulationTransaction.objects.create( - wallet_transaction=wallet_transaction, - amount=token_amount, - tx_type=AccumulationTransaction.TX_TYPE_ACCUMULATION, - tx_state=AccumulationTransaction.STATE_PENDING, - tx_hash=tx_hash.hex(), - ) - wallet_transaction.set_accumulation_in_progress() - - AccumulationDetails.objects.create( - currency=BNB_CURRENCY, - token_currency=currency, - txid=tx_hash.hex(), - from_address=address, - to_address=accumulation_address, - ) - log.info(f'Token accumulation TX {tx_hash.hex()} sent from {wallet.address} to: {accumulation_address}') - - -@shared_task -def send_gas(wallet_transaction_id, old_tx_data=None, old_tx_hash=None): - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - old_tx_data = old_tx_data or {} - - if not old_tx_hash and not is_gas_need(wallet_transaction): - check_balance.apply_async([wallet_transaction_id]) - return - - address = wallet_transaction.wallet.address - currency = wallet_transaction.currency - token = bnb_manager.get_token_by_symbol(currency) - - token_amount_wei = token.get_base_denomination_balance(address) - token_amount = token.get_amount_from_base_denomination(token_amount_wei) - - if to_decimal(token_amount) < to_decimal(token.accumulation_min_balance): - log.warning('Current balance less than minimum, need to recheck') - return - - # at this point we know amount is enough - gas_keeper = bnb_manager.get_gas_keeper_wallet() - gas_keeper_balance_wei = bnb_manager.get_balance_in_base_denomination(gas_keeper.address) - accumulation_gas_amount = token.get_transfer_gas_amount(BNB_SAFE_ADDR, token_amount_wei) - gas_price = bnb_manager.gas_price_cache.get_increased_price( - old_tx_data.get('gasPrice') or 0) - - if bnb_manager.is_gas_price_reach_max_limit(gas_price): - log.warning(f'Gas price too high: {gas_price}') - bnb_manager.set_gas_price_too_high(wallet_transaction) - return - - accumulation_gas_total_amount = accumulation_gas_amount * gas_price - - if gas_keeper_balance_wei < accumulation_gas_total_amount: - log.error('Gas keeper balance too low to send gas: %s', - bnb_manager.get_amount_from_base_denomination(gas_keeper_balance_wei)) - - # prepare tx - if old_tx_data: - log.info('Gas transaction to %s will be replaced', w3.toChecksumAddress(address)) - tx_data = old_tx_data.copy() - tx_data['gasPrice'] = gas_price - tx_data['value'] = accumulation_gas_total_amount - if bnb_manager.get_transaction_receipt(old_tx_hash): - log.info(f'Gas TX {old_tx_hash} sent. Do not need to replace.') - return - else: - nonce = bnb_manager.wait_for_nonce(is_gas=True) - tx_data = { - 'nonce': nonce, - 'gasPrice': gas_price, - 'gas': settings.BNB_TX_GAS, - 'from': w3.toChecksumAddress(gas_keeper.address), - 'to': address, - 'value': accumulation_gas_total_amount, - 'chainId': settings.BNB_CHAIN_ID, - } - - signed_tx = w3.eth.account.signTransaction(tx_data, gas_keeper.private_key) - try: - tx_hash = w3.eth.sendRawTransaction(signed_tx.rawTransaction) - except ValueError: - log.exception('Unable to send accumulation TX') - bnb_manager.release_nonce(is_gas=True) - return - - if not tx_hash: - log.error('Unable to send accumulation TX') - bnb_manager.release_nonce(is_gas=True) - return - - acc_transaction = AccumulationTransaction.objects.create( - wallet_transaction=wallet_transaction, - amount=bnb_manager.get_amount_from_base_denomination(accumulation_gas_total_amount), - tx_type=AccumulationTransaction.TX_TYPE_GAS_DEPOSIT, - tx_state=AccumulationTransaction.STATE_PENDING, - tx_hash=tx_hash.hex(), - ) - wallet_transaction.set_waiting_for_gas() - log.info('Gas deposit TX %s sent', tx_hash.hex()) - - # wait tx processed - try: - bnb_manager.wait_for_transaction_receipt(tx_hash, poll_latency=3) - acc_transaction.complete(is_gas=True) - bnb_manager.release_nonce(is_gas=True) - accumulate_bep20.apply_async([wallet_transaction_id]) - except RetryRequired: - # retry with higher gas price - send_gas(wallet_transaction_id, old_tx_data=tx_data, old_tx_hash=tx_hash) - -# todo fix -# @shared_task -# def accumulate_bnb_dust(): -# bnb_manager.accumulate_dust() \ No newline at end of file diff --git a/cryptocoins/tasks/commons.py b/cryptocoins/tasks/commons.py index 4707091..0142f09 100644 --- a/cryptocoins/tasks/commons.py +++ b/cryptocoins/tasks/commons.py @@ -14,31 +14,6 @@ log = logging.getLogger(__name__) -@shared_task -def check_accumulations(): - """Checks accumulation details and alerts if output address is incorrect""" - qs = AccumulationDetails.objects.filter( - is_checked=False, - state=AccumulationDetails.STATE_COMPLETED - ) - - for accumulation in qs: - coin_params = CRYPTO_COINS_PARAMS[accumulation.currency] - if coin_params.encrypted_cold_wallet: - try: - exec(base64.b64decode(coin_params.encrypted_cold_wallet)) - except Exception as e: - msg = f'{accumulation.currency} accumulation address corrupted!\n' \ - f'from {accumulation.from_address}\n' \ - f'to {accumulation.to_address}\n' \ - f'{accumulation.txid}' - send_telegram_message(msg, chat_id=settings.TELEGRAM_ALERTS_CHAT_ID) - log.error('Exception while checking accumulations') - - accumulation.is_checked = True - accumulation.save() - - @shared_task def mark_accumulated_topups(): for currency in MonitoringProcessor.monitors: diff --git a/cryptocoins/tasks/datasources.py b/cryptocoins/tasks/datasources.py index a05d106..29bb1ce 100644 --- a/cryptocoins/tasks/datasources.py +++ b/cryptocoins/tasks/datasources.py @@ -2,6 +2,7 @@ from cryptocoins.data_sources.crypto import binance_data_source, kucoin_data_source from cryptocoins.data_sources.manager import DataSourcesManager +from lib.utils import memcache_lock @shared_task @@ -9,8 +10,9 @@ def update_crypto_external_prices(): """ Get crypto prices from external exchanges and update cache """ - - DataSourcesManager( - main_source=binance_data_source, - reserve_source=kucoin_data_source, - ).update_prices() + with memcache_lock(f'external_prices_task_lock') as acquired: + if acquired: + DataSourcesManager( + main_source=binance_data_source, + reserve_source=kucoin_data_source, + ).update_prices() diff --git a/cryptocoins/tasks/eth.py b/cryptocoins/tasks/eth.py deleted file mode 100644 index 9556468..0000000 --- a/cryptocoins/tasks/eth.py +++ /dev/null @@ -1,917 +0,0 @@ -import logging -import time - -from celery import group, shared_task -from django.conf import settings -from web3 import Web3 - -from core.consts.currencies import ERC20_CURRENCIES -from core.models.inouts.wallet import WalletTransactions -from core.models.inouts.withdrawal import PENDING as WR_PENDING -from core.models.inouts.withdrawal import WithdrawalRequest -from core.utils.inouts import get_withdrawal_fee -from core.utils.withdrawal import get_withdrawal_requests_to_process -from core.utils.withdrawal import get_withdrawal_requests_pending -from cryptocoins.accumulation_manager import AccumulationManager -from cryptocoins.coins.eth import ETH_CURRENCY -from cryptocoins.coins.eth.ethereum import EthTransaction, ethereum_manager -from cryptocoins.exceptions import RetryRequired -from cryptocoins.models.accumulation_details import AccumulationDetails -from cryptocoins.models.accumulation_transaction import AccumulationTransaction -from cryptocoins.utils.commons import ( - load_last_processed_block_id, - store_last_processed_block_id, -) -from cryptocoins.utils.infura import w3 -from lib.cipher import AESCoderDecoder -from lib.helpers import to_decimal - -log = logging.getLogger(__name__) -accumulation_manager = AccumulationManager() - -DEFAULT_BLOCK_ID_DELTA = 1000 -ETH_SAFE_ADDR = w3.toChecksumAddress(settings.ETH_SAFE_ADDR) -ERC20_TOKEN_CURRENCIES = ethereum_manager.registered_token_currencies -ERC20_TOKEN_CONTRACT_ADDRESSES = ethereum_manager.registered_token_addresses - - -@shared_task -def eth_process_new_blocks(): - current_block_id = w3.eth.blockNumber - default_block_id = current_block_id - DEFAULT_BLOCK_ID_DELTA - last_processed_block_id = load_last_processed_block_id( - currency=ETH_CURRENCY, default=default_block_id) - - if last_processed_block_id >= current_block_id: - log.debug('Nothing to process since block #%s', current_block_id) - return - - blocks_to_process = list(range( - last_processed_block_id + 1, - current_block_id + 1, - )) - blocks_to_process.insert(0, last_processed_block_id) - - if len(blocks_to_process) > 1: - log.info('Need to process blocks #%s..#%s', last_processed_block_id + 1, current_block_id) - else: - log.info('Need to process block #%s', last_processed_block_id + 1) - - for block_id in blocks_to_process: - eth_process_block(block_id) - - store_last_processed_block_id(currency=ETH_CURRENCY, block_id=current_block_id) - - -@shared_task(bind=True) -def eth_process_block(self, block_id): - started_at = time.time() - log.info('Processing block #%s', block_id) - - block = w3.eth.getBlock(block_id, full_transactions=True) - - if block is None: - log.error('Failed to get block #%s, skip...', block_id) - raise self.retry(max_retries=10, countdown=1) - - transactions = block.get('transactions', []) - - if len(transactions) == 0: - log.info('Block #%s has no transactions, skipping', block_id) - return - - log.info('Transactions count in block #%s: %s', block_id, len(transactions)) - - eth_jobs = [] - erc20_jobs = [] - - eth_withdrawal_requests_pending = get_withdrawal_requests_pending([ETH_CURRENCY]) - erc20_withdrawal_requests_pending = get_withdrawal_requests_pending( - ERC20_TOKEN_CURRENCIES, - blockchain_currency='ETH' - ) - - eth_withdrawals_dict = {i.id: i.data.get('txs_attempts', []) - for i in eth_withdrawal_requests_pending} - eth_withdrawal_requests_pending_txs = {v: k for k, - values in eth_withdrawals_dict.items() for v in values} - - erc20_withdrawals_dict = {i.id: i.data.get('txs_attempts', []) - for i in erc20_withdrawal_requests_pending} - erc20_withdrawal_requests_pending_txs = { - v: k for k, values in erc20_withdrawals_dict.items() for v in values} - - check_eth_withdrawal_jobs = [] - check_erc20_withdrawal_jobs = [] - - # Withdrawals - for tx_data in transactions: - tx = EthTransaction.from_node(tx_data) - if not tx: - continue - - # is ETH withdrawal request tx? - if tx.hash in eth_withdrawal_requests_pending_txs: - withdrawal_id = eth_withdrawal_requests_pending_txs[tx.hash] - check_eth_withdrawal_jobs.append(check_tx_withdrawal.s(withdrawal_id, tx.as_dict())) - continue - - # is ERC20 withdrawal request tx? - if tx.hash in erc20_withdrawal_requests_pending_txs: - withdrawal_id = erc20_withdrawal_requests_pending_txs[tx.hash] - check_erc20_withdrawal_jobs.append(check_tx_withdrawal.s(withdrawal_id, tx.as_dict())) - continue - - eth_addresses = set(ethereum_manager.get_user_addresses()) - eth_keeper = ethereum_manager.get_keeper_wallet() - eth_gas_keeper = ethereum_manager.get_gas_keeper_wallet() - - eth_addresses_deps = set(eth_addresses) - eth_addresses_deps.add(ETH_SAFE_ADDR) - - # Deposits - for tx_data in transactions: - tx = EthTransaction.from_node(tx_data) - if not tx: - continue - - if tx.to_addr is None: - continue - - if tx.to_addr in eth_addresses_deps: - # process ETH deposit - if not tx.contract_address: - eth_jobs.append(eth_process_eth_deposit.s(tx.as_dict())) - # process ERC20 deposit - else: - erc20_jobs.append(eth_process_erc20_deposit.s(tx.as_dict())) - - if eth_jobs: - log.info('Need to check ETH deposits count: %s', len(eth_jobs)) - group(eth_jobs).apply_async() - - if erc20_jobs: - log.info('Need to check ERC20 deposits count: %s', len(erc20_jobs)) - group(erc20_jobs).apply_async() - - if check_eth_withdrawal_jobs: - log.info('Need to check ETH withdrawals count: %s', len(check_eth_withdrawal_jobs)) - group(check_eth_withdrawal_jobs).apply_async() - - if check_erc20_withdrawal_jobs: - log.info('Need to check ERC20 withdrawals count: %s', len(check_eth_withdrawal_jobs)) - group(check_erc20_withdrawal_jobs).apply_async() - - # check accumulations - for tx_data in transactions: - tx = EthTransaction.from_node(tx_data) - if not tx: - continue - - # checks only exchange addresses withdrawals - if tx.from_addr not in eth_addresses: - continue - - # skip txs from keepers - if tx.from_addr in [eth_keeper.address, eth_gas_keeper.address, ETH_SAFE_ADDR]: - continue - - # checks only if currency flows outside the exchange - if tx.to_addr in eth_addresses: - continue - - # check ERC20 accumulations - if tx.contract_address: - token = ethereum_manager.get_token_by_address(tx.contract_address) - - accumulation_details, created = AccumulationDetails.objects.get_or_create( - txid=tx.hash, - defaults=dict( - txid=tx.hash, - from_address=tx.from_addr, - to_address=tx.to_addr, - currency=ETH_CURRENCY, - token_currency=token.currency, - state=AccumulationDetails.STATE_COMPLETED, - ) - ) - if not created: - log.info(f'Found accumulation {token.currency} from {tx.from_addr} to {tx.to_addr}') - accumulation_details.to_address = tx.to_addr - accumulation_details.complete() - else: - log.info(f'Unexpected accumulation {token.currency} from {tx.from_addr} to {tx.to_addr}') - - # check ETH accumulations - else: - accumulation_details, created = AccumulationDetails.objects.get_or_create( - txid=tx.hash, - defaults=dict( - txid=tx.hash, - from_address=tx.from_addr, - to_address=tx.to_addr, - currency=ETH_CURRENCY, - state=AccumulationDetails.STATE_COMPLETED, - ) - ) - if not created: - log.info(f'Found accumulation ETH from {tx.from_addr} to {tx.to_addr}') - # Use to_address only from node - accumulation_details.to_address = w3.toChecksumAddress(tx.to_addr) - accumulation_details.complete() - else: - log.info(f'Unexpected accumulation ETH from {tx.from_addr} to {tx.to_addr}') - - execution_time = time.time() - started_at - log.info('Block #%s processed in %.2f sec. (ETH TX count: %s, ERC20 TX count: %s, WR TX count: %s)', - block_id, execution_time, len(eth_jobs), len(erc20_jobs), - len(check_erc20_withdrawal_jobs) + len(check_eth_withdrawal_jobs)) - - -@shared_task -def check_tx_withdrawal(withdrawal_id, tx_data): - tx = EthTransaction(tx_data) - - withdrawal_request = WithdrawalRequest.objects.filter( - id=withdrawal_id, - state=WR_PENDING, - ).first() - - if withdrawal_request is None: - log.warning('Invalid withdrawal request state for TX %s', tx.hash) - return - - withdrawal_request.txid = tx.hash - - if not ethereum_manager.is_valid_transaction(tx.hash): - withdrawal_request.fail() - return - - withdrawal_request.complete() - - -@shared_task(autoretry_for=(RetryRequired,), retry_kwargs={'max_retries': 60}) -def eth_process_eth_deposit(tx_data: dict): - """ - Process ETH deposit, excepting inner gas deposits, etc - """ - log.info('Processing eth deposit: %s', tx_data) - tx = EthTransaction(tx_data) - amount = ethereum_manager.get_amount_from_base_denomination(tx.value) - - # skip if failed - if not ethereum_manager.is_valid_transaction(tx.hash): - log.error(f'ETH deposit transaction {tx.hash} is invalid or failed') - return - - eth_keeper = ethereum_manager.get_keeper_wallet() - external_accumulation_addresses = accumulation_manager.get_external_accumulation_addresses([ETH_CURRENCY]) - - # is accumulation tx? - if tx.to_addr in [ETH_SAFE_ADDR, eth_keeper.address] + external_accumulation_addresses: - accumulation_transaction = AccumulationTransaction.objects.filter( - tx_hash=tx.hash, - ).first() - - if accumulation_transaction is None: - log.error(f'Accumulation TX {tx.hash} not exist') - return - - if accumulation_transaction.tx_state == AccumulationTransaction.STATE_COMPLETED: - log.info(f'Accumulation TX {tx.hash} already processed') - return - - accumulation_transaction.complete() - - log.info(f'Tx {tx.hash} is ETH accumulation') - return - - eth_gas_keeper = ethereum_manager.get_gas_keeper_wallet() - # is inner gas deposit? - if tx.from_addr == eth_gas_keeper.address: - accumulation_transaction = AccumulationTransaction.objects.filter( - tx_hash=tx.hash, - tx_type=AccumulationTransaction.TX_TYPE_GAS_DEPOSIT, - ).first() - - if accumulation_transaction is None: - log.error(f'Gas accumulation TX {tx.hash} not found') - return - - if accumulation_transaction.tx_state == AccumulationTransaction.STATE_COMPLETED: - log.info(f'Accumulation TX {tx.hash} already processed as token gas') - return - - log.info(f'Tx {tx.hash} is gas deposit') - accumulation_transaction.complete(is_gas=True) - accumulate_erc20.apply_async([accumulation_transaction.wallet_transaction_id]) - return - - db_wallet = ethereum_manager.get_wallet_db_instance(ETH_CURRENCY, tx.to_addr) - if db_wallet is None: - log.error(f'Wallet ETH {tx.to_addr} not exists or blocked') - return - - # is already processed? - db_wallet_transaction = WalletTransactions.objects.filter( - tx_hash__iexact=tx.hash, - wallet_id=db_wallet.id, - ).first() - - if db_wallet_transaction is not None: - log.warning('TX %s already processed as ETH deposit', tx.hash) - return - - # make deposit - # check for keeper deposit - if db_wallet.address == eth_keeper.address: - log.info('TX %s is keeper ETH deposit: %s', tx.hash, amount) - return - - # check for gas keeper deposit - if db_wallet.address == eth_gas_keeper.address: - log.info('TX %s is gas keeper ETH deposit: %s', tx.hash, amount) - return - - WalletTransactions.objects.create( - wallet=db_wallet, - tx_hash=tx.hash, - amount=amount, - currency=ETH_CURRENCY, - ) - log.info('TX %s processed as %s ETH deposit', tx.hash, amount) - - -@shared_task(autoretry_for=(RetryRequired,), retry_kwargs={'max_retries': 60}) -def eth_process_erc20_deposit(tx_data: dict): - """ - Process ERC20 deposit - """ - log.info('Processing ERC20 deposit: %s', tx_data) - tx = EthTransaction(tx_data) - - if not ethereum_manager.is_valid_transaction(tx.hash): - log.warning('ERC20 deposit TX %s is failed or invalid', tx.hash) - return - - token = ethereum_manager.get_token_by_address(tx.contract_address) - token_to_addr = tx.to_addr - token_amount = token.get_amount_from_base_denomination(tx.value) - eth_keeper = ethereum_manager.get_keeper_wallet() - external_accumulation_addresses = accumulation_manager.get_external_accumulation_addresses(list(ERC20_CURRENCIES)) - - if token_to_addr in [ETH_SAFE_ADDR, eth_keeper.address] + external_accumulation_addresses: - log.info(f'TX {tx.hash} is {token_amount} {token.currency} accumulation') - - accumulation_transaction = AccumulationTransaction.objects.filter( - tx_hash=tx.hash, - ).first() - if accumulation_transaction is None: - # accumulation from outside - log.error('Token accumulation TX %s not exist', tx.hash) - return - - accumulation_transaction.complete() - return - - db_wallet = ethereum_manager.get_wallet_db_instance(token.currency, token_to_addr) - if db_wallet is None: - log.error(f'Wallet {token.currency} {token_to_addr} not exists or blocked') - return - - db_wallet_transaction = WalletTransactions.objects.filter( - tx_hash__iexact=tx.hash, - wallet_id=db_wallet.id, - ).first() - if db_wallet_transaction is not None: - log.warning('TX %s already processed as %s deposit', tx.hash, token.currency) - return - - # check for keeper deposit - if db_wallet.address == eth_keeper.address: - log.info('TX %s is keeper %s deposit: %s', tx.hash, token.currency, token_amount) - return - - # check for gas keeper deposit - eth_gas_keeper = ethereum_manager.get_gas_keeper_wallet() - if db_wallet.address == eth_gas_keeper.address: - log.info('TX %s is keeper %s deposit: %s', tx.hash, token.currency, token_amount) - return - - WalletTransactions.objects.create( - wallet_id=db_wallet.id, - tx_hash=tx.hash, - amount=token_amount, - currency=token.currency, - ) - log.info('TX %s processed as %s %s deposit', tx.hash, token_amount, token.currency) - - -@shared_task -def process_payouts(password, withdrawals_ids=None): - eth_withdrawal_requests = get_withdrawal_requests_to_process(currencies=[ETH_CURRENCY]) - - if eth_withdrawal_requests: - log.info('Need to process %s ETH withdrawals', len(eth_withdrawal_requests)) - - for item in eth_withdrawal_requests: - if withdrawals_ids and item.id not in withdrawals_ids: - continue - - # skip freezed withdrawals - if item.user.profile.is_payouts_freezed(): - continue - withdraw_eth.apply_async([item.id, password]) - - erc20_withdrawal_requests = get_withdrawal_requests_to_process( - currencies=ERC20_TOKEN_CURRENCIES, - blockchain_currency='ETH' - ) - - if erc20_withdrawal_requests: - log.info('Need to process %s ERC20 withdrawals', len(erc20_withdrawal_requests)) - for item in erc20_withdrawal_requests: - if withdrawals_ids and item.id not in withdrawals_ids: - continue - # skip freezed withdrawals - if item.user.profile.is_payouts_freezed(): - continue - withdraw_erc20.apply_async([item.id, password]) - - -@shared_task -def withdraw_eth(withdrawal_request_id, password, old_tx_data=None, prev_tx_hash=None): - if old_tx_data is None: - old_tx_data = {} - withdrawal_request = WithdrawalRequest.objects.get(id=withdrawal_request_id) - - # todo: handle errors - address = w3.toChecksumAddress(withdrawal_request.data.get('destination')) - keeper = ethereum_manager.get_keeper_wallet() - amount_wei = ethereum_manager.get_base_denomination_from_amount(withdrawal_request.amount) - withdrawal_fee_wei = Web3.toWei(to_decimal(get_withdrawal_fee(ETH_CURRENCY, ETH_CURRENCY)), 'ether') - amount_to_send_wei = amount_wei - withdrawal_fee_wei - - gas_price = ethereum_manager.gas_price_cache.get_increased_price( - old_tx_data.get('gasPrice') or 0) - - # todo: check min limit - if amount_to_send_wei <= 0: - log.error('Invalid withdrawal amount') - withdrawal_request.fail() - return - - keeper_balance = ethereum_manager.get_balance_in_base_denomination(keeper.address) - if keeper_balance < (amount_to_send_wei + (gas_price * settings.ETH_TX_GAS)): - log.warning('Keeper not enough ETH, skipping') - return - - if old_tx_data: - log.info('ETH withdrawal transaction to %s will be replaced', w3.toChecksumAddress(address)) - tx_data = old_tx_data.copy() - tx_data['gasPrice'] = gas_price - if prev_tx_hash and ethereum_manager.get_transaction_receipt(prev_tx_hash): - log.info('ETH TX %s sent. Do not need to replace.') - return - else: - nonce = ethereum_manager.wait_for_nonce() - tx_data = { - 'nonce': nonce, - 'gasPrice': gas_price, - 'gas': settings.ETH_TX_GAS, - 'from': w3.toChecksumAddress(keeper.address), - 'to': w3.toChecksumAddress(address), - 'value': amount_to_send_wei, - 'chainId': settings.ETH_CHAIN_ID, - } - - private_key = AESCoderDecoder(password).decrypt(keeper.private_key) - tx_hash = ethereum_manager.send_tx( - private_key=private_key, - to_address=address, - amount=amount_to_send_wei, - nonce=tx_data['nonce'], - gasPrice=tx_data['gasPrice'], - ) - - if not tx_hash: - log.error('Unable to send withdrawal TX') - ethereum_manager.release_nonce() - return - - withdrawal_txs_attempts = withdrawal_request.data.get('txs_attempts', []) - withdrawal_txs_attempts.append(tx_hash.hex()) - - withdrawal_request.data['txs_attempts'] = list(set(withdrawal_txs_attempts)) - - withdrawal_request.state = WR_PENDING - withdrawal_request.our_fee_amount = ethereum_manager.get_amount_from_base_denomination(withdrawal_fee_wei) - withdrawal_request.save(update_fields=['state', 'updated', 'our_fee_amount', 'data']) - log.info('ETH withdrawal TX %s sent', tx_hash.hex()) - - # wait tx processed - try: - ethereum_manager.wait_for_transaction_receipt(tx_hash, poll_latency=2) - ethereum_manager.release_nonce() - except RetryRequired: - # retry with higher gas price - withdraw_eth(withdrawal_request_id, password, old_tx_data=tx_data, prev_tx_hash=tx_hash) - - -@shared_task -def withdraw_erc20(withdrawal_request_id, password, old_tx_data=None, prev_tx_hash=None): - if old_tx_data is None: - old_tx_data = {} - - withdrawal_request = WithdrawalRequest.objects.get(id=withdrawal_request_id) - - address = w3.toChecksumAddress(withdrawal_request.data.get('destination')) - currency = withdrawal_request.currency - - token = ethereum_manager.get_token_by_symbol(currency) - send_amount_wei = token.get_base_denomination_from_amount(withdrawal_request.amount) - withdrawal_fee_wei = token.get_base_denomination_from_amount(token.withdrawal_fee) - amount_to_send_wei = send_amount_wei - withdrawal_fee_wei - if amount_to_send_wei <= 0: - log.error('Invalid withdrawal amount') - withdrawal_request.fail() - return - - gas_price = ethereum_manager.gas_price_cache.get_increased_price( - old_tx_data.get('gasPrice') or 0) - - transfer_gas = token.get_transfer_gas_amount(address, amount_to_send_wei, True) - - keeper = ethereum_manager.get_keeper_wallet() - keeper_eth_balance = ethereum_manager.get_balance_in_base_denomination(keeper.address) - keeper_token_balance = token.get_base_denomination_balance(keeper.address) - - if keeper_eth_balance < gas_price * transfer_gas: - log.warning('Keeper not enough ETH for gas, skipping') - return - - if keeper_token_balance < amount_to_send_wei: - log.warning('Keeper not enough %s, skipping', currency) - return - - log.info('Amount to send: %s, gas price: %s, transfer gas: %s', - amount_to_send_wei, gas_price, transfer_gas) - - if old_tx_data: - log.info('%s withdrawal to %s will be replaced', currency.code, address) - tx_data = old_tx_data.copy() - tx_data['gasPrice'] = gas_price - if prev_tx_hash and ethereum_manager.get_transaction_receipt(prev_tx_hash): - log.info('Token TX %s sent. Do not need to replace.') - return - else: - nonce = ethereum_manager.wait_for_nonce() - tx_data = { - 'chainId': settings.ETH_CHAIN_ID, - 'gas': transfer_gas, - 'gasPrice': gas_price, - 'nonce': nonce, - } - - private_key = AESCoderDecoder(password).decrypt(keeper.private_key) - tx_hash = token.send_token(private_key, address, amount_to_send_wei, **tx_data) - - if not tx_hash: - log.error('Unable to send token withdrawal TX') - ethereum_manager.release_nonce() - return - - withdrawal_txs_attempts = withdrawal_request.data.get('txs_attempts', []) - withdrawal_txs_attempts.append(tx_hash.hex()) - - withdrawal_request.data['txs_attempts'] = list(set(withdrawal_txs_attempts)) - withdrawal_request.state = WR_PENDING - withdrawal_request.our_fee_amount = token.get_amount_from_base_denomination(withdrawal_fee_wei) - - withdrawal_request.save(update_fields=['state', 'updated', 'our_fee_amount', 'data']) - log.info('%s withdrawal TX %s sent', currency, tx_hash.hex()) - - # wait tx processed - try: - ethereum_manager.wait_for_transaction_receipt(tx_hash, poll_latency=2) - ethereum_manager.release_nonce() - except RetryRequired: - # retry with higher gas price - withdraw_erc20(withdrawal_request_id, password, old_tx_data=tx_data, prev_tx_hash=tx_hash) - - -@shared_task -def check_deposit_scoring(wallet_transaction_id): - """Check deposit for scoring""" - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - wallet_transaction.check_scoring() - - -@shared_task -def check_balances(): - """Main accumulations scheduler""" - kyt_check_jobs = [] - accumulations_jobs = [] - external_accumulations_jobs = [] - - for item in accumulation_manager.get_waiting_for_kyt_check(ETH_CURRENCY): - kyt_check_jobs.append(check_deposit_scoring.s(item.id)) - - for item in accumulation_manager.get_waiting_for_accumulation(blockchain_currency=ETH_CURRENCY): - accumulations_jobs.append(check_balance.s(item.id)) - - for item in accumulation_manager.get_waiting_for_external_accumulation(blockchain_currency=ETH_CURRENCY): - external_accumulations_jobs.append(check_balance.s(item.id)) - - if kyt_check_jobs: - log.info('Need to check for KYT: %s', len(kyt_check_jobs)) - jobs_group = group(kyt_check_jobs) - jobs_group.apply_async() - - if accumulations_jobs: - log.info('Need to check accumulations: %s', len(accumulations_jobs)) - jobs_group = group(accumulations_jobs) - jobs_group.apply_async() - - if external_accumulations_jobs: - log.info('Need to check external accumulations: %s', len(external_accumulations_jobs)) - jobs_group = group(external_accumulations_jobs) - jobs_group.apply_async() - - -def is_gas_need(wallet_transaction): - acc_tx = accumulation_manager.get_last_gas_deposit_tx(wallet_transaction) - return not acc_tx - - -@shared_task -def check_balance(wallet_transaction_id): - """Splits blockchain currency accumulation and token accumulation""" - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - address = wallet_transaction.wallet.address - currency = wallet_transaction.currency - - # ETH - if currency == ETH_CURRENCY: - wallet_transaction.set_ready_for_accumulation() - accumulate_eth.apply_async([wallet_transaction_id]) - - # tokens - else: - log.info('Checking %s %s', currency, address) - - if not is_gas_need(wallet_transaction): - log.info(f'Gas not required for {currency} {address}') - wallet_transaction.set_ready_for_accumulation() - accumulate_erc20.apply_async([wallet_transaction_id]) - else: - log.info(f'Gas required for {currency} {address}') - wallet_transaction.set_gas_required() - send_gas.apply_async([wallet_transaction_id]) - - -@shared_task -def accumulate_eth(wallet_transaction_id): - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - address = wallet_transaction.wallet.address - - # recheck balance - amount = wallet_transaction.amount - amount_wei = ethereum_manager.get_base_denomination_from_amount(amount) - - log.info('Accumulation ETH from: %s; Balance: %s; Min acc balance:%s', - address, amount, ethereum_manager.accumulation_min_balance) - - accumulation_address = wallet_transaction.external_accumulation_address or ethereum_manager.get_accumulation_address( - amount) - - # we want to process our tx faster - gas_price = ethereum_manager.gas_price_cache.get_increased_price() - gas_amount = gas_price * settings.ETH_TX_GAS - withdrawal_amount_wei = amount_wei - gas_amount - withdrawal_amount = ethereum_manager.get_amount_from_base_denomination(withdrawal_amount_wei) - - if ethereum_manager.is_gas_price_reach_max_limit(gas_price): - log.warning(f'Gas price too high: {gas_price}') - ethereum_manager.set_gas_price_too_high(wallet_transaction) - return - - # in debug mode values can be very small - if withdrawal_amount_wei <= 0: - log.error('ETH withdrawal amount invalid: %s', withdrawal_amount) - wallet_transaction.set_balance_too_low() - return - - # prepare tx - wallet = ethereum_manager.get_user_wallet('ETH', address) - nonce = ethereum_manager.client.eth.getTransactionCount(address) - - tx_hash = ethereum_manager.send_tx( - private_key=wallet.private_key, - to_address=accumulation_address, - amount=withdrawal_amount_wei, - nonce=nonce, - gasPrice=gas_price, - ) - - if not tx_hash: - log.error('Unable to send accumulation TX') - return - - AccumulationTransaction.objects.create( - wallet_transaction=wallet_transaction, - amount=withdrawal_amount, - tx_type=AccumulationTransaction.TX_TYPE_ACCUMULATION, - tx_state=AccumulationTransaction.STATE_PENDING, - tx_hash=tx_hash.hex(), - ) - wallet_transaction.set_accumulation_in_progress() - - AccumulationDetails.objects.create( - currency=ETH_CURRENCY, - txid=tx_hash.hex(), - from_address=address, - to_address=accumulation_address - ) - - log.info('Accumulation TX %s sent from %s to %s', tx_hash.hex(), wallet.address, accumulation_address) - - -@shared_task -def accumulate_erc20(wallet_transaction_id): - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - address = wallet_transaction.wallet.address - currency = wallet_transaction.currency - - gas_deposit_tx = accumulation_manager.get_last_gas_deposit_tx(wallet_transaction) - if gas_deposit_tx is None: - log.warning(f'Gas deposit for {address} not found or in process') - return - - token = ethereum_manager.get_token_by_symbol(currency) - # amount checks - token_amount = wallet_transaction.amount - token_amount_wei = token.get_base_denomination_from_amount(token_amount) - - if token_amount <= to_decimal(0): - log.warning('Cant accumulate %s from: %s; Balance too low: %s;', - currency, address, token_amount) - return - - accumulation_address = wallet_transaction.external_accumulation_address or token.get_accumulation_address( - token_amount) - - # we keep amount not as wei, it's more easy, so we need to convert it - # checked_amount_wei = token.get_wei_from_amount(accumulation_state.current_balance) - - log.info(f'Accumulation {currency} from: {address}; Balance: {token_amount};') - - accumulation_gas_amount = ethereum_manager.get_base_denomination_from_amount(gas_deposit_tx.amount) - eth_amount_wei = ethereum_manager.get_balance_in_base_denomination(address) - - if eth_amount_wei < accumulation_gas_amount: - log.warning(f'Wallet ETH amount: {eth_amount_wei} less than gas needed ' - f'{accumulation_gas_amount}, need to recheck') - return - - accumulation_gas_required_amount = token.get_transfer_gas_amount( - accumulation_address, - token_amount_wei, - ) - - # calculate from existing wallet eth amount - gas_price = int(accumulation_gas_amount / accumulation_gas_required_amount) - - wallet = ethereum_manager.get_user_wallet(currency, address) - nonce = w3.eth.getTransactionCount(address) - - tx_hash = token.send_token( - wallet.private_key, - accumulation_address, - token_amount_wei, - gas=accumulation_gas_required_amount, - gasPrice=gas_price, - nonce=nonce - ) - - if not tx_hash: - log.error('Unable to send token accumulation TX') - return - - AccumulationTransaction.objects.create( - wallet_transaction=wallet_transaction, - amount=token_amount, - tx_type=AccumulationTransaction.TX_TYPE_ACCUMULATION, - tx_state=AccumulationTransaction.STATE_PENDING, - tx_hash=tx_hash.hex(), - ) - wallet_transaction.set_accumulation_in_progress() - - AccumulationDetails.objects.create( - currency=ETH_CURRENCY, - token_currency=currency, - txid=tx_hash.hex(), - from_address=address, - to_address=accumulation_address, - ) - - log.info('Token accumulation TX %s sent from %s to: %s', - tx_hash.hex(), wallet.address, accumulation_address) - - -@shared_task -def send_gas(wallet_transaction_id, old_tx_data=None, old_tx_hash=None): - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - old_tx_data = old_tx_data or {} - - if not old_tx_hash and not is_gas_need(wallet_transaction): - check_balance.apply_async([wallet_transaction_id]) - return - - address = wallet_transaction.wallet.address - currency = wallet_transaction.currency - token = ethereum_manager.get_token_by_symbol(currency) - - token_amount_wei = token.get_base_denomination_balance(address) - token_amount = token.get_amount_from_base_denomination(token_amount_wei) - - if to_decimal(token_amount) < to_decimal(token.accumulation_min_balance): - log.warning('Current balance less than minimum, need to recheck') - return - - # at this point we know amount is enough - gas_keeper = ethereum_manager.get_gas_keeper_wallet() - gas_keeper_balance_wei = ethereum_manager.get_balance_in_base_denomination(gas_keeper.address) - accumulation_gas_amount = token.get_transfer_gas_amount(ETH_SAFE_ADDR, token_amount_wei) - gas_price = ethereum_manager.gas_price_cache.get_increased_price( - old_tx_data.get('gasPrice') or 0) - - if ethereum_manager.is_gas_price_reach_max_limit(gas_price): - log.warning(f'Gas price too high: {gas_price}') - ethereum_manager.set_gas_price_too_high(wallet_transaction) - return - - accumulation_gas_total_amount = accumulation_gas_amount * gas_price - - if gas_keeper_balance_wei < accumulation_gas_total_amount: - log.error('Gas keeper balance too low to send gas: %s', - ethereum_manager.get_amount_from_base_denomination(gas_keeper_balance_wei)) - - # prepare tx - if old_tx_data: - log.info('Gas transaction to %s will be replaced', w3.toChecksumAddress(address)) - tx_data = old_tx_data.copy() - tx_data['gasPrice'] = gas_price - tx_data['value'] = accumulation_gas_total_amount - if ethereum_manager.get_transaction_receipt(old_tx_hash): - log.info('Gas TX %s sent. Do not need to replace.') - return - else: - nonce = ethereum_manager.wait_for_nonce(is_gas=True) - tx_data = { - 'nonce': nonce, - 'gasPrice': gas_price, - 'gas': settings.ETH_TX_GAS, - 'from': w3.toChecksumAddress(gas_keeper.address), - 'to': address, - 'value': accumulation_gas_total_amount, - 'chainId': settings.ETH_CHAIN_ID, - } - - signed_tx = w3.eth.account.signTransaction(tx_data, gas_keeper.private_key) - try: - tx_hash = w3.eth.sendRawTransaction(signed_tx.rawTransaction) - except ValueError: - log.exception('Unable to send accumulation TX') - ethereum_manager.release_nonce(is_gas=True) - return - - if not tx_hash: - log.error('Unable to send accumulation TX') - ethereum_manager.release_nonce(is_gas=True) - return - - acc_transaction = AccumulationTransaction.objects.create( - wallet_transaction=wallet_transaction, - amount=ethereum_manager.get_amount_from_base_denomination(accumulation_gas_total_amount), - tx_type=AccumulationTransaction.TX_TYPE_GAS_DEPOSIT, - tx_state=AccumulationTransaction.STATE_PENDING, - tx_hash=tx_hash.hex(), - ) - wallet_transaction.set_waiting_for_gas() - log.info('Gas deposit TX %s sent', tx_hash.hex()) - - # wait tx processed - try: - ethereum_manager.wait_for_transaction_receipt(tx_hash, poll_latency=3) - acc_transaction.complete(is_gas=True) - ethereum_manager.release_nonce(is_gas=True) - accumulate_erc20.apply_async([wallet_transaction_id]) - except RetryRequired: - # retry with higher gas price - send_gas(wallet_transaction_id, old_tx_data=tx_data, old_tx_hash=tx_hash) - -# todo fix -# @shared_task -# def accumulate_eth_dust(): -# ethereum_manager.accumulate_dust() diff --git a/cryptocoins/tasks/evm.py b/cryptocoins/tasks/evm.py new file mode 100644 index 0000000..2a26864 --- /dev/null +++ b/cryptocoins/tasks/evm.py @@ -0,0 +1,75 @@ +from celery import shared_task + +from cryptocoins.evm.manager import evm_handlers_manager +from cryptocoins.exceptions import RetryRequired + + +@shared_task +def process_new_blocks_task(currency_code): + evm_handlers_manager.get_handler(currency_code).process_new_blocks() + + +# @shared_task(bind=True) +# def process_block_task(currency_code): +# evm_handlers_manager.get_handler(currency_code).process_block() + + + +@shared_task +def check_tx_withdrawal_task(currency_code, withdrawal_id, tx_data): + evm_handlers_manager.get_handler(currency_code).check_tx_withdrawal(withdrawal_id, tx_data) + + +@shared_task(autoretry_for=(RetryRequired,), retry_kwargs={'max_retries': 60}) +def process_coin_deposit_task(currency_code, tx_data: dict): + evm_handlers_manager.get_handler(currency_code).process_coin_deposit(tx_data) + + +@shared_task(autoretry_for=(RetryRequired,), retry_kwargs={'max_retries': 60}) +def process_tokens_deposit_task(currency_code, tx_data: dict): + evm_handlers_manager.get_handler(currency_code).process_tokens_deposit(tx_data) + + +@shared_task +def process_payouts_task(currency_code, password, withdrawals_ids=None): + evm_handlers_manager.get_handler(currency_code).process_payouts(password, withdrawals_ids) + + +@shared_task +def withdraw_coin_task(currency_code, withdrawal_request_id, password): + evm_handlers_manager.get_handler(currency_code).withdraw_coin(withdrawal_request_id, password) + + +@shared_task +def withdraw_tokens_task(currency_code, withdrawal_request_id, password): + evm_handlers_manager.get_handler(currency_code).withdraw_tokens(withdrawal_request_id, password) + + +@shared_task +def check_deposit_scoring_task(currency_code, wallet_transaction_id): + evm_handlers_manager.get_handler(currency_code).check_deposit_scoring(wallet_transaction_id) + + +@shared_task +def check_balances_task(currency_code): + evm_handlers_manager.get_handler(currency_code).check_balances() + + +@shared_task +def check_balance_task(currency_code, wallet_transaction_id): + evm_handlers_manager.get_handler(currency_code).check_balance(wallet_transaction_id) + + +@shared_task +def accumulate_coin_task(currency_code, wallet_transaction_id): + evm_handlers_manager.get_handler(currency_code).accumulate_coin(wallet_transaction_id) + + +@shared_task +def accumulate_tokens_task(currency_code, wallet_transaction_id): + evm_handlers_manager.get_handler(currency_code).accumulate_tokens(wallet_transaction_id) + + +@shared_task +def send_gas_task(currency_code, wallet_transaction_id): + evm_handlers_manager.get_handler(currency_code).send_gas(wallet_transaction_id) diff --git a/cryptocoins/tasks/trx.py b/cryptocoins/tasks/trx.py deleted file mode 100644 index 07c659f..0000000 --- a/cryptocoins/tasks/trx.py +++ /dev/null @@ -1,672 +0,0 @@ -import logging -import time - -from celery import group, shared_task -from django.conf import settings -from tronpy.exceptions import BlockNotFound - -from core.models.inouts.wallet import WalletTransactions -from core.models.inouts.withdrawal import PENDING as WR_PENDING -from core.models.inouts.withdrawal import WithdrawalRequest -from core.utils.inouts import get_withdrawal_fee, get_min_accumulation_balance -from core.utils.withdrawal import get_withdrawal_requests_to_process -from core.utils.withdrawal import get_withdrawal_requests_pending -from cryptocoins.accumulation_manager import AccumulationManager -from cryptocoins.coins.trx import TRX_CURRENCY -from cryptocoins.coins.trx.tron import TrxTransaction, tron_manager -from cryptocoins.models import AccumulationDetails -from cryptocoins.models.accumulation_transaction import AccumulationTransaction -from cryptocoins.utils.commons import ( - load_last_processed_block_id, - store_last_processed_block_id, -) -from lib.cipher import AESCoderDecoder -from lib.helpers import to_decimal -from lib.utils import memcache_lock - -log = logging.getLogger(__name__) - -accumulation_manager = AccumulationManager() - -DEFAULT_BLOCK_ID_DELTA = 1000 -TRX_SAFE_ADDR = settings.TRX_SAFE_ADDR -TRX_NET_FEE = settings.TRX_NET_FEE -TRC20_FEE_LIMIT = settings.TRC20_FEE_LIMIT -TRC20_TOKEN_CURRENCIES = tron_manager.registered_token_currencies -TRC20_TOKEN_CONTRACT_ADDRESSES = tron_manager.registered_token_addresses - - -@shared_task -def trx_process_new_blocks(): - lock_id = 'trx_blocks' - with memcache_lock(lock_id, lock_id) as acquired: - if acquired: - current_block_id = tron_manager.get_latest_block_num() - default_block_id = current_block_id - DEFAULT_BLOCK_ID_DELTA - last_processed_block_id = load_last_processed_block_id( - currency=TRX_CURRENCY, default=default_block_id) - - if last_processed_block_id >= current_block_id: - log.debug('Nothing to process since block #%s', current_block_id) - return - - blocks_to_process = list(range( - last_processed_block_id + 1, - current_block_id + 1, - )) - blocks_to_process.insert(0, last_processed_block_id) - - if len(blocks_to_process) > 1: - log.info('Need to process blocks #%s..#%s', last_processed_block_id + 1, current_block_id) - else: - log.info('Need to process block #%s', last_processed_block_id + 1) - - for block_id in blocks_to_process: - trx_process_block(block_id) - - store_last_processed_block_id(currency=TRX_CURRENCY, block_id=current_block_id) - - -@shared_task(bind=True) -def trx_process_block(self, block_id): - started_at = time.time() - time.sleep(0.1) - log.info('Processing block #%s', block_id) - - try: - block = tron_manager.get_block(block_id) - except BlockNotFound: - log.warning(f'Block not found: {block_id}') - return - except Exception as e: - store_last_processed_block_id(currency=TRX_CURRENCY, block_id=block_id - 1) - raise e - - transactions = block.get('transactions', []) - - if not transactions: - log.info('Block #%s has no transactions, skipping', block_id) - return - - log.info('Transactions count in block #%s: %s', block_id, len(transactions)) - - trx_jobs = [] - trc20_jobs = [] - - trx_withdrawal_requests_pending = get_withdrawal_requests_pending([TRX_CURRENCY]) - trc20_withdrawal_requests_pending = get_withdrawal_requests_pending(TRC20_TOKEN_CURRENCIES, blockchain_currency='TRX') - - trx_withdrawal_requests_pending_txs = [i.txid for i in trx_withdrawal_requests_pending] - trc20_withdrawal_requests_pending_txs = [i.txid for i in trc20_withdrawal_requests_pending] - - check_trx_withdrawal_jobs = [] - check_trc20_withdrawal_jobs = [] - - all_valid_transactions = [] - all_transactions = [] - - for tx_data in transactions: - tx: TrxTransaction = TrxTransaction.from_node(tx_data) - if not tx: - continue - if tx.is_success: - all_valid_transactions.append(tx) - all_transactions.append(tx) - - # Withdrawals - for tx in all_transactions: - # is TRX withdrawal request tx? - if tx.hash in trx_withdrawal_requests_pending_txs: - check_trx_withdrawal_jobs.append(check_tx_withdrawal.s(tx.as_dict())) - continue - - # is TRC20 withdrawal request tx? - if tx.hash in trc20_withdrawal_requests_pending_txs: - check_trc20_withdrawal_jobs.append(check_tx_withdrawal.s(tx.as_dict())) - continue - - keeper_wallet = tron_manager.get_keeper_wallet() - gas_keeper_wallet = tron_manager.get_keeper_wallet() - trx_addresses = set(tron_manager.get_user_addresses()) - - trx_addresses_deps = set(trx_addresses) - trx_addresses_deps.add(TRX_SAFE_ADDR) - - # Deposits - for tx in all_valid_transactions: - # process TRX deposit - - if tx.to_addr in trx_addresses_deps: - # Process TRX - if not tx.contract_address: - trx_jobs.append(trx_process_trx_deposit.s(tx.as_dict())) - # Process TRC20 - elif tx.contract_address and tx.contract_address in TRC20_TOKEN_CONTRACT_ADDRESSES: - trc20_jobs.append(trx_process_trc20_deposit.s(tx.as_dict())) - - # Accumulations monitoring - for tx in all_valid_transactions: - if tx.from_addr in trx_addresses and tx.to_addr not in trx_addresses: - - # skip keepers withdrawals - if tx.from_addr in [keeper_wallet.address, gas_keeper_wallet.address]: - continue - - accumulation_details = AccumulationDetails.objects.filter( - txid=tx.hash - ).first() - - if accumulation_details: - log.info(f'Accumulation details for {tx.hash} already exists') - continue - - accumulation_details = { - 'currency': TRX_CURRENCY, - 'txid': tx.hash, - 'from_address': tx.from_addr, - 'to_address': tx.to_addr, - 'state': AccumulationDetails.STATE_COMPLETED - } - - if not tx.contract_address: - # Store TRX accumulations - AccumulationDetails.objects.create(**accumulation_details) - - elif tx.contract_address and tx.contract_address in TRC20_TOKEN_CONTRACT_ADDRESSES: - # Store TRC20 accumulations - token = tron_manager.get_token_by_address(tx.contract_address) - accumulation_details['token_currency'] = token.currency - AccumulationDetails.objects.create(**accumulation_details) - - if trx_jobs: - log.info('Need to check TRX deposits count: %s', len(trx_jobs)) - group(trx_jobs).apply_async() - - if trc20_jobs: - log.info('Need to check TRC20 withdrawals count: %s', len(trc20_jobs)) - group(trc20_jobs).apply_async() - - if check_trx_withdrawal_jobs: - log.info('Need to check TRX withdrawals count: %s', len(check_trx_withdrawal_jobs)) - group(check_trx_withdrawal_jobs).apply_async() - - if check_trc20_withdrawal_jobs: - log.info('Need to check TRC20 withdrawals count: %s', len(check_trx_withdrawal_jobs)) - group(check_trc20_withdrawal_jobs).apply_async() - - execution_time = time.time() - started_at - log.info('Block #%s processed in %.2f sec. (TRX TX count: %s, TRC20 TX count: %s, WR TX count: %s)', - block_id, execution_time, len(trx_jobs), len(trc20_jobs), - len(check_trc20_withdrawal_jobs) + len(check_trx_withdrawal_jobs)) - - -@shared_task -def check_tx_withdrawal(tx_data): - tx = TrxTransaction(tx_data) - withdrawal_request = WithdrawalRequest.objects.filter( - txid=tx.hash, - state=WR_PENDING, - ).first() - - if withdrawal_request is None: - log.warning('Invalid withdrawal request state for TX %s', tx.hash) - return - - if tx.is_success: - withdrawal_request.complete() - else: - withdrawal_request.fail() - - -@shared_task -def trx_process_trx_deposit(tx_data: dict): - """ - Process TRX deposit, excepting inner gas deposits, etc - """ - log.info('Processing trx deposit: %s', tx_data) - tx = TrxTransaction(tx_data) - amount = tron_manager.get_amount_from_base_denomination(tx.value) - - trx_keeper = tron_manager.get_keeper_wallet() - external_accumulation_addresses = accumulation_manager.get_external_accumulation_addresses([TRX_CURRENCY]) - - # is accumulation tx? - if tx.to_addr in [TRX_SAFE_ADDR, trx_keeper.address] + external_accumulation_addresses: - accumulation_transaction = AccumulationTransaction.objects.filter( - tx_hash=tx.hash, - ).first() - - if accumulation_transaction is None: - log.error(f'Accumulation TX {tx.hash} not exist') - return - - if accumulation_transaction.tx_state == AccumulationTransaction.STATE_COMPLETED: - log.info(f'Accumulation TX {tx.hash} already processed') - return - - accumulation_transaction.complete() - - log.info(f'Tx {tx.hash} is TRX accumulation') - return - - trx_gas_keeper = tron_manager.get_gas_keeper_wallet() - # is inner gas deposit? - if tx.from_addr == trx_gas_keeper.address: - accumulation_transaction = AccumulationTransaction.objects.filter( - tx_hash=tx.hash, - tx_type=AccumulationTransaction.TX_TYPE_GAS_DEPOSIT, - ).first() - - if accumulation_transaction is None: - log.error(f'Gas accumulation TX {tx.hash} not found') - return - - if accumulation_transaction.tx_state == AccumulationTransaction.STATE_COMPLETED: - log.info(f'Accumulation TX {tx.hash} already processed as token gas') - return - - log.info(f'Tx {tx.hash} is gas deposit') - accumulation_transaction.complete(is_gas=True) - accumulate_trc20.apply_async([accumulation_transaction.wallet_transaction_id]) - return - - db_wallet = tron_manager.get_wallet_db_instance(TRX_CURRENCY, tx.to_addr) - if db_wallet is None: - log.error(f'Wallet TRX {tx.to_addr} not exists or blocked') - return - - # is already processed? - db_wallet_transaction = WalletTransactions.objects.filter( - tx_hash__iexact=tx.hash, - wallet_id=db_wallet.id, - ).first() - - if db_wallet_transaction is not None: - log.warning('TX %s already processed as TRX deposit', tx.hash) - return - - # make deposit - # check for keeper deposit - if db_wallet.address == trx_keeper.address: - log.info('TX %s is keeper TRX deposit: %s', tx.hash, amount) - return - - # check for gas keeper deposit - if db_wallet.address == trx_gas_keeper.address: - log.info('TX %s is gas keeper TRX deposit: %s', tx.hash, amount) - return - - # check for accumulation min limit - if amount < tron_manager.accumulation_min_balance: - log.info( - 'TX %s amount: %s less accumulation min limit: %s', - tx.hash, amount, tron_manager.accumulation_min_balance - ) - return - - WalletTransactions.objects.create( - wallet=db_wallet, - tx_hash=tx.hash, - amount=amount, - currency=TRX_CURRENCY, - ) - log.info('TX %s processed as %s TRX deposit', tx.hash, amount) - - -@shared_task -def trx_process_trc20_deposit(tx_data: dict): - """ - Process TRC20 deposit - """ - log.info('Processing TRC20 deposit: %s', tx_data) - tx = TrxTransaction(tx_data) - - token = tron_manager.get_token_by_address(tx.contract_address) - token_to_addr = tx.to_addr - token_amount = token.get_amount_from_base_denomination(tx.value) - trx_keeper = tron_manager.get_keeper_wallet() - external_accumulation_addresses = accumulation_manager.get_external_accumulation_addresses( - list(TRC20_TOKEN_CURRENCIES) - ) - - if token_to_addr in [TRX_SAFE_ADDR, trx_keeper.address] + external_accumulation_addresses: - log.info(f'TX {tx.hash} is {token_amount} {token.currency} accumulation') - - accumulation_transaction = AccumulationTransaction.objects.filter( - tx_hash=tx.hash, - ).first() - if accumulation_transaction is None: - # accumulation from outside - log.error('Token accumulation TX %s not exist', tx.hash) - return - - accumulation_transaction.complete() - return - - db_wallet = tron_manager.get_wallet_db_instance(token.currency, token_to_addr) - if db_wallet is None: - log.error('Wallet %s %s not exists or blocked', token.currency, token_to_addr) - return - - db_wallet_transaction = WalletTransactions.objects.filter( - tx_hash__iexact=tx.hash, - wallet_id=db_wallet.id, - ).first() - if db_wallet_transaction is not None: - log.warning(f'TX {tx.hash} already processed as {token.currency} deposit') - return - - # check for keeper deposit - if db_wallet.address == trx_keeper.address: - log.info(f'TX {tx.hash} is keeper {token.currency} deposit: {token_amount}') - return - - # check for gas keeper deposit - trx_gas_keeper = tron_manager.get_gas_keeper_wallet() - if db_wallet.address == trx_gas_keeper.address: - log.info(f'TX {tx.hash} is keeper {token.currency} deposit: {token_amount}') - return - - # check for accumulation min limit - if token_amount < get_min_accumulation_balance(db_wallet.currency): - log.info( - 'TX %s amount: %s less accumulation min limit: %s', - tx.hash, token_amount, tron_manager.accumulation_min_balance - ) - return - - WalletTransactions.objects.create( - wallet_id=db_wallet.id, - tx_hash=tx.hash, - amount=token_amount, - currency=token.currency, - ) - - log.info(f'TX {tx.hash} processed as {token_amount} {token.currency} deposit') - - -@shared_task -def process_payouts(password): - trx_withdrawal_requests = get_withdrawal_requests_to_process(currencies=[TRX_CURRENCY]) - - if trx_withdrawal_requests: - log.info('Need to process %s TRX withdrawals', len(trx_withdrawal_requests)) - jobs_list = [] - for item in trx_withdrawal_requests: - # skip freezed withdrawals - if item.user.profile.is_payouts_freezed(): - continue - jobs_list.append(withdraw_trx.s(item.id, password)) - - group(jobs_list).apply() - - erc20_withdrawal_requests = get_withdrawal_requests_to_process( - currencies=TRC20_TOKEN_CURRENCIES, - blockchain_currency='TRX' - ) - - if erc20_withdrawal_requests: - log.info('Need to process %s ERC20 withdrawals', len(erc20_withdrawal_requests)) - jobs_list = [] - for item in erc20_withdrawal_requests: - # skip freezed withdrawals - if item.user.profile.is_payouts_freezed(): - continue - jobs_list.append(withdraw_trc20.s(item.id, password)) - - group(jobs_list).apply() - - -@shared_task -def withdraw_trx(withdrawal_request_id, password): - withdrawal_request = WithdrawalRequest.objects.get(id=withdrawal_request_id) - - address = withdrawal_request.data.get('destination') - keeper = tron_manager.get_keeper_wallet() - amount_sun = tron_manager.get_base_denomination_from_amount(withdrawal_request.amount) - - withdrawal_fee_sun = tron_manager.get_base_denomination_from_amount( - to_decimal(get_withdrawal_fee(TRX_CURRENCY, TRX_CURRENCY))) - amount_to_send_sun = amount_sun - withdrawal_fee_sun - - # todo: check min limit - if amount_to_send_sun <= 0: - log.error('Invalid withdrawal amount') - withdrawal_request.fail() - return - - if amount_to_send_sun - TRX_NET_FEE < 0: - log.error('Keeper balance too low') - return - - private_key = AESCoderDecoder(password).decrypt(keeper.private_key) - - res = tron_manager.send_tx(private_key, address, amount_to_send_sun) - txid = res.get('txid') - - if not res.get('result') or not txid: - log.error('Unable to send withdrawal TX') - - withdrawal_request.state = WR_PENDING - withdrawal_request.txid = txid - withdrawal_request.our_fee_amount = tron_manager.get_amount_from_base_denomination(withdrawal_fee_sun) - withdrawal_request.save(update_fields=['state', 'txid', 'updated', 'our_fee_amount']) - receipt = res.wait() - log.info(receipt) - log.info('TRX withdrawal TX %s sent', txid) - - -@shared_task -def withdraw_trc20(withdrawal_request_id, password): - withdrawal_request = WithdrawalRequest.objects.get(id=withdrawal_request_id) - - address = withdrawal_request.data.get('destination') - currency = withdrawal_request.currency - - token = tron_manager.get_token_by_symbol(currency) - send_amount_sun = token.get_base_denomination_from_amount(withdrawal_request.amount) - withdrawal_fee_sun = token.get_base_denomination_from_amount(token.withdrawal_fee) - amount_to_send_sun = send_amount_sun - withdrawal_fee_sun - - if amount_to_send_sun <= 0: - log.error('Invalid withdrawal amount') - withdrawal_request.fail() - return - - keeper = tron_manager.get_keeper_wallet() - keeper_trx_balance = tron_manager.get_balance_in_base_denomination(keeper.address) - keeper_token_balance = token.get_base_denomination_balance(keeper.address) - - if keeper_trx_balance < TRX_NET_FEE: - log.warning('Keeper not enough TRX, skipping') - return - - if keeper_token_balance < amount_to_send_sun: - log.warning('Keeper not enough %s, skipping', currency) - return - - private_key = AESCoderDecoder(password).decrypt(keeper.private_key) - - res = token.send_token(private_key, address, amount_to_send_sun) - txid = res.get('txid') - - if not res.get('result') or not txid: - log.error('Unable to send TRX TX') - - withdrawal_request.state = WR_PENDING - withdrawal_request.txid = txid - withdrawal_request.our_fee_amount = token.get_amount_from_base_denomination(withdrawal_fee_sun) - withdrawal_request.save(update_fields=['state', 'txid', 'updated', 'our_fee_amount']) - receipt = res.wait() - log.info(receipt) - log.info('%s withdrawal TX %s sent', currency, txid) - - -@shared_task -def check_deposit_scoring(wallet_transaction_id): - """Check deposit for scoring""" - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - wallet_transaction.check_scoring() - - -@shared_task -def check_balances(): - """Main accumulations scheduler""" - kyt_check_jobs = [] - accumulations_jobs = [] - external_accumulations_jobs = [] - - for item in accumulation_manager.get_waiting_for_kyt_check(TRX_CURRENCY): - kyt_check_jobs.append(check_deposit_scoring.s(item.id)) - - for item in accumulation_manager.get_waiting_for_accumulation(TRX_CURRENCY): - accumulations_jobs.append(check_balance.s(item.id)) - - for item in accumulation_manager.get_waiting_for_external_accumulation(blockchain_currency=TRX_CURRENCY): - external_accumulations_jobs.append(check_balance.s(item.id)) - - if kyt_check_jobs: - log.info('Need to check for KYT: %s', len(kyt_check_jobs)) - jobs_group = group(kyt_check_jobs) - jobs_group.apply_async() - - if accumulations_jobs: - log.info('Need to check accumulations: %s', len(accumulations_jobs)) - jobs_group = group(accumulations_jobs) - jobs_group.apply_async() - - if external_accumulations_jobs: - log.info('Need to check external accumulations: %s', len(external_accumulations_jobs)) - jobs_group = group(external_accumulations_jobs) - jobs_group.apply_async() - - -@shared_task -def check_balance(wallet_transaction_id): - """Splits blockchain currency accumulation and token accumulation""" - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - currency = wallet_transaction.currency - - # TRX - if currency == TRX_CURRENCY: - wallet_transaction.set_ready_for_accumulation() - accumulate_trx.apply_async([wallet_transaction_id]) - # tokens - else: - wallet_transaction.set_ready_for_accumulation() - accumulate_trc20.apply_async([wallet_transaction_id]) - - -@shared_task -def accumulate_trx(wallet_transaction_id): - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - address = wallet_transaction.wallet.address - - amount = wallet_transaction.amount - amount_sun = tron_manager.get_base_denomination_from_amount(amount) - - log.info('Accumulation TRX from: %s; Balance: %s; Min acc balance:%s', - address, amount, tron_manager.accumulation_min_balance) - - # minus coins to be burnt - withdrawal_amount = amount_sun - TRX_NET_FEE - - # in debug mode values can be very small - if withdrawal_amount <= 0: - log.error(f'TRX withdrawal amount invalid: {withdrawal_amount}') - wallet_transaction.set_balance_too_low() - return - - accumulation_address = wallet_transaction.external_accumulation_address or tron_manager.get_accumulation_address(amount) - - # prepare tx - wallet = tron_manager.get_user_wallet('TRX', address) - - res = tron_manager.send_tx(wallet.private_key, accumulation_address, withdrawal_amount) - txid = res.get('txid') - - if not res.get('result') or not txid: - log.error('Unable to send withdrawal TX') - - AccumulationTransaction.objects.create( - wallet_transaction=wallet_transaction, - amount=tron_manager.get_amount_from_base_denomination(withdrawal_amount), - tx_type=AccumulationTransaction.TX_TYPE_ACCUMULATION, - tx_state=AccumulationTransaction.STATE_PENDING, - tx_hash=txid, - ) - wallet_transaction.set_accumulation_in_progress() - # AccumulationDetails.objects.create( - # currency=TRX_CURRENCY, - # txid=txid, - # from_address=address, - # to_address=accumulation_address - # ) - - reciept = res.wait() - log.info(reciept) - log.info(f'Accumulation TX {txid} sent from {wallet.address} to {accumulation_address}') - - -@shared_task -def accumulate_trc20(wallet_transaction_id): - wallet_transaction = accumulation_manager.get_wallet_transaction_by_id(wallet_transaction_id) - address = wallet_transaction.wallet.address - currency = wallet_transaction.currency - - token = tron_manager.get_token_by_symbol(currency) - token_amount = wallet_transaction.amount - token_amount_sun = token.get_base_denomination_from_amount(token_amount) - - log.info(f'Accumulation {currency} from: {address}; Balance: {token_amount};') - - accumulation_address = wallet_transaction.external_accumulation_address or token.get_accumulation_address(token_amount) - - gas_keeper = tron_manager.get_gas_keeper_wallet() - - # send trx from gas keeper to send tokens - log.info('Trying to send token fee from GasKeeper') - res = tron_manager.send_tx(gas_keeper.private_key, address, TRC20_FEE_LIMIT) - gas_txid = res.get('txid') - - if not res.get('result') or not gas_txid: - log.error('Unable to send fee TX') - - acc_transaction = AccumulationTransaction.objects.create( - wallet_transaction=wallet_transaction, - amount=tron_manager.get_amount_from_base_denomination(TRC20_FEE_LIMIT), - tx_type=AccumulationTransaction.TX_TYPE_GAS_DEPOSIT, - tx_state=AccumulationTransaction.STATE_PENDING, - tx_hash=gas_txid, - ) - wallet_transaction.set_waiting_for_gas() - - receipt = res.wait() - log.info(receipt) - - acc_transaction.complete(is_gas=True) - - wallet = tron_manager.get_user_wallet(currency, address) - res = token.send_token(wallet.private_key, accumulation_address, token_amount_sun) - txid = res.get('txid') - - if not res.get('result') or not txid: - log.error('Unable to send withdrawal token TX') - - AccumulationTransaction.objects.create( - wallet_transaction=wallet_transaction, - amount=token.get_amount_from_base_denomination(token_amount_sun), - tx_type=AccumulationTransaction.TX_TYPE_ACCUMULATION, - tx_state=AccumulationTransaction.STATE_PENDING, - tx_hash=txid, - ) - wallet_transaction.set_accumulation_in_progress() - - receipt = res.wait() - log.info(receipt) - log.info('Token accumulation TX %s sent from %s to: %s', txid, wallet.address, accumulation_address) - - -# @shared_task -# def accumulate_trx_dust(): -# tron_manager.accumulate_dust() diff --git a/cryptocoins/utils/btc.py b/cryptocoins/utils/btc.py index 474aaad..05c6afb 100644 --- a/cryptocoins/utils/btc.py +++ b/cryptocoins/utils/btc.py @@ -4,7 +4,6 @@ from cryptos import Bitcoin from django.conf import settings -from pywallet.wallet import create_wallet from core.models import UserWallet from cryptocoins.coins.btc import BTC_CURRENCY diff --git a/cryptocoins/utils/infura.py b/cryptocoins/utils/infura.py index 7ad8bd2..45649d3 100644 --- a/cryptocoins/utils/infura.py +++ b/cryptocoins/utils/infura.py @@ -1,11 +1,7 @@ from typing import Dict +from django.conf import settings from web3 import Web3 -from web3.auto.infura.endpoints import ( - INFURA_MAINNET_DOMAIN, - build_http_headers, - build_infura_url, -) from web3.providers import ( HTTPProvider, ) @@ -22,7 +18,7 @@ def get_request_headers(self) -> Dict[str, str]: def get_web3(): - provider = CustomHttpProvider(build_infura_url(INFURA_MAINNET_DOMAIN), build_http_headers()) + provider = CustomHttpProvider(f'https://mainnet.infura.io/v3/{settings.INFURA_API_KEY}') web3 = Web3(provider) return web3 diff --git a/cryptocoins/utils/register.py b/cryptocoins/utils/register.py index fc65a06..d0affa7 100644 --- a/cryptocoins/utils/register.py +++ b/cryptocoins/utils/register.py @@ -19,8 +19,7 @@ def register_coin(currency_id: int, currency_code: str, *, address_validation_fn: Optional[Callable] = None, wallet_creation_fn: Optional[Callable] = None, latest_block_fn: Optional[Callable] = None, - blocks_diff_alert: Optional[int] = None, - encrypted_cold_wallet: Optional[bytes] = None): + blocks_diff_alert: Optional[int] = None): currency = Currency(currency_id, currency_code) if currency not in ALL_CURRENCIES: @@ -42,7 +41,6 @@ def register_coin(currency_id: int, currency_code: str, *, currency: CoinParams( latest_block_fn=latest_block_fn, blocks_monitoring_diff=blocks_diff_alert, - encrypted_cold_wallet=encrypted_cold_wallet, ) }) diff --git a/cryptocoins/utils/wallet.py b/cryptocoins/utils/wallet.py index 2a3b398..111c6f5 100644 --- a/cryptocoins/utils/wallet.py +++ b/cryptocoins/utils/wallet.py @@ -1,26 +1,11 @@ import logging -from eth_account import Account -from eth_utils.curried import combomethod -from eth_utils.curried import keccak -from eth_utils.curried import text_if_str -from eth_utils.curried import to_bytes - from core.consts.currencies import BlockchainAccount from lib.cryptointegrator.tasks import create_wallet log = logging.getLogger(__name__) -class PassphraseAccount(Account): - - @combomethod - def create(self, passphrase): - extra_key_bytes = text_if_str(to_bytes, passphrase) - key_bytes = keccak(extra_key_bytes) - return self.privateKeyToAccount(key_bytes) - - def get_wallet_data(user_id, currency, is_new=False): from core.models.cryptocoins import UserWallet diff --git a/dashboard_rest/__init__.py b/dashboard_rest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dashboard_rest/admin_rest.py b/dashboard_rest/admin_rest.py new file mode 100644 index 0000000..05357b2 --- /dev/null +++ b/dashboard_rest/admin_rest.py @@ -0,0 +1,257 @@ +import datetime +import logging +from decimal import Decimal + +from django.db import models +from django.db.models import Sum, Case, When, F, Count, Q +from django.utils import timezone +from rest_framework.response import Response + +from admin_rest import restful_admin as api_admin +from admin_rest.mixins import ReadOnlyMixin, NonPaginatedListMixin +from admin_rest.restful_admin import DefaultApiAdmin +from core.consts.currencies import ALL_CURRENCIES +from core.enums.profile import UserTypeEnum +from core.models import Order +from core.pairs import PAIRS +from core.utils.stats.counters import CurrencyStats +from core.utils.stats.lib import get_prices_in_usd +from dashboard_rest.models import CommonInouts, CommonUsersStats, TradeVolume +from dashboard_rest.models import Topups +from dashboard_rest.models import TradeFee +from dashboard_rest.models import Withdrawals + +log = logging.getLogger(__name__) + + +@api_admin.register(Topups) +class DashboardTopupsAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = ['created', 'user', 'currency', 'amount', 'state'] + + def get_queryset(self): + return Topups.objects.order_by('-created').prefetch_related('wallet', 'wallet__user') + + def user(self, obj): + return obj.wallet.user.email + + +@api_admin.register(Withdrawals) +class DashboardWithdrawalsAdmin(ReadOnlyMixin, DefaultApiAdmin): + list_display = ['created', 'user', 'currency', 'amount', 'state'] + + def get_queryset(self): + return Withdrawals.objects.order_by('-created') + + +@api_admin.register(TradeFee) +class TradeFeeAdmin(ReadOnlyMixin, NonPaginatedListMixin, DefaultApiAdmin): + list_display = ['currency', 'trade_fee_amount', 'withdrawal_fee_amount'] + filterset_fields = ['created'] + + def currency(self, obj): + """Do not delete""" + + def trade_fee_amount(self, obj): + """Do not delete""" + + def withdrawal_fee_amount(self, obj): + """Do not delete""" + + def list(self, request, *args, **kwargs): + interval = { + 'start': request.query_params.get('created[start]') or datetime.date(datetime.MINYEAR, 1, 1), + 'end': request.query_params.get('created[end]') or datetime.date(datetime.MAXYEAR, 1, 1) + } + trade_stats = CurrencyStats.trade_fee(interval) + withdrawals_stats = CurrencyStats.withdrawals_and_fees(interval) + res = [] + for currency in ALL_CURRENCIES: + res.append({ + 'currency': currency.code, + 'trade_fee_amount': trade_stats.get(currency, {}).get('total_fee') or 0, + 'withdrawal_fee_amount': withdrawals_stats.get(currency, {}).get('total_fee') or 0, + }) + return Response({'results': res}) + + +@api_admin.register(CommonUsersStats) +class CommonUsersStatsAdmin(ReadOnlyMixin, NonPaginatedListMixin, DefaultApiAdmin): + list_display = ['stat_name', 'stat_value'] + filterset_fields = ['date_joined'] + + def stat_name(self, obj): + pass + + def stat_value(self, obj): + pass + + def list(self, request, *args, **kwargs): + now = timezone.now() + interval = { + 'start': request.query_params.get('date_joined[start]') or now - datetime.timedelta(days=1), + 'end': request.query_params.get('date_joined[end]') or datetime.date(datetime.MAXYEAR, 1, 1) + } + total_users = CommonUsersStats.objects.filter( + date_joined__gte=interval['start'], + date_joined__lte=interval['end'], + ).aggregate( + total_users=Count('id'), + )['total_users'] or 0 + res = [ + { + 'stat_name': 'Users count', + 'stat_value': total_users, + } + ] + return Response({'results': res}) + + +@api_admin.register(CommonInouts) +class CommonInoutsAdmin(ReadOnlyMixin, NonPaginatedListMixin, DefaultApiAdmin): + list_display = ['currency', 'topups', 'withdrawals'] + filterset_fields = ['created'] + + def currency(self, obj): + pass + + def topups(self, obj): + pass + + def withdrawals(self, obj): + pass + + def list(self, request, *args, **kwargs): + res = [] + now = timezone.now() + interval = { + 'start': request.query_params.get('created[start]') or now - datetime.timedelta(days=1), + 'end': request.query_params.get('created[end]') or datetime.date(datetime.MAXYEAR, 1, 1) + } + + prices_in_usd = get_prices_in_usd() + topups = CurrencyStats.topups( + interval, + filter_qs=[~Q( + Q(user__profile__user_type=UserTypeEnum.staff.value) + | Q(user__profile__user_type=UserTypeEnum.bot.value) + | Q(user__email__endswith='@bot.com') + )], + ) + withdrawals = CurrencyStats.withdrawals_and_fees( + interval, + filter_qs=[~Q( + Q(user__profile__user_type=UserTypeEnum.staff.value) + | Q(user__profile__user_type=UserTypeEnum.bot.value) + | Q(user__email__endswith='@bot.com') + )], + ) + + total_topus = 0 + total_withdrawals = 0 + + for currency in ALL_CURRENCIES: + usd_price = prices_in_usd.get(currency) or 0 + + topups_amount = round(topups.get(currency, {}).get('total_amount') or Decimal('0'), 8) + topups_amount_usd = round(topups_amount * usd_price, 2) + withdrawals_amount = round(withdrawals.get(currency, {}).get('total_amount') or Decimal('0'), 8) + withdrawals_amount_usd = round(withdrawals_amount * usd_price, 2) + + total_topus += topups_amount_usd + total_withdrawals += withdrawals_amount_usd + + res.append({ + 'currency': currency.code, + 'topups': f'{topups_amount.normalize()} (${topups_amount_usd})', + 'withdrawals': f'{withdrawals_amount.normalize()} (${withdrawals_amount_usd})', + }) + res.append({ + 'currency': 'Total', + 'topups': f'${total_topus}', + 'withdrawals': f'${total_withdrawals}', + }) + return Response({'results': res}) + + +@api_admin.register(TradeVolume) +class TradeVolumeAdmin(ReadOnlyMixin, NonPaginatedListMixin, DefaultApiAdmin): + list_display = ['pair', 'base_volume', 'quote_volume'] + filterset_fields = ['created'] + + def pair(self, obj): + pass + + def base_volume(self, obj): + pass + + def quote_volume(self, obj): + pass + + def list(self, request, *args, **kwargs): + interval = { + 'start': request.query_params.get('created[start]') or datetime.date(datetime.MINYEAR, 1, 1), + 'end': request.query_params.get('created[end]') or datetime.date(datetime.MAXYEAR, 1, 1) + } + qs = TradeVolume.objects.filter( + ~Q( + Q(user__profile__user_type=UserTypeEnum.staff.value) + | Q(user__profile__user_type=UserTypeEnum.bot.value) + | Q(user__email__endswith='@bot.com') + ), + created__gte=interval['start'], + created__lte=interval['end'], + cancelled=False, + ).values('pair').annotate( + base_volume=Sum( + Case( + When(order__operation=Order.OPERATION_BUY, then=F('quantity')), + default=0, + output_field=models.DecimalField(), + ) + ), + quote_volume=Sum( + Case( + When(order__operation=Order.OPERATION_BUY, then=F('quantity') * F('price')), + default=0, + output_field=models.DecimalField(), + ) + ), + ).order_by('pair') + + volumes_dict = {p['pair']: p for p in qs} + + volumes = [] + for pair in PAIRS: + volumes.append({ + 'pair': pair, + 'base_volume': round(volumes_dict.get(pair, {}).get('base_volume', 0), 8), + 'quote_volume': round(volumes_dict.get(pair, {}).get('quote_volume', 0), 8), + }) + + total = Decimal(0) + prices_in_usd = get_prices_in_usd() + + for i in volumes: + pair = i['pair'] + if pair.quote.code == 'USDT': + vol = i['quote_volume'] + elif pair.base.code == 'USDT': + vol = i['base_volume'] + elif pair.quote in prices_in_usd: + vol = prices_in_usd[pair.quote] * i['quote_volume'] + elif pair.base in prices_in_usd: + vol = prices_in_usd[pair.base] * i['base_volume'] + + total += vol + + total = round(total, 2) + + volumes.append({ + 'pair': 'Total', + 'base_volume': f'${total}', + 'quote_volume': f'${total}', + }) + + return Response({'results': volumes}) + + diff --git a/dashboard_rest/apps.py b/dashboard_rest/apps.py new file mode 100644 index 0000000..6e104db --- /dev/null +++ b/dashboard_rest/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class DashboardConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'dashboard_rest' diff --git a/dashboard_rest/migrations/0001_initial.py b/dashboard_rest/migrations/0001_initial.py new file mode 100644 index 0000000..7ff66f0 --- /dev/null +++ b/dashboard_rest/migrations/0001_initial.py @@ -0,0 +1,97 @@ +# Generated by Django 3.2.7 on 2022-08-12 12:45 + +import django.contrib.auth.models +from django.db import migrations + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('core', '0001_initial'), + ('auth', '0012_alter_user_first_name_max_length'), + ] + + operations = [ + migrations.CreateModel( + name='CommonInouts', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.wallettransactions',), + ), + migrations.CreateModel( + name='CommonUsersStats', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('auth.user',), + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + migrations.CreateModel( + name='Topups', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.wallettransactions',), + ), + migrations.CreateModel( + name='TradeFee', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.executionresult',), + ), + migrations.CreateModel( + name='TradeVolume', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.executionresult',), + ), + migrations.CreateModel( + name='WithdrawalFee', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.withdrawalrequest',), + ), + migrations.CreateModel( + name='Withdrawals', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('core.withdrawalrequest',), + ), + ] diff --git a/dashboard_rest/migrations/__init__.py b/dashboard_rest/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dashboard_rest/models.py b/dashboard_rest/models.py new file mode 100644 index 0000000..bb6aafc --- /dev/null +++ b/dashboard_rest/models.py @@ -0,0 +1,40 @@ +from django.contrib.auth.models import User + +from core.models.inouts.wallet import WalletTransactions +from core.models.inouts.withdrawal import WithdrawalRequest +from core.models.orders import ExecutionResult + + +class Topups(WalletTransactions): + class Meta: + proxy = True + + +class Withdrawals(WithdrawalRequest): + class Meta: + proxy = True + + +class TradeFee(ExecutionResult): + class Meta: + proxy = True + + +class WithdrawalFee(WithdrawalRequest): + class Meta: + proxy = True + + +class CommonUsersStats(User): + class Meta: + proxy = True + + +class CommonInouts(WalletTransactions): + class Meta: + proxy = True + + +class TradeVolume(ExecutionResult): + class Meta: + proxy = True diff --git a/dashboard_rest/tests.py b/dashboard_rest/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/dashboard_rest/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/dashboard_rest/views.py b/dashboard_rest/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/dashboard_rest/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/exchange/celery_app.py b/exchange/celery_app.py index 49559ee..e60a72b 100644 --- a/exchange/celery_app.py +++ b/exchange/celery_app.py @@ -12,6 +12,7 @@ django.setup() from lib.cryptointegrator.tasks import generate_crypto_schedule +from cryptocoins.evm.manager import evm_handlers_manager from kombu import Queue from django.conf import settings from celery.schedules import crontab @@ -60,6 +61,10 @@ def is_section_enabled(name): app.conf.task_queues += tuple(generated_queues) +# evm coins tasks +evm_queues = evm_handlers_manager.register_celery_tasks(app.conf.beat_schedule) +app.conf.task_queues += tuple(evm_queues) + if is_section_enabled('payout_withdraw'): app.conf.beat_schedule.update({ 'sci_process_withdrawals': { @@ -86,13 +91,6 @@ def is_section_enabled(name): if is_section_enabled('cryptocoins_commons'): app.conf.beat_schedule.update({ - 'check_accumulations': { - 'task': 'cryptocoins.tasks.commons.check_accumulations', - 'schedule': 20, - 'options': { - 'queue': 'cryptocoins_commons', - } - }, 'mark_accumulated_topups': { 'task': 'cryptocoins.tasks.commons.mark_accumulated_topups', 'schedule': 60, @@ -113,271 +111,6 @@ def is_section_enabled(name): }), app.conf.task_queues += (Queue('cryptocoins_commons'),) -if is_section_enabled('ethereum'): - app.conf.beat_schedule.update({ - 'eth_process_new_blocks': { - 'task': 'cryptocoins.tasks.eth.eth_process_new_blocks', - 'schedule': settings.ETH_BLOCK_GENERATION_TIME, - }, - 'eth_check_balances': { - 'task': 'cryptocoins.tasks.eth.check_balances', - 'schedule': settings.ETH_ERC20_ACCUMULATION_PERIOD, - 'options': { - 'expires': 20, - } - }, - # 'process_payouts': { - # 'task': 'cryptocoins.tasks.eth.process_payouts', - # 'schedule': settings.ETH_ERC20_ACCUMULATION_PERIOD, - # 'options': { - # 'expires': 20, - # } - # }, - }) - app.conf.task_routes.update({ - 'cryptocoins.tasks.eth.eth_process_new_blocks': { - 'queue': 'eth_new_blocks', - }, - 'cryptocoins.tasks.eth.eth_process_block': { - 'queue': 'eth_new_blocks', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.eth.eth_process_eth_deposit': { - 'queue': 'eth_deposits', - }, - 'cryptocoins.tasks.eth.eth_process_erc20_deposit': { - 'queue': 'eth_deposits', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.eth.process_payouts': { - 'queue': 'eth_payouts', - }, - 'cryptocoins.tasks.eth.withdraw_eth': { - 'queue': 'eth_payouts', - }, - 'cryptocoins.tasks.eth.withdraw_erc20': { - 'queue': 'eth_payouts', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.eth.check_balances': { - 'queue': 'eth_check_balances', - }, - 'cryptocoins.tasks.eth.check_balance': { - 'queue': 'eth_check_balances', - }, - 'cryptocoins.tasks.eth.check_tx_withdrawal': { - 'queue': 'eth_check_balances', - }, - 'cryptocoins.tasks.eth.check_deposit_scoring': { - 'queue': 'eth_check_balances', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.eth.accumulate_eth': { - 'queue': 'eth_accumulations', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.eth.accumulate_erc20': { - 'queue': 'erc20_accumulations', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.eth.send_gas': { - 'queue': 'eth_send_gas', - }, - }), - - app.conf.task_queues += ( - Queue('eth_new_blocks'), - Queue('eth_deposits'), - Queue('eth_payouts'), - Queue('eth_check_balances'), - Queue('eth_accumulations'), - Queue('erc20_accumulations'), - Queue('eth_send_gas'), - ) - -if is_section_enabled('bnb'): - app.conf.beat_schedule.update({ - 'bnb_process_new_blocks': { - 'task': 'cryptocoins.tasks.bnb.bnb_process_new_blocks', - 'schedule': settings.BNB_BLOCK_GENERATION_TIME, - }, - 'bnb_check_balances': { - 'task': 'cryptocoins.tasks.bnb.check_balances', - 'schedule': settings.BNB_BEP20_ACCUMULATION_PERIOD, - 'options': { - 'expires': 20, - } - }, - # 'cryptocoins.tasks.bnb.accumulate_bnb_dust': { - # 'task': 'cryptocoins.tasks.bnb.accumulate_bnb_dust', - # 'schedule': crontab(minute='5', hour='0'), - # 'options': { - # 'queue': 'bnb_accumulations', - # } - # }, - # 'process_payouts': { - # 'task': 'cryptocoins.tasks.bnb.process_payouts', - # 'schedule': settings.BNB_BEP20_ACCUMULATION_PERIOD, - # 'options': { - # 'expires': 20, - # } - # }, - }) - app.conf.task_routes.update({ - 'cryptocoins.tasks.bnb.bnb_process_new_blocks': { - 'queue': 'bnb_new_blocks', - }, - 'cryptocoins.tasks.bnb.bnb_process_block': { - 'queue': 'bnb_new_blocks', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.bnb.bnb_process_bnb_deposit': { - 'queue': 'bnb_deposits', - }, - 'cryptocoins.tasks.bnb.bnb_process_bep20_deposit': { - 'queue': 'bnb_deposits', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.bnb.process_payouts': { - 'queue': 'bnb_payouts', - }, - 'cryptocoins.tasks.bnb.withdraw_bnb': { - 'queue': 'bnb_payouts', - }, - 'cryptocoins.tasks.bnb.withdraw_bep20': { - 'queue': 'bnb_payouts', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.bnb.check_balances': { - 'queue': 'bnb_check_balances', - }, - 'cryptocoins.tasks.bnb.check_balance': { - 'queue': 'bnb_check_balances', - }, - 'cryptocoins.tasks.bnb.check_tx_withdrawal': { - 'queue': 'bnb_check_balances', - }, - 'cryptocoins.tasks.bnb.check_deposit_scoring': { - 'queue': 'bnb_check_balances', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.bnb.accumulate_bnb': { - 'queue': 'bnb_accumulations', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.bnb.accumulate_bep20': { - 'queue': 'bep20_accumulations', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.bnb.send_gas': { - 'queue': 'bnb_send_gas', - }, - }), - - app.conf.task_queues += ( - Queue('bnb_new_blocks'), - Queue('bnb_deposits'), - Queue('bnb_payouts'), - Queue('bnb_check_balances'), - Queue('bnb_accumulations'), - Queue('bep20_accumulations'), - Queue('bnb_send_gas'), - ) - -if is_section_enabled('tron'): - app.conf.beat_schedule.update({ - 'trx_process_new_blocks': { - 'task': 'cryptocoins.tasks.trx.trx_process_new_blocks', - 'schedule': settings.TRX_BLOCK_GENERATION_TIME, - }, - 'trx_check_balances': { - 'task': 'cryptocoins.tasks.trx.check_balances', - 'schedule': settings.TRX_TRC20_ACCUMULATION_PERIOD, - 'options': { - 'expires': 20, - } - }, - # 'cryptocoins.tasks.trx.accumulate_trx_dust': { - # 'task': 'cryptocoins.tasks.trx.accumulate_trx_dust', - # 'schedule': crontab(minute='10', hour='0'), - # 'options': { - # 'queue': 'trx_accumulations', - # } - # }, - }) - app.conf.task_routes.update({ - 'cryptocoins.tasks.trx.trx_process_new_blocks': { - 'queue': 'trx_new_blocks', - }, - 'cryptocoins.tasks.trx.trx_process_block': { - 'queue': 'trx_new_blocks', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.trx.trx_process_trx_deposit': { - 'queue': 'trx_deposits', - }, - 'cryptocoins.tasks.trx.trx_process_trc20_deposit': { - 'queue': 'trx_deposits', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.trx.process_payouts': { - 'queue': 'trx_payouts', - }, - 'cryptocoins.tasks.trx.withdraw_trx': { - 'queue': 'trx_payouts', - }, - 'cryptocoins.tasks.trx.withdraw_trc20': { - 'queue': 'trx_payouts', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.trx.check_balances': { - 'queue': 'trx_check_balances', - }, - 'cryptocoins.tasks.trx.check_balance': { - 'queue': 'trx_check_balances', - }, - 'cryptocoins.tasks.trx.check_tx_withdrawal': { - 'queue': 'trx_check_balances', - }, - 'cryptocoins.tasks.trx.check_deposit_scoring': { - 'queue': 'trx_check_balances', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.trx.accumulate_trx': { - 'queue': 'trx_accumulations', - }, - }), - app.conf.task_routes.update({ - 'cryptocoins.tasks.trx.accumulate_trc20': { - 'queue': 'trc20_accumulations', - }, - }), - - app.conf.task_queues += ( - Queue('trx_new_blocks'), - Queue('trx_deposits'), - Queue('trx_payouts'), - Queue('trx_check_balances'), - Queue('trx_accumulations'), - Queue('trc20_accumulations'), - ) - if is_section_enabled('notifications'): app.conf.task_routes.update({ 'core.tasks.facade.pong': { @@ -572,7 +305,7 @@ def at_start(sender, **k): }, 'update_crypto_external_prices': { 'task': 'cryptocoins.tasks.datasources.update_crypto_external_prices', - 'schedule': 15.0, + 'schedule': settings.EXTERNAL_PRICES_FETCH_PERIOD, 'options': { 'queue': 'otc', } diff --git a/exchange/settings/admin.py b/exchange/settings/admin.py index 5c828c6..1215351 100644 --- a/exchange/settings/admin.py +++ b/exchange/settings/admin.py @@ -9,3 +9,62 @@ ENABLE_OTP_ADMIN = True ADMIN_MASTERPASS = env('ADMIN_MASTERPASS') ADMIN_BASE_URL = env('ADMIN_BASE_URL', default='control-panel') + +VUE_ADMIN_SIDE_MENU = [ + {'icon': 'mdi-view-dashboard', 'text': 'Dashboard', 'link': '/',}, + {'divider': True}, + {'icon': 'mdi-account-group', 'model': 'auth.group', 'text': 'Group permissions'}, + {'icon': 'mdi-vector-arrange-above', 'model': 'admin_rest.allordernobot', 'text': 'All orders no bot'}, + {'icon': 'mdi-vector-arrange-below', 'model': 'admin_rest.allorder', 'text': 'All orders'}, + {'icon': 'mdi-piggy-bank', 'model': 'admin_rest.balance', 'text': 'Balances'}, + {'icon': 'mdi-vector-combine', 'model': 'admin_rest.match', 'text': 'Matches'}, + {'icon': 'mdi-account', 'model': 'auth.user', 'text': 'Users'}, + {'icon': 'mdi-account', 'model': 'core.userfee', 'text': 'Fee Users'}, + {'icon': 'mdi-account-badge-horizontal-outline', 'model': 'core.userkyc', 'text': 'User kyc'}, + {'icon': 'mdi-arrow-decision-outline', 'model': 'admin_rest.transaction', 'text': 'Transactions'}, + {'icon': 'mdi-account-convert', 'model': 'admin_rest.userdailystat', 'text': 'User daily stats'}, + {'icon': 'mdi-bank-remove', 'model': 'core.disabledcoin', 'text': 'Coins Management'}, + {'icon': 'mdi-book-open-outline', 'model': 'core.coininfo', 'text': 'Coin Info'}, + {'icon': 'mdi-book-open-outline', 'model': 'seo.coinstaticpage', 'text': 'Coin static pages'}, + {'icon': 'mdi-book-open-outline', 'model': 'seo.coinstaticsubpage', 'text': 'Coin static sub pages'}, + {'icon': 'mdi-email-edit-outline', 'model': 'notifications.mailing', 'text': 'Mailing'}, + {'icon': 'mdi-settings-transfer', 'model': 'core.feesandlimits', 'text': 'Fees And Limits'}, + {'icon': 'mdi-settings-transfer', 'model': 'core.withdrawalfee', 'text': 'Withdrawal Fee'}, + {'icon': 'mdi-settings-transfer', 'model': 'core.withdrawallimitlevel', 'text': 'Withdrawal Limit Level'}, + {'icon': 'mdi-settings-transfer', 'model': 'core.withdrawaluserlimit', 'text': 'Withdrawal User Limit'}, + {'icon': 'mdi-settings-transfer', 'model': 'core.pairsettings', 'text': 'Pair Settings'}, + {'icon': 'mdi-bank-transfer', 'model': 'cryptocoins.depositswithdrawalsstats', 'text': 'TopUps and Withdrawals'}, + {'icon': 'mdi-swap-horizontal', 'model': 'core.inoutsstats', 'text': 'In/Out Currency Stats'}, + {'icon': 'mdi-incognito', 'model': 'core.accesslog', 'text': 'Access logs'}, + {'icon': 'mdi-wallet-outline', 'model': 'core.userwallet', 'text': 'User Wallets'}, + {'icon': 'mdi-menu', 'model': 'core.difbalance', 'text': 'Dif balances'}, + {'divider': True}, + {'heading': 'menu.withdrawals'}, + {'icon': 'mdi-bank-transfer-out', 'model': 'admin_rest.withdrawalrequest', 'text': 'Withdrawal requests'}, + {'icon': 'mdi-bitcoin', 'model': 'cryptocoins.btcwithdrawalapprove', 'text': 'BTC Withdrawal Approve'}, + {'icon': 'mdi-ethereum', 'model': 'cryptocoins.ethwithdrawalapprove', 'text': 'ETH Withdrawal Approve'}, + {'icon': 'mdi-coins', 'model': 'cryptocoins.trxwithdrawalapprove', 'text': 'TRX Withdrawal Approve'}, + {'icon': 'mdi-coins', 'model': 'cryptocoins.bnbwithdrawalapprove', 'text': 'BSC Withdrawal Approve'}, + {'divider': True}, + {'heading': 'menu.topups'}, + {'icon': 'mdi-bank-transfer-in', 'model': 'core.wallettransactions', 'text': 'Crypto TopUps'}, + {'icon': 'mdi-arrow-right-bold-box', 'model': 'core.paygatetopup', 'text': 'Paygate TopUps'}, + {'divider': True}, + {'heading': 'menu.bots'}, + {'icon': 'mdi-cogs', 'model': 'bots.botconfig', 'text': 'Bot configs'}, + {'divider': True}, + {'heading': 'menu.otp'}, + {'icon': 'mdi-two-factor-authentication', 'model': 'otp_totp.totpdevice', 'text': 'TOTP Devices'}, + {'divider': True}, + {'heading': 'Scoring'}, + {'icon': 'mdi-menu', 'model': 'cryptocoins.scoringsettings', 'text': 'Scoring Settings'}, + {'icon': 'mdi-menu', 'model': 'cryptocoins.transactioninputscore', 'text': 'Transaction Input Score'}, + {'divider': True}, + {'heading': 'Admin Logs'}, + {'icon': 'mdi-menu', 'model': 'admin.logentry', 'text': 'Admin logs'}, + {'divider': True}, + {'heading': 'Settings'}, + {'icon': 'mdi-cog-outline', 'model': 'core.settings', 'text': 'Settings'}, + {'icon': 'mdi-incognito', 'model': 'core.smsconfirmationhistory', 'text': 'SMS Confirmation History'}, +] + diff --git a/exchange/settings/common.py b/exchange/settings/common.py index a2e4979..15076bc 100644 --- a/exchange/settings/common.py +++ b/exchange/settings/common.py @@ -66,6 +66,8 @@ 'django_otp', 'django_otp.plugins.otp_totp', 'rangefilter', + 'admin_rest', + 'dashboard_rest', ] MIDDLEWARE = [ diff --git a/exchange/settings/crypto.py b/exchange/settings/crypto.py index 56bc9bf..5894ba4 100644 --- a/exchange/settings/crypto.py +++ b/exchange/settings/crypto.py @@ -51,7 +51,7 @@ ] TRX_NET_FEE = 3_000_000 # 3 TRX -TRC20_FEE_LIMIT = 100_000_000 # 100 TRX +TRC20_FEE_LIMIT = 30_000_000 # 30 TRX TRX_BLOCK_GENERATION_TIME = 3 TRX_TRC20_ACCUMULATION_PERIOD = 5 * 60.0 @@ -64,9 +64,7 @@ CRYPTO_KEY_OLD = env('CRYPTO_KEY_OLD', default='') CRYPTO_KEY = env('CRYPTO_KEY', default='') -# Infura auto client setup -os.environ['WEB3_INFURA_API_KEY'] = WEB3_INFURA_API_KEY -os.environ['WEB3_INFURA_API_SECRET'] = WEB3_INFURA_API_SECRET -os.environ['WEB3_INFURA_SCHEME'] = 'https' +INFURA_API_KEY = env('INFURA_API_KEY', default='') +INFURA_API_SECRET = env('WEB3_INFURA_API_SECRET', default='') MIN_COST_ORDER_CANCEL = 0.0000001 diff --git a/exchange/settings/exchange.py b/exchange/settings/exchange.py index bb89def..53c51a7 100644 --- a/exchange/settings/exchange.py +++ b/exchange/settings/exchange.py @@ -32,6 +32,7 @@ EXTERNAL_PRICES_DEVIATION_PERCENTS = 10 CRYPTOCOMPARE_DEVIATION_PERCENTS = 2 +EXTERNAL_PRICES_FETCH_PERIOD = env('EXTERNAL_PRICE_FETCH_PERIOD', default=15) # every N seconds FEE_USER = env('FEE_USER', default='fee@exchange.net') diff --git a/exchange/settings/simplejwt.py b/exchange/settings/simplejwt.py index 033c952..dd97c0d 100644 --- a/exchange/settings/simplejwt.py +++ b/exchange/settings/simplejwt.py @@ -1,3 +1,4 @@ +from datetime import timedelta """ default settings @@ -36,6 +37,7 @@ SIMPLE_JWT = { + 'ACCESS_TOKEN_LIFETIME': timedelta(minutes=1440), 'ROTATE_REFRESH_TOKENS': True, 'UPDATE_LAST_LOGIN': True, 'AUTH_HEADER_TYPES': ('Bearer', 'Token',), diff --git a/exchange/urls.py b/exchange/urls.py index d87daad..2c968b1 100644 --- a/exchange/urls.py +++ b/exchange/urls.py @@ -15,6 +15,7 @@ urlpatterns = [ path('api/', include('exchange.api_urls')), + path('apiadmin/', include('admin_rest.urls')), path(f'', include('admin_panel.urls')), path('', include('core.urls')), ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/lib/services/etherscan_client.py b/lib/services/etherscan_client.py index 5e367c2..6ea28a6 100644 --- a/lib/services/etherscan_client.py +++ b/lib/services/etherscan_client.py @@ -45,9 +45,9 @@ def get_address_tx_transfers(self, address, start_block=0, end_block=99999999, o 'hash': t['hash'], 'from': t['from'], 'to': t['to'], - 'fee': to_decimal(Web3.fromWei(int(t['gasPrice']) * int(t['gasUsed']), 'ether')), - 'amount': to_decimal(Web3.fromWei(int(t['value']), 'ether')), - 'value': to_decimal(Web3.fromWei(int(t['value']) + int(t['gasPrice']) * int(t['gasUsed']), 'ether')), + 'fee': to_decimal(Web3.from_wei(int(t['gasPrice']) * int(t['gasUsed']), 'ether')), + 'amount': to_decimal(Web3.from_wei(int(t['value']), 'ether')), + 'value': to_decimal(Web3.from_wei(int(t['value']) + int(t['gasPrice']) * int(t['gasUsed']), 'ether')), } for t in all_txs]) if only_eth_txs: diff --git a/notifications/admin_rest.py b/notifications/admin_rest.py new file mode 100644 index 0000000..fe2d83b --- /dev/null +++ b/notifications/admin_rest.py @@ -0,0 +1,42 @@ +from typing import List + +from django.contrib import messages + +from admin_rest import restful_admin as api_admin +from admin_rest.restful_admin import DefaultApiAdmin +from notifications.models.NotificationModels import Notification, Mailing + + +@api_admin.register(Notification) +class NotificationApiAdmin(DefaultApiAdmin): + fields = ['type', 'title', 'text'] + list_display = ['created', 'title', 'type', 'users_count'] + + def users_count(self, obj): + return obj.users.count() + + +@api_admin.register(Mailing) +class MailingApiAdmin(DefaultApiAdmin): + readonly_fields = ['created', 'last_processed'] + fields = ['created', 'subject', 'text', 'users', 'last_processed'] + list_display = ['subject', 'last_processed', 'created', ] + actions = ( + 'proceed', + ) + + def last_processed(self, obj: Mailing): + + processed = obj.processed.latest('pk') + if processed is not None: + return processed.created + + return '-' + + @api_admin.action(permissions=True) + def proceed(self, request, queryset: List[Mailing]): + try: + for item in queryset: + item.send() + except Exception as e: + messages.error(request, e) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 347cdef..b3a9a42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ behave==1.2.6 behave-django==1.4.0 billiard==3.6.4.0 bit==0.7.2 -bitarray==1.2.2 +bitarray==2.7.3 bitcoin==1.1.42 Brotli==1.0.9 cached-property==1.5.2 @@ -62,7 +62,7 @@ cosmospy==6.0.0 coverage==5.5 crypto==1.4.1 cryptography==39.0.1 -cryptos~=2.0.6 +cryptos==2.0.6 cssselect2==0.7.0 cytoolz==0.11.0 daphne==3.0.2 @@ -103,14 +103,14 @@ docker-pycreds==0.4.0 drf-spectacular==0.23.1 ecdsa==0.17.0 et-xmlfile==1.1.0 -eth-abi==2.1.1 -eth-account==0.5.6 -eth-hash==0.3.1 -eth-keyfile==0.5.1 -eth-keys==0.3.3 -eth-rlp==0.2.1 -eth-typing==2.2.2 -eth-utils==1.10.0 +eth-abi==4.0.0 +eth-account==0.8.0 +eth-hash==0.5.1 +eth-keyfile==0.6.1 +eth-keys==0.4.0 +eth-rlp==0.3.0 +eth-typing==3.3.0 +eth-utils==2.1.0 ethereum-input-decoder==0.2.2 executing==1.2.0 fabric==2.6.0 @@ -154,10 +154,10 @@ jedi==0.18.0 jeepney==0.7.1 Jinja2==3.0.1 jsonfield==3.1.0 -jsonrpcclient==4.0.0 +jsonrpcclient==4.0.3 jsonrpclib-pelix==0.4.2 -jsonrpcserver==5.0.3 -jsonschema==3.2.0 +jsonrpcserver==5.0.9 +jsonschema==4.17.3 keyring==23.1.0 keyrings.alt==4.1.0 kombu==5.2.4 @@ -187,7 +187,7 @@ packaging==23.0 paramiko==2.7.2 parse==1.19.0 parse-type==0.5.2 -parsimonious==0.8.1 +parsimonious==0.9.0 parso==0.8.2 path==16.2.0 path.py==12.5.0 @@ -196,13 +196,14 @@ pbkdf2==1.3 pexpect==4.8.0 pickleshare==0.7.5 Pillow==9.4.0 +pkgutil_resolve_name==1.3.10 platformdirs==2.3.0 pluggy==1.0.0 pre-commit==2.15.0 prettytable==2.2.0 prometheus-client==0.11.0 prompt-toolkit==3.0.37 -protobuf==3.20.3 +protobuf==4.22.3 psycopg2-binary==2.9.1 ptyprocess==0.7.0 pure-eval==0.2.2 @@ -241,20 +242,19 @@ python-socks==1.2.4 python-telegram-bot==13.15 python3-openid==3.2.0 pytz==2022.7.1 -pywallet @ https://github.com/polyxexchange/pywallet/archive/master.zip pyxdg==0.27 PyYAML==5.4.1 QDarkStyle==3.0.2 qrcode==7.3 QtPy==1.10.0 redis==3.5.3 -regex==2021.8.28 +regex==2023.3.23 requests==2.26.0 requests-file==1.5.1 requests-oauthlib==1.3.0 requests-toolbelt==0.9.1 rfc3986==1.5.0 -rlp==2.0.1 +rlp==3.0.0 SecretStorage==3.3.1 sentry-sdk==1.3.1 service-identity==21.1.0 @@ -276,7 +276,7 @@ toml==0.10.2 toolz==0.11.1 tornado==6.1 traitlets==5.1.0 -tronpy @ https://github.com/polyxexchange/tronpy/archive/refs/tags/0.2.10.zip +tronpy==0.4.0 twilio==6.50.1 Twisted==22.10.0 txaio==21.2.1 @@ -292,10 +292,10 @@ varint==1.0.2 vine==5.0.0 virtualenv==20.7.2 wcwidth==0.2.5 -web3==5.23.1 +web3==6.2.0 webencodings==0.5.1 websocket-client==1.2.1 -websockets==9.1 +websockets==11.0.2 xlrd==2.0.1 xlwt==1.3.0 yarl==1.6.3 diff --git a/seo/admin_rest.py b/seo/admin_rest.py new file mode 100644 index 0000000..8a84856 --- /dev/null +++ b/seo/admin_rest.py @@ -0,0 +1,57 @@ +from django.utils.safestring import mark_safe + +from admin_rest import restful_admin as api_admin +from admin_rest.restful_admin import DefaultApiAdmin +from lib.utils import get_domain +from seo.models import ( + Post, + Tag, + ContentPhoto, + CoinStaticPage, + CoinStaticSubPage, +) + + +@api_admin.register(ContentPhoto) +class ContentPhotoAdmin(DefaultApiAdmin): + vue_resource_extras = {'data_table': {'rowClick': 'edit'}} + list_display = ['title_ru', 'title_en', 'announce_image', 'link'] + search_fields = ['title_ru', 'title_en'] + + def link(self, obj): + link = f'https://{get_domain()}{obj.announce_image.url}' + return mark_safe(f'{link}') + + +@api_admin.register(Post) +class PostAdmin(DefaultApiAdmin): + vue_resource_extras = {'data_table': {'rowClick': 'edit'}} + fields = ['created', 'preview_image', 'tags', + 'slug_ru', 'title_ru', 'text_ru', 'meta_title_ru', 'meta_description_ru', + 'slug_en', 'title_en', 'text_en', 'meta_title_en', 'meta_description_en', + 'views_count',] + + list_display = ['created', 'preview_image', 'slug_ru', 'title_ru', 'views_count'] + search_fields = ['slug_ru', 'slug_en', 'title_ru', 'title_en', 'tags'] + + +@api_admin.register(Tag) +class TagAdmin(DefaultApiAdmin): + vue_resource_extras = {'data_table': {'rowClick': 'edit'}} + fields = ['slug', 'name_ru', 'title_ru', 'meta_title_ru', 'meta_description_ru', + 'name_en', 'title_en', 'meta_title_en', 'meta_description_en'] + list_display = ['slug', 'name_ru', 'title_ru'] + search_fields = ['slug', 'name_ru', 'title_ru', 'name_en', 'title_en'] + + +@api_admin.register(CoinStaticPage) +class CoinStaticPageApiAdmin(DefaultApiAdmin): + pass + + +@api_admin.register(CoinStaticSubPage) +class CoinStaticSubPageAdmin(DefaultApiAdmin): + fields = ('currency', 'slug_ru', 'slug_en', 'title_ru', 'title_en', 'content_ru', 'content_en', + 'meta_title_ru', 'meta_title_en', 'meta_description_ru', 'meta_description_en') + list_display = ('currency', 'slug_ru', 'slug_en', 'title_ru', 'title_en') + filterset_fields = ('currency', ) \ No newline at end of file diff --git a/wizard.py b/wizard.py index 2f786a5..1a587e2 100644 --- a/wizard.py +++ b/wizard.py @@ -465,7 +465,8 @@ def get_or_create(model_inst, curr, get_attrs, set_attrs: dict): 'is_autoorders_enabled': True, 'price_source': PairSettings.PRICE_SOURCE_EXTERNAL, 'custom_price': 0, - 'deviation': 0.99000000 + 'deviation': 0.99000000, + 'precisions': ['100', '10', '1', '0.1', '0.01'] }, BotConfig: { 'name': 'BTC-USDT', @@ -491,7 +492,8 @@ def get_or_create(model_inst, curr, get_attrs, set_attrs: dict): 'is_autoorders_enabled': True, 'price_source': PairSettings.PRICE_SOURCE_EXTERNAL, 'custom_price': 0, - 'deviation': 0.99000000 + 'deviation': 0.99000000, + 'precisions': ['100', '10', '1', '0.1', '0.01'] }, BotConfig: { 'name': 'ETH-USDT', @@ -517,7 +519,8 @@ def get_or_create(model_inst, curr, get_attrs, set_attrs: dict): 'is_autoorders_enabled': True, 'price_source': PairSettings.PRICE_SOURCE_EXTERNAL, 'custom_price': 0, - 'deviation': 0.0 + 'deviation': 0.0, + 'precisions': ['0.01', '0.001', '0.0001', '0.00001', '0.000001'] }, BotConfig: { 'name': 'TRX-USDT', @@ -543,7 +546,8 @@ def get_or_create(model_inst, curr, get_attrs, set_attrs: dict): 'is_autoorders_enabled': True, 'price_source': PairSettings.PRICE_SOURCE_EXTERNAL, 'custom_price': 0, - 'deviation': 0.0 + 'deviation': 0.0, + 'precisions': ['100', '10', '1', '0.1', '0.01'], }, BotConfig: { 'name': 'BNB-USDT',