From 479a563e1848a5f5e6b285458c0f68ed56a2b5e0 Mon Sep 17 00:00:00 2001 From: cryptosharks131 <38626122+cryptosharks131@users.noreply.github.com> Date: Mon, 28 Mar 2022 15:14:02 -0400 Subject: [PATCH] v1.0.5 (#56) --- gui/forms.py | 1 + gui/migrations/0023_repair_closures.py | 23 ++ gui/migrations/0024_autofees.py | 26 +++ .../0025_alter_closures_unique_together.py | 54 +++++ gui/models.py | 19 +- gui/serializers.py | 14 +- gui/templates/advanced.html | 56 ++++- gui/templates/autofees.html | 36 +++ gui/templates/autopilot.html | 2 +- gui/templates/balances.html | 6 +- gui/templates/base.html | 2 +- gui/templates/channel.html | 38 ++- gui/templates/channels.html | 10 +- gui/templates/closures.html | 2 +- gui/templates/fee_rates.html | 21 +- gui/templates/home.html | 23 +- gui/templates/payments.html | 12 +- gui/urls.py | 3 + gui/views.py | 219 ++++++++++++++---- jobs.py | 86 ++++++- rebalancer.py | 18 +- 21 files changed, 577 insertions(+), 94 deletions(-) create mode 100644 gui/migrations/0023_repair_closures.py create mode 100644 gui/migrations/0024_autofees.py create mode 100644 gui/migrations/0025_alter_closures_unique_together.py create mode 100644 gui/templates/autofees.html diff --git a/gui/forms.py b/gui/forms.py index e5d4d450..16c7594c 100644 --- a/gui/forms.py +++ b/gui/forms.py @@ -81,6 +81,7 @@ class AutoRebalanceForm(forms.Form): (5, 'ar_enabled'), (6, 'ar_max_cost'), (7, 'channel_state'), + (8, 'auto_fees'), ] class UpdateChannel(forms.Form): diff --git a/gui/migrations/0023_repair_closures.py b/gui/migrations/0023_repair_closures.py new file mode 100644 index 00000000..dfe387ef --- /dev/null +++ b/gui/migrations/0023_repair_closures.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.7 on 2022-03-11 23:31 + +from django.db import migrations, models +import django.utils.timezone + +class Migration(migrations.Migration): + + dependencies = [ + ('gui', '0022_auto_20220227_2235'), + ] + + operations = [ + migrations.AddField( + model_name='channels', + name='fees_updated', + field=models.DateTimeField(default=django.utils.timezone.now), + ), + migrations.AddField( + model_name='channels', + name='auto_fees', + field=models.BooleanField(default=False), + ), + ] \ No newline at end of file diff --git a/gui/migrations/0024_autofees.py b/gui/migrations/0024_autofees.py new file mode 100644 index 00000000..553be92e --- /dev/null +++ b/gui/migrations/0024_autofees.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2.7 on 2022-03-16 17:12 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('gui', '0023_repair_closures'), + ] + + operations = [ + migrations.CreateModel( + name='Autofees', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(default=django.utils.timezone.now)), + ('chan_id', models.CharField(max_length=20)), + ('peer_alias', models.CharField(max_length=32)), + ('setting', models.CharField(max_length=20)), + ('old_value', models.IntegerField()), + ('new_value', models.IntegerField()), + ], + ), + ] diff --git a/gui/migrations/0025_alter_closures_unique_together.py b/gui/migrations/0025_alter_closures_unique_together.py new file mode 100644 index 00000000..8317d7ad --- /dev/null +++ b/gui/migrations/0025_alter_closures_unique_together.py @@ -0,0 +1,54 @@ +# Generated by Django 3.2.7 on 2022-03-23 14:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gui', '0024_autofees'), + ] + + operations = [ + migrations.DeleteModel( + name='Resolutions', + ), + migrations.DeleteModel( + name='Closures', + ), + migrations.CreateModel( + name='Resolutions', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('chan_id', models.CharField(max_length=20)), + ('resolution_type', models.IntegerField()), + ('outcome', models.IntegerField()), + ('outpoint_tx', models.CharField(max_length=64)), + ('outpoint_index', models.IntegerField()), + ('amount_sat', models.BigIntegerField()), + ('sweep_txid', models.CharField(max_length=64)), + ], + ), + migrations.CreateModel( + name='Closures', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('chan_id', models.CharField(max_length=20)), + ('funding_txid', models.CharField(max_length=64)), + ('funding_index', models.IntegerField()), + ('closing_tx', models.CharField(max_length=64)), + ('remote_pubkey', models.CharField(max_length=66)), + ('capacity', models.BigIntegerField()), + ('close_height', models.IntegerField()), + ('settled_balance', models.BigIntegerField()), + ('time_locked_balance', models.BigIntegerField()), + ('close_type', models.IntegerField()), + ('open_initiator', models.IntegerField()), + ('close_initiator', models.IntegerField()), + ('resolution_count', models.IntegerField()), + ], + options={ + 'unique_together': {('funding_txid', 'funding_index')}, + }, + ), + ] diff --git a/gui/models.py b/gui/models.py index d6e5b703..2ef14957 100644 --- a/gui/models.py +++ b/gui/models.py @@ -96,6 +96,8 @@ class Channels(models.Model): ar_in_target = models.IntegerField(default=100) ar_out_target = models.IntegerField() ar_max_cost = models.IntegerField() + fees_updated = models.DateTimeField(default=timezone.now) + auto_fees = models.BooleanField(default=False) def save(self, *args, **kwargs): if not self.ar_out_target: @@ -169,7 +171,9 @@ class Meta: app_label = 'gui' class Closures(models.Model): - chan_id = models.CharField(max_length=20, primary_key=True) + chan_id = models.CharField(max_length=20) + funding_txid = models.CharField(max_length=64) + funding_index = models.IntegerField() closing_tx = models.CharField(max_length=64) remote_pubkey = models.CharField(max_length=66) capacity = models.BigIntegerField() @@ -182,9 +186,10 @@ class Closures(models.Model): resolution_count = models.IntegerField() class Meta: app_label = 'gui' + unique_together = (('funding_txid', 'funding_index'),) class Resolutions(models.Model): - chan_id = models.ForeignKey('Closures', on_delete=models.CASCADE) + chan_id = models.CharField(max_length=20) resolution_type = models.IntegerField() outcome = models.IntegerField() outpoint_tx = models.CharField(max_length=64) @@ -222,6 +227,16 @@ class Meta: app_label = 'gui' class Autopilot(models.Model): + timestamp = models.DateTimeField(default=timezone.now) + chan_id = models.CharField(max_length=20) + peer_alias = models.CharField(max_length=32) + setting = models.CharField(max_length=20) + old_value = models.IntegerField() + new_value = models.IntegerField() + class Meta: + app_label = 'gui' + +class Autofees(models.Model): timestamp = models.DateTimeField(default=timezone.now) chan_id = models.CharField(max_length=20) peer_alias = models.CharField(max_length=32) diff --git a/gui/serializers.py b/gui/serializers.py index b5ea05e3..c224e63f 100644 --- a/gui/serializers.py +++ b/gui/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers from rest_framework.relations import PrimaryKeyRelatedField -from .models import LocalSettings, Payments, PaymentHops, Invoices, Forwards, Channels, Rebalancer, Peers, Onchain, PendingHTLCs, FailedHTLCs +from .models import LocalSettings, Payments, PaymentHops, Invoices, Forwards, Channels, Rebalancer, Peers, Onchain, PendingHTLCs, FailedHTLCs, Closures, Resolutions ##FUTURE UPDATE 'exclude' TO 'fields' @@ -89,6 +89,18 @@ class Meta: model = Onchain exclude = [] +class ClosuresSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.ReadOnlyField() + class Meta: + model = Closures + exclude = [] + +class ResolutionsSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.ReadOnlyField() + class Meta: + model = Resolutions + exclude = [] + class PaymentHopsSerializer(serializers.HyperlinkedModelSerializer): payment_hash = PrimaryKeyRelatedField(read_only=True) class Meta: diff --git a/gui/templates/advanced.html b/gui/templates/advanced.html index 7b235639..6ff001b9 100644 --- a/gui/templates/advanced.html +++ b/gui/templates/advanced.html @@ -7,6 +7,60 @@

Advanced Channel Settings

+ + + + + + + + + + + + @@ -142,7 +196,7 @@

Update Local Settings

{% elif settings.key == 'LND-RetentionDays' %} - {% elif settings.key|slice:":3" == 'AR-' %} + {% elif settings.key|slice:":3" == 'AR-' or settings.key|slice:":3" == 'AF-' %} {% else %} diff --git a/gui/templates/autofees.html b/gui/templates/autofees.html new file mode 100644 index 00000000..43464197 --- /dev/null +++ b/gui/templates/autofees.html @@ -0,0 +1,36 @@ +{% extends "base.html" %} +{% block title %} {{ block.super }} - Autofees{% endblock %} +{% block content %} +{% load humanize %} +{% if autofees %} +
+

Autofees Logs

+
+
+ {% csrf_token %} + + +
+ 		+
+ {% csrf_token %} + + +
+ 		+
+ {% csrf_token %} + + +
+ 		+
+ {% csrf_token %} + + +
+ 		+
+ {% csrf_token %} + + +
+ 		+
+ {% csrf_token %} + + +
+ 		+
+ {% csrf_token %} + + +
+
Channel ID Peer Alias
+ + + + + + + + + {% for log in autofees %} + + + + + + + + + {% endfor %} +
TimestampChannel IDPeer AliasSettingOld ValueNew Value
{{ log.timestamp|naturaltime }}{{ log.chan_id }}{% if log.peer_alias == '' %}---{% else %}{{ log.peer_alias }}{% endif %}{{ log.setting }}{{ log.old_value }}{{ log.new_value }}
+
+{% endif %} +{% if not autofees %} +
+

No autofees logs to see here yet!

+
Experimental. This will allow LNDg to automatically act upon the suggestions found here.
+
+{% endif %} +{% endblock %} \ No newline at end of file diff --git a/gui/templates/autopilot.html b/gui/templates/autopilot.html index a6b20cec..80924151 100644 --- a/gui/templates/autopilot.html +++ b/gui/templates/autopilot.html @@ -16,7 +16,7 @@

Autopilot Logs

{% for log in autopilot %} - {{ log.timestamp|naturaltime }} + {{ log.timestamp|naturaltime }} {{ log.chan_id }} {% if log.peer_alias == '' %}---{% else %}{{ log.peer_alias }}{% endif %} {{ log.setting }} diff --git a/gui/templates/balances.html b/gui/templates/balances.html index 2c34fa63..8d9ad30d 100644 --- a/gui/templates/balances.html +++ b/gui/templates/balances.html @@ -37,9 +37,9 @@

Transactions

{% for transaction in transactions %} {{ transaction.tx_hash }} - {{ transaction.amount }} - {% if transaction.block_height == 0 %}{{ transaction.block_height }}{% else %}{{ transaction.block_height }}{% endif %} - {{ transaction.fee }} + {{ transaction.amount|intcomma }} + {% if transaction.block_height == 0 %}---{% else %}{{ transaction.block_height|intcomma }}{% endif %} + {{ transaction.fee|intcomma }} {{ transaction.label }} {% endfor %} diff --git a/gui/templates/base.html b/gui/templates/base.html index 05c3f196..d6421e2d 100644 --- a/gui/templates/base.html +++ b/gui/templates/base.html @@ -28,7 +28,7 @@

My Lnd Overview

diff --git a/gui/templates/channel.html b/gui/templates/channel.html index 679fa2f9..6485b3b9 100644 --- a/gui/templates/channel.html +++ b/gui/templates/channel.html @@ -25,15 +25,17 @@

Lifetime Rebalanced Out: {{ channel.amt_rebal_out|intcomma }} ({{ channel.r

Profitability

-

1-Day Revenue: {{ channel.revenue_1day|intcomma }} | Costs: {{ channel.costs_1day|intcomma }} | Profits: {{ channel.profits_1day|intcomma }} [{{ channel.profits_vol_1day }}] | Assisted Revenues: {{ channel.revenue_assist_1day|intcomma }} | APY: {{ channel.apy_1day }}%

-

7-Day Revenue: {{ channel.revenue_7day|intcomma }} | Costs: {{ channel.costs_7day|intcomma }} | Profits: {{ channel.profits_7day|intcomma }} [{{ channel.profits_vol_7day }}] | Assisted Revenues: {{ channel.revenue_assist_7day|intcomma }} | APY: {{ channel.apy_7day }}%

-

30-Day Revenue: {{ channel.revenue_30day|intcomma }} | Costs: {{ channel.costs_30day|intcomma }} | Profits: {{ channel.profits_30day|intcomma }} [{{ channel.profits_vol_30day }}] | Assisted Revenues: {{ channel.revenue_assist_30day|intcomma }} | APY: {{ channel.apy_30day }}%

-

Lifetime Revenue: {{ channel.revenue|intcomma }} | Costs: {{ channel.costs|intcomma }} | Profits: {{ channel.profits|intcomma }} [{{ channel.profits_vol }}] | Assisted Revenues: {{ channel.revenue_assist|intcomma }} | APY: {{ channel.apy }}%

+

1-Day Revenue: {{ channel.revenue_1day|intcomma }} | Costs: {{ channel.costs_1day|intcomma }} | Profits: {{ channel.profits_1day|intcomma }} [{{ channel.profits_vol_1day }}] | Assisted Revenues: {{ channel.revenue_assist_1day|intcomma }} | APY: {{ channel.apy_1day }}% | iAPY: {{ channel.assisted_apy_1day }}% | CV: {{ channel.cv_1day }}%

+

7-Day Revenue: {{ channel.revenue_7day|intcomma }} | Costs: {{ channel.costs_7day|intcomma }} | Profits: {{ channel.profits_7day|intcomma }} [{{ channel.profits_vol_7day }}] | Assisted Revenues: {{ channel.revenue_assist_7day|intcomma }} | APY: {{ channel.apy_7day }}% | iAPY: {{ channel.assisted_apy_7day }}% | CV: {{ channel.cv_7day }}%

+

30-Day Revenue: {{ channel.revenue_30day|intcomma }} | Costs: {{ channel.costs_30day|intcomma }} | Profits: {{ channel.profits_30day|intcomma }} [{{ channel.profits_vol_30day }}] | Assisted Revenues: {{ channel.revenue_assist_30day|intcomma }} | APY: {{ channel.apy_30day }}% | iAPY: {{ channel.assisted_apy_30day }}% | CV: {{ channel.cv_30day }}%

+

Lifetime Revenue: {{ channel.revenue|intcomma }} | Costs: {{ channel.costs|intcomma }} | Profits: {{ channel.profits|intcomma }} [{{ channel.profits_vol }}] | Assisted Revenues: {{ channel.revenue_assist|intcomma }} | APY: {{ channel.apy }}% | iAPY: {{ channel.assisted_apy }}% | CV: {{ channel.cv }}%

Channel Settings

+ + @@ -54,6 +56,16 @@

Channel Settings

+ + @@ -252,13 +264,14 @@

Last 5 Payments Sent

- - - + + + + - - + + {% for payment in payments %} @@ -266,6 +279,7 @@

Last 5 Payments Sent

+ @@ -318,6 +332,7 @@

Last 5 Failed HTLCs

+ @@ -325,11 +340,12 @@

Last 5 Failed HTLCs

{% for failed_htlc in failed_htlcs %} - - + + + diff --git a/gui/templates/channels.html b/gui/templates/channels.html index 753575ea..e5c228cb 100644 --- a/gui/templates/channels.html +++ b/gui/templates/channels.html @@ -20,11 +20,11 @@

Channel Performance

- + - + @@ -34,13 +34,13 @@

Channel Performance

- + - + - + diff --git a/gui/templates/closures.html b/gui/templates/closures.html index adec9a8c..402d83b8 100644 --- a/gui/templates/closures.html +++ b/gui/templates/closures.html @@ -23,7 +23,7 @@

Closures

- + diff --git a/gui/templates/fee_rates.html b/gui/templates/fee_rates.html index fa2b9801..20665bef 100644 --- a/gui/templates/fee_rates.html +++ b/gui/templates/fee_rates.html @@ -25,6 +25,15 @@

Suggested Fee Rates

+ + {% for channel in channels %} @@ -49,9 +58,19 @@

Suggested Fee Rates

- + + + {% endfor %}
AFUpdated 7Day Flow Out Rate Rebal Rate Active
+
+ {% csrf_token %} + + + + +
+ 		{{ channel.fees_updated|naturaltime }} {% if channel.net_routed_7day > 0 %}OUT{% elif channel.net_routed_7day < 0 %}IN{% else %}---{% endif %}{% if channel.net_routed_7day != 0 %} | {{ channel.net_routed_7day }}{% endif %} {{ channel.out_rate }} {{ channel.rebal_ppm }}
Timestamp HashValueFee PaidStatusValueFee PaidPPM PaidStatus Chan Out Alias Chan Out IDRouteKeysendRouteKeysend
{{ payment.payment_hash }} {{ payment.value|add:"0"|intcomma }} {{ payment.fee|intcomma }}{{ payment.ppm|intcomma }} {% if payment.status == 1 %}In-Flight{% elif payment.status == 2 %}Succeeded{% elif payment.status == 3 %}Failed{% else %}{{ payment.status }}{% endif %} {% if payment.status == 2 %}{% if payment.chan_out_alias == '' %}---{% else %}{{ payment.chan_out_alias }}{% endif %}{% else %}N/A{% endif %} {% if payment.status == 2 %}{{ payment.chan_out }}{% else %}N/A{% endif %} Chan In Alias Chan Out Alias Forward AmountActual Outbound Potential Fee HTLC Failure Failure Detail
{{ failed_htlc.timestamp|naturaltime }}{{ failed_htlc.chan_id_in }}{{ failed_htlc.chan_id_out }}{{ failed_htlc.chan_id_in }}{{ failed_htlc.chan_id_out }} {% if failed_htlc.chan_in_alias == '' %}---{% else %}{{ failed_htlc.chan_in_alias }}{% endif %} {% if failed_htlc.chan_out_alias == '' %}---{% else %}{{ failed_htlc.chan_out_alias }}{% endif %} {{ failed_htlc.amount|intcomma }}{{ failed_htlc.chan_out_liq|intcomma }} ({{ failed_htlc.chan_out_pending|intcomma }}) {{ failed_htlc.missed_fee|intcomma }} {% if failed_htlc.wire_failure == 15 %}Temporary Channel Failure{% elif failed_htlc.wire_failure == 18 %}Unknown Next Peer{% elif failed_htlc.wire_failure == 12 %}Fee Insufficient{% else %}{{ failed_htlc.wire_failure }}{% endif %} {% if failed_htlc.failure_detail == 1 %}---{% elif failed_htlc.failure_detail == 5 %}HTLC Exceeds Max{% elif failed_htlc.failure_detail == 6 %}Insufficient Balance{% elif failed_htlc.failure_detail == 20 %}Invalid Keysend{% elif failed_htlc.failure_detail == 22 %}Circular Route{% else %}{{ failed_htlc.failure_detail }}{% endif %} Peer Alias Capacity Outbound FlowAPYAPY | CV Out [Profit] | In Inbound Flow Outbound FlowAPYAPY | CV Out [Profit] | In Inbound Flow Updates
{{ channel.chan_id }} {% if channel.alias == '' %}{{ channel.remote_pubkey|slice:":12" }}{% else %}{{ channel.alias }}{% endif %}{{ channel.capacity }} M{{ channel.mil_capacity }} M {{ channel.amt_routed_out_7day|intcomma }} M ({{ channel.routed_out_7day }}) | {{ channel.amt_rebal_in_7day|intcomma }} M ({{ channel.rebal_in_7day }}){{ channel.apy_7day }}%{{ channel.apy_7day }}% | {{ channel.cv_7day }}% {{ channel.revenue_7day|intcomma }} [{{ channel.profits_7day|intcomma }}] | {{ channel.revenue_assist_7day|intcomma }} {{ channel.amt_routed_in_7day|intcomma }} M ({{ channel.routed_in_7day }}) | {{ channel.amt_rebal_out_7day|intcomma }} M ({{ channel.rebal_out_7day }}) {{ channel.amt_routed_out_30day|intcomma }} M ({{ channel.routed_out_30day }}) | {{ channel.amt_rebal_in_30day|intcomma }} M ({{ channel.rebal_in_30day }}){{ channel.apy_30day }}%{{ channel.apy_30day }}% | {{ channel.cv_30day }}% {{ channel.revenue_30day|intcomma }} [{{ channel.profits_30day|intcomma }}] | {{ channel.revenue_assist_30day|intcomma }} {{ channel.amt_routed_in_30day|intcomma }} M ({{ channel.routed_in_30day }}) | {{ channel.amt_rebal_out_30day|intcomma }} M ({{ channel.rebal_out_30day }}) {{ channel.updates }}%
{% if closure.chan_id == '0' %}---{% else %}{{ closure.chan_id }}{% endif %} {% if closure.alias == '' %}{{ closure.remote_pubkey|slice:":12" }}{% else %}{{ closure.alias }}{% endif %}{{ closure.capacity|intcomma }}{{ closure.capacity|intcomma }} {% if closure.closing_tx == '0000000000000000000000000000000000000000000000000000000000000000' %}---{% else %}{{ closure.closing_tx }}{% endif %} {{ closure.settled_balance|intcomma }} {{ closure.time_locked_balance|intcomma }} Max Cost iRate iBaseUpdated +
+ {% csrf_token %} + + + +
+
{{ channel.local_base_fee|intcomma }}{{ channel.ar_max_cost }}%{{ channel.ar_max_cost }}% {{ channel.remote_fee_rate|intcomma }} {{ channel.remote_base_fee|intcomma }}{{ channel.fees_updated|naturaltime }} +
+ {% csrf_token %} + + + + +
+
diff --git a/gui/templates/home.html b/gui/templates/home.html index a3d6ecc9..865ea030 100644 --- a/gui/templates/home.html +++ b/gui/templates/home.html @@ -4,7 +4,8 @@ {% load humanize %}

{{ node_info.alias }} | {{ node_info.identity_pubkey }}

-

Capacity: {{ capacity|intcomma }} | Active Channels: {{ node_info.num_active_channels }} / {{ total_channels }} | Peers: {{ node_info.num_peers }} | DB Size: {{ db_size }} GB

+

Public Capacity: {{ total_capacity|intcomma }} | Active Channels: {{ node_info.num_active_channels }} / {{ total_channels }} | Peers: {{ node_info.num_peers }} | DB Size: {% if db_size > 0 %}{{ db_size }} GB{% else %}---{% endif %}

+ {% if total_private > 0 %}

Private Capacity: {{ private_capacity|intcomma }} | Locked Liquidity: {{ private_outbound|intcomma }} | Active Private Channels: {{ active_private }} / {{ total_private }}

{% endif %}

Public Address: {% for info in node_info.uris %}{{ info }} | {% endfor %}

Lnd sync: {{ node_info.synced_to_graph }} | chain sync: {{ node_info.synced_to_chain }} | {% for info in node_info.chains %}{{ info }}{% endfor %} | {{ node_info.block_height }} | {{ node_info.block_hash }}

@@ -22,9 +23,11 @@

Lifetime Routed: {{ total_forwards|intcomma }} | Value: {{ total_value_forwa

7-Day Routed: {{ routed_7day|intcomma }} | Value: {{ routed_7day_amt|intcomma }} | Fees Earned: {{ earned_7day|intcomma }} [{{ 7day_routed_ppm|intcomma }}] | Onchain Fees: {{ onchain_costs_7day|intcomma }} | Offchain Fees: {{ total_7day_fees|intcomma }} [{{ 7day_payments_ppm|intcomma }}] | Percent Cost: {{ percent_cost_7day }}% | Profit/Outbound: {{ profit_per_outbound|intcomma }} [{{ profit_per_outbound_real|intcomma }}] | Outbound Utilization: {{ routed_7day_percent }}%

-

Inbound Liquidity: {{ inbound|intcomma }} | Outbound Liquidity: {{ outbound|intcomma }} | Liquidity Ratio: {{ liq_ratio }}%

-

Balance In Limbo: {{ limbo_balance|intcomma }} | Unsettled Liquidity: {{ unsettled|intcomma }} | Pending HTLCs: {{ pending_htlc_count }}

-

Closures | New Peers | AR Actions | Fee Rates | Autopilot Logs | Channel Performance | Keysends | Rebalancing | Advanced Settings

+

Total Inbound Liquidity: {{ sum_inbound|intcomma }} | Outbound Liquidity: {{ sum_outbound|intcomma }} | Liquidity Ratio: {{ liq_ratio }}%

+

Active Inbound Liquidity: {{ inbound|intcomma }} | Outbound Liquidity: {{ outbound|intcomma }} | Unsettled Liquidity: {{ unsettled|intcomma }}

+

Inactive Inbound Liquidity: {{ inactive_inbound|intcomma }} | Outbound Liquidity: {{ inactive_outbound|intcomma }} | Unsettled Liquidity: {{ inactive_unsettled|intcomma }}

+

Balance In Limbo: {{ limbo_balance|intcomma }} | Total Unsettled Liquidity: {{ total_unsettled|intcomma }} | Pending HTLCs: {{ pending_htlc_count }}

+

Closures | New Peers | AR Actions | Fee Rates | Autopilot | Autofees | Channel Performance | Keysends | Rebalancing | Advanced Settings

{% if active_channels %}
@@ -388,13 +391,14 @@

Last 5 Payments Sent

Timestamp Hash - Value - Fee Paid - Status + Value + Fee Paid + PPM Paid + Status Chan Out Alias Chan Out ID - Route - Keysend + Route + Keysend {% for payment in payments %} @@ -402,6 +406,7 @@

Last 5 Payments Sent

{{ payment.payment_hash }} {{ payment.value|add:"0"|intcomma }} {{ payment.fee|intcomma }} + {{ payment.ppm|intcomma }} {% if payment.status == 1 %}In-Flight{% elif payment.status == 2 %}Succeeded{% elif payment.status == 3 %}Failed{% else %}{{ payment.status }}{% endif %} {% if payment.status == 2 %}{% if payment.chan_out_alias == '' %}---{% else %}{{ payment.chan_out_alias }}{% endif %}{% else %}N/A{% endif %} {% if payment.status == 2 %}{{ payment.chan_out }}{% else %}N/A{% endif %} diff --git a/gui/templates/payments.html b/gui/templates/payments.html index 11fd5d60..ed6d4bc1 100644 --- a/gui/templates/payments.html +++ b/gui/templates/payments.html @@ -9,13 +9,14 @@

Last 150 Payments

Timestamp Hash - Value - Fee Paid - Status + Value + Fee Paid + PPM Paid + Status Chan Out Alias Chan Out ID - Route - Keysend + Route + Keysend {% for payment in payments %} @@ -23,6 +24,7 @@

Last 150 Payments

{{ payment.payment_hash }} {{ payment.value|add:"0"|intcomma }} {{ payment.fee|intcomma }} + {{ payment.ppm|intcomma }} {% if payment.status == 1 %}In-Flight{% elif payment.status == 2 %}Succeeded{% elif payment.status == 3 %}Failed{% else %}{{ payment.status }}{% endif %} {% if payment.status == 2 %}{% if payment.chan_out_alias == '' %}---{% else %}{{ payment.chan_out_alias }}{% endif %}{% else %}N/A{% endif %} {% if payment.status == 2 %}{{ payment.chan_out }}{% else %}N/A{% endif %} diff --git a/gui/urls.py b/gui/urls.py index 3d57b4e6..ec0bd058 100644 --- a/gui/urls.py +++ b/gui/urls.py @@ -9,6 +9,8 @@ router.register(r'invoices', views.InvoicesViewSet) router.register(r'forwards', views.ForwardsViewSet) router.register(r'onchain', views.OnchainViewSet) +router.register(r'closures', views.ClosuresViewSet) +router.register(r'resolutions', views.ResolutionsViewSet) router.register(r'peers', views.PeersViewSet) router.register(r'channels', views.ChannelsViewSet) router.register(r'rebalancer', views.RebalancerViewSet) @@ -46,6 +48,7 @@ path('keysends/', views.keysends, name='keysends'), path('channels/', views.channels, name='channels'), path('autopilot/', views.autopilot, name='autopilot'), + path('autofees/', views.autofees, name='autofees'), path('advanced/', views.advanced, name='advanced'), path('api/', include(router.urls), name='api-root'), path('api-auth/', include('rest_framework.urls'), name='api-auth'), diff --git a/gui/views.py b/gui/views.py index 094cfc96..70fb8699 100644 --- a/gui/views.py +++ b/gui/views.py @@ -9,8 +9,8 @@ from rest_framework.response import Response from rest_framework.decorators import api_view from .forms import OpenChannelForm, CloseChannelForm, ConnectPeerForm, AddInvoiceForm, RebalancerForm, ChanPolicyForm, UpdateChannel, UpdateSetting, AutoRebalanceForm -from .models import Payments, PaymentHops, Invoices, Forwards, Channels, Rebalancer, LocalSettings, Peers, Onchain, Closures, Resolutions, PendingHTLCs, FailedHTLCs, Autopilot -from .serializers import ConnectPeerSerializer, FailedHTLCSerializer, LocalSettingsSerializer, OpenChannelSerializer, CloseChannelSerializer, AddInvoiceSerializer, PaymentHopsSerializer, PaymentSerializer, InvoiceSerializer, ForwardSerializer, ChannelSerializer, PendingHTLCSerializer, RebalancerSerializer, UpdateAliasSerializer, PeerSerializer, OnchainSerializer +from .models import Payments, PaymentHops, Invoices, Forwards, Channels, Rebalancer, LocalSettings, Peers, Onchain, Closures, Resolutions, PendingHTLCs, FailedHTLCs, Autopilot, Autofees +from .serializers import ConnectPeerSerializer, FailedHTLCSerializer, LocalSettingsSerializer, OpenChannelSerializer, CloseChannelSerializer, AddInvoiceSerializer, PaymentHopsSerializer, PaymentSerializer, InvoiceSerializer, ForwardSerializer, ChannelSerializer, PendingHTLCSerializer, RebalancerSerializer, UpdateAliasSerializer, PeerSerializer, OnchainSerializer, ClosuresSerializer, ResolutionsSerializer from .lnd_deps import lightning_pb2 as ln from .lnd_deps import lightning_pb2_grpc as lnrpc from gui.lnd_deps import router_pb2 as lnr @@ -50,12 +50,12 @@ def home(request): pending_force_closed = pending_channels.pending_force_closing_channels waiting_for_close = pending_channels.waiting_close_channels #Get recorded payment events - payments = Payments.objects.exclude(status=3).order_by('-creation_date') + payments = Payments.objects.exclude(status=3) total_payments = payments.filter(status=2).count() total_sent = 0 if total_payments == 0 else payments.filter(status=2).aggregate(Sum('value'))['value__sum'] total_fees = 0 if total_payments == 0 else payments.aggregate(Sum('fee'))['fee__sum'] #Get recorded invoice details - invoices = Invoices.objects.exclude(state=2).order_by('-creation_date') + invoices = Invoices.objects.exclude(state=2) total_invoices = invoices.filter(state=1).count() total_received = 0 if total_invoices == 0 else invoices.aggregate(Sum('amt_paid'))['amt_paid__sum'] #Get recorded forwarding events @@ -70,10 +70,10 @@ def home(request): forwards_df_out_count = DataFrame() if forwards_df.empty else forwards_df.groupby('chan_id_out', as_index=True).count() #Get current active channels active_channels = Channels.objects.filter(is_active=True, is_open=True, private=False).annotate(outbound_percent=((Sum('local_balance')+Sum('pending_outbound'))*1000)/Sum('capacity')).annotate(inbound_percent=((Sum('remote_balance')+Sum('pending_inbound'))*1000)/Sum('capacity')).order_by('outbound_percent') - total_capacity = 0 if active_channels.count() == 0 else active_channels.aggregate(Sum('capacity'))['capacity__sum'] - total_inbound = 0 if total_capacity == 0 else active_channels.aggregate(Sum('remote_balance'))['remote_balance__sum'] - total_outbound = 0 if total_capacity == 0 else active_channels.aggregate(Sum('local_balance'))['local_balance__sum'] - total_unsettled = 0 if total_capacity == 0 else active_channels.aggregate(Sum('unsettled_balance'))['unsettled_balance__sum'] + active_capacity = 0 if active_channels.count() == 0 else active_channels.aggregate(Sum('capacity'))['capacity__sum'] + active_inbound = 0 if active_capacity == 0 else active_channels.aggregate(Sum('remote_balance'))['remote_balance__sum'] + active_outbound = 0 if active_capacity == 0 else active_channels.aggregate(Sum('local_balance'))['local_balance__sum'] + active_unsettled = 0 if active_capacity == 0 else active_channels.aggregate(Sum('unsettled_balance'))['unsettled_balance__sum'] filter_7day = datetime.now() - timedelta(days=7) forwards_df_7d = DataFrame.from_records(forwards.filter(forward_date__gte=filter_7day).values()) forwards_df_in_7d_sum = DataFrame() if forwards_df_7d.empty else forwards_df_7d.groupby('chan_id_in', as_index=True).sum() @@ -88,6 +88,7 @@ def home(request): total_7day_fees = 0 if payments_7day.count() == 0 else payments_7day.aggregate(Sum('fee'))['fee__sum'] pending_htlc_count = Channels.objects.filter(is_open=True).aggregate(Sum('htlc_count'))['htlc_count__sum'] if Channels.objects.filter(is_open=True).exists() else 0 pending_outbound = Channels.objects.filter(is_open=True).aggregate(Sum('pending_outbound'))['pending_outbound__sum'] if Channels.objects.filter(is_open=True).exists() else 0 + pending_inbound = Channels.objects.filter(is_open=True).aggregate(Sum('pending_inbound'))['pending_inbound__sum'] if Channels.objects.filter(is_open=True).exists() else 0 detailed_active_channels = [] for channel in active_channels: detailed_channel = {} @@ -124,10 +125,16 @@ def home(request): detailed_active_channels.append(detailed_channel) #Get current inactive channels inactive_channels = Channels.objects.filter(is_active=False, is_open=True, private=False).annotate(outbound_percent=((Sum('local_balance')+Sum('pending_outbound'))*100)/Sum('capacity')).annotate(inbound_percent=((Sum('remote_balance')+Sum('pending_inbound'))*100)/Sum('capacity')).order_by('outbound_percent') + inactive_capacity = 0 if inactive_channels.count() == 0 else inactive_channels.aggregate(Sum('capacity'))['capacity__sum'] private_channels = Channels.objects.filter(is_open=True, private=True).annotate(outbound_percent=((Sum('local_balance')+Sum('pending_outbound'))*100)/Sum('capacity')).annotate(inbound_percent=((Sum('remote_balance')+Sum('pending_inbound'))*100)/Sum('capacity')).order_by('outbound_percent') inactive_outbound = 0 if inactive_channels.count() == 0 else inactive_channels.aggregate(Sum('local_balance'))['local_balance__sum'] - private_outbound = 0 if private_channels.count() == 0 else private_channels.aggregate(Sum('local_balance'))['local_balance__sum'] - sum_outbound = total_outbound + pending_outbound + inactive_outbound + private_outbound + inactive_inbound = 0 if inactive_channels.count() == 0 else inactive_channels.aggregate(Sum('remote_balance'))['remote_balance__sum'] + private_count = private_channels.count() + active_private = private_channels.filter(is_active=True).count() + private_capacity = private_channels.aggregate(Sum('capacity'))['capacity__sum'] + private_outbound = 0 if private_count == 0 else private_channels.aggregate(Sum('local_balance'))['local_balance__sum'] + sum_outbound = active_outbound + pending_outbound + inactive_outbound + sum_inbound = active_inbound + pending_inbound + inactive_inbound onchain_txs = Onchain.objects.all() onchain_costs = 0 if onchain_txs.count() == 0 else onchain_txs.aggregate(Sum('fee'))['fee__sum'] onchain_costs_7day = 0 if onchain_txs.filter(time_stamp__gte=filter_7day).count() == 0 else onchain_txs.filter(time_stamp__gte=filter_7day).aggregate(Sum('fee'))['fee__sum'] @@ -135,7 +142,7 @@ def home(request): total_costs_7day = total_7day_fees + onchain_costs_7day #Get list of recent rebalance requests rebalances = Rebalancer.objects.all().annotate(ppm=(Sum('fee_limit')*1000000)/Sum('value')).order_by('-id') - total_channels = node_info.num_active_channels + node_info.num_inactive_channels + total_channels = node_info.num_active_channels + node_info.num_inactive_channels - private_count local_settings = LocalSettings.objects.filter(key__contains='AR-') try: db_size = round(path.getsize(path.expanduser(LND_DIR_PATH + '/data/graph/' + LND_NETWORK + '/channel.db'))*0.000000001, 3) @@ -146,11 +153,11 @@ def home(request): 'node_info': node_info, 'total_channels': total_channels, 'balances': balances, - 'payments': payments[:6], + 'payments': payments.annotate(ppm=Round((Sum('fee')*1000000)/Sum('value'), output_field=IntegerField())).order_by('-creation_date')[:6], 'total_sent': int(total_sent), 'fees_paid': int(total_fees), 'total_payments': total_payments, - 'invoices': invoices[:6], + 'invoices': invoices.order_by('-creation_date')[:6], 'total_received': total_received, 'total_invoices': total_invoices, 'forwards': forwards_df.head(15).to_dict(orient='records'), @@ -169,10 +176,20 @@ def home(request): 'onchain_costs_7day': onchain_costs_7day, 'total_7day_fees': int(total_7day_fees), 'active_channels': detailed_active_channels, - 'capacity': total_capacity, - 'inbound': total_inbound, - 'outbound': total_outbound, - 'unsettled': total_unsettled, + 'total_capacity': active_capacity + inactive_capacity, + 'active_private': active_private, + 'total_private': private_count, + 'private_outbound': private_outbound, + 'private_capacity': private_capacity, + 'inbound': active_inbound, + 'sum_inbound': sum_inbound, + 'inactive_inbound': inactive_inbound, + 'outbound': active_outbound, + 'sum_outbound': sum_outbound, + 'inactive_outbound': inactive_outbound, + 'unsettled': active_unsettled, + 'inactive_unsettled': pending_outbound + pending_inbound - active_unsettled, + 'total_unsettled': pending_outbound + pending_inbound, 'limbo_balance': limbo_balance, 'inactive_channels': inactive_channels, 'private_channels': private_channels, @@ -189,7 +206,7 @@ def home(request): 'routed_ppm': 0 if total_value_forwards == 0 else int((total_earned/total_value_forwards)*1000000), '7day_routed_ppm': 0 if routed_7day_amt == 0 else int((total_earned_7day/routed_7day_amt)*1000000), '7day_payments_ppm': 0 if payments_7day_amt == 0 else int((total_7day_fees/payments_7day_amt)*1000000), - 'liq_ratio': 0 if total_outbound == 0 else int((total_inbound/sum_outbound)*100), + 'liq_ratio': 0 if sum_outbound == 0 else int((sum_inbound/sum_outbound)*100), 'eligible_count': Channels.objects.filter(is_active=True, is_open=True, private=False, auto_rebalance=True).annotate(inbound_can=((Sum('remote_balance')+Sum('pending_inbound'))*100)/Sum('capacity')).annotate(fee_ratio=(Sum('remote_fee_rate')*100)/Sum('local_fee_rate')).filter(inbound_can__gte=F('ar_in_target'), fee_ratio__lte=F('ar_max_cost')).count(), 'enabled_count': Channels.objects.filter(is_open=True, auto_rebalance=True).count(), 'network': 'testnet/' if LND_NETWORK == 'testnet' else '', @@ -207,8 +224,8 @@ def channels(request): filter_7day = datetime.now() - timedelta(days=7) filter_30day = datetime.now() - timedelta(days=30) forwards = Forwards.objects.filter(forward_date__gte=filter_30day) - payments = Payments.objects.filter(status=2).filter(creation_date__gte=filter_30day).filter(rebal_chan__isnull=False) - invoices = Invoices.objects.filter(state=1).filter(settle_date__gte=filter_30day).filter(r_hash__in=payments.values_list('payment_hash')) + payments = Payments.objects.filter(status=2, creation_date__gte=filter_30day, rebal_chan__isnull=False) + invoices = Invoices.objects.filter(state=1, r_hash__in=payments.values_list('payment_hash')) channels = Channels.objects.filter(is_open=True, private=False) channels_df = DataFrame.from_records(channels.values()) if channels_df.shape[0] > 0: @@ -236,7 +253,7 @@ def channels(request): invoices_df_7d_count = DataFrame() if invoices_df_7d.empty else invoices_df_7d.groupby('chan_in', as_index=True).count() invoice_hashes_7d = DataFrame() if invoices_df_7d.empty else invoices_df_7d.groupby('chan_in', as_index=True)['r_hash'].apply(list) invoice_hashes_30d = DataFrame() if invoices_df_30d.empty else invoices_df_30d.groupby('chan_in', as_index=True)['r_hash'].apply(list) - channels_df['capacity'] = channels_df.apply(lambda row: round(row.capacity/1000000, 1), axis=1) + channels_df['mil_capacity'] = channels_df.apply(lambda row: round(row.capacity/1000000, 1), axis=1) channels_df['local_balance'] = channels_df.apply(lambda row: row.local_balance + row.pending_outbound, axis=1) channels_df['remote_balance'] = channels_df.apply(lambda row: row.remote_balance + row.pending_inbound, axis=1) channels_df['routed_in_7day'] = channels_df.apply(lambda row: forwards_df_in_7d_count.loc[row.chan_id].amt_out_msat if (forwards_df_in_7d_count.index == row.chan_id).any() else 0, axis=1) @@ -274,13 +291,13 @@ def channels(request): outbound_ratio = node_outbound/node_capacity channels_df['apy_7day'] = channels_df.apply(lambda row: round((row['profits_7day']*5214.2857)/(row['capacity']*outbound_ratio), 2), axis=1) channels_df['apy_30day'] = channels_df.apply(lambda row: round((row['profits_30day']*1216.6667)/(row['capacity']*outbound_ratio), 2), axis=1) + channels_df['cv_7day'] = channels_df.apply(lambda row: round((row['revenue_7day']*5214.2857)/(row['capacity']*outbound_ratio) + (row['revenue_assist_7day']*5214.2857)/(row['capacity']*(1-outbound_ratio)), 2), axis=1) + channels_df['cv_30day'] = channels_df.apply(lambda row: round((row['revenue_30day']*1216.6667)/(row['capacity']*outbound_ratio) + (row['revenue_assist_30day']*1216.6667)/(row['capacity']*(1-outbound_ratio)), 2), axis=1) else: - channels_df['attempts_30day'] = 0 - channels_df['success_30day'] = 0 - channels_df['success_rate_30day'] = 0 - channels_df['attempts_7day'] = 0 - channels_df['success_7day'] = 0 - channels_df['success_rate_7day'] = 0 + channels_df['apy_7day'] = 0.0 + channels_df['apy_30day'] = 0.0 + channels_df['cv_7day'] = 0.0 + channels_df['cv_30day'] = 0.0 else: apy_7day = 0 apy_30day = 0 @@ -299,11 +316,12 @@ def channels(request): @login_required(login_url='/lndg-admin/login/?next=/') def fees(request): if request.method == 'GET': + filter_1day = datetime.now() - timedelta(days=1) filter_7day = datetime.now() - timedelta(days=7) - channels = Channels.objects.filter(is_open=True) + channels = Channels.objects.filter(is_open=True, private=False) channels_df = DataFrame.from_records(channels.values()) if channels_df.shape[0] > 0: - failed_htlc_df = DataFrame.from_records(FailedHTLCs.objects.filter(timestamp__gte=filter_7day).order_by('-id').values()) + failed_htlc_df = DataFrame.from_records(FailedHTLCs.objects.filter(timestamp__gte=filter_1day).order_by('-id').values()) if failed_htlc_df.shape[0] > 0: failed_htlc_df = failed_htlc_df[failed_htlc_df['wire_failure']==15][failed_htlc_df['failure_detail']==6][failed_htlc_df['amount']>failed_htlc_df['chan_out_liq']+failed_htlc_df['chan_out_pending']] forwards = Forwards.objects.filter(forward_date__gte=filter_7day, amt_out_msat__gte=1000000) @@ -320,7 +338,7 @@ def fees(request): channels_df['revenue_7day'] = channels_df.apply(lambda row: int(forwards_df_out_7d_sum.loc[row.chan_id].fee) if forwards_df_out_7d_sum.empty == False and (forwards_df_out_7d_sum.index == row.chan_id).any() else 0, axis=1) channels_df['revenue_assist_7day'] = channels_df.apply(lambda row: int(forwards_df_in_7d_sum.loc[row.chan_id].fee) if forwards_df_in_7d_sum.empty == False and (forwards_df_in_7d_sum.index == row.chan_id).any() else 0, axis=1) channels_df['out_rate'] = channels_df.apply(lambda row: int((row['revenue_7day']/row['amt_routed_out_7day'])*1000000) if row['amt_routed_out_7day'] > 0 else 0, axis=1) - channels_df['failed_out_7day'] = 0 if failed_htlc_df.empty else channels_df.apply(lambda row: len(failed_htlc_df[failed_htlc_df['chan_id_out']==row.chan_id]), axis=1) + channels_df['failed_out_1day'] = 0 if failed_htlc_df.empty else channels_df.apply(lambda row: len(failed_htlc_df[failed_htlc_df['chan_id_out']==row.chan_id]), axis=1) payments = Payments.objects.filter(status=2).filter(creation_date__gte=filter_7day).filter(rebal_chan__isnull=False) invoices = Invoices.objects.filter(state=1).filter(settle_date__gte=filter_7day).filter(r_hash__in=payments.values_list('payment_hash')) payments_df_7d = DataFrame.from_records(payments.filter(creation_date__gte=filter_7day).values()) @@ -331,7 +349,7 @@ def fees(request): channels_df['costs_7day'] = channels_df.apply(lambda row: 0 if row['amt_rebal_in_7day'] == 0 else int(payments_df_7d.set_index('payment_hash', inplace=False).loc[invoice_hashes_7d[row.chan_id] if invoice_hashes_7d.empty == False and (invoice_hashes_7d.index == row.chan_id).any() else []]['fee'].sum()), axis=1) channels_df['rebal_ppm'] = channels_df.apply(lambda row: int((row['costs_7day']/row['amt_rebal_in_7day'])*1000000) if row['amt_rebal_in_7day'] > 0 else 0, axis=1) channels_df['max_suggestion'] = channels_df.apply(lambda row: int((row['out_rate'] if row['out_rate'] > 0 else row['local_fee_rate'])*1.15) if row['in_percent'] > 25 else int(row['local_fee_rate']), axis=1) - channels_df['max_suggestion'] = channels_df.apply(lambda row: row['local_fee_rate']+25 if row['max_suggestion'] > (row['local_fee_rate']+25) else row['max_suggestion'], axis=1) + channels_df['max_suggestion'] = channels_df.apply(lambda row: row['local_fee_rate']+25 if row['max_suggestion'] > (row['local_fee_rate']+25) or row['max_suggestion'] == 0 else row['max_suggestion'], axis=1) channels_df['min_suggestion'] = channels_df.apply(lambda row: int((row['out_rate'] if row['out_rate'] > 0 else row['local_fee_rate'])*0.75) if row['out_percent'] > 25 else int(row['local_fee_rate']), axis=1) channels_df['min_suggestion'] = channels_df.apply(lambda row: row['local_fee_rate']-50 if row['min_suggestion'] < (row['local_fee_rate']-50) else row['min_suggestion'], axis=1) channels_df['assisted_ratio'] = channels_df.apply(lambda row: round((row['revenue_assist_7day'] if row['revenue_7day'] == 0 else row['revenue_assist_7day']/row['revenue_7day']), 2), axis=1) @@ -342,10 +360,14 @@ def fees(request): channels_df['fee_rate_only'] = channels_df.apply(lambda row: int(row['local_fee_rate']+row['net_routed_7day']*row['local_fee_rate']*0.05), axis=1) channels_df['new_rate'] = channels_df.apply(lambda row: row['adjusted_out_rate'] if row['net_routed_7day'] != 0 else (row['adjusted_rebal_rate'] if row['rebal_ppm'] > 0 and row['out_rate'] > 0 else (row['out_rate_only'] if row['out_rate'] > 0 else (row['min_suggestion'] if row['net_routed_7day'] == 0 and row['in_percent'] < 25 else row['fee_rate_only']))), axis=1) channels_df['new_rate'] = channels_df.apply(lambda row: 0 if row['new_rate'] < 0 else row['new_rate'], axis=1) - channels_df['new_rate'] = channels_df.apply(lambda row: row['max_suggestion'] if row['max_suggestion'] > 0 and row['new_rate'] > row['max_suggestion'] else row['new_rate'], axis=1) + channels_df['adjustment'] = channels_df.apply(lambda row: int(row['new_rate']-row['local_fee_rate']), axis=1) + channels_df['new_rate'] = channels_df.apply(lambda row: row['local_fee_rate']-10 if row['adjustment']==0 and row['out_percent']>=25 and row['net_routed_7day']==0 else row['new_rate'], axis=1) + channels_df['new_rate'] = channels_df.apply(lambda row: row['local_fee_rate']+25 if row['adjustment']==0 and row['out_percent']<25 and row['failed_out_1day']>25 else row['new_rate'], axis=1) + channels_df['new_rate'] = channels_df.apply(lambda row: row['max_suggestion'] if row['new_rate'] > row['max_suggestion'] else row['new_rate'], axis=1) channels_df['new_rate'] = channels_df.apply(lambda row: row['min_suggestion'] if row['new_rate'] < row['min_suggestion'] else row['new_rate'], axis=1) channels_df['new_rate'] = channels_df.apply(lambda row: int(round(row['new_rate']/5, 0)*5), axis=1) channels_df['adjustment'] = channels_df.apply(lambda row: int(row['new_rate']-row['local_fee_rate']), axis=1) + channels_df['eligible'] = channels_df.apply(lambda row: (datetime.now()-row['fees_updated']).total_seconds() > 86400, axis=1) context = { 'channels': [] if channels_df.empty else channels_df.sort_values(by=['out_percent']).to_dict(orient='records'), 'network': 'testnet/' if LND_NETWORK == 'testnet' else '', @@ -422,13 +444,16 @@ def balances(request): def closures(request): if request.method == 'GET': closures_df = DataFrame.from_records(Closures.objects.all().values()) - channels_df = DataFrame.from_records(Channels.objects.all().values('chan_id', 'alias')) - if channels_df.empty: - merged = closures_df - merged['alias'] = '' + if closures_df.empty: + merged = DataFrame() else: - merged = merge(closures_df, channels_df, on='chan_id', how='left') - merged['alias'] = merged['alias'].fillna('') + channels_df = DataFrame.from_records(Channels.objects.all().values('chan_id', 'alias')) + if channels_df.empty: + merged = closures_df + merged['alias'] = '' + else: + merged = merge(closures_df, channels_df, on='chan_id', how='left') + merged['alias'] = merged['alias'].fillna('') context = { 'closures': [] if merged.empty else merged.sort_values(by=['close_height'], ascending=False).to_dict(orient='records'), 'network': 'testnet/' if LND_NETWORK == 'testnet' else '', @@ -462,7 +487,7 @@ def channel(request): filter_7day = datetime.now() - timedelta(days=7) filter_30day = datetime.now() - timedelta(days=30) forwards_df = DataFrame.from_records(Forwards.objects.filter(Q(chan_id_in=chan_id) | Q(chan_id_out=chan_id)).values()) - payments_df = DataFrame.from_records(Payments.objects.filter(status=2).filter(chan_out=chan_id).filter(rebal_chan__isnull=False).values()) + payments_df = DataFrame.from_records(Payments.objects.filter(status=2).filter(chan_out=chan_id).filter(rebal_chan__isnull=False).annotate(ppm=Round((Sum('fee')*1000000)/Sum('value'), output_field=IntegerField())).values()) invoices_df = DataFrame.from_records(Invoices.objects.filter(state=1).filter(chan_in=chan_id).filter(r_hash__in=Payments.objects.filter(status=2).filter(rebal_chan=chan_id)).values()) channels_df = DataFrame.from_records(Channels.objects.filter(is_open=True).values()) node_outbound = channels_df['local_balance'].sum() @@ -710,7 +735,6 @@ def channel(request): channels_df['profits_vol_30day'] = 0 if channels_df['amt_routed_out_30day'][0] == 0 else int(channels_df['profits_30day'] / (channels_df['amt_routed_out_30day']/1000000)) channels_df['profits_vol_7day'] = 0 if channels_df['amt_routed_out_7day'][0] == 0 else int(channels_df['profits_7day'] / (channels_df['amt_routed_out_7day']/1000000)) channels_df['profits_vol_1day'] = 0 if channels_df['amt_routed_out_1day'][0] == 0 else int(channels_df['profits_1day'] / (channels_df['amt_routed_out_1day']/1000000)) - channels_df['apy'] = 0.0 channels_df = channels_df.copy() channels_df['net_routed_7day'] = round((channels_df['amt_routed_out_7day']-channels_df['amt_routed_in_7day'])/channels_df['capacity'], 1) channels_df['out_rate'] = int((channels_df['revenue_7day']/channels_df['amt_routed_out_7day'])*1000000) if channels_df['amt_routed_out_7day'][0] > 0 else 0 @@ -736,15 +760,35 @@ def channel(request): channels_df['fee_check'] = 1 if channels_df['ar_max_cost'][0] == 0 else int(round(((channels_df['fee_ratio']/channels_df['ar_max_cost'])*1000)/10, 0)) channels_df = channels_df.copy() channels_df['steps'] = 0 if channels_df['inbound_can'][0] < 1 else int(((channels_df['in_percent']-channels_df['ar_in_target'])/((channels_df['ar_amt_target']/channels_df['capacity'])*100))+0.999) + channels_df['apy'] = 0.0 + channels_df['apy_30day'] = 0.0 + channels_df['apy_7day'] = 0.0 + channels_df['apy_1day'] = 0.0 + channels_df['assisted_apy'] = 0.0 + channels_df['assisted_apy_30day'] = 0.0 + channels_df['assisted_apy_7day'] = 0.0 + channels_df['assisted_apy_1day'] = 0.0 + channels_df['cv'] = 0.0 + channels_df['cv_30day'] = 0.0 + channels_df['cv_7day'] = 0.0 + channels_df['cv_1day'] = 0.0 if node_capacity > 0: outbound_ratio = node_outbound/node_capacity if start_date is not None: time_delta = datetime.now() - start_date.to_pydatetime() days_routing = time_delta.days + (time_delta.seconds/86400) channels_df['apy'] = round(((channels_df['profits']/days_routing)*36500)/(channels_df['capacity']*outbound_ratio), 2) + channels_df['assisted_apy'] = round(((channels_df['revenue_assist']/days_routing)*36500)/(channels_df['capacity']*(1-outbound_ratio)), 2) + channels_df['cv'] = round(((channels_df['revenue']/days_routing)*36500)/(channels_df['capacity']*outbound_ratio) + channels_df['assisted_apy'], 2) channels_df['apy_30day'] = round((channels_df['profits_30day']*1216.6667)/(channels_df['capacity']*outbound_ratio), 2) channels_df['apy_7day'] = round((channels_df['profits_7day']*5214.2857)/(channels_df['capacity']*outbound_ratio), 2) channels_df['apy_1day'] = round((channels_df['profits_1day']*36500)/(channels_df['capacity']*outbound_ratio), 2) + channels_df['assisted_apy_30day'] = round((channels_df['revenue_assist_30day']*1216.6667)/(channels_df['capacity']*(1-outbound_ratio)), 2) + channels_df['assisted_apy_7day'] = round((channels_df['revenue_assist_7day']*5214.2857)/(channels_df['capacity']*(1-outbound_ratio)), 2) + channels_df['assisted_apy_1day'] = round((channels_df['revenue_assist_1day']*36500)/(channels_df['capacity']*(1-outbound_ratio)), 2) + channels_df['cv_30day'] = round((channels_df['revenue_30day']*1216.6667)/(channels_df['capacity']*outbound_ratio) + channels_df['assisted_apy_30day'], 2) + channels_df['cv_7day'] = round((channels_df['revenue_7day']*5214.2857)/(channels_df['capacity']*outbound_ratio) + channels_df['assisted_apy_7day'], 2) + channels_df['cv_1day'] = round((channels_df['revenue_1day']*36500)/(channels_df['capacity']*outbound_ratio) + channels_df['assisted_apy_1day'], 2) else: channels_df = DataFrame() forwards_df = DataFrame() @@ -874,7 +918,7 @@ def failed_htlcs(request): def payments(request): if request.method == 'GET': context = { - 'payments': Payments.objects.filter(status=2).order_by('-creation_date')[:150], + 'payments': Payments.objects.filter(status=2).annotate(ppm=Round((Sum('fee')*1000000)/Sum('value'), output_field=IntegerField())).order_by('-creation_date')[:150], } return render(request, 'payments.html', context) else: @@ -956,6 +1000,16 @@ def autopilot(request): else: return redirect('home') +@login_required(login_url='/lndg-admin/login/?next=/') +def autofees(request): + if request.method == 'GET': + context = { + 'autofees': Autofees.objects.all().order_by('-id') + } + return render(request, 'autofees.html', context) + else: + return redirect('home') + @login_required(login_url='/lndg-admin/login/?next=/') def open_channel_form(request): if request.method == 'POST': @@ -1133,6 +1187,8 @@ def update_chan_policy(request): if form.cleaned_data['target_all']: args = {'global': True, 'base_fee_msat': form.cleaned_data['new_base_fee'], 'fee_rate': (form.cleaned_data['new_fee_rate']/1000000), 'time_lock_delta': 40} stub.UpdateChannelPolicy(ln.PolicyUpdateRequest(**args)) + channels = Channels.objects.filter(is_open=True) + channels.update(fees_updated=datetime.now()) elif len(form.cleaned_data['target_chans']) > 0: for channel in form.cleaned_data['target_chans']: channel_point = ln.ChannelPoint() @@ -1140,6 +1196,9 @@ def update_chan_policy(request): channel_point.funding_txid_str = channel.funding_txid channel_point.output_index = channel.output_index stub.UpdateChannelPolicy(ln.PolicyUpdateRequest(chan_point=channel_point, base_fee_msat=form.cleaned_data['new_base_fee'], fee_rate=(form.cleaned_data['new_fee_rate']/1000000), time_lock_delta=40)) + db_channel = Channels.objects.get(chan_id=channel.chan_id) + db_channel.fees_updated = datetime.now() + db_channel.save() else: messages.error(request, 'No channels were specified in the update request!') messages.success(request, 'Channel policies updated! This will be broadcast during the next graph update!') @@ -1272,6 +1331,7 @@ def update_channel(request): channel_point.output_index = db_channel.output_index stub.UpdateChannelPolicy(ln.PolicyUpdateRequest(chan_point=channel_point, base_fee_msat=db_channel.local_base_fee, fee_rate=(target/1000000), time_lock_delta=40)) db_channel.local_fee_rate = target + db_channel.fees_updated = datetime.now() db_channel.save() messages.success(request, 'Fee rate for channel ' + str(db_channel.alias) + ' (' + str(db_channel.chan_id) + ') updated to a value of: ' + str(target)) elif update_target == 2: @@ -1306,6 +1366,10 @@ def update_channel(request): messages.success(request, 'Toggled channel state for channel ' + str(db_channel.alias) + ' (' + str(db_channel.chan_id) + ') to a value of: ' + ('Enabled' if target == 1 else 'Disabled')) if target == 0: messages.warning(request, 'Use with caution, while a channel is disabled (local fees highlighted in red) it will not route out.') + elif update_target == 8: + db_channel.auto_fees = True if db_channel.auto_fees == False else False + db_channel.save() + messages.success(request, 'Auto fees status for chanel ' + str(db_channel.alias) + ' (' + str(db_channel.chan_id) + ') updated to a value of: ' + str(db_channel.auto_fees)) else: messages.error(request, 'Invalid target code. Please try again.') else: @@ -1328,7 +1392,6 @@ def update_setting(request): db_percent_target = LocalSettings.objects.get(key='AR-Target%') db_percent_target.value = target_percent db_percent_target.save() - Channels.objects.all().update(ar_amt_target=Round(F('capacity')*target_percent, output_field=IntegerField())) messages.success(request, 'Updated auto rebalancer target amount for all channels to: ' + str(target_percent)) elif key == 'AR-Time': target_time = int(value) @@ -1359,7 +1422,6 @@ def update_setting(request): db_outbound_target = LocalSettings.objects.get(key='AR-Outbound%') db_outbound_target.value = outbound_percent db_outbound_target.save() - Channels.objects.all().update(ar_out_target=int(outbound_percent*100)) messages.success(request, 'Updated auto rebalancer target outbound percent setting for all channels to: ' + str(outbound_percent)) elif key == 'AR-MaxFeeRate': fee_rate = int(value) @@ -1380,7 +1442,6 @@ def update_setting(request): db_max_cost = LocalSettings.objects.get(key='AR-MaxCost%') db_max_cost.value = max_cost db_max_cost.save() - Channels.objects.all().update(ar_max_cost=int(max_cost*100)) messages.success(request, 'Updated auto rebalancer max cost setting to: ' + str(max_cost)) elif key == 'AR-Autopilot': autopilot = int(value) @@ -1432,6 +1493,68 @@ def update_setting(request): db_retention_days.value = retention_days db_retention_days.save() messages.success(request, 'Updated payment cleanup retention days to: ' + str(retention_days)) + elif key == 'ALL-oRate': + target = int(value) + stub = lnrpc.LightningStub(lnd_connect(settings.LND_DIR_PATH, settings.LND_NETWORK, settings.LND_RPC_SERVER)) + channels = Channels.objects.filter(is_open=True) + for db_channel in channels: + channel_point = ln.ChannelPoint() + channel_point.funding_txid_bytes = bytes.fromhex(db_channel.funding_txid) + channel_point.funding_txid_str = db_channel.funding_txid + channel_point.output_index = db_channel.output_index + stub.UpdateChannelPolicy(ln.PolicyUpdateRequest(chan_point=channel_point, base_fee_msat=db_channel.local_base_fee, fee_rate=(target/1000000), time_lock_delta=40)) + db_channel.local_fee_rate = target + db_channel.fees_updated = datetime.now() + db_channel.save() + messages.success(request, 'Fee rate for all open channels updated to a value of: ' + str(target)) + elif key == 'ALL-oBase': + target = int(value) + stub = lnrpc.LightningStub(lnd_connect(settings.LND_DIR_PATH, settings.LND_NETWORK, settings.LND_RPC_SERVER)) + channels = Channels.objects.filter(is_open=True) + for db_channel in channels: + stub = lnrpc.LightningStub(lnd_connect(settings.LND_DIR_PATH, settings.LND_NETWORK, settings.LND_RPC_SERVER)) + channel_point = ln.ChannelPoint() + channel_point.funding_txid_bytes = bytes.fromhex(db_channel.funding_txid) + channel_point.funding_txid_str = db_channel.funding_txid + channel_point.output_index = db_channel.output_index + stub.UpdateChannelPolicy(ln.PolicyUpdateRequest(chan_point=channel_point, base_fee_msat=target, fee_rate=(db_channel.local_fee_rate/1000000), time_lock_delta=40)) + db_channel.local_base_fee = target + db_channel.save() + messages.success(request, 'Base fee for all channels updated to a value of: ' + str(target)) + elif key == 'ALL-Amts': + target = int(value) + channels = Channels.objects.filter(is_open=True).update(ar_amt_target=target) + messages.success(request, 'AR target amounts for all channels updated to a value of: ' + str(target)) + elif key == 'ALL-MaxCost': + target = int(value) + channels = Channels.objects.filter(is_open=True).update(ar_max_cost=target) + messages.success(request, 'AR max cost %s for all channels updated to a value of: ' + str(target)) + elif key == 'ALL-oTarget': + target = int(value) + channels = Channels.objects.filter(is_open=True).update(ar_out_target=target) + messages.success(request, 'AR outbound liquidity target %s for all channels updated to a value of: ' + str(target)) + elif key == 'ALL-iTarget': + target = int(value) + channels = Channels.objects.filter(is_open=True).update(ar_in_target=target) + messages.success(request, 'AR inbound liquidity target %s for all channels updated to a value of: ' + str(target)) + elif key == 'ALL-AR': + target = int(value) + channels = Channels.objects.filter(is_open=True).update(auto_rebalance=target) + messages.success(request, 'Auto-Rebalance targeting for all channels updated to a value of: ' + str(target)) + elif key == 'ALL-AF': + target = int(value) + channels = Channels.objects.filter(is_open=True, private=False).update(auto_fees=target) + messages.success(request, 'Auto Fees setting for all channels updated to a value of: ' + str(target)) + elif key == 'AF-Enabled': + enabled = int(value) + try: + db_enabled = LocalSettings.objects.get(key='AF-Enabled') + except: + LocalSettings(key='AF-Enabled', value='0').save() + db_enabled = LocalSettings.objects.get(key='AF-Enabled') + db_enabled.value = enabled + db_enabled.save() + messages.success(request, 'Updated autofees enabled setting to: ' + str(enabled)) else: messages.error(request, 'Invalid Request. Please try again.') else: @@ -1462,6 +1585,14 @@ class OnchainViewSet(viewsets.ReadOnlyModelViewSet): queryset = Onchain.objects.all() serializer_class = OnchainSerializer +class ClosuresViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Closures.objects.all() + serializer_class = ClosuresSerializer + +class ResolutionsViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Resolutions.objects.all() + serializer_class = ResolutionsSerializer + class PendingHTLCViewSet(viewsets.ReadOnlyModelViewSet): queryset = PendingHTLCs.objects.all() serializer_class = PendingHTLCSerializer diff --git a/jobs.py b/jobs.py index 7f7a9fd3..47971ef6 100644 --- a/jobs.py +++ b/jobs.py @@ -8,9 +8,10 @@ from gui.lnd_deps.lnd_connect import lnd_connect from lndg import settings from os import environ +from pandas import DataFrame environ['DJANGO_SETTINGS_MODULE'] = 'lndg.settings' django.setup() -from gui.models import Payments, PaymentHops, Invoices, Forwards, Channels, Peers, Onchain, Closures, Resolutions, PendingHTLCs, LocalSettings +from gui.models import Payments, PaymentHops, Invoices, Forwards, Channels, Peers, Onchain, Closures, Resolutions, PendingHTLCs, LocalSettings, FailedHTLCs, Autofees def update_payments(stub): #Remove anything in-flight so we can get most up to date status @@ -283,12 +284,13 @@ def update_closures(stub): counter += 1 if counter > skip: resolution_count = len(closure.resolutions) - db_closure = Closures(chan_id=closure.chan_id, closing_tx=closure.closing_tx_hash, remote_pubkey=closure.remote_pubkey, capacity=closure.capacity, close_height=closure.close_height, settled_balance=closure.settled_balance, time_locked_balance=closure.time_locked_balance, close_type=closure.close_type, open_initiator=closure.open_initiator, close_initiator=closure.close_initiator, resolution_count=resolution_count) + txid, index = closure.channel_point.split(':') + db_closure = Closures(chan_id=closure.chan_id, funding_txid=txid, funding_index=index, closing_tx=closure.closing_tx_hash, remote_pubkey=closure.remote_pubkey, capacity=closure.capacity, close_height=closure.close_height, settled_balance=closure.settled_balance, time_locked_balance=closure.time_locked_balance, close_type=closure.close_type, open_initiator=closure.open_initiator, close_initiator=closure.close_initiator, resolution_count=resolution_count) db_closure.save() if resolution_count > 0: Resolutions.objects.filter(chan_id=closure.chan_id).delete() for resolution in closure.resolutions: - Resolutions(chan_id=db_closure, resolution_type=resolution.resolution_type, outcome=resolution.outcome, outpoint_tx=resolution.outpoint.txid_str, outpoint_index=resolution.outpoint.output_index, amount_sat=resolution.amount_sat, sweep_txid=resolution.sweep_txid).save() + Resolutions(chan_id=closure.chan_id, resolution_type=resolution.resolution_type, outcome=resolution.outcome, outpoint_tx=resolution.outpoint.txid_str, outpoint_index=resolution.outpoint.output_index, amount_sat=resolution.amount_sat, sweep_txid=resolution.sweep_txid).save() def reconnect_peers(stub): inactive_peers = Channels.objects.filter(is_open=True, is_active=False, private=False).values_list('remote_pubkey', flat=True).distinct() @@ -346,19 +348,95 @@ def clean_payments(stub): payment.cleaned = True payment.save() +def auto_fees(stub): + if LocalSettings.objects.filter(key='AF-Enabled').exists(): + enabled = int(LocalSettings.objects.filter(key='AF-Enabled')[0].value) + else: + LocalSettings(key='AF-Enabled', value='0').save() + enabled = 0 + if enabled == 1: + filter_1day = datetime.now() - timedelta(days=1) + filter_7day = datetime.now() - timedelta(days=7) + channels = Channels.objects.filter(is_open=True, is_active=True, private=False, auto_fees=True) + channels_df = DataFrame.from_records(channels.values()) + channels_df['eligible'] = channels_df.apply(lambda row: (datetime.now()-row['fees_updated']).total_seconds() > 86400, axis=1) + channels_df = channels_df[channels_df['eligible']==True] + if channels_df.shape[0] > 0: + failed_htlc_df = DataFrame.from_records(FailedHTLCs.objects.filter(timestamp__gte=filter_1day).order_by('-id').values()) + if failed_htlc_df.shape[0] > 0: + failed_htlc_df = failed_htlc_df[failed_htlc_df['wire_failure']==15][failed_htlc_df['failure_detail']==6][failed_htlc_df['amount']>failed_htlc_df['chan_out_liq']+failed_htlc_df['chan_out_pending']] + forwards = Forwards.objects.filter(forward_date__gte=filter_7day, amt_out_msat__gte=1000000) + forwards_df_7d = DataFrame.from_records(forwards.values()) + forwards_df_in_7d_sum = DataFrame() if forwards_df_7d.empty else forwards_df_7d.groupby('chan_id_in', as_index=True).sum() + forwards_df_out_7d_sum = DataFrame() if forwards_df_7d.empty else forwards_df_7d.groupby('chan_id_out', as_index=True).sum() + channels_df['local_balance'] = channels_df.apply(lambda row: row.local_balance + row.pending_outbound, axis=1) + channels_df['remote_balance'] = channels_df.apply(lambda row: row.remote_balance + row.pending_inbound, axis=1) + channels_df['in_percent'] = channels_df.apply(lambda row: int(round((row['remote_balance']/row['capacity'])*100, 0)), axis=1) + channels_df['out_percent'] = channels_df.apply(lambda row: int(round((row['local_balance']/row['capacity'])*100, 0)), axis=1) + channels_df['amt_routed_in_7day'] = channels_df.apply(lambda row: int(forwards_df_in_7d_sum.loc[row.chan_id].amt_out_msat/1000) if (forwards_df_in_7d_sum.index == row.chan_id).any() else 0, axis=1) + channels_df['amt_routed_out_7day'] = channels_df.apply(lambda row: int(forwards_df_out_7d_sum.loc[row.chan_id].amt_out_msat/1000) if (forwards_df_out_7d_sum.index == row.chan_id).any() else 0, axis=1) + channels_df['net_routed_7day'] = channels_df.apply(lambda row: round((row['amt_routed_out_7day']-row['amt_routed_in_7day'])/row['capacity'], 1), axis=1) + channels_df['revenue_7day'] = channels_df.apply(lambda row: int(forwards_df_out_7d_sum.loc[row.chan_id].fee) if forwards_df_out_7d_sum.empty == False and (forwards_df_out_7d_sum.index == row.chan_id).any() else 0, axis=1) + channels_df['revenue_assist_7day'] = channels_df.apply(lambda row: int(forwards_df_in_7d_sum.loc[row.chan_id].fee) if forwards_df_in_7d_sum.empty == False and (forwards_df_in_7d_sum.index == row.chan_id).any() else 0, axis=1) + channels_df['out_rate'] = channels_df.apply(lambda row: int((row['revenue_7day']/row['amt_routed_out_7day'])*1000000) if row['amt_routed_out_7day'] > 0 else 0, axis=1) + channels_df['failed_out_1day'] = 0 if failed_htlc_df.empty else channels_df.apply(lambda row: len(failed_htlc_df[failed_htlc_df['chan_id_out']==row.chan_id]), axis=1) + payments = Payments.objects.filter(status=2).filter(creation_date__gte=filter_7day).filter(rebal_chan__isnull=False) + invoices = Invoices.objects.filter(state=1).filter(settle_date__gte=filter_7day).filter(r_hash__in=payments.values_list('payment_hash')) + payments_df_7d = DataFrame.from_records(payments.filter(creation_date__gte=filter_7day).values()) + invoices_df_7d = DataFrame.from_records(invoices.filter(settle_date__gte=filter_7day).values()) + invoices_df_7d_sum = DataFrame() if invoices_df_7d.empty else invoices_df_7d.groupby('chan_in', as_index=True).sum() + invoice_hashes_7d = DataFrame() if invoices_df_7d.empty else invoices_df_7d.groupby('chan_in', as_index=True)['r_hash'].apply(list) + channels_df['amt_rebal_in_7day'] = channels_df.apply(lambda row: int(invoices_df_7d_sum.loc[row.chan_id].amt_paid) if invoices_df_7d_sum.empty == False and (invoices_df_7d_sum.index == row.chan_id).any() else 0, axis=1) + channels_df['costs_7day'] = channels_df.apply(lambda row: 0 if row['amt_rebal_in_7day'] == 0 else int(payments_df_7d.set_index('payment_hash', inplace=False).loc[invoice_hashes_7d[row.chan_id] if invoice_hashes_7d.empty == False and (invoice_hashes_7d.index == row.chan_id).any() else []]['fee'].sum()), axis=1) + channels_df['rebal_ppm'] = channels_df.apply(lambda row: int((row['costs_7day']/row['amt_rebal_in_7day'])*1000000) if row['amt_rebal_in_7day'] > 0 else 0, axis=1) + channels_df['max_suggestion'] = channels_df.apply(lambda row: int((row['out_rate'] if row['out_rate'] > 0 else row['local_fee_rate'])*1.15) if row['in_percent'] > 25 else int(row['local_fee_rate']), axis=1) + channels_df['max_suggestion'] = channels_df.apply(lambda row: row['local_fee_rate']+25 if row['max_suggestion'] > (row['local_fee_rate']+25) or row['max_suggestion'] == 0 else row['max_suggestion'], axis=1) + channels_df['min_suggestion'] = channels_df.apply(lambda row: int((row['out_rate'] if row['out_rate'] > 0 else row['local_fee_rate'])*0.75) if row['out_percent'] > 25 else int(row['local_fee_rate']), axis=1) + channels_df['min_suggestion'] = channels_df.apply(lambda row: row['local_fee_rate']-50 if row['min_suggestion'] < (row['local_fee_rate']-50) else row['min_suggestion'], axis=1) + channels_df['assisted_ratio'] = channels_df.apply(lambda row: round((row['revenue_assist_7day'] if row['revenue_7day'] == 0 else row['revenue_assist_7day']/row['revenue_7day']), 2), axis=1) + channels_df['profit_margin'] = channels_df.apply(lambda row: row['out_rate']*((100-row['ar_max_cost'])/100), axis=1) + channels_df['adjusted_out_rate'] = channels_df.apply(lambda row: int(row['out_rate']+row['net_routed_7day']*row['assisted_ratio']), axis=1) + channels_df['adjusted_rebal_rate'] = channels_df.apply(lambda row: int(row['rebal_ppm']+row['profit_margin']), axis=1) + channels_df['out_rate_only'] = channels_df.apply(lambda row: int(row['out_rate']+row['net_routed_7day']*row['out_rate']*0.02), axis=1) + channels_df['fee_rate_only'] = channels_df.apply(lambda row: int(row['local_fee_rate']+row['net_routed_7day']*row['local_fee_rate']*0.05), axis=1) + channels_df['new_rate'] = channels_df.apply(lambda row: row['adjusted_out_rate'] if row['net_routed_7day'] != 0 else (row['adjusted_rebal_rate'] if row['rebal_ppm'] > 0 and row['out_rate'] > 0 else (row['out_rate_only'] if row['out_rate'] > 0 else (row['min_suggestion'] if row['net_routed_7day'] == 0 and row['in_percent'] < 25 else row['fee_rate_only']))), axis=1) + channels_df['new_rate'] = channels_df.apply(lambda row: 0 if row['new_rate'] < 0 else row['new_rate'], axis=1) + channels_df['adjustment'] = channels_df.apply(lambda row: int(row['new_rate']-row['local_fee_rate']), axis=1) + channels_df['new_rate'] = channels_df.apply(lambda row: row['local_fee_rate']-10 if row['adjustment']==0 and row['out_percent']>=25 and row['net_routed_7day']==0 else row['new_rate'], axis=1) + channels_df['new_rate'] = channels_df.apply(lambda row: row['local_fee_rate']+25 if row['adjustment']==0 and row['out_percent']<25 and row['failed_out_1day']>25 else row['new_rate'], axis=1) + channels_df['new_rate'] = channels_df.apply(lambda row: row['max_suggestion'] if row['new_rate'] > row['max_suggestion'] else row['new_rate'], axis=1) + channels_df['new_rate'] = channels_df.apply(lambda row: row['min_suggestion'] if row['new_rate'] < row['min_suggestion'] else row['new_rate'], axis=1) + channels_df['new_rate'] = channels_df.apply(lambda row: int(round(row['new_rate']/5, 0)*5), axis=1) + channels_df['adjustment'] = channels_df.apply(lambda row: int(row['new_rate']-row['local_fee_rate']), axis=1) + update_df = channels_df[channels_df['adjustment']!=0] + if not update_df.empty: + for target_channel in update_df.to_dict(orient='records'): + print('Updating fees for channel ' + str(target_channel['chan_id']) + ' to a value of: ' + str(target_channel['new_rate'])) + channel = Channels.objects.filter(chan_id=target_channel['chan_id'])[0] + channel_point = ln.ChannelPoint() + channel_point.funding_txid_bytes = bytes.fromhex(channel.funding_txid) + channel_point.funding_txid_str = channel.funding_txid + channel_point.output_index = channel.output_index + stub.UpdateChannelPolicy(ln.PolicyUpdateRequest(chan_point=channel_point, base_fee_msat=channel.local_base_fee, fee_rate=(target_channel['new_rate']/1000000), time_lock_delta=40)) + channel.local_fee_rate = target_channel['new_rate'] + channel.fees_updated = datetime.now() + channel.save() + Autofees(chan_id=channel.chan_id, peer_alias=channel.alias, setting='Fee Rate', old_value=target_channel['local_fee_rate'], new_value=target_channel['new_rate']).save() + def main(): try: stub = lnrpc.LightningStub(lnd_connect(settings.LND_DIR_PATH, settings.LND_NETWORK, settings.LND_RPC_SERVER)) #Update data update_channels(stub) update_peers(stub) - update_payments(stub) update_invoices(stub) + update_payments(stub) update_forwards(stub) update_onchain(stub) update_closures(stub) reconnect_peers(stub) clean_payments(stub) + auto_fees(stub) except Exception as e: print('Error processing background data: ' + str(e)) diff --git a/rebalancer.py b/rebalancer.py index 4bc72539..21a97cfd 100644 --- a/rebalancer.py +++ b/rebalancer.py @@ -37,6 +37,7 @@ def run_rebalancer(rebalance): elif payment_response.status == 2: #SUCCESSFUL rebalance.status = 2 + successful_out = payment_response.htlcs[0].route.hops[0].pub_key elif payment_response.status == 3: #FAILURE if payment_response.failure_reason == 1: @@ -67,12 +68,12 @@ def run_rebalancer(rebalance): rebalance.stop = datetime.now() rebalance.save() if rebalance.status == 2: - update_channel(stub, rebalance.last_hop_pubkey) + update_channels(stub, rebalance.last_hop_pubkey, successful_out) auto_rebalance_channels = Channels.objects.filter(is_active=True, is_open=True, private=False).annotate(percent_outbound=((Sum('local_balance')+Sum('pending_outbound'))*100)/Sum('capacity')).annotate(inbound_can=(((Sum('remote_balance')+Sum('pending_inbound'))*100)/Sum('capacity'))/Sum('ar_in_target')) inbound_cans = auto_rebalance_channels.filter(remote_pubkey=rebalance.last_hop_pubkey).filter(auto_rebalance=True, inbound_can__gte=1) if len(inbound_cans) > 0: outbound_cans = list(auto_rebalance_channels.filter(auto_rebalance=False, percent_outbound__gte=F('ar_out_target')).values_list('chan_id', flat=True)) - next_rebalance = Rebalancer(value=rebalance.value, fee_limit=rebalance.fee_limit, outgoing_chan_ids=str(outbound_cans).replace('\'', ''), last_hop_pubkey=rebalance.last_hop_pubkey, target_alias=rebalance.target_alias, duration=rebalance.duration) + next_rebalance = Rebalancer(value=rebalance.value, fee_limit=rebalance.fee_limit, outgoing_chan_ids=str(outbound_cans).replace('\'', ''), last_hop_pubkey=rebalance.last_hop_pubkey, target_alias=rebalance.target_alias, duration=1) next_rebalance.save() else: next_rebalance = None @@ -80,8 +81,15 @@ def run_rebalancer(rebalance): next_rebalance = None return next_rebalance -def update_channel(stub, peer_pubkey): - channel = stub.ListChannels(ln.ListChannelsRequest(peer=bytes.fromhex(peer_pubkey))).channels[0] +def update_channels(stub, incoming_channel, outgoing_channel): + # Incoming channel update + channel = stub.ListChannels(ln.ListChannelsRequest(peer=bytes.fromhex(incoming_channel))).channels[0] + db_channel = Channels.objects.filter(chan_id=channel.chan_id)[0] + db_channel.local_balance = channel.local_balance + db_channel.remote_balance = channel.remote_balance + db_channel.save() + # Outgoing channel update + channel = stub.ListChannels(ln.ListChannelsRequest(peer=bytes.fromhex(outgoing_channel))).channels[0] db_channel = Channels.objects.filter(chan_id=channel.chan_id)[0] db_channel.local_balance = channel.local_balance db_channel.remote_balance = channel.remote_balance @@ -127,7 +135,7 @@ def auto_schedule(): # TLDR: willing to pay 1 sat for every value_per_fee sats moved if Rebalancer.objects.filter(last_hop_pubkey=target.remote_pubkey).exclude(status=0).exists(): last_rebalance = Rebalancer.objects.filter(last_hop_pubkey=target.remote_pubkey).exclude(status=0).order_by('-id')[0] - if not (last_rebalance.value != target_value or last_rebalance.status in [2, 6] or (last_rebalance.status in [3, 4, 5, 7, 400, 408] and (int((datetime.now() - last_rebalance.stop).total_seconds() / 60) > 30)) or (last_rebalance.status == 1 and (int((datetime.now() - last_rebalance.start).total_seconds() / 60) > 30))): + if not (last_rebalance.value != target_value or last_rebalance.status == 2 or (last_rebalance.status in [3, 4, 5, 6, 7, 400, 408] and (int((datetime.now() - last_rebalance.stop).total_seconds() / 60) > 30)) or (last_rebalance.status == 1 and (int((datetime.now() - last_rebalance.start).total_seconds() / 60) > 30))): continue print('Creating Auto Rebalance Request') print('Request for:', target.chan_id)