diff --git a/.gitignore b/.gitignore index d8640d6c..6b65c096 100644 --- a/.gitignore +++ b/.gitignore @@ -143,4 +143,7 @@ GitHub.sublime-settings .history # Tailwind -node_modules/ \ No newline at end of file +node_modules/ + +# Domains +deploy/letsencrypt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e1dbb2ae..9f0db1e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,4 +55,8 @@ COPY --from=node-builder /app ./ RUN python manage.py collectstatic --noinput --clear -i tailwindcss # Runtime command that executes when "docker run" is called. -CMD ["uwsgi", "--ini", "/app/wsgi.ini"] \ No newline at end of file + +# Check traefik + etcd configs to running domains +ENV ENABLE_CHECK_TRAEFIK=True + +CMD ["uwsgi", "--ini", "/app/wsgi.ini"] diff --git a/app/contrib/domains/__init__.py b/app/contrib/domains/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/contrib/domains/route53/__init__.py b/app/contrib/domains/route53/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/contrib/domains/route53/admin.py b/app/contrib/domains/route53/admin.py new file mode 100644 index 00000000..2137f111 --- /dev/null +++ b/app/contrib/domains/route53/admin.py @@ -0,0 +1,56 @@ +from django.contrib import admin +from django.http.request import HttpRequest + +from django.utils.html import format_html + + +from .models import VPS, HostedZone, RecordSet + + +class VPSAdmin(admin.ModelAdmin): + list_display = ("name", "static_ip") + # list_filter = ("name", ) + +admin.site.register(VPS, VPSAdmin) + + +class RecordSetAdmin(admin.ModelAdmin): + search_fields = ("name", ) + + +admin.site.register(RecordSet, RecordSetAdmin) + + +class RecordSetInline(admin.TabularInline): + model = RecordSet + extra = 0 + + +class HostedZoneAdmin(admin.ModelAdmin): + list_display = ("name", "vps", "healtcheck") + search_fields = ("name", ) + list_filter = ("healtcheck", ) + readonly_fields = ("healtcheck", ) + inlines = [RecordSetInline, ] + + class Media: + css = { + 'all': ('css/route53/tags.css',) + } + + def has_add_permission(self, request: HttpRequest) -> bool: + return False + + def has_change_permission(self, request: HttpRequest, obj=None) -> bool: + return False + + def vps(self, obj): + html = "" + + return format_html(html) + + +admin.site.register(HostedZone, HostedZoneAdmin) \ No newline at end of file diff --git a/app/contrib/domains/route53/apps.py b/app/contrib/domains/route53/apps.py new file mode 100644 index 00000000..262e5cac --- /dev/null +++ b/app/contrib/domains/route53/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig +from django.db.models.signals import pre_delete, pre_save + + +class Route53AppConfig(AppConfig): + name = "contrib.domains.route53" + + def ready(self): + from . import signals, models + + # pre_save.connect(signals.create_or_update_route53, models.HostedZone) + + pre_delete.connect(signals.delete_on_route53, models.RecordSet) + pre_delete.connect(signals.delete_on_route53, models.HostedZone) diff --git a/app/contrib/domains/route53/entries.py b/app/contrib/domains/route53/entries.py new file mode 100644 index 00000000..d90ba7b8 --- /dev/null +++ b/app/contrib/domains/route53/entries.py @@ -0,0 +1,51 @@ +from django.conf import settings + + +class Manager: + def __init__(self, name=None, fields=None): + import boto3 + + self.client = boto3.client( + "route53", + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + # aws_region=settings.AWS_REGION + ) + + def list_hosted_zones(self): + response = self.client.list_hosted_zones() + hosted_zones = response.get("HostedZones", []) + + return list( + map( + lambda x: HostedZone(id=x.get("Id", ""), name=x.get("Name", "")), + hosted_zones, + ) + ) + + def get_hosted_zone(self, id): + return self.client.get_hosted_zone(Id=id) + + def list_resource_record_sets(self, id): + return self.client.list_resource_record_sets(HostedZoneId=id) + + def test_dns_answer(self, id, record_name, record_type): + return self.client.test_dns_answer( + HostedZoneId=id, RecordName=record_name, RecordType=record_type + ) + + +class HostedZone(object): + id: str + name: str + + def __init__(self, id, name): + self.id = id + self.name = name + + objects = Manager() + + @property + def delegation_set(self): + response = self.objects.client.get_hosted_zone(Id=self.id) + return response.get("DelegationSet") diff --git a/app/contrib/domains/route53/management/__init__.py b/app/contrib/domains/route53/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/contrib/domains/route53/management/commands/__init__.py b/app/contrib/domains/route53/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/contrib/domains/route53/management/commands/update_healthcheck.py b/app/contrib/domains/route53/management/commands/update_healthcheck.py new file mode 100644 index 00000000..5d28a83c --- /dev/null +++ b/app/contrib/domains/route53/management/commands/update_healthcheck.py @@ -0,0 +1,24 @@ +import boto3 + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from contrib.domains.route53.models import HostedZone, RecordSet + + +class Command(BaseCommand): + help = "" + + def handle(self, *args, **options): + for hz in HostedZone.objects.all(): + hz.healtcheck = hz.check_ns() + hz.save() + + is_ok = HostedZone.objects.filter(healtcheck=True).count() + is_fail = HostedZone.objects.filter(healtcheck=False).count() + + self.stdout.write( + self.style.SUCCESS( + f"Successfully to update HostedZone: {is_ok} ok | {is_fail} fail." + ) + ) diff --git a/app/contrib/domains/route53/management/commands/update_hosted_zones.py b/app/contrib/domains/route53/management/commands/update_hosted_zones.py new file mode 100644 index 00000000..c8920487 --- /dev/null +++ b/app/contrib/domains/route53/management/commands/update_hosted_zones.py @@ -0,0 +1,79 @@ +import boto3 + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from contrib.domains.route53.models import HostedZone, RecordSet +from contrib.domains.route53.utils import get_record_set + +# from route53.entries import HostedZone as HostedZoneEntry + + +class Command(BaseCommand): + help = "" + + def get_list_resource_record_sets(self, hosted_zone_id, next_marker=None): + params = {"Marker": next_marker} if next_marker else {} + data = self.route53.list_resource_record_sets( + HostedZoneId=hosted_zone_id, **params + ) + + yield self.prepare_record_set_list(data["ResourceRecordSets"]) + + if data["IsTruncated"] == True and "NextMarker" in data.keys(): + yield from self.get_list_resource_record_sets( + hosted_zone_id, next_marker=data["NextMarker"] + ) + + def get_list_hosted_zones(self, next_marker=None): + params = {"Marker": next_marker} if next_marker else {} + data = self.route53.list_hosted_zones(**params) + + yield self.prepare_hosted_zone_list(data["HostedZones"]) + + if data["IsTruncated"] == True: + # Chama função até não existir mais páginas + yield from self.get_list_hosted_zones(next_marker=data["NextMarker"]) + + def prepare_record_set_list(self, records=[]): + return list(map(get_record_set, records)) + + def prepare_hosted_zone_list(self, hosted_zones=[]): + return list( + map( + lambda x: dict(id=x.get("Id", ""), name=x.get("Name", "")), + hosted_zones, + ) + ) + + def handle(self, *args, **options): + self.count = 0 + + self.route53 = boto3.client( + "route53", + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + # aws_region=settings.AWS_REGION + ) + + # First called + hosted_zones_next = self.get_list_hosted_zones() + for hosted_zones in hosted_zones_next: + for hosted_zone in hosted_zones: + obj, created = HostedZone.objects.get_or_create(**hosted_zone) + + records_next = self.get_list_resource_record_sets(obj.id) + for records in records_next: + for record in records: + obj, created = RecordSet.objects.get_or_create( + hosted_zone_id=hosted_zone["id"], **record + ) + + if created: + self.count += 1 + + self.stdout.write( + self.style.SUCCESS( + f"Successfully to create HostedZone: {self.count} objects." + ) + ) diff --git a/app/contrib/domains/route53/migrations/0001_initial.py b/app/contrib/domains/route53/migrations/0001_initial.py new file mode 100644 index 00000000..cea8a111 --- /dev/null +++ b/app/contrib/domains/route53/migrations/0001_initial.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.6 on 2023-10-31 19:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='HostedZone', + fields=[ + ('id', models.CharField(max_length=50, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=100)), + ], + ), + ] diff --git a/app/contrib/domains/route53/migrations/0002_vps.py b/app/contrib/domains/route53/migrations/0002_vps.py new file mode 100644 index 00000000..0b188cf5 --- /dev/null +++ b/app/contrib/domains/route53/migrations/0002_vps.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.6 on 2023-11-01 15:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('route53', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='VPS', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=40)), + ('static_ip', models.GenericIPAddressField(unique=True)), + ], + ), + ] diff --git a/app/contrib/domains/route53/migrations/0003_recordset.py b/app/contrib/domains/route53/migrations/0003_recordset.py new file mode 100644 index 00000000..cd1b3777 --- /dev/null +++ b/app/contrib/domains/route53/migrations/0003_recordset.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.6 on 2023-11-06 20:42 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('route53', '0002_vps'), + ] + + operations = [ + migrations.CreateModel( + name='RecordSet', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ('record_type', models.CharField(choices=[('SOA', 'Soa'), ('A', 'A'), ('TXT', 'Txt'), ('NS', 'Ns'), ('CNAME', 'Cname'), ('MX', 'Mx'), ('NAPTR', 'Naptr'), ('PTR', 'Ptr'), ('SRV', 'Srv'), ('SPF', 'Spf'), ('AAAA', 'Aaaa'), ('CAA', 'Caa'), ('DS', 'Ds')], max_length=5)), + ('resource', models.JSONField(blank=True, null=True)), + ('hosted_zone', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='route53.hostedzone')), + ], + ), + ] diff --git a/app/contrib/domains/route53/migrations/0004_hostedzone_healtcheck.py b/app/contrib/domains/route53/migrations/0004_hostedzone_healtcheck.py new file mode 100644 index 00000000..da4dda64 --- /dev/null +++ b/app/contrib/domains/route53/migrations/0004_hostedzone_healtcheck.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.6 on 2023-11-06 22:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('route53', '0003_recordset'), + ] + + operations = [ + migrations.AddField( + model_name='hostedzone', + name='healtcheck', + field=models.BooleanField(default=False), + ), + ] diff --git a/app/contrib/domains/route53/migrations/0005_alter_vps_options.py b/app/contrib/domains/route53/migrations/0005_alter_vps_options.py new file mode 100644 index 00000000..4f2144fe --- /dev/null +++ b/app/contrib/domains/route53/migrations/0005_alter_vps_options.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.6 on 2023-11-30 14:45 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('route53', '0004_hostedzone_healtcheck'), + ] + + operations = [ + migrations.AlterModelOptions( + name='vps', + options={'verbose_name': 'Instância', 'verbose_name_plural': 'Instâncias'}, + ), + ] diff --git a/app/contrib/domains/route53/migrations/__init__.py b/app/contrib/domains/route53/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/contrib/domains/route53/models.py b/app/contrib/domains/route53/models.py new file mode 100644 index 00000000..9d84cd3d --- /dev/null +++ b/app/contrib/domains/route53/models.py @@ -0,0 +1,148 @@ +from django.conf import settings +from django.db import models + +import boto3 +import dns.resolver +from botocore.exceptions import ClientError + + +class HostedZone(models.Model): + id = models.CharField(max_length=50, primary_key=True, auto_created=False) + name = models.CharField(max_length=100) + healtcheck = models.BooleanField(default=False) + + def __str__(self): + return self.name + + def related_instances(self): + rs1 = self.recordset_set.filter( + record_type=RecordType.A, name=self.name + ).first() + rs2 = self.recordset_set.filter( + record_type=RecordType.A, name="\\052." + self.name + ).first() + + if ( + rs1 + and rs2 + and len(rs1.resource) > 0 + and len(rs2.resource) > 0 + and rs1.resource[0] == rs2.resource[0] + ): + return VPS.objects.filter(static_ip=rs1.resource[0]) + + # if self.name == 'meurecife.org.br.': + # import ipdb;ipdb.set_trace() + + # lista = list( + # map( + # lambda x: x.vps, list(filter(lambda x: x.vps, self.recordset_set.all())) + # ) + # ) + + return VPS.objects.none() + + def check_ns(self): + record_set = self.recordset_set.filter(record_type="NS").first() + + try: + for rdata in dns.resolver.query(self.name[:-1], "NS"): + if str(rdata) in record_set.resource: + return True + except Exception: + return False + + return False + + def delete_on_route53(self): + client = boto3.client( + "route53", + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + # aws_region=settings.AWS_REGION + ) + + response = client.delete_hosted_zone(Id=self.id) + + return response["ChangeInfo"]["Status"] + + +class RecordType(models.TextChoices): + SOA = "SOA" + A = "A" + TXT = "TXT" + NS = "NS" + CNAME = "CNAME" + MX = "MX" + NAPTR = "NAPTR" + PTR = "PTR" + SRV = "SRV" + SPF = "SPF" + AAAA = "AAAA" + CAA = "CAA" + DS = "DS" + + +class RecordSet(models.Model): + hosted_zone = models.ForeignKey(HostedZone, on_delete=models.CASCADE) + name = models.CharField(max_length=100) + record_type = models.CharField(max_length=5, choices=RecordType.choices) + resource = models.JSONField(blank=True, null=True) # [""] + + def __str__(self): + return self.name + + @property + def vps(self): + if self.record_type == RecordType.A and len(self.resource) > 0: + vps = VPS.objects.filter(static_ip=self.resource[0]).first() + if vps: + return vps.name + + return None + + def delete_on_route53(self): + + if self.record_type != RecordType.NS and self.record_type != RecordType.SOA: + client = boto3.client( + "route53", + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + # aws_region=settings.AWS_REGION + ) + + try: + response = client.change_resource_record_sets( + HostedZoneId=self.hosted_zone.id, + ChangeBatch={ + "Changes": [ + { + "Action": "DELETE", + "ResourceRecordSet": { + "Name": self.name, + "Type": self.record_type, + "TTL": 300, + "ResourceRecords": list( + map(lambda x: dict(Value=x), self.resource) + ), + }, + } + ] + }, + ) + + return response["ChangeInfo"]["Status"] + except ClientError as err: + print(err) + + +class VPS(models.Model): + name = models.CharField(max_length=40) + static_ip = models.GenericIPAddressField(unique=True) + + class Meta: + verbose_name = "Instância" + verbose_name_plural = "Instâncias" + + def __str__(self): + return self.name diff --git a/app/contrib/domains/route53/signals.py b/app/contrib/domains/route53/signals.py new file mode 100644 index 00000000..2f528900 --- /dev/null +++ b/app/contrib/domains/route53/signals.py @@ -0,0 +1,4 @@ + + +def delete_on_route53(sender, instance, **kwargs): + instance.delete_on_route53() \ No newline at end of file diff --git a/app/contrib/domains/route53/static/css/route53/tags.css b/app/contrib/domains/route53/static/css/route53/tags.css new file mode 100644 index 00000000..f8b14fc7 --- /dev/null +++ b/app/contrib/domains/route53/static/css/route53/tags.css @@ -0,0 +1,15 @@ +ul.tags { + margin: 0; + padding: 0; +} + +.tags li { + list-style-type: none; + color: var(--primary-fg); + background-color: var(--primary); + display: inline-block; + padding: 2px 10px; + font-size: smaller; + font-weight: bold; + border-radius: 50px; +} \ No newline at end of file diff --git a/app/contrib/domains/route53/utils.py b/app/contrib/domains/route53/utils.py new file mode 100644 index 00000000..1291ba67 --- /dev/null +++ b/app/contrib/domains/route53/utils.py @@ -0,0 +1,12 @@ +from .models import VPS + + +def get_record_set(x): + resource_records = list(map(lambda y: y.get("Value"), x.get("ResourceRecords", []))) + record = { + "name": x.get("Name"), + "record_type": x.get("Type"), + "resource": resource_records, + } + + return record diff --git a/app/contrib/domains/traefik/__init__.py b/app/contrib/domains/traefik/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/contrib/domains/traefik/admin.py b/app/contrib/domains/traefik/admin.py new file mode 100644 index 00000000..22928216 --- /dev/null +++ b/app/contrib/domains/traefik/admin.py @@ -0,0 +1,11 @@ +from django.contrib import admin + +from .models import Route + + +class RouteAdmin(admin.ModelAdmin): + list_display = ("dns", "subdomain", "instance", "service") + autocomplete_fields = ("dns", "subdomain") + + +admin.site.register(Route, RouteAdmin) \ No newline at end of file diff --git a/app/contrib/domains/traefik/apps.py b/app/contrib/domains/traefik/apps.py new file mode 100644 index 00000000..8668d672 --- /dev/null +++ b/app/contrib/domains/traefik/apps.py @@ -0,0 +1,76 @@ +from django.apps import AppConfig +from django.db.models.signals import post_save, post_delete +from django.conf import settings + + +from etcd3 import Client + +initial_config = [ + # Setup inicial + ( + "traefik/tls/options/default/cipherSuites/0", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + ), + ( + "traefik/tls/options/default/cipherSuites/1", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + ), + ( + "traefik/tls/options/default/cipherSuites/2", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + ), + ( + "traefik/tls/options/default/cipherSuites/3", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + ), + ( + "traefik/tls/options/default/cipherSuites/4", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + ), + ( + "traefik/tls/options/default/cipherSuites/5", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + ), + ("traefik/tls/options/default/minVersion", "VersionTLS12"), + ("traefik/http/middlewares/securityHeader/headers/contenttypenosniff", "true"), + ("traefik/http/middlewares/securityHeader/headers/framedeny", "false"), + ("traefik/http/middlewares/securityHeader/headers/sslredirect", "true"), + ("traefik/http/middlewares/securityHeader/headers/stsincludesubdomains", "true"), + ("traefik/http/middlewares/securityHeader/headers/stspreload", "true"), + ("traefik/http/middlewares/securityHeader/headers/stsseconds", "63072000"), +] + + +# initial_web_application_config = [ +# ("traefik/http/routers/0-public-staging-bonde-org/tls", "true"), +# ("traefik/http/routers/0-public-staging-bonde-org/tls/certresolver", "myresolver"), +# ("traefik/http/routers/0-public-staging-bonde-org/service", "webpage@docker"), +# # Configura todas as entradas possíveis +# ("traefik/http/routers/0-public-staging-bonde-org/rule", "HostRegexp(`{host:.+}`)"), +# ("traefik/http/routers/0-public-staging-bonde-org/tls/domains/0/main", "staging.bonde.org"), +# ("traefik/http/routers/0-public-staging-bonde-org/tls/domains/0/sans/0", "*.staging.bonde.org"), +# ] + + +class TraefikAppConfig(AppConfig): + name = "contrib.domains.traefik" + + def ready(self): + import os + + if os.getenv("ENABLE_CHECK_TRAEFIK", False): + client = Client(host=settings.ETCD_HOST, port=settings.ETCD_PORT) + + configs = [] + configs.extend(initial_config) + # configs.extend(initial_web_application_config) + + for key_value in configs: + client.put(key=key_value[0], value=key_value[1]) + + + # Signals configuration + from . import signals, models + + post_save.connect(signals.update_traefik_config, sender=models.Route) + post_delete.connect(signals.delete_traefik_config, sender=models.Route) diff --git a/app/contrib/domains/traefik/migrations/0001_initial.py b/app/contrib/domains/traefik/migrations/0001_initial.py new file mode 100644 index 00000000..410bebc5 --- /dev/null +++ b/app/contrib/domains/traefik/migrations/0001_initial.py @@ -0,0 +1,26 @@ +# Generated by Django 4.2.6 on 2023-11-30 14:45 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('route53', '0005_alter_vps_options'), + ] + + operations = [ + migrations.CreateModel( + name='Site', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('service', models.CharField(choices=[('webpage@docker', 'Public'), ('cms@docker', 'CMS')], max_length=25)), + ('dns', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='route53.hostedzone')), + ('instance', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='route53.vps')), + ('subdomain', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='route53.recordset')), + ], + ), + ] diff --git a/app/contrib/domains/traefik/migrations/0002_auto_20231206_1951.py b/app/contrib/domains/traefik/migrations/0002_auto_20231206_1951.py new file mode 100644 index 00000000..0ba53162 --- /dev/null +++ b/app/contrib/domains/traefik/migrations/0002_auto_20231206_1951.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2 on 2023-12-06 19:51 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('route53', '0005_alter_vps_options'), + ('traefik', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Route', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('service', models.CharField(choices=[('webpage@docker', 'Public'), ('cms@docker', 'CMS')], max_length=25)), + ('dns', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='route53.hostedzone')), + ('instance', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='route53.vps')), + ('subdomain', models.ForeignKey(blank=True, limit_choices_to={'record_type': 'A'}, null=True, on_delete=django.db.models.deletion.SET_NULL, to='route53.recordset')), + ], + ), + migrations.DeleteModel( + name='Site', + ), + ] diff --git a/app/contrib/domains/traefik/migrations/__init__.py b/app/contrib/domains/traefik/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/contrib/domains/traefik/models.py b/app/contrib/domains/traefik/models.py new file mode 100644 index 00000000..6cbc23d6 --- /dev/null +++ b/app/contrib/domains/traefik/models.py @@ -0,0 +1,65 @@ +from django.conf import settings +from django.db import models +from etcd3 import Client + +from contrib.domains.route53.models import HostedZone, RecordSet, VPS, RecordType + + +class Container(models.TextChoices): + webpage = "webpage@docker", "Public" + djangocms = "cms@docker", "CMS" + + +class Route(models.Model): + dns = models.ForeignKey(HostedZone, on_delete=models.CASCADE) + subdomain = models.ForeignKey( + RecordSet, + on_delete=models.SET_NULL, + null=True, + blank=True, + limit_choices_to={"record_type": RecordType.A}, + ) + instance = models.ForeignKey(VPS, on_delete=models.SET_NULL, null=True, blank=True) + service = models.CharField(max_length=25, choices=Container.choices) + + def get_traefik_config(self): + domain = ( + self.subdomain.name + if self.subdomain.name[-1] != "." + else self.subdomain.name[:-1] + ) + prefix = f"traefik/http/routers/{self.id}-{domain.replace('.', '-')}" + + configs = ( + ("/tls", "true"), + ("/tls/certresolver", "myresolver"), + ("/service", self.service), + ( + "/rule", + f"Host(`{domain}`,`www.{domain}`)" + ) + ) + + return list(map(lambda x: (prefix + x[0], x[1]), configs)) + + def update_traefik_config(self): + try: + client = Client(host=settings.ETCD_HOST, port=settings.ETCD_PORT) + + for config in self.get_traefik_config(): + client.put(key=config[0], value=config[1]) + + return True + except Exception: + return False + + def delete_traefik_config(self): + try: + client = Client(host=settings.ETCD_HOST, port=settings.ETCD_PORT) + + for config in self.get_traefik_config(): + client.delete_range(key=config[0]) + + return True + except Exception: + return False \ No newline at end of file diff --git a/app/contrib/domains/traefik/signals.py b/app/contrib/domains/traefik/signals.py new file mode 100644 index 00000000..23c35f4b --- /dev/null +++ b/app/contrib/domains/traefik/signals.py @@ -0,0 +1,8 @@ + + +def update_traefik_config(sender, instance, **kwargs): + instance.update_traefik_config() + + +def delete_traefik_config(sender, instance, **kwargs): + instance.delete_traefik_config() \ No newline at end of file diff --git a/app/project/settings/base.py b/app/project/settings/base.py index 0e189e1d..9e77b4c0 100644 --- a/app/project/settings/base.py +++ b/app/project/settings/base.py @@ -19,7 +19,12 @@ ALLOWED_HOSTS=(list, None), DISABLE_RECAPTCHA=(bool, False), RECAPTCHA_PUBLIC_KEY=(str, "MyRecaptchaKey123"), - RECAPTCHA_PRIVATE_KEY=(str, "MyRecaptchaPrivateKey456") + RECAPTCHA_PRIVATE_KEY=(str, "MyRecaptchaPrivateKey456"), + AWS_ACCESS_KEY_ID=(str, ""), + AWS_SECRET_ACCESS_KEY=(str, ""), + AWS_REGION=(str, ""), + ETCD_HOST=(str, "127.0.0.1"), + ETCD_PORT=(int, 2379) ) # Build paths inside the project like this: BASE_DIR / 'subdir'. @@ -81,6 +86,9 @@ "contrib.frontend.grid", "contrib.frontend.maps", "contrib.ga", + # + "contrib.domains.route53", + "contrib.domains.traefik", # Experimentação "eleicao", "django_social_share", @@ -245,3 +253,18 @@ RECAPTCHA_PRIVATE_KEY = env("RECAPTCHA_PRIVATE_KEY") SILENCED_SYSTEM_CHECKS = ['captcha.recaptcha_test_key_error'] + + +# Bonde Router + +# AWS +AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID") + +AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY") + +AWS_REGION = env("AWS_REGION") + +# Etcd +ETCD_HOST = env("ETCD_HOST") + +ETCD_PORT = env("ETCD_PORT") \ No newline at end of file diff --git a/app/requirements.txt b/app/requirements.txt index aaa5664a..0637d5a0 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -5,7 +5,7 @@ django-select2 django-formtools django-recaptcha==3 # Django CMS -django-cms +django-cms>=3 djangocms-text-ckeditor djangocms-picture django-colorfield @@ -20,3 +20,6 @@ reportlab django-social-share pyjwt requests +# Domains +etcd3-py>=0.1.6 +dnspython>=2.4.2 diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 56005a0a..6f170d87 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,64 +1,129 @@ services: - cms: - image: ${DOCKER_IMAGE:-nossas/cms:main} - restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" - pull_policy: always - environment: - - DEBUG=${DEBUG:-True} - - ALLOWED_HOSTS=${ALLOWED_HOSTS:-"docker.localhost"} - - CMS_DATABASE_URL=${CMS_DATABASE_URL} - - BONDE_DATABASE_URL=${BONDE_DATABASE_URL} - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - - AWS_STORAGE_BUCKET_NAME=${AWS_STORAGE_BUCKET_NAME} - - RECAPTCHA_PUBLIC_KEY=${RECAPTCHA_PUBLIC_KEY} - - RECAPTCHA_PRIVATE_KEY=${RECAPTCHA_PRIVATE_KEY} - - DISABLE_RECAPTCHA=${DISABLE_RECAPTCHA} - - BONDE_ACTION_API_URL=${BONDE_ACTION_API_URL} - - BONDE_ACTION_SECRET_KEY=${BONDE_ACTION_SECRET_KEY} - labels: - - traefik.enable=true - - traefik.http.routers.cms.priority=10 - - traefik.http.services.cms.loadbalancer.server.port=8000 - - traefik.http.routers.cms.tls=true - - traefik.http.routers.cms.tls.certresolver=myresolver - - traefik.http.routers.cms.rule=${TRAEFIK_ROUTERS_RULE:-"HostRegexp(`cms.staging.bonde.org`)"} + # traefik: + # image: "traefik:v2.9" + # depends_on: + # - etcd + # command: + # # API + # - "--global.checknewversion=${TRAEFIK_CHECK_NEW_VERSION:-false}" + # - "--global.sendanonymoususage=${TRAEFIK_SEND_ANONYMOUS_USAGE:-false}" + # - "--api.insecure=true" + # # Providers + # - "--providers.docker=true" + # - "--providers.docker.exposedbydefault=false" + # - "--providers.docker.defaultRule=Host(`{{ index .Labels \"com.docker.compose.service\"}}.${DEFAULT_DOMAIN_RULE:-staging.bonde.org}`)" - # prometheus: - # image: prom/prometheus - # restart: 'no' - # user: root - # volumes: - # - prometheus_data:/prometheus - # # - ./prometheus.yml:/etc/prometheus/prometheus.yml - # labels: - # - traefik.enable=true - # - traefik.http.services.prometheus.loadbalancer.server.port=9090 - # - traefik.http.routers.prometheus.tls=true - # - traefik.http.routers.prometheus.tls.certresolver=myresolver - - # grafana: - # image: grafana/grafana - # user: root - # environment: - # GF_INSTALL_PLUGINS: "grafana-clock-panel,grafana-simple-json-datasource" - # restart: 'no' - # volumes: - # - grafana_data:/var/lib/grafana - # depends_on: - # - prometheus - # labels: - # - traefik.enable=true - # - traefik.http.services.grafana.loadbalancer.server.port=3000 - # - traefik.http.routers.grafana.tls=true - # - traefik.http.routers.grafana.tls.certresolver=myresolver + # - "--providers.etcd=true" + # - "--providers.etcd.endpoints=etcd:2379" + # - "--providers.etcd.rootkey=traefik" + # # Logs + # - "--log.filepath=/logs/traefik.log" + # - "--log.format=json" + # - "--log.level=${TRAEFIK_LOG_LEVEL:-ERROR}" + # - "--metrics.prometheus" + # - "--accesslog.filepath=/logs/access.log" + # - "--accesslog.format=json" + # # Entrypoints + # - "--entrypoints.web.address=:80" + # - "--entrypoints.web.http.redirections.entryPoint.to=websecure" + # - "--entrypoints.web.http.redirections.entryPoint.scheme=https" + # - "--entrypoints.web.http.redirections.entrypoint.permanent=true" + # - "--entrypoints.websecure.address=:443" + # - "--entrypoints.websecure.http.middlewares=securityHeader@etcd" + # # + # - "--pilot.token=${TRAEFIK_PILOT_TOKEN:-}" + # - "--ping" + # - "--certificatesresolvers.myresolver.acme.tlschallenge=true" + # - "--certificatesresolvers.myresolver.acme.email=${DEFAULT_EMAIL_ACME:-tech@bonde.devel}" + # - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" + # - "--certificatesresolvers.myresolver.acme.dnschallenge.provider=route53" + # restart: always + # healthcheck: + # test: ['CMD', 'traefik', 'healthcheck', '--ping'] + # interval: 10s + # timeout: 10s + # retries: 5 + # ports: + # - "80:80" + # - "443:443" + # # - "8080:8080" + # # networks: + # # - bonde + # volumes: + # - "/var/run/docker.sock:/var/run/docker.sock:ro" + # - letsencrypt:/letsencrypt + # # env_file: + # # - .env + # # environment: + # # AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-xxxxxxx} + # # AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-xxxxxx} + # # AWS_REGION: ${AWS_REGION:-us-east-1} + # labels: + # - traefik.enable=true + # # global redirection: https (www.) to https + # - traefik.http.routers.wwwsecure-catchall.rule=HostRegexp(`{host:(www\\.).+}`) + # - traefik.http.routers.wwwsecure-catchall.entrypoints=websecure + # - traefik.http.routers.wwwsecure-catchall.tls=true + # - traefik.http.routers.wwwsecure-catchall.middlewares=wwwtohttps + # # middleware: http(s)://(www.) to https:// + # - traefik.http.middlewares.wwwtohttps.redirectregex.regex=^https?://(?:www\\.)?(.+) + # - traefik.http.middlewares.wwwtohttps.redirectregex.replacement=https://$${1} + # - traefik.http.middlewares.wwwtohttps.redirectregex.permanent=true + # # export traefik dashboard + # - traefik.http.services.traefik.loadbalancer.server.port=8080 + # - traefik.http.routers.traefik.tls=true + # - traefik.http.routers.traefik.tls.certresolver=myresolver -networks: - default: - name: bonde + # etcd: + # image: 'bitnami/etcd:latest' + # environment: + # - ALLOW_NONE_AUTHENTICATION=yes + # - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + # # - ETCD_PROXY=on + # - ETCD_ENABLE_V2=true + # # - ETCD_ + # - ETCDCTL_API=3 + # volumes: + # - etcd_data:/bitnami/etcd + # ports: + # - 2379:2379 + # - 2380:2380 + + cms: + image: ${DOCKER_IMAGE:-nossas/cms:main} + restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" + pull_policy: always + # depends_on: + # - etcd + environment: + - DEBUG=${DEBUG:-True} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-"docker.localhost"} + - CMS_DATABASE_URL=${CMS_DATABASE_URL} + - BONDE_DATABASE_URL=${BONDE_DATABASE_URL} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_STORAGE_BUCKET_NAME=${AWS_STORAGE_BUCKET_NAME} + - RECAPTCHA_PUBLIC_KEY=${RECAPTCHA_PUBLIC_KEY} + - RECAPTCHA_PRIVATE_KEY=${RECAPTCHA_PRIVATE_KEY} + - DISABLE_RECAPTCHA=${DISABLE_RECAPTCHA} + - BONDE_ACTION_API_URL=${BONDE_ACTION_API_URL} + - BONDE_ACTION_SECRET_KEY=${BONDE_ACTION_SECRET_KEY} + - ETCD_HOST=${ETCD_HOST:-"etcd"} + - ETCD_PORT=${ETCD_PORT:-2379} + labels: + - traefik.enable=true + - traefik.http.routers.cms.priority=10 + - traefik.http.services.cms.loadbalancer.server.port=8000 + - traefik.http.routers.cms.tls=true + - traefik.http.routers.cms.tls.certresolver=myresolver + - traefik.http.routers.cms.rule=${TRAEFIK_ROUTERS_RULE:-"HostRegexp(`cms.staging.bonde.org`)"} # volumes: -# prometheus_data: +# letsencrypt: # driver: local -# grafana_data: +# etcd_data: # driver: local + +networks: + default: + name: bonde diff --git a/docker-compose.yml b/docker-compose.yml index 6f03f9ad..c33de851 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,77 +1,165 @@ services: - # web: - # build: - # context: . - # dockerfile: Dockerfile.local - # # image: nossas/cms:main - # networks: - # - bonde - # ports: - # - "80:8000" - # env_file: - # - .env-local - # depends_on: - # - db - - # db: - # image: postgres:latest - # restart: always - # environment: - # - POSTGRES_USER=postgres - # - POSTGRES_PASSWORD=postgres - # - POSTGRES_DB=cms - # # logging: - # # options: - # # max-size: 10m - # # max-file: "3" - # networks: - # - bonde - # ports: - # - '5432:5432' - # volumes: - # - /tmp/data:/var/lib/postgresql/data + traefik: + image: "traefik:v2.9" + depends_on: + - etcd + command: + # API + - "--global.checknewversion=${TRAEFIK_CHECK_NEW_VERSION:-false}" + - "--global.sendanonymoususage=${TRAEFIK_SEND_ANONYMOUS_USAGE:-false}" + - "--api.insecure=true" + # Providers + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--providers.docker.defaultRule=Host(`{{ index .Labels \"com.docker.compose.service\"}}.${DEFAULT_DOMAIN_RULE:-staging.bonde.org}`)" - prometheus: - image: prom/prometheus - restart: 'no' - user: root - volumes: - - /tmp/prometheus_data:/prometheus - - ./deploy/prometheus.yml:/etc/prometheus/prometheus.yml + - "--providers.etcd=true" + - "--providers.etcd.endpoints=etcd:2379" + - "--providers.etcd.rootkey=traefik" + # Logs + - "--log.filepath=/logs/traefik.log" + - "--log.format=json" + - "--log.level=${TRAEFIK_LOG_LEVEL:-ERROR}" + - "--metrics.prometheus" + - "--accesslog.filepath=/logs/access.log" + - "--accesslog.format=json" + # Entrypoints + - "--entrypoints.web.address=:80" + - "--entrypoints.web.http.redirections.entryPoint.to=websecure" + - "--entrypoints.web.http.redirections.entryPoint.scheme=https" + - "--entrypoints.web.http.redirections.entrypoint.permanent=true" + - "--entrypoints.websecure.address=:443" + - "--entrypoints.websecure.http.middlewares=securityHeader@etcd" + # + - "--pilot.token=${TRAEFIK_PILOT_TOKEN:-}" + - "--ping" + - "--certificatesresolvers.myresolver.acme.tlschallenge=true" + - "--certificatesresolvers.myresolver.acme.email=${DEFAULT_EMAIL_ACME:-tech@bonde.devel}" + - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" + - "--certificatesresolvers.myresolver.acme.dnschallenge.provider=route53" + restart: always + healthcheck: + test: ['CMD', 'traefik', 'healthcheck', '--ping'] + interval: 10s + timeout: 10s + retries: 5 + ports: + - "80:80" + - "443:443" + # - "8080:8080" # networks: # - bonde - # ports: - # - 9090:9090 - network_mode: "host" - # depends_on: - # - web + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" + - "/letsencrypt:/letsencrypt" + env_file: + - .env + # environment: + # AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-xxxxxxx} + # AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-xxxxxx} + # AWS_REGION: ${AWS_REGION:-us-east-1} + labels: + - traefik.enable=true + # global redirection: https (www.) to https + - traefik.http.routers.wwwsecure-catchall.rule=HostRegexp(`{host:(www\\.).+}`) + - traefik.http.routers.wwwsecure-catchall.entrypoints=websecure + - traefik.http.routers.wwwsecure-catchall.tls=true + - traefik.http.routers.wwwsecure-catchall.middlewares=wwwtohttps + # middleware: http(s)://(www.) to https:// + - traefik.http.middlewares.wwwtohttps.redirectregex.regex=^https?://(?:www\\.)?(.+) + - traefik.http.middlewares.wwwtohttps.redirectregex.replacement=https://$${1} + - traefik.http.middlewares.wwwtohttps.redirectregex.permanent=true + # export traefik dashboard + - traefik.http.services.traefik.loadbalancer.server.port=8080 + - traefik.http.routers.traefik.tls=true + - traefik.http.routers.traefik.tls.certresolver=myresolver - grafana: - image: grafana/grafana - user: root + etcd: + image: 'bitnami/etcd:latest' environment: - GF_INSTALL_PLUGINS: "grafana-clock-panel,grafana-simple-json-datasource" - restart: 'no' + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + # - ETCD_PROXY=on + - ETCD_ENABLE_V2=true + # - ETCD_ + - ETCDCTL_API=3 volumes: - - /tmp/grafana_data:/var/lib/grafana - # networks: - # - bonde - # ports: - # - 3000:3000 - network_mode: "host" + - etcd_data:/bitnami/etcd + ports: + - 2379:2379 + - 2380:2380 + + cms: + image: ${DOCKER_IMAGE:-nossas/cms:main} + restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" + pull_policy: always + depends_on: + - traefik + environment: + - DEBUG=${DEBUG:-True} + - ALLOWED_HOSTS=${ALLOWED_HOSTS} + - CMS_DATABASE_URL=${CMS_DATABASE_URL} + - BONDE_DATABASE_URL=${BONDE_DATABASE_URL} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_STORAGE_BUCKET_NAME=${AWS_STORAGE_BUCKET_NAME} + - RECAPTCHA_PUBLIC_KEY=${RECAPTCHA_PUBLIC_KEY} + - RECAPTCHA_PRIVATE_KEY=${RECAPTCHA_PRIVATE_KEY} + - DISABLE_RECAPTCHA=${DISABLE_RECAPTCHA:-False} + - BONDE_ACTION_API_URL=${BONDE_ACTION_API_URL} + - BONDE_ACTION_SECRET_KEY=${BONDE_ACTION_SECRET_KEY} + - ETCD_HOST=${ETCD_HOST:-"etcd"} + - ETCD_PORT=${ETCD_PORT:-2379} + labels: + - traefik.enable=true + - traefik.http.services.cms.loadbalancer.server.port=8000 + # - traefik.http.routers.cms.priority=10 + # - traefik.http.routers.cms.tls=true + # - traefik.http.routers.cms.tls.certresolver=myresolver + # - traefik.http.routers.cms.rule=${TRAEFIK_ROUTERS_RULE:-HostRegexp(`cms.staging.bonde.org`)} + + webpage: + image: ${DOCKER_IMAGE:-nossas/bonde-clients:v7.6.5} + command: pnpm --filter webpage-client start + restart: "${DOCKER_RESTART_POLICY:-unless-stopped}" + pull_policy: always depends_on: - - prometheus + - traefik + environment: + - PORT=3000 + - NODE_ENV=${PUBLIC_NODE_ENV:-development} + - ACTION_SECRET_KEY=${PUBLIC_ACTION_SECRET_KEY} + - REACT_APP_API_GRAPHQL_SECRET=${PUBLIC_HASURA_SECRET} + - REACT_APP_PAGARME_KEY=${PUBLIC_PAGARME_KEY} + - REACT_APP_DOMAIN_API_ACTIVISTS=${PUBLIC_HASURA_API} + - REACT_APP_DOMAIN_API_GRAPHQL=${PUBLIC_HASURA_API} + - REACT_APP_DOMAIN_API_REST=${PUBLIC_REST_API} + - REACT_APP_DOMAIN_PUBLIC=${DEFAULT_DOMAIN_RULE:-staging.bonde.org} + - REACT_APP_ACTIVE_API_CACHE=${ACTIVE_API_CACHE:-false} + # - REACT_APP_DOMAIN_IMAGINARY=${PUBLIC_IMAGINARY:-http://imaginary.bonde.devel} + healthcheck: + test: "${DOCKER_WEB_HEALTHCHECK_TEST:-wget -qO- localhost:3000/api/ping}" + interval: "60s" + timeout: "3s" + start_period: "50s" + retries: 3 + labels: + - traefik.enable=true + - traefik.http.services.webpage.loadbalancer.server.port=3000 + # - traefik.http.routers.public.priority=-1 + # - traefik.http.routers.public.tls=true + # - traefik.http.routers.public.tls.certresolver=myresolver + # - traefik.http.routers.public.rule=HostRegexp(`{host:.+}`) + # - traefik.http.routers.public.tls.domains[0].main=${DEFAULT_DOMAIN_RULE:-bonde.devel} + # - traefik.http.routers.public.tls.domains[0].sans=*.${DEFAULT_DOMAIN_RULE:-bonde.devel} - redis: - image: redis:6.2-alpine - restart: always - command: redis-server --save 20 1 --loglevel warning --requirepass eYVX7EwVmmxKPCDmwMtyKVge8oLd2t81 - # ports: - # - '6379:6379' - network_mode: "host" - volumes: - - /tmp/redis_data:/data +volumes: + letsencrypt: + driver: local + etcd_data: + driver: local -# networks: -# bonde: -# external: True \ No newline at end of file +networks: + default: + name: bonde + external: true \ No newline at end of file