From 40ee146a791fef9de9a4191e448313f8d4cc3f14 Mon Sep 17 00:00:00 2001 From: James Greenhill Date: Tue, 15 Aug 2023 16:55:59 -0700 Subject: [PATCH] add backup page and api --- .env.example | 3 + docker-compose.dev.yml | 3 + docker-compose.yml | 4 + frontend/src/pages/Backups/Backups.tsx | 96 +++++++++---------- housewatch/api/backups.py | 35 +++++++ housewatch/clickhouse/backups.py | 65 +++++++++++++ housewatch/settings/__init__.py | 6 ++ .../tests/test_backup_table_fixture.sql | 9 ++ housewatch/urls.py | 2 + 9 files changed, 174 insertions(+), 49 deletions(-) create mode 100644 .env.example create mode 100644 housewatch/api/backups.py create mode 100644 housewatch/clickhouse/backups.py create mode 100644 housewatch/tests/test_backup_table_fixture.sql diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5d09e68 --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY_ID +AWS_SECRET_ACCESS_KEY=YOUR_AWS_SECRET_ACCESS_KEY +AWS_DEFAULT_REGION=YOUR_AWS_DEFAULT_REGION \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 0426dc9..c78dd4d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -16,6 +16,9 @@ services: CLICKHOUSE_SECURE: false CLICKHOUSE_VERIFY: false CLICKHOUSE_CA: "" + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY + AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION command: - bash - -c diff --git a/docker-compose.yml b/docker-compose.yml index e7a95fa..9065fce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,10 @@ services: CLICKHOUSE_SECURE: $CLICKHOUSE_SECURE CLICKHOUSE_VERIFY: $CLICKHOUSE_VERIFY CLICKHOUSE_CA: $CLICKHOUSE_CA + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY + AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION + command: - bash - -c diff --git a/frontend/src/pages/Backups/Backups.tsx b/frontend/src/pages/Backups/Backups.tsx index 2a5cd3f..4575555 100644 --- a/frontend/src/pages/Backups/Backups.tsx +++ b/frontend/src/pages/Backups/Backups.tsx @@ -1,44 +1,40 @@ import React, { useEffect, useState } from 'react' import { ColumnType } from 'antd/es/table' -import { Table, Col, Row, Tooltip, notification } from 'antd' +import { Table, Button, Col, Row, Tooltip, notification } from 'antd' -interface ClusterNode { - cluster: string - shard_num: number - shard_weight: number - replica_num: number - host_name: string - host_address: string - port: number - is_local: boolean - user: string - default_database: string - errors_count: number - slowdowns_count: number - estimated_recovery_time: number +interface BackupRow { + id: string + name: string + status: string + error: string + start_time: string + end_time: string + num_files: number + total_size: number + num_entries: number + uncompressed_size: number + compressed_size: number + files_read: number + bytes_read: number } -interface Cluster { - cluster: string - nodes: ClusterNode[] -} - -interface Clusters { - clusters: Cluster[] +interface Backups { + backups: BackupRow[] } export default function Backups() { - const [clusters, setClusters] = useState({ - clusters: [], + const [backups, setBackups] = useState({ + backups: [], }) - const [loadingClusters, setLoadingClusters] = useState(false) + const [loadingBackups, setLoadingBackups] = useState(false) const loadData = async () => { try { - const res = await fetch('/api/clusters') + const res = await fetch('/api/backups') const resJson = await res.json() - const clusters = { clusters: resJson } - setClusters(clusters) + const backups = { backups: resJson } + console.log(backups) + setBackups(backups) } catch (err) { notification.error({ message: 'Failed to load data' }) } @@ -48,34 +44,36 @@ export default function Backups() { loadData() }, []) - const columns: ColumnType[] = [ - { title: 'Cluster', dataIndex: 'cluster' }, - { title: 'Shard Number', dataIndex: 'shard_num' }, - { title: 'Shard Weight', dataIndex: 'shard_weight' }, - { title: 'Replica Number', dataIndex: 'replica_num' }, - { title: 'Host Name', dataIndex: 'host_name' }, - { title: 'Host Address', dataIndex: 'host_address' }, - { title: 'Port', dataIndex: 'port' }, - { title: 'Is Local', dataIndex: 'is_local' }, - { title: 'User', dataIndex: 'user' }, - { title: 'Default Database', dataIndex: 'default_database' }, - { title: 'Errors Count', dataIndex: 'errors_count' }, - { title: 'Slowdowns Count', dataIndex: 'slowdowns_count' }, - { title: 'Recovery Time', dataIndex: 'estimated_recovery_time' }, + const columns: ColumnType[] = [ + { title: 'UUID', dataIndex: 'id' }, + { title: 'Name', dataIndex: 'name' }, + { title: 'Status', dataIndex: 'status' }, + { title: 'Error', dataIndex: 'error' }, + { title: 'Start', dataIndex: 'start_time' }, + { title: 'End', dataIndex: 'end_time' }, + { title: 'Size', dataIndex: 'total_size' }, + { title: 'Entries', dataIndex: 'num_entries' }, + { title: 'Uncompressed Size', dataIndex: 'uncompressed_size' }, + { title: 'Compressed Size', dataIndex: 'compressed_size' }, + { title: 'Files Read', dataIndex: 'files_read' }, + { title: 'Bytes Read', dataIndex: 'bytes_read' }, ] return (
-

Clusters

+

Backups

+
+
    - {clusters.clusters.map((cluster) => ( - <> -

    {cluster.cluster}

    - - - ))} +

    diff --git a/housewatch/api/backups.py b/housewatch/api/backups.py new file mode 100644 index 0000000..b887a79 --- /dev/null +++ b/housewatch/api/backups.py @@ -0,0 +1,35 @@ +import structlog +from rest_framework.decorators import action +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.viewsets import GenericViewSet +from housewatch.clickhouse import backups + + +logger = structlog.get_logger(__name__) + + +class BackupViewset(GenericViewSet): + def list(self, request: Request) -> Response: + cluster = request.query_params.get("cluster") + return Response(backups.get_backups(cluster=cluster)) + + def retrieve(self, request: Request, pk: str) -> Response: + cluster = request.query_params.get("cluster") + return Response(backups.get_backup(pk, cluster=cluster)) + + @action(detail=True, methods=["post"]) + def restore(self, request: Request, pk: str) -> Response: + backups.restore_backup(pk) + return Response() + + def create(self, request: Request) -> Response: + database = request.data.get("database") + table = request.data.get("table") + bucket = request.data.get("bucket") + path = request.data.get("path") + if table: + res = backups.create_table_backup(database, table, bucket, path) + else: + res = backups.create_database_backup(database, bucket, path) + return Response(res) diff --git a/housewatch/clickhouse/backups.py b/housewatch/clickhouse/backups.py new file mode 100644 index 0000000..9c3b2ba --- /dev/null +++ b/housewatch/clickhouse/backups.py @@ -0,0 +1,65 @@ +from collections import defaultdict +from housewatch.clickhouse.client import run_query + +from django.conf import settings + + +def get_backups(cluster=None): + if cluster: + QUERY = """SELECT * FROM clusterAllReplicas(%(cluster)s, system.backups)""" + else: + QUERY = """SELECT * FROM system.backups""" + res = run_query(QUERY, {"cluster": cluster}) + return res + + +def get_backup(backup, cluster=None): + if cluster: + QUERY = """Select * FROM clusterAllReplicas(%(cluster)s, system.backups) WHERE id = '%(uuid)s' """ + return run_query(QUERY, {"cluster": cluster, "uuid": backup}) + else: + QUERY = """Select * FROM system.backups WHERE id = '%(uuid)s' """ + return run_query(QUERY, {"uuid": backup}) + + +def create_table_backup(database, table, bucket, path, aws_key=None, aws_secret=None): + if aws_key is None or aws_secret is None: + aws_key = settings.AWS_ACCESS_KEY_ID + aws_secret = settings.AWS_SECRET_ACCESS_KEY + QUERY = """BACKUP TABLE %(database)s.%(table)s + TO S3('https://%(bucket)s.s3.amazonaws.com/%(path)s', '%(aws_key)s', '%(aws_secret)s') + ASYNC""" + return run_query( + QUERY, + { + "database": database, + "table": table, + "bucket": bucket, + "path": path, + "aws_key": aws_key, + "aws_secret": aws_secret, + }, + ) + + +def create_database_backup(database, bucket, path, aws_key=None, aws_secret=None): + if aws_key is None or aws_secret is None: + aws_key = settings.AWS_ACCESS_KEY_ID + aws_secret = settings.AWS_SECRET_ACCESS_KEY + QUERY = """BACKUP DATABASE %(database)s + TO S3('https://%(bucket)s.s3.amazonaws.com/%(path)s', '%(aws_key)s', '%(aws_secret)s') + ASYNC""" + return run_query( + QUERY, + { + "database": database, + "bucket": bucket, + "path": path, + "aws_key": aws_key, + "aws_secret": aws_secret, + }, + ) + + +def restore_backup(backup): + pass diff --git a/housewatch/settings/__init__.py b/housewatch/settings/__init__.py index 696ec42..e2e3226 100644 --- a/housewatch/settings/__init__.py +++ b/housewatch/settings/__init__.py @@ -253,3 +253,9 @@ def get_from_env(key: str, default: Any = None, *, optional: bool = False, type_ CLICKHOUSE_DATABASE = get_from_env("CLICKHOUSE_DATABASE", "defaul") CLICKHOUSE_USER = get_from_env("CLICKHOUSE_USER", "default") CLICKHOUSE_PASSWORD = get_from_env("CLICKHOUSE_PASSWORD", "") + + +# AWS settings for Backups +AWS_ACCESS_KEY_ID = get_from_env("AWS_ACCESS_KEY_ID", "") +AWS_SECRET_ACCESS_KEY = get_from_env("AWS_SECRET_ACCESS_KEY", "") +AWS_DEFAULT_REGION = get_from_env("AWS_DEFAULT_REGION", "us-east-1") diff --git a/housewatch/tests/test_backup_table_fixture.sql b/housewatch/tests/test_backup_table_fixture.sql new file mode 100644 index 0000000..fb00df8 --- /dev/null +++ b/housewatch/tests/test_backup_table_fixture.sql @@ -0,0 +1,9 @@ +CREATE TABLE test_backup ( + id UUID DEFAULT generateUUIDv4(), + name String, + timestamp DateTime DEFAULT now() +) ENGINE = MergeTree() +ORDER BY id; +INSERT INTO test_backup (name) +SELECT substring(toString(rand() * 1000000000), 1, 5) AS random_string +FROM numbers(100); \ No newline at end of file diff --git a/housewatch/urls.py b/housewatch/urls.py index 4269fb9..4e066b5 100644 --- a/housewatch/urls.py +++ b/housewatch/urls.py @@ -4,6 +4,7 @@ from rest_framework_extensions.routers import ExtendedDefaultRouter from housewatch.api.instance import InstanceViewset from housewatch.api.cluster import ClusterViewset +from housewatch.api.backups import BackupViewset from housewatch.api.analyze import AnalyzeViewset from housewatch.api.async_migration import AsyncMigrationsViewset from housewatch.views import healthz @@ -21,6 +22,7 @@ def __init__(self, *args, **kwargs): router = DefaultRouterPlusPlus() router.register(r"api/instance", InstanceViewset, basename="instance") router.register(r"api/clusters", ClusterViewset, basename="cluster") +router.register(r"api/backups", BackupViewset, basename="backup") router.register(r"api/analyze", AnalyzeViewset, basename="analyze") router.register(r"api/async_migrations", AsyncMigrationsViewset, basename="async_migrations") router.register(r"api/saved_queries", SavedQueryViewset, basename="saved_queries")