VERY WIP gcp

djmitche · Dec 24, 2023 · 65ea5e8 · 65ea5e8
1 parent 1380d79
commit 65ea5e8
Show file tree

Hide file tree

Showing 26 changed files with 3,294 additions and 193 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -27,19 +27,21 @@ env_logger = "^0.10.0"
 ffizz-header = "0.5"
 flate2 = "1"
 futures = "^0.3.25"
+google-cloud-storage = { version = "0.15.0", default-features = false, features = ["rustls-tls", "auth"] }
 lazy_static = "1"
 libc = "0.2.136"
 log = "^0.4.17"
 pretty_assertions = "1"
 proptest = "^1.4.0"
-ring = "0.16"
+ring = "0.17"
 rstest = "0.17"
 rusqlite = { version = "0.29", features = ["bundled"] }
 serde_json = "^1.0"
 serde = { version = "^1.0.147", features = ["derive"] }
 strum = "0.25"
 strum_macros = "0.25"
 tempfile = "3"
+tokio = { version = "1", features = ["rt-multi-thread"] }
 thiserror = "1.0"
-ureq = "^2.9.0"
+ureq = { version = "^2.8.0", features = ["tls"] }
 uuid = { version = "^1.6.0", features = ["serde", "v4"] }
diff --git a/NOTES.txt b/NOTES.txt
@@ -0,0 +1,18 @@
+# TODO
+
+  - snapshots
+  - cleanup
+    - delete invalid versions by constructing the chain and deleting everything not on it
+    - delete old versions (older than some particular age, older than latest snapshot)
+    - delete old snapshots
+  - update TC docs, manpage
+
+# CAS notes
+
+  S3: DynamoDB? But can it store big blobs? No, 400k max, so just use its locks.
+
+  Azure:
+   - read(name)
+   - if content != old, fail
+   - if not found, write new value unconditionally, start over with compare_and_swap(name, new, new)
+   - write with condition on etag from first step
diff --git a/doc/man/task-sync.5.in b/doc/man/task-sync.5.in
@@ -60,6 +60,10 @@ For synchronization to a server, a better solution is to run
 periodically, such as via
 .BR cron (8) .
 
+.SS Cloud Services
+
+TODO: document sync.gcp etc.
+
 .SS Local Synchronization
 
 In order to take advantage of synchronization's side effect of saving disk

diff --git a/src/Context.cpp b/src/Context.cpp
@@ -283,6 +283,7 @@ std::string configurationDefaults =
   "#sync.server.encryption_secret                 # Encryption secret for sync to a server\n"
   "#sync.server.origin                            # Origin of the sync server\n"
   "#sync.local.server_dir                         # Directory for local sync\n"
+  "#sync.gcp.bucket                               # Bucket for sync to GCP\n"
   "\n"
   "# Aliases - alternate names for commands\n"
   "alias.rm=delete                                # Alias for the delete command\n"

diff --git a/src/commands/CmdShow.cpp b/src/commands/CmdShow.cpp
@@ -193,6 +193,7 @@ int CmdShow::execute (std::string& output)
     " sugar"
     " summary.all.projects"
     " sync.local.server_dir"
+    " sync.gcp.bucket"
     " sync.server.client_id"
     " sync.server.encryption_secret"
     " sync.server.origin"

diff --git a/src/commands/CmdSync.cpp b/src/commands/CmdSync.cpp
@@ -63,17 +63,32 @@ int CmdSync::execute (std::string& output)
 
   // If no server is set up, quit.
   std::string origin = Context::getContext ().config.get ("sync.server.origin");
-  std::string client_id = Context::getContext ().config.get ("sync.server.client_id");
-  std::string encryption_secret = Context::getContext ().config.get ("sync.server.encryption_secret");
   std::string server_dir = Context::getContext ().config.get ("sync.local.server_dir");
+  std::string gcp_bucket = Context::getContext ().config.get ("sync.gcp.bucket");
   if (server_dir != "") {
-    server = tc::Server (server_dir);
+    server = tc::Server::new_local (server_dir);
     server_ident = server_dir;
-  } else if (origin != "" && client_id != "" && encryption_secret != "") {
-    server = tc::Server (origin, client_id, encryption_secret);
-    server_ident = origin;
+  } else if (gcp_bucket != "") {
+    std::string encryption_secret = Context::getContext ().config.get ("sync.gcp.encryption_secret");
+    if (encryption_secret == "") {
+      throw std::string ("sync.gcp.encryption_secret is required");
+    }
+    server = tc::Server::new_gcp (gcp_bucket, encryption_secret);
+    std::ostringstream os;
+    os << "GCP bucket " << gcp_bucket;
+    server_ident = os.str();
+  } else if (origin != "") {
+    std::string client_id = Context::getContext ().config.get ("sync.server.client_id");
+    std::string encryption_secret = Context::getContext ().config.get ("sync.server.encryption_secret");
+    if (client_id == "" || encryption_secret == "") {
+      throw std::string ("sync.server.client_id and encryption_secret are required");
+    }
+    server = tc::Server::new_sync (origin, client_id, encryption_secret);
+    std::ostringstream os;
+    os << "Sync server at " << origin;
+    server_ident = os.str();
   } else {
-    throw std::string ("Neither sync.server nor sync.local are configured.");
+    throw std::string ("No sync.* settings are configured.");
   }
 
   std::stringstream out;

diff --git a/src/tc/Server.cpp b/src/tc/Server.cpp
@@ -32,7 +32,8 @@
 using namespace tc::ffi;
 
 ////////////////////////////////////////////////////////////////////////////////
-tc::Server::Server (const std::string &server_dir)
+tc::Server
+tc::Server::new_local (const std::string &server_dir)
 {
   TCString tc_server_dir = tc_string_borrow (server_dir.c_str ());
   TCString error;
@@ -43,18 +44,17 @@ tc::Server::Server (const std::string &server_dir)
     tc_string_free (&error);
     throw errmsg;
   }
-  inner = unique_tcserver_ptr (
+  return Server (unique_tcserver_ptr (
       tcserver,
-      [](TCServer* rep) { tc_server_free (rep); });
+      [](TCServer* rep) { tc_server_free (rep); }));
 }
 
 ////////////////////////////////////////////////////////////////////////////////
-tc::Server::Server (const std::string &origin, const std::string &client_id, const std::string &encryption_secret)
+tc::Server
+tc::Server::new_sync (const std::string &origin, const std::string &client_id, const std::string &encryption_secret)
 {
   TCString tc_origin = tc_string_borrow (origin.c_str ());
-
   TCString tc_client_id = tc_string_borrow (client_id.c_str ());
-
   TCString tc_encryption_secret = tc_string_borrow (encryption_secret.c_str ());
 
   TCUuid tc_client_uuid;
@@ -65,16 +65,36 @@ tc::Server::Server (const std::string &origin, const std::string &client_id, con
   }
 
   TCString error;
-  auto tcserver = tc_server_new_remote (tc_origin, tc_client_uuid, tc_encryption_secret, &error);
+  auto tcserver = tc_server_new_sync (tc_origin, tc_client_uuid, tc_encryption_secret, &error);
   if (!tcserver) {
     auto errmsg = format ("Could not configure connection to server at {1}: {2}",
         origin, tc_string_content (&error));
     tc_string_free (&error);
     throw errmsg;
   }
-  inner = unique_tcserver_ptr (
+  return Server (unique_tcserver_ptr (
       tcserver,
-      [](TCServer* rep) { tc_server_free (rep); });
+      [](TCServer* rep) { tc_server_free (rep); }));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+tc::Server
+tc::Server::new_gcp (const std::string &bucket, const std::string &encryption_secret)
+{
+  TCString tc_bucket = tc_string_borrow (bucket.c_str ());
+  TCString tc_encryption_secret = tc_string_borrow (encryption_secret.c_str ());
+
+  TCString error;
+  auto tcserver = tc_server_new_gcp (tc_bucket, tc_encryption_secret, &error);
+  if (!tcserver) {
+    auto errmsg = format ("Could not configure connection to GCP bucket {1}: {2}",
+        bucket, tc_string_content (&error));
+    tc_string_free (&error);
+    throw errmsg;
+  }
+  return Server (unique_tcserver_ptr (
+      tcserver,
+      [](TCServer* rep) { tc_server_free (rep); }));
 }
 
 ////////////////////////////////////////////////////////////////////////////////

diff --git a/src/tc/Server.h b/src/tc/Server.h
@@ -43,18 +43,21 @@ namespace tc {
 
   // Server wraps the TCServer type, managing its memory, errors, and so on.
   //
-  // Except as noted, method names match the suffix to `tc_replica_..`.
+  // Except as noted, method names match the suffix to `tc_server_..`.
   class Server
   {
   public:
     // Construct a null server
     Server () = default;
 
     // Construct a local server (tc_server_new_local).
-    Server (const std::string& server_dir);
+    static Server new_local (const std::string& server_dir);
 
-    // Construct a remote server (tc_server_new_remote).
-    Server (const std::string &origin, const std::string &client_id, const std::string &encryption_secret);
+    // Construct a remote server (tc_server_new_sync).
+    static Server new_sync (const std::string &origin, const std::string &client_id, const std::string &encryption_secret);
+
+    // Construct a GCP server (tc_server_new_gcp).
+    static Server new_gcp (const std::string &bucket, const std::string &encryption_secret);
 
     // This object "owns" inner, so copy is not allowed.
     Server (const Server &) = delete;
@@ -65,6 +68,8 @@ namespace tc {
     Server &operator=(Server &&) noexcept;
 
   protected:
+    Server (unique_tcserver_ptr inner) : inner(std::move(inner)) {};
+
     unique_tcserver_ptr inner;
 
     // Replica accesses the inner pointer to call tc_replica_sync