Skip to content

Commit

Permalink
Improved backup and restore usage of disk space
Browse files Browse the repository at this point in the history
  • Loading branch information
@bautisflow
bautisflow committed Oct 24, 2023
1 parent d7b499e commit 1ce0b32
Show file tree
Hide file tree
Showing 12 changed files with 919 additions and 544 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/static-analysis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
sudo apt update
sudo apt install -y libarchive-tools libarchive-dev libboost-all-dev
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis

Expand All @@ -45,7 +45,7 @@ jobs:
fi
- name: Install sonar-scanner and build-wrapper
uses: sonarsource/sonarcloud-github-c-cpp@v1
uses: sonarsource/sonarcloud-github-c-cpp@v2

- name: Run build-wrapper
run: |
Expand All @@ -69,4 +69,4 @@ jobs:
sonar-scanner \
--define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" \
--define sonar.projectVersion="${{ env.version }}"
fi
fi
8 changes: 8 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
# Changelog
All notable changes to this project will be documented in this file.

## [1.1.0] - 2023-10-04
### Changed
- Mongodb backup is compressed in gzip by default
- Influxdb and mongodb backups are created in default mounted directory of container

### Removed
- Backup of influxdb v1 is no longer needed

## [1.0.0] - 2023-08-22
### Changed
- Set thinger-iotmp dependency as production version 1.1.0
Expand Down
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ FetchContent_Declare(
GIT_REPOSITORY https://github.com/gabime/spdlog.git
GIT_TAG v1.11.0
)
add_compile_definitions(SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_TRACE)
FetchContent_MakeAvailable(spdlog)

# json
Expand Down
2 changes: 1 addition & 1 deletion install/install_thinger_monitor.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ case `uname -m` in
esac

usage() {
echo "usage: install_$_monitor.sh [-h] [-t TOKEN] [-s SERVER] [-k] [-u]"
echo "usage: install_$_module.sh [-h] [-t TOKEN] [-s SERVER] [-k] [-u]"
echo
echo "Installs and runs $_module binary"
echo
Expand Down
8 changes: 7 additions & 1 deletion src/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,11 +154,17 @@ int main(int argc, char *argv[]) {

thinger::monitor::Client monitor(client, config);


// Retrieve properties on connection and any update
for (auto const &property: config.remote_properties) {
client.property_stream(property.c_str(), true) = [&config, &property, &monitor](iotmp::input& in) {
LOG_INFO(fmt::format("Received property {0} update value", property));
config.update(property, in["value"]);

if ( in["value"].is_empty() ) {
config.update(property, in);
} else {
config.update(property, in["value"]);
}
monitor.reload_configuration(property);
};
}
Expand Down
164 changes: 90 additions & 74 deletions src/thinger/client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,11 @@ namespace thinger::monitor {
config_(config)
{

/*if ( ! config_.get_backup().empty() ) {
resources_.insert( {"backup", client["backup"]});
resources_.insert( {"restore", client["restore"]});
}*/

// Executed only once
system::retrieve_hostname(hostname);
system::retrieve_os_version(os_version);
Expand Down Expand Up @@ -126,112 +131,122 @@ namespace thinger::monitor {
};
}

resources_.at("backup") = [this, &client](iotmp::input& in, iotmp::output& out) {
//if ( ! config_.get_backup().empty() ) {
resources_.at("backup") = [this, &client](iotmp::input &in, iotmp::output &out) {

if (!config_.get_backup().empty()) {

std::string tag = in["tag"];
std::string endpoint = in["endpoint"];
std::string tag = in["tag"];
std::string endpoint = in["endpoint"];

auto today = Date();
in["tag"] = today.to_iso8601();
in["endpoint"] = "backup_finished";
auto today = Date();
in["tag"] = today.to_iso8601();
in["endpoint"] = "backup_finished";

if ( f1.future.valid() && f1.future.wait_for(std::chrono::seconds(0)) != std::future_status::ready ) {
if ( f1.task == "backup" )
out["status"] = "Already executing";
else
out["status"] = ("Executing: "+f1.task).c_str();
return;
}
if (f1.future.valid() && f1.future.wait_for(std::chrono::seconds(0)) != std::future_status::ready) {
if (f1.task == "backup")
out["status"] = "Already executing";
else
out["status"] = ("Executing: " + f1.task).c_str();
return;
}

// future from a packaged_task
std::packaged_task<void(std::string,std::string)> task([this, &client](const std::string& task_tag, const std::string& task_endpoint) {
// future from a packaged_task
std::packaged_task<void(std::string, std::string)> task(
[this, &client](const std::string &task_tag, const std::string &task_endpoint) {

std::unique_ptr<ThingerMonitorBackup> backup{}; // as nullptr
// Add new possible options for backup systems
if (config_.get_backup() == "platform") {
backup = std::make_unique<PlatformBackup>(config_, hostname, task_tag);
backup = std::make_unique<PlatformBackup>(config_, hostname, task_tag);
}

json data;
data["device"] = config_.get_id();
data["hostname"] = hostname;
data["backup"] = {};
data["device"] = config_.get_id();
data["hostname"] = hostname;
data["backup"] = {};
data["backup"]["operations"] = {};

LOG_INFO("[_BACKUP] Creating backup");
data["backup"]["operations"]["backup"] = backup->backup();
LOG_INFO("[_BACKUP] Uploading backup");
data["backup"]["operations"]["upload"] = backup->upload();
LOG_INFO("[_BACKUP] Cleaning backup temporary files");
data["backup"]["operations"]["clean"] = backup->clean();

// Clean if upload succeded
if ( data["backup"]["operations"]["upload"].get<bool>() ) {
LOG_INFO("[_BACKUP] Cleaning backup temporary files");
data["backup"]["operations"]["clean"] = backup->clean();
} else {
LOG_WARNING("[_BACKUP] Left local copy of backup due to failed upload");
}

data["backup"]["status"] = true;
for (auto& element : data["backup"]["operations"]) {
if (!element["status"].get<bool>()) {
data["backup"]["status"] = false;
break;
}
for (auto &element: data["backup"]["operations"]) {
if (!element["status"].get<bool>()) {
data["backup"]["status"] = false;
break;
}
}

//LOG_LEVEL(1, "[_BACKUP] Backup status: {0}", data.dump());
LOG_LEVEL(1, "[_BACKUP] Backup status: %s", data.dump());
if (!task_endpoint.empty()) {
protoson::pson payload;
protoson::json_decoder::parse(data, payload);
client.call_endpoint(task_endpoint.c_str(), payload);
protoson::pson payload;
protoson::json_decoder::parse(data, payload);
client.call_endpoint(task_endpoint.c_str(), payload);
}

});
});

if (!tag.empty()) {
out["status"] = "Launched";
f1.task = "backup";
f1.future = task.get_future(); // get a future
std::thread thread(std::move(task), tag, endpoint);
thread.detach();
}
if (!tag.empty()) {
out["status"] = "Launched";
f1.task = "backup";
f1.future = task.get_future(); // get a future
std::thread thread(std::move(task), tag, endpoint);
thread.detach();
}

out["status"] = "Ready to be launched";
out["status"] = "Ready to be launched";

} else {
out["status"] = "ERROR";
out["error"] = "Can't launch backup. Set backups property.";
out["status"] = "ERROR";
out["error"] = "Can't launch backup. Set backups property.";
}
};
};

resources_.at("restore") = [this, &client](iotmp::input& in, iotmp::output& out) {
resources_.at("restore") = [this, &client](iotmp::input &in, iotmp::output &out) {

if (!config_.get_backup().empty()) {

std::string tag = in["tag"];
std::string endpoint = in["endpoint"];
std::string tag = in["tag"];
std::string endpoint = in["endpoint"];

auto today = Date();
in["tag"] = today.to_iso8601();
in["endpoint"] = "restore_finished";
auto today = Date();
in["tag"] = today.to_iso8601();
in["endpoint"] = "restore_finished";

if ( f1.future.valid() && f1.future.wait_for(std::chrono::seconds(0)) != std::future_status::ready ) {
if ( f1.task == "restore" )
out["status"] = "Already executing";
else
out["status"] = ("Executing: "+f1.task).c_str();
return;
}
if (f1.future.valid() && f1.future.wait_for(std::chrono::seconds(0)) != std::future_status::ready) {
if (f1.task == "restore")
out["status"] = "Already executing";
else
out["status"] = ("Executing: " + f1.task).c_str();
return;
}

std::packaged_task<void(std::string, std::string)> task([this, &client](const std::string& task_tag, const std::string& task_endpoint) {
// TODO: Catch exception in thread
std::packaged_task<void(std::string, std::string)> task(
[this, &client](const std::string &task_tag, const std::string &task_endpoint) {

std::unique_ptr<ThingerMonitorRestore> restore{};
// Add new possible options for backup systems
if (config_.get_backup() == "platform") {
restore = std::make_unique<PlatformRestore>(config_, hostname, task_tag);
restore = std::make_unique<PlatformRestore>(config_, hostname, task_tag);
}

json data;
data["device"] = config_.get_id();
data["hostname"] = hostname;
data["restore"] = {};
data["device"] = config_.get_id();
data["hostname"] = hostname;
data["restore"] = {};
data["restore"]["operations"] = {};

LOG_INFO("[___RSTR] Downloading backup");
Expand All @@ -243,28 +258,29 @@ namespace thinger::monitor {
//LOG_LEVEL(1, "[___RSTR] Restore status: {0}", data.dump());
LOG_LEVEL(1, "[___RSTR] Restore status: %s", data.dump());
if (!task_endpoint.empty()) {
protoson::pson payload;
protoson::json_decoder::parse(data, payload);
client.call_endpoint(task_endpoint.c_str(), payload);
protoson::pson payload;
protoson::json_decoder::parse(data, payload);
client.call_endpoint(task_endpoint.c_str(), payload);
}

});
});

if (!tag.empty()) {
out["status"] = "Launched";
f1.task = "restore";
f1.future = task.get_future();
std::thread thread(std::move(task), tag, endpoint);
thread.detach();
}
if (!tag.empty()) {
out["status"] = "Launched";
f1.task = "restore";
f1.future = task.get_future();
std::thread thread(std::move(task), tag, endpoint);
thread.detach();
}

out["status"] = "Ready to be launched";
out["status"] = "Ready to be launched";

} else {
out["status"] = "ERROR";
out["error"] = "Can't launch restore. Set backups property.";
out["status"] = "ERROR";
out["error"] = "Can't launch restore. Set backups property.";
}
};
};
//}

resources_.at("monitor") = [this](iotmp::output& out) {

Expand Down
Loading

0 comments on commit 1ce0b32

Please sign in to comment.