From cc505e488184e958bcaedad6fed86f91d128e6bd Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 16 Dec 2024 20:24:45 -0500 Subject: [PATCH] Support importing Taskwarrior v2.x data files (#3724) This should ease the pain of upgrading from v2.x to v3.x. --- doc/man/task.1.in | 5 + src/CMakeLists.txt | 1 + src/TF2.cpp | 184 +++++++++++++++++++++++++++++++++++ src/TF2.h | 66 +++++++++++++ src/commands/CMakeLists.txt | 1 + src/commands/CmdCustom.cpp | 3 +- src/commands/CmdImportV2.cpp | 135 +++++++++++++++++++++++++ src/commands/CmdImportV2.h | 46 +++++++++ src/commands/Command.cpp | 3 + test/CMakeLists.txt | 1 + test/import-v2.test.py | 81 +++++++++++++++ 11 files changed, 525 insertions(+), 1 deletion(-) create mode 100644 src/TF2.cpp create mode 100644 src/TF2.h create mode 100644 src/commands/CmdImportV2.cpp create mode 100644 src/commands/CmdImportV2.h create mode 100755 test/import-v2.test.py diff --git a/doc/man/task.1.in b/doc/man/task.1.in index e7e8a71b9..52bbcaab5 100644 --- a/doc/man/task.1.in +++ b/doc/man/task.1.in @@ -414,6 +414,11 @@ few example scripts, such as: import-yaml.pl .fi +.TP +.B task import-v2 +Imports tasks from the Taskwarrior v2.x format. This is used when upgrading from +version 2.x to version 3.x. + .TP .B task log Adds a new task that is already completed, to the task list. It is affected by diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9bb7f2e38..45b027c4a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,6 +14,7 @@ add_library (task STATIC CLI2.cpp CLI2.h Hooks.cpp Hooks.h Lexer.cpp Lexer.h Operation.cpp Operation.h + TF2.cpp TF2.h TDB2.cpp TDB2.h Task.cpp Task.h Variant.cpp Variant.h diff --git a/src/TF2.cpp b/src/TF2.cpp new file mode 100644 index 000000000..9b7e2f940 --- /dev/null +++ b/src/TF2.cpp @@ -0,0 +1,184 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Copyright 2006 - 2021, Tomas Babej, Paul Beckingham, Federico Hernandez. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// https://www.opensource.org/licenses/mit-license.php +// +//////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define STRING_TDB2_REVERTED "Modified task reverted." + +//////////////////////////////////////////////////////////////////////////////// +TF2::TF2() : _loaded_tasks(false), _loaded_lines(false) {} + +//////////////////////////////////////////////////////////////////////////////// +TF2::~TF2() {} + +//////////////////////////////////////////////////////////////////////////////// +void TF2::target(const std::string& f) { _file = File(f); } + +//////////////////////////////////////////////////////////////////////////////// +const std::vector>& TF2::get_tasks() { + if (!_loaded_tasks) load_tasks(); + + return _tasks; +} + +//////////////////////////////////////////////////////////////////////////////// +// Attempt an FF4 parse. +// +// Note that FF1, FF2, FF3, and JSON are no longer supported. +// +// start --> [ --> Att --> ] --> end +// ^ | +// +-------+ +// +std::map TF2::load_task(const std::string& input) { + std::map data; + + // File format version 4, from 2009-5-16 - now, v1.7.1+ + // This is the parse format tried first, because it is most used. + data.clear(); + + if (input[0] == '[') { + // Not using Pig to parse here (which would be idiomatic), because we + // don't need to differentiate betwen utf-8 and normal characters. + // Pig's scanning the string can be expensive. + auto ending_bracket = input.find_last_of(']'); + if (ending_bracket != std::string::npos) { + std::string line = input.substr(1, ending_bracket); + + if (line.length() == 0) throw std::string("Empty record in input."); + + Pig attLine(line); + std::string name; + std::string value; + while (!attLine.eos()) { + if (attLine.getUntilAscii(':', name) && attLine.skip(':') && + attLine.getQuoted('"', value)) { +#ifdef PRODUCT_TASKWARRIOR + legacyAttributeMap(name); +#endif + + data[name] = decode(json::decode(value)); + } + + attLine.skip(' '); + } + + std::string remainder; + attLine.getRemainder(remainder); + if (remainder.length()) throw std::string("Unrecognized characters at end of line."); + } + } else { + throw std::string("Record not recognized as format 4."); + } + + // for compatibility, include all tags in `tags` as `tag_..` attributes + if (data.find("tags") != data.end()) { + for (auto& tag : split(data["tags"], ',')) { + data[Task::tag2Attr(tag)] = "x"; + } + } + + // same for `depends` / `dep_..` + if (data.find("depends") != data.end()) { + for (auto& dep : split(data["depends"], ',')) { + data[Task::dep2Attr(dep)] = "x"; + } + } + + return data; +} + +//////////////////////////////////////////////////////////////////////////////// +// Decode values after parse. +// [ <- &open; +// ] <- &close; +const std::string TF2::decode(const std::string& value) const { + if (value.find('&') == std::string::npos) return value; + + auto modified = str_replace(value, "&open;", "["); + return str_replace(modified, "&close;", "]"); +} + +//////////////////////////////////////////////////////////////////////////////// +void TF2::load_tasks() { + Timer timer; + + if (!_loaded_lines) { + load_lines(); + } + + // Reduce unnecessary allocations/copies. + // Calling it on _tasks is the right thing to do even when from_gc is set. + _tasks.reserve(_lines.size()); + + int line_number = 0; // Used for error message in catch block. + try { + for (auto& line : _lines) { + ++line_number; + auto task = load_task(line); + _tasks.push_back(task); + } + + _loaded_tasks = true; + } + + catch (const std::string& e) { + throw e + format(" in {1} at line {2}", _file._data, line_number); + } + + Context::getContext().time_load_us += timer.total_us(); +} + +//////////////////////////////////////////////////////////////////////////////// +void TF2::load_lines() { + if (_file.open()) { + if (Context::getContext().config.getBoolean("locking")) _file.lock(); + + _file.read(_lines); + _file.close(); + _loaded_lines = true; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// vim: ts=2 et sw=2 diff --git a/src/TF2.h b/src/TF2.h new file mode 100644 index 000000000..fbec1f12b --- /dev/null +++ b/src/TF2.h @@ -0,0 +1,66 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Copyright 2006 - 2024, Tomas Babej, Paul Beckingham, Federico Hernandez. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// https://www.opensource.org/licenses/mit-license.php +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef INCLUDED_TF2 +#define INCLUDED_TF2 + +#include +#include +#include + +#include +#include +#include +#include +#include + +// TF2 Class represents a single 2.x-style file in the task database. +// +// This is only used for importing tasks from 2.x. It only reads format 4, based +// on a stripped-down version of the TF2 class from v2.6.2. +class TF2 { + public: + TF2(); + ~TF2(); + + void target(const std::string&); + + const std::vector>& get_tasks(); + + std::map load_task(const std::string&); + void load_tasks(); + void load_lines(); + const std::string decode(const std::string& value) const; + + bool _loaded_tasks; + bool _loaded_lines; + std::vector> _tasks; + std::vector _lines; + File _file; +}; + +#endif +//////////////////////////////////////////////////////////////////////////////// diff --git a/src/commands/CMakeLists.txt b/src/commands/CMakeLists.txt index e12f5cc0e..720b6cbc9 100644 --- a/src/commands/CMakeLists.txt +++ b/src/commands/CMakeLists.txt @@ -35,6 +35,7 @@ set (commands_SRCS Command.cpp Command.h CmdHistory.cpp CmdHistory.h CmdIDs.cpp CmdIDs.h CmdImport.cpp CmdImport.h + CmdImportV2.cpp CmdImportV2.h CmdInfo.cpp CmdInfo.h CmdLog.cpp CmdLog.h CmdLogo.cpp CmdLogo.h diff --git a/src/commands/CmdCustom.cpp b/src/commands/CmdCustom.cpp index 911ac719d..f2ee70c66 100644 --- a/src/commands/CmdCustom.cpp +++ b/src/commands/CmdCustom.cpp @@ -241,7 +241,8 @@ int CmdCustom::execute(std::string& output) { Color warning = Color(Context::getContext().config.get("color.warning")); std::cerr << warning.colorize(format("Found existing '*.data' files in {1}", location)) << "\n"; std::cerr << " Taskwarrior's storage format changed in 3.0, requiring a manual migration.\n"; - std::cerr << " See https://taskwarrior.org/docs/upgrade-3/\n"; + std::cerr << " See https://taskwarrior.org/docs/upgrade-3/. Run `task import-v2` to import\n"; + std::cerr << " the tasks into the Taskwarrior-3.x format\n"; } feedback_backlog(); diff --git a/src/commands/CmdImportV2.cpp b/src/commands/CmdImportV2.cpp new file mode 100644 index 000000000..e34e597a2 --- /dev/null +++ b/src/commands/CmdImportV2.cpp @@ -0,0 +1,135 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Copyright 2006 - 2021, Tomas Babej, Paul Beckingham, Federico Hernandez. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// https://www.opensource.org/licenses/mit-license.php +// +//////////////////////////////////////////////////////////////////////////////// + +#include +// cmake.h include header must come first + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +//////////////////////////////////////////////////////////////////////////////// +CmdImportV2::CmdImportV2() { + _keyword = "import-v2"; + _usage = "task import-v2"; + _description = "Imports Taskwarrior v2.x files"; + _read_only = false; + _displays_id = false; + _needs_gc = false; + _uses_context = false; + _accepts_filter = false; + _accepts_modifications = false; + _accepts_miscellaneous = true; + _category = Command::Category::migration; +} + +//////////////////////////////////////////////////////////////////////////////// +int CmdImportV2::execute(std::string&) { + std::vector> task_data; + + std::string location = (Context::getContext().data_dir); + File pending_file = File(location + "/pending.data"); + if (pending_file.exists()) { + TF2 pending_tf; + pending_tf.target(pending_file); + auto& pending_tasks = pending_tf.get_tasks(); + task_data.insert(task_data.end(), pending_tasks.begin(), pending_tasks.end()); + } + File completed_file = File(location + "/completed.data"); + if (completed_file.exists()) { + TF2 completed_tf; + completed_tf.target(completed_file); + auto& completed_tasks = completed_tf.get_tasks(); + task_data.insert(task_data.end(), completed_tasks.begin(), completed_tasks.end()); + } + + auto count = import(task_data); + + Context::getContext().footnote( + format("Imported {1} tasks from `*.data` files. You may now delete these files.", count)); + return 0; +} + +//////////////////////////////////////////////////////////////////////////////// +int CmdImportV2::import(const std::vector>& task_data) { + auto count = 0; + const std::string uuid_key = "uuid"; + const std::string id_key = "id"; + const std::string descr_key = "description"; + auto& replica = Context::getContext().tdb2.replica(); + rust::Vec ops; + tc::add_undo_point(ops); + + for (auto& task : task_data) { + auto uuid_iter = task.find(uuid_key); + if (uuid_iter == task.end()) { + std::cout << " err - Task with no UUID\n"; + continue; + } + auto uuid_str = uuid_iter->second; + auto uuid = tc::uuid_from_string(uuid_str); + + bool added_task = false; + auto maybe_task_data = replica->get_task_data(uuid); + auto task_data = maybe_task_data.is_some() ? maybe_task_data.take() : [&]() { + added_task = true; + return tc::create_task(uuid, ops); + }(); + + for (auto& attr : task) { + if (attr.first == uuid_key || attr.first == id_key) { + continue; + } + task_data->update(attr.first, attr.second, ops); + } + count++; + + if (added_task) { + std::cout << " add "; + } else { + std::cout << " mod "; + } + std::cout << uuid_str << ' '; + if (auto descr_iter = task.find(descr_key); descr_iter != task.end()) { + std::cout << descr_iter->second; + } else { + std::cout << "(no description)"; + } + std::cout << "\n"; + } + + replica->commit_operations(std::move(ops)); + return count; +} + +//////////////////////////////////////////////////////////////////////////////// diff --git a/src/commands/CmdImportV2.h b/src/commands/CmdImportV2.h new file mode 100644 index 000000000..049d66d0e --- /dev/null +++ b/src/commands/CmdImportV2.h @@ -0,0 +1,46 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Copyright 2006 - 2021, Tomas Babej, Paul Beckingham, Federico Hernandez. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// https://www.opensource.org/licenses/mit-license.php +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef INCLUDED_CMDIMPORTV2 +#define INCLUDED_CMDIMPORTV2 + +#include +#include + +#include +#include + +class CmdImportV2 : public Command { + public: + CmdImportV2(); + int execute(std::string &); + + private: + int import(const std::vector> &task_data); +}; + +#endif +//////////////////////////////////////////////////////////////////////////////// diff --git a/src/commands/Command.cpp b/src/commands/Command.cpp index 0dc215b01..dc6b36d87 100644 --- a/src/commands/Command.cpp +++ b/src/commands/Command.cpp @@ -66,6 +66,7 @@ #include #include #include +#include #include #include #include @@ -188,6 +189,8 @@ void Command::factory(std::map& all) { all[c->keyword()] = c; c = new CmdImport(); all[c->keyword()] = c; + c = new CmdImportV2(); + all[c->keyword()] = c; c = new CmdInfo(); all[c->keyword()] = c; c = new CmdLog(); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0de39af75..08051bab2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -136,6 +136,7 @@ set (pythonTests hyphenate.test.py ids.test.py import.test.py + import-v2.test.py info.test.py limit.test.py list.all.projects.test.py diff --git a/test/import-v2.test.py b/test/import-v2.test.py new file mode 100755 index 000000000..063feedae --- /dev/null +++ b/test/import-v2.test.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +############################################################################### +# +# Copyright 2006 - 2021, Tomas Babej, Paul Beckingham, Federico Hernandez. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# https://www.opensource.org/licenses/mit-license.php +# +############################################################################### + +import sys +import os +import unittest +import json + +# Ensure python finds the local simpletap module +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from basetest import Task, TestCase +from basetest.utils import mkstemp + + +class TestImport(TestCase): + def setUp(self): + self.t = Task() + self.t.config("dateformat", "m/d/Y") + + # Multiple tasks. + self.pending = """\ +[description:"bing" due:"1734480000" entry:"1734397061" modified:"1734397061" status:"pending" uuid:"ad7f7585-bff3-4b57-a116-abfc9f71ee4a"] +[description:"baz" entry:"1734397063" modified:"1734397063" status:"pending" uuid:"591ccfee-dd8d-44e9-908a-40618257cf54"]\ +""" + self.completed = """\ +[description:"foo" end:"1734397073" entry:"1734397054" modified:"1734397074" status:"deleted" uuid:"6849568f-55d7-4152-8db0-00356e39f0bb"] +[description:"bar" end:"1734397065" entry:"1734397056" modified:"1734397065" status:"completed" uuid:"51921813-7abb-412d-8ada-7c1417d01209"]\ +""" + + def test_import_v2(self): + with open(os.path.join(self.t.datadir, "pending.data"), "w") as f: + f.write(self.pending) + with open(os.path.join(self.t.datadir, "completed.data"), "w") as f: + f.write(self.completed) + code, out, err = self.t("import-v2") + self.assertIn("Imported 4 tasks", err) + + code, out, err = self.t("list") + self.assertIn("bing", out) + self.assertIn("baz", out) + self.assertNotIn("foo", out) + self.assertNotIn("bar", out) + + code, out, err = self.t("completed") + self.assertNotIn("bing", out) + self.assertNotIn("baz", out) + self.assertNotIn("foo", out) # deleted, not in the completed report + self.assertIn("bar", out) + + +if __name__ == "__main__": + from simpletap import TAPTestRunner + + unittest.main(testRunner=TAPTestRunner()) + +# vim: ai sts=4 et sw=4 ft=python