removing trailling space from output

singnet · Nov 13, 2024 · 67ef49a · 67ef49a
1 parent 191060a
commit 67ef49a
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 9 deletions.
diff --git a/extra/cpp/tokenizers/src/metta_tokenizer.cc b/extra/cpp/tokenizers/src/metta_tokenizer.cc
@@ -20,7 +20,7 @@ using namespace std;
  */
 pair<size_t, string> _tokenize(const string& expression, size_t cursor = 0) {
     string output;
-    string header = "LINK Expression ";
+    string header = "LINK Expression";
     int target_count = 0;
     string token;
     char ch;
@@ -32,13 +32,13 @@ pair<size_t, string> _tokenize(const string& expression, size_t cursor = 0) {
         if (ch == '(') {
             if (cursor > start) {
                 tie(cursor, token) = _tokenize(expression, cursor);
-                output += token;
+                output += " " + token;
                 target_count++;
             }
             continue;
 
         } else if (ch == ')') {
-            return make_pair(cursor, header + to_string(target_count) + " " + output);
+            return make_pair(cursor, header + " " + to_string(target_count) + output);
 
         } else if (isspace(ch)) {
             continue;
@@ -56,11 +56,11 @@ pair<size_t, string> _tokenize(const string& expression, size_t cursor = 0) {
             --cursor;
 
             if (token[0] == '$') {
-                header = "LINK_TEMPLATE Expression ";
-                output += "VARIABLE " + token.substr(1) + " ";
+                header = "LINK_TEMPLATE Expression";
+                output += " VARIABLE " + token.substr(1);
                 target_count++;
             } else {
-                output += "NODE Symbol " + token + " ";
+                output += " NODE Symbol " + token;
                 target_count++;
             }
         }

diff --git a/extra/cpp/tokenizers/tests/test_metta_tokenizer.cc b/extra/cpp/tokenizers/tests/test_metta_tokenizer.cc
@@ -6,19 +6,16 @@ TEST(MettaTokenizerTest, BasicAssertions) {
     string expected = "LINK_TEMPLATE Expression 3 NODE Symbol Similarity LINK Expression 2 NODE Symbol Concept NODE Symbol \"human\" VARIABLE v1";
     string expression = "(Similarity (Concept \"human\") $v1)";
     string actual = tokenize(expression);
-    actual.erase(actual.find_last_not_of(' ') + 1);  // trim trailing spaces
     EXPECT_EQ(actual, expected);
 
     expected = "LINK_TEMPLATE Expression 4 NODE Symbol Similarity VARIABLE v0 LINK Expression 2 NODE Symbol Concept NODE Symbol \"human\" VARIABLE v1";
     expression = "(Similarity $v0 (Concept \"human\") $v1)";
     actual = tokenize(expression);
-    actual.erase(actual.find_last_not_of(' ') + 1);  // trim trailing spaces
     EXPECT_EQ(actual, expected);
 
     expected = "LINK_TEMPLATE Expression 3 NODE Symbol Similarity LINK_TEMPLATE Expression 2 NODE Symbol Concept VARIABLE v0 VARIABLE v1";
     expression = "(Similarity (Concept $v0) $v1)";
     actual = tokenize(expression);
-    actual.erase(actual.find_last_not_of(' ') + 1);  // trim trailing spaces
     EXPECT_EQ(actual, expected);
 }