Fixes to make line breaks in python statements compatible with the Ca…

…dabra display shortcut ';'.

client_server/Server.cc

@@ -220,7 +220,6 @@ std::string Server::run_string(const std::string& blk, bool handle_output)
 
 	// Preparse input block.
 	// std::cerr << "RAW:\n" << blk << std::endl;
-
 	auto newblk = cadabra::cdb2python_string(blk, true);
 
 	// std::cerr << "PREPARSED:\n" << newblk << std::endl;

core/CMakeLists.txt

@@ -270,13 +270,15 @@ set(IMAGES
 
 include_directories(
    "."
+   "${CADABRA_LIBS_DIR}/pybind11/include"
    "${CADABRA_LIBS_DIR}/internal/include"
    "${CADABRA_LIBS_DIR}/whereami"
    "${CADABRA_LIBS_DIR}/base64"   
    "${CADABRA_LIBS_DIR}/dbg"   
    "${CADABRA_LIBS_DIR}/linenoise"
    "${CADABRA_LIBS_DIR}/nlohmann"     
    ${Boost_INCLUDE_DIRS}
+   ${Python_INCLUDE_DIRS}
    )
 
 

core/CdbPython.cc

@@ -6,6 +6,8 @@
 #include <iomanip>
 #include <iostream>
 #include "CdbPython.hh"
+#include <pybind11/pybind11.h>
+#include <pybind11/embed.h>
 
 #ifndef CDBPYTHON_NO_NOTEBOOK
 #include "DataCell.hh"
@@ -57,18 +59,108 @@ std::string cadabra::cdb2python_string(const std::string& blk, bool display)
 	{
 	std::stringstream str(blk);
 	std::string line;
-	std::string newblk;
+	std::string tmpblk;
+	std::vector<std::string> newblks;
 	ConvertData cv;
-//	std::string lhs, rhs, op, indent;
+
+	std::pair<std::string, std::string> res;
+
+	// We collect lines into blocks until they pass python
+	// validation. If we ever hit an indentation error, we add
+	// previous successful blocks and re-try, successively,
+	// until the block compiles again.
+
 	while(std::getline(str, line, '\n')) {
-		std::string res=cadabra::convert_line(line, cv, display); // lhs, rhs, op, indent, display);
-		// std::cerr << "preparsed : " + res << std::endl;
-		if(res!="::empty")
-			newblk += res+"\n";
+		res = cadabra::convert_line(line, cv, display);
+
+//		if(res.second!="::empty")
+//			tmpblk += res.first + res.second + "\n";
+
+		if(res.second!="::empty")
+			tmpblk += res.second;
+
+		while(true) {
+			// std::cerr << "CHECK:---\n" << tmpblk << "\n---" << std::endl;
+			int ic = is_python_code_complete(tmpblk);
+			if(ic==1) {
+				newblks.push_back(res.first + tmpblk + "\n");
+				tmpblk = "";
+				break;
+				}
+			if(ic==0) {
+				tmpblk += "\n";
+				break;
+				}
+			if(ic==-1) {
+				if(newblks.size()==0)
+					break;
+				// Grow block by adding previously ok block,
+				// then try compiling again.
+				tmpblk = newblks.back() + tmpblk;
+				newblks.pop_back();
+				}
+			}
 		}
+
+	// Collect.
+	std::string newblk;
+	for(const auto& blk: newblks)
+		newblk += blk;
+
+	// Add anything still left and cross fingers.
+	newblk += tmpblk+"\n";
+
 	return newblk;
 	}
 
+//  1: complete
+//  0: incomplete
+// -1: indentation error, need backtracking
+
+int cadabra::is_python_code_complete(const std::string& code)
+	{
+	// The following code prints None, <code object> and <code object>.
+	// Make sure that the string you feed here does *not* include the
+	// newline on the last line.
+	//
+	// 
+   //  from codeop import *
+   //  
+   //  str1='''def fun():
+   //     print("hello")'''
+   //  str2='''def fun():
+   //     print("hello")
+   //  '''
+   //  str3='''print("hello")'''
+   //  
+   //  print(compile_command(str1, "<string>", "single"))
+   //  print(compile_command(str2, "<string>", "single"))
+   //  print(compile_command(str3, "<string>", "single"))
+
+   //	pybind11::scoped_interpreter guard{}; // Ensures the Python interpreter is initialized.
+
+	// Import the 'codeop' module.
+	pybind11::object codeop = pybind11::module_::import("codeop");
+	pybind11::object compile_command = codeop.attr("compile_command");
+
+	// std::cerr << "CHECK:\n" << code << "END" << std::endl;
+	try {
+		pybind11::object result = compile_command(code, "<string>", "single");
+		if( result.is_none() ) return 0;
+		else                   return 1;
+		}
+	catch (pybind11::error_already_set& e) {
+		// std::cerr << "EXCEPTION: " << e.what() << std::endl;
+		if (std::string(e.what()).find("unexpected EOF") != std::string::npos) {
+			return -1;
+			}
+		if (std::string(e.what()).find("Indentation") != std::string::npos) {
+			return -1;
+			}
+		throw;
+		}
+	}
+
 cadabra::ConvertData::ConvertData()
 	{
 	}
@@ -79,9 +171,9 @@ cadabra::ConvertData::ConvertData(const std::string& lhs_, const std::string& rh
 	{
 	}
 
-std::string cadabra::convert_line(const std::string& line, ConvertData& cv, bool display) // std::string& lhs, std::string& rhs, std::string& op, std::string& indent, bool display)
+std::pair<std::string, std::string> cadabra::convert_line(const std::string& line, ConvertData& cv, bool display)
 	{
-	std::string ret;
+	std::string ret, prefix;
 
 	auto& lhs    = cv.lhs;
 	auto& rhs    = cv.rhs;
@@ -104,19 +196,19 @@ std::string cadabra::convert_line(const std::string& line, ConvertData& cv, bool
 		}
 
 	if(line_stripped.size()==0) {
-		return "";
+		return std::make_pair(prefix, "");
 		}
 
 	// Do not do anything with comment lines.
-	if(line_stripped[0]=='#') return line;
+	if(line_stripped[0]=='#') return std::make_pair(prefix, line);
 
 	// Bare ';' gets replaced with 'display(_)' but *only* if we have no
 	// preceding lines which have not finished parsing.
 	if(line_stripped==";" && lhs=="") {
 		if(display)
-			return indent_line+"display(_)";
+			return std::make_pair(prefix, indent_line+"display(_)");
 		else
-			return indent_line;
+			return std::make_pair(prefix, indent_line);
 		}
 
 	// 'lastchar' is either a Cadabra termination character, or empty.
@@ -139,15 +231,15 @@ std::string cadabra::convert_line(const std::string& line, ConvertData& cv, bool
 			lhs="";
 			rhs="";
 			op="";
-			return ret;
+			return std::make_pair(prefix, ret);
 			}
 		}
 	else {
 		// If we are a Cadabra continuation, add to the rhs without further processing
 		// and return an empty line immediately.
 		if(lhs!="") {
 			rhs += line_stripped+" ";
-			return "::empty";
+			return std::make_pair(prefix, "::empty");
 			}
 		}
 
@@ -165,7 +257,7 @@ std::string cadabra::convert_line(const std::string& line, ConvertData& cv, bool
 	if(std::regex_match(line_stripped, converge_res, converge_match)) {
 		ret = indent_line+std::string(converge_res[1])+std::string(converge_res[2])+".reset(); _="+std::string(converge_res[2])+"\n"
 		      + indent_line+std::string(converge_res[1])+"while "+std::string(converge_res[2])+".changed():";
-		return ret;
+		return std::make_pair(prefix, ret);
 		}
 
 	size_t found = line_stripped.find(":=");
@@ -176,7 +268,7 @@ std::string cadabra::convert_line(const std::string& line, ConvertData& cv, bool
 			lhs=line_stripped.substr(0,found);
 			rhs=line_stripped.substr(found+2);
 			op=":=";
-			return "::empty";
+			return std::make_pair(prefix, "::empty");
 			}
 		else {
 			line_stripped=line_stripped.substr(0,line_stripped.size()-1);
@@ -218,13 +310,16 @@ std::string cadabra::convert_line(const std::string& line, ConvertData& cv, bool
 				}
 			}
 		else {
-			if(lastchar==";" && display)
-				ret = indent_line + "_ = " + line_stripped + " display(_)";
-			else
+			if(lastchar==";" && display) {
+				prefix = "_ = " + prefix;
+				ret = indent_line + line_stripped + " display(_)";
+				}
+			else {
 				ret = indent_line + line_stripped;
+				}
 			}
 		}
-	return ret+end_of_line;
+	return std::make_pair(prefix, ret+end_of_line);
 	}
 
 #ifndef CDBPYTHON_NO_NOTEBOOK
@@ -270,9 +365,10 @@ std::string cadabra::cnb2python(const std::string& in_name, bool for_standalone)
 				ConvertData cv;
 //				std::string line, lhs, rhs, op, indent;
 				while (std::getline(s, line)) {
-					auto res = convert_line(line, cv, for_standalone); // lhs, rhs, op, indent, for_standalone);
-					if(res!="::empty")
-						ofs << res << '\n';
+					std::pair<std::string, std::string> res
+						= convert_line(line, cv, for_standalone); // lhs, rhs, op, indent, for_standalone);
+					if(res.second!="::empty")
+						ofs << res.second << '\n';
 					}
 				}
 			}

core/CdbPython.hh

@@ -18,6 +18,14 @@ namespace cadabra {
 
 	std::string cdb2python_string(const std::string&, bool display);	
 
+	/// Test whether a block of Python code is complete or requires
+	/// more input lines. This uses `codeop.compile_command`.
+	///     1: complete
+   ///     0: incomplete
+   ///    -1: indentation error, need backtracking
+
+	int         is_python_code_complete(const std::string&);
+
 	/// \ingroup files
 	/// Object to store pre-parsing intermediate results. Necessary
 	/// to keep things tidy but also in order to avoid the fact that
@@ -42,8 +50,10 @@ namespace cadabra {
    /// 
    /// TODO: make ';' at the end of '::' line result the print statement printing 
    /// property objects using their readable form; addresses one issue report).
+	///
+	/// Returns two strings, one which should be prefixed to the block so far, the other postfixed.
 
-	std::string convert_line(const std::string&, ConvertData& cv, bool display); //std::string& lhs, std::string& rhs, std::string& op, std::string& indent, bool display);
+	std::pair<std::string, std::string> convert_line(const std::string&, ConvertData& cv, bool display); //std::string& lhs, std::string& rhs, std::string& op, std::string& indent, bool display);
 
 	/// \ingroup files
 	/// Convert a Cadabra notebook file to pure Python. This gets

core/cadabra2-cli.cc

@@ -400,8 +400,9 @@ void Shell::process_ps1(const std::string& line)
 	// Convert cadabra to python
 	bool display = !(flags & Flags::IgnoreSemicolons);
 	cadabra::ConvertData cv;
-//	std::string lhs, rhs, op, indent;
-	std::string output = cadabra::convert_line(line, cv, display);
+	std::pair<std::string, std::string> res = cadabra::convert_line(line, cv, display);
+	// FIXME: we need to process res.first
+	const std::string& output = res.second;
 	if (output == "::empty") {
 		// Cadabra continuation line, add to collect
 		collect += line + "\n";

core/cadabra2.in

@@ -85,7 +85,7 @@ class Shell(InteractiveConsole):
     def push(self, line):
 
         # line = self.preprocess(line, True)
-        line = convert_line(line, self.convert_data, True)
+        prefix, line = convert_line(line, self.convert_data, True)
         if self.convert_data.lhs == "":
             # print('executing: ')
             # print(line) # line is generically multiple lines
@@ -136,7 +136,7 @@ if __name__ == '__main__':
                 collect=""
                 for line in f:
                     line=line.rstrip()
-                    res = convert_line(line, sh.convert_data, True)
+                    prefix, res = convert_line(line, sh.convert_data, True)
                     if res!="::empty":
                        collect += res+"\n"		    
                 # print("----\n"+collect+"----\n")

frontend/gtkmm/CodeInput.cc

@@ -47,15 +47,26 @@ CodeInput::CodeInput(DTree::iterator it, const std::string& txt, double s, const
 	init(prefs);
 	}
 
+void CodeInput::on_size_allocate(Gtk::Allocation& allocation)
+	{
+//	allocation.set_width(400); // Fixed width
+	Gtk::Box::on_size_allocate(allocation);
+	}
+
 void CodeInput::init(const Prefs& prefs)
 	{
 	//	scroll_.set_size_request(-1,200);
 	//	scroll_.set_border_width(1);
 	//	scroll_.set_policy(Gtk::POLICY_NEVER, Gtk::POLICY_ALWAYS);
-	edit.set_wrap_mode(Gtk::WRAP_NONE); // WRAP_WORD_CHAR); wrapping leads to weird effects
+//	edit.set_wrap_mode(Gtk::WRAP_NONE); // WRAP_WORD_CHAR); wrapping leads to weird effects
 	edit.set_pixels_above_lines(1);
 	edit.set_pixels_below_lines(1);
 	edit.set_pixels_inside_wrap(1);
+
+	// Fix the size of the outer box, we re-fix this on window resize.
+//	set_hexpand(false);
+//	set_size_request(400, -1);
+
 	// The following two are margins around the vbox which contains the
 	// text input and the LaTeX output(s).
 	set_margin_top(10);
@@ -88,9 +99,13 @@ void CodeInput::init(const Prefs& prefs)
 	if (prefs.highlight) {
 		using namespace std::string_literals;
 		switch (edit.datacell->cell_type) {
-			// Fallthrough
-			case DataCell::CellType::python:
 			case DataCell::CellType::latex:
+				edit.set_wrap_mode(Gtk::WRAP_WORD_CHAR);
+				std::cerr << "wrapping!" << std::endl;
+				enable_highlighting(edit.datacell->cell_type, prefs);
+				break;
+			case DataCell::CellType::python:
+				edit.set_wrap_mode(Gtk::WRAP_NONE);
 				enable_highlighting(edit.datacell->cell_type, prefs);
 				break;
 			default:
@@ -101,7 +116,7 @@ void CodeInput::init(const Prefs& prefs)
 
 	auto dummy_adj = Gtk::Adjustment::create(0,0,0);
 	edit.set_focus_hadjustment(dummy_adj);
-	
+
 	add(edit);
 	//	set_border_width(3);
 	show_all();
@@ -727,15 +742,25 @@ bool CodeInput::exp_input_tv::on_draw(const Cairo::RefPtr<Cairo::Context>& cr)
 
 bool CodeInput::exp_input_tv::on_focus_in_event(GdkEventFocus *event)
 	{
+	// std::cerr << "FOCUS IN" << std::endl;
 	cell_got_focus(datacell);
-	vadjustment->set_value(previous_value);
-	return false;
+	if(previous_value>=0) 
+		vadjustment->set_value(previous_value);
+	return Gtk::TextView::on_focus_in_event(event);
+	}
+
+bool CodeInput::exp_input_tv::on_focus_out_event(GdkEventFocus *event)
+	{
+	// std::cerr << "FOCUS OUT" << std::endl;
+	previous_value = -99.0;
+	return Gtk::TextView::on_focus_out_event(event);
 	}
 
 bool CodeInput::exp_input_tv::on_motion_notify_event(GdkEventMotion* event)
 	{
+	// std::cerr << "MOTION" << std::endl;
 	previous_value = vadjustment->get_value();
-	return false;
+	return Gtk::TextView::on_motion_notify_event(event);
 	}
 
 // bool CodeInput::exp_input_tv::on_move_cursor_event(Glib::RefPtr<Gtk::TextBuffer::Mark> iter, Gtk::MovementStep step, bool extend_selection)