Can 'send' a .glb file to the frontend and have it display using fila…

…ment.

client_server/ComputeThread.cc

@@ -412,7 +412,9 @@ void ComputeThread::on_message(websocketpp::connection_hdl hdl, message_ptr msg)
 					docthread->queue_action(action);
 					}
 				else if (msg_type == "graphics_view") {
-					std::cerr << "received graphics cell " << content["output"].get<std::string>() << std::endl;
+					// std::cerr << "received graphics cell " << content["output"].get<std::string>() << std::endl;
+					// The data is a base64 encoded .glb file. We send this straight
+					// to the action; FIXME: could perhaps decode already here?
 					DataCell result(cell_id, DataCell::CellType::graphics_view, content["output"].get<std::string>());
 					std::shared_ptr<ActionBase> action =
 					   std::make_shared<ActionAddCell>(result, parent_id, ActionAddCell::Position::child);

frontend/gtkmm/CMakeLists.txt

@@ -83,18 +83,78 @@ if(NOT EXISTS ${FILAMENT}/out/release/filament/lib/${CMAKE_SYSTEM_PROCESSOR}/lib
 endif()
 
 set(FILAMENT_LIBRARIES
+  filament-iblprefilter
+  geometry
+  meshoptimizer
+  mikktspace
+  imageio
+  png
+  tinyexr
+  z
+  basis_encoder
+  filamat
   filament
+  image
   ibl
   backend
-  geometry
   bluegl
   bluevk
   vkshaders
   filabridge
   filaflat
   smol-v
-  image
+  stb
   utils
+  gltfio
+  viewer
+)
+set(FILAMENT_ALL
+  ${FILAMENT}/out/cmake-release/libs/gltfio/libgltfio.a
+  ${FILAMENT}/out/cmake-release/libs/gltfio/libgltfio_core.a
+  ${FILAMENT}/out/cmake-release/third_party/draco/tnt/libdracodec.a
+
+  ${FILAMENT}/out/cmake-release/libs/camutils/libcamutils.a
+  ${FILAMENT}/out/cmake-release/libs/filagui/libfilagui.a
+  ${FILAMENT}/out/cmake-release/libs/filamat/libfilamat.a
+  ${FILAMENT}/out/cmake-release/shaders/libshaders.a
+  ${FILAMENT}/out/cmake-release/third_party/glslang/tnt/SPIRV/libSPIRV.a
+  ${FILAMENT}/out/cmake-release/third_party/glslang/tnt/glslang/libglslang.a
+  ${FILAMENT}/out/cmake-release/third_party/glslang/tnt/OGLCompilersDLL/libOGLCompiler.a
+  ${FILAMENT}/out/cmake-release/third_party/glslang/tnt/glslang/OSDependent/Unix/libOSDependent.a
+  ${FILAMENT}/out/cmake-release/third_party/spirv-tools/source/opt/libSPIRV-Tools-opt.a
+  ${FILAMENT}/out/cmake-release/third_party/spirv-tools/source/libSPIRV-Tools.a
+  ${FILAMENT}/out/cmake-release/third_party/spirv-cross/tnt/libspirv-cross-glsl.a
+  ${FILAMENT}/out/cmake-release/third_party/spirv-cross/tnt/libspirv-cross-msl.a
+  ${FILAMENT}/out/cmake-release/third_party/spirv-cross/tnt/libspirv-cross-core.a
+  ${FILAMENT}/out/cmake-release/libs/iblprefilter/libfilament-iblprefilter.a
+  ${FILAMENT}/out/cmake-release/libs/geometry/libgeometry.a
+  ${FILAMENT}/out/cmake-release/third_party/meshoptimizer/tnt/libmeshoptimizer.a
+  ${FILAMENT}/out/cmake-release/third_party/mikktspace/libmikktspace.a
+  ${FILAMENT}/out/cmake-release/third_party/getopt/libgetopt.a
+  ${FILAMENT}/out/cmake-release/libs/imageio/libimageio.a
+  ${FILAMENT}/out/cmake-release/third_party/libpng/tnt/libpng.a
+  ${FILAMENT}/out/cmake-release/third_party/tinyexr/tnt/libtinyexr.a
+  ${FILAMENT}/out/cmake-release/third_party/libz/tnt/libz.a
+  ${FILAMENT}/out/cmake-release/third_party/basisu/tnt/libbasis_encoder.a
+  ${FILAMENT}/out/cmake-release/third_party/imgui/tnt/libimgui.a
+  ${FILAMENT}/out/cmake-release/libs/ktxreader/libktxreader.a
+  ${FILAMENT}/out/cmake-release/filament/libfilament.a
+  ${FILAMENT}/out/cmake-release/filament/backend/libbackend.a
+  ${FILAMENT}/out/cmake-release/libs/bluegl/libbluegl.a
+  ${FILAMENT}/out/cmake-release/libs/bluevk/libbluevk.a
+  ${FILAMENT}/out/cmake-release/filament/backend/libvkshaders.a
+  ${FILAMENT}/out/cmake-release/libs/filaflat/libfilaflat.a
+  ${FILAMENT}/out/cmake-release/libs/filabridge/libfilabridge.a
+  ${FILAMENT}/out/cmake-release/third_party/smol-v/tnt/libsmol-v.a
+  ${FILAMENT}/out/cmake-release/libs/ibl/libibl-lite.a
+  ${FILAMENT}/out/cmake-release/libs/image/libimage.a
+  ${FILAMENT}/out/cmake-release/third_party/basisu/tnt/libbasis_transcoder.a
+  ${FILAMENT}/out/cmake-release/third_party/basisu/tnt/libzstd.a
+  ${FILAMENT}/out/cmake-release/libs/math/libmath.a
+  ${FILAMENT}/out/cmake-release/third_party/libsdl2/tnt/libsdl2.a
+  ${FILAMENT}/out/cmake-release/third_party/stb/tnt/libstb.a
+  ${FILAMENT}/out/cmake-release/libs/utils/libutils.a
+#  libs/filamentapp/libfilamentapp-resources.a
 )
 
 link_directories(${FILAMENT}/out/release/filament/lib/${CMAKE_SYSTEM_PROCESSOR})
@@ -239,14 +299,14 @@ target_link_libraries(
   cadabra2-gtk
   PUBLIC
   cadabra_client
-  ${FILAMENT_LIBRARIES}
   ${Boost_LIBRARIES} 
   ${GLIBMM_LIBRARIES} 
   ${GTKMM3_LIBRARIES} 
   ${SQLITE3_LIBRARIES} 
   Threads::Threads
   c++
 )
+target_link_libraries(cadabra2-gtk PRIVATE "-Wl,--start-group" ${FILAMENT_ALL} "-Wl,--end-group")
 
 if(USE_MICROTEX)
   if(WIN32)

frontend/gtkmm/GraphicsView.cc

@@ -5,9 +5,14 @@
 #include <iostream>
 #include <fstream>
 #include <gdk/gdkx.h>
+#include <filament/FilamentAPI.h>
+#include <filament/Engine.h>
 #include <filament/Viewport.h>
 #include <filament/ColorGrading.h>
-
+#include <gltfio/AssetLoader.h>
+#include <gltfio/ResourceLoader.h>
+#include <filament/LightManager.h>
+
 using namespace cadabra;
 
 // LIBGL_ALWAYS_SOFTWARE=true cadabra2-gtk
@@ -34,6 +39,11 @@ GraphicsView::GraphicsView(filament::Engine *engine_)
 	show_all();
 	}
 
+void GraphicsView::set_gltf(const std::string& str)
+	{
+	glview.set_gltf(str);
+	}
+
 GraphicsView::GLView::GLView(filament::Engine *engine_)
 	: engine(engine_), zoom(1.0)
 	{
@@ -56,6 +66,11 @@ static constexpr uint16_t TRIANGLE_INDICES[3] = { 0, 1, 2 };
 using utils::Entity;
 using utils::EntityManager;
 
+void GraphicsView::GLView::set_gltf(const std::string& str)
+	{
+	gltf_str = 	Glib::Base64::decode(str);
+	}
+
 void GraphicsView::GLView::first_render()
 	{
 	// Setup swapchain.
@@ -67,7 +82,6 @@ void GraphicsView::GLView::first_render()
 
 	swapChain = engine->createSwapChain((void*)x11_window_id, 
 													filament::SwapChain::CONFIG_TRANSPARENT);
-	std::cerr << swapChain << std::endl;
 
 	// Setup buffers.
 	vb = filament::VertexBuffer::Builder()
@@ -101,12 +115,21 @@ void GraphicsView::GLView::first_render()
 	scene = engine->createScene();
 	view = engine->createView();
 	view->setViewport(filament::Viewport(0, 0, 400, 400));
-	scene->addEntity(renderable);
 	camera = utils::EntityManager::get().create();
 	cam = engine->createCamera(camera);
 	view->setCamera(cam);
 	view->setScene(scene);
 
+	mainlight = utils::EntityManager::get().create();
+	filament::LightManager::Builder(filament::LightManager::Type::DIRECTIONAL)
+		.color(filament::Color::toLinear<filament::ACCURATE>(filament::sRGBColor(1.0f, 1.0f, 1.0f)))
+		.intensity(100000.0)
+		.position({0.0, 10.0, 0.0})
+		.direction({0.0, -1.0, 0.0})
+		.castShadows(true)
+		.build(*engine, mainlight);
+	scene->addEntity(mainlight);
+
 //	skybox = filament::Skybox::Builder().color({0.1, 0.125, 0.25, 1.0}).build(*engine);
 	skybox = filament::Skybox::Builder().color({4.0, 4.0, 4.0, 1.0}).build(*engine);	
 	scene->setSkybox(skybox);
@@ -122,7 +145,17 @@ void GraphicsView::GLView::first_render()
 //	co.clearColor=filament::math::float4({1.0,1.0,0.0,0.5});
 	co.clearColor=filament::math::float4({1.0,1.0,1.0,1.0});	
 	renderer->setClearOptions(co);
-//	view->setClearTargets(filament::backend::TargetBufferFlags::COLOR | filament::backend::TargetBufferFlags::DEPTH);
+
+	if(gltf_str.size()>0) {
+		if(!load(gltf_str)) {
+			std::cerr << "GraphicsView::GLView::first_render: adding test triangle." << std::endl;
+			scene->addEntity(renderable);
+			}
+		}
+	else {
+		std::cerr << "GraphicsView::GLView::first_render: adding a test triangle." << std::endl;		
+		scene->addEntity(renderable);
+		}
 	}
 
 //bool GraphicsView::GLView::on_render(const Glib::RefPtr< Gdk::GLContext > &context)
@@ -197,21 +230,55 @@ void GraphicsView::GLView::setup_camera()
 	view->setViewport({0, 0, w, h});
 
 	// setup view matrix
-	const filament::math::float3 eye(0.f, 0.f, 1.f);
+//	const filament::math::float3 eye(0.f, 0.f, 1.f);
+	const filament::math::float3 eye(2.f, 2.f, 1.f);
 	const filament::math::float3 target(0.f, 0.f, 0.f);
 	const filament::math::float3 up(0.f, 1.f, 0.f);
 	cam->lookAt(eye, target, up);
 
 	// setup projection matrix
 	const float aspect = float(w) / h;
-	std::cerr << "aspect = " << aspect << ", zoom = " << zoom << std::endl;
-	cam->setProjection(filament::Camera::Projection::ORTHO,
-								 -aspect * zoom,
-								 aspect * zoom,
-								 -zoom,
-								 zoom,
-								 0,
-								 10.0);
+	// std::cerr << "aspect = " << aspect << ", zoom = " << zoom << std::endl;
+	// cam->setProjection(filament::Camera::Projection::PERSPECTIVE,
+	// 							 -aspect * zoom,
+	// 							 aspect * zoom,
+	// 							 -zoom,
+	// 							 zoom,
+	// 							 0,
+	// 							 100.0);
+	cam->setProjection(80.0, // fov
+							 aspect,
+							 0.01,
+							 100.0
+							 );
+//	cam->setExposure(1); //, 1.2, 100.0);
 	}
 
+bool GraphicsView::GLView::load(const std::string& gltf)
+	{
+	auto materials = filament::gltfio::createJitShaderProvider(engine);
+//	auto decoder = filament::gltfio::createStbProvider(engine);
+	auto loader = filament::gltfio::AssetLoader::create({engine, materials});
+
+	// Parse the glTF content and create Filament entities.
+	filament::gltfio::FilamentAsset* asset = loader->createAsset((uint8_t *)gltf.data(), gltf.size());
+	if(!asset) {
+		std::cerr << "GraphicsView::GLView::load: failed to parse/convert gltf." << std::endl;
+		return false;
+		}
+
+	// Load buffers and textures from disk.
+	filament::gltfio::ResourceLoader resourceLoader({engine, ".", true});
+//	resourceLoader.addTextureProvider("image/png", decoder);
+//	resourceLoader.addTextureProvider("image/jpeg", decoder);
+	resourceLoader.loadResources(asset);
+
+	// Free the glTF hierarchy as it is no longer needed.
+	asset->releaseSourceData();
+
+	// Add renderables to the scene.
+	scene->addEntities(asset->getEntities(), asset->getEntityCount());
 
+	std::cerr << "GraphicsView::GLView::load: " << asset->getEntityCount() << " gltf entities added." << std::endl;
+	return true;
+	}

frontend/gtkmm/GraphicsView.hh

@@ -1,16 +1,13 @@
 
 #pragma once
 
-#include <gtkmm/box.h>
-#include <gtkmm/glarea.h>
-#include <gtkmm/eventbox.h>
-#include <gtkmm/drawingarea.h>
-
+#include <filament/FilamentAPI.h>
 #include <filament/Engine.h>
 #include <filament/SwapChain.h>
 #include <filament/Camera.h>
 #include <filament/Engine.h>
 #include <filament/IndexBuffer.h>
+#include <filament/LightManager.h>
 #include <filament/Material.h>
 #include <filament/MaterialInstance.h>
 #include <filament/RenderableManager.h>
@@ -21,6 +18,11 @@
 #include <filament/View.h>
 #include <utils/EntityManager.h>
 
+#include <gtkmm/box.h>
+#include <gtkmm/glarea.h>
+#include <gtkmm/eventbox.h>
+#include <gtkmm/drawingarea.h>
+
 namespace cadabra {
 
 	/// \ingroup frontend
@@ -36,6 +38,9 @@ namespace cadabra {
 			virtual bool on_button_press_event(GdkEventButton *event) override;
 			virtual bool on_button_release_event(GdkEventButton *event) override;
 
+			/// Set the content of this 3d view to be the gltf (json).
+			void set_gltf(const std::string&);
+
 			class GLView :
 				public Gtk::DrawingArea  {
 //				public Gtk::GLArea  {
@@ -44,10 +49,14 @@ namespace cadabra {
 					//virtual bool on_render (const Glib::RefPtr< Gdk::GLContext > &context);
 					virtual bool on_draw(const Cairo::RefPtr<Cairo::Context>& cr) override;
 
+					void set_gltf(const std::string&);
 				private:
 					void first_render();
 					void setup_camera();
+					bool load(const std::string& gltf_data);
 
+					std::string gltf_str;
+
 					// Filament things. The engine is owned by the NotebookWindow and passed
 					// in on creation of GraphicsView. The swapchain, on the other hand, is
 					// per-view, so we are responsible for it.
@@ -64,6 +73,7 @@ namespace cadabra {
 					filament::View*         view;
 					utils::Entity           camera;
 					utils::Entity           renderable;
+					utils::Entity           mainlight;
 					filament::Renderer     *renderer;
 
 					float zoom;

frontend/gtkmm/NotebookWindow.cc

@@ -1261,6 +1261,7 @@ void NotebookWindow::add_cell(const DTree& tr, DTree::iterator it, bool visible)
 				}
 			case DataCell::CellType::graphics_view: {
 				newcell.graphicsbox = manage( new GraphicsView(filament_engine) );
+				newcell.graphicsbox->set_gltf(it->textbuf);
 				w=newcell.graphicsbox;
 				break;
 				}