From e8d234d3722b27d1501f2fc45e045b5e022fff1b Mon Sep 17 00:00:00 2001 From: Alvaro Huarte Date: Sun, 15 Apr 2018 10:20:52 +0200 Subject: [PATCH] Refactoring of code to easily write new output formats This defines a set of abstract classes that anyone can override to implement new formats when serializing the terrain tiles. --- src/CMakeLists.txt | 10 ++ src/CTBFileOutputStream.cpp | 43 +++++++++ src/CTBFileOutputStream.hpp | 60 ++++++++++++ src/CTBFileTileSerializer.cpp | 169 ++++++++++++++++++++++++++++++++++ src/CTBFileTileSerializer.hpp | 74 +++++++++++++++ src/CTBOutputStream.hpp | 40 ++++++++ src/CTBZOutputStream.cpp | 72 +++++++++++++++ src/CTBZOutputStream.hpp | 55 +++++++++++ src/GDALSerializer.hpp | 52 +++++++++++ src/MeshSerializer.hpp | 50 ++++++++++ src/MeshTile.cpp | 94 +++++++++---------- src/MeshTile.hpp | 5 + src/TerrainSerializer.hpp | 50 ++++++++++ src/TerrainTile.cpp | 40 +++----- src/TerrainTile.hpp | 5 + tools/ctb-tile.cpp | 120 ++++++------------------ 16 files changed, 770 insertions(+), 169 deletions(-) create mode 100644 src/CTBFileOutputStream.cpp create mode 100644 src/CTBFileOutputStream.hpp create mode 100644 src/CTBFileTileSerializer.cpp create mode 100644 src/CTBFileTileSerializer.hpp create mode 100644 src/CTBOutputStream.hpp create mode 100644 src/CTBZOutputStream.cpp create mode 100644 src/CTBZOutputStream.hpp create mode 100644 src/GDALSerializer.hpp create mode 100644 src/MeshSerializer.hpp create mode 100644 src/TerrainSerializer.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 49d39d0..30be95c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,6 +21,9 @@ add_library(ctb SHARED GDALTile.cpp GDALTiler.cpp GDALDatasetReader.cpp + CTBFileTileSerializer.cpp + CTBFileOutputStream.cpp + CTBZOutputStream.cpp TerrainTiler.cpp TerrainTile.cpp MeshTiler.cpp @@ -35,9 +38,14 @@ set(HEADERS BoundingSphere.hpp Coordinate.hpp Coordinate3D.hpp + GDALSerializer.hpp GDALTile.hpp GDALTiler.hpp GDALDatasetReader.hpp + CTBFileTileSerializer.hpp + CTBFileOutputStream.hpp + CTBOutputStream.hpp + CTBZOutputStream.hpp GlobalGeodetic.hpp GlobalMercator.hpp Grid.hpp @@ -45,12 +53,14 @@ set(HEADERS HeightFieldChunker.hpp Mesh.hpp MeshIterator.hpp + MeshSerializer.hpp MeshTile.hpp MeshTiler.hpp RasterIterator.hpp RasterTiler.hpp CTBException.hpp TerrainIterator.hpp + TerrainSerializer.hpp TerrainTile.hpp TerrainTiler.hpp Tile.hpp diff --git a/src/CTBFileOutputStream.cpp b/src/CTBFileOutputStream.cpp new file mode 100644 index 0000000..a0dc4a3 --- /dev/null +++ b/src/CTBFileOutputStream.cpp @@ -0,0 +1,43 @@ +/******************************************************************************* + * Copyright 2018 GeoData + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *******************************************************************************/ + +/** + * @file CTBFileOutputStream.cpp + * @brief This defines the `CTBFileOutputStream` class + */ + +#include "CTBFileOutputStream.hpp" + +using namespace ctb; + +/** + * @details + * Writes a sequence of memory pointed by ptr into the FILE*. + */ +uint32_t +ctb::CTBFileOutputStream::write(const void *ptr, uint32_t size) { + return (uint32_t)fwrite(ptr, size, 1, fp); +} + +/** + * @details + * Writes a sequence of memory pointed by ptr into the ostream. + */ +uint32_t +ctb::CTBStdOutputStream::write(const void *ptr, uint32_t size) { + mstream.write((const char *)ptr, size); + return size; +} diff --git a/src/CTBFileOutputStream.hpp b/src/CTBFileOutputStream.hpp new file mode 100644 index 0000000..a1bc3bd --- /dev/null +++ b/src/CTBFileOutputStream.hpp @@ -0,0 +1,60 @@ +#ifndef CTBFILEOUTPUTSTREAM_HPP +#define CTBFILEOUTPUTSTREAM_HPP + +/******************************************************************************* + * Copyright 2018 GeoData + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *******************************************************************************/ + +/** + * @file CTBFileOutputStream.hpp + * @brief This declares and defines the `CTBFileOutputStream` and `CTBStdOutputStream` classes + */ + +#include +#include +#include "CTBOutputStream.hpp" + +namespace ctb { + class CTBFileOutputStream; + class CTBStdOutputStream; +} + +/// Implements CTBOutputStream for `FILE*` objects +class CTB_DLL ctb::CTBFileOutputStream : public ctb::CTBOutputStream { +public: + CTBFileOutputStream(FILE *fptr): fp(fptr) {} + + /// Writes a sequence of memory pointed by ptr into the stream + virtual uint32_t write(const void *ptr, uint32_t size); + +protected: + /// The underlying FILE* + FILE *fp; +}; + +/// Implements CTBOutputStream for `std::ostream` objects +class CTB_DLL ctb::CTBStdOutputStream : public ctb::CTBOutputStream { +public: + CTBStdOutputStream(std::ostream &stream): mstream(stream) {} + + /// Writes a sequence of memory pointed by ptr into the stream + virtual uint32_t write(const void *ptr, uint32_t size); + +protected: + /// The underlying std::ostream + std::ostream &mstream; +}; + +#endif /* CTBFILEOUTPUTSTREAM_HPP */ diff --git a/src/CTBFileTileSerializer.cpp b/src/CTBFileTileSerializer.cpp new file mode 100644 index 0000000..ef7582e --- /dev/null +++ b/src/CTBFileTileSerializer.cpp @@ -0,0 +1,169 @@ +/******************************************************************************* + * Copyright 2018 GeoData + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *******************************************************************************/ + +/** + * @file CTBFileTileSerializer.cpp + * @brief This defines the `CTBFileTileSerializer` class + */ + +#include +#include +#include + +#include "../deps/concat.hpp" +#include "cpl_vsi.h" +#include "CTBException.hpp" +#include "CTBFileTileSerializer.hpp" + +#include "CTBFileOutputStream.hpp" +#include "CTBZOutputStream.hpp" + +using namespace std; +using namespace ctb; + +#ifdef _WIN32 +static const char *osDirSep = "\\"; +#else +static const char *osDirSep = "/"; +#endif + + +/// Create a filename for a tile coordinate +std::string +ctb::CTBFileTileSerializer::getTileFilename(const TileCoordinate *coord, const string dirname, const char *extension) { + static mutex mutex; + VSIStatBufL stat; + string filename = concat(dirname, coord->zoom, osDirSep, coord->x); + + lock_guard lock(mutex); + + // Check whether the `{zoom}/{x}` directory exists or not + if (VSIStatExL(filename.c_str(), &stat, VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG)) { + filename = concat(dirname, coord->zoom); + + // Check whether the `{zoom}` directory exists or not + if (VSIStatExL(filename.c_str(), &stat, VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG)) { + // Create the `{zoom}` directory + if (VSIMkdir(filename.c_str(), 0755)) + throw CTBException("Could not create the zoom level directory"); + + } else if (!VSI_ISDIR(stat.st_mode)) { + throw CTBException("Zoom level file path is not a directory"); + } + + // Create the `{zoom}/{x}` directory + filename += concat(osDirSep, coord->x); + if (VSIMkdir(filename.c_str(), 0755)) + throw CTBException("Could not create the x level directory"); + + } else if (!VSI_ISDIR(stat.st_mode)) { + throw CTBException("X level file path is not a directory"); + } + + // Create the filename itself, adding the extension if required + filename += concat(osDirSep, coord->y); + if (extension != NULL) { + filename += "."; + filename += extension; + } + + return filename; +} + +/// Check if file exists +static bool +fileExists(const std::string& filename) { + VSIStatBufL statbuf; + return VSIStatExL(filename.c_str(), &statbuf, VSI_STAT_EXISTS_FLAG) == 0; +} + + +/** + * @details + * Returns if the specified Tile Coordinate should be serialized + */ +bool ctb::CTBFileTileSerializer::mustSerializeCoordinate(const ctb::TileCoordinate *coordinate, const char *extension) { + if (!mresume) + return true; + + const string filename = getTileFilename(coordinate, moutputDir, extension ? extension : "terrain"); + return !fileExists(filename); +} + +/** + * @details + * Serialize a GDALTile to the Directory store + */ +bool +ctb::CTBFileTileSerializer::serializeTile(const ctb::GDALTile *tile, GDALDriver *driver, const char *extension, const CPLStringList &creationOptions) { + const TileCoordinate *coordinate = tile; + const string filename = getTileFilename(coordinate, moutputDir, extension); + const string temp_filename = concat(filename, ".tmp"); + + GDALDataset *poDstDS; + poDstDS = driver->CreateCopy(temp_filename.c_str(), tile->dataset, FALSE, creationOptions.List(), NULL, NULL); + + // Close the datasets, flushing data to destination + if (poDstDS == NULL) { + throw CTBException("Could not create GDAL tile"); + } + GDALClose(poDstDS); + + if (VSIRename(temp_filename.c_str(), filename.c_str()) != 0) { + throw new CTBException("Could not rename temporary file"); + } + return true; +} + +/** + * @details + * Serialize a TerrainTile to the Directory store + */ +bool +ctb::CTBFileTileSerializer::serializeTile(const ctb::TerrainTile *tile) { + const TileCoordinate *coordinate = tile; + const string filename = getTileFilename(tile, moutputDir, "terrain"); + const string temp_filename = concat(filename, ".tmp"); + + CTBZFileOutputStream ostream(temp_filename.c_str()); + tile->writeFile(ostream); + ostream.close(); + + if (VSIRename(temp_filename.c_str(), filename.c_str()) != 0) { + throw new CTBException("Could not rename temporary file"); + } + return true; +} + +/** + * @details + * Serialize a MeshTile to the Directory store + */ +bool +ctb::CTBFileTileSerializer::serializeTile(const ctb::MeshTile *tile, bool writeVertexNormals) { + const TileCoordinate *coordinate = tile; + const string filename = getTileFilename(coordinate, moutputDir, "terrain"); + const string temp_filename = concat(filename, ".tmp"); + + CTBZFileOutputStream ostream(temp_filename.c_str()); + tile->writeFile(ostream, writeVertexNormals); + ostream.close(); + + if (VSIRename(temp_filename.c_str(), filename.c_str()) != 0) { + throw new CTBException("Could not rename temporary file"); + } + return true; +} diff --git a/src/CTBFileTileSerializer.hpp b/src/CTBFileTileSerializer.hpp new file mode 100644 index 0000000..22a885c --- /dev/null +++ b/src/CTBFileTileSerializer.hpp @@ -0,0 +1,74 @@ +#ifndef CTBFILETILESERIALIZER_HPP +#define CTBFILETILESERIALIZER_HPP + +/******************************************************************************* + * Copyright 2018 GeoData + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *******************************************************************************/ + +/** + * @file CTBFileTileSerializer.hpp + * @brief This declares and defines the `CTBFileTileSerializer` class + */ + +#include + +#include "TileCoordinate.hpp" +#include "GDALSerializer.hpp" +#include "TerrainSerializer.hpp" +#include "MeshSerializer.hpp" + +namespace ctb { + class CTBFileTileSerializer; +} + +/// Implements a serializer of `Tile`s based in a directory of files +class CTB_DLL ctb::CTBFileTileSerializer : + public ctb::GDALSerializer, + public ctb::TerrainSerializer, + public ctb::MeshSerializer { +public: + CTBFileTileSerializer(const std::string &outputDir, bool resume): + moutputDir(outputDir), + mresume(resume) {} + + /// Start a new serialization task + virtual void startSerialization() {}; + + /// Returns if the specified Tile Coordinate should be serialized + virtual bool mustSerializeCoordinate(const ctb::TileCoordinate *coordinate, const char *extension = NULL); + + /// Serialize a GDALTile to the store + virtual bool serializeTile(const ctb::GDALTile *tile, GDALDriver *driver, const char *extension, const CPLStringList &creationOptions); + /// Serialize a TerrainTile to the store + virtual bool serializeTile(const ctb::TerrainTile *tile); + /// Serialize a MeshTile to the store + virtual bool serializeTile(const ctb::MeshTile *tile, bool writeVertexNormals = false); + + /// Serialization finished, releases any resources loaded + virtual void endSerialization() {}; + + + /// Create a filename for a tile coordinate + static std::string + getTileFilename(const TileCoordinate *coord, const std::string dirname, const char *extension); + +protected: + /// The target directory where serializing + std::string moutputDir; + /// Do not overwrite existing files + bool mresume; +}; + +#endif /* CTBFILETILESERIALIZER_HPP */ diff --git a/src/CTBOutputStream.hpp b/src/CTBOutputStream.hpp new file mode 100644 index 0000000..7d0d10f --- /dev/null +++ b/src/CTBOutputStream.hpp @@ -0,0 +1,40 @@ +#ifndef CTBOUTPUTSTREAM_HPP +#define CTBOUTPUTSTREAM_HPP + +/******************************************************************************* + * Copyright 2018 GeoData + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *******************************************************************************/ + +/** + * @file CTBOutputStream.hpp + * @brief This declares and defines the `CTBOutputStream` class + */ + +#include "config.hpp" +#include "types.hpp" + +namespace ctb { + class CTBOutputStream; +} + +/// This represents a generic CTB output stream to write raw data +class CTB_DLL ctb::CTBOutputStream { +public: + + /// Writes a sequence of memory pointed by ptr into the stream + virtual uint32_t write(const void *ptr, uint32_t size) = 0; +}; + +#endif /* CTBOUTPUTSTREAM_HPP */ diff --git a/src/CTBZOutputStream.cpp b/src/CTBZOutputStream.cpp new file mode 100644 index 0000000..5420b5a --- /dev/null +++ b/src/CTBZOutputStream.cpp @@ -0,0 +1,72 @@ +/******************************************************************************* + * Copyright 2018 GeoData + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *******************************************************************************/ + +/** + * @file CTBZOutputStream.cpp + * @brief This defines the `CTBZOutputStream` and `CTBZFileOutputStream` classes + */ + +#include "CTBException.hpp" +#include "CTBZOutputStream.hpp" + +using namespace ctb; + +/** + * @details + * Writes a sequence of memory pointed by ptr into the GZFILE*. + */ +uint32_t +ctb::CTBZOutputStream::write(const void *ptr, uint32_t size) { + if (size == 1) { + int c = *((const char *)ptr); + return gzputc(fp, c) == -1 ? 0 : 1; + } + else { + return gzwrite(fp, ptr, size) == 0 ? 0 : size; + } +} + +ctb::CTBZFileOutputStream::CTBZFileOutputStream(const char *fileName) : CTBZOutputStream(NULL) { + gzFile file = gzopen(fileName, "wb"); + + if (file == NULL) { + throw CTBException("Failed to open file"); + } + fp = file; +} + +ctb::CTBZFileOutputStream::~CTBZFileOutputStream() { + close(); +} + +void +ctb::CTBZFileOutputStream::close() { + + // Try and close the file + if (fp) { + switch (gzclose(fp)) { + case Z_OK: + break; + case Z_STREAM_ERROR: + case Z_ERRNO: + case Z_MEM_ERROR: + case Z_BUF_ERROR: + default: + throw CTBException("Failed to close file"); + } + fp = NULL; + } +} diff --git a/src/CTBZOutputStream.hpp b/src/CTBZOutputStream.hpp new file mode 100644 index 0000000..b4890ba --- /dev/null +++ b/src/CTBZOutputStream.hpp @@ -0,0 +1,55 @@ +#ifndef CTBZOUTPUTSTREAM_HPP +#define CTBZOUTPUTSTREAM_HPP + +/******************************************************************************* + * Copyright 2018 GeoData + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *******************************************************************************/ + +/** + * @file CTBZOutputStream.hpp + * @brief This declares and defines the `CTBZOutputStream` class + */ + +#include "zlib.h" +#include "CTBOutputStream.hpp" + +namespace ctb { + class CTBZFileOutputStream; + class CTBZOutputStream; +} + +/// Implements CTBOutputStream for `GZFILE` object +class CTB_DLL ctb::CTBZOutputStream : public ctb::CTBOutputStream { +public: + CTBZOutputStream(gzFile gzptr): fp(gzptr) {} + + /// Writes a sequence of memory pointed by ptr into the stream + virtual uint32_t write(const void *ptr, uint32_t size); + +protected: + /// The underlying GZFILE* + gzFile fp; +}; + +/// Implements CTBOutputStream for gzipped files +class CTB_DLL ctb::CTBZFileOutputStream : public ctb::CTBZOutputStream { +public: + CTBZFileOutputStream(const char *fileName); + ~CTBZFileOutputStream(); + + void close(); +}; + +#endif /* CTBZOUTPUTSTREAM_HPP */ diff --git a/src/GDALSerializer.hpp b/src/GDALSerializer.hpp new file mode 100644 index 0000000..115375d --- /dev/null +++ b/src/GDALSerializer.hpp @@ -0,0 +1,52 @@ +#ifndef GDALSERIALIZER_HPP +#define GDALSERIALIZER_HPP + +/******************************************************************************* + * Copyright 2018 GeoData + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *******************************************************************************/ + +/** + * @file GDALSerializer.hpp + * @brief This declares and defines the `GDALSerializer` class + */ + +#include "config.hpp" +#include "TileCoordinate.hpp" +#include "GDALTile.hpp" + +#include "cpl_string.h" + +namespace ctb { + class GDALSerializer; +} + +/// Store `GDALTile`s from a GDAL Dataset +class CTB_DLL ctb::GDALSerializer { +public: + + /// Start a new serialization task + virtual void startSerialization() = 0; + + /// Returns if the specified Tile Coordinate should be serialized + virtual bool mustSerializeCoordinate(const ctb::TileCoordinate *coordinate, const char *extension = NULL) = 0; + + /// Serialize a GDALTile to the store + virtual bool serializeTile(const ctb::GDALTile *tile, GDALDriver *driver, const char *extension, const CPLStringList &creationOptions) = 0; + + /// Serialization finished, releases any resources loaded + virtual void endSerialization() = 0; +}; + +#endif /* GDALSERIALIZER_HPP */ diff --git a/src/MeshSerializer.hpp b/src/MeshSerializer.hpp new file mode 100644 index 0000000..c496200 --- /dev/null +++ b/src/MeshSerializer.hpp @@ -0,0 +1,50 @@ +#ifndef MESHSERIALIZER_HPP +#define MESHSERIALIZER_HPP + +/******************************************************************************* + * Copyright 2018 GeoData + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *******************************************************************************/ + +/** + * @file MeshSerializer.hpp + * @brief This declares and defines the `MeshSerializer` class + */ + +#include "config.hpp" +#include "TileCoordinate.hpp" +#include "MeshTile.hpp" + +namespace ctb { + class MeshSerializer; +} + +/// Store `MeshTile`s from a GDAL Dataset +class CTB_DLL ctb::MeshSerializer { +public: + + /// Start a new serialization task + virtual void startSerialization() = 0; + + /// Returns if the specified Tile Coordinate should be serialized + virtual bool mustSerializeCoordinate(const ctb::TileCoordinate *coordinate, const char *extension = NULL) = 0; + + /// Serialize a MeshTile to the store + virtual bool serializeTile(const ctb::MeshTile *tile, bool writeVertexNormals = false) = 0; + + /// Serialization finished, releases any resources loaded + virtual void endSerialization() = 0; +}; + +#endif /* MESHSERIALIZER_HPP */ diff --git a/src/MeshTile.cpp b/src/MeshTile.cpp index 11ad15f..7173dfb 100644 --- a/src/MeshTile.cpp +++ b/src/MeshTile.cpp @@ -24,11 +24,11 @@ #include #include #include "cpl_conv.h" -#include "zlib.h" #include "CTBException.hpp" #include "MeshTile.hpp" #include "BoundingSphere.hpp" +#include "CTBZOutputStream.hpp" using namespace ctb; @@ -117,7 +117,7 @@ static inline int quantizeIndices(const double &origin, const double &factor, co } // Write the edge indices of the mesh -template int writeEdgeIndices(gzFile terrainFile, const Mesh &mesh, double edgeCoord, int componentIndex) { +template int writeEdgeIndices(CTBOutputStream &ostream, const Mesh &mesh, double edgeCoord, int componentIndex) { std::vector indices; std::map ihash; @@ -136,11 +136,11 @@ template int writeEdgeIndices(gzFile terrainFile, const Mesh &mesh, } int edgeCount = indices.size(); - gzwrite(terrainFile, &edgeCount, sizeof(int)); + ostream.write(&edgeCount, sizeof(int)); for (size_t i = 0; i < edgeCount; i++) { T indice = (T)indices[i]; - gzwrite(terrainFile, &indice, sizeof(T)); + ostream.write(&indice, sizeof(T)); } return indices.size(); } @@ -207,11 +207,15 @@ MeshTile::MeshTile(const TileCoordinate &coord): */ void MeshTile::writeFile(const char *fileName, bool writeVertexNormals) const { - gzFile terrainFile = gzopen(fileName, "wb"); + CTBZFileOutputStream ostream(fileName); + writeFile(ostream); +} - if (terrainFile == NULL) { - throw CTBException("Failed to open file"); - } +/** + * @details This writes raw terrain data to an output stream. + */ +void +MeshTile::writeFile(CTBOutputStream &ostream, bool writeVertexNormals) const { // Calculate main header mesh data std::vector cartesianVertices; @@ -236,37 +240,37 @@ MeshTile::writeFile(const char *fileName, bool writeVertexNormals) const { double centerX = cartesianBounds.min.x + 0.5 * (cartesianBounds.max.x - cartesianBounds.min.x); double centerY = cartesianBounds.min.y + 0.5 * (cartesianBounds.max.y - cartesianBounds.min.y); double centerZ = cartesianBounds.min.z + 0.5 * (cartesianBounds.max.z - cartesianBounds.min.z); - gzwrite(terrainFile, ¢erX, sizeof(double)); - gzwrite(terrainFile, ¢erY, sizeof(double)); - gzwrite(terrainFile, ¢erZ, sizeof(double)); + ostream.write(¢erX, sizeof(double)); + ostream.write(¢erY, sizeof(double)); + ostream.write(¢erZ, sizeof(double)); // // The minimum and maximum heights in the area covered by this tile. float minimumHeight = (float)bounds.min.z; float maximumHeight = (float)bounds.max.z; - gzwrite(terrainFile, &minimumHeight, sizeof(float)); - gzwrite(terrainFile, &maximumHeight, sizeof(float)); + ostream.write(&minimumHeight, sizeof(float)); + ostream.write(&maximumHeight, sizeof(float)); // - // The tile’s bounding sphere. The X,Y,Z coordinates are again expressed + // The tile's bounding sphere. The X,Y,Z coordinates are again expressed // in Earth-centered Fixed coordinates, and the radius is in meters. double boundingSphereCenterX = cartesianBoundingSphere.center.x; double boundingSphereCenterY = cartesianBoundingSphere.center.y; double boundingSphereCenterZ = cartesianBoundingSphere.center.z; double boundingSphereRadius = cartesianBoundingSphere.radius; - gzwrite(terrainFile, &boundingSphereCenterX, sizeof(double)); - gzwrite(terrainFile, &boundingSphereCenterY, sizeof(double)); - gzwrite(terrainFile, &boundingSphereCenterZ, sizeof(double)); - gzwrite(terrainFile, &boundingSphereRadius , sizeof(double)); + ostream.write(&boundingSphereCenterX, sizeof(double)); + ostream.write(&boundingSphereCenterY, sizeof(double)); + ostream.write(&boundingSphereCenterZ, sizeof(double)); + ostream.write(&boundingSphereRadius , sizeof(double)); // // The horizon occlusion point, expressed in the ellipsoid-scaled Earth-centered Fixed frame. CRSVertex horizonOcclusionPoint = ocp_fromPoints(cartesianVertices, cartesianBoundingSphere); - gzwrite(terrainFile, &horizonOcclusionPoint.x, sizeof(double)); - gzwrite(terrainFile, &horizonOcclusionPoint.y, sizeof(double)); - gzwrite(terrainFile, &horizonOcclusionPoint.z, sizeof(double)); + ostream.write(&horizonOcclusionPoint.x, sizeof(double)); + ostream.write(&horizonOcclusionPoint.y, sizeof(double)); + ostream.write(&horizonOcclusionPoint.z, sizeof(double)); // # Write mesh vertices (X Y Z components of each vertex): int vertexCount = mMesh.vertices.size(); - gzwrite(terrainFile, &vertexCount, sizeof(int)); + ostream.write(&vertexCount, sizeof(int)); for (int c = 0; c < 3; c++) { double origin = bounds.min[c]; double factor = 0; @@ -275,20 +279,20 @@ MeshTile::writeFile(const char *fileName, bool writeVertexNormals) const { // Move the initial value int u0 = quantizeIndices(origin, factor, mMesh.vertices[0][c]), u1, ud; uint16_t sval = zigZagEncode(u0); - gzwrite(terrainFile, &sval, sizeof(uint16_t)); + ostream.write(&sval, sizeof(uint16_t)); for (size_t i = 1, icount = mMesh.vertices.size(); i < icount; i++) { u1 = quantizeIndices(origin, factor, mMesh.vertices[i][c]); ud = u1 - u0; sval = zigZagEncode(ud); - gzwrite(terrainFile, &sval, sizeof(uint16_t)); + ostream.write(&sval, sizeof(uint16_t)); u0 = u1; } } // # Write mesh indices: int triangleCount = mMesh.indices.size() / 3; - gzwrite(terrainFile, &triangleCount, sizeof(int)); + ostream.write(&triangleCount, sizeof(int)); if (vertexCount > BYTESPLIT) { uint32_t highest = 0; uint32_t code; @@ -296,15 +300,15 @@ MeshTile::writeFile(const char *fileName, bool writeVertexNormals) const { // Write main indices for (size_t i = 0, icount = mMesh.indices.size(); i < icount; i++) { code = highest - mMesh.indices[i]; - gzwrite(terrainFile, &code, sizeof(uint32_t)); + ostream.write(&code, sizeof(uint32_t)); if (code == 0) highest++; } // Write all vertices on the edge of the tile (W, S, E, N) - writeEdgeIndices(terrainFile, mMesh, bounds.min.x, 0); - writeEdgeIndices(terrainFile, mMesh, bounds.min.y, 1); - writeEdgeIndices(terrainFile, mMesh, bounds.max.x, 0); - writeEdgeIndices(terrainFile, mMesh, bounds.max.y, 1); + writeEdgeIndices(ostream, mMesh, bounds.min.x, 0); + writeEdgeIndices(ostream, mMesh, bounds.min.y, 1); + writeEdgeIndices(ostream, mMesh, bounds.max.x, 0); + writeEdgeIndices(ostream, mMesh, bounds.max.y, 1); } else { uint16_t highest = 0; @@ -313,23 +317,23 @@ MeshTile::writeFile(const char *fileName, bool writeVertexNormals) const { // Write main indices for (size_t i = 0, icount = mMesh.indices.size(); i < icount; i++) { code = highest - mMesh.indices[i]; - gzwrite(terrainFile, &code, sizeof(uint16_t)); + ostream.write(&code, sizeof(uint16_t)); if (code == 0) highest++; } // Write all vertices on the edge of the tile (W, S, E, N) - writeEdgeIndices(terrainFile, mMesh, bounds.min.x, 0); - writeEdgeIndices(terrainFile, mMesh, bounds.min.y, 1); - writeEdgeIndices(terrainFile, mMesh, bounds.max.x, 0); - writeEdgeIndices(terrainFile, mMesh, bounds.max.y, 1); + writeEdgeIndices(ostream, mMesh, bounds.min.x, 0); + writeEdgeIndices(ostream, mMesh, bounds.min.y, 1); + writeEdgeIndices(ostream, mMesh, bounds.max.x, 0); + writeEdgeIndices(ostream, mMesh, bounds.max.y, 1); } // # Write 'Oct-Encoded Per-Vertex Normals' for Terrain Lighting: if (writeVertexNormals && triangleCount > 0) { unsigned char extensionId = 1; - gzwrite(terrainFile, &extensionId, sizeof(unsigned char)); + ostream.write(&extensionId, sizeof(unsigned char)); int extensionLength = 2 * vertexCount; - gzwrite(terrainFile, &extensionLength, sizeof(int)); + ostream.write(&extensionLength, sizeof(int)); std::vector normalsPerVertex(vertexCount); std::vector normalsPerFace(triangleCount); @@ -358,22 +362,10 @@ MeshTile::writeFile(const char *fileName, bool writeVertexNormals) const { } for (size_t i = 0; i < vertexCount; i++) { Coordinate xy = octEncode(normalsPerVertex[i].normalize()); - gzwrite(terrainFile, &xy.x, sizeof(unsigned char)); - gzwrite(terrainFile, &xy.y, sizeof(unsigned char)); + ostream.write(&xy.x, sizeof(unsigned char)); + ostream.write(&xy.y, sizeof(unsigned char)); } } - - // Try and close the file - switch (gzclose(terrainFile)) { - case Z_OK: - break; - case Z_STREAM_ERROR: - case Z_ERRNO: - case Z_MEM_ERROR: - case Z_BUF_ERROR: - default: - throw CTBException("Failed to close file"); - } } bool diff --git a/src/MeshTile.hpp b/src/MeshTile.hpp index 5cd3fc9..2300c37 100644 --- a/src/MeshTile.hpp +++ b/src/MeshTile.hpp @@ -27,6 +27,7 @@ #include "Mesh.hpp" #include "TileCoordinate.hpp" #include "Tile.hpp" +#include "CTBOutputStream.hpp" namespace ctb { class MeshTile; @@ -55,6 +56,10 @@ class CTB_DLL ctb::MeshTile : void writeFile(const char *fileName, bool writeVertexNormals = false) const; + /// Write terrain data to an output stream + void + writeFile(CTBOutputStream &ostream, bool writeVertexNormals = false) const; + /// Does the terrain tile have child tiles? bool hasChildren() const; diff --git a/src/TerrainSerializer.hpp b/src/TerrainSerializer.hpp new file mode 100644 index 0000000..e21a7fc --- /dev/null +++ b/src/TerrainSerializer.hpp @@ -0,0 +1,50 @@ +#ifndef TERRAINSERIALIZER_HPP +#define TERRAINSERIALIZER_HPP + +/******************************************************************************* + * Copyright 2018 GeoData + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + *******************************************************************************/ + +/** + * @file TerrainSerializer.hpp + * @brief This declares and defines the `TerrainSerializer` class + */ + +#include "config.hpp" +#include "TileCoordinate.hpp" +#include "TerrainTile.hpp" + +namespace ctb { + class TerrainSerializer; +} + +/// Store `TerrainTile`s from a GDAL Dataset +class CTB_DLL ctb::TerrainSerializer { +public: + + /// Start a new serialization task + virtual void startSerialization() = 0; + + /// Returns if the specified Tile Coordinate should be serialized + virtual bool mustSerializeCoordinate(const ctb::TileCoordinate *coordinate, const char *extension = NULL) = 0; + + /// Serialize a TerrainTile to the store + virtual bool serializeTile(const ctb::TerrainTile *tile) = 0; + + /// Serialization finished, releases any resources loaded + virtual void endSerialization() = 0; +}; + +#endif /* TERRAINSERIALIZER_HPP */ diff --git a/src/TerrainTile.cpp b/src/TerrainTile.cpp index 7b78262..393765c 100644 --- a/src/TerrainTile.cpp +++ b/src/TerrainTile.cpp @@ -28,6 +28,8 @@ #include "TerrainTile.hpp" #include "GlobalGeodetic.hpp" #include "Bounds.hpp" +#include "CTBFileOutputStream.hpp" +#include "CTBZOutputStream.hpp" using namespace ctb; @@ -136,9 +138,8 @@ Terrain::readFile(const char *fileName) { */ void Terrain::writeFile(FILE *fp) const { - fwrite(mHeights.data(), TILE_CELL_SIZE * 2, 1, fp); - fwrite(&mChildren, 1, 1, fp); - fwrite(mMask, mMaskLength, 1, fp); + CTBFileOutputStream ostream(fp); + writeFile(ostream); } /** @@ -146,41 +147,30 @@ Terrain::writeFile(FILE *fp) const { */ void Terrain::writeFile(const char *fileName) const { - gzFile terrainFile = gzopen(fileName, "wb"); + CTBZFileOutputStream ostream(fileName); + writeFile(ostream); +} - if (terrainFile == NULL) { - throw CTBException("Failed to open file"); - } +/** + * @details This writes raw terrain data to an output stream. + */ +void +Terrain::writeFile(CTBOutputStream &ostream) const { // Write the height data - if (gzwrite(terrainFile, mHeights.data(), TILE_CELL_SIZE * 2) == 0) { - gzclose(terrainFile); + if (ostream.write((const void *)mHeights.data(), TILE_CELL_SIZE * 2) != TILE_CELL_SIZE * 2) { throw CTBException("Failed to write height data"); } // Write the child flags - if (gzputc(terrainFile, mChildren) == -1) { - gzclose(terrainFile); + if (ostream.write(&mChildren, 1) != 1) { throw CTBException("Failed to write child flags"); } // Write the water mask - if (gzwrite(terrainFile, mMask, mMaskLength) == 0) { - gzclose(terrainFile); + if (ostream.write(&mMask, mMaskLength) != mMaskLength) { throw CTBException("Failed to write water mask"); } - - // Try and close the file - switch (gzclose(terrainFile)) { - case Z_OK: - break; - case Z_STREAM_ERROR: - case Z_ERRNO: - case Z_MEM_ERROR: - case Z_BUF_ERROR: - default: - throw CTBException("Failed to close file"); - } } std::vector diff --git a/src/TerrainTile.hpp b/src/TerrainTile.hpp index 5941166..ce739ad 100644 --- a/src/TerrainTile.hpp +++ b/src/TerrainTile.hpp @@ -29,6 +29,7 @@ #include "config.hpp" #include "Tile.hpp" #include "TileCoordinate.hpp" +#include "CTBOutputStream.hpp" namespace ctb { class Terrain; @@ -65,6 +66,10 @@ class CTB_DLL ctb::Terrain { void writeFile(const char *fileName) const; + /// Write terrain data to an output stream + void + writeFile(CTBOutputStream &ostream) const; + /// Get the water mask as a boolean mask std::vector mask(); diff --git a/tools/ctb-tile.cpp b/tools/ctb-tile.cpp index 3851ba0..dcc7fee 100644 --- a/tools/ctb-tile.cpp +++ b/tools/ctb-tile.cpp @@ -52,6 +52,7 @@ #include "TerrainIterator.hpp" #include "MeshIterator.hpp" #include "GDALDatasetReader.hpp" +#include "CTBFileTileSerializer.hpp" using namespace std; using namespace ctb; @@ -245,52 +246,6 @@ class TerrainBuild : public Command { bool vertexNormals; }; -/** - * Create a filename for a tile coordinate - * - * This also creates the tile directory structure. - */ -static string -getTileFilename(const TileCoordinate *coord, const string dirname, const char *extension) { - static mutex mutex; - VSIStatBufL stat; - string filename = concat(dirname, coord->zoom, osDirSep, coord->x); - - lock_guard lock(mutex); - - // Check whether the `{zoom}/{x}` directory exists or not - if (VSIStatExL(filename.c_str(), &stat, VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG)) { - filename = concat(dirname, coord->zoom); - - // Check whether the `{zoom}` directory exists or not - if (VSIStatExL(filename.c_str(), &stat, VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG)) { - // Create the `{zoom}` directory - if (VSIMkdir(filename.c_str(), 0755)) - throw CTBException("Could not create the zoom level directory"); - - } else if (!VSI_ISDIR(stat.st_mode)) { - throw CTBException("Zoom level file path is not a directory"); - } - - // Create the `{zoom}/{x}` directory - filename += concat(osDirSep, coord->x); - if (VSIMkdir(filename.c_str(), 0755)) - throw CTBException("Could not create the x level directory"); - - } else if (!VSI_ISDIR(stat.st_mode)) { - throw CTBException("X level file path is not a directory"); - } - - // Create the filename itself, adding the extension if required - filename += concat(osDirSep, coord->y); - if (extension != NULL) { - filename += "."; - filename += extension; - } - - return filename; -} - /** * Increment a TilerIterator whilst cooperating between threads * @@ -363,6 +318,10 @@ showProgress(int currentIndex, string filename) { return progressFunc(currentIndex / (double) iteratorSize, message.c_str(), NULL); } +int +showProgress(int currentIndex) { + return progressFunc(currentIndex / (double) iteratorSize, NULL, NULL); +} static bool fileExists(const std::string& filename) { @@ -541,7 +500,7 @@ class TerrainMetadata { /// Output GDAL tiles represented by a tiler to a directory static void -buildGDAL(const RasterTiler &tiler, TerrainBuild *command, TerrainMetadata *metadata) { +buildGDAL(GDALSerializer &serializer, const RasterTiler &tiler, TerrainBuild *command, TerrainMetadata *metadata) { GDALDriver *poDriver = GetGDALDriverManager()->GetDriverByName(command->outputFormat); if (poDriver == NULL) { @@ -553,7 +512,6 @@ buildGDAL(const RasterTiler &tiler, TerrainBuild *command, TerrainMetadata *meta } const char *extension = poDriver->GetMetadataItem(GDAL_DMD_EXTENSION); - const string dirname = string(command->outputDir) + osDirSep; i_zoom startZoom = (command->startZoom < 0) ? tiler.maxZoomLevel() : command->startZoom, endZoom = (command->endZoom < 0) ? 0 : command->endZoom; @@ -563,38 +521,22 @@ buildGDAL(const RasterTiler &tiler, TerrainBuild *command, TerrainMetadata *meta while (!iter.exhausted()) { const TileCoordinate *coordinate = iter.GridIterator::operator*(); - GDALDataset *poDstDS; - const string filename = getTileFilename(coordinate, dirname, extension); if (metadata) metadata->add(tiler.grid(), coordinate); - if( !command->resume || !fileExists(filename) ) { + if (serializer.mustSerializeCoordinate(coordinate, extension)) { GDALTile *tile = *iter; - const string temp_filename = concat(filename, ".tmp"); - poDstDS = poDriver->CreateCopy(temp_filename.c_str(), tile->dataset, FALSE, - command->creationOptions.List(), NULL, NULL ); + serializer.serializeTile(tile, poDriver, extension, command->creationOptions); delete tile; - - // Close the datasets, flushing data to destination - if (poDstDS == NULL) { - throw CTBException("Could not create GDAL tile"); - } - - GDALClose(poDstDS); - - if (VSIRename(temp_filename.c_str(), filename.c_str()) != 0) { - throw new CTBException("Could not rename temporary file"); - } } currentIndex = incrementIterator(iter, currentIndex); - showProgress(currentIndex, filename); + showProgress(currentIndex); } } /// Output terrain tiles represented by a tiler to a directory static void -buildTerrain(const TerrainTiler &tiler, TerrainBuild *command, TerrainMetadata *metadata) { - const string dirname = string(command->outputDir) + osDirSep; +buildTerrain(TerrainSerializer &serializer, const TerrainTiler &tiler, TerrainBuild *command, TerrainMetadata *metadata) { i_zoom startZoom = (command->startZoom < 0) ? tiler.maxZoomLevel() : command->startZoom, endZoom = (command->endZoom < 0) ? 0 : command->endZoom; @@ -605,35 +547,28 @@ buildTerrain(const TerrainTiler &tiler, TerrainBuild *command, TerrainMetadata * while (!iter.exhausted()) { const TileCoordinate *coordinate = iter.GridIterator::operator*(); - const string filename = getTileFilename(coordinate, dirname, "terrain"); if (metadata) metadata->add(tiler.grid(), coordinate); - if( !command->resume || !fileExists(filename) ) { + if (serializer.mustSerializeCoordinate(coordinate)) { TerrainTile *tile = iter.operator*(&reader); - const string temp_filename = concat(filename, ".tmp"); - - tile->writeFile(temp_filename.c_str()); + serializer.serializeTile(tile); delete tile; - - if (VSIRename(temp_filename.c_str(), filename.c_str()) != 0) { - throw new CTBException("Could not rename temporary file"); - } } currentIndex = incrementIterator(iter, currentIndex); - showProgress(currentIndex, filename); + showProgress(currentIndex); } } /// Output mesh tiles represented by a tiler to a directory static void -buildMesh(const MeshTiler &tiler, TerrainBuild *command, TerrainMetadata *metadata, bool writeVertexNormals = false) { - const string dirname = string(command->outputDir) + osDirSep; +buildMesh(MeshSerializer &serializer, const MeshTiler &tiler, TerrainBuild *command, TerrainMetadata *metadata, bool writeVertexNormals = false) { i_zoom startZoom = (command->startZoom < 0) ? tiler.maxZoomLevel() : command->startZoom, endZoom = (command->endZoom < 0) ? 0 : command->endZoom; // DEBUG Chunker: #if 0 + const string dirname = string(command->outputDir) + osDirSep; TileCoordinate coordinate(13, 8102, 6047); MeshTile *tile = tiler.createMesh(tiler.dataset(), coordinate); // @@ -660,23 +595,16 @@ buildMesh(const MeshTiler &tiler, TerrainBuild *command, TerrainMetadata *metada while (!iter.exhausted()) { const TileCoordinate *coordinate = iter.GridIterator::operator*(); - const string filename = getTileFilename(coordinate, dirname, "terrain"); if (metadata) metadata->add(tiler.grid(), coordinate); - if( !command->resume || !fileExists(filename) ) { + if (serializer.mustSerializeCoordinate(coordinate)) { MeshTile *tile = iter.operator*(&reader); - const string temp_filename = concat(filename, ".tmp"); - - tile->writeFile(temp_filename.c_str(), writeVertexNormals); + serializer.serializeTile(tile, writeVertexNormals); delete tile; - - if (VSIRename(temp_filename.c_str(), filename.c_str()) != 0) { - throw new CTBException("Could not rename temporary file"); - } } currentIndex = incrementIterator(iter, currentIndex); - showProgress(currentIndex, filename); + showProgress(currentIndex); } } @@ -717,24 +645,30 @@ runTiler(TerrainBuild *command, Grid *grid, TerrainMetadata *metadata) { // Metadata of only this thread, it will be joined to global later TerrainMetadata *threadMetadata = metadata ? new TerrainMetadata() : NULL; + // Choose serializer of tiles (Directory of files, MBTiles store...) + CTBFileTileSerializer serializer(string(command->outputDir) + osDirSep, command->resume); + try { + serializer.startSerialization(); + if (command->metadata) { const RasterTiler tiler(poDataset, *grid, command->tilerOptions); buildMetadata(tiler, command, threadMetadata); } else if (strcmp(command->outputFormat, "Terrain") == 0) { const TerrainTiler tiler(poDataset, *grid); - buildTerrain(tiler, command, threadMetadata); + buildTerrain(serializer, tiler, command, threadMetadata); } else if (strcmp(command->outputFormat, "Mesh") == 0) { const MeshTiler tiler(poDataset, *grid, command->tilerOptions, command->meshQualityFactor); - buildMesh(tiler, command, threadMetadata, command->vertexNormals); + buildMesh(serializer, tiler, command, threadMetadata, command->vertexNormals); } else { // it's a GDAL format const RasterTiler tiler(poDataset, *grid, command->tilerOptions); - buildGDAL(tiler, command, threadMetadata); + buildGDAL(serializer, tiler, command, threadMetadata); } } catch (CTBException &e) { cerr << "Error: " << e.what() << endl; } + serializer.endSerialization(); GDALClose(poDataset);