Implement --input-tileset (gbdev#1464)

As discussed in gbdev#575 (comment)

contrib/bash_compl/_rgbgfx.bash

@@ -21,6 +21,7 @@ _rgbgfx_completions() {
 		[b]="base-tiles:unk"
 		[c]="colors:unk"
 		[d]="depth:unk"
+		[i]="input-tileset:glob-*.2bpp"
 		[L]="slice:unk"
 		[N]="nb-tiles:unk"
 		[n]="nb-palettes:unk"

contrib/zsh_compl/_rgbgfx

@@ -30,6 +30,7 @@ local args=(
 	'(-b --base-tiles)'{-b,--base-tiles}'+[Base tile IDs for tile map output]:base tile IDs:'
 	'(-c --colors)'{-c,--colors}'+[Specify color palettes]:palette spec:'
 	'(-d --depth)'{-d,--depth}'+[Set bit depth]:bit depth:_depths'
+	'(-i --input-tileset)'{-i,--input-tileset}'+[Use specific tiles]:tileset file:_files -g "*.2bpp"'
 	'(-L --slice)'{-L,--slice}'+[Only process a portion of the image]:input slice:'
 	'(-N --nb-tiles)'{-N,--nb-tiles}'+[Limit number of tiles]:tile count:'
 	'(-n --nb-palettes)'{-n,--nb-palettes}'+[Limit number of palettes]:palette count:'

include/gfx/main.hpp

@@ -28,7 +28,8 @@ struct Options {
 		EMBEDDED,
 	} palSpecType = NO_SPEC; // -c
 	std::vector<std::array<std::optional<Rgba>, 4>> palSpec{};
-	uint8_t bitDepth = 2; // -d
+	uint8_t bitDepth = 2;       // -d
+	std::string inputTileset{}; // -i
 	struct {
 		uint16_t left;
 		uint16_t top;

man/rgbgfx.1

@@ -16,6 +16,7 @@
 .Op Fl b Ar base_ids
 .Op Fl c Ar pal_spec
 .Op Fl d Ar depth
+.Op Fl i Ar input_tiles
 .Op Fl L Ar slice
 .Op Fl N Ar nb_tiles
 .Op Fl n Ar nb_pals
@@ -164,6 +165,57 @@ for a list of formats and their descriptions.
 .It Fl d Ar depth , Fl \-depth Ar depth
 Set the bit depth of the output tile data, in bits per pixel (bpp), either 1 or 2 (the default).
 This changes how tile data is output, and the maximum number of colors per palette (2 and 4 respectively).
+.It Fl i Ar input_tiles , Fl \-input-tileset Ar input_tiles
+Use the specified input tiles in addition to having
+.Nm
+automatically determine some.
+The input tiles will always be first in the
+.Fl o
+image output, and will always get the first IDs in the
+.Fl t
+tilemap output.
+.Ar input_tiles
+must contain 1bpp or 2bpp tile data
+.Pq whichever matches the Fl d No option used here ,
+as could be previously generated with the
+.Fl o
+option.
+.Pp
+If the
+.Fl o
+option is also specified, then the input tiles will be assigned the first tile IDs, and any tiles from the input image that are not in the input tileset will be assigned subsequent IDs.
+But if the
+.Fl o
+option is
+.Em not
+specified, then the tile map can
+.Em only
+use tiles from the input tileset.
+Using
+.Fl o
+with
+.Fl i
+is useful if you want to precisely control the tile IDs of its tile map.
+Using
+.Fl i
+alone is more useful if you want several images to use a subset of shared tiles.
+.Pp
+If the image will use more than one color palette, it is
+.Em strongly
+advised to generate the palette set along with the input tile data, and pass
+.Fl c Cm gbc: Ns Ar input_palette
+along with
+.Fl i Ar input_tiles .
+This is because
+.Nm
+might not generate the same palette set for this image as it did for its input tileset.
+.Pp
+See
+.Sx EXAMPLES
+for examples of how to use this option.
+.Pp
+This option is ignored in
+.Sx REVERSE MODE .
 .It Fl L Ar slice , Fl \-slice Ar slice
 Only process a given rectangle of the image.
 This is useful for example if the input image is a sheet of some sort, and you want to convert each cel individually.
@@ -637,7 +689,13 @@ without needing an input image.
 .Pp
 .Dl $ rgbgfx -c '#fff,#ff0,#f80,#000' -p colors.pal
 .Pp
-TODO: more examples.
+The following will convert two level images using the same tileset, and error out if any of them contain tiles not in the tileset.
+.Pp
+.Bd -literal -offset Ds
+$ rgbgfx tileset.png -o tileset.2bpp -O -P
+$ rgbgfx level1.png -i tileset.2bpp -c gbc:tileset.pal -t level1.tilemap -a level1.attrmap
+$ rgbgfx level2.png -i tileset.2bpp -c gbc:tileset.pal -t level2.tilemap -a level2.attrmap
+.Ed
 .Sh BUGS
 Please report bugs and mistakes in this man page on
 .Lk https://github.com/gbdev/rgbds/issues GitHub .

src/gfx/main.cpp

@@ -108,7 +108,7 @@ void Options::verbosePrint(uint8_t level, char const *fmt, ...) const {
 }
 
 // Short options
-static char const *optstring = "-Aa:b:Cc:Dd:FfhL:mN:n:Oo:Pp:Qq:r:s:Tt:U:uVvx:Z";
+static char const *optstring = "-Aa:b:Cc:Dd:Ffhi:L:mN:n:Oo:Pp:Qq:r:s:Tt:U:uVvx:Z";
 
 /*
  * Equivalent long options
@@ -127,6 +127,7 @@ static option const longopts[] = {
     {"color-curve",      no_argument,       nullptr, 'C'},
     {"colors",           required_argument, nullptr, 'c'},
     {"depth",            required_argument, nullptr, 'd'},
+    {"input-tileset",    required_argument, nullptr, 'i'},
     {"slice",            required_argument, nullptr, 'L'},
     {"mirror-tiles",     no_argument,       nullptr, 'm'},
     {"nb-tiles",         required_argument, nullptr, 'N'},
@@ -154,9 +155,10 @@ static option const longopts[] = {
 static void printUsage() {
 	fputs(
 	    "Usage: rgbgfx [-r stride] [-CmOuVXYZ] [-v [-v ...]] [-a <attr_map> | -A]\n"
-	    "       [-b <base_ids>] [-c <colors>] [-d <depth>] [-L <slice>] [-N <nb_tiles>]\n"
-	    "       [-n <nb_pals>] [-o <out_file>] [-p <pal_file> | -P] [-q <pal_map> | -Q]\n"
-	    "       [-s <nb_colors>] [-t <tile_map> | -T] [-x <nb_tiles>] <file>\n"
+	    "       [-b <base_ids>] [-c <colors>] [-d <depth>] [-i <tileset_file>]\n"
+	    "       [-L <slice>] [-N <nb_tiles>] [-n <nb_pals>] [-o <out_file>]\n"
+	    "       [-p <pal_file> | -P] [-q <pal_map> | -Q] [-s <nb_colors>]\n"
+	    "       [-t <tile_map> | -T] [-x <nb_tiles>] <file>\n"
 	    "Useful options:\n"
 	    "    -m, --mirror-tiles    optimize out mirrored tiles\n"
 	    "    -o, --output <path>   output the tile data to this path\n"
@@ -427,6 +429,11 @@ static char *parseArgv(int argc, char *argv[]) {
 				options.bitDepth = 2;
 			}
 			break;
+		case 'i':
+			if (!options.inputTileset.empty())
+				warning("Overriding input tileset file %s", options.inputTileset.c_str());
+			options.inputTileset = musl_optarg;
+			break;
 		case 'L':
 			options.inputSlice.left = parseNumber(arg, "Input slice left coordinate");
 			if (options.inputSlice.left > INT16_MAX) {

src/gfx/process.cpp

@@ -692,7 +692,7 @@ static void outputPalettes(std::vector<Palette> const &palettes) {
 	if (!options.palettes.empty()) {
 		File output;
 		if (!output.open(options.palettes, std::ios_base::out | std::ios_base::binary)) {
-			fatal("Failed to open \"%s\": %s", output.c_str(options.palettes), strerror(errno));
+			fatal("Failed to create \"%s\": %s", output.c_str(options.palettes), strerror(errno));
 		}
 
 		for (Palette const &palette : palettes) {
@@ -706,6 +706,17 @@ static void outputPalettes(std::vector<Palette> const &palettes) {
 	}
 }
 
+static void hashBitplanes(uint16_t bitplanes, uint16_t &hash) {
+	hash ^= bitplanes;
+	if (options.allowMirroringX) {
+		// Count the line itself as mirrored, which ensures the same hash as the tile's horizontal
+		// flip; vertical mirroring is already taken care of because the symmetric line will be
+		// XOR'd the same way. (This can trivially create some collisions, but real-world tile data
+		// generally doesn't trigger them.)
+		hash ^= flipTable[bitplanes >> 8] << 8 | flipTable[bitplanes & 0xFF];
+	}
+}
+
 class TileData {
 	std::array<uint8_t, 16> _data;
 	// The hash is a bit lax: it's the XOR of all lines, and every other nibble is identical
@@ -736,23 +747,23 @@ class TileData {
 		return row;
 	}
 
+	TileData(std::array<uint8_t, 16> &&raw) : _data(raw), _hash(0) {
+		for (uint8_t y = 0; y < 8; ++y) {
+			uint16_t bitplanes = _data[y * 2] | _data[y * 2 + 1] << 8;
+			hashBitplanes(bitplanes, _hash);
+		}
+	}
+
 	TileData(Png::TilesVisitor::Tile const &tile, Palette const &palette) : _hash(0) {
 		size_t writeIndex = 0;
 		for (uint32_t y = 0; y < 8; ++y) {
 			uint16_t bitplanes = rowBitplanes(tile, palette, y);
+			hashBitplanes(bitplanes, _hash);
+
 			_data[writeIndex++] = bitplanes & 0xFF;
 			if (options.bitDepth == 2) {
 				_data[writeIndex++] = bitplanes >> 8;
 			}
-
-			// Update the hash
-			_hash ^= bitplanes;
-			if (options.allowMirroringX) {
-				// Count the line itself as mirrorred horizontally; vertical mirroring is already
-				// taken care of because the symmetric line will be XOR'd the same way.
-				// (This reduces the hash's efficiency, but seems benign with most real-world data.)
-				_hash ^= flipTable[bitplanes >> 8] << 8 | flipTable[bitplanes & 0xFF];
-			}
 		}
 	}
 
@@ -836,7 +847,7 @@ static void outputTileData(
 ) {
 	File output;
 	if (!output.open(options.output, std::ios_base::out | std::ios_base::binary)) {
-		fatal("Failed to open \"%s\": %s", output.c_str(options.output), strerror(errno));
+		fatal("Failed to create \"%s\": %s", output.c_str(options.output), strerror(errno));
 	}
 
 	uint16_t widthTiles = options.inputSlice.width ? options.inputSlice.width : png.getWidth() / 8;
@@ -875,7 +886,7 @@ static void outputMaps(
 		if (!path.empty()) {
 			file.emplace();
 			if (!file->open(path, std::ios_base::out | std::ios_base::binary)) {
-				fatal("Failed to open \"%s\": %s", file->c_str(options.tilemap), strerror(errno));
+				fatal("Failed to create \"%s\": %s", file->c_str(options.tilemap), strerror(errno));
 			}
 		}
 	};
@@ -923,12 +934,10 @@ struct UniqueTiles {
 	/*
 	 * Adds a tile to the collection, and returns its ID
 	 */
-	std::tuple<uint16_t, TileData::MatchType>
-	    addTile(Png::TilesVisitor::Tile const &tile, Palette const &palette) {
-		TileData newTile(tile, palette);
+	std::tuple<uint16_t, TileData::MatchType> addTile(TileData newTile) {
 		auto [tileData, inserted] = tileset.insert(newTile);
 
-		TileData::MatchType matchType = TileData::EXACT;
+		TileData::MatchType matchType = TileData::NOPE;
 		if (inserted) {
 			// Give the new tile the next available unique ID
 			tileData->tileID = static_cast<uint16_t>(tiles.size());
@@ -963,8 +972,57 @@ static UniqueTiles dedupTiles(
 	// by caching the full tile data anyway, so we might as well.)
 	UniqueTiles tiles;
 
+	if (!options.inputTileset.empty()) {
+		File inputTileset;
+		if (!inputTileset.open(options.inputTileset, std::ios::in | std::ios::binary)) {
+			fatal("Failed to open \"%s\": %s", options.inputTileset.c_str(), strerror(errno));
+		}
+
+		std::array<uint8_t, 16> tile;
+		size_t const tileSize = options.bitDepth * 8;
+		for (;;) {
+			// It's okay to cast between character types.
+			size_t len = inputTileset->sgetn(reinterpret_cast<char *>(tile.data()), tileSize);
+			if (len == 0) { // EOF!
+				break;
+			} else if (len != tileSize) {
+				fatal(
+				    "\"%s\" does not contain a multiple of %zu bytes; is it actually tile data?",
+				    options.inputTileset.c_str(),
+				    tileSize
+				);
+			} else if (len == 8) {
+				// Expand the tile data to 2bpp.
+				for (size_t i = 8; i--;) {
+					tile[i * 2 + 1] = 0;
+					tile[i * 2] = tile[i];
+				}
+			}
+
+			auto [tileID, matchType] = tiles.addTile(std::move(tile));
+
+			if (matchType != TileData::NOPE) {
+				error(
+				    "The input tileset's tile #%hu was deduplicated; please check that your "
+				    "deduplication flags (`-u`, `-m`) are consistent with what was used to "
+				    "generate the input tileset",
+				    tileID
+				);
+			}
+		}
+	}
+
 	for (auto [tile, attr] : zip(png.visitAsTiles(), attrmap)) {
-		auto [tileID, matchType] = tiles.addTile(tile, palettes[mappings[attr.protoPaletteID]]);
+		auto [tileID, matchType] = tiles.addTile({tile, palettes[mappings[attr.protoPaletteID]]});
+
+		if (matchType == TileData::NOPE && options.output.empty()) {
+			error(
+			    "Tile at (%" PRIu32 ", %" PRIu32
+			    ") is not within the input tileset, and `-o` was not given!",
+			    tile.x,
+			    tile.y
+			);
+		}
 
 		attr.xFlip = matchType == TileData::HFLIP || matchType == TileData::VHFLIP;
 		attr.yFlip = matchType == TileData::VFLIP || matchType == TileData::VHFLIP;
@@ -1186,6 +1244,12 @@ continue_visiting_tiles:;
 			);
 		}
 
+		// I currently cannot figure out useful semantics for this combination of flags.
+		if (!options.inputTileset.empty()) {
+			fatal("Input tilesets are not supported without `-u`\nPlease consider explaining your "
+			      "use case to RGBDS' developers!");
+		}
+
 		if (!options.output.empty()) {
 			options.verbosePrint(Options::VERB_LOG_ACT, "Generating unoptimized tile data...\n");
 			unoptimized::outputTileData(png, attrmap, palettes, mappings);

test/gfx/input_tileset.flags

@@ -0,0 +1,2 @@
+-i input_tileset.in.2bpp
+-u

test/gfx/input_tileset_deduped.err

@@ -0,0 +1,2 @@
+error: The input tileset's tile #7 was deduplicated; please check that your deduplication flags (`-u`, `-m`) are consistent with what was used to generate the input tileset
+Conversion aborted after 1 error

test/gfx/input_tileset_deduped.flags

@@ -0,0 +1,3 @@
+-i input_tileset_deduped.in.2bpp
+-u
+-o result.2bpp

test/gfx/input_tileset_extra.err

@@ -0,0 +1,4 @@
+error: Tile at (0, 0) is not within the input tileset, and `-o` was not given!
+error: Tile at (0, 8) is not within the input tileset, and `-o` was not given!
+error: Tile at (8, 8) is not within the input tileset, and `-o` was not given!
+Conversion aborted after 3 errors

test/gfx/input_tileset_extra.flags

@@ -0,0 +1,2 @@
+-i input_tileset.in.2bpp
+-u