Resolve file write race condition, batch frequent tile updates

src/mapview/tileDataEventHandler.js

@@ -2,14 +2,20 @@
 const sharp = require("sharp");
 const path = require("path");
 sharp.cache(false);
-const messages = require("../../messages");
+const sleep = require("../util/sleep");
+
+const fileLocks = {}; // Used to prevent multiple writes to the same file
+const updates = new Map();
 
 module.exports = async function tileDataEventHandler({ type, data, size, position }) {
 	// console.log(type, size, position);
-	const updates = new Map();
 	// Image tiles are 512x512 pixels arranged in a grid, starting at 0,0
 	const TILE_SIZE = 512;
 	if (type === "pixels") {
+		if (data.length % 3 !== 0) {
+			this.logger.error(`Invalid pixel data length: ${data.length}`);
+			return;
+		}
 		for (let i = 0; i < data.length; i += 3) {
 			const x = Math.floor(data[i]); // Convert from string and strip decimals
 			const y = Math.floor(data[i + 1]); // Convert from string and strip decimals
@@ -56,44 +62,71 @@ module.exports = async function tileDataEventHandler({ type, data, size, positio
 			]);
 		}
 	}
+
 	// Perform image updates
 	for (const [filename, pixels] of updates) {
-		// console.log("updating", filename);
-		// Check if image exists, otherwise create one
-		let raw;
-		try {
-			raw = await sharp(path.resolve(this._tilesPath, filename))
-				.raw()
-				.toBuffer();
-		} catch (e) {
-			raw = await sharp({
-				create: {
+		// Wait for previous calls to complete before we take over the file locks
+		const imagePath = path.resolve(this._tilesPath, filename);
+		if (!fileLocks[imagePath]) {
+			fileLocks[imagePath] = [];
+		}
+		// Already waiting, cancel
+		if (fileLocks[imagePath].length > 1) {
+			return;
+		}
+		if (fileLocks[imagePath].length > 0) {
+			// const timer = `Waiting for tile lock ${filename}`;
+			// console.time(timer);
+			fileLocks[imagePath].push(true); // Since we are waiting, we don't need to let later calls also wait
+			await Promise.all(fileLocks[imagePath]);
+			// console.timeEnd(timer);
+		}
+		fileLocks[imagePath] = []; // No need to leak memory
+		fileLocks[imagePath].push(new Promise(async (resolve) => {
+			// Check if image exists, otherwise create one
+			let raw;
+			try {
+				raw = await sharp(imagePath)
+					.raw()
+					.toBuffer();
+			} catch (e) {
+				raw = await sharp({
+					create: {
+						width: TILE_SIZE,
+						height: TILE_SIZE,
+						channels: 4,
+						background: { r: 20, g: 20, b: 20, alpha: 0 },
+					},
+				})
+					.raw()
+					.toBuffer();
+			}
+			// Write pixels
+			for (const [x, y, rgba] of pixels) {
+				const index = (y * TILE_SIZE + x) * 4;
+				raw[index] = rgba[0];
+				raw[index + 1] = rgba[1];
+				raw[index + 2] = rgba[2];
+				raw[index + 3] = rgba[3];
+			}
+			// Clear processed pixels. MUST be done before further async operations
+			pixels.clear();
+			// Convert back to sharp and write to file
+			await sharp(raw, {
+				raw: {
 					width: TILE_SIZE,
 					height: TILE_SIZE,
 					channels: 4,
-					background: { r: 20, g: 20, b: 20, alpha: 0 },
 				},
 			})
-				.raw()
-				.toBuffer();
-		}
-		// Write pixels
-		for (const [x, y, rgba] of pixels) {
-			const index = (y * TILE_SIZE + x) * 4;
-			raw[index] = rgba[0];
-			raw[index + 1] = rgba[1];
-			raw[index + 2] = rgba[2];
-			raw[index + 3] = rgba[3];
-		}
-		// Convert back to sharp and write to file
-		await sharp(raw, {
-			raw: {
-				width: TILE_SIZE,
-				height: TILE_SIZE,
-				channels: 4,
-			},
-		})
-			.png()
-			.toFile(path.resolve(this._tilesPath, filename));
+				.png()
+				.toFile(imagePath);
+			await sleep(250); // Reduce IOPS a bit by batching updates
+			resolve();
+			if (fileLocks[imagePath].length > 0) {
+				// Nothing is waiting, clear the lock queue
+				fileLocks[imagePath] = [];
+			}
+		}));
 	}
 };