feat: allow creating index files based on steam depot configs

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.24)
 
 project( verifier
 	DESCRIPTION "A tool used to verify a game's install."
-	VERSION 0.3.1
+	VERSION 0.3.2
 )
 set( CMAKE_CXX_STANDARD 20 )
 set( CMAKE_CXX_STANDARD_REQUIRED ON )
@@ -44,7 +44,7 @@ target_compile_definitions( ${PROJECT_NAME} PRIVATE $<$<CONFIG:Debug>:DEBUG> "VE
 
 # Link to CLI dependencies
 include( "${CMAKE_CURRENT_SOURCE_DIR}/src/thirdparty/CMakeLists.txt" )
-target_link_libraries( ${PROJECT_NAME} PRIVATE Argumentum::argumentum cryptopp::cryptopp fmt::fmt vpkpp)
+target_link_libraries( ${PROJECT_NAME} PRIVATE Argumentum::argumentum cryptopp::cryptopp fmt::fmt kvpp vpkpp)
 
 # Create GUI executable
 if( VERIFIER_BUILD_GUI )

src/create.cpp

@@ -14,7 +14,8 @@
 #include <cryptopp/filters.h>
 #include <cryptopp/hex.h>
 #include <cryptopp/sha.h>
-#include <sourcepp/parser/Text.h>
+#include <kvpp/kvpp.h>
+#include <sourcepp/fs/FS.h>
 #include <sourcepp/string/String.h>
 #include <vpkpp/format/VPK.h>
 
@@ -23,6 +24,7 @@
 static auto enterVPK( std::ofstream& writer, std::string_view vpkPath, std::string_view vpkPathRel, const std::vector<std::regex>& excludes, const std::vector<std::regex>& includes, unsigned int& count ) -> bool;
 static auto buildRegexCollection( const std::vector<std::string>& regexStrings, std::string_view collectionType ) -> std::vector<std::regex>;
 static auto matchPath( const std::string& path, const std::vector<std::regex>& regexes ) -> bool;
+static auto globToRegex( std::string_view glob ) -> std::string;
 
 auto createFromRoot( std::string_view root_, std::string_view indexLocation, bool skipArchives,
 					 const std::vector<std::string>& fileExcludes, const std::vector<std::string>& fileIncludes,
@@ -40,10 +42,10 @@ auto createFromRoot( std::string_view root_, std::string_view indexLocation, boo
 		return 1;
 	}
 
-	std::vector<std::regex> archiveExclusionREs = buildRegexCollection(archiveExcludes, "archive exclusion");
-	std::vector<std::regex> archiveInclusionREs = buildRegexCollection(archiveIncludes, "archive inclusion");
-	std::vector<std::regex> fileExclusionREs = buildRegexCollection(fileExcludes, "file exclusion");
-	std::vector<std::regex> fileInclusionREs = buildRegexCollection(fileIncludes, "file inclusion");
+	std::vector<std::regex> archiveExclusionREs = buildRegexCollection( archiveExcludes, "archive exclusion" );
+	std::vector<std::regex> archiveInclusionREs = buildRegexCollection( archiveIncludes, "archive inclusion" );
+	std::vector<std::regex> fileExclusionREs = buildRegexCollection( fileExcludes, "file exclusion" );
+	std::vector<std::regex> fileInclusionREs = buildRegexCollection( fileIncludes, "file inclusion" );
 
 	// We always pass some regexes in from main.cpp, so not need for an ugly check if we actually
 	// compiled anything - fileExclusionREs will always be non-empty.
@@ -76,22 +78,23 @@ auto createFromRoot( std::string_view root_, std::string_view indexLocation, boo
 				continue;
 			}
 
-			if ( !archiveExclusionREs.empty() && !matchPath( pathRel, archiveInclusionREs ) ) {
+			if ( !archiveInclusionREs.empty() && !matchPath( pathRel, archiveInclusionREs ) ) {
 				continue;
 			}
 
+
 			if ( enterVPK( writer, path, pathRel, fileExclusionREs, fileInclusionREs, count ) ) {
 				Log_Info( "Processed VPK at `{}`", path );
 				continue;
 			}
 
 			Log_Warn( "Unable to open VPK at `{}`. Treating as a regular file...", path );
 		} else {
-			if ( !fileInclusionREs.empty() && matchPath( pathRel, fileExclusionREs ) ) {
+			if ( !fileExclusionREs.empty() && matchPath( pathRel, fileExclusionREs ) ) {
 				continue;
 			}
 
-			if ( !fileExclusionREs.empty() && !matchPath( pathRel, fileInclusionREs ) ) {
+			if ( !fileInclusionREs.empty() && !matchPath( pathRel, fileInclusionREs ) ) {
 				continue;
 			}
 		}
@@ -118,7 +121,7 @@ auto createFromRoot( std::string_view root_, std::string_view indexLocation, boo
 		CryptoPP::SHA1 sha1er{};
 		CryptoPP::CRC32 crc32er{};
 
-		unsigned char buffer[2048];
+		unsigned char buffer[ 2048 ];
 		while ( auto bufCount = std::fread( buffer, 1, sizeof( buffer ), file ) ) {
 			sha1er.Update( buffer, bufCount );
 			crc32er.Update( buffer, bufCount );
@@ -149,6 +152,111 @@ auto createFromRoot( std::string_view root_, std::string_view indexLocation, boo
 	return 0;
 }
 
+auto createFromSteamDepotConfigs( const std::string& configPath, const std::vector<std::string>& depotIDs, std::string_view indexLocation,
+								  bool skipArchives, const std::vector<std::string>& fileExcludes, const std::vector<std::string>& fileIncludes,
+								  const std::vector<std::string>& archiveExcludes, const std::vector<std::string>& archiveIncludes ) -> int {
+	using namespace kvpp;
+
+	/*
+	 * We don't support the full depot config spec (most of it though!)
+	 *
+	 * Assumptions are as follows:
+	 *  - ContentRoot (wherever it appears) is relative
+	 *  - DepotBuildConfig/FileMapping/LocalPath is relative
+	 *  - DepotBuildConfig/FileMapping/LocalPath contains files directly under DepotBuildConfig/FileMapping/DepotPath
+	 *    (for example, LocalPath is ".\p2ce\x.txt" and DepotPath is ".\p2ce\")
+	 *  - DepotBuildConfig/FileMapping/Recursive is true when LocalPath is a directory
+	 */
+	Log_Warn( "The Steam depot config parser is incomplete and fine-tuned to work with Portal 2: Community Edition. Use at your own risk." );
+
+	auto start{ std::chrono::high_resolution_clock::now() };
+	unsigned int configs = 0;
+
+	if (! std::filesystem::exists( configPath ) ) {
+		Log_Error( "Depot config at `{}` does not exist!", configPath );
+		return 1;
+	}
+
+	KV1 configKeyvalues{ sourcepp::fs::readFileText( configPath ) };
+	if ( configKeyvalues.hasChild( "DepotBuildConfig" ) ) {
+		Log_Error( R"(Depot config at `{}` has root key "DepotBuildConfig". Use the config with the root key "AppBuild" instead.)", configPath );
+		return 1;
+	}
+
+	const auto& appBuildConfig = configKeyvalues[ "AppBuild" ];
+	if ( appBuildConfig.isInvalid() ) {
+		Log_Error( "Depot config at `{}` is invalid!", configPath );
+		return 1;
+	}
+
+	auto contentRoot{ std::filesystem::path{ configPath }.parent_path() / appBuildConfig[ "ContentRoot" ].getValue() };
+	const auto& depots = appBuildConfig[ "Depots" ];
+	for ( const auto& depot : depots.getChildren() ) {
+		if ( std::find( depotIDs.begin(), depotIDs.end(), depot.getKey() ) == depotIDs.end() ) {
+			continue;
+		}
+
+		const auto createFromSteamDepotConfig{ [ &configPath, &indexLocation, skipArchives, &fileExcludes, &contentRoot, &fileIncludes, &archiveExcludes, &archiveIncludes ]( const KV1Element& depotBuildConfig ) {
+			std::vector<std::string> exclusionRegexes;
+			exclusionRegexes.insert( exclusionRegexes.end(), fileExcludes.begin(), fileExcludes.end() );
+			for ( int i = 0; i < depotBuildConfig.getChildCount( "FileExclusion" ); i++ ) {
+				std::string exclusion{ depotBuildConfig( "FileExclusion", i ).getValue() };
+				sourcepp::string::normalizeSlashes( exclusion );
+				if ( exclusion.starts_with( "./" ) )
+					exclusion = exclusion.substr(2);
+
+				exclusionRegexes.emplace_back( globToRegex( exclusion ) );
+			}
+
+			std::vector<std::string> inclusionRegexes{ fileIncludes };
+			for ( int i = 0; i < depotBuildConfig.getChildCount( "FileMapping" ); i++ ) {
+				std::string inclusion{ depotBuildConfig( "FileMapping", i )[ "LocalPath" ].getValue() };
+				sourcepp::string::normalizeSlashes( inclusion );
+				if ( inclusion.starts_with( "./" ) )
+					inclusion = inclusion.substr(2);
+
+				inclusionRegexes.emplace_back( globToRegex( inclusion ) );
+			}
+
+			createFromRoot(
+				depotBuildConfig.hasChild( "ContentRoot" ) ? ( std::filesystem::path{ configPath }.parent_path() / depotBuildConfig[ "ContentRoot" ].getValue() ).string() : contentRoot.string(),
+				indexLocation,
+				skipArchives,
+				exclusionRegexes,
+				inclusionRegexes,
+				archiveExcludes,
+				archiveIncludes
+			);
+		} };
+
+		if ( depot.getChildCount() > 0 ) {
+			createFromSteamDepotConfig( depot );
+		} else {
+			auto depotPath{ ( std::filesystem::path{ configPath }.parent_path() / depot.getValue() ).string() };
+			if ( !std::filesystem::exists( depotPath ) ) {
+				Log_Error( "Failed to load depot with ID `{}`: could not find depot config file at `{}`!", depot.getKey(), depotPath );
+				continue;
+			}
+
+			KV1 depotBuildConfigKeyvalues{ sourcepp::fs::readFileText( depotPath ) };
+
+			const auto& depotBuildConfig = depotBuildConfigKeyvalues[ "DepotBuildConfig" ];
+			if ( depotBuildConfig.isInvalid() ) {
+				Log_Error( "Depot config with ID `{}` at `{}` is invalid!", depot.getKey(), depotPath );
+				continue;
+			}
+
+			createFromSteamDepotConfig( depotBuildConfig );
+		}
+
+		Log_Info( "Finished processing depot with ID `{}`.", depot.getKey() );
+		configs++;
+	}
+
+	Log_Info( "Finished processing {} depot configs in {}.", configs, std::chrono::duration_cast<std::chrono::seconds>( std::chrono::high_resolution_clock::now() - start ) );
+	return 0;
+}
+
 static auto enterVPK( std::ofstream& writer, std::string_view vpkPath, std::string_view vpkPathRel, const std::vector<std::regex>& excludes, const std::vector<std::regex>& includes, unsigned int& count ) -> bool {
 	using namespace vpkpp;
 
@@ -197,7 +305,7 @@ static auto enterVPK( std::ofstream& writer, std::string_view vpkPath, std::stri
 }
 
 static auto buildRegexCollection( const std::vector<std::string>& regexStrings, std::string_view collectionType ) -> std::vector<std::regex> {
-	std::vector<std::regex> collection {};
+	std::vector<std::regex> collection{};
 
 	if ( !regexStrings.empty() ) {
 		Log_Info( "Compiling {} regexes...", collectionType );
@@ -216,3 +324,23 @@ static auto matchPath( const std::string& path, const std::vector<std::regex>& r
 		return std::regex_match( path, item );
 	});
 }
+
+static auto globToRegex( std::string_view glob ) -> std::string {
+	static constexpr std::string_view SPECIAL_CHARS{ R"(.+^$()[]{}|-+:'"<>\#&!)" };
+
+	std::string out;
+	for ( char c : glob ) {
+		if ( SPECIAL_CHARS.find( c ) != std::string_view::npos ) {
+			out += '\\';
+			out += c;
+		} else if ( c == '?' ) {
+			out += '.';
+		} else if ( c == '*' ) {
+			out += ".*?";
+		} else {
+			out += c;
+		}
+	}
+
+	return out;
+}

src/create.hpp

@@ -3,9 +3,14 @@
 //
 #pragma once
 
+#include <string>
 #include <string_view>
 #include <vector>
 
 auto createFromRoot( std::string_view root_, std::string_view indexLocation, bool skipArchives,
 					 const std::vector<std::string>& fileExcludes, const std::vector<std::string>& fileIncludes,
 					 const std::vector<std::string>& archiveExcludes, const std::vector<std::string>& archiveIncludes) -> int;
+
+auto createFromSteamDepotConfigs( const std::string& configPath, const std::vector<std::string>& depotIDs, std::string_view indexLocation,
+								  bool skipArchives, const std::vector<std::string>& fileExcludes, const std::vector<std::string>& fileIncludes,
+								  const std::vector<std::string>& archiveExcludes, const std::vector<std::string>& archiveIncludes ) -> int;

src/main.cpp

@@ -43,6 +43,8 @@ auto main( int argc, char* argv[] ) -> int {
 	std::vector<std::string> fileIncludes;
 	std::vector<std::string> archiveExcludes;
 	std::vector<std::string> archiveIncludes;
+	std::string steamDepotConfig;
+	std::vector<std::string> steamDepotIDs;
 	std::string indexLocation;
 	bool overwrite{ false };
 	const auto programFile{ std::filesystem::path( argv[ 0 ] ).filename() };
@@ -71,15 +73,22 @@ auto main( int argc, char* argv[] ) -> int {
 		.metavar( "excluded" )
 		.minargs( 1 );
 	params.add_parameter( fileIncludes, "--include" )
-		.help( "RegExp pattern(s) to include files when creating the index. If not present, all files not matching an exclusion will be included." )
+		.help( "RegExp pattern(s) to include files when creating the index. If not present, all files not matching an exclusion will be included.")
 		.metavar( "included" );
 	params.add_parameter( archiveExcludes, "--exclude-archives", "-E" )
-		.help( "RegExp pattern(s) to exclude VPKs when creating the index." )
+		.help( "RegExp pattern(s) to exclude VPKs when creating the index. Ignored if `--steam-depot-config` is present." )
 		.metavar( "excluded-archives" )
 		.minargs( 1 );
 	params.add_parameter( archiveIncludes, "--include-archives" )
 		.help( "RegExp pattern(s) to include VPKs when creating the index. If not present, all VPKs not matching an exclusion will be included." )
 		.metavar( "included-archives" );
+	params.add_parameter( steamDepotConfig, "--steam-depot-config" )
+		.help( "Use a Steam depot configuration file to include/exclude content. Pair this option with `--steam-depot-ids`." )
+		.metavar( "steam-depot-config" )
+		.maxargs( 1 );
+	params.add_parameter( steamDepotIDs, "--steam-depot-ids" )
+		.help( "The Steam depot IDs to include the content of. These should correspond with keys in the depots section of the Steam depot configuration file. Pair this option with `--steam-depot-config`." )
+		.metavar( "steam-depot-ids" );
 	params.add_parameter( indexLocation, "--index", "-i" )
 		.help( "The index file to use." )
 		.metavar( "index-loc" )
@@ -109,7 +118,6 @@ auto main( int argc, char* argv[] ) -> int {
 		Log_Info( "`{}` started at {:02d}:{:02d}:{:02d}", programFile.string(), localPtr->tm_hour, localPtr->tm_min, localPtr->tm_sec );
 	}
 
-	int ret;
 	if ( newIndex ) {
 		if ( const auto indexPath{ std::filesystem::path{ root } / indexLocation }; std::filesystem::exists( indexPath ) ) {
 			if (! overwrite ) {
@@ -121,7 +129,7 @@ auto main( int argc, char* argv[] ) -> int {
 					return 1;
 				}
 			} else {
-				Log_Warn( "Index file `{}` already exists, will be overwritten.", indexPath.string() );
+				Log_Warn( "Index file `{}` already exists, it will be overwritten.", indexPath.string() );
 			}
 			// clear contents
 			std::ofstream writer{ indexPath, std::ios::out | std::ios::trunc };
@@ -134,15 +142,39 @@ auto main( int argc, char* argv[] ) -> int {
 		fileExcludes.emplace_back( ".*\\.log" );
 		fileExcludes.emplace_back( ".*verifier_index\\.rsv" );
 
-		ret = createFromRoot( root, indexLocation, skipArchives, fileExcludes, fileIncludes, archiveExcludes, archiveIncludes );
-	} else {
-		if ( overwrite )
-			Log_Error( "current action doesn't support `--overwrite`, please remove it." );
-		if ( !fileExcludes.empty() )
-			Log_Error( "current action doesn't support `--exclude`, please remove it." );
+		// create from a steam depot config
+		if ( !steamDepotConfig.empty() || !steamDepotIDs.empty() ) {
+			if ( steamDepotConfig.empty() && !steamDepotIDs.empty() ) {
+				Log_Warn( "`--steam-depot-config` must be set when `--steam-depot-ids` is used." );
+				return 1;
+			}
+			if ( !steamDepotConfig.empty() && steamDepotIDs.empty() ) {
+				Log_Warn( "`--steam-depot-ids` must be set when `--steam-depot-config` is used." );
+				return 1;
+			}
 
-		ret = verify( root, indexLocation );
+			return createFromSteamDepotConfigs( steamDepotConfig, steamDepotIDs, indexLocation, skipArchives, fileExcludes, fileIncludes, archiveExcludes, archiveIncludes );
+		}
+
+		return createFromRoot( root, indexLocation, skipArchives, fileExcludes, fileIncludes, archiveExcludes, archiveIncludes );
 	}
 
-	return ret;
+	if ( skipArchives )
+		Log_Warn( "The current action doesn't support `--skip-archives`, it will be ignored." );
+	if (! fileExcludes.empty() )
+		Log_Warn( "The current action doesn't support `--exclude`, it will be ignored." );
+	if (! fileIncludes.empty() )
+		Log_Warn( "The current action doesn't support `--include`, it will be ignored." );
+	if (! archiveExcludes.empty() )
+		Log_Warn( "The current action doesn't support `--exclude-archives`, it will be ignored." );
+	if (! archiveIncludes.empty() )
+		Log_Warn( "The current action doesn't support `--include-archives`, it will be ignored." );
+	if (! steamDepotConfig.empty() )
+		Log_Warn( "The current action doesn't support `--steam-depot-config`, it will be ignored." );
+	if (! steamDepotIDs.empty() )
+		Log_Warn( "The current action doesn't support `--steam-depot-ids`, it will be ignored." );
+	if ( overwrite )
+		Log_Warn( "The current action doesn't support `--overwrite`, it will be ignored." );
+
+	return verify( root, indexLocation );
 }

src/thirdparty/CMakeLists.txt

@@ -9,4 +9,11 @@ add_subdirectory( "${CMAKE_CURRENT_LIST_DIR}/argumentum" SYSTEM )
 add_subdirectory( "${CMAKE_CURRENT_LIST_DIR}/fmt" SYSTEM )
 
 # sourcepp
+set( SOURCEPP_USE_BSPPP OFF CACHE INTERNAL "" )
+set( SOURCEPP_USE_DMXPP OFF CACHE INTERNAL "" )
+set( SOURCEPP_USE_FGDPP OFF CACHE INTERNAL "" )
+set( SOURCEPP_USE_MDLPP OFF CACHE INTERNAL "" )
+set( SOURCEPP_USE_STEAMPP OFF CACHE INTERNAL "" )
+set( SOURCEPP_USE_VICEPP OFF CACHE INTERNAL "" )
+set( SOURCEPP_USE_VTFPP OFF CACHE INTERNAL "" )
 add_subdirectory( "${CMAKE_CURRENT_LIST_DIR}/sourcepp" SYSTEM )

src/verify.cpp

@@ -107,7 +107,7 @@ auto verify( std::string_view root_, std::string_view indexLocation ) -> int {
 		CryptoPP::SHA1 sha1er{};
 		CryptoPP::CRC32 crc32er{};
 
-		unsigned char buffer[2048];
+		unsigned char buffer[ 2048 ];
 		while ( auto count = std::fread( buffer, 1, sizeof( buffer ), file ) ) {
 			sha1er.Update( buffer, count );
 			crc32er.Update( buffer, count );