Add support for multiple program modules in a single symbol database

README.md

@@ -108,6 +108,7 @@ Use of a code formatter such as `clang-format` or `astyle` on the output is reco
 
 | Format Version | Release | Changes |
 | - | - | - |
+| 10 | | Added modules as their own symbol type. Removed the text_address property of source file symbols. |
 | 9 | | Added optional is_virtual_base_class property to nodes in base class lists. |
 | 8 | | Overhauled the format based on the structure of the new symbol database. An error AST node type has been added. The data, function definition, initializer list, source file and variable AST node types have been removed and replaced. |
 | 7 | v1.x | Base classes are now no longer doubly nested inside two JSON objects. Added acccess_specifier property. |

src/ccc/elf.cpp

@@ -149,10 +149,10 @@ Result<ElfFile> parse_elf_file(std::vector<u8> image)
 }
 
 Result<void> import_elf_section_headers(
-	SymbolDatabase& database, const ElfFile& elf, SymbolSourceHandle source)
+	SymbolDatabase& database, const ElfFile& elf, SymbolSourceHandle source, const Module* module_symbol)
 {
 	for(const ElfSection& section : elf.sections) {
-		Result<Section*> symbol = database.sections.create_symbol(section.name, source, section.address);
+		Result<Section*> symbol = database.sections.create_symbol(section.name, source, module_symbol, section.address);
 		CCC_RETURN_IF_ERROR(symbol);
 
 		(*symbol)->set_size(section.size);
@@ -161,7 +161,7 @@ Result<void> import_elf_section_headers(
 	return Result<void>();
 }
 
-Result<void> read_virtual(u8* dest, u32 address, u32 size, const std::vector<ElfFile*>& elves)
+Result<void> read_virtual(u8* dest, u32 address, u32 size, const std::vector<const ElfFile*>& elves)
 {
 	while(size > 0) {
 		bool mapped = false;

src/ccc/elf.h

@@ -57,12 +57,12 @@ struct ElfFile {
 // Parse the ELF file header, section headers and program headers.
 Result<ElfFile> parse_elf_file(std::vector<u8> image);
 
-Result<void> import_elf_section_headers(SymbolDatabase& database, const ElfFile& elf, SymbolSourceHandle source);
+Result<void> import_elf_section_headers(SymbolDatabase& database, const ElfFile& elf, SymbolSourceHandle source, const Module* module_symbol);
 
-Result<void> read_virtual(u8* dest, u32 address, u32 size, const std::vector<ElfFile*>& elves);
+Result<void> read_virtual(u8* dest, u32 address, u32 size, const std::vector<const ElfFile*>& elves);
 
 template <typename T>
-Result<std::vector<T>> read_virtual_vector(u32 address, u32 count, const std::vector<ElfFile*>& elves)
+Result<std::vector<T>> read_virtual_vector(u32 address, u32 count, const std::vector<const ElfFile*>& elves)
 {
 	std::vector<T> vector(count);
 	Result<void> result = read_virtual((u8*) vector.data(), address, count * sizeof(T), elves);

src/ccc/elf_symtab.cpp

@@ -54,6 +54,7 @@ static const char* symbol_visibility_to_string(SymbolVisibility visibility);
 Result<void> import_symbols(
 	SymbolDatabase& database,
 	SymbolSourceHandle source,
+	const Module* module_symbol,
 	std::span<const u8> symtab,
 	std::span<const u8> strtab,
 	u32 importer_flags,
@@ -78,13 +79,14 @@ Result<void> import_symbols(
 		switch(symbol->type()) {
 			case SymbolType::NOTYPE: {
 				Result<Label*> label = database.labels.create_symbol(
-					string, source, address, importer_flags, demangler);
+					string, source, module_symbol, address, importer_flags, demangler);
 				CCC_RETURN_IF_ERROR(label);
+
 				break;
 			}
 			case SymbolType::OBJECT: {
 				Result<GlobalVariable*> global_variable = database.global_variables.create_symbol(
-					string, source, address, importer_flags, demangler);
+					string, source, module_symbol, address, importer_flags, demangler);
 				CCC_RETURN_IF_ERROR(global_variable);
 
 				if(*global_variable) {
@@ -95,7 +97,7 @@ Result<void> import_symbols(
 			}
 			case SymbolType::FUNC: {
 				Result<Function*> function = database.functions.create_symbol(
-					string, source, address, importer_flags, demangler);
+					string, source, module_symbol, address, importer_flags, demangler);
 				CCC_RETURN_IF_ERROR(function);
 
 				if(*function) {
@@ -105,7 +107,7 @@ Result<void> import_symbols(
 				break;
 			}
 			case SymbolType::FILE: {
-				Result<SourceFile*> source_file = database.source_files.create_symbol(string, source);
+				Result<SourceFile*> source_file = database.source_files.create_symbol(string, source, module_symbol);
 				CCC_RETURN_IF_ERROR(source_file);
 
 				break;

src/ccc/elf_symtab.h

@@ -10,6 +10,7 @@ namespace ccc::elf {
 Result<void> import_symbols(
 	SymbolDatabase& database,
 	SymbolSourceHandle source,
+	const Module* module_symbol,
 	std::span<const u8> symtab,
 	std::span<const u8> strtab,
 	u32 importer_flags,

src/ccc/mdebug_analysis.cpp

@@ -14,7 +14,6 @@ Result<void> LocalSymbolTableAnalyser::stab_magic(const char* magic)
 
 Result<void> LocalSymbolTableAnalyser::source_file(const char* path, Address text_address)
 {
-	m_source_file.text_address = text_address;
 	if(m_next_relative_path.empty()) {
 		m_next_relative_path = m_source_file.command_line_path;
 	}
@@ -43,7 +42,8 @@ Result<void> LocalSymbolTableAnalyser::data_type(const ParsedSymbol& symbol)
 	StabsTypeNumber number = symbol.name_colon_type.type->type_number;
 
 	if(m_context.importer_flags & DONT_DEDUPLICATE_TYPES) {
-		Result<DataType*> data_type = m_database.data_types.create_symbol(name, m_context.symbol_source);
+		Result<DataType*> data_type = m_database.data_types.create_symbol(
+			name, m_context.symbol_source, m_context.module_symbol);
 		CCC_RETURN_IF_ERROR(data_type);
 
 		m_source_file.stabs_type_number_to_handle[number] = (*data_type)->handle();
@@ -52,7 +52,7 @@ Result<void> LocalSymbolTableAnalyser::data_type(const ParsedSymbol& symbol)
 		(*data_type)->files = {m_source_file.handle()};
 	} else {
 		Result<ccc::DataType*> type = m_database.create_data_type_if_unique(
-			std::move(*node), number, name, m_source_file, m_context.symbol_source);
+			std::move(*node), number, name, m_source_file, m_context.symbol_source, m_context.module_symbol);
 		CCC_RETURN_IF_ERROR(type);
 	}
 
@@ -63,14 +63,12 @@ Result<void> LocalSymbolTableAnalyser::global_variable(
 	const char* mangled_name, Address address, const StabsType& type, bool is_static, GlobalStorageLocation location)
 {
 	Result<GlobalVariable*> global = m_database.global_variables.create_symbol(
-		mangled_name, m_context.symbol_source, address, m_context.importer_flags, m_context.demangler);
+		mangled_name, m_context.symbol_source, m_context.module_symbol, address, m_context.importer_flags, m_context.demangler);
 	CCC_RETURN_IF_ERROR(global);
 	CCC_ASSERT(*global);
 
 	m_global_variables.expand_to_include((*global)->handle());
 
-
-
 	Result<std::unique_ptr<ast::Node>> node = stabs_type_to_ast(type, nullptr, m_stabs_to_ast_state, 0, true, false);
 	CCC_RETURN_IF_ERROR(node);
 
@@ -171,8 +169,10 @@ Result<void> LocalSymbolTableAnalyser::parameter(
 {
 	CCC_CHECK(m_current_function, "Parameter symbol before first func/proc symbol.");
 
-	Result<ParameterVariable*> parameter_variable = m_database.parameter_variables.create_symbol(name, m_context.symbol_source);
+	Result<ParameterVariable*> parameter_variable = m_database.parameter_variables.create_symbol(
+		name, m_context.symbol_source, m_context.module_symbol);
 	CCC_RETURN_IF_ERROR(parameter_variable);
+
 	m_current_parameter_variables.expand_to_include((*parameter_variable)->handle());
 
 	Result<std::unique_ptr<ast::Node>> node = stabs_type_to_ast(type, nullptr, m_stabs_to_ast_state, 0, true, true);
@@ -199,8 +199,10 @@ Result<void> LocalSymbolTableAnalyser::local_variable(
 	}
 
 	Address address = (desc == StabsSymbolDescriptor::STATIC_LOCAL_VARIABLE) ? value : Address();
-	Result<LocalVariable*> local_variable = m_database.local_variables.create_symbol(name, m_context.symbol_source, address);
+	Result<LocalVariable*> local_variable = m_database.local_variables.create_symbol(
+		name, m_context.symbol_source, m_context.module_symbol, address);
 	CCC_RETURN_IF_ERROR(local_variable);
+
 	m_current_local_variables.expand_to_include((*local_variable)->handle());
 	m_pending_local_variables.emplace_back((*local_variable)->handle());
 
@@ -235,7 +237,7 @@ Result<void> LocalSymbolTableAnalyser::lbrac(s32 begin_offset)
 {
 	for(LocalVariableHandle local_variable_handle : m_pending_local_variables) {
 		if(LocalVariable* local_variable = m_database.local_variables.symbol_from_handle(local_variable_handle)) {
-			local_variable->live_range.low = m_source_file.text_address.value + begin_offset;
+			local_variable->live_range.low = m_source_file.address().value + begin_offset;
 		}
 	}
 
@@ -252,7 +254,7 @@ Result<void> LocalSymbolTableAnalyser::rbrac(s32 end_offset)
 	std::vector<LocalVariableHandle>& variables = m_blocks.back();
 	for(LocalVariableHandle local_variable_handle : variables) {
 		if(LocalVariable* local_variable = m_database.local_variables.symbol_from_handle(local_variable_handle)) {
-			local_variable->live_range.high = m_source_file.text_address.value + end_offset;
+			local_variable->live_range.high = m_source_file.address().value + end_offset;
 		}
 	}
 
@@ -285,7 +287,7 @@ Result<void> LocalSymbolTableAnalyser::create_function(const char* mangled_name,
 	}
 
 	Result<Function*> function = m_database.functions.create_symbol(
-		mangled_name, m_context.symbol_source, address, m_context.importer_flags, m_context.demangler);
+		mangled_name, m_context.symbol_source, m_context.module_symbol, address, m_context.importer_flags, m_context.demangler);
 	CCC_RETURN_IF_ERROR(function);
 	CCC_ASSERT(*function);
 	m_current_function = *function;

src/ccc/mdebug_analysis.h

@@ -16,6 +16,7 @@ struct AnalysisContext {
 	const mdebug::SymbolTableReader* reader = nullptr;
 	const std::map<std::string, const mdebug::Symbol*>* globals;
 	SymbolSourceHandle symbol_source;
+	const Module* module_symbol = nullptr;
 	u32 importer_flags = NO_IMPORTER_FLAGS;
 	DemanglerFunctions demangler;
 };

src/ccc/mdebug_importer.cpp

@@ -5,15 +5,22 @@
 
 namespace ccc::mdebug {
 
-static Result<void> resolve_type_names(SymbolDatabase& database, SymbolSourceHandle source, u32 importer_flags);
-static Result<void> resolve_type_name(ast::TypeName& type_name, SymbolDatabase& database, SymbolSourceHandle source, u32 importer_flags);
+static Result<void> resolve_type_names(
+	SymbolDatabase& database, SymbolSourceHandle source, const Module* module_symbol, u32 importer_flags);
+static Result<void> resolve_type_name(
+	ast::TypeName& type_name,
+	SymbolDatabase& database,
+	SymbolSourceHandle source,
+	const Module* module_symbol,
+	u32 importer_flags);
 static void compute_size_bytes(ast::Node& node, SymbolDatabase& database);
 
 Result<void> import_symbol_table(
 	SymbolDatabase& database,
 	std::span<const u8> elf,
 	s32 section_offset,
 	SymbolSourceHandle source,
+	const Module* module_symbol,
 	u32 importer_flags,
 	const DemanglerFunctions& demangler)
 {
@@ -39,7 +46,8 @@ Result<void> import_symbol_table(
 	AnalysisContext context;
 	context.reader = &reader;
 	context.globals = &globals;
-	context.symbol_source = (source);
+	context.symbol_source = source;
+	context.module_symbol = module_symbol;
 	context.importer_flags = importer_flags;
 	context.demangler = demangler;
 
@@ -71,7 +79,7 @@ Result<void> import_files(SymbolDatabase& database, const AnalysisContext& conte
 	}
 
 	// Lookup data types and store data type handles in type names.
-	Result<void> type_name_result = resolve_type_names(database, context.symbol_source, context.importer_flags);
+	Result<void> type_name_result = resolve_type_names(database, context.symbol_source, context.module_symbol, context.importer_flags);
 	CCC_RETURN_IF_ERROR(type_name_result);
 
 	// Compute the size in bytes of all the AST nodes.
@@ -106,20 +114,6 @@ Result<void> import_file(SymbolDatabase& database, const mdebug::File& input, co
 	// about this case for .mdebug sections so just make sure it never happens.
 	CCC_ASSERT(context.importer_flags & DONT_DEDUPLICATE_SYMBOLS);
 
-	Result<SourceFile*> source_file = database.source_files.create_symbol(input.full_path, context.symbol_source);
-	CCC_RETURN_IF_ERROR(source_file);
-
-	(*source_file)->working_dir = input.working_dir;
-	(*source_file)->command_line_path = input.command_line_path;
-
-	// Sometimes the INFO symbols contain information about what toolchain
-	// version was used for building the executable.
-	for(const mdebug::Symbol& symbol : input.symbols) {
-		if(symbol.symbol_class == mdebug::SymbolClass::INFO && strcmp(symbol.string, "@stabs") != 0) {
-			(*source_file)->toolchain_version_info.emplace(symbol.string);
-		}
-	}
-
 	// Parse the stab strings into a data structure that's vaguely
 	// one-to-one with the text-based representation.
 	u32 importer_flags_for_this_file = context.importer_flags;
@@ -135,6 +129,30 @@ Result<void> import_file(SymbolDatabase& database, const mdebug::File& input, co
 		}
 	}
 
+	// Find the address of the source file.
+	Address text_address;
+	for(const ParsedSymbol& symbol : *symbols) {
+		if(symbol.type == ParsedSymbolType::SOURCE_FILE) {
+			text_address = symbol.raw->value;
+			break;
+		}
+	}
+
+	Result<SourceFile*> source_file = database.source_files.create_symbol(
+		input.full_path, context.symbol_source, context.module_symbol, text_address);
+	CCC_RETURN_IF_ERROR(source_file);
+
+	(*source_file)->working_dir = input.working_dir;
+	(*source_file)->command_line_path = input.command_line_path;
+
+	// Sometimes the INFO symbols contain information about what toolchain
+	// version was used for building the executable.
+	for(const mdebug::Symbol& symbol : input.symbols) {
+		if(symbol.symbol_class == mdebug::SymbolClass::INFO && strcmp(symbol.string, "@stabs") != 0) {
+			(*source_file)->toolchain_version_info.emplace(symbol.string);
+		}
+	}
+
 	StabsToAstState stabs_to_ast_state;
 	stabs_to_ast_state.file_handle = (*source_file)->handle().value;
 	stabs_to_ast_state.stabs_types = &stabs_types;
@@ -274,14 +292,15 @@ Result<void> import_file(SymbolDatabase& database, const mdebug::File& input, co
 	return Result<void>();
 }
 
-static Result<void> resolve_type_names(SymbolDatabase& database, SymbolSourceHandle source, u32 importer_flags)
+static Result<void> resolve_type_names(
+	SymbolDatabase& database, SymbolSourceHandle source, const Module* module_symbol, u32 importer_flags)
 {
 	Result<void> result;
 	database.for_each_symbol([&](ccc::Symbol& symbol) {
 		if(symbol.source() == source && symbol.type()) {
 			ast::for_each_node(*symbol.type(), ast::PREORDER_TRAVERSAL, [&](ast::Node& node) {
 				if(node.descriptor == ast::TYPE_NAME) {
-					Result<void> type_name_result = resolve_type_name(node.as<ast::TypeName>(), database, source, importer_flags);
+					Result<void> type_name_result = resolve_type_name(node.as<ast::TypeName>(), database, source, module_symbol, importer_flags);
 					if(!type_name_result.success()) {
 						result = std::move(type_name_result);
 					}
@@ -293,7 +312,12 @@ static Result<void> resolve_type_names(SymbolDatabase& database, SymbolSourceHan
 	return result;
 }
 
-static Result<void> resolve_type_name(ast::TypeName& type_name, SymbolDatabase& database, SymbolSourceHandle source, u32 importer_flags)
+static Result<void> resolve_type_name(
+	ast::TypeName& type_name,
+	SymbolDatabase& database,
+	SymbolSourceHandle source,
+	const Module* module_symbol,
+	u32 importer_flags)
 {
 	ast::TypeName::UnresolvedStabs* unresolved_stabs = type_name.unresolved_stabs.get();
 	if(!unresolved_stabs) {
@@ -368,7 +392,7 @@ static Result<void> resolve_type_name(ast::TypeName& type_name, SymbolDatabase&
 	}
 
 	if(forward_declared_node) {
-		Result<DataType*> forward_declared_type = database.data_types.create_symbol(unresolved_stabs->type_name, source);
+		Result<DataType*> forward_declared_type = database.data_types.create_symbol(unresolved_stabs->type_name, source, module_symbol);
 		CCC_RETURN_IF_ERROR(forward_declared_type);
 
 		(*forward_declared_type)->set_type(std::move(forward_declared_node));

src/ccc/mdebug_importer.h

@@ -16,6 +16,7 @@ Result<void> import_symbol_table(
 	std::span<const u8> elf,
 	s32 section_offset,
 	SymbolSourceHandle source,
+	const Module* module_symbol,
 	u32 importer_flags,
 	const DemanglerFunctions& demangler);
 Result<void> import_files(SymbolDatabase& database, const AnalysisContext& context);

src/ccc/mdebug_symbols.cpp

@@ -60,7 +60,8 @@ Result<std::vector<ParsedSymbol>> parse_symbols(const std::vector<mdebug::Symbol
 								CCC_WARN("%s Symbol string: %s", STAB_TRUNCATED_ERROR_MESSAGE, string);
 								importer_flags &= ~STRICT_PARSING;
 							} else {
-								return parse_result;
+								return CCC_FAILURE("%s Symbol string: %s",
+									parse_result.error().message.c_str(), string);
 							}
 						}
 					} else {

src/ccc/sndll.cpp

@@ -113,6 +113,7 @@ Result<void> import_sndll_symbols(
 	SymbolDatabase& database,
 	const SNDLLFile& sndll,
 	SymbolSourceHandle source,
+	const Module* module_symbol,
 	u32 importer_flags,
 	DemanglerFunctions demangler)
 {
@@ -121,15 +122,17 @@ Result<void> import_sndll_symbols(
 			switch(symbol.type) {
 				case SNDLLSymbolType::RELATIVE:
 				case SNDLLSymbolType::WEAK: {
-					Result<Label*> result = database.labels.create_symbol(
-						symbol.string, source, sndll.address.get_or_zero() + symbol.value, importer_flags, demangler);
-					CCC_RETURN_IF_ERROR(result);
+					Result<Label*> label = database.labels.create_symbol(
+						symbol.string, source, module_symbol, sndll.address.get_or_zero() + symbol.value, importer_flags, demangler);
+					CCC_RETURN_IF_ERROR(label);
+
 					break;
 				}
 				case SNDLLSymbolType::ABSOLUTE: {
-					Result<Label*> result = database.labels.create_symbol(
-						symbol.string, source, symbol.value, importer_flags, demangler);
-					CCC_RETURN_IF_ERROR(result);
+					Result<Label*> label = database.labels.create_symbol(
+						symbol.string, source, module_symbol,symbol.value, importer_flags, demangler);
+					CCC_RETURN_IF_ERROR(label);
+
 					break;
 				}
 				case SNDLLSymbolType::NIL:

src/ccc/sndll.h

@@ -41,6 +41,7 @@ Result<void> import_sndll_symbols(
 	SymbolDatabase& database,
 	const SNDLLFile& sndll,
 	SymbolSourceHandle source,
+	const Module* module_symbol,
 	u32 importer_flags,
 	DemanglerFunctions demangler);