Changeset: 85 Work on codegen (a big blob of changes about various things, giving up on writing a clear commit message)

- stuff along the lines of cleaning up store process
- remove completed TODOs
- move code generation out of parser loop
- ^^^ also introduce some weird bugs of DeclXxx::name field disappearing -- maybe fixed, maybe didn't, can't reliably reproduce
- add infra to mangle (not included in codegen yet, also not tested)
- convert SourceFile storage map to node map, ensuring pointer stability (was broken before)
- buildsystem asan and UBsan applying to all targest

1 files changed, 99 insertions, 58 deletions

diff --git a/source/20-codegen-compiler/main.cpp b/source/20-codegen-compiler/main.cpp
index 5350f8e..af51900 100644
--- a/source/20-codegen-compiler/main.cpp
+++ b/source/20-codegen-compiler/main.cpp
@@ -2,7 +2,6 @@
 #include "CodegenDecl.hpp"
 #include "CodegenLexer.hpp"
 #include "CodegenModel.hpp"
-#include "CodegenModelArchive.hpp"
 #include "CodegenOutput.hpp"
 #include "CodegenUtils.hpp"
 
@@ -30,9 +29,12 @@ namespace fs = std::filesystem;
 // TOOD maybe switch to libclang, maintaining this parser is just too painful
 
 struct AppState {
-	CodegenModel* model;
-	CodegenModelArchive* modelArchive;
-	robin_hood::unordered_map<std::string, SourceFile, StringHash, StringEqual> sourceFiles;
+	CodegenRuntimeModel* runtimeModel;
+	CodegenArchiveModel* archiveModel;
+	// NOTE: decl objects reference the SourceFile objects by pointer
+	robin_hood::unordered_node_map<std::string, SourceFile, StringHash, StringEqual> sourceFiles;
+	std::vector<DeclEnum*> enumsToRevisit;
+	std::vector<DeclStruct*> structsToRevisit;
 	std::string_view outputDir;
 	std::string_view databaseFilePath;
 
@@ -42,10 +44,11 @@ struct AppState {
 			return iter->second;
 		} else {
 			auto [iter, success] = sourceFiles.try_emplace(std::string(filename), SourceFile{});
-			auto& filename = iter->first; // NOTE: shadows the parameter `filename`
-			auto& sourceFile = iter->second;
-			sourceFile.filename = filename;
-			return sourceFile;
+			// NOTE: "persistent" means pointer stable below
+			auto& persistentFilename = iter->first;
+			auto& persistentSourceFile = iter->second;
+			persistentSourceFile.filename = persistentFilename;
+			return persistentSourceFile;
 		}
 	}
 };
@@ -536,11 +539,13 @@ void GenerateForClassMetadata(
 	if (!decl.baseClasses.empty()) {
 		// Forward declare the variables (which may appear before this section, after this section, or in another TU)
 		for (auto& baseClass : decl.baseClasses) {
-			APPEND_FMT_LN(data.text, "extern const TypeInfo gCGtype_%s_TypeInfo;", declIdName);
+			auto baseClassIdName = baseClass->name.c_str();
+			APPEND_FMT_LN(data.text, "extern const TypeInfo gCGtype_%s_TypeInfo;", baseClassIdName);
 		}
 		APPEND_FMT_LN(data.text, "const TypeInfo* const gCGtype_%s_BaseClasses[] = {", declIdName);
 		for (auto& baseClass : decl.baseClasses) {
-			APPEND_FMT_LN(data.text, "gCGtype_%s_TypeInfo,", declIdName);
+			auto baseClassIdName = baseClass->name.c_str();
+			APPEND_FMT_LN(data.text, "gCGtype_%s_TypeInfo,", baseClassIdName);
 		}
 		APPEND_LIT_LN(data.text, "};");
 	}
@@ -593,10 +598,6 @@ struct ParserState {
 };
 
 struct ParserOutput {
-	CodegenOutput headerOutput;
-	CodegenOutput sourceOutput;
-	CodegenOutput standaloneSourceOutput;
-
 	// Example:
 	// namespace std::details {
 	//   /* [stack top].ns = std::details */
@@ -706,11 +707,6 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 	// TODO move lexedTokens and consumption related functions to ParserState struct
 
 	ParserOutput po;
-	{
-		INPLACE_FMT(hpp, "%.*s.gh.inl", PRINTF_STRING_VIEW(filenameStem));
-		INPLACE_FMT(cpp, "%.*s.gs.inl", PRINTF_STRING_VIEW(filenameStem));
-		Utils::ProduceGeneratedHeader(hpp, po.headerOutput, cpp, po.sourceOutput);
-	}
 
 	auto& tokens = ls.tokens;
 	auto& idx = ls.idx;
@@ -744,7 +740,7 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 								break;
 							}
 
-							po.currentNamespace = as.model->AddNamespace(DeclNamespace{
+							po.currentNamespace = as.runtimeModel->AddNamespace(DeclNamespace{
 								.container = po.currentNamespace,
 								.name = tokens[idx].text,
 							});
@@ -816,8 +812,10 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 
 								// TODO support namespace qualified names
 								auto baseClassFullname = Utils::MakeFullName(idenTok.text, po.currentNamespace);
-								auto baseClassDecl = as.model->FindStruct(baseClassFullname);
+								auto baseClassDecl = as.runtimeModel->FindStruct(baseClassFullname);
 								if (baseClassDecl) {
+									// TODO retreive class from database
+									// ---- Or just silent create it, and assume the code was valid?
 									// We silently ignore a non-existent base class, because they may reside in a file that we didn't scan
 									structDecl.baseClasses.push_back(baseClassDecl);
 								}
@@ -843,7 +841,7 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 
 						{
 							// Get a pointer to the decl inside CodegenInput's storage
-							auto decl = as.model->AddStruct(std::move(fullname), std::move(structDecl));
+							auto decl = as.runtimeModel->AddStruct(std::move(fullname), std::move(structDecl));
 							po.currentStruct = decl;
 							po.currentStructBraceDepth = po.currentBraceDepth;
 						}
@@ -940,12 +938,16 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 							++idx;
 						}
 
+						// if (po.currentEnum->generating) {
+						// 	GenerateForEnum(po.headerOutput, po.sourceOutput, *po.currentEnum);
+						// }
 						if (po.currentEnum->generating) {
-							GenerateForEnum(po.headerOutput, po.sourceOutput, *po.currentEnum);
+							as.enumsToRevisit.push_back(po.currentEnum);
 						}
 
 						{
-							auto decl = as.model->AddEnum(std::move(fullname), std::move(enumDecl));
+							auto decl = as.runtimeModel->AddEnum(std::move(fullname), std::move(enumDecl));
+							// Fix pointers
 							po.currentEnum = decl;
 							po.currentEnumBraceDepth = po.currentBraceDepth;
 						}
@@ -1069,15 +1071,8 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 										// NOTE: intentionally shadowing
 										INPLACE_FMT(getterName, "Get%s", GetPascalCasedName().c_str());
 
-										// TODO work with pass-by-value vs pass-by-reference
-										// this probably needs libclang to detect the size and existance of trivial copy-ctors
-										CodegenOutputThing data;
-										APPEND_FMT_LN(data.text, "const %s& %.*s::%s() const {", decl.type.c_str(), PRINTF_STRING_VIEW(decl.containerStruct->fullname), getterName);
-										APPEND_FMT_LN(data.text, "    return %s;", decl.name.c_str());
-										APPEND_LIT_LN(data.text, "}");
-
-										po.sourceOutput.AddOutputThing(std::move(data));
 										decl.getterName = getterName;
+										decl.isGetterGenerated = true;
 									} else {
 										decl.getterName = getterName;
 									}
@@ -1090,13 +1085,8 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 										// NOTE: intentionally shadowing
 										INPLACE_FMT(setterName, "Set%s", GetPascalCasedName().c_str());
 
-										CodegenOutputThing data;
-										APPEND_FMT_LN(data.text, "void %.*s::%s(const %s& value) const {", PRINTF_STRING_VIEW(decl.containerStruct->fullname), setterName, decl.type.c_str());
-										APPEND_FMT_LN(data.text, "    this->%s = value;", decl.name.c_str());
-										APPEND_LIT_LN(data.text, "}");
-
-										po.sourceOutput.AddOutputThing(std::move(data));
 										decl.setterName = setterName;
+										decl.isSetterGenerated = true;
 									} else {
 										decl.setterName = setterName;
 									}
@@ -1157,11 +1147,7 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 					// Exit struct
 
 					if (po.currentStruct->generating) {
-						GenerateForClassMetadata(po.headerOutput, po.sourceOutput, *po.currentStruct);
-					}
-					if (po.currentStruct->generatingInheritanceHiearchy) {
-						// NOTE: this option is transitive to all child classes (as long as they have the basic annotation)
-						// TODO
+						as.structsToRevisit.push_back(po.currentStruct);
 					}
 
 					po.currentStruct = nullptr;
@@ -1170,6 +1156,11 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 				if (po.currentEnum && po.currentBraceDepth == po.currentEnumBraceDepth) {
 					// Exit enum
 
+					// TODO this is unused currently, see CKw_Enum branch
+					if (po.currentEnum->generating) {
+						as.enumsToRevisit.push_back(po.currentEnum);
+					}
+
 					po.currentEnum = nullptr;
 					po.currentEnumBraceDepth = -1;
 				}
@@ -1185,14 +1176,7 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 		printf("[WARNING] unbalanced brace at end of file\n");
 	}
 
-	INPLACE_FMT(generatedHeaderInlName, "%.*s/%.*s.gh.inl", PRINTF_STRING_VIEW(as.outputDir), PRINTF_STRING_VIEW(filenameStem));
-	Utils::WriteOutputFile(po.headerOutput, generatedHeaderInlName);
-	INPLACE_FMT(generatedSourceInlName, "%.*s/%.*s.gs.inl", PRINTF_STRING_VIEW(as.outputDir), PRINTF_STRING_VIEW(filenameStem));
-	Utils::WriteOutputFile(po.sourceOutput, generatedSourceInlName);
-	INPLACE_FMT(generatedCppName, "%.*s/%.*s.g.cpp", PRINTF_STRING_VIEW(as.outputDir), PRINTF_STRING_VIEW(filenameStem));
-	Utils::WriteOutputFile(po.standaloneSourceOutput, generatedCppName);
-
-	as.modelArchive->DeleteDeclsRelatedToFile(filenameStem);
+	as.archiveModel->DeleteDeclsRelatedToFile(filenameStem);
 	// as.modelArchive->Store(po.model);
 }
 
@@ -1346,11 +1330,10 @@ where   --output-dir=<path>: the *directory* to write generated contents to. Thi
 	DEBUG_PRINTF("Outputting to directory %.*s.\n", PRINTF_STRING_VIEW(as.outputDir));
 	DEBUG_PRINTF("Databse file: %.*s.\n", PRINTF_STRING_VIEW(as.databaseFilePath));
 
-	// TODO make every input command share the same CodegenModel
 	// TODO move the actual output logic after processing all input commands, based on SQLite batabase model instead of the in-memory CodegenModel model
 	//      this allows better consistency between direct in-file entities (like enums) vs. multi-file entities (like struct inheritance hierarchy)
 	//      this would also mean almost rewriting the whole codegen logic, to work on a changelist fetched from SQLite database instead of being embedded inside the parser loop
-	// 	    TODO how do we detect the case of 
+	// 	    TODO how do we detect the case of
 	//           1. has: Foo.hpp Bar.hpp
 	//           2. struct Foo; struct Bar : Foo;
 	//           3. struct Foo is removed from Foo.hpp, but our parser only recieves Foo.hpp as file changed--and can't figure out that there is still a reference to Foo in Bar.hpp
@@ -1358,11 +1341,14 @@ where   --output-dir=<path>: the *directory* to write generated contents to. Thi
 	//           - use some kind of database scanner to review all references to a class when removing (e.g. detect for logic error on foreign key linked columns)
 	//           - follow the file links in database, and propagate parsing to those files in the hierarchy
 	//             - pretty much defeats the purpose of using an incremental parser: some classes like GameObject will have links throughout a very large portion of the project code
-	CodegenModel model;
-	CodegenModelArchive modelArchive(as.databaseFilePath);
+	// - [x] out of parser generation
+	// - [ ] database readback
+	// - [ ] full database based generation (tentative)
+	CodegenRuntimeModel runtimeModel;
+	CodegenArchiveModel archiveModel(as.databaseFilePath);
 
-	as.model = &model;
-	as.modelArchive = &modelArchive;
+	as.runtimeModel = &runtimeModel;
+	as.archiveModel = &archiveModel;
 
 	// Positional argument pass
 	for (int i = 1; i < argc; ++i) {
@@ -1383,7 +1369,62 @@ where   --output-dir=<path>: the *directory* to write generated contents to. Thi
 		}
 	}
 
-	modelArchive.Store(model);
+	for (auto decl : as.enumsToRevisit) {
+		auto& headerOutput = decl->sourceFile->postHeaderOutput;
+		auto& sourceOutput = decl->sourceFile->postSourceOutput;
+		GenerateForEnum(headerOutput, sourceOutput, *decl);
+	}
+	for (auto decl : as.structsToRevisit) {
+		auto& headerOutput = decl->sourceFile->postHeaderOutput;
+		auto& sourceOutput = decl->sourceFile->postSourceOutput;
+
+		// Always-on metdata
+		GenerateForClassMetadata(headerOutput, sourceOutput, *decl);
+
+		if (decl->generatingInheritanceHiearchy) {
+			// TODO
+		}
+
+		for (auto& property : decl->memberVariables) {
+			if (property.isGetterGenerated) {
+				// TODO work with pass-by-value vs pass-by-reference
+				// this probably needs libclang to detect the size and existance of trivial copy-ctors
+				CodegenOutputThing data;
+				APPEND_FMT_LN(data.text, "const %s& %.*s::%s() const {", property.type.c_str(), PRINTF_STRING_VIEW(property.containerStruct->fullname), property.getterName.c_str());
+				APPEND_FMT_LN(data.text, "    return %s;", property.name.c_str());
+				APPEND_LIT_LN(data.text, "}");
+
+				sourceOutput.AddOutputThing(std::move(data));
+			}
+			if (property.isSetterGenerated) {
+				CodegenOutputThing data;
+				APPEND_FMT_LN(data.text, "void %.*s::%s(const %s& value) const {", PRINTF_STRING_VIEW(property.containerStruct->fullname), property.setterName.c_str(), property.type.c_str());
+				APPEND_FMT_LN(data.text, "    this->%s = value;", property.name.c_str());
+				APPEND_LIT_LN(data.text, "}");
+
+				sourceOutput.AddOutputThing(std::move(data));
+			}
+		}
+		for (auto& method : decl->memberFunctions) {
+			// TODO
+		}
+	}
+
+	archiveModel.Store(runtimeModel);
+
+	// Write output files
+	for (auto&& [_, sourceFile] : as.sourceFiles) {
+		INPLACE_FMT(hpp, "%.*s.gh.inl", PRINTF_STRING_VIEW(sourceFile.filename));
+		INPLACE_FMT(cpp, "%.*s.gs.inl", PRINTF_STRING_VIEW(sourceFile.filename));
+		Utils::ProduceGeneratedHeader(hpp, sourceFile.postHeaderOutput, cpp, sourceFile.postSourceOutput);
+
+		INPLACE_FMT(generatedHeaderInlName, "%.*s/%s", PRINTF_STRING_VIEW(as.outputDir), hpp);
+		Utils::WriteOutputFile(sourceFile.postHeaderOutput, generatedHeaderInlName);
+		INPLACE_FMT(generatedSourceInlName, "%.*s/%s", PRINTF_STRING_VIEW(as.outputDir), cpp);
+		Utils::WriteOutputFile(sourceFile.postSourceOutput, generatedSourceInlName);
+		INPLACE_FMT(generatedCppName, "%.*s/%.*s.g.cpp", PRINTF_STRING_VIEW(as.outputDir), PRINTF_STRING_VIEW(sourceFile.filename));
+		Utils::WriteOutputFile(sourceFile.tuOutput, generatedCppName);
+	}
 
 	return 0;
 }