Changeset: 79 Initial work on fixing codegen for the main project, add enum underlying type (EUT) scanning capability

1 files changed, 131 insertions, 41 deletions

diff --git a/source/20-codegen-compiler/main.cpp b/source/20-codegen-compiler/main.cpp
index 2068962..e9a449c 100644
--- a/source/20-codegen-compiler/main.cpp
+++ b/source/20-codegen-compiler/main.cpp
@@ -27,9 +27,10 @@ using namespace std::literals;
 namespace fs = std::filesystem;
 
 // TODO support codegen target in .cpp files
+// TOOD maybe switch to libclang, maintaining this parser is just too painful
 
 struct AppState {
-	/*nullable*/ CodegenModelArchive* modelArchive = nullptr;
+	CodegenModelArchive* modelArchive;
 	robin_hood::unordered_map<std::string, SourceFile, StringHash, StringEqual> sourceFiles;
 	std::string_view outputDir;
 	std::string_view databaseFilePath;
@@ -149,6 +150,8 @@ enum CppKeyword {
 	CKw_Protected,
 	CKw_Private,
 	CKw_Virtual,
+	CKw_Using,
+	CKw_Template,
 	CKw_COUNT,
 };
 
@@ -162,6 +165,8 @@ RSTR_LUT_DECL(CppKeyword, 0, CKw_COUNT) {
 	RSTR_LUT_MAP(CKw_Protected, "protected");
 	RSTR_LUT_MAP(CKw_Private, "private");
 	RSTR_LUT_MAP(CKw_Virtual, "virtual");
+	RSTR_LUT_MAP(CKw_Using, "using");
+	RSTR_LUT_MAP(CKw_Template, "template");
 }
 
 enum CodegenDirective {
@@ -299,6 +304,30 @@ found:
 	return result;
 }
 
+EnumUnderlyingType TryConsumeEnumUnderlyingType(CodegenLexer& lexer) {
+	// Try 1, 2, 3, 4 tokens from the current position
+	// NOTE: see the FSTR map initialization code for reference that there is max 4 tokens
+	for (int i = 4; i >= 1; --i) {
+		auto text = CombineTokens(std::span(&lexer.Current(), i), " "sv);
+		auto iter = RSTR_LUT(EnumUnderlyingType).find(text);
+		if (iter != RSTR_LUT(EnumUnderlyingType).end()) {
+			lexer.idx += i;
+			return iter->second;
+		}
+	}
+	return EUT_COUNT;
+}
+
+// Also includes the ':' token in the front
+EnumUnderlyingType TryConsumeEnumUnderlyingTypeClause(CodegenLexer& lexer) {
+	if (lexer.Current().text != ":") {
+		return EUT_COUNT;
+	}
+
+	++lexer.idx;
+	return TryConsumeEnumUnderlyingType(lexer);
+}
+
 enum StructMetaGenOptions {
 	SMGO_InheritanceHiearchy,
 	SMGO_COUNT,
@@ -445,7 +474,7 @@ void GenerateForEnum(CodegenOutput& headerOut, CodegenOutput& sourceOut, const D
 		{
 			auto& o = lookupTable.text;
 			// TODO use correct underlying type
-			APPEND_FMT_LN(o, "constinit frozen::unordered_map<frozen::string, uint64_t, %" PRId64 "> %s = {", filteredElements.size(), str2ValName);
+			APPEND_FMT_LN(o, "constinit frozen::unordered_map<frozen::string, %s, %" PRId64 "> %s = {", decl.underlyingTypeStr.c_str(), filteredElements.size(), str2ValName);
 			for (auto& elm : filteredElements) {
 				APPEND_FMT_LN(o, "{\"%s\", %" PRId64 "},", elm.name.c_str(), elm.value);
 			}
@@ -481,7 +510,11 @@ void GenerateForEnum(CodegenOutput& headerOut, CodegenOutput& sourceOut, const D
 }
 
 void GenerateClassProperty(CodegenOutput& headerOutput, CodegenOutput& sourceOutput) {
-	// TODO
+	CodegenOutputThing thing;
+	APPEND_FMT_LN(thing.text, "TypePropertyInfo gCGtype_%s_%s_Property = {", "TODO", "TODO");
+	APPEND_LIT_LN(thing.text, "};");
+
+	sourceOutput.AddOutputThing(std::move(thing));
 }
 
 void GenerateClassFunction(CodegenOutput& headerOutput, CodegenOutput& sourceOutput) {
@@ -498,20 +531,41 @@ void GenerateForClassMetadata(
 
 	CodegenOutputThing data;
 	// TODO generate type id, this needs global scanning
-	APPEND_FMT_LN(data.text, "const TypeInfo* const gCGtype_%s_BaseClasses[] = {", declIdName);
-	for (auto& baseClass : decl.baseClasses) {
-		// TODO get ptr to TypeInfo, this needs global scanning for non-file local classes
+
+	if (!decl.baseClasses.empty()) {
+		// Forward declare the variables (which may appear before this section, after this section, or in another TU)
+		for (auto& baseClass : decl.baseClasses) {
+			APPEND_FMT_LN(data.text, "extern const TypeInfo gCGtype_%s_TypeInfo;", declIdName);
+		}
+		APPEND_FMT_LN(data.text, "const TypeInfo* const gCGtype_%s_BaseClasses[] = {", declIdName);
+		for (auto& baseClass : decl.baseClasses) {
+			APPEND_FMT_LN(data.text, "gCGtype_%s_TypeInfo,", declIdName);
+		}
+		APPEND_LIT_LN(data.text, "};");
 	}
-	APPEND_LIT_LN(data.text, "};");
-	APPEND_FMT_LN(data.text, "const TypePropertyInfo gCGtype_%s_Properties[] = {", declIdName);
-	for (auto& property : decl.memberVariables) {
-		APPEND_FMT_LN(data.text, "{.name=\"%s\"sv, .getterName=\"%s\"sv, .setterName=\"%s\"sv},", property.name.c_str(), property.getterName.c_str(), property.setterName.c_str());
+
+	if (!decl.memberVariables.empty()) {
+		APPEND_FMT_LN(data.text, "const TypePropertyInfo gCGtype_%s_Properties[] = {", declIdName);
+		for (auto& property : decl.memberVariables) {
+			APPEND_FMT_LN(data.text, "{.name=\"%s\"sv, .getterName=\"%s\"sv, .setterName=\"%s\"sv},", property.name.c_str(), property.getterName.c_str(), property.setterName.c_str());
+		}
+		APPEND_LIT_LN(data.text, "};");
 	}
-	APPEND_LIT_LN(data.text, "};");
+
+	if (!decl.memberFunctions.empty()) {
+		APPEND_FMT_LN(data.text, "const TypeMethodInfo gCGtype_%s_Methods[] = {", declIdName);
+		for (auto& method : decl.memberFunctions) {
+			// TODO
+		}
+		APPEND_LIT_LN(data.text, "};");
+	}
+
 	APPEND_FMT_LN(data.text, "const TypeInfo gCGtype_%s_TypeInfo = {", declIdName);
 	APPEND_FMT_LN(data.text, ".name = \"%s\"sv,", declIdName);
-	APPEND_FMT_LN(data.text, ".parents = gCGtype_%s_BaseClasses,", declIdName);
-	APPEND_FMT_LN(data.text, ".properties = gCGtype_%s_Properties};", declIdName);
+	if (!decl.baseClasses.empty()) APPEND_FMT_LN(data.text, ".parents = gCGtype_%s_BaseClasses,", declIdName);
+	if (!decl.memberVariables.empty()) APPEND_FMT_LN(data.text, ".properties = gCGtype_%s_Properties,", declIdName);
+	if (!decl.memberFunctions.empty()) APPEND_FMT_LN(data.text, ".methods = gCGtype_%s_Methods,", declIdName);
+	APPEND_LIT_LN(data.text, "};");
 
 	CodegenOutputThing queryFunc;
 	APPEND_FMT(queryFunc.text,
@@ -664,16 +718,7 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 		auto& token = ls.Current();
 
 		bool incrementTokenIdx = true;
-
-		// Reamalgamate token type and single char tokens
-		int tokenKey;
-		if (token.type == CLEX_ext_single_char) {
-			tokenKey = token.text[0];
-		} else {
-			tokenKey = token.type;
-		}
-
-		switch (tokenKey) {
+		switch (token.Reamalgamate()) {
 			case CLEX_id: {
 				CppKeyword keyword;
 				{
@@ -797,6 +842,11 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 					}
 
 					case CKw_Enum: {
+						if (po.currentStruct) {
+							// TODO parsing enums inside classes is currently broken (1. model database is not modeled for this 2. codegen logic is not modeled)
+							break;
+						}
+
 						// Consume the "enum" keyword
 						++idx;
 						incrementTokenIdx = false;
@@ -817,9 +867,8 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 						DeclEnum enumDecl;
 						enumDecl.sourceFile = &sourceFile;
 						enumDecl.container = po.currentNamespace;
-						enumDecl.underlyingType = EUT_Int32; // TODO
-						enumDecl.underlyingTypeStr = "int";
 						enumDecl.name = name;
+						// Setting underlying type: see below
 
 						// Temporarily bind the pointers to local variable, HandleDirectiveEnum() and other functions expect these to the set
 						po.currentEnum = &enumDecl;
@@ -828,11 +877,26 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 						// Consume the enum name identifier
 						++idx;
 
+						// Setting underlying type
+						// NOTE: these two variables are only valid if we enter the EUT found branch
+						size_t eutClauseBegin = idx + /* The ':' token */ 1;
+						size_t eutClauseEnd;
+						if (auto eut = TryConsumeEnumUnderlyingTypeClause(ls);
+							eut != EUT_COUNT)
+						{
+							eutClauseEnd = idx;
+							enumDecl.underlyingType = eut;
+							enumDecl.underlyingTypeStr = CombineTokens(std::span(tokens.begin() + eutClauseBegin, eutClauseEnd - eutClauseBegin), " "sv);
+						} else {
+							enumDecl.underlyingType = EUT_Int32;
+							enumDecl.underlyingTypeStr = "int";
+						}
+
 						int enumClosingBraceCount = 0;
 						int enumBraceDepth = 0;
 						while (enumClosingBraceCount == 0 && idx < tokens.size()) {
 							auto& token = tokens[idx];
-							switch (token.type) {
+							switch (token.Reamalgamate()) {
 								case CLEX_id: {
 									if (token.text == "BRUSSEL_ENUM") {
 										// Consume the argument list and skip advancing index: this function already consumed all the tokens about BRUSSEL_ENUM
@@ -858,17 +922,13 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 									}
 								} break;
 
-								case CLEX_ext_single_char: {
-									switch (token.text[0]) {
-										case '{': {
-											++enumBraceDepth;
-										} break;
+								case '{': {
+									++enumBraceDepth;
+								} break;
 
-										case '}': {
-											--enumBraceDepth;
-											++enumClosingBraceCount;
-										} break;
-									}
+								case '}': {
+									--enumBraceDepth;
+									++enumClosingBraceCount;
 								} break;
 							}
 
@@ -892,6 +952,18 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 						goto endCaseCLEX_id;
 					}
 
+					// Consume the whole statement, because this statement may contain `enum` or `class` keywords that will pollute the parser
+					case CKw_Template: {
+						// `template` is either a template list which we don't care about, or a part of a type which we don't care about,
+						// unless it's a part of a function declaration, where the tokens are handled inside CG_ClassMethod parsing
+						// TODO handle nested templates or operator> inside template expression
+						ls.SkipUntilTokenSingleChar('>');
+					} break;
+					case CKw_Using: {
+						// `using` indicates a type alias or namespace import which we don't care about
+						ls.SkipUntilTokenSingleChar(';');
+					} break;
+
 					// We don't care about these keywords
 					case CKw_Public:
 					case CKw_Protected:
@@ -963,6 +1035,7 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 							break;
 						}
 						auto& decl = declOpt.value();
+						decl.containerStruct = po.currentStruct;
 
 						// Different option's common logic
 						std::string pascalCaseName;
@@ -985,14 +1058,20 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 							if (iter == lut.end()) continue;
 							switch (iter->second) {
 								case SPO_Getter: {
-									// TODO I'm too lazy to write error checks, just let the codegen crash
+									// NOTE: I'm too lazy to write error checks, just let the codegen crash if syntax is invalid
 									auto& getterName = arg.at(1)->text;
 									if (getterName == "auto") {
 										// NOTE: intentionally shadowing
 										INPLACE_FMT(getterName, "Get%s", GetPascalCasedName().c_str());
 
-										// TODO generate getter function
+										// TODO work with pass-by-value vs pass-by-reference
+										// this probably needs libclang to detect the size and existance of trivial copy-ctors
+										CodegenOutputThing data;
+										APPEND_FMT_LN(data.text, "const %s& %.*s::%s() const {", decl.type.c_str(), PRINTF_STRING_VIEW(decl.containerStruct->fullname), getterName);
+										APPEND_FMT_LN(data.text, "    return %s;", decl.name.c_str());
+										APPEND_LIT_LN(data.text, "}");
 
+										po.sourceOutput.AddOutputThing(std::move(data));
 										decl.getterName = getterName;
 									} else {
 										decl.getterName = getterName;
@@ -1000,14 +1079,18 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s
 								} break;
 
 								case SPO_Setter: {
-									// TODO
+									// NOTE: I'm too lazy to write error checks, just let the codegen crash if syntax is invalid
 									auto& setterName = arg.at(1)->text;
 									if (setterName == "auto") {
 										// NOTE: intentionally shadowing
 										INPLACE_FMT(setterName, "Set%s", GetPascalCasedName().c_str());
 
-										// TODO generate setter function
+										CodegenOutputThing data;
+										APPEND_FMT_LN(data.text, "void %.*s::%s(const %s& value) const {", PRINTF_STRING_VIEW(decl.containerStruct->fullname), setterName, decl.type.c_str());
+										APPEND_FMT_LN(data.text, "    this->%s = value;", decl.name.c_str());
+										APPEND_LIT_LN(data.text, "}");
 
+										po.sourceOutput.AddOutputThing(std::move(data));
 										decl.setterName = setterName;
 									} else {
 										decl.setterName = setterName;
@@ -1162,7 +1245,9 @@ void HandleArgument(AppState& as, InputOpcode opcode, std::string_view operand)
 				DEBUG_PRINTF("File-list file does not exist, silently skipping.\n");
 				break;
 			}
-			DEFER { fclose(fileList); };
+			DEFER {
+				fclose(fileList);
+			};
 
 			std::string line;
 			while (Utils::ReadCstdioLine(fileList, line)) {
@@ -1258,8 +1343,13 @@ where   --output-dir=<path>: the *directory* to write generated contents to. Thi
 	DEBUG_PRINTF("Outputting to directory %.*s.\n", PRINTF_STRING_VIEW(as.outputDir));
 	DEBUG_PRINTF("Databse file: %.*s.\n", PRINTF_STRING_VIEW(as.databaseFilePath));
 
+#define BRUSSEL_ENABLE_CODEGEN_ARCHIVE 0
+#if BRUSSEL_ENABLE_CODEGEN_ARCHIVE
 	CodegenModelArchive archive(as.databaseFilePath);
 	as.modelArchive = &archive;
+#else
+	as.modelArchive = nullptr;
+#endif
 
 	// Positional argument pass
 	for (int i = 1; i < argc; ++i) {