From d18a28a9659092952aef70a30a47726e7c16d31a Mon Sep 17 00:00:00 2001 From: rtk0c Date: Tue, 24 May 2022 21:47:55 -0700 Subject: Changeset: 38 Branch comment: [] Add infrastructure for codegen --- buildtools/codegen/README.md | 5 +++ buildtools/codegen/main.cpp | 102 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 buildtools/codegen/README.md create mode 100644 buildtools/codegen/main.cpp (limited to 'buildtools/codegen') diff --git a/buildtools/codegen/README.md b/buildtools/codegen/README.md new file mode 100644 index 0000000..7164132 --- /dev/null +++ b/buildtools/codegen/README.md @@ -0,0 +1,5 @@ +# Code Generator +The main code generator. + +## Folder structure +The main program's source files are all located in this folder directly. Text tempaltes are located in `templates/` and none of the files are compiled (even if they end with .c or .cpp). diff --git a/buildtools/codegen/main.cpp b/buildtools/codegen/main.cpp new file mode 100644 index 0000000..4a1d486 --- /dev/null +++ b/buildtools/codegen/main.cpp @@ -0,0 +1,102 @@ +#include "ScopeGuard.hpp" +#include "Utils.hpp" + +#include +#include +#include +#include + +using namespace std::literals; +namespace fs = std::filesystem; + +enum InputOpcode { + IOP_ProcessSingleFile, + IOP_ProcessRecursively, + IOP_COUNT, +}; + +InputOpcode ParseInputOpcode(std::string_view text) { + if (text == "single"sv) { + return IOP_ProcessSingleFile; + } else if (text == "rec"sv) { + return IOP_ProcessRecursively; + } + return IOP_COUNT; +} + +void HandleInputFile(std::string_view source) { + stb_lexer lexer; + char stringStorage[65536]; + const char* srcBegin = source.data(); + const char* srcEnd = srcBegin + source.length(); + stb_c_lexer_init(&lexer, srcBegin, srcEnd, stringStorage, sizeof(stringStorage)); + + // TODO +} + +std::string ReadFileAtOnce(const fs::path& path) { + auto file = Utils::OpenCstdioFile(path, Utils::Read); + if (!file) throw std::runtime_error("Failed to open source file."); + DEFER { fclose(file); }; + + fseek(file, 0, SEEK_END); + auto fileSize = ftell(file); + rewind(file); + + std::string result(fileSize, '\0'); + fread(result.data(), fileSize, 1, file); + + return result; +} + +void HandleArgument(InputOpcode opcode, std::string_view operand) { + switch (opcode) { + case IOP_ProcessSingleFile: { + fs::path filePath(operand); + auto source = ReadFileAtOnce(filePath); + HandleInputFile(source); + } break; + + case IOP_ProcessRecursively: { + fs::path startPath(operand); + for (auto& item : fs::directory_iterator(startPath)) { + if (!item.is_regular_file()) { + continue; + } + + auto& path = item.path(); + auto filename = path.filename().string(); + if (filename != ".c" || + filename != ".cpp") + { + continue; + } + + auto source = ReadFileAtOnce(path); + HandleInputFile(source); + } + } break; + + case IOP_COUNT: break; + } +} + +int main(int argc, char* argv[]) { + // TODO better arg parser + // option 1: use cxxopts and positional arguments + // option 1: take one argument only, being a json objecet + + // If no cli is provided (argv[0]), this loop will do nothing + // Otherwise, start with the 2nd element which is the 1st argument + for (int i = 1; i < argc; ++i) { + std::string_view arg(argv[i]); + auto separatorLoc = arg.find(':'); + if (separatorLoc != std::string_view::npos) { + auto opcode = ParseInputOpcode(arg.substr(0, separatorLoc)); + auto operand = arg.substr(separatorLoc); + HandleArgument(opcode, operand); + } + } + + return 0; +} -- cgit v1.2.3-70-g09d2 From 30e7501b006e55bdeec0db18709d3fd4c5db86b5 Mon Sep 17 00:00:00 2001 From: rtk0c Date: Fri, 27 May 2022 13:47:40 -0700 Subject: Changeset: 40 Add custom token typing mechanism --- buildtools/codegen/CodegenLookupTable.h | 19 +++ buildtools/codegen/main.cpp | 270 +++++++++++++++++++++++++++++++- conanfile.txt | 1 + source-common/MacrosCodegen.hpp | 7 +- 4 files changed, 293 insertions(+), 4 deletions(-) create mode 100644 buildtools/codegen/CodegenLookupTable.h (limited to 'buildtools/codegen') diff --git a/buildtools/codegen/CodegenLookupTable.h b/buildtools/codegen/CodegenLookupTable.h new file mode 100644 index 0000000..02c0c7a --- /dev/null +++ b/buildtools/codegen/CodegenLookupTable.h @@ -0,0 +1,19 @@ +#pragma once + +#define LUT_DECL_VAR(name, aType, aCount, bType, bCount) \ + int name##A2B[aCount]; \ + int name##B2A[bCount]; \ + using name##AType = aType; \ + using name##BType = bType; \ + void InitializeLookupTable_##name() + +#define LUT_MAP_FOR(name) \ + int* lutMappingA2B = name##A2B; \ + int* lutMappingB2A = name##B2A +#define LUT_MAP(from, to) \ + lutMappingA2B[from] = to; \ + lutMappingB2A[to] = from + +#define LUT_INIT(name) InitializeLookupTable_##name() +#define LUT_LOOKUP(name, from) (name##BType)(name##A2B[from]) +#define LUT_REV_LOOKUP(name, to) (name##AType)(name##B2A[to]) diff --git a/buildtools/codegen/main.cpp b/buildtools/codegen/main.cpp index 4a1d486..cf31bd8 100644 --- a/buildtools/codegen/main.cpp +++ b/buildtools/codegen/main.cpp @@ -1,8 +1,14 @@ +#include "CodegenLookupTable.h" +#include "Macros.hpp" #include "ScopeGuard.hpp" #include "Utils.hpp" +#include #include +#include +#include #include +#include #include #include @@ -15,6 +21,26 @@ enum InputOpcode { IOP_COUNT, }; +enum CodegenDirectives { + CD_ClassInfo, // BRUSSEL_CLASS + CD_EnumInfo, // BRUSSEL_ENUM + // TODO implement + CD_GlobalSequencer, // BRUSSEL_INIT + CD_COUNT, +}; + +enum EnumUnderlyingType { + EUT_Int8, + EUT_Int16, + EUT_Int32, + EUT_Int64, + EUT_Uint8, + EUT_Uint16, + EUT_Uint32, + EUT_Uint64, + EUT_COUNT, +}; + InputOpcode ParseInputOpcode(std::string_view text) { if (text == "single"sv) { return IOP_ProcessSingleFile; @@ -24,6 +50,147 @@ InputOpcode ParseInputOpcode(std::string_view text) { return IOP_COUNT; } +struct InputDefinitionStruct { + std::string name; +}; + +struct InputDefinitionEnumElement { + std::string name; + uint64_t value; +}; + +struct InputDefinitionEnum { + std::string name; + std::vector elements; + EnumUnderlyingType underlyingType; +}; + +enum LexedTokenType { + // stb_c_lexer token types, ported over + LTT_Identifier, + LTT_IntLiteral, + LTT_FloatLiteral, + LTT_DqString, + LTT_SqString, + LTT_CharLiteral, + LTT_OperEquals, + LTT_OperNotEquals, + LTT_OperLessOrEqual, + LTT_OperGreaterOrEqual, + LTT_OperAndAnd, + LTT_OperOrOr, + LTT_OperShiftLeft, + LTT_OperShiftRight, + LTT_OperIncrement, + LTT_OperDecrement, + LTT_OperAddAssign, + LTT_OperSubAssign, + LTT_OperMulAssign, + LTT_OperDivAssign, + LTT_OperModAssign, + LTT_OperAndAssign, + LTT_OperOrAssign, + LTT_OperXorAssign, + LTT_OperArrow, + LTT_OperEqualArrow, + LTT_OperShiftLeflAssign, + LTT_OperShiftRightAssign, + + // Custom token types + LTT_OperAdd, + LTT_OperSub, + LTT_OperMul, + LTT_OperDiv, + LTT_OperMod, + LTT_ParenOpen, + LTT_ParenClose, + LTT_BracketOpen, + LTT_BracketClose, + LTT_BraceOpen, + LTT_BraceClose, + + LTT_COUNT, +}; + +// NOTE: maintain with CLEX_* defined in stb_c_lexer.h +LUT_DECL_VAR(gClexTokens, int, CLEX_first_unused_token, LexedTokenType, LTT_COUNT) { + LUT_MAP_FOR(gClexTokens); + LUT_MAP(CLEX_id, LTT_Identifier); + LUT_MAP(CLEX_intlit, LTT_IntLiteral); + LUT_MAP(CLEX_floatlit, LTT_FloatLiteral); + LUT_MAP(CLEX_dqstring, LTT_DqString); + LUT_MAP(CLEX_sqstring, LTT_SqString); + LUT_MAP(CLEX_charlit, LTT_CharLiteral); + LUT_MAP(CLEX_eq, LTT_OperEquals); + LUT_MAP(CLEX_noteq, LTT_OperNotEquals); + LUT_MAP(CLEX_lesseq, LTT_OperLessOrEqual); + LUT_MAP(CLEX_greatereq, LTT_OperGreaterOrEqual); + LUT_MAP(CLEX_andand, LTT_OperAndAnd); + LUT_MAP(CLEX_oror, LTT_OperOrOr); + LUT_MAP(CLEX_shl, LTT_OperShiftLeft); + LUT_MAP(CLEX_shr, LTT_OperShiftRight); + LUT_MAP(CLEX_plusplus, LTT_OperIncrement); + LUT_MAP(CLEX_minusminus, LTT_OperDecrement); + LUT_MAP(CLEX_pluseq, LTT_OperAddAssign); + LUT_MAP(CLEX_minuseq, LTT_OperSubAssign); + LUT_MAP(CLEX_muleq, LTT_OperMulAssign); + LUT_MAP(CLEX_diveq, LTT_OperDivAssign); + LUT_MAP(CLEX_modeq, LTT_OperModAssign); + LUT_MAP(CLEX_andeq, LTT_OperAndAssign); + LUT_MAP(CLEX_oreq, LTT_OperOrAssign); + LUT_MAP(CLEX_xoreq, LTT_OperXorAssign); + LUT_MAP(CLEX_arrow, LTT_OperArrow); + LUT_MAP(CLEX_eqarrow, LTT_OperEqualArrow); + LUT_MAP(CLEX_shleq, LTT_OperShiftLeflAssign); + LUT_MAP(CLEX_shreq, LTT_OperShiftRightAssign); +} + +LUT_DECL_VAR(gSingleCharTokens, char, std::numeric_limits::max() + 1, LexedTokenType, LTT_COUNT) { + LUT_MAP_FOR(gSingleCharTokens); + LUT_MAP('+', LTT_OperAdd); + LUT_MAP('-', LTT_OperSub); + LUT_MAP('*', LTT_OperMul); + LUT_MAP('/', LTT_OperDiv); + LUT_MAP('%', LTT_OperMod); + LUT_MAP('(', LTT_ParenOpen); + LUT_MAP(')', LTT_ParenClose); + LUT_MAP('[', LTT_BracketOpen); + LUT_MAP(']', LTT_BracketClose); + LUT_MAP('{', LTT_BraceOpen); + LUT_MAP('}', LTT_BraceClose); +} + +// See stb_c_lexer.h's comments, here are a few additinos that aren't made clear in the file: +// - `lexer->token` (noted as "token" below) after calling stb_c_lexer_get_token() contains either: +// 1. 0 <= token < 256: an ASCII character (more precisely a single char that the lexer ate; technically can be an incomplete code unit) +// 2. token < 0: an unknown token +// 3. One of the `CLEX_*` enums: a special, recognized token such as an operator +LexedTokenType MapFromStb(const stb_lexer& lexer) { + if (lexer.token >= 0 && lexer.token < 256) { + // Single char token + char c = lexer.token; + return LUT_LOOKUP(gSingleCharTokens, lexer.token); + } + + return LUT_LOOKUP(gClexTokens, lexer.token); +} +int MapToStb(LexedTokenType token) { + // TODO + + return LUT_REV_LOOKUP(gClexTokens, token); +} + +struct StbLexerToken { + std::string text; + LexedTokenType type; +}; + +void CheckBraceDepth(int braceDpeth) { + if (braceDpeth < 0) { + printf("[WARNING] unbalanced brace"); + } +} + void HandleInputFile(std::string_view source) { stb_lexer lexer; char stringStorage[65536]; @@ -31,7 +198,103 @@ void HandleInputFile(std::string_view source) { const char* srcEnd = srcBegin + source.length(); stb_c_lexer_init(&lexer, srcBegin, srcEnd, stringStorage, sizeof(stringStorage)); - // TODO + std::vector tokens; + std::vector foundStructs; + std::vector foundEnums; + + enum NextMatchingConstruct { + NMC_None, + NMC_Enum, + NMC_StructClass, + } matchingConstruct = NMC_None; + + int bracePairDepth = 0; + + while (true) { + int stbToken = stb_c_lexer_get_token(&lexer); + if (stbToken == 0) { + // EOF + break; + } + + // TODO needed? + // StbLexerToken token; + // token.type = lexer.token; + // token.text = std::string(lexer.string, lexer.string_len); + // tokens.push_back(token); + + switch (lexer.token) { + case CLEX_id: { + std::string_view idenText(lexer.string, lexer.string_len); + if (idenText == "struct"sv || idenText == "class"sv) { + // TODO + matchingConstruct = NMC_StructClass; + } else if (idenText == "enum"sv) { + // TODO + matchingConstruct = NMC_Enum; + } + } break; + + case CLEX_intlit: + case CLEX_floatlit: + case CLEX_dqstring: + case CLEX_sqstring: + case CLEX_charlit: + case CLEX_eq: + case CLEX_noteq: + case CLEX_lesseq: + case CLEX_greatereq: + case CLEX_andand: + case CLEX_oror: + case CLEX_shl: + case CLEX_shr: + case CLEX_plusplus: + case CLEX_minusminus: + case CLEX_pluseq: + case CLEX_minuseq: + case CLEX_muleq: + case CLEX_diveq: + case CLEX_modeq: + case CLEX_andeq: + case CLEX_oreq: + case CLEX_xoreq: + case CLEX_arrow: + case CLEX_eqarrow: + case CLEX_shleq: + case CLEX_shreq: { + + } break; + + case CLEX_parse_error: { + fprintf(stderr, "[ERROR] stb_c_lexer countered a parse error."); + // TODO how to handle? + } break; + + default: { + if (lexer.token >= 0 && lexer.token < 256) { + // Single char token + char c = lexer.token; + switch (c) { + case '{': { + bracePairDepth++; + CheckBraceDepth(bracePairDepth); + } break; + + case '}': { + bracePairDepth--; + CheckBraceDepth(bracePairDepth); + } break; + } + } else { + fprintf(stderr, "[ERROR] Encountered unknown token %ld.", lexer.token); + } + } break; + } + } + + if (bracePairDepth != 0) { + printf("[WARNING] unbalanced brace at end of file."); + } } std::string ReadFileAtOnce(const fs::path& path) { @@ -59,7 +322,7 @@ void HandleArgument(InputOpcode opcode, std::string_view operand) { case IOP_ProcessRecursively: { fs::path startPath(operand); - for (auto& item : fs::directory_iterator(startPath)) { + for (auto& item : fs::recursive_directory_iterator(startPath)) { if (!item.is_regular_file()) { continue; } @@ -82,6 +345,9 @@ void HandleArgument(InputOpcode opcode, std::string_view operand) { } int main(int argc, char* argv[]) { + LUT_INIT(gClexTokens); + LUT_INIT(gSingleCharTokens); + // TODO better arg parser // option 1: use cxxopts and positional arguments // option 1: take one argument only, being a json objecet diff --git a/conanfile.txt b/conanfile.txt index f80978c..2b03fbb 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -5,6 +5,7 @@ assimp/5.2.2 rapidjson/cci.20211112 stb/cci.20210910 robin-hood-hashing/3.11.5 +frozen/1.1.1 [generators] cmake diff --git a/source-common/MacrosCodegen.hpp b/source-common/MacrosCodegen.hpp index 6c93d09..d6d5c6f 100644 --- a/source-common/MacrosCodegen.hpp +++ b/source-common/MacrosCodegen.hpp @@ -1,4 +1,7 @@ +// NOTE: contents of this file is coupled with buildtools/codegen/ +// when updating, change both sides at the same time + #pragma once -#define BRUSSEL_METACLASS -#define BRUSSEL_METAENUM(name) +#define BRUSSEL_CLASS +#define BRUSSEL_ENUM(name, options) -- cgit v1.2.3-70-g09d2 From 6c3007295a6a8c6803933392834c974ec5f56aa0 Mon Sep 17 00:00:00 2001 From: rtk0c Date: Fri, 27 May 2022 17:31:04 -0700 Subject: Changeset: 41 Add logic to parse enums --- CMakeLists.txt | 4 + buildtools/cmake/RTTI.cmake | 31 +++ buildtools/codegen/CodegenDefinition.hpp | 1 + buildtools/codegen/CodegenOutput.cpp | 7 + buildtools/codegen/CodegenOutput.hpp | 17 ++ buildtools/codegen/main.cpp | 257 +++++++++++-------------- buildtools/codegen/tests/examples/TestEnum.hpp | 8 + 7 files changed, 178 insertions(+), 147 deletions(-) create mode 100644 buildtools/cmake/RTTI.cmake create mode 100644 buildtools/codegen/CodegenDefinition.hpp create mode 100644 buildtools/codegen/CodegenOutput.cpp create mode 100644 buildtools/codegen/CodegenOutput.hpp create mode 100644 buildtools/codegen/tests/examples/TestEnum.hpp (limited to 'buildtools/codegen') diff --git a/CMakeLists.txt b/CMakeLists.txt index 3d349c5..19518bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,8 @@ project(ProjectBrussel LANGUAGES C CXX) include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) conan_basic_setup() +include(buildtools/cmake/RTTI.cmake) + find_package(OpenGL REQUIRED) add_subdirectory(3rdparty/glfw) add_subdirectory(3rdparty/glm) @@ -53,6 +55,8 @@ target_link_libraries(meta_codegen PRIVATE commonthings ) +target_flag_rtti(meta_codegen OFF) + # ============================================================================== # add_executable requires at least one source file diff --git a/buildtools/cmake/RTTI.cmake b/buildtools/cmake/RTTI.cmake new file mode 100644 index 0000000..b948497 --- /dev/null +++ b/buildtools/cmake/RTTI.cmake @@ -0,0 +1,31 @@ +function(target_flag_rtti_msvc TARGET_NAME ENABLED) + if(ENABLED) + target_compile_options(${TARGET_NAME} PRIVATE /GR) + else() + target_compile_options(${TARGET_NAME} PRIVATE /GR-) + endif() +endfunction() + +function(target_flag_rtti_gcc TARGET_NAME ENABLED) + if(ENABLED) + target_compile_options(${TARGET_NAME} PRIVATE -frtti) + else() + target_compile_options(${TARGET_NAME} PRIVATE -fno-rtti) + endif() +endfunction() + +function(target_flag_rtti TARGET_NAME ENABLED) + if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + target_flag_rtti_msvc(${TARGET_NAME} ${ENABLED}) + elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT MATCHES "MSVC") + target_flag_rtti_msvc(${TARGET_NAME} ${ENABLED}) + elseif(CMAKE_CXX_COMPILER_FRONTEND_VARIANT MATCHES "GNU") + target_flag_rtti_gcc(${TARGET_NAME} ${ENABLED}) + endif() + elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + target_flag_rtti_gcc(${TARGET_NAME} ${ENABLED}) + else() + message(FATAL "target_flag_rtti(): Unknown compiler ${CMAKE_CXX_COMPILER_ID}") + endif() +endfunction() diff --git a/buildtools/codegen/CodegenDefinition.hpp b/buildtools/codegen/CodegenDefinition.hpp new file mode 100644 index 0000000..6f70f09 --- /dev/null +++ b/buildtools/codegen/CodegenDefinition.hpp @@ -0,0 +1 @@ +#pragma once diff --git a/buildtools/codegen/CodegenOutput.cpp b/buildtools/codegen/CodegenOutput.cpp new file mode 100644 index 0000000..214ded6 --- /dev/null +++ b/buildtools/codegen/CodegenOutput.cpp @@ -0,0 +1,7 @@ +#include "CodegenOutput.hpp" + +#include + +void CodegenOutput::AddOutputThing(CodegenOutputThing thing) { + mOutThings.push_back(std::move(thing)); +} diff --git a/buildtools/codegen/CodegenOutput.hpp b/buildtools/codegen/CodegenOutput.hpp new file mode 100644 index 0000000..660feb1 --- /dev/null +++ b/buildtools/codegen/CodegenOutput.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +// A generic "thing" (could be anything, comments, string-concated functionsm, etc.) to spit into the output file +struct CodegenOutputThing { + std::string text; +}; + +class CodegenOutput { +private: + std::vector mOutThings; + +public: + void AddOutputThing(CodegenOutputThing thing); +}; diff --git a/buildtools/codegen/main.cpp b/buildtools/codegen/main.cpp index cf31bd8..9f89191 100644 --- a/buildtools/codegen/main.cpp +++ b/buildtools/codegen/main.cpp @@ -1,4 +1,5 @@ #include "CodegenLookupTable.h" +#include "CodegenOutput.hpp" #include "Macros.hpp" #include "ScopeGuard.hpp" #include "Utils.hpp" @@ -65,124 +66,10 @@ struct InputDefinitionEnum { EnumUnderlyingType underlyingType; }; -enum LexedTokenType { - // stb_c_lexer token types, ported over - LTT_Identifier, - LTT_IntLiteral, - LTT_FloatLiteral, - LTT_DqString, - LTT_SqString, - LTT_CharLiteral, - LTT_OperEquals, - LTT_OperNotEquals, - LTT_OperLessOrEqual, - LTT_OperGreaterOrEqual, - LTT_OperAndAnd, - LTT_OperOrOr, - LTT_OperShiftLeft, - LTT_OperShiftRight, - LTT_OperIncrement, - LTT_OperDecrement, - LTT_OperAddAssign, - LTT_OperSubAssign, - LTT_OperMulAssign, - LTT_OperDivAssign, - LTT_OperModAssign, - LTT_OperAndAssign, - LTT_OperOrAssign, - LTT_OperXorAssign, - LTT_OperArrow, - LTT_OperEqualArrow, - LTT_OperShiftLeflAssign, - LTT_OperShiftRightAssign, - - // Custom token types - LTT_OperAdd, - LTT_OperSub, - LTT_OperMul, - LTT_OperDiv, - LTT_OperMod, - LTT_ParenOpen, - LTT_ParenClose, - LTT_BracketOpen, - LTT_BracketClose, - LTT_BraceOpen, - LTT_BraceClose, - - LTT_COUNT, -}; - -// NOTE: maintain with CLEX_* defined in stb_c_lexer.h -LUT_DECL_VAR(gClexTokens, int, CLEX_first_unused_token, LexedTokenType, LTT_COUNT) { - LUT_MAP_FOR(gClexTokens); - LUT_MAP(CLEX_id, LTT_Identifier); - LUT_MAP(CLEX_intlit, LTT_IntLiteral); - LUT_MAP(CLEX_floatlit, LTT_FloatLiteral); - LUT_MAP(CLEX_dqstring, LTT_DqString); - LUT_MAP(CLEX_sqstring, LTT_SqString); - LUT_MAP(CLEX_charlit, LTT_CharLiteral); - LUT_MAP(CLEX_eq, LTT_OperEquals); - LUT_MAP(CLEX_noteq, LTT_OperNotEquals); - LUT_MAP(CLEX_lesseq, LTT_OperLessOrEqual); - LUT_MAP(CLEX_greatereq, LTT_OperGreaterOrEqual); - LUT_MAP(CLEX_andand, LTT_OperAndAnd); - LUT_MAP(CLEX_oror, LTT_OperOrOr); - LUT_MAP(CLEX_shl, LTT_OperShiftLeft); - LUT_MAP(CLEX_shr, LTT_OperShiftRight); - LUT_MAP(CLEX_plusplus, LTT_OperIncrement); - LUT_MAP(CLEX_minusminus, LTT_OperDecrement); - LUT_MAP(CLEX_pluseq, LTT_OperAddAssign); - LUT_MAP(CLEX_minuseq, LTT_OperSubAssign); - LUT_MAP(CLEX_muleq, LTT_OperMulAssign); - LUT_MAP(CLEX_diveq, LTT_OperDivAssign); - LUT_MAP(CLEX_modeq, LTT_OperModAssign); - LUT_MAP(CLEX_andeq, LTT_OperAndAssign); - LUT_MAP(CLEX_oreq, LTT_OperOrAssign); - LUT_MAP(CLEX_xoreq, LTT_OperXorAssign); - LUT_MAP(CLEX_arrow, LTT_OperArrow); - LUT_MAP(CLEX_eqarrow, LTT_OperEqualArrow); - LUT_MAP(CLEX_shleq, LTT_OperShiftLeflAssign); - LUT_MAP(CLEX_shreq, LTT_OperShiftRightAssign); -} - -LUT_DECL_VAR(gSingleCharTokens, char, std::numeric_limits::max() + 1, LexedTokenType, LTT_COUNT) { - LUT_MAP_FOR(gSingleCharTokens); - LUT_MAP('+', LTT_OperAdd); - LUT_MAP('-', LTT_OperSub); - LUT_MAP('*', LTT_OperMul); - LUT_MAP('/', LTT_OperDiv); - LUT_MAP('%', LTT_OperMod); - LUT_MAP('(', LTT_ParenOpen); - LUT_MAP(')', LTT_ParenClose); - LUT_MAP('[', LTT_BracketOpen); - LUT_MAP(']', LTT_BracketClose); - LUT_MAP('{', LTT_BraceOpen); - LUT_MAP('}', LTT_BraceClose); -} - -// See stb_c_lexer.h's comments, here are a few additinos that aren't made clear in the file: -// - `lexer->token` (noted as "token" below) after calling stb_c_lexer_get_token() contains either: -// 1. 0 <= token < 256: an ASCII character (more precisely a single char that the lexer ate; technically can be an incomplete code unit) -// 2. token < 0: an unknown token -// 3. One of the `CLEX_*` enums: a special, recognized token such as an operator -LexedTokenType MapFromStb(const stb_lexer& lexer) { - if (lexer.token >= 0 && lexer.token < 256) { - // Single char token - char c = lexer.token; - return LUT_LOOKUP(gSingleCharTokens, lexer.token); - } - - return LUT_LOOKUP(gClexTokens, lexer.token); -} -int MapToStb(LexedTokenType token) { - // TODO - - return LUT_REV_LOOKUP(gClexTokens, token); -} - struct StbLexerToken { std::string text; - LexedTokenType type; + // Can either be CLEX_* values, or just chars for single character tokens + int type; }; void CheckBraceDepth(int braceDpeth) { @@ -200,17 +87,37 @@ void HandleInputFile(std::string_view source) { std::vector tokens; std::vector foundStructs; + InputDefinitionStruct currStruct; std::vector foundEnums; + InputDefinitionEnum currEnum; + + auto PushFoundStruct = [&]() { + foundStructs.push_back(std::move(currStruct)); + currStruct = {}; + }; + auto PushFoundEnum = [&]() { + foundEnums.push_back(std::move(currEnum)); + currEnum = {}; + }; enum NextMatchingConstruct { NMC_None, NMC_Enum, NMC_StructClass, } matchingConstruct = NMC_None; + bool matchingConstructInBody = false; + + bool matchingDirectiveParams = false; int bracePairDepth = 0; while (true) { + // See stb_c_lexer.h's comments, here are a few additinos that aren't made clear in the file: + // - `lexer->token` (noted as "token" below) after calling stb_c_lexer_get_token() contains either: + // 1. 0 <= token < 256: an ASCII character (more precisely a single char that the lexer ate; technically can be an incomplete code unit) + // 2. token < 0: an unknown token + // 3. One of the `CLEX_*` enums: a special, recognized token such as an operator + int stbToken = stb_c_lexer_get_token(&lexer); if (stbToken == 0) { // EOF @@ -225,13 +132,53 @@ void HandleInputFile(std::string_view source) { switch (lexer.token) { case CLEX_id: { - std::string_view idenText(lexer.string, lexer.string_len); - if (idenText == "struct"sv || idenText == "class"sv) { - // TODO - matchingConstruct = NMC_StructClass; - } else if (idenText == "enum"sv) { - // TODO - matchingConstruct = NMC_Enum; + // WORKAROUND: stb_c_lexer doens't set string_len properly when parsing identifiers + std::string_view idenText(lexer.string); + // std::string_view idenText(lexer.string, lexer.string_len); + switch (matchingConstruct) { + case NMC_StructClass: { + if (matchingConstructInBody) { + // TODO + } + } break; + + case NMC_Enum: { + if (matchingConstructInBody) { + printf("[DEBUG] found enum element '%s'\n", lexer.string); + currEnum.elements.push_back(InputDefinitionEnumElement{ + .name = std::string(idenText), + .value = 0, // TODO parse + }); + } else { + currEnum.name = std::string(idenText); + } + } break; + + default: { + if (idenText == "struct"sv || idenText == "class"sv) { + printf("[DEBUG] found struct named\n"); + matchingConstruct = NMC_StructClass; + } else if (idenText == "enum"sv) { + printf("[DEBUG] found enum\n"); + matchingConstruct = NMC_Enum; + } else if (idenText == "BRUSSEL_CLASS"sv) { + // TODO + printf("[DEBUG] found BRUSSEL_CLASS\n"); + } else if (idenText == "BRUSSEL_ENUM"sv) { + matchingDirectiveParams = true; + printf("[DEBUG] found BRUSSEL_ENUM\n"); + } + + if (matchingDirectiveParams) { + for (auto& foundEnum : foundEnums) { + if (foundEnum.name == idenText) { + // TODO generate data + break; + } + } + matchingDirectiveParams = false; + } + } break; } } break; @@ -265,30 +212,49 @@ void HandleInputFile(std::string_view source) { } break; - case CLEX_parse_error: { - fprintf(stderr, "[ERROR] stb_c_lexer countered a parse error."); - // TODO how to handle? + case '{': { + bracePairDepth++; + CheckBraceDepth(bracePairDepth); + + switch (matchingConstruct) { + case NMC_StructClass: + case NMC_Enum: { + matchingConstructInBody = true; + } break; + + default: break; + } } break; - default: { - if (lexer.token >= 0 && lexer.token < 256) { - // Single char token - char c = lexer.token; - switch (c) { - case '{': { - bracePairDepth++; - CheckBraceDepth(bracePairDepth); - } break; - - case '}': { - bracePairDepth--; - CheckBraceDepth(bracePairDepth); - } break; - } - } else { - fprintf(stderr, "[ERROR] Encountered unknown token %ld.", lexer.token); + case '}': { + bracePairDepth--; + CheckBraceDepth(bracePairDepth); + + switch (matchingConstruct) { + case NMC_StructClass: { + matchingConstruct = NMC_None; + matchingConstructInBody = false; + } break; + + case NMC_Enum: { + printf("[DEBUG] committed enum '%s'\n", currEnum.name.c_str()); + for (auto& elm : currEnum.elements) { + printf(" - element %s = %" PRId64 "\n", elm.name.c_str(), elm.value); + } + + matchingConstruct = NMC_None; + matchingConstructInBody = false; + PushFoundEnum(); + } break; + + default: break; } } break; + + case CLEX_parse_error: { + fprintf(stderr, "[ERROR] stb_c_lexer countered a parse error."); + // TODO how to handle? + } break; } } @@ -345,9 +311,6 @@ void HandleArgument(InputOpcode opcode, std::string_view operand) { } int main(int argc, char* argv[]) { - LUT_INIT(gClexTokens); - LUT_INIT(gSingleCharTokens); - // TODO better arg parser // option 1: use cxxopts and positional arguments // option 1: take one argument only, being a json objecet @@ -359,7 +322,7 @@ int main(int argc, char* argv[]) { auto separatorLoc = arg.find(':'); if (separatorLoc != std::string_view::npos) { auto opcode = ParseInputOpcode(arg.substr(0, separatorLoc)); - auto operand = arg.substr(separatorLoc); + auto operand = arg.substr(separatorLoc + 1); HandleArgument(opcode, operand); } } diff --git a/buildtools/codegen/tests/examples/TestEnum.hpp b/buildtools/codegen/tests/examples/TestEnum.hpp new file mode 100644 index 0000000..aaa3d74 --- /dev/null +++ b/buildtools/codegen/tests/examples/TestEnum.hpp @@ -0,0 +1,8 @@ +#include "MacrosCodegen.hpp" + +enum MyEnum { + EnumElement1, + EnumElement2, + EnumElement3, +}; +BRUSSEL_ENUM(MyEnum, GenInfo); -- cgit v1.2.3-70-g09d2 From 1a6f1ea3b76c3ed4cad5aba5502af390ce50a2c0 Mon Sep 17 00:00:00 2001 From: rtk0c Date: Sat, 28 May 2022 20:52:42 -0700 Subject: Changeset: 42 Change codegen input parsing to lookahead based; lookup table infra; input/output decl infra --- buildtools/codegen/CodegenConfig.hpp | 9 + buildtools/codegen/CodegenDecl.hpp | 32 ++ buildtools/codegen/CodegenDefinition.hpp | 1 - buildtools/codegen/CodegenInput.inl | 40 ++ buildtools/codegen/CodegenLookupTable.h | 19 - buildtools/codegen/CodegenLookupTable.hpp | 50 +++ buildtools/codegen/CodegenOutput.cpp | 7 - buildtools/codegen/CodegenOutput.hpp | 17 - buildtools/codegen/CodegenOutput.inl | 31 ++ buildtools/codegen/main.cpp | 517 ++++++++++++++++--------- buildtools/codegen/tests/examples/TestEnum.hpp | 2 +- source-common/Enum.hpp | 12 +- source-common/MacrosCodegen.hpp | 2 +- 13 files changed, 501 insertions(+), 238 deletions(-) create mode 100644 buildtools/codegen/CodegenConfig.hpp create mode 100644 buildtools/codegen/CodegenDecl.hpp delete mode 100644 buildtools/codegen/CodegenDefinition.hpp create mode 100644 buildtools/codegen/CodegenInput.inl delete mode 100644 buildtools/codegen/CodegenLookupTable.h create mode 100644 buildtools/codegen/CodegenLookupTable.hpp delete mode 100644 buildtools/codegen/CodegenOutput.cpp delete mode 100644 buildtools/codegen/CodegenOutput.hpp create mode 100644 buildtools/codegen/CodegenOutput.inl (limited to 'buildtools/codegen') diff --git a/buildtools/codegen/CodegenConfig.hpp b/buildtools/codegen/CodegenConfig.hpp new file mode 100644 index 0000000..4ed576a --- /dev/null +++ b/buildtools/codegen/CodegenConfig.hpp @@ -0,0 +1,9 @@ +#pragma once + +#define CODEGEN_DEBUG_PRINT 1 + +#if CODEGEN_DEBUG_PRINT +# define DEBUG_PRINTF(...) printf(__VA_ARGS__) +#else +# define DEBUG_PRINTF(...) +#endif diff --git a/buildtools/codegen/CodegenDecl.hpp b/buildtools/codegen/CodegenDecl.hpp new file mode 100644 index 0000000..3414d80 --- /dev/null +++ b/buildtools/codegen/CodegenDecl.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +// Structs or classes +struct DeclStruct { + std::string name; +}; + +enum EnumUnderlyingType { + EUT_Int8, + EUT_Int16, + EUT_Int32, + EUT_Int64, + EUT_Uint8, + EUT_Uint16, + EUT_Uint32, + EUT_Uint64, + EUT_COUNT, +}; + +struct DeclEnumElement { + std::string name; + uint64_t value; +}; + +struct DeclEnum { + std::string name; + std::vector elements; + EnumUnderlyingType underlyingType; +}; diff --git a/buildtools/codegen/CodegenDefinition.hpp b/buildtools/codegen/CodegenDefinition.hpp deleted file mode 100644 index 6f70f09..0000000 --- a/buildtools/codegen/CodegenDefinition.hpp +++ /dev/null @@ -1 +0,0 @@ -#pragma once diff --git a/buildtools/codegen/CodegenInput.inl b/buildtools/codegen/CodegenInput.inl new file mode 100644 index 0000000..9fae43c --- /dev/null +++ b/buildtools/codegen/CodegenInput.inl @@ -0,0 +1,40 @@ +#pragma once + +#include "CodegenConfig.hpp" +#include "CodegenDecl.hpp" + +#include + +#include +#include +#include +#include + +class CodegenInput { +private: + std::vector mEnums; + robin_hood::unordered_map mDeclByName; + +public: + void AddEnum(DeclEnum decl) { +#if CODEGEN_DEBUG_PRINT + printf("Committed enum '%s'\n", decl.name.c_str()); + for (auto& elm : decl.elements) { + printf(" - element %s = %" PRId64 "\n", elm.name.c_str(), elm.value); + } +#endif + + mDeclByName.try_emplace(decl.name, mEnums.size()); + mEnums.push_back(std::move(decl)); + } + + const DeclEnum* FindEnumByName(std::string_view name) const { + // TODO handle multiple kinds of decl + auto iter = mDeclByName.find(name); + if (iter != mDeclByName.end()) { + return &mEnums[iter->second]; + } else { + return nullptr; + } + } +}; diff --git a/buildtools/codegen/CodegenLookupTable.h b/buildtools/codegen/CodegenLookupTable.h deleted file mode 100644 index 02c0c7a..0000000 --- a/buildtools/codegen/CodegenLookupTable.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#define LUT_DECL_VAR(name, aType, aCount, bType, bCount) \ - int name##A2B[aCount]; \ - int name##B2A[bCount]; \ - using name##AType = aType; \ - using name##BType = bType; \ - void InitializeLookupTable_##name() - -#define LUT_MAP_FOR(name) \ - int* lutMappingA2B = name##A2B; \ - int* lutMappingB2A = name##B2A -#define LUT_MAP(from, to) \ - lutMappingA2B[from] = to; \ - lutMappingB2A[to] = from - -#define LUT_INIT(name) InitializeLookupTable_##name() -#define LUT_LOOKUP(name, from) (name##BType)(name##A2B[from]) -#define LUT_REV_LOOKUP(name, to) (name##AType)(name##B2A[to]) diff --git a/buildtools/codegen/CodegenLookupTable.hpp b/buildtools/codegen/CodegenLookupTable.hpp new file mode 100644 index 0000000..2b6a993 --- /dev/null +++ b/buildtools/codegen/CodegenLookupTable.hpp @@ -0,0 +1,50 @@ +#pragma once + +// BIDI stands for bi-directional +#define BIDI_LUT_DECL(name, aType, aCount, bType, bCount) \ + int gLut_##name##_A2B[aCount]; \ + int gLut_##name##_B2A[bCount]; \ + using name##AType = aType; \ + using name##BType = bType; \ + void InitializeLut##name() +#define BIDI_LUT_MAP_FOR(name) \ + int* lutMappingA2B = gLut_##name##_A2B; \ + int* lutMappingB2A = gLut_##name##_B2A +#define BIDI_LUT_MAP(from, to) \ + lutMappingA2B[from] = to; \ + lutMappingB2A[to] = from +#define BIDI_LUT_INIT(name) InitializeLut##name() +#define BIDI_LUT_A2B_LOOKUP(name, from) (name##BType)(gLut_##name##_A2B[from]) +#define BIDI_LUT_B2A_LOOKUP(name, to) (name##AType)(gLut_##name##_B2A[to]) + +#define STR_LUT_DECL(name, enumMinValue, enumMaxValue) \ + constexpr int kLutMinVal_##name = enumMinValue; \ + const char* gLut_##name[(int)enumMaxValue - (int)enumMinValue]; \ + void InitializeLut##name() +#define STR_LUT_MAP_FOR(name) \ + const char** lutMapping = gLut_##name; \ + int lutMappingMinValue = kLutMinVal_##name +#define STR_LUT_MAP(value, text) lutMapping[value - lutMappingMinValue] = text +#define STR_LUT_MAP_ENUM(enumValue) STR_LUT_MAP(enumValue, #enumValue) +#define STR_LUT_LOOKUP(name, enumValue) gLut_##name[enumValue - kLutMinVal_##name] +#define STR_LUT_INIT(name) InitializeLut##name() + +// BSTR stands for bi-directional string +#define BSTR_LUT_DECL(name, enumMinValue, enumMaxValue) \ + constexpr int kLutMinVal_##name = enumMinValue; \ + const char* gLut_##name##_V2S[(int)enumMaxValue - (int)enumMinValue]; \ + robin_hood::unordered_flat_map gLut_##name##_S2V; \ + void InitializeLut##name() +#define BSTR_LUT_MAP_FOR(name) \ + const char** lutMappingV2S = gLut_##name##_V2S; \ + auto& lutMappingS2V = gLut_##name##_S2V; \ + int lutMappingMinValue = kLutMinVal_##name +#define BSTR_LUT_MAP(value, text) \ + lutMappingV2S[value - lutMappingMinValue] = text; \ + lutMappingS2V.insert_or_assign(std::string_view(text), value); +#define BSTR_LUT_MAP_ENUM(enumValue) BSTR_LUT_MAP(enumValue, #enumValue) +#define BSTR_LUT_V2S(name) gLut_##name##_V2S +#define BSTR_LUT_S2V(name) gLut_##name##_S2V +#define BSTR_LUT_V2S_LOOKUP(name, enumValue) gLut_##name##_V2S[enumValue - kLutMinVal_##name] +#define BSTR_LUT_S2V_LOOKUP(name, string) gLut_##name##_S2V.find(std::string_view(text)) +#define BSTR_LUT_INIT(name) InitializeLut##name() diff --git a/buildtools/codegen/CodegenOutput.cpp b/buildtools/codegen/CodegenOutput.cpp deleted file mode 100644 index 214ded6..0000000 --- a/buildtools/codegen/CodegenOutput.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "CodegenOutput.hpp" - -#include - -void CodegenOutput::AddOutputThing(CodegenOutputThing thing) { - mOutThings.push_back(std::move(thing)); -} diff --git a/buildtools/codegen/CodegenOutput.hpp b/buildtools/codegen/CodegenOutput.hpp deleted file mode 100644 index 660feb1..0000000 --- a/buildtools/codegen/CodegenOutput.hpp +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include -#include - -// A generic "thing" (could be anything, comments, string-concated functionsm, etc.) to spit into the output file -struct CodegenOutputThing { - std::string text; -}; - -class CodegenOutput { -private: - std::vector mOutThings; - -public: - void AddOutputThing(CodegenOutputThing thing); -}; diff --git a/buildtools/codegen/CodegenOutput.inl b/buildtools/codegen/CodegenOutput.inl new file mode 100644 index 0000000..6d59301 --- /dev/null +++ b/buildtools/codegen/CodegenOutput.inl @@ -0,0 +1,31 @@ +#pragma once + +#include "CodegenDecl.hpp" + +#include +#include +#include +#include + +// A generic "thing" (could be anything, comments, string-concated functionsm, etc.) to spit into the output file +struct CodegenOutputThing { + std::string text; +}; + +class CodegenOutput { +private: + std::vector mOutThings; + +public: + void AddOutputThing(CodegenOutputThing thing) { + mOutThings.push_back(std::move(thing)); + } + + void MergeContents(CodegenOutput other) { + std::move(other.mOutThings.begin(), other.mOutThings.end(), this->mOutThings.begin()); + } + + void Write(FILE* file) { + // TODO + } +}; diff --git a/buildtools/codegen/main.cpp b/buildtools/codegen/main.cpp index 9f89191..74acd1c 100644 --- a/buildtools/codegen/main.cpp +++ b/buildtools/codegen/main.cpp @@ -1,84 +1,170 @@ -#include "CodegenLookupTable.h" -#include "CodegenOutput.hpp" -#include "Macros.hpp" -#include "ScopeGuard.hpp" -#include "Utils.hpp" +#include "CodegenConfig.hpp" +#include "CodegenDecl.hpp" +#include "CodegenLookupTable.hpp" -#include +#include "CodegenInput.inl" +#include "CodegenOutput.inl" + +#include +#include +#include +#include + +#include #include #include #include #include #include +#include #include #include using namespace std::literals; namespace fs = std::filesystem; -enum InputOpcode { - IOP_ProcessSingleFile, - IOP_ProcessRecursively, - IOP_COUNT, +struct AppState { + CodegenOutput mainOutput; }; -enum CodegenDirectives { - CD_ClassInfo, // BRUSSEL_CLASS - CD_EnumInfo, // BRUSSEL_ENUM - // TODO implement - CD_GlobalSequencer, // BRUSSEL_INIT - CD_COUNT, +enum { + CLEX_ext_single_char = CLEX_first_unused_token, + CLEX_ext_COUNT, }; -enum EnumUnderlyingType { - EUT_Int8, - EUT_Int16, - EUT_Int32, - EUT_Int64, - EUT_Uint8, - EUT_Uint16, - EUT_Uint32, - EUT_Uint64, - EUT_COUNT, -}; - -InputOpcode ParseInputOpcode(std::string_view text) { - if (text == "single"sv) { - return IOP_ProcessSingleFile; - } else if (text == "rec"sv) { - return IOP_ProcessRecursively; - } - return IOP_COUNT; +STR_LUT_DECL(ClexNames, CLEX_eof, CLEX_ext_COUNT) { + STR_LUT_MAP_FOR(ClexNames); + STR_LUT_MAP_ENUM(CLEX_intlit); + STR_LUT_MAP_ENUM(CLEX_floatlit); + STR_LUT_MAP_ENUM(CLEX_id); + STR_LUT_MAP_ENUM(CLEX_dqstring); + STR_LUT_MAP_ENUM(CLEX_sqstring); + STR_LUT_MAP_ENUM(CLEX_charlit); + STR_LUT_MAP_ENUM(CLEX_eq); + STR_LUT_MAP_ENUM(CLEX_noteq); + STR_LUT_MAP_ENUM(CLEX_lesseq); + STR_LUT_MAP_ENUM(CLEX_greatereq); + STR_LUT_MAP_ENUM(CLEX_andand); + STR_LUT_MAP_ENUM(CLEX_oror); + STR_LUT_MAP_ENUM(CLEX_shl); + STR_LUT_MAP_ENUM(CLEX_shr); + STR_LUT_MAP_ENUM(CLEX_plusplus); + STR_LUT_MAP_ENUM(CLEX_minusminus); + STR_LUT_MAP_ENUM(CLEX_pluseq); + STR_LUT_MAP_ENUM(CLEX_minuseq); + STR_LUT_MAP_ENUM(CLEX_muleq); + STR_LUT_MAP_ENUM(CLEX_diveq); + STR_LUT_MAP_ENUM(CLEX_modeq); + STR_LUT_MAP_ENUM(CLEX_andeq); + STR_LUT_MAP_ENUM(CLEX_oreq); + STR_LUT_MAP_ENUM(CLEX_xoreq); + STR_LUT_MAP_ENUM(CLEX_arrow); + STR_LUT_MAP_ENUM(CLEX_eqarrow); + STR_LUT_MAP_ENUM(CLEX_shleq); + STR_LUT_MAP_ENUM(CLEX_shreq); + STR_LUT_MAP_ENUM(CLEX_ext_single_char); } -struct InputDefinitionStruct { - std::string name; +enum CppKeyword { + CKw_Struct, + CKw_Class, + CKw_Enum, + CKw_COUNT, }; -struct InputDefinitionEnumElement { - std::string name; - uint64_t value; -}; +BSTR_LUT_DECL(CppKeyword, 0, CKw_COUNT) { + BSTR_LUT_MAP_FOR(CppKeyword); + BSTR_LUT_MAP(CKw_Struct, "struct"); + BSTR_LUT_MAP(CKw_Class, "class"); +} -struct InputDefinitionEnum { - std::string name; - std::vector elements; - EnumUnderlyingType underlyingType; +enum CodegenDirective { + CD_ClassInfo, + CD_EnumInfo, + CD_COUNT, }; +BSTR_LUT_DECL(CodegenDirective, 0, CD_COUNT) { + BSTR_LUT_MAP_FOR(CodegenDirective); + BSTR_LUT_MAP(CD_ClassInfo, "BRUSSEL_CLASS"); + BSTR_LUT_MAP(CD_EnumInfo, "BRUSSEL_ENUM"); +} + struct StbLexerToken { std::string text; - // Can either be CLEX_* values, or just chars for single character tokens + // Can either be CLEX_* or CLEX_ext_* values int type; }; +bool StbTokenIsSingleChar(int lexerToken) { + return lexerToken >= 0 && lexerToken < 256; +} + +bool StbTokenIsMultiChar(int lexerToken) { + return !StbTokenIsMultiChar(lexerToken); +} + void CheckBraceDepth(int braceDpeth) { if (braceDpeth < 0) { printf("[WARNING] unbalanced brace"); } } -void HandleInputFile(std::string_view source) { +const StbLexerToken* +PeekTokenOfTypeAt(const std::vector& tokens, size_t idx, int type) { + auto& token = tokens[idx]; + if (token.type != type) { + return nullptr; + } + + return &token; +} + +std::pair +PeekTokenOfType(const std::vector& tokens, size_t current, int type) { + for (size_t i = current; i < tokens.size(); ++i) { + if (auto token = PeekTokenOfTypeAt(tokens, i, type)) { + return { token, i }; + } + } + return { nullptr, current }; +} + +std::pair>, size_t> +PeekDirectiveArgumentList(const std::vector& tokens, size_t current) { + std::vector> result; + decltype(result)::value_type currentArg; + + size_t i = current; + int parenDepth = 0; + for (; i < tokens.size(); ++i) { + auto& token = tokens[i]; + if (token.text[0] == '(') { + if (parenDepth > 0) { + currentArg.push_back(&token); + } + ++parenDepth; + } else if (token.text[0] == ')') { + --parenDepth; + if (parenDepth == 0) { + // End of argument list + break; + } + } else if (parenDepth > 0) { + // Parse these only if we are inside the argument list + if (token.text[0] == ',') { + result.push_back(std::move(currentArg)); + currentArg = {}; + } else { + currentArg.push_back(&token); + } + } + } + + return { result, i }; +} + +std::vector RecordTokens(std::string_view source) { stb_lexer lexer; char stringStorage[65536]; const char* srcBegin = source.data(); @@ -86,31 +172,6 @@ void HandleInputFile(std::string_view source) { stb_c_lexer_init(&lexer, srcBegin, srcEnd, stringStorage, sizeof(stringStorage)); std::vector tokens; - std::vector foundStructs; - InputDefinitionStruct currStruct; - std::vector foundEnums; - InputDefinitionEnum currEnum; - - auto PushFoundStruct = [&]() { - foundStructs.push_back(std::move(currStruct)); - currStruct = {}; - }; - auto PushFoundEnum = [&]() { - foundEnums.push_back(std::move(currEnum)); - currEnum = {}; - }; - - enum NextMatchingConstruct { - NMC_None, - NMC_Enum, - NMC_StructClass, - } matchingConstruct = NMC_None; - bool matchingConstructInBody = false; - - bool matchingDirectiveParams = false; - - int bracePairDepth = 0; - while (true) { // See stb_c_lexer.h's comments, here are a few additinos that aren't made clear in the file: // - `lexer->token` (noted as "token" below) after calling stb_c_lexer_get_token() contains either: @@ -124,137 +185,169 @@ void HandleInputFile(std::string_view source) { break; } - // TODO needed? - // StbLexerToken token; - // token.type = lexer.token; - // token.text = std::string(lexer.string, lexer.string_len); - // tokens.push_back(token); + if (lexer.token == CLEX_parse_error) { + printf("[ERROR] stb_c_lexer countered a parse error."); + // TODO how to handle? + continue; + } + + StbLexerToken token; + if (StbTokenIsSingleChar(lexer.token)) { + token.type = CLEX_ext_single_char; + token.text = std::string(1, lexer.token); + } else { + token.type = lexer.token; + // WORKAROUND: use null terminated string, stb_c_lexer doens't set string_len properly when parsing identifiers + token.text = std::string(lexer.string); + } + tokens.push_back(std::move(token)); + token = {}; + } + return tokens; +} + +enum StructMetaGenOptions { + SMGO_InheritanceHiearchy, + SMGO_PublicFields, + SMGO_ProtectedFields, + SMGO_PrivateFields, + SMGO_COUNT, +}; + +BSTR_LUT_DECL(StructMetaGenOptions, 0, SMGO_COUNT) { + BSTR_LUT_MAP_FOR(StructMetaGenOptions); + BSTR_LUT_MAP(SMGO_InheritanceHiearchy, "GenInheritanceHiearchy"); + BSTR_LUT_MAP(SMGO_PublicFields, "GenPublicFields"); + BSTR_LUT_MAP(SMGO_ProtectedFields, "GenProtectedFields"); + BSTR_LUT_MAP(SMGO_PrivateFields, "GenPrivateFields"); +} + +enum EnumMetaGenOptions { + EMGO_Basic, + EMGO_COUNT, +}; + +BSTR_LUT_DECL(EnumMetaGenOptions, 0, EMGO_COUNT) { + BSTR_LUT_MAP_FOR(EnumMetaGenOptions); + BSTR_LUT_MAP(EMGO_Basic, "GenBasic"); +} + +void GenerateForEnum(CodegenOutput& out, const DeclEnum& decl, EnumFlags options) { +} + +void HandleInputFile(AppState& state, std::string_view source) { + auto tokens = RecordTokens(source); + size_t tokenIdx = 0; + +#if CODEGEN_DEBUG_PRINT + printf("BEGIN tokens\n"); + for (auto& token : tokens) { + printf(" token %-32s '%s'\n", STR_LUT_LOOKUP(ClexNames, token.type), token.text.c_str()); + } + printf("END tokens\n"); +#endif + + CodegenInput input; + CodegenOutput output; - switch (lexer.token) { + int bracePairDepth = 0; + while (tokenIdx < tokens.size()) { + auto& token = tokens[tokenIdx]; + + bool incrementTokenIdx = true; + + switch (token.type) { case CLEX_id: { - // WORKAROUND: stb_c_lexer doens't set string_len properly when parsing identifiers - std::string_view idenText(lexer.string); - // std::string_view idenText(lexer.string, lexer.string_len); - switch (matchingConstruct) { - case NMC_StructClass: { - if (matchingConstructInBody) { - // TODO - } + CppKeyword keyword; + { + auto& map = BSTR_LUT_S2V(CppKeyword); + auto iter = map.find(token.text); + if (iter != map.end()) { + keyword = iter->second; + } else { + break; + } + } + switch (keyword) { + case CKw_Struct: + case CKw_Class: { + auto& idenTok = tokens[tokenIdx + 1]; // TODO handle end of list + DEBUG_PRINTF("[DEBUG] found struct named %s\n", idenTok.text.c_str()); } break; - case NMC_Enum: { - if (matchingConstructInBody) { - printf("[DEBUG] found enum element '%s'\n", lexer.string); - currEnum.elements.push_back(InputDefinitionEnumElement{ - .name = std::string(idenText), - .value = 0, // TODO parse - }); + case CKw_Enum: { + StbLexerToken* idenTok = &token + 1; // TODO handle end of list + if (idenTok->text == "class") { + idenTok += 1; + DEBUG_PRINTF("[DEBUG] found enum class named %s\n", idenTok->text.c_str()); } else { - currEnum.name = std::string(idenText); + DEBUG_PRINTF("[DEBUG] found enum named %s\n", idenTok->text.c_str()); } } break; - default: { - if (idenText == "struct"sv || idenText == "class"sv) { - printf("[DEBUG] found struct named\n"); - matchingConstruct = NMC_StructClass; - } else if (idenText == "enum"sv) { - printf("[DEBUG] found enum\n"); - matchingConstruct = NMC_Enum; - } else if (idenText == "BRUSSEL_CLASS"sv) { - // TODO - printf("[DEBUG] found BRUSSEL_CLASS\n"); - } else if (idenText == "BRUSSEL_ENUM"sv) { - matchingDirectiveParams = true; - printf("[DEBUG] found BRUSSEL_ENUM\n"); + case CKw_COUNT: break; + } + + CodegenDirective directive; + { + auto& map = BSTR_LUT_S2V(CodegenDirective); + auto iter = map.find(token.text); + if (iter != map.end()) { + directive = iter->second; + } else { + break; + } + } + switch (directive) { + case CD_ClassInfo: { + // TODO + } break; + + case CD_EnumInfo: { + auto& optionsStrMap = BSTR_LUT_S2V(EnumMetaGenOptions); + auto [argList, newIdx] = PeekDirectiveArgumentList(tokens, tokenIdx); + if (argList.size() < 1) { + printf("[ERROR] invalid syntax for BRUSSEL_ENUM\n"); + break; } - if (matchingDirectiveParams) { - for (auto& foundEnum : foundEnums) { - if (foundEnum.name == idenText) { - // TODO generate data - break; - } + auto& enumName = argList[0][0]->text; + auto enumDecl = input.FindEnumByName(enumName); + + auto& directiveOptions = argList[1]; + EnumFlags options; + for (auto optionTok : directiveOptions) { + auto iter = optionsStrMap.find(optionTok->text); + if (iter != optionsStrMap.end()) { + options |= iter->second; + } else { + printf("[ERROR] invalid option '%s' for BRUSSEL_ENUM", optionTok->text.c_str()); } - matchingDirectiveParams = false; } - } break; - } - } break; - case CLEX_intlit: - case CLEX_floatlit: - case CLEX_dqstring: - case CLEX_sqstring: - case CLEX_charlit: - case CLEX_eq: - case CLEX_noteq: - case CLEX_lesseq: - case CLEX_greatereq: - case CLEX_andand: - case CLEX_oror: - case CLEX_shl: - case CLEX_shr: - case CLEX_plusplus: - case CLEX_minusminus: - case CLEX_pluseq: - case CLEX_minuseq: - case CLEX_muleq: - case CLEX_diveq: - case CLEX_modeq: - case CLEX_andeq: - case CLEX_oreq: - case CLEX_xoreq: - case CLEX_arrow: - case CLEX_eqarrow: - case CLEX_shleq: - case CLEX_shreq: { + GenerateForEnum(output, *enumDecl, options); + tokenIdx = newIdx; + incrementTokenIdx = false; + } break; + + case CD_COUNT: break; + } } break; case '{': { bracePairDepth++; CheckBraceDepth(bracePairDepth); - - switch (matchingConstruct) { - case NMC_StructClass: - case NMC_Enum: { - matchingConstructInBody = true; - } break; - - default: break; - } } break; case '}': { bracePairDepth--; CheckBraceDepth(bracePairDepth); - - switch (matchingConstruct) { - case NMC_StructClass: { - matchingConstruct = NMC_None; - matchingConstructInBody = false; - } break; - - case NMC_Enum: { - printf("[DEBUG] committed enum '%s'\n", currEnum.name.c_str()); - for (auto& elm : currEnum.elements) { - printf(" - element %s = %" PRId64 "\n", elm.name.c_str(), elm.value); - } - - matchingConstruct = NMC_None; - matchingConstructInBody = false; - PushFoundEnum(); - } break; - - default: break; - } } break; + } - case CLEX_parse_error: { - fprintf(stderr, "[ERROR] stb_c_lexer countered a parse error."); - // TODO how to handle? - } break; + if (incrementTokenIdx) { + ++tokenIdx; } } @@ -278,12 +371,18 @@ std::string ReadFileAtOnce(const fs::path& path) { return result; } -void HandleArgument(InputOpcode opcode, std::string_view operand) { +enum InputOpcode { + IOP_ProcessSingleFile, + IOP_ProcessRecursively, + IOP_COUNT, +}; + +void HandleArgument(AppState& state, InputOpcode opcode, std::string_view operand) { switch (opcode) { case IOP_ProcessSingleFile: { fs::path filePath(operand); auto source = ReadFileAtOnce(filePath); - HandleInputFile(source); + HandleInputFile(state, source); } break; case IOP_ProcessRecursively: { @@ -302,7 +401,7 @@ void HandleArgument(InputOpcode opcode, std::string_view operand) { } auto source = ReadFileAtOnce(path); - HandleInputFile(source); + HandleInputFile(state, source); } } break; @@ -310,21 +409,67 @@ void HandleArgument(InputOpcode opcode, std::string_view operand) { } } +InputOpcode ParseInputOpcode(std::string_view text) { + if (text == "single"sv) { + return IOP_ProcessSingleFile; + } else if (text == "rec"sv) { + return IOP_ProcessRecursively; + } + return IOP_COUNT; +} + int main(int argc, char* argv[]) { + STR_LUT_INIT(ClexNames); + BSTR_LUT_INIT(CppKeyword); + BSTR_LUT_INIT(CodegenDirective); + BSTR_LUT_INIT(StructMetaGenOptions); + BSTR_LUT_INIT(EnumMetaGenOptions); + // TODO better arg parser // option 1: use cxxopts and positional arguments - // option 1: take one argument only, being a json objecet + // option 2: take one argument only, being a json objecet + + AppState state; + + // If no cli is provided (argv[0] conventionally but not mandatorily the cli), this will do thing + // Otherwise, start with the 2nd element in the array, which is the 1st actual argument + if (argc < 2) { + // NOTE: keep in sync with various enum options and parser code + printf(&R"""( +USAGE: codegen.exe [:]... +where : the _file_ to write generated contents to + is one of: + "single" process this file only + "rec" starting at the given directory , recursively process all .h .c .hpp .cpp files +)"""[1]); + return -1; + } + + const char* outputFilePath = argv[1]; + DEBUG_PRINTF("Outputting to file %s.\n", outputFilePath); - // If no cli is provided (argv[0]), this loop will do nothing - // Otherwise, start with the 2nd element which is the 1st argument - for (int i = 1; i < argc; ++i) { + for (int i = 2; i < argc; ++i) { std::string_view arg(argv[i]); auto separatorLoc = arg.find(':'); if (separatorLoc != std::string_view::npos) { - auto opcode = ParseInputOpcode(arg.substr(0, separatorLoc)); + auto opcodeString = arg.substr(0, separatorLoc); + auto opcode = ParseInputOpcode(opcodeString); auto operand = arg.substr(separatorLoc + 1); - HandleArgument(opcode, operand); + + DEBUG_PRINTF("Processing input command %.*s at path %.*s\n", (int)opcodeString.size(), opcodeString.data(), (int)operand.size(), operand.data()); + + HandleArgument(state, opcode, operand); + } + } + + { + auto outputFile = Utils::OpenCstdioFile(outputFilePath, Utils::WriteTruncate); + if (!outputFile) { + printf("[ERROR] unable to open output file %s", outputFilePath); + return -1; } + DEFER { fclose(outputFile); }; + state.mainOutput.Write(outputFile); } return 0; diff --git a/buildtools/codegen/tests/examples/TestEnum.hpp b/buildtools/codegen/tests/examples/TestEnum.hpp index aaa3d74..2a93c01 100644 --- a/buildtools/codegen/tests/examples/TestEnum.hpp +++ b/buildtools/codegen/tests/examples/TestEnum.hpp @@ -5,4 +5,4 @@ enum MyEnum { EnumElement2, EnumElement3, }; -BRUSSEL_ENUM(MyEnum, GenInfo); +BRUSSEL_ENUM(MyEnum, GenBasic); diff --git a/source-common/Enum.hpp b/source-common/Enum.hpp index 5e106fe..e8750f2 100644 --- a/source-common/Enum.hpp +++ b/source-common/Enum.hpp @@ -61,32 +61,32 @@ public: } } - EnumFlags& operator|=(EnumFlags that) const { + EnumFlags& operator|=(EnumFlags that) { mValue |= that.mValue; return *this; } - EnumFlags& operator&=(EnumFlags that) const { + EnumFlags& operator&=(EnumFlags that) { mValue &= that.mValue; return *this; } - EnumFlags& operator^=(EnumFlags that) const { + EnumFlags& operator^=(EnumFlags that) { mValue ^= that.mValue; return *this; } - EnumFlags& operator|=(TEnum e) const { + EnumFlags& operator|=(TEnum e) { mValue |= 1 << static_cast(e); return *this; } - EnumFlags& operator&=(TEnum e) const { + EnumFlags& operator&=(TEnum e) { mValue &= 1 << static_cast(e); return *this; } - EnumFlags& operator^=(TEnum e) const { + EnumFlags& operator^=(TEnum e) { mValue ^= 1 << static_cast(e); return *this; } diff --git a/source-common/MacrosCodegen.hpp b/source-common/MacrosCodegen.hpp index d6d5c6f..6803023 100644 --- a/source-common/MacrosCodegen.hpp +++ b/source-common/MacrosCodegen.hpp @@ -3,5 +3,5 @@ #pragma once -#define BRUSSEL_CLASS +#define BRUSSEL_CLASS(name, options) #define BRUSSEL_ENUM(name, options) -- cgit v1.2.3-70-g09d2 From 66b63ae887f553e1cac813546a6b827a9c85d80c Mon Sep 17 00:00:00 2001 From: rtk0c Date: Sun, 29 May 2022 16:14:26 -0700 Subject: Changeset: 43 Add tostring code gen for enums --- buildtools/codegen/CodegenDecl.cpp | 48 ++++++ buildtools/codegen/CodegenDecl.hpp | 33 ++++ buildtools/codegen/CodegenInput.inl | 4 +- buildtools/codegen/CodegenOutput.inl | 18 ++- buildtools/codegen/main.cpp | 212 +++++++++++++++++++++---- buildtools/codegen/tests/examples/TestEnum.hpp | 17 +- source-common/Enum.hpp | 2 +- 7 files changed, 295 insertions(+), 39 deletions(-) create mode 100644 buildtools/codegen/CodegenDecl.cpp (limited to 'buildtools/codegen') diff --git a/buildtools/codegen/CodegenDecl.cpp b/buildtools/codegen/CodegenDecl.cpp new file mode 100644 index 0000000..ac6bb01 --- /dev/null +++ b/buildtools/codegen/CodegenDecl.cpp @@ -0,0 +1,48 @@ +#include "CodegenDecl.hpp" + +#include + +static EnumValuePattern NextPattern(EnumValuePattern val) { + return (EnumValuePattern)(val + 1); +} + +EnumValuePattern DeclEnum::CalcPattern() const { + if (elements.empty()) return EVP_Continuous; + + auto pattern = EVP_Continuous; +restart: + auto lastVal = elements[0].value; + for (size_t i = 1; i < elements.size(); ++i) { + auto currVal = elements[i].value; + switch (pattern) { + case EVP_Continuous: { + bool satisfy = lastVal + 1 == currVal; + if (!satisfy) { + pattern = NextPattern(pattern); + goto restart; + } + } break; + + case EVP_Bits: { + bool satisfy = (lastVal << 1) == currVal; + if (!satisfy) { + pattern = NextPattern(pattern); + goto restart; + } + } break; + + // A random pattern can match anything + case EVP_Random: + case EVP_COUNT: break; + } + lastVal = currVal; + } + + return pattern; +} + +void DeclEnum::CalcPatternIfNecessary() { + if (pattern == EVP_COUNT) { + CalcPattern(); + } +} diff --git a/buildtools/codegen/CodegenDecl.hpp b/buildtools/codegen/CodegenDecl.hpp index 3414d80..c2d8dbb 100644 --- a/buildtools/codegen/CodegenDecl.hpp +++ b/buildtools/codegen/CodegenDecl.hpp @@ -20,8 +20,22 @@ enum EnumUnderlyingType { EUT_COUNT, }; +enum EnumValuePattern { + // The numbers cover n..m with no gaps + EVP_Continuous, + // The numbers cover for i in n..m, 1 << i + // e.g. [0] = 1 << 0, + // [1] = 1 << 1. + // [2] = 1 << 2. etc. + EVP_Bits, + // The numbesr don't have a particular pattern + EVP_Random, + EVP_COUNT, +}; + struct DeclEnumElement { std::string name; + // TODO support int64_t, etc. enum underlying types uint64_t value; }; @@ -29,4 +43,23 @@ struct DeclEnum { std::string name; std::vector elements; EnumUnderlyingType underlyingType; + // Start with invalid value, calculate on demand + EnumValuePattern pattern = EVP_COUNT; + + EnumValuePattern CalcPattern() const; + void CalcPatternIfNecessary(); +}; + +struct DeclFunctionArgument { + std::string type; + std::string name; +}; + +struct DeclFunction { + // Things like extern, static, etc. that gets written before the function return type + std::string prefix; + std::string name; + std::string returnType; + std::vector arguments; + std::string body; }; diff --git a/buildtools/codegen/CodegenInput.inl b/buildtools/codegen/CodegenInput.inl index 9fae43c..80a39d0 100644 --- a/buildtools/codegen/CodegenInput.inl +++ b/buildtools/codegen/CodegenInput.inl @@ -13,7 +13,7 @@ class CodegenInput { private: std::vector mEnums; - robin_hood::unordered_map mDeclByName; + robin_hood::unordered_map mDeclByName; public: void AddEnum(DeclEnum decl) { @@ -24,7 +24,7 @@ public: } #endif - mDeclByName.try_emplace(decl.name, mEnums.size()); + mDeclByName.try_emplace(std::string(decl.name), mEnums.size()); mEnums.push_back(std::move(decl)); } diff --git a/buildtools/codegen/CodegenOutput.inl b/buildtools/codegen/CodegenOutput.inl index 6d59301..752682c 100644 --- a/buildtools/codegen/CodegenOutput.inl +++ b/buildtools/codegen/CodegenOutput.inl @@ -15,6 +15,14 @@ struct CodegenOutputThing { class CodegenOutput { private: std::vector mOutThings; + std::vector mOutStructs; + std::vector mOutEnums; + std::vector mOutFunctions; + +public: + std::string optionOutPrefix; + // Whether to add prefixes mOutPrefix to all global names or not + bool optionAutoAddPrefix : 1 = false; public: void AddOutputThing(CodegenOutputThing thing) { @@ -22,10 +30,16 @@ public: } void MergeContents(CodegenOutput other) { - std::move(other.mOutThings.begin(), other.mOutThings.end(), this->mOutThings.begin()); + std::move(other.mOutThings.begin(), other.mOutThings.end(), std::back_inserter(this->mOutThings)); } void Write(FILE* file) { - // TODO +#define WRITE_LITERAL(str) fwrite(str, sizeof(char), sizeof(str) - 1, file) + for (auto& thing : mOutThings) { + WRITE_LITERAL("// Output thing\n"); + fwrite(thing.text.c_str(), sizeof(char), thing.text.size(), file); + WRITE_LITERAL("\n"); + } +#undef WRITE_LITERAL } }; diff --git a/buildtools/codegen/main.cpp b/buildtools/codegen/main.cpp index 74acd1c..ed8cbbe 100644 --- a/buildtools/codegen/main.cpp +++ b/buildtools/codegen/main.cpp @@ -76,6 +76,7 @@ BSTR_LUT_DECL(CppKeyword, 0, CKw_COUNT) { BSTR_LUT_MAP_FOR(CppKeyword); BSTR_LUT_MAP(CKw_Struct, "struct"); BSTR_LUT_MAP(CKw_Class, "class"); + BSTR_LUT_MAP(CKw_Enum, "enum"); } enum CodegenDirective { @@ -161,6 +162,10 @@ PeekDirectiveArgumentList(const std::vector& tokens, size_t curre } } + if (!currentArg.empty()) { + result.push_back(std::move(currentArg)); + } + return { result, i }; } @@ -223,21 +228,95 @@ BSTR_LUT_DECL(StructMetaGenOptions, 0, SMGO_COUNT) { } enum EnumMetaGenOptions { - EMGO_Basic, + EMGO_ToString, + EMGO_ExcludeUseHeuristics, EMGO_COUNT, }; BSTR_LUT_DECL(EnumMetaGenOptions, 0, EMGO_COUNT) { BSTR_LUT_MAP_FOR(EnumMetaGenOptions); - BSTR_LUT_MAP(EMGO_Basic, "GenBasic"); + BSTR_LUT_MAP(EMGO_ToString, "ToString"); + BSTR_LUT_MAP(EMGO_ExcludeUseHeuristics, "ExcludeHeuristics"); +} + +// I give up, hopefully nothing overflows this buffer +#define APPEND_SPRINTF(out, format, ...) \ + { \ + char buffer[65536]; \ + snprintf(buffer, sizeof(buffer), format, __VA_ARGS__); \ + out += buffer; \ + } + +std::string GenerateEnumStringArray(CodegenOutput& out, const DeclEnum& decl, bool useHeruistics) { + std::string arrayName; + APPEND_SPRINTF(arrayName, "gCG_%s_StringTable", decl.name.c_str()); + + CodegenOutputThing thing; + APPEND_SPRINTF(thing.text, "const char* %s[] = {\n", arrayName.c_str()); + for (auto& elm : decl.elements) { + if (useHeruistics && elm.name.ends_with("COUNT")) { + continue; + } + + APPEND_SPRINTF(thing.text, "\"%s\",\n", elm.name.c_str()); + } + thing.text += "};\n"; + out.AddOutputThing(std::move(thing)); + + return arrayName; +} + +std::string GenerateEnumStringMap(CodegenOutput& out, const DeclEnum& decl, bool useHeruistics) { + std::string mapName; + // TODO + + return mapName; } void GenerateForEnum(CodegenOutput& out, const DeclEnum& decl, EnumFlags options) { + bool useExcludeHeuristics = options.IsSet(EMGO_ExcludeUseHeuristics); + + if (options.IsSet(EMGO_ToString)) { + // Generate name lookup table + + switch (decl.CalcPattern()) { + case EVP_Continuous: { + auto arrayName = GenerateEnumStringArray(out, decl, useExcludeHeuristics); + int minVal = decl.elements.front().value; + int maxVal = decl.elements.back().value; + if (useExcludeHeuristics && + decl.elements.back().name.ends_with("COUNT") && + decl.elements.size() >= 2) + { + // Skip the last *_COUNT element if instructed to use heuristics + maxVal = decl.elements[decl.elements.size() - 2].value; + } + + CodegenOutputThing lookupFunction; + APPEND_SPRINTF(lookupFunction.text, "std::string_view Stringify%s(%s value) {\n", decl.name.c_str(), decl.name.c_str()); + APPEND_SPRINTF(lookupFunction.text, " if (value < 0 || value >= %d) return {};\n", maxVal); + APPEND_SPRINTF(lookupFunction.text, " return %s[value - %d];\n", arrayName.c_str(), minVal); + lookupFunction.text += "}\n"; + + out.AddOutputThing(std::move(lookupFunction)); + } break; + + case EVP_Bits: { + auto arrayName = GenerateEnumStringArray(out, decl, useExcludeHeuristics); + } break; + + case EVP_Random: { + auto mapName = GenerateEnumStringMap(out, decl, useExcludeHeuristics); + } break; + + case EVP_COUNT: break; + } + } } void HandleInputFile(AppState& state, std::string_view source) { auto tokens = RecordTokens(source); - size_t tokenIdx = 0; + size_t idx = 0; #if CODEGEN_DEBUG_PRINT printf("BEGIN tokens\n"); @@ -247,12 +326,12 @@ void HandleInputFile(AppState& state, std::string_view source) { printf("END tokens\n"); #endif - CodegenInput input; - CodegenOutput output; + CodegenInput fileInput; + CodegenOutput fileOutput; int bracePairDepth = 0; - while (tokenIdx < tokens.size()) { - auto& token = tokens[tokenIdx]; + while (idx < tokens.size()) { + auto& token = tokens[idx]; bool incrementTokenIdx = true; @@ -265,25 +344,77 @@ void HandleInputFile(AppState& state, std::string_view source) { if (iter != map.end()) { keyword = iter->second; } else { - break; + keyword = CKw_COUNT; // Skip keyword section } } switch (keyword) { case CKw_Struct: case CKw_Class: { - auto& idenTok = tokens[tokenIdx + 1]; // TODO handle end of list + auto& idenTok = tokens[idx + 1]; // TODO handle end of list DEBUG_PRINTF("[DEBUG] found struct named %s\n", idenTok.text.c_str()); - } break; + goto endIdenCase; + } case CKw_Enum: { - StbLexerToken* idenTok = &token + 1; // TODO handle end of list - if (idenTok->text == "class") { - idenTok += 1; - DEBUG_PRINTF("[DEBUG] found enum class named %s\n", idenTok->text.c_str()); + // Consume the "enum" keyword + ++idx; + incrementTokenIdx = false; + + DeclEnum enumDecl; + enumDecl.underlyingType = EUT_Int32; // TODO + + if (tokens[idx].text == "class") { + // Consume the "class" keyword + ++idx; + DEBUG_PRINTF("[DEBUG] found enum class named %s\n", tokens[idx].text.c_str()); } else { - DEBUG_PRINTF("[DEBUG] found enum named %s\n", idenTok->text.c_str()); + DEBUG_PRINTF("[DEBUG] found enum named %s\n", tokens[idx].text.c_str()); } - } break; + + // Consume the enum name identifier + enumDecl.name = tokens[idx].text; + ++idx; + + int enumClosingBraceCount = 0; + int enumBraceDepth = 0; + while (enumClosingBraceCount == 0 && idx < tokens.size()) { + auto& token = tokens[idx]; + switch (token.type) { + case CLEX_id: { + auto& vec = enumDecl.elements; + // Set to the previous enum element's value + 1, or starting from 0 if this is the first + // Also overridden in the CLEX_intlit branch + auto value = vec.empty() ? 0 : vec.back().value + 1; + vec.push_back(DeclEnumElement{ + .name = token.text, + .value = value, + }); + } break; + + case CLEX_intlit: { + + } break; + + case CLEX_ext_single_char: { + switch (token.text[0]) { + case '{': { + ++enumBraceDepth; + } break; + + case '}': { + --enumBraceDepth; + ++enumClosingBraceCount; + } break; + } + } break; + } + + ++idx; + } + + fileInput.AddEnum(std::move(enumDecl)); + goto endIdenCase; + } case CKw_COUNT: break; } @@ -295,24 +426,33 @@ void HandleInputFile(AppState& state, std::string_view source) { if (iter != map.end()) { directive = iter->second; } else { - break; + directive = CD_COUNT; // Skip directive section } } switch (directive) { case CD_ClassInfo: { // TODO - } break; + goto endIdenCase; + } case CD_EnumInfo: { + // Consume the directive + ++idx; + incrementTokenIdx = false; + auto& optionsStrMap = BSTR_LUT_S2V(EnumMetaGenOptions); - auto [argList, newIdx] = PeekDirectiveArgumentList(tokens, tokenIdx); + auto [argList, newIdx] = PeekDirectiveArgumentList(tokens, idx); if (argList.size() < 1) { printf("[ERROR] invalid syntax for BRUSSEL_ENUM\n"); - break; + break; // TODO handle this error case gracefully (advance to semicolon?) } auto& enumName = argList[0][0]->text; - auto enumDecl = input.FindEnumByName(enumName); + auto enumDecl = fileInput.FindEnumByName(enumName); + if (!enumDecl) { + printf("[ERROR] BRUSSEL_ENUM: referring to non-existent enum '%s'\n", enumName.c_str()); + break; + } auto& directiveOptions = argList[1]; EnumFlags options; @@ -321,19 +461,23 @@ void HandleInputFile(AppState& state, std::string_view source) { if (iter != optionsStrMap.end()) { options |= iter->second; } else { - printf("[ERROR] invalid option '%s' for BRUSSEL_ENUM", optionTok->text.c_str()); + printf("[ERROR] BRUSSEL_ENUM: invalid option '%s'\n", optionTok->text.c_str()); } } - GenerateForEnum(output, *enumDecl, options); + GenerateForEnum(fileOutput, *enumDecl, options); - tokenIdx = newIdx; + idx = newIdx; incrementTokenIdx = false; - } break; + goto endIdenCase; + } case CD_COUNT: break; } - } break; + + endIdenCase: + break; + } case '{': { bracePairDepth++; @@ -347,13 +491,15 @@ void HandleInputFile(AppState& state, std::string_view source) { } if (incrementTokenIdx) { - ++tokenIdx; + ++idx; } } if (bracePairDepth != 0) { printf("[WARNING] unbalanced brace at end of file."); } + + state.mainOutput.MergeContents(std::move(fileOutput)); } std::string ReadFileAtOnce(const fs::path& path) { @@ -436,12 +582,12 @@ int main(int argc, char* argv[]) { if (argc < 2) { // NOTE: keep in sync with various enum options and parser code printf(&R"""( -USAGE: codegen.exe [:]... -where : the _file_ to write generated contents to - is one of: - "single" process this file only - "rec" starting at the given directory , recursively process all .h .c .hpp .cpp files -)"""[1]); + USAGE: codegen.exe [:]... + where : the _file_ to write generated contents to + is one of: + "single" process this file only + "rec" starting at the given directory , recursively process all .h .c .hpp .cpp files + )"""[1]); return -1; } diff --git a/buildtools/codegen/tests/examples/TestEnum.hpp b/buildtools/codegen/tests/examples/TestEnum.hpp index 2a93c01..c498cd9 100644 --- a/buildtools/codegen/tests/examples/TestEnum.hpp +++ b/buildtools/codegen/tests/examples/TestEnum.hpp @@ -5,4 +5,19 @@ enum MyEnum { EnumElement2, EnumElement3, }; -BRUSSEL_ENUM(MyEnum, GenBasic); +BRUSSEL_ENUM(MyEnum, ToString); + +enum CountedEnumAll { + CEA_Foo, + CEA_Bar, + CEA_COUNT, +}; +BRUSSEL_ENUM(CountedEnumAll, ToString); + +enum CountedEnum { + CE_Foo, + CE_Bar, + CE_FooBar, + CE_COUNT, +}; +BRUSSEL_ENUM(CountedEnum, ToString ExcludeHeuristics); diff --git a/source-common/Enum.hpp b/source-common/Enum.hpp index e8750f2..8ad75ba 100644 --- a/source-common/Enum.hpp +++ b/source-common/Enum.hpp @@ -18,7 +18,7 @@ public: } EnumFlags(TEnum e) - : mValue{ 1 << static_cast(e) } { + : mValue{ static_cast(1) << static_cast(e) } { } bool IsSet(EnumFlags mask) const { -- cgit v1.2.3-70-g09d2 From 32f00c086b35c5bc32d800b96253b378b9367580 Mon Sep 17 00:00:00 2001 From: rtk0c Date: Sun, 29 May 2022 16:31:29 -0700 Subject: Changeset: 46 Move lookup table infra to source-common/ --- buildtools/codegen/CodegenLookupTable.hpp | 50 ----------------------------- buildtools/codegen/main.cpp | 2 +- source-common/LookupTable.hpp | 53 +++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 51 deletions(-) delete mode 100644 buildtools/codegen/CodegenLookupTable.hpp create mode 100644 source-common/LookupTable.hpp (limited to 'buildtools/codegen') diff --git a/buildtools/codegen/CodegenLookupTable.hpp b/buildtools/codegen/CodegenLookupTable.hpp deleted file mode 100644 index 2b6a993..0000000 --- a/buildtools/codegen/CodegenLookupTable.hpp +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -// BIDI stands for bi-directional -#define BIDI_LUT_DECL(name, aType, aCount, bType, bCount) \ - int gLut_##name##_A2B[aCount]; \ - int gLut_##name##_B2A[bCount]; \ - using name##AType = aType; \ - using name##BType = bType; \ - void InitializeLut##name() -#define BIDI_LUT_MAP_FOR(name) \ - int* lutMappingA2B = gLut_##name##_A2B; \ - int* lutMappingB2A = gLut_##name##_B2A -#define BIDI_LUT_MAP(from, to) \ - lutMappingA2B[from] = to; \ - lutMappingB2A[to] = from -#define BIDI_LUT_INIT(name) InitializeLut##name() -#define BIDI_LUT_A2B_LOOKUP(name, from) (name##BType)(gLut_##name##_A2B[from]) -#define BIDI_LUT_B2A_LOOKUP(name, to) (name##AType)(gLut_##name##_B2A[to]) - -#define STR_LUT_DECL(name, enumMinValue, enumMaxValue) \ - constexpr int kLutMinVal_##name = enumMinValue; \ - const char* gLut_##name[(int)enumMaxValue - (int)enumMinValue]; \ - void InitializeLut##name() -#define STR_LUT_MAP_FOR(name) \ - const char** lutMapping = gLut_##name; \ - int lutMappingMinValue = kLutMinVal_##name -#define STR_LUT_MAP(value, text) lutMapping[value - lutMappingMinValue] = text -#define STR_LUT_MAP_ENUM(enumValue) STR_LUT_MAP(enumValue, #enumValue) -#define STR_LUT_LOOKUP(name, enumValue) gLut_##name[enumValue - kLutMinVal_##name] -#define STR_LUT_INIT(name) InitializeLut##name() - -// BSTR stands for bi-directional string -#define BSTR_LUT_DECL(name, enumMinValue, enumMaxValue) \ - constexpr int kLutMinVal_##name = enumMinValue; \ - const char* gLut_##name##_V2S[(int)enumMaxValue - (int)enumMinValue]; \ - robin_hood::unordered_flat_map gLut_##name##_S2V; \ - void InitializeLut##name() -#define BSTR_LUT_MAP_FOR(name) \ - const char** lutMappingV2S = gLut_##name##_V2S; \ - auto& lutMappingS2V = gLut_##name##_S2V; \ - int lutMappingMinValue = kLutMinVal_##name -#define BSTR_LUT_MAP(value, text) \ - lutMappingV2S[value - lutMappingMinValue] = text; \ - lutMappingS2V.insert_or_assign(std::string_view(text), value); -#define BSTR_LUT_MAP_ENUM(enumValue) BSTR_LUT_MAP(enumValue, #enumValue) -#define BSTR_LUT_V2S(name) gLut_##name##_V2S -#define BSTR_LUT_S2V(name) gLut_##name##_S2V -#define BSTR_LUT_V2S_LOOKUP(name, enumValue) gLut_##name##_V2S[enumValue - kLutMinVal_##name] -#define BSTR_LUT_S2V_LOOKUP(name, string) gLut_##name##_S2V.find(std::string_view(text)) -#define BSTR_LUT_INIT(name) InitializeLut##name() diff --git a/buildtools/codegen/main.cpp b/buildtools/codegen/main.cpp index ed8cbbe..334b195 100644 --- a/buildtools/codegen/main.cpp +++ b/buildtools/codegen/main.cpp @@ -1,11 +1,11 @@ #include "CodegenConfig.hpp" #include "CodegenDecl.hpp" -#include "CodegenLookupTable.hpp" #include "CodegenInput.inl" #include "CodegenOutput.inl" #include +#include #include #include #include diff --git a/source-common/LookupTable.hpp b/source-common/LookupTable.hpp new file mode 100644 index 0000000..c50e28e --- /dev/null +++ b/source-common/LookupTable.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include +#include + +// BIDI stands for bi-directional +#define BIDI_LUT_DECL(name, aType, aCount, bType, bCount) \ + int gLut_##name##_A2B[aCount]; \ + int gLut_##name##_B2A[bCount]; \ + using name##AType = aType; \ + using name##BType = bType; \ + void InitializeLut##name() +#define BIDI_LUT_MAP_FOR(name) \ + int* lutMappingA2B = gLut_##name##_A2B; \ + int* lutMappingB2A = gLut_##name##_B2A +#define BIDI_LUT_MAP(from, to) \ + lutMappingA2B[from] = to; \ + lutMappingB2A[to] = from +#define BIDI_LUT_INIT(name) InitializeLut##name() +#define BIDI_LUT_A2B_LOOKUP(name, from) (name##BType)(gLut_##name##_A2B[from]) +#define BIDI_LUT_B2A_LOOKUP(name, to) (name##AType)(gLut_##name##_B2A[to]) + +#define STR_LUT_DECL(name, enumMinValue, enumMaxValue) \ + constexpr int kLutMinVal_##name = enumMinValue; \ + const char* gLut_##name[(int)enumMaxValue - (int)enumMinValue]; \ + void InitializeLut##name() +#define STR_LUT_MAP_FOR(name) \ + const char** lutMapping = gLut_##name; \ + int lutMappingMinValue = kLutMinVal_##name +#define STR_LUT_MAP(value, text) lutMapping[value - lutMappingMinValue] = text +#define STR_LUT_MAP_ENUM(enumValue) STR_LUT_MAP(enumValue, #enumValue) +#define STR_LUT_LOOKUP(name, enumValue) gLut_##name[enumValue - kLutMinVal_##name] +#define STR_LUT_INIT(name) InitializeLut##name() + +// BSTR stands for bi-directional string +#define BSTR_LUT_DECL(name, enumMinValue, enumMaxValue) \ + constexpr int kLutMinVal_##name = enumMinValue; \ + const char* gLut_##name##_V2S[(int)enumMaxValue - (int)enumMinValue]; \ + robin_hood::unordered_flat_map gLut_##name##_S2V; \ + void InitializeLut##name() +#define BSTR_LUT_MAP_FOR(name) \ + const char** lutMappingV2S = gLut_##name##_V2S; \ + auto& lutMappingS2V = gLut_##name##_S2V; \ + int lutMappingMinValue = kLutMinVal_##name +#define BSTR_LUT_MAP(value, text) \ + lutMappingV2S[value - lutMappingMinValue] = text; \ + lutMappingS2V.insert_or_assign(std::string_view(text), value); +#define BSTR_LUT_MAP_ENUM(enumValue) BSTR_LUT_MAP(enumValue, #enumValue) +#define BSTR_LUT_V2S(name) gLut_##name##_V2S +#define BSTR_LUT_S2V(name) gLut_##name##_S2V +#define BSTR_LUT_V2S_LOOKUP(name, enumValue) gLut_##name##_V2S[enumValue - kLutMinVal_##name] +#define BSTR_LUT_S2V_LOOKUP(name, string) gLut_##name##_S2V.find(std::string_view(text)) +#define BSTR_LUT_INIT(name) InitializeLut##name() -- cgit v1.2.3-70-g09d2 From 5112858f9a4adcf76240bcddad19179a5c56d4f3 Mon Sep 17 00:00:00 2001 From: rtk0c Date: Sun, 29 May 2022 18:30:03 -0700 Subject: Changeset: 47 Add fromstring codegen for enums --- buildtools/codegen/CodegenDecl.cpp | 5 +- buildtools/codegen/CodegenDecl.hpp | 4 +- buildtools/codegen/CodegenMacros.hpp | 30 ++++++++ buildtools/codegen/CodegenOutput.inl | 25 +++++-- buildtools/codegen/main.cpp | 98 +++++++++++++++++++------- buildtools/codegen/tests/examples/TestEnum.hpp | 6 +- 6 files changed, 130 insertions(+), 38 deletions(-) create mode 100644 buildtools/codegen/CodegenMacros.hpp (limited to 'buildtools/codegen') diff --git a/buildtools/codegen/CodegenDecl.cpp b/buildtools/codegen/CodegenDecl.cpp index ac6bb01..7cf21ce 100644 --- a/buildtools/codegen/CodegenDecl.cpp +++ b/buildtools/codegen/CodegenDecl.cpp @@ -41,8 +41,9 @@ restart: return pattern; } -void DeclEnum::CalcPatternIfNecessary() { +EnumValuePattern DeclEnum::GetPattern() const { if (pattern == EVP_COUNT) { - CalcPattern(); + pattern = CalcPattern(); } + return pattern; } diff --git a/buildtools/codegen/CodegenDecl.hpp b/buildtools/codegen/CodegenDecl.hpp index c2d8dbb..b7c1ee4 100644 --- a/buildtools/codegen/CodegenDecl.hpp +++ b/buildtools/codegen/CodegenDecl.hpp @@ -44,10 +44,10 @@ struct DeclEnum { std::vector elements; EnumUnderlyingType underlyingType; // Start with invalid value, calculate on demand - EnumValuePattern pattern = EVP_COUNT; + mutable EnumValuePattern pattern = EVP_COUNT; EnumValuePattern CalcPattern() const; - void CalcPatternIfNecessary(); + EnumValuePattern GetPattern() const; }; struct DeclFunctionArgument { diff --git a/buildtools/codegen/CodegenMacros.hpp b/buildtools/codegen/CodegenMacros.hpp new file mode 100644 index 0000000..84c9d09 --- /dev/null +++ b/buildtools/codegen/CodegenMacros.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +// I give up, hopefully nothing overflows this buffer +// TODO handle buffer sizing properly + +#define APPEND_LIT(out, str) out += str + +#define APPEND_FMT(out, format, ...) \ + { \ + char buffer[65536]; \ + snprintf(buffer, sizeof(buffer), format, __VA_ARGS__); \ + out += buffer; \ + } + +#define WRITE_LIT(file, str) fwrite(str, sizeof(char), sizeof(str) - 1, file) + +// NOTE: snprintf() returns the size written (given an infinite buffer) not including \0 +#define WRITE_FMT(file, format, ...) \ + { \ + char buffer[65536]; \ + int size = snprintf(buffer, sizeof(buffer), format, __VA_ARGS__); \ + fwrite(buffer, sizeof(char), std::min(size, sizeof(buffer)), file); \ + } + +#define APPEND_LIT_LN(out, str) APPEND_LIT(out, (str "\n")) +#define APPEND_FMT_LN(out, format, ...) APPEND_FMT(out, (format "\n"), __VA_ARGS__) +#define WRITE_LIT_LN(out, str) WRITE_LIT(out, (str "\n")) +#define WRITE_FMT_LN(out, format, ...) WRITE_FMT(out, (format "\n"), __VA_ARGS__) diff --git a/buildtools/codegen/CodegenOutput.inl b/buildtools/codegen/CodegenOutput.inl index 752682c..d1f2e50 100644 --- a/buildtools/codegen/CodegenOutput.inl +++ b/buildtools/codegen/CodegenOutput.inl @@ -1,7 +1,9 @@ #pragma once #include "CodegenDecl.hpp" +#include "CodegenMacros.hpp" +#include #include #include #include @@ -31,15 +33,30 @@ public: void MergeContents(CodegenOutput other) { std::move(other.mOutThings.begin(), other.mOutThings.end(), std::back_inserter(this->mOutThings)); + std::move(other.mOutStructs.begin(), other.mOutStructs.end(), std::back_inserter(this->mOutStructs)); + std::move(other.mOutEnums.begin(), other.mOutEnums.end(), std::back_inserter(this->mOutEnums)); + std::move(other.mOutFunctions.begin(), other.mOutFunctions.end(), std::back_inserter(this->mOutFunctions)); } void Write(FILE* file) { -#define WRITE_LITERAL(str) fwrite(str, sizeof(char), sizeof(str) - 1, file) for (auto& thing : mOutThings) { - WRITE_LITERAL("// Output thing\n"); + WRITE_LIT(file, "// Output thing\n"); fwrite(thing.text.c_str(), sizeof(char), thing.text.size(), file); - WRITE_LITERAL("\n"); + WRITE_LIT(file, "\n"); + } + + for (auto& declStruct : mOutStructs) { + WRITE_FMT(file, "struct %s {\n", declStruct.name.c_str()); + // TODO + WRITE_LIT(file, "}\n"); + } + + for (auto& declEnum : mOutEnums) { + // TODO + } + + for (auto& declFunc : mOutFunctions) { + // TODO } -#undef WRITE_LITERAL } }; diff --git a/buildtools/codegen/main.cpp b/buildtools/codegen/main.cpp index 334b195..df52b66 100644 --- a/buildtools/codegen/main.cpp +++ b/buildtools/codegen/main.cpp @@ -1,5 +1,6 @@ #include "CodegenConfig.hpp" #include "CodegenDecl.hpp" +#include "CodegenMacros.hpp" #include "CodegenInput.inl" #include "CodegenOutput.inl" @@ -10,6 +11,8 @@ #include #include +#include +#include #include #include #include @@ -23,8 +26,11 @@ using namespace std::literals; namespace fs = std::filesystem; +// TODO handle namespace + struct AppState { - CodegenOutput mainOutput; + CodegenOutput headerOutput; + CodegenOutput sourceOutput; }; enum { @@ -229,6 +235,7 @@ BSTR_LUT_DECL(StructMetaGenOptions, 0, SMGO_COUNT) { enum EnumMetaGenOptions { EMGO_ToString, + EMGO_FromString, EMGO_ExcludeUseHeuristics, EMGO_COUNT, }; @@ -236,31 +243,24 @@ enum EnumMetaGenOptions { BSTR_LUT_DECL(EnumMetaGenOptions, 0, EMGO_COUNT) { BSTR_LUT_MAP_FOR(EnumMetaGenOptions); BSTR_LUT_MAP(EMGO_ToString, "ToString"); + BSTR_LUT_MAP(EMGO_FromString, "FromString"); BSTR_LUT_MAP(EMGO_ExcludeUseHeuristics, "ExcludeHeuristics"); } -// I give up, hopefully nothing overflows this buffer -#define APPEND_SPRINTF(out, format, ...) \ - { \ - char buffer[65536]; \ - snprintf(buffer, sizeof(buffer), format, __VA_ARGS__); \ - out += buffer; \ - } - std::string GenerateEnumStringArray(CodegenOutput& out, const DeclEnum& decl, bool useHeruistics) { std::string arrayName; - APPEND_SPRINTF(arrayName, "gCG_%s_StringTable", decl.name.c_str()); + APPEND_FMT(arrayName, "gCG_%s_Val2Str", decl.name.c_str()); CodegenOutputThing thing; - APPEND_SPRINTF(thing.text, "const char* %s[] = {\n", arrayName.c_str()); + APPEND_FMT_LN(thing.text, "const char* %s[] = {", arrayName.c_str()); for (auto& elm : decl.elements) { if (useHeruistics && elm.name.ends_with("COUNT")) { continue; } - APPEND_SPRINTF(thing.text, "\"%s\",\n", elm.name.c_str()); + APPEND_FMT_LN(thing.text, "\"%s\",", elm.name.c_str()); } - thing.text += "};\n"; + APPEND_LIT_LN(thing.text, "};"); out.AddOutputThing(std::move(thing)); return arrayName; @@ -277,9 +277,9 @@ void GenerateForEnum(CodegenOutput& out, const DeclEnum& decl, EnumFlags string lookup table and function - switch (decl.CalcPattern()) { + switch (decl.GetPattern()) { case EVP_Continuous: { auto arrayName = GenerateEnumStringArray(out, decl, useExcludeHeuristics); int minVal = decl.elements.front().value; @@ -293,25 +293,57 @@ void GenerateForEnum(CodegenOutput& out, const DeclEnum& decl, EnumFlags= %d) return {};\n", maxVal); - APPEND_SPRINTF(lookupFunction.text, " return %s[value - %d];\n", arrayName.c_str(), minVal); - lookupFunction.text += "}\n"; + auto& o = lookupFunction.text; + APPEND_FMT_LN(o, "std::string_view EnumToString_%s(%s value) {", decl.name.c_str(), decl.name.c_str()); + APPEND_FMT_LN(o, " if (value < 0 || value >= %d) return {};", maxVal); + APPEND_FMT_LN(o, " return %s[value - %d];", arrayName.c_str(), minVal); + APPEND_LIT_LN(o, "}"); out.AddOutputThing(std::move(lookupFunction)); } break; case EVP_Bits: { auto arrayName = GenerateEnumStringArray(out, decl, useExcludeHeuristics); + // TODO } break; case EVP_Random: { auto mapName = GenerateEnumStringMap(out, decl, useExcludeHeuristics); + // TODO } break; case EVP_COUNT: break; } } + + if (options.IsSet(EMGO_FromString)) { + // Generate string -> value lookup table + char mapName[1024]; + snprintf(mapName, sizeof(mapName), "gCG_%s_Str2Val", decl.name.c_str()); + + CodegenOutputThing lookupTable; + auto& o1 = lookupTable.text; + APPEND_FMT_LN(o1, "constinit frozen::unordered_map %s = {", decl.name.c_str(), decl.elements.size(), mapName); + for (auto& elm : decl.elements) { + APPEND_FMT_LN(o1, "{\"%s\", %s::%s},", elm.name.c_str(), decl.name.c_str(), elm.name.c_str()); + } + APPEND_LIT_LN(o1, "};"); + + // Generate lookup function + CodegenOutputThing lookupFunction; + auto& o2 = lookupFunction.text; + APPEND_FMT_LN(o2, "std::optional<%s> EnumFromString_%s(std::string_view value) {", decl.name.c_str(), decl.name.c_str()); + APPEND_FMT_LN(o2, " auto iter = %s.find(value);", mapName); + APPEND_FMT_LN(o2, " if (iter != %s.end()) {", mapName); + APPEND_LIT_LN(o2, " return iter->second;"); + APPEND_LIT_LN(o2, " } else {"); + APPEND_LIT_LN(o2, " return {};"); + APPEND_LIT_LN(o2, " }"); + APPEND_LIT_LN(o2, "}"); + + out.AddOutputThing(std::move(lookupTable)); + out.AddOutputThing(std::move(lookupFunction)); + } } void HandleInputFile(AppState& state, std::string_view source) { @@ -499,7 +531,7 @@ void HandleInputFile(AppState& state, std::string_view source) { printf("[WARNING] unbalanced brace at end of file."); } - state.mainOutput.MergeContents(std::move(fileOutput)); + state.sourceOutput.MergeContents(std::move(fileOutput)); } std::string ReadFileAtOnce(const fs::path& path) { @@ -577,17 +609,29 @@ int main(int argc, char* argv[]) { AppState state; + state.sourceOutput.AddOutputThing(CodegenOutputThing{ + .text = &R"""( +// This file is generated. Any changes will be overidden when building. + +#include +#include +#include +#include +using namespace std::literals; + )"""[1], + }); + // If no cli is provided (argv[0] conventionally but not mandatorily the cli), this will do thing // Otherwise, start with the 2nd element in the array, which is the 1st actual argument if (argc < 2) { // NOTE: keep in sync with various enum options and parser code printf(&R"""( - USAGE: codegen.exe [:]... - where : the _file_ to write generated contents to - is one of: - "single" process this file only - "rec" starting at the given directory , recursively process all .h .c .hpp .cpp files - )"""[1]); +USAGE: codegen.exe [:]... +where : the _file_ to write generated contents to + is one of: + "single" process this file only + "rec" starting at the given directory , recursively process all .h .c .hpp .cpp files +)"""[1]); return -1; } @@ -615,7 +659,7 @@ int main(int argc, char* argv[]) { return -1; } DEFER { fclose(outputFile); }; - state.mainOutput.Write(outputFile); + state.sourceOutput.Write(outputFile); } return 0; diff --git a/buildtools/codegen/tests/examples/TestEnum.hpp b/buildtools/codegen/tests/examples/TestEnum.hpp index c498cd9..6b4ab33 100644 --- a/buildtools/codegen/tests/examples/TestEnum.hpp +++ b/buildtools/codegen/tests/examples/TestEnum.hpp @@ -5,14 +5,14 @@ enum MyEnum { EnumElement2, EnumElement3, }; -BRUSSEL_ENUM(MyEnum, ToString); +BRUSSEL_ENUM(MyEnum, ToString FromString); enum CountedEnumAll { CEA_Foo, CEA_Bar, CEA_COUNT, }; -BRUSSEL_ENUM(CountedEnumAll, ToString); +BRUSSEL_ENUM(CountedEnumAll, ToString FromString); enum CountedEnum { CE_Foo, @@ -20,4 +20,4 @@ enum CountedEnum { CE_FooBar, CE_COUNT, }; -BRUSSEL_ENUM(CountedEnum, ToString ExcludeHeuristics); +BRUSSEL_ENUM(CountedEnum, ToString FromString ExcludeHeuristics); -- cgit v1.2.3-70-g09d2 From 6f29abe5571eb68207986bdadb97b207264ac958 Mon Sep 17 00:00:00 2001 From: rtk0c Date: Sun, 29 May 2022 21:57:53 -0700 Subject: Changeset: 49 Convert codegen output to use template specialization --- CMakeLists.txt | 45 +++++-- README.md | 9 ++ buildtools/codegen/CodegenOutput.inl | 3 +- buildtools/codegen/main.cpp | 148 ++++++++++++++------- buildtools/codegen/tests/examples/TestEnum.hpp | 23 ---- buildtools/codegen/tests/examples/TestEnum.hpp.txt | 21 +++ source-codegen-base/MetadataBase.hpp | 14 ++ 7 files changed, 180 insertions(+), 83 deletions(-) delete mode 100644 buildtools/codegen/tests/examples/TestEnum.hpp create mode 100644 buildtools/codegen/tests/examples/TestEnum.hpp.txt create mode 100644 source-codegen-base/MetadataBase.hpp (limited to 'buildtools/codegen') diff --git a/CMakeLists.txt b/CMakeLists.txt index 19518bf..0e8dc26 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,10 +15,10 @@ add_subdirectory(3rdparty/tracy) # ============================================================================== -file(GLOB_RECURSE commonthings_SOURCES source-common/*.c source-common/*.cpp) -add_library(commonthings OBJECT ${commonthings_SOURCES}) +file(GLOB_RECURSE things_common_SOURCES source-common/*.c source-common/*.cpp) +add_library(things_common OBJECT ${things_common_SOURCES}) -set_target_properties(commonthings PROPERTIES +set_target_properties(things_common PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF @@ -27,8 +27,8 @@ set_target_properties(commonthings PROPERTIES UNITY_BUILD OFF ) -target_include_directories(commonthings PUBLIC source-common) -target_link_libraries(commonthings PUBLIC +target_include_directories(things_common PUBLIC source-common) +target_link_libraries(things_common PUBLIC # External dependencies ${CONAN_LIBS} glm::glm @@ -37,25 +37,45 @@ target_link_libraries(commonthings PUBLIC # ============================================================================== # NOTE: delibrately not recursive, see README.md in the folder for details -file(GLOB meta_codegen_SOURCES buildtools/codegen/*.c buildtools/codegen/*.cpp) -add_executable(meta_codegen ${meta_codegen_SOURCES}) +file(GLOB codegen_SOURCES buildtools/codegen/*.c buildtools/codegen/*.cpp) +add_executable(codegen ${codegen_SOURCES}) -set_target_properties(meta_codegen PROPERTIES +set_target_properties(codegen PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF UNITY_BUILD OFF ) -target_link_libraries(meta_codegen PRIVATE +target_link_libraries(codegen PRIVATE # External dependencies ${CONAN_LIBS} # Project internal components - commonthings + things_common ) -target_flag_rtti(meta_codegen OFF) +target_flag_rtti(codegen OFF) + +# ============================================================================== + +file(GLOB_RECURSE things_codegen_base_SOURCES source-common/*.c source-common/*.cpp) +add_library(things_codegen_base OBJECT ${things_codegen_base_SOURCES}) + +set_target_properties(things_common PROPERTIES + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF +) + +target_include_directories(things_codegen_base PUBLIC source-codegen-base) +target_link_libraries(things_codegen_base PUBLIC + # External dependencies + ${CONAN_LIBS} + + # Project internal components + things_common +) # ============================================================================== @@ -95,7 +115,8 @@ target_link_libraries(${PROJECT_NAME} PRIVATE tracy # Project internal components - commonthings + things_common + things_metadata ) option(BRUSSEL_ENABLE_PROFILING "Whether profiling support is enabled or not." OFF) diff --git a/README.md b/README.md index dc0b4c1..5caf9f7 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,12 @@ + [Urho3D](https://urho3d.io/) + [godot](https://godotengine.org/) + Everything else in `3rdparty/` + +## Project Structure +- `buildtools` + - `cmake`: CMake scripts consumed by the root `CMakeLists.txt`. + - `codegen`: Code generator similar to Qt MOC. + Currently runs on: `source`. +- `source-common`: Code that's compiled as a part of all targets. Check each target's build script for details. +- `source-codegen-base`: Code that's consumed along with output of `buildtools/codegen`. +- `source`: The main game. diff --git a/buildtools/codegen/CodegenOutput.inl b/buildtools/codegen/CodegenOutput.inl index d1f2e50..edd9abc 100644 --- a/buildtools/codegen/CodegenOutput.inl +++ b/buildtools/codegen/CodegenOutput.inl @@ -38,9 +38,8 @@ public: std::move(other.mOutFunctions.begin(), other.mOutFunctions.end(), std::back_inserter(this->mOutFunctions)); } - void Write(FILE* file) { + void Write(FILE* file) const { for (auto& thing : mOutThings) { - WRITE_LIT(file, "// Output thing\n"); fwrite(thing.text.c_str(), sizeof(char), thing.text.size(), file); WRITE_LIT(file, "\n"); } diff --git a/buildtools/codegen/main.cpp b/buildtools/codegen/main.cpp index df52b66..e759c31 100644 --- a/buildtools/codegen/main.cpp +++ b/buildtools/codegen/main.cpp @@ -273,7 +273,7 @@ std::string GenerateEnumStringMap(CodegenOutput& out, const DeclEnum& decl, bool return mapName; } -void GenerateForEnum(CodegenOutput& out, const DeclEnum& decl, EnumFlags options) { +void GenerateForEnum(CodegenOutput& headerOut, CodegenOutput& sourceOut, const DeclEnum& decl, EnumFlags options) { bool useExcludeHeuristics = options.IsSet(EMGO_ExcludeUseHeuristics); if (options.IsSet(EMGO_ToString)) { @@ -281,7 +281,7 @@ void GenerateForEnum(CodegenOutput& out, const DeclEnum& decl, EnumFlags= %d) return {};", maxVal); - APPEND_FMT_LN(o, " return %s[value - %d];", arrayName.c_str(), minVal); - APPEND_LIT_LN(o, "}"); - - out.AddOutputThing(std::move(lookupFunction)); + CodegenOutputThing lookupFunctionDecl; + auto& o1 = lookupFunctionDecl.text; + APPEND_LIT_LN(o1, "template <>"); + APPEND_FMT_LN(o1, "std::string_view Metadata::EnumToString<%s>(%s);", decl.name.c_str(), decl.name.c_str()); + + CodegenOutputThing lookupFunctionDef; + auto& o2 = lookupFunctionDef.text; + APPEND_LIT_LN(o2, "template <>"); + APPEND_FMT_LN(o2, "std::string_view Metadata::EnumToString<%s>(%s value) {", decl.name.c_str(), decl.name.c_str()); + APPEND_FMT_LN(o2, " if (value < 0 || value >= %d) return {};", maxVal); + APPEND_FMT_LN(o2, " return %s[value - %d];", arrayName.c_str(), minVal); + APPEND_LIT_LN(o2, "}"); + + headerOut.AddOutputThing(std::move(lookupFunctionDecl)); + sourceOut.AddOutputThing(std::move(lookupFunctionDef)); } break; case EVP_Bits: { - auto arrayName = GenerateEnumStringArray(out, decl, useExcludeHeuristics); + auto arrayName = GenerateEnumStringArray(sourceOut, decl, useExcludeHeuristics); // TODO } break; case EVP_Random: { - auto mapName = GenerateEnumStringMap(out, decl, useExcludeHeuristics); + auto mapName = GenerateEnumStringMap(sourceOut, decl, useExcludeHeuristics); // TODO } break; @@ -330,19 +337,26 @@ void GenerateForEnum(CodegenOutput& out, const DeclEnum& decl, EnumFlags EnumFromString_%s(std::string_view value) {", decl.name.c_str(), decl.name.c_str()); - APPEND_FMT_LN(o2, " auto iter = %s.find(value);", mapName); - APPEND_FMT_LN(o2, " if (iter != %s.end()) {", mapName); - APPEND_LIT_LN(o2, " return iter->second;"); - APPEND_LIT_LN(o2, " } else {"); - APPEND_LIT_LN(o2, " return {};"); - APPEND_LIT_LN(o2, " }"); - APPEND_LIT_LN(o2, "}"); - - out.AddOutputThing(std::move(lookupTable)); - out.AddOutputThing(std::move(lookupFunction)); + CodegenOutputThing lookupFunctionDecl; + auto& o2 = lookupFunctionDecl.text; + APPEND_LIT_LN(o2, "template <>"); + APPEND_FMT_LN(o2, "std::optional<%s> Metadata::EnumFromString<%s>(std::string_view);", decl.name.c_str(), decl.name.c_str()); + + CodegenOutputThing lookupFunctionDef; + auto& o3 = lookupFunctionDef.text; + APPEND_LIT_LN(o3, "template <>"); + APPEND_FMT_LN(o3, "std::optional<%s> Metadata::EnumFromString<%s>(std::string_view value) {", decl.name.c_str(), decl.name.c_str()); + APPEND_FMT_LN(o3, " auto iter = %s.find(value);", mapName); + APPEND_FMT_LN(o3, " if (iter != %s.end()) {", mapName); + APPEND_LIT_LN(o3, " return iter->second;"); + APPEND_LIT_LN(o3, " } else {"); + APPEND_LIT_LN(o3, " return {};"); + APPEND_LIT_LN(o3, " }"); + APPEND_LIT_LN(o3, "}"); + + sourceOut.AddOutputThing(std::move(lookupTable)); + headerOut.AddOutputThing(std::move(lookupFunctionDecl)); + sourceOut.AddOutputThing(std::move(lookupFunctionDef)); } } @@ -358,8 +372,9 @@ void HandleInputFile(AppState& state, std::string_view source) { printf("END tokens\n"); #endif - CodegenInput fileInput; - CodegenOutput fileOutput; + CodegenInput cgInput; + auto& cgHeaderOutput = state.headerOutput; + auto& cgSourceOutput = state.sourceOutput; int bracePairDepth = 0; while (idx < tokens.size()) { @@ -444,7 +459,7 @@ void HandleInputFile(AppState& state, std::string_view source) { ++idx; } - fileInput.AddEnum(std::move(enumDecl)); + cgInput.AddEnum(std::move(enumDecl)); goto endIdenCase; } @@ -480,7 +495,7 @@ void HandleInputFile(AppState& state, std::string_view source) { } auto& enumName = argList[0][0]->text; - auto enumDecl = fileInput.FindEnumByName(enumName); + auto enumDecl = cgInput.FindEnumByName(enumName); if (!enumDecl) { printf("[ERROR] BRUSSEL_ENUM: referring to non-existent enum '%s'\n", enumName.c_str()); break; @@ -497,7 +512,7 @@ void HandleInputFile(AppState& state, std::string_view source) { } } - GenerateForEnum(fileOutput, *enumDecl, options); + GenerateForEnum(cgHeaderOutput, cgSourceOutput, *enumDecl, options); idx = newIdx; incrementTokenIdx = false; @@ -530,8 +545,6 @@ void HandleInputFile(AppState& state, std::string_view source) { if (bracePairDepth != 0) { printf("[WARNING] unbalanced brace at end of file."); } - - state.sourceOutput.MergeContents(std::move(fileOutput)); } std::string ReadFileAtOnce(const fs::path& path) { @@ -596,6 +609,23 @@ InputOpcode ParseInputOpcode(std::string_view text) { return IOP_COUNT; } +bool WriteOutputFile(const CodegenOutput& output, const char* dirPath, const char* fileName) { + char path[2048]; + snprintf(path, sizeof(path), "%s/%s", dirPath, fileName); + + auto outputFile = Utils::OpenCstdioFile(path, Utils::WriteTruncate); + if (!outputFile) { + printf("[ERROR] unable to open output file %s", path); + return false; + } + DEFER { fclose(outputFile); }; + + DEBUG_PRINTF("Writing output %s", path); + output.Write(outputFile); + + return true; +} + int main(int argc, char* argv[]) { STR_LUT_INIT(ClexNames); BSTR_LUT_INIT(CppKeyword); @@ -609,14 +639,22 @@ int main(int argc, char* argv[]) { AppState state; + state.headerOutput.AddOutputThing(CodegenOutputThing{ + .text = &R"""( +// This file is generated. Any changes will be overidden when building. +#pragma once + +#include +)"""[1], + }); + state.sourceOutput.AddOutputThing(CodegenOutputThing{ .text = &R"""( // This file is generated. Any changes will be overidden when building. +#include "MetadataImpl.hpp" #include #include -#include -#include using namespace std::literals; )"""[1], }); @@ -627,7 +665,7 @@ using namespace std::literals; // NOTE: keep in sync with various enum options and parser code printf(&R"""( USAGE: codegen.exe [:]... -where : the _file_ to write generated contents to +where : the directory to write generated contents to. This will NOT automatically create the directory. is one of: "single" process this file only "rec" starting at the given directory , recursively process all .h .c .hpp .cpp files @@ -635,8 +673,33 @@ where : the _file_ to write generated contents to return -1; } - const char* outputFilePath = argv[1]; - DEBUG_PRINTF("Outputting to file %s.\n", outputFilePath); + const char* outputDir = argv[1]; + DEBUG_PRINTF("Outputting to directory %s.\n", outputDirPath); + + { + char path[2048]; + snprintf(path, sizeof(path), "%s/%s", outputDir, "Metadata.hpp"); + + auto fileContent = R"""( +// This file is generated. Any changes will be overidden when building. +// This file is an umbrella header for all generated metadata. End users simply include this file to get access to everything. +#pragma once + +#include + +#include +)"""sv.substr(1); // Get rid of the \n at the beginning + + auto file = Utils::OpenCstdioFile(path, Utils::WriteTruncate); + if (!file) { + printf("[ERROR] unable to open output file %s", path); + return -1; + } + DEFER { fclose(file); }; + + DEBUG_PRINTF("Writing output file %s", path); + fwrite(fileContent.data(), sizeof(decltype(fileContent)::value_type), fileContent.size(), file); + } for (int i = 2; i < argc; ++i) { std::string_view arg(argv[i]); @@ -652,15 +715,8 @@ where : the _file_ to write generated contents to } } - { - auto outputFile = Utils::OpenCstdioFile(outputFilePath, Utils::WriteTruncate); - if (!outputFile) { - printf("[ERROR] unable to open output file %s", outputFilePath); - return -1; - } - DEFER { fclose(outputFile); }; - state.sourceOutput.Write(outputFile); - } + WriteOutputFile(state.headerOutput, outputDir, "MetadataImpl.hpp"); + WriteOutputFile(state.sourceOutput, outputDir, "MetadataImpl.cpp"); return 0; } diff --git a/buildtools/codegen/tests/examples/TestEnum.hpp b/buildtools/codegen/tests/examples/TestEnum.hpp deleted file mode 100644 index 6b4ab33..0000000 --- a/buildtools/codegen/tests/examples/TestEnum.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "MacrosCodegen.hpp" - -enum MyEnum { - EnumElement1, - EnumElement2, - EnumElement3, -}; -BRUSSEL_ENUM(MyEnum, ToString FromString); - -enum CountedEnumAll { - CEA_Foo, - CEA_Bar, - CEA_COUNT, -}; -BRUSSEL_ENUM(CountedEnumAll, ToString FromString); - -enum CountedEnum { - CE_Foo, - CE_Bar, - CE_FooBar, - CE_COUNT, -}; -BRUSSEL_ENUM(CountedEnum, ToString FromString ExcludeHeuristics); diff --git a/buildtools/codegen/tests/examples/TestEnum.hpp.txt b/buildtools/codegen/tests/examples/TestEnum.hpp.txt new file mode 100644 index 0000000..e596c5e --- /dev/null +++ b/buildtools/codegen/tests/examples/TestEnum.hpp.txt @@ -0,0 +1,21 @@ +enum MyEnum { + EnumElement1, + EnumElement2, + EnumElement3, +}; +BRUSSEL_ENUM(MyEnum, ToString FromString); + +enum CountedEnumAll { + CEA_Foo, + CEA_Bar, + CEA_COUNT, +}; +BRUSSEL_ENUM(CountedEnumAll, ToString FromString); + +enum CountedEnum { + CE_Foo, + CE_Bar, + CE_FooBar, + CE_COUNT, +}; +BRUSSEL_ENUM(CountedEnum, ToString FromString ExcludeHeuristics); diff --git a/source-codegen-base/MetadataBase.hpp b/source-codegen-base/MetadataBase.hpp new file mode 100644 index 0000000..59c61da --- /dev/null +++ b/source-codegen-base/MetadataBase.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +namespace Metadata { + +template +std::string_view EnumToString(TEnum value) = delete; + +template +std::optional EnumFromString(std::string_view str) = delete; + +} // namespace Metadata -- cgit v1.2.3-70-g09d2 From cdd84f25ab1d2a57ee5c7b4c954e35a8d7e2dca3 Mon Sep 17 00:00:00 2001 From: rtk0c Date: Sun, 29 May 2022 22:04:13 -0700 Subject: Changeset: 50 Fix buildsystem mistakes in changeset 49, add debug print option --- CMakeLists.txt | 11 ++++++++--- buildtools/codegen/CodegenConfig.hpp | 4 +++- buildtools/codegen/main.cpp | 2 +- source-codegen-base/MetadataBase.cpp | 1 + 4 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 source-codegen-base/MetadataBase.cpp (limited to 'buildtools/codegen') diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e8dc26..55db14b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,12 +57,17 @@ target_link_libraries(codegen PRIVATE target_flag_rtti(codegen OFF) +option(BRUSSEL_CODEGEN_DEBUG_PRINT "Enable debug printing in the code generator or not." OFF) +if(BRUSSEL_CODEGEN_DEBUG_PRINT) + target_compile_definitions(codegen PRIVATE CODEGEN_DEBUG_PRINT=1) +endif() + # ============================================================================== -file(GLOB_RECURSE things_codegen_base_SOURCES source-common/*.c source-common/*.cpp) +file(GLOB_RECURSE things_codegen_base_SOURCES source-codegen-base/*.c source-codegen-base/*.cpp) add_library(things_codegen_base OBJECT ${things_codegen_base_SOURCES}) -set_target_properties(things_common PROPERTIES +set_target_properties(things_codegen_base PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF @@ -116,7 +121,7 @@ target_link_libraries(${PROJECT_NAME} PRIVATE # Project internal components things_common - things_metadata + things_codegen_base ) option(BRUSSEL_ENABLE_PROFILING "Whether profiling support is enabled or not." OFF) diff --git a/buildtools/codegen/CodegenConfig.hpp b/buildtools/codegen/CodegenConfig.hpp index 4ed576a..b9dc56c 100644 --- a/buildtools/codegen/CodegenConfig.hpp +++ b/buildtools/codegen/CodegenConfig.hpp @@ -1,6 +1,8 @@ #pragma once -#define CODEGEN_DEBUG_PRINT 1 +#ifndef CODEGEN_DEBUG_PRINT +# define CODEGEN_DEBUG_PRINT 0 +#endif #if CODEGEN_DEBUG_PRINT # define DEBUG_PRINTF(...) printf(__VA_ARGS__) diff --git a/buildtools/codegen/main.cpp b/buildtools/codegen/main.cpp index e759c31..298f19e 100644 --- a/buildtools/codegen/main.cpp +++ b/buildtools/codegen/main.cpp @@ -674,7 +674,7 @@ where : the directory to write generated contents to. This will N } const char* outputDir = argv[1]; - DEBUG_PRINTF("Outputting to directory %s.\n", outputDirPath); + DEBUG_PRINTF("Outputting to directory %s.\n", outputDir); { char path[2048]; diff --git a/source-codegen-base/MetadataBase.cpp b/source-codegen-base/MetadataBase.cpp new file mode 100644 index 0000000..3ccf870 --- /dev/null +++ b/source-codegen-base/MetadataBase.cpp @@ -0,0 +1 @@ +#include "MetadataBase.hpp" -- cgit v1.2.3-70-g09d2 From ce9559e8c2b69d46cff064241bd9a04c014af44f Mon Sep 17 00:00:00 2001 From: rtk0c Date: Mon, 30 May 2022 13:09:26 -0700 Subject: Changeset: 51 Add integration into the main game --- CMakeLists.txt | 24 +++++- buildtools/codegen/CodegenOutput.inl | 19 ++++- buildtools/codegen/CodegenUtils.inl | 73 ++++++++++++++++ buildtools/codegen/main.cpp | 154 ++++++++++------------------------ source-codegen-base/MacrosCodegen.hpp | 7 ++ source-codegen-base/Metadata.cpp | 1 + source-codegen-base/Metadata.hpp | 4 + source-codegen-base/MetadataBase.hpp | 4 +- source-common/Macros.hpp | 2 + source-common/MacrosCodegen.hpp | 7 -- source/GraphicsTags.cpp | 2 + source/GraphicsTags.hpp | 12 +++ source/main.cpp | 5 ++ 13 files changed, 191 insertions(+), 123 deletions(-) create mode 100644 buildtools/codegen/CodegenUtils.inl create mode 100644 source-codegen-base/MacrosCodegen.hpp create mode 100644 source-codegen-base/Metadata.cpp create mode 100644 source-codegen-base/Metadata.hpp delete mode 100644 source-common/MacrosCodegen.hpp (limited to 'buildtools/codegen') diff --git a/CMakeLists.txt b/CMakeLists.txt index 55db14b..4800935 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,8 +62,6 @@ if(BRUSSEL_CODEGEN_DEBUG_PRINT) target_compile_definitions(codegen PRIVATE CODEGEN_DEBUG_PRINT=1) endif() -# ============================================================================== - file(GLOB_RECURSE things_codegen_base_SOURCES source-codegen-base/*.c source-codegen-base/*.cpp) add_library(things_codegen_base OBJECT ${things_codegen_base_SOURCES}) @@ -82,6 +80,25 @@ target_link_libraries(things_codegen_base PUBLIC things_common ) +# TODO support reading all files from the target, instead of manually supplying a search dir +function(target_gen_metadata TARGET_NAME SEARCH_DIR) + get_target_property(TARGET_SOURCES ${TARGET_NAME} SOURCES) + + set(OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/${TARGET_NAME}) + set(OUTPUT_FILES + ${OUTPUT_DIR}/generated/GeneratedCode.hpp + ${OUTPUT_DIR}/generated/GeneratedCode.cpp + ) + add_custom_command( + OUTPUT ${OUTPUT_FILES} + COMMAND codegen ${OUTPUT_DIR}/generated rec:${SEARCH_DIR} + DEPENDS ${TARGET_SOURCES} + ) + + target_include_directories(${TARGET_NAME} PRIVATE ${OUTPUT_DIR}) + target_sources(${TARGET_NAME} PRIVATE ${OUTPUT_FILES}) +endfunction() + # ============================================================================== # add_executable requires at least one source file @@ -153,3 +170,6 @@ if(BRUSSEL_ENABLE_ASAN) -fno-omit-frame-pointer ) endif() + +get_filename_component(METADATA_INP_DIR "source" ABSOLUTE) +target_gen_metadata(${PROJECT_NAME} ${METADATA_INP_DIR}) diff --git a/buildtools/codegen/CodegenOutput.inl b/buildtools/codegen/CodegenOutput.inl index edd9abc..ff7b912 100644 --- a/buildtools/codegen/CodegenOutput.inl +++ b/buildtools/codegen/CodegenOutput.inl @@ -3,6 +3,9 @@ #include "CodegenDecl.hpp" #include "CodegenMacros.hpp" +#include + +#include #include #include #include @@ -16,6 +19,7 @@ struct CodegenOutputThing { class CodegenOutput { private: + robin_hood::unordered_set mRequestIncludes; std::vector mOutThings; std::vector mOutStructs; std::vector mOutEnums; @@ -27,6 +31,12 @@ public: bool optionAutoAddPrefix : 1 = false; public: + void AddRequestInclude(std::string_view include) { + if (!mRequestIncludes.contains(include)) { + mRequestIncludes.insert(std::string(include)); + } + } + void AddOutputThing(CodegenOutputThing thing) { mOutThings.push_back(std::move(thing)); } @@ -39,15 +49,20 @@ public: } void Write(FILE* file) const { + for (auto& include : mRequestIncludes) { + // TODO how to resolve to the correct include paths? + WRITE_FMT_LN(file, "#include <%s>", include.c_str()); + } + for (auto& thing : mOutThings) { fwrite(thing.text.c_str(), sizeof(char), thing.text.size(), file); WRITE_LIT(file, "\n"); } for (auto& declStruct : mOutStructs) { - WRITE_FMT(file, "struct %s {\n", declStruct.name.c_str()); + WRITE_FMT_LN(file, "struct %s {", declStruct.name.c_str()); // TODO - WRITE_LIT(file, "}\n"); + WRITE_LIT_LN(file, "};"); } for (auto& declEnum : mOutEnums) { diff --git a/buildtools/codegen/CodegenUtils.inl b/buildtools/codegen/CodegenUtils.inl new file mode 100644 index 0000000..ea46ac1 --- /dev/null +++ b/buildtools/codegen/CodegenUtils.inl @@ -0,0 +1,73 @@ +#pragma once + +#include "CodegenConfig.hpp" +#include "CodegenMacros.hpp" + +#include "CodegenOutput.inl" + +#include +#include + +#include +#include +#include + +namespace Utils { + +std::string ReadFileAsString(const std::filesystem::path& path) { + auto file = Utils::OpenCstdioFile(path, Utils::Read); + if (!file) throw std::runtime_error("Failed to open source file."); + DEFER { fclose(file); }; + + fseek(file, 0, SEEK_END); + auto fileSize = ftell(file); + rewind(file); + + std::string result(fileSize, '\0'); + fread(result.data(), fileSize, 1, file); + + return result; +} + +bool WriteOutputFile(const CodegenOutput& output, std::string_view dir, std::string_view filename, std::string_view additionalSuffix = {}) { + char path[2048]; + snprintf(path, sizeof(path), "%.*s/%.*s%.*s", PRINTF_STRING_VIEW(dir), PRINTF_STRING_VIEW(filename), PRINTF_STRING_VIEW(additionalSuffix)); + + auto outputFile = Utils::OpenCstdioFile(path, Utils::WriteTruncate); + if (!outputFile) { + printf("[ERROR] unable to open output file %s\n", path); + return false; + } + DEFER { fclose(outputFile); }; + + DEBUG_PRINTF("Writing output %s\n", path); + output.Write(outputFile); + + return true; +} + +void ProduceGeneratedHeaderFileHeader(CodegenOutput& output) { + output.AddOutputThing(CodegenOutputThing{ + .text = &R"""( +// This file is generated. Any changes will be overidden when building. +#pragma once + +#include +)"""[1], + }); +} + +void ProduceGeneratedSourceFileHeader(CodegenOutput& output) { + output.AddOutputThing(CodegenOutputThing{ + .text = &R"""( +// This file is generated. Any changes will be overidden when building. +#include "GeneratedCode.hpp" + +#include +#include +using namespace std::literals; + )"""[1], + }); +} + +} // namespace Utils diff --git a/buildtools/codegen/main.cpp b/buildtools/codegen/main.cpp index 298f19e..3a3e443 100644 --- a/buildtools/codegen/main.cpp +++ b/buildtools/codegen/main.cpp @@ -4,6 +4,7 @@ #include "CodegenInput.inl" #include "CodegenOutput.inl" +#include "CodegenUtils.inl" #include #include @@ -27,10 +28,12 @@ using namespace std::literals; namespace fs = std::filesystem; // TODO handle namespace +// TODO support codegen target in .cpp files struct AppState { - CodegenOutput headerOutput; - CodegenOutput sourceOutput; + std::string_view outputDir; + CodegenOutput mainHeaderOutput; + CodegenOutput mainSourceOutput; }; enum { @@ -113,7 +116,7 @@ bool StbTokenIsMultiChar(int lexerToken) { void CheckBraceDepth(int braceDpeth) { if (braceDpeth < 0) { - printf("[WARNING] unbalanced brace"); + printf("[WARNING] unbalanced brace\n"); } } @@ -197,7 +200,7 @@ std::vector RecordTokens(std::string_view source) { } if (lexer.token == CLEX_parse_error) { - printf("[ERROR] stb_c_lexer countered a parse error."); + printf("[ERROR] stb_c_lexer countered a parse error.\n"); // TODO how to handle? continue; } @@ -292,11 +295,6 @@ void GenerateForEnum(CodegenOutput& headerOut, CodegenOutput& sourceOut, const D maxVal = decl.elements[decl.elements.size() - 2].value; } - CodegenOutputThing lookupFunctionDecl; - auto& o1 = lookupFunctionDecl.text; - APPEND_LIT_LN(o1, "template <>"); - APPEND_FMT_LN(o1, "std::string_view Metadata::EnumToString<%s>(%s);", decl.name.c_str(), decl.name.c_str()); - CodegenOutputThing lookupFunctionDef; auto& o2 = lookupFunctionDef.text; APPEND_LIT_LN(o2, "template <>"); @@ -305,7 +303,6 @@ void GenerateForEnum(CodegenOutput& headerOut, CodegenOutput& sourceOut, const D APPEND_FMT_LN(o2, " return %s[value - %d];", arrayName.c_str(), minVal); APPEND_LIT_LN(o2, "}"); - headerOut.AddOutputThing(std::move(lookupFunctionDecl)); sourceOut.AddOutputThing(std::move(lookupFunctionDef)); } break; @@ -337,11 +334,6 @@ void GenerateForEnum(CodegenOutput& headerOut, CodegenOutput& sourceOut, const D APPEND_LIT_LN(o1, "};"); // Generate lookup function - CodegenOutputThing lookupFunctionDecl; - auto& o2 = lookupFunctionDecl.text; - APPEND_LIT_LN(o2, "template <>"); - APPEND_FMT_LN(o2, "std::optional<%s> Metadata::EnumFromString<%s>(std::string_view);", decl.name.c_str(), decl.name.c_str()); - CodegenOutputThing lookupFunctionDef; auto& o3 = lookupFunctionDef.text; APPEND_LIT_LN(o3, "template <>"); @@ -355,12 +347,11 @@ void GenerateForEnum(CodegenOutput& headerOut, CodegenOutput& sourceOut, const D APPEND_LIT_LN(o3, "}"); sourceOut.AddOutputThing(std::move(lookupTable)); - headerOut.AddOutputThing(std::move(lookupFunctionDecl)); sourceOut.AddOutputThing(std::move(lookupFunctionDef)); } } -void HandleInputFile(AppState& state, std::string_view source) { +void HandleInputFile(AppState& state, std::string_view filenameStem, std::string_view source) { auto tokens = RecordTokens(source); size_t idx = 0; @@ -373,8 +364,10 @@ void HandleInputFile(AppState& state, std::string_view source) { #endif CodegenInput cgInput; - auto& cgHeaderOutput = state.headerOutput; - auto& cgSourceOutput = state.sourceOutput; + CodegenOutput cgHeaderOutput; + Utils::ProduceGeneratedHeaderFileHeader(cgHeaderOutput); + CodegenOutput cgSourceOutput; + Utils::ProduceGeneratedSourceFileHeader(cgSourceOutput); int bracePairDepth = 0; while (idx < tokens.size()) { @@ -545,21 +538,9 @@ void HandleInputFile(AppState& state, std::string_view source) { if (bracePairDepth != 0) { printf("[WARNING] unbalanced brace at end of file."); } -} - -std::string ReadFileAtOnce(const fs::path& path) { - auto file = Utils::OpenCstdioFile(path, Utils::Read); - if (!file) throw std::runtime_error("Failed to open source file."); - DEFER { fclose(file); }; - fseek(file, 0, SEEK_END); - auto fileSize = ftell(file); - rewind(file); - - std::string result(fileSize, '\0'); - fread(result.data(), fileSize, 1, file); - - return result; + Utils::WriteOutputFile(cgHeaderOutput, state.outputDir, filenameStem, ".gh.inl"sv); + Utils::WriteOutputFile(cgSourceOutput, state.outputDir, filenameStem, ".gs.inl"sv); } enum InputOpcode { @@ -571,12 +552,17 @@ enum InputOpcode { void HandleArgument(AppState& state, InputOpcode opcode, std::string_view operand) { switch (opcode) { case IOP_ProcessSingleFile: { - fs::path filePath(operand); - auto source = ReadFileAtOnce(filePath); - HandleInputFile(state, source); + DEBUG_PRINTF("Processing single file %.*s\n", PRINTF_STRING_VIEW(operand)); + + fs::path path(operand); + auto filenameStem = path.stem().string(); + auto source = Utils::ReadFileAsString(path); + HandleInputFile(state, filenameStem, source); } break; case IOP_ProcessRecursively: { + DEBUG_PRINTF("Recursively processing folder %.*s\n", PRINTF_STRING_VIEW(operand)); + fs::path startPath(operand); for (auto& item : fs::recursive_directory_iterator(startPath)) { if (!item.is_regular_file()) { @@ -584,15 +570,19 @@ void HandleArgument(AppState& state, InputOpcode opcode, std::string_view operan } auto& path = item.path(); - auto filename = path.filename().string(); - if (filename != ".c" || - filename != ".cpp") + auto pathExt = path.extension(); + auto pathStem = path.stem(); + if (pathExt != ".h" && + pathExt != ".hpp") { continue; } - auto source = ReadFileAtOnce(path); - HandleInputFile(state, source); + DEBUG_PRINTF("Processing subfile %s\n", path.string().c_str()); + + auto filenameStem = pathStem.string(); + auto source = Utils::ReadFileAsString(path); + HandleInputFile(state, filenameStem, source); } } break; @@ -605,25 +595,10 @@ InputOpcode ParseInputOpcode(std::string_view text) { return IOP_ProcessSingleFile; } else if (text == "rec"sv) { return IOP_ProcessRecursively; + } else { + DEBUG_PRINTF("Unknown input opcode %s\n", text.data()); + throw std::runtime_error("Unknown input opcode"); } - return IOP_COUNT; -} - -bool WriteOutputFile(const CodegenOutput& output, const char* dirPath, const char* fileName) { - char path[2048]; - snprintf(path, sizeof(path), "%s/%s", dirPath, fileName); - - auto outputFile = Utils::OpenCstdioFile(path, Utils::WriteTruncate); - if (!outputFile) { - printf("[ERROR] unable to open output file %s", path); - return false; - } - DEFER { fclose(outputFile); }; - - DEBUG_PRINTF("Writing output %s", path); - output.Write(outputFile); - - return true; } int main(int argc, char* argv[]) { @@ -639,25 +614,8 @@ int main(int argc, char* argv[]) { AppState state; - state.headerOutput.AddOutputThing(CodegenOutputThing{ - .text = &R"""( -// This file is generated. Any changes will be overidden when building. -#pragma once - -#include -)"""[1], - }); - - state.sourceOutput.AddOutputThing(CodegenOutputThing{ - .text = &R"""( -// This file is generated. Any changes will be overidden when building. -#include "MetadataImpl.hpp" - -#include -#include -using namespace std::literals; - )"""[1], - }); + Utils::ProduceGeneratedHeaderFileHeader(state.mainHeaderOutput); + Utils::ProduceGeneratedSourceFileHeader(state.mainSourceOutput); // If no cli is provided (argv[0] conventionally but not mandatorily the cli), this will do thing // Otherwise, start with the 2nd element in the array, which is the 1st actual argument @@ -668,55 +626,31 @@ USAGE: codegen.exe [:]... where : the directory to write generated contents to. This will NOT automatically create the directory. is one of: "single" process this file only - "rec" starting at the given directory , recursively process all .h .c .hpp .cpp files + "rec" starting at the given directory , recursively process all .h .hpp files )"""[1]); return -1; } - const char* outputDir = argv[1]; + state.outputDir = std::string_view(argv[1]); DEBUG_PRINTF("Outputting to directory %s.\n", outputDir); - { - char path[2048]; - snprintf(path, sizeof(path), "%s/%s", outputDir, "Metadata.hpp"); - - auto fileContent = R"""( -// This file is generated. Any changes will be overidden when building. -// This file is an umbrella header for all generated metadata. End users simply include this file to get access to everything. -#pragma once - -#include - -#include -)"""sv.substr(1); // Get rid of the \n at the beginning - - auto file = Utils::OpenCstdioFile(path, Utils::WriteTruncate); - if (!file) { - printf("[ERROR] unable to open output file %s", path); - return -1; - } - DEFER { fclose(file); }; - - DEBUG_PRINTF("Writing output file %s", path); - fwrite(fileContent.data(), sizeof(decltype(fileContent)::value_type), fileContent.size(), file); - } - for (int i = 2; i < argc; ++i) { - std::string_view arg(argv[i]); + const char* argRaw = argv[i]; + std::string_view arg(argRaw); + DEBUG_PRINTF("Processing input command %s\n", argRaw); + auto separatorLoc = arg.find(':'); if (separatorLoc != std::string_view::npos) { auto opcodeString = arg.substr(0, separatorLoc); auto opcode = ParseInputOpcode(opcodeString); auto operand = arg.substr(separatorLoc + 1); - DEBUG_PRINTF("Processing input command %.*s at path %.*s\n", (int)opcodeString.size(), opcodeString.data(), (int)operand.size(), operand.data()); - HandleArgument(state, opcode, operand); } } - WriteOutputFile(state.headerOutput, outputDir, "MetadataImpl.hpp"); - WriteOutputFile(state.sourceOutput, outputDir, "MetadataImpl.cpp"); + Utils::WriteOutputFile(state.mainHeaderOutput, state.outputDir, "GeneratedCode.hpp"sv); + Utils::WriteOutputFile(state.mainSourceOutput, state.outputDir, "GeneratedCode.cpp"sv); return 0; } diff --git a/source-codegen-base/MacrosCodegen.hpp b/source-codegen-base/MacrosCodegen.hpp new file mode 100644 index 0000000..6803023 --- /dev/null +++ b/source-codegen-base/MacrosCodegen.hpp @@ -0,0 +1,7 @@ +// NOTE: contents of this file is coupled with buildtools/codegen/ +// when updating, change both sides at the same time + +#pragma once + +#define BRUSSEL_CLASS(name, options) +#define BRUSSEL_ENUM(name, options) diff --git a/source-codegen-base/Metadata.cpp b/source-codegen-base/Metadata.cpp new file mode 100644 index 0000000..ee32054 --- /dev/null +++ b/source-codegen-base/Metadata.cpp @@ -0,0 +1 @@ +#include "Metadata.hpp" diff --git a/source-codegen-base/Metadata.hpp b/source-codegen-base/Metadata.hpp new file mode 100644 index 0000000..a038c15 --- /dev/null +++ b/source-codegen-base/Metadata.hpp @@ -0,0 +1,4 @@ +#pragma once + +#include "MacrosCodegen.hpp" +#include "MetadataBase.hpp" diff --git a/source-codegen-base/MetadataBase.hpp b/source-codegen-base/MetadataBase.hpp index 59c61da..8be668d 100644 --- a/source-codegen-base/MetadataBase.hpp +++ b/source-codegen-base/MetadataBase.hpp @@ -6,9 +6,9 @@ namespace Metadata { template -std::string_view EnumToString(TEnum value) = delete; +std::string_view EnumToString(TEnum value); template -std::optional EnumFromString(std::string_view str) = delete; +std::optional EnumFromString(std::string_view str); } // namespace Metadata diff --git a/source-common/Macros.hpp b/source-common/Macros.hpp index b5d05fa..a255ada 100644 --- a/source-common/Macros.hpp +++ b/source-common/Macros.hpp @@ -14,6 +14,8 @@ #define UNUSED(x) (void)x; +#define PRINTF_STRING_VIEW(s) (int)s.size(), s.data() + #if defined(_MSC_VER) # define UNREACHABLE __assume(0) #elif defined(__GNUC__) || defined(__clang__) diff --git a/source-common/MacrosCodegen.hpp b/source-common/MacrosCodegen.hpp deleted file mode 100644 index 6803023..0000000 --- a/source-common/MacrosCodegen.hpp +++ /dev/null @@ -1,7 +0,0 @@ -// NOTE: contents of this file is coupled with buildtools/codegen/ -// when updating, change both sides at the same time - -#pragma once - -#define BRUSSEL_CLASS(name, options) -#define BRUSSEL_ENUM(name, options) diff --git a/source/GraphicsTags.cpp b/source/GraphicsTags.cpp index b389acf..522a58f 100644 --- a/source/GraphicsTags.cpp +++ b/source/GraphicsTags.cpp @@ -306,3 +306,5 @@ GLenum Tags::FindGLType(std::string_view name) { return GL_NONE; } } + +#include diff --git a/source/GraphicsTags.hpp b/source/GraphicsTags.hpp index 34c0885..09e62bf 100644 --- a/source/GraphicsTags.hpp +++ b/source/GraphicsTags.hpp @@ -5,6 +5,16 @@ #include #include +#include + +enum TestEnum { + TE_Position, + TE_Color, + TE_TexCoord, + TE_COUNT, +}; +BRUSSEL_ENUM(TestEnum, ToString FromString ExcludeHeuristics); + namespace Tags { /// Vertex element semantics, used to identify the meaning of vertex buffer contents enum VertexElementSemantic { @@ -96,3 +106,5 @@ GLenum FindGLType(std::string_view name); constexpr GLuint kInvalidLocation = std::numeric_limits::max(); } // namespace Tags + +#include diff --git a/source/main.cpp b/source/main.cpp index c49fc0b..353746b 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -3,9 +3,11 @@ #include "AppConfig.hpp" #include "CommonVertexIndex.hpp" #include "EditorGuizmo.hpp" +#include "GraphicsTags.hpp" #include "Ires.hpp" #include "Level.hpp" #include "Material.hpp" +#include "Metadata.hpp" #include "Shader.hpp" #define GLFW_INCLUDE_NONE @@ -102,6 +104,9 @@ fs::path GetEnvVar(const char* name, const char* backup) { int main(int argc, char* argv[]) { using namespace Tags; + auto str = Metadata::EnumToString(TE_Color); + printf("%.*s", PRINTF_STRING_VIEW(str)); + constexpr auto kImGuiBackend = "imgui-backend"; constexpr auto kGameDataDir = "game-data-directory"; constexpr auto kGameAssetDir = "game-asset-directory"; -- cgit v1.2.3-70-g09d2 From 7d8bca09b3c4bf1418e758bd3bd0d6f85672153e Mon Sep 17 00:00:00 2001 From: rtk0c Date: Mon, 30 May 2022 15:52:19 -0700 Subject: Changeset: 52 Add support for namespaced enums --- buildtools/codegen/CodegenDecl.hpp | 9 + buildtools/codegen/CodegenInput.inl | 35 +++- buildtools/codegen/CodegenUtils.inl | 33 ++++ buildtools/codegen/main.cpp | 191 ++++++++++++++++----- buildtools/codegen/tests/examples/TestEnum.hpp.txt | 22 +++ source/GraphicsTags.hpp | 11 +- source/main.cpp | 5 - 7 files changed, 245 insertions(+), 61 deletions(-) (limited to 'buildtools/codegen') diff --git a/buildtools/codegen/CodegenDecl.hpp b/buildtools/codegen/CodegenDecl.hpp index b7c1ee4..32d5445 100644 --- a/buildtools/codegen/CodegenDecl.hpp +++ b/buildtools/codegen/CodegenDecl.hpp @@ -3,8 +3,15 @@ #include #include +struct DeclNamespace { + DeclNamespace* container = nullptr; + std::string name; + std::string_view fullname; // View into storage map key +}; + // Structs or classes struct DeclStruct { + DeclNamespace* container = nullptr; std::string name; }; @@ -40,6 +47,7 @@ struct DeclEnumElement { }; struct DeclEnum { + DeclNamespace* container = nullptr; std::string name; std::vector elements; EnumUnderlyingType underlyingType; @@ -56,6 +64,7 @@ struct DeclFunctionArgument { }; struct DeclFunction { + DeclNamespace* container = nullptr; // Things like extern, static, etc. that gets written before the function return type std::string prefix; std::string name; diff --git a/buildtools/codegen/CodegenInput.inl b/buildtools/codegen/CodegenInput.inl index 80a39d0..0809e7f 100644 --- a/buildtools/codegen/CodegenInput.inl +++ b/buildtools/codegen/CodegenInput.inl @@ -3,20 +3,26 @@ #include "CodegenConfig.hpp" #include "CodegenDecl.hpp" +#include "CodegenUtils.inl" + #include #include #include #include +#include #include +using namespace std::literals; + class CodegenInput { private: std::vector mEnums; - robin_hood::unordered_map mDeclByName; + robin_hood::unordered_flat_map mDeclByName; + robin_hood::unordered_node_map mNamespaces; public: - void AddEnum(DeclEnum decl) { + void AddEnum(std::string fullname, DeclEnum decl) { #if CODEGEN_DEBUG_PRINT printf("Committed enum '%s'\n", decl.name.c_str()); for (auto& elm : decl.elements) { @@ -24,10 +30,20 @@ public: } #endif - mDeclByName.try_emplace(std::string(decl.name), mEnums.size()); + mDeclByName.try_emplace(std::move(fullname), mEnums.size()); mEnums.push_back(std::move(decl)); } + DeclNamespace* AddNamespace(DeclNamespace ns) { + auto path = Utils::MakeFullName(""sv, &ns); + auto [iter, success] = mNamespaces.try_emplace(std::move(path), std::move(ns)); + auto& nsRef = iter->second; + if (success) { + nsRef.fullname = iter->first; + } + return &nsRef; + } + const DeclEnum* FindEnumByName(std::string_view name) const { // TODO handle multiple kinds of decl auto iter = mDeclByName.find(name); @@ -37,4 +53,17 @@ public: return nullptr; } } + + const DeclNamespace* FindNamespace(std::string_view fullname) const { + auto iter = mNamespaces.find(fullname); + if (iter != mNamespaces.end()) { + return &iter->second; + } else { + return nullptr; + } + } + + DeclNamespace* FindNamespace(std::string_view name) { + return const_cast(const_cast(this)->FindNamespace(name)); + } }; diff --git a/buildtools/codegen/CodegenUtils.inl b/buildtools/codegen/CodegenUtils.inl index ea46ac1..f9d913e 100644 --- a/buildtools/codegen/CodegenUtils.inl +++ b/buildtools/codegen/CodegenUtils.inl @@ -46,6 +46,34 @@ bool WriteOutputFile(const CodegenOutput& output, std::string_view dir, std::str return true; } +std::string MakeFullName(std::string_view name, DeclNamespace* ns = nullptr) { + size_t length = 0; + std::vector components; + if (!name.empty()) { + components.push_back(name); + length += name.length(); + } + + DeclNamespace* currentNamespace = ns; + while (currentNamespace) { + components.push_back(currentNamespace->name); + length += currentNamespace->name.size() + /*::*/ 2; + currentNamespace = currentNamespace->container; + } + + std::string fullname; + fullname.reserve(length); + for (auto it = components.rbegin(); it != components.rend(); ++it) { + fullname += *it; + fullname += "::"; + } + // Get rid of the last "::" + fullname.pop_back(); + fullname.pop_back(); + + return fullname; +} + void ProduceGeneratedHeaderFileHeader(CodegenOutput& output) { output.AddOutputThing(CodegenOutputThing{ .text = &R"""( @@ -53,6 +81,9 @@ void ProduceGeneratedHeaderFileHeader(CodegenOutput& output) { #pragma once #include + +#include +#include )"""[1], }); } @@ -63,6 +94,8 @@ void ProduceGeneratedSourceFileHeader(CodegenOutput& output) { // This file is generated. Any changes will be overidden when building. #include "GeneratedCode.hpp" +#include +#include #include #include using namespace std::literals; diff --git a/buildtools/codegen/main.cpp b/buildtools/codegen/main.cpp index 3a3e443..2c259a4 100644 --- a/buildtools/codegen/main.cpp +++ b/buildtools/codegen/main.cpp @@ -75,6 +75,7 @@ STR_LUT_DECL(ClexNames, CLEX_eof, CLEX_ext_COUNT) { } enum CppKeyword { + CKw_Namespace, CKw_Struct, CKw_Class, CKw_Enum, @@ -83,6 +84,7 @@ enum CppKeyword { BSTR_LUT_DECL(CppKeyword, 0, CKw_COUNT) { BSTR_LUT_MAP_FOR(CppKeyword); + BSTR_LUT_MAP(CKw_Namespace, "namespace"); BSTR_LUT_MAP(CKw_Struct, "struct"); BSTR_LUT_MAP(CKw_Class, "class"); BSTR_LUT_MAP(CKw_Enum, "enum"); @@ -277,7 +279,27 @@ std::string GenerateEnumStringMap(CodegenOutput& out, const DeclEnum& decl, bool } void GenerateForEnum(CodegenOutput& headerOut, CodegenOutput& sourceOut, const DeclEnum& decl, EnumFlags options) { - bool useExcludeHeuristics = options.IsSet(EMGO_ExcludeUseHeuristics); + char enumName[2048]; + if (decl.container) { + snprintf(enumName, sizeof(enumName), "%.*s::%s", PRINTF_STRING_VIEW(decl.container->fullname), decl.name.c_str()); + } else { + strncpy(enumName, decl.name.c_str(), sizeof(enumName)); + } + + auto useExcludeHeuristics = options.IsSet(EMGO_ExcludeUseHeuristics); + auto filteredElements = [&]() { + if (useExcludeHeuristics) { + decltype(decl.elements) result; + for (auto& elm : decl.elements) { + if (elm.name.ends_with("COUNT")) continue; + + result.push_back(elm); + } + return result; + } else { + return decl.elements; + } + }(); if (options.IsSet(EMGO_ToString)) { // Generate value -> string lookup table and function @@ -285,23 +307,18 @@ void GenerateForEnum(CodegenOutput& headerOut, CodegenOutput& sourceOut, const D switch (decl.GetPattern()) { case EVP_Continuous: { auto arrayName = GenerateEnumStringArray(sourceOut, decl, useExcludeHeuristics); - int minVal = decl.elements.front().value; - int maxVal = decl.elements.back().value; - if (useExcludeHeuristics && - decl.elements.back().name.ends_with("COUNT") && - decl.elements.size() >= 2) - { - // Skip the last *_COUNT element if instructed to use heuristics - maxVal = decl.elements[decl.elements.size() - 2].value; - } + int minVal = filteredElements.empty() ? 0 : filteredElements.front().value; + int maxVal = filteredElements.empty() ? 0 : filteredElements.back().value; CodegenOutputThing lookupFunctionDef; - auto& o2 = lookupFunctionDef.text; - APPEND_LIT_LN(o2, "template <>"); - APPEND_FMT_LN(o2, "std::string_view Metadata::EnumToString<%s>(%s value) {", decl.name.c_str(), decl.name.c_str()); - APPEND_FMT_LN(o2, " if (value < 0 || value >= %d) return {};", maxVal); - APPEND_FMT_LN(o2, " return %s[value - %d];", arrayName.c_str(), minVal); - APPEND_LIT_LN(o2, "}"); + { + auto& o = lookupFunctionDef.text; + APPEND_LIT_LN(o, "template <>"); + APPEND_FMT_LN(o, "std::string_view Metadata::EnumToString<%s>(%s value) {", enumName, enumName); + APPEND_FMT_LN(o, " if (value < %d || value > %d) return {};", minVal, maxVal); + APPEND_FMT_LN(o, " return %s[value - %d];", arrayName.c_str(), minVal); + APPEND_LIT_LN(o, "}"); + } sourceOut.AddOutputThing(std::move(lookupFunctionDef)); } break; @@ -323,28 +340,34 @@ void GenerateForEnum(CodegenOutput& headerOut, CodegenOutput& sourceOut, const D if (options.IsSet(EMGO_FromString)) { // Generate string -> value lookup table char mapName[1024]; + // TODO mangle to prevent name conflicts of enum in different namespaces snprintf(mapName, sizeof(mapName), "gCG_%s_Str2Val", decl.name.c_str()); CodegenOutputThing lookupTable; - auto& o1 = lookupTable.text; - APPEND_FMT_LN(o1, "constinit frozen::unordered_map %s = {", decl.name.c_str(), decl.elements.size(), mapName); - for (auto& elm : decl.elements) { - APPEND_FMT_LN(o1, "{\"%s\", %s::%s},", elm.name.c_str(), decl.name.c_str(), elm.name.c_str()); + { + auto& o = lookupTable.text; + // TODO use correct underlying type + APPEND_FMT_LN(o, "constinit frozen::unordered_map %s = {", filteredElements.size(), mapName); + for (auto& elm : filteredElements) { + APPEND_FMT_LN(o, "{\"%s\", %" PRId64 "},", elm.name.c_str(), elm.value); + } + APPEND_LIT_LN(o, "};"); } - APPEND_LIT_LN(o1, "};"); // Generate lookup function CodegenOutputThing lookupFunctionDef; - auto& o3 = lookupFunctionDef.text; - APPEND_LIT_LN(o3, "template <>"); - APPEND_FMT_LN(o3, "std::optional<%s> Metadata::EnumFromString<%s>(std::string_view value) {", decl.name.c_str(), decl.name.c_str()); - APPEND_FMT_LN(o3, " auto iter = %s.find(value);", mapName); - APPEND_FMT_LN(o3, " if (iter != %s.end()) {", mapName); - APPEND_LIT_LN(o3, " return iter->second;"); - APPEND_LIT_LN(o3, " } else {"); - APPEND_LIT_LN(o3, " return {};"); - APPEND_LIT_LN(o3, " }"); - APPEND_LIT_LN(o3, "}"); + { + auto& o = lookupFunctionDef.text; + APPEND_LIT_LN(o, "template <>"); + APPEND_FMT_LN(o, "std::optional<%s> Metadata::EnumFromString<%s>(std::string_view value) {", enumName, enumName); + APPEND_FMT_LN(o, " auto iter = %s.find(value);", mapName); + APPEND_FMT_LN(o, " if (iter != %s.end()) {", mapName); + APPEND_FMT_LN(o, " return (%s)iter->second;", enumName); + APPEND_LIT_LN(o, " } else {"); + APPEND_LIT_LN(o, " return {};"); + APPEND_LIT_LN(o, " }"); + APPEND_LIT_LN(o, "}"); + } sourceOut.AddOutputThing(std::move(lookupTable)); sourceOut.AddOutputThing(std::move(lookupFunctionDef)); @@ -369,7 +392,32 @@ void HandleInputFile(AppState& state, std::string_view filenameStem, std::string CodegenOutput cgSourceOutput; Utils::ProduceGeneratedSourceFileHeader(cgSourceOutput); - int bracePairDepth = 0; + int currentBraceDepth = 0; + // The current effective namespace, see example + DeclNamespace* currentNamespace = nullptr; + + struct NamespaceStackframe { + // The current namespace that owns the brace level, see example + DeclNamespace* ns = nullptr; + // Brace depth `ns` was created at (e.g. [std::details].depth == 0) + int depth = 0; + }; + std::vector nsStack; + + // Example: + // namespace std::details { + // /* [stack top].ns = std::details */ + // /* [stack top].depth = std */ + // } + // namespace foo { + // /* [stack top].ns = foo */ + // /* [stack top].depth = foo */ + // namespace details { + // /* [stack top].ns = foo::details */ + // /* [stack top].depth = foo::details */ + // } + // } + while (idx < tokens.size()) { auto& token = tokens[idx]; @@ -388,6 +436,40 @@ void HandleInputFile(AppState& state, std::string_view filenameStem, std::string } } switch (keyword) { + case CKw_Namespace: { + ++idx; + incrementTokenIdx = false; + + while (true) { + if (tokens[idx].type != CLEX_id) { + // TODO better error recovery + printf("[ERROR] invalid syntax for namespace\n"); + break; + } + + currentNamespace = cgInput.AddNamespace(DeclNamespace{ + .container = currentNamespace, + .name = tokens[idx].text, + }); + + if (tokens[idx + 1].text[0] == ':' && + tokens[idx + 2].text[0] == ':') + { + // Skip the two ':' tokens, try parse the next identifier + idx += 3; + } else { + break; + } + } + + nsStack.push_back(NamespaceStackframe{ + .ns = currentNamespace, + .depth = currentBraceDepth, + }); + + goto endIdenCase; + } + case CKw_Struct: case CKw_Class: { auto& idenTok = tokens[idx + 1]; // TODO handle end of list @@ -401,6 +483,7 @@ void HandleInputFile(AppState& state, std::string_view filenameStem, std::string incrementTokenIdx = false; DeclEnum enumDecl; + enumDecl.container = currentNamespace; enumDecl.underlyingType = EUT_Int32; // TODO if (tokens[idx].text == "class") { @@ -452,7 +535,8 @@ void HandleInputFile(AppState& state, std::string_view filenameStem, std::string ++idx; } - cgInput.AddEnum(std::move(enumDecl)); + auto fullname = Utils::MakeFullName(enumDecl.name, currentNamespace); + cgInput.AddEnum(std::move(fullname), std::move(enumDecl)); goto endIdenCase; } @@ -488,7 +572,7 @@ void HandleInputFile(AppState& state, std::string_view filenameStem, std::string } auto& enumName = argList[0][0]->text; - auto enumDecl = cgInput.FindEnumByName(enumName); + auto enumDecl = cgInput.FindEnumByName(Utils::MakeFullName(enumName, currentNamespace)); if (!enumDecl) { printf("[ERROR] BRUSSEL_ENUM: referring to non-existent enum '%s'\n", enumName.c_str()); break; @@ -519,15 +603,32 @@ void HandleInputFile(AppState& state, std::string_view filenameStem, std::string break; } - case '{': { - bracePairDepth++; - CheckBraceDepth(bracePairDepth); - } break; - - case '}': { - bracePairDepth--; - CheckBraceDepth(bracePairDepth); - } break; + case CLEX_ext_single_char: + switch (token.text[0]) { + case '{': { + currentBraceDepth++; + CheckBraceDepth(currentBraceDepth); + } break; + + case '}': { + currentBraceDepth--; + CheckBraceDepth(currentBraceDepth); + + if (!nsStack.empty()) { + auto& ns = nsStack.back(); + if (ns.depth == currentBraceDepth) { + nsStack.pop_back(); + + if (!nsStack.empty()) { + currentNamespace = nsStack.back().ns; + } else { + currentNamespace = nullptr; + } + } + } + } break; + } + break; } if (incrementTokenIdx) { @@ -535,7 +636,7 @@ void HandleInputFile(AppState& state, std::string_view filenameStem, std::string } } - if (bracePairDepth != 0) { + if (currentBraceDepth != 0) { printf("[WARNING] unbalanced brace at end of file."); } @@ -632,7 +733,7 @@ where : the directory to write generated contents to. This will N } state.outputDir = std::string_view(argv[1]); - DEBUG_PRINTF("Outputting to directory %s.\n", outputDir); + DEBUG_PRINTF("Outputting to directory %.*s.\n", PRINTF_STRING_VIEW(state.outputDir)); for (int i = 2; i < argc; ++i) { const char* argRaw = argv[i]; diff --git a/buildtools/codegen/tests/examples/TestEnum.hpp.txt b/buildtools/codegen/tests/examples/TestEnum.hpp.txt index e596c5e..441d97c 100644 --- a/buildtools/codegen/tests/examples/TestEnum.hpp.txt +++ b/buildtools/codegen/tests/examples/TestEnum.hpp.txt @@ -19,3 +19,25 @@ enum CountedEnum { CE_COUNT, }; BRUSSEL_ENUM(CountedEnum, ToString FromString ExcludeHeuristics); + +namespace MyNamespace { +enum MyNamespacedEnum { + MNE_Foo, + MNE_Bar, +}; +BRUSSEL_ENUM(MyNamespacedEnum, ToString FromString ExcludeHeuristics); + +namespace details { + enum MyNamespacedEnum { + MNE_Foo, + MNE_Bar, + }; + BRUSSEL_ENUM(MyNamespacedEnum, ToString FromString ExcludeHeuristics); +} +} + +namespace foo::details { +enum Enum { +}; +BRUSSEL_ENUM(Enum, ToString FromString ExcludeHeuristics); +} diff --git a/source/GraphicsTags.hpp b/source/GraphicsTags.hpp index 09e62bf..f83b99c 100644 --- a/source/GraphicsTags.hpp +++ b/source/GraphicsTags.hpp @@ -7,14 +7,6 @@ #include -enum TestEnum { - TE_Position, - TE_Color, - TE_TexCoord, - TE_COUNT, -}; -BRUSSEL_ENUM(TestEnum, ToString FromString ExcludeHeuristics); - namespace Tags { /// Vertex element semantics, used to identify the meaning of vertex buffer contents enum VertexElementSemantic { @@ -42,6 +34,7 @@ enum VertexElementSemantic { VES_Generic, VES_COUNT, }; +BRUSSEL_ENUM(VertexElementSemantic, ToString FromString ExcludeHeuristics); std::string_view NameOf(VertexElementSemantic semantic); VertexElementSemantic FindVertexElementSemantic(std::string_view name); @@ -82,6 +75,8 @@ enum VertexElementType { VET_Ushort4Norm, VET_NORM_END = VET_Ushort4Norm, }; +// TODO this enum isn't continuous, not supported yet +// BRUSSEL_ENUM(VertexElementType, ToString FromString ExcludeHeuristics); int SizeOf(VertexElementType type); int VectorLenOf(VertexElementType type); diff --git a/source/main.cpp b/source/main.cpp index 353746b..c49fc0b 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -3,11 +3,9 @@ #include "AppConfig.hpp" #include "CommonVertexIndex.hpp" #include "EditorGuizmo.hpp" -#include "GraphicsTags.hpp" #include "Ires.hpp" #include "Level.hpp" #include "Material.hpp" -#include "Metadata.hpp" #include "Shader.hpp" #define GLFW_INCLUDE_NONE @@ -104,9 +102,6 @@ fs::path GetEnvVar(const char* name, const char* backup) { int main(int argc, char* argv[]) { using namespace Tags; - auto str = Metadata::EnumToString(TE_Color); - printf("%.*s", PRINTF_STRING_VIEW(str)); - constexpr auto kImGuiBackend = "imgui-backend"; constexpr auto kGameDataDir = "game-data-directory"; constexpr auto kGameAssetDir = "game-asset-directory"; -- cgit v1.2.3-70-g09d2