From a98b0495a957ed15900c052bcad00b3c48367546 Mon Sep 17 00:00:00 2001 From: rtk0c Date: Thu, 7 Jul 2022 19:30:47 -0700 Subject: Changeset: 79 Initial work on fixing codegen for the main project, add enum underlying type (EUT) scanning capability --- source/20-codegen-compiler/CodegenLexer.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'source/20-codegen-compiler/CodegenLexer.cpp') diff --git a/source/20-codegen-compiler/CodegenLexer.cpp b/source/20-codegen-compiler/CodegenLexer.cpp index dab6aea..ecb2186 100644 --- a/source/20-codegen-compiler/CodegenLexer.cpp +++ b/source/20-codegen-compiler/CodegenLexer.cpp @@ -2,6 +2,14 @@ #include +int StbLexerToken::Reamalgamate() const { + if (type == CLEX_ext_single_char) { + return text[0]; + } else { + return type; + } +} + bool StbTokenIsSingleChar(int lexerToken) { return lexerToken >= 0 && lexerToken < 256; } @@ -10,16 +18,27 @@ bool StbTokenIsMultiChar(int lexerToken) { return !StbTokenIsMultiChar(lexerToken); } -std::string CombineTokens(std::span tokens) { +std::string CombineTokens(std::span tokens, std::string_view separator) { + if (tokens.empty()) { + return {}; + } + size_t length = 0; for (auto& token : tokens) { length += token.text.size(); + length += separator.size(); } + // Intentionally counting an extra separator: leave space for the last append below + std::string result; result.reserve(length); for (auto& token : tokens) { result += token.text; + result += separator; } + // Remove the trailing separator + result.resize(result.size() - separator.size()); + return result; } -- cgit v1.2.3-70-g09d2