Changeset: 79 Initial work on fixing codegen for the main project, add enum underlying type (EUT) scanning capability

1 files changed, 20 insertions, 1 deletions

diff --git a/source/20-codegen-compiler/CodegenLexer.cpp b/source/20-codegen-compiler/CodegenLexer.cpp
index dab6aea..ecb2186 100644
--- a/source/20-codegen-compiler/CodegenLexer.cpp
+++ b/source/20-codegen-compiler/CodegenLexer.cpp
@@ -2,6 +2,14 @@
 
 #include <cassert>
 
+int StbLexerToken::Reamalgamate() const {
+	if (type == CLEX_ext_single_char) {
+		return text[0];
+	} else {
+		return type;
+	}
+}
+
 bool StbTokenIsSingleChar(int lexerToken) {
 	return lexerToken >= 0 && lexerToken < 256;
 }
@@ -10,16 +18,27 @@ bool StbTokenIsMultiChar(int lexerToken) {
 	return !StbTokenIsMultiChar(lexerToken);
 }
 
-std::string CombineTokens(std::span<const StbLexerToken> tokens) {
+std::string CombineTokens(std::span<const StbLexerToken> tokens, std::string_view separator) {
+	if (tokens.empty()) {
+		return {};
+	}
+
 	size_t length = 0;
 	for (auto& token : tokens) {
 		length += token.text.size();
+		length += separator.size();
 	}
+	// Intentionally counting an extra separator: leave space for the last append below
+
 	std::string result;
 	result.reserve(length);
 	for (auto& token : tokens) {
 		result += token.text;
+		result += separator;
 	}
+	// Remove the trailing separator
+	result.resize(result.size() - separator.size());
+
 	return result;
 }