From f138311d2d2e0cc9ba0496d523bb46f2c1c9fb73 Mon Sep 17 00:00:00 2001
From: rtk0c <mail.tianyig@gmail.com>
Date: Wed, 20 Sep 2023 23:58:58 -0700
Subject: Copy from the PlasticSCM repo, replace vendored glm wtih conan

---
 source/20-codegen-compiler/CodegenLexer.hpp | 49 +++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 source/20-codegen-compiler/CodegenLexer.hpp

(limited to 'source/20-codegen-compiler/CodegenLexer.hpp')

diff --git a/source/20-codegen-compiler/CodegenLexer.hpp b/source/20-codegen-compiler/CodegenLexer.hpp
new file mode 100644
index 0000000..ec8c8b7
--- /dev/null
+++ b/source/20-codegen-compiler/CodegenLexer.hpp
@@ -0,0 +1,49 @@
+#pragma once
+
+#include <LookupTable.hpp>
+
+#include <stb_c_lexer.h>
+#include <span>
+#include <string>
+#include <string_view>
+#include <vector>
+
+enum {
+	CLEX_ext_single_char = CLEX_first_unused_token,
+	CLEX_ext_double_colon,
+	CLEX_ext_dot_dot_dot,
+	CLEX_ext_COUNT,
+};
+
+struct StbLexerToken {
+	std::string text;
+
+	union {
+		double lexerRealNumber;
+		long lexerIntNumber;
+	};
+
+	// Can either be CLEX_* or CLEX_ext_* values
+	int type;
+
+	int Reamalgamate() const;
+};
+
+bool StbTokenIsSingleChar(int lexerToken);
+bool StbTokenIsMultiChar(int lexerToken);
+std::string CombineTokens(std::span<const StbLexerToken> tokens, std::string_view separator = {});
+
+struct CodegenLexer {
+	std::vector<StbLexerToken> tokens;
+	size_t idx = 0;
+
+	void InitializeFrom(std::string_view source);
+
+	const StbLexerToken& Current() const;
+
+	const StbLexerToken* TryConsumeToken(int type);
+	const StbLexerToken* TryConsumeSingleCharToken(char c);
+
+    void SkipUntilToken(int type);
+    void SkipUntilTokenSingleChar(char c);
+};
-- 
cgit v1.2.3-70-g09d2