Changeset: 70 Fix cmake and codegen infra

- Invoke codegen.exe once with a list of changed files, instead of individually for each changed file (this gives the codegen global access to all the code, allowing more things)
- Initial support for outputting an archive SQLite database that contains all the code info

8 files changed, 464 insertions, 124 deletions

diff --git a/source/10-common/Utils.cpp b/source/10-common/Utils.cpp
index dc76b0a..f0ff76d 100644
--- a/source/10-common/Utils.cpp
+++ b/source/10-common/Utils.cpp
@@ -77,6 +77,29 @@ std::string Utils::ReadFileAsString(const fs::path& path) {
 	return result;
 }
 
+bool Utils::ReadCstdioLine(FILE* file, std::string& buffer) {
+	buffer.clear();
+	while (true) {
+		int c = fgetc(file);
+		if (c == EOF) {
+			if (buffer.empty() || buffer.back() != '\n') {
+				buffer += '\n';
+			}
+			return false;
+		} else if (c == '\n') {
+			buffer += '\n';
+			return true;
+		} else {
+			buffer += c;
+		}
+	}
+}
+
+bool Utils::ReadCstdioLine(FILE* file, char* buffer, size_t bufferSize, size_t* outLineLength) {
+	// TODO
+	assert(false && "Unimplemented");
+}
+
 bool Utils::InRangeInclusive(int n, int lower, int upper) {
 	if (lower > upper) {
 		std::swap(lower, upper);
diff --git a/source/10-common/Utils.hpp b/source/10-common/Utils.hpp
index 9560b57..fdf0f5d 100644
--- a/source/10-common/Utils.hpp
+++ b/source/10-common/Utils.hpp
@@ -19,8 +19,21 @@ enum IoMode {
 FILE* OpenCstdioFile(const std::filesystem::path& path, IoMode mode, bool binary = false);
 FILE* OpenCstdioFile(const char* path, IoMode mode, bool binary = false);
 
+/// Retrieve a whole line (marked by `\n` or EOF) into the buffer. If the line ends with EOF, two things happen:
+/// 1. a `\n` character is appended to the line content, emulating as-if the line ended with `\n`.
+/// 2. `false` is returned
+/// Otherwise, `true` is returned.
+///
+/// Empty lines are not skipped at all, including the very last empty line if it exists.
+bool ReadCstdioLine(FILE* file, std::string& buffer);
+/// Same as the other overload, except working with a fixed-size buffer.
+/// NOTE: this also gives the length of the line compared to `std::fgets`.
+///       `std::fgets` requires us to run `std::strlen` on the output again to find the length
+bool ReadCstdioLine(FILE* file, char* buffer, size_t bufferSize, size_t* outLineLength = nullptr);
+
 std::string ReadFileAsString(const std::filesystem::path& path);
 
+
 constexpr float Abs(float v) noexcept {
 	return v < 0.0f ? -v : v;
 }
diff --git a/source/20-codegen-compiler/CodegenDecl.hpp b/source/20-codegen-compiler/CodegenDecl.hpp
index 60d5a13..739dbe4 100644
--- a/source/20-codegen-compiler/CodegenDecl.hpp
+++ b/source/20-codegen-compiler/CodegenDecl.hpp
@@ -77,6 +77,7 @@ struct DeclEnum {
 	std::string_view fullname;
 	std::vector<DeclEnumElement> elements;
 	EnumUnderlyingType underlyingType;
+	std::string underlyingTypeStr;
 	// Start with invalid value, calculate on demand
 	mutable EnumValuePattern pattern = EVP_COUNT;
 
diff --git a/source/20-codegen-compiler/CodegenInput.cpp b/source/20-codegen-compiler/CodegenInput.cpp
deleted file mode 100644
index 0dced0e..0000000
--- a/source/20-codegen-compiler/CodegenInput.cpp
+++ /dev/null
@@ -1,99 +0,0 @@
-#include "CodegenInput.hpp"
-
-#include <Macros.hpp>
-#include <Utils.hpp>
-
-#include <robin_hood.h>
-#include <variant>
-
-struct SomeDecl {
-	std::variant<DeclStruct, DeclFunction, DeclEnum> v;
-};
-
-class CodegenInput::Private {
-public:
-	// We want address stability for everything
-	robin_hood::unordered_node_map<std::string, SomeDecl, StringHash, StringEqual> decls;
-	robin_hood::unordered_node_map<std::string, DeclNamespace, StringHash, StringEqual> namespaces;
-};
-
-CodegenInput::CodegenInput()
-	: m{ new Private() } //
-{
-}
-
-CodegenInput::~CodegenInput() {
-	delete m;
-}
-
-#define STORE_DECL_OF_TYPE(DeclType, fullname, decl)                                                    \
-	auto [iter, success] = m->decls.try_emplace(std::move(fullname), SomeDecl{ .v = std::move(decl) }); \
-	auto& key = iter->first;                                                                            \
-	auto& val = iter->second;                                                                           \
-	auto& declRef = std::get<DeclType>(val.v);                                                          \
-	declRef.fullname = key;                                                                             \
-	return &declRef
-
-DeclEnum* CodegenInput::AddEnum(std::string fullname, DeclEnum decl) {
-#if CODEGEN_DEBUG_PRINT
-	printf("Committed enum '%s'\n", decl.name.c_str());
-	for (auto& elm : decl.elements) {
-		printf("  - element %s = %" PRId64 "\n", elm.name.c_str(), elm.value);
-	}
-#endif
-
-	STORE_DECL_OF_TYPE(DeclEnum, fullname, decl);
-}
-
-DeclStruct* CodegenInput::AddStruct(std::string fullname, DeclStruct decl) {
-#if CODEGEN_DEBUG_PRINT
-	printf("Committed struct '%s'\n", decl.name.c_str());
-	printf("  Base classes:\n");
-	for (auto& base : decl.baseClasses) {
-		printf("  - %.*s\n", PRINTF_STRING_VIEW(base->name));
-	}
-#endif
-
-	STORE_DECL_OF_TYPE(DeclStruct, fullname, decl);
-}
-
-#define FIND_DECL_OF_TYPE(DeclType)                     \
-	auto iter = m->decls.find(name);                    \
-	if (iter != m->decls.end()) {                       \
-		auto& some = iter->second.v;                    \
-		if (auto decl = std::get_if<DeclType>(&some)) { \
-			return decl;                                \
-		}                                               \
-	}                                                   \
-	return nullptr
-
-const DeclEnum* CodegenInput::FindEnum(std::string_view name) const {
-	FIND_DECL_OF_TYPE(DeclEnum);
-}
-
-const DeclStruct* CodegenInput::FindStruct(std::string_view name) const {
-	FIND_DECL_OF_TYPE(DeclStruct);
-}
-
-DeclNamespace* CodegenInput::AddNamespace(DeclNamespace ns) {
-	auto path = Utils::MakeFullName(""sv, &ns);
-	auto [iter, success] = m->namespaces.try_emplace(std::move(path), std::move(ns));
-	auto& nsRef = iter->second;
-	if (success) {
-		nsRef.fullname = iter->first;
-	}
-	return &nsRef;
-}
-
-const DeclNamespace* CodegenInput::FindNamespace(std::string_view fullname) const {
-	auto iter = m->namespaces.find(fullname);
-	if (iter != m->namespaces.end()) {
-		return &iter->second;
-	} else {
-		return nullptr;
-	}
-}
-
-DeclNamespace* CodegenInput::FindNamespace(std::string_view name) {
-	return const_cast<DeclNamespace*>(const_cast<const CodegenInput*>(this)->FindNamespace(name));
-}
diff --git a/source/20-codegen-compiler/CodegenModel.cpp b/source/20-codegen-compiler/CodegenModel.cpp
new file mode 100644
index 0000000..d075c1d
--- /dev/null
+++ b/source/20-codegen-compiler/CodegenModel.cpp
@@ -0,0 +1,293 @@
+#include "CodegenModel.hpp"
+#include "CodegenModelArchive.hpp"
+
+#include "CodegenUtils.hpp"
+
+#include <Macros.hpp>
+#include <ScopeGuard.hpp>
+#include <Utils.hpp>
+
+#include <robin_hood.h>
+#include <cassert>
+#include <cstdint>
+#include <stdexcept>
+#include <string>
+#include <variant>
+
+using namespace std::literals;
+
+struct SomeDecl {
+	std::variant<DeclStruct, DeclFunction, DeclEnum> v;
+};
+
+class CodegenModel::Private {
+	friend class CodegenModelArchive;
+
+public:
+	// We want address stability for everything
+	robin_hood::unordered_node_map<std::string, SomeDecl, StringHash, StringEqual> decls;
+	robin_hood::unordered_node_map<std::string, DeclNamespace, StringHash, StringEqual> namespaces;
+};
+
+class CodegenModelArchive::Private {
+	friend class CodegenModel;
+
+public:
+	sqlite3* database = nullptr;
+	sqlite3_stmt* beginTransactionStmt = nullptr;
+	sqlite3_stmt* commitTransactionStmt = nullptr;
+	sqlite3_stmt* rollbackTransactionStmt = nullptr;
+	sqlite3_stmt* storeEnumStmt = nullptr;
+	sqlite3_stmt* storeEnumElmStmt = nullptr;
+
+	// See below for definition
+	void InitializeDatabase();
+
+	void BeginTransaction() {
+		int result = sqlite3_step(beginTransactionStmt);
+		assert(result == SQLITE_DONE);
+		sqlite3_reset(beginTransactionStmt);
+	}
+
+	void CommitTransaction() {
+		int result = sqlite3_step(commitTransactionStmt);
+		assert(result == SQLITE_DONE);
+		sqlite3_reset(commitTransactionStmt);
+	}
+
+	void RollbackTransaction() {
+		int result = sqlite3_step(rollbackTransactionStmt);
+		assert(result == SQLITE_DONE);
+		sqlite3_reset(rollbackTransactionStmt);
+	}
+};
+
+CodegenModel::CodegenModel()
+	: m{ new Private() } //
+{
+}
+
+CodegenModel::~CodegenModel() {
+	delete m;
+}
+
+#define STORE_DECL_OF_TYPE(DeclType, fullname, decl)                                                    \
+	auto [iter, success] = m->decls.try_emplace(std::move(fullname), SomeDecl{ .v = std::move(decl) }); \
+	auto& key = iter->first;                                                                            \
+	auto& val = iter->second;                                                                           \
+	auto& declRef = std::get<DeclType>(val.v);                                                          \
+	declRef.fullname = key;                                                                             \
+	return &declRef
+
+DeclEnum* CodegenModel::AddEnum(std::string fullname, DeclEnum decl) {
+#if CODEGEN_DEBUG_PRINT
+	printf("Committed enum '%s'\n", decl.name.c_str());
+	for (auto& elm : decl.elements) {
+		printf("  - element %s = %" PRId64 "\n", elm.name.c_str(), elm.value);
+	}
+#endif
+
+	STORE_DECL_OF_TYPE(DeclEnum, fullname, decl);
+}
+
+DeclStruct* CodegenModel::AddStruct(std::string fullname, DeclStruct decl) {
+#if CODEGEN_DEBUG_PRINT
+	printf("Committed struct '%s'\n", decl.name.c_str());
+	printf("  Base classes:\n");
+	for (auto& base : decl.baseClasses) {
+		printf("  - %.*s\n", PRINTF_STRING_VIEW(base->name));
+	}
+#endif
+
+	STORE_DECL_OF_TYPE(DeclStruct, fullname, decl);
+}
+
+#define FIND_DECL_OF_TYPE(DeclType)                     \
+	auto iter = m->decls.find(name);                    \
+	if (iter != m->decls.end()) {                       \
+		auto& some = iter->second.v;                    \
+		if (auto decl = std::get_if<DeclType>(&some)) { \
+			return decl;                                \
+		}                                               \
+	}                                                   \
+	return nullptr
+
+const DeclEnum* CodegenModel::FindEnum(std::string_view name) const {
+	FIND_DECL_OF_TYPE(DeclEnum);
+}
+
+const DeclStruct* CodegenModel::FindStruct(std::string_view name) const {
+	FIND_DECL_OF_TYPE(DeclStruct);
+}
+
+DeclNamespace* CodegenModel::AddNamespace(DeclNamespace ns) {
+	auto path = Utils::MakeFullName(""sv, &ns);
+	auto [iter, success] = m->namespaces.try_emplace(std::move(path), std::move(ns));
+	auto& nsRef = iter->second;
+	if (success) {
+		nsRef.fullname = iter->first;
+	}
+	return &nsRef;
+}
+
+const DeclNamespace* CodegenModel::FindNamespace(std::string_view fullname) const {
+	auto iter = m->namespaces.find(fullname);
+	if (iter != m->namespaces.end()) {
+		return &iter->second;
+	} else {
+		return nullptr;
+	}
+}
+
+DeclNamespace* CodegenModel::FindNamespace(std::string_view name) {
+	return const_cast<DeclNamespace*>(const_cast<const CodegenModel*>(this)->FindNamespace(name));
+}
+
+sqlite3_stmt* Utils::NewSqliteStatement(sqlite3* database, std::string_view sql) {
+	sqlite3_stmt* stmt;
+	int result = sqlite3_prepare(database, sql.data(), sql.size(), &stmt, nullptr);
+	if (result != SQLITE_OK) {
+		INPLACE_FMT(msg, "Failed to prepare statement, error message: %s", sqlite3_errmsg(database));
+		throw std::runtime_error(msg);
+	}
+
+	return stmt;
+}
+
+void CodegenModelArchive::Private::InitializeDatabase() {
+	int result = sqlite3_exec(database, "PRAGMA user_version = 1", nullptr, nullptr, nullptr);
+	assert(result == SQLITE_OK);
+
+	result = sqlite3_exec(database, R"""(
+BEGIN TRANSACTION;
+CREATE TABLE DeclEnums(
+    Id INT PRIMARY KEY,
+    Name TEXT,
+    UnderlyingType TEXT
+);
+CREATE TABLE DeclEnumElements(
+    EnumId INT,
+    Name TEXT,
+    Value INT,
+    FOREIGN KEY (EnumId) REFERENCES DeclEnums(Id)
+);
+COMMIT TRANSACTION;
+)""",
+		nullptr,
+		nullptr,
+		nullptr);
+	assert(result == SQLITE_OK);
+}
+
+CodegenModelArchive::CodegenModelArchive(std::string_view dbPath)
+	: m{ new Private() } //
+{
+	std::string zstrPath(dbPath);
+	int reuslt = sqlite3_open(zstrPath.c_str(), &m->database);
+	if (reuslt != SQLITE_OK) {
+		std::string msg;
+		msg += "Failed to open SQLite3 database, error message:\n";
+		msg += sqlite3_errmsg(m->database);
+		throw std::runtime_error(msg);
+	}
+
+	// Keep the same as `PRAGMA user_version` in the CodegenMetadataArchive::Private::InitializeDatabase()
+	constexpr int kDatabaseVersion = 1;
+	int currentDatabaseVersion;
+	{
+		auto readVersionStmt = Utils::NewSqliteStatement(m->database, "PRAGMA user_version");
+		DEFER { sqlite3_finalize(readVersionStmt); };
+
+		int result = sqlite3_step(readVersionStmt);
+		assert(result == SQLITE_ROW);
+		currentDatabaseVersion = sqlite3_column_int(readVersionStmt, 0);
+
+		result = sqlite3_step(readVersionStmt);
+		assert(result == SQLITE_DONE);
+
+		if (currentDatabaseVersion == 0) {
+			// Newly created database, initialize it
+			m->InitializeDatabase();
+		} else if (currentDatabaseVersion == kDatabaseVersion) {
+			// Same version, no need to do anything
+		} else {
+			INPLACE_FMT(msg, "Incompatbile database versions %d (in file) vs %d (expected).", currentDatabaseVersion, kDatabaseVersion);
+			throw std::runtime_error(msg);
+		}
+	}
+
+	m->beginTransactionStmt = Utils::NewSqliteStatement(m->database, "BEGIN TRANSACTION");
+	m->commitTransactionStmt = Utils::NewSqliteStatement(m->database, "COMMIT TRANSACTION");
+	m->rollbackTransactionStmt = Utils::NewSqliteStatement(m->database, "ROLLBACK TRANSACTION");
+
+	m->storeEnumStmt = Utils::NewSqliteStatement(m->database, "INSERT INTO DeclEnums(Name, UnderlyingType) VALUES(?, ?)"sv);
+	m->storeEnumElmStmt = Utils::NewSqliteStatement(m->database, "INSERT INTO DeclEnumElements(EnumId, Name, Value) VALUES(?, ?, ?)"sv);
+}
+
+CodegenModelArchive::~CodegenModelArchive() {
+	sqlite3_close(m->database);
+	sqlite3_finalize(m->beginTransactionStmt);
+	sqlite3_finalize(m->commitTransactionStmt);
+	sqlite3_finalize(m->rollbackTransactionStmt);
+	sqlite3_finalize(m->storeEnumStmt);
+	sqlite3_finalize(m->storeEnumElmStmt);
+
+	delete m;
+}
+
+void CodegenModelArchive::Store(const CodegenModel& cgInput) {
+	auto& cgm = cgInput.GetPimpl();
+
+	struct Visiter {
+		CodegenModelArchive* self;
+
+		void operator()(const DeclStruct& decl) const {
+			self->StoreStruct(decl);
+		}
+		void operator()(const DeclFunction& decl) const {
+			self->StoreFunction(decl);
+		}
+		void operator()(const DeclEnum& decl) const {
+			self->StoreEnum(decl);
+		}
+	} visiter;
+	visiter.self = this;
+
+	m->BeginTransaction();
+
+	for (auto&& [DISCARD, value] : cgm.decls) {
+		std::visit(visiter, value.v);
+	}
+
+	m->CommitTransaction();
+}
+
+void CodegenModelArchive::StoreStruct(const DeclStruct& decl) {
+	// TODO
+}
+
+void CodegenModelArchive::StoreFunction(const DeclFunction& decl) {
+	// TODO
+}
+
+void CodegenModelArchive::StoreEnum(const DeclEnum& decl) {
+	// TOOD only update
+	sqlite3_bind_text(m->storeEnumStmt, 1, decl.name.c_str(), decl.name.size(), nullptr);
+	sqlite3_bind_text(m->storeEnumStmt, 2, decl.underlyingTypeStr.c_str(), decl.underlyingTypeStr.size(), nullptr);
+	int result = sqlite3_step(m->storeEnumStmt);
+	assert(result == SQLITE_DONE);
+	sqlite3_reset(m->storeEnumStmt);
+	sqlite3_clear_bindings(m->storeEnumStmt);
+
+	auto enumRowId = sqlite3_last_insert_rowid(m->database);
+
+	for (auto& elm : decl.elements) {
+		sqlite3_bind_int64(m->storeEnumElmStmt, 1, enumRowId);
+		sqlite3_bind_text(m->storeEnumElmStmt, 2, elm.name.c_str(), elm.name.size(), nullptr);
+		sqlite3_bind_int64(m->storeEnumElmStmt, 3, elm.value);
+		int result = sqlite3_step(m->storeEnumElmStmt);
+		assert(result == SQLITE_DONE);
+		sqlite3_reset(m->storeEnumElmStmt);
+		sqlite3_clear_bindings(m->storeEnumElmStmt);
+	}
+}
diff --git a/source/20-codegen-compiler/CodegenInput.hpp b/source/20-codegen-compiler/CodegenModel.hpp
index 63c2673..31ca04f 100644
--- a/source/20-codegen-compiler/CodegenInput.hpp
+++ b/source/20-codegen-compiler/CodegenModel.hpp
@@ -10,14 +10,17 @@
 
 using namespace std::literals;
 
-class CodegenInput {
+class CodegenModel {
 private:
 	class Private;
 	Private* m;
 
 public:
-	CodegenInput();
-	~CodegenInput();
+	CodegenModel();
+	~CodegenModel();
+
+	// Implementation detail helper, don't use outside
+	Private& GetPimpl() const { return *m; }
 
 	DeclEnum* AddEnum(std::string fullname, DeclEnum decl);
 	DeclStruct* AddStruct(std::string fullname, DeclStruct decl);
diff --git a/source/20-codegen-compiler/CodegenModelArchive.hpp b/source/20-codegen-compiler/CodegenModelArchive.hpp
new file mode 100644
index 0000000..261d82b
--- /dev/null
+++ b/source/20-codegen-compiler/CodegenModelArchive.hpp
@@ -0,0 +1,29 @@
+#pragma once
+
+#include "CodegenDecl.hpp"
+#include "CodegenModel.hpp"
+
+#include <sqlite3.h>
+#include <string_view>
+
+namespace Utils {
+sqlite3_stmt* NewSqliteStatement(sqlite3* database, std::string_view sql);
+} // namespace Utils
+
+class CodegenModelArchive {
+private:
+	class Private;
+	Private* m;
+
+public:
+	CodegenModelArchive(std::string_view dbPath);
+	~CodegenModelArchive();
+
+	// Implementation detail helper, don't use outside
+	Private& GetPimpl() const { return *m; }
+
+	void Store(const CodegenModel& cgInput);
+	void StoreStruct(const DeclStruct& decl);
+	void StoreFunction(const DeclFunction& decl);
+	void StoreEnum(const DeclEnum& decl);
+};
diff --git a/source/20-codegen-compiler/main.cpp b/source/20-codegen-compiler/main.cpp
index c1559ec..1431ae5 100644
--- a/source/20-codegen-compiler/main.cpp
+++ b/source/20-codegen-compiler/main.cpp
@@ -1,7 +1,8 @@
 #include "CodegenConfig.hpp"
 #include "CodegenDecl.hpp"
-#include "CodegenInput.hpp"
 #include "CodegenLexer.hpp"
+#include "CodegenModel.hpp"
+#include "CodegenModelArchive.hpp"
 #include "CodegenOutput.hpp"
 #include "CodegenUtils.hpp"
 
@@ -28,7 +29,9 @@ namespace fs = std::filesystem;
 // TODO support codegen target in .cpp files
 
 struct AppState {
+	/*nullable*/ CodegenModelArchive* modelArchive = nullptr;
 	std::string_view outputDir;
+	std::string_view databaseFilePath;
 };
 
 FSTR_LUT_DECL(ClexNames, CLEX_eof, CLEX_ext_COUNT) {
@@ -511,7 +514,7 @@ struct NamespaceStackframe {
 };
 
 struct ParserState {
-	CodegenInput input;
+	CodegenModel model;
 	CodegenOutput headerOutput;
 	CodegenOutput sourceOutput;
 	CodegenOutput standaloneSourceOutput;
@@ -663,7 +666,7 @@ void HandleInputFile(AppState& as, std::string_view filenameStem, std::string_vi
 								break;
 							}
 
-							ps.currentNamespace = ps.input.AddNamespace(DeclNamespace{
+							ps.currentNamespace = ps.model.AddNamespace(DeclNamespace{
 								.container = ps.currentNamespace,
 								.name = tokens[idx].text,
 							});
@@ -723,7 +726,7 @@ void HandleInputFile(AppState& as, std::string_view filenameStem, std::string_vi
 
 								// TODO support namespace qualified names
 								auto baseClassFullname = Utils::MakeFullName(idenTok.text, ps.currentNamespace);
-								auto baseClassDecl = ps.input.FindStruct(baseClassFullname);
+								auto baseClassDecl = ps.model.FindStruct(baseClassFullname);
 								if (baseClassDecl) {
 									// We silently ignore a non-existent base class, because they may reside in a file that we didn't scan
 									structDecl.baseClasses.push_back(baseClassDecl);
@@ -750,7 +753,7 @@ void HandleInputFile(AppState& as, std::string_view filenameStem, std::string_vi
 
 						{
 							// Get a pointer to the decl inside CodegenInput's storage
-							auto decl = ps.input.AddStruct(std::move(fullname), std::move(structDecl));
+							auto decl = ps.model.AddStruct(std::move(fullname), std::move(structDecl));
 							ps.currentStruct = decl;
 							ps.currentStructBraceDepth = ps.currentBraceDepth;
 						}
@@ -780,6 +783,7 @@ void HandleInputFile(AppState& as, std::string_view filenameStem, std::string_vi
 						DeclEnum enumDecl;
 						enumDecl.container = ps.currentNamespace;
 						enumDecl.underlyingType = EUT_Int32; // TODO
+						enumDecl.underlyingTypeStr = "int";
 						enumDecl.name = name;
 
 						// Temporarily bind the pointers to local variable, HandleDirectiveEnum() and other functions expect these to the set
@@ -841,7 +845,7 @@ void HandleInputFile(AppState& as, std::string_view filenameStem, std::string_vi
 						}
 
 						{
-							auto decl = ps.input.AddEnum(std::move(fullname), std::move(enumDecl));
+							auto decl = ps.model.AddEnum(std::move(fullname), std::move(enumDecl));
 							ps.currentEnum = decl;
 							ps.currentEnumBraceDepth = ps.currentBraceDepth;
 						}
@@ -1064,15 +1068,20 @@ void HandleInputFile(AppState& as, std::string_view filenameStem, std::string_vi
 	Utils::WriteOutputFile(ps.sourceOutput, generatedSourceInlName);
 	INPLACE_FMT(generatedCppName, "%.*s/%.*s.g.cpp", PRINTF_STRING_VIEW(as.outputDir), PRINTF_STRING_VIEW(filenameStem));
 	Utils::WriteOutputFile(ps.standaloneSourceOutput, generatedCppName);
+
+	if (as.modelArchive) {
+		as.modelArchive->Store(ps.model);
+	}
 }
 
 enum InputOpcode {
 	IOP_ProcessSingleFile,
 	IOP_ProcessRecursively,
+	IOP_ProcessFileList,
 	IOP_COUNT,
 };
 
-void HandleArgument(AppState& state, InputOpcode opcode, std::string_view operand) {
+void HandleArgument(AppState& as, InputOpcode opcode, std::string_view operand) {
 	switch (opcode) {
 		case IOP_ProcessSingleFile: {
 			DEBUG_PRINTF("Processing single file %.*s\n", PRINTF_STRING_VIEW(operand));
@@ -1080,7 +1089,7 @@ void HandleArgument(AppState& state, InputOpcode opcode, std::string_view operan
 			fs::path path(operand);
 			auto filenameStem = path.stem().string();
 			auto source = Utils::ReadFileAsString(path);
-			HandleInputFile(state, filenameStem, source);
+			HandleInputFile(as, filenameStem, source);
 		} break;
 
 		case IOP_ProcessRecursively: {
@@ -1105,7 +1114,33 @@ void HandleArgument(AppState& state, InputOpcode opcode, std::string_view operan
 
 				auto filenameStem = pathStem.string();
 				auto source = Utils::ReadFileAsString(path);
-				HandleInputFile(state, filenameStem, source);
+				HandleInputFile(as, filenameStem, source);
+			}
+		} break;
+
+		case IOP_ProcessFileList: {
+			DEBUG_PRINTF("Processing file list %.*s\n", PRINTF_STRING_VIEW(operand));
+
+			fs::path fileListPath(operand);
+			auto fileList = Utils::OpenCstdioFile(fileListPath, Utils::Read);
+			if (!fileList) {
+				// NOTE: need this because our dirty-file-list generation algorithm in CMakeLists.txt doesn't produce a file when nothing is changed
+				DEBUG_PRINTF("File-list file does not exist, silently skipping.\n");
+				break;
+			}
+			DEFER { fclose(fileList); };
+
+			std::string line;
+			while (Utils::ReadCstdioLine(fileList, line)) {
+				// Remove '\n'
+				line.pop_back();
+
+				DEBUG_PRINTF("Processing file in list %.*s\n", line.c_str());
+
+				fs::path path(line);
+				auto filenameStem = path.stem().string();
+				auto source = Utils::ReadFileAsString(path);
+				HandleInputFile(as, filenameStem, source);
 			}
 		} break;
 
@@ -1118,9 +1153,11 @@ InputOpcode ParseInputOpcode(std::string_view text) {
 		return IOP_ProcessSingleFile;
 	} else if (text == "rec"sv) {
 		return IOP_ProcessRecursively;
+	} else if (text == "fileList"sv) {
+		return IOP_ProcessFileList;
 	} else {
-		DEBUG_PRINTF("Unknown input opcode %s\n", text.data());
-		throw std::runtime_error("Unknown input opcode");
+		INPLACE_FMT(msg, "Unknown input opcode %s\n", text.data());
+		throw std::runtime_error(msg);
 	}
 }
 
@@ -1139,15 +1176,16 @@ int main(int argc, char* argv[]) {
 	//      option 1: use cxxopts and positional arguments
 	//      option 2: take one argument only, being a json objecet
 
-	AppState state;
+	AppState as;
 
 	// If no cli is provided (argv[0] conventionally but not mandatorily the cli), this will do thing
 	// Otherwise, start with the 2nd element in the array, which is the 1st actual argument
-	if (argc < 2) {
+	if (argc <= 1) {
 		// NOTE: keep in sync with various enum options and parser code
 		printf(&R"""(
-USAGE: codegen.exe <output path> [<opcode>:<input path>]...
-where   <output path>: the directory to write generated contents to. This will NOT automatically create the directory.
+USAGE: codegen.exe --output-dir=<path> [--database=<path>] [<opcode>:<input path>]...
+where   --output-dir=<path>: the *directory* to write generated contents to. This will NOT automatically create the directory.
+        --database=<path>: the *file* to use for the code model database.
         <opcode> is one of:
             "single" process this <input path> file only
             "rec" starting at the given directory <input path>, recursively process all .h .hpp files
@@ -1155,13 +1193,52 @@ where   <output path>: the directory to write generated contents to. This will N
 		return -1;
 	}
 
-	state.outputDir = std::string_view(argv[1]);
-	DEBUG_PRINTF("Outputting to directory %.*s.\n", PRINTF_STRING_VIEW(state.outputDir));
+	// Named argument pass
+	robin_hood::unordered_map<std::string_view, std::string_view*> namedArguments{
+		{ "output-dir"sv, &as.outputDir },
+		{ "database"sv, &as.databaseFilePath },
+	};
+	for (int i = 1; i < argc; ++i) {
+		std::string_view arg(argv[i]);
+		if (!arg.starts_with("--")) {
+			// Convention: a "--" argument indicates everything afterwords are positional arguments
+			if (arg.size() == 2) {
+				break;
+			} else {
+				continue;
+			}
+		}
+
+		size_t equalLoc = arg.find('=');
+		auto oper = arg.substr(/*--*/ 2, equalLoc - 2);
+		auto iter = namedArguments.find(oper);
+		if (iter != namedArguments.end()) {
+			auto storage = iter->second;
+			if (storage) {
+				if (equalLoc == std::string_view::npos) {
+					*storage = ""sv;
+				} else {
+					*storage = arg.substr(equalLoc + 1);
+				}
+			}
+		}
+	}
+
+	DEBUG_PRINTF("Outputting to directory %.*s.\n", PRINTF_STRING_VIEW(as.outputDir));
+	DEBUG_PRINTF("Databse file: %.*s.\n", PRINTF_STRING_VIEW(as.databaseFilePath));
+
+	// TODO model archive is broken right now, see each TODO in CodegenModel.cpp
+	// CodegenModelArchive archive(as.databaseFilePath);
+	// as.modelArchive = &archive;
+
+	// Positional argument pass
+	for (int i = 1; i < argc; ++i) {
+		std::string_view arg(argv[i]);
+		if (arg.starts_with("--")) {
+			continue;
+		}
 
-	for (int i = 2; i < argc; ++i) {
-		const char* argRaw = argv[i];
-		std::string_view arg(argRaw);
-		DEBUG_PRINTF("Processing input command %s\n", argRaw);
+		DEBUG_PRINTF("Processing input command %s\n", argv[i]);
 
 		auto separatorLoc = arg.find(':');
 		if (separatorLoc != std::string_view::npos) {
@@ -1169,7 +1246,7 @@ where   <output path>: the directory to write generated contents to. This will N
 			auto opcode = ParseInputOpcode(opcodeString);
 			auto operand = arg.substr(separatorLoc + 1);
 
-			HandleArgument(state, opcode, operand);
+			HandleArgument(as, opcode, operand);
 		}
 	}