3 files changed, 244 insertions, 77 deletions

diff --git a/source/20-codegen-compiler/CodegenModel.cpp b/source/20-codegen-compiler/CodegenModel.cpp
index d075c1d..08323bc 100644
--- a/source/20-codegen-compiler/CodegenModel.cpp
+++ b/source/20-codegen-compiler/CodegenModel.cpp
@@ -29,19 +29,123 @@ public:
 	robin_hood::unordered_node_map<std::string, DeclNamespace, StringHash, StringEqual> namespaces;
 };
 
+// A number for `PRAGMA user_vesrion`, representing the current database version. Increment when the table format changes.
+#define CURRENT_DATABASE_VERSION 1
+constexpr int64_t kGlobalNamespaceRowId = 1;
+
+struct SQLiteDatabase {
+	sqlite3* database = nullptr;
+
+	~SQLiteDatabase() {
+		// NOTE: calling with NULL is a harmless no-op
+		int result = sqlite3_close(database);
+		assert(result == SQLITE_OK);
+	}
+
+	operator sqlite3*() const { return database; }
+	sqlite3** operator&() { return &database; }
+};
+
+struct SQLiteStatement {
+	sqlite3_stmt* stmt = nullptr;
+
+	~SQLiteStatement() {
+		// NOTE: calling with NULL is a harmless no-op
+		// NOTE: we don't care about the error code, because they are returned if the statement has errored in the most recent execution
+		//       but deleting it will succeeed anyways
+		sqlite3_finalize(stmt);
+	}
+
+	operator sqlite3_stmt*() const { return stmt; }
+	sqlite3_stmt** operator&() { return &stmt; }
+
+	bool InitializeLazily(sqlite3* database, std::string_view sql) {
+		if (!stmt) {
+			int result = sqlite3_prepare(database, sql.data(), sql.size(), &stmt, nullptr);
+			if (result != SQLITE_OK) {
+				INPLACE_FMT(msg, "Failed to prepare statement, error message: %s", sqlite3_errmsg(database));
+				throw std::runtime_error(msg);
+			}
+
+			return true;
+		}
+		return false;
+	}
+};
+
+namespace {
+void PrintErrMsgIfPresent(char*& errMsg) {
+	if (errMsg) {
+		printf("SQLite error: %s\n", errMsg);
+		sqlite3_free(errMsg);
+	}
+}
+} // namespace
+
 class CodegenModelArchive::Private {
 	friend class CodegenModel;
 
 public:
-	sqlite3* database = nullptr;
-	sqlite3_stmt* beginTransactionStmt = nullptr;
-	sqlite3_stmt* commitTransactionStmt = nullptr;
-	sqlite3_stmt* rollbackTransactionStmt = nullptr;
-	sqlite3_stmt* storeEnumStmt = nullptr;
-	sqlite3_stmt* storeEnumElmStmt = nullptr;
+	// NOTE: this must be the first field, because we want it to destruct after all other statement fields
+	SQLiteDatabase database;
+	/* Core Statements */
+	SQLiteStatement beginTransactionStmt;
+	SQLiteStatement commitTransactionStmt;
+	SQLiteStatement rollbackTransactionStmt;
+	/* Component Statements, initalized on demand */
+	SQLiteStatement findNamespaceStmt;
+	SQLiteStatement findOrStoreNamespaceStmt;
+	SQLiteStatement storeEnumStmt;
+	SQLiteStatement storeEnumElmStmt;
 
-	// See below for definition
-	void InitializeDatabase();
+	void InitializeCoreStatements() {
+		char* errMsg = nullptr;
+
+		int result = sqlite3_exec(database, "PRAGMA user_version = " STRINGIFY(CURRENT_DATABASE_VERSION), nullptr, nullptr, &errMsg);
+		PrintErrMsgIfPresent(errMsg);
+		assert(result == SQLITE_OK);
+
+		result = sqlite3_exec(database, R"""(
+BEGIN TRANSACTION;
+CREATE TABLE Namespaces(
+	ParentNamespaceRowId INTEGER REFERENCES Namespaces(rowid),
+	Name TEXT,
+	UNIQUE (ParentNamespaceRowId, Name)
+);
+
+CREATE TABLE DeclStructs(
+	NamespaceRowId INTEGER REFERENCES Namespaces(rowid),
+	ParentStructRowId INTEGER REFERENCES DeclStructs(rowid),
+	Name TEXT,
+	UNIQUE (NamespaceRowId, Name)
+);
+
+CREATE TABLE DeclEnums(
+	NamespaceRowId INTEGER REFERENCES Namespaces(rowid),
+	Name TEXT,
+	UnderlyingType TEXT,
+	UNIQUE (NamespaceRowId, Name)
+);
+CREATE TABLE DeclEnumElements(
+	EnumRowId INTEGER REFERENCES DeclEnums(rowid),
+	Name TEXT,
+	Value INTEGER,
+	UNIQUE (EnumRowId, Name)
+);
+
+INSERT INTO Namespaces(rowid, ParentNamespaceRowId, Name)
+VALUES
+	-- Special global namespace that has no parent, and rowid should always be 1
+	(1, NULL, "<global namespace>");
+
+COMMIT TRANSACTION;
+)""",
+			nullptr,
+			nullptr,
+			&errMsg);
+		PrintErrMsgIfPresent(errMsg);
+		assert(result == SQLITE_OK);
+	}
 
 	void BeginTransaction() {
 		int result = sqlite3_step(beginTransactionStmt);
@@ -60,6 +164,92 @@ public:
 		assert(result == SQLITE_DONE);
 		sqlite3_reset(rollbackTransactionStmt);
 	}
+
+	/// \return Row ID of the namespace, or 0 if it currently doesn't exist.
+	int64_t FindNamespace(const DeclNamespace* ns) {
+		if (!ns) {
+			return kGlobalNamespaceRowId;
+		}
+
+		InitializeStmt_findNamespaceStmt();
+		return FindNamespaceImpl(*ns);
+	}
+
+	/// \return Row ID of the namespace.
+	int64_t FindOrStoreNamespace(const DeclNamespace* ns) {
+		if (!ns) {
+			return kGlobalNamespaceRowId;
+		}
+
+		InitializeStmt_findNamespaceStmt();
+		InitializeStmt_findOrStoreNamespaceStmt();
+		return FindOrStoreNamespaceImpl(*ns);
+	}
+
+private:
+	void InitializeStmt_findNamespaceStmt() {
+		findNamespaceStmt.InitializeLazily(database, R"""(
+SELECT rowid FROM Namespaces WHERE ParentNamespaceRowId = ?1 AND Name = ?2;
+)"""sv);
+	}
+
+	void InitializeStmt_findOrStoreNamespaceStmt() {
+		findOrStoreNamespaceStmt.InitializeLazily(database, R"""(
+INSERT INTO Namespaces(ParentNamespaceRowId, Name)
+	VALUES (?1, ?2)
+	RETURNING rowid;
+)"""sv);
+	}
+
+	int64_t FindNamespaceImpl(const DeclNamespace& ns) {
+		int64_t parentNsRowId;
+		if (ns.container) {
+			parentNsRowId = FindNamespaceImpl(*ns.container);
+			if (parentNsRowId == 0) {
+				// Parent namespace doesn't exist in database, shortcircuit
+				return 0;
+			}
+		} else {
+			parentNsRowId = kGlobalNamespaceRowId;
+		}
+
+		return FindNamespaceImpl(ns, parentNsRowId);
+	}
+
+	int64_t FindNamespaceImpl(const DeclNamespace& ns, int64_t parentNsRowId) {
+		sqlite3_stmt* stmt = findNamespaceStmt;
+		sqlite3_bind_int64(stmt, 1, parentNsRowId);
+		sqlite3_bind_text(stmt, 2, ns.name.c_str(), ns.name.size(), nullptr);
+		int result = sqlite3_step(stmt);
+		if (result == SQLITE_ROW) {
+			int64_t rowId = sqlite3_column_int64(stmt, 0);
+			sqlite3_reset(stmt);
+
+			return rowId;
+		} else {
+			return 0;
+		}
+	}
+
+	int64_t FindOrStoreNamespaceImpl(const DeclNamespace& ns) {
+		if (auto rowId = FindNamespaceImpl(ns); rowId != 0) {
+			return rowId;
+		}
+
+		sqlite3_stmt* stmt = findOrStoreNamespaceStmt;
+		int64_t parentRowId = ns.container
+			? FindOrStoreNamespaceImpl(*ns.container)
+			: kGlobalNamespaceRowId;
+		sqlite3_bind_int64(stmt, 1, parentRowId);
+		sqlite3_bind_text(stmt, 2, ns.name.c_str(), ns.name.size(), nullptr);
+		int result = sqlite3_step(stmt);
+		assert(result == SQLITE_ROW);
+
+		auto rowId = sqlite3_column_int64(stmt, 0);
+		sqlite3_reset(stmt);
+
+		return rowId;
+	}
 };
 
 CodegenModel::CodegenModel()
@@ -143,42 +333,6 @@ DeclNamespace* CodegenModel::FindNamespace(std::string_view name) {
 	return const_cast<DeclNamespace*>(const_cast<const CodegenModel*>(this)->FindNamespace(name));
 }
 
-sqlite3_stmt* Utils::NewSqliteStatement(sqlite3* database, std::string_view sql) {
-	sqlite3_stmt* stmt;
-	int result = sqlite3_prepare(database, sql.data(), sql.size(), &stmt, nullptr);
-	if (result != SQLITE_OK) {
-		INPLACE_FMT(msg, "Failed to prepare statement, error message: %s", sqlite3_errmsg(database));
-		throw std::runtime_error(msg);
-	}
-
-	return stmt;
-}
-
-void CodegenModelArchive::Private::InitializeDatabase() {
-	int result = sqlite3_exec(database, "PRAGMA user_version = 1", nullptr, nullptr, nullptr);
-	assert(result == SQLITE_OK);
-
-	result = sqlite3_exec(database, R"""(
-BEGIN TRANSACTION;
-CREATE TABLE DeclEnums(
-    Id INT PRIMARY KEY,
-    Name TEXT,
-    UnderlyingType TEXT
-);
-CREATE TABLE DeclEnumElements(
-    EnumId INT,
-    Name TEXT,
-    Value INT,
-    FOREIGN KEY (EnumId) REFERENCES DeclEnums(Id)
-);
-COMMIT TRANSACTION;
-)""",
-		nullptr,
-		nullptr,
-		nullptr);
-	assert(result == SQLITE_OK);
-}
-
 CodegenModelArchive::CodegenModelArchive(std::string_view dbPath)
 	: m{ new Private() } //
 {
@@ -191,47 +345,39 @@ CodegenModelArchive::CodegenModelArchive(std::string_view dbPath)
 		throw std::runtime_error(msg);
 	}
 
-	// Keep the same as `PRAGMA user_version` in the CodegenMetadataArchive::Private::InitializeDatabase()
-	constexpr int kDatabaseVersion = 1;
-	int currentDatabaseVersion;
 	{
-		auto readVersionStmt = Utils::NewSqliteStatement(m->database, "PRAGMA user_version");
-		DEFER { sqlite3_finalize(readVersionStmt); };
+		SQLiteStatement readVersionStmt;
+		readVersionStmt.InitializeLazily(m->database, "PRAGMA user_version"sv);
 
 		int result = sqlite3_step(readVersionStmt);
 		assert(result == SQLITE_ROW);
-		currentDatabaseVersion = sqlite3_column_int(readVersionStmt, 0);
+		int currentDatabaseVersion = sqlite3_column_int(readVersionStmt, 0);
 
 		result = sqlite3_step(readVersionStmt);
 		assert(result == SQLITE_DONE);
 
 		if (currentDatabaseVersion == 0) {
 			// Newly created database, initialize it
-			m->InitializeDatabase();
-		} else if (currentDatabaseVersion == kDatabaseVersion) {
+			m->InitializeCoreStatements();
+		} else if (currentDatabaseVersion == CURRENT_DATABASE_VERSION) {
 			// Same version, no need to do anything
 		} else {
-			INPLACE_FMT(msg, "Incompatbile database versions %d (in file) vs %d (expected).", currentDatabaseVersion, kDatabaseVersion);
+			INPLACE_FMT(msg, "Incompatbile database versions %d (in file) vs %d (expected).", currentDatabaseVersion, CURRENT_DATABASE_VERSION);
 			throw std::runtime_error(msg);
 		}
 	}
 
-	m->beginTransactionStmt = Utils::NewSqliteStatement(m->database, "BEGIN TRANSACTION");
-	m->commitTransactionStmt = Utils::NewSqliteStatement(m->database, "COMMIT TRANSACTION");
-	m->rollbackTransactionStmt = Utils::NewSqliteStatement(m->database, "ROLLBACK TRANSACTION");
+	// This database is used for a buildsystem and can be regenerated at any time. We don't care for the slightest about data integrity, we just want fast updates
+	// NOTE: These pragmas are not persistent, so we need to set them every time
+	sqlite3_exec(m->database, "PRAGMA synchronous = OFF", nullptr, nullptr, nullptr);
+	sqlite3_exec(m->database, "PRAGMA journal_mode = MEMORY", nullptr, nullptr, nullptr);
 
-	m->storeEnumStmt = Utils::NewSqliteStatement(m->database, "INSERT INTO DeclEnums(Name, UnderlyingType) VALUES(?, ?)"sv);
-	m->storeEnumElmStmt = Utils::NewSqliteStatement(m->database, "INSERT INTO DeclEnumElements(EnumId, Name, Value) VALUES(?, ?, ?)"sv);
+	m->beginTransactionStmt.InitializeLazily(m->database, "BEGIN TRANSACTION");
+	m->commitTransactionStmt.InitializeLazily(m->database, "COMMIT TRANSACTION");
+	m->rollbackTransactionStmt.InitializeLazily(m->database, "ROLLBACK TRANSACTION");
 }
 
 CodegenModelArchive::~CodegenModelArchive() {
-	sqlite3_close(m->database);
-	sqlite3_finalize(m->beginTransactionStmt);
-	sqlite3_finalize(m->commitTransactionStmt);
-	sqlite3_finalize(m->rollbackTransactionStmt);
-	sqlite3_finalize(m->storeEnumStmt);
-	sqlite3_finalize(m->storeEnumElmStmt);
-
 	delete m;
 }
 
@@ -255,6 +401,10 @@ void CodegenModelArchive::Store(const CodegenModel& cgInput) {
 
 	m->BeginTransaction();
 
+	for (auto&& [DISCARD, ns] : cgm.namespaces) {
+		// This will insert the namespace if it doesn't exist, or no-op (fetches data) if it already exists
+		m->FindOrStoreNamespace(&ns);
+	}
 	for (auto&& [DISCARD, value] : cgm.decls) {
 		std::visit(visiter, value.v);
 	}
@@ -271,16 +421,37 @@ void CodegenModelArchive::StoreFunction(const DeclFunction& decl) {
 }
 
 void CodegenModelArchive::StoreEnum(const DeclEnum& decl) {
-	// TOOD only update
-	sqlite3_bind_text(m->storeEnumStmt, 1, decl.name.c_str(), decl.name.size(), nullptr);
-	sqlite3_bind_text(m->storeEnumStmt, 2, decl.underlyingTypeStr.c_str(), decl.underlyingTypeStr.size(), nullptr);
+	// -Argument-	-Description-
+	// ?1			Namespace ID
+	// ?2			Enum name
+	// ?3			Enum underlying type
+	m->storeEnumStmt.InitializeLazily(m->database, R"""(
+INSERT INTO DeclEnums(NamespaceRowId, Name, UnderlyingType)
+	VALUES (?1, ?2, ?3)
+	ON CONFLICT DO UPDATE SET UnderlyingType=?3
+	RETURNING rowid;
+)"""sv);
+
+	// -Argument-	-Description-
+	// ?1			Container enum's rowid
+	// ?2			Enum element name
+	// ?3			Enum element value
+	m->storeEnumElmStmt.InitializeLazily(m->database, R"""(
+INSERT INTO DeclEnumElements(EnumRowId, Name, Value)
+	VALUES (?1, ?2, ?3)
+	ON CONFLICT DO UPDATE SET Value=?3;
+)"""sv);
+
+	// TODO delete non-existent enums
+	sqlite3_bind_int(m->storeEnumStmt, 1, m->FindNamespace(decl.container));
+	sqlite3_bind_text(m->storeEnumStmt, 2, decl.name.c_str(), decl.name.size(), nullptr);
+	sqlite3_bind_text(m->storeEnumStmt, 3, decl.underlyingTypeStr.c_str(), decl.underlyingTypeStr.size(), nullptr);
 	int result = sqlite3_step(m->storeEnumStmt);
-	assert(result == SQLITE_DONE);
+	assert(result == SQLITE_ROW);
+	auto enumRowId = sqlite3_column_int64(m->storeEnumStmt, 0);
 	sqlite3_reset(m->storeEnumStmt);
 	sqlite3_clear_bindings(m->storeEnumStmt);
 
-	auto enumRowId = sqlite3_last_insert_rowid(m->database);
-
 	for (auto& elm : decl.elements) {
 		sqlite3_bind_int64(m->storeEnumElmStmt, 1, enumRowId);
 		sqlite3_bind_text(m->storeEnumElmStmt, 2, elm.name.c_str(), elm.name.size(), nullptr);
diff --git a/source/20-codegen-compiler/CodegenModelArchive.hpp b/source/20-codegen-compiler/CodegenModelArchive.hpp
index 261d82b..0e0e5ad 100644
--- a/source/20-codegen-compiler/CodegenModelArchive.hpp
+++ b/source/20-codegen-compiler/CodegenModelArchive.hpp
@@ -6,10 +6,6 @@
 #include <sqlite3.h>
 #include <string_view>
 
-namespace Utils {
-sqlite3_stmt* NewSqliteStatement(sqlite3* database, std::string_view sql);
-} // namespace Utils
-
 class CodegenModelArchive {
 private:
 	class Private;
diff --git a/source/20-codegen-compiler/main.cpp b/source/20-codegen-compiler/main.cpp
index 1431ae5..5a59b5d 100644
--- a/source/20-codegen-compiler/main.cpp
+++ b/source/20-codegen-compiler/main.cpp
@@ -1228,8 +1228,8 @@ where   --output-dir=<path>: the *directory* to write generated contents to. Thi
 	DEBUG_PRINTF("Databse file: %.*s.\n", PRINTF_STRING_VIEW(as.databaseFilePath));
 
 	// TODO model archive is broken right now, see each TODO in CodegenModel.cpp
-	// CodegenModelArchive archive(as.databaseFilePath);
-	// as.modelArchive = &archive;
+	CodegenModelArchive archive(as.databaseFilePath);
+	as.modelArchive = &archive;
 
 	// Positional argument pass
 	for (int i = 1; i < argc; ++i) {