#include "CodegenModel.hpp" #include "CodegenModelArchive.hpp" #include "CodegenUtils.hpp" #include #include #include #include #include #include #include #include #include using namespace std::literals; struct SomeDecl { std::variant v; }; class CodegenModel::Private { friend class CodegenModelArchive; public: // We want address stability for everything robin_hood::unordered_node_map decls; robin_hood::unordered_node_map namespaces; }; // A number for `PRAGMA user_vesrion`, representing the current database version. Increment when the table format changes. #define CURRENT_DATABASE_VERSION 1 constexpr int64_t kGlobalNamespaceId = 1; struct SQLiteDatabase { sqlite3* database = nullptr; ~SQLiteDatabase() { // NOTE: calling with NULL is a harmless no-op int result = sqlite3_close(database); assert(result == SQLITE_OK); } operator sqlite3*() const { return database; } sqlite3** operator&() { return &database; } }; struct SQLiteStatement { sqlite3_stmt* stmt = nullptr; SQLiteStatement(const SQLiteStatement&) = delete; SQLiteStatement& operator=(const SQLiteStatement&) = delete; SQLiteStatement() = default; ~SQLiteStatement() { // NOTE: calling with NULL is a harmless no-op // NOTE: we don't care about the error code, because they are returned if the statement has errored in the most recent execution // but deleting it will succeeed anyways sqlite3_finalize(stmt); } operator sqlite3_stmt*() const { return stmt; } sqlite3_stmt** operator&() { return &stmt; } bool InitializeLazily(sqlite3* database, std::string_view sql) { if (!stmt) { int result = sqlite3_prepare(database, sql.data(), sql.size(), &stmt, nullptr); if (result != SQLITE_OK) { INPLACE_FMT(msg, "Failed to prepare statement, error message: %s", sqlite3_errmsg(database)); throw std::runtime_error(msg); } return true; } return false; } }; namespace { void PrintErrMsgIfPresent(char*& errMsg) { if (errMsg) { printf("SQLite error: %s\n", errMsg); sqlite3_free(errMsg); } } } // namespace class CodegenModelArchive::Private { friend class CodegenModel; public: // NOTE: this must be the first field, because we want it to destruct after all other statement fields SQLiteDatabase database; /* Core Statements */ SQLiteStatement beginTransactionStmt; SQLiteStatement commitTransactionStmt; SQLiteStatement rollbackTransactionStmt; /* Component Statements, initalized on demand */ SQLiteStatement findNamespaceStmt; SQLiteStatement findOrStoreNamespaceStmt; SQLiteStatement storeStructStmt; SQLiteStatement storeStructBaseClassStmt; SQLiteStatement storeStructPropertyStmt; // TODO store method SQLiteStatement storeEnumStmt; SQLiteStatement storeEnumElmStmt; SQLiteStatement deleteFunctionDeclByFilenameStmt; SQLiteStatement deleteStructDeclByFilenameStmt; SQLiteStatement deleteEnumDeclByFilenameStmt; void InitializeDatabase() { char* errMsg = nullptr; int result = sqlite3_exec(database, "PRAGMA user_version = " STRINGIFY(CURRENT_DATABASE_VERSION), nullptr, nullptr, &errMsg); PrintErrMsgIfPresent(errMsg); assert(result == SQLITE_OK); // TODO create a table of file names // TODO unique with overloading, and container structs result = sqlite3_exec(database, R"""( BEGIN TRANSACTION; CREATE TABLE Files( -- NOTE: SQLite forbids foreign keys referencing the implicit `rowid` column, we have to create an alias for it Id INTEGER PRIMARY KEY, FileName TEXT, UNIQUE (FileName) ); CREATE TABLE Namespaces( Id INTEGER PRIMARY KEY, ParentNamespaceId INTEGER REFERENCES Namespaces(Id), Name TEXT, UNIQUE (ParentNamespaceId, Name) ); CREATE TABLE DeclFunctions( Id INTEGER PRIMARY KEY, NamespaceId INTEGER REFERENCES Namespaces(Id), Name TEXT, FileId INTEGER REFERENCES Files(Id) ); CREATE TABLE DeclFunctionParameters( FunctionId INTEGER REFERENCES DeclFunctions(Id) ON DELETE CASCADE, Name TEXT, Type TEXT, UNIQUE (FunctionId, Name) ); CREATE TABLE DeclStructs( Id INTEGER PRIMARY KEY, NamespaceId INTEGER REFERENCES Namespaces(Id), Name TEXT, FileId INTEGER REFERENCES Files(Id), IsMetadataMarked INTEGER ); CREATE TABLE DeclStructBaseClassRelations( StructId INTEGER REFERENCES DeclStructs(Id) ON DELETE CASCADE, -- NOTE: intentionally not foreign keys, because we want relations to still exist even if the base class is deleted -- we do validation after a complete regeneration pass, on reads ParentStructNamespaceId INTEGER, ParentStructName TEXT, UNIQUE (StructId, ParentStructNamespaceId, ParentStructName) ); CREATE TABLE DeclStructProperties( StructId INTEGER REFERENCES DeclStructs(Id) ON DELETE CASCADE, Name TEXT, Type TEXT, -- NOTE: getter and setter may or may not be methods; search the DeclStructMethods table if needed GetterName TEXT, SetterName TEXT, IsPlainField INTEGER GENERATED ALWAYS AS (GetterName = '' AND SetterName = '') VIRTUAL, IsMetadataMarked INTEGER ); CREATE TABLE DeclStructMethods( Id INTEGER PRIMARY KEY, StructId INTEGER REFERENCES DeclStructs(Id) ON DELETE CASCADE, Name TEXT, Type TEXT, IsConst INTEGER, IsMetadataMarked INTEGER ); CREATE TABLE DeclStructMethodParameters( MethodId INTEGER REFERENCES DeclStructMethods(Id) ON DELETE CASCADE, Name TEXT, Type TEXT, UNIQUE (MethodId, Name) ); CREATE TABLE DeclEnums( Id INTEGER PRIMARY KEY, NamespaceId INTEGER REFERENCES Namespaces(Id), Name TEXT, UnderlyingType TEXT, FileId INTEGER REFERENCES Files(Id) ); CREATE TABLE DeclEnumElements( EnumId INTEGER REFERENCES DeclEnums(Id) ON DELETE CASCADE, Name TEXT, Value INTEGER, UNIQUE (EnumId, Name) ); CREATE INDEX Index_DeclFunctions_FileId ON DeclFunctions(FileId); CREATE INDEX Index_DeclStructs_FileId ON DeclStructs(FileId); CREATE INDEX Index_DeclEnums_FileId ON DeclEnums(FileId); CREATE UNIQUE INDEX Index_DeclFunctions_Identity ON DeclFunctions(NamespaceId, Name); CREATE UNIQUE INDEX Index_DeclStruct_Identity ON DeclStructs(NamespaceId, Name); CREATE UNIQUE INDEX Index_DeclStructProperties_Identity ON DeclStructProperties(StructId, Name); CREATE UNIQUE INDEX Index_DeclStructMethods_Identity ON DeclStructMethods(StructId, Name); CREATE UNIQUE INDEX Index_DeclEnums_Identity ON DeclEnums(NamespaceId, Name); INSERT INTO Namespaces(Id, ParentNamespaceId, Name) VALUES -- Special global namespace that has no parent, and Id should always be 1 (1, NULL, ""); COMMIT TRANSACTION; )""", nullptr, nullptr, &errMsg); PrintErrMsgIfPresent(errMsg); assert(result == SQLITE_OK); } void BeginTransaction() { int result = sqlite3_step(beginTransactionStmt); assert(result == SQLITE_DONE); sqlite3_reset(beginTransactionStmt); } void CommitTransaction() { int result = sqlite3_step(commitTransactionStmt); assert(result == SQLITE_DONE); sqlite3_reset(commitTransactionStmt); } void RollbackTransaction() { int result = sqlite3_step(rollbackTransactionStmt); assert(result == SQLITE_DONE); sqlite3_reset(rollbackTransactionStmt); } /// \return Row ID of the namespace, or 0 if it currently doesn't exist. int64_t FindNamespace(const DeclNamespace* ns) { if (!ns) { return kGlobalNamespaceId; } InitializeStmt_findNamespaceStmt(); return FindNamespaceImpl(*ns); } /// \return Row ID of the namespace. int64_t FindOrStoreNamespace(const DeclNamespace* ns) { if (!ns) { return kGlobalNamespaceId; } InitializeStmt_findNamespaceStmt(); InitializeStmt_findOrStoreNamespaceStmt(); return FindOrStoreNamespaceImpl(*ns); } private: void InitializeStmt_findNamespaceStmt() { findNamespaceStmt.InitializeLazily(database, R"""( SELECT Id FROM Namespaces WHERE ParentNamespaceId = ?1 AND Name = ?2; )"""sv); } void InitializeStmt_findOrStoreNamespaceStmt() { findOrStoreNamespaceStmt.InitializeLazily(database, R"""( INSERT INTO Namespaces(ParentNamespaceId, Name) VALUES (?1, ?2) RETURNING Id; )"""sv); } int64_t FindNamespaceImpl(const DeclNamespace& ns) { int64_t parentNsRowId; if (ns.container) { parentNsRowId = FindNamespaceImpl(*ns.container); if (parentNsRowId == 0) { // Parent namespace doesn't exist in database, shortcircuit return 0; } } else { parentNsRowId = kGlobalNamespaceId; } return FindNamespaceImpl(ns, parentNsRowId); } int64_t FindNamespaceImpl(const DeclNamespace& ns, int64_t parentNsRowId) { sqlite3_stmt* stmt = findNamespaceStmt; sqlite3_bind_int64(stmt, 1, parentNsRowId); sqlite3_bind_text(stmt, 2, ns.name.c_str(), ns.name.size(), nullptr); int result = sqlite3_step(stmt); if (result == SQLITE_ROW) { int64_t rowId = sqlite3_column_int64(stmt, 0); sqlite3_reset(stmt); return rowId; } else { return 0; } } int64_t FindOrStoreNamespaceImpl(const DeclNamespace& ns) { if (auto rowId = FindNamespaceImpl(ns); rowId != 0) { return rowId; } sqlite3_stmt* stmt = findOrStoreNamespaceStmt; int64_t parentRowId = ns.container ? FindOrStoreNamespaceImpl(*ns.container) : kGlobalNamespaceId; sqlite3_bind_int64(stmt, 1, parentRowId); sqlite3_bind_text(stmt, 2, ns.name.c_str(), ns.name.size(), nullptr); int result = sqlite3_step(stmt); assert(result == SQLITE_ROW); auto rowId = sqlite3_column_int64(stmt, 0); sqlite3_reset(stmt); return rowId; } }; CodegenModel::CodegenModel() : m{ new Private() } // { } CodegenModel::~CodegenModel() { delete m; } #define STORE_DECL_OF_TYPE(DeclType, fullname, decl) \ auto [iter, success] = m->decls.try_emplace(std::move(fullname), SomeDecl{ .v = std::move(decl) }); \ auto& key = iter->first; \ auto& val = iter->second; \ auto& declRef = std::get(val.v); \ declRef.fullname = key; \ return &declRef DeclEnum* CodegenModel::AddEnum(std::string fullname, DeclEnum decl) { #if CODEGEN_DEBUG_PRINT printf("Committed enum '%s'\n", decl.name.c_str()); for (auto& elm : decl.elements) { printf(" - element %s = %" PRId64 "\n", elm.name.c_str(), elm.value); } #endif STORE_DECL_OF_TYPE(DeclEnum, fullname, decl); } DeclStruct* CodegenModel::AddStruct(std::string fullname, DeclStruct decl) { #if CODEGEN_DEBUG_PRINT printf("Committed struct '%s'\n", decl.name.c_str()); printf(" Base classes:\n"); for (auto& base : decl.baseClasses) { printf(" - %.*s\n", PRINTF_STRING_VIEW(base->name)); } #endif STORE_DECL_OF_TYPE(DeclStruct, fullname, decl); } #define FIND_DECL_OF_TYPE(DeclType) \ auto iter = m->decls.find(name); \ if (iter != m->decls.end()) { \ auto& some = iter->second.v; \ if (auto decl = std::get_if(&some)) { \ return decl; \ } \ } \ return nullptr const DeclEnum* CodegenModel::FindEnum(std::string_view name) const { FIND_DECL_OF_TYPE(DeclEnum); } const DeclStruct* CodegenModel::FindStruct(std::string_view name) const { FIND_DECL_OF_TYPE(DeclStruct); } DeclNamespace* CodegenModel::AddNamespace(DeclNamespace ns) { auto path = Utils::MakeFullName(""sv, &ns); auto [iter, success] = m->namespaces.try_emplace(std::move(path), std::move(ns)); auto& nsRef = iter->second; if (success) { nsRef.fullname = iter->first; } return &nsRef; } const DeclNamespace* CodegenModel::FindNamespace(std::string_view fullname) const { auto iter = m->namespaces.find(fullname); if (iter != m->namespaces.end()) { return &iter->second; } else { return nullptr; } } DeclNamespace* CodegenModel::FindNamespace(std::string_view name) { return const_cast(const_cast(this)->FindNamespace(name)); } CodegenModelArchive::CodegenModelArchive(std::string_view dbPath) : m{ new Private() } // { std::string zstrPath(dbPath); int reuslt = sqlite3_open(zstrPath.c_str(), &m->database); if (reuslt != SQLITE_OK) { std::string msg; msg += "Failed to open SQLite3 database, error message:\n"; msg += sqlite3_errmsg(m->database); throw std::runtime_error(msg); } // NOTE: These pragmas are not persistent, so we need to set them every time // As of SQLite3 3.38.5, it defaults to foreign_keys = OFF, so we need this to be on for ON DELETE CASCADE and etc. to work sqlite3_exec(m->database, "PRAGMA foreign_keys = ON", nullptr, nullptr, nullptr); // This database is used for a buildsystem and can be regenerated at any time. We don't care for the slightest about data integrity, we just want fast updates sqlite3_exec(m->database, "PRAGMA synchronous = OFF", nullptr, nullptr, nullptr); sqlite3_exec(m->database, "PRAGMA journal_mode = MEMORY", nullptr, nullptr, nullptr); { SQLiteStatement readVersionStmt; readVersionStmt.InitializeLazily(m->database, "PRAGMA user_version"sv); int result = sqlite3_step(readVersionStmt); assert(result == SQLITE_ROW); int currentDatabaseVersion = sqlite3_column_int(readVersionStmt, 0); result = sqlite3_step(readVersionStmt); assert(result == SQLITE_DONE); if (currentDatabaseVersion == 0) { // Newly created database, initialize it m->InitializeDatabase(); } else if (currentDatabaseVersion == CURRENT_DATABASE_VERSION) { // Same version, no need to do anything } else { INPLACE_FMT(msg, "Incompatbile database versions %d (in file) vs %d (expected).", currentDatabaseVersion, CURRENT_DATABASE_VERSION); throw std::runtime_error(msg); } } // Initialize core statements m->beginTransactionStmt.InitializeLazily(m->database, "BEGIN TRANSACTION"); m->commitTransactionStmt.InitializeLazily(m->database, "COMMIT TRANSACTION"); m->rollbackTransactionStmt.InitializeLazily(m->database, "ROLLBACK TRANSACTION"); } CodegenModelArchive::~CodegenModelArchive() { delete m; } void CodegenModelArchive::DeleteDeclsRelatedToFile(std::string_view filename) { // -Argument- -Description- // ?1 The filename to delete m->deleteFunctionDeclByFilenameStmt.InitializeLazily(m->database, "DELETE FROM DeclFunctions WHERE FileId = (SELECT Id FROM Files WHERE FileName = ?1)"sv); m->deleteStructDeclByFilenameStmt.InitializeLazily(m->database, "DELETE FROM DeclStructs WHERE FileId = (SELECT Id FROM Files WHERE FileName = ?1);"sv); m->deleteEnumDeclByFilenameStmt.InitializeLazily(m->database, "DELETE FROM DeclEnums WHERE FileId = (SELECT Id FROM Files WHERE FileName = ?1);"sv); m->BeginTransaction(); auto stmtList = { m->deleteFunctionDeclByFilenameStmt.stmt, m->deleteStructDeclByFilenameStmt.stmt, m->deleteEnumDeclByFilenameStmt.stmt, }; for (auto& stmt : stmtList) { sqlite3_bind_text(stmt, 1, filename.data(), filename.size(), nullptr); int result = sqlite3_step(stmt); assert(result == SQLITE_DONE); sqlite3_reset(stmt); sqlite3_clear_bindings(stmt); } m->CommitTransaction(); } void CodegenModelArchive::Store(const CodegenModel& cgInput) { auto& cgm = cgInput.GetPimpl(); struct Visiter { CodegenModelArchive* self; void operator()(const DeclStruct& decl) const { self->StoreStruct(decl); } void operator()(const DeclFunction& decl) const { self->StoreFunction(decl); } void operator()(const DeclEnum& decl) const { self->StoreEnum(decl); } } visiter; visiter.self = this; m->BeginTransaction(); for (auto&& [DISCARD, ns] : cgm.namespaces) { // This will insert the namespace if it doesn't exist, or no-op (fetches data) if it already exists m->FindOrStoreNamespace(&ns); } for (auto&& [DISCARD, value] : cgm.decls) { std::visit(visiter, value.v); } m->CommitTransaction(); } void CodegenModelArchive::StoreStruct(const DeclStruct& decl) { // -Argument- -Description- // ?1 Namespace ID // ?2 Struct name // ?3 File containing the struct m->storeStructStmt.InitializeLazily(m->database, R"""( INSERT OR IGNORE INTO Files(FileName) VALUES (?3); WITH const AS (SELECT Id AS fileId FROM Files WHERE FileName = ?3) INSERT INTO DeclStructs(NamespaceId, Name, FileId, IsMetadataMarked) VALUES (?1, ?2, const.fileId, TRUE) ON CONFLICT DO UPDATE SET FileId = const.fileId RETURNING Id )"""sv); // -Argument- -Description- // ?1 Struct ID // ?2 Parent struct's namespace ID // ?3 Parent struct's name m->storeStructBaseClassStmt.InitializeLazily(m->database, R"""( INSERT INTO DeclStructBaseClassRelations(StructId, ParentStructNamespaceId, ParentStructName) VALUES (?1, ?2, ?3) )"""sv); // -Argument- -Description- // ?1 Struct ID // ?2 Property name // ?3 Property type // ?4 Getter name (optional) // ?5 Setter name (optional) m->storeStructPropertyStmt.InitializeLazily(m->database, R"""( INSERT INTO DeclStructProperties(StructId, Name, Type, GetterName, SetterName, IsMetadataMarked) VALUES (?1, ?2, ?3, ?4, ?5, TRUE) )"""sv); int result; sqlite3_bind_int64(m->storeStructStmt, 1, m->FindNamespace(decl.container)); sqlite3_bind_text(m->storeStructStmt, 2, decl.name.c_str(), decl.name.size(), nullptr); if (decl.sourceFile) { sqlite3_bind_text(m->storeStructStmt, 3, decl.sourceFile->filename.data(), decl.sourceFile->filename.size(), nullptr); } else { sqlite3_bind_text(m->storeStructStmt, 3, "", 0, nullptr); } result = sqlite3_step(m->storeStructStmt); result = sqlite3_step(m->storeStructStmt); assert(result == SQLITE_ROW); int64_t structId = sqlite3_column_int64(m->storeStructStmt, 0); sqlite3_reset(m->storeStructStmt); sqlite3_clear_bindings(m->storeStructStmt); for (auto& baseClass : decl.baseClasses) { sqlite3_bind_int64(m->storeStructBaseClassStmt, 1, structId); sqlite3_bind_int64(m->storeStructBaseClassStmt, 2, m->FindNamespace(baseClass->container)); sqlite3_bind_text(m->storeStructBaseClassStmt, 3, baseClass->name.c_str(), baseClass->name.size(), nullptr); int result = sqlite3_step(m->storeStructBaseClassStmt); assert(result == SQLITE_DONE); sqlite3_reset(m->storeStructBaseClassStmt); sqlite3_clear_bindings(m->storeStructBaseClassStmt); } for (auto& property : decl.memberVariables) { sqlite3_bind_int64(m->storeStructPropertyStmt, 1, structId); sqlite3_bind_text(m->storeStructPropertyStmt, 2, property.name.c_str(), property.name.size(), nullptr); sqlite3_bind_text(m->storeStructPropertyStmt, 3, property.type.c_str(), property.type.size(), nullptr); sqlite3_bind_text(m->storeStructPropertyStmt, 4, property.getterName.c_str(), property.getterName.size(), nullptr); sqlite3_bind_text(m->storeStructPropertyStmt, 5, property.setterName.c_str(), property.setterName.size(), nullptr); int result = sqlite3_step(m->storeStructPropertyStmt); assert(result == SQLITE_DONE); sqlite3_reset(m->storeStructPropertyStmt); sqlite3_clear_bindings(m->storeStructPropertyStmt); } for (auto& method : decl.memberFunctions) { // TODO } } void CodegenModelArchive::StoreFunction(const DeclFunction& decl) { // TODO } void CodegenModelArchive::StoreEnum(const DeclEnum& decl) { // -Argument- -Description- // ?1 Namespace ID // ?2 Enum name // ?3 Enum underlying type // ?4 File containing the enum // TODO this is breaking sqlite: cannot find const.fileId m->storeEnumStmt.InitializeLazily(m->database, R"""( INSERT OR IGNORE INTO Files(FileName) VALUES (?4); WITH const AS (SELECT Id AS fileId FROM Files WHERE FileName = ?4) INSERT INTO DeclEnums(NamespaceId, Name, UnderlyingType, FileId) VALUES (?1, ?2, ?3, const.fileId) ON CONFLICT DO UPDATE SET UnderlyingType = ?3, FileId = const.fileId RETURNING Id )"""sv); // -Argument- -Description- // ?1 Container enum's id // ?2 Enum element name // ?3 Enum element value m->storeEnumElmStmt.InitializeLazily(m->database, R"""( INSERT INTO DeclEnumElements(EnumId, Name, Value) VALUES (?1, ?2, ?3) ON CONFLICT DO UPDATE SET Value=?3 )"""sv); int result; sqlite3_bind_int(m->storeEnumStmt, 1, m->FindNamespace(decl.container)); sqlite3_bind_text(m->storeEnumStmt, 2, decl.name.c_str(), decl.name.size(), nullptr); sqlite3_bind_text(m->storeEnumStmt, 3, decl.underlyingTypeStr.c_str(), decl.underlyingTypeStr.size(), nullptr); if (decl.sourceFile) { sqlite3_bind_text(m->storeEnumStmt, 4, decl.sourceFile->filename.data(), decl.sourceFile->filename.size(), nullptr); } else { sqlite3_bind_text(m->storeEnumStmt, 4, "", 0, nullptr); } result = sqlite3_step(m->storeEnumStmt); result = sqlite3_step(m->storeEnumStmt); assert(result == SQLITE_ROW); auto enumId = sqlite3_column_int64(m->storeEnumStmt, 0); sqlite3_reset(m->storeEnumStmt); sqlite3_clear_bindings(m->storeEnumStmt); for (auto& elm : decl.elements) { sqlite3_bind_int64(m->storeEnumElmStmt, 1, enumId); sqlite3_bind_text(m->storeEnumElmStmt, 2, elm.name.c_str(), elm.name.size(), nullptr); sqlite3_bind_int64(m->storeEnumElmStmt, 3, elm.value); int result = sqlite3_step(m->storeEnumElmStmt); assert(result == SQLITE_DONE); sqlite3_reset(m->storeEnumElmStmt); sqlite3_clear_bindings(m->storeEnumElmStmt); } }