From a417c79a08feeabe555d4ab9e213294599557efc Mon Sep 17 00:00:00 2001 From: rtk0c Date: Fri, 8 Jul 2022 17:10:23 -0700 Subject: Changeset: 82 Fix compiling the main game with codegen, add Files table to replace embedding filename in each decl table directly --- source/20-codegen-compiler/CodegenModel.cpp | 214 ++++++++++++++++------------ source/20-codegen-compiler/main.cpp | 13 +- 2 files changed, 132 insertions(+), 95 deletions(-) (limited to 'source') diff --git a/source/20-codegen-compiler/CodegenModel.cpp b/source/20-codegen-compiler/CodegenModel.cpp index b466d6e..0017b62 100644 --- a/source/20-codegen-compiler/CodegenModel.cpp +++ b/source/20-codegen-compiler/CodegenModel.cpp @@ -32,6 +32,7 @@ public: // A number for `PRAGMA user_vesrion`, representing the current database version. Increment when the table format changes. #define CURRENT_DATABASE_VERSION 1 constexpr int64_t kGlobalNamespaceId = 1; +constexpr int64_t kNoFileId = 1; struct SQLiteDatabase { sqlite3* database = nullptr; @@ -64,14 +65,14 @@ struct SQLiteStatement { operator sqlite3_stmt*() const { return stmt; } sqlite3_stmt** operator&() { return &stmt; } + void Initialize(sqlite3* database, std::string_view sql) { + int result = sqlite3_prepare(database, sql.data(), sql.size(), &stmt, nullptr); + assert(result == SQLITE_OK); + } + bool InitializeLazily(sqlite3* database, std::string_view sql) { if (!stmt) { - int result = sqlite3_prepare(database, sql.data(), sql.size(), &stmt, nullptr); - if (result != SQLITE_OK) { - INPLACE_FMT(msg, "Failed to prepare statement, error message: %s", sqlite3_errmsg(database)); - throw std::runtime_error(msg); - } - + Initialize(database, sql); return true; } return false; @@ -97,9 +98,11 @@ public: SQLiteStatement beginTransactionStmt; SQLiteStatement commitTransactionStmt; SQLiteStatement rollbackTransactionStmt; - /* Component Statements, initalized on demand */ + SQLiteStatement storeFileStmt; + SQLiteStatement findFileStmt; SQLiteStatement findNamespaceStmt; - SQLiteStatement findOrStoreNamespaceStmt; + SQLiteStatement storeNamespaceStmt; + /* Component Statements, initalized on demand */ SQLiteStatement storeStructStmt; SQLiteStatement storeStructBaseClassStmt; SQLiteStatement storeStructPropertyStmt; @@ -137,9 +140,9 @@ CREATE TABLE Namespaces( CREATE TABLE DeclFunctions( Id INTEGER PRIMARY KEY, + FileId INTEGER REFERENCES Files(Id), NamespaceId INTEGER REFERENCES Namespaces(Id), - Name TEXT, - FileId INTEGER REFERENCES Files(Id) + Name TEXT ); CREATE TABLE DeclFunctionParameters( FunctionId INTEGER REFERENCES DeclFunctions(Id) ON DELETE CASCADE, @@ -150,9 +153,9 @@ CREATE TABLE DeclFunctionParameters( CREATE TABLE DeclStructs( Id INTEGER PRIMARY KEY, + FileId INTEGER REFERENCES Files(Id), NamespaceId INTEGER REFERENCES Namespaces(Id), Name TEXT, - FileId INTEGER REFERENCES Files(Id), IsMetadataMarked INTEGER ); CREATE TABLE DeclStructBaseClassRelations( @@ -190,10 +193,10 @@ CREATE TABLE DeclStructMethodParameters( CREATE TABLE DeclEnums( Id INTEGER PRIMARY KEY, + FileId INTEGER REFERENCES Files(Id), NamespaceId INTEGER REFERENCES Namespaces(Id), Name TEXT, - UnderlyingType TEXT, - FileId INTEGER REFERENCES Files(Id) + UnderlyingType TEXT ); CREATE TABLE DeclEnumElements( EnumId INTEGER REFERENCES DeclEnums(Id) ON DELETE CASCADE, @@ -214,10 +217,12 @@ CREATE UNIQUE INDEX Index_DeclStructMethods_Identity ON DeclStructMethods(Struct CREATE UNIQUE INDEX Index_DeclEnums_Identity ON DeclEnums(NamespaceId, Name); +INSERT INTO Files(Id, FileName) +VALUES (1, ''); + +-- Special global namespace that has no parent, and Id should always be 1 INSERT INTO Namespaces(Id, ParentNamespaceId, Name) -VALUES - -- Special global namespace that has no parent, and Id should always be 1 - (1, NULL, ""); +VALUES (1, NULL, ''); COMMIT TRANSACTION; )""", @@ -252,7 +257,6 @@ COMMIT TRANSACTION; return kGlobalNamespaceId; } - InitializeStmt_findNamespaceStmt(); return FindNamespaceImpl(*ns); } @@ -262,26 +266,74 @@ COMMIT TRANSACTION; return kGlobalNamespaceId; } - InitializeStmt_findNamespaceStmt(); - InitializeStmt_findOrStoreNamespaceStmt(); - return FindOrStoreNamespaceImpl(*ns); + if (auto rowId = FindNamespaceImpl(*ns); rowId != 0) { + return rowId; + } + + sqlite3_stmt* stmt = storeNamespaceStmt; + int64_t parentRowId = FindOrStoreNamespace(ns->container); + sqlite3_bind_int64(stmt, 1, parentRowId); + sqlite3_bind_text(stmt, 2, ns->name.c_str(), ns->name.size(), nullptr); + + int result = sqlite3_step(stmt); + DEFER { + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + }; + assert(result == SQLITE_ROW); + + auto nsId = sqlite3_column_int64(stmt, 0); + return nsId; } -private: - void InitializeStmt_findNamespaceStmt() { - findNamespaceStmt.InitializeLazily(database, R"""( -SELECT Id FROM Namespaces WHERE ParentNamespaceId = ?1 AND Name = ?2; -)"""sv); + /// \return Row ID of the file, or 0 if it currently doesn't exist. + int64_t FindFile(std::string_view filename) { + sqlite3_stmt* stmt = findFileStmt; + sqlite3_bind_text(stmt, 1, filename.data(), filename.size(), nullptr); + + int result = sqlite3_step(stmt); + DEFER { + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + }; + + if (result == SQLITE_ROW) { + auto fileId = sqlite3_column_int64(stmt, 0); + return fileId; + } else { + return 0; + } } - void InitializeStmt_findOrStoreNamespaceStmt() { - findOrStoreNamespaceStmt.InitializeLazily(database, R"""( -INSERT INTO Namespaces(ParentNamespaceId, Name) - VALUES (?1, ?2) - RETURNING Id; -)"""sv); + /// \return Row ID of the file + int64_t FindOrStoreFile(std::string_view filename) { + if (auto id = FindFile(filename); id != 0) { + return id; + } + + sqlite3_stmt* stmt = storeFileStmt; + sqlite3_bind_text(stmt, 1, filename.data(), filename.size(), nullptr); + + int result = sqlite3_step(stmt); + DEFER { + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + }; + assert(result == SQLITE_ROW); + + auto fileId = sqlite3_column_int64(stmt, 0); + return fileId; } + /// \return Row ID of the file + int64_t FindOrStoreFile(/*nullable*/ const SourceFile* file) { + if (!file) { + return kNoFileId; + } + return FindOrStoreFile(file->filename); + } + +private: int64_t FindNamespaceImpl(const DeclNamespace& ns) { int64_t parentNsRowId; if (ns.container) { @@ -301,36 +353,20 @@ INSERT INTO Namespaces(ParentNamespaceId, Name) sqlite3_stmt* stmt = findNamespaceStmt; sqlite3_bind_int64(stmt, 1, parentNsRowId); sqlite3_bind_text(stmt, 2, ns.name.c_str(), ns.name.size(), nullptr); + int result = sqlite3_step(stmt); - if (result == SQLITE_ROW) { - int64_t rowId = sqlite3_column_int64(stmt, 0); + DEFER { sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + }; - return rowId; + if (result == SQLITE_ROW) { + int64_t nsId = sqlite3_column_int64(stmt, 0); + return nsId; } else { return 0; } } - - int64_t FindOrStoreNamespaceImpl(const DeclNamespace& ns) { - if (auto rowId = FindNamespaceImpl(ns); rowId != 0) { - return rowId; - } - - sqlite3_stmt* stmt = findOrStoreNamespaceStmt; - int64_t parentRowId = ns.container - ? FindOrStoreNamespaceImpl(*ns.container) - : kGlobalNamespaceId; - sqlite3_bind_int64(stmt, 1, parentRowId); - sqlite3_bind_text(stmt, 2, ns.name.c_str(), ns.name.size(), nullptr); - int result = sqlite3_step(stmt); - assert(result == SQLITE_ROW); - - auto rowId = sqlite3_column_int64(stmt, 0); - sqlite3_reset(stmt); - - return rowId; - } }; CodegenModel::CodegenModel() @@ -456,9 +492,13 @@ CodegenModelArchive::CodegenModelArchive(std::string_view dbPath) } // Initialize core statements - m->beginTransactionStmt.InitializeLazily(m->database, "BEGIN TRANSACTION"); - m->commitTransactionStmt.InitializeLazily(m->database, "COMMIT TRANSACTION"); - m->rollbackTransactionStmt.InitializeLazily(m->database, "ROLLBACK TRANSACTION"); + m->beginTransactionStmt.Initialize(m->database, "BEGIN TRANSACTION"); + m->commitTransactionStmt.Initialize(m->database, "COMMIT TRANSACTION"); + m->rollbackTransactionStmt.Initialize(m->database, "ROLLBACK TRANSACTION"); + m->findFileStmt.Initialize(m->database, "SELECT Id FROM Files WHERE FileName = ?1"); + m->storeFileStmt.Initialize(m->database, "INSERT INTO Files(FileName) VALUES (?1) RETURNING Id"); + m->findNamespaceStmt.Initialize(m->database, "SELECT Id FROM Namespaces WHERE ParentNamespaceId = ?1 AND Name = ?2"); + m->storeNamespaceStmt.Initialize(m->database, "INSERT INTO Namespaces(ParentNamespaceId, Name) VALUES (?1, ?2) RETURNING Id"); } CodegenModelArchive::~CodegenModelArchive() { @@ -508,6 +548,9 @@ void CodegenModelArchive::Store(const CodegenModel& cgInput) { m->BeginTransaction(); + // TODO why are some namepsaces not found in StoreStruct or StoreEnum calls, if we don't use FindOrStoreNamespace there? + // since we store all currently scanned namespaces into the model here, does that mean the parser is not correctly picking some of those up? + // but if the parser is failing, than the DeclThing objects should have a null `container` field? for (auto&& [DISCARD, ns] : cgm.namespaces) { // This will insert the namespace if it doesn't exist, or no-op (fetches data) if it already exists m->FindOrStoreNamespace(&ns); @@ -523,17 +566,14 @@ void CodegenModelArchive::StoreStruct(const DeclStruct& decl) { // -Argument- -Description- // ?1 Namespace ID // ?2 Struct name - // ?3 File containing the struct + // ?3 File ID containing the struct + // ?4 Is this struct marked for metadata generation? m->storeStructStmt.InitializeLazily(m->database, R"""( -INSERT OR IGNORE INTO Files(FileName) -VALUES (?3); - -WITH - const AS (SELECT Id AS fileId FROM Files WHERE FileName = ?3) INSERT INTO DeclStructs(NamespaceId, Name, FileId, IsMetadataMarked) -VALUES (?1, ?2, const.fileId, TRUE) +VALUES (?1, ?2, ?3, ?4) ON CONFLICT DO UPDATE SET - FileId = const.fileId + FileId = ?3, + IsMetadataMarked = ?4 RETURNING Id )"""sv); @@ -552,21 +592,18 @@ VALUES (?1, ?2, ?3) // ?3 Property type // ?4 Getter name (optional) // ?5 Setter name (optional) + // ?6 Is this property marked for metadata generation? m->storeStructPropertyStmt.InitializeLazily(m->database, R"""( INSERT INTO DeclStructProperties(StructId, Name, Type, GetterName, SetterName, IsMetadataMarked) -VALUES (?1, ?2, ?3, ?4, ?5, TRUE) +VALUES (?1, ?2, ?3, ?4, ?5, ?6) )"""sv); int result; - sqlite3_bind_int64(m->storeStructStmt, 1, m->FindNamespace(decl.container)); + sqlite3_bind_int64(m->storeStructStmt, 1, m->FindOrStoreNamespace(decl.container)); sqlite3_bind_text(m->storeStructStmt, 2, decl.name.c_str(), decl.name.size(), nullptr); - if (decl.sourceFile) { - sqlite3_bind_text(m->storeStructStmt, 3, decl.sourceFile->filename.data(), decl.sourceFile->filename.size(), nullptr); - } else { - sqlite3_bind_text(m->storeStructStmt, 3, "", 0, nullptr); - } - result = sqlite3_step(m->storeStructStmt); + sqlite3_bind_int64(m->storeStructStmt, 3, m->FindOrStoreFile(decl.sourceFile)); + sqlite3_bind_int(m->storeStructStmt, 4, decl.generating); result = sqlite3_step(m->storeStructStmt); assert(result == SQLITE_ROW); int64_t structId = sqlite3_column_int64(m->storeStructStmt, 0); @@ -575,7 +612,7 @@ VALUES (?1, ?2, ?3, ?4, ?5, TRUE) for (auto& baseClass : decl.baseClasses) { sqlite3_bind_int64(m->storeStructBaseClassStmt, 1, structId); - sqlite3_bind_int64(m->storeStructBaseClassStmt, 2, m->FindNamespace(baseClass->container)); + sqlite3_bind_int64(m->storeStructBaseClassStmt, 2, m->FindOrStoreNamespace(baseClass->container)); sqlite3_bind_text(m->storeStructBaseClassStmt, 3, baseClass->name.c_str(), baseClass->name.size(), nullptr); int result = sqlite3_step(m->storeStructBaseClassStmt); assert(result == SQLITE_DONE); @@ -589,6 +626,8 @@ VALUES (?1, ?2, ?3, ?4, ?5, TRUE) sqlite3_bind_text(m->storeStructPropertyStmt, 3, property.type.c_str(), property.type.size(), nullptr); sqlite3_bind_text(m->storeStructPropertyStmt, 4, property.getterName.c_str(), property.getterName.size(), nullptr); sqlite3_bind_text(m->storeStructPropertyStmt, 5, property.setterName.c_str(), property.setterName.size(), nullptr); + // Since DeclMemberVariable entries currently only exist if it's marked BRUSSEL_PROPERTY + sqlite3_bind_int(m->storeStructPropertyStmt, 6, true); int result = sqlite3_step(m->storeStructPropertyStmt); assert(result == SQLITE_DONE); sqlite3_reset(m->storeStructPropertyStmt); @@ -609,19 +648,13 @@ void CodegenModelArchive::StoreEnum(const DeclEnum& decl) { // ?1 Namespace ID // ?2 Enum name // ?3 Enum underlying type - // ?4 File containing the enum - // TODO this is breaking sqlite: cannot find const.fileId + // ?4 File ID containing the enum m->storeEnumStmt.InitializeLazily(m->database, R"""( -INSERT OR IGNORE INTO Files(FileName) -VALUES (?4); - -WITH - const AS (SELECT Id AS fileId FROM Files WHERE FileName = ?4) INSERT INTO DeclEnums(NamespaceId, Name, UnderlyingType, FileId) -VALUES (?1, ?2, ?3, const.fileId) +VALUES (?1, ?2, ?3, ?4) ON CONFLICT DO UPDATE SET UnderlyingType = ?3, - FileId = const.fileId + FileId = ?4 RETURNING Id )"""sv); @@ -635,19 +668,12 @@ VALUES (?1, ?2, ?3) ON CONFLICT DO UPDATE SET Value=?3 )"""sv); - int result; - auto eutName = decl.GetUnderlyingTypeName(); - - sqlite3_bind_int(m->storeEnumStmt, 1, m->FindNamespace(decl.container)); + sqlite3_bind_int(m->storeEnumStmt, 1, m->FindOrStoreNamespace(decl.container)); sqlite3_bind_text(m->storeEnumStmt, 2, decl.name.c_str(), decl.name.size(), nullptr); + auto eutName = decl.GetUnderlyingTypeName(); sqlite3_bind_text(m->storeEnumStmt, 3, eutName.data(), eutName.size(), nullptr); - if (decl.sourceFile) { - sqlite3_bind_text(m->storeEnumStmt, 4, decl.sourceFile->filename.data(), decl.sourceFile->filename.size(), nullptr); - } else { - sqlite3_bind_text(m->storeEnumStmt, 4, "", 0, nullptr); - } - result = sqlite3_step(m->storeEnumStmt); - result = sqlite3_step(m->storeEnumStmt); + sqlite3_bind_int64(m->storeEnumStmt, 4, m->FindOrStoreFile(decl.sourceFile)); + int result = sqlite3_step(m->storeEnumStmt); assert(result == SQLITE_ROW); auto enumId = sqlite3_column_int64(m->storeEnumStmt, 0); sqlite3_reset(m->storeEnumStmt); diff --git a/source/20-codegen-compiler/main.cpp b/source/20-codegen-compiler/main.cpp index 348d60e..dcec0d3 100644 --- a/source/20-codegen-compiler/main.cpp +++ b/source/20-codegen-compiler/main.cpp @@ -774,6 +774,17 @@ void ParseInputFileAndGenerate(AppState& as, CodegenLexer& /*lexingState*/ ls, s ++idx; incrementTokenIdx = false; + // For forward declarations, there are always 2 tokens after `class`: an identifier, and the ';' token + // Example: + // class MyClass; + if (tokens[idx + 0].type == CLEX_id && + tokens[idx + 1].text == ";") + { + // Skip class forward declarations + idx += 2; + break; + } + auto& idenTok = tokens[idx]; if (idenTok.type != CLEX_id) { printf("[ERROR] invalid syntax for struct or class\n"); @@ -1337,7 +1348,7 @@ where --output-dir=: the *directory* to write generated contents to. Thi DEBUG_PRINTF("Outputting to directory %.*s.\n", PRINTF_STRING_VIEW(as.outputDir)); DEBUG_PRINTF("Databse file: %.*s.\n", PRINTF_STRING_VIEW(as.databaseFilePath)); -#define BRUSSEL_ENABLE_CODEGEN_ARCHIVE 0 +#define BRUSSEL_ENABLE_CODEGEN_ARCHIVE 1 #if BRUSSEL_ENABLE_CODEGEN_ARCHIVE CodegenModelArchive archive(as.databaseFilePath); as.modelArchive = &archive; -- cgit v1.2.3-70-g09d2