From bd93d70925f00f6036ea66226b612f3ee52127c7 Mon Sep 17 00:00:00 2001 From: Vinayak Iyer Date: Mon, 21 Apr 2025 11:49:06 +0530 Subject: [PATCH] (feat): pg and env --- .env.example | 10 ++ .gitignore | 1 + Dockerfile | 45 +++++ docker-compose.yml | 19 ++ engine/database/database.cc | 47 +++++ engine/database/database.h | 28 ++- engine/database/engines.cc | 349 ++++++++++++++++++++++-------------- engine/database/engines.h | 69 ++----- engine/utils/env_utils.cc | 53 ++++++ engine/utils/env_utils.h | 14 ++ 10 files changed, 440 insertions(+), 195 deletions(-) create mode 100644 .env.example create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 engine/database/database.cc create mode 100644 engine/utils/env_utils.cc create mode 100644 engine/utils/env_utils.h diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..864af76ee --- /dev/null +++ b/.env.example @@ -0,0 +1,10 @@ +# Database Configuration +DB_TYPE=postgresql +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_DB=cortex +POSTGRES_USER=cortex +POSTGRES_PASSWORD=cortex + +# Application Configuration +MODELS_DIR=/app/models \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8f10ea41e..8ff120bd1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ dist node_modules .turbo package-lock.json +.env # CI - Test - Coverage cortex.log diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..a8537b981 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +FROM ubuntu:22.04 + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + git \ + curl \ + tar \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /app + +# Copy source code +COPY . . + +# Setup vcpkg +RUN cd vcpkg && \ + ./bootstrap-vcpkg.sh && \ + ./vcpkg install + +# Create build directory and build +RUN mkdir -p build && cd build && \ + cmake .. -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake && \ + make -j$(nproc) + +# Create directory for models +RUN mkdir -p /app/models + +# Set environment variables +ENV DB_TYPE=postgresql +ENV POSTGRES_HOST=postgres +ENV POSTGRES_PORT=5432 +ENV POSTGRES_DB=cortex +ENV POSTGRES_USER=cortex +ENV POSTGRES_PASSWORD=cortex +ENV PORT=3432 + +# Expose port if needed +EXPOSE ${PORT} + +# Command to run the application +CMD ["./build/cortex"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..745e817ac --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: '3.8' + +services: + app: + build: . + ports: + - "8080:8080" + volumes: + - ./models:/app/models + - ./.env:/app/.env + env_file: + - .env + networks: + - cortex-network + + +networks: + cortex-network: + driver: bridge \ No newline at end of file diff --git a/engine/database/database.cc b/engine/database/database.cc new file mode 100644 index 000000000..01e4bc7dc --- /dev/null +++ b/engine/database/database.cc @@ -0,0 +1,47 @@ +#include "database.h" +#include "utils/env_utils.h" +#include + +namespace cortex::db { + +Database::Database() : type_(DatabaseType::SQLite), pg_conn_(nullptr) { + // Check if PostgreSQL is configured in environment + if (auto db_type = cortex::utils::EnvUtils::GetEnv("DB_TYPE")) { + if (*db_type == "postgresql") { + Initialize(DatabaseType::PostgreSQL, cortex::utils::EnvUtils::GetPostgresConnectionString()); + return; + } + } + // Default to SQLite + Initialize(DatabaseType::SQLite); +} + +Database::~Database() { + if (pg_conn_) { + PQfinish(pg_conn_); + } +} + +void Database::Initialize(DatabaseType type, const std::string& connection_string) { + type_ = type; + + if (type == DatabaseType::SQLite) { + sqlite_db_ = std::make_unique( + file_manager_utils::GetCortexDataPath() / "cortex.db", + SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE); + } else if (type == DatabaseType::PostgreSQL) { + if (connection_string.empty()) { + throw std::runtime_error("PostgreSQL connection string is required"); + } + + pg_conn_ = PQconnectdb(connection_string.c_str()); + if (PQstatus(pg_conn_) != CONNECTION_OK) { + std::string error = PQerrorMessage(pg_conn_); + PQfinish(pg_conn_); + pg_conn_ = nullptr; + throw std::runtime_error("Failed to connect to PostgreSQL: " + error); + } + } +} + +} // namespace cortex::db \ No newline at end of file diff --git a/engine/database/database.h b/engine/database/database.h index dbe58cc4b..f94c923c7 100644 --- a/engine/database/database.h +++ b/engine/database/database.h @@ -1,27 +1,43 @@ #pragma once #include +#include #include "SQLiteCpp/SQLiteCpp.h" #include "utils/file_manager_utils.h" +#include namespace cortex::db { + +enum class DatabaseType { + SQLite, + PostgreSQL +}; + class Database { public: Database(Database const&) = delete; Database& operator=(Database const&) = delete; - ~Database() {} + ~Database(); static Database& GetInstance() { static Database db; return db; } - SQLite::Database& db() { return db_; } + // Initialize database with configuration + void Initialize(DatabaseType type, const std::string& connection_string = ""); + + // Get database connection based on type + SQLite::Database& GetSQLite() { return *sqlite_db_; } + PGconn* GetPostgreSQL() { return pg_conn_; } + + DatabaseType GetType() const { return type_; } private: - Database() - : db_(file_manager_utils::GetCortexDataPath() / "cortex.db", - SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE) {} - SQLite::Database db_; + Database(); + + DatabaseType type_; + std::unique_ptr sqlite_db_; + PGconn* pg_conn_; }; } // namespace cortex::db diff --git a/engine/database/engines.cc b/engine/database/engines.cc index a4d13ef79..37c5bc83c 100644 --- a/engine/database/engines.cc +++ b/engine/database/engines.cc @@ -1,17 +1,61 @@ #include "engines.h" #include #include "database.h" +#include namespace cortex::db { -void CreateTable(SQLite::Database& db) {} +void Engines::CreateTable() { + if (Database::GetInstance().GetType() == DatabaseType::SQLite) { + db_.exec(R"( + CREATE TABLE IF NOT EXISTS engines ( + engine_name TEXT NOT NULL, + type TEXT NOT NULL, + api_key TEXT, + url TEXT, + version TEXT, + variant TEXT NOT NULL, + status TEXT, + metadata TEXT, + PRIMARY KEY (engine_name, variant) + ) + )"); + } +} -Engines::Engines() : db_(cortex::db::Database::GetInstance().db()) { - CreateTable(db_); +void Engines::CreatePostgreSQLTable() { + if (Database::GetInstance().GetType() == DatabaseType::PostgreSQL) { + pg_conn_ = Database::GetInstance().GetPostgreSQL(); + const char* query = R"( + CREATE TABLE IF NOT EXISTS engines ( + engine_name TEXT NOT NULL, + type TEXT NOT NULL, + api_key TEXT, + url TEXT, + version TEXT, + variant TEXT NOT NULL, + status TEXT, + metadata TEXT, + PRIMARY KEY (engine_name, variant) + ) + )"; + + PGresult* res = PQexec(pg_conn_, query); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + PQclear(res); + throw std::runtime_error("Failed to create PostgreSQL table: " + std::string(PQerrorMessage(pg_conn_))); + } + PQclear(res); + } +} + +Engines::Engines() : db_(Database::GetInstance().GetSQLite()) { + CreateTable(); + CreatePostgreSQLTable(); } Engines::Engines(SQLite::Database& db) : db_(db) { - CreateTable(db_); + CreateTable(); } Engines::~Engines() {} @@ -21,152 +65,191 @@ std::optional Engines::UpsertEngine( const std::string& api_key, const std::string& url, const std::string& version, const std::string& variant, const std::string& status, const std::string& metadata) { - try { - SQLite::Statement query( - db_, - "INSERT INTO engines (engine_name, type, api_key, url, version, " - "variant, status, metadata) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " - "ON CONFLICT(engine_name, variant) DO UPDATE SET " - "type = excluded.type, " - "api_key = excluded.api_key, " - "url = excluded.url, " - "version = excluded.version, " - "status = excluded.status, " - "metadata = excluded.metadata, " - "date_updated = CURRENT_TIMESTAMP " - "RETURNING id, engine_name, type, api_key, url, version, variant, " - "status, metadata, date_created, date_updated;"); - - query.bind(1, engine_name); - query.bind(2, type); - query.bind(3, api_key); - query.bind(4, url); - query.bind(5, version); - query.bind(6, variant); - query.bind(7, status); - query.bind(8, metadata); - - if (query.executeStep()) { - return EngineEntry{ - query.getColumn(0).getInt(), query.getColumn(1).getString(), - query.getColumn(2).getString(), query.getColumn(3).getString(), - query.getColumn(4).getString(), query.getColumn(5).getString(), - query.getColumn(6).getString(), query.getColumn(7).getString(), - query.getColumn(8).getString(), query.getColumn(9).getString(), - query.getColumn(10).getString()}; - } else { + if (Database::GetInstance().GetType() == DatabaseType::SQLite) { + try { + SQLite::Statement query( + db_, + "INSERT INTO engines (engine_name, type, api_key, url, version, " + "variant, status, metadata) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT(engine_name, variant) DO UPDATE SET " + "type = excluded.type, " + "api_key = excluded.api_key, " + "url = excluded.url, " + "version = excluded.version, " + "status = excluded.status, " + "metadata = excluded.metadata"); + + query.bind(1, engine_name); + query.bind(2, type); + query.bind(3, api_key); + query.bind(4, url); + query.bind(5, version); + query.bind(6, variant); + query.bind(7, status); + query.bind(8, metadata); + + query.exec(); + + return GetEngine(engine_name, variant); + } catch (const std::exception& e) { return std::nullopt; } - } catch (const std::exception& e) { - return std::nullopt; - } -} - -std::optional> Engines::GetEngines() const { - try { - SQLite::Statement query( - db_, - "SELECT id, engine_name, type, api_key, url, version, variant, status, " - "metadata, date_created, date_updated " - "FROM engines " - "WHERE status = 'Default' " - "ORDER BY date_updated DESC"); - - std::vector engines; - while (query.executeStep()) { - engines.push_back(EngineEntry{ - query.getColumn(0).getInt(), query.getColumn(1).getString(), - query.getColumn(2).getString(), query.getColumn(3).getString(), - query.getColumn(4).getString(), query.getColumn(5).getString(), - query.getColumn(6).getString(), query.getColumn(7).getString(), - query.getColumn(8).getString(), query.getColumn(9).getString(), - query.getColumn(10).getString()}); + } else { + // PostgreSQL implementation + const char* query = "INSERT INTO engines (engine_name, type, api_key, url, version, variant, status, metadata) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8) " + "ON CONFLICT (engine_name, variant) DO UPDATE SET " + "type = EXCLUDED.type, " + "api_key = EXCLUDED.api_key, " + "url = EXCLUDED.url, " + "version = EXCLUDED.version, " + "status = EXCLUDED.status, " + "metadata = EXCLUDED.metadata"; + + const char* values[8] = { + engine_name.c_str(), + type.c_str(), + api_key.c_str(), + url.c_str(), + version.c_str(), + variant.c_str(), + status.c_str(), + metadata.c_str() + }; + + PGresult* res = PQexecParams(pg_conn_, query, 8, nullptr, values, nullptr, nullptr, 0); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + PQclear(res); + return std::nullopt; } - - return engines; - } catch (const std::exception& e) { - return std::nullopt; + PQclear(res); + + return GetEngine(engine_name, variant); } } -std::optional Engines::GetEngineById(int id) const { - try { - SQLite::Statement query( - db_, - "SELECT id, engine_name, type, api_key, url, version, variant, status, " - "metadata, date_created, date_updated " - "FROM engines " - "WHERE id = ? AND status = 'Default' " - "ORDER BY date_updated DESC LIMIT 1"); - - query.bind(1, id); - - if (query.executeStep()) { - return EngineEntry{ - query.getColumn(0).getInt(), query.getColumn(1).getString(), - query.getColumn(2).getString(), query.getColumn(3).getString(), - query.getColumn(4).getString(), query.getColumn(5).getString(), - query.getColumn(6).getString(), query.getColumn(7).getString(), - query.getColumn(8).getString(), query.getColumn(9).getString(), - query.getColumn(10).getString()}; - } else { +std::optional Engines::GetEngine(const std::string& engine_name, + const std::string& variant) { + if (Database::GetInstance().GetType() == DatabaseType::SQLite) { + try { + SQLite::Statement query(db_, + "SELECT * FROM engines WHERE engine_name = ? AND variant = ?"); + query.bind(1, engine_name); + query.bind(2, variant); + + if (query.executeStep()) { + EngineEntry entry; + entry.engine_name = query.getColumn(0).getString(); + entry.type = query.getColumn(1).getString(); + entry.api_key = query.getColumn(2).getString(); + entry.url = query.getColumn(3).getString(); + entry.version = query.getColumn(4).getString(); + entry.variant = query.getColumn(5).getString(); + entry.status = query.getColumn(6).getString(); + entry.metadata = query.getColumn(7).getString(); + return entry; + } + } catch (const std::exception& e) { + return std::nullopt; + } + } else { + // PostgreSQL implementation + const char* query = "SELECT * FROM engines WHERE engine_name = $1 AND variant = $2"; + const char* values[2] = {engine_name.c_str(), variant.c_str()}; + + PGresult* res = PQexecParams(pg_conn_, query, 2, nullptr, values, nullptr, nullptr, 0); + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + PQclear(res); return std::nullopt; } - } catch (const std::exception& e) { - return std::nullopt; + + if (PQntuples(res) > 0) { + EngineEntry entry; + entry.engine_name = PQgetvalue(res, 0, 0); + entry.type = PQgetvalue(res, 0, 1); + entry.api_key = PQgetvalue(res, 0, 2); + entry.url = PQgetvalue(res, 0, 3); + entry.version = PQgetvalue(res, 0, 4); + entry.variant = PQgetvalue(res, 0, 5); + entry.status = PQgetvalue(res, 0, 6); + entry.metadata = PQgetvalue(res, 0, 7); + PQclear(res); + return entry; + } + PQclear(res); } + return std::nullopt; } -std::optional Engines::GetEngineByNameAndVariant( - const std::string& engine_name, - const std::optional variant) const { - try { - std::string queryStr = - "SELECT id, engine_name, type, api_key, url, version, variant, status, " - "metadata, date_created, date_updated " - "FROM engines " - "WHERE engine_name = ? AND status = 'Default' "; - - if (variant) { - queryStr += "AND variant = ? "; +std::vector Engines::GetAllEngines() { + std::vector entries; + + if (Database::GetInstance().GetType() == DatabaseType::SQLite) { + try { + SQLite::Statement query(db_, "SELECT * FROM engines"); + while (query.executeStep()) { + EngineEntry entry; + entry.engine_name = query.getColumn(0).getString(); + entry.type = query.getColumn(1).getString(); + entry.api_key = query.getColumn(2).getString(); + entry.url = query.getColumn(3).getString(); + entry.version = query.getColumn(4).getString(); + entry.variant = query.getColumn(5).getString(); + entry.status = query.getColumn(6).getString(); + entry.metadata = query.getColumn(7).getString(); + entries.push_back(entry); + } + } catch (const std::exception& e) { + return entries; } - - queryStr += "ORDER BY date_updated DESC LIMIT 1"; - - SQLite::Statement query(db_, queryStr); - - query.bind(1, engine_name); - - if (variant) { - query.bind(2, variant.value()); + } else { + // PostgreSQL implementation + PGresult* res = PQexec(pg_conn_, "SELECT * FROM engines"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + PQclear(res); + return entries; } - - if (query.executeStep()) { - return EngineEntry{ - query.getColumn(0).getInt(), query.getColumn(1).getString(), - query.getColumn(2).getString(), query.getColumn(3).getString(), - query.getColumn(4).getString(), query.getColumn(5).getString(), - query.getColumn(6).getString(), query.getColumn(7).getString(), - query.getColumn(8).getString(), query.getColumn(9).getString(), - query.getColumn(10).getString()}; - } else { - return std::nullopt; + + int rows = PQntuples(res); + for (int i = 0; i < rows; i++) { + EngineEntry entry; + entry.engine_name = PQgetvalue(res, i, 0); + entry.type = PQgetvalue(res, i, 1); + entry.api_key = PQgetvalue(res, i, 2); + entry.url = PQgetvalue(res, i, 3); + entry.version = PQgetvalue(res, i, 4); + entry.variant = PQgetvalue(res, i, 5); + entry.status = PQgetvalue(res, i, 6); + entry.metadata = PQgetvalue(res, i, 7); + entries.push_back(entry); } - } catch (const std::exception& e) { - return std::nullopt; + PQclear(res); } + + return entries; } -std::optional Engines::DeleteEngineById(int id) { - try { - SQLite::Statement query(db_, "DELETE FROM engines WHERE id = ?"); - - query.bind(1, id); - query.exec(); - return std::nullopt; - } catch (const std::exception& e) { - return std::string("Failed to delete engine: ") + e.what(); +bool Engines::DeleteEngine(const std::string& engine_name, const std::string& variant) { + if (Database::GetInstance().GetType() == DatabaseType::SQLite) { + try { + SQLite::Statement query(db_, + "DELETE FROM engines WHERE engine_name = ? AND variant = ?"); + query.bind(1, engine_name); + query.bind(2, variant); + return query.exec() > 0; + } catch (const std::exception& e) { + return false; + } + } else { + // PostgreSQL implementation + const char* query = "DELETE FROM engines WHERE engine_name = $1 AND variant = $2"; + const char* values[2] = {engine_name.c_str(), variant.c_str()}; + + PGresult* res = PQexecParams(pg_conn_, query, 2, nullptr, values, nullptr, nullptr, 0); + bool success = (PQresultStatus(res) == PGRES_COMMAND_OK && PQcmdTuples(res)[0] != '0'); + PQclear(res); + return success; } } diff --git a/engine/database/engines.h b/engine/database/engines.h index 1312a9c67..780b21e05 100644 --- a/engine/database/engines.h +++ b/engine/database/engines.h @@ -1,16 +1,13 @@ #pragma once -#include -#include -#include -#include #include +#include #include +#include "database.h" namespace cortex::db { struct EngineEntry { - int id; std::string engine_name; std::string type; std::string api_key; @@ -19,55 +16,12 @@ struct EngineEntry { std::string variant; std::string status; std::string metadata; - std::string date_created; - std::string date_updated; - Json::Value ToJson() const { - Json::Value root; - Json::Reader reader; - - // Convert basic fields - root["id"] = id; - root["engine"] = engine_name; - root["type"] = type; - root["api_key"] = api_key; - root["url"] = url; - root["version"] = version; - root["variant"] = variant; - root["status"] = status; - root["date_created"] = date_created; - root["date_updated"] = date_updated; - - // Parse metadata string into JSON object - Json::Value metadataJson; - if (!metadata.empty()) { - bool success = reader.parse(metadata, metadataJson, - false); // false = don't collect comments - if (success) { - root["metadata"] = metadataJson; - } else { - root["metadata"] = Json::Value::null; - } - } else { - root["metadata"] = Json::Value(Json::objectValue); // empty object - } - - return root; - } }; class Engines { - private: - SQLite::Database& db_; - - bool IsUnique(const std::vector& entries, - const std::string& model_id, - const std::string& model_alias) const; - - std::optional> LoadModelListNoLock() const; - public: Engines(); - Engines(SQLite::Database& db); + explicit Engines(SQLite::Database& db); ~Engines(); std::optional UpsertEngine( @@ -76,13 +30,16 @@ class Engines { const std::string& version, const std::string& variant, const std::string& status, const std::string& metadata); - std::optional> GetEngines() const; - std::optional GetEngineById(int id) const; - std::optional GetEngineByNameAndVariant( - const std::string& engine_name, - const std::optional variant = std::nullopt) const; + std::optional GetEngine(const std::string& engine_name, + const std::string& variant); + std::vector GetAllEngines(); + bool DeleteEngine(const std::string& engine_name, const std::string& variant); - std::optional DeleteEngineById(int id); + private: + void CreateTable(); + void CreatePostgreSQLTable(); + + SQLite::Database& db_; + PGconn* pg_conn_{nullptr}; }; - } // namespace cortex::db \ No newline at end of file diff --git a/engine/utils/env_utils.cc b/engine/utils/env_utils.cc new file mode 100644 index 000000000..0ddd5c8c5 --- /dev/null +++ b/engine/utils/env_utils.cc @@ -0,0 +1,53 @@ +#include "env_utils.h" +#include +#include + +namespace cortex::utils { + +std::optional EnvUtils::GetEnv(const std::string& key) { + const char* value = std::getenv(key.c_str()); + if (value == nullptr) { + return std::nullopt; + } + return std::string(value); +} + +std::string EnvUtils::GetPostgresConnectionString() { + // Try to get the full connection string first + if (auto conn_str = GetEnv("POSTGRES_CONNECTION_STRING")) { + return *conn_str; + } + + // If not found, construct from individual components + std::string conn_str = "postgresql://"; + + if (auto user = GetEnv("POSTGRES_USER")) { + conn_str += *user; + if (auto password = GetEnv("POSTGRES_PASSWORD")) { + conn_str += ":" + *password; + } + conn_str += "@"; + } + + if (auto host = GetEnv("POSTGRES_HOST")) { + conn_str += *host; + } else { + conn_str += "localhost"; + } + + if (auto port = GetEnv("POSTGRES_PORT")) { + conn_str += ":" + *port; + } else { + conn_str += ":5432"; + } + + if (auto dbname = GetEnv("POSTGRES_DB")) { + conn_str += "/" + *dbname; + } else { + conn_str += "/cortex"; + } + + return conn_str; +} + +} // namespace cortex::utils \ No newline at end of file diff --git a/engine/utils/env_utils.h b/engine/utils/env_utils.h new file mode 100644 index 000000000..6160dc5e9 --- /dev/null +++ b/engine/utils/env_utils.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +namespace cortex::utils { + +class EnvUtils { + public: + static std::optional GetEnv(const std::string& key); + static std::string GetPostgresConnectionString(); +}; + +} // namespace cortex::utils \ No newline at end of file