Merge pull request #45 from Pythagora-io/sqlite

add sqlite db

.gitignore

@@ -163,5 +163,8 @@ cython_debug/
 # Logger
 /pilot/logger/debug.log
 
+#sqlite
+/pilot/gpt-pilot
+
 # workspace
 workspace

README.md

@@ -39,7 +39,7 @@ Obviously, it still can't create any production-ready app but the general concep
 
 
 - **Python**
-- **PostgreSQL**
+- **PostgreSQL** (optional, projects default is SQLite)
    - DB is needed for multiple reasons like continuing app development if you had to stop at any point or app crashed, going back to specific step so you can change some later steps in development, easier debugging, for future we will add functionality to update project (change some things in existing project or add new features to the project and so on)...
 
 
@@ -52,7 +52,8 @@ After you have Python and PostgreSQL installed, follow these steps:
 5. `pip install -r requirements.txt` (install the dependencies)
 6. `cd pilot`
 7. `mv .env.example .env` (create the .env file)
-8. Add your environment (OpenAI/Azure), your API key and the PostgreSQL database info to the `.env` file
+8. Add your environment (OpenAI/Azure), your API key and the SQLite/PostgreSQL database info to the `.env` file
+   - to change from SQLite to PostgreSQL in your .env just set `DATABASE_TYPE=postgres`
 9. `python db_init.py` (initialize the database)
 10. `python main.py` (start GPT Pilot)
 

pilot/database/config.py

@@ -0,0 +1,8 @@
+import os
+
+DATABASE_TYPE = os.getenv("DATABASE_TYPE", "sqlite")
+DB_NAME = os.getenv("DB_NAME")
+DB_HOST = os.getenv("DB_HOST")
+DB_PORT = os.getenv("DB_PORT")
+DB_USER = os.getenv("DB_USER")
+DB_PASSWORD = os.getenv("DB_PASSWORD")

pilot/database/connection/postgres.py

@@ -0,0 +1,22 @@
+import psycopg2
+from peewee import PostgresqlDatabase
+from psycopg2.extensions import quote_ident
+from database.config import DB_NAME, DB_HOST, DB_PORT, DB_USER, DB_PASSWORD
+
+def get_postgres_database():
+    return PostgresqlDatabase(DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST, port=DB_PORT)
+
+def create_postgres_database():
+    conn = psycopg2.connect(
+        dbname='postgres',
+        user=DB_USER,
+        password=DB_PASSWORD,
+        host=DB_HOST,
+        port=DB_PORT
+    )
+    conn.autocommit = True
+    cursor = conn.cursor()
+    safe_db_name = quote_ident(DB_NAME, conn)
+    cursor.execute(f"CREATE DATABASE {safe_db_name}")
+    cursor.close()
+    conn.close()

pilot/database/connection/sqlite.py

@@ -0,0 +1,5 @@
+from peewee import SqliteDatabase
+from database.config import DB_NAME
+
+def get_sqlite_database():
+    return SqliteDatabase(DB_NAME)

pilot/database/database.py

@@ -4,12 +4,12 @@ from termcolor import colored
 from functools import reduce
 import operator
 import psycopg2
-import os
 from const.common import PROMPT_DATA_TO_IGNORE
 from logger.logger import logger
 from psycopg2.extensions import quote_ident
 
 from utils.utils import hash_data
+from database.config import DB_NAME, DB_HOST, DB_PORT, DB_USER, DB_PASSWORD, DATABASE_TYPE
 from database.models.components.base_models import database
 from database.models.user import User
 from database.models.app import App
@@ -26,12 +26,6 @@ from database.models.command_runs import CommandRuns
 from database.models.user_inputs import UserInputs
 from database.models.files import File
 
-DB_NAME = os.getenv("DB_NAME")
-DB_HOST = os.getenv("DB_HOST")
-DB_PORT = os.getenv("DB_PORT")
-DB_USER = os.getenv("DB_USER")
-DB_PASSWORD = os.getenv("DB_PASSWORD")
-
 
 def save_user(user_id, email, password):
     try:
@@ -383,7 +377,14 @@ def drop_tables():
             UserInputs,
             File,
         ]:
-            database.execute_sql(f'DROP TABLE IF EXISTS "{table._meta.table_name}" CASCADE')
+            if DATABASE_TYPE == "postgresql":
+                sql = f'DROP TABLE IF EXISTS "{table._meta.table_name}" CASCADE'
+            elif DATABASE_TYPE == "sqlite":
+                sql = f'DROP TABLE IF EXISTS "{table._meta.table_name}"'
+            else:
+                raise ValueError(f"Unsupported DATABASE_TYPE: {DATABASE_TYPE}")
+
+            database.execute_sql(sql)
 
 
 def database_exists():
@@ -396,35 +397,42 @@ def database_exists():
 
 
 def create_database():
-    # Connect to the default 'postgres' database to create a new database
-    conn = psycopg2.connect(
-        dbname='postgres',
-        user=DB_USER,
-        password=DB_PASSWORD,
-        host=DB_HOST,
-        port=DB_PORT
-    )
-    conn.autocommit = True
-    cursor = conn.cursor()
+    if DATABASE_TYPE == "postgres":
+        # Connect to the default 'postgres' database to create a new database
+        conn = psycopg2.connect(
+            dbname='postgres',
+            user=DB_USER,
+            password=DB_PASSWORD,
+            host=DB_HOST,
+            port=DB_PORT
+        )
+        conn.autocommit = True
+        cursor = conn.cursor()
 
-    # Safely quote the database name
-    safe_db_name = quote_ident(DB_NAME, conn)
+        # Safely quote the database name
+        safe_db_name = quote_ident(DB_NAME, conn)
 
-    # Use the safely quoted database name in the SQL query
-    cursor.execute(f"CREATE DATABASE {safe_db_name}")
+        # Use the safely quoted database name in the SQL query
+        cursor.execute(f"CREATE DATABASE {safe_db_name}")
 
-    cursor.close()
-    conn.close()
+        cursor.close()
+        conn.close()
+    else:
+        pass
 
 
 def tables_exist():
     tables = [User, App, ProjectDescription, UserStories, UserTasks, Architecture, DevelopmentPlanning,
               DevelopmentSteps, EnvironmentSetup, Development, FileSnapshot, CommandRuns, UserInputs, File]
-    for table in tables:
-        try:
-            database.get_tables().index(table._meta.table_name)
-        except ValueError:
-            return False
+
+    if DATABASE_TYPE == "postgres":
+        for table in tables:
+            try:
+                database.get_tables().index(table._meta.table_name)
+            except ValueError:
+                return False
+    else:
+        pass
     return True
 
 

pilot/database/models/architecture.py

@@ -1,10 +1,15 @@
 from peewee import *
-
+from database.config import DATABASE_TYPE
 from database.models.components.progress_step import ProgressStep
+from database.models.components.sqlite_middlewares import JSONField
 from playhouse.postgres_ext import BinaryJSONField
 
 
 class Architecture(ProgressStep):
-    architecture = BinaryJSONField()
+    if DATABASE_TYPE == 'postgres':
+        architecture = BinaryJSONField()
+    else:
+        architecture = JSONField()  # Custom JSON field for SQLite
+
     class Meta:
         db_table = 'architecture'

pilot/database/models/components/base_models.py

@@ -1,23 +1,17 @@
-import os
 from peewee import *
 from datetime import datetime
 from uuid import uuid4
 
-DB_NAME = os.getenv("DB_NAME")
-DB_HOST = os.getenv("DB_HOST")
-DB_PORT = os.getenv("DB_PORT")
-DB_USER = os.getenv("DB_USER")
-DB_PASSWORD = os.getenv("DB_PASSWORD")
+from database.config import DATABASE_TYPE
+from database.connection.postgres import get_postgres_database
+from database.connection.sqlite import get_sqlite_database
 
 
 # Establish connection to the database
-database = PostgresqlDatabase(
-    DB_NAME,
-    user=DB_USER,
-    password=DB_PASSWORD,
-    host=DB_HOST,
-    port=DB_PORT
-)
+if DATABASE_TYPE == "postgres":
+    database = get_postgres_database()
+else:
+    database = get_sqlite_database()
 
 
 class BaseModel(Model):

pilot/database/models/components/progress_step.py

@@ -1,16 +1,23 @@
 from peewee import *
-
-from playhouse.postgres_ext import BinaryJSONField
-
+from database.config import DATABASE_TYPE
 from database.models.components.base_models import BaseModel
 from database.models.app import App
+from database.models.components.sqlite_middlewares import JSONField
+from playhouse.postgres_ext import BinaryJSONField
 
 
 class ProgressStep(BaseModel):
     app = ForeignKeyField(App, primary_key=True, on_delete='CASCADE')
     step = CharField()
-    data = BinaryJSONField(null=True)
-    messages = BinaryJSONField(null=True)
-    app_data = BinaryJSONField()
+
+    if DATABASE_TYPE == 'postgres':
+        app_data = BinaryJSONField()
+        data = BinaryJSONField(null=True)
+        messages = BinaryJSONField(null=True)
+    else:
+        app_data = JSONField()
+        data = JSONField(null=True)
+        messages = JSONField(null=True)
+
     completed = BooleanField(default=False)
     completed_at = DateTimeField(null=True)

pilot/database/models/components/sqlite_middlewares.py

@@ -0,0 +1,14 @@
+import json
+from peewee import TextField
+
+
+class JSONField(TextField):
+    def python_value(self, value):
+        if value is not None:
+            return json.loads(value)
+        return value
+
+    def db_value(self, value):
+        if value is not None:
+            return json.dumps(value)
+        return value

pilot/database/models/development_planning.py

@@ -1,11 +1,15 @@
 from peewee import *
-
+from database.config import DATABASE_TYPE
 from database.models.components.progress_step import ProgressStep
+from database.models.components.sqlite_middlewares import JSONField
 from playhouse.postgres_ext import BinaryJSONField
 
 
 class DevelopmentPlanning(ProgressStep):
-    development_plan = BinaryJSONField()
+    if DATABASE_TYPE == 'postgres':
+        development_plan = BinaryJSONField()
+    else:
+        development_plan = JSONField()  # Custom JSON field for SQLite
 
     class Meta:
         db_table = 'development_planning'

pilot/database/models/development_steps.py

@@ -1,21 +1,26 @@
 from peewee import *
-
-from playhouse.postgres_ext import BinaryJSONField
-
+from database.config import DATABASE_TYPE
 from database.models.components.base_models import BaseModel
 from database.models.app import App
-
+from database.models.components.sqlite_middlewares import JSONField
+from playhouse.postgres_ext import BinaryJSONField
 
 class DevelopmentSteps(BaseModel):
     id = AutoField()  # This will serve as the primary key
     app = ForeignKeyField(App, on_delete='CASCADE')
     hash_id = CharField(null=False)
-    messages = BinaryJSONField(null=True)
-    llm_response = BinaryJSONField(null=False)
+
+    if DATABASE_TYPE == 'postgres':
+        messages = BinaryJSONField(null=True)
+        llm_response = BinaryJSONField(null=False)
+    else:
+        messages = JSONField(null=True)  # Custom JSON field for SQLite
+        llm_response = JSONField(null=False)  # Custom JSON field for SQLite
+
     previous_step = ForeignKeyField('self', null=True, column_name='previous_step')
 
     class Meta:
         db_table = 'development_steps'
         indexes = (
             (('app', 'hash_id'), True),
-        )
+        )

pilot/database/models/project_description.py

@@ -1,7 +1,4 @@
 from peewee import *
-
-from playhouse.postgres_ext import BinaryJSONField
-
 from database.models.components.progress_step import ProgressStep
 
 

pilot/database/models/user_stories.py

@@ -1,10 +1,14 @@
 from peewee import *
-
+from database.config import DATABASE_TYPE
 from database.models.components.progress_step import ProgressStep
+from database.models.components.sqlite_middlewares import JSONField
 from playhouse.postgres_ext import BinaryJSONField
 
 
 class UserStories(ProgressStep):
-    user_stories = BinaryJSONField()
+    if DATABASE_TYPE == 'postgres':
+        user_stories = BinaryJSONField()
+    else:
+        user_stories = JSONField()  # Custom JSON field for SQLite
     class Meta:
         db_table = 'user_stories'

pilot/database/models/user_tasks.py

@@ -1,10 +1,15 @@
 from peewee import *
-
+from database.config import DATABASE_TYPE
 from database.models.components.progress_step import ProgressStep
+from database.models.components.sqlite_middlewares import JSONField
 from playhouse.postgres_ext import BinaryJSONField
 
 
 class UserTasks(ProgressStep):
-    user_tasks = BinaryJSONField()
+    if DATABASE_TYPE == 'postgres':
+        user_tasks = BinaryJSONField()
+    else:
+        user_tasks = JSONField()  # Custom JSON field for SQLite
+
     class Meta:
         db_table = 'user_tasks'