Merge branch 'main' of github.com:Pythagora-io/copilot

.gitignore

@@ -161,4 +161,7 @@ cython_debug/
 
 
 # Logger
-/euclid/logger/debug.log
+/euclid/logger/debug.log
+
+#dev
+/euclid/brija.py

euclid/const/db.py

@@ -2,4 +2,4 @@ DB_NAME = 'euclid'
 DB_HOST = 'localhost'
 DB_PORT = '5432'
 DB_USER = 'admin'
-DB_PASSWORD = 'admin'
+DB_PASSWORD = 'admin'

euclid/database/database.py

@@ -1,167 +1,163 @@
-# database.py
+from playhouse.shortcuts import model_to_dict
+from peewee import *
 
-import psycopg2
-import json
-from psycopg2 import sql
-from psycopg2.extras import RealDictCursor
-from const import db
-from logger.logger import logger
+from utils.utils import hash_data
+from database.models.components.base_models import database
+from database.models.user import User
+from database.models.app import App
+from database.models.project_description import ProjectDescription
+from database.models.user_stories import UserStories
+from database.models.user_tasks import UserTasks
+from database.models.architecture import Architecture
+from database.models.development_planning import DevelopmentPlanning
+from database.models.development_steps import DevelopmentSteps
+from database.models.environment_setup import EnvironmentSetup
+from database.models.development import Development
 
 
-def create_connection():
-    conn = psycopg2.connect(
-        host=db.DB_HOST,
-        database=db.DB_NAME,
-        port=db.DB_PORT,
-        user=db.DB_USER,
-        password=db.DB_PASSWORD,
-        cursor_factory=RealDictCursor
-    )
-    return conn
-
-
-def create_tables():
-    commands = (
-        """
-        DROP TABLE IF EXISTS progress_steps;
-        DROP TABLE IF EXISTS apps;
-        DROP TABLE IF EXISTS users;
-        """,
-        """
-        CREATE TABLE users (
-            id UUID PRIMARY KEY,
-            username VARCHAR(255) NOT NULL,
-            email VARCHAR(255) NOT NULL,
-            password VARCHAR(255) NOT NULL,
-            created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-            updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
-        )
-        """,
-        """
-        CREATE TABLE apps (
-            id SERIAL PRIMARY KEY,
-            user_id UUID NOT NULL,
-            app_id UUID NOT NULL UNIQUE,
-            app_type VARCHAR(255) NOT NULL,
-            status VARCHAR(255) NOT NULL,
-            created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-            updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (user_id)
-                REFERENCES users (id)
-                ON UPDATE CASCADE ON DELETE CASCADE
-        )
-        """,
-        """
-        CREATE TABLE progress_steps (
-            id SERIAL PRIMARY KEY,
-            app_id UUID NOT NULL,
-            step VARCHAR(255) NOT NULL,
-            data TEXT,
-            completed BOOLEAN NOT NULL,
-            completed_at TIMESTAMP,
-            created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-            updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (app_id)
-                REFERENCES apps (app_id)
-                ON UPDATE CASCADE ON DELETE CASCADE,
-            UNIQUE (app_id, step)
-        )
-        """)
-
-    conn = None
+def save_user(user_id, email="email", password="password"):
     try:
-        conn = create_connection()
-        cur = conn.cursor()
-        for command in commands:
-            cur.execute(command)
-        cur.close()
-        conn.commit()
-    except (Exception, psycopg2.DatabaseError) as error:
-        print(error)
-    finally:
-        if conn is not None:
-            conn.close()
+        user = User.get(User.id == user_id)
+        return user
+    except DoesNotExist:
+        return User.create(id=user_id, email=email, password=password)
 
 
 def save_app(user_id, app_id, app_type):
-    conn = create_connection()
-    cursor = conn.cursor()
+    try:
+        app = App.get(App.id == app_id)
+    except DoesNotExist:
+        user = save_user(user_id)
+        app = App.create(id=app_id, user=user, app_type=app_type)
 
-    cursor.execute("SELECT * FROM users WHERE id = %s", (str(user_id),))
-    if cursor.fetchone() is None:
-        # If user doesn't exist, create a new user
-        cursor.execute("INSERT INTO users (id, username, email, password) VALUES (%s, 'username', 'email', 'password')",
-                       (str(user_id),))
-
-    # Now save or update the app
-    cursor.execute("""
-        INSERT INTO apps (user_id, app_id, app_type, status)
-        VALUES (%s, %s, %s, 'started')
-        ON CONFLICT (app_id) DO UPDATE SET
-        user_id = EXCLUDED.user_id, app_type = EXCLUDED.app_type, status = EXCLUDED.status
-        RETURNING id
-    """, (str(user_id), str(app_id), app_type))
-
-    conn.commit()
-    cursor.close()
-    conn.close()
-
-    logger.info('App saved')
-
-    return
+    return app
 
 
 def save_progress(app_id, step, data):
-    conn = create_connection()
-    cursor = conn.cursor()
+    progress_table_map = {
+        'project_description': ProjectDescription,
+        'user_stories': UserStories,
+        'user_tasks': UserTasks,
+        'architecture': Architecture,
+        'development_planning': DevelopmentPlanning,
+        'environment_setup': EnvironmentSetup,
+        'development': Development,
+    }
 
-    # Check if the data is a dictionary. If it is, convert it to a JSON string.
-    if isinstance(data, dict):
-        data = json.dumps(data)
+    data['step'] = step
 
-    # INSERT the data, but on conflict (if the app_id and step combination already exists) UPDATE the data
-    insert = sql.SQL(
-        """INSERT INTO progress_steps (app_id, step, data, completed)
-           VALUES (%s, %s, %s, false)
-           ON CONFLICT (app_id, step) DO UPDATE
-           SET data = excluded.data, completed = excluded.completed"""
-    )
+    ProgressTable = progress_table_map.get(step)
+    if not ProgressTable:
+        raise ValueError(f"Invalid step: {step}")
 
-    cursor.execute(insert, (app_id, step, data))
+    app = get_app(app_id)
 
-    conn.commit()
-    cursor.close()
-    conn.close()
+    # Use the get_or_create method, which attempts to retrieve a record
+    # or creates a new one if it does not exist.
+    progress, created = ProgressTable.get_or_create(app=app, defaults=data)
+
+    # If the record was not created, it already existed and we should update it
+    if not created:
+        for key, value in data.items():
+            setattr(progress, key, value)
+        progress.save()
+
+    return progress
 
 
-def get_apps_by_id(app_id):
-    conn = create_connection()
-    cursor = conn.cursor()
-
-    cursor.execute("SELECT * FROM apps WHERE app_id = %s", (str(app_id),))
-    apps = cursor.fetchall()
-
-    cursor.close()
-    conn.close()
-
-    return apps
+def get_app(app_id):
+    try:
+        app = App.get(App.id == app_id)
+        return app
+    except DoesNotExist:
+        raise ValueError(f"No app with id: {app_id}")
 
 
 def get_progress_steps(app_id, step=None):
-    conn = create_connection()
-    cursor = conn.cursor()
+    progress_table_map = {
+        'project_description': ProjectDescription,
+        'user_stories': UserStories,
+        'user_tasks': UserTasks,
+        'architecture': Architecture,
+        'development_planning': DevelopmentPlanning,
+        'environment_setup': EnvironmentSetup,
+        'development': Development,
+    }
 
     if step:
-        cursor.execute("SELECT * FROM progress_steps WHERE app_id = %s AND step = %s", (app_id, step))
+        ProgressTable = progress_table_map.get(step)
+        if not ProgressTable:
+            raise ValueError(f"Invalid step: {step}")
+
+        try:
+            progress = ProgressTable.get(ProgressTable.app_id == app_id)
+            return model_to_dict(progress)
+        except DoesNotExist:
+            return None
     else:
-        cursor.execute("SELECT * FROM progress_steps WHERE app_id = %s", (app_id,))
-    steps = cursor.fetchall()
+        steps = {}
+        for step, ProgressTable in progress_table_map.items():
+            try:
+                progress = ProgressTable.get(ProgressTable.app_id == app_id)
+                steps[step] = model_to_dict(progress)
+            except DoesNotExist:
+                steps[step] = None
 
-    cursor.close()
-    conn.close()
+        return steps
 
-    return steps
+
+def save_development_step(app_id, messages):
+    app = get_app(app_id)
+    hash_id = hash_data(messages)
+    try:
+        dev_step = DevelopmentSteps.create(app=app, hash_id=hash_id, messages=messages)
+    except IntegrityError:
+        print(f"A Development Step with hash_id {hash_id} already exists.")
+        return None
+    return dev_step
+
+
+def get_development_step_by_hash_id(hash_id):
+    try:
+        dev_step = DevelopmentSteps.get(DevelopmentSteps.hash_id == hash_id)
+    except DoesNotExist:
+        print(f"No Development Step found with hash_id {hash_id}")
+        return None
+    return dev_step
+
+
+def create_tables():
+    with database:
+        database.create_tables([
+            User,
+            App,
+            ProjectDescription,
+            UserStories,
+            UserTasks,
+            Architecture,
+            DevelopmentPlanning,
+            DevelopmentSteps,
+            EnvironmentSetup,
+            Development
+        ])
+
+
+def drop_tables():
+    with database:
+        database.drop_tables([
+            User,
+            App,
+            ProjectDescription,
+            UserStories,
+            UserTasks,
+            Architecture,
+            DevelopmentPlanning,
+            DevelopmentSteps,
+            EnvironmentSetup,
+            Development
+        ])
 
 
 if __name__ == "__main__":
+    drop_tables()
     create_tables()

euclid/database/models/app.py

@@ -0,0 +1,10 @@
+from peewee import *
+
+from database.models.components.base_models import BaseModel
+from database.models.user import User
+
+
+class App(BaseModel):
+    user = ForeignKeyField(User, backref='apps')
+    app_type = CharField()
+    status = CharField(default='started')

euclid/database/models/architecture.py

@@ -0,0 +1,9 @@
+from peewee import *
+
+from database.models.components.progress_step import ProgressStep
+
+
+class Architecture(ProgressStep):
+    architecture = TextField()
+    class Meta:
+        db_table = 'architecture'

euclid/database/models/components/base_models.py

@@ -0,0 +1,24 @@
+from peewee import *
+from datetime import datetime
+from uuid import uuid4
+
+from const import db
+
+
+# Establish connection to the database
+database = PostgresqlDatabase(
+    db.DB_NAME,
+    user=db.DB_USER,
+    password=db.DB_PASSWORD,
+    host=db.DB_HOST,
+    port=db.DB_PORT
+)
+
+
+class BaseModel(Model):
+    id = UUIDField(primary_key=True, default=uuid4)
+    created_at = DateTimeField(default=datetime.now)
+    updated_at = DateTimeField(default=datetime.now)
+
+    class Meta:
+        database = database

euclid/database/models/components/progress_step.py

@@ -0,0 +1,16 @@
+from peewee import *
+
+from playhouse.postgres_ext import BinaryJSONField
+
+from database.models.components.base_models import BaseModel
+from database.models.app import App
+
+
+class ProgressStep(BaseModel):
+    app = ForeignKeyField(App, primary_key=True)
+    step = CharField()
+    data = BinaryJSONField(null=True)
+    messages = BinaryJSONField(null=True)
+    app_data = BinaryJSONField()
+    completed = BooleanField(default=False)
+    completed_at = DateTimeField(null=True)

euclid/database/models/development.py

@@ -0,0 +1,8 @@
+from peewee import *
+
+from database.models.components.progress_step import ProgressStep
+
+
+class Development(ProgressStep):
+    class Meta:
+        db_table = 'development'

euclid/database/models/development_planning.py

@@ -0,0 +1,10 @@
+from peewee import *
+
+from database.models.components.progress_step import ProgressStep
+
+
+class DevelopmentPlanning(ProgressStep):
+    architecture = TextField()
+
+    class Meta:
+        db_table = 'development_planning'

euclid/database/models/development_steps.py

@@ -0,0 +1,12 @@
+from peewee import *
+
+from playhouse.postgres_ext import BinaryJSONField
+
+from database.models.components.base_models import BaseModel
+from database.models.app import App
+
+
+class DevelopmentSteps(BaseModel):
+    app = ForeignKeyField(App, primary_key=True)
+    hash_id = CharField(unique=True, null=False)
+    messages = BinaryJSONField(null=True)

euclid/database/models/environment_setup.py

@@ -0,0 +1,6 @@
+from database.models.components.progress_step import ProgressStep
+
+
+class EnvironmentSetup(ProgressStep):
+    class Meta:
+        db_table = 'environment_setup'

euclid/database/models/project_description.py

@@ -0,0 +1,13 @@
+from peewee import *
+
+from playhouse.postgres_ext import BinaryJSONField
+
+from database.models.components.progress_step import ProgressStep
+
+
+class ProjectDescription(ProgressStep):
+    prompt = TextField()
+    summary = TextField()
+
+    class Meta:
+        db_table = 'project_description'

euclid/database/models/user.py

@@ -0,0 +1,8 @@
+from peewee import *
+
+from database.models.components.base_models import BaseModel
+
+
+class User(BaseModel):
+    email = CharField(unique=True)
+    password = CharField()

euclid/database/models/user_stories.py

@@ -0,0 +1,9 @@
+from peewee import *
+
+from database.models.components.progress_step import ProgressStep
+
+
+class UserStories(ProgressStep):
+    user_stories = TextField()
+    class Meta:
+        db_table = 'user_stories'

euclid/database/models/user_tasks.py

@@ -0,0 +1,9 @@
+from peewee import *
+
+from database.models.components.progress_step import ProgressStep
+
+
+class UserTasks(ProgressStep):
+    user_tasks = TextField()
+    class Meta:
+        db_table = 'user_tasks'

euclid/helpers/agents/Architect.py

@@ -1,3 +1,4 @@
+from utils.utils import step_already_finished
 from helpers.Agent import Agent
 import json
 from termcolor import colored
@@ -18,17 +19,10 @@ class Architect(Agent):
         self.convo_architecture = AgentConvo(self)
 
         # If this app_id already did this step, just get all data from DB and don't ask user again
-        steps = get_progress_steps(self.project.args['app_id'], self.project.current_step)
-        if steps and not execute_step(self.project.args['step'], self.project.current_step):
-            first_step = steps[0]
-            data = json.loads(first_step['data'])
-
-            architecture = data.get('architecture')
-
-            message = f"Architecture already done for this app_id: {self.project.args['app_id']}. Moving to next step..."
-            print(colored(message, "green"))
-            logger.info(message)
-            return architecture
+        step = get_progress_steps(self.project.args['app_id'], self.project.current_step)
+        if step and not execute_step(self.project.args['step'], self.project.current_step):
+            step_already_finished(self.project.args, step)
+            return step['architecture']
 
         # ARCHITECTURE
         print(colored(f"Planning project architecture...\n", "green"))

euclid/helpers/agents/ProductOwner.py

@@ -5,7 +5,7 @@ from helpers.AgentConvo import AgentConvo
 
 from logger.logger import logger
 from database.database import save_progress, save_app, get_progress_steps
-from utils.utils import execute_step, generate_app_data
+from utils.utils import execute_step, generate_app_data, step_already_finished
 from prompts.prompts import ask_for_app_type, ask_for_main_app_definition, get_additional_info_from_openai, \
     generate_messages_from_description, get_additional_info_from_user
 from const.function_calls import USER_STORIES, USER_TASKS
@@ -19,39 +19,31 @@ class ProductOwner(Agent):
         convo_project_description = AgentConvo(self)
 
         # If this app_id already did this step, just get all data from DB and don't ask user again
-        steps = get_progress_steps(self.project.args['app_id'], self.project.current_step)
-        if steps and not execute_step(self.project.args['step'], self.project.current_step):
-            first_step = steps[0]
-            data = json.loads(first_step['data'])
-
-            summary = data.get('summary')
-            app_data = data.get('app_data')
-            self.project.args.update(app_data)
-
-            message = f"Project summary already done for this app_id: {self.project.args['app_id']}. Moving to next step..."
-            print(colored(message, "green"))
-            logger.info(message)
-
-            self.project_description = summary
-            return summary
+        step = get_progress_steps(self.project.args['app_id'], self.project.current_step)
+        if step and not execute_step(self.project.args['step'], self.project.current_step):
+            step_already_finished(self.project.args, step)
+            return step['summary'], step['messages']
 
         # PROJECT DESCRIPTION
         self.project.args['app_type'] = ask_for_app_type()
 
         save_app(self.project.args['user_id'], self.project.args['app_id'], self.project.args['app_type'])
 
-        description = ask_for_main_app_definition()
+        main_prompt = ask_for_main_app_definition()
 
         high_level_messages = get_additional_info_from_openai(
-            generate_messages_from_description(description, self.project.args['app_type']))
+            generate_messages_from_description(main_prompt, self.project.args['app_type']))
 
         high_level_summary = convo_project_description.send_message('utils/summary.prompt',
                                                 {'conversation': '\n'.join(
                                                     [f"{msg['role']}: {msg['content']}" for msg in high_level_messages])})
 
-        save_progress(self.project.args['app_id'], self.project.current_step,
-                    {"messages": high_level_messages, "summary": high_level_summary, "app_data": generate_app_data(self.project.args)
-                    })
+        save_progress(self.project.args['app_id'], self.project.current_step, {
+            "prompt": main_prompt,
+            "messages": high_level_messages,
+            "summary": high_level_summary,
+            "app_data": generate_app_data(self.project.args)
+        })
 
         self.project_description = high_level_summary
         return high_level_summary
@@ -63,28 +55,19 @@ class ProductOwner(Agent):
         self.convo_user_stories = AgentConvo(self)
         
         # If this app_id already did this step, just get all data from DB and don't ask user again
-        steps = get_progress_steps(self.project.args['app_id'], self.project.current_step)
-        if steps and not execute_step(self.project.args['step'], self.project.current_step):
-            first_step = steps[0]
-            data = json.loads(first_step['data'])
-
-            user_stories = data.get('user_stories')
-            app_data = data.get('app_data')
-            if app_data is not None:
-                self.project.args.update(app_data)
-
-            message = f"User stories already done for this app_id: {self.project.args['app_id']}. Moving to next step..."
-            print(colored(message, "green"))
-            logger.info(message)
-            return user_stories
+        step = get_progress_steps(self.project.args['app_id'], self.project.current_step)
+        if step and not execute_step(self.project.args['step'], self.project.current_step):
+            step_already_finished(self.project.args, step)
+            return step['user_stories'], step['messages']
 
         # USER STORIES
         print(colored(f"Generating user stories...\n", "green"))
         logger.info(f"Generating user stories...")
 
-        user_stories = self.convo_user_stories.send_message('user_stories/specs.prompt',
-                                            {'prompt': self.project_description, 'app_type': self.project.args['app_type']},
-                                            USER_STORIES)
+        user_stories = self.convo_user_stories.send_message('user_stories/specs.prompt', {
+            'prompt': self.project_description,
+            'app_type': self.project.args['app_type']
+        }, USER_STORIES)
 
         logger.info(user_stories)
         user_stories = get_additional_info_from_user(user_stories, 'product_owner')
@@ -92,7 +75,9 @@ class ProductOwner(Agent):
         logger.info(f"Final user stories: {user_stories}")
 
         save_progress(self.project.args['app_id'], self.project.current_step, {
-            "messages": self.convo_user_stories.messages, "user_stories": user_stories, "app_data": generate_app_data(self.project.args)
+            "messages": self.convo_user_stories.messages,
+            "user_stories": user_stories,
+            "app_data": generate_app_data(self.project.args)
         })
 
         return user_stories
@@ -103,19 +88,10 @@ class ProductOwner(Agent):
         current_step = 'user_tasks'
 
         # If this app_id already did this step, just get all data from DB and don't ask user again
-        steps = get_progress_steps(self.project.args['app_id'], current_step)
-        if steps and not execute_step(self.project.args['step'], current_step):
-            first_step = steps[0]
-            data = json.loads(first_step['data'])
-
-            app_data = data.get('app_data')
-            if app_data is not None:
-                self.project.args.update(app_data)
-
-            message = f"User tasks already done for this app_id: {self.project.args['app_id']}. Moving to next step..."
-            print(colored(message, "green"))
-            logger.info(message)
-            return data.get('user_tasks')
+        step = get_progress_steps(self.project.args['app_id'], current_step)
+        if step and not execute_step(self.project.args['step'], current_step):
+            step_already_finished(self.project.args, step)
+            return step['user_tasks']
 
         # USER TASKS
         print(colored(f"Generating user tasks...\n", "green"))
@@ -130,7 +106,9 @@ class ProductOwner(Agent):
         logger.info(f"Final user tasks: {user_tasks}")
 
         save_progress(self.project.args['app_id'], current_step, {
-            "messages": self.convo_user_stories.messages,"user_tasks": user_tasks, "app_data": generate_app_data(self.project.args)
+            "messages": self.convo_user_stories.messages,
+            "user_tasks": user_tasks,
+            "app_data": generate_app_data(self.project.args)
         })
 
         return user_tasks

euclid/utils/utils.py

@@ -5,11 +5,15 @@ import os
 import platform
 import distro
 import uuid
+import json
+import hashlib
 import re
 from jinja2 import Environment, FileSystemLoader
+from termcolor import colored
 
 from const.llm import MAX_QUESTIONS, END_RESPONSE
 from const.common import ROLES, STEPS
+from logger.logger import logger
 
 
 def get_arguments():
@@ -143,8 +147,20 @@ def execute_step(matching_step, current_step):
     return matching_step_index is not None and current_step_index is not None and current_step_index >= matching_step_index
 
 
+def step_already_finished(args, step):
+    args.update(step['app_data'])
+
+    message = f"{capitalize_first_word_with_underscores(step['step'])} already done for this app_id: {args['app_id']}. Moving to next step..."
+    print(colored(message, "green"))
+    logger.info(message)
+
+
 def generate_app_data(args):
     return {'app_id': args['app_id'], 'app_type': args['app_type']}
 
 def array_of_objects_to_string(array):
-    return '\n'.join([f'{key}: {value}' for key, value in array.items()])
+    return '\n'.join([f'{key}: {value}' for key, value in array.items()])
+
+def hash_data(data):
+    serialized_data = json.dumps(data, sort_keys=True).encode('utf-8')
+    return hashlib.sha256(serialized_data).hexdigest()

requirements.txt

@@ -5,6 +5,7 @@ distro==1.8.0
 idna==3.4
 Jinja2==3.1.2
 MarkupSafe==2.1.3
+peewee==3.16.2
 prompt-toolkit==3.0.39
 psycopg2==2.9.6
 python-dotenv==1.0.0