merge

euclid/const/llm.py

@@ -1,4 +1,4 @@
-MIN_TOKENS_FOR_GPT_RESPONSE = 60
+MIN_TOKENS_FOR_GPT_RESPONSE = 600
 MAX_GPT_MODEL_TOKENS = 8192
 MAX_QUESTIONS = 3
 END_RESPONSE = "EVERYTHING_CLEAR"

euclid/database/database.py

@@ -179,11 +179,11 @@ def hash_and_save_step(Model, app_id, hash_data_args, data_fields, message):
 
     try:
         inserted_id = (Model
-                        .insert(**data_to_insert)
-                        .on_conflict(conflict_target=[Model.app, Model.hash_id],
-                                    preserve=fields_to_preserve,
-                                    update={})
-                        .execute())
+            .insert(**data_to_insert)
+            .on_conflict(conflict_target=[Model.app, Model.hash_id],
+                preserve=fields_to_preserve,
+                update=data_fields)
+            .execute())
 
         record = Model.get_by_id(inserted_id)
         print(colored(f"{message} with id {record.id}", "yellow"))
@@ -266,18 +266,19 @@ def get_user_input_from_hash_id(project, query):
     return user_input
 
 def delete_all_subsequent_steps(project):
-    delete_subsequent_steps(DevelopmentSteps, project.checkpoints['last_development_step'], 'previous_step')
-    delete_subsequent_steps(CommandRuns, project.checkpoints['last_command_run'], 'previous_step')
-    delete_subsequent_steps(UserInputs, project.checkpoints['last_user_input'], 'previous_step')
+    delete_subsequent_steps(DevelopmentSteps, project.checkpoints['last_development_step'])
+    delete_subsequent_steps(CommandRuns, project.checkpoints['last_command_run'])
+    delete_subsequent_steps(UserInputs, project.checkpoints['last_user_input'])
 
-def delete_subsequent_steps(model, step, step_field_name):
+def delete_subsequent_steps(model, step):
     if step is None:
         return
     print(colored(f"Deleting subsequent {model.__name__} steps after {step.id}", "red"))
-    subsequent_step = model.get_or_none(**{step_field_name: step.id})
-    if subsequent_step:
-        delete_subsequent_steps(model, subsequent_step, step_field_name)
-        subsequent_step.delete_instance()
+    subsequent_steps = model.select().where(model.previous_step == step.id)
+    for subsequent_step in subsequent_steps:
+        if subsequent_step:
+            delete_subsequent_steps(model, subsequent_step)
+            subsequent_step.delete_instance()
 
 def get_all_connected_steps(step, previous_step_field_name):
     """Recursively get all steps connected to the given step."""
@@ -288,6 +289,11 @@ def get_all_connected_steps(step, previous_step_field_name):
         prev_step = getattr(prev_step, previous_step_field_name)
     return connected_steps
 
+def delete_all_app_development_data(app):
+    models = [DevelopmentSteps, CommandRuns, UserInputs, File, FileSnapshot]
+    for model in models:
+        model.delete().where(model.app == app).execute()
+
 def delete_unconnected_steps_from(step, previous_step_field_name):
     if step is None:
         return
@@ -303,6 +309,14 @@ def delete_unconnected_steps_from(step, previous_step_field_name):
         print(colored(f"Deleting unconnected {step.__class__.__name__} step {unconnected_step.id}", "red"))
         unconnected_step.delete_instance()
 
+def save_file_description(project, path, name, description):
+    (File.insert(app=project.app, path=path, name=name, description=description)
+        .on_conflict(
+            conflict_target=[File.app, File.name, File.path],
+            preserve=[],
+            update={'description': description})
+        .execute())
+
 def create_tables():
     with database:
         database.create_tables([

euclid/database/models/file_snapshot.py

@@ -2,14 +2,17 @@ from peewee import *
 
 from database.models.components.base_models import BaseModel
 from database.models.development_steps import DevelopmentSteps
+from database.models.app import App
+from database.models.files import File
 
 class FileSnapshot(BaseModel):
+    app = ForeignKeyField(App, on_delete='CASCADE')
     development_step = ForeignKeyField(DevelopmentSteps, backref='files', on_delete='CASCADE')
-    name = CharField()
+    file = ForeignKeyField(File, on_delete='CASCADE', null=True)
     content = TextField()
 
     class Meta:
         db_table = 'file_snapshot'
         indexes = (
-            (('development_step', 'name'), True),
+            (('development_step', 'file'), True),
         )

euclid/database/models/files.py

@@ -5,10 +5,12 @@ from database.models.development_steps import DevelopmentSteps
 from database.models.app import App
 
 class File(BaseModel):
+    id = AutoField()
     app = ForeignKeyField(App, on_delete='CASCADE')
     name = CharField()
     path = CharField()
-    description = TextField()
+    full_path = CharField()
+    description = TextField(null=True)
 
     class Meta:
         indexes = (

euclid/helpers/AgentConvo.py

@@ -22,12 +22,14 @@ class AgentConvo:
         self.messages.append(get_sys_message(self.agent.role))
 
     def send_message(self, prompt_path=None, prompt_data=None, function_calls=None):
-
         # craft message
         if prompt_path is not None and prompt_data is not None:
             prompt = get_prompt(prompt_path, prompt_data)
             self.messages.append({"role": "user", "content": prompt})
 
+        if function_calls is not None and 'function_calls' in function_calls:
+            self.messages[-1]['content'] += '\nMAKE SURE THAT YOU RESPOND WITH A CORRECT JSON FORMAT!!!'
+
         # check if we already have the LLM response saved
         self.agent.project.llm_req_num += 1
         development_step = get_development_step_from_hash_id(self.agent.project, prompt_path, prompt_data, self.agent.project.llm_req_num)
@@ -130,3 +132,6 @@ class AgentConvo:
             content = file.read()
         process = subprocess.Popen('pbcopy', stdin=subprocess.PIPE)
         process.communicate(content.replace('{{messages}}', str(self.messages)).encode('utf-8'))
+
+    def remove_last_x_messages(self, x):
+        self.messages = self.messages[:-x]

euclid/helpers/Project.py

@@ -3,9 +3,9 @@ import os
 from termcolor import colored
 from const.common import IGNORE_FOLDERS
 from database.models.app import App
-from database.database import get_app, delete_unconnected_steps_from
+from database.database import get_app, delete_unconnected_steps_from, delete_all_app_development_data
 from utils.questionary import styled_text
-from helpers.files import get_files_content, clear_directory
+from helpers.files import get_files_content, clear_directory, update_file
 from helpers.cli import build_directory_tree
 from helpers.agents.TechLead import TechLead
 from helpers.agents.Developer import Developer
@@ -30,8 +30,6 @@ class Project:
             'last_command_run': None,
             'last_development_step': None,
         }
-        self.skip_steps = False if ('skip_until_dev_step' in args and args['skip_until_dev_step'] == '0') else True
-        self.skip_until_dev_step = args['skip_until_dev_step'] if 'skip_until_dev_step' in args else None
         # TODO make flexible
         # self.root_path = get_parent_folder('euclid')
         self.root_path = ''
@@ -64,6 +62,20 @@ class Project:
         self.tech_lead = TechLead(self)
         self.development_plan = self.tech_lead.create_development_plan()
 
+        # TODO move to constructor eventually
+        if 'skip_until_dev_step' in self.args:
+            self.skip_until_dev_step = self.args['skip_until_dev_step']
+            if self.args['skip_until_dev_step'] == '0':
+                clear_directory(self.root_path, IGNORE_FOLDERS)
+                delete_all_app_development_data(self.args['app_id'])
+                self.skip_steps = False
+        else:
+            self.skip_until_dev_step = None
+            self.skip_steps = True
+
+        self.skip_steps = False if ('skip_until_dev_step' in self.args and self.args['skip_until_dev_step'] == '0') else True
+        # TODO END
+
         self.developer = Developer(self)
         self.developer.set_up_environment();
 
@@ -71,21 +83,27 @@ class Project:
 
     def get_directory_tree(self, with_descriptions=False):
         files = {}
-        if with_descriptions:
+        if with_descriptions and False:
             files = File.select().where(File.app_id == self.args['app_id'])
             files = {snapshot.name: snapshot for snapshot in files}
-        return build_directory_tree(self.root_path + '/', ignore=IGNORE_FOLDERS, files=files, add_descriptions=True)
+        return build_directory_tree(self.root_path + '/', ignore=IGNORE_FOLDERS, files=files, add_descriptions=False)
 
     def get_test_directory_tree(self):
         # TODO remove hardcoded path
         return build_directory_tree(self.root_path + '/tests', ignore=IGNORE_FOLDERS)
 
+    def get_all_coded_files(self):
+        files = File.select().where(File.app_id == self.args['app_id'])
+        files = self.get_files([file.path + file.name for file in files])
+        return files
+
     def get_files(self, files):
         files_with_content = []
         for file in files:
             # TODO this is a hack, fix it
             try:
-                file_content = open(self.get_full_file_path('', file), 'r').read()
+                relative_path, full_path = self.get_full_file_path('', file)
+                file_content = open(full_path, 'r').read()
             except:
                 file_content = ''
 
@@ -95,20 +113,48 @@ class Project:
             })
         return files_with_content
 
+    def save_file(self, data):
+        data['path'], data['full_path'] = self.get_full_file_path(data['path'], data['name'])
+        update_file(data['full_path'], data['content'])
+
+        file_in_db, created = File.get_or_create(
+            app=self.app,
+            name=data['name'],
+            path=data['path'],
+            full_path=data['full_path'],
+        )
+
     def get_full_file_path(self, file_path, file_name):
         file_path = file_path.replace('./', '', 1).rstrip(file_name)
+
         if not file_path.endswith('/'):
             file_path = file_path + '/'
-        return self.root_path + file_path + file_name
+
+        if file_name.startswith('/'):
+            file_name = file_name[1:]
+
+        if not file_path.startswith('/'):
+            file_path = '/' + file_path
+
+        return (file_path, self.root_path + file_path + file_name)
 
     def save_files_snapshot(self, development_step_id):
         files = get_files_content(self.root_path, ignore=IGNORE_FOLDERS)
         development_step, created = DevelopmentSteps.get_or_create(id=development_step_id)
 
         for file in files:
-            file_snapshot, created = FileSnapshot.get_or_create(
-                development_step=development_step,
+            # TODO this can be optimized so we don't go to the db each time
+            file_in_db, created = File.get_or_create(
+                app=self.app,
                 name=file['name'],
+                path=file['path'],
+                full_path=file['full_path'],
+            )
+
+            file_snapshot, created = FileSnapshot.get_or_create(
+                app=self.app,
+                development_step=development_step,
+                file=file_in_db,
                 defaults={'content': file.get('content', '')}
             )
             file_snapshot.content = content = file['content']
@@ -120,7 +166,7 @@ class Project:
 
         clear_directory(self.root_path, IGNORE_FOLDERS)
         for file_snapshot in file_snapshots:
-            full_path = self.root_path + '/' + file_snapshot.name
+            full_path = self.root_path + file_snapshot.file.path + '/' + file_snapshot.file.name
             # Ensure directory exists
             os.makedirs(os.path.dirname(full_path), exist_ok=True)
 

euclid/helpers/agents/CodeMonkey.py

@@ -28,16 +28,6 @@ class CodeMonkey(Agent):
         }, IMPLEMENT_CHANGES)
 
         for file_data in changes:
-            file_data['full_path'] = self.project.get_full_file_path(file_data['path'], file_data['name'])
-
-            if file_data['description'] != '':
-                    (File.insert(app=self.project.app, path=file_data['path'], name=file_data['name'], description=file_data['description'])
-                        .on_conflict(
-                            conflict_target=[File.app, File.name, File.path],
-                            preserve=[],
-                            update={'description': file_data['description']})
-                        .execute())
-
-            update_file(file_data['full_path'], file_data['content'])
+            self.project.save_file(file_data)
 
         return convo

euclid/helpers/agents/Developer.py

@@ -1,6 +1,7 @@
 import json
 import uuid
 from termcolor import colored
+from helpers.files import update_file
 from utils.utils import step_already_finished
 from helpers.agents.CodeMonkey import CodeMonkey
 from logger.logger import logger
@@ -98,7 +99,7 @@ class Developer(Agent):
         if step and not execute_step(self.project.args['step'], self.project.current_step):
             step_already_finished(self.project.args, step)
             return
-        
+
         # ENVIRONMENT SETUP
         print(colored(f"Setting up the environment...\n", "green"))
         logger.info(f"Setting up the environment...")
@@ -154,19 +155,19 @@ class Developer(Agent):
             'development/task/step_check.prompt',
             {},
             GET_TEST_TYPE)
-        
+
         if test_type == 'command_test':
             run_command_until_success(command['command'], command['timeout'], convo)
         elif test_type == 'automated_test':
             code_monkey.implement_code_changes(convo, automated_test_description, 0)
         elif test_type == 'manual_test':
             # TODO make the message better
-            response = self.project.ask_for_human_intervention(
+            user_feedback = self.project.ask_for_human_intervention(
                 'Message from Euclid: I need your help. Can you please test if this was successful?',
                 manual_test_description
             )
-            if response is not None and response != 'DONE':
-                self.test_code_changes(code_monkey, convo)
+            if user_feedback is not None:
+                debug(convo, user_input=user_feedback, issue_description=manual_test_description)
 
     def implement_step(self, convo, step_index, type, description):
         # TODO remove hardcoded folder path

euclid/helpers/cli.py

@@ -45,6 +45,12 @@ def execute_command(project, command, timeout=5000):
     timeout = min(max(timeout, MIN_COMMAND_RUN_TIME), MAX_COMMAND_RUN_TIME)
 
     print(colored(f'Can i execute the command: `{command}` with {timeout}ms timeout?', 'white', attrs=['bold']))
+
+    answer = styled_text(
+        project,
+        'If yes, just press ENTER and if not, please paste the output of running this command here and press ENTER'
+    )
+
     project.command_runs_count += 1
     command_run = get_command_run_from_hash_id(project, command)
     if command_run is not None and project.skip_steps:
@@ -53,11 +59,6 @@ def execute_command(project, command, timeout=5000):
         print(colored(f'Restoring command run response id {command_run.id}:\n```\n{command_run.cli_response}```', 'yellow'))
         return command_run.cli_response
 
-    answer = styled_text(
-        project,
-        'If yes, just press ENTER and if not, please paste the output of running this command here and press ENTER'
-    )
-
     return_value = None
 
     if answer != '':

euclid/helpers/files.py

@@ -28,9 +28,12 @@ def get_files_content(directory, ignore=[]):
             with open(path, 'r', encoding='utf-8', errors='ignore') as f:
                 file_content = f.read()
 
+            file_name = path.replace(directory + '/', '')
             return_array.append({
-                'name': path.replace(directory + '/', ''),
-                'content': file_content
+                'name': file_name,
+                'path': '/' + file.replace(file_name, ''),
+                'content': file_content,
+                'full_path': path,
             })
 
     return return_array

euclid/prompts/dev_ops/ran_command.prompt

@@ -1,4 +1,4 @@
-I ran the command `{{ command }}` and for this response from CLI:
+{{ additional_info }}I ran the command `{{ command }}` and for this response from CLI:
 ```
 {{ cli_response }}
 ```

euclid/prompts/development/implement_changes.prompt

@@ -9,6 +9,6 @@ Here is how files look now:
 {% endfor %}
 {% endif %}
 
-Now, think step by step and apply the needed changes into appropriate files and return the changed files.
+Now, think step by step and apply the needed changes for step #{{ step_index }} - {{ step_description }}.
 
 Within the file modifications, anything needs to be written by the user, add the comment in the same line as the code that starts with `// INPUT_REQUIRED {input_description}` where `input_description` is a description of what needs to be added here by the user. Finally, you can save the modified files on the disk by calling `save_files` function.

euclid/utils/llm_connection.py

@@ -194,9 +194,11 @@ def stream_gpt_completion(data, req_type):
                 if 'name' in json_line['function_call']:
                     function_calls['name'] = json_line['function_call']['name']
                     print(f'Function call: {function_calls["name"]}')
+
                 if 'arguments' in json_line['function_call']:
                     function_calls['arguments'] += json_line['function_call']['arguments']
                     print(json_line['function_call']['arguments'], end='', flush=True)
+
             if 'content' in json_line:
                 content = json_line.get('content')
                 if content: