From 14187041864d641a808d216673a2f78fc0b759e9 Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Wed, 30 Aug 2023 23:16:17 +0200 Subject: [PATCH 1/6] Initial setup for IPC Client and logging --- pilot/const/ipc.py | 5 ++++ pilot/helpers/Project.py | 23 ++++++++++++++++++ pilot/helpers/ipc.py | 52 ++++++++++++++++++++++++++++++++++++++++ pilot/utils/arguments.py | 4 ---- 4 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 pilot/const/ipc.py create mode 100644 pilot/helpers/ipc.py diff --git a/pilot/const/ipc.py b/pilot/const/ipc.py new file mode 100644 index 0000000..ab1bd5b --- /dev/null +++ b/pilot/const/ipc.py @@ -0,0 +1,5 @@ +MESSAGE_TYPE = { + 'verbose': 'verbose', + 'gpt_stream': 'gpt_stream', + 'user_input_request': 'user_input_request' +} \ No newline at end of file diff --git a/pilot/helpers/Project.py b/pilot/helpers/Project.py index 2a4283c..1b54ef4 100644 --- a/pilot/helpers/Project.py +++ b/pilot/helpers/Project.py @@ -1,9 +1,12 @@ import os +import time from termcolor import colored from const.common import IGNORE_FOLDERS from database.models.app import App from database.database import get_app, delete_unconnected_steps_from, delete_all_app_development_data +from helpers.ipc import IPCClient +from const.ipc import MESSAGE_TYPE from utils.questionary import styled_text from helpers.files import get_files_content, clear_directory, update_file from helpers.cli import build_directory_tree @@ -51,6 +54,15 @@ class Project: # if development_plan is not None: # self.development_plan = development_plan + if '--external-log-process' in args: + self.ipc_client_instance = IPCClient() + + print('..'*20) + self.log('\n------------------ STARTING NEW PROJECT ----------------------', 'verbose') + self.log(f"If you wish to continue with this project in future run:", 'verbose') + self.log(f'python main.py app_id={args["app_id"]}', 'verbose') + self.log('--------------------------------------------------------------\n', 'verbose') + def start(self): self.project_manager = ProductOwner(self) self.project_manager.get_project_description() @@ -195,3 +207,14 @@ class Project: return cbs[answer]() elif answer != '': return answer + + def log(self, text, message_type, cb=None): + if self.ipc_client_instance is None or self.ipc_client_instance.client is None: + print(text) + else: + self.ipc_client_instance.send({ + 'type': MESSAGE_TYPE[message_type], + 'content': str(text), + }) + if cb is not None: + self.ipc_client_instance.listen(lambda response: cb(response['content'])) \ No newline at end of file diff --git a/pilot/helpers/ipc.py b/pilot/helpers/ipc.py new file mode 100644 index 0000000..785d37b --- /dev/null +++ b/pilot/helpers/ipc.py @@ -0,0 +1,52 @@ +# ipc.py +import socket +import json +import time + +class IPCClient: + def __init__(self): + self.ready = False + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + print("Connecting to the external process...") + try: + client.connect(('localhost', 8124)) + self.client = client + print("Connected!") + except ConnectionRefusedError: + self.client = None + print("Connection refused, make sure you started the external process") + + def handle_request(self, message_content): + print(f"Received request from the external process: {message_content}") + return message_content # For demonstration, we're just echoing back the content + + def listen(self, cb): + if self.client is None: + print("Not connected to the external process!") + return + try: + while True: + data = self.client.recv(4096) + message = json.loads(data) + + if message['type'] == 'request': + cb(message['content']) + if message['type'] == 'request': + response_content = self.handle_request(message['content']) + response = { + 'type': 'response', + 'content': response_content + } + self.client.sendall(json.dumps(response).encode('utf-8')) + time.sleep(0.1) + + serialized_data = json.dumps(data) + self.client.sendall(serialized_data.encode('utf-8')) + + finally: + self.client.close() + + def send(self, data): + serialized_data = json.dumps(data) + self.client.sendall(serialized_data.encode('utf-8')) + time.sleep(0.1) diff --git a/pilot/utils/arguments.py b/pilot/utils/arguments.py index 8590373..a756f4b 100644 --- a/pilot/utils/arguments.py +++ b/pilot/utils/arguments.py @@ -49,8 +49,4 @@ def get_arguments(): if 'step' not in arguments: arguments['step'] = None - print(colored('\n------------------ STARTING NEW PROJECT ----------------------', 'green', attrs=['bold'])) - print(f"If you wish to continue with this project in future run:") - print(colored(f'python main.py app_id={arguments["app_id"]}', 'green', attrs=['bold'])) - print(colored('--------------------------------------------------------------\n', 'green', attrs=['bold'])) return arguments From bdb4d0dff8a3ea50cd4e5143b68d5653bf08dd57 Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Thu, 31 Aug 2023 08:38:37 +0200 Subject: [PATCH 2/6] Enabled getting user input from the external process --- pilot/helpers/Project.py | 18 +++++++++--------- pilot/helpers/agents/ProductOwner.py | 7 ++++--- pilot/helpers/ipc.py | 27 +++++++-------------------- pilot/utils/questionary.py | 14 ++++++++++---- 4 files changed, 30 insertions(+), 36 deletions(-) diff --git a/pilot/helpers/Project.py b/pilot/helpers/Project.py index 1b54ef4..f54d4e1 100644 --- a/pilot/helpers/Project.py +++ b/pilot/helpers/Project.py @@ -1,6 +1,7 @@ import os import time +from fabulous.color import bold, green, yellow from termcolor import colored from const.common import IGNORE_FOLDERS from database.models.app import App @@ -57,11 +58,10 @@ class Project: if '--external-log-process' in args: self.ipc_client_instance = IPCClient() - print('..'*20) - self.log('\n------------------ STARTING NEW PROJECT ----------------------', 'verbose') + self.log(green(bold('\n------------------ STARTING NEW PROJECT ----------------------')), 'verbose') self.log(f"If you wish to continue with this project in future run:", 'verbose') - self.log(f'python main.py app_id={args["app_id"]}', 'verbose') - self.log('--------------------------------------------------------------\n', 'verbose') + self.log(green(bold(f'python main.py app_id={args["app_id"]}')), 'verbose') + self.log(green(bold('--------------------------------------------------------------\n')), 'verbose') def start(self): self.project_manager = ProductOwner(self) @@ -193,9 +193,9 @@ class Project: delete_unconnected_steps_from(self.checkpoints['last_user_input'], 'previous_step') def ask_for_human_intervention(self, message, description=None, cbs={}): - print(colored(message, "yellow", attrs=['bold'])) + self.log(yellow(bold(message))) if description is not None: - print(description) + self.log(description) answer = '' while answer != 'continue': answer = styled_text( @@ -208,7 +208,7 @@ class Project: elif answer != '': return answer - def log(self, text, message_type, cb=None): + def log(self, text, message_type): if self.ipc_client_instance is None or self.ipc_client_instance.client is None: print(text) else: @@ -216,5 +216,5 @@ class Project: 'type': MESSAGE_TYPE[message_type], 'content': str(text), }) - if cb is not None: - self.ipc_client_instance.listen(lambda response: cb(response['content'])) \ No newline at end of file + if message_type == MESSAGE_TYPE['user_input_request']: + return self.ipc_client_instance.listen() \ No newline at end of file diff --git a/pilot/helpers/agents/ProductOwner.py b/pilot/helpers/agents/ProductOwner.py index d592fb3..84039c1 100644 --- a/pilot/helpers/agents/ProductOwner.py +++ b/pilot/helpers/agents/ProductOwner.py @@ -1,4 +1,5 @@ from termcolor import colored +from fabulous.color import bold, green, yellow from helpers.AgentConvo import AgentConvo from helpers.Agent import Agent @@ -43,7 +44,7 @@ class ProductOwner(Agent): self.project, generate_messages_from_description(main_prompt, self.project.args['app_type'], self.project.args['name'])) - print(colored('Project Summary:\n', 'green', attrs=['bold'])) + self.project.log(green(bold('Project Summary:\n'))) high_level_summary = convo_project_description.send_message('utils/summary.prompt', {'conversation': '\n'.join([f"{msg['role']}: {msg['content']}" for msg in high_level_messages])}) @@ -73,7 +74,7 @@ class ProductOwner(Agent): # USER STORIES msg = f"User Stories:\n" - print(colored(msg, "green", attrs=['bold'])) + self.project.log(green(bold(msg))) logger.info(msg) self.project.user_stories = self.convo_user_stories.continuous_conversation('user_stories/specs.prompt', { @@ -107,7 +108,7 @@ class ProductOwner(Agent): # USER TASKS msg = f"User Tasks:\n" - print(colored(msg, "green", attrs=['bold'])) + self.project.log(green(bold(msg))) logger.info(msg) self.project.user_tasks = self.convo_user_stories.continuous_conversation('user_stories/user_tasks.prompt', diff --git a/pilot/helpers/ipc.py b/pilot/helpers/ipc.py index 785d37b..bd47b63 100644 --- a/pilot/helpers/ipc.py +++ b/pilot/helpers/ipc.py @@ -20,31 +20,18 @@ class IPCClient: print(f"Received request from the external process: {message_content}") return message_content # For demonstration, we're just echoing back the content - def listen(self, cb): + def listen(self): if self.client is None: print("Not connected to the external process!") return - try: - while True: - data = self.client.recv(4096) - message = json.loads(data) - if message['type'] == 'request': - cb(message['content']) - if message['type'] == 'request': - response_content = self.handle_request(message['content']) - response = { - 'type': 'response', - 'content': response_content - } - self.client.sendall(json.dumps(response).encode('utf-8')) - time.sleep(0.1) + while True: + data = self.client.recv(4096) + message = json.loads(data) - serialized_data = json.dumps(data) - self.client.sendall(serialized_data.encode('utf-8')) - - finally: - self.client.close() + if message['type'] == 'response': + # self.client.close() + return message['content'] def send(self, data): serialized_data = json.dumps(data) diff --git a/pilot/utils/questionary.py b/pilot/utils/questionary.py index a9a9506..be1a902 100644 --- a/pilot/utils/questionary.py +++ b/pilot/utils/questionary.py @@ -3,6 +3,7 @@ import questionary from termcolor import colored from database.database import save_user_input, get_user_input_from_hash_id +from const.ipc import MESSAGE_TYPE custom_style = Style.from_dict({ 'question': '#FFFFFF bold', # the color and style of the question @@ -28,10 +29,15 @@ def styled_text(project, question): print(colored(f'{user_input.user_input}', 'yellow', attrs=['bold'])) return user_input.user_input - config = { - 'style': custom_style, - } - response = questionary.text(question, **config).unsafe_ask() # .ask() is included here + if project.ipc_client_instance is None or project.ipc_client_instance.client is None: + config = { + 'style': custom_style, + } + response = questionary.text(question, **config).unsafe_ask() # .ask() is included here + else: + response = project.log(question, MESSAGE_TYPE['user_input_request']) + project.log(response, MESSAGE_TYPE['verbose']) + user_input = save_user_input(project, question, response) print('\n\n', end='') From ca58c4958d148a6868e07a43e91b75b60e2c96de Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Fri, 1 Sep 2023 18:27:00 +0200 Subject: [PATCH 3/6] Implemented final version of IPC communication --- pilot/const/ipc.py | 6 ++-- pilot/database/database.py | 16 ++++++++++ pilot/helpers/Project.py | 35 ++++++++++++++++----- pilot/helpers/ipc.py | 12 ++++++-- pilot/main.py | 58 ++++++++++++++++++++++++++++++++--- pilot/utils/llm_connection.py | 5 ++- pilot/utils/utils.py | 11 +++++++ 7 files changed, 123 insertions(+), 20 deletions(-) diff --git a/pilot/const/ipc.py b/pilot/const/ipc.py index ab1bd5b..22729e5 100644 --- a/pilot/const/ipc.py +++ b/pilot/const/ipc.py @@ -1,5 +1,7 @@ MESSAGE_TYPE = { 'verbose': 'verbose', - 'gpt_stream': 'gpt_stream', - 'user_input_request': 'user_input_request' + 'stream': 'stream', + 'user_input_request': 'user_input_request', + 'info': 'info', + 'local': 'local', } \ No newline at end of file diff --git a/pilot/database/database.py b/pilot/database/database.py index eb5020c..ff4e9b0 100644 --- a/pilot/database/database.py +++ b/pilot/database/database.py @@ -32,6 +32,22 @@ DB_PORT = os.getenv("DB_PORT") DB_USER = os.getenv("DB_USER") DB_PASSWORD = os.getenv("DB_PASSWORD") +def get_created_apps(): + return [model_to_dict(app) for app in App.select()] + +def get_created_apps_with_steps(): + apps = get_created_apps() + for app in apps: + app['id'] = str(app['id']) + app['steps'] = get_progress_steps(app['id']) + app['development_steps'] = get_all_app_development_steps(app['id']) + # TODO this is a quick way to remove the unnecessary fields from the response + app['steps'] = {outer_k: {k: v for k, v in inner_d.items() if k in {'created_at', 'completeted_at', 'completed'}} if inner_d is not None else None for outer_k, inner_d in app['steps'].items()} + app['development_steps'] = [{k: v for k, v in dev_step.items() if k in {'id', 'created_at'}} for dev_step in app['development_steps']] + return apps + +def get_all_app_development_steps(app_id): + return [model_to_dict(dev_step) for dev_step in DevelopmentSteps.select().where(DevelopmentSteps.app == app_id)] def save_user(user_id, email, password): try: diff --git a/pilot/helpers/Project.py b/pilot/helpers/Project.py index f54d4e1..0a37b7c 100644 --- a/pilot/helpers/Project.py +++ b/pilot/helpers/Project.py @@ -1,3 +1,4 @@ +import json import os import time @@ -24,7 +25,7 @@ from utils.files import get_parent_folder class Project: def __init__(self, args, name=None, description=None, user_stories=None, user_tasks=None, architecture=None, - development_plan=None, current_step=None): + development_plan=None, current_step=None, ipc_client_instance=None): self.args = args self.llm_req_num = 0 self.command_runs_count = 0 @@ -38,6 +39,9 @@ class Project: self.root_path = '' self.skip_until_dev_step = None self.skip_steps = None + + self.ipc_client_instance = ipc_client_instance + # self.restore_files({dev_step_id_to_start_from}) if current_step is not None: @@ -55,20 +59,31 @@ class Project: # if development_plan is not None: # self.development_plan = development_plan - if '--external-log-process' in args: - self.ipc_client_instance = IPCClient() + # if '--external-log-process' in args: + # self.ipc_client_instance = IPCClient() + # else: + # self.ipc_client_instance = None - self.log(green(bold('\n------------------ STARTING NEW PROJECT ----------------------')), 'verbose') - self.log(f"If you wish to continue with this project in future run:", 'verbose') - self.log(green(bold(f'python main.py app_id={args["app_id"]}')), 'verbose') - self.log(green(bold('--------------------------------------------------------------\n')), 'verbose') + print(green(bold('\n------------------ STARTING NEW PROJECT ----------------------'))) + print(f"If you wish to continue with this project in future run:") + print(green(bold(f'python main.py app_id={args["app_id"]}'))) + print(green(bold('--------------------------------------------------------------\n'))) def start(self): self.project_manager = ProductOwner(self) + print(json.dumps({ + "project_stage": "project_description" + }), type='info') self.project_manager.get_project_description() + print(json.dumps({ + "project_stage": "user_stories" + }), type='info') self.user_stories = self.project_manager.get_user_stories() # self.user_tasks = self.project_manager.get_user_tasks() + print(json.dumps({ + "project_stage": "architecture" + }), type='info') self.architect = Architect(self) self.architecture = self.architect.get_architecture() @@ -88,8 +103,14 @@ class Project: # TODO END self.developer = Developer(self) + print(json.dumps({ + "project_stage": "environment_setup" + }), type='info') self.developer.set_up_environment(); + print(json.dumps({ + "project_stage": "coding" + }), type='info') self.developer.start_coding() def get_directory_tree(self, with_descriptions=False): diff --git a/pilot/helpers/ipc.py b/pilot/helpers/ipc.py index bd47b63..07c2f5e 100644 --- a/pilot/helpers/ipc.py +++ b/pilot/helpers/ipc.py @@ -3,13 +3,15 @@ import socket import json import time +from utils.utils import json_serial + class IPCClient: - def __init__(self): + def __init__(self, port): self.ready = False client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) print("Connecting to the external process...") try: - client.connect(('localhost', 8124)) + client.connect(('localhost', int(port))) self.client = client print("Connected!") except ConnectionRefusedError: @@ -34,6 +36,10 @@ class IPCClient: return message['content'] def send(self, data): - serialized_data = json.dumps(data) + serialized_data = json.dumps(data, default=json_serial) + print(serialized_data, type='local') + + data_length = len(serialized_data) + self.client.sendall(data_length.to_bytes(4, byteorder='big')) self.client.sendall(serialized_data.encode('utf-8')) time.sleep(0.1) diff --git a/pilot/main.py b/pilot/main.py index b483ce7..402ad8b 100644 --- a/pilot/main.py +++ b/pilot/main.py @@ -1,15 +1,20 @@ # main.py from __future__ import print_function, unicode_literals +import builtins +import json from dotenv import load_dotenv + +from helpers.ipc import IPCClient +from const.ipc import MESSAGE_TYPE +from utils.utils import json_serial load_dotenv() from helpers.Project import Project from utils.arguments import get_arguments from logger.logger import logger -from database.database import database_exists, create_database, tables_exist, create_tables - +from database.database import database_exists, create_database, tables_exist, create_tables, get_created_apps_with_steps def init(): # Check if the "euclid" database exists, if not, create it @@ -27,9 +32,52 @@ def init(): return arguments + + +def get_custom_print(args): + built_in_print = builtins.print + + def print_to_external_process(*args, **kwargs): + # message = " ".join(map(str, args)) + message = args[0] + + if 'type' not in kwargs: + kwargs['type'] = 'verbose' + elif kwargs['type'] == MESSAGE_TYPE['local']: + local_print(*args, **kwargs) + return + + ipc_client_instance.send({ + 'type': MESSAGE_TYPE[kwargs['type']], + 'content': message, + }) + if kwargs['type'] == MESSAGE_TYPE['user_input_request']: + return ipc_client_instance.listen() + + def local_print(*args, **kwargs): + message = " ".join(map(str, args)) + if 'type' in kwargs: + if kwargs['type'] == MESSAGE_TYPE['info']: + return + del kwargs['type'] + + built_in_print(message, **kwargs) + + ipc_client_instance = None + if '--external-log-process-port' in args: + ipc_client_instance = IPCClient(args['--external-log-process-port']) + return print_to_external_process, ipc_client_instance + else: + return local_print, ipc_client_instance + if __name__ == "__main__": args = init() - # TODO get checkpoint from database and fill the project with it - project = Project(args) - project.start() \ No newline at end of file + builtins.print, ipc_client_instance = get_custom_print(args) + + if '--get-created-apps-with-steps' in args: + print({ 'db_data': get_created_apps_with_steps() }, type='info') + else: + # TODO get checkpoint from database and fill the project with it + project = Project(args, ipc_client_instance=ipc_client_instance) + project.start() \ No newline at end of file diff --git a/pilot/utils/llm_connection.py b/pilot/utils/llm_connection.py index c191d8e..4a0f946 100644 --- a/pilot/utils/llm_connection.py +++ b/pilot/utils/llm_connection.py @@ -74,8 +74,6 @@ def num_tokens_from_functions(functions, model="gpt-4"): for o in v['enum']: function_tokens += 3 function_tokens += len(encoding.encode(o)) - # else: - # print(f"Warning: not supported field {field}") function_tokens += 11 num_tokens += function_tokens @@ -85,7 +83,8 @@ def num_tokens_from_functions(functions, model="gpt-4"): def create_gpt_chat_completion(messages: List[dict], req_type, min_tokens=MIN_TOKENS_FOR_GPT_RESPONSE, - function_calls=None): + function_calls=None): + tokens_in_messages = round(get_tokens_in_messages(messages) * 1.2) # add 20% to account for not 100% accuracy if function_calls is not None: tokens_in_messages += round( diff --git a/pilot/utils/utils.py b/pilot/utils/utils.py index 771dafb..deb710d 100644 --- a/pilot/utils/utils.py +++ b/pilot/utils/utils.py @@ -1,7 +1,9 @@ # utils/utils.py +import datetime import os import platform +import uuid import distro import json import hashlib @@ -167,3 +169,12 @@ def clean_filename(filename): cleaned_filename = re.sub(r'\s', '_', cleaned_filename) return cleaned_filename + +def json_serial(obj): + """JSON serializer for objects not serializable by default json code""" + if isinstance(obj, (datetime.datetime, datetime.date)): + return obj.isoformat() + elif isinstance(obj, uuid.UUID): + return str(obj) + else: + return str(obj) \ No newline at end of file From 85ac7e8276d111af96b1c811892632adcf486b5c Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Fri, 1 Sep 2023 18:28:20 +0200 Subject: [PATCH 4/6] Refactored all prints to be colored with fabulous and not termcolor --- pilot/database/database.py | 8 ++++---- pilot/helpers/AgentConvo.py | 8 ++++---- pilot/helpers/Project.py | 5 ++--- pilot/helpers/agents/Architect.py | 4 ++-- pilot/helpers/agents/Developer.py | 12 ++++++------ pilot/helpers/agents/ProductOwner.py | 7 +++---- pilot/helpers/agents/TechLead.py | 4 ++-- pilot/helpers/cli.py | 20 ++++++++++---------- pilot/helpers/files.py | 4 ++-- pilot/prompts/prompts.py | 9 ++++----- pilot/utils/arguments.py | 2 -- pilot/utils/llm_connection.py | 21 ++++++++------------- pilot/utils/questionary.py | 10 +++++----- pilot/utils/utils.py | 4 ++-- 14 files changed, 54 insertions(+), 64 deletions(-) diff --git a/pilot/database/database.py b/pilot/database/database.py index ff4e9b0..51f96f9 100644 --- a/pilot/database/database.py +++ b/pilot/database/database.py @@ -1,6 +1,6 @@ from playhouse.shortcuts import model_to_dict from peewee import * -from termcolor import colored +from fabulous.color import yellow, red from functools import reduce import operator import psycopg2 @@ -210,7 +210,7 @@ def hash_and_save_step(Model, app_id, hash_data_args, data_fields, message): .execute()) record = Model.get_by_id(inserted_id) - logger.debug(colored(f"{message} with id {record.id}", "yellow")) + logger.debug(yellow(f"{message} with id {record.id}")) except IntegrityError: print(f"A record with hash_id {hash_id} already exists for {Model.__name__}.") return None @@ -308,7 +308,7 @@ def delete_all_subsequent_steps(project): def delete_subsequent_steps(model, step): if step is None: return - logger.info(colored(f"Deleting subsequent {model.__name__} steps after {step.id}", "red")) + logger.info(red(f"Deleting subsequent {model.__name__} steps after {step.id}")) subsequent_steps = model.select().where(model.previous_step == step.id) for subsequent_step in subsequent_steps: if subsequent_step: @@ -344,7 +344,7 @@ def delete_unconnected_steps_from(step, previous_step_field_name): ).order_by(DevelopmentSteps.id.desc()) for unconnected_step in unconnected_steps: - print(colored(f"Deleting unconnected {step.__class__.__name__} step {unconnected_step.id}", "red")) + print(red(f"Deleting unconnected {step.__class__.__name__} step {unconnected_step.id}")) unconnected_step.delete_instance() diff --git a/pilot/helpers/AgentConvo.py b/pilot/helpers/AgentConvo.py index b5bac8e..5bb7a19 100644 --- a/pilot/helpers/AgentConvo.py +++ b/pilot/helpers/AgentConvo.py @@ -1,5 +1,5 @@ import subprocess -from termcolor import colored +from fabulous.color import yellow, bold from database.database import get_development_step_from_hash_id, save_development_step, delete_all_subsequent_steps from utils.utils import array_of_objects_to_string @@ -34,7 +34,7 @@ class AgentConvo: development_step = get_development_step_from_hash_id(self.agent.project, prompt_path, prompt_data, self.agent.project.llm_req_num) if development_step is not None and self.agent.project.skip_steps: # if we do, use it - print(colored(f'Restoring development step with id {development_step.id}', 'yellow')) + print(yellow(f'Restoring development step with id {development_step.id}')) self.agent.project.checkpoints['last_development_step'] = development_step self.agent.project.restore_files(development_step.id) response = development_step.llm_response @@ -90,7 +90,7 @@ class AgentConvo: # Continue conversation until GPT response equals END_RESPONSE while response != END_RESPONSE: - print(colored("Do you want to add anything else? If not, ", 'yellow') + colored('just press ENTER.', 'yellow', attrs=['bold'])) + print(yellow("Do you want to add anything else? If not, ") + yellow(bold('just press ENTER.'))) user_message = ask_user(self.agent.project, response, False) if user_message == "": @@ -125,7 +125,7 @@ class AgentConvo: print_msg = capitalize_first_word_with_underscores(self.high_level_step) if self.log_to_user: if self.agent.project.checkpoints['last_development_step'] is not None: - print(colored("\nDev step ", 'yellow') + colored(self.agent.project.checkpoints['last_development_step'], 'yellow', attrs=['bold']) + '\n', end='') + print(yellow("\nDev step ") + yellow(bold(self.agent.project.checkpoints['last_development_step'])) + '\n', end='') print(f"\n{content}\n") logger.info(f"{print_msg}: {content}\n") diff --git a/pilot/helpers/Project.py b/pilot/helpers/Project.py index 0a37b7c..c666b31 100644 --- a/pilot/helpers/Project.py +++ b/pilot/helpers/Project.py @@ -3,7 +3,6 @@ import os import time from fabulous.color import bold, green, yellow -from termcolor import colored from const.common import IGNORE_FOLDERS from database.models.app import App from database.database import get_app, delete_unconnected_steps_from, delete_all_app_development_data @@ -214,9 +213,9 @@ class Project: delete_unconnected_steps_from(self.checkpoints['last_user_input'], 'previous_step') def ask_for_human_intervention(self, message, description=None, cbs={}): - self.log(yellow(bold(message))) + print(yellow(bold(message))) if description is not None: - self.log(description) + print(description) answer = '' while answer != 'continue': answer = styled_text( diff --git a/pilot/helpers/agents/Architect.py b/pilot/helpers/agents/Architect.py index 6237c4b..b1ac8b9 100644 --- a/pilot/helpers/agents/Architect.py +++ b/pilot/helpers/agents/Architect.py @@ -1,7 +1,7 @@ from utils.utils import step_already_finished from helpers.Agent import Agent import json -from termcolor import colored +from fabulous.color import green, bold from const.function_calls import ARCHITECTURE from utils.utils import execute_step, find_role_from_step, generate_app_data @@ -27,7 +27,7 @@ class Architect(Agent): return step['architecture'] # ARCHITECTURE - print(colored(f"Planning project architecture...\n", "green", attrs=['bold'])) + print(green(bold(f"Planning project architecture...\n"))) logger.info(f"Planning project architecture...") architecture = self.convo_architecture.send_message('architecture/technologies.prompt', diff --git a/pilot/helpers/agents/Developer.py b/pilot/helpers/agents/Developer.py index cb451e4..db06269 100644 --- a/pilot/helpers/agents/Developer.py +++ b/pilot/helpers/agents/Developer.py @@ -1,6 +1,6 @@ import json import uuid -from termcolor import colored +from fabulous.color import yellow, green, red, bold from utils.questionary import styled_text from helpers.files import update_file from utils.utils import step_already_finished @@ -26,7 +26,7 @@ class Developer(Agent): self.project.skip_steps = False if ('skip_until_dev_step' in self.project.args and self.project.args['skip_until_dev_step'] == '0') else True # DEVELOPMENT - print(colored(f"Ok, great, now, let's start with the actual development...\n", "green", attrs=['bold'])) + print(green(bold(f"Ok, great, now, let's start with the actual development...\n"))) logger.info(f"Starting to create the actual code...") self.implement_task() @@ -100,9 +100,9 @@ class Developer(Agent): elif should_rerun_command == 'YES': cli_response, llm_response = execute_command_and_check_cli_response(test_command['command'], test_command['timeout'], convo) if llm_response == 'NEEDS_DEBUGGING': - print(colored(f'Got incorrect CLI response:', 'red')) + print(red(f'Got incorrect CLI response:')) print(cli_response) - print(colored('-------------------', 'red')) + print(red('-------------------')) if llm_response == 'DONE': return True @@ -119,7 +119,7 @@ class Developer(Agent): while True: # TODO add description about how can the user check if the app works user_feedback = self.project.ask_for_human_intervention( - 'Can you check if the app works?\nIf you want to run the app, ' + colored('just type "r" and press ENTER', 'yellow', attrs=['bold']), + 'Can you check if the app works?\nIf you want to run the app, ' + yellow(bold('just type "r" and press ENTER')), cbs={ 'r': lambda: run_command_until_success(self.run_command, None, iteration_convo, force=True) }) if user_feedback == 'continue': @@ -165,7 +165,7 @@ class Developer(Agent): }) return # ENVIRONMENT SETUP - print(colored(f"Setting up the environment...\n", "green")) + print(green(f"Setting up the environment...\n")) logger.info(f"Setting up the environment...") os_info = get_os_info() diff --git a/pilot/helpers/agents/ProductOwner.py b/pilot/helpers/agents/ProductOwner.py index 84039c1..1e8b8de 100644 --- a/pilot/helpers/agents/ProductOwner.py +++ b/pilot/helpers/agents/ProductOwner.py @@ -1,4 +1,3 @@ -from termcolor import colored from fabulous.color import bold, green, yellow from helpers.AgentConvo import AgentConvo @@ -44,7 +43,7 @@ class ProductOwner(Agent): self.project, generate_messages_from_description(main_prompt, self.project.args['app_type'], self.project.args['name'])) - self.project.log(green(bold('Project Summary:\n'))) + print(green(bold('Project Summary:\n'))) high_level_summary = convo_project_description.send_message('utils/summary.prompt', {'conversation': '\n'.join([f"{msg['role']}: {msg['content']}" for msg in high_level_messages])}) @@ -74,7 +73,7 @@ class ProductOwner(Agent): # USER STORIES msg = f"User Stories:\n" - self.project.log(green(bold(msg))) + print(green(bold(msg))) logger.info(msg) self.project.user_stories = self.convo_user_stories.continuous_conversation('user_stories/specs.prompt', { @@ -108,7 +107,7 @@ class ProductOwner(Agent): # USER TASKS msg = f"User Tasks:\n" - self.project.log(green(bold(msg))) + print(green(bold(msg))) logger.info(msg) self.project.user_tasks = self.convo_user_stories.continuous_conversation('user_stories/user_tasks.prompt', diff --git a/pilot/helpers/agents/TechLead.py b/pilot/helpers/agents/TechLead.py index 9ec5e89..fa0cefc 100644 --- a/pilot/helpers/agents/TechLead.py +++ b/pilot/helpers/agents/TechLead.py @@ -1,7 +1,7 @@ from utils.utils import step_already_finished from helpers.Agent import Agent import json -from termcolor import colored +from fabulous.color import green, bold from const.function_calls import DEV_STEPS from helpers.cli import build_directory_tree from helpers.AgentConvo import AgentConvo @@ -29,7 +29,7 @@ class TechLead(Agent): return step['development_plan'] # DEVELOPMENT PLANNING - print(colored(f"Starting to create the action plan for development...\n", "green", attrs=['bold'])) + print(green(bold(f"Starting to create the action plan for development...\n"))) logger.info(f"Starting to create the action plan for development...") # TODO add clarifications diff --git a/pilot/helpers/cli.py b/pilot/helpers/cli.py index 48413c6..df6508a 100644 --- a/pilot/helpers/cli.py +++ b/pilot/helpers/cli.py @@ -7,7 +7,7 @@ import time import uuid import platform -from termcolor import colored +from fabulous.color import yellow, green, white, red, bold from database.database import get_command_run_from_hash_id, save_command_run from const.function_calls import DEBUG_STEPS_BREAKDOWN @@ -74,8 +74,8 @@ def execute_command(project, command, timeout=None, force=False): timeout = min(max(timeout, MIN_COMMAND_RUN_TIME), MAX_COMMAND_RUN_TIME) if not force: - print(colored(f'\n--------- EXECUTE COMMAND ----------', 'yellow', attrs=['bold'])) - print(colored(f'Can i execute the command: `') + colored(command, 'yellow', attrs=['bold']) + colored(f'` with {timeout}ms timeout?')) + print(yellow(bold(f'\n--------- EXECUTE COMMAND ----------'))) + print(f'Can i execute the command: `' + yellow(bold(command)) + f'` with {timeout}ms timeout?') answer = styled_text( project, @@ -87,7 +87,7 @@ def execute_command(project, command, timeout=None, force=False): if command_run is not None and project.skip_steps: # if we do, use it project.checkpoints['last_command_run'] = command_run - print(colored(f'Restoring command run response id {command_run.id}:\n```\n{command_run.cli_response}```', 'yellow')) + print(yellow(f'Restoring command run response id {command_run.id}:\n```\n{command_run.cli_response}```')) return command_run.cli_response return_value = None @@ -105,7 +105,7 @@ def execute_command(project, command, timeout=None, force=False): while True and return_value is None: elapsed_time = time.time() - start_time if timeout is not None: - print(colored(f'\rt: {round(elapsed_time * 1000)}ms : ', 'white', attrs=['bold']), end='', flush=True) + print(white(bold(f'\rt: {round(elapsed_time * 1000)}ms : ')), end='', flush=True) # Check if process has finished if process.poll() is not None: @@ -114,7 +114,7 @@ def execute_command(project, command, timeout=None, force=False): while not q.empty(): output_line = q.get_nowait() if output_line not in output: - print(colored('CLI OUTPUT:', 'green') + output_line, end='') + print(green('CLI OUTPUT:') + output_line, end='') output += output_line break @@ -131,7 +131,7 @@ def execute_command(project, command, timeout=None, force=False): if line: output += line - print(colored('CLI OUTPUT:', 'green') + line, end='') + print(green('CLI OUTPUT:') + line, end='') # Read stderr try: @@ -141,7 +141,7 @@ def execute_command(project, command, timeout=None, force=False): if stderr_line: stderr_output += stderr_line - print(colored('CLI ERROR:', 'red') + stderr_line, end='') # Print with different color for distinction + print(red('CLI ERROR:') + stderr_line, end='') # Print with different color for distinction except (KeyboardInterrupt, TimeoutError) as e: interrupted = True @@ -215,9 +215,9 @@ def run_command_until_success(command, timeout, convo, additional_message=None, {'cli_response': cli_response, 'command': command, 'additional_message': additional_message}) if response != 'DONE': - print(colored(f'Got incorrect CLI response:', 'red')) + print(red(f'Got incorrect CLI response:')) print(cli_response) - print(colored('-------------------', 'red')) + print(red('-------------------')) debug(convo, {'command': command, 'timeout': timeout}) diff --git a/pilot/helpers/files.py b/pilot/helpers/files.py index 24f2d75..9145e99 100644 --- a/pilot/helpers/files.py +++ b/pilot/helpers/files.py @@ -1,4 +1,4 @@ -from termcolor import colored +from fabulous.color import green import os @@ -11,7 +11,7 @@ def update_file(path, new_content): # Write content to the file with open(path, 'w') as file: file.write(new_content) - print(colored(f"Updated file {path}", "green")) + print(green(f"Updated file {path}")) def get_files_content(directory, ignore=[]): return_array = [] diff --git a/pilot/prompts/prompts.py b/pilot/prompts/prompts.py index c667268..fa59535 100644 --- a/pilot/prompts/prompts.py +++ b/pilot/prompts/prompts.py @@ -1,6 +1,6 @@ # prompts/prompts.py -from termcolor import colored +from fabulous.color import yellow import questionary from const import common @@ -102,14 +102,13 @@ def get_additional_info_from_user(project, messages, role): while True: if isinstance(message, dict) and 'text' in message: message = message['text'] - print(colored( - f"Please check this message and say what needs to be changed. If everything is ok just press ENTER", - "yellow")) + print(yellow(f"Please check this message and say what needs to be changed. If everything is ok just press ENTER",)) answer = ask_user(project, message, False) if answer.lower() == '': break response = create_gpt_chat_completion( - generate_messages_from_custom_conversation(role, [get_prompt('utils/update.prompt'), message, answer], 'user'), 'additional_info') + generate_messages_from_custom_conversation(role, [get_prompt('utils/update.prompt'), message, answer], 'user'), + 'additional_info') message = response diff --git a/pilot/utils/arguments.py b/pilot/utils/arguments.py index a756f4b..14e0718 100644 --- a/pilot/utils/arguments.py +++ b/pilot/utils/arguments.py @@ -1,8 +1,6 @@ import sys import uuid -from termcolor import colored - from database.database import get_app diff --git a/pilot/utils/llm_connection.py b/pilot/utils/llm_connection.py index 4a0f946..345b765 100644 --- a/pilot/utils/llm_connection.py +++ b/pilot/utils/llm_connection.py @@ -10,15 +10,11 @@ from jinja2 import Environment, FileSystemLoader from const.llm import MIN_TOKENS_FOR_GPT_RESPONSE, MAX_GPT_MODEL_TOKENS, MAX_QUESTIONS, END_RESPONSE from logger.logger import logger -from termcolor import colored +from fabulous.color import red from utils.utils import get_prompt_components, fix_json from utils.spinner import spinner_start, spinner_stop -def connect_to_llm(): - pass - - def get_prompt(prompt_name, data=None): if data is None: data = {} @@ -115,8 +111,7 @@ def create_gpt_chat_completion(messages: List[dict], req_type, min_tokens=MIN_TO response = stream_gpt_completion(gpt_data, req_type) return response except Exception as e: - print( - 'The request to OpenAI API failed. Here is the error message:') + print('The request to OpenAI API failed. Here is the error message:') print(e) @@ -139,7 +134,7 @@ def retry_on_exception(func): try: return func(*args, **kwargs) except Exception as e: - print(colored(f'There was a problem with request to openai API:', 'red')) + print(red(f'There was a problem with request to openai API:')) print(str(e)) user_message = questionary.text( "Do you want to try make the same request again? If yes, just press ENTER. Otherwise, type 'no'.", @@ -168,8 +163,7 @@ def stream_gpt_completion(data, req_type): delete_last_n_lines(lines_printed) return result_data - # spinner = spinner_start(colored("Waiting for OpenAI API response...", 'yellow')) - # print(colored("Stream response from OpenAI:", 'yellow')) + # spinner = spinner_start(yellow("Waiting for OpenAI API response...")) api_key = os.getenv("OPENAI_API_KEY") logger.info(f'Request data: {data}') @@ -191,6 +185,7 @@ def stream_gpt_completion(data, req_type): gpt_response = '' function_calls = {'name': '', 'arguments': ''} + for line in response.iter_lines(): # Ignore keep-alive new lines if line: @@ -226,7 +221,7 @@ def stream_gpt_completion(data, req_type): if 'arguments' in json_line['function_call']: function_calls['arguments'] += json_line['function_call']['arguments'] - print(json_line['function_call']['arguments'], end='', flush=True) + print(json_line['function_call']['arguments'], type='stream', end='', flush=True) if 'content' in json_line: content = json_line.get('content') @@ -239,9 +234,9 @@ def stream_gpt_completion(data, req_type): buffer = "" # reset the buffer gpt_response += content - print(content, end='', flush=True) + print(content, type='stream', end='', flush=True) - print('\n') + print('\n', type='stream') if function_calls['arguments'] != '': logger.info(f'Response via function call: {function_calls["arguments"]}') function_calls['arguments'] = load_data_to_json(function_calls['arguments']) diff --git a/pilot/utils/questionary.py b/pilot/utils/questionary.py index be1a902..50f53a0 100644 --- a/pilot/utils/questionary.py +++ b/pilot/utils/questionary.py @@ -1,6 +1,6 @@ from prompt_toolkit.styles import Style import questionary -from termcolor import colored +from fabulous.color import yellow, bold from database.database import save_user_input, get_user_input_from_hash_id from const.ipc import MESSAGE_TYPE @@ -25,8 +25,8 @@ def styled_text(project, question): if user_input is not None and user_input.user_input is not None and project.skip_steps: # if we do, use it project.checkpoints['last_user_input'] = user_input - print(colored(f'Restoring user input id {user_input.id}: ', 'yellow'), end='') - print(colored(f'{user_input.user_input}', 'yellow', attrs=['bold'])) + print(yellow(bold(f'Restoring user input id {user_input.id}: ')), end='') + print(yellow(bold(f'{user_input.user_input}'))) return user_input.user_input if project.ipc_client_instance is None or project.ipc_client_instance.client is None: @@ -35,8 +35,8 @@ def styled_text(project, question): } response = questionary.text(question, **config).unsafe_ask() # .ask() is included here else: - response = project.log(question, MESSAGE_TYPE['user_input_request']) - project.log(response, MESSAGE_TYPE['verbose']) + response = print(question, type='user_input_request') + print(response) user_input = save_user_input(project, question, response) diff --git a/pilot/utils/utils.py b/pilot/utils/utils.py index deb710d..994cf0d 100644 --- a/pilot/utils/utils.py +++ b/pilot/utils/utils.py @@ -9,7 +9,7 @@ import json import hashlib import re from jinja2 import Environment, FileSystemLoader -from termcolor import colored +from fabulous.color import green from const.llm import MAX_QUESTIONS, END_RESPONSE from const.common import ROLES, STEPS @@ -121,7 +121,7 @@ def step_already_finished(args, step): args.update(step['app_data']) message = f"{capitalize_first_word_with_underscores(step['step'])} already done for this app_id: {args['app_id']}. Moving to next step..." - print(colored(message, "green")) + print(green(message)) logger.info(message) From 28d014353611aa9c3abb3e8fb8ebc6dddc387327 Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Fri, 1 Sep 2023 18:29:31 +0200 Subject: [PATCH 5/6] TEMP fix --- pilot/utils/llm_connection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pilot/utils/llm_connection.py b/pilot/utils/llm_connection.py index 345b765..86a9bbe 100644 --- a/pilot/utils/llm_connection.py +++ b/pilot/utils/llm_connection.py @@ -151,7 +151,8 @@ def retry_on_exception(func): @retry_on_exception def stream_gpt_completion(data, req_type): - terminal_width = os.get_terminal_size().columns + # TODO add type dynamically - this isn't working when connected to the external process + terminal_width = 50#os.get_terminal_size().columns lines_printed = 2 buffer = "" # A buffer to accumulate incoming data From 0488358c9baec8dc26002202b10064ede4e60fe1 Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Fri, 1 Sep 2023 18:29:49 +0200 Subject: [PATCH 6/6] Remove of unnecessary comments --- pilot/helpers/Project.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pilot/helpers/Project.py b/pilot/helpers/Project.py index c666b31..c2dbf90 100644 --- a/pilot/helpers/Project.py +++ b/pilot/helpers/Project.py @@ -58,11 +58,6 @@ class Project: # if development_plan is not None: # self.development_plan = development_plan - # if '--external-log-process' in args: - # self.ipc_client_instance = IPCClient() - # else: - # self.ipc_client_instance = None - print(green(bold('\n------------------ STARTING NEW PROJECT ----------------------'))) print(f"If you wish to continue with this project in future run:") print(green(bold(f'python main.py app_id={args["app_id"]}')))