diff --git a/pilot/const/ipc.py b/pilot/const/ipc.py index ab1bd5b..22729e5 100644 --- a/pilot/const/ipc.py +++ b/pilot/const/ipc.py @@ -1,5 +1,7 @@ MESSAGE_TYPE = { 'verbose': 'verbose', - 'gpt_stream': 'gpt_stream', - 'user_input_request': 'user_input_request' + 'stream': 'stream', + 'user_input_request': 'user_input_request', + 'info': 'info', + 'local': 'local', } \ No newline at end of file diff --git a/pilot/database/database.py b/pilot/database/database.py index eb5020c..ff4e9b0 100644 --- a/pilot/database/database.py +++ b/pilot/database/database.py @@ -32,6 +32,22 @@ DB_PORT = os.getenv("DB_PORT") DB_USER = os.getenv("DB_USER") DB_PASSWORD = os.getenv("DB_PASSWORD") +def get_created_apps(): + return [model_to_dict(app) for app in App.select()] + +def get_created_apps_with_steps(): + apps = get_created_apps() + for app in apps: + app['id'] = str(app['id']) + app['steps'] = get_progress_steps(app['id']) + app['development_steps'] = get_all_app_development_steps(app['id']) + # TODO this is a quick way to remove the unnecessary fields from the response + app['steps'] = {outer_k: {k: v for k, v in inner_d.items() if k in {'created_at', 'completeted_at', 'completed'}} if inner_d is not None else None for outer_k, inner_d in app['steps'].items()} + app['development_steps'] = [{k: v for k, v in dev_step.items() if k in {'id', 'created_at'}} for dev_step in app['development_steps']] + return apps + +def get_all_app_development_steps(app_id): + return [model_to_dict(dev_step) for dev_step in DevelopmentSteps.select().where(DevelopmentSteps.app == app_id)] def save_user(user_id, email, password): try: diff --git a/pilot/helpers/Project.py b/pilot/helpers/Project.py index f54d4e1..0a37b7c 100644 --- a/pilot/helpers/Project.py +++ b/pilot/helpers/Project.py @@ -1,3 +1,4 @@ +import json import os import time @@ -24,7 +25,7 @@ from utils.files import get_parent_folder class Project: def __init__(self, args, name=None, description=None, user_stories=None, user_tasks=None, architecture=None, - development_plan=None, current_step=None): + development_plan=None, current_step=None, ipc_client_instance=None): self.args = args self.llm_req_num = 0 self.command_runs_count = 0 @@ -38,6 +39,9 @@ class Project: self.root_path = '' self.skip_until_dev_step = None self.skip_steps = None + + self.ipc_client_instance = ipc_client_instance + # self.restore_files({dev_step_id_to_start_from}) if current_step is not None: @@ -55,20 +59,31 @@ class Project: # if development_plan is not None: # self.development_plan = development_plan - if '--external-log-process' in args: - self.ipc_client_instance = IPCClient() + # if '--external-log-process' in args: + # self.ipc_client_instance = IPCClient() + # else: + # self.ipc_client_instance = None - self.log(green(bold('\n------------------ STARTING NEW PROJECT ----------------------')), 'verbose') - self.log(f"If you wish to continue with this project in future run:", 'verbose') - self.log(green(bold(f'python main.py app_id={args["app_id"]}')), 'verbose') - self.log(green(bold('--------------------------------------------------------------\n')), 'verbose') + print(green(bold('\n------------------ STARTING NEW PROJECT ----------------------'))) + print(f"If you wish to continue with this project in future run:") + print(green(bold(f'python main.py app_id={args["app_id"]}'))) + print(green(bold('--------------------------------------------------------------\n'))) def start(self): self.project_manager = ProductOwner(self) + print(json.dumps({ + "project_stage": "project_description" + }), type='info') self.project_manager.get_project_description() + print(json.dumps({ + "project_stage": "user_stories" + }), type='info') self.user_stories = self.project_manager.get_user_stories() # self.user_tasks = self.project_manager.get_user_tasks() + print(json.dumps({ + "project_stage": "architecture" + }), type='info') self.architect = Architect(self) self.architecture = self.architect.get_architecture() @@ -88,8 +103,14 @@ class Project: # TODO END self.developer = Developer(self) + print(json.dumps({ + "project_stage": "environment_setup" + }), type='info') self.developer.set_up_environment(); + print(json.dumps({ + "project_stage": "coding" + }), type='info') self.developer.start_coding() def get_directory_tree(self, with_descriptions=False): diff --git a/pilot/helpers/ipc.py b/pilot/helpers/ipc.py index bd47b63..07c2f5e 100644 --- a/pilot/helpers/ipc.py +++ b/pilot/helpers/ipc.py @@ -3,13 +3,15 @@ import socket import json import time +from utils.utils import json_serial + class IPCClient: - def __init__(self): + def __init__(self, port): self.ready = False client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) print("Connecting to the external process...") try: - client.connect(('localhost', 8124)) + client.connect(('localhost', int(port))) self.client = client print("Connected!") except ConnectionRefusedError: @@ -34,6 +36,10 @@ class IPCClient: return message['content'] def send(self, data): - serialized_data = json.dumps(data) + serialized_data = json.dumps(data, default=json_serial) + print(serialized_data, type='local') + + data_length = len(serialized_data) + self.client.sendall(data_length.to_bytes(4, byteorder='big')) self.client.sendall(serialized_data.encode('utf-8')) time.sleep(0.1) diff --git a/pilot/main.py b/pilot/main.py index b483ce7..402ad8b 100644 --- a/pilot/main.py +++ b/pilot/main.py @@ -1,15 +1,20 @@ # main.py from __future__ import print_function, unicode_literals +import builtins +import json from dotenv import load_dotenv + +from helpers.ipc import IPCClient +from const.ipc import MESSAGE_TYPE +from utils.utils import json_serial load_dotenv() from helpers.Project import Project from utils.arguments import get_arguments from logger.logger import logger -from database.database import database_exists, create_database, tables_exist, create_tables - +from database.database import database_exists, create_database, tables_exist, create_tables, get_created_apps_with_steps def init(): # Check if the "euclid" database exists, if not, create it @@ -27,9 +32,52 @@ def init(): return arguments + + +def get_custom_print(args): + built_in_print = builtins.print + + def print_to_external_process(*args, **kwargs): + # message = " ".join(map(str, args)) + message = args[0] + + if 'type' not in kwargs: + kwargs['type'] = 'verbose' + elif kwargs['type'] == MESSAGE_TYPE['local']: + local_print(*args, **kwargs) + return + + ipc_client_instance.send({ + 'type': MESSAGE_TYPE[kwargs['type']], + 'content': message, + }) + if kwargs['type'] == MESSAGE_TYPE['user_input_request']: + return ipc_client_instance.listen() + + def local_print(*args, **kwargs): + message = " ".join(map(str, args)) + if 'type' in kwargs: + if kwargs['type'] == MESSAGE_TYPE['info']: + return + del kwargs['type'] + + built_in_print(message, **kwargs) + + ipc_client_instance = None + if '--external-log-process-port' in args: + ipc_client_instance = IPCClient(args['--external-log-process-port']) + return print_to_external_process, ipc_client_instance + else: + return local_print, ipc_client_instance + if __name__ == "__main__": args = init() - # TODO get checkpoint from database and fill the project with it - project = Project(args) - project.start() \ No newline at end of file + builtins.print, ipc_client_instance = get_custom_print(args) + + if '--get-created-apps-with-steps' in args: + print({ 'db_data': get_created_apps_with_steps() }, type='info') + else: + # TODO get checkpoint from database and fill the project with it + project = Project(args, ipc_client_instance=ipc_client_instance) + project.start() \ No newline at end of file diff --git a/pilot/utils/llm_connection.py b/pilot/utils/llm_connection.py index c191d8e..4a0f946 100644 --- a/pilot/utils/llm_connection.py +++ b/pilot/utils/llm_connection.py @@ -74,8 +74,6 @@ def num_tokens_from_functions(functions, model="gpt-4"): for o in v['enum']: function_tokens += 3 function_tokens += len(encoding.encode(o)) - # else: - # print(f"Warning: not supported field {field}") function_tokens += 11 num_tokens += function_tokens @@ -85,7 +83,8 @@ def num_tokens_from_functions(functions, model="gpt-4"): def create_gpt_chat_completion(messages: List[dict], req_type, min_tokens=MIN_TOKENS_FOR_GPT_RESPONSE, - function_calls=None): + function_calls=None): + tokens_in_messages = round(get_tokens_in_messages(messages) * 1.2) # add 20% to account for not 100% accuracy if function_calls is not None: tokens_in_messages += round( diff --git a/pilot/utils/utils.py b/pilot/utils/utils.py index 771dafb..deb710d 100644 --- a/pilot/utils/utils.py +++ b/pilot/utils/utils.py @@ -1,7 +1,9 @@ # utils/utils.py +import datetime import os import platform +import uuid import distro import json import hashlib @@ -167,3 +169,12 @@ def clean_filename(filename): cleaned_filename = re.sub(r'\s', '_', cleaned_filename) return cleaned_filename + +def json_serial(obj): + """JSON serializer for objects not serializable by default json code""" + if isinstance(obj, (datetime.datetime, datetime.date)): + return obj.isoformat() + elif isinstance(obj, uuid.UUID): + return str(obj) + else: + return str(obj) \ No newline at end of file