Implemented final version of IPC communication

pilot/const/ipc.py

@@ -1,5 +1,7 @@
 MESSAGE_TYPE = {
     'verbose': 'verbose',
-    'gpt_stream': 'gpt_stream',
-    'user_input_request': 'user_input_request'
+    'stream': 'stream',
+    'user_input_request': 'user_input_request',
+    'info': 'info',
+    'local': 'local',
 }

pilot/database/database.py

@@ -32,6 +32,22 @@ DB_PORT = os.getenv("DB_PORT")
 DB_USER = os.getenv("DB_USER")
 DB_PASSWORD = os.getenv("DB_PASSWORD")
 
+def get_created_apps():
+    return [model_to_dict(app) for app in App.select()]
+
+def get_created_apps_with_steps():
+    apps = get_created_apps()
+    for app in apps:
+        app['id'] = str(app['id'])
+        app['steps'] = get_progress_steps(app['id'])
+        app['development_steps'] = get_all_app_development_steps(app['id'])
+        # TODO this is a quick way to remove the unnecessary fields from the response
+        app['steps'] = {outer_k: {k: v for k, v in inner_d.items() if k in {'created_at', 'completeted_at', 'completed'}} if inner_d is not None else None for outer_k, inner_d in app['steps'].items()}
+        app['development_steps'] = [{k: v for k, v in dev_step.items() if k in {'id', 'created_at'}} for dev_step in app['development_steps']]
+    return apps
+
+def get_all_app_development_steps(app_id):
+    return [model_to_dict(dev_step) for dev_step in DevelopmentSteps.select().where(DevelopmentSteps.app == app_id)]
 
 def save_user(user_id, email, password):
     try:

pilot/helpers/Project.py

@@ -1,3 +1,4 @@
+import json
 import os
 import time
 
@@ -24,7 +25,7 @@ from utils.files import get_parent_folder
 
 class Project:
     def __init__(self, args, name=None, description=None, user_stories=None, user_tasks=None, architecture=None,
-                 development_plan=None, current_step=None):
+                 development_plan=None, current_step=None, ipc_client_instance=None):
         self.args = args
         self.llm_req_num = 0
         self.command_runs_count = 0
@@ -38,6 +39,9 @@ class Project:
         self.root_path = ''
         self.skip_until_dev_step = None
         self.skip_steps = None
+
+        self.ipc_client_instance = ipc_client_instance
+
         # self.restore_files({dev_step_id_to_start_from})
 
         if current_step is not None:
@@ -55,20 +59,31 @@ class Project:
         # if development_plan is not None:
         #     self.development_plan = development_plan
 
-        if '--external-log-process' in args:
-            self.ipc_client_instance = IPCClient()
+        # if '--external-log-process' in args:
+        #     self.ipc_client_instance = IPCClient()
+        # else:
+        #     self.ipc_client_instance = None
 
-        self.log(green(bold('\n------------------ STARTING NEW PROJECT ----------------------')), 'verbose')
-        self.log(f"If you wish to continue with this project in future run:", 'verbose')
-        self.log(green(bold(f'python main.py app_id={args["app_id"]}')), 'verbose')
-        self.log(green(bold('--------------------------------------------------------------\n')), 'verbose')
+        print(green(bold('\n------------------ STARTING NEW PROJECT ----------------------')))
+        print(f"If you wish to continue with this project in future run:")
+        print(green(bold(f'python main.py app_id={args["app_id"]}')))
+        print(green(bold('--------------------------------------------------------------\n')))
 
     def start(self):
         self.project_manager = ProductOwner(self)
+        print(json.dumps({
+            "project_stage": "project_description"
+        }), type='info')
         self.project_manager.get_project_description()
+        print(json.dumps({
+            "project_stage": "user_stories"
+        }), type='info')
         self.user_stories = self.project_manager.get_user_stories()
         # self.user_tasks = self.project_manager.get_user_tasks()
 
+        print(json.dumps({
+            "project_stage": "architecture"
+        }), type='info')
         self.architect = Architect(self)
         self.architecture = self.architect.get_architecture()
 
@@ -88,8 +103,14 @@ class Project:
         # TODO END
 
         self.developer = Developer(self)
+        print(json.dumps({
+            "project_stage": "environment_setup"
+        }), type='info')
         self.developer.set_up_environment();
 
+        print(json.dumps({
+            "project_stage": "coding"
+        }), type='info')
         self.developer.start_coding()
 
     def get_directory_tree(self, with_descriptions=False):

pilot/helpers/ipc.py

@@ -3,13 +3,15 @@ import socket
 import json
 import time
 
+from utils.utils import json_serial
+
 class IPCClient:
-    def __init__(self):
+    def __init__(self, port):
         self.ready = False
         client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         print("Connecting to the external process...")
         try:
-            client.connect(('localhost', 8124))
+            client.connect(('localhost', int(port)))
             self.client = client
             print("Connected!")
         except ConnectionRefusedError:
@@ -34,6 +36,10 @@ class IPCClient:
                 return message['content']
 
     def send(self, data):
-        serialized_data = json.dumps(data)
+        serialized_data = json.dumps(data, default=json_serial)
+        print(serialized_data, type='local')
+
+        data_length = len(serialized_data)
+        self.client.sendall(data_length.to_bytes(4, byteorder='big'))
         self.client.sendall(serialized_data.encode('utf-8'))
         time.sleep(0.1)

pilot/main.py

@@ -1,15 +1,20 @@
 # main.py
 from __future__ import print_function, unicode_literals
+import builtins
+import json
 
 from dotenv import load_dotenv
+
+from helpers.ipc import IPCClient
+from const.ipc import MESSAGE_TYPE
+from utils.utils import json_serial
 load_dotenv()
 
 from helpers.Project import Project
 
 from utils.arguments import get_arguments
 from logger.logger import logger
-from database.database import database_exists, create_database, tables_exist, create_tables
-
+from database.database import database_exists, create_database, tables_exist, create_tables, get_created_apps_with_steps
 
 def init():
     # Check if the "euclid" database exists, if not, create it
@@ -27,9 +32,52 @@ def init():
     return arguments
 
 
+        
+    
+def get_custom_print(args):
+    built_in_print = builtins.print
+
+    def print_to_external_process(*args, **kwargs):
+        # message = " ".join(map(str, args))
+        message = args[0]
+
+        if 'type' not in kwargs:
+            kwargs['type'] = 'verbose'
+        elif kwargs['type'] == MESSAGE_TYPE['local']:
+            local_print(*args, **kwargs)
+            return
+
+        ipc_client_instance.send({
+            'type': MESSAGE_TYPE[kwargs['type']],
+            'content': message,
+        })
+        if kwargs['type'] == MESSAGE_TYPE['user_input_request']:
+            return ipc_client_instance.listen()
+        
+    def local_print(*args, **kwargs):
+        message = " ".join(map(str, args))
+        if 'type' in kwargs:
+            if kwargs['type'] == MESSAGE_TYPE['info']:
+                return
+            del kwargs['type']
+
+        built_in_print(message, **kwargs)
+
+    ipc_client_instance = None
+    if '--external-log-process-port' in args:
+        ipc_client_instance = IPCClient(args['--external-log-process-port'])
+        return print_to_external_process, ipc_client_instance
+    else:
+        return local_print, ipc_client_instance
+
 if __name__ == "__main__":
     args = init()
 
-    # TODO get checkpoint from database and fill the project with it
-    project = Project(args)
-    project.start()
+    builtins.print, ipc_client_instance = get_custom_print(args)
+
+    if '--get-created-apps-with-steps' in args:
+        print({ 'db_data': get_created_apps_with_steps() }, type='info')
+    else:
+        # TODO get checkpoint from database and fill the project with it
+        project = Project(args, ipc_client_instance=ipc_client_instance)
+        project.start()

pilot/utils/llm_connection.py

@@ -74,8 +74,6 @@ def num_tokens_from_functions(functions, model="gpt-4"):
                             for o in v['enum']:
                                 function_tokens += 3
                                 function_tokens += len(encoding.encode(o))
-                        # else:
-                        #     print(f"Warning: not supported field {field}")
                 function_tokens += 11
 
         num_tokens += function_tokens
@@ -85,7 +83,8 @@ def num_tokens_from_functions(functions, model="gpt-4"):
 
 
 def create_gpt_chat_completion(messages: List[dict], req_type, min_tokens=MIN_TOKENS_FOR_GPT_RESPONSE,
-                               function_calls=None):
+    function_calls=None):
+
     tokens_in_messages = round(get_tokens_in_messages(messages) * 1.2)  # add 20% to account for not 100% accuracy
     if function_calls is not None:
         tokens_in_messages += round(

pilot/utils/utils.py

@@ -1,7 +1,9 @@
 # utils/utils.py
 
+import datetime
 import os
 import platform
+import uuid
 import distro
 import json
 import hashlib
@@ -167,3 +169,12 @@ def clean_filename(filename):
     cleaned_filename = re.sub(r'\s', '_', cleaned_filename)
 
     return cleaned_filename
+
+def json_serial(obj):
+    """JSON serializer for objects not serializable by default json code"""
+    if isinstance(obj, (datetime.datetime, datetime.date)):
+        return obj.isoformat()
+    elif isinstance(obj, uuid.UUID):
+        return str(obj)
+    else:
+        return str(obj)