From 4d91f381c13a278c78dc81e40cde15bd0520fdef Mon Sep 17 00:00:00 2001 From: Nicholas Albion Date: Fri, 22 Sep 2023 19:11:50 +1000 Subject: [PATCH] Tested Developer.install_technology before refactoring without function_calling --- .github/workflows/ci.yml | 2 +- pilot/helpers/agents/test_Developer.py | 61 ++++++++++++++ pilot/test/mock_questionary.py | 25 ++++++ pilot/test_main_e2e.py | 27 +----- pilot/utils/test_function_calling.py | 112 +++++++++++++++++++++++++ pilot/utils/test_llm_connection.py | 56 +------------ pytest.ini | 1 + 7 files changed, 203 insertions(+), 81 deletions(-) create mode 100644 pilot/helpers/agents/test_Developer.py create mode 100644 pilot/test/mock_questionary.py create mode 100644 pilot/utils/test_function_calling.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54cf21e..c27c8fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,4 +43,4 @@ jobs: run: | pip install pytest cd pilot - PYTHONPATH=. pytest -m "not slow" + PYTHONPATH=. pytest -m "not slow and not uses_tokens" diff --git a/pilot/helpers/agents/test_Developer.py b/pilot/helpers/agents/test_Developer.py new file mode 100644 index 0000000..0dc603b --- /dev/null +++ b/pilot/helpers/agents/test_Developer.py @@ -0,0 +1,61 @@ +import builtins +import os +from unittest.mock import patch, Mock + +from helpers.AgentConvo import AgentConvo +from dotenv import load_dotenv +load_dotenv() + +from main import get_custom_print +from .Developer import Developer, ENVIRONMENT_SETUP_STEP +from helpers.Project import Project + + +def mock_terminal_size(): + mock_size = Mock() + mock_size.columns = 80 # or whatever width you want + return mock_size + + +class TestDeveloper: + def setup_method(self): + builtins.print, ipc_client_instance = get_custom_print({}) + + name = 'TestDeveloper' + self.project = Project({ + 'app_id': 'test-developer', + 'name': name, + 'app_type': '' + }, + name=name, + architecture=[], + user_stories=[] + ) + + self.project.root_path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), + '../../../workspace/TestDeveloper')) + self.project.technologies = [] + self.project.current_step = ENVIRONMENT_SETUP_STEP + self.developer = Developer(self.project) + + # @pytest.mark.uses_tokens + @patch('helpers.AgentConvo.get_saved_development_step') + @patch('helpers.AgentConvo.save_development_step') + @patch('helpers.AgentConvo.create_gpt_chat_completion', + return_value={'function_calls': { + 'name': 'execute_command', + 'arguments': {'command': 'python --version', 'timeout': 10} + }}) + @patch('helpers.cli.styled_text', return_value='no') + @patch('helpers.cli.execute_command', return_value=('', 'DONE')) + def test_install_technology(self, mock_execute_command, mock_styled_text, + mock_completion, mock_save, mock_get_saved_step): + # Given + self.developer.convo_os_specific_tech = AgentConvo(self.developer) + + # When + llm_response = self.developer.install_technology('python') + + # Then + assert llm_response == 'DONE' + mock_execute_command.assert_called_once_with(self.project, 'python --version', 10) diff --git a/pilot/test/mock_questionary.py b/pilot/test/mock_questionary.py new file mode 100644 index 0000000..56f1ed9 --- /dev/null +++ b/pilot/test/mock_questionary.py @@ -0,0 +1,25 @@ +class MockQuestionary: + def __init__(self, answers=None): + if answers is None: + answers = [] + self.answers = iter(answers) + self.state = 'project_description' + + def text(self, question: str, style=None): + print('AI: ' + question) + if question.startswith('User Story'): + self.state = 'user_stories' + elif question.endswith('write "DONE"'): + self.state = 'DONE' + return self + + def unsafe_ask(self): + if self.state == 'user_stories': + answer = '' + elif self.state == 'DONE': + answer = 'DONE' + else: # if self.state == 'project_description': + answer = next(self.answers, '') + + print('User:', answer) + return answer diff --git a/pilot/test_main_e2e.py b/pilot/test_main_e2e.py index 4df59d5..e865e70 100644 --- a/pilot/test_main_e2e.py +++ b/pilot/test_main_e2e.py @@ -6,6 +6,7 @@ load_dotenv() from database.database import create_tables from helpers.Project import Project +from test.mock_questionary import MockQuestionary from .main import init, get_custom_print @@ -21,32 +22,8 @@ def test_init(): assert args[field] is None -class MockQuestionary(): - def __init__(self, answers=[]): - self.answers = iter(answers) - self.state = 'project_description' - - def text(self, question: str, style=None): - print('AI: ' + question) - if question.startswith('User Story'): - self.state = 'user_stories' - elif question.endswith('write "DONE"'): - self.state = 'DONE' - return self - - def unsafe_ask(self): - if self.state == 'user_stories': - answer = '' - elif self.state == 'DONE': - answer = 'DONE' - else: # if self.state == 'project_description': - answer = next(self.answers, '') - - print('User:', answer) - return answer - - @pytest.mark.slow +@pytest.mark.uses_tokens @pytest.mark.skip(reason="Uses lots of tokens") def test_end_to_end(): # Given diff --git a/pilot/utils/test_function_calling.py b/pilot/utils/test_function_calling.py new file mode 100644 index 0000000..978e68a --- /dev/null +++ b/pilot/utils/test_function_calling.py @@ -0,0 +1,112 @@ +from local_llm_function_calling.prompter import CompletionModelPrompter, InstructModelPrompter + +from const.function_calls import ARCHITECTURE, DEV_STEPS +from .function_calling import JsonPrompter + + +def test_completion_function_prompt(): + # Given + prompter = CompletionModelPrompter() + + # When + prompt = prompter.prompt('Create a web-based chat app', ARCHITECTURE['definitions']) # , 'process_technologies') + + # Then + assert prompt == '''Create a web-based chat app + +Available functions: +process_technologies - Print the list of technologies that are created. +```jsonschema +{ + "technologies": { + "type": "array", + "description": "List of technologies that are created in a list.", + "items": { + "type": "string", + "description": "technology" + } + } +} +``` + +Function call: + +Function call: ''' + + +def test_instruct_function_prompter(): + # Given + prompter = InstructModelPrompter() + + # When + prompt = prompter.prompt('Create a web-based chat app', ARCHITECTURE['definitions']) # , 'process_technologies') + + # Then + assert prompt == '''Your task is to call a function when needed. You will be provided with a list of functions. Available functions: +process_technologies - Print the list of technologies that are created. +```jsonschema +{ + "technologies": { + "type": "array", + "description": "List of technologies that are created in a list.", + "items": { + "type": "string", + "description": "technology" + } + } +} +``` + +Create a web-based chat app + +Function call: ''' + + +def test_json_prompter(): + # Given + prompter = JsonPrompter() + + # When + prompt = prompter.prompt('Create a web-based chat app', ARCHITECTURE['definitions']) # , 'process_technologies') + + # Then + assert prompt == '''[INST] <> +Help choose the appropriate function to call to answer the user's question. +In your response you must only use JSON output and provide no notes or commentary. + +Available functions: +- process_technologies - Print the list of technologies that are created. +<> + +Create a web-based chat app [/INST]''' + + +def test_llama_instruct_function_prompter_named(): + # Given + prompter = LlamaInstructPrompter() + + # When + prompt = prompter.prompt('Create a web-based chat app', ARCHITECTURE['definitions'], 'process_technologies') + + # Then + assert prompt == '''[INST] <> +Define the arguments for process_technologies to answer the user's question. +In your response you must only use JSON output and provide no notes or commentary. + +Function description: Print the list of technologies that are created. +Function parameters should follow this schema: +```jsonschema +{ + "technologies": { + "type": "array", + "description": "List of technologies that are created in a list.", + "items": { + "type": "string", + "description": "technology" + } + } +} +``` +<> + +Create a web-based chat app [/INST]''' diff --git a/pilot/utils/test_llm_connection.py b/pilot/utils/test_llm_connection.py index 3c1ea32..a5c82da 100644 --- a/pilot/utils/test_llm_connection.py +++ b/pilot/utils/test_llm_connection.py @@ -2,7 +2,7 @@ import builtins import os from dotenv import load_dotenv from unittest.mock import patch -from local_llm_function_calling.prompter import CompletionModelPrompter, InstructModelPrompter + from const.function_calls import ARCHITECTURE, DEV_STEPS from helpers.AgentConvo import AgentConvo @@ -97,61 +97,7 @@ class TestLlmConnection: # response = response['function_calls']['arguments']['technologies'] assert 'Node.js' in response - def test_completion_function_prompt(self): - # Given - prompter = CompletionModelPrompter() - # When - prompt = prompter.prompt('Create a web-based chat app', ARCHITECTURE['definitions']) # , 'process_technologies') - - # Then - assert prompt == '''Create a web-based chat app - -Available functions: -process_technologies - Print the list of technologies that are created. -```jsonschema -{ - "technologies": { - "type": "array", - "description": "List of technologies that are created in a list.", - "items": { - "type": "string", - "description": "technology" - } - } -} -``` - -Function call: - -Function call: ''' - - def test_instruct_function_prompter(self): - # Given - prompter = InstructModelPrompter() - - # When - prompt = prompter.prompt('Create a web-based chat app', ARCHITECTURE['definitions']) # , 'process_technologies') - - # Then - assert prompt == '''Your task is to call a function when needed. You will be provided with a list of functions. Available functions: -process_technologies - Print the list of technologies that are created. -```jsonschema -{ - "technologies": { - "type": "array", - "description": "List of technologies that are created in a list.", - "items": { - "type": "string", - "description": "technology" - } - } -} -``` - -Create a web-based chat app - -Function call: ''' def _create_convo(self, agent): convo = AgentConvo(agent) \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index a3b504a..b0c4c73 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,4 +4,5 @@ python_files = test_*.py markers = slow: marks tests as slow (deselect with '-m "not slow"') + uses_tokens: Integration tests which use tokens daily: tests which should be run daily