OMGeeky/gpt-pilot
1
0
Fork 0
mirror of https://github.com/OMGeeky/gpt-pilot.git synced 2026-02-23 15:49:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

avoid getting stuck in a loop if LLM can't conform to schema.

Browse Source
This commit is contained in:
Nicholas Albion
2023-10-04 15:23:36 +11:00
parent 0d8a4c7fee
commit cbac991bd9
2 changed files with 16 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
pilot/helpers/agents/test_Developer.py
View File

@@ -144,7 +144,7 @@ class TestDeveloper:
self.project.developer = self.developer
# we send a GET_TEST_TYPE spec, but the 1st response is invalid
types_in_response = ['command', 'command_test']
types_in_response = ['command', 'wrong_again', 'command_test']
json_received = []
def generate_response(*args, **kwargs):
@@ -169,12 +169,12 @@ class TestDeveloper:
mock_questionary = MockQuestionary([''])
# with patch('utils.questionary.questionary', mock_questionary):
# When
result = self.developer.test_code_changes(monkey, convo)
with patch('utils.questionary.questionary', mock_questionary):
# When
result = self.developer.test_code_changes(monkey, convo)
# Then
assert result == {'success': True, 'cli_response': 'stdout:\n```\n\n```'}
assert mock_requests_post.call_count == 2
assert "The JSON is invalid at $.type - 'command' is not one of ['automated_test', 'command_test', 'manual_test', 'no_test']" in json_received[1]['messages'][3]['content']
assert mock_execute.call_count == 1
# Then
assert result == {'success': True, 'cli_response': 'stdout:\n```\n\n```'}
assert mock_requests_post.call_count == 3
assert "The JSON is invalid at $.type - 'command' is not one of ['automated_test', 'command_test', 'manual_test', 'no_test']" in json_received[1]['messages'][3]['content']
assert mock_execute.call_count == 1

8
pilot/utils/llm_connection.py
View File

@@ -163,13 +163,19 @@ def retry_on_exception(func):
args[0]['function_buffer'] = e.doc
continue
elif isinstance(e, ValidationError):
function_error_count = 1 if 'function_error' not in args[0] else args[0]['function_error_count'] + 1
args[0]['function_error_count'] = function_error_count
logger.warning('Received invalid JSON response from LLM. Asking to retry...')
logger.info(f' at {e.json_path} {e.message}')
# eg:
# json_path: '$.type'
# message: "'command' is not one of ['automated_test', 'command_test', 'manual_test', 'no_test']"
args[0]['function_error'] = f'at {e.json_path} - {e.message}'
continue
# Attempt retry if the JSON schema is invalid, but avoid getting stuck in a loop
if function_error_count < 3:
continue
if "context_length_exceeded" in err_str:
# spinner_stop(spinner)
raise TokenLimitError(get_tokens_in_messages_from_openai_error(err_str), MAX_GPT_MODEL_TOKENS)