diff --git a/app/models/assistant.rb b/app/models/assistant.rb index 46939ace..5d377d38 100644 --- a/app/models/assistant.rb +++ b/app/models/assistant.rb @@ -18,7 +18,12 @@ class Assistant provider = get_model_provider(message.ai_model) - response = provider.chat_response(message, instructions: instructions, available_functions: functions) + response = provider.chat_response( + message, + instructions: instructions, + available_functions: functions, + streamer: streamer + ) stop_thinking @@ -36,6 +41,13 @@ class Assistant end private + def streamer + proc do |data| + puts data + # TODO process data + end + end + def stop_thinking sleep artificial_thinking_delay chat.broadcast_remove target: "thinking-indicator" diff --git a/app/models/assistant/provideable.rb b/app/models/assistant/provideable.rb index 94bbd84a..e8764f3e 100644 --- a/app/models/assistant/provideable.rb +++ b/app/models/assistant/provideable.rb @@ -6,7 +6,7 @@ module Assistant::Provideable ChatResponseFunctionExecution = Data.define(:id, :call_id, :name, :arguments, :result) ChatResponse = Data.define(:id, :messages, :functions, :model) - def chat_response(message, instructions: nil, available_functions: []) + def chat_response(message, instructions: nil, available_functions: [], streamer: nil) raise NotImplementedError, "Subclasses must implement #chat_response" end diff --git a/app/models/provider/openai.rb b/app/models/provider/openai.rb index 055447f9..56359f18 100644 --- a/app/models/provider/openai.rb +++ b/app/models/provider/openai.rb @@ -24,11 +24,12 @@ class Provider::Openai < Provider MODELS.include?(model) end - def chat_response(message, instructions: nil, available_functions: []) + def chat_response(message, instructions: nil, available_functions: [], streamer: nil) provider_response do processor = ChatResponseProcessor.new( client: client, message: message, + streamer: streamer, instructions: instructions, available_functions: available_functions ) diff --git a/app/models/provider/openai/chat_response_processor.rb b/app/models/provider/openai/chat_response_processor.rb index 7021ba06..7531830b 100644 --- a/app/models/provider/openai/chat_response_processor.rb +++ b/app/models/provider/openai/chat_response_processor.rb @@ -1,9 +1,10 @@ class Provider::Openai::ChatResponseProcessor - def initialize(message:, client:, instructions: nil, available_functions: []) + def initialize(message:, client:, instructions: nil, available_functions: [], streamer: nil) @client = client @message = message @instructions = instructions @available_functions = available_functions + @streamer = streamer end def process @@ -22,8 +23,9 @@ class Provider::Openai::ChatResponseProcessor end private - attr_reader :client, :message, :instructions, :available_functions + attr_reader :client, :message, :instructions, :available_functions, :streamer + StreamChunk = Data.define(:type, :data) PendingFunction = Data.define(:id, :call_id, :name, :arguments) # Expected response interface for an "LLM Provider" @@ -45,14 +47,57 @@ class Provider::Openai::ChatResponseProcessor # No need to pass tools for follow-up messages that provide function results prepared_tools = executed_functions.empty? ? tools : [] - raw_response = client.responses.create(parameters: { + raw_response = nil + + internal_streamer = proc do |chunk| + type = chunk.dig("type") + + if streamer.present? + case type + when "response.output_text.delta", "response.refusal.delta" + # We don't distinguish between text and refusal yet, so stream both the same + streamer.call(StreamChunk.new(type: "output_text", data: chunk.dig("delta"))) + when "response.function_call_arguments.done" + streamer.call(StreamChunk.new(type: "function_request", data: chunk.dig("arguments"))) + when "response.completed" + res = chunk.dig("response") + res_output = res.dig("output") + + functions_output = if executed_functions.any? + executed_functions + else + extract_pending_functions(res_output) + end + + data = Response.new( + id: res.dig("id"), + messages: extract_messages(res_output), + functions: functions_output, + model: res.dig("model") + ) + + streamer.call(StreamChunk.new(type: "response", data: data)) + end + end + + if type == "response.completed" + raw_response = chunk.dig("response") + end + end + + client.responses.create(parameters: { model: model, input: prepared_input, instructions: instructions, tools: prepared_tools, - previous_response_id: previous_response_id + previous_response_id: previous_response_id, + stream: internal_streamer }) + if raw_response.dig("status") == "failed" || raw_response.dig("status") == "incomplete" + raise Provider::Openai::Error.new("OpenAI returned a failed or incomplete response", { chunk: chunk }) + end + response_output = raw_response.dig("output") functions_output = if executed_functions.any? diff --git a/test/models/provider/openai_test.rb b/test/models/provider/openai_test.rb index f6b3667e..7f3e8e90 100644 --- a/test/models/provider/openai_test.rb +++ b/test/models/provider/openai_test.rb @@ -23,8 +23,25 @@ class Provider::OpenaiTest < ActiveSupport::TestCase end end + test "provides basic chat response 2" do + VCR.use_cassette("openai/chat/basic_response", record: :all) do + chat = chats(:two) + message = chat.messages.create!( + type: "UserMessage", + content: "This is a chat test. If it's working, respond with a single word: Yes", + ai_model: @subject_model + ) + + response = @subject.chat_response(message) + + assert response.success? + assert_equal 1, response.data.messages.size + assert_includes response.data.messages.first.content, "Yes" + end + end + test "handles chat response with tool calls" do - VCR.use_cassette("openai/chat/tool_calls") do + VCR.use_cassette("openai/chat/tool_calls", record: :all) do class PredictableToolFunction < Assistant::Function class << self def expected_test_result diff --git a/test/vcr_cassettes/openai/chat/basic_response.yml b/test/vcr_cassettes/openai/chat/basic_response.yml index ecf0eb46..a38d8271 100644 --- a/test/vcr_cassettes/openai/chat/basic_response.yml +++ b/test/vcr_cassettes/openai/chat/basic_response.yml @@ -6,7 +6,7 @@ http_interactions: body: encoding: UTF-8 string: '{"model":"gpt-4o","input":[{"role":"user","content":"This is a chat - test. If it''s working, respond with a single word: Yes"}],"instructions":null,"tools":[],"previous_response_id":null}' + test. If it''s working, respond with a single word: Yes"}],"instructions":null,"tools":[],"previous_response_id":null,"stream":true}' headers: Content-Type: - application/json @@ -24,9 +24,9 @@ http_interactions: message: OK headers: Date: - - Wed, 26 Mar 2025 16:39:37 GMT + - Wed, 26 Mar 2025 20:38:53 GMT Content-Type: - - application/json + - text/event-stream; charset=utf-8 Transfer-Encoding: - chunked Connection: @@ -36,85 +36,57 @@ http_interactions: Openai-Organization: - "" X-Request-Id: - - req_a68d4bb0ba66d6ce1b2b13d1eaeaf06a + - req_019d20b30a7aad658a848109a7b1d9a7 Openai-Processing-Ms: - - '747' + - '78' Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload Cf-Cache-Status: - DYNAMIC Set-Cookie: - - __cf_bm=VgfPaGp9lqdx25NaTMYGRW4ff.Wev64.Cz23yZHoYVw-1743007177-1.0.1.1-g8pHwGJVfZnNc8G04_Sgt_TgYMARdmgSILVBCMbsDASBaSU1jzjMeHktjLenfVMpmtr9Dif1xJ.1fvXpL9UoMrnA2Kf5yh1km3gqcNQ0p3Q; - path=/; expires=Wed, 26-Mar-25 17:09:37 GMT; domain=.api.openai.com; HttpOnly; + - __cf_bm=8GIBHnpZz7OLpPiIZN_57apWdMo55at4QPDQ3B0S61U-1743021533-1.0.1.1-W0sXULgGh8mIqmbx8GqF3UJZ4UND3vTOJa4P8wov9R85I_G2SK631xhONoDmhVONgpt41yj08ZXVV4z7oNLWmfwMhRqB.IXZ0ODtCeELmmk; + path=/; expires=Wed, 26-Mar-25 21:08:53 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - - _cfuvid=AVQYDcJPgy.gzoxqSZg_eO3DQJ1ZtdfWXPka_62DImU-1743007177953-0.0.1.1-604800000; + - _cfuvid=XwbFBEN.O70FMLzBgac6U3S0thWeu6FsBxqrGi7ejpM-1743021533036-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None X-Content-Type-Options: - nosniff Server: - cloudflare Cf-Ray: - - 926815c6eddfe1eb-ORD + - 926974406901cf47-CMH Alt-Svc: - h3=":443"; ma=86400 body: - encoding: ASCII-8BIT - string: |- - { - "id": "resp_67e42dc92c98819293b83b2b13ba97e208e0dce32cfcc9c2", - "object": "response", - "created_at": 1743007177, - "status": "completed", - "error": null, - "incomplete_details": null, - "instructions": null, - "max_output_tokens": null, - "model": "gpt-4o-2024-08-06", - "output": [ - { - "type": "message", - "id": "msg_67e42dc9bba881929e195486a9ad78ea08e0dce32cfcc9c2", - "status": "completed", - "role": "assistant", - "content": [ - { - "type": "output_text", - "text": "Yes", - "annotations": [] - } - ] - } - ], - "parallel_tool_calls": true, - "previous_response_id": null, - "reasoning": { - "effort": null, - "generate_summary": null - }, - "store": true, - "temperature": 1.0, - "text": { - "format": { - "type": "text" - } - }, - "tool_choice": "auto", - "tools": [], - "top_p": 1.0, - "truncation": "disabled", - "usage": { - "input_tokens": 43, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens": 2, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 45 - }, - "user": null, - "metadata": {} - } - recorded_at: Wed, 26 Mar 2025 16:39:37 GMT + encoding: UTF-8 + string: |+ + event: response.created + data: {"type":"response.created","response":{"id":"resp_67e465dcebe88192af1708aeffc86a250cda4bd56b1d6516","object":"response","created_at":1743021532,"status":"in_progress","error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"generate_summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + + event: response.in_progress + data: {"type":"response.in_progress","response":{"id":"resp_67e465dcebe88192af1708aeffc86a250cda4bd56b1d6516","object":"response","created_at":1743021532,"status":"in_progress","error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"generate_summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + + event: response.output_item.added + data: {"type":"response.output_item.added","output_index":0,"item":{"type":"message","id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","status":"in_progress","role":"assistant","content":[]}} + + event: response.content_part.added + data: {"type":"response.content_part.added","item_id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","output_index":0,"content_index":0,"part":{"type":"output_text","text":"","annotations":[]}} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","item_id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","output_index":0,"content_index":0,"delta":"Yes"} + + event: response.output_text.done + data: {"type":"response.output_text.done","item_id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","output_index":0,"content_index":0,"text":"Yes"} + + event: response.content_part.done + data: {"type":"response.content_part.done","item_id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","output_index":0,"content_index":0,"part":{"type":"output_text","text":"Yes","annotations":[]}} + + event: response.output_item.done + data: {"type":"response.output_item.done","output_index":0,"item":{"type":"message","id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Yes","annotations":[]}]}} + + event: response.completed + data: {"type":"response.completed","response":{"id":"resp_67e465dcebe88192af1708aeffc86a250cda4bd56b1d6516","object":"response","created_at":1743021532,"status":"completed","error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"model":"gpt-4o-2024-08-06","output":[{"type":"message","id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Yes","annotations":[]}]}],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"generate_summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":43,"input_tokens_details":{"cached_tokens":0},"output_tokens":2,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":45},"user":null,"metadata":{}}} + + recorded_at: Wed, 26 Mar 2025 20:38:53 GMT recorded_with: VCR 6.3.1 +... diff --git a/test/vcr_cassettes/openai/chat/error.yml b/test/vcr_cassettes/openai/chat/error.yml deleted file mode 100644 index c4e5b0fa..00000000 --- a/test/vcr_cassettes/openai/chat/error.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -http_interactions: -- request: - method: post - uri: https://api.openai.com/v1/responses - body: - encoding: UTF-8 - string: '{"model":"invalid-model-that-will-trigger-api-error","input":[{"role":"user","content":"Error - test"}],"instructions":null,"tools":[],"previous_response_id":null}' - headers: - Content-Type: - - application/json - Authorization: - - Bearer - Accept-Encoding: - - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 - Accept: - - "*/*" - User-Agent: - - Ruby - response: - status: - code: 400 - message: Bad Request - headers: - Date: - - Wed, 26 Mar 2025 16:39:37 GMT - Content-Type: - - application/json - Content-Length: - - '207' - Connection: - - keep-alive - Openai-Version: - - '2020-10-01' - Openai-Organization: - - "" - X-Request-Id: - - req_fbea740269fd866bf13c8bdc17798de1 - Openai-Processing-Ms: - - '118' - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Cf-Cache-Status: - - DYNAMIC - Set-Cookie: - - __cf_bm=TK4DOS84TE1BBqD6wwFrE92fkTeBLmwOfHxDcTbP0d4-1743007177-1.0.1.1-80iHD_mSYrsI18Be_VOwaYhL8aFPjMP94B5lg3sAbqkkDtfpOP5VVTeVeCC56H_O17klQzRVVY4Tk_GQ3QFUJ280DLY.AKuRDiMnVwFKGt4; - path=/; expires=Wed, 26-Mar-25 17:09:37 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=z1RZlzcsco0CA7w7LzJoa3ZUt6Amz1p81E5aZN3V7QM-1743007177428-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - X-Content-Type-Options: - - nosniff - Server: - - cloudflare - Cf-Ray: - - 926815c6e89a0293-ORD - Alt-Svc: - - h3=":443"; ma=86400 - body: - encoding: UTF-8 - string: |- - { - "error": { - "message": "The requested model 'invalid-model-that-will-trigger-api-error' does not exist.", - "type": "invalid_request_error", - "param": "model", - "code": "model_not_found" - } - } - recorded_at: Wed, 26 Mar 2025 16:39:37 GMT -recorded_with: VCR 6.3.1 diff --git a/test/vcr_cassettes/openai/chat/tool_calls.yml b/test/vcr_cassettes/openai/chat/tool_calls.yml deleted file mode 100644 index 687a8ea9..00000000 --- a/test/vcr_cassettes/openai/chat/tool_calls.yml +++ /dev/null @@ -1,248 +0,0 @@ ---- -http_interactions: -- request: - method: post - uri: https://api.openai.com/v1/responses - body: - encoding: UTF-8 - string: '{"model":"gpt-4o","input":[{"role":"user","content":"What is my net - worth?"}],"instructions":"Use the tools available to you to answer the user''s - question.","tools":[{"type":"function","name":"get_net_worth","description":"Gets - user net worth data","parameters":{"type":"object","properties":{},"required":[],"additionalProperties":false},"strict":true}],"previous_response_id":null}' - headers: - Content-Type: - - application/json - Authorization: - - Bearer - Accept-Encoding: - - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 - Accept: - - "*/*" - User-Agent: - - Ruby - response: - status: - code: 200 - message: OK - headers: - Date: - - Wed, 26 Mar 2025 16:39:37 GMT - Content-Type: - - application/json - Transfer-Encoding: - - chunked - Connection: - - keep-alive - Openai-Version: - - '2020-10-01' - Openai-Organization: - - "" - X-Request-Id: - - req_ceb289fde2c231617955a1555a4cd79e - Openai-Processing-Ms: - - '768' - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Cf-Cache-Status: - - DYNAMIC - Set-Cookie: - - __cf_bm=fQqPxoLXAE3ef6cr1S4DPM2iTPS59q6ZCPlKepx7mug-1743007177-1.0.1.1-_dol0qexmAEgTXHGgO_328asGX25U_pnypZQ1M2Sgs0C99rTu0foM9K95DypgMBhcrSeuOniz3AUC1iA9otoNKfSxZH_LhnNrh27.BOhFKg; - path=/; expires=Wed, 26-Mar-25 17:09:37 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=5ONQI6E2u9UonIMtOb2a5j3E.VRss6iDq.sEIyobkpg-1743007177879-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - X-Content-Type-Options: - - nosniff - Server: - - cloudflare - Cf-Ray: - - 926815c6ec42fa06-ORD - Alt-Svc: - - h3=":443"; ma=86400 - body: - encoding: ASCII-8BIT - string: |- - { - "id": "resp_67e42dc91c508192970794d670e0578e0177b37425b2541e", - "object": "response", - "created_at": 1743007177, - "status": "completed", - "error": null, - "incomplete_details": null, - "instructions": "Use the tools available to you to answer the user's question.", - "max_output_tokens": null, - "model": "gpt-4o-2024-08-06", - "output": [ - { - "type": "function_call", - "id": "fc_67e42dc9ba808192b7cd3a9aa23b9e090177b37425b2541e", - "call_id": "call_7hAWwco32nLu12yi5NUtdhVp", - "name": "get_net_worth", - "arguments": "{}", - "status": "completed" - } - ], - "parallel_tool_calls": true, - "previous_response_id": null, - "reasoning": { - "effort": null, - "generate_summary": null - }, - "store": true, - "temperature": 1.0, - "text": { - "format": { - "type": "text" - } - }, - "tool_choice": "auto", - "tools": [ - { - "type": "function", - "description": "Gets user net worth data", - "name": "get_net_worth", - "parameters": { - "type": "object", - "properties": {}, - "required": [], - "additionalProperties": false - }, - "strict": true - } - ], - "top_p": 1.0, - "truncation": "disabled", - "usage": { - "input_tokens": 271, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens": 13, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 284 - }, - "user": null, - "metadata": {} - } - recorded_at: Wed, 26 Mar 2025 16:39:37 GMT -- request: - method: post - uri: https://api.openai.com/v1/responses - body: - encoding: UTF-8 - string: '{"model":"gpt-4o","input":[{"role":"user","content":"What is my net - worth?"},{"type":"function_call_output","call_id":"call_7hAWwco32nLu12yi5NUtdhVp","output":"\"$124,200\""}],"instructions":"Use - the tools available to you to answer the user''s question.","tools":[],"previous_response_id":"resp_67e42dc91c508192970794d670e0578e0177b37425b2541e"}' - headers: - Content-Type: - - application/json - Authorization: - - Bearer - Accept-Encoding: - - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 - Accept: - - "*/*" - User-Agent: - - Ruby - response: - status: - code: 200 - message: OK - headers: - Date: - - Wed, 26 Mar 2025 16:39:39 GMT - Content-Type: - - application/json - Transfer-Encoding: - - chunked - Connection: - - keep-alive - Openai-Version: - - '2020-10-01' - Openai-Organization: - - "" - X-Request-Id: - - req_d9c14a951d94d246f53d27a2f68114a1 - Openai-Processing-Ms: - - '1297' - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Cf-Cache-Status: - - DYNAMIC - Set-Cookie: - - __cf_bm=xf4vcPVWTbUItUeVwaSWKuZ1t1iKC6vhUnx3gMXgr0o-1743007179-1.0.1.1-XNh1mngDDGHGDjdN3lqmsUXRLUYrDN7PWhCVFVOICtmjzkVHFzWH2jGcmb5wVCGp_QkniP78ElxT4em1UB15UjOw9zNIs3_XnSERacSreLI; - path=/; expires=Wed, 26-Mar-25 17:09:39 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=0wWDs.bI2DAGf5YSsnwQy_qPsw8ijrFs3wNf2AfUEjo-1743007179480-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - X-Content-Type-Options: - - nosniff - Server: - - cloudflare - Cf-Ray: - - 926815cf190c6193-ORD - Alt-Svc: - - h3=":443"; ma=86400 - body: - encoding: ASCII-8BIT - string: |- - { - "id": "resp_67e42dca2c0081928305a7b42a0c69620177b37425b2541e", - "object": "response", - "created_at": 1743007178, - "status": "completed", - "error": null, - "incomplete_details": null, - "instructions": "Use the tools available to you to answer the user's question.", - "max_output_tokens": null, - "model": "gpt-4o-2024-08-06", - "output": [ - { - "type": "message", - "id": "msg_67e42dcb320c81928ce5dc97145fa2770177b37425b2541e", - "status": "completed", - "role": "assistant", - "content": [ - { - "type": "output_text", - "text": "Your net worth is $124,200.", - "annotations": [] - } - ] - } - ], - "parallel_tool_calls": true, - "previous_response_id": "resp_67e42dc91c508192970794d670e0578e0177b37425b2541e", - "reasoning": { - "effort": null, - "generate_summary": null - }, - "store": true, - "temperature": 1.0, - "text": { - "format": { - "type": "text" - } - }, - "tool_choice": "auto", - "tools": [], - "top_p": 1.0, - "truncation": "disabled", - "usage": { - "input_tokens": 85, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens": 10, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 95 - }, - "user": null, - "metadata": {} - } - recorded_at: Wed, 26 Mar 2025 16:39:39 GMT -recorded_with: VCR 6.3.1