gavin/maybe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 28 Wiki Activity

basic stream implementation

Browse Source
This commit is contained in:
Zach Gollwitzer
2025-03-26 17:09:12 -04:00
parent 866a504003
commit e03f88a18c
8 changed files with 123 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
app/models/assistant.rb
View File

@@ -18,7 +18,12 @@ class Assistant
provider = get_model_provider(message.ai_model)
response = provider.chat_response(message, instructions: instructions, available_functions: functions)
response = provider.chat_response(
message,
instructions: instructions,
available_functions: functions,
streamer: streamer
)
stop_thinking
@@ -36,6 +41,13 @@ class Assistant
end
private
def streamer
proc do |data|
puts data
# TODO process data
end
end
def stop_thinking
sleep artificial_thinking_delay
chat.broadcast_remove target: "thinking-indicator"

2
app/models/assistant/provideable.rb
View File

@@ -6,7 +6,7 @@ module Assistant::Provideable
ChatResponseFunctionExecution = Data.define(:id, :call_id, :name, :arguments, :result)
ChatResponse = Data.define(:id, :messages, :functions, :model)
def chat_response(message, instructions: nil, available_functions: [])
def chat_response(message, instructions: nil, available_functions: [], streamer: nil)
raise NotImplementedError, "Subclasses must implement #chat_response"
end

3
app/models/provider/openai.rb
View File

@@ -24,11 +24,12 @@ class Provider::Openai < Provider
MODELS.include?(model)
end
def chat_response(message, instructions: nil, available_functions: [])
def chat_response(message, instructions: nil, available_functions: [], streamer: nil)
provider_response do
processor = ChatResponseProcessor.new(
client: client,
message: message,
streamer: streamer,
instructions: instructions,
available_functions: available_functions
)

53
app/models/provider/openai/chat_response_processor.rb
View File

@@ -1,9 +1,10 @@
class Provider::Openai::ChatResponseProcessor
def initialize(message:, client:, instructions: nil, available_functions: [])
def initialize(message:, client:, instructions: nil, available_functions: [], streamer: nil)
@client = client
@message = message
@instructions = instructions
@available_functions = available_functions
@streamer = streamer
end
def process
@@ -22,8 +23,9 @@ class Provider::Openai::ChatResponseProcessor
end
private
attr_reader :client, :message, :instructions, :available_functions
attr_reader :client, :message, :instructions, :available_functions, :streamer
StreamChunk = Data.define(:type, :data)
PendingFunction = Data.define(:id, :call_id, :name, :arguments)
# Expected response interface for an "LLM Provider"
@@ -45,14 +47,57 @@ class Provider::Openai::ChatResponseProcessor
# No need to pass tools for follow-up messages that provide function results
prepared_tools = executed_functions.empty? ? tools : []
raw_response = client.responses.create(parameters: {
raw_response = nil
internal_streamer = proc do |chunk|
type = chunk.dig("type")
if streamer.present?
case type
when "response.output_text.delta", "response.refusal.delta"
# We don't distinguish between text and refusal yet, so stream both the same
streamer.call(StreamChunk.new(type: "output_text", data: chunk.dig("delta")))
when "response.function_call_arguments.done"
streamer.call(StreamChunk.new(type: "function_request", data: chunk.dig("arguments")))
when "response.completed"
res = chunk.dig("response")
res_output = res.dig("output")
functions_output = if executed_functions.any?
executed_functions
else
extract_pending_functions(res_output)
end
data = Response.new(
id: res.dig("id"),
messages: extract_messages(res_output),
functions: functions_output,
model: res.dig("model")
)
streamer.call(StreamChunk.new(type: "response", data: data))
end
end
if type == "response.completed"
raw_response = chunk.dig("response")
end
end
client.responses.create(parameters: {
model: model,
input: prepared_input,
instructions: instructions,
tools: prepared_tools,
previous_response_id: previous_response_id
previous_response_id: previous_response_id,
stream: internal_streamer
})
if raw_response.dig("status") == "failed" || raw_response.dig("status") == "incomplete"
raise Provider::Openai::Error.new("OpenAI returned a failed or incomplete response", { chunk: chunk })
end
response_output = raw_response.dig("output")
functions_output = if executed_functions.any?

19
test/models/provider/openai_test.rb
View File

@@ -23,8 +23,25 @@ class Provider::OpenaiTest < ActiveSupport::TestCase
end
end
test "provides basic chat response 2" do
VCR.use_cassette("openai/chat/basic_response", record: :all) do
chat = chats(:two)
message = chat.messages.create!(
type: "UserMessage",
content: "This is a chat test. If it's working, respond with a single word: Yes",
ai_model: @subject_model
)
response = @subject.chat_response(message)
assert response.success?
assert_equal 1, response.data.messages.size
assert_includes response.data.messages.first.content, "Yes"
end
end
test "handles chat response with tool calls" do
VCR.use_cassette("openai/chat/tool_calls") do
VCR.use_cassette("openai/chat/tool_calls", record: :all) do
class PredictableToolFunction < Assistant::Function
class << self
def expected_test_result

108
test/vcr_cassettes/openai/chat/basic_response.yml
View File

@@ -6,7 +6,7 @@ http_interactions:
body:
encoding: UTF-8
string: '{"model":"gpt-4o","input":[{"role":"user","content":"This is a chat
test. If it''s working, respond with a single word: Yes"}],"instructions":null,"tools":[],"previous_response_id":null}'
test. If it''s working, respond with a single word: Yes"}],"instructions":null,"tools":[],"previous_response_id":null,"stream":true}'
headers:
Content-Type:
- application/json
@@ -24,9 +24,9 @@ http_interactions:
message: OK
headers:
Date:
- Wed, 26 Mar 2025 16:39:37 GMT
- Wed, 26 Mar 2025 20:38:53 GMT
Content-Type:
- application/json
- text/event-stream; charset=utf-8
Transfer-Encoding:
- chunked
Connection:
@@ -36,85 +36,57 @@ http_interactions:
Openai-Organization:
- "<OPENAI_ORGANIZATION_ID>"
X-Request-Id:
- req_a68d4bb0ba66d6ce1b2b13d1eaeaf06a
- req_019d20b30a7aad658a848109a7b1d9a7
Openai-Processing-Ms:
- '747'
- '78'
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
Cf-Cache-Status:
- DYNAMIC
Set-Cookie:
- __cf_bm=VgfPaGp9lqdx25NaTMYGRW4ff.Wev64.Cz23yZHoYVw-1743007177-1.0.1.1-g8pHwGJVfZnNc8G04_Sgt_TgYMARdmgSILVBCMbsDASBaSU1jzjMeHktjLenfVMpmtr9Dif1xJ.1fvXpL9UoMrnA2Kf5yh1km3gqcNQ0p3Q;
path=/; expires=Wed, 26-Mar-25 17:09:37 GMT; domain=.api.openai.com; HttpOnly;
- __cf_bm=8GIBHnpZz7OLpPiIZN_57apWdMo55at4QPDQ3B0S61U-1743021533-1.0.1.1-W0sXULgGh8mIqmbx8GqF3UJZ4UND3vTOJa4P8wov9R85I_G2SK631xhONoDmhVONgpt41yj08ZXVV4z7oNLWmfwMhRqB.IXZ0ODtCeELmmk;
path=/; expires=Wed, 26-Mar-25 21:08:53 GMT; domain=.api.openai.com; HttpOnly;
Secure; SameSite=None
- _cfuvid=AVQYDcJPgy.gzoxqSZg_eO3DQJ1ZtdfWXPka_62DImU-1743007177953-0.0.1.1-604800000;
- _cfuvid=XwbFBEN.O70FMLzBgac6U3S0thWeu6FsBxqrGi7ejpM-1743021533036-0.0.1.1-604800000;
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
X-Content-Type-Options:
- nosniff
Server:
- cloudflare
Cf-Ray:
- 926815c6eddfe1eb-ORD
- 926974406901cf47-CMH
Alt-Svc:
- h3=":443"; ma=86400
body:
encoding: ASCII-8BIT
string: |-
{
"id": "resp_67e42dc92c98819293b83b2b13ba97e208e0dce32cfcc9c2",
"object": "response",
"created_at": 1743007177,
"status": "completed",
"error": null,
"incomplete_details": null,
"instructions": null,
"max_output_tokens": null,
"model": "gpt-4o-2024-08-06",
"output": [
{
"type": "message",
"id": "msg_67e42dc9bba881929e195486a9ad78ea08e0dce32cfcc9c2",
"status": "completed",
"role": "assistant",
"content": [
{
"type": "output_text",
"text": "Yes",
"annotations": []
}
]
}
],
"parallel_tool_calls": true,
"previous_response_id": null,
"reasoning": {
"effort": null,
"generate_summary": null
},
"store": true,
"temperature": 1.0,
"text": {
"format": {
"type": "text"
}
},
"tool_choice": "auto",
"tools": [],
"top_p": 1.0,
"truncation": "disabled",
"usage": {
"input_tokens": 43,
"input_tokens_details": {
"cached_tokens": 0
},
"output_tokens": 2,
"output_tokens_details": {
"reasoning_tokens": 0
},
"total_tokens": 45
},
"user": null,
"metadata": {}
}
recorded_at: Wed, 26 Mar 2025 16:39:37 GMT
encoding: UTF-8
string: |+
event: response.created
data: {"type":"response.created","response":{"id":"resp_67e465dcebe88192af1708aeffc86a250cda4bd56b1d6516","object":"response","created_at":1743021532,"status":"in_progress","error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"generate_summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}}
event: response.in_progress
data: {"type":"response.in_progress","response":{"id":"resp_67e465dcebe88192af1708aeffc86a250cda4bd56b1d6516","object":"response","created_at":1743021532,"status":"in_progress","error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"generate_summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}}
event: response.output_item.added
data: {"type":"response.output_item.added","output_index":0,"item":{"type":"message","id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","status":"in_progress","role":"assistant","content":[]}}
event: response.content_part.added
data: {"type":"response.content_part.added","item_id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","output_index":0,"content_index":0,"part":{"type":"output_text","text":"","annotations":[]}}
event: response.output_text.delta
data: {"type":"response.output_text.delta","item_id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","output_index":0,"content_index":0,"delta":"Yes"}
event: response.output_text.done
data: {"type":"response.output_text.done","item_id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","output_index":0,"content_index":0,"text":"Yes"}
event: response.content_part.done
data: {"type":"response.content_part.done","item_id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","output_index":0,"content_index":0,"part":{"type":"output_text","text":"Yes","annotations":[]}}
event: response.output_item.done
data: {"type":"response.output_item.done","output_index":0,"item":{"type":"message","id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Yes","annotations":[]}]}}
event: response.completed
data: {"type":"response.completed","response":{"id":"resp_67e465dcebe88192af1708aeffc86a250cda4bd56b1d6516","object":"response","created_at":1743021532,"status":"completed","error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"model":"gpt-4o-2024-08-06","output":[{"type":"message","id":"msg_67e465dd69588192b17f5d89aae448060cda4bd56b1d6516","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Yes","annotations":[]}]}],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"generate_summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":43,"input_tokens_details":{"cached_tokens":0},"output_tokens":2,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":45},"user":null,"metadata":{}}}
recorded_at: Wed, 26 Mar 2025 20:38:53 GMT
recorded_with: VCR 6.3.1
...

72
test/vcr_cassettes/openai/chat/error.yml
View File

@@ -1,72 +0,0 @@
---
http_interactions:
- request:
method: post
uri: https://api.openai.com/v1/responses
body:
encoding: UTF-8
string: '{"model":"invalid-model-that-will-trigger-api-error","input":[{"role":"user","content":"Error
test"}],"instructions":null,"tools":[],"previous_response_id":null}'
headers:
Content-Type:
- application/json
Authorization:
- Bearer <OPENAI_ACCESS_TOKEN>
Accept-Encoding:
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
Accept:
- "*/*"
User-Agent:
- Ruby
response:
status:
code: 400
message: Bad Request
headers:
Date:
- Wed, 26 Mar 2025 16:39:37 GMT
Content-Type:
- application/json
Content-Length:
- '207'
Connection:
- keep-alive
Openai-Version:
- '2020-10-01'
Openai-Organization:
- "<OPENAI_ORGANIZATION_ID>"
X-Request-Id:
- req_fbea740269fd866bf13c8bdc17798de1
Openai-Processing-Ms:
- '118'
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
Cf-Cache-Status:
- DYNAMIC
Set-Cookie:
- __cf_bm=TK4DOS84TE1BBqD6wwFrE92fkTeBLmwOfHxDcTbP0d4-1743007177-1.0.1.1-80iHD_mSYrsI18Be_VOwaYhL8aFPjMP94B5lg3sAbqkkDtfpOP5VVTeVeCC56H_O17klQzRVVY4Tk_GQ3QFUJ280DLY.AKuRDiMnVwFKGt4;
path=/; expires=Wed, 26-Mar-25 17:09:37 GMT; domain=.api.openai.com; HttpOnly;
Secure; SameSite=None
- _cfuvid=z1RZlzcsco0CA7w7LzJoa3ZUt6Amz1p81E5aZN3V7QM-1743007177428-0.0.1.1-604800000;
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
X-Content-Type-Options:
- nosniff
Server:
- cloudflare
Cf-Ray:
- 926815c6e89a0293-ORD
Alt-Svc:
- h3=":443"; ma=86400
body:
encoding: UTF-8
string: |-
{
"error": {
"message": "The requested model 'invalid-model-that-will-trigger-api-error' does not exist.",
"type": "invalid_request_error",
"param": "model",
"code": "model_not_found"
}
}
recorded_at: Wed, 26 Mar 2025 16:39:37 GMT
recorded_with: VCR 6.3.1

248
test/vcr_cassettes/openai/chat/tool_calls.yml
View File

@@ -1,248 +0,0 @@
---
http_interactions:
- request:
method: post
uri: https://api.openai.com/v1/responses
body:
encoding: UTF-8
string: '{"model":"gpt-4o","input":[{"role":"user","content":"What is my net
worth?"}],"instructions":"Use the tools available to you to answer the user''s
question.","tools":[{"type":"function","name":"get_net_worth","description":"Gets
user net worth data","parameters":{"type":"object","properties":{},"required":[],"additionalProperties":false},"strict":true}],"previous_response_id":null}'
headers:
Content-Type:
- application/json
Authorization:
- Bearer <OPENAI_ACCESS_TOKEN>
Accept-Encoding:
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
Accept:
- "*/*"
User-Agent:
- Ruby
response:
status:
code: 200
message: OK
headers:
Date:
- Wed, 26 Mar 2025 16:39:37 GMT
Content-Type:
- application/json
Transfer-Encoding:
- chunked
Connection:
- keep-alive
Openai-Version:
- '2020-10-01'
Openai-Organization:
- "<OPENAI_ORGANIZATION_ID>"
X-Request-Id:
- req_ceb289fde2c231617955a1555a4cd79e
Openai-Processing-Ms:
- '768'
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
Cf-Cache-Status:
- DYNAMIC
Set-Cookie:
- __cf_bm=fQqPxoLXAE3ef6cr1S4DPM2iTPS59q6ZCPlKepx7mug-1743007177-1.0.1.1-_dol0qexmAEgTXHGgO_328asGX25U_pnypZQ1M2Sgs0C99rTu0foM9K95DypgMBhcrSeuOniz3AUC1iA9otoNKfSxZH_LhnNrh27.BOhFKg;
path=/; expires=Wed, 26-Mar-25 17:09:37 GMT; domain=.api.openai.com; HttpOnly;
Secure; SameSite=None
- _cfuvid=5ONQI6E2u9UonIMtOb2a5j3E.VRss6iDq.sEIyobkpg-1743007177879-0.0.1.1-604800000;
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
X-Content-Type-Options:
- nosniff
Server:
- cloudflare
Cf-Ray:
- 926815c6ec42fa06-ORD
Alt-Svc:
- h3=":443"; ma=86400
body:
encoding: ASCII-8BIT
string: |-
{
"id": "resp_67e42dc91c508192970794d670e0578e0177b37425b2541e",
"object": "response",
"created_at": 1743007177,
"status": "completed",
"error": null,
"incomplete_details": null,
"instructions": "Use the tools available to you to answer the user's question.",
"max_output_tokens": null,
"model": "gpt-4o-2024-08-06",
"output": [
{
"type": "function_call",
"id": "fc_67e42dc9ba808192b7cd3a9aa23b9e090177b37425b2541e",
"call_id": "call_7hAWwco32nLu12yi5NUtdhVp",
"name": "get_net_worth",
"arguments": "{}",
"status": "completed"
}
],
"parallel_tool_calls": true,
"previous_response_id": null,
"reasoning": {
"effort": null,
"generate_summary": null
},
"store": true,
"temperature": 1.0,
"text": {
"format": {
"type": "text"
}
},
"tool_choice": "auto",
"tools": [
{
"type": "function",
"description": "Gets user net worth data",
"name": "get_net_worth",
"parameters": {
"type": "object",
"properties": {},
"required": [],
"additionalProperties": false
},
"strict": true
}
],
"top_p": 1.0,
"truncation": "disabled",
"usage": {
"input_tokens": 271,
"input_tokens_details": {
"cached_tokens": 0
},
"output_tokens": 13,
"output_tokens_details": {
"reasoning_tokens": 0
},
"total_tokens": 284
},
"user": null,
"metadata": {}
}
recorded_at: Wed, 26 Mar 2025 16:39:37 GMT
- request:
method: post
uri: https://api.openai.com/v1/responses
body:
encoding: UTF-8
string: '{"model":"gpt-4o","input":[{"role":"user","content":"What is my net
worth?"},{"type":"function_call_output","call_id":"call_7hAWwco32nLu12yi5NUtdhVp","output":"\"$124,200\""}],"instructions":"Use
the tools available to you to answer the user''s question.","tools":[],"previous_response_id":"resp_67e42dc91c508192970794d670e0578e0177b37425b2541e"}'
headers:
Content-Type:
- application/json
Authorization:
- Bearer <OPENAI_ACCESS_TOKEN>
Accept-Encoding:
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
Accept:
- "*/*"
User-Agent:
- Ruby
response:
status:
code: 200
message: OK
headers:
Date:
- Wed, 26 Mar 2025 16:39:39 GMT
Content-Type:
- application/json
Transfer-Encoding:
- chunked
Connection:
- keep-alive
Openai-Version:
- '2020-10-01'
Openai-Organization:
- "<OPENAI_ORGANIZATION_ID>"
X-Request-Id:
- req_d9c14a951d94d246f53d27a2f68114a1
Openai-Processing-Ms:
- '1297'
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
Cf-Cache-Status:
- DYNAMIC
Set-Cookie:
- __cf_bm=xf4vcPVWTbUItUeVwaSWKuZ1t1iKC6vhUnx3gMXgr0o-1743007179-1.0.1.1-XNh1mngDDGHGDjdN3lqmsUXRLUYrDN7PWhCVFVOICtmjzkVHFzWH2jGcmb5wVCGp_QkniP78ElxT4em1UB15UjOw9zNIs3_XnSERacSreLI;
path=/; expires=Wed, 26-Mar-25 17:09:39 GMT; domain=.api.openai.com; HttpOnly;
Secure; SameSite=None
- _cfuvid=0wWDs.bI2DAGf5YSsnwQy_qPsw8ijrFs3wNf2AfUEjo-1743007179480-0.0.1.1-604800000;
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
X-Content-Type-Options:
- nosniff
Server:
- cloudflare
Cf-Ray:
- 926815cf190c6193-ORD
Alt-Svc:
- h3=":443"; ma=86400
body:
encoding: ASCII-8BIT
string: |-
{
"id": "resp_67e42dca2c0081928305a7b42a0c69620177b37425b2541e",
"object": "response",
"created_at": 1743007178,
"status": "completed",
"error": null,
"incomplete_details": null,
"instructions": "Use the tools available to you to answer the user's question.",
"max_output_tokens": null,
"model": "gpt-4o-2024-08-06",
"output": [
{
"type": "message",
"id": "msg_67e42dcb320c81928ce5dc97145fa2770177b37425b2541e",
"status": "completed",
"role": "assistant",
"content": [
{
"type": "output_text",
"text": "Your net worth is $124,200.",
"annotations": []
}
]
}
],
"parallel_tool_calls": true,
"previous_response_id": "resp_67e42dc91c508192970794d670e0578e0177b37425b2541e",
"reasoning": {
"effort": null,
"generate_summary": null
},
"store": true,
"temperature": 1.0,
"text": {
"format": {
"type": "text"
}
},
"tool_choice": "auto",
"tools": [],
"top_p": 1.0,
"truncation": "disabled",
"usage": {
"input_tokens": 85,
"input_tokens_details": {
"cached_tokens": 0
},
"output_tokens": 10,
"output_tokens_details": {
"reasoning_tokens": 0
},
"total_tokens": 95
},
"user": null,
"metadata": {}
}
recorded_at: Wed, 26 Mar 2025 16:39:39 GMT
recorded_with: VCR 6.3.1