Adjust streaming timeout for OpenAI models in OaiStreamHandler
- Implemented conditional logic to double the streaming timeout for models starting with "o1" or "o3". - Improved handling of streaming timeout configuration to enhance performance based on model type.
This commit is contained in:
@@ -65,8 +65,12 @@ func OaiStreamHandler(c *gin.Context, resp *http.Response, info *relaycommon.Rel
|
||||
scanner.Split(bufio.ScanLines)
|
||||
|
||||
service.SetEventStreamHeaders(c)
|
||||
|
||||
ticker := time.NewTicker(time.Duration(constant.StreamingTimeout) * time.Second)
|
||||
streamingTimeout := time.Duration(constant.StreamingTimeout) * time.Second
|
||||
if strings.HasPrefix(info.UpstreamModelName, "o1") || strings.HasPrefix(info.UpstreamModelName, "o3") {
|
||||
// twice timeout for o1 model
|
||||
streamingTimeout *= 2
|
||||
}
|
||||
ticker := time.NewTicker(streamingTimeout)
|
||||
defer ticker.Stop()
|
||||
|
||||
stopChan := make(chan bool)
|
||||
|
||||
Reference in New Issue
Block a user