Adjust streaming timeout for OpenAI models in OaiStreamHandler
- Implemented conditional logic to double the streaming timeout for models starting with "o1" or "o3". - Improved handling of streaming timeout configuration to enhance performance based on model type.
This commit is contained in:
@@ -65,8 +65,12 @@ func OaiStreamHandler(c *gin.Context, resp *http.Response, info *relaycommon.Rel
|
|||||||
scanner.Split(bufio.ScanLines)
|
scanner.Split(bufio.ScanLines)
|
||||||
|
|
||||||
service.SetEventStreamHeaders(c)
|
service.SetEventStreamHeaders(c)
|
||||||
|
streamingTimeout := time.Duration(constant.StreamingTimeout) * time.Second
|
||||||
ticker := time.NewTicker(time.Duration(constant.StreamingTimeout) * time.Second)
|
if strings.HasPrefix(info.UpstreamModelName, "o1") || strings.HasPrefix(info.UpstreamModelName, "o3") {
|
||||||
|
// twice timeout for o1 model
|
||||||
|
streamingTimeout *= 2
|
||||||
|
}
|
||||||
|
ticker := time.NewTicker(streamingTimeout)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
|
||||||
stopChan := make(chan bool)
|
stopChan := make(chan bool)
|
||||||
|
|||||||
Reference in New Issue
Block a user