feat: Improve embedding request handling and support across channels
- Update EmbeddingRequest DTO to support more flexible input types - Add input parsing method to handle various input formats - Implement ConvertEmbeddingRequest for multiple channel adaptors - Remove relayMode parameter from EmbeddingHelper - Add input validation for embedding requests - Simplify embedding request conversion for different channels
This commit is contained in:
@@ -4,7 +4,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/pkoukk/tiktoken-go"
|
||||
"image"
|
||||
"log"
|
||||
"math"
|
||||
@@ -14,6 +13,8 @@ import (
|
||||
relaycommon "one-api/relay/common"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/pkoukk/tiktoken-go"
|
||||
)
|
||||
|
||||
// tokenEncoderMap won't grow after initialization
|
||||
@@ -323,6 +324,12 @@ func CountTokenInput(input any, model string) (int, error) {
|
||||
text += s
|
||||
}
|
||||
return CountTextToken(text, model)
|
||||
case []interface{}:
|
||||
text := ""
|
||||
for _, item := range v {
|
||||
text += fmt.Sprintf("%v", item)
|
||||
}
|
||||
return CountTextToken(text, model)
|
||||
}
|
||||
return CountTokenInput(fmt.Sprintf("%v", input), model)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user