feat: Improve embedding request handling and support across channels

- Update EmbeddingRequest DTO to support more flexible input types
- Add input parsing method to handle various input formats
- Implement ConvertEmbeddingRequest for multiple channel adaptors
- Remove relayMode parameter from EmbeddingHelper
- Add input validation for embedding requests
- Simplify embedding request conversion for different channels
This commit is contained in:
1808837298@qq.com
2025-02-12 14:39:36 +08:00
parent eceb6afcdd
commit f5e3063f33
14 changed files with 84 additions and 64 deletions

View File

@@ -4,7 +4,6 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/pkoukk/tiktoken-go"
"image"
"log"
"math"
@@ -14,6 +13,8 @@ import (
relaycommon "one-api/relay/common"
"strings"
"unicode/utf8"
"github.com/pkoukk/tiktoken-go"
)
// tokenEncoderMap won't grow after initialization
@@ -323,6 +324,12 @@ func CountTokenInput(input any, model string) (int, error) {
text += s
}
return CountTextToken(text, model)
case []interface{}:
text := ""
for _, item := range v {
text += fmt.Sprintf("%v", item)
}
return CountTextToken(text, model)
}
return CountTokenInput(fmt.Sprintf("%v", input), model)
}