From 6c4f64c397e0fa22b4d39f06c39fa411bf420b7d Mon Sep 17 00:00:00 2001
From: CaIon <1808837298@qq.com>
Date: Tue, 10 Jun 2025 18:51:26 +0800
Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20fix(token=5Fcounter):=20refactor?=
 =?UTF-8?q?=20token=20encoder=20initialization=20and=20retrieval=20logic?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 service/token_counter.go | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/service/token_counter.go b/service/token_counter.go
index 29f15fec..636dbd94 100644
--- a/service/token_counter.go
+++ b/service/token_counter.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/tiktoken-go/tokenizer"
+	"github.com/tiktoken-go/tokenizer/codec"
 	"image"
 	"log"
 	"math"
@@ -21,20 +22,16 @@ var defaultTokenEncoder tokenizer.Codec
 
 func InitTokenEncoders() {
 	common.SysLog("initializing token encoders")
-	cl100TokenEncoder, err := tokenizer.Get(tokenizer.Cl100kBase)
-	if err != nil {
-		common.FatalLog(fmt.Sprintf("failed to get gpt-3.5-turbo token encoder: %s", err.Error()))
-	}
-	defaultTokenEncoder = cl100TokenEncoder
+	defaultTokenEncoder = codec.NewCl100kBase()
 	common.SysLog("token encoders initialized")
 }
 
 func getTokenEncoder(model string) tokenizer.Codec {
-	codec, err := tokenizer.ForModel(tokenizer.Model(model))
+	modelCodec, err := tokenizer.ForModel(tokenizer.Model(model))
 	if err != nil {
 		return defaultTokenEncoder
 	}
-	return codec
+	return modelCodec
 }
 
 func getTokenNum(tokenEncoder tokenizer.Codec, text string) int {