diff --git a/relay/channel/gemini/relay-gemini.go b/relay/channel/gemini/relay-gemini.go index 9ab167b1..bf1ece57 100644 --- a/relay/channel/gemini/relay-gemini.go +++ b/relay/channel/gemini/relay-gemini.go @@ -18,6 +18,24 @@ import ( "github.com/gin-gonic/gin" ) +var geminiSupportedMimeTypes = map[string]bool{ + "application/pdf": true, + "audio/mpeg": true, + "audio/mp3": true, + "audio/wav": true, + "image/png": true, + "image/jpeg": true, + "text/plain": true, + "video/mov": true, + "video/mpeg": true, + "video/mp4": true, + "video/mpg": true, + "video/avi": true, + "video/wmv": true, + "video/mpegps": true, + "video/flv": true, +} + // Setting safety to the lowest possible values since Gemini is already powerless enough func CovertGemini2OpenAI(textRequest dto.GeneralOpenAIRequest, info *relaycommon.RelayInfo) (*GeminiChatRequest, error) { @@ -215,14 +233,20 @@ func CovertGemini2OpenAI(textRequest dto.GeneralOpenAIRequest, info *relaycommon } // 判断是否是url if strings.HasPrefix(part.GetImageMedia().Url, "http") { - // 是url,获取图片的类型和base64编码的数据 + // 是url,获取文件的类型和base64编码的数据 fileData, err := service.GetFileBase64FromUrl(part.GetImageMedia().Url) if err != nil { - return nil, fmt.Errorf("get file base64 from url failed: %s", err.Error()) + return nil, fmt.Errorf("get file base64 from url '%s' failed: %w", part.GetImageMedia().Url, err) } + + // 校验 MimeType 是否在 Gemini 支持的白名单中 + if _, ok := geminiSupportedMimeTypes[strings.ToLower(fileData.MimeType)]; !ok { + return nil, fmt.Errorf("MIME type '%s' from URL '%s' is not supported by Gemini. Supported types are: %v", fileData.MimeType, part.GetImageMedia().Url, getSupportedMimeTypesList()) + } + parts = append(parts, GeminiPart{ InlineData: &GeminiInlineData{ - MimeType: fileData.MimeType, + MimeType: fileData.MimeType, // 使用原始的 MimeType,因为大小写可能对API有意义 Data: fileData.Base64Data, }, }) @@ -291,100 +315,126 @@ func CovertGemini2OpenAI(textRequest dto.GeneralOpenAIRequest, info *relaycommon return &geminiRequest, nil } +// Helper function to get a list of supported MIME types for error messages +func getSupportedMimeTypesList() []string { + keys := make([]string, 0, len(geminiSupportedMimeTypes)) + for k := range geminiSupportedMimeTypes { + keys = append(keys, k) + } + return keys +} + // cleanFunctionParameters recursively removes unsupported fields from Gemini function parameters. func cleanFunctionParameters(params interface{}) interface{} { if params == nil { return nil } - paramMap, ok := params.(map[string]interface{}) - if !ok { - // Not a map, return as is (e.g., could be an array or primitive) - return params - } + switch v := params.(type) { + case map[string]interface{}: + // Create a copy to avoid modifying the original + cleanedMap := make(map[string]interface{}) + for k, val := range v { + cleanedMap[k] = val + } - // Create a copy to avoid modifying the original - cleanedMap := make(map[string]interface{}) - for k, v := range paramMap { - cleanedMap[k] = v - } + // Remove unsupported root-level fields + delete(cleanedMap, "default") + delete(cleanedMap, "exclusiveMaximum") + delete(cleanedMap, "exclusiveMinimum") + delete(cleanedMap, "$schema") + delete(cleanedMap, "additionalProperties") - // Remove unsupported root-level fields - delete(cleanedMap, "default") - delete(cleanedMap, "exclusiveMaximum") - delete(cleanedMap, "exclusiveMinimum") - delete(cleanedMap, "$schema") - delete(cleanedMap, "additionalProperties") - - // Clean properties - if props, ok := cleanedMap["properties"].(map[string]interface{}); ok && props != nil { - cleanedProps := make(map[string]interface{}) - for propName, propValue := range props { - propMap, ok := propValue.(map[string]interface{}) - if !ok { - cleanedProps[propName] = propValue // Keep non-map properties - continue - } - - // Create a copy of the property map - cleanedPropMap := make(map[string]interface{}) - for k, v := range propMap { - cleanedPropMap[k] = v - } - - // Remove unsupported fields - delete(cleanedPropMap, "default") - delete(cleanedPropMap, "exclusiveMaximum") - delete(cleanedPropMap, "exclusiveMinimum") - delete(cleanedPropMap, "$schema") - delete(cleanedPropMap, "additionalProperties") - - // Check and clean 'format' for string types - if propType, typeExists := cleanedPropMap["type"].(string); typeExists && propType == "string" { - if formatValue, formatExists := cleanedPropMap["format"].(string); formatExists { - if formatValue != "enum" && formatValue != "date-time" { - delete(cleanedPropMap, "format") - } + // Check and clean 'format' for string types + if propType, typeExists := cleanedMap["type"].(string); typeExists && propType == "string" { + if formatValue, formatExists := cleanedMap["format"].(string); formatExists { + if formatValue != "enum" && formatValue != "date-time" { + delete(cleanedMap, "format") } } + } - // Recursively clean nested properties within this property if it's an object/array - // Check the type before recursing - if propType, typeExists := cleanedPropMap["type"].(string); typeExists && (propType == "object" || propType == "array") { - cleanedProps[propName] = cleanFunctionParameters(cleanedPropMap) - } else { - cleanedProps[propName] = cleanedPropMap // Assign the cleaned map back if not recursing + // Clean properties + if props, ok := cleanedMap["properties"].(map[string]interface{}); ok && props != nil { + cleanedProps := make(map[string]interface{}) + for propName, propValue := range props { + cleanedProps[propName] = cleanFunctionParameters(propValue) } - + cleanedMap["properties"] = cleanedProps } - cleanedMap["properties"] = cleanedProps - } - // Recursively clean items in arrays if needed (e.g., type: array, items: { ... }) - if items, ok := cleanedMap["items"].(map[string]interface{}); ok && items != nil { - cleanedMap["items"] = cleanFunctionParameters(items) - } - // Also handle items if it's an array of schemas - if itemsArray, ok := cleanedMap["items"].([]interface{}); ok { - cleanedItemsArray := make([]interface{}, len(itemsArray)) - for i, item := range itemsArray { - cleanedItemsArray[i] = cleanFunctionParameters(item) + // Recursively clean items in arrays + if items, ok := cleanedMap["items"].(map[string]interface{}); ok && items != nil { + cleanedMap["items"] = cleanFunctionParameters(items) } - cleanedMap["items"] = cleanedItemsArray - } - - // Recursively clean other schema composition keywords if necessary - for _, field := range []string{"allOf", "anyOf", "oneOf"} { - if nested, ok := cleanedMap[field].([]interface{}); ok { - cleanedNested := make([]interface{}, len(nested)) - for i, item := range nested { - cleanedNested[i] = cleanFunctionParameters(item) + // Also handle items if it's an array of schemas + if itemsArray, ok := cleanedMap["items"].([]interface{}); ok { + cleanedItemsArray := make([]interface{}, len(itemsArray)) + for i, item := range itemsArray { + cleanedItemsArray[i] = cleanFunctionParameters(item) } - cleanedMap[field] = cleanedNested + cleanedMap["items"] = cleanedItemsArray } - } - return cleanedMap + // Recursively clean other schema composition keywords + for _, field := range []string{"allOf", "anyOf", "oneOf"} { + if nested, ok := cleanedMap[field].([]interface{}); ok { + cleanedNested := make([]interface{}, len(nested)) + for i, item := range nested { + cleanedNested[i] = cleanFunctionParameters(item) + } + cleanedMap[field] = cleanedNested + } + } + + // Recursively clean patternProperties + if patternProps, ok := cleanedMap["patternProperties"].(map[string]interface{}); ok { + cleanedPatternProps := make(map[string]interface{}) + for pattern, schema := range patternProps { + cleanedPatternProps[pattern] = cleanFunctionParameters(schema) + } + cleanedMap["patternProperties"] = cleanedPatternProps + } + + // Recursively clean definitions + if definitions, ok := cleanedMap["definitions"].(map[string]interface{}); ok { + cleanedDefinitions := make(map[string]interface{}) + for defName, defSchema := range definitions { + cleanedDefinitions[defName] = cleanFunctionParameters(defSchema) + } + cleanedMap["definitions"] = cleanedDefinitions + } + + // Recursively clean $defs (newer JSON Schema draft) + if defs, ok := cleanedMap["$defs"].(map[string]interface{}); ok { + cleanedDefs := make(map[string]interface{}) + for defName, defSchema := range defs { + cleanedDefs[defName] = cleanFunctionParameters(defSchema) + } + cleanedMap["$defs"] = cleanedDefs + } + + // Clean conditional keywords + for _, field := range []string{"if", "then", "else", "not"} { + if nested, ok := cleanedMap[field]; ok { + cleanedMap[field] = cleanFunctionParameters(nested) + } + } + + return cleanedMap + + case []interface{}: + // Handle arrays of schemas + cleanedArray := make([]interface{}, len(v)) + for i, item := range v { + cleanedArray[i] = cleanFunctionParameters(item) + } + return cleanedArray + + default: + // Not a map or array, return as is (e.g., could be a primitive) + return params + } } func removeAdditionalPropertiesWithDepth(schema interface{}, depth int) interface{} { diff --git a/relay/relay-image.go b/relay/relay-image.go index 36b4b9f8..dc63cce8 100644 --- a/relay/relay-image.go +++ b/relay/relay-image.go @@ -41,6 +41,9 @@ func getAndValidImageRequest(c *gin.Context, info *relaycommon.RelayInfo) (*dto. imageRequest.Quality = "standard" } } + if imageRequest.N == 0 { + imageRequest.N = 1 + } default: err := common.UnmarshalBodyReusable(c, imageRequest) if err != nil {