x

services/fetch_manager.py

@@ -0,0 +1,46 @@
+import asyncio
+from typing import Any, Dict, Optional
+
+from loguru import logger
+
+from core.config import Config
+
+from .fetch_service import FetchService
+
+
+class FetchManager:
+    def __init__(self, config: Config):
+        self.config = config
+        self.fetch_service = FetchService()
+        self.semaphore = asyncio.Semaphore(config.global_config.max_concurrency)
+        
+    async def request(
+        self, 
+        method: str, 
+        url: str, 
+        proxy: Optional[str] = None, 
+        **kwargs
+    ) -> Dict[str, Any]:
+        """
+        使用信号量控制并发的请求方法
+        """
+        async with self.semaphore:
+            for _ in range(self.config.global_config.retry_times):
+                try:
+                    response = await self.fetch_service.request(
+                        method=method,
+                        url=url,
+                        proxy=proxy,
+                        timeout=self.config.global_config.timeout,
+                        **kwargs
+                    )
+                    
+                    if 'error' not in response:
+                        return response
+                        
+                except asyncio.TimeoutError:
+                    logger.warning(f"请求超时,正在重试: {url}")
+                    continue
+                    
+            logger.error(f"达到最大重试次数: {url}")
+            return {'error': 'Max retries exceeded'}