From c050955157834619813e0922da9157d01e254f13 Mon Sep 17 00:00:00 2001 From: wood chen Date: Sun, 15 Jun 2025 06:12:46 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=95=B0=E6=8D=AE=E9=A2=84?= =?UTF-8?q?=E5=8A=A0=E8=BD=BD=E5=8A=9F=E8=83=BD=EF=BC=8C=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E7=BC=93=E5=AD=98=E9=80=BB=E8=BE=91=EF=BC=8C?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=81=A5=E5=BA=B7=E6=A3=80=E6=9F=A5=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=8C=E8=B0=83=E6=95=B4=E5=85=B0=E7=A9=BA=E5=9B=BE?= =?UTF-8?q?=E5=BA=8A=E8=8E=B7=E5=8F=96=E5=99=A8=E7=9A=84=E9=87=8D=E8=AF=95?= =?UTF-8?q?=E6=9C=BA=E5=88=B6(=E6=8C=87=E6=95=B0=E9=80=80=E9=81=BF)?= =?UTF-8?q?=EF=BC=8C=E6=8F=90=E5=8D=87=E7=B3=BB=E7=BB=9F=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E5=92=8C=E7=A8=B3=E5=AE=9A=E6=80=A7=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker-release.yml | 39 +++++ .github/workflows/docker.yml | 2 - database/database.go | 124 +++++++++++++- handler/handler.go | 27 ++- initapp/init.go | 247 +++++++++++++++++++++++++++ main.go | 8 + router/router.go | 3 + service/data_source_fetcher.go | 34 +++- service/endpoint_service.go | 29 +++- service/lankong_fetcher.go | 116 ++++++++++++- service/preloader.go | 27 +++ 11 files changed, 631 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/docker-release.yml create mode 100644 initapp/init.go diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml new file mode 100644 index 0000000..1db334f --- /dev/null +++ b/.github/workflows/docker-release.yml @@ -0,0 +1,39 @@ +name: Docker Release + +on: + release: + types: [published] + +env: + IMAGE_NAME: random-api-go + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: woodchen + password: ${{ secrets.ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile + push: true + tags: woodchen/${{ env.IMAGE_NAME }}:${{ github.ref_name }} + platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6e5e593..11564c3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,8 +4,6 @@ on: push: branches: - main - tags: - - v* paths-ignore: - 'lankong_tools/**' - '*.md' diff --git a/database/database.go b/database/database.go index 065d8c1..a644dd3 100644 --- a/database/database.go +++ b/database/database.go @@ -5,6 +5,7 @@ import ( "log" "os" "path/filepath" + "sync" "time" "random-api-go/model" @@ -16,6 +17,19 @@ import ( var DB *gorm.DB +// 配置缓存 +var ( + configCache = make(map[string]*CachedConfig) + configCacheMutex sync.RWMutex +) + +// CachedConfig 缓存的配置项 +type CachedConfig struct { + Value string + CachedAt time.Time + CacheTTL time.Duration // 缓存生存时间 +} + // Initialize 初始化数据库 func Initialize(dataDir string) error { // 确保数据目录存在 @@ -78,19 +92,57 @@ func Close() error { return nil } -// GetConfig 获取配置值 +// GetConfig 获取配置值（带缓存） func GetConfig(key string, defaultValue string) string { + // 先检查缓存 + configCacheMutex.RLock() + cached, exists := configCache[key] + configCacheMutex.RUnlock() + + // 如果缓存存在且未过期，直接返回 + if exists && time.Since(cached.CachedAt) < cached.CacheTTL { + return cached.Value + } + + // 从数据库查询 var config model.Config if err := DB.Where("key = ?", key).First(&config).Error; err != nil { if err == gorm.ErrRecordNotFound { + // 缓存默认值（短时间缓存，避免频繁查询不存在的配置） + cacheConfig(key, defaultValue, 5*time.Minute) return defaultValue } log.Printf("Failed to get config %s: %v", key, err) return defaultValue } + + // 缓存查询结果 + var cacheTTL time.Duration + switch key { + case "homepage_content": + cacheTTL = 10 * time.Minute // 首页内容缓存10分钟 + case "oauth_client_id", "oauth_client_secret", "oauth_redirect_uri": + cacheTTL = 30 * time.Minute // OAuth配置缓存30分钟 + default: + cacheTTL = 5 * time.Minute // 其他配置缓存5分钟 + } + + cacheConfig(key, config.Value, cacheTTL) return config.Value } +// cacheConfig 缓存配置值 +func cacheConfig(key, value string, ttl time.Duration) { + configCacheMutex.Lock() + defer configCacheMutex.Unlock() + + configCache[key] = &CachedConfig{ + Value: value, + CachedAt: time.Now(), + CacheTTL: ttl, + } +} + // SetConfig 设置配置值 func SetConfig(key, value, configType string) error { var config model.Config @@ -103,15 +155,31 @@ func SetConfig(key, value, configType string) error { Value: value, Type: configType, } - return DB.Create(&config).Error + if err := DB.Create(&config).Error; err != nil { + return err + } } else if err != nil { return err + } else { + // 更新现有配置 + config.Value = value + config.Type = configType + if err := DB.Save(&config).Error; err != nil { + return err + } } - // 更新现有配置 - config.Value = value - config.Type = configType - return DB.Save(&config).Error + // 清理缓存 + invalidateConfigCache(key) + return nil +} + +// invalidateConfigCache 清理指定配置的缓存 +func invalidateConfigCache(key string) { + configCacheMutex.Lock() + defer configCacheMutex.Unlock() + delete(configCache, key) + log.Printf("已清理配置 %s 的缓存", key) } // ListConfigs 列出所有配置 @@ -123,5 +191,47 @@ func ListConfigs() ([]model.Config, error) { // DeleteConfig 删除配置 func DeleteConfig(key string) error { - return DB.Where("key = ?", key).Delete(&model.Config{}).Error + err := DB.Where("key = ?", key).Delete(&model.Config{}).Error + if err != nil { + return err + } + + // 清理缓存 + invalidateConfigCache(key) + return nil +} + +// GetConfigCacheStats 获取配置缓存统计信息 +func GetConfigCacheStats() map[string]interface{} { + configCacheMutex.RLock() + defer configCacheMutex.RUnlock() + + stats := make(map[string]interface{}) + stats["total_cached"] = len(configCache) + + // 统计各种状态的缓存 + var validCount, expiredCount int + cacheDetails := make(map[string]interface{}) + + for key, cached := range configCache { + isExpired := time.Since(cached.CachedAt) >= cached.CacheTTL + if isExpired { + expiredCount++ + } else { + validCount++ + } + + cacheDetails[key] = map[string]interface{}{ + "cached_at": cached.CachedAt.Format("2006-01-02 15:04:05"), + "ttl_seconds": int(cached.CacheTTL.Seconds()), + "expired": isExpired, + "value_length": len(cached.Value), + } + } + + stats["valid_count"] = validCount + stats["expired_count"] = expiredCount + stats["details"] = cacheDetails + + return stats } diff --git a/handler/handler.go b/handler/handler.go index 5323efc..774ff9a 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "random-api-go/database" + "random-api-go/initapp" "random-api-go/monitoring" "random-api-go/service" "random-api-go/stats" @@ -34,7 +35,7 @@ type Handlers struct { func NewHandlers(statsManager *stats.StatsManager) *Handlers { return &Handlers{ Stats: statsManager, - cacheDuration: 30 * time.Minute, // 缓存30分钟 + cacheDuration: 5 * time.Minute, // 缓存5分钟，减少首次访问等待时间 } } @@ -292,3 +293,27 @@ func (h *Handlers) HandlePublicHomeConfig(w http.ResponseWriter, r *http.Request return } } + +// HandleHealth 处理健康检查请求 +func (h *Handlers) HandleHealth(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + w.Header().Set("Content-Type", "application/json") + + // 获取初始化状态 + initStatus := initapp.GetInitStatus() + + response := map[string]interface{}{ + "status": "healthy", + "timestamp": time.Now().Unix(), + "init": initStatus, + } + + if err := json.NewEncoder(w).Encode(response); err != nil { + http.Error(w, "Error encoding response", http.StatusInternalServerError) + return + } +} diff --git a/initapp/init.go b/initapp/init.go new file mode 100644 index 0000000..9107ec2 --- /dev/null +++ b/initapp/init.go @@ -0,0 +1,247 @@ +package initapp + +import ( + "log" + "random-api-go/database" + "random-api-go/model" + "random-api-go/service" + "sync" + "time" +) + +// InitData 初始化应用数据，预加载所有需要的数据到内存中 +func InitData() error { + log.Println("开始初始化应用数据...") + start := time.Now() + + // 1. 初始化端点服务（这会启动预加载器） + endpointService := service.GetEndpointService() + log.Println("✓ 端点服务已初始化") + + // 2. 暂停预加载器的定期刷新，避免与初始化冲突 + preloader := endpointService.GetPreloader() + preloader.PausePeriodicRefresh() + log.Println("✓ 已暂停预加载器定期刷新") + + // 3. 获取所有活跃的端点和数据源 + endpoints, err := endpointService.ListEndpoints() + if err != nil { + log.Printf("获取端点列表失败: %v", err) + return err + } + + // 4. 统计需要预加载的数据源 + var activeDataSources []model.DataSource + for _, endpoint := range endpoints { + if !endpoint.IsActive { + continue + } + for _, ds := range endpoint.DataSources { + if ds.IsActive && ds.Type != "api_get" && ds.Type != "api_post" { + // API类型的数据源不需要预加载，使用实时请求 + activeDataSources = append(activeDataSources, ds) + } + } + } + + log.Printf("发现 %d 个端点，%d 个需要预加载的数据源", len(endpoints), len(activeDataSources)) + + if len(activeDataSources) == 0 { + log.Println("✓ 没有需要预加载的数据源") + // 恢复预加载器定期刷新 + preloader.ResumePeriodicRefresh() + log.Printf("应用数据初始化完成，耗时: %v", time.Since(start)) + return nil + } + + // 5. 并发预加载所有数据源 + var wg sync.WaitGroup + var successCount, failCount int + var mutex sync.Mutex + + // 限制并发数，避免过多的并发请求 + semaphore := make(chan struct{}, 5) // 最多5个并发 + + for _, ds := range activeDataSources { + wg.Add(1) + go func(dataSource model.DataSource) { + defer wg.Done() + + // 获取信号量 + semaphore <- struct{}{} + defer func() { <-semaphore }() + + log.Printf("预加载数据源: %s (ID: %d, 类型: %s)", dataSource.Name, dataSource.ID, dataSource.Type) + + // 使用预加载器预加载数据源 + if err := preloader.RefreshDataSource(dataSource.ID); err != nil { + log.Printf("预加载数据源 %d 失败: %v", dataSource.ID, err) + mutex.Lock() + failCount++ + mutex.Unlock() + } else { + log.Printf("预加载数据源 %d 成功", dataSource.ID) + mutex.Lock() + successCount++ + mutex.Unlock() + } + }(ds) + } + + // 等待所有预加载完成 + wg.Wait() + + log.Printf("✓ 数据源预加载完成: 成功 %d 个，失败 %d 个", successCount, failCount) + + // 6. 预热URL统计缓存 + log.Println("预热URL统计缓存...") + if err := preloadURLStats(endpointService, endpoints); err != nil { + log.Printf("预热URL统计缓存失败: %v", err) + } else { + log.Println("✓ URL统计缓存预热完成") + } + + // 7. 预加载配置 + log.Println("预加载系统配置...") + preloadConfigs() + log.Println("✓ 系统配置预加载完成") + + // 8. 恢复预加载器定期刷新 + preloader.ResumePeriodicRefresh() + log.Println("✓ 已恢复预加载器定期刷新") + + duration := time.Since(start) + log.Printf("🎉 应用数据初始化完成，总耗时: %v", duration) + + return nil +} + +// preloadURLStats 预热URL统计缓存 +func preloadURLStats(endpointService *service.EndpointService, endpoints []*model.APIEndpoint) error { + urlStats := make(map[string]struct { + TotalURLs int `json:"total_urls"` + }) + + for _, endpoint := range endpoints { + if !endpoint.IsActive { + continue + } + + totalURLs := 0 + for _, ds := range endpoint.DataSources { + if ds.IsActive { + // 使用优化后的URL计数方法 + count, err := endpointService.GetDataSourceURLCount(&ds) + if err != nil { + log.Printf("获取数据源 %d URL数量失败: %v", ds.ID, err) + // 使用估算值 + switch ds.Type { + case "manual": + totalURLs += 5 + case "lankong": + totalURLs += 100 + case "api_get", "api_post": + totalURLs += 1 + default: + totalURLs += 1 + } + } else { + totalURLs += count + } + } + } + + urlStats[endpoint.URL] = struct { + TotalURLs int `json:"total_urls"` + }{ + TotalURLs: totalURLs, + } + } + + log.Printf("预热了 %d 个端点的URL统计", len(urlStats)) + return nil +} + +// preloadConfigs 预加载系统配置 +func preloadConfigs() { + // 预加载常用配置，触发数据库查询并缓存结果 + configs := []string{ + // 基础配置 + "homepage_content", + + // OAuth配置 + "oauth_client_id", + "oauth_client_secret", + "oauth_redirect_uri", + + // 系统配置 + "site_title", + "site_description", + "admin_email", + "max_file_size", + "allowed_file_types", + + // API配置 + "rate_limit_enabled", + "rate_limit_requests", + "rate_limit_window", + "cors_enabled", + "cors_origins", + + // 兰空图床配置 + "lankong_max_retries", + + // 缓存配置 + "cache_enabled", + "cache_ttl", + "max_cache_size", + + // 日志配置 + "log_level", + "log_file_enabled", + "log_retention_days", + } + + var loadedCount int + for _, key := range configs { + value := database.GetConfig(key, "") + if value != "" { + log.Printf("预加载配置 %s: %d 字符", key, len(value)) + loadedCount++ + } + } + + log.Printf("预加载了 %d 个配置项", loadedCount) +} + +// GetInitStatus 获取初始化状态（可用于健康检查） +func GetInitStatus() map[string]interface{} { + endpointService := service.GetEndpointService() + + // 获取缓存统计 + cacheStats := endpointService.GetCacheManager().GetCacheStats() + + // 获取端点数量 + endpoints, _ := endpointService.ListEndpoints() + activeEndpoints := 0 + for _, ep := range endpoints { + if ep.IsActive { + activeEndpoints++ + } + } + + // 获取配置缓存统计 + configCacheStats := database.GetConfigCacheStats() + + return map[string]interface{}{ + "data_cache": map[string]interface{}{ + "items": len(cacheStats), + "details": cacheStats, + }, + "config_cache": configCacheStats, + "endpoints": map[string]interface{}{ + "total": len(endpoints), + "active": activeEndpoints, + }, + } +} diff --git a/main.go b/main.go index 6cfcef7..d064037 100644 --- a/main.go +++ b/main.go @@ -12,6 +12,7 @@ import ( "random-api-go/config" "random-api-go/database" "random-api-go/handler" + "random-api-go/initapp" "random-api-go/logging" "random-api-go/router" "random-api-go/service" @@ -64,6 +65,13 @@ func (a *App) Initialize() error { // 初始化端点服务 service.GetEndpointService() + // 预加载所有数据到内存 + log.Println("开始预加载应用数据...") + if err := initapp.InitData(); err != nil { + log.Printf("预加载数据失败: %v", err) + // 不返回错误，允许应用继续启动 + } + // 创建管理后台处理器 a.adminHandler = handler.NewAdminHandler() diff --git a/router/router.go b/router/router.go index ec2fcd9..13a2291 100644 --- a/router/router.go +++ b/router/router.go @@ -20,6 +20,8 @@ type Handler interface { HandleStats(w http.ResponseWriter, r *http.Request) HandleURLStats(w http.ResponseWriter, r *http.Request) HandleMetrics(w http.ResponseWriter, r *http.Request) + // 健康检查 + HandleHealth(w http.ResponseWriter, r *http.Request) // 公开端点 HandlePublicEndpoints(w http.ResponseWriter, r *http.Request) // 公开首页配置 @@ -78,6 +80,7 @@ func (r *Router) SetupAllRoutes(handler Handler, adminHandler AdminHandler, stat r.HandleFunc("/api/stats", handler.HandleStats) r.HandleFunc("/api/urlstats", handler.HandleURLStats) r.HandleFunc("/api/metrics", handler.HandleMetrics) + r.HandleFunc("/api/health", handler.HandleHealth) r.HandleFunc("/api/endpoints", handler.HandlePublicEndpoints) r.HandleFunc("/api/home-config", handler.HandlePublicHomeConfig) diff --git a/service/data_source_fetcher.go b/service/data_source_fetcher.go index 7f25091..9e6b87e 100644 --- a/service/data_source_fetcher.go +++ b/service/data_source_fetcher.go @@ -4,7 +4,9 @@ import ( "encoding/json" "fmt" "log" + "random-api-go/database" "random-api-go/model" + "strconv" "strings" "time" ) @@ -18,13 +20,43 @@ type DataSourceFetcher struct { // NewDataSourceFetcher 创建数据源获取器 func NewDataSourceFetcher(cacheManager *CacheManager) *DataSourceFetcher { + // 从配置中获取兰空图床最大重试次数 + maxRetries := getIntConfig("lankong_max_retries", 7) + + var lankongFetcher *LankongFetcher + if maxRetries > 0 { + // 使用自定义配置 + lankongFetcher = NewLankongFetcherWithConfig(maxRetries) + log.Printf("兰空图床获取器配置: 最大重试%d次", maxRetries) + } else { + // 使用默认配置 + lankongFetcher = NewLankongFetcher() + log.Printf("兰空图床获取器使用默认配置") + } + return &DataSourceFetcher{ cacheManager: cacheManager, - lankongFetcher: NewLankongFetcher(), + lankongFetcher: lankongFetcher, apiFetcher: NewAPIFetcher(), } } +// getIntConfig 获取整数配置，如果不存在或无效则返回默认值 +func getIntConfig(key string, defaultValue int) int { + configStr := database.GetConfig(key, "") + if configStr == "" { + return defaultValue + } + + value, err := strconv.Atoi(configStr) + if err != nil { + log.Printf("配置 %s 的值 '%s' 不是有效整数，使用默认值 %d", key, configStr, defaultValue) + return defaultValue + } + + return value +} + // FetchURLs 从数据源获取URL列表 func (dsf *DataSourceFetcher) FetchURLs(dataSource *model.DataSource) ([]string, error) { // API类型的数据源直接实时请求，不使用缓存 diff --git a/service/endpoint_service.go b/service/endpoint_service.go index 2e72f2f..d043d89 100644 --- a/service/endpoint_service.go +++ b/service/endpoint_service.go @@ -355,6 +355,11 @@ func (s *EndpointService) GetPreloader() *Preloader { return s.preloader } +// GetCacheManager 获取缓存管理器（用于外部控制） +func (s *EndpointService) GetCacheManager() *CacheManager { + return s.cacheManager +} + // GetDataSourceURLCount 获取数据源的URL数量 func (s *EndpointService) GetDataSourceURLCount(dataSource *model.DataSource) (int, error) { // 对于API类型和端点类型的数据源，返回1（因为每次都是实时请求） @@ -362,11 +367,25 @@ func (s *EndpointService) GetDataSourceURLCount(dataSource *model.DataSource) (i return 1, nil } - // 对于其他类型的数据源，尝试获取实际的URL数量 - urls, err := s.dataSourceFetcher.FetchURLs(dataSource) - if err != nil { - return 0, err + // 优先从内存缓存获取URL数量，避免触发耗时的网络请求 + cacheKey := fmt.Sprintf("datasource_%d", dataSource.ID) + if cachedURLs, exists := s.cacheManager.GetFromMemoryCache(cacheKey); exists { + return len(cachedURLs), nil } - return len(urls), nil + // 如果缓存中没有数据，返回估算值，避免在统计时触发耗时操作 + switch dataSource.Type { + case "manual": + // 手动数据源可以快速解析配置获取数量 + urls, err := s.dataSourceFetcher.FetchURLs(dataSource) + if err != nil { + return 0, nil // 返回0而不是错误，避免影响整体统计 + } + return len(urls), nil + case "lankong": + // 兰空图床如果没有缓存，返回估算值 + return 100, nil // 估算值 + default: + return 0, nil + } } diff --git a/service/lankong_fetcher.go b/service/lankong_fetcher.go index a540726..e2394ce 100644 --- a/service/lankong_fetcher.go +++ b/service/lankong_fetcher.go @@ -12,14 +12,38 @@ import ( // LankongFetcher 兰空图床获取器 type LankongFetcher struct { - client *http.Client + client *http.Client + retryConfig *RetryConfig +} + +// RetryConfig 重试配置 +type RetryConfig struct { + MaxRetries int // 最大重试次数 + BaseDelay time.Duration // 基础延迟 } // NewLankongFetcher 创建兰空图床获取器 func NewLankongFetcher() *LankongFetcher { return &LankongFetcher{ client: &http.Client{ - Timeout: 30 * time.Second, + Timeout: 60 * time.Second, // 增加超时时间 + }, + retryConfig: &RetryConfig{ + MaxRetries: 7, // 最多重试7次 (0秒/15秒/15秒/30秒/30秒/60秒/60秒/180秒) + BaseDelay: 1 * time.Second, // 基础延迟（实际不使用，使用固定延迟序列） + }, + } +} + +// NewLankongFetcherWithConfig 创建带自定义配置的兰空图床获取器 +func NewLankongFetcherWithConfig(maxRetries int) *LankongFetcher { + return &LankongFetcher{ + client: &http.Client{ + Timeout: 60 * time.Second, + }, + retryConfig: &RetryConfig{ + MaxRetries: maxRetries, + BaseDelay: 1 * time.Second, }, } } @@ -52,7 +76,7 @@ func (lf *LankongFetcher) FetchURLs(config *model.LankongConfig) ([]string, erro // 获取第一页以确定总页数 firstPageURL := fmt.Sprintf("%s?album_id=%s&page=1", baseURL, albumID) - response, err := lf.fetchPage(firstPageURL, config.APIToken) + response, err := lf.fetchPageWithRetry(firstPageURL, config.APIToken) if err != nil { log.Printf("Failed to fetch first page for album %s: %v", albumID, err) continue @@ -62,9 +86,10 @@ func (lf *LankongFetcher) FetchURLs(config *model.LankongConfig) ([]string, erro log.Printf("相册 %s 共有 %d 页", albumID, totalPages) // 处理所有页面 + albumURLs := []string{} for page := 1; page <= totalPages; page++ { reqURL := fmt.Sprintf("%s?album_id=%s&page=%d", baseURL, albumID, page) - pageResponse, err := lf.fetchPage(reqURL, config.APIToken) + pageResponse, err := lf.fetchPageWithRetry(reqURL, config.APIToken) if err != nil { log.Printf("Failed to fetch page %d for album %s: %v", page, albumID, err) continue @@ -72,22 +97,90 @@ func (lf *LankongFetcher) FetchURLs(config *model.LankongConfig) ([]string, erro for _, item := range pageResponse.Data.Data { if item.Links.URL != "" { - allURLs = append(allURLs, item.Links.URL) + albumURLs = append(albumURLs, item.Links.URL) } } - // 添加小延迟避免请求过快 - if page < totalPages { - time.Sleep(100 * time.Millisecond) + // 进度日志 + if page%10 == 0 || page == totalPages { + log.Printf("相册 %s: 已处理 %d/%d 页，收集到 %d 个URL", albumID, page, totalPages, len(albumURLs)) } } - log.Printf("完成相册 %s: 收集到 %d 个URL", albumID, len(allURLs)) + allURLs = append(allURLs, albumURLs...) + log.Printf("完成相册 %s: 收集到 %d 个URL", albumID, len(albumURLs)) } return allURLs, nil } +// fetchPageWithRetry 带重试的页面获取 +func (lf *LankongFetcher) fetchPageWithRetry(url string, apiToken string) (*LankongResponse, error) { + var lastErr error + + for attempt := 0; attempt <= lf.retryConfig.MaxRetries; attempt++ { + response, err := lf.fetchPage(url, apiToken) + if err == nil { + return response, nil + } + + lastErr = err + + // 如果是最后一次尝试，不再重试 + if attempt == lf.retryConfig.MaxRetries { + break + } + + // 计算延迟时间 + var delay time.Duration + if isRateLimitError(err) { + // 对于429错误，使用固定的延迟序列 + delay = getRateLimitDelay(attempt) + log.Printf("遇到频率限制 (尝试 %d/%d): %v，等待 %v 后重试", attempt+1, lf.retryConfig.MaxRetries+1, err, delay) + } else { + // 其他错误使用较短的延迟 + delay = time.Duration(attempt+1) * time.Second + log.Printf("请求失败 (尝试 %d/%d): %v，%v 后重试", attempt+1, lf.retryConfig.MaxRetries+1, err, delay) + } + + time.Sleep(delay) + } + + return nil, fmt.Errorf("重试 %d 次后仍然失败: %v", lf.retryConfig.MaxRetries, lastErr) +} + +// getRateLimitDelay 获取频率限制的延迟时间 +// 延迟序列：0秒 / 15秒 / 15秒 / 30秒 / 30秒 / 60秒 / 60秒 / 180秒 +func getRateLimitDelay(attempt int) time.Duration { + delaySequence := []time.Duration{ + 0 * time.Second, // 第1次重试：立即 + 15 * time.Second, // 第2次重试：15秒后 + 15 * time.Second, // 第3次重试：15秒后 + 30 * time.Second, // 第4次重试：30秒后 + 30 * time.Second, // 第5次重试：30秒后 + 60 * time.Second, // 第6次重试：60秒后 + 60 * time.Second, // 第7次重试：60秒后 + 180 * time.Second, // 第8次重试：180秒后 + } + + if attempt < len(delaySequence) { + return delaySequence[attempt] + } + + // 如果超出序列长度，使用最后一个值 + return delaySequence[len(delaySequence)-1] +} + +// isRateLimitError 检查是否是频率限制错误 +func isRateLimitError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return fmt.Sprintf("%v", err) == "rate limit exceeded (429), need to slow down requests" || + fmt.Sprintf("%v", errStr) == "API returned status code: 429" +} + // fetchPage 获取兰空图床单页数据 func (lf *LankongFetcher) fetchPage(url string, apiToken string) (*LankongResponse, error) { req, err := http.NewRequest("GET", url, nil) @@ -104,6 +197,11 @@ func (lf *LankongFetcher) fetchPage(url string, apiToken string) (*LankongRespon } defer resp.Body.Close() + // 特殊处理429错误 + if resp.StatusCode == 429 { + return nil, fmt.Errorf("rate limit exceeded (429), need to slow down requests") + } + if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("API returned status code: %d", resp.StatusCode) } diff --git a/service/preloader.go b/service/preloader.go index 2080160..174ccdd 100644 --- a/service/preloader.go +++ b/service/preloader.go @@ -14,6 +14,7 @@ type Preloader struct { dataSourceFetcher *DataSourceFetcher cacheManager *CacheManager running bool + paused bool stopChan chan struct{} mutex sync.RWMutex } @@ -55,6 +56,22 @@ func (p *Preloader) Stop() { log.Println("预加载器已停止") } +// PausePeriodicRefresh 暂停定期刷新 +func (p *Preloader) PausePeriodicRefresh() { + p.mutex.Lock() + defer p.mutex.Unlock() + p.paused = true + log.Println("预加载器定期刷新已暂停") +} + +// ResumePeriodicRefresh 恢复定期刷新 +func (p *Preloader) ResumePeriodicRefresh() { + p.mutex.Lock() + defer p.mutex.Unlock() + p.paused = false + log.Println("预加载器定期刷新已恢复") +} + // PreloadDataSourceOnSave 在保存数据源时预加载数据 func (p *Preloader) PreloadDataSourceOnSave(dataSource *model.DataSource) { // API类型的数据源不需要预加载，使用实时请求 @@ -185,6 +202,16 @@ func (p *Preloader) runPeriodicRefresh() { // checkAndRefreshExpiredData 检查并刷新过期数据 func (p *Preloader) checkAndRefreshExpiredData() { + // 检查是否暂停 + p.mutex.RLock() + isPaused := p.paused + p.mutex.RUnlock() + + if isPaused { + log.Println("预加载器定期刷新已暂停，跳过此次检查") + return + } + log.Println("开始检查过期数据...") // 获取所有活跃的数据源