diff --git a/database/database.go b/database/database.go index c13e86a..065d8c1 100644 --- a/database/database.go +++ b/database/database.go @@ -62,7 +62,6 @@ func autoMigrate() error { &model.APIEndpoint{}, &model.DataSource{}, &model.URLReplaceRule{}, - &model.CachedURL{}, &model.Config{}, ) } @@ -79,11 +78,6 @@ func Close() error { return nil } -// CleanExpiredCache 清理过期缓存 -func CleanExpiredCache() error { - return DB.Where("expires_at < ?", time.Now()).Delete(&model.CachedURL{}).Error -} - // GetConfig 获取配置值 func GetConfig(key string, defaultValue string) string { var config model.Config diff --git a/handler/admin_handler.go b/handler/admin_handler.go index c6073d7..42f6d70 100644 --- a/handler/admin_handler.go +++ b/handler/admin_handler.go @@ -391,9 +391,7 @@ func (h *AdminHandler) UpdateDataSource(w http.ResponseWriter, r *http.Request) if updateData.Config != "" { dataSource.Config = updateData.Config } - if updateData.CacheDuration != 0 { - dataSource.CacheDuration = updateData.CacheDuration - } + dataSource.IsActive = updateData.IsActive // 使用服务更新数据源（会自动预加载） diff --git a/handler/handler.go b/handler/handler.go index 2eb0232..5323efc 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -34,7 +34,7 @@ type Handlers struct { func NewHandlers(statsManager *stats.StatsManager) *Handlers { return &Handlers{ Stats: statsManager, - cacheDuration: 5 * time.Minute, // 缓存5分钟 + cacheDuration: 30 * time.Minute, // 缓存30分钟 } } diff --git a/model/api_endpoint.go b/model/api_endpoint.go index 8ecd8f7..2353763 100644 --- a/model/api_endpoint.go +++ b/model/api_endpoint.go @@ -26,21 +26,19 @@ type APIEndpoint struct { // DataSource 数据源模型 type DataSource struct { - ID uint `json:"id" gorm:"primaryKey"` - EndpointID uint `json:"endpoint_id" gorm:"not null;index"` - Name string `json:"name" gorm:"not null"` - Type string `json:"type" gorm:"not null;check:type IN ('lankong', 'manual', 'api_get', 'api_post', 'endpoint')"` - Config string `json:"config" gorm:"not null"` - CacheDuration int `json:"cache_duration" gorm:"default:3600"` // 缓存时长（秒） - IsActive bool `json:"is_active" gorm:"default:true"` - LastSync *time.Time `json:"last_sync,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - DeletedAt gorm.DeletedAt `json:"-" gorm:"index"` + ID uint `json:"id" gorm:"primaryKey"` + EndpointID uint `json:"endpoint_id" gorm:"not null;index"` + Name string `json:"name" gorm:"not null"` + Type string `json:"type" gorm:"not null;check:type IN ('lankong', 'manual', 'api_get', 'api_post', 'endpoint')"` + Config string `json:"config" gorm:"not null"` + IsActive bool `json:"is_active" gorm:"default:true"` + LastSync *time.Time `json:"last_sync,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DeletedAt gorm.DeletedAt `json:"-" gorm:"index"` // 关联 - Endpoint APIEndpoint `json:"-" gorm:"foreignKey:EndpointID"` - CachedURLs []CachedURL `json:"-" gorm:"foreignKey:DataSourceID"` + Endpoint APIEndpoint `json:"-" gorm:"foreignKey:EndpointID"` } // URLReplaceRule URL替换规则模型 @@ -59,19 +57,6 @@ type URLReplaceRule struct { Endpoint *APIEndpoint `json:"endpoint,omitempty" gorm:"foreignKey:EndpointID"` } -// CachedURL 缓存URL模型 -type CachedURL struct { - ID uint `json:"id" gorm:"primaryKey"` - DataSourceID uint `json:"data_source_id" gorm:"not null;index"` - OriginalURL string `json:"original_url" gorm:"not null"` - FinalURL string `json:"final_url" gorm:"not null"` - ExpiresAt time.Time `json:"expires_at" gorm:"index"` - CreatedAt time.Time `json:"created_at"` - - // 关联 - DataSource DataSource `json:"-" gorm:"foreignKey:DataSourceID"` -} - // Config 通用配置表 type Config struct { ID uint `json:"id" gorm:"primaryKey"` diff --git a/service/cache_manager.go b/service/cache_manager.go index d6d1e25..424b092 100644 --- a/service/cache_manager.go +++ b/service/cache_manager.go @@ -1,30 +1,28 @@ package service import ( + "fmt" "log" - "random-api-go/database" - "random-api-go/model" "sync" - "time" ) // CacheManager 缓存管理器 type CacheManager struct { - memoryCache map[string]*CachedEndpoint + memoryCache map[string]*CachedItem mutex sync.RWMutex } -// 注意：CachedEndpoint 类型定义在 endpoint_service.go 中 +// CachedItem 缓存项（永久缓存，只在数据变动时刷新） +type CachedItem struct { + URLs []string +} // NewCacheManager 创建缓存管理器 func NewCacheManager() *CacheManager { cm := &CacheManager{ - memoryCache: make(map[string]*CachedEndpoint), + memoryCache: make(map[string]*CachedItem), } - // 启动定期清理过期缓存的协程 - go cm.cleanupExpiredCache() - return cm } @@ -41,12 +39,12 @@ func (cm *CacheManager) GetFromMemoryCache(key string) ([]string, bool) { return cached.URLs, true } -// SetMemoryCache 设置内存缓存（duration参数保留以兼容现有接口，但不再使用） -func (cm *CacheManager) SetMemoryCache(key string, urls []string, duration time.Duration) { +// SetMemoryCache 设置内存缓存 +func (cm *CacheManager) SetMemoryCache(key string, urls []string) { cm.mutex.Lock() defer cm.mutex.Unlock() - cm.memoryCache[key] = &CachedEndpoint{ + cm.memoryCache[key] = &CachedItem{ URLs: urls, } } @@ -59,118 +57,28 @@ func (cm *CacheManager) InvalidateMemoryCache(key string) { delete(cm.memoryCache, key) } -// GetFromDBCache 从数据库缓存获取URL -func (cm *CacheManager) GetFromDBCache(dataSourceID uint) ([]string, error) { - var cachedURLs []model.CachedURL - if err := database.DB.Where("data_source_id = ? AND expires_at > ?", dataSourceID, time.Now()). - Find(&cachedURLs).Error; err != nil { - return nil, err - } - - var urls []string - for _, cached := range cachedURLs { - urls = append(urls, cached.FinalURL) - } - - return urls, nil -} - -// SetDBCache 设置数据库缓存 -func (cm *CacheManager) SetDBCache(dataSourceID uint, urls []string, duration time.Duration) error { - // 先删除旧的缓存 - if err := database.DB.Where("data_source_id = ?", dataSourceID).Delete(&model.CachedURL{}).Error; err != nil { - log.Printf("Failed to delete old cache for data source %d: %v", dataSourceID, err) - } - - // 插入新的缓存 - expiresAt := time.Now().Add(duration) - for _, url := range urls { - cachedURL := model.CachedURL{ - DataSourceID: dataSourceID, - OriginalURL: url, - FinalURL: url, - ExpiresAt: expiresAt, - } - if err := database.DB.Create(&cachedURL).Error; err != nil { - log.Printf("Failed to cache URL: %v", err) - } - } - - return nil -} - -// UpdateDBCacheIfChanged 只有当数据变化时才更新数据库缓存，并返回是否需要清理内存缓存 -func (cm *CacheManager) UpdateDBCacheIfChanged(dataSourceID uint, newURLs []string, duration time.Duration) (bool, error) { - // 获取现有缓存 - existingURLs, err := cm.GetFromDBCache(dataSourceID) - if err != nil { - // 如果获取失败，直接设置新缓存 - return true, cm.SetDBCache(dataSourceID, newURLs, duration) - } - - // 比较URL列表是否相同 - if cm.urlSlicesEqual(existingURLs, newURLs) { - // 数据没有变化，只更新过期时间 - expiresAt := time.Now().Add(duration) - if err := database.DB.Model(&model.CachedURL{}). - Where("data_source_id = ?", dataSourceID). - Update("expires_at", expiresAt).Error; err != nil { - log.Printf("Failed to update cache expiry for data source %d: %v", dataSourceID, err) - } - return false, nil - } - - // 数据有变化，更新缓存 - return true, cm.SetDBCache(dataSourceID, newURLs, duration) -} - // InvalidateMemoryCacheForDataSource 清理与数据源相关的内存缓存 -func (cm *CacheManager) InvalidateMemoryCacheForDataSource(dataSourceID uint) error { - // 获取数据源信息 - var dataSource model.DataSource - if err := database.DB.Preload("Endpoint").First(&dataSource, dataSourceID).Error; err != nil { - return err - } - - // 清理该端点的内存缓存 - cm.InvalidateMemoryCache(dataSource.Endpoint.URL) - log.Printf("已清理端点 %s 的内存缓存（数据源 %d 数据发生变化）", dataSource.Endpoint.URL, dataSourceID) - - return nil +func (cm *CacheManager) InvalidateMemoryCacheForDataSource(dataSourceID uint) { + cacheKey := fmt.Sprintf("datasource_%d", dataSourceID) + cm.InvalidateMemoryCache(cacheKey) + log.Printf("已清理数据源 %d 的内存缓存", dataSourceID) } -// urlSlicesEqual 比较两个URL切片是否相等 -func (cm *CacheManager) urlSlicesEqual(a, b []string) bool { - if len(a) != len(b) { - return false - } - - // 创建map来比较 - urlMap := make(map[string]bool) - for _, url := range a { - urlMap[url] = true - } - - for _, url := range b { - if !urlMap[url] { - return false - } - } - - return true +// InvalidateMemoryCacheForEndpoint 清理与端点相关的内存缓存 +func (cm *CacheManager) InvalidateMemoryCacheForEndpoint(endpointURL string) { + cm.InvalidateMemoryCache(endpointURL) + log.Printf("已清理端点 %s 的内存缓存", endpointURL) } -// cleanupExpiredCache 定期清理过期的数据库缓存（内存缓存不再自动过期） -func (cm *CacheManager) cleanupExpiredCache() { - ticker := time.NewTicker(10 * time.Minute) - defer ticker.Stop() +// GetCacheStats 获取缓存统计信息 +func (cm *CacheManager) GetCacheStats() map[string]int { + cm.mutex.RLock() + defer cm.mutex.RUnlock() - for range ticker.C { - now := time.Now() - - // 内存缓存不再自动过期，只清理数据库中的过期缓存 - if err := database.DB.Where("expires_at < ?", now).Delete(&model.CachedURL{}).Error; err != nil { - log.Printf("Failed to cleanup expired cache: %v", err) - } + stats := make(map[string]int) + for key, cached := range cm.memoryCache { + stats[key] = len(cached.URLs) } + + return stats } diff --git a/service/data_source_fetcher.go b/service/data_source_fetcher.go index 5363647..7f25091 100644 --- a/service/data_source_fetcher.go +++ b/service/data_source_fetcher.go @@ -33,9 +33,12 @@ func (dsf *DataSourceFetcher) FetchURLs(dataSource *model.DataSource) ([]string, return dsf.fetchAPIURLs(dataSource) } - // 其他类型的数据源先检查数据库缓存 - if cachedURLs, err := dsf.cacheManager.GetFromDBCache(dataSource.ID); err == nil && len(cachedURLs) > 0 { - log.Printf("从数据库缓存获取到 %d 个URL (数据源ID: %d)", len(cachedURLs), dataSource.ID) + // 构建内存缓存的key（使用数据源ID） + cacheKey := fmt.Sprintf("datasource_%d", dataSource.ID) + + // 先检查内存缓存 + if cachedURLs, exists := dsf.cacheManager.GetFromMemoryCache(cacheKey); exists && len(cachedURLs) > 0 { + log.Printf("从内存缓存获取到 %d 个URL (数据源ID: %d)", len(cachedURLs), dataSource.ID) return cachedURLs, nil } @@ -64,20 +67,9 @@ func (dsf *DataSourceFetcher) FetchURLs(dataSource *model.DataSource) ([]string, return urls, nil } - // 缓存结果到数据库 - cacheDuration := time.Duration(dataSource.CacheDuration) * time.Second - changed, err := dsf.cacheManager.UpdateDBCacheIfChanged(dataSource.ID, urls, cacheDuration) - if err != nil { - log.Printf("Failed to cache URLs for data source %d: %v", dataSource.ID, err) - } else if changed { - log.Printf("数据源 %d 的数据已更新，缓存了 %d 个URL", dataSource.ID, len(urls)) - // 数据发生变化，清理相关的内存缓存 - if err := dsf.cacheManager.InvalidateMemoryCacheForDataSource(dataSource.ID); err != nil { - log.Printf("Failed to invalidate memory cache for data source %d: %v", dataSource.ID, err) - } - } else { - log.Printf("数据源 %d 的数据未变化，仅更新了过期时间", dataSource.ID) - } + // 缓存结果到内存 + dsf.cacheManager.SetMemoryCache(cacheKey, urls) + log.Printf("数据源 %d 已缓存 %d 个URL到内存", dataSource.ID, len(urls)) // 更新最后同步时间 now := time.Now() diff --git a/service/endpoint_service.go b/service/endpoint_service.go index cd7dc24..2e72f2f 100644 --- a/service/endpoint_service.go +++ b/service/endpoint_service.go @@ -12,12 +12,6 @@ import ( "time" ) -// CachedEndpoint 缓存的端点数据 -type CachedEndpoint struct { - URLs []string - // 移除ExpiresAt字段，内存缓存不再自动过期 -} - // EndpointService API端点服务 type EndpointService struct { cacheManager *CacheManager diff --git a/service/preloader.go b/service/preloader.go index f16101b..2080160 100644 --- a/service/preloader.go +++ b/service/preloader.go @@ -1,6 +1,7 @@ package service import ( + "fmt" "log" "random-api-go/database" "random-api-go/model" @@ -202,9 +203,10 @@ func (p *Preloader) checkAndRefreshExpiredData() { continue } - // 检查缓存是否即将过期（提前5分钟刷新） - cachedURLs, err := p.cacheManager.GetFromDBCache(dataSource.ID) - if err != nil || len(cachedURLs) == 0 { + // 检查内存缓存是否存在 + cacheKey := fmt.Sprintf("datasource_%d", dataSource.ID) + cachedURLs, exists := p.cacheManager.GetFromMemoryCache(cacheKey) + if !exists || len(cachedURLs) == 0 { // 没有缓存数据，需要刷新 refreshCount++ wg.Add(1) diff --git a/web/components/admin/DataSourceManagement.tsx b/web/components/admin/DataSourceManagement.tsx index 082e789..0b6fdf1 100644 --- a/web/components/admin/DataSourceManagement.tsx +++ b/web/components/admin/DataSourceManagement.tsx @@ -28,7 +28,6 @@ export default function DataSourceManagement({ name: '', type: 'manual' as 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint', config: '', - cache_duration: 3600, is_active: true }) @@ -56,7 +55,7 @@ export default function DataSourceManagement({ if (response.ok) { onUpdate() - setFormData({ name: '', type: 'manual' as const, config: '', cache_duration: 3600, is_active: true }) + setFormData({ name: '', type: 'manual' as const, config: '', is_active: true }) setShowCreateForm(false) alert('数据源创建成功') } else { @@ -111,7 +110,7 @@ export default function DataSourceManagement({ if (response.ok) { onUpdate() - setFormData({ name: '', type: 'manual' as const, config: '', cache_duration: 3600, is_active: true }) + setFormData({ name: '', type: 'manual' as const, config: '', is_active: true }) setEditingDataSource(null) alert('数据源更新成功') } else { @@ -145,7 +144,6 @@ export default function DataSourceManagement({ name: dataSource.name, type: dataSource.type, config: config, - cache_duration: dataSource.cache_duration, is_active: dataSource.is_active }) setShowCreateForm(false) // 关闭创建表单 @@ -153,7 +151,7 @@ export default function DataSourceManagement({ const cancelEdit = () => { setEditingDataSource(null) - setFormData({ name: '', type: 'manual' as const, config: '', cache_duration: 3600, is_active: true }) + setFormData({ name: '', type: 'manual' as const, config: '', is_active: true }) } const deleteDataSource = async (dataSourceId: number) => { @@ -217,7 +215,7 @@ export default function DataSourceManagement({ onClick={() => { setShowCreateForm(true) setEditingDataSource(null) - setFormData({ name: '', type: 'manual' as const, config: '', cache_duration: 3600, is_active: true }) + setFormData({ name: '', type: 'manual' as const, config: '', is_active: true }) }} size="sm" > @@ -264,19 +262,6 @@ export default function DataSourceManagement({ onChange={(config) => setFormData({ ...formData, config })} />
-
- - setFormData({ ...formData, cache_duration: parseInt(e.target.value) || 0 })} - min="0" - /> -

- 设置为0表示不缓存，建议设置3600秒(1小时)以上 -

-
名称 类型 状态 - 缓存时长 最后同步 操作 @@ -336,9 +320,6 @@ export default function DataSourceManagement({ {dataSource.is_active ? '启用' : '禁用'} - - {dataSource.cache_duration > 0 ? `${dataSource.cache_duration}秒` : '不缓存'} - {dataSource.last_sync ? new Date(dataSource.last_sync).toLocaleString() : '未同步'} diff --git a/web/types/admin.ts b/web/types/admin.ts index 93df7c5..3a54e49 100644 --- a/web/types/admin.ts +++ b/web/types/admin.ts @@ -23,7 +23,6 @@ export interface DataSource { name: string type: 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint' config: string - cache_duration: number is_active: boolean last_sync?: string created_at: string