wood/random-api-go
1
0
Fork 0
mirror of https://github.com/woodchen-ink/random-api-go.git synced 2025-07-18 05:42:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

移除CachedURL模型及相关缓存逻辑，更新数据源处理逻辑以使用内存缓存，调整缓存时长设置，优化数据源管理界面，简化代码结构，提升性能和可维护性。

Browse Source
This commit is contained in:
wood chen 2025-06-14 22:24:30 +08:00
parent 310f627a9e
commit c3dc214f9b
10 changed files with 58 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
database/database.go
View File

@ -62,7 +62,6 @@ func autoMigrate() error {
&model.APIEndpoint{},
&model.DataSource{},
&model.URLReplaceRule{},
&model.CachedURL{},
&model.Config{},
)
}
@ -79,11 +78,6 @@ func Close() error {
return nil
}
// CleanExpiredCache 清理过期缓存
func CleanExpiredCache() error {
return DB.Where("expires_at < ?", time.Now()).Delete(&model.CachedURL{}).Error
}
// GetConfig 获取配置值
func GetConfig(key string, defaultValue string) string {
var config model.Config

4
handler/admin_handler.go
View File

@ -391,9 +391,7 @@ func (h *AdminHandler) UpdateDataSource(w http.ResponseWriter, r *http.Request)
if updateData.Config != "" {
dataSource.Config = updateData.Config
}
if updateData.CacheDuration != 0 {
dataSource.CacheDuration = updateData.CacheDuration
}
dataSource.IsActive = updateData.IsActive
// 使用服务更新数据源（会自动预加载）

2
handler/handler.go
View File

@ -34,7 +34,7 @@ type Handlers struct {
func NewHandlers(statsManager *stats.StatsManager) *Handlers {
return &Handlers{
Stats: statsManager,
cacheDuration: 5 * time.Minute, // 缓存5分钟
cacheDuration: 30 * time.Minute, // 缓存30分钟
}
}

37
model/api_endpoint.go
View File

@ -26,21 +26,19 @@ type APIEndpoint struct {
// DataSource 数据源模型
type DataSource struct {
ID uint `json:"id" gorm:"primaryKey"`
EndpointID uint `json:"endpoint_id" gorm:"not null;index"`
Name string `json:"name" gorm:"not null"`
Type string `json:"type" gorm:"not null;check:type IN ('lankong', 'manual', 'api_get', 'api_post', 'endpoint')"`
Config string `json:"config" gorm:"not null"`
CacheDuration int `json:"cache_duration" gorm:"default:3600"` // 缓存时长（秒）
IsActive bool `json:"is_active" gorm:"default:true"`
LastSync *time.Time `json:"last_sync,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DeletedAt gorm.DeletedAt `json:"-" gorm:"index"`
ID uint `json:"id" gorm:"primaryKey"`
EndpointID uint `json:"endpoint_id" gorm:"not null;index"`
Name string `json:"name" gorm:"not null"`
Type string `json:"type" gorm:"not null;check:type IN ('lankong', 'manual', 'api_get', 'api_post', 'endpoint')"`
Config string `json:"config" gorm:"not null"`
IsActive bool `json:"is_active" gorm:"default:true"`
LastSync *time.Time `json:"last_sync,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DeletedAt gorm.DeletedAt `json:"-" gorm:"index"`
// 关联
Endpoint APIEndpoint `json:"-" gorm:"foreignKey:EndpointID"`
CachedURLs []CachedURL `json:"-" gorm:"foreignKey:DataSourceID"`
Endpoint APIEndpoint `json:"-" gorm:"foreignKey:EndpointID"`
}
// URLReplaceRule URL替换规则模型
@ -59,19 +57,6 @@ type URLReplaceRule struct {
Endpoint *APIEndpoint `json:"endpoint,omitempty" gorm:"foreignKey:EndpointID"`
}
// CachedURL 缓存URL模型
type CachedURL struct {
ID uint `json:"id" gorm:"primaryKey"`
DataSourceID uint `json:"data_source_id" gorm:"not null;index"`
OriginalURL string `json:"original_url" gorm:"not null"`
FinalURL string `json:"final_url" gorm:"not null"`
ExpiresAt time.Time `json:"expires_at" gorm:"index"`
CreatedAt time.Time `json:"created_at"`
// 关联
DataSource DataSource `json:"-" gorm:"foreignKey:DataSourceID"`
}
// Config 通用配置表
type Config struct {
ID uint `json:"id" gorm:"primaryKey"`

146
service/cache_manager.go
View File

@ -1,30 +1,28 @@
package service
import (
"fmt"
"log"
"random-api-go/database"
"random-api-go/model"
"sync"
"time"
)
// CacheManager 缓存管理器
type CacheManager struct {
memoryCache map[string]*CachedEndpoint
memoryCache map[string]*CachedItem
mutex sync.RWMutex
}
// 注意：CachedEndpoint 类型定义在 endpoint_service.go 中
// CachedItem 缓存项（永久缓存，只在数据变动时刷新）
type CachedItem struct {
URLs []string
}
// NewCacheManager 创建缓存管理器
func NewCacheManager() *CacheManager {
cm := &CacheManager{
memoryCache: make(map[string]*CachedEndpoint),
memoryCache: make(map[string]*CachedItem),
}
// 启动定期清理过期缓存的协程
go cm.cleanupExpiredCache()
return cm
}
@ -41,12 +39,12 @@ func (cm *CacheManager) GetFromMemoryCache(key string) ([]string, bool) {
return cached.URLs, true
}
// SetMemoryCache 设置内存缓存（duration参数保留以兼容现有接口，但不再使用）
func (cm *CacheManager) SetMemoryCache(key string, urls []string, duration time.Duration) {
// SetMemoryCache 设置内存缓存
func (cm *CacheManager) SetMemoryCache(key string, urls []string) {
cm.mutex.Lock()
defer cm.mutex.Unlock()
cm.memoryCache[key] = &CachedEndpoint{
cm.memoryCache[key] = &CachedItem{
URLs: urls,
}
}
@ -59,118 +57,28 @@ func (cm *CacheManager) InvalidateMemoryCache(key string) {
delete(cm.memoryCache, key)
}
// GetFromDBCache 从数据库缓存获取URL
func (cm *CacheManager) GetFromDBCache(dataSourceID uint) ([]string, error) {
var cachedURLs []model.CachedURL
if err := database.DB.Where("data_source_id = ? AND expires_at > ?", dataSourceID, time.Now()).
Find(&cachedURLs).Error; err != nil {
return nil, err
}
var urls []string
for _, cached := range cachedURLs {
urls = append(urls, cached.FinalURL)
}
return urls, nil
}
// SetDBCache 设置数据库缓存
func (cm *CacheManager) SetDBCache(dataSourceID uint, urls []string, duration time.Duration) error {
// 先删除旧的缓存
if err := database.DB.Where("data_source_id = ?", dataSourceID).Delete(&model.CachedURL{}).Error; err != nil {
log.Printf("Failed to delete old cache for data source %d: %v", dataSourceID, err)
}
// 插入新的缓存
expiresAt := time.Now().Add(duration)
for _, url := range urls {
cachedURL := model.CachedURL{
DataSourceID: dataSourceID,
OriginalURL: url,
FinalURL: url,
ExpiresAt: expiresAt,
}
if err := database.DB.Create(&cachedURL).Error; err != nil {
log.Printf("Failed to cache URL: %v", err)
}
}
return nil
}
// UpdateDBCacheIfChanged 只有当数据变化时才更新数据库缓存，并返回是否需要清理内存缓存
func (cm *CacheManager) UpdateDBCacheIfChanged(dataSourceID uint, newURLs []string, duration time.Duration) (bool, error) {
// 获取现有缓存
existingURLs, err := cm.GetFromDBCache(dataSourceID)
if err != nil {
// 如果获取失败，直接设置新缓存
return true, cm.SetDBCache(dataSourceID, newURLs, duration)
}
// 比较URL列表是否相同
if cm.urlSlicesEqual(existingURLs, newURLs) {
// 数据没有变化，只更新过期时间
expiresAt := time.Now().Add(duration)
if err := database.DB.Model(&model.CachedURL{}).
Where("data_source_id = ?", dataSourceID).
Update("expires_at", expiresAt).Error; err != nil {
log.Printf("Failed to update cache expiry for data source %d: %v", dataSourceID, err)
}
return false, nil
}
// 数据有变化，更新缓存
return true, cm.SetDBCache(dataSourceID, newURLs, duration)
}
// InvalidateMemoryCacheForDataSource 清理与数据源相关的内存缓存
func (cm *CacheManager) InvalidateMemoryCacheForDataSource(dataSourceID uint) error {
// 获取数据源信息
var dataSource model.DataSource
if err := database.DB.Preload("Endpoint").First(&dataSource, dataSourceID).Error; err != nil {
return err
}
// 清理该端点的内存缓存
cm.InvalidateMemoryCache(dataSource.Endpoint.URL)
log.Printf("已清理端点 %s 的内存缓存（数据源 %d 数据发生变化）", dataSource.Endpoint.URL, dataSourceID)
return nil
func (cm *CacheManager) InvalidateMemoryCacheForDataSource(dataSourceID uint) {
cacheKey := fmt.Sprintf("datasource_%d", dataSourceID)
cm.InvalidateMemoryCache(cacheKey)
log.Printf("已清理数据源 %d 的内存缓存", dataSourceID)
}
// urlSlicesEqual 比较两个URL切片是否相等
func (cm *CacheManager) urlSlicesEqual(a, b []string) bool {
if len(a) != len(b) {
return false
}
// 创建map来比较
urlMap := make(map[string]bool)
for _, url := range a {
urlMap[url] = true
}
for _, url := range b {
if !urlMap[url] {
return false
}
}
return true
// InvalidateMemoryCacheForEndpoint 清理与端点相关的内存缓存
func (cm *CacheManager) InvalidateMemoryCacheForEndpoint(endpointURL string) {
cm.InvalidateMemoryCache(endpointURL)
log.Printf("已清理端点 %s 的内存缓存", endpointURL)
}
// cleanupExpiredCache 定期清理过期的数据库缓存（内存缓存不再自动过期）
func (cm *CacheManager) cleanupExpiredCache() {
ticker := time.NewTicker(10 * time.Minute)
defer ticker.Stop()
// GetCacheStats 获取缓存统计信息
func (cm *CacheManager) GetCacheStats() map[string]int {
cm.mutex.RLock()
defer cm.mutex.RUnlock()
for range ticker.C {
now := time.Now()
// 内存缓存不再自动过期，只清理数据库中的过期缓存
if err := database.DB.Where("expires_at < ?", now).Delete(&model.CachedURL{}).Error; err != nil {
log.Printf("Failed to cleanup expired cache: %v", err)
}
stats := make(map[string]int)
for key, cached := range cm.memoryCache {
stats[key] = len(cached.URLs)
}
return stats
}

26
service/data_source_fetcher.go
View File

@ -33,9 +33,12 @@ func (dsf *DataSourceFetcher) FetchURLs(dataSource *model.DataSource) ([]string,
return dsf.fetchAPIURLs(dataSource)
}
// 其他类型的数据源先检查数据库缓存
if cachedURLs, err := dsf.cacheManager.GetFromDBCache(dataSource.ID); err == nil && len(cachedURLs) > 0 {
log.Printf("从数据库缓存获取到 %d 个URL (数据源ID: %d)", len(cachedURLs), dataSource.ID)
// 构建内存缓存的key（使用数据源ID）
cacheKey := fmt.Sprintf("datasource_%d", dataSource.ID)
// 先检查内存缓存
if cachedURLs, exists := dsf.cacheManager.GetFromMemoryCache(cacheKey); exists && len(cachedURLs) > 0 {
log.Printf("从内存缓存获取到 %d 个URL (数据源ID: %d)", len(cachedURLs), dataSource.ID)
return cachedURLs, nil
}
@ -64,20 +67,9 @@ func (dsf *DataSourceFetcher) FetchURLs(dataSource *model.DataSource) ([]string,
return urls, nil
}
// 缓存结果到数据库
cacheDuration := time.Duration(dataSource.CacheDuration) * time.Second
changed, err := dsf.cacheManager.UpdateDBCacheIfChanged(dataSource.ID, urls, cacheDuration)
if err != nil {
log.Printf("Failed to cache URLs for data source %d: %v", dataSource.ID, err)
} else if changed {
log.Printf("数据源 %d 的数据已更新，缓存了 %d 个URL", dataSource.ID, len(urls))
// 数据发生变化，清理相关的内存缓存
if err := dsf.cacheManager.InvalidateMemoryCacheForDataSource(dataSource.ID); err != nil {
log.Printf("Failed to invalidate memory cache for data source %d: %v", dataSource.ID, err)
}
} else {
log.Printf("数据源 %d 的数据未变化，仅更新了过期时间", dataSource.ID)
}
// 缓存结果到内存
dsf.cacheManager.SetMemoryCache(cacheKey, urls)
log.Printf("数据源 %d 已缓存 %d 个URL到内存", dataSource.ID, len(urls))
// 更新最后同步时间
now := time.Now()

6
service/endpoint_service.go
View File

@ -12,12 +12,6 @@ import (
"time"
)
// CachedEndpoint 缓存的端点数据
type CachedEndpoint struct {
URLs []string
// 移除ExpiresAt字段，内存缓存不再自动过期
}
// EndpointService API端点服务
type EndpointService struct {
cacheManager *CacheManager

8
service/preloader.go
View File

@ -1,6 +1,7 @@
package service
import (
"fmt"
"log"
"random-api-go/database"
"random-api-go/model"
@ -202,9 +203,10 @@ func (p *Preloader) checkAndRefreshExpiredData() {
continue
}
// 检查缓存是否即将过期（提前5分钟刷新）
cachedURLs, err := p.cacheManager.GetFromDBCache(dataSource.ID)
if err != nil || len(cachedURLs) == 0 {
// 检查内存缓存是否存在
cacheKey := fmt.Sprintf("datasource_%d", dataSource.ID)
cachedURLs, exists := p.cacheManager.GetFromMemoryCache(cacheKey)
if !exists || len(cachedURLs) == 0 {
// 没有缓存数据，需要刷新
refreshCount++
wg.Add(1)

27
web/components/admin/DataSourceManagement.tsx
View File

@ -28,7 +28,6 @@ export default function DataSourceManagement({
name: '',
type: 'manual' as 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint',
config: '',
cache_duration: 3600,
is_active: true
})
@ -56,7 +55,7 @@ export default function DataSourceManagement({
if (response.ok) {
onUpdate()
setFormData({ name: '', type: 'manual' as const, config: '', cache_duration: 3600, is_active: true })
setFormData({ name: '', type: 'manual' as const, config: '', is_active: true })
setShowCreateForm(false)
alert('数据源创建成功')
} else {
@ -111,7 +110,7 @@ export default function DataSourceManagement({
if (response.ok) {
onUpdate()
setFormData({ name: '', type: 'manual' as const, config: '', cache_duration: 3600, is_active: true })
setFormData({ name: '', type: 'manual' as const, config: '', is_active: true })
setEditingDataSource(null)
alert('数据源更新成功')
} else {
@ -145,7 +144,6 @@ export default function DataSourceManagement({
name: dataSource.name,
type: dataSource.type,
config: config,
cache_duration: dataSource.cache_duration,
is_active: dataSource.is_active
})
setShowCreateForm(false) // 关闭创建表单
@ -153,7 +151,7 @@ export default function DataSourceManagement({
const cancelEdit = () => {
setEditingDataSource(null)
setFormData({ name: '', type: 'manual' as const, config: '', cache_duration: 3600, is_active: true })
setFormData({ name: '', type: 'manual' as const, config: '', is_active: true })
}
const deleteDataSource = async (dataSourceId: number) => {
@ -217,7 +215,7 @@ export default function DataSourceManagement({
onClick={() => {
setShowCreateForm(true)
setEditingDataSource(null)
setFormData({ name: '', type: 'manual' as const, config: '', cache_duration: 3600, is_active: true })
setFormData({ name: '', type: 'manual' as const, config: '', is_active: true })
}}
size="sm"
>
@ -264,19 +262,6 @@ export default function DataSourceManagement({
onChange={(config) => setFormData({ ...formData, config })}
/>
<div className="grid grid-cols-2 gap-3">
<div className="space-y-1">
<Label htmlFor="ds-cache">缓存时长(秒)</Label>
<Input
id="ds-cache"
type="number"
value={formData.cache_duration}
onChange={(e) => setFormData({ ...formData, cache_duration: parseInt(e.target.value) || 0 })}
min="0"
/>
<p className="text-xs text-muted-foreground">
设置为0表示不缓存，建议设置3600秒(1小时)以上
</p>
</div>
<div className="flex items-center space-x-2 pt-6">
<Switch
id="ds-active"
@ -310,7 +295,6 @@ export default function DataSourceManagement({
<TableHead>名称</TableHead>
<TableHead>类型</TableHead>
<TableHead>状态</TableHead>
<TableHead>缓存时长</TableHead>
<TableHead>最后同步</TableHead>
<TableHead>操作</TableHead>
</TableRow>
@ -336,9 +320,6 @@ export default function DataSourceManagement({
{dataSource.is_active ? '启用' : '禁用'}
</span>
</TableCell>
<TableCell>
{dataSource.cache_duration > 0 ? `${dataSource.cache_duration}秒` : '不缓存'}
</TableCell>
<TableCell>
{dataSource.last_sync ? new Date(dataSource.last_sync).toLocaleString() : '未同步'}
</TableCell>

1
web/types/admin.ts
View File

@ -23,7 +23,6 @@ export interface DataSource {
name: string
type: 'lankong' | 'manual' | 'api_get' | 'api_post' | 'endpoint'
config: string
cache_duration: number
is_active: boolean
last_sync?: string
created_at: string