From ef2ab55fe6b52bf8356935da8895c7b456720d0c Mon Sep 17 00:00:00 2001 From: wood chen Date: Fri, 11 Jul 2025 20:08:00 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=BC=95=E7=94=A8=E6=9D=A5?= =?UTF-8?q?=E6=BA=90=E7=BB=9F=E8=AE=A1=E9=80=BB=E8=BE=91=EF=BC=8C=E7=A7=BB?= =?UTF-8?q?=E9=99=A4=E6=8C=81=E4=B9=85=E5=8C=96=E5=AD=98=E5=82=A8=EF=BC=8C?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=9C=80=E5=90=8E=E8=AE=BF=E9=97=AE=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E5=AD=97=E6=AE=B5=EF=BC=8C=E6=8F=90=E5=8D=87=E4=BB=AA?= =?UTF-8?q?=E8=A1=A8=E6=9D=BF=E5=B1=95=E7=A4=BA=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/metrics/collector.go | 185 ++++++++------------------------ internal/metrics/persistence.go | 75 +++---------- internal/models/metrics.go | 2 + web/app/dashboard/page.tsx | 82 +++++++++++--- 4 files changed, 126 insertions(+), 218 deletions(-) diff --git a/internal/metrics/collector.go b/internal/metrics/collector.go index 8e4609b..60e807c 100644 --- a/internal/metrics/collector.go +++ b/internal/metrics/collector.go @@ -10,7 +10,6 @@ import ( "proxy-go/internal/utils" "runtime" "sort" - "strings" "sync" "sync/atomic" "time" @@ -152,34 +151,24 @@ func (c *Collector) RecordRequest(path string, status int, latency time.Duration c.latencyBuckets.Store(bucketKey, counter) } - // 更新引用来源统计 - if r != nil { - referer := r.Header.Get("Referer") - if referer != "" { - // 简化引用来源，只保留域名部分 - referer = simplifyReferer(referer) - - if value, ok := c.refererStats.Load(referer); ok { - stat := value.(*models.PathMetrics) - stat.AddRequest() - if status >= 400 { - stat.AddError() - } - stat.AddLatency(int64(latency)) - stat.AddBytes(bytes) - } else { - newStat := &models.PathMetrics{ - Path: referer, - } - newStat.RequestCount.Store(1) - if status >= 400 { - newStat.ErrorCount.Store(1) - } - newStat.TotalLatency.Store(int64(latency)) - newStat.BytesTransferred.Store(bytes) - c.refererStats.Store(referer, newStat) - } + // 记录引用来源 + if referer := r.Referer(); referer != "" { + var refererMetrics *models.PathMetrics + if existingMetrics, ok := c.refererStats.Load(referer); ok { + refererMetrics = existingMetrics.(*models.PathMetrics) + } else { + refererMetrics = &models.PathMetrics{Path: referer} + c.refererStats.Store(referer, refererMetrics) } + + refererMetrics.AddRequest() + if status >= 400 { + refererMetrics.AddError() + } + refererMetrics.AddBytes(bytes) + refererMetrics.AddLatency(latency.Nanoseconds()) + // 更新最后访问时间 + refererMetrics.LastAccessTime.Store(time.Now().Unix()) } // 更新最近请求记录 @@ -478,126 +467,36 @@ func (c *Collector) getBandwidthHistory() map[string]string { return history } -// simplifyReferer 简化引用来源URL，只保留域名部分 -func simplifyReferer(referer string) string { - // 移除协议部分 - if idx := strings.Index(referer, "://"); idx != -1 { - referer = referer[idx+3:] - } - - // 只保留域名部分 - if idx := strings.Index(referer, "/"); idx != -1 { - referer = referer[:idx] - } - - return referer -} - // startCleanupTask 启动定期清理任务 func (c *Collector) startCleanupTask() { go func() { - // 先立即执行一次清理 - c.cleanupOldData() - - ticker := time.NewTicker(15 * time.Minute) // 每15分钟清理一次 + ticker := time.NewTicker(1 * time.Hour) defer ticker.Stop() - for range ticker.C { - c.cleanupOldData() + for { + <-ticker.C + oneDayAgo := time.Now().Add(-24 * time.Hour).Unix() + + // 清理超过24小时的引用来源统计 + var keysToDelete []interface{} + c.refererStats.Range(func(key, value interface{}) bool { + metrics := value.(*models.PathMetrics) + if metrics.LastAccessTime.Load() < oneDayAgo { + keysToDelete = append(keysToDelete, key) + } + return true + }) + + for _, key := range keysToDelete { + c.refererStats.Delete(key) + } + + if len(keysToDelete) > 0 { + log.Printf("[Collector] 已清理 %d 条过期的引用来源统计", len(keysToDelete)) + } + + // 强制GC + runtime.GC() } }() } - -// cleanupOldData 清理旧数据 -func (c *Collector) cleanupOldData() { - log.Printf("[Metrics] 开始清理旧数据...") - - // 清理引用来源统计 - 类似地处理 - var referersToRemove []string - var referersCount int - var totalRefererRequests int64 - - // 先收集所有引用来源及其请求数 - type refererInfo struct { - referer string - count int64 - } - var referers []refererInfo - - c.refererStats.Range(func(key, value interface{}) bool { - referer := key.(string) - stats := value.(*models.PathMetrics) - count := stats.GetRequestCount() - referersCount++ - totalRefererRequests += count - referers = append(referers, refererInfo{referer, count}) - return true - }) - - // 按请求数排序 - sort.Slice(referers, func(i, j int) bool { - return referers[i].count > referers[j].count - }) - - // 只保留前50个请求数最多的引用来源，或者请求数占总请求数2%以上的引用来源 - refThreshold := totalRefererRequests / 50 // 2%的阈值 - if refThreshold < 5 { - refThreshold = 5 // 至少保留请求数>=5的引用来源 - } - - // 标记要删除的引用来源 - for _, ri := range referers { - if len(referers)-len(referersToRemove) <= 50 { - // 已经只剩下50个引用来源了，不再删除 - break - } - - if ri.count < refThreshold { - referersToRemove = append(referersToRemove, ri.referer) - } - } - - // 删除标记的引用来源 - for _, referer := range referersToRemove { - c.refererStats.Delete(referer) - } - - // 清理带宽历史 - 只保留最近的记录 - c.bandwidthStats.Lock() - if len(c.bandwidthStats.history) > 10 { - // 找出最旧的记录并删除 - var oldestKeys []string - var oldestTimes []time.Time - - for k := range c.bandwidthStats.history { - t, err := time.Parse("01-02 15:04", k) - if err != nil { - continue - } - oldestTimes = append(oldestTimes, t) - oldestKeys = append(oldestKeys, k) - } - - // 按时间排序 - sort.Slice(oldestKeys, func(i, j int) bool { - return oldestTimes[i].Before(oldestTimes[j]) - }) - - // 删除最旧的记录，只保留最近10条 - for i := 0; i < len(oldestKeys)-10; i++ { - delete(c.bandwidthStats.history, oldestKeys[i]) - } - } - c.bandwidthStats.Unlock() - - // 强制进行一次GC - runtime.GC() - - // 打印内存使用情况 - var mem runtime.MemStats - runtime.ReadMemStats(&mem) - - log.Printf("[Metrics] 清理完成: 删除了 %d/%d 个引用来源, 当前内存使用: %s", - len(referersToRemove), referersCount, - utils.FormatBytes(int64(mem.Alloc))) -} diff --git a/internal/metrics/persistence.go b/internal/metrics/persistence.go index 40e9ce4..a40f728 100644 --- a/internal/metrics/persistence.go +++ b/internal/metrics/persistence.go @@ -6,7 +6,6 @@ import ( "log" "os" "path/filepath" - "proxy-go/internal/models" "proxy-go/internal/utils" "runtime" "sync" @@ -16,15 +15,14 @@ import ( // MetricsStorage 指标存储结构 type MetricsStorage struct { - collector *Collector - saveInterval time.Duration - dataDir string - stopChan chan struct{} - wg sync.WaitGroup - lastSaveTime time.Time - mutex sync.RWMutex - statusCodeFile string - refererStatsFile string + collector *Collector + saveInterval time.Duration + dataDir string + stopChan chan struct{} + wg sync.WaitGroup + lastSaveTime time.Time + mutex sync.RWMutex + statusCodeFile string } // NewMetricsStorage 创建新的指标存储 @@ -34,12 +32,11 @@ func NewMetricsStorage(collector *Collector, dataDir string, saveInterval time.D } return &MetricsStorage{ - collector: collector, - saveInterval: saveInterval, - dataDir: dataDir, - stopChan: make(chan struct{}), - statusCodeFile: filepath.Join(dataDir, "status_codes.json"), - refererStatsFile: filepath.Join(dataDir, "referer_stats.json"), + collector: collector, + saveInterval: saveInterval, + dataDir: dataDir, + stopChan: make(chan struct{}), + statusCodeFile: filepath.Join(dataDir, "status_codes.json"), } } @@ -107,11 +104,7 @@ func (ms *MetricsStorage) SaveMetrics() error { return fmt.Errorf("保存状态码统计失败: %v", err) } - // 保存引用来源统计 - 限制数量 - topReferers := stats["top_referers"] - if err := saveJSONToFile(ms.refererStatsFile, topReferers); err != nil { - return fmt.Errorf("保存引用来源统计失败: %v", err) - } + // 不再保存引用来源统计，因为它现在只保存在内存中 // 单独保存延迟分布 if latencyStats, ok := stats["latency_stats"].(map[string]interface{}); ok { @@ -166,45 +159,7 @@ func (ms *MetricsStorage) LoadMetrics() error { } } - // 2. 加载引用来源统计（如果文件存在） - if fileExists(ms.refererStatsFile) { - var refererStats []map[string]interface{} - if err := loadJSONFromFile(ms.refererStatsFile, &refererStats); err != nil { - log.Printf("[MetricsStorage] 加载引用来源统计失败: %v", err) - } else { - // 只加载前20个引用来源统计 - maxReferers := 20 - if len(refererStats) > maxReferers { - refererStats = refererStats[:maxReferers] - } - - for _, refererStat := range refererStats { - referer, ok := refererStat["path"].(string) - if !ok { - continue - } - - requestCount, _ := refererStat["request_count"].(float64) - errorCount, _ := refererStat["error_count"].(float64) - bytesTransferred, _ := refererStat["bytes_transferred"].(float64) - - // 创建或更新引用来源统计 - var refererMetrics *models.PathMetrics - if existingMetrics, ok := ms.collector.refererStats.Load(referer); ok { - refererMetrics = existingMetrics.(*models.PathMetrics) - } else { - refererMetrics = &models.PathMetrics{Path: referer} - ms.collector.refererStats.Store(referer, refererMetrics) - } - - // 设置统计值 - refererMetrics.RequestCount.Store(int64(requestCount)) - refererMetrics.ErrorCount.Store(int64(errorCount)) - refererMetrics.BytesTransferred.Store(int64(bytesTransferred)) - } - log.Printf("[MetricsStorage] 加载了 %d 条引用来源统计", len(refererStats)) - } - } + // 不再加载引用来源统计，因为它现在只保存在内存中 // 3. 加载延迟分布（如果文件存在） latencyDistributionFile := filepath.Join(ms.dataDir, "latency_distribution.json") diff --git a/internal/models/metrics.go b/internal/models/metrics.go index 727b390..2bb68a2 100644 --- a/internal/models/metrics.go +++ b/internal/models/metrics.go @@ -19,6 +19,7 @@ type PathMetrics struct { TotalLatency atomic.Int64 `json:"-"` BytesTransferred atomic.Int64 `json:"bytes_transferred"` AvgLatency string `json:"avg_latency"` + LastAccessTime atomic.Int64 `json:"last_access_time"` // 最后访问时间戳 } // PathMetricsJSON 用于 JSON 序列化的路径统计信息 @@ -28,6 +29,7 @@ type PathMetricsJSON struct { ErrorCount int64 `json:"error_count"` BytesTransferred int64 `json:"bytes_transferred"` AvgLatency string `json:"avg_latency"` + LastAccessTime int64 `json:"last_access_time"` // 最后访问时间戳 } // GetRequestCount 获取请求数 diff --git a/web/app/dashboard/page.tsx b/web/app/dashboard/page.tsx index 5d6d3aa..d64494e 100644 --- a/web/app/dashboard/page.tsx +++ b/web/app/dashboard/page.tsx @@ -40,6 +40,7 @@ interface Metrics { error_count: number avg_latency: string bytes_transferred: number + last_access_time: number // 添加最后访问时间字段 }> } @@ -321,7 +322,12 @@ export default function DashboardPage() { {metrics.top_referers && metrics.top_referers.length > 0 && ( - 引用来源统计 (Top {metrics.top_referers.length}) + + 引用来源统计 + + (近24小时, 共 {metrics.top_referers.length} 条记录) + +
@@ -331,24 +337,49 @@ export default function DashboardPage() { 来源域名 请求数 错误数 + 错误率 平均延迟 传输大小 + 最后访问 - {metrics.top_referers.map((referer, index) => ( - - - - {referer.path} - - - {referer.request_count} - {referer.error_count} - {referer.avg_latency} - {formatBytes(referer.bytes_transferred)} - - ))} + {metrics.top_referers + .sort((a, b) => b.request_count - a.request_count) + .map((referer, index) => { + const errorRate = ((referer.error_count / referer.request_count) * 100).toFixed(1); + const lastAccessTime = new Date(referer.last_access_time * 1000); + const timeAgo = getTimeAgo(lastAccessTime); + + return ( + + + + {referer.path} + + + {referer.request_count} + {referer.error_count} + + + {errorRate}% + + + {referer.avg_latency} + {formatBytes(referer.bytes_transferred)} + + + {timeAgo} + + + + ); + })}
@@ -444,9 +475,30 @@ function formatLatency(nanoseconds: number) { } } +function getTimeAgo(date: Date) { + const now = new Date(); + const diffInSeconds = Math.floor((now.getTime() - date.getTime()) / 1000); + + if (diffInSeconds < 60) { + return `${diffInSeconds}秒前`; + } + + const diffInMinutes = Math.floor(diffInSeconds / 60); + if (diffInMinutes < 60) { + return `${diffInMinutes}分钟前`; + } + + const diffInHours = Math.floor(diffInMinutes / 60); + if (diffInHours < 24) { + return `${diffInHours}小时前`; + } + + return date.toLocaleString(); +} + function getStatusColor(status: number) { if (status >= 500) return "bg-red-100 text-red-800" if (status >= 400) return "bg-yellow-100 text-yellow-800" if (status >= 300) return "bg-blue-100 text-blue-800" return "bg-green-100 text-green-800" -} \ No newline at end of file +} \ No newline at end of file