wood/proxy-go
1
0
Fork 0
mirror of https://github.com/woodchen-ink/proxy-go.git synced 2025-07-18 00:21:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

移除路径统计相关代码和数据存储，更新指标处理逻辑，调整引用来源统计的加载数量限制，以简化代码和提高性能。

Browse Source
This commit is contained in:
wood chen 2025-03-23 12:28:15 +08:00
parent ef1bec7710
commit 5418e89e3b
4 changed files with 8 additions and 249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
internal/handler/metrics.go
View File

@ -35,9 +35,6 @@ type Metrics struct {
// 状态码统计
StatusCodeStats map[string]int64 `json:"status_code_stats"`
// 路径统计
TopPaths []models.PathMetricsJSON `json:"top_paths"`
// 最近请求
RecentRequests []models.RequestLog `json:"recent_requests"`
@ -76,7 +73,6 @@ func (h *ProxyHandler) MetricsHandler(w http.ResponseWriter, r *http.Request) {
"bytes_per_second": float64(0),
"requests_per_second": float64(0),
"status_code_stats": make(map[string]int64),
"top_paths": make([]models.PathMetrics, 0),
"recent_requests": make([]models.RequestLog, 0),
"top_referers": make([]models.PathMetrics, 0),
"latency_stats": map[string]interface{}{
@ -124,7 +120,6 @@ func (h *ProxyHandler) MetricsHandler(w http.ResponseWriter, r *http.Request) {
BytesPerSecond: float64(totalBytes) / utils.MaxFloat64(uptimeSeconds, 1),
RequestsPerSecond: float64(totalRequests) / utils.MaxFloat64(uptimeSeconds, 1),
StatusCodeStats: statusCodeStats,
TopPaths: models.SafePathMetrics(stats["top_paths"]),
RecentRequests: models.SafeRequestLogs(stats["recent_requests"]),
TopReferers: models.SafePathMetrics(stats["top_referers"]),
BandwidthHistory: bandwidthHistory,

148
internal/metrics/collector.go
View File

@ -24,7 +24,6 @@ type Collector struct {
latencySum int64
maxLatency int64 // 最大响应时间
minLatency int64 // 最小响应时间
pathStats sync.Map
statusCodeStats sync.Map
latencyBuckets sync.Map // 响应时间分布
refererStats sync.Map // 引用来源统计
@ -36,7 +35,6 @@ type Collector struct {
history map[string]int64
}
recentRequests *models.RequestQueue
pathStatsMutex sync.RWMutex
config *config.Config
}
@ -154,30 +152,6 @@ func (c *Collector) RecordRequest(path string, status int, latency time.Duration
c.latencyBuckets.Store(bucketKey, counter)
}
// 更新路径统计
c.pathStatsMutex.Lock()
if value, ok := c.pathStats.Load(path); ok {
stat := value.(*models.PathMetrics)
stat.AddRequest()
if status >= 400 {
stat.AddError()
}
stat.AddLatency(int64(latency))
stat.AddBytes(bytes)
} else {
newStat := &models.PathMetrics{
Path: path,
}
newStat.RequestCount.Store(1)
if status >= 400 {
newStat.ErrorCount.Store(1)
}
newStat.TotalLatency.Store(int64(latency))
newStat.BytesTransferred.Store(bytes)
c.pathStats.Store(path, newStat)
}
c.pathStatsMutex.Unlock()
// 更新引用来源统计
if r != nil {
referer := r.Header.Get("Referer")
@ -277,45 +251,6 @@ func (c *Collector) GetStats() map[string]interface{} {
return true
})
// 收集路径统计
var pathMetrics []*models.PathMetrics
pathCount := 0
c.pathStats.Range(func(key, value interface{}) bool {
stats := value.(*models.PathMetrics)
requestCount := stats.GetRequestCount()
if requestCount > 0 {
totalLatency := stats.GetTotalLatency()
avgLatencyMs := float64(totalLatency) / float64(requestCount) / float64(time.Millisecond)
stats.AvgLatency = fmt.Sprintf("%.2fms", avgLatencyMs)
pathMetrics = append(pathMetrics, stats)
}
// 限制遍历的数量，避免过多数据导致内存占用过高
pathCount++
return pathCount < 100 // 最多遍历100个路径
})
// 按请求数降序排序，请求数相同时按路径字典序排序
sort.Slice(pathMetrics, func(i, j int) bool {
countI := pathMetrics[i].GetRequestCount()
countJ := pathMetrics[j].GetRequestCount()
if countI != countJ {
return countI > countJ
}
return pathMetrics[i].Path < pathMetrics[j].Path
})
// 只保留前10个
if len(pathMetrics) > 10 {
pathMetrics = pathMetrics[:10]
}
// 转换为值切片
pathMetricsValues := make([]models.PathMetricsJSON, len(pathMetrics))
for i, metric := range pathMetrics {
pathMetricsValues[i] = metric.ToJSON()
}
// 收集引用来源统计
var refererMetrics []*models.PathMetrics
refererCount := 0
@ -344,9 +279,9 @@ func (c *Collector) GetStats() map[string]interface{} {
return refererMetrics[i].Path < refererMetrics[j].Path
})
// 只保留前10个
if len(refererMetrics) > 10 {
refererMetrics = refererMetrics[:10]
// 只保留前20个
if len(refererMetrics) > 20 {
refererMetrics = refererMetrics[:20]
}
// 转换为值切片
@ -396,7 +331,6 @@ func (c *Collector) GetStats() map[string]interface{} {
"requests_per_second": requestsPerSecond,
"bytes_per_second": float64(atomic.LoadInt64(&c.totalBytes)) / totalRuntime.Seconds(),
"status_code_stats": statusCodeStats,
"top_paths": pathMetricsValues,
"top_referers": refererMetricsValues,
"recent_requests": recentRequests,
"latency_stats": map[string]interface{}{
@ -452,29 +386,6 @@ func (c *Collector) validateLoadedData() error {
return true
})
// 验证路径统计
var totalPathRequests int64
c.pathStats.Range(func(_, value interface{}) bool {
stats := value.(*models.PathMetrics)
requestCount := stats.GetRequestCount()
errorCount := stats.GetErrorCount()
if requestCount < 0 || errorCount < 0 {
return false
}
if errorCount > requestCount {
return false
}
totalPathRequests += requestCount
return true
})
// 由于我们限制了路径统计的收集数量，路径统计总数可能小于状态码统计总数
// 因此，我们只需要确保路径统计总数不超过状态码统计总数即可
if float64(totalPathRequests) > float64(statusCodeTotal)*1.1 { // 允许10%的误差
return fmt.Errorf("path stats total (%d) significantly exceeds status code total (%d)",
totalPathRequests, statusCodeTotal)
}
return nil
}
@ -601,56 +512,6 @@ func (c *Collector) startCleanupTask() {
func (c *Collector) cleanupOldData() {
log.Printf("[Metrics] 开始清理旧数据...")
// 清理路径统计 - 只保留有请求且请求数较多的路径
var pathsToRemove []string
var pathsCount int
var totalRequests int64
// 先收集所有路径及其请求数
type pathInfo struct {
path string
count int64
}
var paths []pathInfo
c.pathStats.Range(func(key, value interface{}) bool {
path := key.(string)
stats := value.(*models.PathMetrics)
count := stats.GetRequestCount()
pathsCount++
totalRequests += count
paths = append(paths, pathInfo{path, count})
return true
})
// 按请求数排序
sort.Slice(paths, func(i, j int) bool {
return paths[i].count > paths[j].count
})
// 只保留前100个请求数最多的路径，或者请求数占总请求数1%以上的路径
threshold := totalRequests / 100 // 1%的阈值
if threshold < 10 {
threshold = 10 // 至少保留请求数>=10的路径
}
// 标记要删除的路径
for _, pi := range paths {
if len(paths)-len(pathsToRemove) <= 100 {
// 已经只剩下100个路径了，不再删除
break
}
if pi.count < threshold {
pathsToRemove = append(pathsToRemove, pi.path)
}
}
// 删除标记的路径
for _, path := range pathsToRemove {
c.pathStats.Delete(path)
}
// 清理引用来源统计 - 类似地处理
var referersToRemove []string
var referersCount int
@ -736,8 +597,7 @@ func (c *Collector) cleanupOldData() {
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
log.Printf("[Metrics] 清理完成: 删除了 %d/%d 个路径, %d/%d 个引用来源, 当前内存使用: %s",
len(pathsToRemove), pathsCount,
log.Printf("[Metrics] 清理完成: 删除了 %d/%d 个引用来源, 当前内存使用: %s",
len(referersToRemove), referersCount,
utils.FormatBytes(int64(mem.Alloc)))
}

56
internal/metrics/persistence.go
View File

@ -24,7 +24,6 @@ type MetricsStorage struct {
lastSaveTime time.Time
mutex sync.RWMutex
metricsFile string
pathStatsFile string
statusCodeFile string
refererStatsFile string
}
@ -41,7 +40,6 @@ func NewMetricsStorage(collector *Collector, dataDir string, saveInterval time.D
dataDir: dataDir,
stopChan: make(chan struct{}),
metricsFile: filepath.Join(dataDir, "metrics.json"),
pathStatsFile: filepath.Join(dataDir, "path_stats.json"),
statusCodeFile: filepath.Join(dataDir, "status_codes.json"),
refererStatsFile: filepath.Join(dataDir, "referer_stats.json"),
}
@ -124,12 +122,6 @@ func (ms *MetricsStorage) SaveMetrics() error {
return fmt.Errorf("保存基本指标失败: %v", err)
}
// 保存路径统计 - 限制数量
topPaths := stats["top_paths"]
if err := saveJSONToFile(ms.pathStatsFile, topPaths); err != nil {
return fmt.Errorf("保存路径统计失败: %v", err)
}
// 保存状态码统计
if err := saveJSONToFile(ms.statusCodeFile, stats["status_code_stats"]); err != nil {
return fmt.Errorf("保存状态码统计失败: %v", err)
@ -188,47 +180,7 @@ func (ms *MetricsStorage) LoadMetrics() error {
atomic.StoreInt64(&ms.collector.totalBytes, int64(totalBytes))
}
// 2. 加载路径统计（如果文件存在）
if fileExists(ms.pathStatsFile) {
var pathStats []map[string]interface{}
if err := loadJSONFromFile(ms.pathStatsFile, &pathStats); err != nil {
log.Printf("[MetricsStorage] 加载路径统计失败: %v", err)
} else {
// 只加载前10个路径统计
maxPaths := 10
if len(pathStats) > maxPaths {
pathStats = pathStats[:maxPaths]
}
for _, pathStat := range pathStats {
path, ok := pathStat["path"].(string)
if !ok {
continue
}
requestCount, _ := pathStat["request_count"].(float64)
errorCount, _ := pathStat["error_count"].(float64)
bytesTransferred, _ := pathStat["bytes_transferred"].(float64)
// 创建或更新路径统计
var pathMetrics *models.PathMetrics
if existingMetrics, ok := ms.collector.pathStats.Load(path); ok {
pathMetrics = existingMetrics.(*models.PathMetrics)
} else {
pathMetrics = &models.PathMetrics{Path: path}
ms.collector.pathStats.Store(path, pathMetrics)
}
// 设置统计值
pathMetrics.RequestCount.Store(int64(requestCount))
pathMetrics.ErrorCount.Store(int64(errorCount))
pathMetrics.BytesTransferred.Store(int64(bytesTransferred))
}
log.Printf("[MetricsStorage] 加载了 %d 条路径统计", len(pathStats))
}
}
// 3. 加载状态码统计（如果文件存在）
// 2. 加载状态码统计（如果文件存在）
if fileExists(ms.statusCodeFile) {
var statusCodeStats map[string]interface{}
if err := loadJSONFromFile(ms.statusCodeFile, &statusCodeStats); err != nil {
@ -253,14 +205,14 @@ func (ms *MetricsStorage) LoadMetrics() error {
}
}
// 4. 加载引用来源统计（如果文件存在）
// 3. 加载引用来源统计（如果文件存在）
if fileExists(ms.refererStatsFile) {
var refererStats []map[string]interface{}
if err := loadJSONFromFile(ms.refererStatsFile, &refererStats); err != nil {
log.Printf("[MetricsStorage] 加载引用来源统计失败: %v", err)
} else {
// 只加载前10个引用来源统计
maxReferers := 10
// 只加载前20个引用来源统计
maxReferers := 20
if len(refererStats) > maxReferers {
refererStats = refererStats[:maxReferers]
}

48
web/app/dashboard/page.tsx
View File

@ -17,13 +17,6 @@ interface Metrics {
bytes_per_second: number
error_rate: number
status_code_stats: Record<string, number>
top_paths: Array<{
path: string
request_count: number
error_count: number
avg_latency: string
bytes_transferred: number
}>
recent_requests: Array<{
Time: string
Path: string
@ -359,47 +352,6 @@ export default function DashboardPage() {
</Card>
)}
<Card>
<CardHeader>
<CardTitle>热门路径 (Top 10)</CardTitle>
</CardHeader>
<CardContent>
<div className="overflow-x-auto">
<table className="w-full">
<thead>
<tr className="border-b">
<th className="text-left p-2">路径</th>
<th className="text-left p-2">请求数</th>
<th className="text-left p-2">错误数</th>
<th className="text-left p-2">平均延迟</th>
<th className="text-left p-2">传输大小</th>
</tr>
</thead>
<tbody>
{(metrics.top_paths || []).map((path, index) => (
<tr key={index} className="border-b">
<td className="p-2 max-w-xs truncate">
<a
href={path.path}
target="_blank"
rel="noopener noreferrer"
className="text-blue-600 hover:text-blue-800 hover:underline"
>
{path.path}
</a>
</td>
<td className="p-2">{path.request_count}</td>
<td className="p-2">{path.error_count}</td>
<td className="p-2">{path.avg_latency}</td>
<td className="p-2">{formatBytes(path.bytes_transferred)}</td>
</tr>
))}
</tbody>
</table>
</div>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle>最近请求</CardTitle>