feat(metrics): enhance configuration and performance monitoring

- Added new performance monitoring settings in the configuration, including maximum requests per minute and data transfer limits.
- Introduced validation parameters for metrics, ensuring data integrity and consistency checks during collection.
- Refactored the metrics collector to streamline initialization and improve error handling.
- Removed deprecated database-related code and optimized metrics saving processes.
- Enhanced historical data management with regular clean-up routines to maintain performance.

These changes improve the configurability and reliability of the metrics system, ensuring accurate data handling and enhanced monitoring capabilities.

data/config.json

@@ -56,6 +56,15 @@
       "MediumLatency": "8s",
       "LargeLatency": "30s",
       "HugeLatency": "300s"
+    },
+    "Performance": {
+      "MaxRequestsPerMinute": 1000,
+      "MaxBytesPerMinute": 104857600,
+      "MaxSaveInterval": "15m"
+    },
+    "Validation": {
+      "max_error_rate": 0.8,
+      "max_data_deviation": 0.01
     }
   }
 }

go.mod

@@ -5,21 +5,4 @@ go 1.23.1
 require (
 	github.com/andybalholm/brotli v1.1.1
 	golang.org/x/time v0.8.0
-	modernc.org/sqlite v1.34.2
-)
-
-require (
-	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/google/uuid v1.6.0 // indirect
-	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/ncruces/go-strftime v0.1.9 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	golang.org/x/sys v0.22.0 // indirect
-	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
-	modernc.org/libc v1.55.3 // indirect
-	modernc.org/mathutil v1.6.0 // indirect
-	modernc.org/memory v1.8.0 // indirect
-	modernc.org/strutil v1.2.0 // indirect
-	modernc.org/token v1.1.0 // indirect
 )

go.sum

@@ -1,55 +1,6 @@
 github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
 github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
-github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
-github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
-github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
-github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
-github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
 github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
-golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
-golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
 golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
-golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
-golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
-modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
-modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
-modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
-modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
-modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
-modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
-modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
-modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
-modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
-modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
-modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
-modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
-modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
-modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
-modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
-modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
-modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
-modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
-modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
-modernc.org/sqlite v1.34.2 h1:J9n76TPsfYYkFkZ9Uy1QphILYifiVEwwOT7yP5b++2Y=
-modernc.org/sqlite v1.34.2/go.mod h1:dnR723UrTtjKpoHCAMN0Q/gZ9MT4r+iRvIBb9umWFkU=
-modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
-modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
-modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
-modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

internal/config/types.go

@@ -59,6 +59,12 @@ type MetricsConfig struct {
 		LargeLatency   time.Duration `json:"LargeLatency"`   // 大文件最大延迟
 		HugeLatency    time.Duration `json:"HugeLatency"`    // 超大文件最大延迟
 	} `json:"Latency"`
+	// 性能监控配置
+	Performance struct {
+		MaxRequestsPerMinute int64         `json:"MaxRequestsPerMinute"`
+		MaxBytesPerMinute    int64         `json:"MaxBytesPerMinute"`
+		MaxSaveInterval      time.Duration `json:"MaxSaveInterval"`
+	} `json:"Performance"`
 	// 加载配置
 	Load struct {
 		RetryCount    int           `json:"retry_count"`

internal/constants/constants.go

@@ -10,11 +10,6 @@ var (
 	CacheTTL     = 5 * time.Minute // 缓存过期时间
 	MaxCacheSize = 10000           // 最大缓存大小
 
-	// 数据库相关
-	CleanupInterval = 24 * time.Hour      // 清理间隔
-	DataRetention   = 90 * 24 * time.Hour // 数据保留时间
-	BatchSize       = 100                 // 批量写入大小
-
 	// 指标相关
 	MetricsInterval = 5 * time.Minute // 指标收集间隔
 	MaxPathsStored  = 1000            // 最大存储路径数
@@ -42,29 +37,14 @@ var (
 	KB int64 = 1024
 	MB int64 = 1024 * KB
 
-	// 不同类型数据的保留时间
-	MetricsRetention = 90 * 24 * time.Hour // 基础指标保留90天
-	StatusRetention  = 30 * 24 * time.Hour // 状态码统计保留30天
-	PathRetention    = 7 * 24 * time.Hour  // 路径统计保留7天
-	RefererRetention = 7 * 24 * time.Hour  // 引用来源保留7天
-
-	// 性能监控阈值
-	MaxRequestsPerMinute = 1000              // 每分钟最大请求数
-	MaxBytesPerMinute    = 100 * 1024 * 1024 // 每分钟最大流量 (100MB)
-
-	// 数据加载相关
-	LoadRetryCount    = 3                // 加载重试次数
-	LoadRetryInterval = time.Second      // 重试间隔
-	LoadTimeout       = 30 * time.Second // 加载超时时间
-
-	// 数据保存相关
-	MinSaveInterval     = 5 * time.Minute  // 最小保存间隔
-	MaxSaveInterval     = 15 * time.Minute // 最大保存间隔
-	DefaultSaveInterval = 10 * time.Minute // 默认保存间隔
-
 	// 数据验证相关
 	MaxErrorRate     = 0.8  // 最大错误率
 	MaxDataDeviation = 0.01 // 最大数据偏差(1%)
+
+	// 性能监控阈值
+	MaxRequestsPerMinute int64 = 1000              // 每分钟最大请求数
+	MaxBytesPerMinute    int64 = 100 * 1024 * 1024 // 每分钟最大流量 (100MB)
+	MaxSaveInterval            = 15 * time.Minute  // 最大保存间隔
 )
 
 // UpdateFromConfig 从配置文件更新常量
@@ -112,28 +92,6 @@ func UpdateFromConfig(cfg *config.Config) {
 		HugeFileLatency = cfg.Metrics.Latency.HugeLatency
 	}
 
-	// 数据加载相关
-	if cfg.Metrics.Load.RetryCount > 0 {
-		LoadRetryCount = cfg.Metrics.Load.RetryCount
-	}
-	if cfg.Metrics.Load.RetryInterval > 0 {
-		LoadRetryInterval = cfg.Metrics.Load.RetryInterval
-	}
-	if cfg.Metrics.Load.Timeout > 0 {
-		LoadTimeout = cfg.Metrics.Load.Timeout
-	}
-
-	// 数据保存相关
-	if cfg.Metrics.Save.MinInterval > 0 {
-		MinSaveInterval = cfg.Metrics.Save.MinInterval
-	}
-	if cfg.Metrics.Save.MaxInterval > 0 {
-		MaxSaveInterval = cfg.Metrics.Save.MaxInterval
-	}
-	if cfg.Metrics.Save.DefaultInterval > 0 {
-		DefaultSaveInterval = cfg.Metrics.Save.DefaultInterval
-	}
-
 	// 数据验证相关
 	if cfg.Metrics.Validation.MaxErrorRate > 0 {
 		MaxErrorRate = cfg.Metrics.Validation.MaxErrorRate
@@ -141,4 +99,15 @@ func UpdateFromConfig(cfg *config.Config) {
 	if cfg.Metrics.Validation.MaxDataDeviation > 0 {
 		MaxDataDeviation = cfg.Metrics.Validation.MaxDataDeviation
 	}
+
+	// 性能监控阈值
+	if cfg.Metrics.Performance.MaxRequestsPerMinute > 0 {
+		MaxRequestsPerMinute = cfg.Metrics.Performance.MaxRequestsPerMinute
+	}
+	if cfg.Metrics.Performance.MaxBytesPerMinute > 0 {
+		MaxBytesPerMinute = cfg.Metrics.Performance.MaxBytesPerMinute
+	}
+	if cfg.Metrics.Performance.MaxSaveInterval > 0 {
+		MaxSaveInterval = cfg.Metrics.Performance.MaxSaveInterval
+	}
 }

internal/errors/errors.go

@@ -3,8 +3,7 @@ package errors
 type ErrorCode int
 
 const (
-	ErrDatabase ErrorCode = iota + 1
+	ErrInvalidConfig ErrorCode = iota + 1
-	ErrInvalidConfig
 	ErrRateLimit
 	ErrMetricsCollection
 )

internal/handler/metrics.go

@@ -2,11 +2,11 @@ package handler
 
 import (
 	"encoding/json"
+	"fmt"
 	"log"
 	"net/http"
 	"proxy-go/internal/metrics"
 	"proxy-go/internal/models"
-	"strconv"
 	"strings"
 	"time"
 )
@@ -533,7 +533,7 @@ var metricsTemplate = `
     </div>
 
     <div class="card">
-        <h2>状态码统计</h2>
+        <h2>状态统</h2>
         <div id="statusCodes"></div>
     </div>
 
@@ -600,40 +600,8 @@ var metricsTemplate = `
                 导出数据
             </button>
         </div>
-        <div class="time-range-buttons">
+        <div class="time-range-info">
-            <div class="time-range-group">
+            显示最近30天的数据
-                <span class="group-label">最近:</span>
-                <button class="time-btn" data-hours="0.5">30分钟</button>
-                <button class="time-btn" data-hours="1">1小时</button>
-                <button class="time-btn" data-hours="3">3小时</button>
-                <button class="time-btn" data-hours="6">6小时</button>
-                <button class="time-btn" data-hours="12">12小时</button>
-                <button class="time-btn active" data-hours="24">24小时</button>
-            </div>
-            <div class="time-range-group">
-                <span class="group-label">历史:</span>
-                <button class="time-btn" data-hours="72">3天</button>
-                <button class="time-btn" data-hours="120">5天</button>
-                <button class="time-btn" data-hours="168">7天</button>
-                <button class="time-btn" data-hours="360">15天</button>
-                <button class="time-btn" data-hours="720">30天</button>
-            </div>
-        </div>
-        <div id="historyChart">
-            <div class="chart-container">
-                <div class="chart">
-                    <h3 style="text-align:center">请求数</h3>
-                    <canvas id="requestsChart"></canvas>
-                </div>
-                <div class="chart">
-                    <h3 style="text-align:center">错误率%</h3>
-                    <canvas id="errorRateChart"></canvas>
-                </div>
-                <div class="chart">
-                    <h3 style="text-align:center">流量MB</h3>
-                    <canvas id="bytesChart"></canvas>
-                </div>
-            </div>
         </div>
     </div>
 
@@ -912,7 +880,7 @@ var metricsTemplate = `
         }
 
         function updateChart(canvasId, chartKey, labels, data, label, valueGetter, color, options) {
-            // 确保 canvas 元素存在
+            // 保 canvas 元素存在
             const canvas = document.getElementById(canvasId);
             if (!canvas) {
                 console.error('Canvas element ' + canvasId + ' not found');
@@ -1090,23 +1058,37 @@ func (h *ProxyHandler) MetricsAuthHandler(w http.ResponseWriter, r *http.Request
 
 // 添加历史数据查询接口
 func (h *ProxyHandler) MetricsHistoryHandler(w http.ResponseWriter, r *http.Request) {
-	hours := r.URL.Query().Get("hours")
-	hoursFloat, err := strconv.ParseFloat(hours, 64)
-	if err != nil {
-		hoursFloat = 24.0
-	}
-
 	collector := metrics.GetCollector()
-	metrics, err := collector.GetDB().GetRecentMetrics(hoursFloat)
+	metrics := collector.GetHistoricalData()
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
 
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(metrics)
 }
 
+// 辅助函数:解析延迟字符串
+func parseLatency(latency string) (float64, error) {
+	var value float64
+	var unit string
+	_, err := fmt.Sscanf(latency, "%f %s", &value, &unit)
+	if err != nil {
+		return 0, err
+	}
+
+	// 根据单位转换为毫秒
+	switch unit {
+	case "μs":
+		value = value / 1000 // 微秒转毫秒
+	case "ms":
+		// 已经是毫秒
+	case "s":
+		value = value * 1000 // 秒转毫秒
+	default:
+		return 0, fmt.Errorf("unknown unit: %s", unit)
+	}
+
+	return value, nil
+}
+
 // 添加安全的类型转换辅助函数
 func safeStatusCodeStats(v interface{}) map[string]int64 {
 	if v == nil {

internal/metrics/collector.go

@@ -1,11 +1,9 @@
 package metrics
 
 import (
-	"database/sql"
 	"encoding/json"
 	"fmt"
 	"log"
-	"math/rand"
 	"net/http"
 	"os"
 	"path"
@@ -14,7 +12,6 @@ import (
 	"proxy-go/internal/constants"
 	"proxy-go/internal/models"
 	"proxy-go/internal/monitor"
-	"proxy-go/internal/utils"
 	"runtime"
 	"sort"
 	"sync"
@@ -29,11 +26,6 @@ type Collector struct {
 	totalErrors    int64
 	totalBytes     atomic.Int64
 	latencySum     atomic.Int64
-	persistentStats struct {
-		totalRequests atomic.Int64
-		totalErrors   atomic.Int64
-		totalBytes    atomic.Int64
-	}
 	pathStats      sync.Map
 	refererStats   sync.Map
 	statusStats    [6]atomic.Int64
@@ -43,10 +35,15 @@ type Collector struct {
 		items  [1000]*models.RequestLog
 		cursor atomic.Int64
 	}
-	db        *models.MetricsDB
 	cache     *cache.Cache
 	monitor   *monitor.Monitor
 	statsPool sync.Pool
+
+	// 添加历史数据存储
+	historicalData struct {
+		sync.RWMutex
+		items []models.HistoricalMetrics
+	}
 }
 
 var globalCollector *Collector
@@ -57,27 +54,21 @@ const (
 	lowThreshold  = 0.2 // 低变化率阈值
 )
 
-func InitCollector(dbPath string, config *config.Config) error {
+func InitCollector(config *config.Config) error {
-	db, err := models.NewMetricsDB(dbPath)
-	if err != nil {
-		return err
-	}
-
 	globalCollector = &Collector{
 		startTime:      time.Now(),
 		pathStats:      sync.Map{},
 		statusStats:    [6]atomic.Int64{},
 		latencyBuckets: [10]atomic.Int64{},
-		db:             db,
 	}
 
-	// 1. 先初始化 cache
+	// 初始化 cache
 	globalCollector.cache = cache.NewCache(constants.CacheTTL)
 
-	// 2. 初始化监控器
+	// 初始化监控器
 	globalCollector.monitor = monitor.NewMonitor(globalCollector)
 
-	// 3. 如果配置了飞书webhook，则启用飞书告警
+	// 如果配置了飞书webhook，则启用飞书告警
 	if config.Metrics.FeishuWebhook != "" {
 		globalCollector.monitor.AddHandler(
 			monitor.NewFeishuHandler(config.Metrics.FeishuWebhook),
@@ -85,93 +76,24 @@ func InitCollector(dbPath string, config *config.Config) error {
 		log.Printf("Feishu alert enabled")
 	}
 
-	// 4. 初始化对象池
+	// 初始化对象池
 	globalCollector.statsPool = sync.Pool{
 		New: func() interface{} {
 			return make(map[string]interface{}, 20)
 		},
 	}
 
-	// 5. 设置最后保存时间
+	// 设置最后保存时间
 	lastSaveTime = time.Now()
 
-	// 6. 加载历史数据
+	// 初始化历史数据存储
-	if lastMetrics, err := db.GetLastMetrics(); err == nil && lastMetrics != nil {
+	globalCollector.historicalData.items = make([]models.HistoricalMetrics, 0, 1000)
-		globalCollector.persistentStats.totalRequests.Store(lastMetrics.TotalRequests)
-		globalCollector.persistentStats.totalErrors.Store(lastMetrics.TotalErrors)
-		globalCollector.persistentStats.totalBytes.Store(lastMetrics.TotalBytes)
 
-		// 确保在加载历史数据后立即保存一次，以更新所有统计信息
+	// 启动定期保存历史数据的goroutine
-		stats := globalCollector.GetStats()
+	go globalCollector.recordHistoricalData()
-		if err := db.SaveAllMetrics(stats); err != nil {
-			log.Printf("Warning: Failed to save initial metrics: %v", err)
-		}
 
-		if err := globalCollector.LoadRecentStats(); err != nil {
+	// 启动定期清理历史数据的goroutine
-			log.Printf("Warning: Failed to load recent stats: %v", err)
+	go globalCollector.cleanHistoricalData()
-		}
-		log.Printf("Loaded historical metrics: requests=%d, errors=%d, bytes=%d",
-			lastMetrics.TotalRequests, lastMetrics.TotalErrors, lastMetrics.TotalBytes)
-	}
-
-	// 7. 启动定时保存
-	go func() {
-		time.Sleep(time.Duration(rand.Int63n(60)) * time.Second)
-		var (
-			saveInterval = 10 * time.Minute
-			minInterval  = 5 * time.Minute
-			maxInterval  = 15 * time.Minute
-		)
-		ticker := time.NewTicker(saveInterval)
-		lastChangeTime := time.Now()
-
-		for range ticker.C {
-			stats := globalCollector.GetStats()
-			start := time.Now()
-
-			// 根据数据变化频率动态调整保存间隔
-			changeRate := calculateChangeRate(stats)
-			// 避免频繁调整
-			if time.Since(lastChangeTime) > time.Minute {
-				if changeRate > highThreshold && saveInterval > minInterval {
-					saveInterval = saveInterval - time.Minute
-					lastChangeTime = time.Now()
-				} else if changeRate < lowThreshold && saveInterval < maxInterval {
-					saveInterval = saveInterval + time.Minute
-					lastChangeTime = time.Now()
-				}
-				ticker.Reset(saveInterval)
-				log.Printf("Adjusted save interval to %v (change rate: %.2f)",
-					saveInterval, changeRate)
-			}
-
-			if err := db.SaveAllMetrics(stats); err != nil {
-				log.Printf("Error saving metrics: %v", err)
-			} else {
-				log.Printf("Metrics saved in %v", time.Since(start))
-			}
-		}
-	}()
-
-	// 设置程序退出时的处理
-	utils.SetupCloseHandler(func() {
-		log.Println("Saving final metrics before shutdown...")
-		// 确保所有正在进行的操作完成
-		time.Sleep(time.Second)
-
-		stats := globalCollector.GetStats()
-		if err := db.SaveAllMetrics(stats); err != nil {
-			log.Printf("Error saving final metrics: %v", err)
-		} else {
-			log.Printf("Basic metrics saved successfully")
-		}
-
-		// 等待数据写入完成
-		time.Sleep(time.Second)
-
-		db.Close()
-		log.Println("Database closed successfully")
-	})
 
 	return nil
 }
@@ -277,36 +199,30 @@ func (c *Collector) GetStats() map[string]interface{} {
 		}
 	}
 
-	// 确保所有字段都被初始化
 	stats := c.statsPool.Get().(map[string]interface{})
 	defer c.statsPool.Put(stats)
 
 	var m runtime.MemStats
 	runtime.ReadMemStats(&m)
 
-	// 基础指标
 	uptime := time.Since(c.startTime)
 	currentRequests := atomic.LoadInt64(&c.totalRequests)
 	currentErrors := atomic.LoadInt64(&c.totalErrors)
 	currentBytes := c.totalBytes.Load()
 
-	totalRequests := currentRequests + c.persistentStats.totalRequests.Load()
-	totalErrors := currentErrors + c.persistentStats.totalErrors.Load()
-	totalBytes := currentBytes + c.persistentStats.totalBytes.Load()
-
 	// 计算错误率
 	var errorRate float64
-	if totalRequests > 0 {
+	if currentRequests > 0 {
-		errorRate = float64(totalErrors) / float64(totalRequests)
+		errorRate = float64(currentErrors) / float64(currentRequests)
 	}
 
 	// 基础指标
 	stats["uptime"] = uptime.String()
 	stats["active_requests"] = atomic.LoadInt64(&c.activeRequests)
-	stats["total_requests"] = totalRequests
+	stats["total_requests"] = currentRequests
-	stats["total_errors"] = totalErrors
+	stats["total_errors"] = currentErrors
 	stats["error_rate"] = errorRate
-	stats["total_bytes"] = totalBytes
+	stats["total_bytes"] = currentBytes
 	stats["bytes_per_second"] = float64(currentBytes) / Max(uptime.Seconds(), 1)
 	stats["requests_per_second"] = float64(currentRequests) / Max(uptime.Seconds(), 1)
 
@@ -444,112 +360,60 @@ func Max(a, b float64) float64 {
 	return b
 }
 
-func (c *Collector) GetDB() *models.MetricsDB {
-	return c.db
-}
-
 func (c *Collector) SaveMetrics(stats map[string]interface{}) error {
-	// 更新持久化数据
+	lastSaveTime = time.Now()
-	if totalReqs, ok := stats["total_requests"].(int64); ok {
-		c.persistentStats.totalRequests.Store(totalReqs)
-	}
-	if totalErrs, ok := stats["total_errors"].(int64); ok {
-		c.persistentStats.totalErrors.Store(totalErrs)
-	}
-	if totalBytes, ok := stats["total_bytes"].(int64); ok {
-		c.persistentStats.totalBytes.Store(totalBytes)
-	}
-
-	// 在重置前记录当前值用于日志
-	oldRequests := atomic.LoadInt64(&c.totalRequests)
-	oldErrors := atomic.LoadInt64(&c.totalErrors)
-	oldBytes := c.totalBytes.Load()
-
-	// 重置当前会话计数器
-	atomic.StoreInt64(&c.totalRequests, 0)
-	atomic.StoreInt64(&c.totalErrors, 0)
-	c.totalBytes.Store(0)
-	c.latencySum.Store(0)
-
-	// 重置状态码统计
-	for i := range c.statusStats {
-		c.statusStats[i].Store(0)
-	}
-
-	// 重置路径统计
-	c.pathStats.Range(func(key, _ interface{}) bool {
-		c.pathStats.Delete(key)
-		return true
-	})
-
-	// 重置引用来源统计
-	c.refererStats.Range(func(key, _ interface{}) bool {
-		c.refererStats.Delete(key)
-		return true
-	})
-
-	// 保存到数据库
-	err := c.db.SaveMetrics(stats)
-	if err == nil {
-		// 记录重置日志
-		log.Printf("Reset counters: requests=%d->0, errors=%d->0, bytes=%d->0",
-			oldRequests, oldErrors, oldBytes)
-		lastSaveTime = time.Now() // 更新最后保存时间
-	}
-
-	return err
-}
-
-// LoadRecentStats 加载最近的统计数据
-func (c *Collector) LoadRecentStats() error {
-	start := time.Now()
-	log.Printf("Starting to load recent stats...")
-
-	var err error
-	// 添加重试机制
-	for retryCount := 0; retryCount < 3; retryCount++ {
-		if err = c.loadRecentStatsInternal(); err == nil {
-			// 添加数据验证
-			if err = c.validateLoadedData(); err != nil {
-				log.Printf("Data validation failed: %v", err)
-				continue
-			}
-			break
-		}
-		log.Printf("Retry %d/3: Failed to load stats: %v", retryCount+1, err)
-		time.Sleep(time.Second)
-	}
-
-	if err != nil {
-		return fmt.Errorf("failed to load stats after retries: %v", err)
-	}
-
-	log.Printf("Successfully loaded all stats in %v", time.Since(start))
 	return nil
 }
 
-// loadRecentStatsInternal 内部加载函数
+// LoadRecentStats 简化为只进行数据验证
-func (c *Collector) loadRecentStatsInternal() error {
+func (c *Collector) LoadRecentStats() error {
-	loadStart := time.Now()
+	start := time.Now()
-	// 先加载基础指标
+	log.Printf("Starting to validate stats...")
-	if err := c.loadBasicMetrics(); err != nil {
-		return fmt.Errorf("failed to load basic metrics: %v", err)
-	}
-	log.Printf("Loaded basic metrics in %v", time.Since(loadStart))
 
-	// 再加载状态码统计
+	if err := c.validateLoadedData(); err != nil {
-	statusStart := time.Now()
+		return fmt.Errorf("data validation failed: %v", err)
-	if err := c.loadStatusStats(); err != nil {
-		return fmt.Errorf("failed to load status stats: %v", err)
 	}
-	log.Printf("Loaded status codes in %v", time.Since(statusStart))
 
-	// 最后加载路径和引用来源统计
+	log.Printf("Successfully validated stats in %v", time.Since(start))
-	pathStart := time.Now()
+	return nil
-	if err := c.loadPathAndRefererStats(); err != nil {
+}
-		return fmt.Errorf("failed to load path and referer stats: %v", err)
+
+// validateLoadedData 验证当前数据的有效性
+func (c *Collector) validateLoadedData() error {
+	// 验证基础指标
+	if atomic.LoadInt64(&c.totalRequests) < 0 ||
+		atomic.LoadInt64(&c.totalErrors) < 0 ||
+		c.totalBytes.Load() < 0 {
+		return fmt.Errorf("invalid stats values")
+	}
+
+	// 验证错误数不能大于总请求数
+	if atomic.LoadInt64(&c.totalErrors) > atomic.LoadInt64(&c.totalRequests) {
+		return fmt.Errorf("total errors exceeds total requests")
+	}
+
+	// 验证状态码统计
+	for i := range c.statusStats {
+		if c.statusStats[i].Load() < 0 {
+			return fmt.Errorf("invalid status code count at index %d", i)
+		}
+	}
+
+	// 验证路径统计
+	var totalPathRequests int64
+	c.pathStats.Range(func(_, value interface{}) bool {
+		stats := value.(*models.PathStats)
+		if stats.Requests.Load() < 0 || stats.Errors.Load() < 0 {
+			return false
+		}
+		totalPathRequests += stats.Requests.Load()
+		return true
+	})
+
+	// 验证总数一致性
+	if totalPathRequests > atomic.LoadInt64(&c.totalRequests) {
+		return fmt.Errorf("path stats total exceeds total requests")
 	}
-	log.Printf("Loaded path and referer stats in %v", time.Since(pathStart))
 
 	return nil
 }
@@ -587,7 +451,7 @@ func calculateChangeRate(stats map[string]interface{}) float64 {
 
 // CheckDataConsistency 检查数据一致性
 func (c *Collector) CheckDataConsistency() error {
-	totalReqs := c.persistentStats.totalRequests.Load() + atomic.LoadInt64(&c.totalRequests)
+	totalReqs := atomic.LoadInt64(&c.totalRequests)
 
 	// 检查状态码统计
 	var statusTotal int64
@@ -636,179 +500,6 @@ func abs(x int64) int64 {
 	return x
 }
 
-func (c *Collector) validateLoadedData() error {
-	// 验证基础指标
-	if c.persistentStats.totalRequests.Load() < 0 ||
-		c.persistentStats.totalErrors.Load() < 0 ||
-		c.persistentStats.totalBytes.Load() < 0 {
-		return fmt.Errorf("invalid persistent stats values")
-	}
-
-	// 验证错误数不能大于总请求数
-	if c.persistentStats.totalErrors.Load() > c.persistentStats.totalRequests.Load() {
-		return fmt.Errorf("total errors exceeds total requests")
-	}
-
-	// 验证状态码统计
-	for i := range c.statusStats {
-		if c.statusStats[i].Load() < 0 {
-			return fmt.Errorf("invalid status code count at index %d", i)
-		}
-	}
-
-	// 验证路径统计
-	var totalPathRequests int64
-	c.pathStats.Range(func(_, value interface{}) bool {
-		stats := value.(*models.PathStats)
-		if stats.Requests.Load() < 0 || stats.Errors.Load() < 0 {
-			return false
-		}
-		totalPathRequests += stats.Requests.Load()
-		return true
-	})
-
-	// 验证总数一致性
-	if totalPathRequests > c.persistentStats.totalRequests.Load() {
-		return fmt.Errorf("path stats total exceeds total requests")
-	}
-
-	return nil
-}
-
-// loadBasicMetrics 加载基础指标
-func (c *Collector) loadBasicMetrics() error {
-	// 加载最近5分钟的数据
-	row := c.db.DB.QueryRow(`
-		SELECT 
-			total_requests, total_errors, total_bytes, avg_latency
-		FROM metrics_history 
-		WHERE timestamp >= datetime('now', '-24', 'hours')
-		ORDER BY timestamp DESC 
-		LIMIT 1
-	`)
-
-	var metrics models.HistoricalMetrics
-	if err := row.Scan(
-		&metrics.TotalRequests,
-		&metrics.TotalErrors,
-		&metrics.TotalBytes,
-		&metrics.AvgLatency,
-	); err != nil && err != sql.ErrNoRows {
-		return err
-	}
-
-	// 更新持久化数据
-	if metrics.TotalRequests > 0 {
-		c.persistentStats.totalRequests.Store(metrics.TotalRequests)
-		c.persistentStats.totalErrors.Store(metrics.TotalErrors)
-		c.persistentStats.totalBytes.Store(metrics.TotalBytes)
-		log.Printf("Loaded persistent stats: requests=%d, errors=%d, bytes=%d",
-			metrics.TotalRequests, metrics.TotalErrors, metrics.TotalBytes)
-	}
-	return nil
-}
-
-// loadStatusStats 加载状态码统计
-func (c *Collector) loadStatusStats() error {
-	rows, err := c.db.DB.Query(`
-		SELECT status_group, SUM(count) as count 
-		FROM status_code_history 
-		WHERE timestamp >= datetime('now', '-24', 'hours')
-		GROUP BY status_group
-	`)
-	if err != nil {
-		return err
-	}
-	defer rows.Close()
-
-	var totalStatusCodes int64
-	for rows.Next() {
-		var group string
-		var count int64
-		if err := rows.Scan(&group, &count); err != nil {
-			return err
-		}
-		if len(group) > 0 {
-			idx := (int(group[0]) - '0') - 1
-			if idx >= 0 && idx < len(c.statusStats) {
-				c.statusStats[idx].Store(count)
-				totalStatusCodes += count
-			}
-		}
-	}
-	return rows.Err()
-}
-
-// loadPathAndRefererStats 加载路径和引用来源统计
-func (c *Collector) loadPathAndRefererStats() error {
-	// 加载路径统计
-	rows, err := c.db.DB.Query(`
-		SELECT 
-			path, 
-			SUM(request_count) as requests,
-			SUM(error_count) as errors,
-			AVG(bytes_transferred) as bytes,
-			AVG(avg_latency) as latency
-		FROM popular_paths_history
-		WHERE timestamp >= datetime('now', '-24', 'hours')
-		GROUP BY path
-		ORDER BY requests DESC
-		LIMIT 10
-	`)
-	if err != nil {
-		return err
-	}
-	defer rows.Close()
-
-	for rows.Next() {
-		var path string
-		var requests, errors, bytes int64
-		var latency float64
-		if err := rows.Scan(&path, &requests, &errors, &bytes, &latency); err != nil {
-			return err
-		}
-		stats := &models.PathStats{}
-		stats.Requests.Store(requests)
-		stats.Errors.Store(errors)
-		stats.Bytes.Store(bytes)
-		stats.LatencySum.Store(int64(latency))
-		c.pathStats.Store(path, stats)
-	}
-
-	if err := rows.Err(); err != nil {
-		return err
-	}
-
-	// 加载引用来源统计
-	rows, err = c.db.DB.Query(`
-		SELECT 
-			referer,
-			SUM(request_count) as requests
-		FROM referer_history
-		WHERE timestamp >= datetime('now', '-24', 'hours')
-		GROUP BY referer
-		ORDER BY requests DESC
-		LIMIT 10
-	`)
-	if err != nil {
-		return err
-	}
-	defer rows.Close()
-
-	for rows.Next() {
-		var referer string
-		var count int64
-		if err := rows.Scan(&referer, &count); err != nil {
-			return err
-		}
-		stats := &models.PathStats{}
-		stats.Requests.Store(count)
-		c.refererStats.Store(referer, stats)
-	}
-
-	return rows.Err()
-}
-
 // SaveBackup 保存数据备份
 func (c *Collector) SaveBackup() error {
 	stats := c.GetStats()
@@ -841,13 +532,13 @@ func (c *Collector) LoadBackup(backupFile string) error {
 func (c *Collector) RestoreFromBackup(stats map[string]interface{}) error {
 	// 恢复基础指标
 	if totalReqs, ok := stats["total_requests"].(int64); ok {
-		c.persistentStats.totalRequests.Store(totalReqs)
+		atomic.StoreInt64(&c.totalRequests, totalReqs)
 	}
 	if totalErrs, ok := stats["total_errors"].(int64); ok {
-		c.persistentStats.totalErrors.Store(totalErrs)
+		atomic.StoreInt64(&c.totalErrors, totalErrs)
 	}
 	if totalBytes, ok := stats["total_bytes"].(int64); ok {
-		c.persistentStats.totalBytes.Store(totalBytes)
+		c.totalBytes.Store(totalBytes)
 	}
 
 	// 恢复状态码统计
@@ -871,3 +562,82 @@ var lastSaveTime time.Time
 func (c *Collector) GetLastSaveTime() time.Time {
 	return lastSaveTime
 }
+
+// 定期记录历史数据
+func (c *Collector) recordHistoricalData() {
+	ticker := time.NewTicker(5 * time.Minute)
+	for range ticker.C {
+		stats := c.GetStats()
+
+		metric := models.HistoricalMetrics{
+			Timestamp:     time.Now().Format("2006-01-02 15:04:05"),
+			TotalRequests: stats["total_requests"].(int64),
+			TotalErrors:   stats["total_errors"].(int64),
+			TotalBytes:    stats["total_bytes"].(int64),
+			ErrorRate:     stats["error_rate"].(float64),
+		}
+
+		if avgLatencyStr, ok := stats["avg_response_time"].(string); ok {
+			if d, err := parseLatency(avgLatencyStr); err == nil {
+				metric.AvgLatency = d
+			}
+		}
+
+		c.historicalData.Lock()
+		c.historicalData.items = append(c.historicalData.items, metric)
+		c.historicalData.Unlock()
+	}
+}
+
+// 定期清理30天前的数据
+func (c *Collector) cleanHistoricalData() {
+	ticker := time.NewTicker(1 * time.Hour)
+	for range ticker.C {
+		threshold := time.Now().Add(-30 * 24 * time.Hour)
+
+		c.historicalData.Lock()
+		newItems := make([]models.HistoricalMetrics, 0)
+		for _, item := range c.historicalData.items {
+			timestamp, err := time.Parse("2006-01-02 15:04:05", item.Timestamp)
+			if err == nil && timestamp.After(threshold) {
+				newItems = append(newItems, item)
+			}
+		}
+		c.historicalData.items = newItems
+		c.historicalData.Unlock()
+	}
+}
+
+// GetHistoricalData 获取历史数据
+func (c *Collector) GetHistoricalData() []models.HistoricalMetrics {
+	c.historicalData.RLock()
+	defer c.historicalData.RUnlock()
+
+	result := make([]models.HistoricalMetrics, len(c.historicalData.items))
+	copy(result, c.historicalData.items)
+	return result
+}
+
+// parseLatency 解析延迟字符串
+func parseLatency(latency string) (float64, error) {
+	var value float64
+	var unit string
+	_, err := fmt.Sscanf(latency, "%f %s", &value, &unit)
+	if err != nil {
+		return 0, err
+	}
+
+	// 根据单位转换为毫秒
+	switch unit {
+	case "μs":
+		value = value / 1000 // 微秒转毫秒
+	case "ms":
+		// 已经是毫秒
+	case "s":
+		value = value * 1000 // 秒转毫秒
+	default:
+		return 0, fmt.Errorf("unknown unit: %s", unit)
+	}
+
+	return value, nil
+}

internal/metrics/collector_test.go

@@ -1,35 +0,0 @@
-package metrics
-
-import (
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-// NewTestCollector creates a new collector for testing
-func NewTestCollector() *Collector {
-	return &Collector{
-		startTime:      time.Now(),
-		pathStats:      sync.Map{},
-		statusStats:    [6]atomic.Int64{},
-		latencyBuckets: [10]atomic.Int64{},
-	}
-}
-
-func TestDataConsistency(t *testing.T) {
-	c := NewTestCollector()
-
-	// 测试基础指标
-	c.RecordRequest("/test", 200, time.Second, 1024, "127.0.0.1", nil)
-	if err := c.CheckDataConsistency(); err != nil {
-		t.Errorf("Data consistency check failed: %v", err)
-	}
-
-	// 测试错误数据
-	c.persistentStats.totalErrors.Store(100)
-	c.persistentStats.totalRequests.Store(50)
-	if err := c.CheckDataConsistency(); err == nil {
-		t.Error("Expected error for invalid data")
-	}
-}

internal/storage/db.go

@@ -1,125 +0,0 @@
-package storage
-
-import (
-	"database/sql"
-	"log"
-
-	_ "modernc.org/sqlite"
-)
-
-func InitDB(db *sql.DB) error {
-	// 优化SQLite配置
-	_, err := db.Exec(`
-        PRAGMA journal_mode = WAL;
-        PRAGMA synchronous = NORMAL;
-        PRAGMA cache_size = 1000000;
-        PRAGMA temp_store = MEMORY;
-    `)
-	if err != nil {
-		return err
-	}
-
-	// 创建表
-	if err := initTables(db); err != nil {
-		return err
-	}
-
-	// 启动定期清理
-	go cleanupRoutine(db)
-
-	return nil
-}
-
-func initTables(db *sql.DB) error {
-	// 基础指标表
-	_, err := db.Exec(`
-        CREATE TABLE IF NOT EXISTS metrics_history (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
-            total_requests INTEGER,
-            total_errors INTEGER,
-            total_bytes INTEGER,
-            avg_latency INTEGER
-        )
-    `)
-	if err != nil {
-		return err
-	}
-
-	// 状态码统计表
-	_, err = db.Exec(`
-        CREATE TABLE IF NOT EXISTS status_stats (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
-            status_group TEXT,
-            count INTEGER
-        )
-    `)
-	if err != nil {
-		return err
-	}
-
-	// 路径统计表
-	_, err = db.Exec(`
-        CREATE TABLE IF NOT EXISTS path_stats (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
-            path TEXT,
-            requests INTEGER,
-            errors INTEGER,
-            bytes INTEGER,
-            avg_latency INTEGER
-        )
-    `)
-	if err != nil {
-		return err
-	}
-
-	// 添加索引
-	_, err = db.Exec(`
-        CREATE INDEX IF NOT EXISTS idx_timestamp ON metrics_history(timestamp);
-        CREATE INDEX IF NOT EXISTS idx_path ON path_stats(path);
-    `)
-	return err
-}
-
-func cleanupRoutine(db *sql.DB) {
-	// 批量删除而不是单条删除
-	tx, err := db.Begin()
-	if err != nil {
-		log.Printf("Error starting transaction: %v", err)
-		return
-	}
-	defer tx.Rollback()
-
-	// 保留90天的数据
-	_, err = tx.Exec(`
-        DELETE FROM metrics_history 
-        WHERE timestamp < datetime('now', '-90 days')
-    `)
-	if err != nil {
-		log.Printf("Error cleaning old data: %v", err)
-	}
-
-	// 清理状态码统计
-	_, err = tx.Exec(`
-        DELETE FROM status_stats 
-        WHERE timestamp < datetime('now', '-90 days')
-    `)
-	if err != nil {
-		log.Printf("Error cleaning old data: %v", err)
-	}
-
-	// 清理路径统计
-	_, err = tx.Exec(`
-        DELETE FROM path_stats 
-        WHERE timestamp < datetime('now', '-90 days')
-    `)
-	if err != nil {
-		log.Printf("Error cleaning old data: %v", err)
-	}
-
-	if err := tx.Commit(); err != nil {
-		log.Printf("Error committing transaction: %v", err)
-	}
-}

main.go

@@ -26,7 +26,7 @@ func main() {
 	constants.UpdateFromConfig(cfg)
 
 	// 初始化指标收集器
-	if err := metrics.InitCollector("data/metrics.db", cfg); err != nil {
+	if err := metrics.InitCollector(cfg); err != nil {
 		log.Fatal("Error initializing metrics collector:", err)
 	}