From 97a195c9ddc465e0fde7756ef6275c1af34f7700 Mon Sep 17 00:00:00 2001 From: wood chen Date: Thu, 5 Dec 2024 08:15:18 +0800 Subject: [PATCH] feat(metrics): add recent status stats loading and optimize database transaction settings - Implemented a new function to load recent status statistics from the database, enhancing the collector's ability to track real-time metrics. - Added error logging for failed status stats loading to improve monitoring and debugging capabilities. - Moved database transaction optimization settings outside of the transaction scope in SaveMetrics, improving performance during metric saving operations. - Updated SQL queries in GetRecentMetrics to streamline time filtering logic, ensuring accurate retrieval of recent metrics. These changes enhance the metrics collection process and improve the overall performance and reliability of the metrics dashboard. --- internal/metrics/collector.go | 30 ++++++++++++++++++++++++++++++ internal/models/metrics.go | 34 +++++++++++++--------------------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/internal/metrics/collector.go b/internal/metrics/collector.go index 81d302b..2cb6c7c 100644 --- a/internal/metrics/collector.go +++ b/internal/metrics/collector.go @@ -66,6 +66,9 @@ func InitCollector(dbPath string, config *config.Config) error { globalCollector.persistentStats.totalRequests.Store(lastMetrics.TotalRequests) globalCollector.persistentStats.totalErrors.Store(lastMetrics.TotalErrors) globalCollector.persistentStats.totalBytes.Store(lastMetrics.TotalBytes) + if err := loadRecentStatusStats(db); err != nil { + log.Printf("Warning: Failed to load recent status stats: %v", err) + } log.Printf("Loaded historical metrics: requests=%d, errors=%d, bytes=%d", lastMetrics.TotalRequests, lastMetrics.TotalErrors, lastMetrics.TotalBytes) } @@ -586,3 +589,30 @@ func formatAvgLatency(latencySum, requests int64) string { } return FormatDuration(time.Duration(latencySum / requests)) } + +func loadRecentStatusStats(db *models.MetricsDB) error { + rows, err := db.DB.Query(` + SELECT status_group, count + FROM status_stats + WHERE timestamp >= datetime('now', '-5', 'minutes') + `) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var group string + var count int64 + if err := rows.Scan(&group, &count); err != nil { + return err + } + if len(group) > 0 { + idx := (int(group[0]) - '0') - 1 + if idx >= 0 && idx < len(globalCollector.statusStats) { + globalCollector.statusStats[idx].Store(count) + } + } + } + return rows.Err() +} diff --git a/internal/models/metrics.go b/internal/models/metrics.go index a37be00..e8b4569 100644 --- a/internal/models/metrics.go +++ b/internal/models/metrics.go @@ -285,7 +285,7 @@ func cleanupRoutine(db *sql.DB) { continue } - // 优化清理性能 + // 优化理性能 if _, err := tx.Exec("PRAGMA synchronous = NORMAL"); err != nil { log.Printf("Error setting synchronous mode: %v", err) } @@ -367,20 +367,20 @@ func getDBSize(db *sql.DB) int64 { } func (db *MetricsDB) SaveMetrics(stats map[string]interface{}) error { + // 设置事务优化参数 - 移到事务外 + if _, err := db.DB.Exec("PRAGMA synchronous = NORMAL"); err != nil { + return fmt.Errorf("failed to set synchronous mode: %v", err) + } + if _, err := db.DB.Exec("PRAGMA journal_mode = WAL"); err != nil { + return fmt.Errorf("failed to set journal mode: %v", err) + } + tx, err := db.DB.Begin() if err != nil { return err } defer tx.Rollback() - // 设置事务优化参数 - if _, err := tx.Exec("PRAGMA synchronous = NORMAL"); err != nil { - return fmt.Errorf("failed to set synchronous mode: %v", err) - } - if _, err := tx.Exec("PRAGMA journal_mode = WAL"); err != nil { - return fmt.Errorf("failed to set journal mode: %v", err) - } - // 使用预处理语句提高性能 stmt, err := tx.Prepare(` INSERT INTO metrics_history ( @@ -464,14 +464,6 @@ func (db *MetricsDB) GetRecentMetrics(hours float64) ([]HistoricalMetrics, error return nil, fmt.Errorf("failed to commit transaction: %v", err) } - // 计算查询时间范围 - endTime := time.Now() - startTime := endTime.Add(-time.Duration(hours * float64(time.Hour.Nanoseconds()))) - - // 格式化时间 - startTimeStr := startTime.Format("2006-01-02 15:04:05") - endTimeStr := endTime.Format("2006-01-02 15:04:05") - // 处理小于1小时的情况 if hours <= 0 { hours = 0.5 // 30分钟 @@ -507,11 +499,11 @@ func (db *MetricsDB) GetRecentMetrics(hours float64) ([]HistoricalMetrics, error rows, err := db.DB.Query(` WITH RECURSIVE time_series(time_point) AS ( - SELECT datetime(?, 'localtime') + SELECT datetime('now', 'localtime') UNION ALL SELECT datetime(time_point, '-' || ? || ' minutes') FROM time_series - WHERE time_point > datetime(?, 'localtime') + WHERE time_point > datetime('now', '-' || ? || ' hours', 'localtime') LIMIT 1000 ), grouped_metrics AS ( @@ -522,7 +514,7 @@ func (db *MetricsDB) GetRecentMetrics(hours float64) ([]HistoricalMetrics, error MAX(total_bytes) as period_bytes, AVG(avg_latency) as avg_latency FROM metrics_history - WHERE timestamp BETWEEN ? AND ? + WHERE timestamp >= datetime('now', '-' || ? || ' hours', 'localtime') GROUP BY group_time ) SELECT @@ -535,7 +527,7 @@ func (db *MetricsDB) GetRecentMetrics(hours float64) ([]HistoricalMetrics, error LEFT JOIN grouped_metrics m ON strftime(?, ts.time_point, 'localtime') = m.group_time ORDER BY timestamp DESC LIMIT 1000 - `, endTimeStr, timeStep, startTimeStr, interval, startTimeStr, endTimeStr, interval, interval) + `, timeStep, hours, interval, hours, interval, interval) if err != nil { return nil, err }