feat(docker, config, api): update docker-compose for logging, enhance app structure, and add system metrics display

- Updated docker-compose.yml to mount logs directory.
- Changed BASE_URL environment variable to point to the new API endpoint.
- Refactored main.go to implement a structured App type, improving initialization and graceful shutdown.
- Enhanced config management with JSON loading and environment variable support.
- Added monitoring capabilities in api_handler for logging request metrics.
- Introduced new metrics display in index.html with corresponding CSS styles for better visualization.
This commit is contained in:
wood chen 2024-11-30 23:23:58 +08:00
parent f31ff5caa7
commit e70ca4cf52
15 changed files with 691 additions and 51 deletions

17
config.json Normal file
View File

@ -0,0 +1,17 @@
{
"server": {
"port": ":5003",
"read_timeout": "30s",
"write_timeout": "30s",
"max_header_bytes": 1048576
},
"storage": {
"data_dir": "/root/data",
"stats_file": "/root/data/stats.json",
"log_file": "/var/log/random-api/server.log"
},
"api": {
"base_url": "",
"request_timeout": "10s"
}
}

View File

@ -1,20 +1,66 @@
package config package config
import ( import (
"encoding/json"
"math/rand" "math/rand"
"os"
"time" "time"
) )
const ( const (
Port = ":5003"
RequestTimeout = 10 * time.Second
EnvBaseURL = "BASE_URL" EnvBaseURL = "BASE_URL"
DefaultPort = ":5003"
RequestTimeout = 10 * time.Second
) )
type Config struct {
Server struct {
Port string `json:"port"`
ReadTimeout time.Duration `json:"read_timeout"`
WriteTimeout time.Duration `json:"write_timeout"`
MaxHeaderBytes int `json:"max_header_bytes"`
} `json:"server"`
Storage struct {
DataDir string `json:"data_dir"`
StatsFile string `json:"stats_file"`
LogFile string `json:"log_file"`
} `json:"storage"`
API struct {
BaseURL string `json:"base_url"`
RequestTimeout time.Duration `json:"request_timeout"`
} `json:"api"`
}
var ( var (
cfg Config
RNG *rand.Rand RNG *rand.Rand
) )
func Load(configFile string) error {
file, err := os.Open(configFile)
if err != nil {
return err
}
defer file.Close()
decoder := json.NewDecoder(file)
if err := decoder.Decode(&cfg); err != nil {
return err
}
if envBaseURL := os.Getenv(EnvBaseURL); envBaseURL != "" {
cfg.API.BaseURL = envBaseURL
}
return nil
}
func Get() *Config {
return &cfg
}
func InitRNG(r *rand.Rand) { func InitRNG(r *rand.Rand) {
RNG = r RNG = r
} }

17
data/config.json Normal file
View File

@ -0,0 +1,17 @@
{
"server": {
"port": ":5003",
"read_timeout": "30s",
"write_timeout": "30s",
"max_header_bytes": 1048576
},
"storage": {
"data_dir": "/root/data",
"stats_file": "/root/data/stats.json",
"log_file": "/var/log/random-api/server.log"
},
"api": {
"base_url": "",
"request_timeout": "10s"
}
}

View File

@ -6,8 +6,9 @@ services:
- "5003:5003" - "5003:5003"
volumes: volumes:
- ./public:/root/public - ./public:/root/public
- ./logs:/var/log/random-api
- ./data:/root/data - ./data:/root/data
environment: environment:
- TZ=Asia/Shanghai - TZ=Asia/Shanghai
- BASE_URL=https://example.com/csvfile - BASE_URL=https://github-file.czl.net/random-api.czl.net
restart: unless-stopped restart: unless-stopped

View File

@ -6,6 +6,7 @@ import (
"log" "log"
"net/http" "net/http"
"net/url" "net/url"
"random-api-go/monitoring"
"random-api-go/services" "random-api-go/services"
"random-api-go/stats" "random-api-go/stats"
"random-api-go/utils" "random-api-go/utils"
@ -31,9 +32,7 @@ func HandleAPIRequest(w http.ResponseWriter, r *http.Request) {
sourceInfo := "direct" sourceInfo := "direct"
if referer != "" { if referer != "" {
if parsedURL, err := url.Parse(referer); err == nil { if parsedURL, err := url.Parse(referer); err == nil {
// 包含主机名和路径
sourceInfo = parsedURL.Host + parsedURL.Path sourceInfo = parsedURL.Host + parsedURL.Path
// 如果有查询参数,也可以加上
if parsedURL.RawQuery != "" { if parsedURL.RawQuery != "" {
sourceInfo += "?" + parsedURL.RawQuery sourceInfo += "?" + parsedURL.RawQuery
} }
@ -44,6 +43,15 @@ func HandleAPIRequest(w http.ResponseWriter, r *http.Request) {
pathSegments := strings.Split(path, "/") pathSegments := strings.Split(path, "/")
if len(pathSegments) < 2 { if len(pathSegments) < 2 {
monitoring.LogRequest(monitoring.RequestLog{
Time: time.Now(),
Path: r.URL.Path,
Method: r.Method,
StatusCode: http.StatusNotFound,
Latency: float64(time.Since(start).Microseconds()) / 1000,
IP: realIP,
Referer: sourceInfo,
})
http.NotFound(w, r) http.NotFound(w, r)
return return
} }
@ -80,6 +88,17 @@ func HandleAPIRequest(w http.ResponseWriter, r *http.Request) {
duration := time.Since(start) duration := time.Since(start)
// 记录请求日志
monitoring.LogRequest(monitoring.RequestLog{
Time: time.Now(),
Path: r.URL.Path,
Method: r.Method,
StatusCode: http.StatusFound,
Latency: float64(duration.Microseconds()) / 1000, // 转换为毫秒
IP: realIP,
Referer: sourceInfo,
})
log.Printf(" %-12s | %-15s | %-6s | %-20s | %-20s | %-50s", log.Printf(" %-12s | %-15s | %-6s | %-20s | %-20s | %-50s",
duration, // 持续时间 duration, // 持续时间
realIP, // 真实IP realIP, // 真实IP

39
handlers/handlers.go Normal file
View File

@ -0,0 +1,39 @@
package handlers
import (
"net/http"
"random-api-go/router"
"random-api-go/stats"
)
type Router interface {
HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request))
}
type Handlers struct {
Stats *stats.StatsManager
}
func (h *Handlers) HandleAPIRequest(w http.ResponseWriter, r *http.Request) {
HandleAPIRequest(w, r)
}
func (h *Handlers) HandleStats(w http.ResponseWriter, r *http.Request) {
HandleStats(w, r)
}
func (h *Handlers) HandleURLStats(w http.ResponseWriter, r *http.Request) {
HandleURLStats(w, r)
}
func (h *Handlers) HandleMetrics(w http.ResponseWriter, r *http.Request) {
HandleMetrics(w, r)
}
func (h *Handlers) Setup(r *router.Router) {
r.HandleFunc("/pic/", h.HandleAPIRequest)
r.HandleFunc("/video/", h.HandleAPIRequest)
r.HandleFunc("/stats", h.HandleStats)
r.HandleFunc("/urlstats", h.HandleURLStats)
r.HandleFunc("/metrics", h.HandleMetrics)
}

View File

@ -0,0 +1,14 @@
package handlers
import (
"encoding/json"
"net/http"
"random-api-go/monitoring"
)
func HandleMetrics(w http.ResponseWriter, r *http.Request) {
metrics := monitoring.CollectMetrics()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(metrics)
}

134
main.go
View File

@ -1,75 +1,117 @@
package main package main
import ( import (
"context"
"fmt"
"log" "log"
"math/rand"
"net/http" "net/http"
"os" "os"
"os/signal" "os/signal"
"random-api-go/config" "random-api-go/config"
"random-api-go/handlers" "random-api-go/handlers"
"random-api-go/logging" "random-api-go/logging"
"random-api-go/router"
"random-api-go/services" "random-api-go/services"
"random-api-go/stats" "random-api-go/stats"
"syscall" "syscall"
"time" "time"
) )
func init() { type App struct {
if err := os.MkdirAll("data", 0755); err != nil { server *http.Server
log.Fatal("Failed to create data directory:", err) router *router.Router
Stats *stats.StatsManager
}
func NewApp() *App {
return &App{
router: router.New(),
} }
} }
func main() { func (a *App) Initialize() error {
source := rand.NewSource(time.Now().UnixNano()) // 创建必要的目录
config.InitRNG(rand.New(source)) if err := os.MkdirAll(config.Get().Storage.DataDir, 0755); err != nil {
return fmt.Errorf("failed to create data directory: %w", err)
logging.SetupLogging()
statsManager := stats.NewStatsManager("data/stats.json")
// 设置优雅关闭
setupGracefulShutdown(statsManager)
// 初始化handlers
if err := handlers.InitializeHandlers(statsManager); err != nil {
log.Fatal("Failed to initialize handlers:", err)
} }
// 初始化加载所有CSV内容 // 初始化配置
if err := config.Load("/root/data/config.json"); err != nil {
return err
}
// 初始化日志
logging.SetupLogging()
// 初始化统计管理器
a.Stats = stats.NewStatsManager(config.Get().Storage.StatsFile)
// 初始化服务
if err := services.InitializeCSVService(); err != nil { if err := services.InitializeCSVService(); err != nil {
log.Fatal("Failed to initialize CSV Service:", err) return err
}
// 创建 handlers
handlers := &handlers.Handlers{
Stats: a.Stats,
} }
// 设置路由 // 设置路由
setupRoutes() a.router.Setup(handlers)
log.Printf("Server starting on %s...\n", config.Port) // 创建 HTTP 服务器
if err := http.ListenAndServe(config.Port, nil); err != nil { cfg := config.Get().Server
a.server = &http.Server{
Addr: cfg.Port,
Handler: a.router,
ReadTimeout: cfg.ReadTimeout,
WriteTimeout: cfg.WriteTimeout,
MaxHeaderBytes: cfg.MaxHeaderBytes,
}
return nil
}
func (a *App) Run() error {
// 启动服务器
go func() {
log.Printf("Server starting on %s...\n", a.server.Addr)
if err := a.server.ListenAndServe(); err != http.ErrServerClosed {
log.Fatalf("Server failed: %v", err)
}
}()
// 优雅关闭
return a.gracefulShutdown()
}
func (a *App) gracefulShutdown() error {
quit := make(chan os.Signal, 1)
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
<-quit
log.Println("Server is shutting down...")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
a.Stats.Shutdown()
if err := a.server.Shutdown(ctx); err != nil {
return err
}
log.Println("Server shutdown completed")
return nil
}
func main() {
app := NewApp()
if err := app.Initialize(); err != nil {
log.Fatal(err)
}
if err := app.Run(); err != nil {
log.Fatal(err) log.Fatal(err)
} }
} }
func setupGracefulShutdown(statsManager *stats.StatsManager) {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
go func() {
<-c
log.Println("Server is shutting down...")
statsManager.Shutdown()
log.Println("Stats manager shutdown completed")
os.Exit(0)
}()
}
func setupRoutes() {
fs := http.FileServer(http.Dir("./public"))
http.Handle("/", fs)
http.HandleFunc("/pic/", handlers.HandleAPIRequest)
http.HandleFunc("/video/", handlers.HandleAPIRequest)
http.HandleFunc("/stats", handlers.HandleStats)
// 添加URL统计接口
http.HandleFunc("/urlstats", handlers.HandleURLStats)
}

45
middleware/metrics.go Normal file
View File

@ -0,0 +1,45 @@
package middleware
import (
"net/http"
"random-api-go/monitoring"
"random-api-go/utils"
"time"
)
func MetricsMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
// 创建自定义的ResponseWriter来捕获状态码
rw := &responseWriter{
ResponseWriter: w,
statusCode: http.StatusOK,
}
// 处理请求
next.ServeHTTP(rw, r)
// 记录请求数据
duration := time.Since(start)
monitoring.LogRequest(monitoring.RequestLog{
Time: time.Now(),
Path: r.URL.Path,
Method: r.Method,
StatusCode: rw.statusCode,
Latency: float64(duration.Microseconds()) / 1000,
IP: utils.GetRealIP(r),
Referer: r.Referer(),
})
})
}
type responseWriter struct {
http.ResponseWriter
statusCode int
}
func (rw *responseWriter) WriteHeader(statusCode int) {
rw.statusCode = statusCode
rw.ResponseWriter.WriteHeader(statusCode)
}

25
middleware/middleware.go Normal file
View File

@ -0,0 +1,25 @@
package middleware
import "net/http"
// Chain 用于组合多个中间件
func Chain(middlewares ...func(http.Handler) http.Handler) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
for i := len(middlewares) - 1; i >= 0; i-- {
next = middlewares[i](next)
}
return next
}
}
// Recovery 中间件用于捕获 panic
func Recovery(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if err := recover(); err != nil {
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
}
}()
next.ServeHTTP(w, r)
})
}

110
monitoring/metrics.go Normal file
View File

@ -0,0 +1,110 @@
package monitoring
import (
"runtime"
"sync"
"time"
)
type SystemMetrics struct {
// 基础指标
Uptime time.Duration `json:"uptime"`
StartTime time.Time `json:"start_time"`
// 系统指标
NumCPU int `json:"num_cpu"`
NumGoroutine int `json:"num_goroutine"`
MemoryStats struct {
Alloc uint64 `json:"alloc"`
TotalAlloc uint64 `json:"total_alloc"`
Sys uint64 `json:"sys"`
HeapAlloc uint64 `json:"heap_alloc"`
HeapSys uint64 `json:"heap_sys"`
} `json:"memory_stats"`
// 性能指标
RequestCount int64 `json:"request_count"`
AverageLatency float64 `json:"average_latency"`
// 流量统计
TotalBytesIn int64 `json:"total_bytes_in"`
TotalBytesOut int64 `json:"total_bytes_out"`
// 状态码统计
StatusCodes map[int]int64 `json:"status_codes"`
// 路径延迟统计
PathLatencies map[string]float64 `json:"path_latencies"`
// 最近请求
RecentRequests []RequestLog `json:"recent_requests"`
// 热门引用来源
TopReferers map[string]int64 `json:"top_referers"`
}
type RequestLog struct {
Time time.Time `json:"time"`
Path string `json:"path"`
Method string `json:"method"`
StatusCode int `json:"status_code"`
Latency float64 `json:"latency"`
IP string `json:"ip"`
Referer string `json:"referer"`
}
var (
metrics SystemMetrics
mu sync.RWMutex
startTime = time.Now()
)
func init() {
metrics.StatusCodes = make(map[int]int64)
metrics.PathLatencies = make(map[string]float64)
metrics.TopReferers = make(map[string]int64)
metrics.RecentRequests = make([]RequestLog, 0, 100)
}
func CollectMetrics() *SystemMetrics {
mu.Lock()
defer mu.Unlock()
var m runtime.MemStats
runtime.ReadMemStats(&m)
metrics.Uptime = time.Since(startTime)
metrics.StartTime = startTime
metrics.NumCPU = runtime.NumCPU()
metrics.NumGoroutine = runtime.NumGoroutine()
metrics.MemoryStats.Alloc = m.Alloc
metrics.MemoryStats.TotalAlloc = m.TotalAlloc
metrics.MemoryStats.Sys = m.Sys
metrics.MemoryStats.HeapAlloc = m.HeapAlloc
metrics.MemoryStats.HeapSys = m.HeapSys
return &metrics
}
func LogRequest(log RequestLog) {
mu.Lock()
defer mu.Unlock()
metrics.RequestCount++
metrics.StatusCodes[log.StatusCode]++
metrics.TopReferers[log.Referer]++
// 更新路径延迟
if existing, ok := metrics.PathLatencies[log.Path]; ok {
metrics.PathLatencies[log.Path] = (existing + log.Latency) / 2
} else {
metrics.PathLatencies[log.Path] = log.Latency
}
// 保存最近请求记录
metrics.RecentRequests = append(metrics.RecentRequests, log)
if len(metrics.RecentRequests) > 100 {
metrics.RecentRequests = metrics.RecentRequests[1:]
}
}

View File

@ -173,3 +173,95 @@ table {
transform: translate(-50%, -20px); transform: translate(-50%, -20px);
} }
} }
/* 系统监控样式 */
.metrics-container {
background: rgba(0, 0, 0, 0.2);
border-radius: 8px;
padding: 20px;
margin: 20px 0;
}
.metrics-section {
margin-bottom: 20px;
}
.metrics-section h3 {
color: #2196f3;
margin-bottom: 15px;
font-size: 1.1em;
border-bottom: 1px solid rgba(33, 150, 243, 0.2);
padding-bottom: 5px;
}
.metrics-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 10px;
}
.metric-item {
background: rgba(255, 255, 255, 0.1);
padding: 12px;
border-radius: 6px;
font-size: 0.9em;
}
.status-codes {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(120px, 1fr));
gap: 10px;
}
.status-code-item {
background: rgba(255, 255, 255, 0.1);
padding: 8px 12px;
border-radius: 6px;
display: flex;
justify-content: space-between;
align-items: center;
}
.recent-requests table {
width: 100%;
border-collapse: collapse;
font-size: 0.9em;
}
.recent-requests th,
.recent-requests td {
padding: 8px;
text-align: left;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.recent-requests th {
color: #2196f3;
font-weight: 500;
}
.top-referers {
display: grid;
gap: 8px;
}
.referer-item {
background: rgba(255, 255, 255, 0.1);
padding: 8px 12px;
border-radius: 6px;
display: flex;
justify-content: space-between;
align-items: center;
}
.referer {
max-width: 70%;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.count {
color: #2196f3;
font-weight: 500;
}

View File

@ -239,6 +239,137 @@
// 定期更新统计数据 // 定期更新统计数据
setInterval(loadStats, 5 * 1000); setInterval(loadStats, 5 * 1000);
async function loadMetrics() {
try {
const response = await fetch('/metrics');
const metrics = await response.json();
updateMetricsDisplay(metrics);
} catch (error) {
console.error('Error loading metrics:', error);
}
}
function updateMetricsDisplay(metrics) {
const metricsHtml = `
<div class="metrics-container">
<div class="metrics-section">
<h3>基础指标</h3>
<div class="metrics-grid">
<div class="metric-item">运行时间:${formatDuration(metrics.uptime)}</div>
<div class="metric-item">启动时间:${new Date(metrics.start_time).toLocaleString()}</div>
</div>
</div>
<div class="metrics-section">
<h3>系统指标</h3>
<div class="metrics-grid">
<div class="metric-item">CPU核心数${metrics.num_cpu}</div>
<div class="metric-item">Goroutine数${metrics.num_goroutine}</div>
<div class="metric-item">内存使用:${formatBytes(metrics.memory_stats.heap_alloc)}</div>
<div class="metric-item">系统内存:${formatBytes(metrics.memory_stats.heap_sys)}</div>
</div>
</div>
<div class="metrics-section">
<h3>性能指标</h3>
<div class="metrics-grid">
<div class="metric-item">总请求数:${metrics.request_count}</div>
<div class="metric-item">平均延迟:${metrics.average_latency.toFixed(2)}ms</div>
</div>
</div>
<div class="metrics-section">
<h3>状态码统计</h3>
<div class="status-codes">
${Object.entries(metrics.status_codes)
.map(([code, count]) => `
<div class="status-code-item">
<span class="code">${code}</span>
<span class="count">${count}</span>
</div>
`).join('')}
</div>
</div>
<div class="metrics-section">
<h3>最近请求</h3>
<div class="recent-requests">
<table>
<thead>
<tr>
<th>时间</th>
<th>路径</th>
<th>方法</th>
<th>状态</th>
<th>延迟</th>
</tr>
</thead>
<tbody>
${metrics.recent_requests.slice(0, 10).map(req => `
<tr>
<td>${new Date(req.time).toLocaleString()}</td>
<td>${req.path}</td>
<td>${req.method}</td>
<td>${req.status_code}</td>
<td>${req.latency.toFixed(2)}ms</td>
</tr>
`).join('')}
</tbody>
</table>
</div>
</div>
<div class="metrics-section">
<h3>热门引用来源</h3>
<div class="top-referers">
${Object.entries(metrics.top_referers)
.sort(([,a], [,b]) => b - a)
.slice(0, 10)
.map(([referer, count]) => `
<div class="referer-item">
<span class="referer">${referer || '直接访问'}</span>
<span class="count">${count}</span>
</div>
`).join('')}
</div>
</div>
</div>
`;
const metricsElement = document.getElementById('system-metrics');
if (metricsElement) {
metricsElement.innerHTML = metricsHtml;
}
}
function formatBytes(bytes) {
const units = ['B', 'KB', 'MB', 'GB'];
let size = bytes;
let unitIndex = 0;
while (size >= 1024 && unitIndex < units.length - 1) {
size /= 1024;
unitIndex++;
}
return `${size.toFixed(2)} ${units[unitIndex]}`;
}
function formatDuration(ns) {
const duration = ns / 1e9; // 转换为秒
const days = Math.floor(duration / 86400);
const hours = Math.floor((duration % 86400) / 3600);
const minutes = Math.floor((duration % 3600) / 60);
const seconds = Math.floor(duration % 60);
return `${days}天 ${hours}时 ${minutes}分 ${seconds}秒`;
}
// 定期更新监控数据
setInterval(loadMetrics, 5000);
// 初始加载
document.addEventListener('DOMContentLoaded', () => {
loadMetrics();
});
</script> </script>
</body> </body>

View File

@ -5,6 +5,8 @@
<div id="stats-detail"></div> <div id="stats-detail"></div>
</div> </div>
<div id="system-metrics"></div>
--- ---

40
router/router.go Normal file
View File

@ -0,0 +1,40 @@
package router
import (
"net/http"
"random-api-go/middleware"
)
type Router struct {
mux *http.ServeMux
}
type Handler interface {
Setup(r *Router)
}
func New() *Router {
return &Router{
mux: http.NewServeMux(),
}
}
func (r *Router) Setup(h Handler) {
// 静态文件服务
fileServer := http.FileServer(http.Dir("./public"))
r.mux.Handle("/", middleware.Chain(
middleware.Recovery,
middleware.MetricsMiddleware,
)(fileServer))
// 设置API路由
h.Setup(r)
}
func (r *Router) HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) {
r.mux.HandleFunc(pattern, handler)
}
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
r.mux.ServeHTTP(w, req)
}