进一步优化日志

This commit is contained in:
wood chen 2024-10-22 19:32:22 +08:00
parent 4c674c8a9c
commit c8f3d0bf38
5 changed files with 92 additions and 61 deletions

View File

@ -1,11 +1,11 @@
package encoder package encoder
import ( import (
"fmt"
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
"sync" "sync"
"sync/atomic"
"time" "time"
"webp_server_go/config" "webp_server_go/config"
"webp_server_go/helper" "webp_server_go/helper"
@ -16,18 +16,30 @@ import (
func PrefetchImages() { func PrefetchImages() {
sTime := time.Now() sTime := time.Now()
log.Infof("Prefetching using %d cores", config.Jobs) log.Infof("开始预取图像,使用 %d 个核心", config.Jobs)
// 使用固定大小的工作池来限制并发 // 使用固定大小的工作池来限制并发
workerPool := make(chan struct{}, config.Jobs) workerPool := make(chan struct{}, config.Jobs)
var wg sync.WaitGroup var wg sync.WaitGroup
all := helper.FileCount(config.Config.ImgPath) all := helper.FileCount(config.Config.ImgPath)
bar := progressbar.Default(all, "Prefetching...") log.Infof("总共需要处理 %d 个文件", all)
bar := progressbar.Default(all, "预取进度")
var processedCount int32 // 用于计数处理的文件数
err := filepath.Walk(config.Config.ImgPath, err := filepath.Walk(config.Config.ImgPath,
func(picAbsPath string, info os.FileInfo, err error) error { func(picAbsPath string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() || !helper.CheckAllowedType(picAbsPath) { if err != nil {
log.Warnf("访问文件时出错: %s, 错误: %v", picAbsPath, err)
return nil
}
if info.IsDir() {
log.Debugf("跳过目录: %s", picAbsPath)
return nil
}
if !helper.CheckAllowedType(picAbsPath) {
log.Debugf("跳过不支持的文件类型: %s", picAbsPath)
return nil return nil
} }
@ -37,18 +49,25 @@ func PrefetchImages() {
workerPool <- struct{}{} // 获取工作槽 workerPool <- struct{}{} // 获取工作槽
defer func() { <-workerPool }() // 释放工作槽 defer func() { <-workerPool }() // 释放工作槽
log.Debugf("开始处理文件: %s", picAbsPath)
metadata := helper.ReadMetadata(picAbsPath, "", config.LocalHostAlias) metadata := helper.ReadMetadata(picAbsPath, "", config.LocalHostAlias)
avifAbsPath, webpAbsPath, jxlAbsPath := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias) avifAbsPath, webpAbsPath, jxlAbsPath := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias)
_ = os.MkdirAll(path.Dir(avifAbsPath), 0755) if err := os.MkdirAll(path.Dir(avifAbsPath), 0755); err != nil {
log.Warnf("创建目录失败: %s, 错误: %v", path.Dir(avifAbsPath), err)
log.Infof("Prefetching %s", picAbsPath) return
}
supported := map[string]bool{ supported := map[string]bool{
"raw": true, "webp": true, "avif": true, "jxl": true, "raw": true, "webp": true, "avif": true, "jxl": true,
} }
ConvertFilter(picAbsPath, jxlAbsPath, avifAbsPath, webpAbsPath, config.ExtraParams{Width: 0, Height: 0}, supported, nil) ConvertFilter(picAbsPath, jxlAbsPath, avifAbsPath, webpAbsPath, config.ExtraParams{Width: 0, Height: 0}, supported, nil)
atomic.AddInt32(&processedCount, 1)
log.Debugf("文件处理完成: %s (进度: %d/%d)", picAbsPath, atomic.LoadInt32(&processedCount), all)
_ = bar.Add(1) _ = bar.Add(1)
}() }()
@ -58,8 +77,9 @@ func PrefetchImages() {
wg.Wait() // 等待所有工作完成 wg.Wait() // 等待所有工作完成
if err != nil { if err != nil {
log.Errorln(err) log.Errorf("遍历目录时发生错误: %v", err)
} }
elapsed := time.Since(sTime) elapsed := time.Since(sTime)
_, _ = fmt.Fprintf(os.Stdout, "Prefetch complete in %s\n\n", elapsed) log.Infof("预取完成,共处理 %d 个文件,耗时 %s", atomic.LoadInt32(&processedCount), elapsed)
} }

View File

@ -17,14 +17,14 @@ func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error {
imgHeightWidthRatio := float32(imageHeight) / float32(imageWidth) imgHeightWidthRatio := float32(imageHeight) / float32(imageWidth)
// Here we have width, height and max_width, max_height //这里我们有宽度、高度和 max_width、max_height
// Both pairs cannot be used at the same time //两对不能同时使用
// max_height and max_width are used to make sure bigger images are resized to max_height and max_width //max_height 和 max_width 用于确保将更大的图像调整为 max_height 和 max_width
// e.g, 500x500px image with max_width=200,max_height=100 will be resized to 100x100 //例如max_width=200,max_height=100 的 500x500px 图像将调整为 100x100
// while smaller images are untouched //而较小的图像则保持不变
// If both are used, we will use width and height //如果两者都使用,我们将使用宽度和高度
if extraParams.MaxHeight > 0 && extraParams.MaxWidth > 0 { if extraParams.MaxHeight > 0 && extraParams.MaxWidth > 0 {
// If any of it exceeds // If any of it exceeds
@ -32,8 +32,8 @@ func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error {
// Check which dimension exceeds most // Check which dimension exceeds most
heightExceedRatio := float32(imageHeight) / float32(extraParams.MaxHeight) heightExceedRatio := float32(imageHeight) / float32(extraParams.MaxHeight)
widthExceedRatio := float32(imageWidth) / float32(extraParams.MaxWidth) widthExceedRatio := float32(imageWidth) / float32(extraParams.MaxWidth)
// If height exceeds more, like 500x500 -> 200x100 (2.5 < 5) // 如果高度超过更多,例如 500x500 -> 200x100 (2.5 < 5)
// Take max_height as new height ,resize and retain ratio // 以max_height为新高度调整大小并保留比例
if heightExceedRatio > widthExceedRatio { if heightExceedRatio > widthExceedRatio {
err := img.Thumbnail(int(float32(extraParams.MaxHeight)/imgHeightWidthRatio), extraParams.MaxHeight, 0) err := img.Thumbnail(int(float32(extraParams.MaxHeight)/imgHeightWidthRatio), extraParams.MaxHeight, 0)
if err != nil { if err != nil {
@ -104,65 +104,95 @@ func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error {
} }
func ResizeItself(raw, dest string, extraParams config.ExtraParams) { func ResizeItself(raw, dest string, extraParams config.ExtraParams) {
log.Infof("Resize %s itself to %s", raw, dest) log.Infof("开始调整图像大小: 源文件=%s, 目标文件=%s", raw, dest)
// we need to create dir first // 创建目标目录
var err = os.MkdirAll(path.Dir(dest), 0755) if err := os.MkdirAll(path.Dir(dest), 0755); err != nil {
if err != nil { log.Errorf("创建目标目录失败: %v", err)
log.Error(err.Error()) return
} }
// 加载图像
img, err := vips.LoadImageFromFile(raw, &vips.ImportParams{ img, err := vips.LoadImageFromFile(raw, &vips.ImportParams{
FailOnError: boolFalse, FailOnError: boolFalse,
}) })
if err != nil { if err != nil {
log.Warnf("Could not load %s: %s", raw, err) log.Warnf("加载图像失败: 文件=%s, 错误=%v", raw, err)
return return
} }
_ = resizeImage(img, extraParams) defer img.Close()
// 调整图像大小
if err := resizeImage(img, extraParams); err != nil {
log.Warnf("调整图像大小失败: %v", err)
return
}
// 移除元数据(如果配置要求)
if config.Config.StripMetadata { if config.Config.StripMetadata {
log.Debug("正在移除图像元数据")
img.RemoveMetadata() img.RemoveMetadata()
} }
buf, _, _ := img.ExportNative()
_ = os.WriteFile(dest, buf, 0600) // 导出图像
img.Close() buf, _, err := img.ExportNative()
if err != nil {
log.Errorf("导出图像失败: %v", err)
return
}
// 写入文件
if err := os.WriteFile(dest, buf, 0600); err != nil {
log.Errorf("写入目标文件失败: 文件=%s, 错误=%v", dest, err)
return
}
log.Infof("图像大小调整成功: 目标文件=%s", dest)
} }
// Pre-process image(auto rotate, resize, etc.) // Pre-process image(auto rotate, resize, etc.)
func preProcessImage(img *vips.ImageRef, imageType string, extraParams config.ExtraParams) error { func preProcessImage(img *vips.ImageRef, imageType string, extraParams config.ExtraParams) error {
// Check Width/Height and ignore image formats log.Debugf("开始预处理图像: 类型=%s, 宽度=%d, 高度=%d", imageType, img.Metadata().Width, img.Metadata().Height)
// 检查宽度/高度并忽略特定图像格式
switch imageType { switch imageType {
case "webp": case "webp":
if img.Metadata().Width > config.WebpMax || img.Metadata().Height > config.WebpMax { if img.Metadata().Width > config.WebpMax || img.Metadata().Height > config.WebpMax {
log.Warnf("WebP图像尺寸超限: 宽度=%d, 高度=%d, 最大限制=%d", img.Metadata().Width, img.Metadata().Height, config.WebpMax)
return errors.New("WebP图像太大") return errors.New("WebP图像太大")
} }
imageFormat := img.Format() if slices.Contains(webpIgnore, img.Format()) {
if slices.Contains(webpIgnore, imageFormat) { log.Infof("WebP编码器忽略图像类型: %s", img.Format())
// Return err to render original image
return errors.New("WebP 编码器:忽略图像类型") return errors.New("WebP 编码器:忽略图像类型")
} }
case "avif": case "avif":
if img.Metadata().Width > config.AvifMax || img.Metadata().Height > config.AvifMax { if img.Metadata().Width > config.AvifMax || img.Metadata().Height > config.AvifMax {
log.Warnf("AVIF图像尺寸超限: 宽度=%d, 高度=%d, 最大限制=%d", img.Metadata().Width, img.Metadata().Height, config.AvifMax)
return errors.New("AVIF图像太大") return errors.New("AVIF图像太大")
} }
imageFormat := img.Format() if slices.Contains(avifIgnore, img.Format()) {
if slices.Contains(avifIgnore, imageFormat) { log.Infof("AVIF编码器忽略图像类型: %s", img.Format())
// Return err to render original image
return errors.New("AVIF 编码器:忽略图像类型") return errors.New("AVIF 编码器:忽略图像类型")
} }
} }
// Auto rotate // 自动旋转
err := img.AutoRotate() if err := img.AutoRotate(); err != nil {
if err != nil { log.Errorf("图像自动旋转失败: %v", err)
return err return err
} }
log.Debug("图像自动旋转完成")
// 额外参数处理
if config.Config.EnableExtraParams { if config.Config.EnableExtraParams {
err = resizeImage(img, extraParams) log.Debug("开始应用额外图像处理参数")
if err != nil { if err := resizeImage(img, extraParams); err != nil {
log.Errorf("应用额外图像处理参数失败: %v", err)
return err return err
} }
log.Debug("额外图像处理参数应用完成")
} }
log.Debug("图像预处理完成")
return nil return nil
} }

View File

@ -42,7 +42,7 @@ func downloadFile(filepath string, url string) error {
if resp.StatusCode != fiber.StatusOK { if resp.StatusCode != fiber.StatusOK {
log.Errorf("获取远程图像时远程返回 %s", resp.Status) log.Errorf("获取远程图像时远程返回 %s", resp.Status)
return fmt.Errorf("unexpected status: %s", resp.Status) return fmt.Errorf("意外的状态: %s", resp.Status)
} }
// 创建目标文件 // 创建目标文件

View File

@ -108,25 +108,6 @@ func Convert(c *fiber.Ctx) error {
} }
} }
// 新增检查是否为WebP格式
// if strings.ToLower(path.Ext(filename)) == ".webp" {
// log.Infof("原始图像已经是WebP格式: %s", reqURI)
// var webpImagePath string
// if proxyMode {
// // 对于代理模式,确保文件已经被下载
// metadata = fetchRemoteImg(realRemoteAddr, targetHostName)
// webpImagePath = path.Join(config.Config.RemoteRawPath, targetHostName, metadata.Id)
// } else {
// webpImagePath = path.Join(config.Config.ImgPath, reqURI)
// }
// // 检查文件是否存在
// if helper.FileExists(webpImagePath) {
// // 直接返回原WebP图片
// return c.SendFile(webpImagePath)
// }
// }
if !helper.CheckAllowedType(filename) { if !helper.CheckAllowedType(filename) {
msg := "不允许的文件扩展名 " + filename msg := "不允许的文件扩展名 " + filename
log.Warn(msg) log.Warn(msg)
@ -220,7 +201,7 @@ func Convert(c *fiber.Ctx) error {
return c.SendFile(dest) return c.SendFile(dest)
} }
// Check the original image for existence, // 检查原始图像是否存在,
if !helper.ImageExists(rawImageAbs) { if !helper.ImageExists(rawImageAbs) {
helper.DeleteMetadata(reqURIwithQuery, targetHostName) helper.DeleteMetadata(reqURIwithQuery, targetHostName)
msg := "Image not found!" msg := "Image not found!"

View File

@ -180,7 +180,7 @@ func FindSmallestFiles(files []string) string {
for _, f := range files { for _, f := range files {
stat, err := os.Stat(f) stat, err := os.Stat(f)
if err != nil { if err != nil {
log.Warnf("%s not found on filesystem", f) log.Warnf("在文件系统上找不到 %s", f)
continue continue
} }
if stat.Size() < small || small == 0 { if stat.Size() < small || small == 0 {