diff --git a/README.md b/README.md index a6770d7..35f0ca3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,28 @@ # 自定义webp-server-go 1. 如果文件非图片类型, 302重定向到源文件 +2. 改进对比逻辑, 不仅仅依靠etag + 主要修改: + 增加了文件大小和最后修改时间的检查。 + 保留了原有的 ETag 对比逻辑。 + 使用多重条件判断文件是否需要更新。 + 主要好处: + + 提高了文件变更检测的准确性: + + 不仅依赖 ETag,还考虑了文件大小和修改时间。 + 能够捕获一些 ETag 未能反映的变化。 + 保持了效率: + + 优先使用缓存的 ETag,减少不必要的网络请求。 + 只有在确实需要时才进行文件下载。 + 增强了适应性: + + 即使某些信息(如 ETag)不可用,也能依靠其他指标进行判断。 + 改进了日志记录: + + 提供更详细的信息,说明为什么文件需要更新。 + 总的来说,这些修改在保持原有功能的基础上,提高了文件更新检测的准确性和鲁棒性,同时保持了系统的效率。
diff --git a/handler/remote.go b/handler/remote.go
index 64bd607..c4e1801 100644
--- a/handler/remote.go
+++ b/handler/remote.go
@@ -6,7 +6,9 @@ import (
"os"
"path"
"path/filepath"
+ "strconv"
"strings"
+ "time"
"webp_server_go/config"
"webp_server_go/helper"
@@ -77,67 +79,94 @@ func downloadFile(filepath string, url string) {
}
func fetchRemoteImg(url string, subdir string) config.MetaFile {
- // url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
- // How do we know if the remote img is changed? we're using hash(etag+length)
- var etag string
-
cacheKey := subdir + ":" + helper.HashString(url)
- if val, found := config.RemoteCache.Get(cacheKey); found {
- if etagVal, ok := val.(string); ok {
- log.Infof("Using cache for remote addr: %s", url)
- etag = etagVal
- } else {
- config.RemoteCache.Delete(cacheKey)
- }
- }
+ var metadata config.MetaFile
+ var etag string
+ var size int64
+ var lastModified time.Time
- if etag == "" {
+ if cachedETag, found := config.RemoteCache.Get(cacheKey); found {
+ etag = cachedETag.(string)
+ log.Infof("Using cached ETag for remote addr: %s", url)
+ } else {
log.Infof("Remote Addr is %s, pinging for info...", url)
- etag = pingURL(url)
+ etag, size, lastModified = pingURL(url)
if etag != "" {
config.RemoteCache.Set(cacheKey, etag, cache.DefaultExpiration)
}
}
- metadata := helper.ReadMetadata(url, etag, subdir)
+ metadata = helper.ReadMetadata(url, etag, subdir)
localRawImagePath := path.Join(config.Config.RemoteRawPath, subdir, metadata.Id)
localExhaustImagePath := path.Join(config.Config.ExhaustPath, subdir, metadata.Id)
- if !helper.ImageExists(localRawImagePath) || metadata.Checksum != helper.HashString(etag) {
- cleanProxyCache(localExhaustImagePath)
- if metadata.Checksum != helper.HashString(etag) {
- // remote file has changed
- log.Info("Remote file changed, updating metadata and fetching image source...")
- helper.DeleteMetadata(url, subdir)
- helper.WriteMetadata(url, etag, subdir)
+ needUpdate := false
+ if !helper.ImageExists(localRawImagePath) {
+ log.Info("Remote file not found in remote-raw, fetching...")
+ needUpdate = true
+ } else {
+ localFileInfo, err := os.Stat(localRawImagePath)
+ if err == nil {
+ if size > 0 && size != localFileInfo.Size() {
+ log.Info("File size changed, updating...")
+ needUpdate = true
+ } else if !lastModified.IsZero() && lastModified.After(localFileInfo.ModTime()) {
+ log.Info("Remote file is newer, updating...")
+ needUpdate = true
+ } else if metadata.Checksum != helper.HashString(etag) {
+ log.Info("ETag changed, updating...")
+ needUpdate = true
+ }
} else {
- // local file not exists
- log.Info("Remote file not found in remote-raw, re-fetching...")
+ log.Warnf("Error checking local file: %v", err)
+ needUpdate = true
}
- downloadFile(localRawImagePath, url)
}
+
+ if needUpdate {
+ cleanProxyCache(localExhaustImagePath)
+ helper.DeleteMetadata(url, subdir)
+ helper.WriteMetadata(url, etag, subdir)
+ downloadFile(localRawImagePath, url)
+ // 重新读取更新后的元数据
+ metadata = helper.ReadMetadata(url, etag, subdir)
+ }
+
return metadata
}
-func pingURL(url string) string {
- // this function will try to return identifiable info, currently include etag, content-length as string
- // anything goes wrong, will return ""
- var etag, length string
+func pingURL(url string) (string, int64, time.Time) {
+ var etag string
+ var size int64
+ var lastModified time.Time
+
resp, err := http.Head(url)
if err != nil {
- log.Errorln("Connection to remote error when pingUrl!")
- return ""
+ log.Errorf("Connection to remote error when pingUrl: %v", err)
+ return "", 0, time.Time{}
}
defer resp.Body.Close()
if resp.StatusCode == fiber.StatusOK {
- etag = resp.Header.Get("etag")
- length = resp.Header.Get("content-length")
- }
- if etag == "" {
- log.Info("Remote didn't return etag in header when getRemoteImageInfo, please check.")
- }
- return etag + length
-}
+ etag = resp.Header.Get("ETag")
+ sizeStr := resp.Header.Get("Content-Length")
+ size, _ = strconv.ParseInt(sizeStr, 10, 64)
+ lastModifiedStr := resp.Header.Get("Last-Modified")
+ lastModified, _ = time.Parse(time.RFC1123, lastModifiedStr)
+ if etag == "" {
+ log.Warn("Remote didn't return ETag in header, using Last-Modified if available")
+ etag = lastModifiedStr
+ }
+
+ if etag == "" && lastModified.IsZero() {
+ log.Warn("Neither ETag nor Last-Modified available, using Content-Length as fallback")
+ etag = sizeStr
+ }
+ } else {
+ log.Warnf("Unexpected status code: %d when pinging URL: %s", resp.StatusCode, url)
+ }
+
+ return etag, size, lastModified
+}