diff --git a/.gitignore b/.gitignore index 51782b8..224bc68 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ remote-raw/ coverage.txt .DS_Store /webp_server_go +/metadata/* diff --git a/config/config.go b/config/config.go index ecc0087..69a5299 100644 --- a/config/config.go +++ b/config/config.go @@ -3,7 +3,6 @@ package config import ( "encoding/json" "flag" - "fmt" "os" "regexp" "runtime" @@ -59,10 +58,18 @@ var ( ProxyMode bool Prefetch bool Config jsonFile - Version = "0.9.3" + Version = "0.9.4" WriteLock = cache.New(5*time.Minute, 10*time.Minute) ) +const Metadata = "metadata" + +type MetaFile struct { + Id string `json:"id"` // hash of below path️, also json file name id.webp + Path string `json:"path"` // local: path with width and height, proxy: full url + Checksum string `json:"checksum"` // hash of original file or hash(etag). Use this to identify changes +} + type jsonFile struct { Host string `json:"HOST"` Port string `json:"PORT"` @@ -81,7 +88,6 @@ func init() { flag.BoolVar(&DumpConfig, "dump-config", false, "Print sample config.json") flag.BoolVar(&DumpSystemd, "dump-systemd", false, "Print sample systemd service file.") flag.BoolVar(&ShowVersion, "V", false, "Show version information.") - } func LoadConfig() { @@ -100,11 +106,6 @@ type ExtraParams struct { Height int // in px } -// String : convert ExtraParams to string, used to generate cache path -func (e *ExtraParams) String() string { - return fmt.Sprintf("_width=%d&height=%d", e.Width, e.Height) -} - func switchProxyMode() { matched, _ := regexp.MatchString(`^https?://`, Config.ImgPath) if matched { diff --git a/config/config_test.go b/config/config_test.go index 53b1a77..dbb82eb 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,8 +1,9 @@ package config import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) func TestMain(m *testing.M) { @@ -21,15 +22,6 @@ func TestLoadConfig(t *testing.T) { assert.Equal(t, Config.ExhaustPath, "./exhaust") } -func TestExtraParamsString(t *testing.T) { - param := ExtraParams{ - Width: 100, - Height: 100, - } - assert.Equal(t, param.String(), "_width=100&height=100") - -} - func TestSwitchProxyMode(t *testing.T) { switchProxyMode() assert.False(t, ProxyMode) diff --git a/encoder/encoder.go b/encoder/encoder.go index 3006777..fcae293 100644 --- a/encoder/encoder.go +++ b/encoder/encoder.go @@ -4,9 +4,7 @@ import ( "errors" "os" "path" - "path/filepath" "runtime" - "strings" "sync" "webp_server_go/config" "webp_server_go/helper" @@ -32,7 +30,7 @@ func init() { func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error { imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width) if extraParams.Width > 0 && extraParams.Height > 0 { - err := img.Thumbnail(extraParams.Width, extraParams.Height, 0) + err := img.Thumbnail(extraParams.Width, extraParams.Height, vips.InterestingAttention) if err != nil { return err } @@ -85,27 +83,20 @@ func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParam } } +func ResizeItself(raw, dest string, extraParams config.ExtraParams) { + log.Infof("Resize %s itself to %s", raw, dest) + img, _ := vips.LoadImageFromFile(raw, &vips.ImportParams{ + FailOnError: boolFalse, + }) + _ = resizeImage(img, extraParams) + buf, _, _ := img.ExportNative() + _ = os.WriteFile(dest, buf, 0600) + img.Close() +} + func convertImage(raw, optimized, imageType string, extraParams config.ExtraParams) error { - // we don't have /path/to/tsuki.jpg.1582558990.webp, maybe we have /path/to/tsuki.jpg.1082008000.webp - // delete the old converted pic and convert a new one. - // optimized: /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - // we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.imageType - // If contain extraParams like tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200 - - s := strings.Split(path.Base(optimized), ".") - pattern := path.Join(path.Dir(optimized), s[0]+"."+s[1]+".*."+s[len(s)-1]) - - matches, err := filepath.Glob(pattern) - if err != nil { - log.Error(err.Error()) - } else { - for _, p := range matches { - _ = os.Remove(p) - } - } - // we need to create dir first - err = os.MkdirAll(path.Dir(optimized), 0755) + var err = os.MkdirAll(path.Dir(optimized), 0755) if err != nil { log.Error(err.Error()) } diff --git a/encoder/prefetch.go b/encoder/prefetch.go index 01d04f9..dfb4256 100644 --- a/encoder/prefetch.go +++ b/encoder/prefetch.go @@ -5,7 +5,6 @@ import ( "os" "path" "path/filepath" - "strings" "time" "webp_server_go/config" "webp_server_go/helper" @@ -35,8 +34,8 @@ func PrefetchImages() { return nil } // RawImagePath string, ImgFilename string, reqURI string - proposedURI := strings.Replace(picAbsPath, config.Config.ImgPath, "", 1) - avif, webp := helper.GenOptimizedAbsPath(picAbsPath, proposedURI, config.ExtraParams{Width: 0, Height: 0}) + metadata := helper.ReadMetadata(picAbsPath, "") + avif, webp := helper.GenOptimizedAbsPath(metadata) _ = os.MkdirAll(path.Dir(avif), 0755) log.Infof("Prefetching %s", picAbsPath) go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan) diff --git a/handler/remote.go b/handler/remote.go index 55669e8..66f9665 100644 --- a/handler/remote.go +++ b/handler/remote.go @@ -75,28 +75,21 @@ func downloadFile(filepath string, url string) { } -func fetchRemoteImg(url string) string { +func fetchRemoteImg(url string) config.MetaFile { // url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 - // How do we know if the remote img is changed? we're using hash(url+etag) as key. - // if this exists in local system, means the remote img is not changed, we can use it directly. - // otherwise, we need to fetch it from remote and store it in local system. + // How do we know if the remote img is changed? we're using hash(etag+length) log.Infof("Remote Addr is %s, pinging for info...", url) - // identifiable is etag + length - identifiable := pingURL(url) - // For store the remote raw image, /home/webp_server/remote-raw/3a42ab801f669d64-b8f999ab5acd69d03f5e904b1b84eb79210536 - // Which 3a42ab801f669d64 is hash(url), b8f999ab5acd69d03f5e904b1b84eb79 is etag and 210536 is length - localRawImagePath := path.Join(config.RemoteRaw, helper.HashString(url)+"-"+identifiable) + etag := pingURL(url) + metadata := helper.ReadMetadata(url, etag) + localRawImagePath := path.Join(config.RemoteRaw, metadata.Id) - if helper.ImageExists(localRawImagePath) { - return localRawImagePath - } else { - // Temporary store of remote file. - cleanProxyCache(config.RemoteRaw + helper.HashString(url) + "*") + if !helper.ImageExists(localRawImagePath) || metadata.Checksum != helper.HashString(etag) { + // remote file has changed or local file not exists log.Info("Remote file not found in remote-raw, re-fetching...") + cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id+"*")) downloadFile(localRawImagePath, url) - return localRawImagePath } - + return metadata } func pingURL(url string) string { @@ -117,7 +110,5 @@ func pingURL(url string) string { if etag == "" { log.Info("Remote didn't return etag in header when getRemoteImageInfo, please check.") } - // Remove " from etag - etag = strings.ReplaceAll(etag, "\"", "") return etag + length } diff --git a/handler/router.go b/handler/router.go index 5793f80..71d1fd9 100644 --- a/handler/router.go +++ b/handler/router.go @@ -3,7 +3,6 @@ package handler import ( "net/http" "net/url" - "os" "webp_server_go/config" "webp_server_go/encoder" "webp_server_go/helper" @@ -40,31 +39,42 @@ func Convert(c *fiber.Ctx) error { reqURI = path.Clean(reqURI) reqURIwithQuery = path.Clean(reqURIwithQuery) - WidthInt, err := strconv.Atoi(c.Query("width")) - if err != nil { - WidthInt = 0 - } - HeightInt, err := strconv.Atoi(c.Query("height")) - if err != nil { - HeightInt = 0 - } + width, _ := strconv.Atoi(c.Query("width")) + height, _ := strconv.Atoi(c.Query("height")) + var extraParams = config.ExtraParams{ - Width: WidthInt, - Height: HeightInt, + Width: width, + Height: height, } var rawImageAbs string + var metadata = config.MetaFile{} if config.ProxyMode { // this is proxyMode, we'll have to use this url to download and save it to local path, which also gives us rawImageAbs // https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 - rawImageAbs = fetchRemoteImg(config.Config.ImgPath + reqURIwithQuery) - + metadata = fetchRemoteImg(config.Config.ImgPath + reqURIwithQuery) + rawImageAbs = path.Join(config.RemoteRaw, metadata.Id) } else { // not proxyMode, we'll use local path - rawImageAbs = path.Join(config.Config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg + metadata = helper.ReadMetadata(reqURIwithQuery, "") + rawImageAbs = path.Join(config.Config.ImgPath, reqURI) + // detect if source file has changed + if metadata.Checksum != helper.HashFile(rawImageAbs) { + log.Info("Source file has changed, re-encoding...") + helper.WriteMetadata(reqURIwithQuery, "") + cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id)) + } } goodFormat := helper.GuessSupportedFormat(&c.Request().Header) + // resize itself and return if only one format(raw) is supported + if len(goodFormat) == 1 { + dest := path.Join(config.Config.ExhaustPath, metadata.Id) + if !helper.ImageExists(dest) { + encoder.ResizeItself(rawImageAbs, dest, extraParams) + } + return c.SendFile(dest) + } // Check the original image for existence, if !helper.ImageExists(rawImageAbs) { @@ -75,11 +85,7 @@ func Convert(c *fiber.Ctx) error { return nil } - // generate with timestamp to make sure files are update-to-date - // If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - // If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0 - // If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0 - avifAbs, webpAbs := helper.GenOptimizedAbsPath(rawImageAbs, reqURI, extraParams) + avifAbs, webpAbs := helper.GenOptimizedAbsPath(metadata) encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil) var availableFiles = []string{rawImageAbs} @@ -93,9 +99,7 @@ func Convert(c *fiber.Ctx) error { } finalFilename := helper.FindSmallestFiles(availableFiles) - - buf, _ := os.ReadFile(finalFilename) - contentType := helper.GetFileContentType(buf) + contentType := helper.GetFileContentType(finalFilename) c.Set("Content-Type", contentType) c.Set("X-Compression-Rate", helper.GetCompressionRate(rawImageAbs, finalFilename)) diff --git a/helper/helper.go b/helper/helper.go index 607d85a..32cddc7 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -1,7 +1,6 @@ package helper import ( - "bytes" "fmt" "os" "path" @@ -10,34 +9,25 @@ import ( "time" "webp_server_go/config" - "github.com/cespare/xxhash" "github.com/h2non/filetype" + + "github.com/cespare/xxhash" "github.com/valyala/fasthttp" log "github.com/sirupsen/logrus" ) -var _ = filetype.AddMatcher(filetype.NewType("avif", "image/avif"), avifMatcher) - -func avifMatcher(buf []byte) bool { - // use hexdump on macOS to see the magic number - // 0000001c 66747970 61766966 00000000 61766966 6d696631 6d696166 - magicHeader := []byte{ - 0x0, 0x0, 0x0, 0x1c, - 0x66, 0x74, 0x79, 0x70, - 0x61, 0x76, 0x69, 0x66, - 0x0, 0x0, 0x0, 0x0, - 0x61, 0x76, 0x69, 0x66, - 0x6d, 0x69, 0x66, 0x31, - 0x6d, 0x69, 0x61, 0x66, +func GetFileContentType(filename string) string { + if strings.HasSuffix(filename, ".webp") { + return "image/webp" + } else if strings.HasSuffix(filename, ".avif") { + return "image/avif" + } else { + // raw image, need to use filetype to determine + buf, _ := os.ReadFile(filename) + kind, _ := filetype.Match(buf) + return kind.MIME.Value } - - return len(buf) > 1 && bytes.Equal(buf[:28], magicHeader) || strings.Contains(string(buf), "ftypavif") -} - -func GetFileContentType(buffer []byte) string { - kind, _ := filetype.Match(buffer) - return kind.MIME.Value } func FileCount(dir string) int64 { @@ -99,37 +89,11 @@ func CheckAllowedType(imgFilename string) bool { return false } -func GenOptimizedAbsPath(rawImagePath, reqURI string, extraParams config.ExtraParams) (string, string) { - // imageName is not needed, we can use reqURI - // get file mod time - var ( - imageName = path.Base(reqURI) - exhaustPath = config.Config.ExhaustPath - ) - STAT, err := os.Stat(rawImagePath) - if err != nil { - log.Error(err.Error()) - return "", "" - } - ModifiedTime := STAT.ModTime().Unix() - // TODO: just hash it? - // webpFilename: abc.jpg.png -> abc.jpg.png.1582558990.webp - webpFilename := fmt.Sprintf("%s.%d.webp", imageName, ModifiedTime) - // avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif - avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime) - - // If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - // If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0 - // If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0 - if config.Config.EnableExtraParams { - webpFilename = webpFilename + extraParams.String() - avifFilename = avifFilename + extraParams.String() - } - - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - // Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp - webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename)) - avifAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), avifFilename)) +func GenOptimizedAbsPath(metadata config.MetaFile) (string, string) { + webpFilename := fmt.Sprintf("%s.webp", metadata.Id) + avifFilename := fmt.Sprintf("%s.avif", metadata.Id) + webpAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, webpFilename)) + avifAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, avifFilename)) return avifAbsolutePath, webpAbsolutePath } @@ -214,3 +178,8 @@ func HashString(uri string) string { // xxhash supports cross compile return fmt.Sprintf("%x", xxhash.Sum64String(uri)) } + +func HashFile(filepath string) string { + buf, _ := os.ReadFile(filepath) + return fmt.Sprintf("%x", xxhash.Sum64(buf)) +} diff --git a/helper/helper_test.go b/helper/helper_test.go index 2a5306f..1469918 100644 --- a/helper/helper_test.go +++ b/helper/helper_test.go @@ -18,7 +18,7 @@ func TestMain(m *testing.M) { func TestFileCount(t *testing.T) { // test helper dir count := FileCount("./") - assert.Equal(t, int64(3), count) + assert.Equal(t, int64(4), count) } func TestImageExists(t *testing.T) { @@ -26,10 +26,6 @@ func TestImageExists(t *testing.T) { assert.False(t, ImageExists("dgyuaikdsa")) }) - t.Run("file size incorrect", func(t *testing.T) { - assert.False(t, ImageExists("test.txt")) - }) - // TODO: how to test lock? t.Run("test dir", func(t *testing.T) { @@ -43,7 +39,7 @@ func TestImageExists(t *testing.T) { func TestCheckAllowedType(t *testing.T) { t.Run("not allowed type", func(t *testing.T) { - assert.False(t, CheckAllowedType("test.txt")) + assert.False(t, CheckAllowedType("./helper_test.go")) }) t.Run("allowed type", func(t *testing.T) { diff --git a/helper/metadata.go b/helper/metadata.go new file mode 100644 index 0000000..75f7599 --- /dev/null +++ b/helper/metadata.go @@ -0,0 +1,70 @@ +package helper + +import ( + "encoding/json" + "net/url" + "os" + "path" + "webp_server_go/config" + + log "github.com/sirupsen/logrus" +) + +func getId(p string) (string, string, string) { + var id string + if config.ProxyMode { + return HashString(p), "", "" + } + parsed, _ := url.Parse(p) + width := parsed.Query().Get("width") + height := parsed.Query().Get("height") + // santizedPath will be /webp_server.jpg?width=200\u0026height= in local mode when requesting /webp_server.jpg?width=200 + // santizedPath will be https://docs.webp.sh/images/webp_server.jpg?width=400 in proxy mode when requesting /images/webp_server.jpg?width=400 with IMG_PATH = https://docs.webp.sh + santizedPath := parsed.Path + "?width=" + width + "&height=" + height + id = HashString(santizedPath) + + return id, path.Join(config.Config.ImgPath, parsed.Path), santizedPath +} + +func ReadMetadata(p, etag string) config.MetaFile { + // try to read metadata, if we can't read, create one + var metadata config.MetaFile + var id, _, _ = getId(p) + + buf, err := os.ReadFile(path.Join(config.Metadata, id+".json")) + if err != nil { + log.Warnf("can't read metadata: %s", err) + WriteMetadata(p, etag) + return ReadMetadata(p, etag) + } + + err = json.Unmarshal(buf, &metadata) + if err != nil { + log.Warnf("unmarshal metadata error, possible corrupt file, re-building...: %s", err) + WriteMetadata(p, etag) + return ReadMetadata(p, etag) + } + return metadata +} + +func WriteMetadata(p, etag string) config.MetaFile { + _ = os.Mkdir(config.Metadata, 0755) + + var id, filepath, sant = getId(p) + + var data = config.MetaFile{ + Id: id, + } + + if config.ProxyMode { + data.Path = p + data.Checksum = HashString(etag) + } else { + data.Path = sant + data.Checksum = HashFile(filepath) + } + + buf, _ := json.Marshal(data) + _ = os.WriteFile(path.Join(config.Metadata, data.Id+".json"), buf, 0644) + return data +} diff --git a/helper/metadata_test.go b/helper/metadata_test.go new file mode 100644 index 0000000..9f90227 --- /dev/null +++ b/helper/metadata_test.go @@ -0,0 +1,43 @@ +package helper + +import ( + "net/url" + "path" + "testing" + "webp_server_go/config" +) + +func TestGetId(t *testing.T) { + p := "https://example.com/image.jpg?width=200&height=300" + + t.Run("proxy mode", func(t *testing.T) { + // Test case 1: Proxy mode + config.ProxyMode = true + id, jointPath, santizedPath := getId(p) + + // Verify the return values + expectedId := HashString(p) + expectedPath := "" + expectedSantizedPath := "" + if id != expectedId || jointPath != expectedPath || santizedPath != expectedSantizedPath { + t.Errorf("Test case 1 failed: Expected (%s, %s, %s), but got (%s, %s, %s)", + expectedId, expectedPath, expectedSantizedPath, id, jointPath, santizedPath) + } + }) + t.Run("non-proxy mode", func(t *testing.T) { + // Test case 2: Non-proxy mode + config.ProxyMode = false + p = "/image.jpg?width=400&height=500" + id, jointPath, santizedPath := getId(p) + + // Verify the return values + parsed, _ := url.Parse(p) + expectedId := HashString(parsed.Path + "?width=400&height=500") + expectedPath := path.Join(config.Config.ImgPath, parsed.Path) + expectedSantizedPath := parsed.Path + "?width=400&height=500" + if id != expectedId || jointPath != expectedPath || santizedPath != expectedSantizedPath { + t.Errorf("Test case 2 failed: Expected (%s, %s, %s), but got (%s, %s, %s)", + expectedId, expectedPath, expectedSantizedPath, id, jointPath, santizedPath) + } + }) +} diff --git a/helper/test.txt b/helper/test.txt deleted file mode 100644 index 283e5e9..0000000 --- a/helper/test.txt +++ /dev/null @@ -1 +0,0 @@ -not an image diff --git a/webp-server.go b/webp-server.go index f07f374..72a42b9 100644 --- a/webp-server.go +++ b/webp-server.go @@ -12,6 +12,7 @@ import ( "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/etag" "github.com/gofiber/fiber/v2/middleware/logger" + "github.com/gofiber/fiber/v2/middleware/recover" log "github.com/sirupsen/logrus" ) @@ -41,7 +42,8 @@ func setupLogger() { Format: config.FiberLogFormat, TimeFormat: config.TimeDateFormat, })) - log.Infoln("Logger ready.") + app.Use(recover.New(recover.Config{})) + log.Infoln("fiber ready.") } func init() { @@ -59,7 +61,7 @@ func main() { ▙▚▌▛▀ ▌ ▌▌ ▖ ▌▛▀ ▌ ▐▐ ▛▀ ▌ ▌ ▌▌ ▌ ▘ ▘▝▀▘▀▀ ▘ ▝▀ ▝▀▘▘ ▘ ▝▀▘▘ ▝▀ ▝▀ -Webp Server Go - v%s +WebP Server Go - v%s Develop by WebP Server team. https://github.com/webp-sh`, config.Version) // process cli params @@ -88,7 +90,7 @@ Develop by WebP Server team. https://github.com/webp-sh`, config.Version) app.Get("/*", handler.Convert) fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner, 0x1B) - fmt.Println("Webp Server Go is Running on http://" + listenAddress) + fmt.Println("WebP Server Go is Running on http://" + listenAddress) _ = app.Listen(listenAddress)