mirror of
https://github.com/woodchen-ink/webp_server_go.git
synced 2025-07-18 05:32:02 +08:00
Metadata (#251)
* recover middleware * simplify Atoi * metadata data prototype * InterestingAttention * resize itself * Bump version to 0.9.4 Added some comments Removed String() for Extraparams * Add metadata test * Fix CI * Remove unnecessary tests * Update file count * use t.Run to get test case --------- Co-authored-by: n0vad3v <n0vad3v@riseup.net>
This commit is contained in:
parent
a5e3282ea1
commit
a7b5992662
1
.gitignore
vendored
1
.gitignore
vendored
@ -25,3 +25,4 @@ remote-raw/
|
|||||||
coverage.txt
|
coverage.txt
|
||||||
.DS_Store
|
.DS_Store
|
||||||
/webp_server_go
|
/webp_server_go
|
||||||
|
/metadata/*
|
||||||
|
@ -3,7 +3,6 @@ package config
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
|
||||||
"os"
|
"os"
|
||||||
"regexp"
|
"regexp"
|
||||||
"runtime"
|
"runtime"
|
||||||
@ -59,10 +58,18 @@ var (
|
|||||||
ProxyMode bool
|
ProxyMode bool
|
||||||
Prefetch bool
|
Prefetch bool
|
||||||
Config jsonFile
|
Config jsonFile
|
||||||
Version = "0.9.3"
|
Version = "0.9.4"
|
||||||
WriteLock = cache.New(5*time.Minute, 10*time.Minute)
|
WriteLock = cache.New(5*time.Minute, 10*time.Minute)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const Metadata = "metadata"
|
||||||
|
|
||||||
|
type MetaFile struct {
|
||||||
|
Id string `json:"id"` // hash of below path️, also json file name id.webp
|
||||||
|
Path string `json:"path"` // local: path with width and height, proxy: full url
|
||||||
|
Checksum string `json:"checksum"` // hash of original file or hash(etag). Use this to identify changes
|
||||||
|
}
|
||||||
|
|
||||||
type jsonFile struct {
|
type jsonFile struct {
|
||||||
Host string `json:"HOST"`
|
Host string `json:"HOST"`
|
||||||
Port string `json:"PORT"`
|
Port string `json:"PORT"`
|
||||||
@ -81,7 +88,6 @@ func init() {
|
|||||||
flag.BoolVar(&DumpConfig, "dump-config", false, "Print sample config.json")
|
flag.BoolVar(&DumpConfig, "dump-config", false, "Print sample config.json")
|
||||||
flag.BoolVar(&DumpSystemd, "dump-systemd", false, "Print sample systemd service file.")
|
flag.BoolVar(&DumpSystemd, "dump-systemd", false, "Print sample systemd service file.")
|
||||||
flag.BoolVar(&ShowVersion, "V", false, "Show version information.")
|
flag.BoolVar(&ShowVersion, "V", false, "Show version information.")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func LoadConfig() {
|
func LoadConfig() {
|
||||||
@ -100,11 +106,6 @@ type ExtraParams struct {
|
|||||||
Height int // in px
|
Height int // in px
|
||||||
}
|
}
|
||||||
|
|
||||||
// String : convert ExtraParams to string, used to generate cache path
|
|
||||||
func (e *ExtraParams) String() string {
|
|
||||||
return fmt.Sprintf("_width=%d&height=%d", e.Width, e.Height)
|
|
||||||
}
|
|
||||||
|
|
||||||
func switchProxyMode() {
|
func switchProxyMode() {
|
||||||
matched, _ := regexp.MatchString(`^https?://`, Config.ImgPath)
|
matched, _ := regexp.MatchString(`^https?://`, Config.ImgPath)
|
||||||
if matched {
|
if matched {
|
||||||
|
@ -1,8 +1,9 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMain(m *testing.M) {
|
func TestMain(m *testing.M) {
|
||||||
@ -21,15 +22,6 @@ func TestLoadConfig(t *testing.T) {
|
|||||||
assert.Equal(t, Config.ExhaustPath, "./exhaust")
|
assert.Equal(t, Config.ExhaustPath, "./exhaust")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExtraParamsString(t *testing.T) {
|
|
||||||
param := ExtraParams{
|
|
||||||
Width: 100,
|
|
||||||
Height: 100,
|
|
||||||
}
|
|
||||||
assert.Equal(t, param.String(), "_width=100&height=100")
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSwitchProxyMode(t *testing.T) {
|
func TestSwitchProxyMode(t *testing.T) {
|
||||||
switchProxyMode()
|
switchProxyMode()
|
||||||
assert.False(t, ProxyMode)
|
assert.False(t, ProxyMode)
|
||||||
|
@ -4,9 +4,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"webp_server_go/config"
|
"webp_server_go/config"
|
||||||
"webp_server_go/helper"
|
"webp_server_go/helper"
|
||||||
@ -32,7 +30,7 @@ func init() {
|
|||||||
func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error {
|
func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error {
|
||||||
imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width)
|
imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width)
|
||||||
if extraParams.Width > 0 && extraParams.Height > 0 {
|
if extraParams.Width > 0 && extraParams.Height > 0 {
|
||||||
err := img.Thumbnail(extraParams.Width, extraParams.Height, 0)
|
err := img.Thumbnail(extraParams.Width, extraParams.Height, vips.InterestingAttention)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -85,27 +83,20 @@ func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParam
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ResizeItself(raw, dest string, extraParams config.ExtraParams) {
|
||||||
|
log.Infof("Resize %s itself to %s", raw, dest)
|
||||||
|
img, _ := vips.LoadImageFromFile(raw, &vips.ImportParams{
|
||||||
|
FailOnError: boolFalse,
|
||||||
|
})
|
||||||
|
_ = resizeImage(img, extraParams)
|
||||||
|
buf, _, _ := img.ExportNative()
|
||||||
|
_ = os.WriteFile(dest, buf, 0600)
|
||||||
|
img.Close()
|
||||||
|
}
|
||||||
|
|
||||||
func convertImage(raw, optimized, imageType string, extraParams config.ExtraParams) error {
|
func convertImage(raw, optimized, imageType string, extraParams config.ExtraParams) error {
|
||||||
// we don't have /path/to/tsuki.jpg.1582558990.webp, maybe we have /path/to/tsuki.jpg.1082008000.webp
|
|
||||||
// delete the old converted pic and convert a new one.
|
|
||||||
// optimized: /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
|
|
||||||
// we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.imageType
|
|
||||||
// If contain extraParams like tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200
|
|
||||||
|
|
||||||
s := strings.Split(path.Base(optimized), ".")
|
|
||||||
pattern := path.Join(path.Dir(optimized), s[0]+"."+s[1]+".*."+s[len(s)-1])
|
|
||||||
|
|
||||||
matches, err := filepath.Glob(pattern)
|
|
||||||
if err != nil {
|
|
||||||
log.Error(err.Error())
|
|
||||||
} else {
|
|
||||||
for _, p := range matches {
|
|
||||||
_ = os.Remove(p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// we need to create dir first
|
// we need to create dir first
|
||||||
err = os.MkdirAll(path.Dir(optimized), 0755)
|
var err = os.MkdirAll(path.Dir(optimized), 0755)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(err.Error())
|
log.Error(err.Error())
|
||||||
}
|
}
|
||||||
|
@ -5,7 +5,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
"webp_server_go/config"
|
"webp_server_go/config"
|
||||||
"webp_server_go/helper"
|
"webp_server_go/helper"
|
||||||
@ -35,8 +34,8 @@ func PrefetchImages() {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// RawImagePath string, ImgFilename string, reqURI string
|
// RawImagePath string, ImgFilename string, reqURI string
|
||||||
proposedURI := strings.Replace(picAbsPath, config.Config.ImgPath, "", 1)
|
metadata := helper.ReadMetadata(picAbsPath, "")
|
||||||
avif, webp := helper.GenOptimizedAbsPath(picAbsPath, proposedURI, config.ExtraParams{Width: 0, Height: 0})
|
avif, webp := helper.GenOptimizedAbsPath(metadata)
|
||||||
_ = os.MkdirAll(path.Dir(avif), 0755)
|
_ = os.MkdirAll(path.Dir(avif), 0755)
|
||||||
log.Infof("Prefetching %s", picAbsPath)
|
log.Infof("Prefetching %s", picAbsPath)
|
||||||
go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan)
|
go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan)
|
||||||
|
@ -75,28 +75,21 @@ func downloadFile(filepath string, url string) {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchRemoteImg(url string) string {
|
func fetchRemoteImg(url string) config.MetaFile {
|
||||||
// url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
|
// url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
|
||||||
// How do we know if the remote img is changed? we're using hash(url+etag) as key.
|
// How do we know if the remote img is changed? we're using hash(etag+length)
|
||||||
// if this exists in local system, means the remote img is not changed, we can use it directly.
|
|
||||||
// otherwise, we need to fetch it from remote and store it in local system.
|
|
||||||
log.Infof("Remote Addr is %s, pinging for info...", url)
|
log.Infof("Remote Addr is %s, pinging for info...", url)
|
||||||
// identifiable is etag + length
|
etag := pingURL(url)
|
||||||
identifiable := pingURL(url)
|
metadata := helper.ReadMetadata(url, etag)
|
||||||
// For store the remote raw image, /home/webp_server/remote-raw/3a42ab801f669d64-b8f999ab5acd69d03f5e904b1b84eb79210536
|
localRawImagePath := path.Join(config.RemoteRaw, metadata.Id)
|
||||||
// Which 3a42ab801f669d64 is hash(url), b8f999ab5acd69d03f5e904b1b84eb79 is etag and 210536 is length
|
|
||||||
localRawImagePath := path.Join(config.RemoteRaw, helper.HashString(url)+"-"+identifiable)
|
|
||||||
|
|
||||||
if helper.ImageExists(localRawImagePath) {
|
if !helper.ImageExists(localRawImagePath) || metadata.Checksum != helper.HashString(etag) {
|
||||||
return localRawImagePath
|
// remote file has changed or local file not exists
|
||||||
} else {
|
|
||||||
// Temporary store of remote file.
|
|
||||||
cleanProxyCache(config.RemoteRaw + helper.HashString(url) + "*")
|
|
||||||
log.Info("Remote file not found in remote-raw, re-fetching...")
|
log.Info("Remote file not found in remote-raw, re-fetching...")
|
||||||
|
cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id+"*"))
|
||||||
downloadFile(localRawImagePath, url)
|
downloadFile(localRawImagePath, url)
|
||||||
return localRawImagePath
|
|
||||||
}
|
}
|
||||||
|
return metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
func pingURL(url string) string {
|
func pingURL(url string) string {
|
||||||
@ -117,7 +110,5 @@ func pingURL(url string) string {
|
|||||||
if etag == "" {
|
if etag == "" {
|
||||||
log.Info("Remote didn't return etag in header when getRemoteImageInfo, please check.")
|
log.Info("Remote didn't return etag in header when getRemoteImageInfo, please check.")
|
||||||
}
|
}
|
||||||
// Remove " from etag
|
|
||||||
etag = strings.ReplaceAll(etag, "\"", "")
|
|
||||||
return etag + length
|
return etag + length
|
||||||
}
|
}
|
||||||
|
@ -3,7 +3,6 @@ package handler
|
|||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
|
||||||
"webp_server_go/config"
|
"webp_server_go/config"
|
||||||
"webp_server_go/encoder"
|
"webp_server_go/encoder"
|
||||||
"webp_server_go/helper"
|
"webp_server_go/helper"
|
||||||
@ -40,31 +39,42 @@ func Convert(c *fiber.Ctx) error {
|
|||||||
reqURI = path.Clean(reqURI)
|
reqURI = path.Clean(reqURI)
|
||||||
reqURIwithQuery = path.Clean(reqURIwithQuery)
|
reqURIwithQuery = path.Clean(reqURIwithQuery)
|
||||||
|
|
||||||
WidthInt, err := strconv.Atoi(c.Query("width"))
|
width, _ := strconv.Atoi(c.Query("width"))
|
||||||
if err != nil {
|
height, _ := strconv.Atoi(c.Query("height"))
|
||||||
WidthInt = 0
|
|
||||||
}
|
|
||||||
HeightInt, err := strconv.Atoi(c.Query("height"))
|
|
||||||
if err != nil {
|
|
||||||
HeightInt = 0
|
|
||||||
}
|
|
||||||
var extraParams = config.ExtraParams{
|
var extraParams = config.ExtraParams{
|
||||||
Width: WidthInt,
|
Width: width,
|
||||||
Height: HeightInt,
|
Height: height,
|
||||||
}
|
}
|
||||||
|
|
||||||
var rawImageAbs string
|
var rawImageAbs string
|
||||||
|
var metadata = config.MetaFile{}
|
||||||
if config.ProxyMode {
|
if config.ProxyMode {
|
||||||
// this is proxyMode, we'll have to use this url to download and save it to local path, which also gives us rawImageAbs
|
// this is proxyMode, we'll have to use this url to download and save it to local path, which also gives us rawImageAbs
|
||||||
// https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
|
// https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
|
||||||
rawImageAbs = fetchRemoteImg(config.Config.ImgPath + reqURIwithQuery)
|
metadata = fetchRemoteImg(config.Config.ImgPath + reqURIwithQuery)
|
||||||
|
rawImageAbs = path.Join(config.RemoteRaw, metadata.Id)
|
||||||
} else {
|
} else {
|
||||||
// not proxyMode, we'll use local path
|
// not proxyMode, we'll use local path
|
||||||
rawImageAbs = path.Join(config.Config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg
|
metadata = helper.ReadMetadata(reqURIwithQuery, "")
|
||||||
|
rawImageAbs = path.Join(config.Config.ImgPath, reqURI)
|
||||||
|
// detect if source file has changed
|
||||||
|
if metadata.Checksum != helper.HashFile(rawImageAbs) {
|
||||||
|
log.Info("Source file has changed, re-encoding...")
|
||||||
|
helper.WriteMetadata(reqURIwithQuery, "")
|
||||||
|
cleanProxyCache(path.Join(config.Config.ExhaustPath, metadata.Id))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
goodFormat := helper.GuessSupportedFormat(&c.Request().Header)
|
goodFormat := helper.GuessSupportedFormat(&c.Request().Header)
|
||||||
|
// resize itself and return if only one format(raw) is supported
|
||||||
|
if len(goodFormat) == 1 {
|
||||||
|
dest := path.Join(config.Config.ExhaustPath, metadata.Id)
|
||||||
|
if !helper.ImageExists(dest) {
|
||||||
|
encoder.ResizeItself(rawImageAbs, dest, extraParams)
|
||||||
|
}
|
||||||
|
return c.SendFile(dest)
|
||||||
|
}
|
||||||
|
|
||||||
// Check the original image for existence,
|
// Check the original image for existence,
|
||||||
if !helper.ImageExists(rawImageAbs) {
|
if !helper.ImageExists(rawImageAbs) {
|
||||||
@ -75,11 +85,7 @@ func Convert(c *fiber.Ctx) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// generate with timestamp to make sure files are update-to-date
|
avifAbs, webpAbs := helper.GenOptimizedAbsPath(metadata)
|
||||||
// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
|
|
||||||
// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
|
|
||||||
// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
|
|
||||||
avifAbs, webpAbs := helper.GenOptimizedAbsPath(rawImageAbs, reqURI, extraParams)
|
|
||||||
encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil)
|
encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil)
|
||||||
|
|
||||||
var availableFiles = []string{rawImageAbs}
|
var availableFiles = []string{rawImageAbs}
|
||||||
@ -93,9 +99,7 @@ func Convert(c *fiber.Ctx) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
finalFilename := helper.FindSmallestFiles(availableFiles)
|
finalFilename := helper.FindSmallestFiles(availableFiles)
|
||||||
|
contentType := helper.GetFileContentType(finalFilename)
|
||||||
buf, _ := os.ReadFile(finalFilename)
|
|
||||||
contentType := helper.GetFileContentType(buf)
|
|
||||||
c.Set("Content-Type", contentType)
|
c.Set("Content-Type", contentType)
|
||||||
|
|
||||||
c.Set("X-Compression-Rate", helper.GetCompressionRate(rawImageAbs, finalFilename))
|
c.Set("X-Compression-Rate", helper.GetCompressionRate(rawImageAbs, finalFilename))
|
||||||
|
@ -1,7 +1,6 @@
|
|||||||
package helper
|
package helper
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
@ -10,34 +9,25 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
"webp_server_go/config"
|
"webp_server_go/config"
|
||||||
|
|
||||||
"github.com/cespare/xxhash"
|
|
||||||
"github.com/h2non/filetype"
|
"github.com/h2non/filetype"
|
||||||
|
|
||||||
|
"github.com/cespare/xxhash"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
var _ = filetype.AddMatcher(filetype.NewType("avif", "image/avif"), avifMatcher)
|
func GetFileContentType(filename string) string {
|
||||||
|
if strings.HasSuffix(filename, ".webp") {
|
||||||
func avifMatcher(buf []byte) bool {
|
return "image/webp"
|
||||||
// use hexdump on macOS to see the magic number
|
} else if strings.HasSuffix(filename, ".avif") {
|
||||||
// 0000001c 66747970 61766966 00000000 61766966 6d696631 6d696166
|
return "image/avif"
|
||||||
magicHeader := []byte{
|
} else {
|
||||||
0x0, 0x0, 0x0, 0x1c,
|
// raw image, need to use filetype to determine
|
||||||
0x66, 0x74, 0x79, 0x70,
|
buf, _ := os.ReadFile(filename)
|
||||||
0x61, 0x76, 0x69, 0x66,
|
kind, _ := filetype.Match(buf)
|
||||||
0x0, 0x0, 0x0, 0x0,
|
return kind.MIME.Value
|
||||||
0x61, 0x76, 0x69, 0x66,
|
|
||||||
0x6d, 0x69, 0x66, 0x31,
|
|
||||||
0x6d, 0x69, 0x61, 0x66,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return len(buf) > 1 && bytes.Equal(buf[:28], magicHeader) || strings.Contains(string(buf), "ftypavif")
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetFileContentType(buffer []byte) string {
|
|
||||||
kind, _ := filetype.Match(buffer)
|
|
||||||
return kind.MIME.Value
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func FileCount(dir string) int64 {
|
func FileCount(dir string) int64 {
|
||||||
@ -99,37 +89,11 @@ func CheckAllowedType(imgFilename string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func GenOptimizedAbsPath(rawImagePath, reqURI string, extraParams config.ExtraParams) (string, string) {
|
func GenOptimizedAbsPath(metadata config.MetaFile) (string, string) {
|
||||||
// imageName is not needed, we can use reqURI
|
webpFilename := fmt.Sprintf("%s.webp", metadata.Id)
|
||||||
// get file mod time
|
avifFilename := fmt.Sprintf("%s.avif", metadata.Id)
|
||||||
var (
|
webpAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, webpFilename))
|
||||||
imageName = path.Base(reqURI)
|
avifAbsolutePath := path.Clean(path.Join(config.Config.ExhaustPath, avifFilename))
|
||||||
exhaustPath = config.Config.ExhaustPath
|
|
||||||
)
|
|
||||||
STAT, err := os.Stat(rawImagePath)
|
|
||||||
if err != nil {
|
|
||||||
log.Error(err.Error())
|
|
||||||
return "", ""
|
|
||||||
}
|
|
||||||
ModifiedTime := STAT.ModTime().Unix()
|
|
||||||
// TODO: just hash it?
|
|
||||||
// webpFilename: abc.jpg.png -> abc.jpg.png.1582558990.webp
|
|
||||||
webpFilename := fmt.Sprintf("%s.%d.webp", imageName, ModifiedTime)
|
|
||||||
// avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif
|
|
||||||
avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime)
|
|
||||||
|
|
||||||
// If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
|
|
||||||
// If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0
|
|
||||||
// If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0
|
|
||||||
if config.Config.EnableExtraParams {
|
|
||||||
webpFilename = webpFilename + extraParams.String()
|
|
||||||
avifFilename = avifFilename + extraParams.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
|
|
||||||
// Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp
|
|
||||||
webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename))
|
|
||||||
avifAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), avifFilename))
|
|
||||||
return avifAbsolutePath, webpAbsolutePath
|
return avifAbsolutePath, webpAbsolutePath
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -214,3 +178,8 @@ func HashString(uri string) string {
|
|||||||
// xxhash supports cross compile
|
// xxhash supports cross compile
|
||||||
return fmt.Sprintf("%x", xxhash.Sum64String(uri))
|
return fmt.Sprintf("%x", xxhash.Sum64String(uri))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func HashFile(filepath string) string {
|
||||||
|
buf, _ := os.ReadFile(filepath)
|
||||||
|
return fmt.Sprintf("%x", xxhash.Sum64(buf))
|
||||||
|
}
|
||||||
|
@ -18,7 +18,7 @@ func TestMain(m *testing.M) {
|
|||||||
func TestFileCount(t *testing.T) {
|
func TestFileCount(t *testing.T) {
|
||||||
// test helper dir
|
// test helper dir
|
||||||
count := FileCount("./")
|
count := FileCount("./")
|
||||||
assert.Equal(t, int64(3), count)
|
assert.Equal(t, int64(4), count)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestImageExists(t *testing.T) {
|
func TestImageExists(t *testing.T) {
|
||||||
@ -26,10 +26,6 @@ func TestImageExists(t *testing.T) {
|
|||||||
assert.False(t, ImageExists("dgyuaikdsa"))
|
assert.False(t, ImageExists("dgyuaikdsa"))
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("file size incorrect", func(t *testing.T) {
|
|
||||||
assert.False(t, ImageExists("test.txt"))
|
|
||||||
})
|
|
||||||
|
|
||||||
// TODO: how to test lock?
|
// TODO: how to test lock?
|
||||||
|
|
||||||
t.Run("test dir", func(t *testing.T) {
|
t.Run("test dir", func(t *testing.T) {
|
||||||
@ -43,7 +39,7 @@ func TestImageExists(t *testing.T) {
|
|||||||
|
|
||||||
func TestCheckAllowedType(t *testing.T) {
|
func TestCheckAllowedType(t *testing.T) {
|
||||||
t.Run("not allowed type", func(t *testing.T) {
|
t.Run("not allowed type", func(t *testing.T) {
|
||||||
assert.False(t, CheckAllowedType("test.txt"))
|
assert.False(t, CheckAllowedType("./helper_test.go"))
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("allowed type", func(t *testing.T) {
|
t.Run("allowed type", func(t *testing.T) {
|
||||||
|
70
helper/metadata.go
Normal file
70
helper/metadata.go
Normal file
@ -0,0 +1,70 @@
|
|||||||
|
package helper
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"webp_server_go/config"
|
||||||
|
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
func getId(p string) (string, string, string) {
|
||||||
|
var id string
|
||||||
|
if config.ProxyMode {
|
||||||
|
return HashString(p), "", ""
|
||||||
|
}
|
||||||
|
parsed, _ := url.Parse(p)
|
||||||
|
width := parsed.Query().Get("width")
|
||||||
|
height := parsed.Query().Get("height")
|
||||||
|
// santizedPath will be /webp_server.jpg?width=200\u0026height= in local mode when requesting /webp_server.jpg?width=200
|
||||||
|
// santizedPath will be https://docs.webp.sh/images/webp_server.jpg?width=400 in proxy mode when requesting /images/webp_server.jpg?width=400 with IMG_PATH = https://docs.webp.sh
|
||||||
|
santizedPath := parsed.Path + "?width=" + width + "&height=" + height
|
||||||
|
id = HashString(santizedPath)
|
||||||
|
|
||||||
|
return id, path.Join(config.Config.ImgPath, parsed.Path), santizedPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func ReadMetadata(p, etag string) config.MetaFile {
|
||||||
|
// try to read metadata, if we can't read, create one
|
||||||
|
var metadata config.MetaFile
|
||||||
|
var id, _, _ = getId(p)
|
||||||
|
|
||||||
|
buf, err := os.ReadFile(path.Join(config.Metadata, id+".json"))
|
||||||
|
if err != nil {
|
||||||
|
log.Warnf("can't read metadata: %s", err)
|
||||||
|
WriteMetadata(p, etag)
|
||||||
|
return ReadMetadata(p, etag)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = json.Unmarshal(buf, &metadata)
|
||||||
|
if err != nil {
|
||||||
|
log.Warnf("unmarshal metadata error, possible corrupt file, re-building...: %s", err)
|
||||||
|
WriteMetadata(p, etag)
|
||||||
|
return ReadMetadata(p, etag)
|
||||||
|
}
|
||||||
|
return metadata
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteMetadata(p, etag string) config.MetaFile {
|
||||||
|
_ = os.Mkdir(config.Metadata, 0755)
|
||||||
|
|
||||||
|
var id, filepath, sant = getId(p)
|
||||||
|
|
||||||
|
var data = config.MetaFile{
|
||||||
|
Id: id,
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.ProxyMode {
|
||||||
|
data.Path = p
|
||||||
|
data.Checksum = HashString(etag)
|
||||||
|
} else {
|
||||||
|
data.Path = sant
|
||||||
|
data.Checksum = HashFile(filepath)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf, _ := json.Marshal(data)
|
||||||
|
_ = os.WriteFile(path.Join(config.Metadata, data.Id+".json"), buf, 0644)
|
||||||
|
return data
|
||||||
|
}
|
43
helper/metadata_test.go
Normal file
43
helper/metadata_test.go
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
package helper
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/url"
|
||||||
|
"path"
|
||||||
|
"testing"
|
||||||
|
"webp_server_go/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetId(t *testing.T) {
|
||||||
|
p := "https://example.com/image.jpg?width=200&height=300"
|
||||||
|
|
||||||
|
t.Run("proxy mode", func(t *testing.T) {
|
||||||
|
// Test case 1: Proxy mode
|
||||||
|
config.ProxyMode = true
|
||||||
|
id, jointPath, santizedPath := getId(p)
|
||||||
|
|
||||||
|
// Verify the return values
|
||||||
|
expectedId := HashString(p)
|
||||||
|
expectedPath := ""
|
||||||
|
expectedSantizedPath := ""
|
||||||
|
if id != expectedId || jointPath != expectedPath || santizedPath != expectedSantizedPath {
|
||||||
|
t.Errorf("Test case 1 failed: Expected (%s, %s, %s), but got (%s, %s, %s)",
|
||||||
|
expectedId, expectedPath, expectedSantizedPath, id, jointPath, santizedPath)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("non-proxy mode", func(t *testing.T) {
|
||||||
|
// Test case 2: Non-proxy mode
|
||||||
|
config.ProxyMode = false
|
||||||
|
p = "/image.jpg?width=400&height=500"
|
||||||
|
id, jointPath, santizedPath := getId(p)
|
||||||
|
|
||||||
|
// Verify the return values
|
||||||
|
parsed, _ := url.Parse(p)
|
||||||
|
expectedId := HashString(parsed.Path + "?width=400&height=500")
|
||||||
|
expectedPath := path.Join(config.Config.ImgPath, parsed.Path)
|
||||||
|
expectedSantizedPath := parsed.Path + "?width=400&height=500"
|
||||||
|
if id != expectedId || jointPath != expectedPath || santizedPath != expectedSantizedPath {
|
||||||
|
t.Errorf("Test case 2 failed: Expected (%s, %s, %s), but got (%s, %s, %s)",
|
||||||
|
expectedId, expectedPath, expectedSantizedPath, id, jointPath, santizedPath)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
@ -1 +0,0 @@
|
|||||||
not an image
|
|
@ -12,6 +12,7 @@ import (
|
|||||||
"github.com/gofiber/fiber/v2"
|
"github.com/gofiber/fiber/v2"
|
||||||
"github.com/gofiber/fiber/v2/middleware/etag"
|
"github.com/gofiber/fiber/v2/middleware/etag"
|
||||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||||
|
"github.com/gofiber/fiber/v2/middleware/recover"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -41,7 +42,8 @@ func setupLogger() {
|
|||||||
Format: config.FiberLogFormat,
|
Format: config.FiberLogFormat,
|
||||||
TimeFormat: config.TimeDateFormat,
|
TimeFormat: config.TimeDateFormat,
|
||||||
}))
|
}))
|
||||||
log.Infoln("Logger ready.")
|
app.Use(recover.New(recover.Config{}))
|
||||||
|
log.Infoln("fiber ready.")
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
@ -59,7 +61,7 @@ func main() {
|
|||||||
▙▚▌▛▀ ▌ ▌▌ ▖ ▌▛▀ ▌ ▐▐ ▛▀ ▌ ▌ ▌▌ ▌
|
▙▚▌▛▀ ▌ ▌▌ ▖ ▌▛▀ ▌ ▐▐ ▛▀ ▌ ▌ ▌▌ ▌
|
||||||
▘ ▘▝▀▘▀▀ ▘ ▝▀ ▝▀▘▘ ▘ ▝▀▘▘ ▝▀ ▝▀
|
▘ ▘▝▀▘▀▀ ▘ ▝▀ ▝▀▘▘ ▘ ▝▀▘▘ ▝▀ ▝▀
|
||||||
|
|
||||||
Webp Server Go - v%s
|
WebP Server Go - v%s
|
||||||
Develop by WebP Server team. https://github.com/webp-sh`, config.Version)
|
Develop by WebP Server team. https://github.com/webp-sh`, config.Version)
|
||||||
|
|
||||||
// process cli params
|
// process cli params
|
||||||
@ -88,7 +90,7 @@ Develop by WebP Server team. https://github.com/webp-sh`, config.Version)
|
|||||||
app.Get("/*", handler.Convert)
|
app.Get("/*", handler.Convert)
|
||||||
|
|
||||||
fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner, 0x1B)
|
fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner, 0x1B)
|
||||||
fmt.Println("Webp Server Go is Running on http://" + listenAddress)
|
fmt.Println("WebP Server Go is Running on http://" + listenAddress)
|
||||||
|
|
||||||
_ = app.Listen(listenAddress)
|
_ = app.Listen(listenAddress)
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user