diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 49691ab..f201017 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -26,7 +26,7 @@ jobs: - name: Start the container run: | - cp tests/glib_malloc/docker-compose.yml ./ + cp malloc_tests/glib_malloc/docker-compose.yml ./ docker-compose up -d - name: Send Requests to Server @@ -55,7 +55,7 @@ jobs: - name: Start the container run: | - cp tests/jemalloc/docker-compose.yml ./ + cp malloc_tests/jemalloc/docker-compose.yml ./ docker-compose up -d - name: Send Requests to Server @@ -65,4 +65,4 @@ jobs: - name: Get container RAM stats run: | - docker stats --no-stream \ No newline at end of file + docker stats --no-stream diff --git a/.gitignore b/.gitignore index 6938dec..51782b8 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ builds/ /.idea/webp_server_go.iml remote-raw/ coverage.txt +.DS_Store +/webp_server_go diff --git a/Makefile b/Makefile index 720a608..11c465c 100644 --- a/Makefile +++ b/Makefile @@ -35,8 +35,8 @@ test: static-check go test -v -coverprofile=coverage.txt -covermode=atomic clean: - rm -rf builds - rm -rf prefetch + rm -rf builds prefetch remote-raw exhaust tools coverage.txt + docker: - DOCKER_BUILDKIT=1 docker build -t webpsh/webps . \ No newline at end of file + DOCKER_BUILDKIT=1 docker build -t webpsh/webps . diff --git a/config.go b/config.go deleted file mode 100644 index ada1855..0000000 --- a/config.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import "fmt" - -type Config struct { - Host string `json:"HOST"` - Port string `json:"PORT"` - ImgPath string `json:"IMG_PATH"` - Quality int `json:"QUALITY,string"` - AllowedTypes []string `json:"ALLOWED_TYPES"` - ExhaustPath string `json:"EXHAUST_PATH"` - EnableAVIF bool `json:"ENABLE_AVIF"` - EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"` -} - -type ExtraParams struct { - Width int // in px - Height int // in px -} - -// String : convert ExtraParams to string, used to generate cache path -func (e *ExtraParams) String() string { - return fmt.Sprintf("_width=%d&height=%d", e.Width, e.Height) -} - -var ( - configPath string - jobs int - dumpConfig, dumpSystemd bool - verboseMode, showVersion bool - prefetch, proxyMode bool - remoteRaw = "remote-raw" - config Config - version = "0.9.0" -) - -const ( - sampleConfig = ` -{ - "HOST": "127.0.0.1", - "PORT": "3333", - "QUALITY": "80", - "IMG_PATH": "./pics", - "EXHAUST_PATH": "./exhaust", - "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","gif"], - "ENABLE_AVIF": false, - "ENABLE_EXTRA_PARAMS": false -}` - - sampleSystemd = ` -[Unit] -Description=WebP Server Go -Documentation=https://github.com/webp-sh/webp_server_go -After=nginx.target - -[Service] -Type=simple -StandardError=journal -WorkingDirectory=/opt/webps -ExecStart=/opt/webps/webp-server --config /opt/webps/config.json -Restart=always -RestartSec=3s - -[Install] -WantedBy=multi-user.target` -) - -const ( - webpMax = 16383 - avifMax = 65536 -) diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..5e533fa --- /dev/null +++ b/config/config.go @@ -0,0 +1,115 @@ +package config + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "regexp" + "runtime" + "time" + + "github.com/patrickmn/go-cache" + log "github.com/sirupsen/logrus" +) + +const ( + TimeDateFormat = "2006-01-02 15:04:05" + FiberLogFormat = "${ip} - [${time}] ${method} ${url} ${status} ${referer} ${ua}\n" + WebpMax = 16383 + AvifMax = 65536 + RemoteRaw = "remote-raw" + + SampleConfig = ` +{ + "HOST": "127.0.0.1", + "PORT": "3333", + "QUALITY": "80", + "IMG_PATH": "./pics", + "EXHAUST_PATH": "./exhaust", + "ALLOWED_TYPES": ["jpg","png","jpeg","bmp"], + "ENABLE_AVIF": false, + "ENABLE_EXTRA_PARAMS": false +}` + + SampleSystemd = ` +[Unit] +Description=WebP Server Go +Documentation=https://github.com/webp-sh/webp_server_go +After=nginx.target + +[Service] +Type=simple +StandardError=journal +WorkingDirectory=/opt/webps +ExecStart=/opt/webps/webp-server --config /opt/webps/config.json +Restart=always +RestartSec=3s + +[Install] +WantedBy=multi-user.target` +) + +var ( + configPath string + Jobs int + DumpSystemd bool + DumpConfig bool + ShowVersion bool + ProxyMode bool + Prefetch bool + Config jsonFile + Version = "0.9.0" + WriteLock = cache.New(5*time.Minute, 10*time.Minute) +) + +type jsonFile struct { + Host string `json:"HOST"` + Port string `json:"PORT"` + ImgPath string `json:"IMG_PATH"` + Quality int `json:"QUALITY,string"` + AllowedTypes []string `json:"ALLOWED_TYPES"` + ExhaustPath string `json:"EXHAUST_PATH"` + EnableAVIF bool `json:"ENABLE_AVIF"` + EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"` +} + +func init() { + flag.StringVar(&configPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)") + flag.BoolVar(&Prefetch, "prefetch", false, "Prefetch and convert image to webp") + flag.IntVar(&Jobs, "jobs", runtime.NumCPU(), "Prefetch thread, default is all.") + flag.BoolVar(&DumpConfig, "dump-config", false, "Print sample config.json") + flag.BoolVar(&DumpSystemd, "dump-systemd", false, "Print sample systemd service file.") + flag.BoolVar(&ShowVersion, "V", false, "Show version information.") + flag.Parse() + Config = loadConfig() + switchProxyMode() +} + +func loadConfig() (config jsonFile) { + jsonObject, err := os.Open(configPath) + if err != nil { + log.Fatal(err) + } + decoder := json.NewDecoder(jsonObject) + _ = decoder.Decode(&config) + _ = jsonObject.Close() + return config +} + +type ExtraParams struct { + Width int // in px + Height int // in px +} + +// String : convert ExtraParams to string, used to generate cache path +func (e *ExtraParams) String() string { + return fmt.Sprintf("_width=%d&height=%d", e.Width, e.Height) +} + +func switchProxyMode() { + matched, _ := regexp.MatchString(`^https?://`, Config.ImgPath) + if matched { + ProxyMode = true + } +} diff --git a/encoder.go b/encoder/encoder.go similarity index 72% rename from encoder.go rename to encoder/encoder.go index 4e3a807..7ffe2ea 100644 --- a/encoder.go +++ b/encoder/encoder.go @@ -1,18 +1,35 @@ -package main +package encoder import ( "errors" "os" "path" "path/filepath" + "runtime" "strings" "sync" + "webp_server_go/config" + "webp_server_go/helper" "github.com/davidbyttow/govips/v2/vips" log "github.com/sirupsen/logrus" ) -func resizeImage(img *vips.ImageRef, extraParams ExtraParams) error { +var ( + boolFalse vips.BoolParameter + intMinusOne vips.IntParameter +) + +func init() { + vips.Startup(&vips.Config{ + ConcurrencyLevel: runtime.NumCPU(), + }) + boolFalse.Set(false) + intMinusOne.Set(-1) + +} + +func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error { imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width) if extraParams.Width > 0 && extraParams.Height > 0 { err := img.Thumbnail(extraParams.Width, extraParams.Height, 0) @@ -33,12 +50,12 @@ func resizeImage(img *vips.ImageRef, extraParams ExtraParams) error { return nil } -func convertFilter(raw, avifPath string, webpPath string, extraParams ExtraParams, c chan int) { +func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParams, c chan int) { // all absolute paths var wg sync.WaitGroup wg.Add(2) - if !imageExists(avifPath) && config.EnableAVIF { + if !helper.ImageExists(avifPath) && config.Config.EnableAVIF { go func() { err := convertImage(raw, avifPath, "avif", extraParams) if err != nil { @@ -50,7 +67,7 @@ func convertFilter(raw, avifPath string, webpPath string, extraParams ExtraParam wg.Done() } - if !imageExists(webpPath) { + if !helper.ImageExists(webpPath) { go func() { err := convertImage(raw, webpPath, "webp", extraParams) if err != nil { @@ -68,11 +85,11 @@ func convertFilter(raw, avifPath string, webpPath string, extraParams ExtraParam } } -func convertImage(raw, optimized, itype string, extraParams ExtraParams) error { +func convertImage(raw, optimized, imageType string, extraParams config.ExtraParams) error { // we don't have /path/to/tsuki.jpg.1582558990.webp, maybe we have /path/to/tsuki.jpg.1082008000.webp // delete the old converted pic and convert a new one. // optimized: /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - // we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.itype + // we'll delete file starts with /home/webp_server/exhaust/path/to/tsuki.jpg.ts.imageType // If contain extraParams like tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200 s := strings.Split(path.Base(optimized), ".") @@ -93,20 +110,33 @@ func convertImage(raw, optimized, itype string, extraParams ExtraParams) error { log.Error(err.Error()) } - switch itype { + switch imageType { case "webp": - err = webpEncoder(raw, optimized, config.Quality, extraParams) + err = webpEncoder(raw, optimized, extraParams) case "avif": - err = avifEncoder(raw, optimized, config.Quality, extraParams) + err = avifEncoder(raw, optimized, extraParams) } return err } -func avifEncoder(p1, p2 string, quality int, extraParams ExtraParams) error { +func imageIgnore(imageFormat vips.ImageType) bool { + // Ignore Unknown, WebP, AVIF + ignoreList := []vips.ImageType{vips.ImageTypeUnknown, vips.ImageTypeWEBP, vips.ImageTypeAVIF} + for _, ignore := range ignoreList { + if imageFormat == ignore { + // Return err to render original image + return true + } + } + return false +} + +func avifEncoder(p1, p2 string, extraParams config.ExtraParams) error { // if convert fails, return error; success nil - var buf []byte - var boolFalse vips.BoolParameter - boolFalse.Set(false) + var ( + buf []byte + quality = config.Config.Quality + ) img, err := vips.LoadImageFromFile(p1, &vips.ImportParams{ FailOnError: boolFalse, }) @@ -114,18 +144,11 @@ func avifEncoder(p1, p2 string, quality int, extraParams ExtraParams) error { return err } - // Ignore Unknown, WebP, AVIF - ignoreList := []vips.ImageType{vips.ImageTypeUnknown, vips.ImageTypeWEBP, vips.ImageTypeAVIF} - - imageFormat := img.Format() - for _, ignore := range ignoreList { - if imageFormat == ignore { - // Return err to render original image - return errors.New("encoder: ignore image type") - } + if imageIgnore(img.Format()) { + return errors.New("encoder: ignore image type") } - if config.EnableExtraParams { + if config.Config.EnableExtraParams { err = resizeImage(img, extraParams) if err != nil { return err @@ -133,7 +156,7 @@ func avifEncoder(p1, p2 string, quality int, extraParams ExtraParams) error { } // AVIF has a maximum resolution of 65536 x 65536 pixels. - if img.Metadata().Width > avifMax || img.Metadata().Height > avifMax { + if img.Metadata().Width > config.AvifMax || img.Metadata().Height > config.AvifMax { return errors.New("AVIF: image too large") } @@ -171,13 +194,13 @@ func avifEncoder(p1, p2 string, quality int, extraParams ExtraParams) error { return nil } -func webpEncoder(p1, p2 string, quality int, extraParams ExtraParams) error { +func webpEncoder(p1, p2 string, extraParams config.ExtraParams) error { // if convert fails, return error; success nil - var buf []byte - var boolFalse vips.BoolParameter - boolFalse.Set(false) - var intMinusOne vips.IntParameter - intMinusOne.Set(-1) + var ( + buf []byte + quality = config.Config.Quality + ) + img, err := vips.LoadImageFromFile(p1, &vips.ImportParams{ FailOnError: boolFalse, NumPages: intMinusOne, @@ -186,18 +209,11 @@ func webpEncoder(p1, p2 string, quality int, extraParams ExtraParams) error { return err } - // Ignore Unknown, WebP, AVIF - ignoreList := []vips.ImageType{vips.ImageTypeUnknown, vips.ImageTypeWEBP, vips.ImageTypeAVIF} - - imageFormat := img.Format() - for _, ignore := range ignoreList { - if imageFormat == ignore { - // Return err to render original image - return errors.New("encoder: ignore image type") - } + if imageIgnore(img.Format()) { + return errors.New("encoder: ignore image type") } - if config.EnableExtraParams { + if config.Config.EnableExtraParams { err = resizeImage(img, extraParams) if err != nil { return err @@ -205,8 +221,7 @@ func webpEncoder(p1, p2 string, quality int, extraParams ExtraParams) error { } // The maximum pixel dimensions of a WebP image is 16383 x 16383. - // But GIF is exception, it can be larger than 16383 - if (img.Metadata().Width > webpMax || img.Metadata().Height > webpMax) && imageFormat != vips.ImageTypeGIF { + if (img.Metadata().Width > config.WebpMax || img.Metadata().Height > config.WebpMax) && img.Format() != vips.ImageTypeGIF { return errors.New("WebP: image too large") } @@ -218,14 +233,16 @@ func webpEncoder(p1, p2 string, quality int, extraParams ExtraParams) error { // If quality >= 100, we use lossless mode if quality >= 100 { buf, _, err = img.ExportWebp(&vips.WebpExportParams{ - Lossless: true, - StripMetadata: true, + Lossless: true, + StripMetadata: true, + ReductionEffort: 2, }) } else { buf, _, err = img.ExportWebp(&vips.WebpExportParams{ - Quality: quality, - Lossless: false, - StripMetadata: true, + Quality: quality, + Lossless: false, + StripMetadata: true, + ReductionEffort: 2, }) } diff --git a/prefetch.go b/encoder/prefetch.go similarity index 52% rename from prefetch.go rename to encoder/prefetch.go index 67c2584..01d04f9 100644 --- a/prefetch.go +++ b/encoder/prefetch.go @@ -1,4 +1,4 @@ -package main +package encoder import ( "fmt" @@ -7,24 +7,26 @@ import ( "path/filepath" "strings" "time" + "webp_server_go/config" + "webp_server_go/helper" "github.com/schollz/progressbar/v3" log "github.com/sirupsen/logrus" ) -func prefetchImages(confImgPath string, ExhaustPath string) { +func PrefetchImages() { // maximum ongoing prefetch is depending on your core of CPU var sTime = time.Now() - log.Infof("Prefetching using %d cores", jobs) - var finishChan = make(chan int, jobs) - for i := 0; i < jobs; i++ { + log.Infof("Prefetching using %d cores", config.Jobs) + var finishChan = make(chan int, config.Jobs) + for i := 0; i < config.Jobs; i++ { finishChan <- 1 } //prefetch, recursive through the dir - all := fileCount(confImgPath) + all := helper.FileCount(config.Config.ImgPath) var bar = progressbar.Default(all, "Prefetching...") - err := filepath.Walk(confImgPath, + err := filepath.Walk(config.Config.ImgPath, func(picAbsPath string, info os.FileInfo, err error) error { if err != nil { return err @@ -33,11 +35,11 @@ func prefetchImages(confImgPath string, ExhaustPath string) { return nil } // RawImagePath string, ImgFilename string, reqURI string - proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1) - avif, webp := genOptimizedAbsPath(picAbsPath, ExhaustPath, info.Name(), proposedURI, ExtraParams{Width: 0, Height: 0}) + proposedURI := strings.Replace(picAbsPath, config.Config.ImgPath, "", 1) + avif, webp := helper.GenOptimizedAbsPath(picAbsPath, proposedURI, config.ExtraParams{Width: 0, Height: 0}) _ = os.MkdirAll(path.Dir(avif), 0755) log.Infof("Prefetching %s", picAbsPath) - go convertFilter(picAbsPath, avif, webp, ExtraParams{Width: 0, Height: 0}, finishChan) + go ConvertFilter(picAbsPath, avif, webp, config.ExtraParams{Width: 0, Height: 0}, finishChan) _ = bar.Add(<-finishChan) return nil }) @@ -46,6 +48,6 @@ func prefetchImages(confImgPath string, ExhaustPath string) { log.Errorln(err) } elapsed := time.Since(sTime) - _, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y in %s\n\n", elapsed) + _, _ = fmt.Fprintf(os.Stdout, "Prefetch complete in %s\n\n", elapsed) } diff --git a/encoder_test.go b/encoder_test.go deleted file mode 100644 index 6cfc911..0000000 --- a/encoder_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package main - -import ( - "github.com/davidbyttow/govips/v2/vips" - "github.com/stretchr/testify/assert" - - "os" - "path" - "path/filepath" - "strings" - "testing" -) - -var dest = "/tmp/test-result" - -func walker() []string { - var list []string - _ = filepath.Walk("./pics", func(p string, info os.FileInfo, err error) error { - if !info.IsDir() && !strings.HasPrefix(path.Base(p), ".") { - list = append(list, p) - } - return nil - }) - return list -} - -func TestResizeImage(t *testing.T) { - // Create a test image with specific dimensions - testImage, _ := vips.NewImageFromFile("./pics/png.jpg") - - // Test case 1: Both width and height are greater than 0 - params1 := ExtraParams{ - Width: 100, - Height: 100, - } - err := resizeImage(testImage, params1) - if err != nil { - t.Errorf("Error occurred while resizing image: %v", err) - } - - // Assert the resized image has the expected dimensions - resizedWidth1 := testImage.Width() - resizedHeight1 := testImage.Height() - expectedWidth1 := params1.Width - expectedHeight1 := params1.Height - // If both width and height are provided, follow Width and keep aspect ratio - if resizedWidth1 != expectedWidth1 { - t.Errorf("Resized image dimensions do not match. Expected: %dx%d, Actual: %dx%d", - expectedWidth1, expectedHeight1, resizedWidth1, resizedHeight1) - } - - // Test case 2: Only width is greater than 0 - params2 := ExtraParams{ - Width: 100, - Height: 0, - } - err = resizeImage(testImage, params2) - if err != nil { - t.Errorf("Error occurred while resizing image: %v", err) - } - - // Assert the resized image has the expected width - resizedWidth2 := testImage.Width() - expectedWidth2 := params2.Width - if resizedWidth2 != expectedWidth2 { - t.Errorf("Resized image width does not match. Expected: %d, Actual: %d", - expectedWidth2, resizedWidth2) - } - - // Test case 3: Only height is greater than 0 - params3 := ExtraParams{ - Width: 0, - Height: 100, - } - err = resizeImage(testImage, params3) - if err != nil { - t.Errorf("Error occurred while resizing image: %v", err) - } - - // Assert the resized image has the expected height - resizedHeight3 := testImage.Height() - expectedHeight3 := params3.Height - if resizedHeight3 != expectedHeight3 { - t.Errorf("Resized image height does not match. Expected: %d, Actual: %d", - expectedHeight3, resizedHeight3) - } - -} - -func TestWebPEncoder(t *testing.T) { - // Go through every files - var target = walker() - for _, f := range target { - runEncoder(t, f, dest) - } - _ = os.Remove(dest) -} - -func TestAnimatedGIFWithWebPEncoder(t *testing.T) { - runEncoder(t, "./pics/gif-animated.gif", dest) - _ = os.Remove(dest) - runEncoder(t, "./pics/no.gif", dest) - _ = os.Remove(dest) -} - -func TestAvifEncoder(t *testing.T) { - // Only one file: img_over_16383px.jpg might cause memory issues on CI environment - assert.Nil(t, avifEncoder("./pics/big.jpg", dest, 80, ExtraParams{Width: 0, Height: 0})) - assertType(t, dest, "image/avif") -} - -func TestNonExistImage(t *testing.T) { - assert.NotNil(t, webpEncoder("./pics/empty.jpg", dest, 80, ExtraParams{Width: 0, Height: 0})) - assert.NotNil(t, avifEncoder("./pics/empty.jpg", dest, 80, ExtraParams{Width: 0, Height: 0})) -} - -func TestHighResolutionImage(t *testing.T) { - assert.NotNil(t, webpEncoder("./pics/img_over_16383px.jpg", dest, 80, ExtraParams{Width: 0, Height: 0})) - assert.Nil(t, avifEncoder("./pics/img_over_16383px.jpg", dest, 80, ExtraParams{Width: 0, Height: 0})) -} - -func runEncoder(t *testing.T, file string, dest string) { - if file == "pics/empty.jpg" { - t.Log("Empty file, that's okay.") - } - _ = webpEncoder(file, dest, 80, ExtraParams{Width: 0, Height: 0}) - assertType(t, dest, "image/webp") - -} - -func assertType(t *testing.T, dest, mime string) { - data, _ := os.ReadFile(dest) - types := getFileContentType(data[:512]) - assert.Equalf(t, mime, types, "File %s should be %s", dest, mime) -} diff --git a/go.mod b/go.mod index fbec450..0294839 100644 --- a/go.mod +++ b/go.mod @@ -3,32 +3,35 @@ module webp_server_go go 1.20 require ( + github.com/cespare/xxhash v1.1.0 github.com/davidbyttow/govips/v2 v2.13.0 - github.com/gofiber/fiber/v2 v2.4.0 + github.com/gofiber/fiber/v2 v2.46.0 github.com/h2non/filetype v1.1.3 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/schollz/progressbar/v3 v3.13.1 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.8.4 github.com/valyala/fasthttp v1.47.0 ) require ( github.com/andybalholm/brotli v1.0.5 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/klauspost/compress v1.16.3 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.18 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rivo/uniseg v0.4.3 // indirect + github.com/philhofer/fwd v1.1.2 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect + github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect + github.com/stretchr/testify v1.8.4 // indirect + github.com/tinylib/msgp v1.1.8 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect - golang.org/x/image v0.7.0 // indirect + golang.org/x/image v0.5.0 // indirect golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect ) - -replace github.com/gofiber/fiber/v2 v2.4.0 => github.com/webp-sh/fiber/v2 v2.4.0 diff --git a/go.sum b/go.sum index 34c44cb..a15d44f 100644 --- a/go.sum +++ b/go.sum @@ -1,109 +1,131 @@ -github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidbyttow/govips/v2 v2.13.0 h1:5MK9ZcXZC5GzUR9Ca8fJwOYqMgll/H096ec0PJP59QM= github.com/davidbyttow/govips/v2 v2.13.0/go.mod h1:LPTrwWtNa5n4yl9UC52YBOEGdZcY5hDTP4Ms2QWasTw= +github.com/gofiber/fiber/v2 v2.46.0 h1:wkkWotblsGVlLjXj2dpgKQAYHtXumsK/HyFugQM68Ns= +github.com/gofiber/fiber/v2 v2.46.0/go.mod h1:DNl0/c37WLe0g92U6lx1VMQuxGUQY5V7EIaVoEsUffc= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= -github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= -github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4= +github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8= +github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4= +github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk= +github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g= github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= +github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= +github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c= github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/webp-sh/fiber/v2 v2.4.0 h1:JtkW0HAqHCExodZMZnG7GrLiJuK2YbNYw8eXo55+tr8= -github.com/webp-sh/fiber/v2 v2.4.0/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= -golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw= -golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handler/remote.go b/handler/remote.go new file mode 100644 index 0000000..55669e8 --- /dev/null +++ b/handler/remote.go @@ -0,0 +1,123 @@ +package handler + +import ( + "bytes" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "webp_server_go/config" + "webp_server_go/helper" + + "github.com/gofiber/fiber/v2" + "github.com/h2non/filetype" + log "github.com/sirupsen/logrus" +) + +// Given /path/to/node.png +// Delete /path/to/node.png* +func cleanProxyCache(cacheImagePath string) { + // Delete /node.png* + files, err := filepath.Glob(cacheImagePath + "*") + if err != nil { + log.Infoln(err) + } + for _, f := range files { + if err := os.Remove(f); err != nil { + log.Info(err) + } + } +} + +func downloadFile(filepath string, url string) { + resp, err := http.Get(url) + if err != nil { + log.Errorln("Connection to remote error when downloadFile!") + return + } + defer resp.Body.Close() + + if resp.StatusCode != fiber.StatusOK { + log.Errorf("remote returned %s when fetching remote image", resp.Status) + return + } + + // Copy bytes here + bodyBytes := new(bytes.Buffer) + _, err = bodyBytes.ReadFrom(resp.Body) + if err != nil { + return + } + + // Check if remote content-type is image using check by filetype instead of content-type returned by origin + kind, _ := filetype.Match(bodyBytes.Bytes()) + mime := kind.MIME.Value + if !strings.Contains(mime, "image") { + log.Errorf("remote file %s is not image, remote content has MIME type of %s", url, mime) + return + } + + _ = os.MkdirAll(path.Dir(filepath), 0755) + + // Create Cache here as a lock, so we can prevent incomplete file from being read + // Key: filepath, Value: true + config.WriteLock.Set(filepath, true, -1) + + err = os.WriteFile(filepath, bodyBytes.Bytes(), 0600) + if err != nil { + // not likely to happen + return + } + + // Delete lock here + config.WriteLock.Delete(filepath) + +} + +func fetchRemoteImg(url string) string { + // url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 + // How do we know if the remote img is changed? we're using hash(url+etag) as key. + // if this exists in local system, means the remote img is not changed, we can use it directly. + // otherwise, we need to fetch it from remote and store it in local system. + log.Infof("Remote Addr is %s, pinging for info...", url) + // identifiable is etag + length + identifiable := pingURL(url) + // For store the remote raw image, /home/webp_server/remote-raw/3a42ab801f669d64-b8f999ab5acd69d03f5e904b1b84eb79210536 + // Which 3a42ab801f669d64 is hash(url), b8f999ab5acd69d03f5e904b1b84eb79 is etag and 210536 is length + localRawImagePath := path.Join(config.RemoteRaw, helper.HashString(url)+"-"+identifiable) + + if helper.ImageExists(localRawImagePath) { + return localRawImagePath + } else { + // Temporary store of remote file. + cleanProxyCache(config.RemoteRaw + helper.HashString(url) + "*") + log.Info("Remote file not found in remote-raw, re-fetching...") + downloadFile(localRawImagePath, url) + return localRawImagePath + } + +} + +func pingURL(url string) string { + // this function will try to return identifiable info, currently include etag, content-length as string + // anything goes wrong, will return "" + var etag, length string + resp, err := http.Head(url) + if err != nil { + log.Errorln("Connection to remote error when pingUrl!") + return "" + } + defer resp.Body.Close() + + if resp.StatusCode == fiber.StatusOK { + etag = resp.Header.Get("etag") + length = resp.Header.Get("content-length") + } + if etag == "" { + log.Info("Remote didn't return etag in header when getRemoteImageInfo, please check.") + } + // Remove " from etag + etag = strings.ReplaceAll(etag, "\"", "") + return etag + length +} diff --git a/handler/router.go b/handler/router.go new file mode 100644 index 0000000..e84ffd2 --- /dev/null +++ b/handler/router.go @@ -0,0 +1,104 @@ +package handler + +import ( + "net/http" + "net/url" + "strings" + "webp_server_go/config" + "webp_server_go/encoder" + "webp_server_go/helper" + + "path" + "strconv" + + "github.com/gofiber/fiber/v2" + log "github.com/sirupsen/logrus" +) + +func Convert(c *fiber.Ctx) error { + // this function need to do: + // 1. get request path, query string + // 2. generate rawImagePath, could be local path or remote url(possible with query string) + // 3. pass it to encoder, get the result, send it back + + var ( + reqURI, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg + reqURIwithQuery, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200 + filename = path.Base(reqURI) + ) + + if !helper.CheckAllowedType(filename) { + msg := "File extension not allowed! " + filename + log.Warn(msg) + c.Status(http.StatusBadRequest) + _ = c.Send([]byte(msg)) + return nil + } + + // Sometimes reqURIwithQuery can be https://example.tld/mypic/123.jpg?someother=200&somebugs=200, we need to extract it. + // delete ../ in reqURI to mitigate directory traversal + reqURI = path.Clean(reqURI) + reqURIwithQuery = path.Clean(reqURIwithQuery) + + WidthInt, err := strconv.Atoi(c.Query("width")) + if err != nil { + WidthInt = 0 + } + HeightInt, err := strconv.Atoi(c.Query("height")) + if err != nil { + HeightInt = 0 + } + var extraParams = config.ExtraParams{ + Width: WidthInt, + Height: HeightInt, + } + + var rawImageAbs string + if config.ProxyMode { + // this is proxyMode, we'll have to use this url to download and save it to local path, which also gives us rawImageAbs + // https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 + rawImageAbs = fetchRemoteImg(config.Config.ImgPath + reqURIwithQuery) + + } else { + // not proxyMode, we'll use local path + rawImageAbs = path.Join(config.Config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg + } + + goodFormat := helper.GuessSupportedFormat(&c.Request().Header) + + // Check the original image for existence, + if !helper.ImageExists(rawImageAbs) { + msg := "image not found" + _ = c.Send([]byte(msg)) + log.Warn(msg) + _ = c.SendStatus(404) + return nil + } + + // generate with timestamp to make sure files are update-to-date + // If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + // If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0 + // If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0 + avifAbs, webpAbs := helper.GenOptimizedAbsPath(rawImageAbs, reqURI, extraParams) + encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil) + + var availableFiles = []string{rawImageAbs} + for _, v := range goodFormat { + if v == "avif" { + availableFiles = append(availableFiles, avifAbs) + } + if v == "webp" { + availableFiles = append(availableFiles, webpAbs) + } + } + + finalFilename := helper.FindSmallestFiles(availableFiles) + if strings.HasSuffix(finalFilename, ".webp ") { + c.Set("Content-Type", "image/webp") + } else if strings.HasSuffix(finalFilename, ".avif") { + c.Set("Content-Type", "image/avif") + } + + c.Set("X-Compression-Rate", helper.GetCompressionRate(rawImageAbs, finalFilename)) + return c.SendFile(finalFilename) +} diff --git a/helper.go b/helper.go deleted file mode 100644 index 4b6aa66..0000000 --- a/helper.go +++ /dev/null @@ -1,279 +0,0 @@ -package main - -import ( - "bytes" - "crypto/sha1" //#nosec - "encoding/hex" - "fmt" - "net/http" - "os" - "path" - "path/filepath" - "time" - - "github.com/h2non/filetype" - - "github.com/valyala/fasthttp" - - "strings" - - log "github.com/sirupsen/logrus" -) - -func avifMatcher(buf []byte) bool { - // 0000001c 66747970 61766966 00000000 61766966 6d696631 6d696166 - return len(buf) > 1 && bytes.Equal(buf[:28], []byte{ - 0x0, 0x0, 0x0, 0x1c, - 0x66, 0x74, 0x79, 0x70, - 0x61, 0x76, 0x69, 0x66, - 0x0, 0x0, 0x0, 0x0, - 0x61, 0x76, 0x69, 0x66, - 0x6d, 0x69, 0x66, 0x31, - 0x6d, 0x69, 0x61, 0x66, - }) -} -func getFileContentType(buffer []byte) string { - // TODO deprecated. - var avifType = filetype.NewType("avif", "image/avif") - filetype.AddMatcher(avifType, avifMatcher) - kind, _ := filetype.Match(buffer) - return kind.MIME.Value -} - -func fileCount(dir string) int64 { - var count int64 = 0 - _ = filepath.Walk(dir, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - count += 1 - } - return nil - }) - return count -} - -func imageExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - if info.Size() < 100 { - // means something wrong in exhaust file system - return false - } - - // Check if there is lock in cache, retry after 1 second - maxRetries := 3 - retryDelay := 100 * time.Millisecond // Initial retry delay - - for retry := 0; retry < maxRetries; retry++ { - if _, found := WriteLock.Get(filename); found { - log.Infof("file %s is locked, retrying in %s", filename, retryDelay) - time.Sleep(retryDelay) - retryDelay *= 2 // Exponential backoff - } else { - return !info.IsDir() - } - } - - log.Debugf("file %s exists!", filename) - return !info.IsDir() -} - -func checkAllowedType(imgFilename string) bool { - imgFilename = strings.ToLower(imgFilename) - for _, allowedType := range config.AllowedTypes { - if allowedType == "*" { - return true - } - allowedType = "." + strings.ToLower(allowedType) - if strings.HasSuffix(imgFilename, allowedType) { - return true - } - } - return false -} - -// Check for remote filepath, e.g: https://test.webp.sh/node.png -// return StatusCode, etagValue and length -func getRemoteImageInfo(fileURL string) (int, string, string) { - resp, err := http.Head(fileURL) - if err != nil { - log.Errorln("Connection to remote error when getRemoteImageInfo!") - return http.StatusInternalServerError, "", "" - } - defer resp.Body.Close() - if resp.StatusCode == 200 { - etagValue := resp.Header.Get("etag") - if etagValue == "" { - log.Info("Remote didn't return etag in header when getRemoteImageInfo, please check.") - } else { - return resp.StatusCode, etagValue, resp.Header.Get("content-length") - } - } - - return resp.StatusCode, "", resp.Header.Get("content-length") -} - -func fetchRemoteImage(filepath string, url string) error { - resp, err := http.Get(url) - if err != nil { - log.Errorln("Connection to remote error when fetchRemoteImage!") - return err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("remote returned %s when fetching remote image", resp.Status) - } - - // Copy bytes here - bodyBytes := new(bytes.Buffer) - _, err = bodyBytes.ReadFrom(resp.Body) - if err != nil { - return err - } - - // Check if remote content-type is image using check by filetype instead of content-type returned by origin - kind, _ := filetype.Match(bodyBytes.Bytes()) - if kind == filetype.Unknown || !strings.Contains(kind.MIME.Value, "image") { - return fmt.Errorf("remote file %s is not image, remote content has MIME type of %s", url, kind.MIME.Value) - } - - _ = os.MkdirAll(path.Dir(filepath), 0755) - - // Create Cache here as a lock - // Key: filepath, Value: true - WriteLock.Set(filepath, true, -1) - - err = os.WriteFile(filepath, bodyBytes.Bytes(), 0600) - if err != nil { - return err - } - - // Delete lock here - WriteLock.Delete(filepath) - - return nil -} - -// Given /path/to/node.png -// Delete /path/to/node.png* -func cleanProxyCache(cacheImagePath string) { - // Delete /node.png* - files, err := filepath.Glob(cacheImagePath + "*") - if err != nil { - log.Infoln(err) - } - for _, f := range files { - if err := os.Remove(f); err != nil { - log.Info(err) - } - } -} - -func genOptimizedAbsPath(rawImagePath string, exhaustPath string, imageName string, reqURI string, extraParams ExtraParams) (string, string) { - // get file mod time - STAT, err := os.Stat(rawImagePath) - if err != nil { - log.Error(err.Error()) - return "", "" - } - ModifiedTime := STAT.ModTime().Unix() - // webpFilename: abc.jpg.png -> abc.jpg.png.1582558990.webp - webpFilename := fmt.Sprintf("%s.%d.webp", imageName, ModifiedTime) - // avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif - avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime) - - // If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - // If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0 - // If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0 - if config.EnableExtraParams { - webpFilename = webpFilename + extraParams.String() - avifFilename = avifFilename + extraParams.String() - } - - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - // Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp - webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename)) - avifAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), avifFilename)) - return avifAbsolutePath, webpAbsolutePath -} - -func getCompressionRate(RawImagePath string, optimizedImg string) string { - originFileInfo, err := os.Stat(RawImagePath) - if err != nil { - log.Warnf("Failed to get raw image %v", err) - return "" - } - optimizedFileInfo, err := os.Stat(optimizedImg) - if err != nil { - log.Warnf("Failed to get optimized image %v", err) - return "" - } - compressionRate := float64(optimizedFileInfo.Size()) / float64(originFileInfo.Size()) - log.Debugf("The compression rate is %d/%d=%.2f", originFileInfo.Size(), optimizedFileInfo.Size(), compressionRate) - return fmt.Sprintf(`%.2f`, compressionRate) -} - -func guessSupportedFormat(header *fasthttp.RequestHeader) []string { - var supported = map[string]bool{ - "raw": true, - "webp": false, - "avif": false} - - var ua = string(header.Peek("user-agent")) - var accept = strings.ToLower(string(header.Peek("accept"))) - log.Debugf("%s\t%s\n", ua, accept) - - if strings.Contains(accept, "image/webp") { - supported["webp"] = true - } - if strings.Contains(accept, "image/avif") { - supported["avif"] = true - } - - // chrome on iOS will not send valid image accept header - if strings.Contains(ua, "iPhone OS 14") || strings.Contains(ua, "CPU OS 14") || - strings.Contains(ua, "iPhone OS 15") || strings.Contains(ua, "CPU OS 15") { - supported["webp"] = true - } else if strings.Contains(ua, "Android") || strings.Contains(ua, "Linux") { - supported["webp"] = true - } - - var accepted []string - for k, v := range supported { - if v { - accepted = append(accepted, k) - } - } - return accepted -} - -func findSmallestFiles(files []string) string { - // walk files - var small int64 - var final string - for _, f := range files { - stat, err := os.Stat(f) - if err != nil { - log.Warnf("%s not found on filesystem", f) - continue - } - if stat.Size() < small || small == 0 { - small = stat.Size() - final = f - } - } - return final -} - -func Sha1Path(uri string) string { - /* #nosec */ - h := sha1.New() - h.Write([]byte(uri)) - return hex.EncodeToString(h.Sum(nil)) -} diff --git a/helper/helper.go b/helper/helper.go new file mode 100644 index 0000000..fecea57 --- /dev/null +++ b/helper/helper.go @@ -0,0 +1,182 @@ +package helper + +import ( + "fmt" + "github.com/cespare/xxhash" + "github.com/valyala/fasthttp" + "os" + "path" + "path/filepath" + "strings" + "time" + "webp_server_go/config" + + log "github.com/sirupsen/logrus" +) + +func FileCount(dir string) int64 { + var count int64 = 0 + _ = filepath.Walk(dir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + count += 1 + } + return nil + }) + return count +} + +func ImageExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + // if file size is less than 100 bytes, we assume it's invalid file + // png starts with an 8-byte signature, follow by 4 chunks 58 bytes. + // JPG is 134 bytes. + // webp is 33 bytes. + if info.Size() < 100 { + // means something wrong in exhaust file system + return false + } + + // Check if there is lock in cache, retry after 1 second + maxRetries := 3 + retryDelay := 100 * time.Millisecond // Initial retry delay + + for retry := 0; retry < maxRetries; retry++ { + if _, found := config.WriteLock.Get(filename); found { + log.Infof("file %s is locked, retrying in %s", filename, retryDelay) + time.Sleep(retryDelay) + retryDelay *= 2 // Exponential backoff + } else { + return !info.IsDir() + } + } + + return !info.IsDir() +} + +func CheckAllowedType(imgFilename string) bool { + for _, allowedType := range config.Config.AllowedTypes { + if allowedType == "*" { + return true + } + allowedType = "." + strings.ToLower(allowedType) + if strings.HasSuffix(strings.ToLower(imgFilename), allowedType) { + return true + } + } + return false +} + +func GenOptimizedAbsPath(rawImagePath, reqURI string, extraParams config.ExtraParams) (string, string) { + // imageName is not needed, we can use reqURI + // get file mod time + var ( + imageName = path.Base(reqURI) + exhaustPath = config.Config.ExhaustPath + ) + STAT, err := os.Stat(rawImagePath) + if err != nil { + log.Error(err.Error()) + return "", "" + } + ModifiedTime := STAT.ModTime().Unix() + // TODO: just hash it? + // webpFilename: abc.jpg.png -> abc.jpg.png.1582558990.webp + webpFilename := fmt.Sprintf("%s.%d.webp", imageName, ModifiedTime) + // avifFilename: abc.jpg.png -> abc.jpg.png.1582558990.avif + avifFilename := fmt.Sprintf("%s.%d.avif", imageName, ModifiedTime) + + // If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + // If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0 + // If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0 + if config.Config.EnableExtraParams { + webpFilename = webpFilename + extraParams.String() + avifFilename = avifFilename + extraParams.String() + } + + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + // Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp + webpAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), webpFilename)) + avifAbsolutePath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), avifFilename)) + return avifAbsolutePath, webpAbsolutePath +} + +func GetCompressionRate(RawImagePath string, optimizedImg string) string { + originFileInfo, err := os.Stat(RawImagePath) + if err != nil { + log.Warnf("Failed to get raw image %v", err) + return "" + } + optimizedFileInfo, err := os.Stat(optimizedImg) + if err != nil { + log.Warnf("Failed to get optimized image %v", err) + return "" + } + compressionRate := float64(optimizedFileInfo.Size()) / float64(originFileInfo.Size()) + return fmt.Sprintf(`%.2f`, compressionRate) +} + +func GuessSupportedFormat(header *fasthttp.RequestHeader) []string { + var ( + supported = map[string]bool{ + "raw": true, + "webp": false, + "avif": false, + } + + ua = string(header.Peek("user-agent")) + accept = strings.ToLower(string(header.Peek("accept"))) + ) + + if strings.Contains(accept, "image/webp") { + supported["webp"] = true + } + if strings.Contains(accept, "image/avif") { + supported["avif"] = true + } + + // chrome on iOS will not send valid image accept header + if strings.Contains(ua, "iPhone OS 14") || strings.Contains(ua, "CPU OS 14") || + strings.Contains(ua, "iPhone OS 15") || strings.Contains(ua, "CPU OS 15") || + strings.Contains(ua, "Android") || strings.Contains(ua, "Linux") { + supported["webp"] = true + } + + // save true value's key to slice + var accepted []string + for k, v := range supported { + if v { + accepted = append(accepted, k) + } + } + return accepted +} + +func FindSmallestFiles(files []string) string { + // walk files + var small int64 + var final string + for _, f := range files { + stat, err := os.Stat(f) + if err != nil { + log.Warnf("%s not found on filesystem", f) + continue + } + if stat.Size() < small || small == 0 { + small = stat.Size() + final = f + } + } + return final +} + +func HashString(uri string) string { + // xxhash supports cross compile + return fmt.Sprintf("%x", xxhash.Sum64String(uri)) +} diff --git a/helper_test.go b/helper_test.go deleted file mode 100644 index 988d7e3..0000000 --- a/helper_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package main - -import ( - "fmt" - "path" - "time" - - log "github.com/sirupsen/logrus" - "github.com/valyala/fasthttp" - - "net/http" - "os" - "path/filepath" - "sort" - "testing" - - "github.com/stretchr/testify/assert" -) - -// test all files: go test -v -cover . -// test one case: go test -v -run TestSelectFormat - -func TestGetFileContentType(t *testing.T) { - var data = []byte("remember remember the 5th of november") - var expected = "" - var result = getFileContentType(data) - assert.Equalf(t, result, expected, "Result: [%s], Expected: [%s]", result, expected) -} - -func TestFileCount(t *testing.T) { - var data = "scripts" - var expected int64 = 2 - var result = fileCount(data) - assert.Equalf(t, result, expected, "Result: [%d], Expected: [%d]", result, expected) -} - -func TestImageExists(t *testing.T) { - var data = "./pics/empty.jpg" - var result = imageExists(data) - - if result { - t.Errorf("Result: [%v], Expected: [%v]", result, false) - } - data = ".pics/empty2.jpg" - result = imageExists(data) - - assert.Falsef(t, result, "Result: [%v], Expected: [%v]", result, false) - -} - -func TestGenOptimizedAbsPath(t *testing.T) { - // Create a temporary file for testing - tempFile, err := os.CreateTemp("", "test_image.*") - if err != nil { - t.Fatalf("Failed to create temporary file: %v", err) - } - defer os.Remove(tempFile.Name()) - - // Set the modification time for the temporary file - modTime := time.Now() - if err := os.Chtimes(tempFile.Name(), modTime, modTime); err != nil { - t.Fatalf("Failed to set modification time for the temporary file: %v", err) - } - - rawImagePath := tempFile.Name() - exhaustPath := "/path/to/exhaust" - imageName := "tsuki.jpg" - reqURI := "/path/to/tsuki.jpg" - extraParams := ExtraParams{Width: 200, Height: 0} - - // Test if config.EnableExtraParams is false - config.EnableExtraParams = false - - avifAbsolutePath, webpAbsolutePath := genOptimizedAbsPath(rawImagePath, exhaustPath, imageName, reqURI, extraParams) - - expectedAvifPath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.avif", imageName, modTime.Unix()))) - expectedWebpPath := path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.webp", imageName, modTime.Unix()))) - - if avifAbsolutePath != expectedAvifPath { - t.Errorf("Avif absolute path is incorrect. Expected: %s, Got: %s", expectedAvifPath, avifAbsolutePath) - } - if webpAbsolutePath != expectedWebpPath { - t.Errorf("Webp absolute path is incorrect. Expected: %s, Got: %s", expectedWebpPath, webpAbsolutePath) - } - - // Test if config.EnableExtraParams is true and extraParams is not 0 - config.EnableExtraParams = true - - avifAbsolutePath, webpAbsolutePath = genOptimizedAbsPath(rawImagePath, exhaustPath, imageName, reqURI, extraParams) - - expectedAvifPath = path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.avif_width=%d&height=%d", imageName, modTime.Unix(), extraParams.Width, extraParams.Height))) - expectedWebpPath = path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.webp_width=%d&height=%d", imageName, modTime.Unix(), extraParams.Width, extraParams.Height))) - - if avifAbsolutePath != expectedAvifPath { - t.Errorf("Avif absolute path is incorrect. Expected: %s, Got: %s", expectedAvifPath, avifAbsolutePath) - } - if webpAbsolutePath != expectedWebpPath { - t.Errorf("Webp absolute path is incorrect. Expected: %s, Got: %s", expectedWebpPath, webpAbsolutePath) - } - - // Test if config.EnableExtraParams is true and extraParams is 0 - config.EnableExtraParams = true - extraParams = ExtraParams{Width: 200, Height: 0} - - avifAbsolutePath, webpAbsolutePath = genOptimizedAbsPath(rawImagePath, exhaustPath, imageName, reqURI, extraParams) - - expectedAvifPath = path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.avif_width=%d&height=%d", imageName, modTime.Unix(), extraParams.Width, extraParams.Height))) - expectedWebpPath = path.Clean(path.Join(exhaustPath, path.Dir(reqURI), fmt.Sprintf("%s.%d.webp_width=%d&height=%d", imageName, modTime.Unix(), extraParams.Width, extraParams.Height))) - - if avifAbsolutePath != expectedAvifPath { - t.Errorf("Avif absolute path is incorrect. Expected: %s, Got: %s", expectedAvifPath, avifAbsolutePath) - } - if webpAbsolutePath != expectedWebpPath { - t.Errorf("Webp absolute path is incorrect. Expected: %s, Got: %s", expectedWebpPath, webpAbsolutePath) - } -} - -func TestSelectFormat(t *testing.T) { - // this is a complete test case for webp compatibility - // func goOrigin(header, ua string) bool - // UNLESS YOU KNOW WHAT YOU ARE DOING, DO NOT CHANGE THE TEST CASE MAPPING HERE. - var fullSupport = []string{"avif", "webp", "raw"} - var webpSupport = []string{"webp", "raw"} - var jpegSupport = []string{"raw"} - var testCase = map[[2]string][]string{ - // Latest Chrome on Windows, macOS, linux, Android and iOS 13 - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"}: fullSupport, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.60 Mobile Safari/537.36"}: fullSupport, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Mozilla/5.0 (Linux; Android 6.0; HTC M8t) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.74 Mobile Safari/537.36"}: fullSupport, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "HTCM8t_LTE/1.0 Android/4.4 release/2013 Browser/WAP2.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 AppleWebKit/534.30"}: webpSupport, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 13_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/83.0.4103.63 Mobile/15E148 Safari/604.1"}: jpegSupport, - - // macOS Catalina Safari - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15"}: jpegSupport, - - // iOS14 Safari and Chrome - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: webpSupport, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: webpSupport, - - // iPadOS 14 Safari and Chrome - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Mobile/15E148 Safari/604.1"}: webpSupport, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPad; CPU OS 14_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1"}: webpSupport, - - // iOS 15 Safari, Firefox and Chrome - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Mobile/15E148 Safari/604.1"}: webpSupport, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/39.0 Mobile/15E148 Safari/605.1.15"}: webpSupport, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/96.0.4664.53 Mobile/15E148 Safari/604.1"}: webpSupport, - - // IE - [2]string{"application/x-ms-application, image/jpeg, application/xaml+xml, image/gif, image/pjpeg, application/x-ms-xbap, */*", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko"}: jpegSupport, - // Others - [2]string{"", "PostmanRuntime/7.26.1"}: jpegSupport, - [2]string{"", "curl/7.64.1"}: jpegSupport, - [2]string{"image/webp", "curl/7.64.1"}: webpSupport, - [2]string{"image/avif,image/webp", "curl/7.64.1"}: fullSupport, - - // some weird browsers - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.16(0x18001033) NetType/WIFI Language/zh_CN"}: webpSupport, - [2]string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Mozilla/5.0 (Linux; Android 6.0; HTC M8t Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/45.0.2454.95 Mobile Safari/537.36 MMWEBID/4285 MicroMessenger/8.0.15.2001(0x28000F41) Process/tools WeChat/arm32 Weixin GPVersion/1 NetType/WIFI Language/zh_CN ABI/arm32"}: webpSupport, - } - for browser, expected := range testCase { - var header fasthttp.RequestHeader - header.Set("accept", browser[0]) - header.Set("user-agent", browser[1]) - guessed := guessSupportedFormat(&header) - - sort.Strings(expected) - sort.Strings(guessed) - log.Infof("%s expected%s --- actual %s", browser, expected, guessed) - assert.Equal(t, expected, guessed) - } - -} - -func TestGetRemoteImageInfo(t *testing.T) { - url := "https://github.com/favicon.ico" - statusCode, etag, length := getRemoteImageInfo(url) - assert.NotEqual(t, "", etag) - assert.NotEqual(t, "0", length) - assert.Equal(t, statusCode, http.StatusOK) - - // test non-exist url - url = "http://sdahjajda.com" - statusCode, etag, length = getRemoteImageInfo(url) - assert.Equal(t, "", etag) - assert.Equal(t, "", length) - assert.Equal(t, statusCode, http.StatusInternalServerError) -} - -func TestFetchRemoteImage(t *testing.T) { - // test the normal one - fp := filepath.Join("./exhaust", "test.ico") - - err := fetchRemoteImage(fp, "http://github.com/favicon.ico") - assert.Equal(t, err, nil) - data, _ := os.ReadFile(fp) - assert.Equal(t, "image/vnd.microsoft.icon", getFileContentType(data)) - - // test can't create file - err = fetchRemoteImage("/", "http://github.com/favicon.ico") - assert.NotNil(t, err) - - // test bad url - err = fetchRemoteImage(fp, "http://ahjdsgdsghja.cya") - assert.NotNil(t, err) - -} - -func TestCleanProxyCache(t *testing.T) { - // test normal situation - fp := filepath.Join("./exhaust", "sample.png.12345.webp") - buf := make([]byte, 0x1000) - _ = os.WriteFile(fp, buf, 0755) - assert.True(t, imageExists(fp)) - cleanProxyCache(fp) - assert.False(t, imageExists(fp)) - - // test bad dir - cleanProxyCache("/aasdyg/dhj2/dagh") -} - -func TestGetCompressionRate(t *testing.T) { - pic1 := "pics/webp_server.bmp" - pic2 := "pics/webp_server.jpg" - var ratio string - - ratio = getCompressionRate(pic1, pic2) - assert.Equal(t, "0.16", ratio) - - ratio = getCompressionRate(pic1, "pic2") - assert.Equal(t, "", ratio) - - ratio = getCompressionRate("pic1", pic2) - assert.Equal(t, "", ratio) -} diff --git a/tests/glib_malloc/docker-compose.yml b/malloc_tests/glib_malloc/docker-compose.yml similarity index 100% rename from tests/glib_malloc/docker-compose.yml rename to malloc_tests/glib_malloc/docker-compose.yml diff --git a/tests/jemalloc/docker-compose.yml b/malloc_tests/jemalloc/docker-compose.yml similarity index 100% rename from tests/jemalloc/docker-compose.yml rename to malloc_tests/jemalloc/docker-compose.yml diff --git a/prefetch_test.go b/prefetch_test.go deleted file mode 100644 index ec8ea80..0000000 --- a/prefetch_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestPrefetchImages(t *testing.T) { - fp := "./prefetch" - _ = os.Mkdir(fp, 0755) - prefetchImages("./pics/dir1/", "./prefetch") - count := fileCount("./prefetch") - assert.Equal(t, int64(1), count) - _ = os.RemoveAll(fp) -} - -func TestBadPrefetch(t *testing.T) { - jobs = 1 - prefetchImages("./pics2", "./prefetch") -} diff --git a/router.go b/router.go deleted file mode 100644 index 524c600..0000000 --- a/router.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "net/http" - "net/url" - - "path" - "strconv" - - "github.com/gofiber/fiber/v2" - log "github.com/sirupsen/logrus" -) - -func convert(c *fiber.Ctx) error { - //basic vars - var ( - reqURI, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg - reqURIwithQuery, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200 - imgFilename = path.Base(reqURI) // pure filename, 123.jpg - ) - // Sometimes reqURIwithQuery can be https://example.tld/mypic/123.jpg?someother=200&somebugs=200, we need to extract it. - u, err := url.Parse(reqURIwithQuery) - if err != nil { - log.Errorln(err) - } - reqURIwithQuery = u.RequestURI() - // delete ../ in reqURI to mitigate directory traversal - reqURI = path.Clean(reqURI) - reqURIwithQuery = path.Clean(reqURIwithQuery) - - // Begin Extra params - var extraParams ExtraParams - Width := c.Query("width") - Height := c.Query("height") - WidthInt, err := strconv.Atoi(Width) - if err != nil { - WidthInt = 0 - } - HeightInt, err := strconv.Atoi(Height) - if err != nil { - HeightInt = 0 - } - extraParams = ExtraParams{ - Width: WidthInt, - Height: HeightInt, - } - // End Extra params - - var rawImageAbs string - if proxyMode { - rawImageAbs = config.ImgPath + reqURIwithQuery // https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 - } else { - rawImageAbs = path.Join(config.ImgPath, reqURI) // /home/xxx/mypic/123.jpg - } - log.Debugf("Incoming connection from %s %s", c.IP(), imgFilename) - - if !checkAllowedType(imgFilename) { - msg := "File extension not allowed! " + imgFilename - log.Warn(msg) - c.Status(http.StatusBadRequest) - _ = c.Send([]byte(msg)) - return nil - } - - goodFormat := guessSupportedFormat(&c.Request().Header) - - if proxyMode { - rawImageAbs, _ = proxyHandler(c, reqURIwithQuery) - } - - log.Debugf("rawImageAbs=%s", rawImageAbs) - - // Check the original image for existence, - if !imageExists(rawImageAbs) { - msg := "image not found" - _ = c.Send([]byte(msg)) - log.Warn(msg) - _ = c.SendStatus(404) - return nil - } - - // generate with timestamp to make sure files are update-to-date - // If extraParams not enabled, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - // If extraParams enabled, and given request at tsuki.jpg?width=200, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=200&height=0 - // If extraParams enabled, and given request at tsuki.jpg, exhaust path will be /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp_width=0&height=0 - avifAbs, webpAbs := genOptimizedAbsPath(rawImageAbs, config.ExhaustPath, imgFilename, reqURI, extraParams) - convertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil) - - var availableFiles = []string{rawImageAbs} - for _, v := range goodFormat { - if v == "avif" { - availableFiles = append(availableFiles, avifAbs) - } - if v == "webp" { - availableFiles = append(availableFiles, webpAbs) - } - } - - var finalFileName = findSmallestFiles(availableFiles) - var finalFileExtension = path.Ext(finalFileName) - if finalFileExtension == ".webp" { - c.Set("Content-Type", "image/webp") - } else if finalFileExtension == ".avif" { - c.Set("Content-Type", "image/avif") - } - - c.Set("X-Compression-Rate", getCompressionRate(rawImageAbs, finalFileName)) - return c.SendFile(finalFileName) -} - -func proxyHandler(c *fiber.Ctx, reqURIwithQuery string) (string, error) { - // https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200 - realRemoteAddr := config.ImgPath + reqURIwithQuery - - // Ping Remote for status code and etag info - log.Infof("Remote Addr is %s, fetching info...", realRemoteAddr) - statusCode, etagValue, _ := getRemoteImageInfo(realRemoteAddr) - - // Since we cannot store file in format of "/mypic/123.jpg?someother=200&somebugs=200", we need to hash it. - reqURIwithQueryHash := Sha1Path(reqURIwithQuery) // 378e740ca56144b7587f3af9debeee544842879a - etagValueHash := Sha1Path(etagValue) // 123e740ca56333b7587f3af9debeee5448428123 - - localRawImagePath := path.Join(remoteRaw, reqURIwithQueryHash+"-etag-"+etagValueHash) // For store the remote raw image, /home/webp_server/remote-raw/378e740ca56144b7587f3af9debeee544842879a-etag-123e740ca56333b7587f3af9debeee5448428123 - - if statusCode == 200 { - if imageExists(localRawImagePath) { - return localRawImagePath, nil - } else { - // Temporary store of remote file. - cleanProxyCache(config.ExhaustPath + reqURIwithQuery + "*") - log.Info("Remote file not found in remote-raw path, fetching...") - err := fetchRemoteImage(localRawImagePath, realRemoteAddr) - return localRawImagePath, err - } - } else { - msg := fmt.Sprintf("Remote returned %d status code!", statusCode) - _ = c.Send([]byte(msg)) - log.Warn(msg) - _ = c.SendStatus(statusCode) - cleanProxyCache(config.ExhaustPath + reqURIwithQuery + "*") - return "", errors.New(msg) - } -} diff --git a/router_test.go b/router_test.go deleted file mode 100644 index d414933..0000000 --- a/router_test.go +++ /dev/null @@ -1,248 +0,0 @@ -package main - -import ( - "io" - "net/http" - "net/http/httptest" - "os" - "testing" - "time" - - "github.com/gofiber/fiber/v2" - "github.com/gofiber/fiber/v2/middleware/etag" - "github.com/patrickmn/go-cache" - "github.com/stretchr/testify/assert" -) - -var ( - chromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36" - acceptWebP = "image/webp,image/apng,image/*,*/*;q=0.8" - acceptAvif = "image/avif,image/*,*/*;q=0.8" - acceptLegacy = "image/jpeg" - safariUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15" - curlUA = "curl/7.64.1" -) - -func TestMain(m *testing.M) { - // setup parameters here... - config.ImgPath = "./pics" - config.ExhaustPath = "./exhaust_test" - config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp"} - - proxyMode = false - remoteRaw = "remote-raw" - WriteLock = cache.New(5*time.Minute, 10*time.Minute) - m.Run() -} - -func requestToServer(url string, app *fiber.App, ua, accept string) (*http.Response, []byte) { - headers := make(map[string]string) - headers["User-Agent"] = ua - headers["Accept"] = accept - return requestToServerHeaders(url, app, headers) -} - -func requestToServerHeaders(url string, app *fiber.App, headers map[string]string) (*http.Response, []byte) { - req := httptest.NewRequest("GET", url, nil) - for header, value := range headers { - req.Header.Set(header, value) - } - resp, err := app.Test(req, 120000) - if err != nil { - return nil, nil - } - data, _ := io.ReadAll(resp.Body) - return resp, data -} - -func TestServerHeaders(t *testing.T) { - var app = fiber.New() - app.Use(etag.New(etag.Config{ - Weak: true, - })) - app.Get("/*", convert) - url := "http://127.0.0.1:3333/webp_server.bmp" - - // test for chrome - response, _ := requestToServer(url, app, chromeUA, acceptWebP) - defer response.Body.Close() - ratio := response.Header.Get("X-Compression-Rate") - etag := response.Header.Get("Etag") - lastModified := response.Header.Get("Last-Modified") - - assert.NotEqual(t, "", ratio) - assert.NotEqual(t, "", etag) - assert.NotEqual(t, "", lastModified) - - // TestServerHeadersNotModified - var headers = map[string]string{ - "User-Agent": chromeUA, - "Accept": acceptWebP, - "If-None-Match": etag, - } - response, _ = requestToServerHeaders(url, app, headers) - defer response.Body.Close() - assert.Equal(t, 304, response.StatusCode) - - headers["If-Modified-Since"] = lastModified - response, _ = requestToServerHeaders(url, app, headers) - defer response.Body.Close() - assert.Equal(t, 304, response.StatusCode) - - headers["If-None-Match"] = "" - headers["If-Modified-Since"] = lastModified - response, _ = requestToServerHeaders(url, app, headers) - defer response.Body.Close() - assert.Equal(t, 304, response.StatusCode) - - // test for safari - response, _ = requestToServer(url, app, safariUA, acceptLegacy) - defer response.Body.Close() - // ratio = response.Header.Get("X-Compression-Rate") - etag = response.Header.Get("Etag") - lastModified = response.Header.Get("Last-Modified") - - assert.NotEqual(t, "", etag) - assert.NotEqual(t, "", lastModified) -} - -func TestConvert(t *testing.T) { - // TODO: old-style test, better update it with accept headers - var testChromeLink = map[string]string{ - "http://127.0.0.1:3333/webp_server.jpg": "image/webp", - "http://127.0.0.1:3333/webp_server.bmp": "image/webp", - "http://127.0.0.1:3333/webp_server.png": "image/webp", - "http://127.0.0.1:3333/empty.jpg": "", - "http://127.0.0.1:3333/png.jpg": "image/webp", - "http://127.0.0.1:3333/12314.jpg": "", - "http://127.0.0.1:3333/dir1/inside.jpg": "image/webp", - "http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/webp", - "http://127.0.0.1:3333/太神啦.png": "image/webp", - } - - var testChromeAvifLink = map[string]string{ - "http://127.0.0.1:3333/webp_server.jpg": "image/avif", - "http://127.0.0.1:3333/webp_server.bmp": "image/avif", - "http://127.0.0.1:3333/webp_server.png": "image/avif", - "http://127.0.0.1:3333/empty.jpg": "", - "http://127.0.0.1:3333/png.jpg": "image/avif", - "http://127.0.0.1:3333/12314.jpg": "", - "http://127.0.0.1:3333/dir1/inside.jpg": "image/avif", - "http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/avif", - "http://127.0.0.1:3333/太神啦.png": "image/avif", - } - - var testSafariLink = map[string]string{ - "http://127.0.0.1:3333/webp_server.jpg": "image/jpeg", - "http://127.0.0.1:3333/webp_server.bmp": "image/bmp", - "http://127.0.0.1:3333/webp_server.png": "image/png", - "http://127.0.0.1:3333/empty.jpg": "", - "http://127.0.0.1:3333/png.jpg": "image/png", - "http://127.0.0.1:3333/12314.jpg": "", - "http://127.0.0.1:3333/dir1/inside.jpg": "image/jpeg", - } - - var app = fiber.New() - app.Get("/*", convert) - - // test Chrome - for url, respType := range testChromeLink { - resp, data := requestToServer(url, app, chromeUA, acceptWebP) - defer resp.Body.Close() - contentType := getFileContentType(data) - assert.Equal(t, respType, contentType) - } - - // test Safari - for url, respType := range testSafariLink { - resp, data := requestToServer(url, app, safariUA, acceptLegacy) - defer resp.Body.Close() - contentType := getFileContentType(data) - assert.Equal(t, respType, contentType) - } - - // test Avif is processed in proxy mode - config.EnableAVIF = true - for url, respType := range testChromeAvifLink { - resp, data := requestToServer(url, app, chromeUA, acceptAvif) - defer resp.Body.Close() - contentType := getFileContentType(data) - assert.NotNil(t, respType) - assert.Equal(t, respType, contentType) - } -} - -func TestConvertNotAllowed(t *testing.T) { - config.AllowedTypes = []string{"jpg", "png", "jpeg"} - - var app = fiber.New() - app.Get("/*", convert) - - // not allowed, but we have the file, this should return File extension not allowed - url := "http://127.0.0.1:3333/webp_server.bmp" - resp, data := requestToServer(url, app, chromeUA, acceptWebP) - defer resp.Body.Close() - assert.Contains(t, string(data), "File extension not allowed") - - // not allowed, random file - url = url + "hagdgd" - resp, data = requestToServer(url, app, chromeUA, acceptWebP) - defer resp.Body.Close() - assert.Contains(t, string(data), "File extension not allowed") - -} - -func TestConvertProxyModeBad(t *testing.T) { - proxyMode = true - - var app = fiber.New() - app.Get("/*", convert) - - // this is local random image, should be 404 - url := "http://127.0.0.1:3333/webp_8888server.jpg" - resp, _ := requestToServer(url, app, chromeUA, acceptWebP) - defer resp.Body.Close() - assert.Equal(t, http.StatusNotFound, resp.StatusCode) - - // this is local random image, test using cURL, should be 404, ref: https://github.com/webp-sh/webp_server_go/issues/197 - resp1, _ := requestToServer(url, app, curlUA, acceptWebP) - defer resp1.Body.Close() - assert.Equal(t, http.StatusNotFound, resp1.StatusCode) -} - -func TestConvertProxyModeWork(t *testing.T) { - proxyMode = true - - var app = fiber.New() - app.Get("/*", convert) - - config.ImgPath = "https://webp.sh" - url := "https://webp.sh/images/cover.jpg" - - resp, data := requestToServer(url, app, chromeUA, acceptWebP) - defer resp.Body.Close() - assert.Equal(t, http.StatusOK, resp.StatusCode) - assert.Equal(t, "image/webp", getFileContentType(data)) - - // test proxyMode with Safari - resp, data = requestToServer(url, app, safariUA, acceptLegacy) - defer resp.Body.Close() - assert.Equal(t, http.StatusOK, resp.StatusCode) - assert.Equal(t, "image/jpeg", getFileContentType(data)) -} - -func TestConvertBigger(t *testing.T) { - proxyMode = false - config.Quality = 100 - config.ImgPath = "./pics" - - var app = fiber.New() - app.Get("/*", convert) - - url := "http://127.0.0.1:3333/big.jpg" - resp, data := requestToServer(url, app, chromeUA, acceptWebP) - defer resp.Body.Close() - assert.Equal(t, "image/jpeg", resp.Header.Get("content-type")) - assert.Equal(t, "image/jpeg", getFileContentType(data)) - _ = os.RemoveAll(config.ExhaustPath) -} diff --git a/webp-server.go b/webp-server.go index ee75089..b81b5c9 100644 --- a/webp-server.go +++ b/webp-server.go @@ -1,77 +1,50 @@ package main import ( - "encoding/json" - "flag" "fmt" "os" - "regexp" "runtime" - "time" + "webp_server_go/config" + "webp_server_go/encoder" + "webp_server_go/handler" - "github.com/davidbyttow/govips/v2/vips" "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/etag" "github.com/gofiber/fiber/v2/middleware/logger" - "github.com/patrickmn/go-cache" log "github.com/sirupsen/logrus" ) -var WriteLock *cache.Cache +var app = fiber.New(fiber.Config{ + ServerHeader: "Webp Server Go", + AppName: "Webp Server Go", + DisableStartupMessage: true, + ProxyHeader: "X-Real-IP", +}) -func loadConfig(path string) Config { - jsonObject, err := os.Open(path) - if err != nil { - log.Fatal(err) - } - decoder := json.NewDecoder(jsonObject) - _ = decoder.Decode(&config) - _ = jsonObject.Close() - return config -} - -func deferInit() { - flag.StringVar(&configPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)") - flag.BoolVar(&prefetch, "prefetch", false, "Prefetch and convert image to webp") - flag.IntVar(&jobs, "jobs", runtime.NumCPU(), "Prefetch thread, default is all.") - flag.BoolVar(&dumpConfig, "dump-config", false, "Print sample config.json") - flag.BoolVar(&dumpSystemd, "dump-systemd", false, "Print sample systemd service file.") - flag.BoolVar(&verboseMode, "v", false, "Verbose, print out debug info.") - flag.BoolVar(&showVersion, "V", false, "Show version information.") - flag.Parse() - // Logrus +func setupLogger() { log.SetOutput(os.Stdout) log.SetReportCaller(true) - Formatter := &log.TextFormatter{ + formatter := &log.TextFormatter{ EnvironmentOverrideColors: true, FullTimestamp: true, - TimestampFormat: "2006-01-02 15:04:05", + TimestampFormat: config.TimeDateFormat, CallerPrettyfier: func(f *runtime.Frame) (string, string) { - return fmt.Sprintf("[%d:%s()]", f.Line, f.Function), "" + return fmt.Sprintf("[%d:%s]", f.Line, f.Function), "" }, } - log.SetFormatter(Formatter) + log.SetFormatter(formatter) + log.SetLevel(log.InfoLevel) - if verboseMode { - log.SetLevel(log.DebugLevel) - log.Debug("Debug mode is enabled!") - } else { - log.SetLevel(log.InfoLevel) - } + // fiber logger format + app.Use(logger.New(logger.Config{ + Format: config.FiberLogFormat, + TimeFormat: config.TimeDateFormat, + })) + log.Infoln("Logger ready.") } -func switchProxyMode() { - // Check for remote address - matched, _ := regexp.MatchString(`^https?://`, config.ImgPath) - proxyMode = false - if matched { - proxyMode = true - } else { - _, err := os.Stat(config.ImgPath) - if err != nil { - log.Fatalf("Your image path %s is incorrect.Please check and confirm.", config.ImgPath) - } - } +func init() { + setupLogger() } func main() { @@ -83,48 +56,32 @@ func main() { ▘ ▘▝▀▘▀▀ ▘ ▝▀ ▝▀▘▘ ▘ ▝▀▘▘ ▝▀ ▝▀ Webp Server Go - v%s -Develop by WebP Server team. https://github.com/webp-sh`, version) +Develop by WebP Server team. https://github.com/webp-sh`, config.Version) - deferInit() // process cli params - if dumpConfig { - fmt.Println(sampleConfig) + if config.DumpConfig { + fmt.Println(config.SampleConfig) os.Exit(0) } - if dumpSystemd { - fmt.Println(sampleSystemd) + if config.DumpSystemd { + fmt.Println(config.SampleSystemd) os.Exit(0) } - if showVersion { + if config.ShowVersion { fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner+"", 0x1B) os.Exit(0) } - config = loadConfig(configPath) - switchProxyMode() - - vips.Startup(&vips.Config{ - ConcurrencyLevel: runtime.NumCPU(), - }) - defer vips.Shutdown() - - WriteLock = cache.New(5*time.Minute, 10*time.Minute) - - if prefetch { - go prefetchImages(config.ImgPath, config.ExhaustPath) + if config.Prefetch { + go encoder.PrefetchImages() } - app := fiber.New(fiber.Config{ - ServerHeader: "Webp Server Go", - DisableStartupMessage: true, - }) app.Use(etag.New(etag.Config{ Weak: true, })) - app.Use(logger.New()) - listenAddress := config.Host + ":" + config.Port - app.Get("/*", convert) + listenAddress := config.Config.Host + ":" + config.Config.Port + app.Get("/*", handler.Convert) fmt.Printf("\n %c[1;32m%s%c[0m\n\n", 0x1B, banner, 0x1B) fmt.Println("Webp Server Go is Running on http://" + listenAddress) diff --git a/webp-server_test.go b/webp-server_test.go deleted file mode 100644 index 316dda2..0000000 --- a/webp-server_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package main - -import ( - "net" - "os" - "runtime" - "testing" - "time" - - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" -) - -// due to test limit, we can't test for cli param part. - -func TestLoadConfig(t *testing.T) { - c := loadConfig("./config.json") - assert.Equal(t, "./exhaust", c.ExhaustPath) - assert.Equal(t, "127.0.0.1", c.Host) - assert.Equal(t, "3333", c.Port) - assert.Equal(t, 80, c.Quality) - assert.Equal(t, "./pics", c.ImgPath) - assert.Equal(t, []string{"jpg", "png", "jpeg", "bmp", "gif"}, c.AllowedTypes) -} - -func TestDeferInit(t *testing.T) { - // test initial value - assert.Equal(t, "", configPath) - assert.False(t, prefetch) - assert.Equal(t, false, dumpSystemd) - assert.Equal(t, false, dumpConfig) - assert.False(t, verboseMode) -} - -func TestMainFunction(t *testing.T) { - // first test verbose mode - assert.False(t, verboseMode) - assert.Equal(t, log.GetLevel(), log.InfoLevel) - os.Args = append(os.Args, "-v", "-prefetch") - - // run main function - go main() - time.Sleep(time.Second * 5) - // verbose, prefetch - assert.Equal(t, log.GetLevel(), log.DebugLevel) - assert.True(t, verboseMode) - assert.True(t, prefetch) - - // test read config value - assert.Equal(t, "config.json", configPath) - assert.Equal(t, runtime.NumCPU(), jobs) - assert.Equal(t, false, dumpSystemd) - assert.Equal(t, false, dumpConfig) - - // test port - conn, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", "3333"), time.Second*2) - assert.Nil(t, err) - assert.NotNil(t, conn) -} - -func TestProxySwitch(t *testing.T) { - // real proxy mode - assert.False(t, proxyMode) - config.ImgPath = "https://z.cn" - switchProxyMode() - assert.True(t, proxyMode) - - // normal - config.ImgPath = os.TempDir() - switchProxyMode() - assert.False(t, proxyMode) -}