wood/webp_server_go
1
0
Fork 0
mirror of https://github.com/woodchen-ink/webp_server_go.git synced 2025-07-18 13:42:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refine convert part (#303)

Browse Source 
* Refine convert part

* Only open image once

* More refine
This commit is contained in:
Nova Kwok 2023-12-17 03:12:42 +08:00 committed by GitHub
parent c771160a05
commit d83f6667cc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 167 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/config.go
View File

@ -46,7 +46,7 @@ var (
ProxyMode bool
Prefetch bool
Config = NewWebPConfig()
Version = "0.10.1"
Version = "0.10.2"
WriteLock = cache.New(5*time.Minute, 10*time.Minute)
RemoteRaw = "./remote-raw"
Metadata = "./metadata"

179
encoder/encoder.go
View File

@ -1,7 +1,6 @@
package encoder
import (
"errors"
"os"
"path"
"runtime"
@ -33,35 +32,13 @@ func init() {
intMinusOne.Set(-1)
}
func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error {
imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width)
if extraParams.Width > 0 && extraParams.Height > 0 {
err := img.Thumbnail(extraParams.Width, extraParams.Height, vips.InterestingAttention)
if err != nil {
return err
}
} else if extraParams.Width > 0 && extraParams.Height == 0 {
err := img.Thumbnail(extraParams.Width, int(float32(extraParams.Width)*imgHeightWidthRatio), 0)
if err != nil {
return err
}
} else if extraParams.Height > 0 && extraParams.Width == 0 {
err := img.Thumbnail(int(float32(extraParams.Height)/imgHeightWidthRatio), extraParams.Height, 0)
if err != nil {
return err
}
}
return nil
}
func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParams, c chan int) {
func ConvertFilter(rawPath, avifPath, webpPath string, extraParams config.ExtraParams, c chan int) {
// all absolute paths
var wg sync.WaitGroup
wg.Add(2)
if !helper.ImageExists(avifPath) && config.Config.EnableAVIF {
go func() {
err := convertImage(raw, avifPath, "avif", extraParams)
err := convertImage(rawPath, avifPath, "avif", extraParams)
if err != nil {
log.Errorln(err)
}
@ -73,7 +50,7 @@ func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParam
if !helper.ImageExists(webpPath) {
go func() {
err := convertImage(raw, webpPath, "webp", extraParams)
err := convertImage(rawPath, webpPath, "webp", extraParams)
if err != nil {
log.Errorln(err)
}
@ -89,94 +66,54 @@ func ConvertFilter(raw, avifPath, webpPath string, extraParams config.ExtraParam
}
}
func ResizeItself(raw, dest string, extraParams config.ExtraParams) {
log.Infof("Resize %s itself to %s", raw, dest)
func convertImage(rawPath, optimizedPath, imageType string, extraParams config.ExtraParams) error {
// we need to create dir first
var err = os.MkdirAll(path.Dir(dest), 0755)
var err = os.MkdirAll(path.Dir(optimizedPath), 0755)
if err != nil {
log.Error(err.Error())
}
img, err := vips.LoadImageFromFile(raw, &vips.ImportParams{
FailOnError: boolFalse,
})
if err != nil {
log.Warnf("Could not load %s: %s", raw, err)
return
}
_ = resizeImage(img, extraParams)
buf, _, _ := img.ExportNative()
_ = os.WriteFile(dest, buf, 0600)
img.Close()
}
func convertImage(raw, optimized, imageType string, extraParams config.ExtraParams) error {
// we need to create dir first
var err = os.MkdirAll(path.Dir(optimized), 0755)
if err != nil {
log.Error(err.Error())
}
// Convert NEF image to JPG first
var convertedRaw, converted = ConvertRawToJPG(raw, optimized)
// If original image is NEF, convert NEF image to JPG first
if strings.HasSuffix(strings.ToLower(rawPath), ".nef") {
var convertedRaw, converted = ConvertRawToJPG(rawPath, optimizedPath)
// If converted, use converted file as raw
if converted {
raw = convertedRaw
}
switch imageType {
case "webp":
err = webpEncoder(raw, optimized, extraParams)
case "avif":
err = avifEncoder(raw, optimized, extraParams)
}
// Use converted file(JPG) as raw input for further convertion
rawPath = convertedRaw
// Remove converted file after convertion
if converted {
defer func() {
log.Infoln("Removing intermediate conversion file:", convertedRaw)
err := os.Remove(convertedRaw)
if err != nil {
log.Warnln("failed to delete converted file", err)
}
}()
}
}
// Image is only opened here
img, err := vips.LoadImageFromFile(rawPath, &vips.ImportParams{
FailOnError: boolFalse,
})
defer img.Close()
// Pre-process image(auto rotate, resize, etc.)
preProcessImage(img, imageType, extraParams)
switch imageType {
case "webp":
err = webpEncoder(img, rawPath, optimizedPath, extraParams)
case "avif":
err = avifEncoder(img, rawPath, optimizedPath, extraParams)
}
return err
}
func avifEncoder(p1, p2 string, extraParams config.ExtraParams) error {
// if convert fails, return error; success nil
func avifEncoder(img *vips.ImageRef, rawPath string, optimizedPath string, extraParams config.ExtraParams) error {
var (
buf []byte
quality = config.Config.Quality
err error
)
img, err := vips.LoadImageFromFile(p1, &vips.ImportParams{
FailOnError: boolFalse,
})
if err != nil {
return err
}
imageFormat := img.Format()
for _, ignore := range avifIgnore {
if imageFormat == ignore {
// Return err to render original image
return errors.New("AVIF encoder: ignore image type")
}
}
if config.Config.EnableExtraParams {
err = resizeImage(img, extraParams)
if err != nil {
return err
}
}
// AVIF has a maximum resolution of 65536 x 65536 pixels.
if img.Metadata().Width > config.AvifMax || img.Metadata().Height > config.AvifMax {
return errors.New("AVIF: image too large")
}
err = img.AutoRotate()
if err != nil {
return err
}
// If quality >= 100, we use lossless mode
if quality >= 100 {
@ -197,55 +134,22 @@ func avifEncoder(p1, p2 string, extraParams config.ExtraParams) error {
return err
}
if err := os.WriteFile(p2, buf, 0600); err != nil {
if err := os.WriteFile(optimizedPath, buf, 0600); err != nil {
log.Error(err)
return err
}
img.Close()
convertLog("AVIF", p1, p2, quality)
convertLog("AVIF", rawPath, optimizedPath, quality)
return nil
}
func webpEncoder(p1, p2 string, extraParams config.ExtraParams) error {
// if convert fails, return error; success nil
func webpEncoder(img *vips.ImageRef, rawPath string, optimizedPath string, extraParams config.ExtraParams) error {
var (
buf []byte
quality = config.Config.Quality
err error
)
img, err := vips.LoadImageFromFile(p1, &vips.ImportParams{
FailOnError: boolFalse,
NumPages: intMinusOne,
})
if err != nil {
return err
}
imageFormat := img.Format()
for _, ignore := range webpIgnore {
if imageFormat == ignore {
// Return err to render original image
return errors.New("WebP encoder: ignore image type")
}
}
if config.Config.EnableExtraParams {
err = resizeImage(img, extraParams)
if err != nil {
return err
}
}
// The maximum pixel dimensions of a WebP image is 16383 x 16383.
if (img.Metadata().Width > config.WebpMax || img.Metadata().Height > config.WebpMax) && img.Format() != vips.ImageTypeGIF {
return errors.New("WebP: image too large")
}
err = img.AutoRotate()
if err != nil {
return err
}
// If quality >= 100, we use lossless mode
if quality >= 100 {
// Lossless mode will not encounter problems as below, because in libvips as code below
@ -283,28 +187,27 @@ func webpEncoder(p1, p2 string, extraParams config.ExtraParams) error {
return err
}
if err := os.WriteFile(p2, buf, 0600); err != nil {
if err := os.WriteFile(optimizedPath, buf, 0600); err != nil {
log.Error(err)
return err
}
img.Close()
convertLog("WebP", p1, p2, quality)
convertLog("WebP", rawPath, optimizedPath, quality)
return nil
}
func convertLog(itype, p1 string, p2 string, quality int) {
oldf, err := os.Stat(p1)
func convertLog(itype, rawPath string, optimizedPath string, quality int) {
oldf, err := os.Stat(rawPath)
if err != nil {
log.Error(err)
return
}
newf, err := os.Stat(p2)
newf, err := os.Stat(optimizedPath)
if err != nil {
log.Error(err)
return
}
log.Infof("%s@%d%%: %s->%s %d->%d %.2f%% deflated", itype, quality,
p1, p2, oldf.Size(), newf.Size(), float32(newf.Size())/float32(oldf.Size())*100)
rawPath, optimizedPath, oldf.Size(), newf.Size(), float32(newf.Size())/float32(oldf.Size())*100)
}

94
encoder/process.go Normal file
View File

@ -0,0 +1,94 @@
package encoder
import (
"errors"
"os"
"path"
"slices"
"webp_server_go/config"
"github.com/davidbyttow/govips/v2/vips"
log "github.com/sirupsen/logrus"
)
func resizeImage(img *vips.ImageRef, extraParams config.ExtraParams) error {
imgHeightWidthRatio := float32(img.Metadata().Height) / float32(img.Metadata().Width)
if extraParams.Width > 0 && extraParams.Height > 0 {
err := img.Thumbnail(extraParams.Width, extraParams.Height, vips.InterestingAttention)
if err != nil {
return err
}
} else if extraParams.Width > 0 && extraParams.Height == 0 {
err := img.Thumbnail(extraParams.Width, int(float32(extraParams.Width)*imgHeightWidthRatio), 0)
if err != nil {
return err
}
} else if extraParams.Height > 0 && extraParams.Width == 0 {
err := img.Thumbnail(int(float32(extraParams.Height)/imgHeightWidthRatio), extraParams.Height, 0)
if err != nil {
return err
}
}
return nil
}
func ResizeItself(raw, dest string, extraParams config.ExtraParams) {
log.Infof("Resize %s itself to %s", raw, dest)
// we need to create dir first
var err = os.MkdirAll(path.Dir(dest), 0755)
if err != nil {
log.Error(err.Error())
}
img, err := vips.LoadImageFromFile(raw, &vips.ImportParams{
FailOnError: boolFalse,
})
if err != nil {
log.Warnf("Could not load %s: %s", raw, err)
return
}
_ = resizeImage(img, extraParams)
buf, _, _ := img.ExportNative()
_ = os.WriteFile(dest, buf, 0600)
img.Close()
}
// Pre-process image(auto rotate, resize, etc.)
func preProcessImage(img *vips.ImageRef, imageType string, extraParams config.ExtraParams) error {
// Check Width/Height and ignore image formats
switch imageType {
case "webp":
if img.Metadata().Width > config.WebpMax || img.Metadata().Height > config.WebpMax {
return errors.New("WebP: image too large")
}
imageFormat := img.Format()
if slices.Contains(webpIgnore, imageFormat) {
// Return err to render original image
return errors.New("WebP encoder: ignore image type")
}
case "avif":
if img.Metadata().Width > config.AvifMax || img.Metadata().Height > config.AvifMax {
return errors.New("AVIF: image too large")
}
imageFormat := img.Format()
if slices.Contains(avifIgnore, imageFormat) {
// Return err to render original image
return errors.New("AVIF encoder: ignore image type")
}
}
// Auto rotate
err := img.AutoRotate()
if err != nil {
return err
}
if config.Config.EnableExtraParams {
err = resizeImage(img, extraParams)
if err != nil {
return err
}
}
return nil
}

5
encoder/rawconvert.go
View File

@ -2,16 +2,11 @@ package encoder
import (
"path/filepath"
"strings"
"github.com/jeremytorres/rawparser"
)
func ConvertRawToJPG(rawPath, optimizedPath string) (string, bool) {
if !strings.HasSuffix(strings.ToLower(rawPath), ".nef") {
// Maybe can use rawParser to convert other raw files to jpg, but I haven't tested it
return rawPath, false
}
parser, _ := rawparser.NewNefParser(true)
info := &rawparser.RawFileInfo{
File: rawPath,

40
handler/router.go
View File

@ -4,6 +4,7 @@ import (
"net/http"
"net/url"
"regexp"
"slices"
"strings"
"webp_server_go/config"
"webp_server_go/encoder"
@ -25,14 +26,26 @@ func Convert(c *fiber.Ctx) error {
var (
reqHostname = c.Hostname()
reqHost = c.Protocol() + "://" + reqHostname // http://www.example.com:8000
reqURI, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg
reqURIwithQuery, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200
reqHeader = &c.Request().Header
reqURIRaw, _ = url.QueryUnescape(c.Path()) // /mypic/123.jpg
reqURIwithQueryRaw, _ = url.QueryUnescape(c.OriginalURL()) // /mypic/123.jpg?someother=200&somebugs=200
reqURI = path.Clean(reqURIRaw) // delete ../ in reqURI to mitigate directory traversal
reqURIwithQuery = path.Clean(reqURIwithQueryRaw) // Sometimes reqURIwithQuery can be https://example.tld/mypic/123.jpg?someother=200&somebugs=200, we need to extract it
filename = path.Base(reqURI)
realRemoteAddr = ""
targetHostName = config.LocalHostAlias
targetHost = config.Config.ImgPath
proxyMode = config.ProxyMode
mapMode = false
width, _ = strconv.Atoi(c.Query("width")) // Extra Params
height, _ = strconv.Atoi(c.Query("height")) // Extra Params
extraParams = config.ExtraParams{
Width: width,
Height: height,
}
)
log.Debugf("Incoming connection from %s %s %s", c.IP(), reqHostname, reqURIwithQuery)
@ -45,19 +58,6 @@ func Convert(c *fiber.Ctx) error {
return nil
}
// Sometimes reqURIwithQuery can be https://example.tld/mypic/123.jpg?someother=200&somebugs=200, we need to extract it.
// delete ../ in reqURI to mitigate directory traversal
reqURI = path.Clean(reqURI)
reqURIwithQuery = path.Clean(reqURIwithQuery)
width, _ := strconv.Atoi(c.Query("width"))
height, _ := strconv.Atoi(c.Query("height"))
var extraParams = config.ExtraParams{
Width: width,
Height: height,
}
// Rewrite the target backend if a mapping rule matches the hostname
if hostMap, hostMapFound := config.Config.ImageMap[reqHost]; hostMapFound {
log.Debugf("Found host mapping %s -> %s", reqHostname, hostMap)
@ -132,9 +132,9 @@ func Convert(c *fiber.Ctx) error {
}
}
goodFormat := helper.GuessSupportedFormat(&c.Request().Header)
supportedFormats := helper.GuessSupportedFormat(reqHeader)
// resize itself and return if only one format(raw) is supported
if len(goodFormat) == 1 {
if len(supportedFormats) == 1 {
dest := path.Join(config.Config.ExhaustPath, targetHostName, metadata.Id)
if !helper.ImageExists(dest) {
encoder.ResizeItself(rawImageAbs, dest, extraParams)
@ -156,14 +156,12 @@ func Convert(c *fiber.Ctx) error {
encoder.ConvertFilter(rawImageAbs, avifAbs, webpAbs, extraParams, nil)
var availableFiles = []string{rawImageAbs}
for _, v := range goodFormat {
if v == "avif" {
if slices.Contains(supportedFormats, "avif") {
availableFiles = append(availableFiles, avifAbs)
}
if v == "webp" {
if slices.Contains(supportedFormats, "webp") {
availableFiles = append(availableFiles, webpAbs)
}
}
finalFilename := helper.FindSmallestFiles(availableFiles)
contentType := helper.GetFileContentType(finalFilename)