wood/webp_server_go
1
0
Fork 0
mirror of https://github.com/woodchen-ink/webp_server_go.git synced 2025-07-18 05:32:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feature: WebP Proxy (#51)

Browse Source 
* feat-webp-proxy

* Fix panic on clear Cache, modified output.

* Optimize etag fetch logic

* Update README for new docs website.

* Bump version to 0.2.0.
This commit is contained in:
Nova Kwok 2020-08-06 14:55:20 +08:00 committed by GitHub
parent 36eb669031
commit 2d2df5571d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 166 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@ -3,7 +3,7 @@
</p>
<img src="https://api.travis-ci.org/webp-sh/webp_server_go.svg?branch=master"/>
[Documentation](https://webp.sh/docs/) | [Website](https://webp.sh/)
[Documentation](https://docs.webp.sh/) | [Website](https://webp.sh/)
This is a Server based on Golang, which allows you to serve WebP images on the fly.
It will convert `jpg,jpeg,png` files by default, this can be customized by editing the `config.json`..
@ -18,7 +18,7 @@ It will convert `jpg,jpeg,png` files by default, this can be customized by editi
## Simple Usage Steps
### 1. Download or build the binary
Download the `webp-server` from [release](https://github.com/n0vad3v/webp_server_go/releases) page.
Download the `webp-server` from [release](https://github.com/webp-sh/webp_server_go/releases) page.
### 2. Dump config file
@ -67,8 +67,7 @@ Let Nginx to `proxy_pass http://localhost:3333/;`, and your webp-server is on-th
## Advanced Usage
For supervisor, Docker sections, please read our documentation at [https://webp.sh/docs/](https://webp.sh/docs/)
For supervisor, Docker sections, please read our documentation at [https://docs.webp.sh/](https://docs.webp.sh/)
## License

53
helper.go
View File

@ -3,6 +3,7 @@ package main
import (
"fmt"
"hash/crc32"
"io"
"io/ioutil"
"net/http"
"os"
@ -48,6 +49,56 @@ func ImageExists(filename string) bool {
return !info.IsDir()
}
// Check for remote filepath, e.g: https://test.webp.sh/node.png
// return StatusCode, etagValue
func GetRemoteImageInfo(fileUrl string) (int, string) {
res, err := http.Head(fileUrl)
if err != nil {
log.Fatal("Connection to remote error!")
}
if res.StatusCode != 404 {
etagValue := res.Header.Get("etag")
if etagValue == "" {
log.Info("Remote didn't return etag in header, please check.")
} else {
return 200, etagValue
}
}
return res.StatusCode, ""
}
func FetchRemoteImage(filepath string, url string) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
_ = os.MkdirAll(path.Dir(filepath), 0755)
out, err := os.Create(filepath)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, resp.Body)
return err
}
// Given /path/to/node.png
// Delete /path/to/node.png*
func CleanProxyCache(cacheImagePath string) {
// Delete /node.png*
files, err := filepath.Glob(cacheImagePath + "*")
if err != nil {
fmt.Println(err)
}
for _, f := range files {
if err := os.Remove(f); err != nil {
log.Info(err)
}
}
}
func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) {
// get file mod time
STAT, err := os.Stat(RawImagePath)
@ -56,7 +107,7 @@ func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, req
}
ModifiedTime := STAT.ModTime().Unix()
// webpFilename: abc.jpg.png -> abc.jpg.png1582558990.webp
var WebpFilename = fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime)
WebpFilename := fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime)
cwd, _ := os.Getwd()
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp

137
router.go
View File

@ -1,6 +1,7 @@
package main
import (
"fmt"
"os"
"path"
"path/filepath"
@ -12,10 +13,10 @@ import (
"github.com/gofiber/fiber"
)
func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY string) func(c *fiber.Ctx) {
func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY string, proxyMode bool) func(c *fiber.Ctx) {
return func(c *fiber.Ctx) {
//basic vars
var reqURI = c.Path() // mypic/123.jpg
var reqURI = c.Path() // /mypic/123.jpg
var RawImageAbs = path.Join(ImgPath, reqURI) // /home/xxx/mypic/123.jpg
var ImgFilename = path.Base(reqURI) // pure filename, 123.jpg
var finalFile string // We'll only need one c.sendFile()
@ -57,52 +58,102 @@ func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY
return
}
// Check the original image for existence,
if !ImageExists(RawImageAbs) {
msg := "Image not found!"
c.Send(msg)
log.Warn(msg)
c.SendStatus(404)
return
}
// Start Proxy Mode
if proxyMode {
// https://test.webp.sh/node.png
realRemoteAddr := ImgPath + reqURI
// Ping Remote for status code and etag info
_, WebpAbsPath := GenWebpAbs(RawImageAbs, ExhaustPath, ImgFilename, reqURI)
if ImageExists(WebpAbsPath) {
finalFile = WebpAbsPath
} else {
// we don't have abc.jpg.png1582558990.webp
// delete the old pic and convert a new one.
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
destHalfFile := path.Clean(path.Join(WebpAbsPath, path.Dir(reqURI), ImgFilename))
matches, err := filepath.Glob(destHalfFile + "*")
if err != nil {
log.Error(err.Error())
} else {
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one
for _, p := range matches {
if strings.Compare(destHalfFile, p) != 0 {
_ = os.Remove(p)
// If status code is 200
// Check for local /node.png-etag-<etagValue>
// if exist
// Send local cache
// else
// Delete local /node.png*
// Fetch and convert to /node.png-etag-<etagValue>
// Send local cache
// else status code is 404
// Delete /node.png*
// Send 404
fmt.Println("Remote Addr is " + realRemoteAddr + ", fetching..")
statusCode, etagValue := GetRemoteImageInfo(realRemoteAddr)
if statusCode == 200 {
// Check local path: /node.png-etag-<etagValue>
localEtagImagePath := ExhaustPath + reqURI + "-etag-" + etagValue
if ImageExists(localEtagImagePath) {
c.SendFile(localEtagImagePath)
} else {
// Temporary store of remote file.
// ./remote-raw/node.png
CleanProxyCache(ExhaustPath + reqURI + "*")
localRemoteTmpPath := "./remote-raw" + reqURI
FetchRemoteImage(localRemoteTmpPath, realRemoteAddr)
q, _ := strconv.ParseFloat(QUALITY, 32)
_ = os.MkdirAll(path.Dir(localEtagImagePath), 0755)
err := WebpEncoder(localRemoteTmpPath, localEtagImagePath, float32(q), true, nil)
if err != nil {
fmt.Println(err)
}
c.SendFile(localEtagImagePath)
}
}
//for webp, we need to create dir first
_ = os.MkdirAll(path.Dir(WebpAbsPath), 0755)
q, _ := strconv.ParseFloat(QUALITY, 32)
err = WebpEncoder(RawImageAbs, WebpAbsPath, float32(q), true, nil)
if err != nil {
log.Error(err)
c.SendStatus(400)
c.Send("Bad file!")
} else {
msg := fmt.Sprintf("Remote returned %d status code!", statusCode)
c.Send(msg)
log.Warn(msg)
c.SendStatus(statusCode)
CleanProxyCache(ExhaustPath + reqURI + "*")
return
}
finalFile = WebpAbsPath
// End Proxy Mode
} else {
// Check the original image for existence,
if !ImageExists(RawImageAbs) {
msg := "Image not found!"
c.Send(msg)
log.Warn(msg)
c.SendStatus(404)
return
}
_, WebpAbsPath := GenWebpAbs(RawImageAbs, ExhaustPath, ImgFilename, reqURI)
if ImageExists(WebpAbsPath) {
finalFile = WebpAbsPath
} else {
// we don't have abc.jpg.png1582558990.webp
// delete the old pic and convert a new one.
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp
destHalfFile := path.Clean(path.Join(WebpAbsPath, path.Dir(reqURI), ImgFilename))
matches, err := filepath.Glob(destHalfFile + "*")
if err != nil {
log.Error(err.Error())
} else {
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed
// /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one
for _, p := range matches {
if strings.Compare(destHalfFile, p) != 0 {
_ = os.Remove(p)
}
}
}
//for webp, we need to create dir first
_ = os.MkdirAll(path.Dir(WebpAbsPath), 0755)
q, _ := strconv.ParseFloat(QUALITY, 32)
err = WebpEncoder(RawImageAbs, WebpAbsPath, float32(q), true, nil)
if err != nil {
log.Error(err)
c.SendStatus(400)
c.Send("Bad file!")
return
}
finalFile = WebpAbsPath
}
etag := GenEtag(finalFile)
c.Set("ETag", etag)
c.SendFile(finalFile)
}
etag := GenEtag(finalFile)
c.Set("ETag", etag)
c.SendFile(finalFile)
}
}

25
webp-server.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"os"
"path"
"regexp"
"runtime"
"github.com/gofiber/fiber"
@ -21,7 +22,7 @@ type Config struct {
ExhaustPath string `json:"EXHAUST_PATH"`
}
const version = "0.1.5"
const version = "0.2.0"
var configPath string
var prefetch bool
@ -65,11 +66,6 @@ func loadConfig(path string) Config {
defer jsonObject.Close()
decoder := json.NewDecoder(jsonObject)
_ = decoder.Decode(&config)
_, err = os.Stat(config.ImgPath)
if err != nil {
log.Fatalf("Your image path %s is incorrect.Please check and confirm.", config.ImgPath)
}
return config
}
@ -118,7 +114,20 @@ func main() {
HOST := config.HOST
PORT := config.PORT
confImgPath := path.Clean(config.ImgPath)
// Check for remote address
matched, _ := regexp.MatchString(`^https?://`, config.ImgPath)
proxyMode := false
confImgPath := ""
if matched {
proxyMode = true
confImgPath = config.ImgPath
} else {
_, err := os.Stat(config.ImgPath)
if err != nil {
log.Fatalf("Your image path %s is incorrect.Please check and confirm.", config.ImgPath)
}
confImgPath = path.Clean(config.ImgPath)
}
QUALITY := config.QUALITY
AllowedTypes := config.AllowedTypes
var ExhaustPath string
@ -141,7 +150,7 @@ func main() {
// Server Info
log.Infof("WebP Server %s %s", version, ListenAddress)
app.Get("/*", Convert(confImgPath, ExhaustPath, AllowedTypes, QUALITY))
app.Get("/*", Convert(confImgPath, ExhaustPath, AllowedTypes, QUALITY, proxyMode))
app.Listen(ListenAddress)
}