diff --git a/encoder.go b/encoder.go new file mode 100644 index 0000000..e79ffdb --- /dev/null +++ b/encoder.go @@ -0,0 +1,68 @@ +package main + +import ( + "bytes" + "errors" + "fmt" + "github.com/chai2010/webp" + "golang.org/x/image/bmp" + "image" + "image/gif" + "image/jpeg" + "image/png" + "io/ioutil" + "log" + "path" + "strings" +) + +func WebpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err error) { + // if convert fails, return error; success nil + var buf bytes.Buffer + var img image.Image + + data, err := ioutil.ReadFile(p1) + if err != nil { + ChanErr(c) + return + } + + contentType := GetFileContentType(data[:512]) + if strings.Contains(contentType, "jpeg") { + img, _ = jpeg.Decode(bytes.NewReader(data)) + } else if strings.Contains(contentType, "png") { + img, _ = png.Decode(bytes.NewReader(data)) + } else if strings.Contains(contentType, "bmp") { + img, _ = bmp.Decode(bytes.NewReader(data)) + } else if strings.Contains(contentType, "gif") { + // TODO: need to support animated webp + img, _ = gif.Decode(bytes.NewReader(data)) + } + + if img == nil { + msg := "image file " + path.Base(p1) + " is corrupted or not supported" + log.Println(msg) + err = errors.New(msg) + ChanErr(c) + return + } + + if err = webp.Encode(&buf, img, &webp.Options{Lossless: false, Quality: quality}); err != nil { + log.Println(err) + ChanErr(c) + return + } + if err = ioutil.WriteFile(p2, buf.Bytes(), 0755); err != nil { + log.Println(err) + ChanErr(c) + return + } + + if Log { + fmt.Printf("Save to %s ok\n", p2) + } + + ChanErr(c) + + return nil +} diff --git a/helper.go b/helper.go new file mode 100644 index 0000000..7f7f29d --- /dev/null +++ b/helper.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + "net/http" + "os" + "path" + "path/filepath" +) + +func ChanErr(ccc chan int) { + if ccc != nil { + ccc <- 1 + } +} + +func GetFileContentType(buffer []byte) string { + // Use the net/http package's handy DectectContentType function. Always returns a valid + // content-type by returning "application/octet-stream" if no others seemed to match. + contentType := http.DetectContentType(buffer) + return contentType +} + +func FileCount(dir string) int { + count := 0 + _ = filepath.Walk(dir, + func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + count += 1 + } + return nil + }) + return count +} + +func ImageExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} + +func GenWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) { + // get file mod time + STAT, err := os.Stat(RawImagePath) + if err != nil { + fmt.Println(err.Error()) + } + ModifiedTime := STAT.ModTime().Unix() + // webpFilename: abc.jpg.png -> abc.jpg.png1582558990.webp + var WebpFilename = fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime) + cwd, _ := os.Getwd() + + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + // Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp + WebpAbsolutePath := path.Clean(path.Join(ExhaustPath, path.Dir(reqURI), WebpFilename)) + return cwd, WebpAbsolutePath +} diff --git a/prefetch.go b/prefetch.go new file mode 100644 index 0000000..30d513b --- /dev/null +++ b/prefetch.go @@ -0,0 +1,51 @@ +package main + +import ( + "bufio" + "fmt" + "log" + "os" + "path" + "path/filepath" + "strconv" + "strings" +) + +func PrefetchImages(confImgPath string, ExhaustPath string, QUALITY string) { + fmt.Println(`Prefetch will convert all your images to webp, it may take some time and consume a lot of CPU resource. Do you want to proceed(Y/n)`) + reader := bufio.NewReader(os.Stdin) + char, _, _ := reader.ReadRune() //y Y enter + // maximum ongoing prefetch is depending on your core of CPU + log.Printf("Prefetching using %d cores", jobs) + var finishChan = make(chan int, jobs) + for i := 0; i < jobs; i++ { + finishChan <- 0 + } + if char == 121 || char == 10 || char == 89 { + //prefetch, recursive through the dir + all := FileCount(confImgPath) + count := 0 + err := filepath.Walk(confImgPath, + func(picAbsPath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // RawImagePath string, ImgFilename string, reqURI string + proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1) + _, p2 := GenWebpAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI) + q, _ := strconv.ParseFloat(QUALITY, 32) + _ = os.MkdirAll(path.Dir(p2), 0755) + go WebpEncoder(picAbsPath, p2, float32(q), false, finishChan) + count += <-finishChan + //progress bar + _, _ = fmt.Fprintf(os.Stdout, "[Webp Server started] - convert in progress: %d/%d\r", count, all) + return nil + }) + if err != nil { + log.Println(err) + } + } + + _, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y\n\n") + +} diff --git a/router.go b/router.go new file mode 100644 index 0000000..1e2c5f8 --- /dev/null +++ b/router.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "github.com/gofiber/fiber" + "os" + "path" + "path/filepath" + "strconv" + "strings" +) + +func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY string) func(c *fiber.Ctx) { + return func(c *fiber.Ctx) { + //basic vars + var reqURI = c.Path() // mypic/123.jpg + var RawImageAbs = path.Join(ImgPath, reqURI) // /home/xxx/mypic/123.jpg + var ImgFilename = path.Base(reqURI) // pure filename, 123.jpg + var finalFile string // We'll only need one c.sendFile() + // Check for Safari users. If they're Safari, just simply ignore everything. + UA := c.Get("User-Agent") + if strings.Contains(UA, "Safari") && !strings.Contains(UA, "Chrome") && + !strings.Contains(UA, "Firefox") { + c.SendFile(RawImageAbs) + return + } + + // check ext + // TODO: may remove this function. Check in Nginx. + var allowed = false + for _, ext := range AllowedTypes { + haystack := strings.ToLower(ImgFilename) + needle := strings.ToLower("." + ext) + if strings.HasSuffix(haystack, needle) { + allowed = true + break + } else { + allowed = false + } + } + if !allowed { + c.Send("File extension not allowed!") + c.SendStatus(403) + return + } + + // Check the original image for existence, + if !ImageExists(RawImageAbs) { + c.Send("Image not found!") + c.SendStatus(404) + return + } + + cwd, WebpAbsPath := GenWebpAbs(RawImageAbs, ExhaustPath, ImgFilename, reqURI) + + if ImageExists(WebpAbsPath) { + finalFile = WebpAbsPath + } else { + // we don't have abc.jpg.png1582558990.webp + // delete the old pic and convert a new one. + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp + destHalfFile := path.Clean(path.Join(cwd, "exhaust", path.Dir(reqURI), ImgFilename)) + matches, err := filepath.Glob(destHalfFile + "*") + if err != nil { + fmt.Println(err.Error()) + } else { + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed + // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one + for _, p := range matches { + if strings.Compare(destHalfFile, p) != 0 { + _ = os.Remove(p) + } + } + } + + //for webp, we need to create dir first + _ = os.MkdirAll(path.Dir(WebpAbsPath), 0755) + q, _ := strconv.ParseFloat(QUALITY, 32) + err = WebpEncoder(RawImageAbs, WebpAbsPath, float32(q), true, nil) + + if err != nil { + fmt.Println(err) + c.SendStatus(400) + c.Send("Bad file!") + return + } + finalFile = WebpAbsPath + } + c.SendFile(finalFile) + } +} diff --git a/webp-server.go b/webp-server.go index 33faa8a..75e4a12 100644 --- a/webp-server.go +++ b/webp-server.go @@ -1,30 +1,14 @@ package main import ( - "bufio" - "bytes" "encoding/json" - "errors" "flag" "fmt" - "image" - "image/gif" - "image/jpeg" - "image/png" - "io/ioutil" + "github.com/gofiber/fiber" "log" - "net/http" "os" "path" - "path/filepath" "runtime" - "strconv" - "strings" - - "golang.org/x/image/bmp" - - "github.com/chai2010/webp" - "github.com/gofiber/fiber" ) type Config struct { @@ -52,8 +36,7 @@ const sampleConfig = ` "IMG_PATH": "/path/to/pics", "EXHAUST_PATH": "", "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","gif"] -} -` +}` const sampleSystemd = ` [Unit] Description=WebP Server @@ -72,8 +55,7 @@ RestartSec=3s [Install] -WantedBy=multi-user.target -` +WantedBy=multi-user.target` func loadConfig(path string) Config { var config Config @@ -87,77 +69,6 @@ func loadConfig(path string) Config { return config } -func imageExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - return !info.IsDir() -} - -func GetFileContentType(buffer []byte) string { - // Use the net/http package's handy DectectContentType function. Always returns a valid - // content-type by returning "application/octet-stream" if no others seemed to match. - contentType := http.DetectContentType(buffer) - return contentType -} - -func chanErr(ccc chan int) { - if ccc != nil { - ccc <- 1 - } -} -func webpEncoder(p1, p2 string, quality float32, Log bool, c chan int) (err error) { - // if convert fails, return error; success nil - var buf bytes.Buffer - var img image.Image - - data, err := ioutil.ReadFile(p1) - if err != nil { - chanErr(c) - return - } - - contentType := GetFileContentType(data[:512]) - if strings.Contains(contentType, "jpeg") { - img, _ = jpeg.Decode(bytes.NewReader(data)) - } else if strings.Contains(contentType, "png") { - img, _ = png.Decode(bytes.NewReader(data)) - } else if strings.Contains(contentType, "bmp") { - img, _ = bmp.Decode(bytes.NewReader(data)) - } else if strings.Contains(contentType, "gif") { - // TODO: need to support animated webp - img, _ = gif.Decode(bytes.NewReader(data)) - } - - if img == nil { - msg := "image file " + path.Base(p1) + " is corrupted or not supported" - log.Println(msg) - err = errors.New(msg) - chanErr(c) - return - } - - if err = webp.Encode(&buf, img, &webp.Options{Lossless: false, Quality: quality}); err != nil { - log.Println(err) - chanErr(c) - return - } - if err = ioutil.WriteFile(p2, buf.Bytes(), 0755); err != nil { - log.Println(err) - chanErr(c) - return - } - - if Log { - fmt.Printf("Save to %s ok\n", p2) - } - - chanErr(c) - - return nil -} - func init() { flag.StringVar(&configPath, "config", "config.json", "/path/to/config.json. (Default: ./config.json)") flag.BoolVar(&prefetch, "prefetch", false, "Prefetch and convert image to webp") @@ -167,203 +78,9 @@ func init() { flag.Parse() } -func Convert(ImgPath string, ExhaustPath string, AllowedTypes []string, QUALITY string) func(c *fiber.Ctx) { - return func(c *fiber.Ctx) { - //basic vars - var reqURI = c.Path() // mypic/123.jpg - var RawImageAbs = path.Join(ImgPath, reqURI) // /home/xxx/mypic/123.jpg - var ImgFilename = path.Base(reqURI) // pure filename, 123.jpg - var finalFile string // We'll only need one c.sendFile() - // Check for Safari users. If they're Safari, just simply ignore everything. - UA := c.Get("User-Agent") - if strings.Contains(UA, "Safari") && !strings.Contains(UA, "Chrome") && - !strings.Contains(UA, "Firefox") { - c.SendFile(RawImageAbs) - return - } - - // check ext - // TODO: may remove this function. Check in Nginx. - var allowed = false - for _, ext := range AllowedTypes { - haystack := strings.ToLower(ImgFilename) - needle := strings.ToLower("." + ext) - if strings.HasSuffix(haystack, needle) { - allowed = true - break - } else { - allowed = false - } - } - if !allowed { - c.Send("File extension not allowed!") - c.SendStatus(403) - return - } - - // Check the original image for existence, - if !imageExists(RawImageAbs) { - c.Send("Image not found!") - c.SendStatus(404) - return - } - - cwd, WebpAbsPath := genWebpAbs(RawImageAbs, ExhaustPath, ImgFilename, reqURI) - - if imageExists(WebpAbsPath) { - finalFile = WebpAbsPath - } else { - // we don't have abc.jpg.png1582558990.webp - // delete the old pic and convert a new one. - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - destHalfFile := path.Clean(path.Join(cwd, "exhaust", path.Dir(reqURI), ImgFilename)) - matches, err := filepath.Glob(destHalfFile + "*") - if err != nil { - fmt.Println(err.Error()) - } else { - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558100.webp <- older ones will be removed - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp <- keep the latest one - for _, p := range matches { - if strings.Compare(destHalfFile, p) != 0 { - _ = os.Remove(p) - } - } - } - - //for webp, we need to create dir first - _ = os.MkdirAll(path.Dir(WebpAbsPath), 0755) - q, _ := strconv.ParseFloat(QUALITY, 32) - err = webpEncoder(RawImageAbs, WebpAbsPath, float32(q), true, nil) - - if err != nil { - fmt.Println(err) - c.SendStatus(400) - c.Send("Bad file!") - return - } - finalFile = WebpAbsPath - } - c.SendFile(finalFile) - } -} - -func fileCount(dir string) int { - count := 0 - _ = filepath.Walk(dir, - func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { - count += 1 - } - return nil - }) - return count -} - -func genWebpAbs(RawImagePath string, ExhaustPath string, ImgFilename string, reqURI string) (string, string) { - // get file mod time - STAT, err := os.Stat(RawImagePath) - if err != nil { - fmt.Println(err.Error()) - } - ModifiedTime := STAT.ModTime().Unix() - // webpFilename: abc.jpg.png -> abc.jpg.png1582558990.webp - var WebpFilename = fmt.Sprintf("%s.%d.webp", ImgFilename, ModifiedTime) - cwd, _ := os.Getwd() - - // /home/webp_server/exhaust/path/to/tsuki.jpg.1582558990.webp - // Custom Exhaust: /path/to/exhaust/web_path/web_to/tsuki.jpg.1582558990.webp - WebpAbsolutePath := path.Clean(path.Join(ExhaustPath, path.Dir(reqURI), WebpFilename)) - return cwd, WebpAbsolutePath -} - -func prefetchImages(confImgPath string, ExhaustPath string, QUALITY string) { - fmt.Println(`Prefetch will convert all your images to webp, it may take some time and consume a lot of CPU resource. Do you want to proceed(Y/n)`) - reader := bufio.NewReader(os.Stdin) - char, _, _ := reader.ReadRune() //y Y enter - // maximum ongoing prefetch is depending on your core of CPU - log.Printf("Prefetching using %d cores", jobs) - var finishChan = make(chan int, jobs) - for i := 0; i < jobs; i++ { - finishChan <- 0 - } - if char == 121 || char == 10 || char == 89 { - //prefetch, recursive through the dir - all := fileCount(confImgPath) - count := 0 - err := filepath.Walk(confImgPath, - func(picAbsPath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - // RawImagePath string, ImgFilename string, reqURI string - proposedURI := strings.Replace(picAbsPath, confImgPath, "", 1) - _, p2 := genWebpAbs(picAbsPath, ExhaustPath, info.Name(), proposedURI) - q, _ := strconv.ParseFloat(QUALITY, 32) - _ = os.MkdirAll(path.Dir(p2), 0755) - go webpEncoder(picAbsPath, p2, float32(q), false, finishChan) - count += <-finishChan - //progress bar - _, _ = fmt.Fprintf(os.Stdout, "[Webp Server started] - convert in progress: %d/%d\r", count, all) - return nil - }) - if err != nil { - log.Println(err) - } - } - - _, _ = fmt.Fprintf(os.Stdout, "Prefetch completeY(^_^)Y\n\n") - -} -func autoUpdate() { - defer func() { - if err := recover(); err != nil { - log.Println("Download error.", err) - } - }() - - var api = "https://api.github.com/repos/webp-sh/webp_server_go/releases/latest" - type Result struct { - TagName string `json:"tag_name"` - } - var res Result - resp1, _ := http.Get(api) - data1, _ := ioutil.ReadAll(resp1.Body) - _ = json.Unmarshal(data1, &res) - - var gitVersion = res.TagName - - if gitVersion > version { - log.Printf("Time to update! New version %s found!", gitVersion) - } else { - log.Println("No new version found.") - return - } - - var filename = fmt.Sprintf("webp-server-%s-%s", runtime.GOOS, runtime.GOARCH) - if runtime.GOARCH == "windows" { - filename += ".exe" - } - var releaseUrl = "https://github.com/webp-sh/webp_server_go/releases/latest/download/" + filename - log.Println("Downloading binary...") - resp, _ := http.Get(releaseUrl) - if resp.StatusCode != 200 { - log.Printf("%s-%s not found on release. "+ - "Contact developers to supply your version", runtime.GOOS, runtime.GOARCH) - return - } - data, _ := ioutil.ReadAll(resp.Body) - _ = os.Mkdir("update", 0755) - err := ioutil.WriteFile(path.Join("update", filename), data, 0755) - - if err == nil { - log.Println("Update complete. Please find your binary from update directory.") - } - _ = resp.Body.Close() -} - func main() { - go autoUpdate() config := loadConfig(configPath) + HOST := config.HOST PORT := config.PORT confImgPath := path.Clean(config.ImgPath) @@ -383,13 +100,11 @@ func main() { } if dumpSystemd { fmt.Println(sampleSystemd) - os.Exit(0) - } if prefetch { - go prefetchImages(confImgPath, ExhaustPath, QUALITY) + go PrefetchImages(confImgPath, ExhaustPath, QUALITY) } app := fiber.New()