commit 6f3a6ec2c0ad25d67bdcbe0d997faedc8d288f50
parent d5920d8e1ac804d553c386cedc0800aa1a2a7ba9
Author: Jack Mordaunt <jackmordaunt@gmail.com>
Date: Fri, 9 Nov 2018 13:39:01 +1300
[~] Persist the produced gif images to a Store.
Diffstat:
4 files changed, 332 insertions(+), 49 deletions(-)
diff --git a/cmd/desktop/giffer.go b/cmd/desktop/giffer.go
@@ -2,13 +2,16 @@ package main
import (
"bytes"
+ "fmt"
"image"
"image/gif"
+ "io"
"log"
"path/filepath"
"strings"
"sync"
+ "github.com/OneOfOne/xxhash"
"github.com/disintegration/imaging"
"github.com/jackmordaunt/giffer"
"github.com/pkg/errors"
@@ -18,6 +21,13 @@ import (
type Giffer struct {
*giffer.Downloader
*giffer.FFMpeg
+ Store GifStore
+}
+
+// GifStore contains Gif files.
+type GifStore interface {
+ Lookup(key string) (*RenderedGif, bool, error)
+ Insert(key string, img *RenderedGif) error
}
// GififyURL downloads the video at url and creates a .gif based on the spcified
@@ -28,6 +38,43 @@ func (g Giffer) GififyURL(
width, height int,
q giffer.Quality,
) (*RenderedGif, error) {
+ if g.Store == nil {
+ return g.make(url, start, end, fps, width, height, q)
+ }
+ hasher := xxhash.New64()
+ input := fmt.Sprintf("%s_%f_%f_%f_%d_%d_%d", url, start, end, fps, width, height, q)
+ _, err := hasher.WriteString(input)
+ if err != nil {
+ return nil, errors.Wrap(err, "hashing input")
+ }
+ hash := fmt.Sprintf("%d", hasher.Sum64())
+ img, ok, err := g.Store.Lookup(hash)
+ if err != nil {
+ return nil, errors.Wrap(err, "store lookup")
+ }
+ if ok && img != nil {
+ return img, nil
+ }
+ img, err = g.make(url, start, end, fps, width, height, q)
+ if err != nil {
+ return nil, err
+ }
+ dup := &RenderedGif{
+ Reader: bytes.NewBuffer([]byte(img.Reader.(*bytes.Buffer).String())),
+ FileName: img.FileName,
+ }
+ if err := g.Store.Insert(hash, dup); err != nil {
+ return nil, errors.Wrap(err, "inserting gif into store")
+ }
+ return img, nil
+}
+
+func (g Giffer) make(
+ url string,
+ start, end, fps float64,
+ width, height int,
+ q giffer.Quality,
+) (*RenderedGif, error) {
videofile, err := g.Download(url, q)
if err != nil {
return nil, errors.Wrap(err, "downloading")
@@ -89,17 +136,17 @@ func (g Giffer) GififyURL(
if err := gif.EncodeAll(buf, cfg); err != nil {
return nil, errors.Wrap(err, "encoding animated gif")
}
- r := &RenderedGif{
- Buffer: buf,
+ img := &RenderedGif{
+ Reader: buf,
// Keep the title but replace the .mp4 extension with .gif
FileName: sanitiseFilepath(strings.Split(filepath.Base(videofile), ".")[0] + ".gif"),
}
- return r, nil
+ return img, nil
}
// RenderedGif wraps the gif data with some metadata.
type RenderedGif struct {
- *bytes.Buffer
+ io.Reader
// FileName is <title>.<ext>
FileName string
}
diff --git a/cmd/desktop/main.go b/cmd/desktop/main.go
@@ -20,12 +20,14 @@ import (
var (
port string
devServer string
+ verbose bool
static http.Handler // responsible for serving UI files.
)
func init() {
flag.StringVar(&port, "p", "8080", "port to serve on")
flag.StringVar(&devServer, "dev-proxy", "", "proxy to forward to (eg, yarn run serve)")
+ flag.BoolVar(&verbose, "v", false, "verbose mode")
flag.Parse()
if devServer != "" {
t, err := url.Parse(devServer)
@@ -45,12 +47,15 @@ func main() {
Dir: "tmp/download",
},
FFMpeg: &giffer.FFMpeg{
- Dir: "tmp/ffmpeg",
- LeaveMess: true,
+ Dir: "tmp/ffmpeg",
+ },
+ Store: &gifdb{
+ Dir: "tmp/gifs",
},
},
- Router: mux.NewRouter(),
- Static: static,
+ Router: mux.NewRouter(),
+ Static: static,
+ Verbose: verbose,
}
svr := &http.Server{
Addr: fmt.Sprintf(":%s", port),
diff --git a/cmd/desktop/server.go b/cmd/desktop/server.go
@@ -9,7 +9,6 @@ import (
"os"
"strings"
"sync"
- "time"
"github.com/OneOfOne/xxhash"
"github.com/jackmordaunt/giffer"
@@ -22,9 +21,10 @@ import (
// UI serves the user interface over http.
type UI struct {
- App *Giffer
- Router *mux.Router
- Static http.Handler
+ App *Giffer
+ Router *mux.Router
+ Static http.Handler
+ Verbose bool
gifmap map[string]http.Handler
init sync.Once
@@ -41,11 +41,13 @@ func (ui *UI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
func (ui *UI) routes() {
- log := Log{
- Logger: log.New(LogWriteHeaderErrors{Out: os.Stdout}, "", 0),
- ShowBody: true,
+ if ui.Verbose {
+ log := Log{
+ Logger: log.New(LogWriteHeaderErrors{Out: os.Stdout}, "", 0),
+ ShowBody: true,
+ }
+ ui.Router.Use(log.Middleware)
}
- ui.Router.Use(log.Middleware)
ui.Router.Handle("/gifify", ui.gifify())
ui.Router.Handle("/gifs/{key}", ui.gifs())
ui.Router.Handle("/gifs/{key}/info", ui.gifs())
@@ -86,6 +88,7 @@ func (ui *UI) gifify() http.HandlerFunc {
WriteBufferSize: 1024,
},
}
+ g.Init()
go g.Process(func() (*RenderedGif, error) {
return ui.App.GififyURL(
req.URL,
@@ -123,40 +126,46 @@ func (ui *UI) gifs() http.HandlerFunc {
}
}
-// Gif handles the serving of a gif file.
-// There are two enpoints:
+// Gif handles the serving of a gif file and a websocket endpoint to push out
+// "gif is ready" messages.
+//
+// 3 parts:
+// 1. A closure that produces the rendered gif and an error (to decouple how we
+// get a hold of the gif from the serving of it).
+// 2. A websocket endpoint that pushes a single message to each connection when
+// the gif is ready to download. If the gif is ready before the connection
+// comes in, the connection receives the message immediately. Otherwise
+// the connection is stored in a map until the gif is ready.
+// 3. A static file endpoint that serves the gif image as an attachment. If the
+// gif isn't ready to be downloaded, an appropriate message is returned.
type Gif struct {
Upgrader *websocket.Upgrader
- Tick time.Duration
- file *RenderedGif
- subs map[*websocket.Conn]struct{}
- subMutex sync.Mutex
- err error
- once sync.Once
+ file *RenderedGif
+ subs map[*websocket.Conn]struct{}
+ done chan *RenderedGif
+ failed chan error
+ connections chan *websocket.Conn
+}
+
+// Init the Gif server.
+func (g *Gif) Init() {
+ g.subs = make(map[*websocket.Conn]struct{})
+ g.done = make(chan *RenderedGif)
+ g.failed = make(chan error)
+ g.connections = make(chan *websocket.Conn)
+ go g.run()
}
// Process runs the specified function and sends a websocket message when it
// completes.
func (g *Gif) Process(fn func() (*RenderedGif, error)) {
- type done struct {
- Err string `json:"error,omitempty"`
- }
- g.file, g.err = fn()
- g.subMutex.Lock()
- var d done
- if g.err != nil {
- d.Err = g.err.Error()
- }
- for s := range g.subs {
- err := s.WriteJSON(d)
- if err != nil {
- s.Close()
- delete(g.subs, s)
- log.Printf("writing json to websocket: %v", err)
- }
+ img, err := fn()
+ if err != nil {
+ g.failed <- err
+ } else {
+ g.done <- img
}
- g.subMutex.Unlock()
}
func (g *Gif) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -173,7 +182,7 @@ func (g *Gif) subscribe(w http.ResponseWriter, r *http.Request) {
log.Printf("upgrading websocket: %v", err)
return
}
- g.append(c)
+ g.connections <- c
}
func (g *Gif) serveFile(w http.ResponseWriter, r *http.Request) {
@@ -189,11 +198,44 @@ func (g *Gif) serveFile(w http.ResponseWriter, r *http.Request) {
}
}
-func (g *Gif) append(c *websocket.Conn) {
- g.once.Do(func() {
- g.subs = make(map[*websocket.Conn]struct{})
- })
- g.subMutex.Lock()
- g.subs[c] = struct{}{}
- g.subMutex.Unlock()
+func (g *Gif) run() {
+ type msg struct {
+ Err string `json:"error,omitempty"`
+ }
+ for {
+ select {
+ case img := <-g.done:
+ g.file = img
+ for s := range g.subs {
+ if err := s.WriteJSON(msg{}); err != nil {
+ log.Printf("writing json to websocket: %v", err)
+ }
+ if err := s.Close(); err != nil {
+ log.Printf("closing websocket: %v", err)
+ }
+ delete(g.subs, s)
+ }
+ case err := <-g.failed:
+ for s := range g.subs {
+ if err := s.WriteJSON(msg{Err: err.Error()}); err != nil {
+ log.Printf("writing json to websocket: %v", err)
+ }
+ if err := s.Close(); err != nil {
+ log.Printf("closing websocket: %v", err)
+ }
+ delete(g.subs, s)
+ }
+ case conn := <-g.connections:
+ if g.file != nil {
+ if err := conn.WriteJSON(msg{}); err != nil {
+ log.Printf("writing json to websocket: %v", err)
+ }
+ if err := conn.Close(); err != nil {
+ log.Printf("closing websocket: %v", err)
+ }
+ } else {
+ g.subs[conn] = struct{}{}
+ }
+ }
+ }
}
diff --git a/cmd/desktop/store.go b/cmd/desktop/store.go
@@ -0,0 +1,189 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "sync"
+
+ "github.com/hashicorp/go-multierror"
+
+ "github.com/pkg/errors"
+)
+
+type gifdb struct {
+ Dir string
+ init sync.Once
+}
+
+// Lookup loads the rendered gif from disk.
+func (db *gifdb) Lookup(key string) (*RenderedGif, bool, error) {
+ var (
+ info os.FileInfo
+ err error
+ )
+ db.init.Do(func() {
+ err = os.MkdirAll(db.Dir, 0755)
+ })
+ if err != nil && !os.IsExist(err) {
+ return nil, false, errors.Wrap(err, "initialising")
+ }
+ meta := filepath.Join(db.Dir, key+".json")
+ info, err = os.Stat(meta)
+ if os.IsNotExist(err) {
+ return nil, false, nil
+ }
+ if err != nil {
+ return nil, false, err
+ }
+ if info.IsDir() {
+ return nil, false, fmt.Errorf("key leads to a directory, not a json file")
+ }
+ img := filepath.Join(db.Dir, key+".gif")
+ info, err = os.Stat(img)
+ if os.IsNotExist(err) {
+ return nil, false, nil
+ }
+ if err != nil {
+ return nil, false, err
+ }
+ if info.IsDir() {
+ return nil, false, fmt.Errorf("key leads to a directory, not a gif file")
+ }
+ var (
+ failed = make(chan error)
+ done = make(chan interface{})
+ wg = sync.WaitGroup{}
+ )
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ if name, err := func() (string, error) {
+ metaf, err := os.Open(meta)
+ if err != nil {
+ return "", errors.Wrap(err, "opening metadata file")
+ }
+ defer metaf.Close()
+ type metadata struct {
+ FileName string `json:"filename"`
+ }
+ var md metadata
+ if err := json.NewDecoder(metaf).Decode(&md); err != nil {
+ return "", errors.Wrap(err, "decoding metadata")
+ }
+ return md.FileName, nil
+ }(); err != nil {
+ failed <- err
+ } else {
+ done <- name
+ }
+ }()
+ go func() {
+ defer wg.Done()
+ if buf, err := func() (*bytes.Buffer, error) {
+ buf := bytes.NewBuffer(nil)
+ file, err := os.Open(img)
+ if err != nil {
+ return nil, errors.Wrap(err, "opening gif file")
+ }
+ defer file.Close()
+ if _, err := io.Copy(buf, file); err != nil {
+ return nil, errors.Wrap(err, "reading gif file")
+ }
+ return buf, nil
+ }(); err != nil {
+ failed <- err
+ } else {
+ done <- buf
+ }
+ }()
+ go func() {
+ wg.Wait()
+ close(failed)
+ }()
+ var failure error
+ go func() {
+ for err := range failed {
+ failure = multierror.Append(failure, err)
+ }
+ close(done)
+ }()
+ r := &RenderedGif{}
+ for v := range done {
+ switch v := v.(type) {
+ case string:
+ r.FileName = v
+ case *bytes.Buffer:
+ r.Reader = v
+ }
+ }
+ if failure != nil {
+ return nil, false, failure
+ }
+ return r, true, nil
+}
+
+// Insert stores the rendered gif on disk.
+func (db *gifdb) Insert(key string, img *RenderedGif) (err error) {
+ db.init.Do(func() {
+ err = os.MkdirAll(db.Dir, 0755)
+ })
+ if err != nil && !os.IsExist(err) {
+ return errors.Wrap(err, "initialising")
+ }
+ var (
+ failed = make(chan error)
+ wg = sync.WaitGroup{}
+ )
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ if err := func() error {
+ imgpath := filepath.Join(db.Dir, key+".gif")
+ imgf, err := os.Create(imgpath)
+ if err != nil {
+ return errors.Wrap(err, "creating gif file")
+ }
+ defer imgf.Close()
+ if _, err := io.Copy(imgf, img); err != nil {
+ return errors.Wrap(err, "persisting gif to disk")
+ }
+ return nil
+ }(); err != nil {
+ failed <- err
+ }
+ }()
+ go func() {
+ defer wg.Done()
+ if err := func() error {
+ meta := filepath.Join(db.Dir, key+".json")
+ metaf, err := os.Create(meta)
+ if err != nil {
+ return errors.Wrap(err, "creating metadata file")
+ }
+ defer metaf.Close()
+ type metadata struct {
+ FileName string `json:"filename"`
+ }
+ if err := json.NewEncoder(metaf).Encode(metadata{
+ FileName: img.FileName,
+ }); err != nil {
+ return errors.Wrap(err, "writing to metadata file")
+ }
+ return nil
+ }(); err != nil {
+ failed <- err
+ }
+ }()
+ go func() {
+ wg.Wait()
+ close(failed)
+ }()
+ for failure := range failed {
+ err = multierror.Append(err, failure)
+ }
+ return err
+}
