perf: cache generated image thumbnails
This commit is contained in:
OpenClaw Agent
+45
-27
@@ -37,25 +37,27 @@ import (
|
||||
var webIndex []byte
|
||||
|
||||
type Server struct {
|
||||
Store storage.Storage
|
||||
Auth *auth.Manager
|
||||
Sessions map[string]*auth.Account
|
||||
SessionExpiry map[string]time.Time
|
||||
SessionMu sync.RWMutex
|
||||
SessionTimeout int
|
||||
JWTSecret string
|
||||
WebDAVRoot string
|
||||
WebDAVFS webdav.FileSystem
|
||||
Allow string
|
||||
MultiRoot bool
|
||||
RemoveMode int
|
||||
ImagePreview bool
|
||||
FolderDownload string
|
||||
HTMLTitle string
|
||||
HTMLNotice string
|
||||
Log *opslog.Logger
|
||||
UserStore *auth.UserStore
|
||||
UserHome string
|
||||
Store storage.Storage
|
||||
Auth *auth.Manager
|
||||
Sessions map[string]*auth.Account
|
||||
SessionExpiry map[string]time.Time
|
||||
SessionMu sync.RWMutex
|
||||
SessionTimeout int
|
||||
JWTSecret string
|
||||
WebDAVRoot string
|
||||
WebDAVFS webdav.FileSystem
|
||||
Allow string
|
||||
MultiRoot bool
|
||||
RemoveMode int
|
||||
ImagePreview bool
|
||||
FolderDownload string
|
||||
HTMLTitle string
|
||||
HTMLNotice string
|
||||
Log *opslog.Logger
|
||||
UserStore *auth.UserStore
|
||||
UserHome string
|
||||
ThumbnailCache *thumbnailCache
|
||||
thumbnailCacheOnce sync.Once
|
||||
}
|
||||
type fileInfo struct {
|
||||
Name string `json:"name"`
|
||||
@@ -1176,20 +1178,36 @@ func (s *Server) thumbnail(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
src, _, err := image.Decode(f)
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
key := thumbnailCacheKey{Path: file, Size: fi.Size(), ModTime: fi.ModTime().UnixNano()}
|
||||
s.thumbnailCacheOnce.Do(func() {
|
||||
if s.ThumbnailCache == nil {
|
||||
s.ThumbnailCache = newThumbnailCache()
|
||||
}
|
||||
})
|
||||
data, err := s.ThumbnailCache.Do(key, func() ([]byte, error) {
|
||||
src, _, err := image.Decode(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
thumb := transform.Resize(src, 320, 240, transform.Linear)
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, thumb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append([]byte(nil), buf.Bytes()...), nil
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, "thumbnail unavailable", http.StatusUnsupportedMediaType)
|
||||
return
|
||||
}
|
||||
thumb := transform.Resize(src, 320, 240, transform.Linear)
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, thumb); err != nil {
|
||||
http.Error(w, "thumbnail unavailable", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
w.Header().Set("Cache-Control", "private, max-age=300")
|
||||
_, _ = w.Write(buf.Bytes())
|
||||
_, _ = w.Write(data)
|
||||
}
|
||||
|
||||
func (s *Server) fileHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type thumbnailCacheKey struct {
|
||||
Path string
|
||||
Size int64
|
||||
ModTime int64
|
||||
}
|
||||
|
||||
type thumbnailCacheEntry struct {
|
||||
key thumbnailCacheKey
|
||||
data []byte
|
||||
}
|
||||
|
||||
type thumbnailFlight struct {
|
||||
done chan struct{}
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
|
||||
type thumbnailCache struct {
|
||||
mu sync.Mutex
|
||||
items map[thumbnailCacheKey]*list.Element
|
||||
order *list.List
|
||||
flights map[thumbnailCacheKey]*thumbnailFlight
|
||||
bytes int64
|
||||
maxBytes int64
|
||||
maxItems int
|
||||
}
|
||||
|
||||
func newThumbnailCache() *thumbnailCache {
|
||||
return &thumbnailCache{
|
||||
items: make(map[thumbnailCacheKey]*list.Element),
|
||||
order: list.New(),
|
||||
flights: make(map[thumbnailCacheKey]*thumbnailFlight),
|
||||
maxBytes: 32 << 20,
|
||||
maxItems: 64,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *thumbnailCache) Do(key thumbnailCacheKey, fn func() ([]byte, error)) ([]byte, error) {
|
||||
c.mu.Lock()
|
||||
if e, ok := c.items[key]; ok {
|
||||
c.order.MoveToFront(e)
|
||||
data := append([]byte(nil), e.Value.(thumbnailCacheEntry).data...)
|
||||
c.mu.Unlock()
|
||||
return data, nil
|
||||
}
|
||||
if f, ok := c.flights[key]; ok {
|
||||
c.mu.Unlock()
|
||||
<-f.done
|
||||
return append([]byte(nil), f.data...), f.err
|
||||
}
|
||||
f := &thumbnailFlight{done: make(chan struct{})}
|
||||
c.flights[key] = f
|
||||
c.mu.Unlock()
|
||||
|
||||
data, err := fn()
|
||||
if err == nil && len(data) > 0 && int64(len(data)) <= c.maxBytes {
|
||||
c.mu.Lock()
|
||||
c.putLocked(key, data)
|
||||
f.data = append([]byte(nil), data...)
|
||||
f.err = err
|
||||
delete(c.flights, key)
|
||||
close(f.done)
|
||||
c.mu.Unlock()
|
||||
return data, nil
|
||||
}
|
||||
c.mu.Lock()
|
||||
f.err = err
|
||||
delete(c.flights, key)
|
||||
close(f.done)
|
||||
c.mu.Unlock()
|
||||
return data, err
|
||||
}
|
||||
|
||||
func (c *thumbnailCache) putLocked(key thumbnailCacheKey, data []byte) {
|
||||
if e, ok := c.items[key]; ok {
|
||||
old := e.Value.(thumbnailCacheEntry)
|
||||
c.bytes -= int64(len(old.data))
|
||||
e.Value = thumbnailCacheEntry{key: key, data: append([]byte(nil), data...)}
|
||||
c.bytes += int64(len(data))
|
||||
c.order.MoveToFront(e)
|
||||
} else {
|
||||
e := c.order.PushFront(thumbnailCacheEntry{key: key, data: append([]byte(nil), data...)})
|
||||
c.items[key] = e
|
||||
c.bytes += int64(len(data))
|
||||
}
|
||||
for c.bytes > c.maxBytes || c.order.Len() > c.maxItems {
|
||||
e := c.order.Back()
|
||||
if e == nil {
|
||||
break
|
||||
}
|
||||
entry := e.Value.(thumbnailCacheEntry)
|
||||
delete(c.items, entry.key)
|
||||
c.bytes -= int64(len(entry.data))
|
||||
c.order.Remove(e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestThumbnailCacheDeduplicatesConcurrentBuilds(t *testing.T) {
|
||||
c := newThumbnailCache()
|
||||
key := thumbnailCacheKey{Path: "photo.jpg", Size: 10, ModTime: 20}
|
||||
var builds atomic.Int32
|
||||
const workers = 24
|
||||
var wg sync.WaitGroup
|
||||
results := make([][]byte, workers)
|
||||
errs := make([]error, workers)
|
||||
wg.Add(workers)
|
||||
for i := 0; i < workers; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
results[i], errs[i] = c.Do(key, func() ([]byte, error) {
|
||||
builds.Add(1)
|
||||
return []byte("thumbnail"), nil
|
||||
})
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
if builds.Load() != 1 {
|
||||
t.Fatalf("thumbnail builder ran %d times, want 1", builds.Load())
|
||||
}
|
||||
for i := range results {
|
||||
if errs[i] != nil || string(results[i]) != "thumbnail" {
|
||||
t.Fatalf("worker %d got result %q, err=%v", i, results[i], errs[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailCacheDoesNotCacheErrors(t *testing.T) {
|
||||
c := newThumbnailCache()
|
||||
key := thumbnailCacheKey{Path: "broken.jpg"}
|
||||
wantErr := errors.New("decode failed")
|
||||
var builds atomic.Int32
|
||||
for i := 0; i < 2; i++ {
|
||||
_, err := c.Do(key, func() ([]byte, error) {
|
||||
builds.Add(1)
|
||||
return nil, wantErr
|
||||
})
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("got err %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
if builds.Load() != 2 {
|
||||
t.Fatalf("error result was cached; builder ran %d times", builds.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailCacheEvictsByBound(t *testing.T) {
|
||||
c := newThumbnailCache()
|
||||
c.maxItems = 2
|
||||
for i := 0; i < 3; i++ {
|
||||
key := thumbnailCacheKey{Path: string(rune('a' + i))}
|
||||
if _, err := c.Do(key, func() ([]byte, error) { return []byte("x"), nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if len(c.items) != 2 || c.order.Len() != 2 {
|
||||
t.Fatalf("cache size=%d/list=%d, want 2/2", len(c.items), c.order.Len())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user