Gek/cache bot search page (#3530)
* feat: add general purpose key/val caching layer * feat: cache bot/metadata response page for 10 seconds
This commit is contained in:
Gabe Kangas
GitHub
parent
9b698336dc
commit
7399bee276
@ -1,15 +1,18 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/owncast/owncast/config"
|
||||
"github.com/owncast/owncast/core"
|
||||
"github.com/owncast/owncast/core/cache"
|
||||
"github.com/owncast/owncast/core/data"
|
||||
"github.com/owncast/owncast/models"
|
||||
"github.com/owncast/owncast/router/middleware"
|
||||
@ -18,6 +21,8 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var gc = cache.GetGlobalCache()
|
||||
|
||||
// IndexHandler handles the default index route.
|
||||
func IndexHandler(w http.ResponseWriter, r *http.Request) {
|
||||
middleware.EnableCors(w)
|
||||
@ -121,6 +126,17 @@ type MetadataPage struct {
|
||||
// Return a basic HTML page with server-rendered metadata from the config
|
||||
// to give to Opengraph clients and web scrapers (bots, web crawlers, etc).
|
||||
func handleScraperMetadataPage(w http.ResponseWriter, r *http.Request) {
|
||||
cacheKey := "bot-scraper-html"
|
||||
cacheHtmlExpiration := time.Duration(10) * time.Second
|
||||
c := gc.GetOrCreateCache(cacheKey, cacheHtmlExpiration)
|
||||
|
||||
cachedHtml := c.GetValueForKey(cacheKey)
|
||||
if cachedHtml != nil {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write(cachedHtml)
|
||||
return
|
||||
}
|
||||
|
||||
tmpl, err := static.GetBotMetadataTemplate()
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
@ -173,11 +189,18 @@ func handleScraperMetadataPage(w http.ResponseWriter, r *http.Request) {
|
||||
SocialHandles: data.GetSocialHandles(),
|
||||
}
|
||||
|
||||
// Cache the rendered HTML
|
||||
var b bytes.Buffer
|
||||
if err := tmpl.Execute(&b, metadata); err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
c.Set(cacheKey, b.Bytes())
|
||||
|
||||
// Set a cache header
|
||||
middleware.SetCachingHeaders(w, r)
|
||||
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
if err := tmpl.Execute(w, metadata); err != nil {
|
||||
if _, err = w.Write(b.Bytes()); err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}
|
||||
|
||||
82
core/cache/cache.go
vendored
Normal file
82
core/cache/cache.go
vendored
Normal file
@ -0,0 +1,82 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/jellydator/ttlcache/v3"
|
||||
)
|
||||
|
||||
// CacheContainer is a container for all caches.
|
||||
type CacheContainer struct {
|
||||
caches map[string]*CacheInstance
|
||||
}
|
||||
|
||||
// CacheInstance is a single cache instance.
|
||||
type CacheInstance struct {
|
||||
cache *ttlcache.Cache[string, []byte]
|
||||
}
|
||||
|
||||
// This is the global singleton instance. (To be removed after refactor).
|
||||
var _instance *CacheContainer
|
||||
|
||||
// NewCache creates a new cache instance.
|
||||
func NewGlobalCache() *CacheContainer {
|
||||
_instance = &CacheContainer{
|
||||
caches: make(map[string]*CacheInstance),
|
||||
}
|
||||
|
||||
return _instance
|
||||
}
|
||||
|
||||
// GetCache returns the cache instance.
|
||||
func GetGlobalCache() *CacheContainer {
|
||||
if _instance != nil {
|
||||
return _instance
|
||||
}
|
||||
return NewGlobalCache()
|
||||
}
|
||||
|
||||
// GetOrCreateCache returns the cache instance or creates a new one.
|
||||
func (c *CacheContainer) GetOrCreateCache(name string, expiration time.Duration) *CacheInstance {
|
||||
if _, ok := c.caches[name]; !ok {
|
||||
c.CreateCache(name, expiration)
|
||||
}
|
||||
return c.caches[name]
|
||||
}
|
||||
|
||||
// CreateCache creates a new cache instance.
|
||||
func (c *CacheContainer) CreateCache(name string, expiration time.Duration) *CacheInstance {
|
||||
cache := ttlcache.New[string, []byte](
|
||||
ttlcache.WithTTL[string, []byte](expiration),
|
||||
ttlcache.WithDisableTouchOnHit[string, []byte](),
|
||||
)
|
||||
ci := &CacheInstance{cache: cache}
|
||||
c.caches[name] = ci
|
||||
go cache.Start()
|
||||
return ci
|
||||
}
|
||||
|
||||
// GetCache returns the cache instance.
|
||||
func (c *CacheContainer) GetCache(name string) *CacheInstance {
|
||||
return c.caches[name]
|
||||
}
|
||||
|
||||
// GetValueForKey returns the value for the given key.
|
||||
func (ci *CacheInstance) GetValueForKey(key string) []byte {
|
||||
value := ci.cache.Get(key, ttlcache.WithDisableTouchOnHit[string, []byte]())
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if value.IsExpired() {
|
||||
return nil
|
||||
}
|
||||
|
||||
val := value.Value()
|
||||
return val
|
||||
}
|
||||
|
||||
// Set sets the value for the given key..
|
||||
func (ci *CacheInstance) Set(key string, value []byte) {
|
||||
ci.cache.Set(key, value, 0)
|
||||
}
|
||||
72
core/cache/cache_test.go
vendored
Normal file
72
core/cache/cache_test.go
vendored
Normal file
@ -0,0 +1,72 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestCache(t *testing.T) {
|
||||
expiration := 5 * time.Second
|
||||
globalCache := GetGlobalCache()
|
||||
assert.NotNil(t, globalCache, "NewGlobalCache should return a non-nil instance")
|
||||
assert.Equal(t, globalCache, GetGlobalCache(), "GetGlobalCache should return the created instance")
|
||||
|
||||
cacheName := "testCache"
|
||||
globalCache.CreateCache(cacheName, expiration)
|
||||
|
||||
createdCache := globalCache.GetCache(cacheName)
|
||||
assert.NotNil(t, createdCache, "GetCache should return a non-nil cache")
|
||||
|
||||
key := "testKey"
|
||||
value := []byte("testValue")
|
||||
createdCache.Set(key, value)
|
||||
|
||||
// Wait for cache to expire
|
||||
time.Sleep(expiration + 1*time.Second)
|
||||
|
||||
// Verify that the cache has expired
|
||||
ci := globalCache.GetCache(cacheName)
|
||||
cachedValue := ci.GetValueForKey(key)
|
||||
assert.Nil(t, cachedValue, "Cache should not contain the value after expiration")
|
||||
}
|
||||
|
||||
func TestConcurrentAccess(t *testing.T) {
|
||||
// Test concurrent access to the cache
|
||||
globalCache := NewGlobalCache()
|
||||
cacheName := "concurrentCache"
|
||||
expiration := 5 * time.Second
|
||||
globalCache.CreateCache(cacheName, expiration)
|
||||
|
||||
// Start multiple goroutines to access the cache concurrently
|
||||
numGoroutines := 10
|
||||
keyPrefix := "key"
|
||||
valuePrefix := "value"
|
||||
|
||||
done := make(chan struct{})
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
go func(index int) {
|
||||
defer func() { done <- struct{}{} }()
|
||||
|
||||
cache := globalCache.GetCache(cacheName)
|
||||
key := keyPrefix + strconv.Itoa(index)
|
||||
value := valuePrefix + strconv.Itoa(index)
|
||||
|
||||
cache.Set(key, []byte(value))
|
||||
|
||||
// Simulate some work
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
ci := globalCache.GetCache(cacheName)
|
||||
cachedValue := string(ci.GetValueForKey(key))
|
||||
assert.Equal(t, value, cachedValue, "Cached value should match the set value")
|
||||
}(i)
|
||||
}
|
||||
|
||||
// Wait for all goroutines to finish
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
<-done
|
||||
}
|
||||
}
|
||||
6
go.mod
6
go.mod
@ -63,18 +63,24 @@ require github.com/SherClockHolmes/webpush-go v1.3.0
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/go-test/deep v1.0.4 // indirect
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
|
||||
github.com/gorilla/css v1.0.0 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/oschwald/maxminddb-golang v1.11.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
golang.org/x/sync v0.3.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/CAFxX/httpcompression v0.0.9
|
||||
github.com/andybalholm/cascadia v1.3.2
|
||||
github.com/jellydator/ttlcache/v3 v3.1.1
|
||||
github.com/mssola/user_agent v0.6.0
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/yuin/goldmark-emoji v1.0.2
|
||||
gopkg.in/evanphx/json-patch.v5 v5.7.0
|
||||
mvdan.cc/xurls v1.1.0
|
||||
|
||||
14
go.sum
14
go.sum
@ -46,6 +46,8 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/
|
||||
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
|
||||
github.com/grafov/m3u8 v0.12.0 h1:T6iTwTsSEtMcwkayef+FJO8kj+Sglr4Lh81Zj8Ked/4=
|
||||
github.com/grafov/m3u8 v0.12.0/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+u080=
|
||||
github.com/jellydator/ttlcache/v3 v3.1.1 h1:RCgYJqo3jgvhl+fEWvjNW8thxGWsgxi+TPhRir1Y9y8=
|
||||
github.com/jellydator/ttlcache/v3 v3.1.1/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
@ -55,6 +57,10 @@ github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUB
|
||||
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
|
||||
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8=
|
||||
github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is=
|
||||
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4=
|
||||
@ -103,6 +109,8 @@ github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwa
|
||||
github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
|
||||
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
|
||||
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/schollz/sqlite3dump v1.3.1 h1:QXizJ7XEJ7hggjqjZ3YRtF3+javm8zKtzNByYtEkPRA=
|
||||
github.com/schollz/sqlite3dump v1.3.1/go.mod h1:mzSTjZpJH4zAb1FN3iNlhWPbbdyeBpOaTW0hukyMHyI=
|
||||
github.com/shirou/gopsutil/v3 v3.23.11 h1:i3jP9NjCPUz7FiZKxlMnODZkdSIp2gnzfrvsu9CuWEQ=
|
||||
@ -142,6 +150,8 @@ github.com/yuin/goldmark-emoji v1.0.2 h1:c/RgTShNgHTtc6xdz2KKI74jJr6rWi7FPgnP9GA
|
||||
github.com/yuin/goldmark-emoji v1.0.2/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY=
|
||||
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
|
||||
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
|
||||
go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
|
||||
golang.org/x/crypto v0.0.0-20180527072434-ab813273cd59/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
@ -167,6 +177,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sys v0.0.0-20180525142821-c11f84a56e43/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@ -210,6 +222,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
|
||||
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
|
||||
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/evanphx/json-patch.v5 v5.7.0 h1:dGKGylPlZ/jus2g1YqhhyzfH0gPy2R8/MYUpW/OslTY=
|
||||
gopkg.in/evanphx/json-patch.v5 v5.7.0/go.mod h1:/kvTRh1TVm5wuM6OkHxqXtE/1nUZZpihg29RtuIyfvk=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user