Connected clients admin API (#217)

* Add support for ending the inbound stream. Closes #191

* Add a simple success response to API requests

* Connected clients API with geo details

* Post-rebase cleanup

* Make setting and reading geo details separate operations to unblock and speed up

* Rename file

* Fire geoip api call behind goroutine

* Add comment

* Post-rebase fixes

* Add support for the MaxMind GeoLite2 GeoIP database
This commit is contained in:
Gabe Kangas 2020-10-06 23:14:33 -07:00 committed by GitHub
parent 1eb7c1985b
commit d7e355bce1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 1926 additions and 37 deletions

1
.gitignore vendored
View File

@ -24,6 +24,7 @@ webroot/hls
webroot/static/content.md
hls/
dist/
data/
transcoder.log
chat.db
.yp.key

60
config-gabe.yaml Normal file
View File

@ -0,0 +1,60 @@
instanceDetails:
name: Localhost Test Instance
title: Owncast Demo Server
summary: "This is Gabe's localhost instance of Owncast."
logo:
small: /img/logo128.png
large: /img/logo256.png
tags:
- software
- music
- animal crossing
# https://github.com/gabek/owncast/blob/master/doc/configuration.md#customization
# for full list of supported social links. All optional.
socialHandles:
- platform: twitter
url: http://twitter.com/owncast
- platform: instagram
url: http://instagram.biz/owncast
- platform: facebook
url: http://facebook.gov/owncast
videoSettings:
# Change this value and keep it secure. Treat it like a password to your live stream.
streamingKey: abc123
# Determine the bitrate of your stream variants.
# See https://github.com/gabek/owncast/blob/master/doc/configuration.md#video-quality for details.
streamQualities:
- high:
videoBitrate: 2000
- medium:
videoBitrate: 800
# s3:
# enabled: true
# endpoint: https://gabevideo.us-east-1.linodeobjects.com
# accessKey: TM24VRAB57SLH72CS0XA
# secret: zKpuJHRNLmOVnzh9gsoQHbRhpYAQt94xCb3Y7pou
# region: us-east-1
# bucket: gabevideo
s3:
enabled: true
endpoint: https://gabevideo.s3-us-west-2.amazonaws.com
accessKey: AKIAZVILNW6ECSTICSPM
secret: 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
region: us-west-2
bucket: gabevideo
acl: public-read
# Enable YP to be listed in the Owncast directory and let people discover your instance.
yp:
enabled: true
ypServiceURL: https://owncast-yp-test.gabek.vercel.app
instanceURL: http://localhost:8080

View File

@ -5,6 +5,7 @@ import "path/filepath"
const (
WebRoot = "webroot"
PrivateHLSStoragePath = "hls"
GeoIPDatabasePath = "data/GeoLite2-City.mmdb"
)
var (

13
controllers/admin.go Normal file
View File

@ -0,0 +1,13 @@
package controllers
import (
"net/http"
"github.com/owncast/owncast/core/rtmp"
)
// DisconnectInboundConnection will force-disconnect an inbound stream
func DisconnectInboundConnection(w http.ResponseWriter, r *http.Request) {
rtmp.Disconnect()
w.WriteHeader(http.StatusOK)
}

View File

@ -0,0 +1,16 @@
package controllers
import (
"encoding/json"
"net/http"
"github.com/owncast/owncast/core"
)
// GetConnectedClients returns currently connected clients
func GetConnectedClients(w http.ResponseWriter, r *http.Request) {
clients := core.GetClients()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(clients)
}

View File

@ -13,6 +13,7 @@ import (
"github.com/owncast/owncast/config"
"github.com/owncast/owncast/core"
"github.com/owncast/owncast/models"
"github.com/owncast/owncast/router/middleware"
"github.com/owncast/owncast/utils"
)
@ -47,8 +48,8 @@ func IndexHandler(w http.ResponseWriter, r *http.Request) {
if path.Ext(r.URL.Path) == ".m3u8" {
middleware.DisableCache(w)
clientID := utils.GenerateClientIDFromRequest(r)
core.SetClientActive(clientID)
client := models.GenerateClientFromRequest(r)
core.SetClientActive(client)
}
// Set a cache control max-age header

View File

@ -70,3 +70,12 @@ func GetMessages() []models.ChatMessage {
return getChatHistory()
}
func GetClient(clientID string) *Client {
for _, client := range _server.Clients {
if client.ClientID == clientID {
return client
}
}
return nil
}

View File

@ -9,6 +9,7 @@ import (
log "github.com/sirupsen/logrus"
"golang.org/x/net/websocket"
"github.com/owncast/owncast/geoip"
"github.com/owncast/owncast/models"
"github.com/owncast/owncast/utils"
@ -21,8 +22,12 @@ const channelBufSize = 100
type Client struct {
ConnectedAt time.Time
MessageCount int
UserAgent string
IPAddress string
Username *string
ClientID string // How we identify unique viewers when counting viewer counts.
Geo *geoip.GeoDetails `json:"geo"`
clientID string // How we identify unique viewers when counting viewer counts.
socketID string // How we identify a single websocket client.
ws *websocket.Conn
ch chan models.ChatMessage
@ -50,10 +55,12 @@ func NewClient(ws *websocket.Conn) *Client {
pingch := make(chan models.PingMessage)
usernameChangeChannel := make(chan models.NameChangeEvent)
ipAddress := utils.GetIPAddressFromRequest(ws.Request())
userAgent := ws.Request().UserAgent()
clientID := utils.GenerateClientIDFromRequest(ws.Request())
socketID, _ := shortid.Generate()
return &Client{time.Now(), 0, clientID, socketID, ws, ch, pingch, usernameChangeChannel, doneCh}
return &Client{time.Now(), 0, userAgent, ipAddress, nil, clientID, nil, socketID, ws, ch, pingch, usernameChangeChannel, doneCh}
}
//GetConnection gets the connection for the client
@ -66,7 +73,7 @@ func (c *Client) Write(msg models.ChatMessage) {
case c.ch <- msg:
default:
_server.remove(c)
_server.err(fmt.Errorf("client %s is disconnected", c.clientID))
_server.err(fmt.Errorf("client %s is disconnected", c.ClientID))
}
}
@ -153,6 +160,7 @@ func (c *Client) userChangedName(data []byte) {
msg.Type = NAMECHANGE
msg.ID = shortid.MustGenerate()
_server.usernameChanged(msg)
c.Username = &msg.NewName
}
func (c *Client) chatMessageReceived(data []byte) {
@ -168,7 +176,21 @@ func (c *Client) chatMessageReceived(data []byte) {
msg.Visible = true
c.MessageCount++
c.Username = &msg.Author
msg.ClientID = c.clientID
msg.ClientID = c.ClientID
_server.SendToAll(msg)
}
// GetViewerClientFromChatClient returns a general models.Client from a chat websocket client.
func (c *Client) GetViewerClientFromChatClient() models.Client {
return models.Client{
ConnectedAt: c.ConnectedAt,
MessageCount: c.MessageCount,
UserAgent: c.UserAgent,
IPAddress: c.IPAddress,
Username: c.Username,
ClientID: c.ClientID,
Geo: geoip.GetGeoFromIP(c.IPAddress),
}
}

View File

@ -79,7 +79,7 @@ func (s *server) onConnection(ws *websocket.Conn) {
client := NewClient(ws)
defer func() {
log.Tracef("The client was connected for %s and sent %d messages (%s)", time.Since(client.ConnectedAt), client.MessageCount, client.clientID)
log.Tracef("The client was connected for %s and sent %d messages (%s)", time.Since(client.ConnectedAt), client.MessageCount, client.ClientID)
if err := ws.Close(); err != nil {
s.errCh <- err
@ -102,13 +102,13 @@ func (s *server) Listen() {
// add new a client
case c := <-s.addCh:
s.Clients[c.socketID] = c
s.listener.ClientAdded(c.clientID)
s.listener.ClientAdded(c.GetViewerClientFromChatClient())
s.sendWelcomeMessageToClient(c)
// remove a client
case c := <-s.delCh:
delete(s.Clients, c.socketID)
s.listener.ClientRemoved(c.clientID)
s.listener.ClientRemoved(c.ClientID)
// broadcast a message to all clients
case msg := <-s.sendAllCh:
@ -138,3 +138,13 @@ func (s *server) sendWelcomeMessageToClient(c *Client) {
}()
}
func (s *server) getClientForClientID(clientID string) *Client {
for _, client := range s.Clients {
if client.ClientID == clientID {
return client
}
}
return nil
}

View File

@ -11,8 +11,8 @@ import (
type ChatListenerImpl struct{}
//ClientAdded is for when a client is added the system
func (cl ChatListenerImpl) ClientAdded(clientID string) {
SetClientActive(clientID)
func (cl ChatListenerImpl) ClientAdded(client models.Client) {
SetClientActive(client)
}
//ClientRemoved is for when a client disconnects/is removed

View File

@ -11,6 +11,8 @@ import (
log "github.com/sirupsen/logrus"
"github.com/owncast/owncast/config"
"github.com/owncast/owncast/core/chat"
"github.com/owncast/owncast/geoip"
"github.com/owncast/owncast/models"
"github.com/owncast/owncast/utils"
)
@ -55,9 +57,13 @@ func setupStats() error {
}
func purgeStaleViewers() {
for clientID, lastConnectedtime := range _stats.Clients {
timeSinceLastActive := time.Since(lastConnectedtime).Minutes()
if timeSinceLastActive > 2 {
for clientID, client := range _stats.Clients {
if client.LastSeen.IsZero() {
continue
}
timeSinceLastActive := time.Since(client.LastSeen).Minutes()
if timeSinceLastActive > 1 {
RemoveClient(clientID)
}
}
@ -80,13 +86,22 @@ func IsStreamConnected() bool {
}
//SetClientActive sets a client as active and connected
func SetClientActive(clientID string) {
// if _, ok := s.clients[clientID]; !ok {
// fmt.Println("Marking client active:", clientID, s.GetViewerCount()+1, "clients connected.")
// }
func SetClientActive(client models.Client) {
l.Lock()
_stats.Clients[clientID] = time.Now()
// If this clientID already exists then update it.
// Otherwise set a new one.
if existingClient, ok := _stats.Clients[client.ClientID]; ok {
existingClient.LastSeen = time.Now()
existingClient.Username = client.Username
existingClient.MessageCount = client.MessageCount
existingClient.Geo = geoip.GetGeoFromIP(existingClient.IPAddress)
_stats.Clients[client.ClientID] = existingClient
} else {
if client.Geo == nil {
geoip.FetchGeoForIP(client.IPAddress)
}
_stats.Clients[client.ClientID] = client
}
l.Unlock()
// Don't update viewer counts if a live stream session is not active.
@ -103,6 +118,19 @@ func RemoveClient(clientID string) {
delete(_stats.Clients, clientID)
}
func GetClients() []models.Client {
clients := make([]models.Client, 0)
for _, client := range _stats.Clients {
chatClient := chat.GetClient(client.ClientID)
if chatClient != nil {
clients = append(clients, chatClient.GetViewerClientFromChatClient())
} else {
clients = append(clients, client)
}
}
return clients
}
func saveStatsToFile() error {
jsonData, err := json.Marshal(_stats)
if err != nil {
@ -125,7 +153,7 @@ func saveStatsToFile() error {
func getSavedStats() (models.Stats, error) {
result := models.Stats{
Clients: make(map[string]time.Time),
Clients: make(map[string]models.Client),
}
if !utils.DoesFileExists(statsFilePath) {

84
geoip/geoip.go Normal file
View File

@ -0,0 +1,84 @@
// This package utilizes the MaxMind GeoLite2 GeoIP database https://dev.maxmind.com/geoip/geoip2/geolite2/.
// You must provide your own copy of this database for it to work.
// Read more about how this works at http://owncast.online/docs/geoip
package geoip
import (
"net"
"github.com/oschwald/geoip2-golang"
"github.com/owncast/owncast/config"
log "github.com/sirupsen/logrus"
)
var _geoIPCache = map[string]GeoDetails{}
var _enabled = true // Try to use GeoIP support it by default.
// GeoDetails stores details about a location
type GeoDetails struct {
CountryCode string `json:"countryCode"`
RegionName string `json:"regionName"`
TimeZone string `json:"timeZone"`
}
// GetGeoFromIP returns geo details associated with an IP address if we
// have previously fetched it.
func GetGeoFromIP(ip string) *GeoDetails {
if cachedGeoDetails, ok := _geoIPCache[ip]; ok {
return &cachedGeoDetails
}
return nil
}
// FetchGeoForIP makes an API call to get geo details for an IP address.
func FetchGeoForIP(ip string) {
// If GeoIP has been disabled then don't try to access it.
if !_enabled {
return
}
// Don't re-fetch if we already have it.
if _, ok := _geoIPCache[ip]; ok {
return
}
go func() {
db, err := geoip2.Open(config.GeoIPDatabasePath)
if err != nil {
log.Traceln("GeoIP support is disabled. visit http://owncast.online/docs/geoip to learn how to enable.", err)
_enabled = false
return
}
defer db.Close()
ipObject := net.ParseIP(ip)
record, err := db.City(ipObject)
if err != nil {
log.Warnln(err)
return
}
// If no country is available then exit
if record.Country.IsoCode == "" {
return
}
// If we believe this IP to be anonymous then no reason to report it
if record.Traits.IsAnonymousProxy {
return
}
response := GeoDetails{
CountryCode: record.Country.IsoCode,
RegionName: record.Subdivisions[0].Names["en"],
TimeZone: record.Location.TimeZone,
}
_geoIPCache[ip] = response
}()
}

1
go.mod
View File

@ -11,6 +11,7 @@ require (
github.com/mssola/user_agent v0.5.2
github.com/nareix/joy5 v0.0.0-20200712071056-a55089207c88
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/oschwald/geoip2-golang v1.4.0
github.com/radovskyb/watcher v1.0.7
github.com/shirou/gopsutil v2.20.7+incompatible
github.com/sirupsen/logrus v1.6.0

6
go.sum
View File

@ -27,6 +27,10 @@ github.com/nareix/joy5 v0.0.0-20200712071056-a55089207c88 h1:CXq5QLPMcfGEZMx8uBM
github.com/nareix/joy5 v0.0.0-20200712071056-a55089207c88/go.mod h1:XmAOs6UJXpNXRwKk+KY/nv5kL6xXYXyellk+A1pTlko=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9PvrpOfz+Ug=
github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng=
github.com/oschwald/maxminddb-golang v1.6.0 h1:KAJSjdHQ8Kv45nFIbtoLGrGWqHFajOIm7skTyz/+Dls=
github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@ -41,6 +45,7 @@ github.com/spf13/cobra v0.0.4-0.20190109003409-7547e83b2d85/go.mod h1:1l0Ry5zgKv
github.com/spf13/pflag v1.0.4-0.20181223182923-24fa6976df40/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf h1:Z2X3Os7oRzpdJ75iPqWZc0HeJWFYNCvKsfpQwFpRNTA=
@ -58,6 +63,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200808120158-1030fc2bf1d9 h1:yi1hN8dcqI9l8klZfy4B8mJvFmmAxJEePIQQFNSd7Cs=

13
list.txt Normal file
View File

@ -0,0 +1,13 @@
file '/Users/gabek/Downloads/MegaManNetworkTransmission_SS_5633_HQ.mp4'
file '/Users/gabek/Downloads/SampleVideo_720x480_10mb.mp4'
file '/Users/gabek/Downloads/big_buck_bunny_720p_surround.mp4'
file '/Users/gabek/Downloads/ed_hd.mp4'
file '/Users/gabek/Downloads/god.mp4'
file '/Users/gabek/Downloads/mixkit-a-boy-and-a-girl-with-a-mask-dancing-nearby-8689.mp4'
file '/Users/gabek/Downloads/mixkit-couple-on-the-dance-floor-having-fun-344.mp4'
file '/Users/gabek/Downloads/mixkit-disco-ball-spinning-1356.mp4'
file '/Users/gabek/Downloads/mixkit-girl-dancing-in-nightclub-302.mp4'
file '/Users/gabek/Downloads/mixkit-hands-raised-high-on-a-nightclub-dance-floor-341.mp4'
file '/Users/gabek/Downloads/mixkit-popping-dancer-wearing-a-mask-with-neon-lights-3613.mp4'
file '/Users/gabek/Downloads/randomdjset.mp4'
file '/Users/gabek/Downloads/testtrailers.mp4'

View File

@ -2,7 +2,7 @@ package models
//ChatListener represents the listener for the chat server
type ChatListener interface {
ClientAdded(clientID string)
ClientAdded(client Client)
ClientRemoved(clientID string)
MessageSent(message ChatMessage)
}

36
models/client.go Normal file
View File

@ -0,0 +1,36 @@
package models
import (
"net/http"
"time"
"github.com/owncast/owncast/geoip"
"github.com/owncast/owncast/utils"
)
type ConnectedClientsResponse struct {
Clients []Client `json:"clients"`
}
type Client struct {
ConnectedAt time.Time `json:"connectedAt"`
LastSeen time.Time `json:"-"`
MessageCount int `json:"messageCount"`
UserAgent string `json:"userAgent"`
IPAddress string `json:"ipAddress"`
Username *string `json:"username"`
ClientID string `json:"clientID"`
Geo *geoip.GeoDetails `json:"geo"`
}
func GenerateClientFromRequest(req *http.Request) Client {
return Client{
ConnectedAt: time.Now(),
LastSeen: time.Now(),
MessageCount: 0,
UserAgent: req.UserAgent(),
IPAddress: utils.GetIPAddressFromRequest(req),
Username: nil,
ClientID: utils.GenerateClientIDFromRequest(req),
}
}

View File

@ -1,8 +1,6 @@
package models
import (
"time"
"github.com/owncast/owncast/utils"
)
@ -12,7 +10,7 @@ type Stats struct {
OverallMaxViewerCount int `json:"overallMaxViewerCount"`
LastDisconnectTime utils.NullTime `json:"lastDisconnectTime"`
StreamConnected bool `json:"-"`
LastConnectTime utils.NullTime `json:"-"`
Clients map[string]time.Time `json:"-"`
StreamConnected bool `json:"-"`
LastConnectTime utils.NullTime `json:"-"`
Clients map[string]Client `json:"-"`
}

1572
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -9,7 +9,6 @@ import (
"github.com/owncast/owncast/config"
"github.com/owncast/owncast/controllers"
"github.com/owncast/owncast/controllers/admin"
"github.com/owncast/owncast/core/chat"
"github.com/owncast/owncast/core/rtmp"
"github.com/owncast/owncast/router/middleware"
@ -67,6 +66,9 @@ func Start() error {
// Get hardware stats
http.HandleFunc("/api/admin/hardwarestats", middleware.RequireAdminAuth(admin.GetHardwareStats))
// Get a a detailed list of currently connected clients
http.HandleFunc("/api/admin/clients", middleware.RequireAdminAuth(controllers.GetConnectedClients))
port := config.Config.GetPublicWebServerPort()
log.Infof("Web server running on port: %d", port)

View File

@ -1,25 +1,41 @@
package utils
import (
"crypto/md5"
"encoding/hex"
"net"
"net/http"
"strings"
log "github.com/sirupsen/logrus"
)
//GenerateClientIDFromRequest generates a client id from the provided request
func GenerateClientIDFromRequest(req *http.Request) string {
var clientID string
ipAddress := GetIPAddressFromRequest(req)
ipAddressComponents := strings.Split(ipAddress, ":")
ipAddressComponents[len(ipAddressComponents)-1] = ""
clientID := strings.Join(ipAddressComponents, ":") + req.UserAgent()
// Create a MD5 hash of this ip + useragent
hasher := md5.New()
hasher.Write([]byte(clientID))
return hex.EncodeToString(hasher.Sum(nil))
}
// GetIPAddressFromRequest returns the IP address from a http request
func GetIPAddressFromRequest(req *http.Request) string {
ipAddressString := req.RemoteAddr
xForwardedFor := req.Header.Get("X-FORWARDED-FOR")
if xForwardedFor != "" {
clientID = xForwardedFor
} else {
ipAddressString := req.RemoteAddr
ipAddressComponents := strings.Split(ipAddressString, ":")
ipAddressComponents[len(ipAddressComponents)-1] = ""
clientID = strings.Join(ipAddressComponents, ":")
ipAddressString = xForwardedFor
}
// fmt.Println("IP address determined to be", ipAddress)
ip, _, err := net.SplitHostPort(ipAddressString)
if err != nil {
log.Errorln(err)
return ""
}
return clientID + req.UserAgent()
return ip
}