From 5e6bc50b59f112ae61ced5fa33b19addb35c6317 Mon Sep 17 00:00:00 2001 From: Gabe Kangas Date: Sun, 6 Mar 2022 17:18:51 -0800 Subject: [PATCH] Handle pagination for the federated actions & followers responses (#1731) * Add pagination for admin social list * Use Paginated API for followers tab on frontend --- activitypub/controllers/followers.go | 2 +- activitypub/outbox/outbox.go | 2 +- activitypub/persistence/followers.go | 12 ++++-- activitypub/persistence/persistence.go | 11 +++-- controllers/admin/federation.go | 13 ++++-- controllers/followers.go | 10 +++-- controllers/pagination.go | 7 ++++ db/query.sql | 3 ++ db/query.sql.go | 11 +++++ router/middleware/pagination.go | 39 ++++++++++++++++++ router/router.go | 6 +-- webroot/js/components/federation/followers.js | 40 +++++++++---------- 12 files changed, 118 insertions(+), 38 deletions(-) create mode 100644 controllers/pagination.go create mode 100644 router/middleware/pagination.go diff --git a/activitypub/controllers/followers.go b/activitypub/controllers/followers.go index 11193f51a..9a1ba7bdf 100644 --- a/activitypub/controllers/followers.go +++ b/activitypub/controllers/followers.go @@ -98,7 +98,7 @@ func getFollowersPage(page string, r *http.Request) (vocab.ActivityStreamsOrdere return nil, errors.Wrap(err, "unable to get follower count") } - followers, err := persistence.GetFederationFollowers(followersPageSize, (pageInt-1)*followersPageSize) + followers, _, err := persistence.GetFederationFollowers(followersPageSize, (pageInt-1)*followersPageSize) if err != nil { return nil, errors.Wrap(err, "unable to get federation followers") } diff --git a/activitypub/outbox/outbox.go b/activitypub/outbox/outbox.go index cf9c2da46..ce79fbe06 100644 --- a/activitypub/outbox/outbox.go +++ b/activitypub/outbox/outbox.go @@ -171,7 +171,7 @@ func getHashtagLinkHTMLFromTagString(baseHashtag string) string { func SendToFollowers(payload []byte) error { localActor := apmodels.MakeLocalIRIForAccount(data.GetDefaultFederationUsername()) - followers, err := persistence.GetFederationFollowers(-1, 0) + followers, _, err := persistence.GetFederationFollowers(-1, 0) if err != nil { log.Errorln("unable to fetch followers to send to", err) return errors.New("unable to fetch followers to send payload to") diff --git a/activitypub/persistence/followers.go b/activitypub/persistence/followers.go index 77863038e..df40c581f 100644 --- a/activitypub/persistence/followers.go +++ b/activitypub/persistence/followers.go @@ -6,6 +6,7 @@ import ( "github.com/owncast/owncast/db" "github.com/owncast/owncast/models" "github.com/owncast/owncast/utils" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -44,14 +45,19 @@ func GetFollowerCount() (int64, error) { } // GetFederationFollowers will return a slice of the followers we keep track of locally. -func GetFederationFollowers(limit int, offset int) ([]models.Follower, error) { +func GetFederationFollowers(limit int, offset int) ([]models.Follower, int, error) { ctx := context.Background() + total, err := _datastore.GetQueries().GetFollowerCount(ctx) + if err != nil { + return nil, 0, errors.Wrap(err, "unable to fetch total number of followers") + } + followersResult, err := _datastore.GetQueries().GetFederationFollowersWithOffset(ctx, db.GetFederationFollowersWithOffsetParams{ Limit: int32(limit), Offset: int32(offset), }) if err != nil { - return nil, err + return nil, 0, err } followers := make([]models.Follower, 0) @@ -69,7 +75,7 @@ func GetFederationFollowers(limit int, offset int) ([]models.Follower, error) { followers = append(followers, singleFollower) } - return followers, nil + return followers, int(total), nil } // GetPendingFollowRequests will return pending follow requests. diff --git a/activitypub/persistence/persistence.go b/activitypub/persistence/persistence.go index 842cce712..68d61d8a0 100644 --- a/activitypub/persistence/persistence.go +++ b/activitypub/persistence/persistence.go @@ -319,18 +319,23 @@ func SaveInboundFediverseActivity(objectIRI string, actorIRI string, eventType s // GetInboundActivities will return a collection of saved, federated activities // limited and offset by the values provided to support pagination. -func GetInboundActivities(limit int, offset int) ([]models.FederatedActivity, error) { +func GetInboundActivities(limit int, offset int) ([]models.FederatedActivity, int, error) { ctx := context.Background() rows, err := _datastore.GetQueries().GetInboundActivitiesWithOffset(ctx, db.GetInboundActivitiesWithOffsetParams{ Limit: int32(limit), Offset: int32(offset), }) if err != nil { - return nil, err + return nil, 0, err } activities := make([]models.FederatedActivity, 0) + total, err := _datastore.GetQueries().GetInboundActivityCount(context.Background()) + if err != nil { + return nil, 0, errors.Wrap(err, "unable to fetch total activity count") + } + for _, row := range rows { singleActivity := models.FederatedActivity{ IRI: row.Iri, @@ -341,7 +346,7 @@ func GetInboundActivities(limit int, offset int) ([]models.FederatedActivity, er activities = append(activities, singleActivity) } - return activities, nil + return activities, int(total), nil } // HasPreviouslyHandledInboundActivity will return if we have previously handled diff --git a/controllers/admin/federation.go b/controllers/admin/federation.go index 43f2aa695..55032df3c 100644 --- a/controllers/admin/federation.go +++ b/controllers/admin/federation.go @@ -160,12 +160,19 @@ func SetFederationBlockDomains(w http.ResponseWriter, r *http.Request) { // GetFederatedActions will return the saved list of accepted inbound // federated activities. -func GetFederatedActions(w http.ResponseWriter, r *http.Request) { - activities, err := persistence.GetInboundActivities(100, 0) +func GetFederatedActions(page int, pageSize int, w http.ResponseWriter, r *http.Request) { + offset := pageSize * page + + activities, total, err := persistence.GetInboundActivities(pageSize, offset) if err != nil { controllers.WriteSimpleResponse(w, false, err.Error()) return } - controllers.WriteResponse(w, activities) + response := controllers.PaginatedResponse{ + Total: total, + Results: activities, + } + + controllers.WriteResponse(w, response) } diff --git a/controllers/followers.go b/controllers/followers.go index 8561d25a4..0f9c98eaa 100644 --- a/controllers/followers.go +++ b/controllers/followers.go @@ -7,12 +7,16 @@ import ( ) // GetFollowers will handle an API request to fetch the list of followers (non-activitypub response). -func GetFollowers(w http.ResponseWriter, r *http.Request) { - followers, err := persistence.GetFederationFollowers(-1, 0) +func GetFollowers(offset int, limit int, w http.ResponseWriter, r *http.Request) { + followers, total, err := persistence.GetFederationFollowers(limit, offset) if err != nil { WriteSimpleResponse(w, false, "unable to fetch followers") return } - WriteResponse(w, followers) + response := PaginatedResponse{ + Total: total, + Results: followers, + } + WriteResponse(w, response) } diff --git a/controllers/pagination.go b/controllers/pagination.go new file mode 100644 index 000000000..9231dd144 --- /dev/null +++ b/controllers/pagination.go @@ -0,0 +1,7 @@ +package controllers + +// PaginatedResponse is a structure for returning a total count with results. +type PaginatedResponse struct { + Total int `json:"total"` + Results interface{} `json:"results"` +} diff --git a/db/query.sql b/db/query.sql index 73cde27fc..5dda612f6 100644 --- a/db/query.sql +++ b/db/query.sql @@ -47,6 +47,9 @@ INSERT INTO ap_outbox(iri, value, type, live_notification) values($1, $2, $3, $4 -- name: AddToAcceptedActivities :exec INSERT INTO ap_accepted_activities(iri, actor, type, timestamp) values($1, $2, $3, $4); +-- name: GetInboundActivityCount :one +SELECT count(*) FROM ap_accepted_activities; + -- name: GetInboundActivitiesWithOffset :many SELECT iri, actor, type, timestamp FROM ap_accepted_activities ORDER BY timestamp DESC LIMIT $1 OFFSET $2; diff --git a/db/query.sql.go b/db/query.sql.go index 9350f69a8..53d59be80 100644 --- a/db/query.sql.go +++ b/db/query.sql.go @@ -280,6 +280,17 @@ func (q *Queries) GetInboundActivitiesWithOffset(ctx context.Context, arg GetInb return items, nil } +const getInboundActivityCount = `-- name: GetInboundActivityCount :one +SELECT count(*) FROM ap_accepted_activities +` + +func (q *Queries) GetInboundActivityCount(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, getInboundActivityCount) + var count int64 + err := row.Scan(&count) + return count, err +} + const getLocalPostCount = `-- name: GetLocalPostCount :one SElECT count(*) FROM ap_outbox ` diff --git a/router/middleware/pagination.go b/router/middleware/pagination.go new file mode 100644 index 000000000..674f71bbe --- /dev/null +++ b/router/middleware/pagination.go @@ -0,0 +1,39 @@ +package middleware + +import ( + "net/http" + "strconv" +) + +// PaginatedHandlerFunc is a handler for endpoints that require pagination. +type PaginatedHandlerFunc func(int, int, http.ResponseWriter, *http.Request) + +// HandlePagination is a middleware handler that pulls pagination values +// and passes them along. +func HandlePagination(handler PaginatedHandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Default 50 items per page + limitString := r.URL.Query().Get("limit") + if limitString == "" { + limitString = "50" + } + limit, err := strconv.Atoi(limitString) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + + // Default first page 0 + offsetString := r.URL.Query().Get("offset") + if offsetString == "" { + offsetString = "0" + } + offset, err := strconv.Atoi(offsetString) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + + handler(offset, limit, w, r) + } +} diff --git a/router/router.go b/router/router.go index 21c237aea..fa8bb34b9 100644 --- a/router/router.go +++ b/router/router.go @@ -77,7 +77,7 @@ func Start() error { http.HandleFunc("/api/remotefollow", controllers.RemoteFollow) // return followers - http.HandleFunc("/api/followers", controllers.GetFollowers) + http.HandleFunc("/api/followers", middleware.HandlePagination(controllers.GetFollowers)) // Authenticated admin requests @@ -127,7 +127,7 @@ func Start() error { http.HandleFunc("/api/admin/chat/users/moderators", middleware.RequireAdminAuth(admin.GetModerators)) // return followers - http.HandleFunc("/api/admin/followers", middleware.RequireAdminAuth(controllers.GetFollowers)) + http.HandleFunc("/api/admin/followers", middleware.RequireAdminAuth(middleware.HandlePagination(controllers.GetFollowers))) // Get a list of pending follow requests http.HandleFunc("/api/admin/followers/pending", middleware.RequireAdminAuth(admin.GetPendingFollowRequests)) @@ -310,7 +310,7 @@ func Start() error { http.HandleFunc("/api/admin/federation/send", middleware.RequireAdminAuth(admin.SendFederatedMessage)) // Return federated activities - http.HandleFunc("/api/admin/federation/actions", middleware.RequireAdminAuth(admin.GetFederatedActions)) + http.HandleFunc("/api/admin/federation/actions", middleware.RequireAdminAuth(middleware.HandlePagination(admin.GetFederatedActions))) // ActivityPub has its own router activitypub.Start(data.GetDatastore()) diff --git a/webroot/js/components/federation/followers.js b/webroot/js/components/federation/followers.js index 6bd6a08b8..89825af62 100644 --- a/webroot/js/components/federation/followers.js +++ b/webroot/js/components/federation/followers.js @@ -2,7 +2,6 @@ import { h, Component } from '/js/web_modules/preact.js'; import htm from '/js/web_modules/htm.js'; import { URL_FOLLOWERS } from '/js/utils/constants.js'; const html = htm.bind(h); -import { paginateArray } from '../../utils/helpers.js'; export default class FollowerList extends Component { constructor(props) { super(props); @@ -10,6 +9,8 @@ export default class FollowerList extends Component { this.state = { followers: [], followersPage: 0, + currentPage: 0, + total: 0, }; } @@ -22,23 +23,26 @@ export default class FollowerList extends Component { } async getFollowers() { - const response = await fetch(URL_FOLLOWERS); + const { currentPage } = this.state; + const limit = 16; + const offset = currentPage * limit; + const u = `${URL_FOLLOWERS}?offset=${offset}&limit=${limit}`; + const response = await fetch(u); const followers = await response.json(); this.setState({ - followers: followers, + followers: followers.results, + total: response.total, }); } changeFollowersPage(page) { - this.setState({ followersPage: page }); + this.setState({ currentPage: page }); + this.getFollowers(); } render() { - const FOLLOWER_PAGE_SIZE = 16; - const { followersPage } = this.state; - - const { followers } = this.state; + const { followers, total, currentPage } = this.state; if (!followers) { return null; } @@ -57,21 +61,15 @@ export default class FollowerList extends Component {

`; - const paginatedFollowers = paginateArray( - followers, - followersPage + 1, - FOLLOWER_PAGE_SIZE - ); - const paginationControls = - paginatedFollowers.totalPages > 1 && - Array(paginatedFollowers.totalPages) + total > 1 && + Array(total) .fill() .map((x, n) => { const activePageClass = - n === followersPage && + n === currentPage && 'bg-indigo-600 rounded-full shadow-md focus:shadow-md text-white'; - return html`
  • + return html`
  • this.changeFollowersPage(n)} @@ -85,13 +83,13 @@ export default class FollowerList extends Component {
    ${followers.length === 0 && noFollowersInfo} - ${paginatedFollowers.items.map((follower) => { + ${followers.map((follower) => { return html` <${SingleFollower} user=${follower} /> `; })}
    -