Chat wire protocol (#3125)
* core: remove file extension from emoji name * web: transform emotes to labels when sending * chat: replace br with line break * core: implement emoji cache * chat: send shortcodes for custom emoji * chat: correct esling errors * core: move emoji injection into dedicated function * emoji: integrate emoji into markdown renderer, fix formatting * chat protocol: correct golangci-lint findings * chat field: specify that the contentEditable is an HTMLElement * admin: mention that emoji should have unique names * Prettified Code! * regenerate pack-lock * chat: correct the emphasis tag, provide fallback for other elements --------- Co-authored-by: jprjr <jprjr@users.noreply.github.com>
This commit is contained in:
John Regan
GitHub
parent
e9a4899686
commit
46ca5223f9
@ -4,15 +4,22 @@ import (
|
||||
"bytes"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
"github.com/teris-io/shortid"
|
||||
"github.com/yuin/goldmark"
|
||||
emoji "github.com/yuin/goldmark-emoji"
|
||||
emojiAst "github.com/yuin/goldmark-emoji/ast"
|
||||
emojiDef "github.com/yuin/goldmark-emoji/definition"
|
||||
"github.com/yuin/goldmark/extension"
|
||||
"github.com/yuin/goldmark/renderer/html"
|
||||
"github.com/yuin/goldmark/util"
|
||||
"mvdan.cc/xurls"
|
||||
|
||||
"github.com/owncast/owncast/core/data"
|
||||
"github.com/owncast/owncast/core/user"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@ -66,6 +73,105 @@ func (e *UserMessageEvent) SetDefaults() {
|
||||
e.RenderAndSanitizeMessageBody()
|
||||
}
|
||||
|
||||
// implements the emojiDef.Emojis interface but uses case-insensitive search.
|
||||
// the .children field isn't currently used, but could be used in a future
|
||||
// implementation of say, emoji packs where a child represents a pack.
|
||||
type emojis struct {
|
||||
list []emojiDef.Emoji
|
||||
names map[string]*emojiDef.Emoji
|
||||
children []emojiDef.Emojis
|
||||
}
|
||||
|
||||
// return a new Emojis set.
|
||||
func newEmojis(emotes ...emojiDef.Emoji) emojiDef.Emojis {
|
||||
self := &emojis{
|
||||
list: emotes,
|
||||
names: map[string]*emojiDef.Emoji{},
|
||||
children: []emojiDef.Emojis{},
|
||||
}
|
||||
|
||||
for i := range self.list {
|
||||
emoji := &self.list[i]
|
||||
for _, s := range emoji.ShortNames {
|
||||
self.names[s] = emoji
|
||||
}
|
||||
}
|
||||
|
||||
return self
|
||||
}
|
||||
|
||||
func (self *emojis) Get(shortName string) (*emojiDef.Emoji, bool) {
|
||||
v, ok := self.names[strings.ToLower(shortName)]
|
||||
if ok {
|
||||
return v, ok
|
||||
}
|
||||
|
||||
for _, child := range self.children {
|
||||
v, ok := child.Get(shortName)
|
||||
if ok {
|
||||
return v, ok
|
||||
}
|
||||
}
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (self *emojis) Add(emotes emojiDef.Emojis) {
|
||||
self.children = append(self.children, emotes)
|
||||
}
|
||||
|
||||
func (self *emojis) Clone() emojiDef.Emojis {
|
||||
clone := &emojis{
|
||||
list: self.list,
|
||||
names: self.names,
|
||||
children: make([]emojiDef.Emojis, len(self.children)),
|
||||
}
|
||||
|
||||
copy(clone.children, self.children)
|
||||
|
||||
return clone
|
||||
}
|
||||
|
||||
var (
|
||||
emojiMu sync.Mutex
|
||||
emojiDefs = newEmojis()
|
||||
emojiHTML = make(map[string]string)
|
||||
emojiModTime time.Time
|
||||
emojiHTMLFormat = `<img src="{{ .URL }}" class="emoji" alt=":{{ .Name }}:" title=":{{ .Name }}:">`
|
||||
emojiHTMLTemplate = template.Must(template.New("emojiHTML").Parse(emojiHTMLFormat))
|
||||
)
|
||||
|
||||
func loadEmoji() {
|
||||
modTime, err := data.UpdateEmojiList(false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if modTime.After(emojiModTime) {
|
||||
emojiMu.Lock()
|
||||
defer emojiMu.Unlock()
|
||||
|
||||
emojiHTML = make(map[string]string)
|
||||
|
||||
emojiList := data.GetEmojiList()
|
||||
emojiArr := make([]emojiDef.Emoji, 0)
|
||||
|
||||
for i := 0; i < len(emojiList); i++ {
|
||||
var buf bytes.Buffer
|
||||
err := emojiHTMLTemplate.Execute(&buf, emojiList[i])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
emojiHTML[strings.ToLower(emojiList[i].Name)] = buf.String()
|
||||
|
||||
emoji := emojiDef.NewEmoji(emojiList[i].Name, nil, strings.ToLower(emojiList[i].Name))
|
||||
emojiArr = append(emojiArr, emoji)
|
||||
}
|
||||
|
||||
emojiDefs = newEmojis(emojiArr...)
|
||||
}
|
||||
}
|
||||
|
||||
// RenderAndSanitizeMessageBody will turn markdown into HTML, sanitize raw user-supplied HTML and standardize
|
||||
// the message into something safe and renderable for clients.
|
||||
func (m *MessageEvent) RenderAndSanitizeMessageBody() {
|
||||
@ -98,6 +204,11 @@ func RenderAndSanitize(raw string) string {
|
||||
|
||||
// RenderMarkdown will return HTML rendered from the string body of a chat message.
|
||||
func RenderMarkdown(raw string) string {
|
||||
loadEmoji()
|
||||
|
||||
emojiMu.Lock()
|
||||
defer emojiMu.Unlock()
|
||||
|
||||
markdown := goldmark.New(
|
||||
goldmark.WithRendererOptions(
|
||||
html.WithUnsafe(),
|
||||
@ -112,6 +223,16 @@ func RenderMarkdown(raw string) string {
|
||||
xurls.Strict,
|
||||
),
|
||||
),
|
||||
emoji.New(
|
||||
emoji.WithEmojis(
|
||||
emojiDefs,
|
||||
),
|
||||
emoji.WithRenderingMethod(emoji.Func),
|
||||
emoji.WithRendererFunc(func(w util.BufWriter, source []byte, n *emojiAst.Emoji, config *emoji.RendererConfig) {
|
||||
baseName := n.Value.ShortNames[0]
|
||||
_, _ = w.WriteString(emojiHTML[baseName])
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@ -6,6 +6,8 @@ import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/owncast/owncast/config"
|
||||
"github.com/owncast/owncast/models"
|
||||
@ -15,29 +17,75 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var emojiCacheMu sync.Mutex
|
||||
var emojiCacheData = make([]models.CustomEmoji, 0)
|
||||
var emojiCacheModTime time.Time
|
||||
|
||||
// UpdateEmojiList will update the cache (if required) and
|
||||
// return the modifiation time.
|
||||
func UpdateEmojiList(force bool) (time.Time, error) {
|
||||
var modTime time.Time
|
||||
|
||||
emojiPathInfo, err := os.Stat(config.CustomEmojiPath)
|
||||
if err != nil {
|
||||
return modTime, err
|
||||
}
|
||||
|
||||
modTime = emojiPathInfo.ModTime()
|
||||
|
||||
if modTime.After(emojiCacheModTime) || force {
|
||||
emojiCacheMu.Lock()
|
||||
defer emojiCacheMu.Unlock()
|
||||
|
||||
// double-check that another thread didn't update this while waiting.
|
||||
if modTime.After(emojiCacheModTime) || force {
|
||||
emojiCacheModTime = modTime
|
||||
if force {
|
||||
emojiCacheModTime = time.Now()
|
||||
}
|
||||
emojiFS := os.DirFS(config.CustomEmojiPath)
|
||||
|
||||
emojiCacheData = make([]models.CustomEmoji, 0)
|
||||
|
||||
walkFunction := func(path string, d os.DirEntry, err error) error {
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
emojiPath := filepath.Join(config.EmojiDir, path)
|
||||
fileName := d.Name()
|
||||
fileBase := fileName[:len(fileName)-len(filepath.Ext(fileName))]
|
||||
singleEmoji := models.CustomEmoji{Name: fileBase, URL: emojiPath}
|
||||
emojiCacheData = append(emojiCacheData, singleEmoji)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := fs.WalkDir(emojiFS, ".", walkFunction); err != nil {
|
||||
log.Errorln("unable to fetch emojis: " + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return modTime, nil
|
||||
}
|
||||
|
||||
// GetEmojiList returns a list of custom emoji from the emoji directory.
|
||||
func GetEmojiList() []models.CustomEmoji {
|
||||
emojiFS := os.DirFS(config.CustomEmojiPath)
|
||||
|
||||
emojiResponse := make([]models.CustomEmoji, 0)
|
||||
|
||||
walkFunction := func(path string, d os.DirEntry, err error) error {
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
emojiPath := filepath.Join(config.EmojiDir, path)
|
||||
singleEmoji := models.CustomEmoji{Name: d.Name(), URL: emojiPath}
|
||||
emojiResponse = append(emojiResponse, singleEmoji)
|
||||
_, err := UpdateEmojiList(false)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := fs.WalkDir(emojiFS, ".", walkFunction); err != nil {
|
||||
log.Errorln("unable to fetch emojis: " + err.Error())
|
||||
return emojiResponse
|
||||
}
|
||||
// Lock to make sure this doesn't get updated in the middle of reading
|
||||
emojiCacheMu.Lock()
|
||||
defer emojiCacheMu.Unlock()
|
||||
|
||||
return emojiResponse
|
||||
// return a copy of cache data, ensures underlying slice isn't affected
|
||||
// by future update
|
||||
emojiData := make([]models.CustomEmoji, len(emojiCacheData))
|
||||
copy(emojiData, emojiCacheData)
|
||||
|
||||
return emojiData
|
||||
}
|
||||
|
||||
// SetupEmojiDirectory sets up the custom emoji directory by copying all built-in
|
||||
|
||||
1
go.mod
1
go.mod
@ -69,6 +69,7 @@ require (
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/oschwald/maxminddb-golang v1.11.0 // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.2 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
|
||||
3
go.sum
3
go.sum
@ -160,9 +160,12 @@ github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYm
|
||||
github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
|
||||
github.com/valyala/gozstd v1.11.0 h1:VV6qQFt+4sBBj9OJ7eKVvsFAMy59Urcs9Lgd+o5FOw0=
|
||||
github.com/valyala/gozstd v1.11.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
|
||||
github.com/yuin/goldmark v1.3.7/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU=
|
||||
github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark-emoji v1.0.2 h1:c/RgTShNgHTtc6xdz2KKI74jJr6rWi7FPgnP9GAsO5s=
|
||||
github.com/yuin/goldmark-emoji v1.0.2/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
|
||||
|
||||
@ -3,6 +3,7 @@ import React, { FC, useEffect, useReducer, useRef, useState } from 'react';
|
||||
import { useRecoilValue } from 'recoil';
|
||||
import ContentEditable from 'react-contenteditable';
|
||||
import sanitizeHtml from 'sanitize-html';
|
||||
import GraphemeSplitter from 'grapheme-splitter';
|
||||
|
||||
import dynamic from 'next/dynamic';
|
||||
import classNames from 'classnames';
|
||||
@ -32,32 +33,118 @@ export type ChatTextFieldProps = {
|
||||
};
|
||||
|
||||
const characterLimit = 300;
|
||||
const maxNodeDepth = 10;
|
||||
const graphemeSplitter = new GraphemeSplitter();
|
||||
|
||||
const getNodeTextContent = (node, depth) => {
|
||||
let text = '';
|
||||
|
||||
if (depth > maxNodeDepth) return text;
|
||||
if (node === null) return text;
|
||||
|
||||
switch (node.nodeType) {
|
||||
case Node.CDATA_SECTION_NODE: // unlikely
|
||||
case Node.TEXT_NODE: {
|
||||
text = node.nodeValue;
|
||||
break;
|
||||
}
|
||||
case Node.ELEMENT_NODE: {
|
||||
switch (node.tagName.toLowerCase()) {
|
||||
case 'img': {
|
||||
text = node.getAttribute('alt') || '';
|
||||
break;
|
||||
}
|
||||
case 'br': {
|
||||
text = '\n';
|
||||
break;
|
||||
}
|
||||
case 'strong':
|
||||
case 'b': {
|
||||
/* markdown representation of bold/strong */
|
||||
text = '**';
|
||||
for (let i = 0; i < node.childNodes.length; i += 1) {
|
||||
text += getNodeTextContent(node.childNodes[i], depth + 1);
|
||||
}
|
||||
text += '**';
|
||||
break;
|
||||
}
|
||||
case 'em':
|
||||
case 'i': {
|
||||
/* markdown representation of italic/emphasis */
|
||||
text = '*';
|
||||
for (let i = 0; i < node.childNodes.length; i += 1) {
|
||||
text += getNodeTextContent(node.childNodes[i], depth + 1);
|
||||
}
|
||||
text += '*';
|
||||
break;
|
||||
}
|
||||
case 'p': {
|
||||
text = '\n';
|
||||
for (let i = 0; i < node.childNodes.length; i += 1) {
|
||||
text += getNodeTextContent(node.childNodes[i], depth + 1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'a':
|
||||
case 'span':
|
||||
case 'div': {
|
||||
for (let i = 0; i < node.childNodes.length; i += 1) {
|
||||
text += getNodeTextContent(node.childNodes[i], depth + 1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
/* nodes which should specifically not be parsed */
|
||||
case 'script':
|
||||
case 'style': {
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
text = node.textContent;
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return text;
|
||||
};
|
||||
|
||||
const getTextContent = node => {
|
||||
const text = getNodeTextContent(node, 0)
|
||||
.replace(/^\s+/, '') /* remove leading whitespace */
|
||||
.replace(/\s+$/, '') /* remove trailing whitespace */
|
||||
.replace(/\n([^\n])/g, ' \n$1'); /* single line break to markdown break */
|
||||
return text;
|
||||
};
|
||||
|
||||
export const ChatTextField: FC<ChatTextFieldProps> = ({ defaultText, enabled, focusInput }) => {
|
||||
const [characterCount, setCharacterCount] = useState(defaultText?.length);
|
||||
const websocketService = useRecoilValue<WebsocketService>(websocketServiceAtom);
|
||||
const text = useRef(defaultText || '');
|
||||
const contentEditable = React.createRef<HTMLElement>();
|
||||
const [customEmoji, setCustomEmoji] = useState([]);
|
||||
|
||||
// This is a bit of a hack to force the component to re-render when the text changes.
|
||||
// By default when updating a ref the component doesn't re-render.
|
||||
const [, forceUpdate] = useReducer(x => x + 1, 0);
|
||||
|
||||
const getCharacterCount = () => text.current.length;
|
||||
const getCharacterCount = () => {
|
||||
const message = getTextContent(contentEditable.current);
|
||||
return graphemeSplitter.countGraphemes(message);
|
||||
};
|
||||
|
||||
const sendMessage = () => {
|
||||
const count = getCharacterCount();
|
||||
|
||||
if (!websocketService) {
|
||||
console.log('websocketService is not defined');
|
||||
return;
|
||||
}
|
||||
|
||||
const message = getTextContent(contentEditable.current);
|
||||
const count = graphemeSplitter.countGraphemes(message);
|
||||
if (count === 0 || count > characterLimit) return;
|
||||
|
||||
let message = text.current;
|
||||
// Strip the opening and closing <p> tags.
|
||||
message = message.replace(/^<p>|<\/p>$/g, '');
|
||||
websocketService.send({ type: MessageType.CHAT, body: message });
|
||||
|
||||
// Clear the input.
|
||||
@ -70,18 +157,19 @@ export const ChatTextField: FC<ChatTextFieldProps> = ({ defaultText, enabled, fo
|
||||
const output = text.current + textToInsert;
|
||||
text.current = output;
|
||||
|
||||
setCharacterCount(getCharacterCount());
|
||||
forceUpdate();
|
||||
};
|
||||
|
||||
// Native emoji
|
||||
const onEmojiSelect = (emoji: string) => {
|
||||
setCharacterCount(getCharacterCount() + 1);
|
||||
insertTextAtEnd(emoji);
|
||||
};
|
||||
|
||||
// Custom emoji images
|
||||
const onCustomEmojiSelect = (name: string, emoji: string) => {
|
||||
const html = `<img src="${emoji}" alt="${name}" title="${name}" class="emoji" />`;
|
||||
const html = `<img src="${emoji}" alt=":${name}:" title=":${name}:" class="emoji" />`;
|
||||
setCharacterCount(getCharacterCount() + name.length + 2);
|
||||
insertTextAtEnd(html);
|
||||
};
|
||||
|
||||
@ -161,6 +249,7 @@ export const ChatTextField: FC<ChatTextFieldProps> = ({ defaultText, enabled, fo
|
||||
style={{ width: '100%' }}
|
||||
role="textbox"
|
||||
aria-label="Chat text input"
|
||||
innerRef={contentEditable}
|
||||
/>
|
||||
{enabled && (
|
||||
<div style={{ display: 'flex', paddingLeft: '5px' }}>
|
||||
|
||||
37204
web/package-lock.json
generated
37204
web/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -30,6 +30,7 @@
|
||||
"chart.js": "^4.2.0",
|
||||
"classnames": "2.3.2",
|
||||
"date-fns": "^2.29.3",
|
||||
"grapheme-splitter": "^1.0.4",
|
||||
"interweave": "^13.0.0",
|
||||
"interweave-autolink": "^5.1.0",
|
||||
"lodash": "4.17.21",
|
||||
|
||||
@ -136,7 +136,8 @@ const Emoji = () => {
|
||||
<Title>Emojis</Title>
|
||||
<Paragraph>
|
||||
Here you can upload new custom emojis for usage in the chat. When uploading a new emoji, the
|
||||
filename will be used as emoji name.
|
||||
filename without extension will be used as emoji name. Additionally, emoji names are
|
||||
case-insensitive. For best results, ensure all emoji have unique names.
|
||||
</Paragraph>
|
||||
<br />
|
||||
<Upload
|
||||
|
||||
Loading…
Reference in New Issue
Block a user