Fix HLS automatic test (#2572)

* mv automated test cleanup to tools.sh

check media file exists before streaming in test/ocTestStream.sh
fix automatic test ffmpeg detection
mv trap to tools
mv update_storage_config() to tools
enable S3 test

* fix video file check in test/ocTestStream.sh

* cleanup ci

* mv auto test owncast build-run to start_owncast()

* suppress cleanup errors

* fix style

* fix Browser Test paths

* use pull_request event for Browser Tests

* explicitly mention when test is skipped

* refactor shell scripts

.github/workflows/automated-end-to-end-api.yaml

@@ -3,12 +3,9 @@ name: Automated API tests
 on:
   push:
     paths-ignore:
-      - 'webroot/**'
       - 'web/**'
-
   pull_request:
     paths-ignore:
-      - 'webroot/**'
       - 'web/**'
 
 jobs:

.github/workflows/browser-testing.yml

@@ -3,10 +3,12 @@ name: Browser Tests
 on:
   push:
     paths:
-      - web/**
-  pull_request_target:
+      - 'web/**'
+      - 'test/automated/browser/**'
+  pull_request:
     paths:
-      - web/**
+      - 'web/**'
+      - 'test/automated/browser/**'
 jobs:
   cypress-run:
     runs-on: ubuntu-latest

.github/workflows/go-lint.yml

@@ -2,12 +2,9 @@ name: Lint
 on:
   push:
     paths-ignore:
-      - 'webroot/**'
       - 'web/**'
-
   pull_request:
     paths-ignore:
-      - 'webroot/**'
       - 'web/**'
 
 permissions:

test/automated/api/run.sh

@@ -1,37 +1,17 @@
 #!/bin/bash
 
-source ../tools.sh
+set -e
 
-TEMP_DB=$(mktemp)
+source ../tools.sh
 
 # Install the node test framework
 npm install --quiet --no-progress
 
-ffmpegInstall
+install_ffmpeg
 
-pushd ../../.. >/dev/null || exit
+start_owncast
 
-# Build and run owncast from source
-go build -o owncast main.go
-./owncast -database "$TEMP_DB" &
-SERVER_PID=$!
-
-popd >/dev/null || exit
-sleep 5
-
-# Start streaming the test file over RTMP to
-# the local owncast instance.
-../../ocTestStream.sh &
-FFMPEG_PID=$!
-
-function finish {
-	kill $SERVER_PID $FFMPEG_PID
-	rm -fr "$TEMP_DB" "$FFMPEG_PATH"
-}
-trap finish EXIT
-
-echo "Waiting..."
-sleep 15
+start_stream
 
 # Run the tests against the instance.
 npm test

test/automated/browser/run.sh

@@ -5,7 +5,6 @@ set -o pipefail
 
 source ../tools.sh
 
-TEMP_DB=$(mktemp)
 BUILD_ID=$((RANDOM % 7200 + 600))
 BROWSER="electron" # Default. Will try to use Google Chrome.
 
@@ -16,61 +15,43 @@ else
 	echo "Google Chrome not found. Using Electron."
 fi
 
-# Change to the root directory of the repository
-pushd "$(git rev-parse --show-toplevel)"
 
 # Bundle the updated web code into the server codebase.
 if [ -z "$SKIP_BUILD" ]; then
 	echo "Bundling web code into server..."
+
+	# Change to the root directory of the repository
+	pushd "$(git rev-parse --show-toplevel)"
+
 	./build/web/bundleWeb.sh >/dev/null
+
+	popd
 else
 	echo "Skipping web build..."
 fi
 
+
 # Install the web test framework
 if [ -z "$SKIP_BUILD" ]; then
 	echo "Installing test dependencies..."
-	pushd test/automated/browser
 	npm install --quiet --no-progress
 
-	popd
 else
 	echo "Skipping dependencies installation"
 fi
 
 set -o nounset
 
-ffmpegInstall
+install_ffmpeg
 
-# Build and run owncast from source
-echo "Building owncast..."
-go build -o owncast main.go
-
-echo "Running owncast..."
-./owncast -database "$TEMP_DB" &
-SERVER_PID=$!
-
-pushd test/automated/browser
+start_owncast
 
 # Run cypress browser tests for desktop
 npx cypress run --browser "$BROWSER" --group "desktop-offline" --env tags=desktop --ci-build-id $BUILD_ID --tag "desktop,offline" --record --key e9c8b547-7a8f-452d-8c53-fd7531491e3b --spec "cypress/e2e/offline/*.cy.js"
 # Run cypress browser tests for mobile
 npx cypress run --browser "$BROWSER" --group "mobile-offline" --ci-build-id $BUILD_ID --tag "mobile,offline" --record --key e9c8b547-7a8f-452d-8c53-fd7531491e3b --spec "cypress/e2e/offline/*.cy.js" --config viewportWidth=375,viewportHeight=667
 
-# Start streaming the test file over RTMP to
-# the local owncast instance.
-echo "Waiting for stream to start..."
-../../ocTestStream.sh &
-STREAMING_CLIENT=$!
-
-function finish {
-	echo "Cleaning up..."
-	kill $SERVER_PID $STREAMING_CLIENT
-	rm -fr "$TEMP_DB" "$FFMPEG_PATH"
-}
-trap finish EXIT SIGHUP SIGINT SIGTERM SIGQUIT SIGABRT SIGTERM
-
-sleep 20
+start_stream
 
 # Run cypress browser tests for desktop
 npx cypress run --browser "$BROWSER" --group "desktop-online" --env tags=desktop --ci-build-id $BUILD_ID --tag "desktop,online" --record --key e9c8b547-7a8f-452d-8c53-fd7531491e3b --spec "cypress/e2e/online/*.cy.js"

test/automated/hls/run.sh

@@ -4,71 +4,31 @@ set -e
 
 source ../tools.sh
 
-function update_storage_config() {
-  echo "Configuring external storage to use ${S3_BUCKET}..."
-
-  # Hard coded to admin:abc123 for auth
-  curl 'http://localhost:8080/api/admin/config/s3' \
-  -H 'Authorization: Basic YWRtaW46YWJjMTIz' \
-  --data-raw "{\"value\":{\"accessKey\":\"${S3_ACCESS_KEY}\",\"acl\":\"\",\"bucket\":\"${S3_BUCKET}\",\"enabled\":true,\"endpoint\":\"${S3_ENDPOINT}\",\"region\":\"${S3_REGION}\",\"secret\":\"${S3_SECRET}\",\"servingEndpoint\":\"\"}}"
-}
-
-TEMP_DB=$(mktemp)
-
 # Install the node test framework
 npm install --silent >/dev/null
 
-ffmpegInstall
+install_ffmpeg
 
-pushd ../../.. >/dev/null
+start_owncast
 
-# Build and run owncast from source
-go build -o owncast main.go
-./owncast -database "$TEMP_DB" &
-SERVER_PID=$!
-
-popd >/dev/null
-sleep 5
-
-# Start the stream.
-../../ocTestStream.sh &
-STREAMING_CLIENT=$!
-
-function finish {
-  echo "Cleaning up..."
-  kill $SERVER_PID $STREAMING_CLIENT
-  rm -fr "$TEMP_DB" "$FFMPEG_PATH"
-}
-trap finish EXIT
-
-echo "Waiting..."
-sleep 13
+start_stream
 
 # Run tests against a fresh install with no settings.
 npm test
 
-# REMOVE this ONCE #2571 IS FIXED
-exit 0
+# Kill the stream.
+kill_with_kids "$STREAM_PID"
 
 # Determine if we should continue testing with S3 configuration.
 if [[ -z "${S3_BUCKET}" ]]; then
-  echo "No S3 configuration set"
+  echo "No S3 configuration is set. Skipping S3 tests!"
   exit 0
 fi
 
-# Kill the stream.
-kill $STREAMING_CLIENT
-sleep 5
-
 # Update the server config to use S3 for storage.
 update_storage_config
 
-# start the stream.
-../../ocTestStream.sh &
-STREAMING_CLIENT=$!
-
-echo "Waiting..."
-sleep 13
+start_stream
 
 # Re-run the HLS test against the external storage configuration.
 npm test

test/automated/tools.sh

@@ -1,22 +1,26 @@
 #!/bin/bash
 
-ffmpegInstall(){
+set -e
+
+function install_ffmpeg() {
     # install a specific version of ffmpeg
 
     FFMPEG_VER="4.4.1"
     FFMPEG_PATH="$(pwd)/ffmpeg-$FFMPEG_VER"
+    PATH=$FFMPEG_PATH:$PATH
 
     if ! [[ -d "$FFMPEG_PATH" ]]; then
         mkdir "$FFMPEG_PATH"
     fi
 
-    pushd "$FFMPEG_PATH" >/dev/null || exit
+    pushd "$FFMPEG_PATH" >/dev/null
 
     if [[ -x "$FFMPEG_PATH/ffmpeg" ]]; then
     
         ffmpeg_version=$("$FFMPEG_PATH/ffmpeg" -version | awk -F 'ffmpeg version' '{print $2}' | awk 'NR==1{print $1}')
 
         if [[ "$ffmpeg_version" == "$FFMPEG_VER-static" ]]; then
+            popd >/dev/null
             return 0
         else
             mv "$FFMPEG_PATH/ffmpeg" "$FFMPEG_PATH/ffmpeg.bk" || rm -f "$FFMPEG_PATH/ffmpeg"
@@ -29,5 +33,64 @@ ffmpegInstall(){
     chmod +x ffmpeg
     PATH=$FFMPEG_PATH:$PATH
 
-    popd >/dev/null || exit
-}
+    popd >/dev/null
+}
+
+function start_owncast() {
+    # Build and run owncast from source
+    echo "Building owncast..."
+    pushd "$(git rev-parse --show-toplevel)" >/dev/null
+    go build -o owncast main.go
+
+    echo "Running owncast..."
+    ./owncast -database "$TEMP_DB" &
+    SERVER_PID=$!
+    popd >/dev/null
+
+    sleep 5
+
+}
+
+function start_stream() {
+    # Start streaming the test file over RTMP to the local owncast instance.
+    ../../ocTestStream.sh &
+    STREAM_PID=$!
+
+    echo "Waiting for stream to start..."
+    sleep 12
+}
+
+function update_storage_config() {
+    echo "Configuring external storage to use ${S3_BUCKET}..."
+
+    # Hard-coded to admin:abc123 for auth
+    curl --fail 'http://localhost:8080/api/admin/config/s3' \
+    -H 'Authorization: Basic YWRtaW46YWJjMTIz' \
+    --data-raw "{\"value\":{\"accessKey\":\"${S3_ACCESS_KEY}\",\"acl\":\"\",\"bucket\":\"${S3_BUCKET}\",\"enabled\":true,\"endpoint\":\"${S3_ENDPOINT}\",\"region\":\"${S3_REGION}\",\"secret\":\"${S3_SECRET}\",\"servingEndpoint\":\"\"}}"
+}
+
+function kill_with_kids() {
+    # kill a process and all its children (by pid)! return no error.
+
+    if [[ -n $1 ]]; then
+        mapfile -t CHILDREN_PID_LIST < <(ps --ppid "$1" -o pid= || true)
+        for child_pid in "${CHILDREN_PID_LIST[@]}"; do
+            kill "$child_pid" &>/dev/null || true
+            wait "$child_pid" &>/dev/null || true
+        done
+        kill "$1" &>/dev/null || true
+        wait "$1" &>/dev/null || true
+    fi
+}
+
+function finish() {
+    echo "Cleaning up..."
+    kill_with_kids "$STREAM_PID"
+    kill "$SERVER_PID" &>/dev/null || true
+    wait "$SERVER_PID" &>/dev/null || true
+	rm -fr "$TEMP_DB"
+}
+
+trap finish EXIT
+
+TEMP_DB=$(mktemp)

test/ocTestStream.sh

@@ -85,7 +85,13 @@ else
   rm -f list.txt
   for file in $CONTENT
   do
-    echo "file '$file'" >> list.txt
+    if [[ -f "$file" ]]; then
+      echo "file '$file'" >> list.txt
+    else
+      echo "ERROR: File not found: $file"
+      exit 1
+    fi
+    
   done
 
   function finish {