Add a workflow for WebTransport E2E test. (#1040)

* Add a workflow for WebTransport E2E test.

* Add karma tests.

* Address comments.

* Remove unnecessary methods from sample service for E2E test.

* Remove owt.js and rest-sample.js.

Author: jianjunz

.github/workflows/webtransport.yaml

@@ -0,0 +1,82 @@
+name: WebTransport E2E Test
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    runs-on: ubuntu-18.04
+    services:
+      rabbitmq:
+          image: rabbitmq:latest
+          ports:
+            - 5672:5672
+    steps:
+      - uses: actions/checkout@v2
+        name: Checkout server code
+      - uses: actions/checkout@v2
+        name: Checkout JavaScript SDK and sample
+        with:
+          repository: open-webrtc-toolkit/owt-client-javascript
+          path: third_party/owt-client-javascript/
+      - uses: actions/setup-node@v2
+        with:
+          node-version: "10"
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+      - run: npm install
+        name: Install dependencies for JavaScript SDK
+        working-directory: third_party/owt-client-javascript/scripts
+      - run: mkdir -p third_party/quic-transport
+        name: Create directory for QUIC SDK
+      - name: Download QUIC SDK
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          github_token: ${{secrets.GITHUB_TOKEN}}
+          workflow: ci.yaml
+          branch: main
+          event: push
+          name: quic-sdk-ubuntu-x64-ci
+          path: third_party/quic-transport
+          repo: open-webrtc-toolkit/owt-sdk-quic
+      - name: Generate certificates
+        run: |
+          wget https://chromium.googlesource.com/chromium/src/+archive/refs/tags/93.0.4575.1/net/tools/quic/certs.tar.gz
+          tar xvzf certs.tar.gz
+          ./generate-certs.sh
+          openssl pkcs12 -inkey out/leaf_cert.key -in out/leaf_cert.pem -export -out out/leaf_cert.pfx -passout pass:abc123
+        working-directory: ./source/agent/quic/e2e-test/scripts
+      - name: Install dependencies
+        run: ./installDepsUnattended.sh
+        working-directory: ./scripts
+      - run: grunt
+        name: Build JavaScript SDK and sample release package
+        working-directory: third_party/owt-client-javascript/scripts
+      - name: Build and pack server
+        run: |
+         ./scripts/build.js -t quic -t internal-io -t logger -c
+         ./scripts/pack.js -t conference-agent -t quic-agent -t management-api -t cluster-manager -t portal -p third_party/owt-client-javascript/dist/samples/conference -i
+      - name: Start server
+        run: |
+          cp ./source/agent/quic/e2e-test/scripts/out/leaf_cert.pfx dist/quic_agent/cert/certificate.pfx
+          cp ./source/agent/quic/e2e-test/config/management_api.toml dist/management_api/management_api.toml
+          cp ./source/agent/quic/e2e-test/config/quic_agent.toml dist/quic_agent/agent.toml
+          cp ./source/agent/quic/e2e-test/config/portal.toml dist/portal/portal.toml
+          cp ./source/agent/quic/e2e-test/service/rest_service.js dist/apps/current_app/samplertcservice.js
+          ./dist/bin/init-webtransport.sh
+      - name: Prepare testing environment
+        run: python3 prepare.py
+        working-directory: ./source/agent/quic/e2e-test/scripts
+      - run: npm install
+        name: Install dependencies for E2E test
+        working-directory: ./source/agent/quic/e2e-test
+      - run: node_modules/karma/bin/karma start e2e-test/config/karma.config.js --cert_fingerprint="$(openssl x509 -noout -fingerprint -sha256 -inform pem -in e2e-test/scripts/out/leaf_cert.pem | cut -c20-124)"
+        working-directory: ./source/agent/quic
+        name: Run E2E tests
+      - uses: actions/upload-artifact@v2
+        name: Upload log
+        with:
+          name: log
+          path: dist/logs
+          retention-days: 1
+        if: ${{ always() }}

scripts/installCommonDeps.sh

@@ -308,27 +308,30 @@ install_quic(){
   tar xzf dist.tgz
   popd
 
-  # QUIC transport
-  local QUIC_SDK_VERSION=`cat ${ROOT}/source/agent/addons/quic/quic_sdk_version`
-  local QUIC_TRANSPORT_PATH=${ROOT}/third_party/quic-transport
-  local QUIC_HEADERS_DIR=${ROOT}/build/libdeps/build/include/owt
-  if [ -d ${QUIC_TRANSPORT_PATH} ]; then
-    rm -r ${QUIC_TRANSPORT_PATH}
-  fi
-  mkdir ${QUIC_TRANSPORT_PATH}
-  pushd ${QUIC_TRANSPORT_PATH}
-  if wget ${QUIC_TRANSPORT_PACKAGE_URL_PREFIX}/linux/${QUIC_SDK_VERSION}.zip; then
-    unzip ${QUIC_SDK_VERSION}.zip
-    rm ${QUIC_SDK_VERSION}.zip
-    cp bin/release/libowt_quic_transport.so ${ROOT}/build/libdeps/build/lib
-    if [ -d ${QUIC_HEADERS_DIR} ]; then
-      rm -r ${QUIC_HEADERS_DIR}
+  # WebTransport
+  if [ "$ENABLE_WEBTRANSPORT" == "true" ]; then
+    local QUIC_SDK_PACKAGE_NAME="quic-sdk-ubuntu-x64-ci"
+    local QUIC_SDK_URL=`cat ${ROOT}/source/agent/addons/quic/quic_sdk_url`
+    local QUIC_TRANSPORT_PATH=${ROOT}/third_party/quic-transport
+    local QUIC_HEADERS_DIR=${ROOT}/build/libdeps/build/include/owt
+    pushd ${QUIC_TRANSPORT_PATH}
+    if [ ! -d ${QUIC_HEADERS_DIR} ]; then
+      mkdir -p ${QUIC_HEADERS_DIR}
     fi
-    cp -r include/owt ${QUIC_HEADERS_DIR}
-  else
-    read -p "Failed to download prebuild QUIC SDK. Please download and compile QUIC SDK version ${QUIC_SDK_VERSION} from https://github.com/open-webrtc-toolkit/owt-deps-quic."
+    if [ ! -d ${ROOT}/build/libdeps/build/lib ]; then
+      mkdir -p ${ROOT}/build/libdeps/build/lib
+    fi
+    if [ -d bin/release ]; then
+      cp bin/release/libowt_web_transport.so ${ROOT}/build/libdeps/build/lib
+      if [ -d ${QUIC_HEADERS_DIR} ]; then
+        rm -r ${QUIC_HEADERS_DIR}
+      fi
+      cp -r include/owt ${QUIC_HEADERS_DIR}
+    else
+      read -p "Unable to find prebuild QUIC SDK. Please download it from ${QUIC_SDK_URL} with a GitHub personal access token or compile it from https://github.com/open-webrtc-toolkit/owt-sdk-quic."
+    fi
+    popd
   fi
-  popd
 }
 
 install_nicer(){

scripts/installDeps.sh

@@ -13,6 +13,7 @@ CLEANUP=false
 NO_INTERNAL=false
 INCR_INSTALL=false
 ONLY_INSTALL=""
+ENABLE_WEBTRANSPORT=false
 SUDO=""
 
 if [[ $EUID -ne 0 ]]; then
@@ -34,6 +35,9 @@ parse_arguments(){
       "--incremental")
         INCR_INSTALL=true
         ;;
+      "--enable-webtransport")
+        ENABLE_WEBTRANSPORT=true
+        ;;
       "--only")
         shift
         ONLY_INSTALL=$1

scripts/installDepsUnattended.sh

@@ -13,8 +13,13 @@ CLEANUP=false
 NIGHTLY=false
 NO_INTERNAL=false
 INCR_INSTALL=false
+ENABLE_WEBTRANSPORT=false
 SUDO=""
 
+if [ "$GITHUB_ACTIONS" == "true" ]; then
+  ENABLE_WEBTRANSPORT=true
+fi
+
 if [[ $EUID -ne 0 ]]; then
   SUDO="sudo -E"
 fi
@@ -74,9 +79,11 @@ then
   fi
 fi
 
-install_node
+if [ "$GITHUB_ACTIONS" != "true" ]; then
+  install_node
+fi
 
-if [ "$NIGHTLY" != "true" ]; then
+if [ "$NIGHTLY" != "true" ] && [ "$GITHUB_ACTIONS" != "true" ]; then
 
   if [ "$DISABLE_NONFREE" = "true" ]; then
     install_mediadeps
@@ -112,7 +119,12 @@ if [ "$NIGHTLY" != "true" ]; then
 
 fi
 
-${NO_INTERNAL} || install_webrtc
+if [ "$GITHUB_ACTIONS" != "true" ]; then
+  ${NO_INTERNAL} || install_webrtc
+else
+  install_node_tools
+  install_quic
+fi
 
 if [ "$CLEANUP" = "true" ]; then
   echo "Cleaning up..."

scripts/installUbuntuDeps.sh

@@ -3,9 +3,23 @@
 install_apt_deps(){
   ${SUDO} apt-get update -y
   ${SUDO} apt-get install git make gcc g++ libglib2.0-dev docbook2x pkg-config -y
-  ${SUDO} apt-get install libboost-regex-dev libboost-thread-dev libboost-system-dev liblog4cxx-dev rabbitmq-server -y
-  ${SUDO} apt-get install mongodb openjdk-8-jre curl libboost-test-dev nasm yasm gyp libx11-dev libkrb5-dev intel-gpu-tools -y
+  ${SUDO} apt-get install libboost-regex-dev libboost-thread-dev libboost-system-dev liblog4cxx-dev -y
+  ${SUDO} apt-get install openjdk-8-jre curl libboost-test-dev nasm yasm gyp libx11-dev libkrb5-dev intel-gpu-tools -y
   ${SUDO} apt-get install m4 autoconf libtool automake cmake libfreetype6-dev libgstreamer-plugins-base1.0-dev -y
+  if [ "$GITHUB_ACTIONS" != "true" ]; then
+    ${SUDO} apt-get install rabbitmq-server mongodb -y
+  else
+    if [ -d $LIB_DIR ]; then
+      echo "Installing mongodb-org from tar"
+      ${SUDO} apt-get install -y libcurl4 openssl liblzma5
+      wget -P $LIB_DIR https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1804-4.4.6.tgz
+      tar -zxvf $LIB_DIR/mongodb-linux-x86_64-ubuntu1804-4.4.6.tgz -C $LIB_DIR
+      ${SUDO} ln -s $LIB_DIR/mongodb-linux-x86_64-ubuntu1804-4.4.6/bin/* /usr/local/bin/
+    else
+      mkdir -p $LIB_DIR
+      install_mongodb
+    fi
+  fi
 }
 
 install_gcc_7(){

scripts/pack.js

@@ -606,6 +606,9 @@ function packScripts() {
   execSync(`cp -a ${rootDir}/scripts/release/package.mcu.json ${distDir}/package.json`);
   execSync(`cp -a ${rootDir}/third_party/NOTICE ${distDir}`);
   execSync(`cp -a ${rootDir}/third_party/ThirdpartyLicenses.txt ${distDir}`);
+  if (process.env.GITHUB_ACTION) {
+    execSync(`cp -a ${rootDir}/scripts/release/init-webtransport.sh ${binDir}`);
+  }
   if (options.binary) {
     execSync(`cp -a ${rootDir}/scripts/release/daemon-bin.sh ${binDir}/daemon.sh`);
   } else {

scripts/release/init-webtransport.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# Copyright (C) <2019> Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+ROOT=`cd "${bin}/.."; pwd`
+
+SUDO=""
+if [[ $EUID -ne 0 ]]; then
+   SUDO="sudo -E"
+fi
+
+OWT_UPDATE_DONE=false
+HAVE_AUTH_UPDATE=false
+
+usage()
+{
+  echo "Usage: [--deps] [--hardware] (Default: without deps and hardware)"
+}
+
+init_software()
+{
+  if ${INSTALL_DEPS}; then
+    echo "Installing dependency..."
+    ${ROOT}/bin/install_node.sh
+    ${ROOT}/bin/init-mongodb.sh --deps
+    ${ROOT}/bin/init-rabbitmq.sh --deps
+    OWT_UPDATE_DONE=true
+    ${ROOT}/management_api/init.sh
+    ${ROOT}/webrtc_agent/install_deps.sh
+    ${ROOT}/video_agent/install_deps.sh
+    ${ROOT}/video_agent/init.sh
+    ${ROOT}/analytics_agent/install_deps.sh
+  else
+    ${SUDO} systemctl start mongod
+    OWT_UPDATE_DONE=true
+    ${ROOT}/management_api/init.sh
+  fi
+}
+
+init_auth()
+{
+  local AUTH_SCRIPT="${ROOT}/cluster_manager/initauth.js"
+  local CIPHER_JS="${ROOT}/cluster_manager/cipher.js"
+  local AUTH_STORE=""
+  if [[ "${HAVE_AUTH_UPDATE}" == "true" ]]; then
+    ${AUTH_SCRIPT}
+    AUTH_STORE=`node -e "console.log(require('${CIPHER_JS}').astore)"`
+    cp ${ROOT}/cluster_manager/${AUTH_STORE} ${ROOT}/portal/
+    cp ${ROOT}/cluster_manager/${AUTH_STORE} ${ROOT}/conference_agent/
+    cp ${ROOT}/cluster_manager/${AUTH_STORE} ${ROOT}/management_api/
+  fi
+}
+
+INSTALL_DEPS=false
+HARDWARE=false
+
+shopt -s extglob
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    *(-)deps )
+      INSTALL_DEPS=true
+      ;;
+    *(-)help )
+      usage
+      exit 0
+      ;;
+  esac
+  shift
+done
+
+echo "Initializing..."
+init_software
+init_auth
+
+bin=`cd "$bin"; pwd`
+${bin}/daemon.sh start management-api
+${bin}/daemon.sh start cluster-manager
+${bin}/daemon.sh start conference-agent
+${bin}/daemon.sh start quic-agent
+${bin}/daemon.sh start portal
+${bin}/daemon.sh start app

source/agent/addons/quic/binding.gyp

@@ -35,7 +35,6 @@
     ],
     'include_dirs': [
       "<!(node -e \"require('nan')\")",
-      'conn_handler',
       '../../../core/common',
       '../../../core/owt_base',
       '$(DEFAULT_DEPENDENCY_PATH)/include',
@@ -44,10 +43,9 @@
     ],
     'ldflags': [
       '-Wl,--no-as-needed',
-      '-Wl,-rpath,/home/jianjunz/code/quic/src/out/debug',
       '-L$(DEFAULT_DEPENDENCY_PATH)/lib',
+      '-Wl,-rpath,$(DEFAULT_DEPENDENCY_PATH)/lib',
       '-fPIC',
-      '-L/home/jianjunz/code/quic/src/out/debug',
     ],
     'cflags_cc!': [
       '-std=gnu++0x',

source/agent/addons/quic/quic_sdk_url

@@ -0,0 +1 @@
+https://api.github.com/repos/open-webrtc-toolkit/owt-sdk-quic/actions/artifacts/71814132/zip