From 223ff330bf25401572c574e6e5c5598a46259a2e Mon Sep 17 00:00:00 2001 From: Kazuyuki Honda Date: Fri, 13 Sep 2019 16:20:50 +0800 Subject: [PATCH 01/37] =?UTF-8?q?GitHub=20Actions=20=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 34 +++++++++++++++++++++++++++++++ .github/workflows/daily_build.yml | 33 ++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/daily_build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..3bb573aa --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,34 @@ +name: Build workflow +on: + push: + # 現状 GitHub Actions には [ci skip] 相当の機能がないので、ドキュメント関連は最初からビルド監視対象から除いておく + # ただし、再帰的に全ファイルにマッチするワイルドカード表現が書けないので、全ファイルを監視対象にするには、 + # 監視が必要なディレクトリ階層分だけ指定しないといけない + # https://help.github.com/en/articles/workflow-syntax-for-github-actions#onpushpull_requestpaths + paths: + - '*' + - '*/*' + - '*/*/*' + - '!*.md' + - '!doc/*' + - '!html/*' + - '!.github/ISSUE_TEMPLATE/*' + - '!.circleci/*' + +jobs: + build_raspbian-buster_armv6: + name: Build momo for raspbian-buster_armv6 + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - run: DOCKER_BUILDKIT=1 NOTTY=1 NOMOUNT=1 make raspbian-buster_armv6 + working-directory: build + timeout-minutes: 120 + build_raspbian-buster_armv7: + name: Build momo for raspbian-buster_armv7 + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - run: DOCKER_BUILDKIT=1 NOTTY=1 NOMOUNT=1 make raspbian-buster_armv7 + working-directory: build + timeout-minutes: 120 \ No newline at end of file diff --git a/.github/workflows/daily_build.yml b/.github/workflows/daily_build.yml new file mode 100644 index 00000000..a4fb60f9 --- /dev/null +++ b/.github/workflows/daily_build.yml @@ -0,0 +1,33 @@ +name: Daily build workflow +on: + schedule: + # UTCで記述する事、この場合は日本時間 9 時にしたいので -9 して 0 にしてある + - cron: "0 0 * * *" + # 明示的な branch の指定はできず、デフォルト branch の latest commit に対して実行される + # https://help.github.com/en/articles/workflow-syntax-for-github-actions#onschedule + +jobs: + build_ubuntu-18_04_armv8: + name: Build momo for build_ubuntu-18.04_armv8 + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - run: DOCKER_BUILDKIT=1 NOTTY=1 NOMOUNT=1 make ubuntu-18.04_armv8 + working-directory: build + timeout-minutes: 120 + build_ubuntu-18_04_x86_64: + name: Build momo for build_ubuntu-18.04_x86_64 + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - run: DOCKER_BUILDKIT=1 NOTTY=1 NOMOUNT=1 make ubuntu-18.04_x86_64 + working-directory: build + timeout-minutes: 120 + build_macos: + name: Build momo for macOS 10.14 + runs-on: macos-10.14 + steps: + - uses: actions/checkout@v1 + - run: make macos + working-directory: build + timeout-minutes: 120 From 45484e50362f272585d283d4bfb1aa3055e45dda Mon Sep 17 00:00:00 2001 From: Kazuyuki Honda Date: Fri, 13 Sep 2019 22:35:40 +0800 Subject: [PATCH 02/37] =?UTF-8?q?build.yml=20=E3=81=8B=E3=82=89=20on.push.?= =?UTF-8?q?paths=20=E3=81=AE=E8=A8=AD=E5=AE=9A=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3bb573aa..16b35819 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,19 +1,5 @@ name: Build workflow -on: - push: - # 現状 GitHub Actions には [ci skip] 相当の機能がないので、ドキュメント関連は最初からビルド監視対象から除いておく - # ただし、再帰的に全ファイルにマッチするワイルドカード表現が書けないので、全ファイルを監視対象にするには、 - # 監視が必要なディレクトリ階層分だけ指定しないといけない - # https://help.github.com/en/articles/workflow-syntax-for-github-actions#onpushpull_requestpaths - paths: - - '*' - - '*/*' - - '*/*/*' - - '!*.md' - - '!doc/*' - - '!html/*' - - '!.github/ISSUE_TEMPLATE/*' - - '!.circleci/*' +on: push jobs: build_raspbian-buster_armv6: From 93c2cbe4f030d11cf25a7f9c6580dce86a68e08a Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sat, 14 Sep 2019 15:42:56 +0900 Subject: [PATCH 03/37] =?UTF-8?q?circleci=20=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .circleci/config.yml | 100 ------------------------------------------- README.md | 1 - 2 files changed, 101 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 22c992c2..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,100 +0,0 @@ -version: 2 -jobs: - build_raspbian-buster_armv7: - docker: - - image: docker:18.09.6 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - version: 18.09.3 - - run: - command: apk add make bash git - - run: - command: DOCKER_BUILDKIT=1 NOTTY=1 NOMOUNT=1 make raspbian-buster_armv7 - working_directory: build - no_output_timeout: 2h - build_raspbian-buster_armv6: - docker: - - image: docker:18.09.6 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - version: 18.09.3 - - run: - command: apk add make bash git - - run: - command: DOCKER_BUILDKIT=1 NOTTY=1 NOMOUNT=1 make raspbian-buster_armv6 - working_directory: build - no_output_timeout: 2h - build_ubuntu-18.04_armv8: - docker: - - image: docker:18.09.6 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - version: 18.09.3 - - run: - command: apk add make bash git - - run: - command: DOCKER_BUILDKIT=1 NOTTY=1 NOMOUNT=1 make ubuntu-18.04_armv8 - working_directory: build - no_output_timeout: 2h - build_ubuntu-18.04_x86_64: - docker: - - image: docker:18.09.6 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - version: 18.09.3 - - run: - command: apk add make bash git - - run: - command: DOCKER_BUILDKIT=1 NOTTY=1 NOMOUNT=1 make ubuntu-18.04_x86_64 - working_directory: build - no_output_timeout: 2h - build_macos: - macos: - xcode: "10.2.1" - steps: - - checkout - - run: - command: make macos - working_directory: build - no_output_timeout: 2h - -workflows: - version: 2 - build_workflow: - jobs: - - build_raspbian-buster_armv6 - - build_raspbian-buster_armv7 - - daily_build_workflow: - triggers: - - schedule: - # UTCで記述する事、この場合は日本時間 9 時にしたいので -9 して 0 にしてある - cron: "0 0 * * *" - filters: - branches: - only: - - develop - jobs: - - build_ubuntu-18.04_armv8 - - build_ubuntu-18.04_x86_64 - - weekly_build_workflow: - triggers: - - schedule: - # macOS はリソースが限られてるので週 1 、日曜日のみビルドする - # UTCで記述する事、この場合は日本時間 9 時にしたいので -9 して 0 にしてある - cron: "0 0 * * 0" - filters: - branches: - only: - - develop - jobs: - - build_macos diff --git a/README.md b/README.md index 51b35425..28b6050b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![GitHub tag (latest SemVer)](https://img.shields.io/github/tag/shiguredo/momo.svg)](https://github.com/shiguredo/momo) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![CircleCI](https://circleci.com/gh/shiguredo/momo.svg?style=svg)](https://circleci.com/gh/shiguredo/momo) ## WebRTC Native Client Momo について From a7f5023c7450c119c951b3e1fa18a6dec92e0863 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sat, 14 Sep 2019 16:16:17 +0900 Subject: [PATCH 04/37] =?UTF-8?q?libwebrt=20=E3=82=92=20m78@{#3}=20?= =?UTF-8?q?=E3=81=AB=E3=82=A2=E3=83=83=E3=83=97=E3=83=87=E3=83=BC=E3=83=88?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 6 ++++++ VERSION | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 36a08aa3..8d8fd3a5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,12 @@ ## develop +- [ADD] CI を CircleCI から GitHub Actions へ切り替える + - macOS の時間制限が OSS の場合はないため Weekly build から Daily build のみにきりかえる +- [UPDATE] libwebrtc M78 コミットポジションを 3 にする + - libwebrtc のハッシュは 68c715dc01cd8cd0ad2726453e7376b5f353fcd1 + - @voluntas + ## 19.09.0 - [ADD] --disable-echo-cancellation オプションを追加 diff --git a/VERSION b/VERSION index 4be02bb9..d59fa49f 100644 --- a/VERSION +++ b/VERSION @@ -2,7 +2,7 @@ # 項目を変更してビルドすれば各環境へ反映される（ように作る） MOMO_VERSION=19.09.0 WEBRTC_VERSION=78 -WEBRTC_COMMIT=5b728cca77c46ed47ae589acba676485a957070b +WEBRTC_COMMIT=68c715dc01cd8cd0ad2726453e7376b5f353fcd1 BOOST_VERSION=1.71.0 JSON_VERSION=3.7.0 CLI11_VERSION=1.8.0 From 935e995a37f193bde73dcaf1b716d16820dffa2f Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sat, 14 Sep 2019 19:24:56 +0900 Subject: [PATCH 05/37] =?UTF-8?q?19.09.1-rc0=20=E3=81=AB=E4=B8=8A=E3=81=92?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d59fa49f..d5b10ba0 100644 --- a/VERSION +++ b/VERSION @@ -1,6 +1,6 @@ # 各種ライブラリのバージョン情報 # 項目を変更してビルドすれば各環境へ反映される（ように作る） -MOMO_VERSION=19.09.0 +MOMO_VERSION=19.09.1-rc0 WEBRTC_VERSION=78 WEBRTC_COMMIT=68c715dc01cd8cd0ad2726453e7376b5f353fcd1 BOOST_VERSION=1.71.0 From 813451b4f733a7801f240ff9084bbd094867fac6 Mon Sep 17 00:00:00 2001 From: tnoho Date: Sat, 13 Jul 2019 18:56:23 +0900 Subject: [PATCH 06/37] =?UTF-8?q?Jetson=20Nano=20=E3=81=A7=E3=81=A8?= =?UTF-8?q?=E3=82=8A=E3=81=82=E3=81=88=E3=81=9A=E5=8B=95=E3=81=84=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + Makefile | 31 +- src/hwenc_jetson/jetson_h264_encoder.cpp | 583 ++++++++++++++++++ src/hwenc_jetson/jetson_h264_encoder.h | 105 ++++ src/rtc/hw_video_encoder_factory.cpp | 6 + src/rtc/manager.cpp | 8 +- .../v4l2_video_capturer.cpp | 1 + 7 files changed, 732 insertions(+), 4 deletions(-) create mode 100644 src/hwenc_jetson/jetson_h264_encoder.cpp create mode 100644 src/hwenc_jetson/jetson_h264_encoder.h diff --git a/.gitignore b/.gitignore index c6b9e6d5..708d479e 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ webrtc_logs_0 /build/package /build/momo/ /build/momo.tar.gz + +/src/hwenc_jetson/Nv* \ No newline at end of file diff --git a/Makefile b/Makefile index 669995dc..b4964e2c 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,7 @@ ifeq ($(PACKAGE_NAME),raspbian-buster_armv6) TARGET_ARCH_ARM ?= armv6 USE_ROS ?= 0 USE_MMAL_ENCODER ?= 1 + USE_JETSON_ENCODER ?= 0 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/raspbian-buster_armv6 @@ -62,6 +63,7 @@ else ifeq ($(PACKAGE_NAME),raspbian-buster_armv7) TARGET_ARCH_ARM ?= armv7 USE_ROS ?= 0 USE_MMAL_ENCODER ?= 1 + USE_JETSON_ENCODER ?= 0 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/raspbian-buster_armv7 @@ -73,6 +75,7 @@ else ifeq ($(PACKAGE_NAME),ubuntu-16.04_armv7_ros) TARGET_ARCH_ARM ?= armv7 USE_ROS ?= 1 USE_MMAL_ENCODER ?= 1 + USE_JETSON_ENCODER ?= 0 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/ubuntu-16.04_armv7_ros @@ -84,6 +87,7 @@ else ifeq ($(PACKAGE_NAME),ubuntu-18.04_armv8) TARGET_ARCH_ARM ?= armv8 USE_ROS ?= 0 USE_MMAL_ENCODER ?= 0 + USE_JETSON_ENCODER ?= 0 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/ubuntu-18.04_armv8 @@ -94,6 +98,7 @@ else ifeq ($(PACKAGE_NAME),ubuntu-16.04_x86_64_ros) TARGET_ARCH ?= x86_64 USE_ROS ?= 1 USE_MMAL_ENCODER ?= 0 + USE_JETSON_ENCODER ?= 0 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/ubuntu-16.04_x86_64_ros @@ -103,6 +108,7 @@ else ifeq ($(PACKAGE_NAME),ubuntu-18.04_x86_64) TARGET_ARCH ?= x86_64 USE_ROS ?= 0 USE_MMAL_ENCODER ?= 0 + USE_JETSON_ENCODER ?= 0 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/ubuntu-18.04_x86_64 @@ -111,6 +117,7 @@ else ifeq ($(PACKAGE_NAME),macos) TARGET_ARCH ?= x86_64 USE_ROS ?= 0 USE_MMAL_ENCODER ?= 0 + USE_JETSON_ENCODER ?= 0 BOOST_ROOT ?= $(CURDIR)/build/macos/boost-$(BOOST_VERSION) # CURDIR を付けると、ar に渡す時に引数が長すぎるって怒られたので、 # 相対パスで指定する。 @@ -157,6 +164,11 @@ else HAS_ERROR = 1 endif + ifndef USE_JETSON_ENCODER + $(info - USE_JETSON_ENCODER が指定されていません) + HAS_ERROR = 1 + endif + ifndef BOOST_ROOT $(info - BOOST_ROOT が指定されていません) HAS_ERROR = 1 @@ -251,7 +263,24 @@ ifeq ($(TARGET_OS),linux) LDFLAGS += -L$(SYSROOT)/usr/lib/$(ARCH_NAME) endif - ifneq ($(TARGET_ARCH_ARM),armv8) + ifeq ($(TARGET_ARCH_ARM),armv8) + # Jetson の設定 + ifeq ($(USE_JETSON_ENCODER),1) + CFLAGS += \ + -DUSE_JETSON_ENCODER=1 \ + -I../../tegra_multimedia_api/include/ + LDFLAGS += \ + -L/mnt/jetson/usr/lib/aarch64-linux-gnu \ + -lv4l2 \ + -L/mnt/jetson/usr/lib/aarch64-linux-gnu/tegra \ + -lnvbuf_utils \ + -lnvddk_vic \ + -lnvrm \ + -lnvrm_graphics \ + -lnvos + SOURCES += $(shell find src/hwenc_jetson -maxdepth 1 -name '*.cpp') + endif + else # armv6, armv7 用 CFLAGS += -mfloat-abi=hard diff --git a/src/hwenc_jetson/jetson_h264_encoder.cpp b/src/hwenc_jetson/jetson_h264_encoder.cpp new file mode 100644 index 00000000..eb83aae3 --- /dev/null +++ b/src/hwenc_jetson/jetson_h264_encoder.cpp @@ -0,0 +1,583 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "jetson_h264_encoder.h" + +#include +#include + +#include "third_party/libyuv/include/libyuv/convert.h" +#include "third_party/libyuv/include/libyuv/convert_from.h" +#include "third_party/libyuv/include/libyuv/video_common.h" + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "system_wrappers/include/metrics.h" + +#include "nvbuf_utils.h" + +#include "rtc/native_buffer.h" + +#define H264HWENC_HEADER_DEBUG 0 +#define INIT_ERROR(cond, desc) if(cond) { \ + RTC_LOG(LS_ERROR) << __FUNCTION__ << desc; \ + Release(); \ + return WEBRTC_VIDEO_CODEC_ERROR; } + +namespace +{ +struct nal_entry +{ + size_t offset; + size_t size; +}; + +const int kLowH264QpThreshold = 34; +const int kHighH264QpThreshold = 40; + +} // namespace + +JetsonH264Encoder::JetsonH264Encoder(const cricket::VideoCodec &codec) + : callback_(nullptr), + encoder_(nullptr), + bitrate_adjuster_(.5, .95), + configured_framerate_(30), + configured_width_(0), + configured_height_(0), + use_mjpeg_(false), + encoded_buffer_length_(0) +{ +} + +JetsonH264Encoder::~JetsonH264Encoder() +{ + Release(); +} + +int32_t JetsonH264Encoder::InitEncode(const webrtc::VideoCodec *codec_settings, + int32_t number_of_cores, + size_t max_payload_size) +{ + RTC_DCHECK(codec_settings); + RTC_DCHECK_EQ(codec_settings->codecType, webrtc::kVideoCodecH264); + + int32_t release_ret = Release(); + if (release_ret != WEBRTC_VIDEO_CODEC_OK) + { + return release_ret; + } + + width_ = codec_settings->width; + height_ = codec_settings->height; + target_bitrate_bps_ = codec_settings->startBitrate * 1000; + bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); + framerate_ = codec_settings->maxFramerate; + if (framerate_ > 30) + { + framerate_ = 30; + } + + RTC_LOG(LS_INFO) << "InitEncode " << framerate_ << "fps " + << target_bitrate_bps_ << "bit/sec"; + + // Initialize encoded image. Default buffer size: size of unencoded data. + encoded_image_._completeFrame = true; + encoded_image_._encodedWidth = 0; + encoded_image_._encodedHeight = 0; + encoded_image_.set_size(0); + encoded_image_.timing_.flags = webrtc::VideoSendTiming::TimingFrameFlags::kInvalid; + encoded_image_.content_type_ = (codec_settings->mode == webrtc::VideoCodecMode::kScreensharing) + ? webrtc::VideoContentType::SCREENSHARE + : webrtc::VideoContentType::UNSPECIFIED; + + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t JetsonH264Encoder::Release() +{ + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t JetsonH264Encoder::JetsonConfigure() +{ + int ret = 0; + + encoder_ = NvVideoEncoder::createVideoEncoder("enc0"); + INIT_ERROR(!encoder_, "Failed to createVideoEncoder"); + + ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_H264, + width_, height_, 2 * 1024 * 1024); + INIT_ERROR(ret < 0, "Failed to setCapturePlaneFormat"); + + ret = encoder_->setOutputPlaneFormat(V4L2_PIX_FMT_YUV420M, + width_, height_); + INIT_ERROR(ret < 0, "Failed to setOutputPlaneFormat"); + + ret = encoder_->setBitrate(bitrate_adjuster_.GetAdjustedBitrateBps()); + INIT_ERROR(ret < 0, "Failed to setBitrate"); + + ret = encoder_->setProfile(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH); + INIT_ERROR(ret < 0, "Failed to setProfile"); + + ret = encoder_->setLevel(V4L2_MPEG_VIDEO_H264_LEVEL_5_1); + INIT_ERROR(ret < 0, "Failed to setLevel"); + + ret = encoder_->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR); + INIT_ERROR(ret < 0, "Failed to setRateControlMode"); + + ret = encoder_->setIDRInterval((1<<31) + 1); + INIT_ERROR(ret < 0, "Failed to setIDRInterval"); + + ret = encoder_->setIFrameInterval((1<<31) + 1); + INIT_ERROR(ret < 0, "Failed to setIFrameInterval"); + + ret = encoder_->setFrameRate(framerate_, 1); + INIT_ERROR(ret < 0, "Failed to setFrameRate"); + + ret = encoder_->setInsertSpsPpsAtIdrEnabled(true); + INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); + + ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_MMAP, 1, true, false); + INIT_ERROR(ret < 0, "Failed to setupPlane at output_plane"); + + ret = encoder_->capture_plane.setupPlane(V4L2_MEMORY_MMAP, 1, true, false); + INIT_ERROR(ret < 0, "Failed to setupPlane at capture_plane"); + + ret = encoder_->output_plane.setStreamStatus(true); + INIT_ERROR(ret < 0, "Failed to setStreamStatus"); + + ret = encoder_->capture_plane.setStreamStatus(true); + INIT_ERROR(ret < 0, "Failed to setStreamStatus"); + + encoder_->capture_plane.setDQThreadCallback(JetsonOutputCallbackFunction); + encoder_->capture_plane.startDQThread(this); + + for (uint32_t i = 0; i < encoder_->capture_plane.getNumBuffers(); i++) + { + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); + v4l2_buf.index = i; + v4l2_buf.m.planes = planes; + ret = encoder_->capture_plane.qBuffer(v4l2_buf, NULL); + INIT_ERROR(ret < 0, "Failed to qBuffer at capture_plane"); + } + + configured_framerate_ = framerate_; + configured_width_ = width_; + configured_height_ = height_; + + return WEBRTC_VIDEO_CODEC_OK; +} + +void JetsonH264Encoder::JetsonRelease() +{ + if (!encoder_) + return; + encoder_->capture_plane.waitForDQThread(2000); + if (encoder_->isInError()) + { + RTC_LOG(LS_ERROR) << "Encoder is in error"; + } + delete encoder_; +} + +bool JetsonH264Encoder::JetsonOutputCallbackFunction(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer, + void *data) +{ + return ((JetsonH264Encoder *)data)->JetsonOutputCallback(v4l2_buf, buffer, shared_buffer); +} + +bool JetsonH264Encoder::JetsonOutputCallback(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer) +{ + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Start"; + if (v4l2_buf == nullptr) + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; + return false; + } + if (buffer->planes[0].bytesused == 0) + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << "buffer size is zero"; + return false; + } + + /*if (buffer->flags & Jetson_BUFFER_HEADER_FLAG_CONFIG) { + memcpy(encoded_image_buffer_.get(), buffer->data, buffer->length); + encoded_buffer_length_ = buffer->length; + mmal_buffer_header_release(buffer); + RTC_LOG(LS_INFO) << "Jetson_BUFFER_HEADER_FLAG_CONFIG"; + return; + }*/ + + uint64_t pts = v4l2_buf->timestamp.tv_sec * rtc::kNumMicrosecsPerSec + + v4l2_buf->timestamp.tv_usec; + RTC_LOG(LS_INFO) << "pts:" << pts + << " bytesused:" << buffer->planes[0].bytesused; + + std::unique_ptr params; + { + rtc::CritScope lock(&frame_params_lock_); + do { + if (frame_params_.empty()) + { + RTC_LOG(LS_WARNING) << __FUNCTION__ + << "Frame parameter is not found. SkipFrame pts:" + << pts; + return true; + } + params = std::move(frame_params_.front()); + frame_params_.pop(); + } while (params->timestamp < pts); + if (params->timestamp != pts) + { + RTC_LOG(LS_WARNING) << __FUNCTION__ + << "Frame parameter is not found. SkipFrame pts:" + << pts; + return true; + } + } + + encoded_image_._encodedWidth = params->width; + encoded_image_._encodedHeight = params->height; + encoded_image_.capture_time_ms_ = params->render_time_ms; + encoded_image_.ntp_time_ms_ = params->ntp_time_ms; + encoded_image_.SetTimestamp(pts / rtc::kNumMicrosecsPerMillisec); + encoded_image_.rotation_ = params->rotation; + encoded_image_.SetColorSpace(params->color_space); + + /* if (encoded_buffer_length_ == 0) + {*/ + SendFrame(buffer->planes[0].data, buffer->planes[0].bytesused); + /* } + else + { + memcpy(encoded_image_buffer_.get() + encoded_buffer_length_, buffer->planes[0].data, buffer->planes[0].bytesused); + encoded_buffer_length_ += buffer->length; + SendFrame(encoded_image_buffer_.get(), encoded_buffer_length_); + encoded_buffer_length_ = 0; + }*/ + + if (encoder_->capture_plane.qBuffer(*v4l2_buf, NULL) < 0) + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to qBuffer at capture_plane"; + return false; + } + + RTC_LOG(LS_ERROR) << __FUNCTION__ << " End"; + return true; +} + +int32_t JetsonH264Encoder::RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback *callback) +{ + callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +void JetsonH264Encoder::SetRates(const RateControlParameters ¶meters) +{ + if (encoder_ == nullptr) + return; + if (parameters.bitrate.get_sum_bps() <= 0 || parameters.framerate_fps <= 0) + return; + + RTC_LOG(LS_INFO) << __FUNCTION__ + << " framerate:" << parameters.framerate_fps + << " bitrate:" << parameters.bitrate.get_sum_bps(); + framerate_ = parameters.framerate_fps; + if (framerate_ > 60) + { + framerate_ = 60; + } + target_bitrate_bps_ = parameters.bitrate.get_sum_bps(); + bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); + return; +} + +void JetsonH264Encoder::SetFramerate(uint32_t framerate) +{ + if (configured_framerate_ == framerate) + { + return; + } + RTC_LOG(LS_INFO) << "SetBitrateBps " << framerate << "fps"; + if (encoder_->setFrameRate(framerate, 1) < 0) + { + RTC_LOG(LS_ERROR) << "Failed to set bitrate"; + return; + } + configured_framerate_ = framerate; +} + +void JetsonH264Encoder::SetBitrateBps(uint32_t bitrate_bps) +{ + if (bitrate_bps < 300000 || configured_bitrate_bps_ == bitrate_bps) + { + return; + } + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << bitrate_bps << "bit/sec"; + if (encoder_->setBitrate(bitrate_bps) < 0) + { + RTC_LOG(LS_ERROR) << "Failed to setBitrate"; + return; + } + configured_bitrate_bps_ = bitrate_bps; +} + +webrtc::VideoEncoder::EncoderInfo JetsonH264Encoder::GetEncoderInfo() const +{ + EncoderInfo info; + info.supports_native_handle = true; + info.implementation_name = "Jetson H264"; + info.scaling_settings = + VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold); + info.is_hardware_accelerated = true; + info.has_internal_source = false; + return info; +} + +int32_t JetsonH264Encoder::Encode( + const webrtc::VideoFrame &input_frame, + const std::vector *frame_types) +{ + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Start"; + if (!callback_) + { + RTC_LOG(LS_WARNING) << "InitEncode() has been called, but a callback function " + << "has not been set with RegisterEncodeCompleteCallback()"; + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + rtc::scoped_refptr frame_buffer = input_frame.video_frame_buffer(); + if (frame_buffer->type() == webrtc::VideoFrameBuffer::Type::kNative) + { + use_mjpeg_ = true; + } + else + { + use_mjpeg_ = false; + } + + + if (frame_buffer->width() != configured_width_ || + frame_buffer->height() != configured_height_) + { + RTC_LOG(LS_INFO) << "Encoder reinitialized from " << configured_width_ + << "x" << configured_height_ << " to " + << frame_buffer->width() << "x" << frame_buffer->height() + << " framerate:" << framerate_; + JetsonRelease(); + if (use_mjpeg_) + { + NativeBuffer* native_buffer = dynamic_cast(frame_buffer.get()); + raw_width_ = native_buffer->raw_width(); + raw_height_ = native_buffer->raw_height(); + } + if (JetsonConfigure() != WEBRTC_VIDEO_CODEC_OK) + { + RTC_LOG(LS_ERROR) << "Failed to JetsonConfigure"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + + bool force_key_frame = false; + if (frame_types != nullptr) + { + RTC_DCHECK_EQ(frame_types->size(), static_cast(1)); + if ((*frame_types)[0] == webrtc::VideoFrameType::kEmptyFrame) + { + return WEBRTC_VIDEO_CODEC_OK; + } + force_key_frame = (*frame_types)[0] == webrtc::VideoFrameType::kVideoFrameKey; + } + + if (force_key_frame) + { + if (encoder_->forceIDR() < 0) + { + RTC_LOG(LS_ERROR) << "Failed to forceIDR"; + } + } + + SetFramerate(framerate_); + SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); + { + rtc::CritScope lock(&frame_params_lock_); + frame_params_.push( + absl::make_unique(frame_buffer->width(), + frame_buffer->height(), + input_frame.render_time_ms(), + input_frame.ntp_time_ms(), + input_frame.timestamp_us(), + input_frame.rotation(), + input_frame.color_space())); + } + + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + NvBuffer *buffer; + + memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + memset(planes, 0, sizeof(planes)); + v4l2_buf.m.planes = planes; + + RTC_LOG(LS_ERROR) << __FUNCTION__ + << " output_plane.getNumBuffers: " << encoder_->output_plane.getNumBuffers() + << " output_plane.getNumQueuedBuffers: " << encoder_->output_plane.getNumQueuedBuffers(); + + if (encoder_->output_plane.getNumQueuedBuffers() == encoder_->output_plane.getNumBuffers()) + { + if (encoder_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) + { + RTC_LOG(LS_ERROR) << "Failed to dqBuffer at output_plane"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + else + { + buffer = encoder_->output_plane.getNthBuffer(encoder_->output_plane.getNumQueuedBuffers()); + v4l2_buf.index = encoder_->output_plane.getNumQueuedBuffers(); + } + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Check 2"; + + rtc::scoped_refptr i420_buffer = frame_buffer->ToI420(); + for (uint32_t i = 0; i < buffer->n_planes; i++) + { + const uint8_t *source_data; + int source_stride; + if (i == 0) { + source_data = i420_buffer->DataY(); + source_stride = i420_buffer->StrideY(); + } else if (i == 1) { + source_data = i420_buffer->DataU(); + source_stride = i420_buffer->StrideU(); + } else if (i == 2) { + source_data = i420_buffer->DataV(); + source_stride = i420_buffer->StrideV(); + }else { + break; + } + NvBuffer::NvBufferPlane &plane = buffer->planes[i]; + std::streamsize bytes_to_read = + plane.fmt.bytesperpixel * plane.fmt.width; + uint8_t *input_data = plane.data; + plane.bytesused = 0; + for (uint32_t j = 0; j < plane.fmt.height; j++) + { + memcpy(input_data, + source_data + (source_stride * j), + bytes_to_read); + input_data += plane.fmt.stride; + } + plane.bytesused = plane.fmt.stride * plane.fmt.height; + } + + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Check 3"; + v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; + v4l2_buf.timestamp.tv_sec = input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; + v4l2_buf.timestamp.tv_usec = input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; + + for (int i = 0; i < MAX_PLANES; i++) + { + if (NvBufferMemSyncForDevice(buffer->planes[i].fd, i, (void **)&buffer->planes[i].data) < 0) + { + RTC_LOG(LS_ERROR) << "Failed to NvBufferMemSyncForDevice"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Check 4"; + + if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) + { + RTC_LOG(LS_ERROR) << "Failed to qBuffer at output_plane"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + RTC_LOG(LS_ERROR) << __FUNCTION__ << " End"; + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t JetsonH264Encoder::SendFrame(unsigned char *buffer, size_t size) +{ + encoded_image_.set_buffer(buffer, size); + encoded_image_.set_size(size); + encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameDelta; + + uint8_t zero_count = 0; + size_t nal_start_idx = 0; + std::vector nals; + for (size_t i = 0; i < size; i++) + { + uint8_t data = buffer[i]; + if ((i != 0) && (i == nal_start_idx)) + { + if ((data & 0x1F) == 0x05) + { + encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameKey; + } + } + if (data == 0x01 && zero_count == 3) + { + if (nal_start_idx != 0) + { + nals.push_back({nal_start_idx, i - nal_start_idx + 1 - 4}); + } + nal_start_idx = i + 1; + } + if (data == 0x00) + { + zero_count++; + } + else + { + zero_count = 0; + } + } + if (nal_start_idx != 0) + { + nals.push_back({nal_start_idx, size - nal_start_idx}); + } + + webrtc::RTPFragmentationHeader frag_header; + frag_header.VerifyAndAllocateFragmentationHeader(nals.size()); + for (size_t i = 0; i < nals.size(); i++) + { + frag_header.fragmentationOffset[i] = nals[i].offset; + frag_header.fragmentationLength[i] = nals[i].size; + } + + webrtc::CodecSpecificInfo codec_specific; + codec_specific.codecType = webrtc::kVideoCodecH264; + codec_specific.codecSpecific.H264.packetization_mode = webrtc::H264PacketizationMode::NonInterleaved; + + h264_bitstream_parser_.ParseBitstream(buffer, size); + h264_bitstream_parser_.GetLastSliceQp(&encoded_image_.qp_); + RTC_LOG(LS_INFO) << __FUNCTION__ + << " last slice qp:" << encoded_image_.qp_; + + webrtc::EncodedImageCallback::Result result = callback_->OnEncodedImage(encoded_image_, &codec_specific, &frag_header); + if (result.error != webrtc::EncodedImageCallback::Result::OK) + { + RTC_LOG(LS_ERROR) << __FUNCTION__ + << " OnEncodedImage failed error:" << result.error; + return WEBRTC_VIDEO_CODEC_ERROR; + } + bitrate_adjuster_.Update(size); + return WEBRTC_VIDEO_CODEC_OK; +} diff --git a/src/hwenc_jetson/jetson_h264_encoder.h b/src/hwenc_jetson/jetson_h264_encoder.h new file mode 100644 index 00000000..4af5f4d4 --- /dev/null +++ b/src/hwenc_jetson/jetson_h264_encoder.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef Jetson_H264_ENCODER_H_ +#define Jetson_H264_ENCODER_H_ + +#include "NvVideoEncoder.h" + +#include +#include +#include +#include + +#include "api/video_codecs/video_encoder.h" +#include "rtc_base/critical_section.h" +#include "common_video/h264/h264_bitstream_parser.h" +#include "common_video/include/bitrate_adjuster.h" +#include "modules/video_coding/codecs/h264/include/h264.h" + +class ProcessThread; + +class JetsonH264Encoder : public webrtc::VideoEncoder +{ +public: + explicit JetsonH264Encoder(const cricket::VideoCodec &codec); + ~JetsonH264Encoder() override; + + int32_t InitEncode(const webrtc::VideoCodec *codec_settings, + int32_t number_of_cores, + size_t max_payload_size) override; + int32_t RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback *callback) override; + int32_t Release() override; + int32_t Encode(const webrtc::VideoFrame &frame, + const std::vector *frame_types) override; + void SetRates(const RateControlParameters ¶meters) override; + webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override; + +private: + struct FrameParams { + FrameParams(int32_t w, + int32_t h, + int64_t rtms, + int64_t ntpms, + int64_t ts, + webrtc::VideoRotation r, + absl::optional c) + : width(w), height(h), render_time_ms(rtms), ntp_time_ms(ntpms), timestamp(ts), rotation(r), color_space(c) {} + + int32_t width; + int32_t height; + int64_t render_time_ms; + int64_t ntp_time_ms; + int64_t timestamp; + webrtc::VideoRotation rotation; + absl::optional color_space; + }; + + int32_t JetsonConfigure(); + void JetsonRelease(); + static bool JetsonOutputCallbackFunction(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer, + void *data); + bool JetsonOutputCallback(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer); + void SetFramerate(uint32_t framerate); + void SetBitrateBps(uint32_t bitrate_bps); + int32_t SendFrame(unsigned char *buffer, size_t size); + + webrtc::EncodedImageCallback *callback_; + NvVideoEncoder * encoder_; + webrtc::BitrateAdjuster bitrate_adjuster_; + uint32_t framerate_; + int32_t configured_framerate_; + uint32_t target_bitrate_bps_; + uint32_t configured_bitrate_bps_; + int32_t raw_width_; + int32_t raw_height_; + int32_t width_; + int32_t height_; + int32_t configured_width_; + int32_t configured_height_; + bool use_mjpeg_; + + webrtc::H264BitstreamParser h264_bitstream_parser_; + + + rtc::CriticalSection frame_params_lock_; + std::queue> frame_params_; + webrtc::EncodedImage encoded_image_; + std::unique_ptr encoded_image_buffer_; + size_t encoded_buffer_length_; +}; + +#endif // Jetson_H264_ENCODER_H_ diff --git a/src/rtc/hw_video_encoder_factory.cpp b/src/rtc/hw_video_encoder_factory.cpp index a9b50f4c..5ef6d453 100644 --- a/src/rtc/hw_video_encoder_factory.cpp +++ b/src/rtc/hw_video_encoder_factory.cpp @@ -13,6 +13,9 @@ #if USE_MMAL_ENCODER #include "hwenc_mmal/mmal_h264_encoder.h" #endif +#if USE_JETSON_ENCODER +#include "hwenc_jetson/jetson_h264_encoder.h" +#endif std::vector HWVideoEncoderFactory::GetSupportedFormats() const { @@ -48,6 +51,9 @@ std::unique_ptr HWVideoEncoderFactory::CreateVideoEncoder( if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) { #if USE_MMAL_ENCODER return std::unique_ptr(absl::make_unique(cricket::VideoCodec(format))); +#endif +#if USE_JETSON_ENCODER + return std::unique_ptr(absl::make_unique(cricket::VideoCodec(format))); #endif } diff --git a/src/rtc/manager.cpp b/src/rtc/manager.cpp index 501bd4ba..2c8f6db6 100644 --- a/src/rtc/manager.cpp +++ b/src/rtc/manager.cpp @@ -32,7 +32,7 @@ #include "ros/ros_audio_device_module.h" #endif -#if USE_MMAL_ENCODER +#if USE_MMAL_ENCODER | USE_JETSON_ENCODER #include "api/video_codecs/video_encoder_factory.h" #include "hw_video_encoder_factory.h" #endif @@ -83,8 +83,10 @@ RTCManager::RTCManager(ConnectionSettings conn_settings, media_dependencies.video_encoder_factory = CreateObjCEncoderFactory(); media_dependencies.video_decoder_factory = CreateObjCDecoderFactory(); #else -#if USE_MMAL_ENCODER - media_dependencies.video_encoder_factory = std::unique_ptr(absl::make_unique()); +#if USE_MMAL_ENCODER | USE_JETSON_ENCODER + media_dependencies.video_encoder_factory = + std::unique_ptr( + absl::make_unique()); #else media_dependencies.video_encoder_factory = webrtc::CreateBuiltinVideoEncoderFactory(); #endif diff --git a/src/v4l2_video_capturer/v4l2_video_capturer.cpp b/src/v4l2_video_capturer/v4l2_video_capturer.cpp index efec1e9e..c24ccfc7 100644 --- a/src/v4l2_video_capturer/v4l2_video_capturer.cpp +++ b/src/v4l2_video_capturer/v4l2_video_capturer.cpp @@ -513,6 +513,7 @@ bool V4L2VideoCapture::CaptureProcess() { .set_video_frame_buffer(dst_buffer) .set_timestamp_rtp(0) .set_timestamp_ms(rtc::TimeMillis()) + .set_timestamp_us(rtc::TimeMicros()) .set_rotation(webrtc::kVideoRotation_0) .build(); OnCapturedFrame(video_frame); From 0d0599209248d5c456b5138e677b71e312ac2839 Mon Sep 17 00:00:00 2001 From: tnoho Date: Sat, 13 Jul 2019 19:12:54 +0900 Subject: [PATCH 07/37] =?UTF-8?q?=E3=82=AA=E3=83=BC=E3=83=88=E3=82=B9?= =?UTF-8?q?=E3=82=B1=E3=83=BC=E3=83=A9=E3=83=BC=E3=81=8C=E5=8B=95=E3=81=84?= =?UTF-8?q?=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_jetson/jetson_h264_encoder.cpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/hwenc_jetson/jetson_h264_encoder.cpp b/src/hwenc_jetson/jetson_h264_encoder.cpp index eb83aae3..8963ca4c 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.cpp +++ b/src/hwenc_jetson/jetson_h264_encoder.cpp @@ -153,11 +153,11 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = encoder_->capture_plane.setupPlane(V4L2_MEMORY_MMAP, 1, true, false); INIT_ERROR(ret < 0, "Failed to setupPlane at capture_plane"); - ret = encoder_->output_plane.setStreamStatus(true); - INIT_ERROR(ret < 0, "Failed to setStreamStatus"); + ret = encoder_->subscribeEvent(V4L2_EVENT_EOS, 0, 0); + INIT_ERROR(ret < 0, "Failed to subscribeEvent V4L2_EVENT_EOS"); - ret = encoder_->capture_plane.setStreamStatus(true); - INIT_ERROR(ret < 0, "Failed to setStreamStatus"); + ret = encoder_->setEncoderCommand(V4L2_ENC_CMD_START, 0); + INIT_ERROR(ret < 0, "Failed to setEncoderCommand START"); encoder_->capture_plane.setDQThreadCallback(JetsonOutputCallbackFunction); encoder_->capture_plane.startDQThread(this); @@ -185,6 +185,10 @@ void JetsonH264Encoder::JetsonRelease() { if (!encoder_) return; + if (encoder_->setEncoderCommand(V4L2_ENC_CMD_STOP, 1) < 0) + { + RTC_LOG(LS_ERROR) << "Failed to setEncoderCommand STOP"; + } encoder_->capture_plane.waitForDQThread(2000); if (encoder_->isInError()) { @@ -211,6 +215,15 @@ bool JetsonH264Encoder::JetsonOutputCallback(struct v4l2_buffer *v4l2_buf, RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; return false; } + if(v4l2_buf->flags & V4L2_BUF_FLAG_LAST) + { + struct v4l2_event ev; + memset(&ev, 0, sizeof(struct v4l2_event)); + if (encoder_->dqEvent(ev, 1000) < 0) + RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to dqEvent"; + if(ev.type == V4L2_EVENT_EOS) + return false; + } if (buffer->planes[0].bytesused == 0) { RTC_LOG(LS_ERROR) << __FUNCTION__ << "buffer size is zero"; @@ -227,7 +240,7 @@ bool JetsonH264Encoder::JetsonOutputCallback(struct v4l2_buffer *v4l2_buf, uint64_t pts = v4l2_buf->timestamp.tv_sec * rtc::kNumMicrosecsPerSec + v4l2_buf->timestamp.tv_usec; - RTC_LOG(LS_INFO) << "pts:" << pts + RTC_LOG(LS_INFO) << __FUNCTION__ << "pts:" << pts << " bytesused:" << buffer->planes[0].bytesused; std::unique_ptr params; From d8aa0df765a2dbd8eb6c4690cc25f3398121e030 Mon Sep 17 00:00:00 2001 From: tnoho Date: Sat, 13 Jul 2019 23:46:17 +0900 Subject: [PATCH 08/37] =?UTF-8?q?=E4=B8=8D=E8=A6=81=E3=81=AA=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_jetson/jetson_h264_encoder.cpp | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/src/hwenc_jetson/jetson_h264_encoder.cpp b/src/hwenc_jetson/jetson_h264_encoder.cpp index 8963ca4c..d739fb5e 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.cpp +++ b/src/hwenc_jetson/jetson_h264_encoder.cpp @@ -147,10 +147,10 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = encoder_->setInsertSpsPpsAtIdrEnabled(true); INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); - ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_MMAP, 1, true, false); + ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_MMAP, 10, true, false); INIT_ERROR(ret < 0, "Failed to setupPlane at output_plane"); - ret = encoder_->capture_plane.setupPlane(V4L2_MEMORY_MMAP, 1, true, false); + ret = encoder_->capture_plane.setupPlane(V4L2_MEMORY_MMAP, 10, true, false); INIT_ERROR(ret < 0, "Failed to setupPlane at capture_plane"); ret = encoder_->subscribeEvent(V4L2_EVENT_EOS, 0, 0); @@ -230,14 +230,6 @@ bool JetsonH264Encoder::JetsonOutputCallback(struct v4l2_buffer *v4l2_buf, return false; } - /*if (buffer->flags & Jetson_BUFFER_HEADER_FLAG_CONFIG) { - memcpy(encoded_image_buffer_.get(), buffer->data, buffer->length); - encoded_buffer_length_ = buffer->length; - mmal_buffer_header_release(buffer); - RTC_LOG(LS_INFO) << "Jetson_BUFFER_HEADER_FLAG_CONFIG"; - return; - }*/ - uint64_t pts = v4l2_buf->timestamp.tv_sec * rtc::kNumMicrosecsPerSec + v4l2_buf->timestamp.tv_usec; RTC_LOG(LS_INFO) << __FUNCTION__ << "pts:" << pts @@ -274,17 +266,7 @@ bool JetsonH264Encoder::JetsonOutputCallback(struct v4l2_buffer *v4l2_buf, encoded_image_.rotation_ = params->rotation; encoded_image_.SetColorSpace(params->color_space); - /* if (encoded_buffer_length_ == 0) - {*/ - SendFrame(buffer->planes[0].data, buffer->planes[0].bytesused); - /* } - else - { - memcpy(encoded_image_buffer_.get() + encoded_buffer_length_, buffer->planes[0].data, buffer->planes[0].bytesused); - encoded_buffer_length_ += buffer->length; - SendFrame(encoded_image_buffer_.get(), encoded_buffer_length_); - encoded_buffer_length_ = 0; - }*/ + SendFrame(buffer->planes[0].data, buffer->planes[0].bytesused); if (encoder_->capture_plane.qBuffer(*v4l2_buf, NULL) < 0) { From 8b82526a5d112cfebce45c0c30e43e4d8bb1c8aa Mon Sep 17 00:00:00 2001 From: tnoho Date: Sun, 14 Jul 2019 00:11:50 +0900 Subject: [PATCH 09/37] =?UTF-8?q?=E9=96=8B=E7=99=BA=E7=94=A8=E3=81=AE?= =?UTF-8?q?=E3=83=87=E3=83=90=E3=83=83=E3=82=B0=E3=83=AD=E3=82=B0=E3=81=AE?= =?UTF-8?q?=E3=83=AD=E3=82=B0=E3=83=AC=E3=83=99=E3=83=AB=E3=82=92=E4=B8=8B?= =?UTF-8?q?=E3=81=92=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_jetson/jetson_h264_encoder.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/hwenc_jetson/jetson_h264_encoder.cpp b/src/hwenc_jetson/jetson_h264_encoder.cpp index d739fb5e..8cd73973 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.cpp +++ b/src/hwenc_jetson/jetson_h264_encoder.cpp @@ -209,7 +209,7 @@ bool JetsonH264Encoder::JetsonOutputCallback(struct v4l2_buffer *v4l2_buf, NvBuffer * buffer, NvBuffer * shared_buffer) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Start"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; if (v4l2_buf == nullptr) { RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; @@ -274,7 +274,7 @@ bool JetsonH264Encoder::JetsonOutputCallback(struct v4l2_buffer *v4l2_buf, return false; } - RTC_LOG(LS_ERROR) << __FUNCTION__ << " End"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; return true; } @@ -351,7 +351,7 @@ int32_t JetsonH264Encoder::Encode( const webrtc::VideoFrame &input_frame, const std::vector *frame_types) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Start"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; if (!callback_) { RTC_LOG(LS_WARNING) << "InitEncode() has been called, but a callback function " @@ -432,7 +432,7 @@ int32_t JetsonH264Encoder::Encode( memset(planes, 0, sizeof(planes)); v4l2_buf.m.planes = planes; - RTC_LOG(LS_ERROR) << __FUNCTION__ + RTC_LOG(LS_INFO) << __FUNCTION__ << " output_plane.getNumBuffers: " << encoder_->output_plane.getNumBuffers() << " output_plane.getNumQueuedBuffers: " << encoder_->output_plane.getNumQueuedBuffers(); @@ -449,7 +449,7 @@ int32_t JetsonH264Encoder::Encode( buffer = encoder_->output_plane.getNthBuffer(encoder_->output_plane.getNumQueuedBuffers()); v4l2_buf.index = encoder_->output_plane.getNumQueuedBuffers(); } - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Check 2"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 2"; rtc::scoped_refptr i420_buffer = frame_buffer->ToI420(); for (uint32_t i = 0; i < buffer->n_planes; i++) @@ -483,7 +483,7 @@ int32_t JetsonH264Encoder::Encode( plane.bytesused = plane.fmt.stride * plane.fmt.height; } - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Check 3"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 3"; v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; v4l2_buf.timestamp.tv_sec = input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; v4l2_buf.timestamp.tv_usec = input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; @@ -496,7 +496,7 @@ int32_t JetsonH264Encoder::Encode( return WEBRTC_VIDEO_CODEC_ERROR; } } - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Check 4"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 4"; if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) { @@ -504,7 +504,7 @@ int32_t JetsonH264Encoder::Encode( return WEBRTC_VIDEO_CODEC_ERROR; } - RTC_LOG(LS_ERROR) << __FUNCTION__ << " End"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; return WEBRTC_VIDEO_CODEC_OK; } From ec30d509bfaa679711cc5742592373eda1a70f5b Mon Sep 17 00:00:00 2001 From: tnoho Date: Sun, 14 Jul 2019 15:43:46 +0900 Subject: [PATCH 10/37] =?UTF-8?q?=E5=88=9D=E5=9B=9E=E3=81=AE=E3=81=BF?= =?UTF-8?q?=E5=8B=95=E4=BD=9C=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 6 +- src/hwenc_jetson/jetson_h264_encoder.cpp | 423 ++++++++++++++++++----- src/hwenc_jetson/jetson_h264_encoder.h | 33 +- src/main.cpp | 4 +- 4 files changed, 372 insertions(+), 94 deletions(-) diff --git a/Makefile b/Makefile index b4964e2c..bf0eb8f1 100644 --- a/Makefile +++ b/Makefile @@ -268,16 +268,20 @@ ifeq ($(TARGET_OS),linux) ifeq ($(USE_JETSON_ENCODER),1) CFLAGS += \ -DUSE_JETSON_ENCODER=1 \ - -I../../tegra_multimedia_api/include/ + -I../../tegra_multimedia_api/include/ \ + -I/mnt/jetson/usr/src/nvidia/tegra_multimedia_api/include/libjpeg-8b/ LDFLAGS += \ -L/mnt/jetson/usr/lib/aarch64-linux-gnu \ -lv4l2 \ -L/mnt/jetson/usr/lib/aarch64-linux-gnu/tegra \ -lnvbuf_utils \ -lnvddk_vic \ + -lnvddk_2d_v2 \ + -lnvjpeg \ -lnvrm \ -lnvrm_graphics \ -lnvos + SOURCES += $(shell find src/v4l2_video_capturer -maxdepth 1 -name '*.cpp') SOURCES += $(shell find src/hwenc_jetson -maxdepth 1 -name '*.cpp') endif else diff --git a/src/hwenc_jetson/jetson_h264_encoder.cpp b/src/hwenc_jetson/jetson_h264_encoder.cpp index 8cd73973..71b027e8 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.cpp +++ b/src/hwenc_jetson/jetson_h264_encoder.cpp @@ -49,12 +49,15 @@ const int kHighH264QpThreshold = 40; JetsonH264Encoder::JetsonH264Encoder(const cricket::VideoCodec &codec) : callback_(nullptr), + decoder_(nullptr), + converter_(nullptr), encoder_(nullptr), bitrate_adjuster_(.5, .95), configured_framerate_(30), configured_width_(0), configured_height_(0), use_mjpeg_(false), + running_(false), encoded_buffer_length_(0) { } @@ -68,6 +71,7 @@ int32_t JetsonH264Encoder::InitEncode(const webrtc::VideoCodec *codec_settings, int32_t number_of_cores, size_t max_payload_size) { + RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; RTC_DCHECK(codec_settings); RTC_DCHECK_EQ(codec_settings->codecType, webrtc::kVideoCodecH264); @@ -100,28 +104,71 @@ int32_t JetsonH264Encoder::InitEncode(const webrtc::VideoCodec *codec_settings, ? webrtc::VideoContentType::SCREENSHARE : webrtc::VideoContentType::UNSPECIFIED; + decoder_ = NvJPEGDecoder::createJPEGDecoder("jpegdec"); + INIT_ERROR(!decoder_, "Failed to createJPEGDecoder"); + + RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; return WEBRTC_VIDEO_CODEC_OK; } int32_t JetsonH264Encoder::Release() { + RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; + JetsonRelease(); + if (decoder_) + { + delete decoder_; + decoder_ = nullptr; + } + RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; return WEBRTC_VIDEO_CODEC_OK; } int32_t JetsonH264Encoder::JetsonConfigure() { + running_ = true; int ret = 0; + if (use_mjpeg_) + { + converter_ = NvVideoConverter::createVideoConverter("conv"); + INIT_ERROR(!decoder_, "Failed to createVideoConverter"); + + ret = converter_->setOutputPlaneFormat(decode_pixfmt_, + raw_width_, raw_height_, + V4L2_NV_BUFFER_LAYOUT_PITCH); + INIT_ERROR(ret < 0, "Failed to converter setOutputPlaneFormat"); + + ret = converter_->setCapturePlaneFormat(V4L2_PIX_FMT_YUV420M, + width_, height_, + V4L2_NV_BUFFER_LAYOUT_PITCH); + INIT_ERROR(ret < 0, "Failed to converter setCapturePlaneFormat"); + + ret = converter_->output_plane.setupPlane(V4L2_MEMORY_DMABUF, 1, false, false); + INIT_ERROR(ret < 0, "Failed to setupPlane at converter output_plane"); + + ret = converter_->capture_plane.setupPlane(V4L2_MEMORY_MMAP, 10, false, false); + INIT_ERROR(ret < 0, "Failed to setupPlane at converter capture_plane"); + + ret = converter_->output_plane.setStreamStatus(true); + INIT_ERROR(ret < 0, "Failed to setStreamStatus at converter output_plane"); + + ret = converter_->capture_plane.setStreamStatus(true); + INIT_ERROR(ret < 0, "Failed to setStreamStatus at converter capture_plane"); + + converter_->capture_plane.setDQThreadCallback(ConvertFinishedCallbackFunction); + } + encoder_ = NvVideoEncoder::createVideoEncoder("enc0"); INIT_ERROR(!encoder_, "Failed to createVideoEncoder"); ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_H264, width_, height_, 2 * 1024 * 1024); - INIT_ERROR(ret < 0, "Failed to setCapturePlaneFormat"); + INIT_ERROR(ret < 0, "Failed to encoder setCapturePlaneFormat"); ret = encoder_->setOutputPlaneFormat(V4L2_PIX_FMT_YUV420M, width_, height_); - INIT_ERROR(ret < 0, "Failed to setOutputPlaneFormat"); + INIT_ERROR(ret < 0, "Failed to encoder setOutputPlaneFormat"); ret = encoder_->setBitrate(bitrate_adjuster_.GetAdjustedBitrateBps()); INIT_ERROR(ret < 0, "Failed to setBitrate"); @@ -147,8 +194,16 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = encoder_->setInsertSpsPpsAtIdrEnabled(true); INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); - ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_MMAP, 10, true, false); - INIT_ERROR(ret < 0, "Failed to setupPlane at output_plane"); + if (use_mjpeg_) + { + ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_DMABUF, 10, false, false); + INIT_ERROR(ret < 0, "Failed to setupPlane at encoder output_plane"); + } + else + { + ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_MMAP, 10, true, false); + INIT_ERROR(ret < 0, "Failed to setupPlane at encoder output_plane"); + } ret = encoder_->capture_plane.setupPlane(V4L2_MEMORY_MMAP, 10, true, false); INIT_ERROR(ret < 0, "Failed to setupPlane at capture_plane"); @@ -159,7 +214,33 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = encoder_->setEncoderCommand(V4L2_ENC_CMD_START, 0); INIT_ERROR(ret < 0, "Failed to setEncoderCommand START"); - encoder_->capture_plane.setDQThreadCallback(JetsonOutputCallbackFunction); + if (use_mjpeg_) + { + converter_->capture_plane.startDQThread(this); + + for (uint32_t i = 0; i < converter_->capture_plane.getNumBuffers(); i++) + { + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); + v4l2_buf.index = i; + v4l2_buf.m.planes = planes; + ret = converter_->capture_plane.qBuffer(v4l2_buf, NULL); + INIT_ERROR(ret < 0, "Failed to qBuffer at converter capture_plane"); + } + + for (uint32_t i = 0; i < encoder_->output_plane.getNumBuffers(); i++) + { + enc0_buffer_queue_.push(encoder_->output_plane.getNthBuffer(i)); + } + encoder_->output_plane.setDQThreadCallback(EncodeOutputCallbackFunction); + } + encoder_->capture_plane.setDQThreadCallback(EncodeFinishedCallbackFunction); + if (use_mjpeg_) + { + encoder_->output_plane.startDQThread(this); + } encoder_->capture_plane.startDQThread(this); for (uint32_t i = 0; i < encoder_->capture_plane.getNumBuffers(); i++) @@ -171,7 +252,7 @@ int32_t JetsonH264Encoder::JetsonConfigure() v4l2_buf.index = i; v4l2_buf.m.planes = planes; ret = encoder_->capture_plane.qBuffer(v4l2_buf, NULL); - INIT_ERROR(ret < 0, "Failed to qBuffer at capture_plane"); + INIT_ERROR(ret < 0, "Failed to qBuffer at encoder capture_plane"); } configured_framerate_ = framerate_; @@ -183,56 +264,193 @@ int32_t JetsonH264Encoder::JetsonConfigure() void JetsonH264Encoder::JetsonRelease() { + RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; + running_ = false; + if (!encoder_) return; + + while (!enc0_buffer_queue_.empty()) enc0_buffer_queue_.pop(); + RTC_LOG(LS_INFO) << __FUNCTION__ << " Chack 1"; if (encoder_->setEncoderCommand(V4L2_ENC_CMD_STOP, 1) < 0) { RTC_LOG(LS_ERROR) << "Failed to setEncoderCommand STOP"; } - encoder_->capture_plane.waitForDQThread(2000); - if (encoder_->isInError()) + //encoder_->capture_plane.waitForDQThread(2000); + delete encoder_; + encoder_ = nullptr; + RTC_LOG(LS_INFO) << __FUNCTION__ << " Chack 2"; + + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + NvBuffer *buffer; + + memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); + v4l2_buf.m.planes = planes; + + RTC_LOG(LS_INFO) << __FUNCTION__ << " Chack 3"; + if (converter_) { - RTC_LOG(LS_ERROR) << "Encoder is in error"; + if (converter_->output_plane.getNumQueuedBuffers() == converter_->output_plane.getNumBuffers()) + { + if (converter_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << "Encoder is in error"; + } + } + planes[0].bytesused = 0; + if (converter_->output_plane.qBuffer(v4l2_buf, NULL) < 0) + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << "Encoder is in error"; + } + RTC_LOG(LS_INFO) << __FUNCTION__ << " Chack 4"; + converter_->capture_plane.waitForDQThread(2000); + + + delete converter_; + converter_ = nullptr; } - delete encoder_; + RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; } -bool JetsonH264Encoder::JetsonOutputCallbackFunction(struct v4l2_buffer *v4l2_buf, +bool JetsonH264Encoder::ConvertFinishedCallbackFunction(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer, + void *data) +{ + return ((JetsonH264Encoder *)data)->ConvertFinishedCallback(v4l2_buf, buffer, shared_buffer); +} + +bool JetsonH264Encoder::ConvertFinishedCallback(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer) +{ + NvBuffer *enc0_buffer; + struct v4l2_buffer enc0_qbuf; + struct v4l2_plane planes[MAX_PLANES]; + + RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; + + if (!v4l2_buf) + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; + return false; + } + { + std::unique_lock lock(enc0_buffer_mtx_); + while (enc0_buffer_queue_.empty()) + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " wait"; + enc0_buffer_cond_.wait(lock, [this] { return enc0_buffer_ready_; }); + enc0_buffer_ready_ = false; + RTC_LOG(LS_INFO) << __FUNCTION__ << " end"; + } + enc0_buffer = enc0_buffer_queue_.front(); + enc0_buffer_queue_.pop(); + } + + memset(&enc0_qbuf, 0, sizeof(enc0_qbuf)); + memset(&planes, 0, sizeof(planes)); + + enc0_qbuf.index = enc0_buffer->index; + enc0_qbuf.m.planes = planes; + + enc0_qbuf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; + enc0_qbuf.timestamp.tv_sec = v4l2_buf->timestamp.tv_sec; + enc0_qbuf.timestamp.tv_usec = v4l2_buf->timestamp.tv_usec; + + if (encoder_->output_plane.qBuffer(enc0_qbuf, buffer) < 0) + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to qBuffer at encoder output_plane"; + return false; + } + + if (v4l2_buf->m.planes[0].bytesused == 0) + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << " buffer size is zero"; + return false; + } + + RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; + return true; +} + +bool JetsonH264Encoder::EncodeOutputCallbackFunction(struct v4l2_buffer *v4l2_buf, NvBuffer * buffer, NvBuffer * shared_buffer, void *data) { - return ((JetsonH264Encoder *)data)->JetsonOutputCallback(v4l2_buf, buffer, shared_buffer); + return ((JetsonH264Encoder *)data)->EncodeOutputCallback(v4l2_buf, buffer, shared_buffer); } -bool JetsonH264Encoder::JetsonOutputCallback(struct v4l2_buffer *v4l2_buf, +bool JetsonH264Encoder::EncodeOutputCallback(struct v4l2_buffer *v4l2_buf, NvBuffer * buffer, NvBuffer * shared_buffer) { + struct v4l2_buffer conv_qbuf; + struct v4l2_plane planes[MAX_PLANES]; + RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; - if (v4l2_buf == nullptr) + + if (!v4l2_buf) { RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; return false; } - if(v4l2_buf->flags & V4L2_BUF_FLAG_LAST) + + memset(&conv_qbuf, 0, sizeof(conv_qbuf)); + memset(&planes, 0, sizeof(planes)); + + conv_qbuf.index = shared_buffer->index; + conv_qbuf.m.planes = planes; + { - struct v4l2_event ev; - memset(&ev, 0, sizeof(struct v4l2_event)); - if (encoder_->dqEvent(ev, 1000) < 0) - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to dqEvent"; - if(ev.type == V4L2_EVENT_EOS) + RTC_LOG(LS_INFO) << __FUNCTION__ << " lock"; + std::unique_lock lock(enc0_buffer_mtx_); + RTC_LOG(LS_INFO) << __FUNCTION__ << " qBuffer"; + if (converter_->capture_plane.qBuffer(conv_qbuf, nullptr) < 0) + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to qBuffer at converter capture_plane"; return false; + } + enc0_buffer_queue_.push(buffer); + enc0_buffer_ready_ = true; + enc0_buffer_cond_.notify_all(); + RTC_LOG(LS_INFO) << __FUNCTION__ << " lock end"; } - if (buffer->planes[0].bytesused == 0) + + RTC_LOG(LS_INFO) << __FUNCTION__ << " End bytesused:" << v4l2_buf->m.planes[0].bytesused; + return true; +} + +bool JetsonH264Encoder::EncodeFinishedCallbackFunction(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer, + void *data) +{ + return ((JetsonH264Encoder *)data)->EncodeFinishedCallback(v4l2_buf, buffer, shared_buffer); +} + +bool JetsonH264Encoder::EncodeFinishedCallback(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer) +{ + RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; + if (!v4l2_buf) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "buffer size is zero"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; + return false; + } + if (buffer->planes[0].bytesused == 0 || !running_) + { + RTC_LOG(LS_ERROR) << __FUNCTION__ << " buffer size is zero"; return false; } uint64_t pts = v4l2_buf->timestamp.tv_sec * rtc::kNumMicrosecsPerSec + v4l2_buf->timestamp.tv_usec; - RTC_LOG(LS_INFO) << __FUNCTION__ << "pts:" << pts + RTC_LOG(LS_INFO) << __FUNCTION__ << " pts:" << pts << " bytesused:" << buffer->planes[0].bytesused; std::unique_ptr params; @@ -311,7 +529,7 @@ void JetsonH264Encoder::SetFramerate(uint32_t framerate) { return; } - RTC_LOG(LS_INFO) << "SetBitrateBps " << framerate << "fps"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " " << framerate << "fps"; if (encoder_->setFrameRate(framerate, 1) < 0) { RTC_LOG(LS_ERROR) << "Failed to set bitrate"; @@ -359,10 +577,19 @@ int32_t JetsonH264Encoder::Encode( return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } + int fd = 0; rtc::scoped_refptr frame_buffer = input_frame.video_frame_buffer(); if (frame_buffer->type() == webrtc::VideoFrameBuffer::Type::kNative) { use_mjpeg_ = true; + NativeBuffer* native_buffer = dynamic_cast(frame_buffer.get()); + int ret = decoder_->decodeToFd(fd, (unsigned char *)native_buffer->Data(), native_buffer->length(), + decode_pixfmt_, raw_width_, raw_height_); + if (ret < 0) + { + RTC_LOG(LS_ERROR) << "Failed to decodeToFd"; + return WEBRTC_VIDEO_CODEC_ERROR; + } } else { @@ -378,12 +605,6 @@ int32_t JetsonH264Encoder::Encode( << frame_buffer->width() << "x" << frame_buffer->height() << " framerate:" << framerate_; JetsonRelease(); - if (use_mjpeg_) - { - NativeBuffer* native_buffer = dynamic_cast(frame_buffer.get()); - raw_width_ = native_buffer->raw_width(); - raw_height_ = native_buffer->raw_height(); - } if (JetsonConfigure() != WEBRTC_VIDEO_CODEC_OK) { RTC_LOG(LS_ERROR) << "Failed to JetsonConfigure"; @@ -426,82 +647,112 @@ int32_t JetsonH264Encoder::Encode( struct v4l2_buffer v4l2_buf; struct v4l2_plane planes[MAX_PLANES]; - NvBuffer *buffer; memset(&v4l2_buf, 0, sizeof(v4l2_buf)); memset(planes, 0, sizeof(planes)); v4l2_buf.m.planes = planes; - RTC_LOG(LS_INFO) << __FUNCTION__ - << " output_plane.getNumBuffers: " << encoder_->output_plane.getNumBuffers() - << " output_plane.getNumQueuedBuffers: " << encoder_->output_plane.getNumQueuedBuffers(); - - if (encoder_->output_plane.getNumQueuedBuffers() == encoder_->output_plane.getNumBuffers()) + if (use_mjpeg_) { - if (encoder_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) + NvBuffer *buffer; + if (converter_->output_plane.getNumQueuedBuffers() == converter_->output_plane.getNumBuffers()) { - RTC_LOG(LS_ERROR) << "Failed to dqBuffer at output_plane"; + if (converter_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) + { + RTC_LOG(LS_ERROR) << "Failed to dqBuffer at converter output_plane"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + + v4l2_buf.index = 0; + planes[0].m.fd = fd; + planes[0].bytesused = 1234; + + v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; + v4l2_buf.timestamp.tv_sec = input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; + v4l2_buf.timestamp.tv_usec = input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; + + if (converter_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) + { + RTC_LOG(LS_ERROR) << "Failed to qBuffer at converter output_plane"; return WEBRTC_VIDEO_CODEC_ERROR; } } else { - buffer = encoder_->output_plane.getNthBuffer(encoder_->output_plane.getNumQueuedBuffers()); - v4l2_buf.index = encoder_->output_plane.getNumQueuedBuffers(); - } - RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 2"; - - rtc::scoped_refptr i420_buffer = frame_buffer->ToI420(); - for (uint32_t i = 0; i < buffer->n_planes; i++) - { - const uint8_t *source_data; - int source_stride; - if (i == 0) { - source_data = i420_buffer->DataY(); - source_stride = i420_buffer->StrideY(); - } else if (i == 1) { - source_data = i420_buffer->DataU(); - source_stride = i420_buffer->StrideU(); - } else if (i == 2) { - source_data = i420_buffer->DataV(); - source_stride = i420_buffer->StrideV(); - }else { - break; + NvBuffer *buffer; + + RTC_LOG(LS_INFO) << __FUNCTION__ + << " output_plane.getNumBuffers: " << encoder_->output_plane.getNumBuffers() + << " output_plane.getNumQueuedBuffers: " << encoder_->output_plane.getNumQueuedBuffers(); + + if (encoder_->output_plane.getNumQueuedBuffers() == encoder_->output_plane.getNumBuffers()) + { + if (encoder_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) + { + RTC_LOG(LS_ERROR) << "Failed to dqBuffer at output_plane"; + return WEBRTC_VIDEO_CODEC_ERROR; + } } - NvBuffer::NvBufferPlane &plane = buffer->planes[i]; - std::streamsize bytes_to_read = - plane.fmt.bytesperpixel * plane.fmt.width; - uint8_t *input_data = plane.data; - plane.bytesused = 0; - for (uint32_t j = 0; j < plane.fmt.height; j++) + else { - memcpy(input_data, - source_data + (source_stride * j), - bytes_to_read); - input_data += plane.fmt.stride; + buffer = encoder_->output_plane.getNthBuffer(encoder_->output_plane.getNumQueuedBuffers()); + v4l2_buf.index = encoder_->output_plane.getNumQueuedBuffers(); + } + RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 2"; + + rtc::scoped_refptr i420_buffer = frame_buffer->ToI420(); + for (uint32_t i = 0; i < buffer->n_planes; i++) + { + const uint8_t *source_data; + int source_stride; + if (i == 0) { + source_data = i420_buffer->DataY(); + source_stride = i420_buffer->StrideY(); + } else if (i == 1) { + source_data = i420_buffer->DataU(); + source_stride = i420_buffer->StrideU(); + } else if (i == 2) { + source_data = i420_buffer->DataV(); + source_stride = i420_buffer->StrideV(); + }else { + break; + } + NvBuffer::NvBufferPlane &plane = buffer->planes[i]; + std::streamsize bytes_to_read = + plane.fmt.bytesperpixel * plane.fmt.width; + uint8_t *input_data = plane.data; + plane.bytesused = 0; + for (uint32_t j = 0; j < plane.fmt.height; j++) + { + memcpy(input_data, + source_data + (source_stride * j), + bytes_to_read); + input_data += plane.fmt.stride; + } + plane.bytesused = plane.fmt.stride * plane.fmt.height; } - plane.bytesused = plane.fmt.stride * plane.fmt.height; - } - RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 3"; - v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; - v4l2_buf.timestamp.tv_sec = input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; - v4l2_buf.timestamp.tv_usec = input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; + RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 3"; + v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; + v4l2_buf.timestamp.tv_sec = input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; + v4l2_buf.timestamp.tv_usec = input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; - for (int i = 0; i < MAX_PLANES; i++) - { - if (NvBufferMemSyncForDevice(buffer->planes[i].fd, i, (void **)&buffer->planes[i].data) < 0) + for (int i = 0; i < MAX_PLANES; i++) { - RTC_LOG(LS_ERROR) << "Failed to NvBufferMemSyncForDevice"; - return WEBRTC_VIDEO_CODEC_ERROR; + if (NvBufferMemSyncForDevice(buffer->planes[i].fd, i, (void **)&buffer->planes[i].data) < 0) + { + RTC_LOG(LS_ERROR) << "Failed to NvBufferMemSyncForDevice"; + return WEBRTC_VIDEO_CODEC_ERROR; + } } - } - RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 4"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 4"; - if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) - { - RTC_LOG(LS_ERROR) << "Failed to qBuffer at output_plane"; - return WEBRTC_VIDEO_CODEC_ERROR; + if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) + { + RTC_LOG(LS_ERROR) << "Failed to qBuffer at encoder output_plane"; + return WEBRTC_VIDEO_CODEC_ERROR; + } } RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; diff --git a/src/hwenc_jetson/jetson_h264_encoder.h b/src/hwenc_jetson/jetson_h264_encoder.h index 4af5f4d4..5c1d3f75 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.h +++ b/src/hwenc_jetson/jetson_h264_encoder.h @@ -12,6 +12,8 @@ #ifndef Jetson_H264_ENCODER_H_ #define Jetson_H264_ENCODER_H_ +#include "NvJpegDecoder.h" +#include "NvVideoConverter.h" #include "NvVideoEncoder.h" #include @@ -66,11 +68,25 @@ class JetsonH264Encoder : public webrtc::VideoEncoder int32_t JetsonConfigure(); void JetsonRelease(); - static bool JetsonOutputCallbackFunction(struct v4l2_buffer *v4l2_buf, + static bool ConvertFinishedCallbackFunction(struct v4l2_buffer *v4l2_buf, NvBuffer * buffer, NvBuffer * shared_buffer, void *data); - bool JetsonOutputCallback(struct v4l2_buffer *v4l2_buf, + bool ConvertFinishedCallback(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer); + static bool EncodeOutputCallbackFunction(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer, + void *data); + bool EncodeOutputCallback(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer); + static bool EncodeFinishedCallbackFunction(struct v4l2_buffer *v4l2_buf, + NvBuffer * buffer, + NvBuffer * shared_buffer, + void *data); + bool EncodeFinishedCallback(struct v4l2_buffer *v4l2_buf, NvBuffer * buffer, NvBuffer * shared_buffer); void SetFramerate(uint32_t framerate); @@ -78,25 +94,32 @@ class JetsonH264Encoder : public webrtc::VideoEncoder int32_t SendFrame(unsigned char *buffer, size_t size); webrtc::EncodedImageCallback *callback_; + NvJPEGDecoder *decoder_; + NvVideoConverter *converter_; NvVideoEncoder * encoder_; webrtc::BitrateAdjuster bitrate_adjuster_; uint32_t framerate_; int32_t configured_framerate_; uint32_t target_bitrate_bps_; uint32_t configured_bitrate_bps_; - int32_t raw_width_; - int32_t raw_height_; + uint32_t decode_pixfmt_; + uint32_t raw_width_; + uint32_t raw_height_; int32_t width_; int32_t height_; int32_t configured_width_; int32_t configured_height_; bool use_mjpeg_; + bool running_; webrtc::H264BitstreamParser h264_bitstream_parser_; - rtc::CriticalSection frame_params_lock_; std::queue> frame_params_; + std::mutex enc0_buffer_mtx_; + std::condition_variable enc0_buffer_cond_; + bool enc0_buffer_ready_ = false; + std::queue enc0_buffer_queue_; webrtc::EncodedImage encoded_image_; std::unique_ptr encoded_image_buffer_; size_t encoded_buffer_length_; diff --git a/src/main.cpp b/src/main.cpp index 9a55f58f..e3e642f2 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -19,7 +19,7 @@ #ifdef __APPLE__ #include "mac_helper/mac_capturer.h" #else -#if USE_MMAL_ENCODER +#if USE_MMAL_ENCODER | USE_JETSON_ENCODER #include "v4l2_video_capturer/v4l2_video_capturer.h" #else #include "rtc/device_video_capturer.h" @@ -83,7 +83,7 @@ int main(int argc, char* argv[]) rtc::scoped_refptr capturer = MacCapturer::Create(cs.getWidth(), cs.getHeight(), cs.framerate, 0); #else -#if USE_MMAL_ENCODER +#if USE_MMAL_ENCODER | USE_JETSON_ENCODER rtc::scoped_refptr capturer = V4L2VideoCapture::Create(cs); #else From 1913bd97f7ef514e4cdd1c0c9e0e2dd9b23620f9 Mon Sep 17 00:00:00 2001 From: tnoho Date: Mon, 15 Jul 2019 08:57:01 +0900 Subject: [PATCH 11/37] =?UTF-8?q?=E6=AD=A3=E5=B8=B8=E3=81=AB=E3=83=AA?= =?UTF-8?q?=E3=83=AA=E3=83=BC=E3=82=B9=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E3=81=AA=E3=81=A3=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_jetson/jetson_h264_encoder.cpp | 131 ++++++++++++----------- src/hwenc_jetson/jetson_h264_encoder.h | 44 ++++---- 2 files changed, 88 insertions(+), 87 deletions(-) diff --git a/src/hwenc_jetson/jetson_h264_encoder.cpp b/src/hwenc_jetson/jetson_h264_encoder.cpp index 71b027e8..a6861859 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.cpp +++ b/src/hwenc_jetson/jetson_h264_encoder.cpp @@ -56,9 +56,7 @@ JetsonH264Encoder::JetsonH264Encoder(const cricket::VideoCodec &codec) configured_framerate_(30), configured_width_(0), configured_height_(0), - use_mjpeg_(false), - running_(false), - encoded_buffer_length_(0) + use_mjpeg_(false) { } @@ -126,11 +124,12 @@ int32_t JetsonH264Encoder::Release() int32_t JetsonH264Encoder::JetsonConfigure() { - running_ = true; int ret = 0; if (use_mjpeg_) { + enc0_buffer_queue_ = new std::queue; + converter_ = NvVideoConverter::createVideoConverter("conv"); INIT_ERROR(!decoder_, "Failed to createVideoConverter"); @@ -191,6 +190,9 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = encoder_->setFrameRate(framerate_, 1); INIT_ERROR(ret < 0, "Failed to setFrameRate"); + ret = encoder_->setHWPresetType(V4L2_ENC_HW_PRESET_ULTRAFAST); + INIT_ERROR(ret < 0, "Failed to setFrameRate"); + ret = encoder_->setInsertSpsPpsAtIdrEnabled(true); INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); @@ -211,8 +213,11 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = encoder_->subscribeEvent(V4L2_EVENT_EOS, 0, 0); INIT_ERROR(ret < 0, "Failed to subscribeEvent V4L2_EVENT_EOS"); - ret = encoder_->setEncoderCommand(V4L2_ENC_CMD_START, 0); - INIT_ERROR(ret < 0, "Failed to setEncoderCommand START"); + ret = encoder_->output_plane.setStreamStatus(true); + INIT_ERROR(ret < 0, "Failed to setStreamStatus at encoder output_plane"); + + ret = encoder_->capture_plane.setStreamStatus(true); + INIT_ERROR(ret < 0, "Failed to setStreamStatus at encoder capture_plane"); if (use_mjpeg_) { @@ -232,7 +237,7 @@ int32_t JetsonH264Encoder::JetsonConfigure() for (uint32_t i = 0; i < encoder_->output_plane.getNumBuffers(); i++) { - enc0_buffer_queue_.push(encoder_->output_plane.getNthBuffer(i)); + enc0_buffer_queue_->push(encoder_->output_plane.getNthBuffer(i)); } encoder_->output_plane.setDQThreadCallback(EncodeOutputCallbackFunction); } @@ -264,54 +269,58 @@ int32_t JetsonH264Encoder::JetsonConfigure() void JetsonH264Encoder::JetsonRelease() { - RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; - running_ = false; - if (!encoder_) return; - - while (!enc0_buffer_queue_.empty()) enc0_buffer_queue_.pop(); - RTC_LOG(LS_INFO) << __FUNCTION__ << " Chack 1"; - if (encoder_->setEncoderCommand(V4L2_ENC_CMD_STOP, 1) < 0) + if (converter_) + { + SendEOS(converter_); + } + else + { + SendEOS(encoder_); + } + encoder_->capture_plane.waitForDQThread(2000); + encoder_->capture_plane.deinitPlane(); + encoder_->output_plane.deinitPlane(); + if (converter_) { - RTC_LOG(LS_ERROR) << "Failed to setEncoderCommand STOP"; + delete enc0_buffer_queue_; } - //encoder_->capture_plane.waitForDQThread(2000); delete encoder_; encoder_ = nullptr; - RTC_LOG(LS_INFO) << __FUNCTION__ << " Chack 2"; + if (converter_) + { + converter_->capture_plane.waitForDQThread(2000); + delete converter_; + converter_ = nullptr; + } +} - struct v4l2_buffer v4l2_buf; - struct v4l2_plane planes[MAX_PLANES]; - NvBuffer *buffer; +void JetsonH264Encoder::SendEOS(NvV4l2Element *element) +{ + if (element->output_plane.getStreamStatus()) + { + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + NvBuffer *buffer; - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); - v4l2_buf.m.planes = planes; + memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); + v4l2_buf.m.planes = planes; - RTC_LOG(LS_INFO) << __FUNCTION__ << " Chack 3"; - if (converter_) - { - if (converter_->output_plane.getNumQueuedBuffers() == converter_->output_plane.getNumBuffers()) + if (element->output_plane.getNumQueuedBuffers() == element->output_plane.getNumBuffers()) { - if (converter_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) + if (element->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Encoder is in error"; + RTC_LOG(LS_ERROR) << "Failed to dqBuffer at encoder output_plane"; } } planes[0].bytesused = 0; - if (converter_->output_plane.qBuffer(v4l2_buf, NULL) < 0) + if (element->output_plane.qBuffer(v4l2_buf, NULL) < 0) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Encoder is in error"; + RTC_LOG(LS_ERROR) << "Failed to qBuffer at encoder output_plane"; } - RTC_LOG(LS_INFO) << __FUNCTION__ << " Chack 4"; - converter_->capture_plane.waitForDQThread(2000); - - - delete converter_; - converter_ = nullptr; } - RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; } bool JetsonH264Encoder::ConvertFinishedCallbackFunction(struct v4l2_buffer *v4l2_buf, @@ -329,8 +338,6 @@ bool JetsonH264Encoder::ConvertFinishedCallback(struct v4l2_buffer *v4l2_buf, NvBuffer *enc0_buffer; struct v4l2_buffer enc0_qbuf; struct v4l2_plane planes[MAX_PLANES]; - - RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; if (!v4l2_buf) { @@ -339,15 +346,13 @@ bool JetsonH264Encoder::ConvertFinishedCallback(struct v4l2_buffer *v4l2_buf, } { std::unique_lock lock(enc0_buffer_mtx_); - while (enc0_buffer_queue_.empty()) + while (enc0_buffer_queue_->empty()) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " wait"; enc0_buffer_cond_.wait(lock, [this] { return enc0_buffer_ready_; }); enc0_buffer_ready_ = false; - RTC_LOG(LS_INFO) << __FUNCTION__ << " end"; } - enc0_buffer = enc0_buffer_queue_.front(); - enc0_buffer_queue_.pop(); + enc0_buffer = enc0_buffer_queue_->front(); + enc0_buffer_queue_->pop(); } memset(&enc0_qbuf, 0, sizeof(enc0_qbuf)); @@ -356,6 +361,11 @@ bool JetsonH264Encoder::ConvertFinishedCallback(struct v4l2_buffer *v4l2_buf, enc0_qbuf.index = enc0_buffer->index; enc0_qbuf.m.planes = planes; + for (int i = 0; i < MAX_PLANES; i++) + { + enc0_qbuf.m.planes[i].bytesused = v4l2_buf->m.planes[i].bytesused; + } + enc0_qbuf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; enc0_qbuf.timestamp.tv_sec = v4l2_buf->timestamp.tv_sec; enc0_qbuf.timestamp.tv_usec = v4l2_buf->timestamp.tv_usec; @@ -368,11 +378,10 @@ bool JetsonH264Encoder::ConvertFinishedCallback(struct v4l2_buffer *v4l2_buf, if (v4l2_buf->m.planes[0].bytesused == 0) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << " buffer size is zero"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " buffer size is zero"; return false; } - - RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; + return true; } @@ -391,8 +400,6 @@ bool JetsonH264Encoder::EncodeOutputCallback(struct v4l2_buffer *v4l2_buf, struct v4l2_buffer conv_qbuf; struct v4l2_plane planes[MAX_PLANES]; - RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; - if (!v4l2_buf) { RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; @@ -406,21 +413,23 @@ bool JetsonH264Encoder::EncodeOutputCallback(struct v4l2_buffer *v4l2_buf, conv_qbuf.m.planes = planes; { - RTC_LOG(LS_INFO) << __FUNCTION__ << " lock"; std::unique_lock lock(enc0_buffer_mtx_); - RTC_LOG(LS_INFO) << __FUNCTION__ << " qBuffer"; if (converter_->capture_plane.qBuffer(conv_qbuf, nullptr) < 0) { RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to qBuffer at converter capture_plane"; return false; } - enc0_buffer_queue_.push(buffer); + enc0_buffer_queue_->push(buffer); enc0_buffer_ready_ = true; enc0_buffer_cond_.notify_all(); - RTC_LOG(LS_INFO) << __FUNCTION__ << " lock end"; } - RTC_LOG(LS_INFO) << __FUNCTION__ << " End bytesused:" << v4l2_buf->m.planes[0].bytesused; + if (conv_qbuf.m.planes[0].bytesused == 0) + { + RTC_LOG(LS_INFO) << __FUNCTION__ << " buffer size is zero"; + return false; + } + return true; } @@ -436,15 +445,14 @@ bool JetsonH264Encoder::EncodeFinishedCallback(struct v4l2_buffer *v4l2_buf, NvBuffer * buffer, NvBuffer * shared_buffer) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; if (!v4l2_buf) { RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; return false; } - if (buffer->planes[0].bytesused == 0 || !running_) + if (buffer->planes[0].bytesused == 0) { - RTC_LOG(LS_ERROR) << __FUNCTION__ << " buffer size is zero"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " buffer size is zero"; return false; } @@ -492,7 +500,6 @@ bool JetsonH264Encoder::EncodeFinishedCallback(struct v4l2_buffer *v4l2_buf, return false; } - RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; return true; } @@ -569,7 +576,6 @@ int32_t JetsonH264Encoder::Encode( const webrtc::VideoFrame &input_frame, const std::vector *frame_types) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; if (!callback_) { RTC_LOG(LS_WARNING) << "InitEncode() has been called, but a callback function " @@ -690,7 +696,7 @@ int32_t JetsonH264Encoder::Encode( { if (encoder_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) { - RTC_LOG(LS_ERROR) << "Failed to dqBuffer at output_plane"; + RTC_LOG(LS_ERROR) << "Failed to dqBuffer at encoder output_plane"; return WEBRTC_VIDEO_CODEC_ERROR; } } @@ -733,7 +739,6 @@ int32_t JetsonH264Encoder::Encode( plane.bytesused = plane.fmt.stride * plane.fmt.height; } - RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 3"; v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; v4l2_buf.timestamp.tv_sec = input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; v4l2_buf.timestamp.tv_usec = input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; @@ -746,7 +751,6 @@ int32_t JetsonH264Encoder::Encode( return WEBRTC_VIDEO_CODEC_ERROR; } } - RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 4"; if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) { @@ -755,7 +759,6 @@ int32_t JetsonH264Encoder::Encode( } } - RTC_LOG(LS_INFO) << __FUNCTION__ << " End"; return WEBRTC_VIDEO_CODEC_OK; } diff --git a/src/hwenc_jetson/jetson_h264_encoder.h b/src/hwenc_jetson/jetson_h264_encoder.h index 5c1d3f75..e937a16f 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.h +++ b/src/hwenc_jetson/jetson_h264_encoder.h @@ -49,12 +49,12 @@ class JetsonH264Encoder : public webrtc::VideoEncoder private: struct FrameParams { FrameParams(int32_t w, - int32_t h, - int64_t rtms, - int64_t ntpms, - int64_t ts, - webrtc::VideoRotation r, - absl::optional c) + int32_t h, + int64_t rtms, + int64_t ntpms, + int64_t ts, + webrtc::VideoRotation r, + absl::optional c) : width(w), height(h), render_time_ms(rtms), ntp_time_ms(ntpms), timestamp(ts), rotation(r), color_space(c) {} int32_t width; @@ -68,27 +68,28 @@ class JetsonH264Encoder : public webrtc::VideoEncoder int32_t JetsonConfigure(); void JetsonRelease(); + void SendEOS(NvV4l2Element *element); static bool ConvertFinishedCallbackFunction(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer, + NvBuffer *buffer, + NvBuffer *shared_buffer, void *data); bool ConvertFinishedCallback(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer); + NvBuffer *buffer, + NvBuffer *shared_buffer); static bool EncodeOutputCallbackFunction(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer, + NvBuffer *buffer, + NvBuffer *shared_buffer, void *data); bool EncodeOutputCallback(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer); + NvBuffer *buffer, + NvBuffer *shared_buffer); static bool EncodeFinishedCallbackFunction(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer, + NvBuffer *buffer, + NvBuffer *shared_buffer, void *data); bool EncodeFinishedCallback(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer); + NvBuffer *buffer, + NvBuffer *shared_buffer); void SetFramerate(uint32_t framerate); void SetBitrateBps(uint32_t bitrate_bps); int32_t SendFrame(unsigned char *buffer, size_t size); @@ -96,7 +97,7 @@ class JetsonH264Encoder : public webrtc::VideoEncoder webrtc::EncodedImageCallback *callback_; NvJPEGDecoder *decoder_; NvVideoConverter *converter_; - NvVideoEncoder * encoder_; + NvVideoEncoder *encoder_; webrtc::BitrateAdjuster bitrate_adjuster_; uint32_t framerate_; int32_t configured_framerate_; @@ -110,7 +111,6 @@ class JetsonH264Encoder : public webrtc::VideoEncoder int32_t configured_width_; int32_t configured_height_; bool use_mjpeg_; - bool running_; webrtc::H264BitstreamParser h264_bitstream_parser_; @@ -119,10 +119,8 @@ class JetsonH264Encoder : public webrtc::VideoEncoder std::mutex enc0_buffer_mtx_; std::condition_variable enc0_buffer_cond_; bool enc0_buffer_ready_ = false; - std::queue enc0_buffer_queue_; + std::queue *enc0_buffer_queue_; webrtc::EncodedImage encoded_image_; - std::unique_ptr encoded_image_buffer_; - size_t encoded_buffer_length_; }; #endif // Jetson_H264_ENCODER_H_ From 369267d323b6841efa8fb7a614e01cab0fe553f1 Mon Sep 17 00:00:00 2001 From: tnoho Date: Mon, 15 Jul 2019 08:57:23 +0900 Subject: [PATCH 12/37] =?UTF-8?q?Sora=E3=81=AB=E6=8E=A5=E7=B6=9A=E3=81=99?= =?UTF-8?q?=E3=82=8B=E9=9A=9B=E3=81=ABH264=E3=82=92=E4=BD=BF=E3=81=88?= =?UTF-8?q?=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index bf0eb8f1..48286cb0 100644 --- a/Makefile +++ b/Makefile @@ -229,6 +229,8 @@ endif ifeq ($(TARGET_OS),linux) ifeq ($(TARGET_ARCH),x86_64) CFLAGS += -DUSE_H264=0 + else ifeq ($(USE_JETSON_ENCODER),1) + CFLAGS += -DUSE_H264=1 else ifeq ($(TARGET_ARCH_ARM),armv8) CFLAGS += -DUSE_H264=0 else From 779679214929f749c163c97c245337def102bc38 Mon Sep 17 00:00:00 2001 From: tnoho Date: Mon, 15 Jul 2019 10:17:51 +0900 Subject: [PATCH 13/37] =?UTF-8?q?4K=E3=83=91=E3=83=83=E3=83=81=E3=82=92?= =?UTF-8?q?=E3=83=87=E3=83=95=E3=82=A9=E3=83=AB=E3=83=88=E3=83=AC=E3=83=BC?= =?UTF-8?q?=E3=83=88=E3=82=82=E5=A4=89=E6=9B=B4=E3=81=99=E3=82=8B=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- patch/4k.patch | 20 ++++++++++++++++++-- src/hwenc_jetson/jetson_h264_encoder.cpp | 10 ++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/patch/4k.patch b/patch/4k.patch index 049d500c..ff51d955 100644 --- a/patch/4k.patch +++ b/patch/4k.patch @@ -1,8 +1,24 @@ +diff --git a/src/media/engine/webrtc_video_engine.cc b/src/media/engine/webrtc_video_engine.cc +index 4219c43..b7b4ffa 100644 +--- a/src/media/engine/webrtc_video_engine.cc ++++ b/src/media/engine/webrtc_video_engine.cc +@@ -244,8 +244,10 @@ static int GetMaxDefaultVideoBitrateKbps(int width, + max_bitrate = 1700; + } else if (width * height <= 960 * 540) { + max_bitrate = 2000; +- } else { ++ } else if (width * height <= 1920 * 1080) { + max_bitrate = 2500; ++ } else { ++ max_bitrate = 15000; + } + if (is_screenshare) + max_bitrate = std::max(max_bitrate, 1200); diff --git a/src/modules/video_capture/linux/device_info_linux.cc b/src/modules/video_capture/linux/device_info_linux.cc -index c138870..e6ec5b4 100644 +index 39f72b0..3cb1226 100644 --- a/src/modules/video_capture/linux/device_info_linux.cc +++ b/src/modules/video_capture/linux/device_info_linux.cc -@@ -225,11 +225,11 @@ int32_t DeviceInfoLinux::FillCapabilities(int fd) { +@@ -224,11 +224,11 @@ int32_t DeviceInfoLinux::FillCapabilities(int fd) { unsigned int videoFormats[] = {V4L2_PIX_FMT_MJPEG, V4L2_PIX_FMT_YUV420, V4L2_PIX_FMT_YUYV, V4L2_PIX_FMT_UYVY}; diff --git a/src/hwenc_jetson/jetson_h264_encoder.cpp b/src/hwenc_jetson/jetson_h264_encoder.cpp index a6861859..36b33c2e 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.cpp +++ b/src/hwenc_jetson/jetson_h264_encoder.cpp @@ -90,7 +90,8 @@ int32_t JetsonH264Encoder::InitEncode(const webrtc::VideoCodec *codec_settings, } RTC_LOG(LS_INFO) << "InitEncode " << framerate_ << "fps " - << target_bitrate_bps_ << "bit/sec"; + << target_bitrate_bps_ << "bit/sec　" + << codec_settings->maxBitrate << "kbit/sec　"; // Initialize encoded image. Default buffer size: size of unencoded data. encoded_image_._completeFrame = true; @@ -190,9 +191,14 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = encoder_->setFrameRate(framerate_, 1); INIT_ERROR(ret < 0, "Failed to setFrameRate"); - ret = encoder_->setHWPresetType(V4L2_ENC_HW_PRESET_ULTRAFAST); + //V4L2_ENC_HW_PRESET_ULTRAFAST が推奨値だけど MEDIUM もフレームレート出てる気がする + ret = encoder_->setHWPresetType(V4L2_ENC_HW_PRESET_MEDIUM); INIT_ERROR(ret < 0, "Failed to setFrameRate"); + //この設定を入れればフレームレートより画質が優先されるが動くとフレームレートが激しく落ちる + //ret = encoder_->setConstantQp(30); + //INIT_ERROR(ret < 0, "Failed to setConstantQp"); + ret = encoder_->setInsertSpsPpsAtIdrEnabled(true); INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); From b05d2137c0cd3f1a33a70414627feeacb1c48efc Mon Sep 17 00:00:00 2001 From: tnoho Date: Sat, 20 Jul 2019 13:13:29 +0900 Subject: [PATCH 14/37] =?UTF-8?q?=E3=82=A4=E3=83=B3=E3=82=BF=E3=83=BC?= =?UTF-8?q?=E3=83=90=E3=83=AB=E3=81=AE=E8=A8=AD=E5=AE=9A=E3=82=92=E8=AA=AD?= =?UTF-8?q?=E3=81=BF=E8=BE=BC=E3=82=80=E3=82=88=E3=81=86=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hwenc_jetson/jetson_h264_encoder.cpp | 44 +++++++++++++----------- src/hwenc_jetson/jetson_h264_encoder.h | 1 + 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/hwenc_jetson/jetson_h264_encoder.cpp b/src/hwenc_jetson/jetson_h264_encoder.cpp index 36b33c2e..757e5f1a 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.cpp +++ b/src/hwenc_jetson/jetson_h264_encoder.cpp @@ -82,12 +82,9 @@ int32_t JetsonH264Encoder::InitEncode(const webrtc::VideoCodec *codec_settings, width_ = codec_settings->width; height_ = codec_settings->height; target_bitrate_bps_ = codec_settings->startBitrate * 1000; + key_frame_interval_ = codec_settings->H264().keyFrameInterval; bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); framerate_ = codec_settings->maxFramerate; - if (framerate_ > 30) - { - framerate_ = 30; - } RTC_LOG(LS_INFO) << "InitEncode " << framerate_ << "fps " << target_bitrate_bps_ << "bit/sec　" @@ -163,7 +160,7 @@ int32_t JetsonH264Encoder::JetsonConfigure() INIT_ERROR(!encoder_, "Failed to createVideoEncoder"); ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_H264, - width_, height_, 2 * 1024 * 1024); + width_, height_, 4 * 1024 * 1024); INIT_ERROR(ret < 0, "Failed to encoder setCapturePlaneFormat"); ret = encoder_->setOutputPlaneFormat(V4L2_PIX_FMT_YUV420M, @@ -182,10 +179,10 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = encoder_->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR); INIT_ERROR(ret < 0, "Failed to setRateControlMode"); - ret = encoder_->setIDRInterval((1<<31) + 1); + ret = encoder_->setIDRInterval(key_frame_interval_); INIT_ERROR(ret < 0, "Failed to setIDRInterval"); - ret = encoder_->setIFrameInterval((1<<31) + 1); + ret = encoder_->setIFrameInterval(key_frame_interval_); INIT_ERROR(ret < 0, "Failed to setIFrameInterval"); ret = encoder_->setFrameRate(framerate_, 1); @@ -193,7 +190,10 @@ int32_t JetsonH264Encoder::JetsonConfigure() //V4L2_ENC_HW_PRESET_ULTRAFAST が推奨値だけど MEDIUM もフレームレート出てる気がする ret = encoder_->setHWPresetType(V4L2_ENC_HW_PRESET_MEDIUM); - INIT_ERROR(ret < 0, "Failed to setFrameRate"); + INIT_ERROR(ret < 0, "Failed to setHWPresetType"); + + ret = encoder_->setNumBFrames(0); + INIT_ERROR(ret < 0, "Failed to setNumBFrames"); //この設定を入れればフレームレートより画質が優先されるが動くとフレームレートが激しく落ちる //ret = encoder_->setConstantQp(30); @@ -202,6 +202,9 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = encoder_->setInsertSpsPpsAtIdrEnabled(true); INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); + ret = encoder_->setInsertVuiEnabled(true); + INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); + if (use_mjpeg_) { ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_DMABUF, 10, false, false); @@ -527,10 +530,6 @@ void JetsonH264Encoder::SetRates(const RateControlParameters ¶meters) << " framerate:" << parameters.framerate_fps << " bitrate:" << parameters.bitrate.get_sum_bps(); framerate_ = parameters.framerate_fps; - if (framerate_ > 60) - { - framerate_ = 60; - } target_bitrate_bps_ = parameters.bitrate.get_sum_bps(); bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); return; @@ -538,6 +537,14 @@ void JetsonH264Encoder::SetRates(const RateControlParameters ¶meters) void JetsonH264Encoder::SetFramerate(uint32_t framerate) { + if (width_ <= 1920 && height_ <= 1080 && framerate > 60) + { + framerate = 60; + } + else if (framerate > 30) + { + framerate = 30; + } if (configured_framerate_ == framerate) { return; @@ -632,14 +639,12 @@ int32_t JetsonH264Encoder::Encode( { return WEBRTC_VIDEO_CODEC_OK; } - force_key_frame = (*frame_types)[0] == webrtc::VideoFrameType::kVideoFrameKey; - } - - if (force_key_frame) - { - if (encoder_->forceIDR() < 0) + if((*frame_types)[0] == webrtc::VideoFrameType::kVideoFrameKey) { - RTC_LOG(LS_ERROR) << "Failed to forceIDR"; + if (encoder_->forceIDR() < 0) + { + RTC_LOG(LS_ERROR) << "Failed to forceIDR"; + } } } @@ -711,7 +716,6 @@ int32_t JetsonH264Encoder::Encode( buffer = encoder_->output_plane.getNthBuffer(encoder_->output_plane.getNumQueuedBuffers()); v4l2_buf.index = encoder_->output_plane.getNumQueuedBuffers(); } - RTC_LOG(LS_INFO) << __FUNCTION__ << " Check 2"; rtc::scoped_refptr i420_buffer = frame_buffer->ToI420(); for (uint32_t i = 0; i < buffer->n_planes; i++) diff --git a/src/hwenc_jetson/jetson_h264_encoder.h b/src/hwenc_jetson/jetson_h264_encoder.h index e937a16f..c528a73c 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.h +++ b/src/hwenc_jetson/jetson_h264_encoder.h @@ -103,6 +103,7 @@ class JetsonH264Encoder : public webrtc::VideoEncoder int32_t configured_framerate_; uint32_t target_bitrate_bps_; uint32_t configured_bitrate_bps_; + int key_frame_interval_; uint32_t decode_pixfmt_; uint32_t raw_width_; uint32_t raw_height_; From 75e43087148a6e28b658f81472fe63c579969a1f Mon Sep 17 00:00:00 2001 From: melpon Date: Sun, 15 Sep 2019 18:19:17 +0900 Subject: [PATCH 15/37] =?UTF-8?q?ubuntu-18.04=5Farmv8=5Fjetson=5Fnano=20?= =?UTF-8?q?=E3=81=AE=E8=A8=AD=E5=AE=9A=E3=82=92=20ubuntu-18.04=5Farmv8=20?= =?UTF-8?q?=E3=81=8B=E3=82=89=E9=81=A9=E5=BD=93=E3=81=AB=E3=82=B3=E3=83=94?= =?UTF-8?q?=E3=83=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 48 +++++++--- VERSION | 1 + .../ubuntu-18.04_armv8_jetson_nano/Dockerfile | 92 +++++++++++++++++++ 3 files changed, 126 insertions(+), 15 deletions(-) create mode 100644 build/ubuntu-18.04_armv8_jetson_nano/Dockerfile diff --git a/Makefile b/Makefile index 48286cb0..4f50a901 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,13 @@ include VERSION # USE_ROS: ROS を使っているかどうか # 有効な値は 0, 1 # -# USE_MMAL_ENCODER: ハードウェアエンコーダを利用するかどうか +# USE_MMAL_ENCODER: MMAL ハードウェアエンコーダを利用するかどうか +# 有効な値は 0, 1 +# +# USE_JETSON_ENCODER: Jetson のハードウェアエンコーダを利用するかどうか +# 有効な値は 0, 1 +# +# USE_H264: H264 を利用するかどうか # 有効な値は 0, 1 # # BOOST_ROOT: Boost のインストール先ディレクトリ @@ -52,6 +58,7 @@ ifeq ($(PACKAGE_NAME),raspbian-buster_armv6) USE_ROS ?= 0 USE_MMAL_ENCODER ?= 1 USE_JETSON_ENCODER ?= 0 + USE_H264 ?= 1 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/raspbian-buster_armv6 @@ -64,6 +71,7 @@ else ifeq ($(PACKAGE_NAME),raspbian-buster_armv7) USE_ROS ?= 0 USE_MMAL_ENCODER ?= 1 USE_JETSON_ENCODER ?= 0 + USE_H264 ?= 1 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/raspbian-buster_armv7 @@ -76,6 +84,7 @@ else ifeq ($(PACKAGE_NAME),ubuntu-16.04_armv7_ros) USE_ROS ?= 1 USE_MMAL_ENCODER ?= 1 USE_JETSON_ENCODER ?= 0 + USE_H264 ?= 1 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/ubuntu-16.04_armv7_ros @@ -88,10 +97,24 @@ else ifeq ($(PACKAGE_NAME),ubuntu-18.04_armv8) USE_ROS ?= 0 USE_MMAL_ENCODER ?= 0 USE_JETSON_ENCODER ?= 0 + USE_H264 ?= 0 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/ubuntu-18.04_armv8 SYSROOT ?= /root/rootfs +else ifeq ($(PACKAGE_NAME),ubuntu-18.04_armv8_jetson_nano) + TARGET_OS ?= linux + TARGET_OS_LINUX ?= ubuntu-18.04 + TARGET_ARCH ?= arm + TARGET_ARCH_ARM ?= armv8 + USE_ROS ?= 0 + USE_MMAL_ENCODER ?= 0 + USE_JETSON_ENCODER ?= 1 + USE_H264 ?= 1 + BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) + WEBRTC_SRC_ROOT ?= /root/webrtc/src + WEBRTC_LIB_ROOT ?= /root/webrtc-build/ubuntu-18.04_armv8_jetson_nano + SYSROOT ?= /root/rootfs else ifeq ($(PACKAGE_NAME),ubuntu-16.04_x86_64_ros) TARGET_OS ?= linux TARGET_OS_LINUX ?= ubuntu-16.04 @@ -99,6 +122,7 @@ else ifeq ($(PACKAGE_NAME),ubuntu-16.04_x86_64_ros) USE_ROS ?= 1 USE_MMAL_ENCODER ?= 0 USE_JETSON_ENCODER ?= 0 + USE_H264 ?= 0 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/ubuntu-16.04_x86_64_ros @@ -109,6 +133,7 @@ else ifeq ($(PACKAGE_NAME),ubuntu-18.04_x86_64) USE_ROS ?= 0 USE_MMAL_ENCODER ?= 0 USE_JETSON_ENCODER ?= 0 + USE_H264 ?= 0 BOOST_ROOT ?= /root/boost-$(BOOST_VERSION) WEBRTC_SRC_ROOT ?= /root/webrtc/src WEBRTC_LIB_ROOT ?= /root/webrtc-build/ubuntu-18.04_x86_64 @@ -118,6 +143,7 @@ else ifeq ($(PACKAGE_NAME),macos) USE_ROS ?= 0 USE_MMAL_ENCODER ?= 0 USE_JETSON_ENCODER ?= 0 + USE_H264 ?= 1 BOOST_ROOT ?= $(CURDIR)/build/macos/boost-$(BOOST_VERSION) # CURDIR を付けると、ar に渡す時に引数が長すぎるって怒られたので、 # 相対パスで指定する。 @@ -169,6 +195,11 @@ else HAS_ERROR = 1 endif + ifndef USE_H264 + $(info - USE_H264 が指定されていません) + HAS_ERROR = 1 + endif + ifndef BOOST_ROOT $(info - BOOST_ROOT が指定されていません) HAS_ERROR = 1 @@ -225,20 +256,7 @@ ifeq ($(USE_ROS),1) endif endif -# x86_64/armv8 の場合は H264 を使わない -ifeq ($(TARGET_OS),linux) - ifeq ($(TARGET_ARCH),x86_64) - CFLAGS += -DUSE_H264=0 - else ifeq ($(USE_JETSON_ENCODER),1) - CFLAGS += -DUSE_H264=1 - else ifeq ($(TARGET_ARCH_ARM),armv8) - CFLAGS += -DUSE_H264=0 - else - CFLAGS += -DUSE_H264=1 - endif -else ifeq ($(TARGET_OS),macos) - CFLAGS += -DUSE_H264=1 -endif +CFLAGS += -DUSE_H264=$(USE_H264) ifeq ($(TARGET_OS),linux) CFLAGS += -fpic diff --git a/VERSION b/VERSION index 4be02bb9..4262f1e6 100644 --- a/VERSION +++ b/VERSION @@ -13,5 +13,6 @@ PACKAGE_NAMES= \ raspbian-buster_armv7 \ ubuntu-16.04_armv7_ros \ ubuntu-18.04_armv8 \ + ubuntu-18.04_armv8_jetson_nano \ ubuntu-16.04_x86_64_ros \ ubuntu-18.04_x86_64 diff --git a/build/ubuntu-18.04_armv8_jetson_nano/Dockerfile b/build/ubuntu-18.04_armv8_jetson_nano/Dockerfile new file mode 100644 index 00000000..08df1942 --- /dev/null +++ b/build/ubuntu-18.04_armv8_jetson_nano/Dockerfile @@ -0,0 +1,92 @@ +# syntax = docker/dockerfile:1.1.1-experimental +FROM ubuntu:18.04 + +#### ---- ここから ubuntu-18.04_armv8 のコピー ----- + +ARG PACKAGE_NAME + +LABEL jp.shiguredo.momo=$PACKAGE_NAME + +RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache + +COPY script/docker.sh /root/ + +# RootFS の構築 + +COPY patch/arm64.conf /root/ + +RUN --mount=type=cache,id=$PACKAGE_NAME,target=/var/cache/apt --mount=type=cache,id=$PACKAGE_NAME,target=/var/lib/apt \ + /bin/bash -c " \ + set -ex \ + && source /root/docker.sh \ + && noninteractive_tzdata \ + && init_rootfs_arm64 /root/rootfs /root/arm64.conf \ + " + +COPY patch/4k.patch /root/ + +# WebRTC の準備 + +ARG WEBRTC_VERSION +ARG WEBRTC_COMMIT + +RUN --mount=type=cache,id=$PACKAGE_NAME,target=/var/cache/apt --mount=type=cache,id=$PACKAGE_NAME,target=/var/lib/apt --mount=type=cache,id=$PACKAGE_NAME,target=/root/webrtc-cache \ + /bin/bash -c " \ + set -ex \ + && source /root/docker.sh \ + && prepare_webrtc_arm /root/webrtc-cache $WEBRTC_COMMIT /root/webrtc \ + && cd /root/webrtc/src \ + && patch -p2 < /root/4k.patch \ + " + +# WebRTC のビルド + +ENV PATH /root/webrtc/depot_tools:$PATH + +RUN \ + set -ex \ + && cd /root/webrtc/src \ + && gn gen /root/webrtc-build/$PACKAGE_NAME --args='target_os="linux" target_cpu="arm64" is_debug=false target_sysroot="/root/rootfs" rtc_use_h264=false rtc_include_tests=false rtc_build_json=true use_rtti=true is_desktop_linux=false' \ + && ninja -C /root/webrtc-build/$PACKAGE_NAME + +# Boost のビルド + +ARG BOOST_VERSION + +RUN \ + /bin/bash -c " \ + set -ex \ + && source /root/docker.sh \ + && setup_boost $BOOST_VERSION /root/boost-source \ + && cd /root/boost-source/source \ + && echo 'using clang : : /root/webrtc/src/third_party/llvm-build/Release+Asserts/bin/clang++ : ;' > project-config.jam \ + && ./b2 \ + cxxflags=' \ + -D_LIBCPP_ABI_UNSTABLE \ + -nostdinc++ \ + -isystem/root/webrtc/src/buildtools/third_party/libc++/trunk/include \ + --target=aarch64-linux-gnu \ + --sysroot=/root/rootfs \ + -I/root/rootfs/usr/include/aarch64-linux-gnu \ + ' \ + linkflags=' \ + -L/root/rootfs/usr/lib/aarch64-linux-gnu \ + -B/root/rootfs/usr/lib/aarch64-linux-gnu \ + ' \ + toolset=clang \ + visibility=global \ + target-os=linux \ + architecture=arm \ + address-model=64 \ + link=static \ + variant=release \ + install \ + -j`nproc` \ + --ignore-site-config \ + --prefix=/root/boost-$BOOST_VERSION \ + --with-filesystem \ + " + +#### ---- ここまで ubuntu-18.04_armv8 のコピー ----- + + From 9ecd0bbf466870b95871b40b4fadb9276a54ed71 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Tue, 17 Sep 2019 17:08:34 +0900 Subject: [PATCH 16/37] =?UTF-8?q?Apple=20/=20AMD=20/=20NVIDIA=20=E3=81=AE?= =?UTF-8?q?=E3=83=A9=E3=82=A4=E3=82=BB=E3=83=B3=E3=82=B9=E6=83=85=E5=A0=B1?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 28b6050b..def430ff 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,12 @@ WebRTC Native Client に対する有料でのサポート契約については W - Raspberry Pi のハードウェアエンコーダのライセンス費用は Raspberry Pi の価格に含まれています - https://www.raspberrypi.org/forums/viewtopic.php?t=200855 +- Apple のライセンス費用は個人利用および非商用利用目的に限る + - https://store.apple.com/Catalog/Japan/Images/EA0270_QTMPEG2.html +- AMD ビデオカードのハードウェアエンコーダのライセンス費用は別途、団体との契約が必要 + - https://github.com/GPUOpen-LibrariesAndSDKs/AMF/blob/master/amf/doc/AMF_API_Reference.pdf +- NVIDIA ビデオカードのハードウェアエンコーダのライセンス費用は別途、団体との契約が必要 + - https://developer.download.nvidia.com/designworks/DesignWorks_SDKs_Samples_Tools_License_distrib_use_rights_2017_06_13.pdf - NVIDIA Jetson Nano のハードウェアエンコーダのライセンス費用は別途、団体との契約が必要 - [NVIDIA Jetson Nano 搭載の H\.264/H\.265 ハードウェアエンコーダのライセンスについて](https://medium.com/@voluntas/nvidia-jetson-nano-%E6%90%AD%E8%BC%89%E3%81%AE-h-264-h-265-%E3%83%8F%E3%83%BC%E3%83%89%E3%82%A6%E3%82%A7%E3%82%A2%E3%82%A8%E3%83%B3%E3%82%B3%E3%83%BC%E3%83%80%E3%81%AE%E3%83%A9%E3%82%A4%E3%82%BB%E3%83%B3%E3%82%B9%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6-ca207af302ee) - Intel Quick Sync Video のハードウェアエンコーダライセンス費用は別途、団体との契約が必要 From 804d52027746482aa4bf0764f410e4173f7b47df Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Wed, 18 Sep 2019 11:28:24 +0900 Subject: [PATCH 17/37] =?UTF-8?q?=E3=83=A9=E3=82=A4=E3=82=BB=E3=83=B3?= =?UTF-8?q?=E3=82=B9=E3=81=AB=E3=81=A4=E3=81=84=E3=81=A6=E8=BF=BD=E8=A8=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index def430ff..6a42adce 100644 --- a/README.md +++ b/README.md @@ -35,9 +35,7 @@ Momo はオープンソースソフトウェアですが、開発については ## バイナリ提供について -Raspberry Pi 向けのバイナリのみ提供を行っています。 - -H.264 ハードウェアエンコーダーのライセンスが Raspberry Pi 以外は不透明というのが理由です。 +Raspberry Pi 向けのバイナリのみ提供を行っています。今後は macOS や Jetson Nano 向けのバイナリ配布も行っていく予定です。 ## 動作環境 @@ -107,9 +105,15 @@ WebRTC Native Client に対する有料でのサポート契約については W ## H.264 のライセンス費用について +H.264 ハードウェアエンコーダ **のみ** を利用している Momo 単体の配布においてはライセンス費用は不要ですが、ハードウェアとセットで配布する場合はライセンス費用を支払う必要があります。 + +ただし、 Raspberry Pi においては H.264 のライセンスがハードウェア費用に含まれているため、配布時にライセンス費用を支払う必要はありません。 + +詳細については [MPEG LA](https://www.mpegla.com/) まで問い合わせる事をおすすめします。 + - Raspberry Pi のハードウェアエンコーダのライセンス費用は Raspberry Pi の価格に含まれています - https://www.raspberrypi.org/forums/viewtopic.php?t=200855 -- Apple のライセンス費用は個人利用および非商用利用目的に限る +- Apple のライセンス費用は個人利用および非商用利用目的に限るため、配布においては別途、団体との契約が必要 - https://store.apple.com/Catalog/Japan/Images/EA0270_QTMPEG2.html - AMD ビデオカードのハードウェアエンコーダのライセンス費用は別途、団体との契約が必要 - https://github.com/GPUOpen-LibrariesAndSDKs/AMF/blob/master/amf/doc/AMF_API_Reference.pdf From 4131e8ba4a84398873df31691326c5df7fcc6c91 Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 18 Sep 2019 17:32:01 +0900 Subject: [PATCH 18/37] =?UTF-8?q?Jetson=20Nano=20=E7=94=A8=E3=83=90?= =?UTF-8?q?=E3=82=A4=E3=83=8A=E3=83=AA=E3=81=8C=20Docker=20=E3=81=A7?= =?UTF-8?q?=E3=82=AF=E3=83=AD=E3=82=B9=E3=82=B3=E3=83=B3=E3=83=91=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E3=81=A7=E3=81=8D=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 33 ++++++++- .../ubuntu-18.04_armv8_jetson_nano/Dockerfile | 15 ++-- .../ubuntu-18.04_armv8_jetson_nano/jetson.sh | 73 +++++++++++++++++++ src/rtc/manager.cpp | 4 +- 4 files changed, 111 insertions(+), 14 deletions(-) create mode 100755 build/ubuntu-18.04_armv8_jetson_nano/jetson.sh diff --git a/Makefile b/Makefile index 4f50a901..eb623d4e 100644 --- a/Makefile +++ b/Makefile @@ -288,13 +288,15 @@ ifeq ($(TARGET_OS),linux) ifeq ($(USE_JETSON_ENCODER),1) CFLAGS += \ -DUSE_JETSON_ENCODER=1 \ - -I../../tegra_multimedia_api/include/ \ - -I/mnt/jetson/usr/src/nvidia/tegra_multimedia_api/include/libjpeg-8b/ + -I$(SYSROOT)/include/libjpeg-8b LDFLAGS += \ - -L/mnt/jetson/usr/lib/aarch64-linux-gnu \ -lv4l2 \ - -L/mnt/jetson/usr/lib/aarch64-linux-gnu/tegra \ + -L$(SYSROOT)/usr/lib/aarch64-linux-gnu/tegra \ + -Wl,-rpath-link=$(SYSROOT)/lib/aarch64-linux-gnu \ + -Wl,-rpath-link=$(SYSROOT)/usr/lib/aarch64-linux-gnu \ + -Wl,-rpath-link=$(SYSROOT)/usr/lib/aarch64-linux-gnu/tegra \ -lnvbuf_utils \ + -lnvbuf_fdmap \ -lnvddk_vic \ -lnvddk_2d_v2 \ -lnvjpeg \ @@ -303,6 +305,17 @@ ifeq ($(TARGET_OS),linux) -lnvos SOURCES += $(shell find src/v4l2_video_capturer -maxdepth 1 -name '*.cpp') SOURCES += $(shell find src/hwenc_jetson -maxdepth 1 -name '*.cpp') + JETSON_ADDITIONAL_SOURCES += \ + $(SYSROOT)/usr/src/nvidia/tegra_multimedia_api/samples/common/classes/NvBuffer.cpp \ + $(SYSROOT)/usr/src/nvidia/tegra_multimedia_api/samples/common/classes/NvElement.cpp \ + $(SYSROOT)/usr/src/nvidia/tegra_multimedia_api/samples/common/classes/NvElementProfiler.cpp \ + $(SYSROOT)/usr/src/nvidia/tegra_multimedia_api/samples/common/classes/NvJpegDecoder.cpp \ + $(SYSROOT)/usr/src/nvidia/tegra_multimedia_api/samples/common/classes/NvJpegEncoder.cpp \ + $(SYSROOT)/usr/src/nvidia/tegra_multimedia_api/samples/common/classes/NvLogging.cpp \ + $(SYSROOT)/usr/src/nvidia/tegra_multimedia_api/samples/common/classes/NvV4l2Element.cpp \ + $(SYSROOT)/usr/src/nvidia/tegra_multimedia_api/samples/common/classes/NvV4l2ElementPlane.cpp \ + $(SYSROOT)/usr/src/nvidia/tegra_multimedia_api/samples/common/classes/NvVideoConverter.cpp \ + $(SYSROOT)/usr/src/nvidia/tegra_multimedia_api/samples/common/classes/NvVideoEncoder.cpp endif else # armv6, armv7 用 @@ -500,6 +513,18 @@ $(BUILD_ROOT)/libwebrtc.a: $(shell find $(WEBRTC_LIB_ROOT)/obj -name '*.o') | $( endif +# Jetson Nano を利用する場合の追加ソースのコンパイル +ifeq ($(USE_JETSON_ENCODER),1) + +$(BUILD_ROOT)/tegra_multimedia_api/%.o: $(SYSROOT)/usr/src/nvidia/tegra_multimedia_api/%.cpp | $(BUILD_ROOT) + @mkdir -p `dirname $@` + $(CXX) $(CFLAGS) $(INCLUDES) -c $< -o $@ + +# OBJECTS にも追加する +OBJECTS += $(addprefix $(BUILD_ROOT)/,$(subst $(SYSROOT)/usr/src/nvidia/,,$(patsubst %.cpp,%.o,$(JETSON_ADDITIONAL_SOURCES)))) + +endif + # src 以下のソースのビルドルール $(BUILD_ROOT)/src/%.o: src/%.cpp | $(BUILD_ROOT) @mkdir -p `dirname $@` diff --git a/build/ubuntu-18.04_armv8_jetson_nano/Dockerfile b/build/ubuntu-18.04_armv8_jetson_nano/Dockerfile index 08df1942..af5d7ba2 100644 --- a/build/ubuntu-18.04_armv8_jetson_nano/Dockerfile +++ b/build/ubuntu-18.04_armv8_jetson_nano/Dockerfile @@ -1,8 +1,6 @@ # syntax = docker/dockerfile:1.1.1-experimental FROM ubuntu:18.04 -#### ---- ここから ubuntu-18.04_armv8 のコピー ----- - ARG PACKAGE_NAME LABEL jp.shiguredo.momo=$PACKAGE_NAME @@ -23,10 +21,15 @@ RUN --mount=type=cache,id=$PACKAGE_NAME,target=/var/cache/apt --mount=type=cache && init_rootfs_arm64 /root/rootfs /root/arm64.conf \ " -COPY patch/4k.patch /root/ +# 構築した RootFS を Jetson Nano のファイルで上書きする + +COPY jetson.sh /root/ +RUN /root/jetson.sh # WebRTC の準備 +COPY patch/4k.patch /root/ + ARG WEBRTC_VERSION ARG WEBRTC_COMMIT @@ -46,7 +49,7 @@ ENV PATH /root/webrtc/depot_tools:$PATH RUN \ set -ex \ && cd /root/webrtc/src \ - && gn gen /root/webrtc-build/$PACKAGE_NAME --args='target_os="linux" target_cpu="arm64" is_debug=false target_sysroot="/root/rootfs" rtc_use_h264=false rtc_include_tests=false rtc_build_json=true use_rtti=true is_desktop_linux=false' \ + && gn gen /root/webrtc-build/$PACKAGE_NAME --args='target_os="linux" target_cpu="arm64" is_debug=false target_sysroot="/root/rootfs" rtc_use_h264=true rtc_include_tests=false rtc_build_json=true use_rtti=true is_desktop_linux=false' \ && ninja -C /root/webrtc-build/$PACKAGE_NAME # Boost のビルド @@ -86,7 +89,3 @@ RUN \ --prefix=/root/boost-$BOOST_VERSION \ --with-filesystem \ " - -#### ---- ここまで ubuntu-18.04_armv8 のコピー ----- - - diff --git a/build/ubuntu-18.04_armv8_jetson_nano/jetson.sh b/build/ubuntu-18.04_armv8_jetson_nano/jetson.sh new file mode 100755 index 00000000..4f466842 --- /dev/null +++ b/build/ubuntu-18.04_armv8_jetson_nano/jetson.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +set -ex + +SYSDIR=/root/rootfs +WORK_DIR=/root/jetson + +mkdir -p $WORK_DIR +cd $WORK_DIR + +# https://developer.nvidia.com/embedded/linux-tegra から必要なファイルをダウンロードしてくる + +if [ ! -e Tegra210_Linux_R32.2.1_aarch64.tbz2 ]; then + curl -LO https://developer.nvidia.com/embedded/dlc/r32-2-1_Release_v1.0/Nano-TX1/Tegra210_Linux_R32.2.1_aarch64.tbz2 +fi +if [ ! -e Tegra_Multimedia_API_R32.2.1_aarch64.tbz2 ]; then + curl -LO https://developer.nvidia.com/embedded/dlc/r32-2-1_Release_v1.0/Nano-TX1/Tegra_Multimedia_API_R32.2.1_aarch64.tbz2 +fi +if [ ! -e Tegra_Linux_Sample-Root-Filesystem_R32.2.1_aarch64.tbz2 ]; then + curl -LO https://developer.nvidia.com/embedded/dlc/r32-2-1_Release_v1.0/Nano-TX1/Tegra_Linux_Sample-Root-Filesystem_R32.2.1_aarch64.tbz2 +fi + +# Tegra Multimedia API +# 必要なファイルだけ展開する +pushd $SYSDIR + tar xvf $WORK_DIR/Tegra_Multimedia_API_R32.2.1_aarch64.tbz2 --strip-components=1 tegra_multimedia_api/include/ +popd + +mkdir -p $SYSDIR/usr/src/nvidia +pushd $SYSDIR/usr/src/nvidia + # ほんとは全部展開するのが正しいけど、必要なのはソースだけなのでそこだけ展開する + tar xvf $WORK_DIR/Tegra_Multimedia_API_R32.2.1_aarch64.tbz2 tegra_multimedia_api/samples/common/classes +popd + +# nvidia drivers +# 何も考えずに全部展開する +tar xvf Tegra210_Linux_R32.2.1_aarch64.tbz2 Linux_for_Tegra/nv_tegra/nvidia_drivers.tbz2 +pushd $SYSDIR + tar xvf $WORK_DIR/Linux_for_Tegra/nv_tegra/nvidia_drivers.tbz2 +popd + +# filesystem +pushd $SYSDIR + # ライブラリ系とインクルード系だけ展開する + tar \ + -xvf $WORK_DIR/Tegra_Linux_Sample-Root-Filesystem_R32.2.1_aarch64.tbz2 \ + --wildcards \ + lib/aarch64-linux-gnu \ + 'usr/lib/aarch64-linux-gnu/*.so*' \ + usr/include \ + --exclude 'lib/aarch64-linux-gnu/*/*' + + pushd usr/lib/aarch64-linux-gnu + # 既存の libdl.so は libdl.so -> ../../../lib/aarch64-linux-gnu/libdl.so.2 なのに対して、 + # Jetson Nano の libdl.so は libdl.so -> /lib/aarch64-linux-gnu/libdl.so.2 になっているため、パスが見つからない。 + # なので symlink を相対パスで貼り直してやる。 + ln -sf ../../../lib/aarch64-linux-gnu/libdl.so.2 libdl.so + # libz.so も同じ。 + ln -sf ../../../lib/aarch64-linux-gnu/libz.so.1.2.11 libz.so + popd +popd + +# symlink +pushd $SYSDIR/usr/lib/aarch64-linux-gnu/ + ln -s libv4l2.so.0.0.999999 libv4l2.so +popd +pushd $SYSDIR/usr/lib/aarch64-linux-gnu/tegra/ + ln -s libnvbuf_utils.so.1.0.0 libnvbuf_utils.so + ln -s libnvbuf_fdmap.so.1.0.0 libnvbuf_fdmap.so +popd + +# cleanup +rm -rf $WORK_DIR diff --git a/src/rtc/manager.cpp b/src/rtc/manager.cpp index 2c8f6db6..5b9fd93c 100644 --- a/src/rtc/manager.cpp +++ b/src/rtc/manager.cpp @@ -32,7 +32,7 @@ #include "ros/ros_audio_device_module.h" #endif -#if USE_MMAL_ENCODER | USE_JETSON_ENCODER +#if USE_MMAL_ENCODER || USE_JETSON_ENCODER #include "api/video_codecs/video_encoder_factory.h" #include "hw_video_encoder_factory.h" #endif @@ -83,7 +83,7 @@ RTCManager::RTCManager(ConnectionSettings conn_settings, media_dependencies.video_encoder_factory = CreateObjCEncoderFactory(); media_dependencies.video_decoder_factory = CreateObjCDecoderFactory(); #else -#if USE_MMAL_ENCODER | USE_JETSON_ENCODER +#if USE_MMAL_ENCODER || USE_JETSON_ENCODER media_dependencies.video_encoder_factory = std::unique_ptr( absl::make_unique()); From 580da243c5d5d5b760a7b0a779abc96c64cff5cd Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 18 Sep 2019 17:49:26 +0900 Subject: [PATCH 19/37] =?UTF-8?q?CHANGES=20=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 36a08aa3..80f8ea7c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,13 @@ ## develop +## xx.yy.zz + +- [ADD] Jetson Nano のハードウェアエンコーダを利用する機能を実装 + - @tnoho +- [ADD] Jetson Nano のビルドを追加 + - @melpon + ## 19.09.0 - [ADD] --disable-echo-cancellation オプションを追加 From a68721394f779da7c91e5db66282bfd2c142e769 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Wed, 18 Sep 2019 19:08:05 +0900 Subject: [PATCH 20/37] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=82=92=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 0ea2b190..a4da8f8c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,7 @@ - @melpon - [ADD] CI を CircleCI から GitHub Actions へ切り替える - macOS の時間制限が OSS の場合はないため Weekly build から Daily build のみにきりかえる + - @hakobera - [UPDATE] libwebrtc M78 コミットポジションを 3 にする - libwebrtc のハッシュは 68c715dc01cd8cd0ad2726453e7376b5f353fcd1 - @voluntas From 5910bb7c2d1b3176ee7c5c5c57ae50f38a8a5ef3 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Wed, 18 Sep 2019 19:09:43 +0900 Subject: [PATCH 21/37] =?UTF-8?q?19.09.1-rc1=20=E3=81=AB=E4=B8=8A=E3=81=92?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 53bf731f..15908038 100644 --- a/VERSION +++ b/VERSION @@ -1,6 +1,6 @@ # 各種ライブラリのバージョン情報 # 項目を変更してビルドすれば各環境へ反映される（ように作る） -MOMO_VERSION=19.09.1-rc0 +MOMO_VERSION=19.09.1-rc1 WEBRTC_VERSION=78 WEBRTC_COMMIT=68c715dc01cd8cd0ad2726453e7376b5f353fcd1 BOOST_VERSION=1.71.0 From 74e6a37a11d04111224a370e12a2acc9a29494d6 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Wed, 18 Sep 2019 19:10:24 +0900 Subject: [PATCH 22/37] =?UTF-8?q?=E3=82=B3=E3=82=B9=E3=83=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 6a42adce..4d4da558 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,6 @@ H.264 ハードウェアエンコーダ **のみ** を利用している Momo - [ラズパイ\+momoでWebRTCで送信するときにマイクの代わりに音声ファイルを使用する \- Qiita](https://qiita.com/tetsu_koba/items/b887c1a0be9f26b795f2) - [ラズパイのmomoでステレオ音声でWebRTCで配信する \- Qiita](https://qiita.com/tetsu_koba/items/6c07129caa5a08d5d172) - ## Windows 版について Windows 版 Momo をビルドするツールを販売しております。興味のある方は以下をご確認ください。 From 4a58b70ac73d514612a3a310b7181030276df94d Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Wed, 18 Sep 2019 19:12:51 +0900 Subject: [PATCH 23/37] =?UTF-8?q?GitHub=20Actions=20=E3=81=AB=20Jetson=20N?= =?UTF-8?q?ano=20=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/daily_build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/daily_build.yml b/.github/workflows/daily_build.yml index a4fb60f9..1087f293 100644 --- a/.github/workflows/daily_build.yml +++ b/.github/workflows/daily_build.yml @@ -15,6 +15,14 @@ jobs: - run: DOCKER_BUILDKIT=1 NOTTY=1 NOMOUNT=1 make ubuntu-18.04_armv8 working-directory: build timeout-minutes: 120 + build_ubuntu-18_04_armv8_jetson_nano: + name: Build momo for build_ubuntu-18.04_armv8_jetson_nano + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - run: DOCKER_BUILDKIT=1 NOTTY=1 NOMOUNT=1 make ubuntu-18.04_armv8_jetson_nano + working-directory: build + timeout-minutes: 120 build_ubuntu-18_04_x86_64: name: Build momo for build_ubuntu-18.04_x86_64 runs-on: ubuntu-18.04 From 083a8c4a98b1da20e43b0eff3c4d601979369397 Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 18 Sep 2019 19:37:56 +0900 Subject: [PATCH 24/37] =?UTF-8?q?=E3=83=A9=E3=82=BA=E3=83=91=E3=82=A4?= =?UTF-8?q?=E3=81=AE=20Dockerfile=20=E3=82=92=20Ubuntu=2018.04=20=E3=83=99?= =?UTF-8?q?=E3=83=BC=E3=82=B9=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/raspbian-buster_armv6/Dockerfile | 3 ++- build/raspbian-buster_armv7/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/build/raspbian-buster_armv6/Dockerfile b/build/raspbian-buster_armv6/Dockerfile index a311177d..5761093e 100644 --- a/build/raspbian-buster_armv6/Dockerfile +++ b/build/raspbian-buster_armv6/Dockerfile @@ -1,5 +1,5 @@ # syntax = docker/dockerfile:1.1.1-experimental -FROM ubuntu:16.04 +FROM ubuntu:18.04 ARG PACKAGE_NAME @@ -17,6 +17,7 @@ RUN --mount=type=cache,id=$PACKAGE_NAME,target=/var/cache/apt --mount=type=cache /bin/bash -c " \ set -ex \ && source /root/docker.sh \ + && noninteractive_tzdata \ && init_rootfs_armhf /root/rootfs /root/rpi-raspbian.conf \ " diff --git a/build/raspbian-buster_armv7/Dockerfile b/build/raspbian-buster_armv7/Dockerfile index edb14253..1d836feb 100644 --- a/build/raspbian-buster_armv7/Dockerfile +++ b/build/raspbian-buster_armv7/Dockerfile @@ -1,5 +1,5 @@ # syntax = docker/dockerfile:1.1.1-experimental -FROM ubuntu:16.04 +FROM ubuntu:18.04 ARG PACKAGE_NAME @@ -17,6 +17,7 @@ RUN --mount=type=cache,id=$PACKAGE_NAME,target=/var/cache/apt --mount=type=cache /bin/bash -c " \ set -ex \ && source /root/docker.sh \ + && noninteractive_tzdata \ && init_rootfs_armhf /root/rootfs /root/rpi-raspbian.conf \ " From bad5a6ae72451e5e6afc83170972ad8492634107 Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 18 Sep 2019 22:42:45 +0900 Subject: [PATCH 25/37] =?UTF-8?q?.clang-format=20=E3=82=92=E7=94=A8?= =?UTF-8?q?=E6=84=8F=E3=81=97=E3=81=A6=E3=82=BD=E3=83=BC=E3=82=B9=E3=82=B3?= =?UTF-8?q?=E3=83=BC=E3=83=89=E5=85=A8=E4=BD=93=E3=81=AB=20clang-format=20?= =?UTF-8?q?=E3=82=92=E9=81=A9=E7=94=A8=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .clang-format | 8 + src/ayame/ayame_server.cpp | 121 ++-- src/ayame/ayame_server.h | 50 +- src/ayame/ayame_websocket_client.cpp | 622 +++++++++--------- src/ayame/ayame_websocket_client.h | 182 ++--- src/connection_settings.h | 14 +- src/hwenc_jetson/jetson_h264_encoder.cpp | 543 +++++++-------- src/hwenc_jetson/jetson_h264_encoder.h | 90 +-- src/hwenc_mmal/mmal_h264_encoder.cpp | 487 ++++++-------- src/hwenc_mmal/mmal_h264_encoder.h | 68 +- src/mac_helper/mac_capturer.h | 6 +- src/mac_helper/mac_capturer.mm | 42 +- src/mac_helper/objc_codec_factory_helper.h | 1 - src/mac_helper/objc_codec_factory_helper.mm | 6 +- src/main.cpp | 93 +-- src/p2p/p2p_connection.cpp | 43 +- src/p2p/p2p_connection.h | 29 +- src/p2p/p2p_server.cpp | 115 ++-- src/p2p/p2p_server.h | 36 +- src/p2p/p2p_session.cpp | 237 ++++--- src/p2p/p2p_session.h | 91 ++- src/p2p/p2p_websocket_session.cpp | 236 +++---- src/p2p/p2p_websocket_session.h | 55 +- src/ros/ros_audio_device.cpp | 287 ++++---- src/ros/ros_audio_device.h | 65 +- src/ros/ros_audio_device_module.cpp | 328 ++++----- src/ros/ros_audio_device_module.h | 68 +- src/ros/ros_log_sink.h | 43 +- src/ros/ros_video_capture.cpp | 96 ++- src/ros/ros_video_capture.h | 21 +- src/rtc/connection.cpp | 155 ++--- src/rtc/connection.h | 28 +- src/rtc/device_video_capturer.cpp | 26 +- src/rtc/device_video_capturer.h | 20 +- src/rtc/hw_video_encoder_factory.cpp | 8 +- src/rtc/hw_video_encoder_factory.h | 18 +- src/rtc/manager.cpp | 136 ++-- src/rtc/manager.h | 11 +- src/rtc/messagesender.h | 13 +- src/rtc/native_buffer.cpp | 89 +-- src/rtc/native_buffer.h | 15 +- src/rtc/observer.cpp | 42 +- src/rtc/observer.h | 82 +-- src/rtc/scalable_track_source.cpp | 28 +- src/signal_listener.cpp | 28 +- src/signal_listener.h | 18 +- src/sora/sora_server.cpp | 135 ++-- src/sora/sora_server.h | 48 +- src/sora/sora_session.cpp | 288 ++++---- src/sora/sora_session.h | 106 ++- src/sora/sora_websocket_client.cpp | 524 +++++++-------- src/sora/sora_websocket_client.h | 178 ++--- src/url_parts.h | 96 ++- src/util.cpp | 273 ++++---- src/util.h | 53 +- .../v4l2_video_capturer.cpp | 91 ++- src/v4l2_video_capturer/v4l2_video_capturer.h | 4 +- src/watchdog.cpp | 15 +- src/watchdog.h | 8 +- src/ws/websocket.cpp | 244 ++++--- src/ws/websocket.h | 85 +-- 61 files changed, 3260 insertions(+), 3688 deletions(-) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..b855004e --- /dev/null +++ b/.clang-format @@ -0,0 +1,8 @@ +--- +Language: Cpp +BasedOnStyle: Chromium +ReflowComments: false +--- +Language: ObjC +BasedOnStyle: Chromium +ReflowComments: false diff --git a/src/ayame/ayame_server.cpp b/src/ayame/ayame_server.cpp index 81af4d26..0030ddda 100644 --- a/src/ayame/ayame_server.cpp +++ b/src/ayame/ayame_server.cpp @@ -2,85 +2,72 @@ #include "util.h" -AyameServer::AyameServer( - boost::asio::io_context& ioc, - boost::asio::ip::tcp::endpoint endpoint, - RTCManager* rtc_manager, - ConnectionSettings conn_settings) - : acceptor_(ioc) - , socket_(ioc) - , rtc_manager_(rtc_manager) - , conn_settings_(conn_settings) - , ws_client_(std::make_shared(ioc, rtc_manager, conn_settings)) -{ - boost::system::error_code ec; +AyameServer::AyameServer(boost::asio::io_context& ioc, + boost::asio::ip::tcp::endpoint endpoint, + RTCManager* rtc_manager, + ConnectionSettings conn_settings) + : acceptor_(ioc), + socket_(ioc), + rtc_manager_(rtc_manager), + conn_settings_(conn_settings), + ws_client_(std::make_shared(ioc, + rtc_manager, + conn_settings)) { + boost::system::error_code ec; - // Open the acceptor - acceptor_.open(endpoint.protocol(), ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "open"); - return; - } + // Open the acceptor + acceptor_.open(endpoint.protocol(), ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "open"); + return; + } - // Allow address reuse - acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "set_option"); - return; - } + // Allow address reuse + acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "set_option"); + return; + } - // Bind to the server address - acceptor_.bind(endpoint, ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "bind"); - return; - } + // Bind to the server address + acceptor_.bind(endpoint, ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "bind"); + return; + } - // Start listening for connections - acceptor_.listen( - boost::asio::socket_base::max_listen_connections, ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "listen"); - return; - } + // Start listening for connections + acceptor_.listen(boost::asio::socket_base::max_listen_connections, ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "listen"); + return; + } } -AyameServer::~AyameServer() -{ - // これをやっておかないと循環参照でリークする - ws_client_->release(); +AyameServer::~AyameServer() { + // これをやっておかないと循環参照でリークする + ws_client_->release(); } -void AyameServer::run() -{ - if (!acceptor_.is_open()) - return; +void AyameServer::run() { + if (!acceptor_.is_open()) + return; - ws_client_->connect(); - doAccept(); + ws_client_->connect(); + doAccept(); } -void AyameServer::doAccept() -{ - acceptor_.async_accept( - socket_, - std::bind( - &AyameServer::onAccept, - shared_from_this(), - std::placeholders::_1)); +void AyameServer::doAccept() { + acceptor_.async_accept(socket_, + std::bind(&AyameServer::onAccept, shared_from_this(), + std::placeholders::_1)); } -void AyameServer::onAccept(boost::system::error_code ec) -{ - if (ec) - { - MOMO_BOOST_ERROR(ec, "accept"); - } +void AyameServer::onAccept(boost::system::error_code ec) { + if (ec) { + MOMO_BOOST_ERROR(ec, "accept"); + } - // Accept another connection - doAccept(); + // Accept another connection + doAccept(); } diff --git a/src/ayame/ayame_server.h b/src/ayame/ayame_server.h index dd61c53d..d06f1faa 100644 --- a/src/ayame/ayame_server.h +++ b/src/ayame/ayame_server.h @@ -2,16 +2,16 @@ #define AYAME_SERVER_H_ #include +#include +#include #include #include #include #include -#include -#include -#include "rtc/manager.h" -#include "connection_settings.h" #include "ayame_websocket_client.h" +#include "connection_settings.h" +#include "rtc/manager.h" /* Ayame と接続する用のサーバ。 @@ -19,28 +19,26 @@ - 指定されたシグナリングサーバに WebSocket で接続 - websocket の挙動は `./ayame_websocket_client` に記述している */ -class AyameServer : public std::enable_shared_from_this -{ - boost::asio::ip::tcp::acceptor acceptor_; - boost::asio::ip::tcp::socket socket_; - - RTCManager* rtc_manager_; - ConnectionSettings conn_settings_; - std::shared_ptr ws_client_; - -public: - AyameServer( - boost::asio::io_context& ioc, - boost::asio::ip::tcp::endpoint endpoint, - RTCManager* rtc_manager, - ConnectionSettings conn_settings); - ~AyameServer(); - - void run(); - -private: - void doAccept(); - void onAccept(boost::system::error_code ec); +class AyameServer : public std::enable_shared_from_this { + boost::asio::ip::tcp::acceptor acceptor_; + boost::asio::ip::tcp::socket socket_; + + RTCManager* rtc_manager_; + ConnectionSettings conn_settings_; + std::shared_ptr ws_client_; + + public: + AyameServer(boost::asio::io_context& ioc, + boost::asio::ip::tcp::endpoint endpoint, + RTCManager* rtc_manager, + ConnectionSettings conn_settings); + ~AyameServer(); + + void run(); + + private: + void doAccept(); + void onAccept(boost::system::error_code ec); }; #endif diff --git a/src/ayame/ayame_websocket_client.cpp b/src/ayame/ayame_websocket_client.cpp index b7f52c40..377feef5 100644 --- a/src/ayame/ayame_websocket_client.cpp +++ b/src/ayame/ayame_websocket_client.cpp @@ -9,247 +9,231 @@ using json = nlohmann::json; bool AyameWebsocketClient::parseURL(URLParts& parts) const { - std::string url = conn_settings_.ayame_signaling_host; + std::string url = conn_settings_.ayame_signaling_host; - if (!URLParts::parse(url, parts)) { - throw std::exception(); - } + if (!URLParts::parse(url, parts)) { + throw std::exception(); + } - std::string default_port; - if (parts.scheme == "wss") { - return true; - } else if (parts.scheme == "ws") { - return false; - } else { - throw std::exception(); - } + std::string default_port; + if (parts.scheme == "wss") { + return true; + } else if (parts.scheme == "ws") { + return false; + } else { + throw std::exception(); + } } boost::asio::ssl::context AyameWebsocketClient::createSSLContext() const { - boost::asio::ssl::context ctx(boost::asio::ssl::context::tlsv12); - ctx.set_default_verify_paths(); - ctx.set_options(boost::asio::ssl::context::default_workarounds | - boost::asio::ssl::context::no_sslv2 | - boost::asio::ssl::context::no_sslv3 | - boost::asio::ssl::context::single_dh_use); - return ctx; + boost::asio::ssl::context ctx(boost::asio::ssl::context::tlsv12); + ctx.set_default_verify_paths(); + ctx.set_options(boost::asio::ssl::context::default_workarounds | + boost::asio::ssl::context::no_sslv2 | + boost::asio::ssl::context::no_sslv3 | + boost::asio::ssl::context::single_dh_use); + return ctx; } -webrtc::PeerConnectionInterface::IceConnectionState AyameWebsocketClient::getRTCConnectionState() const { return rtc_state_; } +webrtc::PeerConnectionInterface::IceConnectionState +AyameWebsocketClient::getRTCConnectionState() const { + return rtc_state_; +} std::shared_ptr AyameWebsocketClient::getRTCConnection() const { - if (rtc_state_ == webrtc::PeerConnectionInterface::IceConnectionState::kIceConnectionConnected) { + if (rtc_state_ == webrtc::PeerConnectionInterface::IceConnectionState:: + kIceConnectionConnected) { return connection_; } else { return nullptr; } } -AyameWebsocketClient::AyameWebsocketClient(boost::asio::io_context& ioc, RTCManager* manager, ConnectionSettings conn_settings) - : ioc_(ioc) - , resolver_(ioc) - , manager_(manager) - , retry_count_(0) - , conn_settings_(conn_settings) - , watchdog_(ioc, std::bind(&AyameWebsocketClient::onWatchdogExpired, this)) -{ - reset(); +AyameWebsocketClient::AyameWebsocketClient(boost::asio::io_context& ioc, + RTCManager* manager, + ConnectionSettings conn_settings) + : ioc_(ioc), + resolver_(ioc), + manager_(manager), + retry_count_(0), + conn_settings_(conn_settings), + watchdog_(ioc, + std::bind(&AyameWebsocketClient::onWatchdogExpired, this)) { + reset(); } void AyameWebsocketClient::reset() { - connection_ = nullptr; - connected_ = false; - ice_servers_.clear(); - - if (parseURL(parts_)) { - auto ssl_ctx = createSSLContext(); - boost::beast::websocket::stream> wss(ioc_, ssl_ctx); - ws_.reset(new Websocket(ioc_, std::move(ssl_ctx))); - // SNI の設定を行う - if (!SSL_set_tlsext_host_name(ws_->nativeSecureSocket().next_layer().native_handle(), parts_.host.c_str())) - { - boost::system::error_code ec{static_cast(::ERR_get_error()), boost::asio::error::get_ssl_category()}; - MOMO_BOOST_ERROR(ec, "SSL_set_tlsext_host_name"); - } - } else { - boost::beast::websocket::stream ws(ioc_); - ws_.reset(new Websocket(ioc_)); + connection_ = nullptr; + connected_ = false; + ice_servers_.clear(); + + if (parseURL(parts_)) { + auto ssl_ctx = createSSLContext(); + boost::beast::websocket::stream< + boost::asio::ssl::stream> + wss(ioc_, ssl_ctx); + ws_.reset(new Websocket(ioc_, std::move(ssl_ctx))); + // SNI の設定を行う + if (!SSL_set_tlsext_host_name( + ws_->nativeSecureSocket().next_layer().native_handle(), + parts_.host.c_str())) { + boost::system::error_code ec{static_cast(::ERR_get_error()), + boost::asio::error::get_ssl_category()}; + MOMO_BOOST_ERROR(ec, "SSL_set_tlsext_host_name"); } + } else { + boost::beast::websocket::stream ws(ioc_); + ws_.reset(new Websocket(ioc_)); + } } void AyameWebsocketClient::release() { - connection_ = nullptr; + connection_ = nullptr; } -bool AyameWebsocketClient::connect() -{ - RTC_LOG(LS_INFO) << __FUNCTION__; +bool AyameWebsocketClient::connect() { + RTC_LOG(LS_INFO) << __FUNCTION__; - if (connected_) { - return false; - } + if (connected_) { + return false; + } - std::string port; - if (parts_.port.empty()) { - port = ws_->isSSL() ? "443" : "80"; - } else { - port = parts_.port; - } + std::string port; + if (parts_.port.empty()) { + port = ws_->isSSL() ? "443" : "80"; + } else { + port = parts_.port; + } - // DNS ルックアップ - resolver_.async_resolve( - parts_.host, - port, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &AyameWebsocketClient::onResolve, - shared_from_this(), - std::placeholders::_1, - std::placeholders::_2))); + // DNS ルックアップ + resolver_.async_resolve( + parts_.host, port, + boost::asio::bind_executor( + ws_->strand(), + std::bind(&AyameWebsocketClient::onResolve, shared_from_this(), + std::placeholders::_1, std::placeholders::_2))); - watchdog_.enable(30); + watchdog_.enable(30); - return true; + return true; } -void AyameWebsocketClient::reconnectAfter() -{ +void AyameWebsocketClient::reconnectAfter() { int interval = 5 * (2 * retry_count_ + 1); - RTC_LOG(LS_INFO) << __FUNCTION__ << " reconnect after " << interval <<" sec"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " reconnect after " << interval << " sec"; watchdog_.enable(interval); retry_count_++; } void AyameWebsocketClient::onWatchdogExpired() { - RTC_LOG(LS_WARNING) << __FUNCTION__; + RTC_LOG(LS_WARNING) << __FUNCTION__; - RTC_LOG(LS_INFO) << __FUNCTION__ << " reconnecting...:"; - reset(); - connect(); + RTC_LOG(LS_INFO) << __FUNCTION__ << " reconnecting...:"; + reset(); + connect(); } -void AyameWebsocketClient::onResolve(boost::system::error_code ec, boost::asio::ip::tcp::resolver::results_type results) { - if (ec) { - reconnectAfter(); - return MOMO_BOOST_ERROR(ec, "resolve"); - } +void AyameWebsocketClient::onResolve( + boost::system::error_code ec, + boost::asio::ip::tcp::resolver::results_type results) { + if (ec) { + reconnectAfter(); + return MOMO_BOOST_ERROR(ec, "resolve"); + } - // DNS ルックアップで得られたエンドポイントに対して接続する - if (ws_->isSSL()) { - boost::asio::async_connect( - ws_->nativeSecureSocket().next_layer().next_layer(), - results.begin(), - results.end(), - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &AyameWebsocketClient::onSSLConnect, - shared_from_this(), - std::placeholders::_1))); - } else { - boost::asio::async_connect( - ws_->nativeSocket().next_layer(), - results.begin(), - results.end(), - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &AyameWebsocketClient::onConnect, - shared_from_this(), - std::placeholders::_1))); - } + // DNS ルックアップで得られたエンドポイントに対して接続する + if (ws_->isSSL()) { + boost::asio::async_connect( + ws_->nativeSecureSocket().next_layer().next_layer(), results.begin(), + results.end(), + boost::asio::bind_executor( + ws_->strand(), + std::bind(&AyameWebsocketClient::onSSLConnect, shared_from_this(), + std::placeholders::_1))); + } else { + boost::asio::async_connect( + ws_->nativeSocket().next_layer(), results.begin(), results.end(), + boost::asio::bind_executor( + ws_->strand(), + std::bind(&AyameWebsocketClient::onConnect, shared_from_this(), + std::placeholders::_1))); + } } void AyameWebsocketClient::onSSLConnect(boost::system::error_code ec) { - if (ec) { - reconnectAfter(); - return MOMO_BOOST_ERROR(ec, "SSLConnect"); - } + if (ec) { + reconnectAfter(); + return MOMO_BOOST_ERROR(ec, "SSLConnect"); + } - // SSL のハンドシェイク - ws_->nativeSecureSocket().next_layer().async_handshake( - boost::asio::ssl::stream_base::client, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &AyameWebsocketClient::onSSLHandshake, - shared_from_this(), - std::placeholders::_1))); + // SSL のハンドシェイク + ws_->nativeSecureSocket().next_layer().async_handshake( + boost::asio::ssl::stream_base::client, + boost::asio::bind_executor( + ws_->strand(), std::bind(&AyameWebsocketClient::onSSLHandshake, + shared_from_this(), std::placeholders::_1))); } void AyameWebsocketClient::onSSLHandshake(boost::system::error_code ec) { - if (ec) { - reconnectAfter(); - return MOMO_BOOST_ERROR(ec, "SSLHandshake"); - } + if (ec) { + reconnectAfter(); + return MOMO_BOOST_ERROR(ec, "SSLHandshake"); + } - // Websocket のハンドシェイク - ws_->nativeSecureSocket().async_handshake(parts_.host, parts_.path_query_fragment, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &AyameWebsocketClient::onHandshake, - shared_from_this(), - std::placeholders::_1))); + // Websocket のハンドシェイク + ws_->nativeSecureSocket().async_handshake( + parts_.host, parts_.path_query_fragment, + boost::asio::bind_executor( + ws_->strand(), std::bind(&AyameWebsocketClient::onHandshake, + shared_from_this(), std::placeholders::_1))); } void AyameWebsocketClient::onConnect(boost::system::error_code ec) { - if (ec) { - reconnectAfter(); - return MOMO_BOOST_ERROR(ec, "connect"); - } - // Websocket のハンドシェイク - ws_->nativeSocket().async_handshake(parts_.host, parts_.path_query_fragment, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &AyameWebsocketClient::onHandshake, - shared_from_this(), - std::placeholders::_1))); + if (ec) { + reconnectAfter(); + return MOMO_BOOST_ERROR(ec, "connect"); + } + // Websocket のハンドシェイク + ws_->nativeSocket().async_handshake( + parts_.host, parts_.path_query_fragment, + boost::asio::bind_executor( + ws_->strand(), std::bind(&AyameWebsocketClient::onHandshake, + shared_from_this(), std::placeholders::_1))); } -void AyameWebsocketClient::onHandshake(boost::system::error_code ec) -{ - if (ec) { - reconnectAfter(); - return MOMO_BOOST_ERROR(ec, "Handshake"); - } +void AyameWebsocketClient::onHandshake(boost::system::error_code ec) { + if (ec) { + reconnectAfter(); + return MOMO_BOOST_ERROR(ec, "Handshake"); + } - connected_ = true; + connected_ = true; - ws_->startToRead(std::bind( - &AyameWebsocketClient::onRead, - this, - std::placeholders::_1, - std::placeholders::_2, - std::placeholders::_3)); + ws_->startToRead(std::bind(&AyameWebsocketClient::onRead, this, + std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3)); - doRegister(); + doRegister(); } -void AyameWebsocketClient::doRegister() -{ - json json_message = { - {"type", "register"}, - {"clientId", conn_settings_.ayame_client_id}, - {"roomId", conn_settings_.ayame_room_id}, - }; - if (conn_settings_.ayame_signaling_key != "") { - json_message["key"] = conn_settings_.ayame_signaling_key; - } - ws_->sendText(json_message.dump()); +void AyameWebsocketClient::doRegister() { + json json_message = { + {"type", "register"}, + {"clientId", conn_settings_.ayame_client_id}, + {"roomId", conn_settings_.ayame_room_id}, + }; + if (conn_settings_.ayame_signaling_key != "") { + json_message["key"] = conn_settings_.ayame_signaling_key; + } + ws_->sendText(json_message.dump()); } void AyameWebsocketClient::doSendPong() { - json json_message = { - {"type", "pong"} - }; - ws_->sendText(json_message.dump()); + json json_message = {{"type", "pong"}}; + ws_->sendText(json_message.dump()); } -void AyameWebsocketClient::createPeerConnection() -{ +void AyameWebsocketClient::createPeerConnection() { if (ice_servers_.empty()) { return; } @@ -260,180 +244,168 @@ void AyameWebsocketClient::createPeerConnection() connection_ = manager_->createConnection(rtc_config, this); } -void AyameWebsocketClient::close() -{ - if (ws_->isSSL()) { - ws_->nativeSecureSocket().async_close(boost::beast::websocket::close_code::normal, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &AyameWebsocketClient::onClose, - shared_from_this(), - std::placeholders::_1))); - } else { - ws_->nativeSocket().async_close(boost::beast::websocket::close_code::normal, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &AyameWebsocketClient::onClose, - shared_from_this(), - std::placeholders::_1))); - } +void AyameWebsocketClient::close() { + if (ws_->isSSL()) { + ws_->nativeSecureSocket().async_close( + boost::beast::websocket::close_code::normal, + boost::asio::bind_executor( + ws_->strand(), + std::bind(&AyameWebsocketClient::onClose, shared_from_this(), + std::placeholders::_1))); + } else { + ws_->nativeSocket().async_close( + boost::beast::websocket::close_code::normal, + boost::asio::bind_executor( + ws_->strand(), + std::bind(&AyameWebsocketClient::onClose, shared_from_this(), + std::placeholders::_1))); + } } -void AyameWebsocketClient::onClose(boost::system::error_code ec) -{ - if (ec) - return MOMO_BOOST_ERROR(ec, "close"); +void AyameWebsocketClient::onClose(boost::system::error_code ec) { + if (ec) + return MOMO_BOOST_ERROR(ec, "close"); } -void AyameWebsocketClient::onRead(boost::system::error_code ec, std::size_t bytes_transferred, std::string text) -{ - RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec; - - boost::ignore_unused(bytes_transferred); - - // 書き込みのために読み込み処理がキャンセルされた時にこのエラーになるので、これはエラーとして扱わない - if (ec == boost::asio::error::operation_aborted) - return; - - if (ec) - return MOMO_BOOST_ERROR(ec, "Read"); - - RTC_LOG(LS_INFO) << __FUNCTION__ << ": text=" << text; - - auto json_message = json::parse(text); - const std::string type = json_message["type"]; - if (type == "accept") - { - if (!connection_) { - // 返却されてきた iceServers を セットする - try { - auto jservers = json_message["iceServers"]; - for (auto jserver : jservers) - { - webrtc::PeerConnectionInterface::IceServer ice_server; - if (jserver.contains("username")) { - ice_server.username = jserver["username"]; - } - if (jserver.contains("credential")) { - ice_server.password = jserver["credential"]; - } - auto jurls = jserver["urls"]; - for (const std::string url : jurls) - { - ice_server.urls.push_back(url); - } - ice_servers_.push_back(ice_server); - } - } - catch (json::type_error &e) - { - // accept 時に iceServers が返却されてこなかった場合 google の stun server を用いる +void AyameWebsocketClient::onRead(boost::system::error_code ec, + std::size_t bytes_transferred, + std::string text) { + RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec; + + boost::ignore_unused(bytes_transferred); + + // 書き込みのために読み込み処理がキャンセルされた時にこのエラーになるので、これはエラーとして扱わない + if (ec == boost::asio::error::operation_aborted) + return; + + if (ec) + return MOMO_BOOST_ERROR(ec, "Read"); + + RTC_LOG(LS_INFO) << __FUNCTION__ << ": text=" << text; + + auto json_message = json::parse(text); + const std::string type = json_message["type"]; + if (type == "accept") { + if (!connection_) { + // 返却されてきた iceServers を セットする + try { + auto jservers = json_message["iceServers"]; + for (auto jserver : jservers) { webrtc::PeerConnectionInterface::IceServer ice_server; - ice_server.uri = "stun:stun.l.google.com:19302"; + if (jserver.contains("username")) { + ice_server.username = jserver["username"]; + } + if (jserver.contains("credential")) { + ice_server.password = jserver["credential"]; + } + auto jurls = jserver["urls"]; + for (const std::string url : jurls) { + ice_server.urls.push_back(url); + } ice_servers_.push_back(ice_server); } - createPeerConnection(); - // peer connection を生成したら offer SDP を生成して送信する - connection_->createOffer(); + } catch (json::type_error& e) { + // accept 時に iceServers が返却されてこなかった場合 google の stun server を用いる + webrtc::PeerConnectionInterface::IceServer ice_server; + ice_server.uri = "stun:stun.l.google.com:19302"; + ice_servers_.push_back(ice_server); } - } - if (type == "offer") { createPeerConnection(); - const std::string sdp = json_message["sdp"]; - connection_->setOffer(sdp); + // peer connection を生成したら offer SDP を生成して送信する + connection_->createOffer(); } - else if (type == "answer") { - if (!connection_) { - return; - } - std::string sdp; - try - { - sdp = json_message["sdp"]; - } - catch (json::type_error &e) - { - return; - } - connection_->setAnswer(sdp); + } + if (type == "offer") { + createPeerConnection(); + const std::string sdp = json_message["sdp"]; + connection_->setOffer(sdp); + } else if (type == "answer") { + if (!connection_) { + return; } - else if (type == "candidate") { - if (!connection_) { - return; - } - int sdp_mlineindex = 0; - std::string sdp_mid, candidate; - json ice = json_message["ice"]; - sdp_mid = ice["sdpMid"]; - sdp_mlineindex = ice["sdpMLineIndex"]; - candidate = ice["candidate"]; - connection_->addIceCandidate(sdp_mid, sdp_mlineindex, candidate); - } else if (type == "ping") { - if (!connection_) { - return; - } - watchdog_.reset(); - doSendPong(); + std::string sdp; + try { + sdp = json_message["sdp"]; + } catch (json::type_error& e) { + return; + } + connection_->setAnswer(sdp); + } else if (type == "candidate") { + if (!connection_) { + return; } + int sdp_mlineindex = 0; + std::string sdp_mid, candidate; + json ice = json_message["ice"]; + sdp_mid = ice["sdpMid"]; + sdp_mlineindex = ice["sdpMLineIndex"]; + candidate = ice["candidate"]; + connection_->addIceCandidate(sdp_mid, sdp_mlineindex, candidate); + } else if (type == "ping") { + if (!connection_) { + return; + } + watchdog_.reset(); + doSendPong(); + } } // WebRTC からのコールバック // これらは別スレッドからやってくるので取り扱い注意 -void AyameWebsocketClient::onIceConnectionStateChange(webrtc::PeerConnectionInterface::IceConnectionState new_state) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " state:" << new_state; - boost::asio::post( - ws_->strand(), - std::bind( - &AyameWebsocketClient::doIceConnectionStateChange, - shared_from_this(), - new_state)); +void AyameWebsocketClient::onIceConnectionStateChange( + webrtc::PeerConnectionInterface::IceConnectionState new_state) { + RTC_LOG(LS_INFO) << __FUNCTION__ << " state:" << new_state; + boost::asio::post(ws_->strand(), + std::bind(&AyameWebsocketClient::doIceConnectionStateChange, + shared_from_this(), new_state)); } -void AyameWebsocketClient::onIceCandidate(const std::string sdp_mid, const int sdp_mlineindex, const std::string sdp) { - // ayame では candidate sdp の交換で `ice` プロパティを用いる。 `candidate` ではないので注意 - json json_message = { +void AyameWebsocketClient::onIceCandidate(const std::string sdp_mid, + const int sdp_mlineindex, + const std::string sdp) { + // ayame では candidate sdp の交換で `ice` プロパティを用いる。 `candidate` ではないので注意 + json json_message = { {"type", "candidate"}, - }; - // ice プロパティの中に object で candidate 情報をセットして送信する - json_message["ice"] = { - {"candidate", sdp}, - {"sdpMLineIndex", sdp_mlineindex}, - {"sdpMid", sdp_mid} - }; - ws_->sendText(json_message.dump()); + }; + // ice プロパティの中に object で candidate 情報をセットして送信する + json_message["ice"] = {{"candidate", sdp}, + {"sdpMLineIndex", sdp_mlineindex}, + {"sdpMid", sdp_mid}}; + ws_->sendText(json_message.dump()); } -void AyameWebsocketClient::onCreateDescription(webrtc::SdpType type, const std::string sdp) { - // sora と異なり ayame モードでは answer 以外 (offer) type の description を送信する場合もある - // なので type は "answer" 固定にしない - json json_message = { - {"type", webrtc::SdpTypeToString(type)}, - {"sdp", sdp} - }; - ws_->sendText(json_message.dump()); +void AyameWebsocketClient::onCreateDescription(webrtc::SdpType type, + const std::string sdp) { + // sora と異なり ayame モードでは answer 以外 (offer) type の description を送信する場合もある + // なので type は "answer" 固定にしない + json json_message = {{"type", webrtc::SdpTypeToString(type)}, {"sdp", sdp}}; + ws_->sendText(json_message.dump()); } void AyameWebsocketClient::onSetDescription(webrtc::SdpType type) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " SdpType: " << webrtc::SdpTypeToString(type); + RTC_LOG(LS_INFO) << __FUNCTION__ + << " SdpType: " << webrtc::SdpTypeToString(type); if (type == webrtc::SdpType::kOffer) { connection_->createAnswer(); } } -void AyameWebsocketClient::doIceConnectionStateChange(webrtc::PeerConnectionInterface::IceConnectionState new_state) { - RTC_LOG(LS_INFO) << __FUNCTION__ << ": newState=" << Util::iceConnectionStateToString(new_state); - - switch (new_state) { - case webrtc::PeerConnectionInterface::IceConnectionState::kIceConnectionConnected: - retry_count_ = 0; - watchdog_.enable(60); - break; - case webrtc::PeerConnectionInterface::IceConnectionState::kIceConnectionFailed: - reconnectAfter(); - break; - default: - break; - } - rtc_state_ = new_state; +void AyameWebsocketClient::doIceConnectionStateChange( + webrtc::PeerConnectionInterface::IceConnectionState new_state) { + RTC_LOG(LS_INFO) << __FUNCTION__ << ": newState=" + << Util::iceConnectionStateToString(new_state); + + switch (new_state) { + case webrtc::PeerConnectionInterface::IceConnectionState:: + kIceConnectionConnected: + retry_count_ = 0; + watchdog_.enable(60); + break; + case webrtc::PeerConnectionInterface::IceConnectionState:: + kIceConnectionFailed: + reconnectAfter(); + break; + default: + break; + } + rtc_state_ = new_state; } diff --git a/src/ayame/ayame_websocket_client.h b/src/ayame/ayame_websocket_client.h index 2d34c896..0872ce68 100644 --- a/src/ayame/ayame_websocket_client.h +++ b/src/ayame/ayame_websocket_client.h @@ -2,99 +2,111 @@ #define AYAME_WEBSOCKET_CLIENT_ #include -#include -#include -#include -#include #include #include #include +#include +#include +#include #include +#include -#include "ws/websocket.h" -#include "rtc/messagesender.h" -#include "rtc/manager.h" #include "connection_settings.h" +#include "rtc/manager.h" +#include "rtc/messagesender.h" #include "url_parts.h" #include "watchdog.h" +#include "ws/websocket.h" -class AyameWebsocketClient : public std::enable_shared_from_this, public RTCMessageSender -{ - boost::asio::io_context& ioc_; - - boost::asio::ip::tcp::resolver resolver_; - - std::unique_ptr ws_; - - URLParts parts_; - - RTCManager* manager_; - std::shared_ptr connection_; - ConnectionSettings conn_settings_; - - int retry_count_; - webrtc::PeerConnectionInterface::IceConnectionState rtc_state_; - - WatchDog watchdog_; - - bool connected_; - - webrtc::PeerConnectionInterface::IceServers ice_servers_; - -private: - bool parseURL(URLParts& parts) const; - boost::asio::ssl::context createSSLContext() const; - -public: - webrtc::PeerConnectionInterface::IceConnectionState getRTCConnectionState() const; - std::shared_ptr getRTCConnection() const; - -public: - AyameWebsocketClient(boost::asio::io_context& ioc, RTCManager* manager, ConnectionSettings conn_settings); - void reset(); - -public: - // connection_ = nullptr すると直ちに onIceConnectionStateChange コールバックが呼ばれるが、 - // この中で使っている shared_from_this() がデストラクタ内で使えないため、デストラクタで connection_ = nullptr すると実行時エラーになる。 - // なのでこのクラスを解放する前に明示的に release() 関数を呼んでもらうことにする。 - void release(); - -public: - bool connect(); - -private: - void reconnectAfter(); - void onWatchdogExpired(); - -private: - void onResolve(boost::system::error_code ec, boost::asio::ip::tcp::resolver::results_type results); - void onSSLConnect(boost::system::error_code ec); - void onSSLHandshake(boost::system::error_code ec); - void onConnect(boost::system::error_code ec); - void onHandshake(boost::system::error_code ec); - void doRegister(); - void doSendPong(); - void createPeerConnection(); - -public: - void close(); - -private: - void onClose(boost::system::error_code ec); - -private: - void onRead(boost::system::error_code ec, std::size_t bytes_transferred, std::string text); - -private: - // WebRTC からのコールバック - // これらは別スレッドからやってくるので取り扱い注意 - void onIceConnectionStateChange(webrtc::PeerConnectionInterface::IceConnectionState new_state) override; - void onIceCandidate(const std::string sdp_mid, const int sdp_mlineindex, const std::string sdp) override; - void onCreateDescription(webrtc::SdpType type, const std::string sdp) override; - void onSetDescription(webrtc::SdpType type) override; - -private: - void doIceConnectionStateChange(webrtc::PeerConnectionInterface::IceConnectionState new_state); +class AyameWebsocketClient + : public std::enable_shared_from_this, + public RTCMessageSender { + boost::asio::io_context& ioc_; + + boost::asio::ip::tcp::resolver resolver_; + + std::unique_ptr ws_; + + URLParts parts_; + + RTCManager* manager_; + std::shared_ptr connection_; + ConnectionSettings conn_settings_; + + int retry_count_; + webrtc::PeerConnectionInterface::IceConnectionState rtc_state_; + + WatchDog watchdog_; + + bool connected_; + + webrtc::PeerConnectionInterface::IceServers ice_servers_; + + private: + bool parseURL(URLParts& parts) const; + boost::asio::ssl::context createSSLContext() const; + + public: + webrtc::PeerConnectionInterface::IceConnectionState getRTCConnectionState() + const; + std::shared_ptr getRTCConnection() const; + + public: + AyameWebsocketClient(boost::asio::io_context& ioc, + RTCManager* manager, + ConnectionSettings conn_settings); + void reset(); + + public: + // connection_ = nullptr すると直ちに onIceConnectionStateChange コールバックが呼ばれるが、 + // この中で使っている shared_from_this() がデストラクタ内で使えないため、デストラクタで connection_ = nullptr すると実行時エラーになる。 + // なのでこのクラスを解放する前に明示的に release() 関数を呼んでもらうことにする。 + void release(); + + public: + bool connect(); + + private: + void reconnectAfter(); + void onWatchdogExpired(); + + private: + void onResolve(boost::system::error_code ec, + boost::asio::ip::tcp::resolver::results_type results); + void onSSLConnect(boost::system::error_code ec); + void onSSLHandshake(boost::system::error_code ec); + void onConnect(boost::system::error_code ec); + void onHandshake(boost::system::error_code ec); + void doRegister(); + void doSendPong(); + void createPeerConnection(); + + public: + void close(); + + private: + void onClose(boost::system::error_code ec); + + private: + void onRead(boost::system::error_code ec, + std::size_t bytes_transferred, + std::string text); + + private: + // WebRTC からのコールバック + // これらは別スレッドからやってくるので取り扱い注意 + void onIceConnectionStateChange( + webrtc::PeerConnectionInterface::IceConnectionState new_state) override; + void onIceCandidate(const std::string sdp_mid, + const int sdp_mlineindex, + const std::string sdp) override; + void onCreateDescription(webrtc::SdpType type, + const std::string sdp) override; + void onSetDescription(webrtc::SdpType type) override; + + private: + void doIceConnectionStateChange( + webrtc::PeerConnectionInterface::IceConnectionState new_state); }; -#endif // AYAME_WEBSOCKET_CLIENT_ +#endif // AYAME_WEBSOCKET_CLIENT_ diff --git a/src/connection_settings.h b/src/connection_settings.h index 628a0da7..2292a943 100644 --- a/src/connection_settings.h +++ b/src/connection_settings.h @@ -1,14 +1,13 @@ #ifndef CONNECTION_SETTINGS_H_ #define CONNECTION_SETTINGS_H_ -#include #include #include +#include #include "api/rtp_parameters.h" -struct ConnectionSettings -{ +struct ConnectionSettings { std::string camera_name = ""; #if USE_ROS bool image_compressed = false; @@ -86,7 +85,8 @@ struct ConnectionSettings return webrtc::DegradationPreference::BALANCED; } - friend std::ostream& operator<<(std::ostream& os, const ConnectionSettings& cs) { + friend std::ostream& operator<<(std::ostream& os, + const ConnectionSettings& cs) { os << "no_video: " << (cs.no_video ? "true" : "false") << "\n"; os << "no_audio: " << (cs.no_audio ? "true" : "false") << "\n"; os << "video_codec: " << cs.video_codec << "\n"; @@ -95,7 +95,8 @@ struct ConnectionSettings os << "audio_bitrate: " << cs.audio_bitrate << "\n"; os << "resolution: " << cs.resolution << "\n"; os << "framerate: " << cs.framerate << "\n"; - os << "fixed_resolution: " << (cs.fixed_resolution ? "true" : "false") << "\n"; + os << "fixed_resolution: " << (cs.fixed_resolution ? "true" : "false") + << "\n"; os << "priority: " << cs.priority << "\n"; os << "port: " << cs.port << "\n"; os << "ayame_signaling_host: " << cs.ayame_signaling_host << "\n"; @@ -103,7 +104,8 @@ struct ConnectionSettings os << "ayame_client_id: " << cs.ayame_client_id << "\n"; os << "sora_signaling_host: " << cs.sora_signaling_host << "\n"; os << "sora_channel_id: " << cs.sora_channel_id << "\n"; - os << "sora_auto_connect: " << (cs.sora_auto_connect ? "true" : "false") << "\n"; + os << "sora_auto_connect: " << (cs.sora_auto_connect ? "true" : "false") + << "\n"; os << "sora_metadata: " << cs.sora_metadata << "\n"; os << "test_document_root: " << cs.test_document_root << "\n"; return os; diff --git a/src/hwenc_jetson/jetson_h264_encoder.cpp b/src/hwenc_jetson/jetson_h264_encoder.cpp index 757e5f1a..5e230959 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.cpp +++ b/src/hwenc_jetson/jetson_h264_encoder.cpp @@ -18,10 +18,10 @@ #include "third_party/libyuv/include/libyuv/convert_from.h" #include "third_party/libyuv/include/libyuv/video_common.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/time_utils.h" -#include "common_video/libyuv/include/webrtc_libyuv.h" #include "system_wrappers/include/metrics.h" #include "nvbuf_utils.h" @@ -29,15 +29,15 @@ #include "rtc/native_buffer.h" #define H264HWENC_HEADER_DEBUG 0 -#define INIT_ERROR(cond, desc) if(cond) { \ - RTC_LOG(LS_ERROR) << __FUNCTION__ << desc; \ - Release(); \ - return WEBRTC_VIDEO_CODEC_ERROR; } - -namespace -{ -struct nal_entry -{ +#define INIT_ERROR(cond, desc) \ + if (cond) { \ + RTC_LOG(LS_ERROR) << __FUNCTION__ << desc; \ + Release(); \ + return WEBRTC_VIDEO_CODEC_ERROR; \ + } + +namespace { +struct nal_entry { size_t offset; size_t size; }; @@ -45,9 +45,9 @@ struct nal_entry const int kLowH264QpThreshold = 34; const int kHighH264QpThreshold = 40; -} // namespace +} // namespace -JetsonH264Encoder::JetsonH264Encoder(const cricket::VideoCodec &codec) +JetsonH264Encoder::JetsonH264Encoder(const cricket::VideoCodec& codec) : callback_(nullptr), decoder_(nullptr), converter_(nullptr), @@ -56,26 +56,21 @@ JetsonH264Encoder::JetsonH264Encoder(const cricket::VideoCodec &codec) configured_framerate_(30), configured_width_(0), configured_height_(0), - use_mjpeg_(false) -{ -} + use_mjpeg_(false) {} -JetsonH264Encoder::~JetsonH264Encoder() -{ +JetsonH264Encoder::~JetsonH264Encoder() { Release(); } -int32_t JetsonH264Encoder::InitEncode(const webrtc::VideoCodec *codec_settings, - int32_t number_of_cores, - size_t max_payload_size) -{ +int32_t JetsonH264Encoder::InitEncode(const webrtc::VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) { RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; RTC_DCHECK(codec_settings); RTC_DCHECK_EQ(codec_settings->codecType, webrtc::kVideoCodecH264); int32_t release_ret = Release(); - if (release_ret != WEBRTC_VIDEO_CODEC_OK) - { + if (release_ret != WEBRTC_VIDEO_CODEC_OK) { return release_ret; } @@ -87,18 +82,20 @@ int32_t JetsonH264Encoder::InitEncode(const webrtc::VideoCodec *codec_settings, framerate_ = codec_settings->maxFramerate; RTC_LOG(LS_INFO) << "InitEncode " << framerate_ << "fps " - << target_bitrate_bps_ << "bit/sec　" - << codec_settings->maxBitrate << "kbit/sec　"; + << target_bitrate_bps_ << "bit/sec　" + << codec_settings->maxBitrate << "kbit/sec　"; // Initialize encoded image. Default buffer size: size of unencoded data. encoded_image_._completeFrame = true; encoded_image_._encodedWidth = 0; encoded_image_._encodedHeight = 0; encoded_image_.set_size(0); - encoded_image_.timing_.flags = webrtc::VideoSendTiming::TimingFrameFlags::kInvalid; - encoded_image_.content_type_ = (codec_settings->mode == webrtc::VideoCodecMode::kScreensharing) - ? webrtc::VideoContentType::SCREENSHARE - : webrtc::VideoContentType::UNSPECIFIED; + encoded_image_.timing_.flags = + webrtc::VideoSendTiming::TimingFrameFlags::kInvalid; + encoded_image_.content_type_ = + (codec_settings->mode == webrtc::VideoCodecMode::kScreensharing) + ? webrtc::VideoContentType::SCREENSHARE + : webrtc::VideoContentType::UNSPECIFIED; decoder_ = NvJPEGDecoder::createJPEGDecoder("jpegdec"); INIT_ERROR(!decoder_, "Failed to createJPEGDecoder"); @@ -107,12 +104,10 @@ int32_t JetsonH264Encoder::InitEncode(const webrtc::VideoCodec *codec_settings, return WEBRTC_VIDEO_CODEC_OK; } -int32_t JetsonH264Encoder::Release() -{ +int32_t JetsonH264Encoder::Release() { RTC_LOG(LS_INFO) << __FUNCTION__ << " Start"; JetsonRelease(); - if (decoder_) - { + if (decoder_) { delete decoder_; decoder_ = nullptr; } @@ -120,31 +115,29 @@ int32_t JetsonH264Encoder::Release() return WEBRTC_VIDEO_CODEC_OK; } -int32_t JetsonH264Encoder::JetsonConfigure() -{ +int32_t JetsonH264Encoder::JetsonConfigure() { int ret = 0; - if (use_mjpeg_) - { - enc0_buffer_queue_ = new std::queue; + if (use_mjpeg_) { + enc0_buffer_queue_ = new std::queue; converter_ = NvVideoConverter::createVideoConverter("conv"); INIT_ERROR(!decoder_, "Failed to createVideoConverter"); - ret = converter_->setOutputPlaneFormat(decode_pixfmt_, - raw_width_, raw_height_, - V4L2_NV_BUFFER_LAYOUT_PITCH); + ret = converter_->setOutputPlaneFormat( + decode_pixfmt_, raw_width_, raw_height_, V4L2_NV_BUFFER_LAYOUT_PITCH); INIT_ERROR(ret < 0, "Failed to converter setOutputPlaneFormat"); - ret = converter_->setCapturePlaneFormat(V4L2_PIX_FMT_YUV420M, - width_, height_, - V4L2_NV_BUFFER_LAYOUT_PITCH); + ret = converter_->setCapturePlaneFormat( + V4L2_PIX_FMT_YUV420M, width_, height_, V4L2_NV_BUFFER_LAYOUT_PITCH); INIT_ERROR(ret < 0, "Failed to converter setCapturePlaneFormat"); - ret = converter_->output_plane.setupPlane(V4L2_MEMORY_DMABUF, 1, false, false); + ret = converter_->output_plane.setupPlane(V4L2_MEMORY_DMABUF, 1, false, + false); INIT_ERROR(ret < 0, "Failed to setupPlane at converter output_plane"); - ret = converter_->capture_plane.setupPlane(V4L2_MEMORY_MMAP, 10, false, false); + ret = converter_->capture_plane.setupPlane(V4L2_MEMORY_MMAP, 10, false, + false); INIT_ERROR(ret < 0, "Failed to setupPlane at converter capture_plane"); ret = converter_->output_plane.setStreamStatus(true); @@ -153,18 +146,18 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = converter_->capture_plane.setStreamStatus(true); INIT_ERROR(ret < 0, "Failed to setStreamStatus at converter capture_plane"); - converter_->capture_plane.setDQThreadCallback(ConvertFinishedCallbackFunction); + converter_->capture_plane.setDQThreadCallback( + ConvertFinishedCallbackFunction); } encoder_ = NvVideoEncoder::createVideoEncoder("enc0"); INIT_ERROR(!encoder_, "Failed to createVideoEncoder"); - ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_H264, - width_, height_, 4 * 1024 * 1024); + ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_H264, width_, height_, + 4 * 1024 * 1024); INIT_ERROR(ret < 0, "Failed to encoder setCapturePlaneFormat"); - ret = encoder_->setOutputPlaneFormat(V4L2_PIX_FMT_YUV420M, - width_, height_); + ret = encoder_->setOutputPlaneFormat(V4L2_PIX_FMT_YUV420M, width_, height_); INIT_ERROR(ret < 0, "Failed to encoder setOutputPlaneFormat"); ret = encoder_->setBitrate(bitrate_adjuster_.GetAdjustedBitrateBps()); @@ -205,13 +198,11 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = encoder_->setInsertVuiEnabled(true); INIT_ERROR(ret < 0, "Failed to setInsertSpsPpsAtIdrEnabled"); - if (use_mjpeg_) - { - ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_DMABUF, 10, false, false); + if (use_mjpeg_) { + ret = + encoder_->output_plane.setupPlane(V4L2_MEMORY_DMABUF, 10, false, false); INIT_ERROR(ret < 0, "Failed to setupPlane at encoder output_plane"); - } - else - { + } else { ret = encoder_->output_plane.setupPlane(V4L2_MEMORY_MMAP, 10, true, false); INIT_ERROR(ret < 0, "Failed to setupPlane at encoder output_plane"); } @@ -228,12 +219,10 @@ int32_t JetsonH264Encoder::JetsonConfigure() ret = encoder_->capture_plane.setStreamStatus(true); INIT_ERROR(ret < 0, "Failed to setStreamStatus at encoder capture_plane"); - if (use_mjpeg_) - { + if (use_mjpeg_) { converter_->capture_plane.startDQThread(this); - for (uint32_t i = 0; i < converter_->capture_plane.getNumBuffers(); i++) - { + for (uint32_t i = 0; i < converter_->capture_plane.getNumBuffers(); i++) { struct v4l2_buffer v4l2_buf; struct v4l2_plane planes[MAX_PLANES]; memset(&v4l2_buf, 0, sizeof(v4l2_buf)); @@ -244,29 +233,26 @@ int32_t JetsonH264Encoder::JetsonConfigure() INIT_ERROR(ret < 0, "Failed to qBuffer at converter capture_plane"); } - for (uint32_t i = 0; i < encoder_->output_plane.getNumBuffers(); i++) - { + for (uint32_t i = 0; i < encoder_->output_plane.getNumBuffers(); i++) { enc0_buffer_queue_->push(encoder_->output_plane.getNthBuffer(i)); } encoder_->output_plane.setDQThreadCallback(EncodeOutputCallbackFunction); } encoder_->capture_plane.setDQThreadCallback(EncodeFinishedCallbackFunction); - if (use_mjpeg_) - { + if (use_mjpeg_) { encoder_->output_plane.startDQThread(this); } encoder_->capture_plane.startDQThread(this); - for (uint32_t i = 0; i < encoder_->capture_plane.getNumBuffers(); i++) - { - struct v4l2_buffer v4l2_buf; - struct v4l2_plane planes[MAX_PLANES]; - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); - v4l2_buf.index = i; - v4l2_buf.m.planes = planes; - ret = encoder_->capture_plane.qBuffer(v4l2_buf, NULL); - INIT_ERROR(ret < 0, "Failed to qBuffer at encoder capture_plane"); + for (uint32_t i = 0; i < encoder_->capture_plane.getNumBuffers(); i++) { + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); + v4l2_buf.index = i; + v4l2_buf.m.planes = planes; + ret = encoder_->capture_plane.qBuffer(v4l2_buf, NULL); + INIT_ERROR(ret < 0, "Failed to qBuffer at encoder capture_plane"); } configured_framerate_ = framerate_; @@ -276,87 +262,75 @@ int32_t JetsonH264Encoder::JetsonConfigure() return WEBRTC_VIDEO_CODEC_OK; } -void JetsonH264Encoder::JetsonRelease() -{ +void JetsonH264Encoder::JetsonRelease() { if (!encoder_) return; - if (converter_) - { + if (converter_) { SendEOS(converter_); - } - else - { + } else { SendEOS(encoder_); } encoder_->capture_plane.waitForDQThread(2000); encoder_->capture_plane.deinitPlane(); encoder_->output_plane.deinitPlane(); - if (converter_) - { + if (converter_) { delete enc0_buffer_queue_; } delete encoder_; encoder_ = nullptr; - if (converter_) - { + if (converter_) { converter_->capture_plane.waitForDQThread(2000); delete converter_; converter_ = nullptr; } } -void JetsonH264Encoder::SendEOS(NvV4l2Element *element) -{ - if (element->output_plane.getStreamStatus()) - { +void JetsonH264Encoder::SendEOS(NvV4l2Element* element) { + if (element->output_plane.getStreamStatus()) { struct v4l2_buffer v4l2_buf; struct v4l2_plane planes[MAX_PLANES]; - NvBuffer *buffer; + NvBuffer* buffer; memset(&v4l2_buf, 0, sizeof(v4l2_buf)); memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane)); v4l2_buf.m.planes = planes; - if (element->output_plane.getNumQueuedBuffers() == element->output_plane.getNumBuffers()) - { - if (element->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) - { + if (element->output_plane.getNumQueuedBuffers() == + element->output_plane.getNumBuffers()) { + if (element->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) { RTC_LOG(LS_ERROR) << "Failed to dqBuffer at encoder output_plane"; } } planes[0].bytesused = 0; - if (element->output_plane.qBuffer(v4l2_buf, NULL) < 0) - { - RTC_LOG(LS_ERROR) << "Failed to qBuffer at encoder output_plane"; + if (element->output_plane.qBuffer(v4l2_buf, NULL) < 0) { + RTC_LOG(LS_ERROR) << "Failed to qBuffer at encoder output_plane"; } } } -bool JetsonH264Encoder::ConvertFinishedCallbackFunction(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer, - void *data) -{ - return ((JetsonH264Encoder *)data)->ConvertFinishedCallback(v4l2_buf, buffer, shared_buffer); +bool JetsonH264Encoder::ConvertFinishedCallbackFunction( + struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer, + void* data) { + return ((JetsonH264Encoder*)data) + ->ConvertFinishedCallback(v4l2_buf, buffer, shared_buffer); } -bool JetsonH264Encoder::ConvertFinishedCallback(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer) -{ - NvBuffer *enc0_buffer; +bool JetsonH264Encoder::ConvertFinishedCallback(struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer) { + NvBuffer* enc0_buffer; struct v4l2_buffer enc0_qbuf; struct v4l2_plane planes[MAX_PLANES]; - - if (!v4l2_buf) - { + + if (!v4l2_buf) { RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; return false; } { std::unique_lock lock(enc0_buffer_mtx_); - while (enc0_buffer_queue_->empty()) - { + while (enc0_buffer_queue_->empty()) { enc0_buffer_cond_.wait(lock, [this] { return enc0_buffer_ready_; }); enc0_buffer_ready_ = false; } @@ -370,23 +344,21 @@ bool JetsonH264Encoder::ConvertFinishedCallback(struct v4l2_buffer *v4l2_buf, enc0_qbuf.index = enc0_buffer->index; enc0_qbuf.m.planes = planes; - for (int i = 0; i < MAX_PLANES; i++) - { + for (int i = 0; i < MAX_PLANES; i++) { enc0_qbuf.m.planes[i].bytesused = v4l2_buf->m.planes[i].bytesused; } enc0_qbuf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; enc0_qbuf.timestamp.tv_sec = v4l2_buf->timestamp.tv_sec; enc0_qbuf.timestamp.tv_usec = v4l2_buf->timestamp.tv_usec; - - if (encoder_->output_plane.qBuffer(enc0_qbuf, buffer) < 0) - { - RTC_LOG(LS_ERROR) << __FUNCTION__ << " Failed to qBuffer at encoder output_plane"; + + if (encoder_->output_plane.qBuffer(enc0_qbuf, buffer) < 0) { + RTC_LOG(LS_ERROR) << __FUNCTION__ + << " Failed to qBuffer at encoder output_plane"; return false; } - - if (v4l2_buf->m.planes[0].bytesused == 0) - { + + if (v4l2_buf->m.planes[0].bytesused == 0) { RTC_LOG(LS_INFO) << __FUNCTION__ << " buffer size is zero"; return false; } @@ -394,23 +366,22 @@ bool JetsonH264Encoder::ConvertFinishedCallback(struct v4l2_buffer *v4l2_buf, return true; } -bool JetsonH264Encoder::EncodeOutputCallbackFunction(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer, - void *data) -{ - return ((JetsonH264Encoder *)data)->EncodeOutputCallback(v4l2_buf, buffer, shared_buffer); +bool JetsonH264Encoder::EncodeOutputCallbackFunction( + struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer, + void* data) { + return ((JetsonH264Encoder*)data) + ->EncodeOutputCallback(v4l2_buf, buffer, shared_buffer); } -bool JetsonH264Encoder::EncodeOutputCallback(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer) -{ +bool JetsonH264Encoder::EncodeOutputCallback(struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer) { struct v4l2_buffer conv_qbuf; struct v4l2_plane planes[MAX_PLANES]; - if (!v4l2_buf) - { + if (!v4l2_buf) { RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; return false; } @@ -423,9 +394,9 @@ bool JetsonH264Encoder::EncodeOutputCallback(struct v4l2_buffer *v4l2_buf, { std::unique_lock lock(enc0_buffer_mtx_); - if (converter_->capture_plane.qBuffer(conv_qbuf, nullptr) < 0) - { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to qBuffer at converter capture_plane"; + if (converter_->capture_plane.qBuffer(conv_qbuf, nullptr) < 0) { + RTC_LOG(LS_ERROR) << __FUNCTION__ + << "Failed to qBuffer at converter capture_plane"; return false; } enc0_buffer_queue_->push(buffer); @@ -433,8 +404,7 @@ bool JetsonH264Encoder::EncodeOutputCallback(struct v4l2_buffer *v4l2_buf, enc0_buffer_cond_.notify_all(); } - if (conv_qbuf.m.planes[0].bytesused == 0) - { + if (conv_qbuf.m.planes[0].bytesused == 0) { RTC_LOG(LS_INFO) << __FUNCTION__ << " buffer size is zero"; return false; } @@ -442,31 +412,29 @@ bool JetsonH264Encoder::EncodeOutputCallback(struct v4l2_buffer *v4l2_buf, return true; } -bool JetsonH264Encoder::EncodeFinishedCallbackFunction(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer, - void *data) -{ - return ((JetsonH264Encoder *)data)->EncodeFinishedCallback(v4l2_buf, buffer, shared_buffer); +bool JetsonH264Encoder::EncodeFinishedCallbackFunction( + struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer, + void* data) { + return ((JetsonH264Encoder*)data) + ->EncodeFinishedCallback(v4l2_buf, buffer, shared_buffer); } -bool JetsonH264Encoder::EncodeFinishedCallback(struct v4l2_buffer *v4l2_buf, - NvBuffer * buffer, - NvBuffer * shared_buffer) -{ - if (!v4l2_buf) - { +bool JetsonH264Encoder::EncodeFinishedCallback(struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer) { + if (!v4l2_buf) { RTC_LOG(LS_INFO) << __FUNCTION__ << " v4l2_buf is null"; return false; } - if (buffer->planes[0].bytesused == 0) - { + if (buffer->planes[0].bytesused == 0) { RTC_LOG(LS_INFO) << __FUNCTION__ << " buffer size is zero"; return false; } - uint64_t pts = v4l2_buf->timestamp.tv_sec * rtc::kNumMicrosecsPerSec - + v4l2_buf->timestamp.tv_usec; + uint64_t pts = v4l2_buf->timestamp.tv_sec * rtc::kNumMicrosecsPerSec + + v4l2_buf->timestamp.tv_usec; RTC_LOG(LS_INFO) << __FUNCTION__ << " pts:" << pts << " bytesused:" << buffer->planes[0].bytesused; @@ -474,25 +442,23 @@ bool JetsonH264Encoder::EncodeFinishedCallback(struct v4l2_buffer *v4l2_buf, { rtc::CritScope lock(&frame_params_lock_); do { - if (frame_params_.empty()) - { - RTC_LOG(LS_WARNING) << __FUNCTION__ - << "Frame parameter is not found. SkipFrame pts:" - << pts; + if (frame_params_.empty()) { + RTC_LOG(LS_WARNING) + << __FUNCTION__ + << "Frame parameter is not found. SkipFrame pts:" << pts; return true; } params = std::move(frame_params_.front()); frame_params_.pop(); } while (params->timestamp < pts); - if (params->timestamp != pts) - { - RTC_LOG(LS_WARNING) << __FUNCTION__ + if (params->timestamp != pts) { + RTC_LOG(LS_WARNING) << __FUNCTION__ << "Frame parameter is not found. SkipFrame pts:" << pts; return true; } } - + encoded_image_._encodedWidth = params->width; encoded_image_._encodedHeight = params->height; encoded_image_.capture_time_ms_ = params->render_time_ms; @@ -502,9 +468,8 @@ bool JetsonH264Encoder::EncodeFinishedCallback(struct v4l2_buffer *v4l2_buf, encoded_image_.SetColorSpace(params->color_space); SendFrame(buffer->planes[0].data, buffer->planes[0].bytesused); - - if (encoder_->capture_plane.qBuffer(*v4l2_buf, NULL) < 0) - { + + if (encoder_->capture_plane.qBuffer(*v4l2_buf, NULL) < 0) { RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to qBuffer at capture_plane"; return false; } @@ -513,21 +478,18 @@ bool JetsonH264Encoder::EncodeFinishedCallback(struct v4l2_buffer *v4l2_buf, } int32_t JetsonH264Encoder::RegisterEncodeCompleteCallback( - webrtc::EncodedImageCallback *callback) -{ + webrtc::EncodedImageCallback* callback) { callback_ = callback; return WEBRTC_VIDEO_CODEC_OK; } -void JetsonH264Encoder::SetRates(const RateControlParameters ¶meters) -{ +void JetsonH264Encoder::SetRates(const RateControlParameters& parameters) { if (encoder_ == nullptr) return; if (parameters.bitrate.get_sum_bps() <= 0 || parameters.framerate_fps <= 0) return; - RTC_LOG(LS_INFO) << __FUNCTION__ - << " framerate:" << parameters.framerate_fps + RTC_LOG(LS_INFO) << __FUNCTION__ << " framerate:" << parameters.framerate_fps << " bitrate:" << parameters.bitrate.get_sum_bps(); framerate_ = parameters.framerate_fps; target_bitrate_bps_ = parameters.bitrate.get_sum_bps(); @@ -535,46 +497,36 @@ void JetsonH264Encoder::SetRates(const RateControlParameters ¶meters) return; } -void JetsonH264Encoder::SetFramerate(uint32_t framerate) -{ - if (width_ <= 1920 && height_ <= 1080 && framerate > 60) - { +void JetsonH264Encoder::SetFramerate(uint32_t framerate) { + if (width_ <= 1920 && height_ <= 1080 && framerate > 60) { framerate = 60; - } - else if (framerate > 30) - { + } else if (framerate > 30) { framerate = 30; } - if (configured_framerate_ == framerate) - { + if (configured_framerate_ == framerate) { return; } RTC_LOG(LS_INFO) << __FUNCTION__ << " " << framerate << "fps"; - if (encoder_->setFrameRate(framerate, 1) < 0) - { + if (encoder_->setFrameRate(framerate, 1) < 0) { RTC_LOG(LS_ERROR) << "Failed to set bitrate"; return; } configured_framerate_ = framerate; } -void JetsonH264Encoder::SetBitrateBps(uint32_t bitrate_bps) -{ - if (bitrate_bps < 300000 || configured_bitrate_bps_ == bitrate_bps) - { +void JetsonH264Encoder::SetBitrateBps(uint32_t bitrate_bps) { + if (bitrate_bps < 300000 || configured_bitrate_bps_ == bitrate_bps) { return; } RTC_LOG(LS_INFO) << __FUNCTION__ << " " << bitrate_bps << "bit/sec"; - if (encoder_->setBitrate(bitrate_bps) < 0) - { + if (encoder_->setBitrate(bitrate_bps) < 0) { RTC_LOG(LS_ERROR) << "Failed to setBitrate"; return; } configured_bitrate_bps_ = bitrate_bps; } -webrtc::VideoEncoder::EncoderInfo JetsonH264Encoder::GetEncoderInfo() const -{ +webrtc::VideoEncoder::EncoderInfo JetsonH264Encoder::GetEncoderInfo() const { EncoderInfo info; info.supports_native_handle = true; info.implementation_name = "Jetson H264"; @@ -586,63 +538,54 @@ webrtc::VideoEncoder::EncoderInfo JetsonH264Encoder::GetEncoderInfo() const } int32_t JetsonH264Encoder::Encode( - const webrtc::VideoFrame &input_frame, - const std::vector *frame_types) -{ - if (!callback_) - { - RTC_LOG(LS_WARNING) << "InitEncode() has been called, but a callback function " - << "has not been set with RegisterEncodeCompleteCallback()"; + const webrtc::VideoFrame& input_frame, + const std::vector* frame_types) { + if (!callback_) { + RTC_LOG(LS_WARNING) + << "InitEncode() has been called, but a callback function " + << "has not been set with RegisterEncodeCompleteCallback()"; return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } int fd = 0; - rtc::scoped_refptr frame_buffer = input_frame.video_frame_buffer(); - if (frame_buffer->type() == webrtc::VideoFrameBuffer::Type::kNative) - { + rtc::scoped_refptr frame_buffer = + input_frame.video_frame_buffer(); + if (frame_buffer->type() == webrtc::VideoFrameBuffer::Type::kNative) { use_mjpeg_ = true; - NativeBuffer* native_buffer = dynamic_cast(frame_buffer.get()); - int ret = decoder_->decodeToFd(fd, (unsigned char *)native_buffer->Data(), native_buffer->length(), - decode_pixfmt_, raw_width_, raw_height_); - if (ret < 0) - { + NativeBuffer* native_buffer = + dynamic_cast(frame_buffer.get()); + int ret = decoder_->decodeToFd(fd, (unsigned char*)native_buffer->Data(), + native_buffer->length(), decode_pixfmt_, + raw_width_, raw_height_); + if (ret < 0) { RTC_LOG(LS_ERROR) << "Failed to decodeToFd"; return WEBRTC_VIDEO_CODEC_ERROR; } - } - else - { + } else { use_mjpeg_ = false; } - if (frame_buffer->width() != configured_width_ || - frame_buffer->height() != configured_height_) - { + frame_buffer->height() != configured_height_) { RTC_LOG(LS_INFO) << "Encoder reinitialized from " << configured_width_ << "x" << configured_height_ << " to " << frame_buffer->width() << "x" << frame_buffer->height() << " framerate:" << framerate_; JetsonRelease(); - if (JetsonConfigure() != WEBRTC_VIDEO_CODEC_OK) - { + if (JetsonConfigure() != WEBRTC_VIDEO_CODEC_OK) { RTC_LOG(LS_ERROR) << "Failed to JetsonConfigure"; return WEBRTC_VIDEO_CODEC_ERROR; } } bool force_key_frame = false; - if (frame_types != nullptr) - { + if (frame_types != nullptr) { RTC_DCHECK_EQ(frame_types->size(), static_cast(1)); - if ((*frame_types)[0] == webrtc::VideoFrameType::kEmptyFrame) - { + if ((*frame_types)[0] == webrtc::VideoFrameType::kEmptyFrame) { return WEBRTC_VIDEO_CODEC_OK; } - if((*frame_types)[0] == webrtc::VideoFrameType::kVideoFrameKey) - { - if (encoder_->forceIDR() < 0) - { + if ((*frame_types)[0] == webrtc::VideoFrameType::kVideoFrameKey) { + if (encoder_->forceIDR() < 0) { RTC_LOG(LS_ERROR) << "Failed to forceIDR"; } } @@ -652,14 +595,11 @@ int32_t JetsonH264Encoder::Encode( SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); { rtc::CritScope lock(&frame_params_lock_); - frame_params_.push( - absl::make_unique(frame_buffer->width(), - frame_buffer->height(), - input_frame.render_time_ms(), - input_frame.ntp_time_ms(), - input_frame.timestamp_us(), - input_frame.rotation(), - input_frame.color_space())); + frame_params_.push(absl::make_unique( + frame_buffer->width(), frame_buffer->height(), + input_frame.render_time_ms(), input_frame.ntp_time_ms(), + input_frame.timestamp_us(), input_frame.rotation(), + input_frame.color_space())); } struct v4l2_buffer v4l2_buf; @@ -669,15 +609,13 @@ int32_t JetsonH264Encoder::Encode( memset(planes, 0, sizeof(planes)); v4l2_buf.m.planes = planes; - if (use_mjpeg_) - { - NvBuffer *buffer; - if (converter_->output_plane.getNumQueuedBuffers() == converter_->output_plane.getNumBuffers()) - { - if (converter_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) - { - RTC_LOG(LS_ERROR) << "Failed to dqBuffer at converter output_plane"; - return WEBRTC_VIDEO_CODEC_ERROR; + if (use_mjpeg_) { + NvBuffer* buffer; + if (converter_->output_plane.getNumQueuedBuffers() == + converter_->output_plane.getNumBuffers()) { + if (converter_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) { + RTC_LOG(LS_ERROR) << "Failed to dqBuffer at converter output_plane"; + return WEBRTC_VIDEO_CODEC_ERROR; } } @@ -686,41 +624,39 @@ int32_t JetsonH264Encoder::Encode( planes[0].bytesused = 1234; v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; - v4l2_buf.timestamp.tv_sec = input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; - v4l2_buf.timestamp.tv_usec = input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; + v4l2_buf.timestamp.tv_sec = + input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; + v4l2_buf.timestamp.tv_usec = + input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; - if (converter_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) - { + if (converter_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) { RTC_LOG(LS_ERROR) << "Failed to qBuffer at converter output_plane"; return WEBRTC_VIDEO_CODEC_ERROR; } - } - else - { - NvBuffer *buffer; + } else { + NvBuffer* buffer; - RTC_LOG(LS_INFO) << __FUNCTION__ - << " output_plane.getNumBuffers: " << encoder_->output_plane.getNumBuffers() - << " output_plane.getNumQueuedBuffers: " << encoder_->output_plane.getNumQueuedBuffers(); + RTC_LOG(LS_INFO) << __FUNCTION__ << " output_plane.getNumBuffers: " + << encoder_->output_plane.getNumBuffers() + << " output_plane.getNumQueuedBuffers: " + << encoder_->output_plane.getNumQueuedBuffers(); - if (encoder_->output_plane.getNumQueuedBuffers() == encoder_->output_plane.getNumBuffers()) - { - if (encoder_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) - { + if (encoder_->output_plane.getNumQueuedBuffers() == + encoder_->output_plane.getNumBuffers()) { + if (encoder_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10) < 0) { RTC_LOG(LS_ERROR) << "Failed to dqBuffer at encoder output_plane"; return WEBRTC_VIDEO_CODEC_ERROR; } - } - else - { - buffer = encoder_->output_plane.getNthBuffer(encoder_->output_plane.getNumQueuedBuffers()); + } else { + buffer = encoder_->output_plane.getNthBuffer( + encoder_->output_plane.getNumQueuedBuffers()); v4l2_buf.index = encoder_->output_plane.getNumQueuedBuffers(); } - rtc::scoped_refptr i420_buffer = frame_buffer->ToI420(); - for (uint32_t i = 0; i < buffer->n_planes; i++) - { - const uint8_t *source_data; + rtc::scoped_refptr i420_buffer = + frame_buffer->ToI420(); + for (uint32_t i = 0; i < buffer->n_planes; i++) { + const uint8_t* source_data; int source_stride; if (i == 0) { source_data = i420_buffer->DataY(); @@ -731,39 +667,35 @@ int32_t JetsonH264Encoder::Encode( } else if (i == 2) { source_data = i420_buffer->DataV(); source_stride = i420_buffer->StrideV(); - }else { + } else { break; } - NvBuffer::NvBufferPlane &plane = buffer->planes[i]; - std::streamsize bytes_to_read = - plane.fmt.bytesperpixel * plane.fmt.width; - uint8_t *input_data = plane.data; + NvBuffer::NvBufferPlane& plane = buffer->planes[i]; + std::streamsize bytes_to_read = plane.fmt.bytesperpixel * plane.fmt.width; + uint8_t* input_data = plane.data; plane.bytesused = 0; - for (uint32_t j = 0; j < plane.fmt.height; j++) - { - memcpy(input_data, - source_data + (source_stride * j), - bytes_to_read); + for (uint32_t j = 0; j < plane.fmt.height; j++) { + memcpy(input_data, source_data + (source_stride * j), bytes_to_read); input_data += plane.fmt.stride; } plane.bytesused = plane.fmt.stride * plane.fmt.height; } v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; - v4l2_buf.timestamp.tv_sec = input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; - v4l2_buf.timestamp.tv_usec = input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; - - for (int i = 0; i < MAX_PLANES; i++) - { - if (NvBufferMemSyncForDevice(buffer->planes[i].fd, i, (void **)&buffer->planes[i].data) < 0) - { + v4l2_buf.timestamp.tv_sec = + input_frame.timestamp_us() / rtc::kNumMicrosecsPerSec; + v4l2_buf.timestamp.tv_usec = + input_frame.timestamp_us() % rtc::kNumMicrosecsPerSec; + + for (int i = 0; i < MAX_PLANES; i++) { + if (NvBufferMemSyncForDevice(buffer->planes[i].fd, i, + (void**)&buffer->planes[i].data) < 0) { RTC_LOG(LS_ERROR) << "Failed to NvBufferMemSyncForDevice"; return WEBRTC_VIDEO_CODEC_ERROR; } } - if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) - { + if (encoder_->output_plane.qBuffer(v4l2_buf, nullptr) < 0) { RTC_LOG(LS_ERROR) << "Failed to qBuffer at encoder output_plane"; return WEBRTC_VIDEO_CODEC_ERROR; } @@ -772,8 +704,7 @@ int32_t JetsonH264Encoder::Encode( return WEBRTC_VIDEO_CODEC_OK; } -int32_t JetsonH264Encoder::SendFrame(unsigned char *buffer, size_t size) -{ +int32_t JetsonH264Encoder::SendFrame(unsigned char* buffer, size_t size) { encoded_image_.set_buffer(buffer, size); encoded_image_.set_size(size); encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameDelta; @@ -781,59 +712,49 @@ int32_t JetsonH264Encoder::SendFrame(unsigned char *buffer, size_t size) uint8_t zero_count = 0; size_t nal_start_idx = 0; std::vector nals; - for (size_t i = 0; i < size; i++) - { + for (size_t i = 0; i < size; i++) { uint8_t data = buffer[i]; - if ((i != 0) && (i == nal_start_idx)) - { - if ((data & 0x1F) == 0x05) - { + if ((i != 0) && (i == nal_start_idx)) { + if ((data & 0x1F) == 0x05) { encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameKey; } } - if (data == 0x01 && zero_count == 3) - { - if (nal_start_idx != 0) - { + if (data == 0x01 && zero_count == 3) { + if (nal_start_idx != 0) { nals.push_back({nal_start_idx, i - nal_start_idx + 1 - 4}); } nal_start_idx = i + 1; } - if (data == 0x00) - { + if (data == 0x00) { zero_count++; - } - else - { + } else { zero_count = 0; } } - if (nal_start_idx != 0) - { + if (nal_start_idx != 0) { nals.push_back({nal_start_idx, size - nal_start_idx}); } webrtc::RTPFragmentationHeader frag_header; frag_header.VerifyAndAllocateFragmentationHeader(nals.size()); - for (size_t i = 0; i < nals.size(); i++) - { + for (size_t i = 0; i < nals.size(); i++) { frag_header.fragmentationOffset[i] = nals[i].offset; frag_header.fragmentationLength[i] = nals[i].size; } webrtc::CodecSpecificInfo codec_specific; codec_specific.codecType = webrtc::kVideoCodecH264; - codec_specific.codecSpecific.H264.packetization_mode = webrtc::H264PacketizationMode::NonInterleaved; + codec_specific.codecSpecific.H264.packetization_mode = + webrtc::H264PacketizationMode::NonInterleaved; h264_bitstream_parser_.ParseBitstream(buffer, size); h264_bitstream_parser_.GetLastSliceQp(&encoded_image_.qp_); - RTC_LOG(LS_INFO) << __FUNCTION__ - << " last slice qp:" << encoded_image_.qp_; + RTC_LOG(LS_INFO) << __FUNCTION__ << " last slice qp:" << encoded_image_.qp_; - webrtc::EncodedImageCallback::Result result = callback_->OnEncodedImage(encoded_image_, &codec_specific, &frag_header); - if (result.error != webrtc::EncodedImageCallback::Result::OK) - { - RTC_LOG(LS_ERROR) << __FUNCTION__ + webrtc::EncodedImageCallback::Result result = + callback_->OnEncodedImage(encoded_image_, &codec_specific, &frag_header); + if (result.error != webrtc::EncodedImageCallback::Result::OK) { + RTC_LOG(LS_ERROR) << __FUNCTION__ << " OnEncodedImage failed error:" << result.error; return WEBRTC_VIDEO_CODEC_ERROR; } diff --git a/src/hwenc_jetson/jetson_h264_encoder.h b/src/hwenc_jetson/jetson_h264_encoder.h index c528a73c..9a35b06b 100644 --- a/src/hwenc_jetson/jetson_h264_encoder.h +++ b/src/hwenc_jetson/jetson_h264_encoder.h @@ -16,37 +16,37 @@ #include "NvVideoConverter.h" #include "NvVideoEncoder.h" +#include #include #include #include -#include #include "api/video_codecs/video_encoder.h" -#include "rtc_base/critical_section.h" #include "common_video/h264/h264_bitstream_parser.h" #include "common_video/include/bitrate_adjuster.h" #include "modules/video_coding/codecs/h264/include/h264.h" +#include "rtc_base/critical_section.h" class ProcessThread; -class JetsonH264Encoder : public webrtc::VideoEncoder -{ -public: - explicit JetsonH264Encoder(const cricket::VideoCodec &codec); +class JetsonH264Encoder : public webrtc::VideoEncoder { + public: + explicit JetsonH264Encoder(const cricket::VideoCodec& codec); ~JetsonH264Encoder() override; - int32_t InitEncode(const webrtc::VideoCodec *codec_settings, + int32_t InitEncode(const webrtc::VideoCodec* codec_settings, int32_t number_of_cores, size_t max_payload_size) override; int32_t RegisterEncodeCompleteCallback( - webrtc::EncodedImageCallback *callback) override; + webrtc::EncodedImageCallback* callback) override; int32_t Release() override; - int32_t Encode(const webrtc::VideoFrame &frame, - const std::vector *frame_types) override; - void SetRates(const RateControlParameters ¶meters) override; + int32_t Encode( + const webrtc::VideoFrame& frame, + const std::vector* frame_types) override; + void SetRates(const RateControlParameters& parameters) override; webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override; -private: + private: struct FrameParams { FrameParams(int32_t w, int32_t h, @@ -55,7 +55,13 @@ class JetsonH264Encoder : public webrtc::VideoEncoder int64_t ts, webrtc::VideoRotation r, absl::optional c) - : width(w), height(h), render_time_ms(rtms), ntp_time_ms(ntpms), timestamp(ts), rotation(r), color_space(c) {} + : width(w), + height(h), + render_time_ms(rtms), + ntp_time_ms(ntpms), + timestamp(ts), + rotation(r), + color_space(c) {} int32_t width; int32_t height; @@ -68,36 +74,36 @@ class JetsonH264Encoder : public webrtc::VideoEncoder int32_t JetsonConfigure(); void JetsonRelease(); - void SendEOS(NvV4l2Element *element); - static bool ConvertFinishedCallbackFunction(struct v4l2_buffer *v4l2_buf, - NvBuffer *buffer, - NvBuffer *shared_buffer, - void *data); - bool ConvertFinishedCallback(struct v4l2_buffer *v4l2_buf, - NvBuffer *buffer, - NvBuffer *shared_buffer); - static bool EncodeOutputCallbackFunction(struct v4l2_buffer *v4l2_buf, - NvBuffer *buffer, - NvBuffer *shared_buffer, - void *data); - bool EncodeOutputCallback(struct v4l2_buffer *v4l2_buf, - NvBuffer *buffer, - NvBuffer *shared_buffer); - static bool EncodeFinishedCallbackFunction(struct v4l2_buffer *v4l2_buf, - NvBuffer *buffer, - NvBuffer *shared_buffer, - void *data); - bool EncodeFinishedCallback(struct v4l2_buffer *v4l2_buf, - NvBuffer *buffer, - NvBuffer *shared_buffer); + void SendEOS(NvV4l2Element* element); + static bool ConvertFinishedCallbackFunction(struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer, + void* data); + bool ConvertFinishedCallback(struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer); + static bool EncodeOutputCallbackFunction(struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer, + void* data); + bool EncodeOutputCallback(struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer); + static bool EncodeFinishedCallbackFunction(struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer, + void* data); + bool EncodeFinishedCallback(struct v4l2_buffer* v4l2_buf, + NvBuffer* buffer, + NvBuffer* shared_buffer); void SetFramerate(uint32_t framerate); void SetBitrateBps(uint32_t bitrate_bps); - int32_t SendFrame(unsigned char *buffer, size_t size); + int32_t SendFrame(unsigned char* buffer, size_t size); - webrtc::EncodedImageCallback *callback_; - NvJPEGDecoder *decoder_; - NvVideoConverter *converter_; - NvVideoEncoder *encoder_; + webrtc::EncodedImageCallback* callback_; + NvJPEGDecoder* decoder_; + NvVideoConverter* converter_; + NvVideoEncoder* encoder_; webrtc::BitrateAdjuster bitrate_adjuster_; uint32_t framerate_; int32_t configured_framerate_; @@ -120,8 +126,8 @@ class JetsonH264Encoder : public webrtc::VideoEncoder std::mutex enc0_buffer_mtx_; std::condition_variable enc0_buffer_cond_; bool enc0_buffer_ready_ = false; - std::queue *enc0_buffer_queue_; + std::queue* enc0_buffer_queue_; webrtc::EncodedImage encoded_image_; }; -#endif // Jetson_H264_ENCODER_H_ +#endif // Jetson_H264_ENCODER_H_ diff --git a/src/hwenc_mmal/mmal_h264_encoder.cpp b/src/hwenc_mmal/mmal_h264_encoder.cpp index 3f0558dd..3e6a90dd 100644 --- a/src/hwenc_mmal/mmal_h264_encoder.cpp +++ b/src/hwenc_mmal/mmal_h264_encoder.cpp @@ -18,9 +18,9 @@ #include "third_party/libyuv/include/libyuv/convert_from.h" #include "third_party/libyuv/include/libyuv/video_common.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" -#include "common_video/libyuv/include/webrtc_libyuv.h" #include "system_wrappers/include/metrics.h" #include "rtc/native_buffer.h" @@ -28,10 +28,8 @@ #define H264HWENC_HEADER_DEBUG 0 #define ROUND_UP_4(num) (((num) + 3) & ~3) -namespace -{ -struct nal_entry -{ +namespace { +struct nal_entry { size_t offset; size_t size; }; @@ -39,14 +37,15 @@ struct nal_entry const int kLowH264QpThreshold = 34; const int kHighH264QpThreshold = 40; -int I420DataSize(const webrtc::I420BufferInterface &frame_buffer) -{ - return frame_buffer.StrideY() * frame_buffer.height() + (frame_buffer.StrideU() + frame_buffer.StrideV()) * ((frame_buffer.height() + 1) / 2); +int I420DataSize(const webrtc::I420BufferInterface& frame_buffer) { + return frame_buffer.StrideY() * frame_buffer.height() + + (frame_buffer.StrideU() + frame_buffer.StrideV()) * + ((frame_buffer.height() + 1) / 2); } -} // namespace +} // namespace -MMALH264Encoder::MMALH264Encoder(const cricket::VideoCodec &codec) +MMALH264Encoder::MMALH264Encoder(const cricket::VideoCodec& codec) : callback_(nullptr), decoder_(nullptr), resizer_(nullptr), @@ -59,24 +58,18 @@ MMALH264Encoder::MMALH264Encoder(const cricket::VideoCodec &codec) configured_height_(0), use_native_(false), use_decoder_(false), - encoded_buffer_length_(0) -{ -} + encoded_buffer_length_(0) {} -MMALH264Encoder::~MMALH264Encoder() -{ -} +MMALH264Encoder::~MMALH264Encoder() {} -int32_t MMALH264Encoder::InitEncode(const webrtc::VideoCodec *codec_settings, - int32_t number_of_cores, - size_t max_payload_size) -{ +int32_t MMALH264Encoder::InitEncode(const webrtc::VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) { RTC_DCHECK(codec_settings); RTC_DCHECK_EQ(codec_settings->codecType, webrtc::kVideoCodecH264); int32_t release_ret = Release(); - if (release_ret != WEBRTC_VIDEO_CODEC_OK) - { + if (release_ret != WEBRTC_VIDEO_CODEC_OK) { return release_ret; } @@ -94,45 +87,42 @@ int32_t MMALH264Encoder::InitEncode(const webrtc::VideoCodec *codec_settings, encoded_image_._encodedWidth = 0; encoded_image_._encodedHeight = 0; encoded_image_.set_size(0); - encoded_image_.timing_.flags = webrtc::VideoSendTiming::TimingFrameFlags::kInvalid; - encoded_image_.content_type_ = (codec_settings->mode == webrtc::VideoCodecMode::kScreensharing) - ? webrtc::VideoContentType::SCREENSHARE - : webrtc::VideoContentType::UNSPECIFIED; + encoded_image_.timing_.flags = + webrtc::VideoSendTiming::TimingFrameFlags::kInvalid; + encoded_image_.content_type_ = + (codec_settings->mode == webrtc::VideoCodecMode::kScreensharing) + ? webrtc::VideoContentType::SCREENSHARE + : webrtc::VideoContentType::UNSPECIFIED; return WEBRTC_VIDEO_CODEC_OK; } -int32_t MMALH264Encoder::Release() -{ +int32_t MMALH264Encoder::Release() { std::lock_guard lock(mtx_); MMALRelease(); return WEBRTC_VIDEO_CODEC_OK; } -int32_t MMALH264Encoder::MMALConfigure() -{ - if (mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_ENCODER, &encoder_) != MMAL_SUCCESS) - { +int32_t MMALH264Encoder::MMALConfigure() { + if (mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_ENCODER, &encoder_) != + MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to create mmal encoder"; Release(); return WEBRTC_VIDEO_CODEC_ERROR; } MMAL_COMPONENT_T* component_in; - MMAL_ES_FORMAT_T *format_in; - if (use_native_) - { - if (mmal_component_create("vc.ril.resize", &resizer_) != MMAL_SUCCESS) - { + MMAL_ES_FORMAT_T* format_in; + if (use_native_) { + if (mmal_component_create("vc.ril.resize", &resizer_) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to create mmal resizer"; Release(); return WEBRTC_VIDEO_CODEC_ERROR; } - if (use_decoder_) - { - if (mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_DECODER, &decoder_) != MMAL_SUCCESS) - { + if (use_decoder_) { + if (mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_DECODER, + &decoder_) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to create mmal decoder"; Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -144,9 +134,7 @@ int32_t MMALH264Encoder::MMALConfigure() format_in->es->video.width = raw_width_; format_in->es->video.height = raw_height_; component_in = decoder_; - } - else - { + } else { format_in = resizer_->input[0]->format; format_in->type = MMAL_ES_TYPE_VIDEO; format_in->encoding = MMAL_ENCODING_I420; @@ -159,7 +147,7 @@ int32_t MMALH264Encoder::MMALConfigure() component_in = resizer_; } - MMAL_ES_FORMAT_T *format_resize; + MMAL_ES_FORMAT_T* format_resize; format_resize = resizer_->output[0]->format; mmal_format_copy(format_resize, resizer_->input[0]->format); format_resize->es->video.width = VCOS_ALIGN_UP(width_, 32); @@ -170,14 +158,11 @@ int32_t MMALH264Encoder::MMALConfigure() format_resize->es->video.crop.height = height_; format_resize->es->video.frame_rate.num = 30; format_resize->es->video.frame_rate.den = 1; - if (mmal_port_format_commit(resizer_->output[0]) != MMAL_SUCCESS) - { + if (mmal_port_format_commit(resizer_->output[0]) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to commit output port format"; return WEBRTC_VIDEO_CODEC_ERROR; } - } - else - { + } else { format_in = encoder_->input[0]->format; format_in->type = MMAL_ES_TYPE_VIDEO; format_in->encoding = MMAL_ENCODING_I420; @@ -189,39 +174,40 @@ int32_t MMALH264Encoder::MMALConfigure() format_in->es->video.crop.height = height_; component_in = encoder_; } - + format_in->es->video.frame_rate.num = 30; format_in->es->video.frame_rate.den = 1; - if (mmal_port_format_commit(component_in->input[0]) != MMAL_SUCCESS) - { + if (mmal_port_format_commit(component_in->input[0]) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to commit input port format"; return WEBRTC_VIDEO_CODEC_ERROR; } /* Output port configure for H264 */ - MMAL_ES_FORMAT_T *format_out = encoder_->output[0]->format; + MMAL_ES_FORMAT_T* format_out = encoder_->output[0]->format; mmal_format_copy(format_out, format_in); encoder_->output[0]->format->type = MMAL_ES_TYPE_VIDEO; encoder_->output[0]->format->encoding = MMAL_ENCODING_H264; encoder_->output[0]->format->es->video.frame_rate.num = 30; encoder_->output[0]->format->es->video.frame_rate.den = 1; - encoder_->output[0]->format->bitrate = bitrate_adjuster_.GetAdjustedBitrateBps(); + encoder_->output[0]->format->bitrate = + bitrate_adjuster_.GetAdjustedBitrateBps(); - if (mmal_port_format_commit(encoder_->output[0]) != MMAL_SUCCESS) - { + if (mmal_port_format_commit(encoder_->output[0]) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to commit output port format"; return WEBRTC_VIDEO_CODEC_ERROR; } - if (mmal_port_parameter_set_boolean(component_in->input[0], MMAL_PARAMETER_ZERO_COPY, MMAL_TRUE) != MMAL_SUCCESS) - { + if (mmal_port_parameter_set_boolean(component_in->input[0], + MMAL_PARAMETER_ZERO_COPY, + MMAL_TRUE) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to set input zero copy"; return WEBRTC_VIDEO_CODEC_ERROR; } - if (mmal_port_parameter_set_boolean(encoder_->output[0], MMAL_PARAMETER_ZERO_COPY, MMAL_TRUE) != MMAL_SUCCESS) - { + if (mmal_port_parameter_set_boolean(encoder_->output[0], + MMAL_PARAMETER_ZERO_COPY, + MMAL_TRUE) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to set output zero copy"; return WEBRTC_VIDEO_CODEC_ERROR; } @@ -233,114 +219,117 @@ int32_t MMALH264Encoder::MMALConfigure() video_profile.profile[0].profile = MMAL_VIDEO_PROFILE_H264_HIGH; video_profile.profile[0].level = MMAL_VIDEO_LEVEL_H264_42; - if (mmal_port_parameter_set(encoder_->output[0], &video_profile.hdr) != MMAL_SUCCESS) - { + if (mmal_port_parameter_set(encoder_->output[0], &video_profile.hdr) != + MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to set H264 profile"; return WEBRTC_VIDEO_CODEC_ERROR; } - if (mmal_port_parameter_set_uint32(encoder_->output[0], MMAL_PARAMETER_INTRAPERIOD, 500) != MMAL_SUCCESS) - { + if (mmal_port_parameter_set_uint32(encoder_->output[0], + MMAL_PARAMETER_INTRAPERIOD, + 500) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to set intra period"; return WEBRTC_VIDEO_CODEC_ERROR; } - if (mmal_port_parameter_set_boolean(encoder_->output[0], MMAL_PARAMETER_VIDEO_ENCODE_INLINE_HEADER, MMAL_TRUE) != MMAL_SUCCESS) - { + if (mmal_port_parameter_set_boolean(encoder_->output[0], + MMAL_PARAMETER_VIDEO_ENCODE_INLINE_HEADER, + MMAL_TRUE) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to set enable inline header"; return WEBRTC_VIDEO_CODEC_ERROR; } - component_in->input[0]->buffer_size = component_in->input[0]->buffer_size_recommended; - if (component_in->input[0]->buffer_size < component_in->input[0]->buffer_size_min) - component_in->input[0]->buffer_size = component_in->input[0]->buffer_size_min; + component_in->input[0]->buffer_size = + component_in->input[0]->buffer_size_recommended; + if (component_in->input[0]->buffer_size < + component_in->input[0]->buffer_size_min) + component_in->input[0]->buffer_size = + component_in->input[0]->buffer_size_min; if (use_decoder_) - component_in->input[0]->buffer_size = component_in->input[0]->buffer_size_recommended * 8; + component_in->input[0]->buffer_size = + component_in->input[0]->buffer_size_recommended * 8; component_in->input[0]->buffer_num = 1; - component_in->input[0]->userdata = (MMAL_PORT_USERDATA_T *)this; + component_in->input[0]->userdata = (MMAL_PORT_USERDATA_T*)this; - if (mmal_port_enable(component_in->input[0], MMALInputCallbackFunction) != MMAL_SUCCESS) - { + if (mmal_port_enable(component_in->input[0], MMALInputCallbackFunction) != + MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to enable input port"; return WEBRTC_VIDEO_CODEC_ERROR; } - pool_in_ = mmal_port_pool_create(component_in->input[0], component_in->input[0]->buffer_num, component_in->input[0]->buffer_size); + pool_in_ = mmal_port_pool_create(component_in->input[0], + component_in->input[0]->buffer_num, + component_in->input[0]->buffer_size); - if (use_native_) - { - if (use_decoder_) - { - if (mmal_connection_create(&conn1_, decoder_->output[0], resizer_->input[0], MMAL_CONNECTION_FLAG_TUNNELLING | MMAL_CONNECTION_FLAG_ALLOCATION_ON_INPUT) != MMAL_SUCCESS) - { + if (use_native_) { + if (use_decoder_) { + if (mmal_connection_create( + &conn1_, decoder_->output[0], resizer_->input[0], + MMAL_CONNECTION_FLAG_TUNNELLING | + MMAL_CONNECTION_FLAG_ALLOCATION_ON_INPUT) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to connect decoder to resizer"; return WEBRTC_VIDEO_CODEC_ERROR; } } - if (mmal_connection_create(&conn2_, resizer_->output[0], encoder_->input[0], MMAL_CONNECTION_FLAG_TUNNELLING) != MMAL_SUCCESS) - { + if (mmal_connection_create(&conn2_, resizer_->output[0], encoder_->input[0], + MMAL_CONNECTION_FLAG_TUNNELLING) != + MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to connect resizer to encoder"; return WEBRTC_VIDEO_CODEC_ERROR; } - if (mmal_component_enable(resizer_) != MMAL_SUCCESS) - { + if (mmal_component_enable(resizer_) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to enable component"; return WEBRTC_VIDEO_CODEC_ERROR; } - if (use_decoder_) - { - if (mmal_component_enable(decoder_) != MMAL_SUCCESS) - { + if (use_decoder_) { + if (mmal_component_enable(decoder_) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to enable component"; return WEBRTC_VIDEO_CODEC_ERROR; } - if (mmal_connection_enable(conn1_) != MMAL_SUCCESS) - { + if (mmal_connection_enable(conn1_) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to enable connection decoder to resizer"; return WEBRTC_VIDEO_CODEC_ERROR; } } - if (mmal_connection_enable(conn2_) != MMAL_SUCCESS) - { + if (mmal_connection_enable(conn2_) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to enable connection resizer to encoder"; return WEBRTC_VIDEO_CODEC_ERROR; } } - encoder_->output[0]->buffer_size = encoder_->output[0]->buffer_size_recommended * 4; + encoder_->output[0]->buffer_size = + encoder_->output[0]->buffer_size_recommended * 4; if (encoder_->output[0]->buffer_size < encoder_->output[0]->buffer_size_min) encoder_->output[0]->buffer_size = encoder_->output[0]->buffer_size_min; encoder_->output[0]->buffer_num = 4; - encoder_->output[0]->userdata = (MMAL_PORT_USERDATA_T *)this; + encoder_->output[0]->userdata = (MMAL_PORT_USERDATA_T*)this; encoded_image_buffer_.reset(new uint8_t[encoder_->output[0]->buffer_size]); - if (mmal_port_enable(encoder_->output[0], MMALOutputCallbackFunction) != MMAL_SUCCESS) - { + if (mmal_port_enable(encoder_->output[0], MMALOutputCallbackFunction) != + MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to enable output port"; return WEBRTC_VIDEO_CODEC_ERROR; } - pool_out_ = mmal_port_pool_create(encoder_->output[0], encoder_->output[0]->buffer_num, encoder_->output[0]->buffer_size); + pool_out_ = mmal_port_pool_create(encoder_->output[0], + encoder_->output[0]->buffer_num, + encoder_->output[0]->buffer_size); - if (mmal_component_enable(encoder_) != MMAL_SUCCESS) - { + if (mmal_component_enable(encoder_) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to enable component"; return WEBRTC_VIDEO_CODEC_ERROR; } configured_width_ = width_; configured_height_ = height_; - if (use_native_) - { + if (use_native_) { stride_width_ = VCOS_ALIGN_UP(raw_width_, 32); stride_height_ = VCOS_ALIGN_UP(raw_height_, 16); - } - else - { + } else { stride_width_ = VCOS_ALIGN_UP(width_, 32); stride_height_ = VCOS_ALIGN_UP(height_, 16); } @@ -348,83 +337,72 @@ int32_t MMALH264Encoder::MMALConfigure() return WEBRTC_VIDEO_CODEC_OK; } -void MMALH264Encoder::MMALRelease() -{ +void MMALH264Encoder::MMALRelease() { if (!encoder_) return; - if (resizer_) - { + if (resizer_) { mmal_component_disable(resizer_); - if (decoder_) - { + if (decoder_) { mmal_component_disable(decoder_); mmal_port_disable(decoder_->input[0]); mmal_port_pool_destroy(decoder_->input[0], pool_in_); - } - else - { + } else { mmal_port_disable(resizer_->input[0]); mmal_port_pool_destroy(resizer_->input[0], pool_in_); } } - if (encoder_) - { + if (encoder_) { mmal_component_disable(encoder_); - if (!resizer_) - { + if (!resizer_) { mmal_port_disable(encoder_->input[0]); mmal_port_pool_destroy(encoder_->input[0], pool_in_); } mmal_port_disable(encoder_->output[0]); mmal_port_pool_destroy(encoder_->output[0], pool_out_); } - if (conn1_) - { + if (conn1_) { mmal_connection_destroy(conn1_); conn1_ = nullptr; } - if (conn2_) - { + if (conn2_) { mmal_connection_destroy(conn2_); conn2_ = nullptr; } - if (encoder_) - { + if (encoder_) { mmal_component_destroy(encoder_); encoder_ = nullptr; } - if (resizer_) - { + if (resizer_) { mmal_component_destroy(resizer_); resizer_ = nullptr; } - if (decoder_) - { + if (decoder_) { mmal_component_destroy(decoder_); decoder_ = nullptr; } - while (!frame_params_.empty()) frame_params_.pop(); + while (!frame_params_.empty()) + frame_params_.pop(); encoded_image_buffer_.reset(); encoded_buffer_length_ = 0; } -void MMALH264Encoder::MMALInputCallbackFunction(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer) -{ - ((MMALH264Encoder *)port->userdata)->MMALInputCallback(port, buffer); +void MMALH264Encoder::MMALInputCallbackFunction(MMAL_PORT_T* port, + MMAL_BUFFER_HEADER_T* buffer) { + ((MMALH264Encoder*)port->userdata)->MMALInputCallback(port, buffer); } -void MMALH264Encoder::MMALInputCallback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer) -{ +void MMALH264Encoder::MMALInputCallback(MMAL_PORT_T* port, + MMAL_BUFFER_HEADER_T* buffer) { mmal_buffer_header_release(buffer); } -void MMALH264Encoder::MMALOutputCallbackFunction(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer) -{ - ((MMALH264Encoder *)port->userdata)->MMALOutputCallback(port, buffer); +void MMALH264Encoder::MMALOutputCallbackFunction(MMAL_PORT_T* port, + MMAL_BUFFER_HEADER_T* buffer) { + ((MMALH264Encoder*)port->userdata)->MMALOutputCallback(port, buffer); } -void MMALH264Encoder::MMALOutputCallback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer) -{ +void MMALH264Encoder::MMALOutputCallback(MMAL_PORT_T* port, + MMAL_BUFFER_HEADER_T* buffer) { if (buffer->length == 0) { mmal_buffer_header_release(buffer); return; @@ -438,8 +416,7 @@ void MMALH264Encoder::MMALOutputCallback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T return; } - RTC_LOG(LS_INFO) << "pts:" << buffer->pts - << " flags:" << buffer->flags + RTC_LOG(LS_INFO) << "pts:" << buffer->pts << " flags:" << buffer->flags << " planes:" << buffer->type->video.planes << " length:" << buffer->length; @@ -447,27 +424,25 @@ void MMALH264Encoder::MMALOutputCallback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T { rtc::CritScope lock(&frame_params_lock_); do { - if (frame_params_.empty()) - { - RTC_LOG(LS_WARNING) << __FUNCTION__ - << "Frame parameter is not found. SkipFrame pts:" - << buffer->pts; + if (frame_params_.empty()) { + RTC_LOG(LS_WARNING) + << __FUNCTION__ + << "Frame parameter is not found. SkipFrame pts:" << buffer->pts; mmal_buffer_header_release(buffer); return; } params = std::move(frame_params_.front()); frame_params_.pop(); } while (params->timestamp < buffer->pts); - if (params->timestamp != buffer->pts) - { - RTC_LOG(LS_WARNING) << __FUNCTION__ + if (params->timestamp != buffer->pts) { + RTC_LOG(LS_WARNING) << __FUNCTION__ << "Frame parameter is not found. SkipFrame pts:" << buffer->pts; mmal_buffer_header_release(buffer); return; } } - + encoded_image_._encodedWidth = params->width; encoded_image_._encodedHeight = params->height; encoded_image_.capture_time_ms_ = params->render_time_ms; @@ -476,59 +451,54 @@ void MMALH264Encoder::MMALOutputCallback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T encoded_image_.rotation_ = params->rotation; encoded_image_.SetColorSpace(params->color_space); - if (encoded_buffer_length_ == 0) - { + if (encoded_buffer_length_ == 0) { SendFrame(buffer->data, buffer->length); - } - else - { - memcpy(encoded_image_buffer_.get() + encoded_buffer_length_, buffer->data, buffer->length); + } else { + memcpy(encoded_image_buffer_.get() + encoded_buffer_length_, buffer->data, + buffer->length); encoded_buffer_length_ += buffer->length; SendFrame(encoded_image_buffer_.get(), encoded_buffer_length_); encoded_buffer_length_ = 0; } - + mmal_buffer_header_release(buffer); } int32_t MMALH264Encoder::RegisterEncodeCompleteCallback( - webrtc::EncodedImageCallback *callback) -{ + webrtc::EncodedImageCallback* callback) { std::lock_guard lock(mtx_); callback_ = callback; return WEBRTC_VIDEO_CODEC_OK; } -void MMALH264Encoder::SetRates(const RateControlParameters ¶meters) -{ +void MMALH264Encoder::SetRates(const RateControlParameters& parameters) { if (encoder_ == nullptr) return; if (parameters.bitrate.get_sum_bps() <= 0 || parameters.framerate_fps <= 0) return; - RTC_LOG(LS_INFO) << __FUNCTION__ << " bitrate:" << parameters.bitrate.get_sum_bps(); + RTC_LOG(LS_INFO) << __FUNCTION__ + << " bitrate:" << parameters.bitrate.get_sum_bps(); target_bitrate_bps_ = parameters.bitrate.get_sum_bps(); bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); return; } -void MMALH264Encoder::SetBitrateBps(uint32_t bitrate_bps) -{ - if (bitrate_bps < 300000 || configured_bitrate_bps_ == bitrate_bps) - { +void MMALH264Encoder::SetBitrateBps(uint32_t bitrate_bps) { + if (bitrate_bps < 300000 || configured_bitrate_bps_ == bitrate_bps) { return; } RTC_LOG(LS_INFO) << "SetBitrateBps " << bitrate_bps << "bit/sec"; - if (mmal_port_parameter_set_uint32(encoder_->output[0], MMAL_PARAMETER_VIDEO_BIT_RATE, bitrate_bps) != MMAL_SUCCESS) - { + if (mmal_port_parameter_set_uint32(encoder_->output[0], + MMAL_PARAMETER_VIDEO_BIT_RATE, + bitrate_bps) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to set bitrate"; return; } configured_bitrate_bps_ = bitrate_bps; } -webrtc::VideoEncoder::EncoderInfo MMALH264Encoder::GetEncoderInfo() const -{ +webrtc::VideoEncoder::EncoderInfo MMALH264Encoder::GetEncoderInfo() const { EncoderInfo info; info.supports_native_handle = true; info.implementation_name = "MMAL H264"; @@ -540,60 +510,55 @@ webrtc::VideoEncoder::EncoderInfo MMALH264Encoder::GetEncoderInfo() const } int32_t MMALH264Encoder::Encode( - const webrtc::VideoFrame &input_frame, - const std::vector *frame_types) -{ + const webrtc::VideoFrame& input_frame, + const std::vector* frame_types) { std::lock_guard lock(mtx_); - if (!callback_) - { - RTC_LOG(LS_WARNING) << "InitEncode() has been called, but a callback function " - << "has not been set with RegisterEncodeCompleteCallback()"; + if (!callback_) { + RTC_LOG(LS_WARNING) + << "InitEncode() has been called, but a callback function " + << "has not been set with RegisterEncodeCompleteCallback()"; return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - rtc::scoped_refptr frame_buffer = input_frame.video_frame_buffer(); + rtc::scoped_refptr frame_buffer = + input_frame.video_frame_buffer(); if (frame_buffer->width() != configured_width_ || - frame_buffer->height() != configured_height_) - { + frame_buffer->height() != configured_height_) { RTC_LOG(LS_INFO) << "Encoder reinitialized from " << configured_width_ << "x" << configured_height_ << " to " << frame_buffer->width() << "x" << frame_buffer->height(); MMALRelease(); - if (frame_buffer->type() == webrtc::VideoFrameBuffer::Type::kNative) - { - NativeBuffer* native_buffer = dynamic_cast(frame_buffer.get()); + if (frame_buffer->type() == webrtc::VideoFrameBuffer::Type::kNative) { + NativeBuffer* native_buffer = + dynamic_cast(frame_buffer.get()); raw_width_ = native_buffer->raw_width(); raw_height_ = native_buffer->raw_height(); use_native_ = true; use_decoder_ = native_buffer->VideoType() == webrtc::VideoType::kMJPEG; - } - else - { + } else { use_native_ = false; } - if (MMALConfigure() != WEBRTC_VIDEO_CODEC_OK) - { + if (MMALConfigure() != WEBRTC_VIDEO_CODEC_OK) { RTC_LOG(LS_ERROR) << "Failed to MMALConfigure"; return WEBRTC_VIDEO_CODEC_ERROR; } } bool force_key_frame = false; - if (frame_types != nullptr) - { + if (frame_types != nullptr) { RTC_DCHECK_EQ(frame_types->size(), static_cast(1)); - if ((*frame_types)[0] == webrtc::VideoFrameType::kEmptyFrame) - { + if ((*frame_types)[0] == webrtc::VideoFrameType::kEmptyFrame) { return WEBRTC_VIDEO_CODEC_OK; } - force_key_frame = (*frame_types)[0] == webrtc::VideoFrameType::kVideoFrameKey; + force_key_frame = + (*frame_types)[0] == webrtc::VideoFrameType::kVideoFrameKey; } - if (force_key_frame) - { - if (mmal_port_parameter_set_boolean(encoder_->output[0], MMAL_PARAMETER_VIDEO_REQUEST_I_FRAME, MMAL_TRUE) != MMAL_SUCCESS) - { + if (force_key_frame) { + if (mmal_port_parameter_set_boolean(encoder_->output[0], + MMAL_PARAMETER_VIDEO_REQUEST_I_FRAME, + MMAL_TRUE) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to request I frame"; } } @@ -601,77 +566,64 @@ int32_t MMALH264Encoder::Encode( SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); { rtc::CritScope lock(&frame_params_lock_); - frame_params_.push( - absl::make_unique(frame_buffer->width(), - frame_buffer->height(), - input_frame.render_time_ms(), - input_frame.ntp_time_ms(), - input_frame.timestamp(), - input_frame.rotation(), - input_frame.color_space())); - } - - MMAL_BUFFER_HEADER_T *buffer; - while ((buffer = mmal_queue_get(pool_out_->queue)) != nullptr) - { - if (mmal_port_send_buffer(encoder_->output[0], buffer) != MMAL_SUCCESS) - { + frame_params_.push(absl::make_unique( + frame_buffer->width(), frame_buffer->height(), + input_frame.render_time_ms(), input_frame.ntp_time_ms(), + input_frame.timestamp(), input_frame.rotation(), + input_frame.color_space())); + } + + MMAL_BUFFER_HEADER_T* buffer; + while ((buffer = mmal_queue_get(pool_out_->queue)) != nullptr) { + if (mmal_port_send_buffer(encoder_->output[0], buffer) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to send output buffer"; return WEBRTC_VIDEO_CODEC_ERROR; } } - while ((buffer = mmal_queue_get(pool_in_->queue)) != nullptr) - { + while ((buffer = mmal_queue_get(pool_in_->queue)) != nullptr) { buffer->pts = buffer->dts = input_frame.timestamp(); buffer->offset = 0; buffer->flags = MMAL_BUFFER_HEADER_FLAG_FRAME; - if (use_decoder_) - { - NativeBuffer* native_buffer = dynamic_cast(frame_buffer.get()); - memcpy(buffer->data, - native_buffer->Data(), - native_buffer->length()); + if (use_decoder_) { + NativeBuffer* native_buffer = + dynamic_cast(frame_buffer.get()); + memcpy(buffer->data, native_buffer->Data(), native_buffer->length()); buffer->length = buffer->alloc_size = native_buffer->length(); - if (mmal_port_send_buffer(decoder_->input[0], buffer) != MMAL_SUCCESS) - { + if (mmal_port_send_buffer(decoder_->input[0], buffer) != MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to send input native buffer"; return WEBRTC_VIDEO_CODEC_ERROR; } - } - else - { + } else { MMAL_COMPONENT_T* component_in; size_t width, height, stride_y, stride_u, stride_v; uint8_t *data_y, *data_u, *data_v; - if (use_native_) - { - NativeBuffer* native_buffer = dynamic_cast(frame_buffer.get()); + if (use_native_) { + NativeBuffer* native_buffer = + dynamic_cast(frame_buffer.get()); width = native_buffer->raw_width(); height = native_buffer->raw_height(); stride_y = width; stride_u = stride_v = width / 2; - data_y = (uint8_t *)native_buffer->Data(); + data_y = (uint8_t*)native_buffer->Data(); data_u = data_y + (width * height); data_v = data_u + (stride_u * (height / 2)); component_in = resizer_; - } - else - { - rtc::scoped_refptr i420_buffer = frame_buffer->ToI420(); + } else { + rtc::scoped_refptr i420_buffer = + frame_buffer->ToI420(); width = i420_buffer->width(); height = i420_buffer->height(); stride_y = i420_buffer->StrideY(); stride_u = i420_buffer->StrideU(); stride_v = i420_buffer->StrideV(); - data_y = (uint8_t *)i420_buffer->DataY(); - data_u = (uint8_t *)i420_buffer->DataU(); - data_v = (uint8_t *)i420_buffer->DataV(); + data_y = (uint8_t*)i420_buffer->DataY(); + data_u = (uint8_t*)i420_buffer->DataU(); + data_v = (uint8_t*)i420_buffer->DataV(); component_in = encoder_; } size_t offset = 0; - for (size_t i = 0; i < height; i++) - { + for (size_t i = 0; i < height; i++) { memcpy(buffer->data + offset, data_y + (stride_y * i), stride_y); offset += stride_width_; } @@ -679,17 +631,17 @@ int32_t MMALH264Encoder::Encode( size_t offset_y = stride_width_ * stride_height_; size_t width_uv = stride_width_ / 2; size_t offset_v = (stride_height_ / 2) * width_uv; - for (size_t i = 0; i < ((height + 1) / 2); i++) - { - memcpy(buffer->data + offset_y + offset, - data_u + (stride_u * i), width_uv); + for (size_t i = 0; i < ((height + 1) / 2); i++) { + memcpy(buffer->data + offset_y + offset, data_u + (stride_u * i), + width_uv); memcpy(buffer->data + offset_y + offset_v + offset, data_v + (stride_v * i), width_uv); offset += width_uv; } - buffer->length = buffer->alloc_size = stride_width_ * stride_height_ * 3 / 2; - if (mmal_port_send_buffer(component_in->input[0], buffer) != MMAL_SUCCESS) - { + buffer->length = buffer->alloc_size = + stride_width_ * stride_height_ * 3 / 2; + if (mmal_port_send_buffer(component_in->input[0], buffer) != + MMAL_SUCCESS) { RTC_LOG(LS_ERROR) << "Failed to send input i420 buffer"; return WEBRTC_VIDEO_CODEC_ERROR; } @@ -699,8 +651,7 @@ int32_t MMALH264Encoder::Encode( return WEBRTC_VIDEO_CODEC_OK; } -int32_t MMALH264Encoder::SendFrame(unsigned char *buffer, size_t size) -{ +int32_t MMALH264Encoder::SendFrame(unsigned char* buffer, size_t size) { encoded_image_.set_buffer(buffer, size); encoded_image_.set_size(size); encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameDelta; @@ -708,59 +659,49 @@ int32_t MMALH264Encoder::SendFrame(unsigned char *buffer, size_t size) uint8_t zero_count = 0; size_t nal_start_idx = 0; std::vector nals; - for (size_t i = 0; i < size; i++) - { + for (size_t i = 0; i < size; i++) { uint8_t data = buffer[i]; - if ((i != 0) && (i == nal_start_idx)) - { - if ((data & 0x1F) == 0x05) - { + if ((i != 0) && (i == nal_start_idx)) { + if ((data & 0x1F) == 0x05) { encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameKey; } } - if (data == 0x01 && zero_count == 3) - { - if (nal_start_idx != 0) - { + if (data == 0x01 && zero_count == 3) { + if (nal_start_idx != 0) { nals.push_back({nal_start_idx, i - nal_start_idx + 1 - 4}); } nal_start_idx = i + 1; } - if (data == 0x00) - { + if (data == 0x00) { zero_count++; - } - else - { + } else { zero_count = 0; } } - if (nal_start_idx != 0) - { + if (nal_start_idx != 0) { nals.push_back({nal_start_idx, size - nal_start_idx}); } webrtc::RTPFragmentationHeader frag_header; frag_header.VerifyAndAllocateFragmentationHeader(nals.size()); - for (size_t i = 0; i < nals.size(); i++) - { + for (size_t i = 0; i < nals.size(); i++) { frag_header.fragmentationOffset[i] = nals[i].offset; frag_header.fragmentationLength[i] = nals[i].size; } webrtc::CodecSpecificInfo codec_specific; codec_specific.codecType = webrtc::kVideoCodecH264; - codec_specific.codecSpecific.H264.packetization_mode = webrtc::H264PacketizationMode::NonInterleaved; + codec_specific.codecSpecific.H264.packetization_mode = + webrtc::H264PacketizationMode::NonInterleaved; h264_bitstream_parser_.ParseBitstream(buffer, size); h264_bitstream_parser_.GetLastSliceQp(&encoded_image_.qp_); - RTC_LOG(LS_INFO) << __FUNCTION__ - << " last slice qp:" << encoded_image_.qp_; + RTC_LOG(LS_INFO) << __FUNCTION__ << " last slice qp:" << encoded_image_.qp_; - webrtc::EncodedImageCallback::Result result = callback_->OnEncodedImage(encoded_image_, &codec_specific, &frag_header); - if (result.error != webrtc::EncodedImageCallback::Result::OK) - { - RTC_LOG(LS_ERROR) << __FUNCTION__ + webrtc::EncodedImageCallback::Result result = + callback_->OnEncodedImage(encoded_image_, &codec_specific, &frag_header); + if (result.error != webrtc::EncodedImageCallback::Result::OK) { + RTC_LOG(LS_ERROR) << __FUNCTION__ << " OnEncodedImage failed error:" << result.error; return WEBRTC_VIDEO_CODEC_ERROR; } diff --git a/src/hwenc_mmal/mmal_h264_encoder.h b/src/hwenc_mmal/mmal_h264_encoder.h index 6ccc4ba7..3bc89197 100644 --- a/src/hwenc_mmal/mmal_h264_encoder.h +++ b/src/hwenc_mmal/mmal_h264_encoder.h @@ -12,15 +12,14 @@ #ifndef MMAL_H264_ENCODER_H_ #define MMAL_H264_ENCODER_H_ -extern "C" -{ +extern "C" { #include "bcm_host.h" #include "interface/mmal/mmal.h" #include "interface/mmal/mmal_format.h" #include "interface/mmal/util/mmal_connection.h" #include "interface/mmal/util/mmal_default_components.h" -#include "interface/mmal/util/mmal_util_params.h" #include "interface/mmal/util/mmal_util.h" +#include "interface/mmal/util/mmal_util_params.h" #include "interface/vcos/vcos.h" } @@ -30,40 +29,46 @@ extern "C" #include #include "api/video_codecs/video_encoder.h" -#include "rtc_base/critical_section.h" #include "common_video/h264/h264_bitstream_parser.h" #include "common_video/include/bitrate_adjuster.h" #include "modules/video_coding/codecs/h264/include/h264.h" +#include "rtc_base/critical_section.h" class ProcessThread; -class MMALH264Encoder : public webrtc::VideoEncoder -{ -public: - explicit MMALH264Encoder(const cricket::VideoCodec &codec); +class MMALH264Encoder : public webrtc::VideoEncoder { + public: + explicit MMALH264Encoder(const cricket::VideoCodec& codec); ~MMALH264Encoder() override; - int32_t InitEncode(const webrtc::VideoCodec *codec_settings, + int32_t InitEncode(const webrtc::VideoCodec* codec_settings, int32_t number_of_cores, size_t max_payload_size) override; int32_t RegisterEncodeCompleteCallback( - webrtc::EncodedImageCallback *callback) override; + webrtc::EncodedImageCallback* callback) override; int32_t Release() override; - int32_t Encode(const webrtc::VideoFrame &frame, - const std::vector *frame_types) override; - void SetRates(const RateControlParameters ¶meters) override; + int32_t Encode( + const webrtc::VideoFrame& frame, + const std::vector* frame_types) override; + void SetRates(const RateControlParameters& parameters) override; webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override; -private: + private: struct FrameParams { FrameParams(int32_t w, - int32_t h, - int64_t rtms, - int64_t ntpms, - int64_t ts, - webrtc::VideoRotation r, - absl::optional c) - : width(w), height(h), render_time_ms(rtms), ntp_time_ms(ntpms), timestamp(ts), rotation(r), color_space(c) {} + int32_t h, + int64_t rtms, + int64_t ntpms, + int64_t ts, + webrtc::VideoRotation r, + absl::optional c) + : width(w), + height(h), + render_time_ms(rtms), + ntp_time_ms(ntpms), + timestamp(ts), + rotation(r), + color_space(c) {} int32_t width; int32_t height; @@ -76,22 +81,24 @@ class MMALH264Encoder : public webrtc::VideoEncoder int32_t MMALConfigure(); void MMALRelease(); - static void MMALInputCallbackFunction(MMAL_PORT_T* port, MMAL_BUFFER_HEADER_T* buffer); + static void MMALInputCallbackFunction(MMAL_PORT_T* port, + MMAL_BUFFER_HEADER_T* buffer); void MMALInputCallback(MMAL_PORT_T* port, MMAL_BUFFER_HEADER_T* buffer); - static void MMALOutputCallbackFunction(MMAL_PORT_T* port, MMAL_BUFFER_HEADER_T* buffer); + static void MMALOutputCallbackFunction(MMAL_PORT_T* port, + MMAL_BUFFER_HEADER_T* buffer); void MMALOutputCallback(MMAL_PORT_T* port, MMAL_BUFFER_HEADER_T* buffer); void SetBitrateBps(uint32_t bitrate_bps); - int32_t SendFrame(unsigned char *buffer, size_t size); + int32_t SendFrame(unsigned char* buffer, size_t size); std::mutex mtx_; - webrtc::EncodedImageCallback *callback_; + webrtc::EncodedImageCallback* callback_; MMAL_COMPONENT_T* decoder_; MMAL_COMPONENT_T* resizer_; MMAL_COMPONENT_T* encoder_; - MMAL_CONNECTION_T *conn1_; - MMAL_CONNECTION_T *conn2_; - MMAL_POOL_T *pool_in_; - MMAL_POOL_T *pool_out_; + MMAL_CONNECTION_T* conn1_; + MMAL_CONNECTION_T* conn2_; + MMAL_POOL_T* pool_in_; + MMAL_POOL_T* pool_out_; webrtc::BitrateAdjuster bitrate_adjuster_; uint32_t target_bitrate_bps_; uint32_t configured_bitrate_bps_; @@ -108,7 +115,6 @@ class MMALH264Encoder : public webrtc::VideoEncoder webrtc::H264BitstreamParser h264_bitstream_parser_; - rtc::CriticalSection frame_params_lock_; std::queue> frame_params_; webrtc::EncodedImage encoded_image_; @@ -116,4 +122,4 @@ class MMALH264Encoder : public webrtc::VideoEncoder size_t encoded_buffer_length_; }; -#endif // MMAL_H264_ENCODER_H_ +#endif // MMAL_H264_ENCODER_H_ diff --git a/src/mac_helper/mac_capturer.h b/src/mac_helper/mac_capturer.h index 9f5537c9..ae4a7290 100644 --- a/src/mac_helper/mac_capturer.h +++ b/src/mac_helper/mac_capturer.h @@ -28,9 +28,9 @@ class MacCapturer : public ScalableVideoTrackSource, public rtc::VideoSinkInterface { public: static rtc::scoped_refptr Create(size_t width, - size_t height, - size_t target_fps, - size_t capture_device_index); + size_t height, + size_t target_fps, + size_t capture_device_index); MacCapturer(size_t width, size_t height, size_t target_fps, diff --git a/src/mac_helper/mac_capturer.mm b/src/mac_helper/mac_capturer.mm index cd65e5ce..040b9347 100644 --- a/src/mac_helper/mac_capturer.mm +++ b/src/mac_helper/mac_capturer.mm @@ -16,13 +16,14 @@ #import "sdk/objc/native/src/objc_frame_buffer.h" @interface RTCVideoSourceAdapter : NSObject -@property(nonatomic) MacCapturer *capturer; +@property(nonatomic) MacCapturer* capturer; @end @implementation RTCVideoSourceAdapter @synthesize capturer = _capturer; -- (void)capturer:(RTCVideoCapturer *)capturer didCaptureVideoFrame:(RTCVideoFrame *)frame { +- (void)capturer:(RTCVideoCapturer*)capturer + didCaptureVideoFrame:(RTCVideoFrame*)frame { const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec; rtc::scoped_refptr buffer = new rtc::RefCountedObject(frame.buffer); @@ -37,15 +38,18 @@ - (void)capturer:(RTCVideoCapturer *)capturer didCaptureVideoFrame:(RTCVideoFram namespace { -AVCaptureDeviceFormat *SelectClosestFormat(AVCaptureDevice *device, size_t width, size_t height) { - NSArray *formats = +AVCaptureDeviceFormat* SelectClosestFormat(AVCaptureDevice* device, + size_t width, + size_t height) { + NSArray* formats = [RTCCameraVideoCapturer supportedFormatsForDevice:device]; - AVCaptureDeviceFormat *selectedFormat = nil; + AVCaptureDeviceFormat* selectedFormat = nil; int currentDiff = INT_MAX; - for (AVCaptureDeviceFormat *format in formats) { - CMVideoDimensions dimension = CMVideoFormatDescriptionGetDimensions(format.formatDescription); - int diff = - std::abs((int64_t)width - dimension.width) + std::abs((int64_t)height - dimension.height); + for (AVCaptureDeviceFormat* format in formats) { + CMVideoDimensions dimension = + CMVideoFormatDescriptionGetDimensions(format.formatDescription); + int diff = std::abs((int64_t)width - dimension.width) + + std::abs((int64_t)height - dimension.height); if (diff < currentDiff) { selectedFormat = format; currentDiff = diff; @@ -65,17 +69,19 @@ - (void)capturer:(RTCVideoCapturer *)capturer didCaptureVideoFrame:(RTCVideoFram capturer_ = [[RTCCameraVideoCapturer alloc] initWithDelegate:adapter_]; - AVCaptureDevice *device = - [[RTCCameraVideoCapturer captureDevices] objectAtIndex:capture_device_index]; - AVCaptureDeviceFormat *format = SelectClosestFormat(device, width, height); + AVCaptureDevice* device = [[RTCCameraVideoCapturer captureDevices] + objectAtIndex:capture_device_index]; + AVCaptureDeviceFormat* format = SelectClosestFormat(device, width, height); [capturer_ startCaptureWithDevice:device format:format fps:target_fps]; } -rtc::scoped_refptr MacCapturer::Create(size_t width, - size_t height, - size_t target_fps, - size_t capture_device_index) { - return new rtc::RefCountedObject(width, height, target_fps, capture_device_index); +rtc::scoped_refptr MacCapturer::Create( + size_t width, + size_t height, + size_t target_fps, + size_t capture_device_index) { + return new rtc::RefCountedObject(width, height, target_fps, + capture_device_index); } void MacCapturer::Destroy() { @@ -86,6 +92,6 @@ - (void)capturer:(RTCVideoCapturer *)capturer didCaptureVideoFrame:(RTCVideoFram Destroy(); } -void MacCapturer::OnFrame(const webrtc::VideoFrame &frame) { +void MacCapturer::OnFrame(const webrtc::VideoFrame& frame) { OnCapturedFrame(frame); } \ No newline at end of file diff --git a/src/mac_helper/objc_codec_factory_helper.h b/src/mac_helper/objc_codec_factory_helper.h index 33ed2b99..1dfcb7fe 100644 --- a/src/mac_helper/objc_codec_factory_helper.h +++ b/src/mac_helper/objc_codec_factory_helper.h @@ -20,4 +20,3 @@ std::unique_ptr CreateObjCEncoderFactory(); std::unique_ptr CreateObjCDecoderFactory(); #endif // MODULES_VIDEO_CODING_CODECS_TEST_OBJC_CODEC_FACTORY_HELPER_H_ - diff --git a/src/mac_helper/objc_codec_factory_helper.mm b/src/mac_helper/objc_codec_factory_helper.mm index d4ee9725..241d9625 100644 --- a/src/mac_helper/objc_codec_factory_helper.mm +++ b/src/mac_helper/objc_codec_factory_helper.mm @@ -16,9 +16,11 @@ #include "sdk/objc/native/api/video_encoder_factory.h" std::unique_ptr CreateObjCEncoderFactory() { - return webrtc::ObjCToNativeVideoEncoderFactory([[RTCDefaultVideoEncoderFactory alloc] init]); + return webrtc::ObjCToNativeVideoEncoderFactory( + [[RTCDefaultVideoEncoderFactory alloc] init]); } std::unique_ptr CreateObjCDecoderFactory() { - return webrtc::ObjCToNativeVideoDecoderFactory([[RTCDefaultVideoDecoderFactory alloc] init]); + return webrtc::ObjCToNativeVideoDecoderFactory( + [[RTCDefaultVideoDecoderFactory alloc] init]); } \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index e3e642f2..e86f80c8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,13 +1,13 @@ #ifndef _MSC_VER #include #endif -#include +#include #include +#include #include -#include -#include #include -#include +#include +#include #include "rtc_base/log_sinks.h" @@ -27,17 +27,16 @@ #endif #endif +#include "ayame/ayame_server.h" #include "connection_settings.h" -#include "util.h" +#include "p2p/p2p_server.h" #include "rtc/manager.h" #include "sora/sora_server.h" -#include "p2p/p2p_server.h" -#include "ayame/ayame_server.h" +#include "util.h" const size_t kDefaultMaxLogFileSize = 10 * 1024 * 1024; -int main(int argc, char* argv[]) -{ +int main(int argc, char* argv[]) { ConnectionSettings cs; bool is_daemon = false; @@ -46,20 +45,18 @@ int main(int argc, char* argv[]) bool use_sora = false; int log_level = rtc::LS_NONE; - Util::parseArgs(argc, argv, is_daemon, use_test, use_ayame, use_sora, log_level, cs); + Util::parseArgs(argc, argv, is_daemon, use_test, use_ayame, use_sora, + log_level, cs); #ifndef _MSC_VER - if (is_daemon) - { - if (daemon(1, 0) == -1) - { + if (is_daemon) { + if (daemon(1, 0) == -1) { std::cerr << "failed to launch momo daemon" << std::endl; return -1; } } #endif - rtc::LogMessage::LogToDebug((rtc::LoggingSeverity)log_level); rtc::LogMessage::LogTimestamps(); rtc::LogMessage::LogThreads(); @@ -68,12 +65,12 @@ int main(int argc, char* argv[]) std::unique_ptr log_sink(new ROSLogSink()); rtc::LogMessage::AddLogToStream(log_sink.get(), rtc::LS_INFO); rtc::scoped_refptr capturer( - new rtc::RefCountedObject(cs)); + new rtc::RefCountedObject(cs)); #else std::unique_ptr log_sink( - new rtc::FileRotatingLogSink("./", "webrtc_logs", kDefaultMaxLogFileSize, 10)); - if (!log_sink->Init()) - { + new rtc::FileRotatingLogSink("./", "webrtc_logs", kDefaultMaxLogFileSize, + 10)); + if (!log_sink->Init()) { RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to open log file"; log_sink.reset(); return 1; @@ -81,49 +78,57 @@ int main(int argc, char* argv[]) rtc::LogMessage::AddLogToStream(log_sink.get(), rtc::LS_INFO); #ifdef __APPLE__ rtc::scoped_refptr capturer = - MacCapturer::Create(cs.getWidth(), cs.getHeight(), cs.framerate, 0); + MacCapturer::Create(cs.getWidth(), cs.getHeight(), cs.framerate, 0); #else #if USE_MMAL_ENCODER | USE_JETSON_ENCODER - rtc::scoped_refptr capturer = - V4L2VideoCapture::Create(cs); + rtc::scoped_refptr capturer = V4L2VideoCapture::Create(cs); #else rtc::scoped_refptr capturer = - DeviceVideoCapturer::Create(cs.getWidth(), cs.getHeight(), cs.framerate); + DeviceVideoCapturer::Create(cs.getWidth(), cs.getHeight(), cs.framerate); #endif #endif - if (!capturer && !cs.no_video) - { + if (!capturer && !cs.no_video) { std::cerr << "failed to create capturer" << std::endl; return 1; } #endif - std::unique_ptr rtc_manager(new RTCManager(cs, std::move(capturer))); + std::unique_ptr rtc_manager( + new RTCManager(cs, std::move(capturer))); { - boost::asio::io_context ioc{1}; + boost::asio::io_context ioc{1}; - boost::asio::signal_set signals(ioc, SIGINT, SIGTERM); - signals.async_wait([&](const boost::system::error_code&, int) { - ioc.stop(); - }); + boost::asio::signal_set signals(ioc, SIGINT, SIGTERM); + signals.async_wait( + [&](const boost::system::error_code&, int) { ioc.stop(); }); - if (use_sora) { - const boost::asio::ip::tcp::endpoint endpoint{boost::asio::ip::make_address("127.0.0.1"), static_cast(cs.port)}; - std::make_shared(ioc, endpoint, rtc_manager.get(), cs)->run(); - } + if (use_sora) { + const boost::asio::ip::tcp::endpoint endpoint{ + boost::asio::ip::make_address("127.0.0.1"), + static_cast(cs.port)}; + std::make_shared(ioc, endpoint, rtc_manager.get(), cs)->run(); + } - if (use_test) { - const boost::asio::ip::tcp::endpoint endpoint{boost::asio::ip::make_address("0.0.0.0"), static_cast(cs.port)}; - std::make_shared(ioc, endpoint, std::make_shared(cs.test_document_root), rtc_manager.get(), cs)->run(); - } + if (use_test) { + const boost::asio::ip::tcp::endpoint endpoint{ + boost::asio::ip::make_address("0.0.0.0"), + static_cast(cs.port)}; + std::make_shared( + ioc, endpoint, std::make_shared(cs.test_document_root), + rtc_manager.get(), cs) + ->run(); + } - if (use_ayame) { - const boost::asio::ip::tcp::endpoint endpoint{boost::asio::ip::make_address("127.0.0.1"), static_cast(cs.port)}; - std::make_shared(ioc, endpoint, rtc_manager.get(), cs)->run(); - } + if (use_ayame) { + const boost::asio::ip::tcp::endpoint endpoint{ + boost::asio::ip::make_address("127.0.0.1"), + static_cast(cs.port)}; + std::make_shared(ioc, endpoint, rtc_manager.get(), cs) + ->run(); + } - ioc.run(); + ioc.run(); } rtc_manager = nullptr; diff --git a/src/p2p/p2p_connection.cpp b/src/p2p/p2p_connection.cpp index d7b50e33..7d859dcd 100644 --- a/src/p2p/p2p_connection.cpp +++ b/src/p2p/p2p_connection.cpp @@ -7,9 +7,9 @@ using json = nlohmann::json; using IceConnectionState = webrtc::PeerConnectionInterface::IceConnectionState; -P2PConnection::P2PConnection(RTCManager* rtc_manager, std::function send) : - _send(send) -{ +P2PConnection::P2PConnection(RTCManager* rtc_manager, + std::function send) + : _send(send) { webrtc::PeerConnectionInterface::RTCConfiguration rtc_config; webrtc::PeerConnectionInterface::IceServers servers; webrtc::PeerConnectionInterface::IceServer ice_server; @@ -19,44 +19,39 @@ P2PConnection::P2PConnection(RTCManager* rtc_manager, std::functioncreateConnection(rtc_config, this); } -void P2PConnection::onIceConnectionStateChange(IceConnectionState new_state) -{ - RTC_LOG(LS_INFO) << __FUNCTION__ << " rtc_state " << Util::iceConnectionStateToString(_rtc_state) << " -> " << Util::iceConnectionStateToString(new_state); +void P2PConnection::onIceConnectionStateChange(IceConnectionState new_state) { + RTC_LOG(LS_INFO) << __FUNCTION__ << " rtc_state " + << Util::iceConnectionStateToString(_rtc_state) << " -> " + << Util::iceConnectionStateToString(new_state); _rtc_state = new_state; } -void P2PConnection::onIceCandidate( - const std::string sdp_mid, const int sdp_mlineindex, const std::string sdp) -{ +void P2PConnection::onIceCandidate(const std::string sdp_mid, + const int sdp_mlineindex, + const std::string sdp) { RTC_LOG(LS_INFO) << __FUNCTION__; - json json_cand = { - {"type", "candidate"} - }; - json_cand["ice"] = { - {"candidate", sdp}, - {"sdpMLineIndex", sdp_mlineindex}, - {"sdpMid", sdp_mid} - }; + json json_cand = {{"type", "candidate"}}; + json_cand["ice"] = {{"candidate", sdp}, + {"sdpMLineIndex", sdp_mlineindex}, + {"sdpMid", sdp_mid}}; std::string str_cand = json_cand.dump(); _send(std::move(str_cand)); } -void P2PConnection::onCreateDescription(webrtc::SdpType type, const std::string sdp) -{ +void P2PConnection::onCreateDescription(webrtc::SdpType type, + const std::string sdp) { RTC_LOG(LS_INFO) << __FUNCTION__; - json json_desc = { - {"type", webrtc::SdpTypeToString(type)}, - {"sdp", sdp} - }; + json json_desc = {{"type", webrtc::SdpTypeToString(type)}, {"sdp", sdp}}; std::string str_desc = json_desc.dump(); _send(std::move(str_desc)); } void P2PConnection::onSetDescription(webrtc::SdpType type) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " SdpType: " << webrtc::SdpTypeToString(type); + RTC_LOG(LS_INFO) << __FUNCTION__ + << " SdpType: " << webrtc::SdpTypeToString(type); if (type == webrtc::SdpType::kOffer) { _connection->createAnswer(); } diff --git a/src/p2p/p2p_connection.h b/src/p2p/p2p_connection.h index 0479da0e..df0d004e 100644 --- a/src/p2p/p2p_connection.h +++ b/src/p2p/p2p_connection.h @@ -5,31 +5,34 @@ #include #include -#include "rtc/manager.h" #include "rtc/connection.h" +#include "rtc/manager.h" #include "rtc/messagesender.h" -class P2PConnection : public RTCMessageSender -{ -public: - P2PConnection(RTCManager* rtc_manager, std::function send); +class P2PConnection : public RTCMessageSender { + public: + P2PConnection(RTCManager* rtc_manager, std::function send); ~P2PConnection() {} - webrtc::PeerConnectionInterface::IceConnectionState getRTCConnectionState() { return _rtc_state; } + webrtc::PeerConnectionInterface::IceConnectionState getRTCConnectionState() { + return _rtc_state; + } std::shared_ptr getRTCConnection() { return _connection; }; -protected: + protected: //WebRTC void onIceConnectionStateChange( - webrtc::PeerConnectionInterface::IceConnectionState new_state) override; - void onIceCandidate( - const std::string sdp_mid, const int sdp_mlineindex, const std::string sdp) override; - void onCreateDescription(webrtc::SdpType type, const std::string sdp) override; + webrtc::PeerConnectionInterface::IceConnectionState new_state) override; + void onIceCandidate(const std::string sdp_mid, + const int sdp_mlineindex, + const std::string sdp) override; + void onCreateDescription(webrtc::SdpType type, + const std::string sdp) override; void onSetDescription(webrtc::SdpType type) override; -private: + private: std::shared_ptr _connection; - std::function _send; + std::function _send; webrtc::PeerConnectionInterface::IceConnectionState _rtc_state; }; #endif diff --git a/src/p2p/p2p_server.cpp b/src/p2p/p2p_server.cpp index 3fa3a336..3d23017f 100644 --- a/src/p2p/p2p_server.cpp +++ b/src/p2p/p2p_server.cpp @@ -3,80 +3,67 @@ #include "p2p_session.h" #include "util.h" -P2PServer::P2PServer( - boost::asio::io_context& ioc, - boost::asio::ip::tcp::endpoint endpoint, - std::shared_ptr const& doc_root, - RTCManager* rtc_manager, - ConnectionSettings conn_settings) - : acceptor_(ioc) - , socket_(ioc) - , doc_root_(doc_root) - , rtc_manager_(rtc_manager) - , conn_settings_(conn_settings) -{ - boost::system::error_code ec; +P2PServer::P2PServer(boost::asio::io_context& ioc, + boost::asio::ip::tcp::endpoint endpoint, + std::shared_ptr const& doc_root, + RTCManager* rtc_manager, + ConnectionSettings conn_settings) + : acceptor_(ioc), + socket_(ioc), + doc_root_(doc_root), + rtc_manager_(rtc_manager), + conn_settings_(conn_settings) { + boost::system::error_code ec; - // Open the acceptor - acceptor_.open(endpoint.protocol(), ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "open"); - return; - } + // Open the acceptor + acceptor_.open(endpoint.protocol(), ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "open"); + return; + } - // Allow address reuse - acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "set_option"); - return; - } + // Allow address reuse + acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "set_option"); + return; + } - // Bind to the server address - acceptor_.bind(endpoint, ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "bind"); - return; - } + // Bind to the server address + acceptor_.bind(endpoint, ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "bind"); + return; + } - // Start listening for connections - acceptor_.listen( - boost::asio::socket_base::max_listen_connections, ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "listen"); - return; - } + // Start listening for connections + acceptor_.listen(boost::asio::socket_base::max_listen_connections, ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "listen"); + return; + } } void P2PServer::run() { - if (!acceptor_.is_open()) - return; - doAccept(); + if (!acceptor_.is_open()) + return; + doAccept(); } -void P2PServer::doAccept() -{ - acceptor_.async_accept( - socket_, - std::bind( - &P2PServer::onAccept, - shared_from_this(), - std::placeholders::_1)); +void P2PServer::doAccept() { + acceptor_.async_accept(socket_, + std::bind(&P2PServer::onAccept, shared_from_this(), + std::placeholders::_1)); } -void P2PServer::onAccept(boost::system::error_code ec) -{ - if (ec) - { - MOMO_BOOST_ERROR(ec, "accept"); - } - else - { - std::make_shared(std::move(socket_), doc_root_, rtc_manager_, conn_settings_)->run(); - } +void P2PServer::onAccept(boost::system::error_code ec) { + if (ec) { + MOMO_BOOST_ERROR(ec, "accept"); + } else { + std::make_shared(std::move(socket_), doc_root_, rtc_manager_, + conn_settings_) + ->run(); + } - doAccept(); + doAccept(); } diff --git a/src/p2p/p2p_server.h b/src/p2p/p2p_server.h index da4a2237..0fde51a8 100644 --- a/src/p2p/p2p_server.h +++ b/src/p2p/p2p_server.h @@ -8,32 +8,30 @@ #include #include -#include "rtc/manager.h" #include "connection_settings.h" +#include "rtc/manager.h" #include "util.h" -class P2PServer : public std::enable_shared_from_this -{ - boost::asio::ip::tcp::acceptor acceptor_; - boost::asio::ip::tcp::socket socket_; - std::shared_ptr doc_root_; +class P2PServer : public std::enable_shared_from_this { + boost::asio::ip::tcp::acceptor acceptor_; + boost::asio::ip::tcp::socket socket_; + std::shared_ptr doc_root_; - RTCManager* rtc_manager_; - ConnectionSettings conn_settings_; + RTCManager* rtc_manager_; + ConnectionSettings conn_settings_; -public: - P2PServer( - boost::asio::io_context& ioc, - boost::asio::ip::tcp::endpoint endpoint, - std::shared_ptr const& doc_root, - RTCManager* rtc_manager, - ConnectionSettings conn_settings); + public: + P2PServer(boost::asio::io_context& ioc, + boost::asio::ip::tcp::endpoint endpoint, + std::shared_ptr const& doc_root, + RTCManager* rtc_manager, + ConnectionSettings conn_settings); - void run(); + void run(); -private: - void doAccept(); - void onAccept(boost::system::error_code ec); + private: + void doAccept(); + void onAccept(boost::system::error_code ec); }; #endif diff --git a/src/p2p/p2p_session.cpp b/src/p2p/p2p_session.cpp index 3c661bab..974fac45 100644 --- a/src/p2p/p2p_session.cpp +++ b/src/p2p/p2p_session.cpp @@ -1,13 +1,13 @@ #include "p2p_session.h" -#include -#include #include -#include #include +#include #include #include +#include #include +#include #ifdef _WIN32 #include @@ -15,155 +15,144 @@ #include "util.h" -P2PSession::P2PSession( - boost::asio::ip::tcp::socket socket, - std::shared_ptr const& doc_root, - RTCManager* rtc_manager, - ConnectionSettings conn_settings) - : socket_(std::move(socket)) - , strand_(socket_.get_executor()) - , doc_root_(doc_root) - , rtc_manager_(rtc_manager) - , conn_settings_(conn_settings) -{ -} +P2PSession::P2PSession(boost::asio::ip::tcp::socket socket, + std::shared_ptr const& doc_root, + RTCManager* rtc_manager, + ConnectionSettings conn_settings) + : socket_(std::move(socket)), + strand_(socket_.get_executor()), + doc_root_(doc_root), + rtc_manager_(rtc_manager), + conn_settings_(conn_settings) {} // Start the asynchronous operation -void P2PSession::run() -{ - doRead(); +void P2PSession::run() { + doRead(); } -void P2PSession::doRead() -{ - // Make the request empty before reading, - // otherwise the operation behavior is undefined. - req_ = {}; - - // Read a request - boost::beast::http::async_read(socket_, buffer_, req_, - boost::asio::bind_executor( - strand_, - std::bind( - &P2PSession::onRead, - shared_from_this(), - std::placeholders::_1, - std::placeholders::_2))); +void P2PSession::doRead() { + // Make the request empty before reading, + // otherwise the operation behavior is undefined. + req_ = {}; + + // Read a request + boost::beast::http::async_read( + socket_, buffer_, req_, + boost::asio::bind_executor( + strand_, std::bind(&P2PSession::onRead, shared_from_this(), + std::placeholders::_1, std::placeholders::_2))); } -void P2PSession::onRead( - boost::system::error_code ec, - std::size_t bytes_transferred) -{ - boost::ignore_unused(bytes_transferred); - - // 接続が切られた - if (ec == boost::beast::http::error::end_of_stream) - return doClose(); - - if (ec) - return MOMO_BOOST_ERROR(ec, "read"); - - // WebSocket の upgrade リクエスト - if (req_.target() == "/ws") - { - if (boost::beast::websocket::is_upgrade(req_)) - { - P2PWebsocketSession::make_shared(std::move(socket_), rtc_manager_, conn_settings_)->run(std::move(req_)); - return; - } - else - { - sendResponse(Util::badRequest(std::move(req_), "Not upgrade request")); - return; - } +void P2PSession::onRead(boost::system::error_code ec, + std::size_t bytes_transferred) { + boost::ignore_unused(bytes_transferred); + + // 接続が切られた + if (ec == boost::beast::http::error::end_of_stream) + return doClose(); + + if (ec) + return MOMO_BOOST_ERROR(ec, "read"); + + // WebSocket の upgrade リクエスト + if (req_.target() == "/ws") { + if (boost::beast::websocket::is_upgrade(req_)) { + P2PWebsocketSession::make_shared(std::move(socket_), rtc_manager_, + conn_settings_) + ->run(std::move(req_)); + return; + } else { + sendResponse(Util::badRequest(std::move(req_), "Not upgrade request")); + return; } + } - handleRequest(); + handleRequest(); } void P2PSession::handleRequest() { - boost::beast::http::request req(std::move(req_)); + boost::beast::http::request req( + std::move(req_)); - // Make sure we can handle the method - if( req.method() != boost::beast::http::verb::get && - req.method() != boost::beast::http::verb::head) - return sendResponse(Util::badRequest(std::move(req), "Unknown HTTP-method")); + // Make sure we can handle the method + if (req.method() != boost::beast::http::verb::get && + req.method() != boost::beast::http::verb::head) + return sendResponse( + Util::badRequest(std::move(req), "Unknown HTTP-method")); - // Request path must be absolute and not contain "..". - if( req.target().empty() || - req.target()[0] != '/' || - req.target().find("..") != boost::beast::string_view::npos) - return sendResponse(Util::badRequest(req, "Illegal request-target")); + // Request path must be absolute and not contain "..". + if (req.target().empty() || req.target()[0] != '/' || + req.target().find("..") != boost::beast::string_view::npos) + return sendResponse(Util::badRequest(req, "Illegal request-target")); - // Build the path to the requested file - boost::filesystem::path path = boost::filesystem::path(*doc_root_) / std::string(req.target()); + // Build the path to the requested file + boost::filesystem::path path = + boost::filesystem::path(*doc_root_) / std::string(req.target()); - if (req.target().back() == '/') - path.append("index.html"); + if (req.target().back() == '/') + path.append("index.html"); #ifdef _WIN32 - path.imbue(std::locale(std::locale(), new std::codecvt_utf8_utf16())); + path.imbue( + std::locale(std::locale(), new std::codecvt_utf8_utf16())); #endif - // Attempt to open the file - boost::beast::error_code ec; - boost::beast::http::file_body::value_type body; - body.open(path.string().c_str(), boost::beast::file_mode::scan, ec); - - // Handle the case where the file doesn't exist - if(ec == boost::system::errc::no_such_file_or_directory) - return sendResponse(Util::notFound(req, req.target())); - - // Handle an unknown error - if(ec) - return sendResponse(Util::serverError(req, ec.message())); - - // Cache the size since we need it after the move - auto const size = body.size(); - - // HEAD リクエスト - if(req.method() == boost::beast::http::verb::head) - { - boost::beast::http::response res{boost::beast::http::status::ok, req.version()}; - res.set(boost::beast::http::field::server, BOOST_BEAST_VERSION_STRING); - res.set(boost::beast::http::field::content_type, Util::mimeType(path.string())); - res.content_length(size); - res.keep_alive(req.keep_alive()); - return sendResponse(std::move(res)); - } - - // GET リクエスト - boost::beast::http::response res{ - std::piecewise_construct, - std::make_tuple(std::move(body)), - std::make_tuple(boost::beast::http::status::ok, req.version())}; + // Attempt to open the file + boost::beast::error_code ec; + boost::beast::http::file_body::value_type body; + body.open(path.string().c_str(), boost::beast::file_mode::scan, ec); + + // Handle the case where the file doesn't exist + if (ec == boost::system::errc::no_such_file_or_directory) + return sendResponse(Util::notFound(req, req.target())); + + // Handle an unknown error + if (ec) + return sendResponse(Util::serverError(req, ec.message())); + + // Cache the size since we need it after the move + auto const size = body.size(); + + // HEAD リクエスト + if (req.method() == boost::beast::http::verb::head) { + boost::beast::http::response res{ + boost::beast::http::status::ok, req.version()}; res.set(boost::beast::http::field::server, BOOST_BEAST_VERSION_STRING); - res.set(boost::beast::http::field::content_type, Util::mimeType(path.string())); + res.set(boost::beast::http::field::content_type, + Util::mimeType(path.string())); res.content_length(size); res.keep_alive(req.keep_alive()); return sendResponse(std::move(res)); + } + + // GET リクエスト + boost::beast::http::response res{ + std::piecewise_construct, std::make_tuple(std::move(body)), + std::make_tuple(boost::beast::http::status::ok, req.version())}; + res.set(boost::beast::http::field::server, BOOST_BEAST_VERSION_STRING); + res.set(boost::beast::http::field::content_type, + Util::mimeType(path.string())); + res.content_length(size); + res.keep_alive(req.keep_alive()); + return sendResponse(std::move(res)); } -void P2PSession::onWrite( - boost::system::error_code ec, - std::size_t bytes_transferred, - bool close) -{ - boost::ignore_unused(bytes_transferred); +void P2PSession::onWrite(boost::system::error_code ec, + std::size_t bytes_transferred, + bool close) { + boost::ignore_unused(bytes_transferred); - if (ec) - return MOMO_BOOST_ERROR(ec, "write"); + if (ec) + return MOMO_BOOST_ERROR(ec, "write"); - if (close) - return doClose(); + if (close) + return doClose(); - res_ = nullptr; + res_ = nullptr; - doRead(); + doRead(); } -void P2PSession::doClose() -{ - boost::system::error_code ec; - socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_send, ec); +void P2PSession::doClose() { + boost::system::error_code ec; + socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_send, ec); } diff --git a/src/p2p/p2p_session.h b/src/p2p/p2p_session.h index dc99bc6c..45d4b592 100644 --- a/src/p2p/p2p_session.h +++ b/src/p2p/p2p_session.h @@ -14,68 +14,59 @@ #include #include -#include "rtc/manager.h" #include "connection_settings.h" -#include "util.h" #include "p2p_websocket_session.h" +#include "rtc/manager.h" +#include "util.h" // 1つの HTTP リクエストを処理するためのクラス -class P2PSession : public std::enable_shared_from_this -{ - boost::asio::ip::tcp::socket socket_; - boost::asio::strand strand_; - boost::beast::flat_buffer buffer_; - std::shared_ptr doc_root_; - boost::beast::http::request req_; - std::shared_ptr res_; +class P2PSession : public std::enable_shared_from_this { + boost::asio::ip::tcp::socket socket_; + boost::asio::strand strand_; + boost::beast::flat_buffer buffer_; + std::shared_ptr doc_root_; + boost::beast::http::request req_; + std::shared_ptr res_; - RTCManager* rtc_manager_; - ConnectionSettings conn_settings_; + RTCManager* rtc_manager_; + ConnectionSettings conn_settings_; -public: - P2PSession( - boost::asio::ip::tcp::socket socket, - std::shared_ptr const& doc_root, - RTCManager* rtc_manager, - ConnectionSettings conn_settings); + public: + P2PSession(boost::asio::ip::tcp::socket socket, + std::shared_ptr const& doc_root, + RTCManager* rtc_manager, + ConnectionSettings conn_settings); - void run(); + void run(); -private: - void doRead(); - void onRead( - boost::system::error_code ec, - std::size_t bytes_transferred); + private: + void doRead(); + void onRead(boost::system::error_code ec, std::size_t bytes_transferred); - void handleRequest(); + void handleRequest(); - template - void sendResponse(boost::beast::http::response msg) { - auto sp = std::make_shared>(std::move(msg)); + template + void sendResponse(boost::beast::http::response msg) { + auto sp = std::make_shared>( + std::move(msg)); - // msg オブジェクトは書き込みが完了するまで生きている必要があるので、 - // メンバに入れてライフタイムを延ばしてやる - res_ = sp; + // msg オブジェクトは書き込みが完了するまで生きている必要があるので、 + // メンバに入れてライフタイムを延ばしてやる + res_ = sp; - // Write the response - boost::beast::http::async_write( - socket_, - *sp, - boost::asio::bind_executor( - strand_, - std::bind( - &P2PSession::onWrite, - shared_from_this(), - std::placeholders::_1, - std::placeholders::_2, - sp->need_eof()))); - } + // Write the response + boost::beast::http::async_write( + socket_, *sp, + boost::asio::bind_executor( + strand_, std::bind(&P2PSession::onWrite, shared_from_this(), + std::placeholders::_1, std::placeholders::_2, + sp->need_eof()))); + } - void onWrite( - boost::system::error_code ec, - std::size_t bytes_transferred, - bool close); - void doClose(); + void onWrite(boost::system::error_code ec, + std::size_t bytes_transferred, + bool close); + void doClose(); }; -#endif // P2P_SESSION_H_ +#endif // P2P_SESSION_H_ diff --git a/src/p2p/p2p_websocket_session.cpp b/src/p2p/p2p_websocket_session.cpp index 3e5016f4..6d163563 100644 --- a/src/p2p/p2p_websocket_session.cpp +++ b/src/p2p/p2p_websocket_session.cpp @@ -1,173 +1,139 @@ #include "p2p_websocket_session.h" -#include #include #include #include +#include #include "util.h" using json = nlohmann::json; -P2PWebsocketSession::P2PWebsocketSession(RTCManager* rtc_manager, ConnectionSettings conn_settings) - : rtc_manager_(rtc_manager) - , conn_settings_(conn_settings) -{ - RTC_LOG(LS_INFO) << __FUNCTION__; +P2PWebsocketSession::P2PWebsocketSession(RTCManager* rtc_manager, + ConnectionSettings conn_settings) + : rtc_manager_(rtc_manager), conn_settings_(conn_settings) { + RTC_LOG(LS_INFO) << __FUNCTION__; } -P2PWebsocketSession::~P2PWebsocketSession() -{ - RTC_LOG(LS_INFO) << __FUNCTION__; +P2PWebsocketSession::~P2PWebsocketSession() { + RTC_LOG(LS_INFO) << __FUNCTION__; } -std::shared_ptr P2PWebsocketSession::make_shared(boost::asio::ip::tcp::socket socket, RTCManager* rtc_manager, ConnectionSettings conn_settings) { - auto p = std::make_shared(rtc_manager, conn_settings); - p->ws_ = std::unique_ptr(new Websocket(std::move(socket))); - return p; +std::shared_ptr P2PWebsocketSession::make_shared( + boost::asio::ip::tcp::socket socket, + RTCManager* rtc_manager, + ConnectionSettings conn_settings) { + auto p = std::make_shared(rtc_manager, conn_settings); + p->ws_ = std::unique_ptr(new Websocket(std::move(socket))); + return p; } -void P2PWebsocketSession::run(boost::beast::http::request req) -{ - RTC_LOG(LS_INFO) << __FUNCTION__; - doAccept(std::move(req)); +void P2PWebsocketSession::run( + boost::beast::http::request req) { + RTC_LOG(LS_INFO) << __FUNCTION__; + doAccept(std::move(req)); } -void P2PWebsocketSession::doAccept(boost::beast::http::request req) -{ - RTC_LOG(LS_INFO) << __FUNCTION__; - // Accept the websocket handshake - ws_->nativeSocket().async_accept( - req, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &P2PWebsocketSession::onAccept, - shared_from_this(), - std::placeholders::_1))); +void P2PWebsocketSession::doAccept( + boost::beast::http::request req) { + RTC_LOG(LS_INFO) << __FUNCTION__; + // Accept the websocket handshake + ws_->nativeSocket().async_accept( + req, + boost::asio::bind_executor( + ws_->strand(), std::bind(&P2PWebsocketSession::onAccept, + shared_from_this(), std::placeholders::_1))); } -void P2PWebsocketSession::onAccept(boost::system::error_code ec) -{ - RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec; +void P2PWebsocketSession::onAccept(boost::system::error_code ec) { + RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec; - if (ec) - return MOMO_BOOST_ERROR(ec, "Accept"); + if (ec) + return MOMO_BOOST_ERROR(ec, "Accept"); - // WebSocket での読み込みを開始 - ws_->startToRead(std::bind( - &P2PWebsocketSession::onRead, - shared_from_this(), - std::placeholders::_1, - std::placeholders::_2, - std::placeholders::_3)); + // WebSocket での読み込みを開始 + ws_->startToRead(std::bind(&P2PWebsocketSession::onRead, shared_from_this(), + std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3)); } -void P2PWebsocketSession::onRead(boost::system::error_code ec, std::size_t bytes_transferred, std::string recv_string) -{ - RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec; +void P2PWebsocketSession::onRead(boost::system::error_code ec, + std::size_t bytes_transferred, + std::string recv_string) { + RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec; - boost::ignore_unused(bytes_transferred); + boost::ignore_unused(bytes_transferred); - if (ec == boost::beast::websocket::error::closed) - return; + if (ec == boost::beast::websocket::error::closed) + return; - if (ec) - return MOMO_BOOST_ERROR(ec, "Read"); + if (ec) + return MOMO_BOOST_ERROR(ec, "Read"); - json recv_message; + json recv_message; - RTC_LOG(LS_INFO) << __FUNCTION__ << ": recv_string=" << recv_string; + RTC_LOG(LS_INFO) << __FUNCTION__ << ": recv_string=" << recv_string; - try - { - recv_message = json::parse(recv_string); - } - catch (json::parse_error &e) - { - return; - } + try { + recv_message = json::parse(recv_string); + } catch (json::parse_error& e) { + return; + } - std::string type; - try - { - type = recv_message["type"]; - } - catch (json::type_error &e) - { - return; - } + std::string type; + try { + type = recv_message["type"]; + } catch (json::type_error& e) { + return; + } - if (type == "offer") - { - std::string sdp; - try - { - sdp = recv_message["sdp"]; - } - catch (json::type_error &e) - { - return; - } - - auto send = std::bind( - [](P2PWebsocketSession* session, std::string str) { - session->ws_->sendText(str); - }, - this, - std::placeholders::_1); - connection_ = std::make_shared(rtc_manager_, send); - std::shared_ptr rtc_conn = connection_->getRTCConnection(); - rtc_conn->setOffer(sdp); + if (type == "offer") { + std::string sdp; + try { + sdp = recv_message["sdp"]; + } catch (json::type_error& e) { + return; } - else if (type == "answer") - { - std::shared_ptr p2p_conn = connection_; - if (!p2p_conn) - { - return; - } - std::string sdp; - try - { - sdp = recv_message["sdp"]; - } - catch (json::type_error &e) - { - return; - } - std::shared_ptr rtc_conn = p2p_conn->getRTCConnection(); - rtc_conn->setAnswer(sdp); + + auto send = std::bind([](P2PWebsocketSession* session, + std::string str) { session->ws_->sendText(str); }, + this, std::placeholders::_1); + connection_ = std::make_shared(rtc_manager_, send); + std::shared_ptr rtc_conn = connection_->getRTCConnection(); + rtc_conn->setOffer(sdp); + } else if (type == "answer") { + std::shared_ptr p2p_conn = connection_; + if (!p2p_conn) { + return; } - else if (type == "candidate") - { - std::shared_ptr p2p_conn = connection_; - if (!p2p_conn) - { - return; - } - int sdp_mlineindex = 0; - std::string sdp_mid, candidate; - try - { - json ice = recv_message["ice"]; - sdp_mid = ice["sdpMid"]; - sdp_mlineindex = ice["sdpMLineIndex"]; - candidate = ice["candidate"]; - } - catch (json::type_error &e) - { - return; - } - std::shared_ptr rtc_conn = p2p_conn->getRTCConnection(); - rtc_conn->addIceCandidate(sdp_mid, sdp_mlineindex, candidate); + std::string sdp; + try { + sdp = recv_message["sdp"]; + } catch (json::type_error& e) { + return; } - else if (type == "close") - { - connection_ = nullptr; + std::shared_ptr rtc_conn = p2p_conn->getRTCConnection(); + rtc_conn->setAnswer(sdp); + } else if (type == "candidate") { + std::shared_ptr p2p_conn = connection_; + if (!p2p_conn) { + return; } - else - { - return; + int sdp_mlineindex = 0; + std::string sdp_mid, candidate; + try { + json ice = recv_message["ice"]; + sdp_mid = ice["sdpMid"]; + sdp_mlineindex = ice["sdpMLineIndex"]; + candidate = ice["candidate"]; + } catch (json::type_error& e) { + return; } + std::shared_ptr rtc_conn = p2p_conn->getRTCConnection(); + rtc_conn->addIceCandidate(sdp_mid, sdp_mlineindex, candidate); + } else if (type == "close") { + connection_ = nullptr; + } else { + return; + } } diff --git a/src/p2p/p2p_websocket_session.h b/src/p2p/p2p_websocket_session.h index 973ba616..e8ab9820 100644 --- a/src/p2p/p2p_websocket_session.h +++ b/src/p2p/p2p_websocket_session.h @@ -11,32 +11,39 @@ #include #include -#include "rtc/manager.h" -#include "ws/websocket.h" #include "connection_settings.h" -#include "util.h" #include "p2p_connection.h" +#include "rtc/manager.h" +#include "util.h" +#include "ws/websocket.h" -class P2PWebsocketSession : public std::enable_shared_from_this -{ - std::unique_ptr ws_; - boost::beast::multi_buffer sending_buffer_; - - RTCManager* rtc_manager_; - ConnectionSettings conn_settings_; - std::shared_ptr connection_; - -public: - P2PWebsocketSession(RTCManager* rtc_manager, ConnectionSettings conn_settings); - ~P2PWebsocketSession(); - static std::shared_ptr make_shared(boost::asio::ip::tcp::socket socket, RTCManager* rtc_manager, ConnectionSettings conn_settings); - void run(boost::beast::http::request req); - -private: - void doAccept(boost::beast::http::request req); - void onAccept(boost::system::error_code ec); - - void onRead(boost::system::error_code ec, std::size_t bytes_transferred, std::string recv_string); +class P2PWebsocketSession + : public std::enable_shared_from_this { + std::unique_ptr ws_; + boost::beast::multi_buffer sending_buffer_; + + RTCManager* rtc_manager_; + ConnectionSettings conn_settings_; + std::shared_ptr connection_; + + public: + P2PWebsocketSession(RTCManager* rtc_manager, + ConnectionSettings conn_settings); + ~P2PWebsocketSession(); + static std::shared_ptr make_shared( + boost::asio::ip::tcp::socket socket, + RTCManager* rtc_manager, + ConnectionSettings conn_settings); + void run(boost::beast::http::request req); + + private: + void doAccept( + boost::beast::http::request req); + void onAccept(boost::system::error_code ec); + + void onRead(boost::system::error_code ec, + std::size_t bytes_transferred, + std::string recv_string); }; -#endif // P2P_WEBSOCKET_SESSION_H_ +#endif // P2P_WEBSOCKET_SESSION_H_ diff --git a/src/ros/ros_audio_device.cpp b/src/ros/ros_audio_device.cpp index 6a56b66e..da337f56 100644 --- a/src/ros/ros_audio_device.cpp +++ b/src/ros/ros_audio_device.cpp @@ -35,51 +35,43 @@ ROSAudioDevice::ROSAudioDevice(ConnectionSettings conn_settings) _lastCallPlayoutMillis(0), _lastCallRecordMillis(0) {} -ROSAudioDevice::~ROSAudioDevice() -{ +ROSAudioDevice::~ROSAudioDevice() { Terminate(); } int32_t ROSAudioDevice::ActiveAudioLayer( - webrtc::AudioDeviceModule::AudioLayer &audioLayer) const -{ + webrtc::AudioDeviceModule::AudioLayer& audioLayer) const { return -1; } -webrtc::AudioDeviceGeneric::InitStatus ROSAudioDevice::Init() -{ +webrtc::AudioDeviceGeneric::InitStatus ROSAudioDevice::Init() { return webrtc::AudioDeviceGeneric::InitStatus::OK; } -int32_t ROSAudioDevice::Terminate() -{ +int32_t ROSAudioDevice::Terminate() { StopRecording(); return 0; } -bool ROSAudioDevice::Initialized() const -{ +bool ROSAudioDevice::Initialized() const { return true; } -int16_t ROSAudioDevice::PlayoutDevices() -{ +int16_t ROSAudioDevice::PlayoutDevices() { return 1; } -int16_t ROSAudioDevice::RecordingDevices() -{ +int16_t ROSAudioDevice::RecordingDevices() { return 1; } -int32_t ROSAudioDevice::PlayoutDeviceName(uint16_t index, - char name[webrtc::kAdmMaxDeviceNameSize], - char guid[webrtc::kAdmMaxGuidSize]) -{ - const char *kName = "dummy_device"; - const char *kGuid = "dummy_device_unique_id"; - if (index < 1) - { +int32_t ROSAudioDevice::PlayoutDeviceName( + uint16_t index, + char name[webrtc::kAdmMaxDeviceNameSize], + char guid[webrtc::kAdmMaxGuidSize]) { + const char* kName = "dummy_device"; + const char* kGuid = "dummy_device_unique_id"; + if (index < 1) { memset(name, 0, webrtc::kAdmMaxDeviceNameSize); memset(guid, 0, webrtc::kAdmMaxGuidSize); memcpy(name, kName, strlen(kName)); @@ -89,14 +81,13 @@ int32_t ROSAudioDevice::PlayoutDeviceName(uint16_t index, return -1; } -int32_t ROSAudioDevice::RecordingDeviceName(uint16_t index, - char name[webrtc::kAdmMaxDeviceNameSize], - char guid[webrtc::kAdmMaxGuidSize]) -{ - const char *kName = "dummy_device"; - const char *kGuid = "dummy_device_unique_id"; - if (index < 1) - { +int32_t ROSAudioDevice::RecordingDeviceName( + uint16_t index, + char name[webrtc::kAdmMaxDeviceNameSize], + char guid[webrtc::kAdmMaxGuidSize]) { + const char* kName = "dummy_device"; + const char* kGuid = "dummy_device_unique_id"; + if (index < 1) { memset(name, 0, webrtc::kAdmMaxDeviceNameSize); memset(guid, 0, webrtc::kAdmMaxGuidSize); memcpy(name, kName, strlen(kName)); @@ -106,10 +97,8 @@ int32_t ROSAudioDevice::RecordingDeviceName(uint16_t index, return -1; } -int32_t ROSAudioDevice::SetPlayoutDevice(uint16_t index) -{ - if (index == 0) - { +int32_t ROSAudioDevice::SetPlayoutDevice(uint16_t index) { + if (index == 0) { _playout_index = index; return 0; } @@ -117,15 +106,12 @@ int32_t ROSAudioDevice::SetPlayoutDevice(uint16_t index) } int32_t ROSAudioDevice::SetPlayoutDevice( - webrtc::AudioDeviceModule::WindowsDeviceType device) -{ + webrtc::AudioDeviceModule::WindowsDeviceType device) { return -1; } -int32_t ROSAudioDevice::SetRecordingDevice(uint16_t index) -{ - if (index == 0) - { +int32_t ROSAudioDevice::SetRecordingDevice(uint16_t index) { + if (index == 0) { _record_index = index; return _record_index; } @@ -133,15 +119,12 @@ int32_t ROSAudioDevice::SetRecordingDevice(uint16_t index) } int32_t ROSAudioDevice::SetRecordingDevice( - webrtc::AudioDeviceModule::WindowsDeviceType device) -{ + webrtc::AudioDeviceModule::WindowsDeviceType device) { return -1; } -int32_t ROSAudioDevice::PlayoutIsAvailable(bool &available) -{ - if (_playout_index == 0) - { +int32_t ROSAudioDevice::PlayoutIsAvailable(bool& available) { + if (_playout_index == 0) { available = true; return _playout_index; } @@ -149,34 +132,28 @@ int32_t ROSAudioDevice::PlayoutIsAvailable(bool &available) return -1; } -int32_t ROSAudioDevice::InitPlayout() -{ +int32_t ROSAudioDevice::InitPlayout() { rtc::CritScope lock(&_critSect); - if (_playing) - { + if (_playing) { return -1; } _playoutFramesIn10MS = static_cast(kPlayoutFixedSampleRate / 100); - if (_ptrAudioBuffer) - { + if (_ptrAudioBuffer) { _ptrAudioBuffer->SetPlayoutSampleRate(kPlayoutFixedSampleRate); _ptrAudioBuffer->SetPlayoutChannels(kPlayoutNumChannels); } return 0; } -bool ROSAudioDevice::PlayoutIsInitialized() const -{ +bool ROSAudioDevice::PlayoutIsInitialized() const { return _playoutFramesIn10MS != 0; } -int32_t ROSAudioDevice::RecordingIsAvailable(bool &available) -{ - if (_record_index == 0) - { +int32_t ROSAudioDevice::RecordingIsAvailable(bool& available) { + if (_record_index == 0) { available = true; return _record_index; } @@ -184,46 +161,39 @@ int32_t ROSAudioDevice::RecordingIsAvailable(bool &available) return -1; } -int32_t ROSAudioDevice::InitRecording() -{ +int32_t ROSAudioDevice::InitRecording() { rtc::CritScope lock(&_critSect); - if (_recording) - { + if (_recording) { return -1; } - _recordingFramesIn10MS = static_cast(_conn_settings.audio_topic_rate / 100); + _recordingFramesIn10MS = + static_cast(_conn_settings.audio_topic_rate / 100); - if (_ptrAudioBuffer) - { + if (_ptrAudioBuffer) { _ptrAudioBuffer->SetRecordingSampleRate(_conn_settings.audio_topic_rate); _ptrAudioBuffer->SetRecordingChannels(_conn_settings.audio_topic_ch); } return 0; } -bool ROSAudioDevice::RecordingIsInitialized() const -{ +bool ROSAudioDevice::RecordingIsInitialized() const { return _recordingFramesIn10MS != 0; } -int32_t ROSAudioDevice::StartPlayout() -{ - if (_playing) - { +int32_t ROSAudioDevice::StartPlayout() { + if (_playing) { return 0; } _playing = true; _playoutFramesLeft = 0; - if (!_playoutBuffer) - { + if (!_playoutBuffer) { _playoutBuffer = new int8_t[kPlayoutBufferSize]; } - if (!_playoutBuffer) - { + if (!_playoutBuffer) { _playing = false; return -1; } @@ -237,15 +207,13 @@ int32_t ROSAudioDevice::StartPlayout() return 0; } -int32_t ROSAudioDevice::StopPlayout() -{ +int32_t ROSAudioDevice::StopPlayout() { { rtc::CritScope lock(&_critSect); _playing = false; } - if (_ptrThreadPlay) - { + if (_ptrThreadPlay) { _ptrThreadPlay->Stop(); _ptrThreadPlay.reset(); } @@ -260,13 +228,11 @@ int32_t ROSAudioDevice::StopPlayout() return 0; } -bool ROSAudioDevice::Playing() const -{ +bool ROSAudioDevice::Playing() const { return _playing; } -int32_t ROSAudioDevice::StartRecording() -{ +int32_t ROSAudioDevice::StartRecording() { if (_recording) { return -1; } @@ -275,11 +241,10 @@ int32_t ROSAudioDevice::StartRecording() // Make sure we only create the buffer once. _recordingBufferSizeIn10MS = _recordingFramesIn10MS * _conn_settings.audio_topic_ch * 2; - if (!_recordingBuffer) - { + if (!_recordingBuffer) { _recordingBuffer = new int8_t[_recordingBufferSizeIn10MS]; } - + ros::NodeHandle nh; _sub = nh.subscribe( _conn_settings.audio_topic_name, 1, @@ -294,8 +259,7 @@ int32_t ROSAudioDevice::StartRecording() return 0; } -int32_t ROSAudioDevice::StopRecording() -{ +int32_t ROSAudioDevice::StopRecording() { { rtc::CritScope lock(&_critSect); if (!_recording) { @@ -304,15 +268,13 @@ int32_t ROSAudioDevice::StopRecording() _recording = false; } - if (_spinner) - { + if (_spinner) { _spinner->stop(); } rtc::CritScope lock(&_critSect); _recordingFramesLeft = 0; - if (_recordingBuffer) - { + if (_recordingBuffer) { delete[] _recordingBuffer; _recordingBuffer = NULL; } @@ -321,151 +283,122 @@ int32_t ROSAudioDevice::StopRecording() return 0; } -bool ROSAudioDevice::Recording() const -{ +bool ROSAudioDevice::Recording() const { return _recording; } -int32_t ROSAudioDevice::InitSpeaker() -{ +int32_t ROSAudioDevice::InitSpeaker() { return 0; } -bool ROSAudioDevice::SpeakerIsInitialized() const -{ +bool ROSAudioDevice::SpeakerIsInitialized() const { return true; } -int32_t ROSAudioDevice::InitMicrophone() -{ +int32_t ROSAudioDevice::InitMicrophone() { return 0; } -bool ROSAudioDevice::MicrophoneIsInitialized() const -{ +bool ROSAudioDevice::MicrophoneIsInitialized() const { return true; } -int32_t ROSAudioDevice::SpeakerVolumeIsAvailable(bool &available) -{ +int32_t ROSAudioDevice::SpeakerVolumeIsAvailable(bool& available) { return -1; } -int32_t ROSAudioDevice::SetSpeakerVolume(uint32_t volume) -{ +int32_t ROSAudioDevice::SetSpeakerVolume(uint32_t volume) { return -1; } -int32_t ROSAudioDevice::SpeakerVolume(uint32_t &volume) const -{ +int32_t ROSAudioDevice::SpeakerVolume(uint32_t& volume) const { return -1; } -int32_t ROSAudioDevice::MaxSpeakerVolume(uint32_t &maxVolume) const -{ +int32_t ROSAudioDevice::MaxSpeakerVolume(uint32_t& maxVolume) const { return -1; } -int32_t ROSAudioDevice::MinSpeakerVolume(uint32_t &minVolume) const -{ +int32_t ROSAudioDevice::MinSpeakerVolume(uint32_t& minVolume) const { return -1; } -int32_t ROSAudioDevice::MicrophoneVolumeIsAvailable(bool &available) -{ +int32_t ROSAudioDevice::MicrophoneVolumeIsAvailable(bool& available) { return -1; } -int32_t ROSAudioDevice::SetMicrophoneVolume(uint32_t volume) -{ +int32_t ROSAudioDevice::SetMicrophoneVolume(uint32_t volume) { return -1; } -int32_t ROSAudioDevice::MicrophoneVolume(uint32_t &volume) const -{ +int32_t ROSAudioDevice::MicrophoneVolume(uint32_t& volume) const { return -1; } -int32_t ROSAudioDevice::MaxMicrophoneVolume(uint32_t &maxVolume) const -{ +int32_t ROSAudioDevice::MaxMicrophoneVolume(uint32_t& maxVolume) const { return -1; } -int32_t ROSAudioDevice::MinMicrophoneVolume(uint32_t &minVolume) const -{ +int32_t ROSAudioDevice::MinMicrophoneVolume(uint32_t& minVolume) const { return -1; } -int32_t ROSAudioDevice::SpeakerMuteIsAvailable(bool &available) -{ +int32_t ROSAudioDevice::SpeakerMuteIsAvailable(bool& available) { return -1; } -int32_t ROSAudioDevice::SetSpeakerMute(bool enable) -{ +int32_t ROSAudioDevice::SetSpeakerMute(bool enable) { return -1; } -int32_t ROSAudioDevice::SpeakerMute(bool &enabled) const -{ +int32_t ROSAudioDevice::SpeakerMute(bool& enabled) const { return -1; } -int32_t ROSAudioDevice::MicrophoneMuteIsAvailable(bool &available) -{ +int32_t ROSAudioDevice::MicrophoneMuteIsAvailable(bool& available) { return -1; } -int32_t ROSAudioDevice::SetMicrophoneMute(bool enable) -{ +int32_t ROSAudioDevice::SetMicrophoneMute(bool enable) { return -1; } -int32_t ROSAudioDevice::MicrophoneMute(bool &enabled) const -{ +int32_t ROSAudioDevice::MicrophoneMute(bool& enabled) const { return -1; } -int32_t ROSAudioDevice::StereoPlayoutIsAvailable(bool &available) -{ +int32_t ROSAudioDevice::StereoPlayoutIsAvailable(bool& available) { available = true; return 0; } -int32_t ROSAudioDevice::SetStereoPlayout(bool enable) -{ +int32_t ROSAudioDevice::SetStereoPlayout(bool enable) { return 0; } -int32_t ROSAudioDevice::StereoPlayout(bool &enabled) const -{ +int32_t ROSAudioDevice::StereoPlayout(bool& enabled) const { enabled = true; return 0; } -int32_t ROSAudioDevice::StereoRecordingIsAvailable(bool &available) -{ +int32_t ROSAudioDevice::StereoRecordingIsAvailable(bool& available) { available = _conn_settings.audio_topic_ch == 2; return 0; } -int32_t ROSAudioDevice::SetStereoRecording(bool enable) -{ +int32_t ROSAudioDevice::SetStereoRecording(bool enable) { return ((_conn_settings.audio_topic_ch == 2) == enable) ? 0 : -1; } -int32_t ROSAudioDevice::StereoRecording(bool &enabled) const -{ +int32_t ROSAudioDevice::StereoRecording(bool& enabled) const { enabled = _conn_settings.audio_topic_ch == 2; return 0; } -int32_t ROSAudioDevice::PlayoutDelay(uint16_t &delayMS) const -{ +int32_t ROSAudioDevice::PlayoutDelay(uint16_t& delayMS) const { return 0; } -void ROSAudioDevice::AttachAudioBuffer(webrtc::AudioDeviceBuffer *audioBuffer) -{ +void ROSAudioDevice::AttachAudioBuffer(webrtc::AudioDeviceBuffer* audioBuffer) { rtc::CritScope lock(&_critSect); _ptrAudioBuffer = audioBuffer; @@ -480,34 +413,32 @@ void ROSAudioDevice::AttachAudioBuffer(webrtc::AudioDeviceBuffer *audioBuffer) } #if defined(WEBRTC_IOS) -int ROSAudioDevice::GetPlayoutAudioParameters(webrtc::AudioParameters* params) const { +int ROSAudioDevice::GetPlayoutAudioParameters( + webrtc::AudioParameters* params) const { RTC_LOG(INFO) << __FUNCTION__; return 0; } -int ROSAudioDevice::GetRecordAudioParameters(webrtc::AudioParameters* params) const { +int ROSAudioDevice::GetRecordAudioParameters( + webrtc::AudioParameters* params) const { RTC_LOG(INFO) << __FUNCTION__; return 0; } -#endif // WEBRTC_IOS +#endif // WEBRTC_IOS -bool ROSAudioDevice::PlayThreadFunc(void *pThis) -{ - return (static_cast(pThis)->PlayThreadProcess()); +bool ROSAudioDevice::PlayThreadFunc(void* pThis) { + return (static_cast(pThis)->PlayThreadProcess()); } -bool ROSAudioDevice::PlayThreadProcess() -{ - if (!_playing) - { +bool ROSAudioDevice::PlayThreadProcess() { + if (!_playing) { return false; } int64_t currentTime = rtc::TimeMillis(); _critSect.Enter(); if (_lastCallPlayoutMillis == 0 || - currentTime - _lastCallPlayoutMillis >= 5) - { + currentTime - _lastCallPlayoutMillis >= 5) { _critSect.Leave(); _ptrAudioBuffer->RequestPlayoutData(_playoutFramesIn10MS); _critSect.Enter(); @@ -520,36 +451,38 @@ bool ROSAudioDevice::PlayThreadProcess() _critSect.Leave(); int64_t deltaTimeMillis = rtc::TimeMillis() - currentTime; - if (deltaTimeMillis < 5) - { + if (deltaTimeMillis < 5) { webrtc::SleepMs(5 - deltaTimeMillis); } return true; } -bool ROSAudioDevice::RecROSCallback(const audio_common_msgs::AudioDataConstPtr &msg) -{ +bool ROSAudioDevice::RecROSCallback( + const audio_common_msgs::AudioDataConstPtr& msg) { size_t copyedDataSize = 0; _critSect.Enter(); - while (_recording && copyedDataSize != msg->data.size()) - { - RTC_LOG(LS_VERBOSE) << "RecROSCallback _recordingBufferSizeIn10MS:" << _recordingBufferSizeIn10MS - << " _writtenBufferSize" << _writtenBufferSize - << " msg->data.size()" << msg->data.size() - << " copyedDataSize" << copyedDataSize; - if (_recordingBufferSizeIn10MS - _writtenBufferSize <= msg->data.size() - copyedDataSize) - { - memcpy(_recordingBuffer + _writtenBufferSize, &msg->data[copyedDataSize], _recordingBufferSizeIn10MS - _writtenBufferSize); + while (_recording && copyedDataSize != msg->data.size()) { + RTC_LOG(LS_VERBOSE) << "RecROSCallback _recordingBufferSizeIn10MS:" + << _recordingBufferSizeIn10MS << " _writtenBufferSize" + << _writtenBufferSize << " msg->data.size()" + << msg->data.size() << " copyedDataSize" + << copyedDataSize; + if (_recordingBufferSizeIn10MS - _writtenBufferSize <= + msg->data.size() - copyedDataSize) { + memcpy(_recordingBuffer + _writtenBufferSize, &msg->data[copyedDataSize], + _recordingBufferSizeIn10MS - _writtenBufferSize); copyedDataSize += _recordingBufferSizeIn10MS - _writtenBufferSize; _writtenBufferSize = 0; - _ptrAudioBuffer->SetRecordedBuffer(_recordingBuffer, _recordingFramesIn10MS); + _ptrAudioBuffer->SetRecordedBuffer(_recordingBuffer, + _recordingFramesIn10MS); _critSect.Leave(); _ptrAudioBuffer->DeliverRecordedData(); webrtc::SleepMs(10); _critSect.Enter(); } else { - memcpy(_recordingBuffer + _writtenBufferSize, &msg->data[copyedDataSize], msg->data.size() - copyedDataSize); + memcpy(_recordingBuffer + _writtenBufferSize, &msg->data[copyedDataSize], + msg->data.size() - copyedDataSize); _writtenBufferSize += msg->data.size() - copyedDataSize; copyedDataSize += msg->data.size() - copyedDataSize; } diff --git a/src/ros/ros_audio_device.h b/src/ros/ros_audio_device.h index 7b9397e3..b814635d 100644 --- a/src/ros/ros_audio_device.h +++ b/src/ros/ros_audio_device.h @@ -14,25 +14,24 @@ #include #include "modules/audio_device/audio_device_generic.h" -#include "rtc_base/platform_thread.h" #include "rtc_base/critical_section.h" +#include "rtc_base/platform_thread.h" #include "rtc_base/system/file_wrapper.h" #include "rtc_base/time_utils.h" -#include "ros/ros.h" #include "audio_common_msgs/AudioData.h" +#include "ros/ros.h" #include "connection_settings.h" -class ROSAudioDevice : public webrtc::AudioDeviceGeneric -{ -public: +class ROSAudioDevice : public webrtc::AudioDeviceGeneric { + public: ROSAudioDevice(ConnectionSettings conn_settings); ~ROSAudioDevice() override; // Retrieve the currently utilized audio layer int32_t ActiveAudioLayer( - webrtc::AudioDeviceModule::AudioLayer &audioLayer) const override; + webrtc::AudioDeviceModule::AudioLayer& audioLayer) const override; // Main initializaton and termination webrtc::AudioDeviceGeneric::InitStatus Init() override; @@ -58,10 +57,10 @@ class ROSAudioDevice : public webrtc::AudioDeviceGeneric webrtc::AudioDeviceModule::WindowsDeviceType device) override; // Audio transport initialization - int32_t PlayoutIsAvailable(bool &available) override; + int32_t PlayoutIsAvailable(bool& available) override; int32_t InitPlayout() override; bool PlayoutIsInitialized() const override; - int32_t RecordingIsAvailable(bool &available) override; + int32_t RecordingIsAvailable(bool& available) override; int32_t InitRecording() override; bool RecordingIsInitialized() const override; @@ -80,59 +79,59 @@ class ROSAudioDevice : public webrtc::AudioDeviceGeneric bool MicrophoneIsInitialized() const override; // Speaker volume controls - int32_t SpeakerVolumeIsAvailable(bool &available) override; + int32_t SpeakerVolumeIsAvailable(bool& available) override; int32_t SetSpeakerVolume(uint32_t volume) override; - int32_t SpeakerVolume(uint32_t &volume) const override; - int32_t MaxSpeakerVolume(uint32_t &maxVolume) const override; - int32_t MinSpeakerVolume(uint32_t &minVolume) const override; + int32_t SpeakerVolume(uint32_t& volume) const override; + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t& minVolume) const override; // Microphone volume controls - int32_t MicrophoneVolumeIsAvailable(bool &available) override; + int32_t MicrophoneVolumeIsAvailable(bool& available) override; int32_t SetMicrophoneVolume(uint32_t volume) override; - int32_t MicrophoneVolume(uint32_t &volume) const override; - int32_t MaxMicrophoneVolume(uint32_t &maxVolume) const override; - int32_t MinMicrophoneVolume(uint32_t &minVolume) const override; + int32_t MicrophoneVolume(uint32_t& volume) const override; + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t& minVolume) const override; // Speaker mute control - int32_t SpeakerMuteIsAvailable(bool &available) override; + int32_t SpeakerMuteIsAvailable(bool& available) override; int32_t SetSpeakerMute(bool enable) override; - int32_t SpeakerMute(bool &enabled) const override; + int32_t SpeakerMute(bool& enabled) const override; // Microphone mute control - int32_t MicrophoneMuteIsAvailable(bool &available) override; + int32_t MicrophoneMuteIsAvailable(bool& available) override; int32_t SetMicrophoneMute(bool enable) override; - int32_t MicrophoneMute(bool &enabled) const override; + int32_t MicrophoneMute(bool& enabled) const override; // Stereo support - int32_t StereoPlayoutIsAvailable(bool &available) override; + int32_t StereoPlayoutIsAvailable(bool& available) override; int32_t SetStereoPlayout(bool enable) override; - int32_t StereoPlayout(bool &enabled) const override; - int32_t StereoRecordingIsAvailable(bool &available) override; + int32_t StereoPlayout(bool& enabled) const override; + int32_t StereoRecordingIsAvailable(bool& available) override; int32_t SetStereoRecording(bool enable) override; - int32_t StereoRecording(bool &enabled) const override; + int32_t StereoRecording(bool& enabled) const override; // Delay information and control - int32_t PlayoutDelay(uint16_t &delayMS) const override; + int32_t PlayoutDelay(uint16_t& delayMS) const override; - void AttachAudioBuffer(webrtc::AudioDeviceBuffer *audioBuffer) override; + void AttachAudioBuffer(webrtc::AudioDeviceBuffer* audioBuffer) override; #if defined(WEBRTC_IOS) int GetPlayoutAudioParameters(webrtc::AudioParameters* params) const override; int GetRecordAudioParameters(webrtc::AudioParameters* params) const override; -#endif // WEBRTC_IOS +#endif // WEBRTC_IOS -private: - static bool PlayThreadFunc(void *); - bool RecROSCallback(const audio_common_msgs::AudioDataConstPtr &msg); + private: + static bool PlayThreadFunc(void*); + bool RecROSCallback(const audio_common_msgs::AudioDataConstPtr& msg); bool PlayThreadProcess(); ConnectionSettings _conn_settings; int32_t _playout_index; int32_t _record_index; - webrtc::AudioDeviceBuffer *_ptrAudioBuffer; - int8_t *_recordingBuffer; // In bytes. - int8_t *_playoutBuffer; // In bytes. + webrtc::AudioDeviceBuffer* _ptrAudioBuffer; + int8_t* _recordingBuffer; // In bytes. + int8_t* _playoutBuffer; // In bytes. uint32_t _recordingFramesLeft; uint32_t _playoutFramesLeft; rtc::CriticalSection _critSect; diff --git a/src/ros/ros_audio_device_module.cpp b/src/ros/ros_audio_device_module.cpp index ea6d0dda..8f345443 100644 --- a/src/ros/ros_audio_device_module.cpp +++ b/src/ros/ros_audio_device_module.cpp @@ -18,51 +18,48 @@ #include "rtc_base/ref_counted_object.h" #include "system_wrappers/include/metrics.h" -ROSAudioDeviceModule::ROSAudioDeviceModule(ConnectionSettings conn_settings, - webrtc::TaskQueueFactory* task_queue_factory) - : _conn_settings(conn_settings), task_queue_factory_(task_queue_factory) -{ +ROSAudioDeviceModule::ROSAudioDeviceModule( + ConnectionSettings conn_settings, + webrtc::TaskQueueFactory* task_queue_factory) + : _conn_settings(conn_settings), task_queue_factory_(task_queue_factory) { RTC_LOG(INFO) << "Current setting use ROS Audio"; } -rtc::scoped_refptr ROSAudioDeviceModule::Create(ConnectionSettings conn_settings, - webrtc::TaskQueueFactory* task_queue_factory) -{ +rtc::scoped_refptr ROSAudioDeviceModule::Create( + ConnectionSettings conn_settings, + webrtc::TaskQueueFactory* task_queue_factory) { RTC_LOG(INFO) << __FUNCTION__; - return new rtc::RefCountedObject(conn_settings, task_queue_factory); + return new rtc::RefCountedObject(conn_settings, + task_queue_factory); } -int32_t ROSAudioDeviceModule::AttachAudioBuffer() -{ +int32_t ROSAudioDeviceModule::AttachAudioBuffer() { RTC_LOG(INFO) << __FUNCTION__; audio_device_->AttachAudioBuffer(audio_device_buffer_.get()); return 0; } -ROSAudioDeviceModule::~ROSAudioDeviceModule() -{ +ROSAudioDeviceModule::~ROSAudioDeviceModule() { RTC_LOG(INFO) << __FUNCTION__; } -int32_t ROSAudioDeviceModule::ActiveAudioLayer(AudioLayer *audioLayer) const -{ +int32_t ROSAudioDeviceModule::ActiveAudioLayer(AudioLayer* audioLayer) const { RTC_LOG(INFO) << __FUNCTION__; AudioLayer activeAudio; - if (audio_device_->ActiveAudioLayer(activeAudio) == -1) - { + if (audio_device_->ActiveAudioLayer(activeAudio) == -1) { return -1; } *audioLayer = activeAudio; return 0; } -int32_t ROSAudioDeviceModule::Init() -{ +int32_t ROSAudioDeviceModule::Init() { RTC_LOG(INFO) << __FUNCTION__; if (initialized_) return 0; - audio_device_buffer_.reset(new webrtc::AudioDeviceBuffer(task_queue_factory_)); + audio_device_buffer_.reset( + new webrtc::AudioDeviceBuffer(task_queue_factory_)); audio_device_.reset(new ROSAudioDevice(_conn_settings)); RTC_CHECK(audio_device_); @@ -72,8 +69,7 @@ int32_t ROSAudioDeviceModule::Init() RTC_HISTOGRAM_ENUMERATION( "WebRTC.Audio.InitializationResult", static_cast(status), static_cast(webrtc::AudioDeviceGeneric::InitStatus::NUM_STATUSES)); - if (status != webrtc::AudioDeviceGeneric::InitStatus::OK) - { + if (status != webrtc::AudioDeviceGeneric::InitStatus::OK) { RTC_LOG(LS_ERROR) << "Audio device initialization failed."; return -1; } @@ -81,46 +77,39 @@ int32_t ROSAudioDeviceModule::Init() return 0; } -int32_t ROSAudioDeviceModule::Terminate() -{ +int32_t ROSAudioDeviceModule::Terminate() { RTC_LOG(INFO) << __FUNCTION__; if (!initialized_) return 0; - if (audio_device_->Terminate() == -1) - { + if (audio_device_->Terminate() == -1) { return -1; } initialized_ = false; return 0; } -bool ROSAudioDeviceModule::Initialized() const -{ +bool ROSAudioDeviceModule::Initialized() const { RTC_LOG(INFO) << __FUNCTION__ << ": " << initialized_; return initialized_; } -int32_t ROSAudioDeviceModule::InitSpeaker() -{ +int32_t ROSAudioDeviceModule::InitSpeaker() { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); return audio_device_->InitSpeaker(); } -int32_t ROSAudioDeviceModule::InitMicrophone() -{ +int32_t ROSAudioDeviceModule::InitMicrophone() { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); return audio_device_->InitMicrophone(); } -int32_t ROSAudioDeviceModule::SpeakerVolumeIsAvailable(bool *available) -{ +int32_t ROSAudioDeviceModule::SpeakerVolumeIsAvailable(bool* available) { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool isAvailable = false; - if (audio_device_->SpeakerVolumeIsAvailable(isAvailable) == -1) - { + if (audio_device_->SpeakerVolumeIsAvailable(isAvailable) == -1) { return -1; } *available = isAvailable; @@ -128,20 +117,17 @@ int32_t ROSAudioDeviceModule::SpeakerVolumeIsAvailable(bool *available) return 0; } -int32_t ROSAudioDeviceModule::SetSpeakerVolume(uint32_t volume) -{ +int32_t ROSAudioDeviceModule::SetSpeakerVolume(uint32_t volume) { RTC_LOG(INFO) << __FUNCTION__ << "(" << volume << ")"; CHECKinitialized_(); return audio_device_->SetSpeakerVolume(volume); } -int32_t ROSAudioDeviceModule::SpeakerVolume(uint32_t *volume) const -{ +int32_t ROSAudioDeviceModule::SpeakerVolume(uint32_t* volume) const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); uint32_t level = 0; - if (audio_device_->SpeakerVolume(level) == -1) - { + if (audio_device_->SpeakerVolume(level) == -1) { return -1; } *volume = level; @@ -149,8 +135,7 @@ int32_t ROSAudioDeviceModule::SpeakerVolume(uint32_t *volume) const return 0; } -bool ROSAudioDeviceModule::SpeakerIsInitialized() const -{ +bool ROSAudioDeviceModule::SpeakerIsInitialized() const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized__BOOL(); bool isInitialized = audio_device_->SpeakerIsInitialized(); @@ -158,8 +143,7 @@ bool ROSAudioDeviceModule::SpeakerIsInitialized() const return isInitialized; } -bool ROSAudioDeviceModule::MicrophoneIsInitialized() const -{ +bool ROSAudioDeviceModule::MicrophoneIsInitialized() const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized__BOOL(); bool isInitialized = audio_device_->MicrophoneIsInitialized(); @@ -167,37 +151,31 @@ bool ROSAudioDeviceModule::MicrophoneIsInitialized() const return isInitialized; } -int32_t ROSAudioDeviceModule::MaxSpeakerVolume(uint32_t *maxVolume) const -{ +int32_t ROSAudioDeviceModule::MaxSpeakerVolume(uint32_t* maxVolume) const { CHECKinitialized_(); uint32_t maxVol = 0; - if (audio_device_->MaxSpeakerVolume(maxVol) == -1) - { + if (audio_device_->MaxSpeakerVolume(maxVol) == -1) { return -1; } *maxVolume = maxVol; return 0; } -int32_t ROSAudioDeviceModule::MinSpeakerVolume(uint32_t *minVolume) const -{ +int32_t ROSAudioDeviceModule::MinSpeakerVolume(uint32_t* minVolume) const { CHECKinitialized_(); uint32_t minVol = 0; - if (audio_device_->MinSpeakerVolume(minVol) == -1) - { + if (audio_device_->MinSpeakerVolume(minVol) == -1) { return -1; } *minVolume = minVol; return 0; } -int32_t ROSAudioDeviceModule::SpeakerMuteIsAvailable(bool *available) -{ +int32_t ROSAudioDeviceModule::SpeakerMuteIsAvailable(bool* available) { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool isAvailable = false; - if (audio_device_->SpeakerMuteIsAvailable(isAvailable) == -1) - { + if (audio_device_->SpeakerMuteIsAvailable(isAvailable) == -1) { return -1; } *available = isAvailable; @@ -205,20 +183,17 @@ int32_t ROSAudioDeviceModule::SpeakerMuteIsAvailable(bool *available) return 0; } -int32_t ROSAudioDeviceModule::SetSpeakerMute(bool enable) -{ +int32_t ROSAudioDeviceModule::SetSpeakerMute(bool enable) { RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; CHECKinitialized_(); return audio_device_->SetSpeakerMute(enable); } -int32_t ROSAudioDeviceModule::SpeakerMute(bool *enabled) const -{ +int32_t ROSAudioDeviceModule::SpeakerMute(bool* enabled) const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool muted = false; - if (audio_device_->SpeakerMute(muted) == -1) - { + if (audio_device_->SpeakerMute(muted) == -1) { return -1; } *enabled = muted; @@ -226,13 +201,11 @@ int32_t ROSAudioDeviceModule::SpeakerMute(bool *enabled) const return 0; } -int32_t ROSAudioDeviceModule::MicrophoneMuteIsAvailable(bool *available) -{ +int32_t ROSAudioDeviceModule::MicrophoneMuteIsAvailable(bool* available) { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool isAvailable = false; - if (audio_device_->MicrophoneMuteIsAvailable(isAvailable) == -1) - { + if (audio_device_->MicrophoneMuteIsAvailable(isAvailable) == -1) { return -1; } *available = isAvailable; @@ -240,20 +213,17 @@ int32_t ROSAudioDeviceModule::MicrophoneMuteIsAvailable(bool *available) return 0; } -int32_t ROSAudioDeviceModule::SetMicrophoneMute(bool enable) -{ +int32_t ROSAudioDeviceModule::SetMicrophoneMute(bool enable) { RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; CHECKinitialized_(); return (audio_device_->SetMicrophoneMute(enable)); } -int32_t ROSAudioDeviceModule::MicrophoneMute(bool *enabled) const -{ +int32_t ROSAudioDeviceModule::MicrophoneMute(bool* enabled) const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool muted = false; - if (audio_device_->MicrophoneMute(muted) == -1) - { + if (audio_device_->MicrophoneMute(muted) == -1) { return -1; } *enabled = muted; @@ -261,13 +231,11 @@ int32_t ROSAudioDeviceModule::MicrophoneMute(bool *enabled) const return 0; } -int32_t ROSAudioDeviceModule::MicrophoneVolumeIsAvailable(bool *available) -{ +int32_t ROSAudioDeviceModule::MicrophoneVolumeIsAvailable(bool* available) { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool isAvailable = false; - if (audio_device_->MicrophoneVolumeIsAvailable(isAvailable) == -1) - { + if (audio_device_->MicrophoneVolumeIsAvailable(isAvailable) == -1) { return -1; } *available = isAvailable; @@ -275,20 +243,17 @@ int32_t ROSAudioDeviceModule::MicrophoneVolumeIsAvailable(bool *available) return 0; } -int32_t ROSAudioDeviceModule::SetMicrophoneVolume(uint32_t volume) -{ +int32_t ROSAudioDeviceModule::SetMicrophoneVolume(uint32_t volume) { RTC_LOG(INFO) << __FUNCTION__ << "(" << volume << ")"; CHECKinitialized_(); return (audio_device_->SetMicrophoneVolume(volume)); } -int32_t ROSAudioDeviceModule::MicrophoneVolume(uint32_t *volume) const -{ +int32_t ROSAudioDeviceModule::MicrophoneVolume(uint32_t* volume) const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); uint32_t level = 0; - if (audio_device_->MicrophoneVolume(level) == -1) - { + if (audio_device_->MicrophoneVolume(level) == -1) { return -1; } *volume = level; @@ -297,13 +262,11 @@ int32_t ROSAudioDeviceModule::MicrophoneVolume(uint32_t *volume) const } int32_t ROSAudioDeviceModule::StereoRecordingIsAvailable( - bool *available) const -{ + bool* available) const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool isAvailable = false; - if (audio_device_->StereoRecordingIsAvailable(isAvailable) == -1) - { + if (audio_device_->StereoRecordingIsAvailable(isAvailable) == -1) { return -1; } *available = isAvailable; @@ -311,36 +274,30 @@ int32_t ROSAudioDeviceModule::StereoRecordingIsAvailable( return 0; } -int32_t ROSAudioDeviceModule::SetStereoRecording(bool enable) -{ +int32_t ROSAudioDeviceModule::SetStereoRecording(bool enable) { RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; CHECKinitialized_(); - if (audio_device_->RecordingIsInitialized()) - { + if (audio_device_->RecordingIsInitialized()) { RTC_LOG(WARNING) << "recording in stereo is not supported"; return -1; } - if (audio_device_->SetStereoRecording(enable) == -1) - { + if (audio_device_->SetStereoRecording(enable) == -1) { RTC_LOG(WARNING) << "failed to change stereo recording"; return -1; } int8_t nChannels(1); - if (enable) - { + if (enable) { nChannels = 2; } audio_device_buffer_.get()->SetRecordingChannels(nChannels); return 0; } -int32_t ROSAudioDeviceModule::StereoRecording(bool *enabled) const -{ +int32_t ROSAudioDeviceModule::StereoRecording(bool* enabled) const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool stereo = false; - if (audio_device_->StereoRecording(stereo) == -1) - { + if (audio_device_->StereoRecording(stereo) == -1) { return -1; } *enabled = stereo; @@ -348,13 +305,11 @@ int32_t ROSAudioDeviceModule::StereoRecording(bool *enabled) const return 0; } -int32_t ROSAudioDeviceModule::StereoPlayoutIsAvailable(bool *available) const -{ +int32_t ROSAudioDeviceModule::StereoPlayoutIsAvailable(bool* available) const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool isAvailable = false; - if (audio_device_->StereoPlayoutIsAvailable(isAvailable) == -1) - { + if (audio_device_->StereoPlayoutIsAvailable(isAvailable) == -1) { return -1; } *available = isAvailable; @@ -362,37 +317,31 @@ int32_t ROSAudioDeviceModule::StereoPlayoutIsAvailable(bool *available) const return 0; } -int32_t ROSAudioDeviceModule::SetStereoPlayout(bool enable) -{ +int32_t ROSAudioDeviceModule::SetStereoPlayout(bool enable) { RTC_LOG(INFO) << __FUNCTION__ << "(" << enable << ")"; CHECKinitialized_(); - if (audio_device_->PlayoutIsInitialized()) - { + if (audio_device_->PlayoutIsInitialized()) { RTC_LOG(LERROR) << "unable to set stereo mode while playing side is initialized"; return -1; } - if (audio_device_->SetStereoPlayout(enable)) - { + if (audio_device_->SetStereoPlayout(enable)) { RTC_LOG(WARNING) << "stereo playout is not supported"; return -1; } int8_t nChannels(1); - if (enable) - { + if (enable) { nChannels = 2; } audio_device_buffer_.get()->SetPlayoutChannels(nChannels); return 0; } -int32_t ROSAudioDeviceModule::StereoPlayout(bool *enabled) const -{ +int32_t ROSAudioDeviceModule::StereoPlayout(bool* enabled) const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool stereo = false; - if (audio_device_->StereoPlayout(stereo) == -1) - { + if (audio_device_->StereoPlayout(stereo) == -1) { return -1; } *enabled = stereo; @@ -400,13 +349,11 @@ int32_t ROSAudioDeviceModule::StereoPlayout(bool *enabled) const return 0; } -int32_t ROSAudioDeviceModule::PlayoutIsAvailable(bool *available) -{ +int32_t ROSAudioDeviceModule::PlayoutIsAvailable(bool* available) { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool isAvailable = false; - if (audio_device_->PlayoutIsAvailable(isAvailable) == -1) - { + if (audio_device_->PlayoutIsAvailable(isAvailable) == -1) { return -1; } *available = isAvailable; @@ -414,13 +361,11 @@ int32_t ROSAudioDeviceModule::PlayoutIsAvailable(bool *available) return 0; } -int32_t ROSAudioDeviceModule::RecordingIsAvailable(bool *available) -{ +int32_t ROSAudioDeviceModule::RecordingIsAvailable(bool* available) { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); bool isAvailable = false; - if (audio_device_->RecordingIsAvailable(isAvailable) == -1) - { + if (audio_device_->RecordingIsAvailable(isAvailable) == -1) { return -1; } *available = isAvailable; @@ -428,32 +373,27 @@ int32_t ROSAudioDeviceModule::RecordingIsAvailable(bool *available) return 0; } -int32_t ROSAudioDeviceModule::MaxMicrophoneVolume(uint32_t *maxVolume) const -{ +int32_t ROSAudioDeviceModule::MaxMicrophoneVolume(uint32_t* maxVolume) const { CHECKinitialized_(); uint32_t maxVol(0); - if (audio_device_->MaxMicrophoneVolume(maxVol) == -1) - { + if (audio_device_->MaxMicrophoneVolume(maxVol) == -1) { return -1; } *maxVolume = maxVol; return 0; } -int32_t ROSAudioDeviceModule::MinMicrophoneVolume(uint32_t *minVolume) const -{ +int32_t ROSAudioDeviceModule::MinMicrophoneVolume(uint32_t* minVolume) const { CHECKinitialized_(); uint32_t minVol(0); - if (audio_device_->MinMicrophoneVolume(minVol) == -1) - { + if (audio_device_->MinMicrophoneVolume(minVol) == -1) { return -1; } *minVolume = minVol; return 0; } -int16_t ROSAudioDeviceModule::PlayoutDevices() -{ +int16_t ROSAudioDeviceModule::PlayoutDevices() { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); uint16_t nPlayoutDevices = audio_device_->PlayoutDevices(); @@ -461,15 +401,13 @@ int16_t ROSAudioDeviceModule::PlayoutDevices() return (int16_t)(nPlayoutDevices); } -int32_t ROSAudioDeviceModule::SetPlayoutDevice(uint16_t index) -{ +int32_t ROSAudioDeviceModule::SetPlayoutDevice(uint16_t index) { RTC_LOG(INFO) << __FUNCTION__ << "(" << index << ")"; CHECKinitialized_(); return audio_device_->SetPlayoutDevice(index); } -int32_t ROSAudioDeviceModule::SetPlayoutDevice(WindowsDeviceType device) -{ +int32_t ROSAudioDeviceModule::SetPlayoutDevice(WindowsDeviceType device) { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); return audio_device_->SetPlayoutDevice(device); @@ -478,24 +416,19 @@ int32_t ROSAudioDeviceModule::SetPlayoutDevice(WindowsDeviceType device) int32_t ROSAudioDeviceModule::PlayoutDeviceName( uint16_t index, char name[webrtc::kAdmMaxDeviceNameSize], - char guid[webrtc::kAdmMaxGuidSize]) -{ + char guid[webrtc::kAdmMaxGuidSize]) { RTC_LOG(INFO) << __FUNCTION__ << "(" << index << ", ...)"; CHECKinitialized_(); - if (name == NULL) - { + if (name == NULL) { return -1; } - if (audio_device_->PlayoutDeviceName(index, name, guid) == -1) - { + if (audio_device_->PlayoutDeviceName(index, name, guid) == -1) { return -1; } - if (name != NULL) - { + if (name != NULL) { RTC_LOG(INFO) << "output: name = " << name; } - if (guid != NULL) - { + if (guid != NULL) { RTC_LOG(INFO) << "output: guid = " << guid; } return 0; @@ -504,31 +437,25 @@ int32_t ROSAudioDeviceModule::PlayoutDeviceName( int32_t ROSAudioDeviceModule::RecordingDeviceName( uint16_t index, char name[webrtc::kAdmMaxDeviceNameSize], - char guid[webrtc::kAdmMaxGuidSize]) -{ + char guid[webrtc::kAdmMaxGuidSize]) { RTC_LOG(INFO) << __FUNCTION__ << "(" << index << ", ...)"; CHECKinitialized_(); - if (name == NULL) - { + if (name == NULL) { return -1; } - if (audio_device_->RecordingDeviceName(index, name, guid) == -1) - { + if (audio_device_->RecordingDeviceName(index, name, guid) == -1) { return -1; } - if (name != NULL) - { + if (name != NULL) { RTC_LOG(INFO) << "output: name = " << name; } - if (guid != NULL) - { + if (guid != NULL) { RTC_LOG(INFO) << "output: guid = " << guid; } return 0; } -int16_t ROSAudioDeviceModule::RecordingDevices() -{ +int16_t ROSAudioDeviceModule::RecordingDevices() { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); uint16_t nRecordingDevices = audio_device_->RecordingDevices(); @@ -536,26 +463,22 @@ int16_t ROSAudioDeviceModule::RecordingDevices() return (int16_t)nRecordingDevices; } -int32_t ROSAudioDeviceModule::SetRecordingDevice(uint16_t index) -{ +int32_t ROSAudioDeviceModule::SetRecordingDevice(uint16_t index) { RTC_LOG(INFO) << __FUNCTION__ << "(" << index << ")"; CHECKinitialized_(); return audio_device_->SetRecordingDevice(index); } -int32_t ROSAudioDeviceModule::SetRecordingDevice(WindowsDeviceType device) -{ +int32_t ROSAudioDeviceModule::SetRecordingDevice(WindowsDeviceType device) { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); return audio_device_->SetRecordingDevice(device); } -int32_t ROSAudioDeviceModule::InitPlayout() -{ +int32_t ROSAudioDeviceModule::InitPlayout() { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); - if (PlayoutIsInitialized()) - { + if (PlayoutIsInitialized()) { return 0; } int32_t result = audio_device_->InitPlayout(); @@ -565,12 +488,10 @@ int32_t ROSAudioDeviceModule::InitPlayout() return result; } -int32_t ROSAudioDeviceModule::InitRecording() -{ +int32_t ROSAudioDeviceModule::InitRecording() { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); - if (RecordingIsInitialized()) - { + if (RecordingIsInitialized()) { return 0; } int32_t result = audio_device_->InitRecording(); @@ -580,26 +501,22 @@ int32_t ROSAudioDeviceModule::InitRecording() return result; } -bool ROSAudioDeviceModule::PlayoutIsInitialized() const -{ +bool ROSAudioDeviceModule::PlayoutIsInitialized() const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized__BOOL(); return audio_device_->PlayoutIsInitialized(); } -bool ROSAudioDeviceModule::RecordingIsInitialized() const -{ +bool ROSAudioDeviceModule::RecordingIsInitialized() const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized__BOOL(); return audio_device_->RecordingIsInitialized(); } -int32_t ROSAudioDeviceModule::StartPlayout() -{ +int32_t ROSAudioDeviceModule::StartPlayout() { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); - if (Playing()) - { + if (Playing()) { return 0; } audio_device_buffer_.get()->StartPlayout(); @@ -610,8 +527,7 @@ int32_t ROSAudioDeviceModule::StartPlayout() return result; } -int32_t ROSAudioDeviceModule::StopPlayout() -{ +int32_t ROSAudioDeviceModule::StopPlayout() { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); int32_t result = audio_device_->StopPlayout(); @@ -622,19 +538,16 @@ int32_t ROSAudioDeviceModule::StopPlayout() return result; } -bool ROSAudioDeviceModule::Playing() const -{ +bool ROSAudioDeviceModule::Playing() const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized__BOOL(); return audio_device_->Playing(); } -int32_t ROSAudioDeviceModule::StartRecording() -{ +int32_t ROSAudioDeviceModule::StartRecording() { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); - if (Recording()) - { + if (Recording()) { return 0; } audio_device_buffer_.get()->StartRecording(); @@ -645,8 +558,7 @@ int32_t ROSAudioDeviceModule::StartRecording() return result; } -int32_t ROSAudioDeviceModule::StopRecording() -{ +int32_t ROSAudioDeviceModule::StopRecording() { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized_(); int32_t result = audio_device_->StopRecording(); @@ -657,26 +569,22 @@ int32_t ROSAudioDeviceModule::StopRecording() return result; } -bool ROSAudioDeviceModule::Recording() const -{ +bool ROSAudioDeviceModule::Recording() const { RTC_LOG(INFO) << __FUNCTION__; CHECKinitialized__BOOL(); return audio_device_->Recording(); } int32_t ROSAudioDeviceModule::RegisterAudioCallback( - webrtc::AudioTransport *audioCallback) -{ + webrtc::AudioTransport* audioCallback) { RTC_LOG(INFO) << __FUNCTION__; return audio_device_buffer_.get()->RegisterAudioCallback(audioCallback); } -int32_t ROSAudioDeviceModule::PlayoutDelay(uint16_t *delayMS) const -{ +int32_t ROSAudioDeviceModule::PlayoutDelay(uint16_t* delayMS) const { CHECKinitialized_(); uint16_t delay = 0; - if (audio_device_->PlayoutDelay(delay) == -1) - { + if (audio_device_->PlayoutDelay(delay) == -1) { RTC_LOG(LERROR) << "failed to retrieve the playout delay"; return -1; } @@ -684,40 +592,33 @@ int32_t ROSAudioDeviceModule::PlayoutDelay(uint16_t *delayMS) const return 0; } -bool ROSAudioDeviceModule::BuiltInAECIsAvailable() const -{ +bool ROSAudioDeviceModule::BuiltInAECIsAvailable() const { return true; } -int32_t ROSAudioDeviceModule::EnableBuiltInAEC(bool enable) -{ +int32_t ROSAudioDeviceModule::EnableBuiltInAEC(bool enable) { return 0; } -bool ROSAudioDeviceModule::BuiltInAGCIsAvailable() const -{ +bool ROSAudioDeviceModule::BuiltInAGCIsAvailable() const { return true; } -int32_t ROSAudioDeviceModule::EnableBuiltInAGC(bool enable) -{ +int32_t ROSAudioDeviceModule::EnableBuiltInAGC(bool enable) { return 0; } -bool ROSAudioDeviceModule::BuiltInNSIsAvailable() const -{ +bool ROSAudioDeviceModule::BuiltInNSIsAvailable() const { return true; } -int32_t ROSAudioDeviceModule::EnableBuiltInNS(bool enable) -{ +int32_t ROSAudioDeviceModule::EnableBuiltInNS(bool enable) { return 0; } #if defined(WEBRTC_IOS) int ROSAudioDeviceModule::GetPlayoutAudioParameters( - webrtc::AudioParameters *params) const -{ + webrtc::AudioParameters* params) const { RTC_LOG(INFO) << __FUNCTION__; int r = audio_device_->GetPlayoutAudioParameters(params); RTC_LOG(INFO) << "output: " << r; @@ -725,11 +626,10 @@ int ROSAudioDeviceModule::GetPlayoutAudioParameters( } int ROSAudioDeviceModule::GetRecordAudioParameters( - webrtc::AudioParameters *params) const -{ + webrtc::AudioParameters* params) const { RTC_LOG(INFO) << __FUNCTION__; int r = audio_device_->GetRecordAudioParameters(params); RTC_LOG(INFO) << "output: " << r; return r; } -#endif // WEBRTC_IOS \ No newline at end of file +#endif // WEBRTC_IOS \ No newline at end of file diff --git a/src/ros/ros_audio_device_module.h b/src/ros/ros_audio_device_module.h index 86b7d427..c452c626 100644 --- a/src/ros/ros_audio_device_module.h +++ b/src/ros/ros_audio_device_module.h @@ -6,30 +6,28 @@ #include "api/task_queue/task_queue_factory.h" #include "modules/audio_device/include/audio_device.h" -#include "ros_audio_device.h" #include "connection_settings.h" +#include "ros_audio_device.h" #define CHECKinitialized_() \ { \ - if (!initialized_) \ - { \ + if (!initialized_) { \ return -1; \ } \ } #define CHECKinitialized__BOOL() \ { \ - if (!initialized_) \ - { \ + if (!initialized_) { \ return false; \ } \ } -class ROSAudioDeviceModule : public webrtc::AudioDeviceModule -{ -public: - static rtc::scoped_refptr Create(ConnectionSettings conn_settings, - webrtc::TaskQueueFactory* task_queue_factory); +class ROSAudioDeviceModule : public webrtc::AudioDeviceModule { + public: + static rtc::scoped_refptr Create( + ConnectionSettings conn_settings, + webrtc::TaskQueueFactory* task_queue_factory); int32_t AttachAudioBuffer(); ROSAudioDeviceModule(ConnectionSettings conn_settings, @@ -37,10 +35,10 @@ class ROSAudioDeviceModule : public webrtc::AudioDeviceModule ~ROSAudioDeviceModule() override; // Retrieve the currently utilized audio layer - int32_t ActiveAudioLayer(AudioLayer *audioLayer) const override; + int32_t ActiveAudioLayer(AudioLayer* audioLayer) const override; // Full-duplex transportation of PCM audio - int32_t RegisterAudioCallback(webrtc::AudioTransport *audioCallback) override; + int32_t RegisterAudioCallback(webrtc::AudioTransport* audioCallback) override; // Main initializaton and termination int32_t Init() override; @@ -64,10 +62,10 @@ class ROSAudioDeviceModule : public webrtc::AudioDeviceModule int32_t SetRecordingDevice(WindowsDeviceType device) override; // Audio transport initialization - int32_t PlayoutIsAvailable(bool *available) override; + int32_t PlayoutIsAvailable(bool* available) override; int32_t InitPlayout() override; bool PlayoutIsInitialized() const override; - int32_t RecordingIsAvailable(bool *available) override; + int32_t RecordingIsAvailable(bool* available) override; int32_t InitRecording() override; bool RecordingIsInitialized() const override; @@ -86,39 +84,39 @@ class ROSAudioDeviceModule : public webrtc::AudioDeviceModule bool MicrophoneIsInitialized() const override; // Speaker volume controls - int32_t SpeakerVolumeIsAvailable(bool *available) override; + int32_t SpeakerVolumeIsAvailable(bool* available) override; int32_t SetSpeakerVolume(uint32_t volume) override; - int32_t SpeakerVolume(uint32_t *volume) const override; - int32_t MaxSpeakerVolume(uint32_t *maxVolume) const override; - int32_t MinSpeakerVolume(uint32_t *minVolume) const override; + int32_t SpeakerVolume(uint32_t* volume) const override; + int32_t MaxSpeakerVolume(uint32_t* maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t* minVolume) const override; // Microphone volume controls - int32_t MicrophoneVolumeIsAvailable(bool *available) override; + int32_t MicrophoneVolumeIsAvailable(bool* available) override; int32_t SetMicrophoneVolume(uint32_t volume) override; - int32_t MicrophoneVolume(uint32_t *volume) const override; - int32_t MaxMicrophoneVolume(uint32_t *maxVolume) const override; - int32_t MinMicrophoneVolume(uint32_t *minVolume) const override; + int32_t MicrophoneVolume(uint32_t* volume) const override; + int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t* minVolume) const override; // Speaker mute control - int32_t SpeakerMuteIsAvailable(bool *available) override; + int32_t SpeakerMuteIsAvailable(bool* available) override; int32_t SetSpeakerMute(bool enable) override; - int32_t SpeakerMute(bool *enabled) const override; + int32_t SpeakerMute(bool* enabled) const override; // Microphone mute control - int32_t MicrophoneMuteIsAvailable(bool *available) override; + int32_t MicrophoneMuteIsAvailable(bool* available) override; int32_t SetMicrophoneMute(bool enable) override; - int32_t MicrophoneMute(bool *enabled) const override; + int32_t MicrophoneMute(bool* enabled) const override; // Stereo support - int32_t StereoPlayoutIsAvailable(bool *available) const override; + int32_t StereoPlayoutIsAvailable(bool* available) const override; int32_t SetStereoPlayout(bool enable) override; - int32_t StereoPlayout(bool *enabled) const override; - int32_t StereoRecordingIsAvailable(bool *available) const override; + int32_t StereoPlayout(bool* enabled) const override; + int32_t StereoRecordingIsAvailable(bool* available) const override; int32_t SetStereoRecording(bool enable) override; - int32_t StereoRecording(bool *enabled) const override; + int32_t StereoRecording(bool* enabled) const override; // Delay information and control - int32_t PlayoutDelay(uint16_t *delayMS) const override; + int32_t PlayoutDelay(uint16_t* delayMS) const override; bool BuiltInAECIsAvailable() const override; int32_t EnableBuiltInAEC(bool enable) override; @@ -128,10 +126,10 @@ class ROSAudioDeviceModule : public webrtc::AudioDeviceModule int32_t EnableBuiltInNS(bool enable) override; #if defined(WEBRTC_IOS) - int GetPlayoutAudioParameters(webrtc::AudioParameters *params) const override; - int GetRecordAudioParameters(webrtc::AudioParameters *params) const override; -#endif // WEBRTC_IOS -private: + int GetPlayoutAudioParameters(webrtc::AudioParameters* params) const override; + int GetRecordAudioParameters(webrtc::AudioParameters* params) const override; +#endif // WEBRTC_IOS + private: ConnectionSettings _conn_settings; webrtc::TaskQueueFactory* task_queue_factory_; bool initialized_ = false; diff --git a/src/ros/ros_log_sink.h b/src/ros/ros_log_sink.h index a421629c..0a878871 100644 --- a/src/ros/ros_log_sink.h +++ b/src/ros/ros_log_sink.h @@ -3,35 +3,32 @@ #include -#include "rtc_base/logging.h" #include "rtc_base/log_sinks.h" +#include "rtc_base/logging.h" -class ROSLogSink : public rtc::LogSink -{ -public: - void OnLogMessage(const std::string &message) override { +class ROSLogSink : public rtc::LogSink { + public: + void OnLogMessage(const std::string& message) override { ROS_INFO_STREAM(message); } - void OnLogMessage(const std::string &message, + void OnLogMessage(const std::string& message, rtc::LoggingSeverity severity, - const char *tag) override - { - switch (severity) - { - case rtc::LS_VERBOSE: - case rtc::LS_NONE: - ROS_DEBUG_STREAM(message); - break; - case rtc::LS_INFO: - ROS_INFO_STREAM(message); - break; - case rtc::LS_WARNING: - ROS_WARN_STREAM(message); - break; - case rtc::LS_ERROR: - ROS_ERROR_STREAM(message); - break; + const char* tag) override { + switch (severity) { + case rtc::LS_VERBOSE: + case rtc::LS_NONE: + ROS_DEBUG_STREAM(message); + break; + case rtc::LS_INFO: + ROS_INFO_STREAM(message); + break; + case rtc::LS_WARNING: + ROS_WARN_STREAM(message); + break; + case rtc::LS_ERROR: + ROS_ERROR_STREAM(message); + break; } } }; diff --git a/src/ros/ros_video_capture.cpp b/src/ros/ros_video_capture.cpp index a4e1a7ad..66261e1a 100644 --- a/src/ros/ros_video_capture.cpp +++ b/src/ros/ros_video_capture.cpp @@ -5,93 +5,89 @@ #include "api/video/i420_buffer.h" #include "rtc_base/log_sinks.h" -#include "third_party/libyuv/include/libyuv.h" #include "sensor_msgs/image_encodings.h" +#include "third_party/libyuv/include/libyuv.h" -ROSVideoCapture::ROSVideoCapture(ConnectionSettings cs) -{ +ROSVideoCapture::ROSVideoCapture(ConnectionSettings cs) { ros::NodeHandle nh; - if (cs.image_compressed) - { - sub_ = nh.subscribe(cs.camera_name, 1, boost::bind(&ROSVideoCapture::ROSCallbackCompressed, this, _1)); - } - else - { - sub_ = nh.subscribe(cs.camera_name, 1, boost::bind(&ROSVideoCapture::ROSCallbackRaw, this, _1)); + if (cs.image_compressed) { + sub_ = nh.subscribe( + cs.camera_name, 1, + boost::bind(&ROSVideoCapture::ROSCallbackCompressed, this, _1)); + } else { + sub_ = nh.subscribe( + cs.camera_name, 1, + boost::bind(&ROSVideoCapture::ROSCallbackRaw, this, _1)); } spinner_ = new ros::AsyncSpinner(1); spinner_->start(); } -ROSVideoCapture::~ROSVideoCapture() -{ +ROSVideoCapture::~ROSVideoCapture() { Destroy(); } -void ROSVideoCapture::Destroy() -{ +void ROSVideoCapture::Destroy() { spinner_->stop(); } -void ROSVideoCapture::ROSCallbackRaw(const sensor_msgs::ImageConstPtr &image) -{ - ROSCallback(image->header.stamp, image->data.data(), image->data.size(), image->width, image->height, ConvertEncodingType(image->encoding)); +void ROSVideoCapture::ROSCallbackRaw(const sensor_msgs::ImageConstPtr& image) { + ROSCallback(image->header.stamp, image->data.data(), image->data.size(), + image->width, image->height, + ConvertEncodingType(image->encoding)); } -void ROSVideoCapture::ROSCallbackCompressed(const sensor_msgs::CompressedImageConstPtr &image) -{ +void ROSVideoCapture::ROSCallbackCompressed( + const sensor_msgs::CompressedImageConstPtr& image) { int width, height; - if (libyuv::MJPGSize(image->data.data(), image->data.size(), &width, &height) < 0) - { + if (libyuv::MJPGSize(image->data.data(), image->data.size(), &width, + &height) < 0) { RTC_LOG(LS_ERROR) << "MJPGSize Failed"; return; } - ROSCallback(image->header.stamp, image->data.data(), image->data.size(), width, height, libyuv::FOURCC_MJPG); + ROSCallback(image->header.stamp, image->data.data(), image->data.size(), + width, height, libyuv::FOURCC_MJPG); } -uint32_t ROSVideoCapture::ConvertEncodingType(const std::string encoding) -{ - if (encoding == sensor_msgs::image_encodings::RGB8) - { +uint32_t ROSVideoCapture::ConvertEncodingType(const std::string encoding) { + if (encoding == sensor_msgs::image_encodings::RGB8) { return libyuv::FOURCC_RAW; - } - else if (encoding == sensor_msgs::image_encodings::BGR8) - { + } else if (encoding == sensor_msgs::image_encodings::BGR8) { return libyuv::FOURCC_24BG; - } - else if (encoding == sensor_msgs::image_encodings::RGBA8) - { + } else if (encoding == sensor_msgs::image_encodings::RGBA8) { return libyuv::FOURCC_BGRA; - } - else if (encoding == sensor_msgs::image_encodings::BGRA8) - { + } else if (encoding == sensor_msgs::image_encodings::BGRA8) { return libyuv::FOURCC_RGBA; } return libyuv::FOURCC_ANY; } -void ROSVideoCapture::ROSCallback(ros::Time ros_time, const uint8_t *sample, size_t sample_size, int src_width, int src_height, uint32_t fourcc) -{ - rtc::scoped_refptr dst_buffer(webrtc::I420Buffer::Create(src_width, src_height)); +void ROSVideoCapture::ROSCallback(ros::Time ros_time, + const uint8_t* sample, + size_t sample_size, + int src_width, + int src_height, + uint32_t fourcc) { + rtc::scoped_refptr dst_buffer( + webrtc::I420Buffer::Create(src_width, src_height)); dst_buffer->InitializeData(); - if (libyuv::ConvertToI420(sample, sample_size, - dst_buffer.get()->MutableDataY(), dst_buffer.get()->StrideY(), - dst_buffer.get()->MutableDataU(), dst_buffer.get()->StrideU(), - dst_buffer.get()->MutableDataV(), dst_buffer.get()->StrideV(), - 0, 0, src_width, src_height, src_width, src_height, - libyuv::kRotate0, fourcc) < 0) - { + if (libyuv::ConvertToI420( + sample, sample_size, dst_buffer.get()->MutableDataY(), + dst_buffer.get()->StrideY(), dst_buffer.get()->MutableDataU(), + dst_buffer.get()->StrideU(), dst_buffer.get()->MutableDataV(), + dst_buffer.get()->StrideV(), 0, 0, src_width, src_height, src_width, + src_height, libyuv::kRotate0, fourcc) < 0) { RTC_LOG(LS_ERROR) << "ConvertToI420 Failed"; return; } webrtc::VideoFrame captureFrame = - webrtc::VideoFrame::Builder() - .set_video_frame_buffer(dst_buffer) - .set_rotation(webrtc::kVideoRotation_0) - .set_timestamp_us((int64_t)(ros_time.toNSec() / 1000)) - .build(); + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(dst_buffer) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us((int64_t)(ros_time.toNSec() / 1000)) + .build(); OnCapturedFrame(captureFrame); } \ No newline at end of file diff --git a/src/ros/ros_video_capture.h b/src/ros/ros_video_capture.h index a69d761c..99635b6d 100644 --- a/src/ros/ros_video_capture.h +++ b/src/ros/ros_video_capture.h @@ -3,28 +3,31 @@ #include "ros/ros.h" -#include "sensor_msgs/Image.h" #include "sensor_msgs/CompressedImage.h" - +#include "sensor_msgs/Image.h" #include "connection_settings.h" #include "rtc/scalable_track_source.h" -class ROSVideoCapture : public ScalableVideoTrackSource -{ -public: +class ROSVideoCapture : public ScalableVideoTrackSource { + public: explicit ROSVideoCapture(ConnectionSettings cs); ~ROSVideoCapture(); void Destroy(); // ROS Callback - void ROSCallbackRaw(const sensor_msgs::ImageConstPtr &image); - void ROSCallbackCompressed(const sensor_msgs::CompressedImageConstPtr &image); + void ROSCallbackRaw(const sensor_msgs::ImageConstPtr& image); + void ROSCallbackCompressed(const sensor_msgs::CompressedImageConstPtr& image); -private: + private: static uint32_t ConvertEncodingType(const std::string encoding); - void ROSCallback(ros::Time ros_time, const uint8_t* sample, size_t sample_size, int src_width, int src_height, uint32_t fourcc); + void ROSCallback(ros::Time ros_time, + const uint8_t* sample, + size_t sample_size, + int src_width, + int src_height, + uint32_t fourcc); ros::AsyncSpinner* spinner_; ros::Subscriber sub_; diff --git a/src/rtc/connection.cpp b/src/rtc/connection.cpp index fb4e45a8..ae7980d7 100644 --- a/src/rtc/connection.cpp +++ b/src/rtc/connection.cpp @@ -2,138 +2,132 @@ #include "connection.h" -RTCConnection::~RTCConnection() -{ +RTCConnection::~RTCConnection() { _connection->Close(); } -void RTCConnection::createOffer() -{ - using RTCOfferAnswerOptions = webrtc::PeerConnectionInterface::RTCOfferAnswerOptions; +void RTCConnection::createOffer() { + using RTCOfferAnswerOptions = + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions; RTCOfferAnswerOptions options = RTCOfferAnswerOptions(); - options.offer_to_receive_video = RTCOfferAnswerOptions::kOfferToReceiveMediaTrue; - options.offer_to_receive_audio = RTCOfferAnswerOptions::kOfferToReceiveMediaTrue; + options.offer_to_receive_video = + RTCOfferAnswerOptions::kOfferToReceiveMediaTrue; + options.offer_to_receive_audio = + RTCOfferAnswerOptions::kOfferToReceiveMediaTrue; _connection->CreateOffer( - CreateSessionDescriptionObserver::Create(_sender, _connection), options); + CreateSessionDescriptionObserver::Create(_sender, _connection), options); } -void RTCConnection::setOffer(const std::string sdp) -{ +void RTCConnection::setOffer(const std::string sdp) { webrtc::SdpParseError error; - std::unique_ptr session_description - = webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp, &error); - if (!session_description) - { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to create session description: " - << error.description.c_str() << "\nline: " << error.line.c_str(); + std::unique_ptr session_description = + webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp, &error); + if (!session_description) { + RTC_LOG(LS_ERROR) << __FUNCTION__ + << "Failed to create session description: " + << error.description.c_str() + << "\nline: " << error.line.c_str(); return; } _connection->SetRemoteDescription( - SetSessionDescriptionObserver::Create( - session_description->GetType(), _sender), - session_description.release()); + SetSessionDescriptionObserver::Create(session_description->GetType(), + _sender), + session_description.release()); } -void RTCConnection::createAnswer() -{ +void RTCConnection::createAnswer() { _connection->CreateAnswer( - CreateSessionDescriptionObserver::Create(_sender, _connection), - webrtc::PeerConnectionInterface::RTCOfferAnswerOptions()); + CreateSessionDescriptionObserver::Create(_sender, _connection), + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions()); } -void RTCConnection::setAnswer(const std::string sdp) -{ +void RTCConnection::setAnswer(const std::string sdp) { webrtc::SdpParseError error; - std::unique_ptr session_description - = webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp, &error); - if (!session_description) - { - RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to create session description: " - << error.description.c_str() << "\nline: " << error.line.c_str(); + std::unique_ptr session_description = + webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp, &error); + if (!session_description) { + RTC_LOG(LS_ERROR) << __FUNCTION__ + << "Failed to create session description: " + << error.description.c_str() + << "\nline: " << error.line.c_str(); return; } _connection->SetRemoteDescription( - SetSessionDescriptionObserver::Create( - session_description->GetType(), _sender), - session_description.release()); + SetSessionDescriptionObserver::Create(session_description->GetType(), + _sender), + session_description.release()); } -void RTCConnection::addIceCandidate( - const std::string sdp_mid, const int sdp_mlineindex, const std::string sdp) -{ +void RTCConnection::addIceCandidate(const std::string sdp_mid, + const int sdp_mlineindex, + const std::string sdp) { webrtc::SdpParseError error; std::unique_ptr candidate( - webrtc::CreateIceCandidate(sdp_mid, sdp_mlineindex, sdp, &error)); - if (!candidate.get()) - { + webrtc::CreateIceCandidate(sdp_mid, sdp_mlineindex, sdp, &error)); + if (!candidate.get()) { RTC_LOG(LS_ERROR) << "Can't parse received candidate message: " - << error.description.c_str() << "\nline: " << error.line.c_str(); + << error.description.c_str() + << "\nline: " << error.line.c_str(); return; } - if (!_connection->AddIceCandidate(candidate.get())) - { - RTC_LOG(LS_WARNING) << __FUNCTION__ << "Failed to apply the received candidate : " << sdp; + if (!_connection->AddIceCandidate(candidate.get())) { + RTC_LOG(LS_WARNING) << __FUNCTION__ + << "Failed to apply the received candidate : " << sdp; return; } } -bool RTCConnection::setAudioEnabled(bool enabled) -{ +bool RTCConnection::setAudioEnabled(bool enabled) { return setMediaEnabled(getLocalAudioTrack(), enabled); } -bool RTCConnection::setVideoEnabled(bool enabled) -{ +bool RTCConnection::setVideoEnabled(bool enabled) { return setMediaEnabled(getLocalVideoTrack(), enabled); } -bool RTCConnection::isAudioEnabled() -{ +bool RTCConnection::isAudioEnabled() { return isMediaEnabled(getLocalAudioTrack()); } -bool RTCConnection::isVideoEnabled() -{ +bool RTCConnection::isVideoEnabled() { return isMediaEnabled(getLocalVideoTrack()); } -rtc::scoped_refptr RTCConnection::getLocalStream() -{ +rtc::scoped_refptr +RTCConnection::getLocalStream() { return _connection->local_streams()->at(0); } -rtc::scoped_refptr RTCConnection::getLocalAudioTrack() -{ - rtc::scoped_refptr local_stream = getLocalStream(); - if (!local_stream) - { +rtc::scoped_refptr +RTCConnection::getLocalAudioTrack() { + rtc::scoped_refptr local_stream = + getLocalStream(); + if (!local_stream) { return nullptr; } - if (local_stream->GetAudioTracks().size() > 0) - { - rtc::scoped_refptr audio_track(local_stream->GetAudioTracks()[0]); - if (audio_track) - { + if (local_stream->GetAudioTracks().size() > 0) { + rtc::scoped_refptr audio_track( + local_stream->GetAudioTracks()[0]); + if (audio_track) { return audio_track; } } return nullptr; } -rtc::scoped_refptr RTCConnection::getLocalVideoTrack() -{ - rtc::scoped_refptr local_stream = getLocalStream(); - if (!local_stream) - { +rtc::scoped_refptr +RTCConnection::getLocalVideoTrack() { + rtc::scoped_refptr local_stream = + getLocalStream(); + if (!local_stream) { return nullptr; } - if (local_stream->GetVideoTracks().size() > 0) - { - rtc::scoped_refptr video_track(local_stream->GetVideoTracks()[0]); - if (video_track) - { + if (local_stream->GetVideoTracks().size() > 0) { + rtc::scoped_refptr video_track( + local_stream->GetVideoTracks()[0]); + if (video_track) { return video_track; } } @@ -141,20 +135,17 @@ rtc::scoped_refptr RTCConnection::getLocalVideoTrac } bool RTCConnection::setMediaEnabled( - rtc::scoped_refptr track, bool enabled) -{ - if (track) - { + rtc::scoped_refptr track, + bool enabled) { + if (track) { return track->set_enabled(enabled); } return false; } bool RTCConnection::isMediaEnabled( - rtc::scoped_refptr track) -{ - if (track) - { + rtc::scoped_refptr track) { + if (track) { return track->enabled(); } return false; diff --git a/src/rtc/connection.h b/src/rtc/connection.h index 4d8f63f4..1def26f1 100644 --- a/src/rtc/connection.h +++ b/src/rtc/connection.h @@ -4,36 +4,38 @@ #include "observer.h" -class RTCConnection -{ -public: - RTCConnection(RTCMessageSender *sender, - std::unique_ptr observer, - rtc::scoped_refptr connection) - : _sender(sender), _observer(std::move(observer)), _connection(connection) {}; +class RTCConnection { + public: + RTCConnection(RTCMessageSender* sender, + std::unique_ptr observer, + rtc::scoped_refptr connection) + : _sender(sender), + _observer(std::move(observer)), + _connection(connection){}; ~RTCConnection(); void createOffer(); void setOffer(const std::string sdp); void createAnswer(); void setAnswer(const std::string sdp); void addIceCandidate(const std::string sdp_mid, - const int sdp_mlineindex, const std::string sdp); + const int sdp_mlineindex, + const std::string sdp); bool setAudioEnabled(bool enabled); bool setVideoEnabled(bool enabled); bool isAudioEnabled(); bool isVideoEnabled(); -private: + private: rtc::scoped_refptr getLocalStream(); rtc::scoped_refptr getLocalAudioTrack(); rtc::scoped_refptr getLocalVideoTrack(); bool setMediaEnabled( - rtc::scoped_refptr track, - bool enabled); + rtc::scoped_refptr track, + bool enabled); bool isMediaEnabled( - rtc::scoped_refptr track); + rtc::scoped_refptr track); - RTCMessageSender *_sender; + RTCMessageSender* _sender; std::unique_ptr _observer; rtc::scoped_refptr _connection; }; diff --git a/src/rtc/device_video_capturer.cpp b/src/rtc/device_video_capturer.cpp index 2ea882c9..386dec61 100644 --- a/src/rtc/device_video_capturer.cpp +++ b/src/rtc/device_video_capturer.cpp @@ -24,9 +24,9 @@ DeviceVideoCapturer::~DeviceVideoCapturer() { } bool DeviceVideoCapturer::Init(size_t width, - size_t height, - size_t target_fps, - size_t capture_device_index) { + size_t height, + size_t target_fps, + size_t capture_device_index) { std::unique_ptr device_info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); @@ -62,10 +62,8 @@ bool DeviceVideoCapturer::Init(size_t width, return true; } -rtc::scoped_refptr DeviceVideoCapturer::Create(size_t width, - size_t height, - size_t target_fps) -{ +rtc::scoped_refptr +DeviceVideoCapturer::Create(size_t width, size_t height, size_t target_fps) { rtc::scoped_refptr capturer; std::unique_ptr info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); @@ -86,12 +84,13 @@ rtc::scoped_refptr DeviceVideoCapturer::Create(size_t width return nullptr; } -rtc::scoped_refptr DeviceVideoCapturer::Create(size_t width, - size_t height, - size_t target_fps, - size_t capture_device_index) { +rtc::scoped_refptr DeviceVideoCapturer::Create( + size_t width, + size_t height, + size_t target_fps, + size_t capture_device_index) { rtc::scoped_refptr vcm_capturer( - new rtc::RefCountedObject()); + new rtc::RefCountedObject()); if (!vcm_capturer->Init(width, height, target_fps, capture_device_index)) { RTC_LOG(LS_WARNING) << "Failed to create DeviceVideoCapturer(w = " << width << ", h = " << height << ", fps = " << target_fps @@ -111,7 +110,6 @@ void DeviceVideoCapturer::Destroy() { vcm_ = nullptr; } -void DeviceVideoCapturer::OnFrame(const webrtc::VideoFrame &frame) -{ +void DeviceVideoCapturer::OnFrame(const webrtc::VideoFrame& frame) { OnCapturedFrame(frame); } \ No newline at end of file diff --git a/src/rtc/device_video_capturer.h b/src/rtc/device_video_capturer.h index d407e284..d398806b 100644 --- a/src/rtc/device_video_capturer.h +++ b/src/rtc/device_video_capturer.h @@ -14,20 +14,22 @@ #include #include "api/scoped_refptr.h" -#include "rtc_base/ref_counted_object.h" #include "modules/video_capture/video_capture.h" +#include "rtc_base/ref_counted_object.h" #include "rtc/scalable_track_source.h" -class DeviceVideoCapturer : public ScalableVideoTrackSource, public rtc::VideoSinkInterface { +class DeviceVideoCapturer : public ScalableVideoTrackSource, + public rtc::VideoSinkInterface { public: static rtc::scoped_refptr Create(size_t width, - size_t height, - size_t target_fps); - static rtc::scoped_refptr Create(size_t width, - size_t height, - size_t target_fps, - size_t capture_device_index); + size_t height, + size_t target_fps); + static rtc::scoped_refptr Create( + size_t width, + size_t height, + size_t target_fps, + size_t capture_device_index); DeviceVideoCapturer(); virtual ~DeviceVideoCapturer(); @@ -39,7 +41,7 @@ class DeviceVideoCapturer : public ScalableVideoTrackSource, public rtc::VideoSi void Destroy(); // rtc::VideoSinkInterface interface. - void OnFrame(const webrtc::VideoFrame &frame) override; + void OnFrame(const webrtc::VideoFrame& frame) override; rtc::scoped_refptr vcm_; webrtc::VideoCaptureCapability capability_; diff --git a/src/rtc/hw_video_encoder_factory.cpp b/src/rtc/hw_video_encoder_factory.cpp index 5ef6d453..e1bc9ec9 100644 --- a/src/rtc/hw_video_encoder_factory.cpp +++ b/src/rtc/hw_video_encoder_factory.cpp @@ -1,7 +1,7 @@ #include "hw_video_encoder_factory.h" -#include "absl/strings/match.h" #include "absl/memory/memory.h" +#include "absl/strings/match.h" #include "api/video_codecs/sdp_video_format.h" #include "media/base/codec.h" #include "media/base/media_constants.h" @@ -50,10 +50,12 @@ std::unique_ptr HWVideoEncoderFactory::CreateVideoEncoder( if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) { #if USE_MMAL_ENCODER - return std::unique_ptr(absl::make_unique(cricket::VideoCodec(format))); + return std::unique_ptr( + absl::make_unique(cricket::VideoCodec(format))); #endif #if USE_JETSON_ENCODER - return std::unique_ptr(absl::make_unique(cricket::VideoCodec(format))); + return std::unique_ptr( + absl::make_unique(cricket::VideoCodec(format))); #endif } diff --git a/src/rtc/hw_video_encoder_factory.h b/src/rtc/hw_video_encoder_factory.h index ef64d841..bd36548e 100644 --- a/src/rtc/hw_video_encoder_factory.h +++ b/src/rtc/hw_video_encoder_factory.h @@ -8,18 +8,18 @@ #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory.h" -class HWVideoEncoderFactory : public webrtc::VideoEncoderFactory -{ - public: - HWVideoEncoderFactory() {} - virtual ~HWVideoEncoderFactory() {} +class HWVideoEncoderFactory : public webrtc::VideoEncoderFactory { + public: + HWVideoEncoderFactory() {} + virtual ~HWVideoEncoderFactory() {} - std::vector GetSupportedFormats() const override; + std::vector GetSupportedFormats() const override; - CodecInfo QueryVideoEncoder(const webrtc::SdpVideoFormat& format) const override; + CodecInfo QueryVideoEncoder( + const webrtc::SdpVideoFormat& format) const override; - std::unique_ptr CreateVideoEncoder( + std::unique_ptr CreateVideoEncoder( const webrtc::SdpVideoFormat& format) override; }; -#endif // HW_VIDEO_ENCODER_FACTORY_H_ \ No newline at end of file +#endif // HW_VIDEO_ENCODER_FACTORY_H_ \ No newline at end of file diff --git a/src/rtc/manager.cpp b/src/rtc/manager.cpp index 5b9fd93c..efac9925 100644 --- a/src/rtc/manager.cpp +++ b/src/rtc/manager.cpp @@ -1,24 +1,24 @@ #include +#include "absl/memory/memory.h" #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" #include "api/create_peerconnection_factory.h" +#include "api/rtc_event_log/rtc_event_log_factory.h" #include "api/task_queue/default_task_queue_factory.h" #include "api/video_track_source_proxy.h" -#include "api/rtc_event_log/rtc_event_log_factory.h" #include "media/engine/webrtc_media_engine.h" #include "modules/audio_device/include/audio_device.h" #include "modules/audio_processing/include/audio_processing.h" #include "modules/video_capture/video_capture.h" #include "modules/video_capture/video_capture_factory.h" -#include "rtc_base/ssl_adapter.h" #include "rtc_base/logging.h" -#include "absl/memory/memory.h" +#include "rtc_base/ssl_adapter.h" -#include "scalable_track_source.h" #include "manager.h" #include "observer.h" +#include "scalable_track_source.h" #include "util.h" #ifdef __APPLE__ @@ -37,10 +37,10 @@ #include "hw_video_encoder_factory.h" #endif -RTCManager::RTCManager(ConnectionSettings conn_settings, - rtc::scoped_refptr video_track_source) - : _conn_settings(conn_settings) -{ +RTCManager::RTCManager( + ConnectionSettings conn_settings, + rtc::scoped_refptr video_track_source) + : _conn_settings(conn_settings) { rtc::InitializeSSL(); _networkThread = rtc::Thread::CreateWithSocketServer(); @@ -51,12 +51,13 @@ RTCManager::RTCManager(ConnectionSettings conn_settings, _signalingThread->Start(); #if __linux__ - webrtc::AudioDeviceModule::AudioLayer audio_layer = webrtc::AudioDeviceModule::kLinuxAlsaAudio; + webrtc::AudioDeviceModule::AudioLayer audio_layer = + webrtc::AudioDeviceModule::kLinuxAlsaAudio; #else - webrtc::AudioDeviceModule::AudioLayer audio_layer = webrtc::AudioDeviceModule::kPlatformDefaultAudio; + webrtc::AudioDeviceModule::AudioLayer audio_layer = + webrtc::AudioDeviceModule::kPlatformDefaultAudio; #endif - if (_conn_settings.no_audio) - { + if (_conn_settings.no_audio) { audio_layer = webrtc::AudioDeviceModule::kDummyAudio; } @@ -66,41 +67,49 @@ RTCManager::RTCManager(ConnectionSettings conn_settings, dependencies.signaling_thread = _signalingThread.get(); dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); dependencies.call_factory = webrtc::CreateCallFactory(); - dependencies.event_log_factory = absl::make_unique( - dependencies.task_queue_factory.get()); + dependencies.event_log_factory = + absl::make_unique( + dependencies.task_queue_factory.get()); // media_dependencies cricket::MediaEngineDependencies media_dependencies; media_dependencies.task_queue_factory = dependencies.task_queue_factory.get(); #if USE_ROS - media_dependencies.adm = ROSAudioDeviceModule::Create(_conn_settings, dependencies.task_queue_factory.get()); + media_dependencies.adm = ROSAudioDeviceModule::Create( + _conn_settings, dependencies.task_queue_factory.get()); #else - media_dependencies.adm = webrtc::AudioDeviceModule::Create(audio_layer, dependencies.task_queue_factory.get()); + media_dependencies.adm = webrtc::AudioDeviceModule::Create( + audio_layer, dependencies.task_queue_factory.get()); #endif - media_dependencies.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory(); - media_dependencies.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory(); + media_dependencies.audio_encoder_factory = + webrtc::CreateBuiltinAudioEncoderFactory(); + media_dependencies.audio_decoder_factory = + webrtc::CreateBuiltinAudioDecoderFactory(); #ifdef __APPLE__ - media_dependencies.video_encoder_factory = CreateObjCEncoderFactory(); - media_dependencies.video_decoder_factory = CreateObjCDecoderFactory(); + media_dependencies.video_encoder_factory = CreateObjCEncoderFactory(); + media_dependencies.video_decoder_factory = CreateObjCDecoderFactory(); #else #if USE_MMAL_ENCODER || USE_JETSON_ENCODER media_dependencies.video_encoder_factory = std::unique_ptr( absl::make_unique()); #else - media_dependencies.video_encoder_factory = webrtc::CreateBuiltinVideoEncoderFactory(); + media_dependencies.video_encoder_factory = + webrtc::CreateBuiltinVideoEncoderFactory(); #endif - media_dependencies.video_decoder_factory = webrtc::CreateBuiltinVideoDecoderFactory(); + media_dependencies.video_decoder_factory = + webrtc::CreateBuiltinVideoDecoderFactory(); #endif media_dependencies.audio_mixer = nullptr; - media_dependencies.audio_processing = webrtc::AudioProcessingBuilder().Create(); + media_dependencies.audio_processing = + webrtc::AudioProcessingBuilder().Create(); dependencies.media_engine = cricket::CreateMediaEngine(std::move(media_dependencies)); - _factory = webrtc::CreateModularPeerConnectionFactory(std::move(dependencies)); - if (!_factory.get()) - { + _factory = + webrtc::CreateModularPeerConnectionFactory(std::move(dependencies)); + if (!_factory.get()) { RTC_LOG(LS_ERROR) << __FUNCTION__ << ": Failed to initialize PeerConnectionFactory"; exit(1); @@ -112,9 +121,7 @@ RTCManager::RTCManager(ConnectionSettings conn_settings, factory_options.ssl_max_version = rtc::SSL_PROTOCOL_DTLS_12; _factory->SetOptions(factory_options); - - if (!_conn_settings.no_audio) - { + if (!_conn_settings.no_audio) { cricket::AudioOptions ao; if (_conn_settings.disable_echo_cancellation) ao.echo_cancellation = false; @@ -129,33 +136,29 @@ RTCManager::RTCManager(ConnectionSettings conn_settings, RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ao.ToString(); _audio_track = _factory->CreateAudioTrack(Util::generateRandomChars(), _factory->CreateAudioSource(ao)); - if (!_audio_track) - { + if (!_audio_track) { RTC_LOG(LS_WARNING) << __FUNCTION__ << ": Cannot create audio_track"; } } - - if (video_track_source && !_conn_settings.no_video) - { - rtc::scoped_refptr video_source = - webrtc::VideoTrackSourceProxy::Create(_signalingThread.get(), _workerThread.get(), video_track_source); - _video_track = _factory->CreateVideoTrack(Util::generateRandomChars(), video_source); - if (_video_track) - { + if (video_track_source && !_conn_settings.no_video) { + rtc::scoped_refptr video_source = + webrtc::VideoTrackSourceProxy::Create( + _signalingThread.get(), _workerThread.get(), video_track_source); + _video_track = + _factory->CreateVideoTrack(Util::generateRandomChars(), video_source); + if (_video_track) { if (_conn_settings.fixed_resolution) { - _video_track->set_content_hint(webrtc::VideoTrackInterface::ContentHint::kText); + _video_track->set_content_hint( + webrtc::VideoTrackInterface::ContentHint::kText); } - } - else - { + } else { RTC_LOG(LS_WARNING) << __FUNCTION__ << ": Cannot create video_track"; } } } -RTCManager::~RTCManager() -{ +RTCManager::~RTCManager() { _audio_track = nullptr; _video_track = nullptr; _factory = nullptr; @@ -167,40 +170,36 @@ RTCManager::~RTCManager() } std::shared_ptr RTCManager::createConnection( - webrtc::PeerConnectionInterface::RTCConfiguration rtc_config, - RTCMessageSender *sender) -{ + webrtc::PeerConnectionInterface::RTCConfiguration rtc_config, + RTCMessageSender* sender) { rtc_config.enable_dtls_srtp = true; rtc_config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; - std::unique_ptr observer(new PeerConnectionObserver(sender)); - rtc::scoped_refptr connection = - _factory->CreatePeerConnection( - rtc_config, nullptr, nullptr, observer.get()); - if (!connection) - { + std::unique_ptr observer( + new PeerConnectionObserver(sender)); + rtc::scoped_refptr connection = + _factory->CreatePeerConnection(rtc_config, nullptr, nullptr, + observer.get()); + if (!connection) { RTC_LOG(LS_ERROR) << __FUNCTION__ << ": CreatePeerConnection failed"; return nullptr; } std::string stream_id = Util::generateRandomChars(); - if (_audio_track) - { - webrtc::RTCErrorOr > audio_sender = - connection->AddTrack(_audio_track, {stream_id}); - if (!audio_sender.ok()) - { + if (_audio_track) { + webrtc::RTCErrorOr > + audio_sender = connection->AddTrack(_audio_track, {stream_id}); + if (!audio_sender.ok()) { RTC_LOG(LS_WARNING) << __FUNCTION__ << ": Cannot add _audio_track"; } } - if (_video_track) - { - webrtc::RTCErrorOr > video_add_result = - connection->AddTrack(_video_track, {stream_id}); - if (video_add_result.ok()) - { - rtc::scoped_refptr video_sender = video_add_result.value(); + if (_video_track) { + webrtc::RTCErrorOr > + video_add_result = connection->AddTrack(_video_track, {stream_id}); + if (video_add_result.ok()) { + rtc::scoped_refptr video_sender = + video_add_result.value(); webrtc::RtpParameters parameters = video_sender->GetParameters(); parameters.degradation_preference = _conn_settings.getPriority(); video_sender->SetParameters(parameters); @@ -209,5 +208,6 @@ std::shared_ptr RTCManager::createConnection( } } - return std::make_shared(sender, std::move(observer), connection); + return std::make_shared(sender, std::move(observer), + connection); } diff --git a/src/rtc/manager.h b/src/rtc/manager.h index 1da8e1d7..693b129f 100644 --- a/src/rtc/manager.h +++ b/src/rtc/manager.h @@ -7,17 +7,16 @@ #include "connection_settings.h" #include "scalable_track_source.h" -class RTCManager -{ -public: +class RTCManager { + public: RTCManager(ConnectionSettings conn_settings, rtc::scoped_refptr video_track_source); ~RTCManager(); std::shared_ptr createConnection( - webrtc::PeerConnectionInterface::RTCConfiguration rtc_config, - RTCMessageSender *sender); + webrtc::PeerConnectionInterface::RTCConfiguration rtc_config, + RTCMessageSender* sender); -private: + private: rtc::scoped_refptr _factory; rtc::scoped_refptr _audio_track; rtc::scoped_refptr _video_track; diff --git a/src/rtc/messagesender.h b/src/rtc/messagesender.h index f37ca54a..d1dea7ac 100644 --- a/src/rtc/messagesender.h +++ b/src/rtc/messagesender.h @@ -4,14 +4,15 @@ #include #include "api/peer_connection_interface.h" -class RTCMessageSender -{ -public: +class RTCMessageSender { + public: virtual void onIceConnectionStateChange( - webrtc::PeerConnectionInterface::IceConnectionState new_state) = 0; + webrtc::PeerConnectionInterface::IceConnectionState new_state) = 0; virtual void onIceCandidate(const std::string sdp_mid, - const int sdp_mlineindex, const std::string sdp) = 0; - virtual void onCreateDescription(webrtc::SdpType type, const std::string sdp) = 0; + const int sdp_mlineindex, + const std::string sdp) = 0; + virtual void onCreateDescription(webrtc::SdpType type, + const std::string sdp) = 0; virtual void onSetDescription(webrtc::SdpType type) = 0; }; diff --git a/src/rtc/native_buffer.cpp b/src/rtc/native_buffer.cpp index f5e774b7..b97b84b4 100644 --- a/src/rtc/native_buffer.cpp +++ b/src/rtc/native_buffer.cpp @@ -1,95 +1,81 @@ #include "native_buffer.h" -#include "rtc_base/checks.h" #include "api/video/i420_buffer.h" +#include "rtc_base/checks.h" #include "third_party/libyuv/include/libyuv.h" static const int kBufferAlignment = 64; namespace { - int ArgbDataSize(int height, int width) { - return width * height * 4; - } +int ArgbDataSize(int height, int width) { + return width * height * 4; +} } // namespace -rtc::scoped_refptr NativeBuffer::Create(webrtc::VideoType video_type, int width, int height) -{ +rtc::scoped_refptr +NativeBuffer::Create(webrtc::VideoType video_type, int width, int height) { return new rtc::RefCountedObject(video_type, width, height); } -webrtc::VideoFrameBuffer::Type NativeBuffer::type() const -{ +webrtc::VideoFrameBuffer::Type NativeBuffer::type() const { return Type::kNative; } -void NativeBuffer::InitializeData() -{ - memset(data_.get(), 0, ArgbDataSize(raw_height_, raw_width_)); +void NativeBuffer::InitializeData() { + memset(data_.get(), 0, ArgbDataSize(raw_height_, raw_width_)); } -int NativeBuffer::width() const -{ +int NativeBuffer::width() const { return scaled_width_; } -int NativeBuffer::height() const -{ +int NativeBuffer::height() const { return scaled_height_; } -rtc::scoped_refptr NativeBuffer::ToI420() -{ +rtc::scoped_refptr NativeBuffer::ToI420() { rtc::scoped_refptr i420_buffer = webrtc::I420Buffer::Create(raw_width_, raw_height_); const int conversionResult = libyuv::ConvertToI420( - data_.get(), length_, - i420_buffer.get()->MutableDataY(), i420_buffer.get()->StrideY(), - i420_buffer.get()->MutableDataU(), i420_buffer.get()->StrideU(), - i420_buffer.get()->MutableDataV(), i420_buffer.get()->StrideV(), - 0, 0, raw_width_, raw_height_, - raw_width_, raw_height_, - libyuv::kRotate0, ConvertVideoType(video_type_)); + data_.get(), length_, i420_buffer.get()->MutableDataY(), + i420_buffer.get()->StrideY(), i420_buffer.get()->MutableDataU(), + i420_buffer.get()->StrideU(), i420_buffer.get()->MutableDataV(), + i420_buffer.get()->StrideV(), 0, 0, raw_width_, raw_height_, raw_width_, + raw_height_, libyuv::kRotate0, ConvertVideoType(video_type_)); rtc::scoped_refptr scaled_buffer = webrtc::I420Buffer::Create(scaled_width_, scaled_height_); - scaled_buffer->ScaleFrom(*i420_buffer->ToI420()); + scaled_buffer->ScaleFrom(*i420_buffer->ToI420()); return scaled_buffer; } -int NativeBuffer::raw_width() const -{ +int NativeBuffer::raw_width() const { return raw_width_; } -int NativeBuffer::raw_height() const -{ +int NativeBuffer::raw_height() const { return raw_height_; } -void NativeBuffer::SetScaledSize(int scaled_width, int scaled_height) -{ +void NativeBuffer::SetScaledSize(int scaled_width, int scaled_height) { scaled_width_ = scaled_width; scaled_height_ = scaled_height; } -void NativeBuffer::SetLength(size_t length) -{ +void NativeBuffer::SetLength(size_t length) { length_ = length; } -size_t NativeBuffer::length() -{ +size_t NativeBuffer::length() { return length_; } -webrtc::VideoType NativeBuffer::VideoType() const -{ +webrtc::VideoType NativeBuffer::VideoType() const { return video_type_; } -const uint8_t* NativeBuffer::Data() const -{ +const uint8_t* NativeBuffer::Data() const { return data_.get(); } @@ -98,17 +84,14 @@ uint8_t* NativeBuffer::MutableData() { } NativeBuffer::NativeBuffer(webrtc::VideoType video_type, int width, int height) - : raw_width_(width), - raw_height_(height), - scaled_width_(width), - scaled_height_(height), - length_(ArgbDataSize(height, width)), - video_type_(video_type), - data_(static_cast( - webrtc::AlignedMalloc(ArgbDataSize(height, width), kBufferAlignment))) -{ -} - -NativeBuffer::~NativeBuffer() -{ -} \ No newline at end of file + : raw_width_(width), + raw_height_(height), + scaled_width_(width), + scaled_height_(height), + length_(ArgbDataSize(height, width)), + video_type_(video_type), + data_(static_cast( + webrtc::AlignedMalloc(ArgbDataSize(height, width), + kBufferAlignment))) {} + +NativeBuffer::~NativeBuffer() {} \ No newline at end of file diff --git a/src/rtc/native_buffer.h b/src/rtc/native_buffer.h index 83098f4e..1188eb42 100644 --- a/src/rtc/native_buffer.h +++ b/src/rtc/native_buffer.h @@ -6,10 +6,11 @@ #include "common_video/libyuv/include/webrtc_libyuv.h" #include "rtc_base/memory/aligned_malloc.h" -class NativeBuffer : public webrtc::VideoFrameBuffer -{ -public: - static rtc::scoped_refptr Create(webrtc::VideoType video_type, int width, int height); +class NativeBuffer : public webrtc::VideoFrameBuffer { + public: + static rtc::scoped_refptr Create(webrtc::VideoType video_type, + int width, + int height); void InitializeData(); @@ -27,11 +28,11 @@ class NativeBuffer : public webrtc::VideoFrameBuffer const uint8_t* Data() const; uint8_t* MutableData(); -protected: + protected: NativeBuffer(webrtc::VideoType video_type, int width, int height); ~NativeBuffer() override; -private: + private: const int raw_width_; const int raw_height_; int scaled_width_; @@ -40,4 +41,4 @@ class NativeBuffer : public webrtc::VideoFrameBuffer const webrtc::VideoType video_type_; const std::unique_ptr data_; }; -#endif // NATIVE_BUFFER_H_ +#endif // NATIVE_BUFFER_H_ diff --git a/src/rtc/observer.cpp b/src/rtc/observer.cpp index 06816bae..a644ca62 100644 --- a/src/rtc/observer.cpp +++ b/src/rtc/observer.cpp @@ -3,59 +3,49 @@ #include "observer.h" - void PeerConnectionObserver::OnIceConnectionChange( - webrtc::PeerConnectionInterface::IceConnectionState new_state) -{ + webrtc::PeerConnectionInterface::IceConnectionState new_state) { _sender->onIceConnectionStateChange(new_state); } -void PeerConnectionObserver::OnIceCandidate(const webrtc::IceCandidateInterface *candidate) -{ +void PeerConnectionObserver::OnIceCandidate( + const webrtc::IceCandidateInterface* candidate) { std::string sdp; - if (candidate->ToString(&sdp)) - { - if (_sender != nullptr) - { - _sender->onIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), sdp); + if (candidate->ToString(&sdp)) { + if (_sender != nullptr) { + _sender->onIceCandidate(candidate->sdp_mid(), + candidate->sdp_mline_index(), sdp); } - } - else - { + } else { RTC_LOG(LS_ERROR) << "Failed to serialize candidate"; } } -void CreateSessionDescriptionObserver::OnSuccess(webrtc::SessionDescriptionInterface *desc) -{ +void CreateSessionDescriptionObserver::OnSuccess( + webrtc::SessionDescriptionInterface* desc) { std::string sdp; desc->ToString(&sdp); RTC_LOG(LS_INFO) << "Created session description : " << sdp; _connection->SetLocalDescription( - SetSessionDescriptionObserver::Create(desc->GetType(), _sender), desc); - if (_sender != nullptr) - { + SetSessionDescriptionObserver::Create(desc->GetType(), _sender), desc); + if (_sender != nullptr) { _sender->onCreateDescription(desc->GetType(), sdp); } } -void CreateSessionDescriptionObserver::OnFailure(webrtc::RTCError error) -{ +void CreateSessionDescriptionObserver::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << "Failed to create session description : " << ToString(error.type()) << ": " << error.message(); } -void SetSessionDescriptionObserver::OnSuccess() -{ +void SetSessionDescriptionObserver::OnSuccess() { RTC_LOG(LS_INFO) << "Set local description success!"; - if (_sender != nullptr) - { + if (_sender != nullptr) { _sender->onSetDescription(_type); } } -void SetSessionDescriptionObserver::OnFailure(webrtc::RTCError error) -{ +void SetSessionDescriptionObserver::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << "Failed to set local description : " << ToString(error.type()) << ": " << error.message(); } \ No newline at end of file diff --git a/src/rtc/observer.h b/src/rtc/observer.h index 9c90a0e1..7bfae0cf 100644 --- a/src/rtc/observer.h +++ b/src/rtc/observer.h @@ -5,70 +5,70 @@ #include "messagesender.h" -class PeerConnectionObserver : public webrtc::PeerConnectionObserver -{ -public: - PeerConnectionObserver(RTCMessageSender *sender) : _sender(sender) {}; -protected: +class PeerConnectionObserver : public webrtc::PeerConnectionObserver { + public: + PeerConnectionObserver(RTCMessageSender* sender) : _sender(sender){}; + + protected: void OnSignalingChange( - webrtc::PeerConnectionInterface::SignalingState new_state) override {} + webrtc::PeerConnectionInterface::SignalingState new_state) override {} void OnAddStream( - rtc::scoped_refptr stream) override {} + rtc::scoped_refptr stream) override {} void OnRemoveStream( - rtc::scoped_refptr stream) override {} + rtc::scoped_refptr stream) override {} void OnDataChannel( - rtc::scoped_refptr data_channel) override {} + rtc::scoped_refptr data_channel) override {} void OnRenegotiationNeeded() override {} void OnIceConnectionChange( - webrtc::PeerConnectionInterface::IceConnectionState new_state) override; + webrtc::PeerConnectionInterface::IceConnectionState new_state) override; void OnIceGatheringChange( - webrtc::PeerConnectionInterface::IceGatheringState new_state) override {}; - void OnIceCandidate(const webrtc::IceCandidateInterface *candidate) override; + webrtc::PeerConnectionInterface::IceGatheringState new_state) override{}; + void OnIceCandidate(const webrtc::IceCandidateInterface* candidate) override; void OnIceConnectionReceivingChange(bool receiving) override {} -private: - RTCMessageSender *_sender; + private: + RTCMessageSender* _sender; }; -class CreateSessionDescriptionObserver : public webrtc::CreateSessionDescriptionObserver -{ -public: - static CreateSessionDescriptionObserver *Create( - RTCMessageSender *sender, webrtc::PeerConnectionInterface *connection) - { - return new rtc::RefCountedObject(sender, connection); +class CreateSessionDescriptionObserver + : public webrtc::CreateSessionDescriptionObserver { + public: + static CreateSessionDescriptionObserver* Create( + RTCMessageSender* sender, + webrtc::PeerConnectionInterface* connection) { + return new rtc::RefCountedObject( + sender, connection); } -protected: - CreateSessionDescriptionObserver( - RTCMessageSender *sender, webrtc::PeerConnectionInterface *connection) - : _sender(sender), _connection(connection) {}; - void OnSuccess(webrtc::SessionDescriptionInterface *desc) override; + protected: + CreateSessionDescriptionObserver(RTCMessageSender* sender, + webrtc::PeerConnectionInterface* connection) + : _sender(sender), _connection(connection){}; + void OnSuccess(webrtc::SessionDescriptionInterface* desc) override; void OnFailure(webrtc::RTCError error) override; -private: - RTCMessageSender *_sender; - webrtc::PeerConnectionInterface *_connection; + private: + RTCMessageSender* _sender; + webrtc::PeerConnectionInterface* _connection; }; class SetSessionDescriptionObserver - : public webrtc::SetSessionDescriptionObserver -{ -public: - static SetSessionDescriptionObserver *Create( - webrtc::SdpType type, RTCMessageSender *sender) - { - return new rtc::RefCountedObject(type, sender); + : public webrtc::SetSessionDescriptionObserver { + public: + static SetSessionDescriptionObserver* Create(webrtc::SdpType type, + RTCMessageSender* sender) { + return new rtc::RefCountedObject(type, + sender); } -protected: - SetSessionDescriptionObserver(webrtc::SdpType type, RTCMessageSender *sender) - : _type(type), _sender(sender) {}; + protected: + SetSessionDescriptionObserver(webrtc::SdpType type, RTCMessageSender* sender) + : _type(type), _sender(sender){}; void OnSuccess() override; void OnFailure(webrtc::RTCError error) override; -private: - RTCMessageSender *_sender; + private: + RTCMessageSender* _sender; webrtc::SdpType _type; }; #endif \ No newline at end of file diff --git a/src/rtc/scalable_track_source.cpp b/src/rtc/scalable_track_source.cpp index 282af93c..d4e62d73 100644 --- a/src/rtc/scalable_track_source.cpp +++ b/src/rtc/scalable_track_source.cpp @@ -20,7 +20,8 @@ #include "native_buffer.h" -ScalableVideoTrackSource::ScalableVideoTrackSource() : AdaptedVideoTrackSource(4) {} +ScalableVideoTrackSource::ScalableVideoTrackSource() + : AdaptedVideoTrackSource(4) {} ScalableVideoTrackSource::~ScalableVideoTrackSource() {} bool ScalableVideoTrackSource::is_screencast() const { @@ -31,7 +32,8 @@ absl::optional ScalableVideoTrackSource::needs_denoising() const { return false; } -webrtc::MediaSourceInterface::SourceState ScalableVideoTrackSource::state() const { +webrtc::MediaSourceInterface::SourceState ScalableVideoTrackSource::state() + const { return SourceState::kLive; } @@ -39,7 +41,8 @@ bool ScalableVideoTrackSource::remote() const { return false; } -void ScalableVideoTrackSource::OnCapturedFrame(const webrtc::VideoFrame& frame) { +void ScalableVideoTrackSource::OnCapturedFrame( + const webrtc::VideoFrame& frame) { const int64_t timestamp_us = frame.timestamp_us(); const int64_t translated_timestamp_us = timestamp_aligner_.TranslateTimestamp(timestamp_us, rtc::TimeMicros()); @@ -50,26 +53,29 @@ void ScalableVideoTrackSource::OnCapturedFrame(const webrtc::VideoFrame& frame) int crop_height; int crop_x; int crop_y; - if (!AdaptFrame(frame.width(), frame.height(), timestamp_us, - &adapted_width, &adapted_height, - &crop_width, &crop_height, &crop_x, &crop_y)) { + if (!AdaptFrame(frame.width(), frame.height(), timestamp_us, &adapted_width, + &adapted_height, &crop_width, &crop_height, &crop_x, + &crop_y)) { return; } - if (useNativeBuffer() && frame.video_frame_buffer()->type() == webrtc::VideoFrameBuffer::Type::kNative) - { - NativeBuffer* frame_buffer = dynamic_cast(frame.video_frame_buffer().get()); + if (useNativeBuffer() && frame.video_frame_buffer()->type() == + webrtc::VideoFrameBuffer::Type::kNative) { + NativeBuffer* frame_buffer = + dynamic_cast(frame.video_frame_buffer().get()); frame_buffer->SetScaledSize(adapted_width, adapted_height); OnFrame(frame); return; } - rtc::scoped_refptr buffer = frame.video_frame_buffer(); + rtc::scoped_refptr buffer = + frame.video_frame_buffer(); if (adapted_width != frame.width() || adapted_height != frame.height()) { // Video adapter has requested a down-scale. Allocate a new buffer and // return scaled version. - rtc::scoped_refptr i420_buffer = webrtc::I420Buffer::Create(adapted_width, adapted_height); + rtc::scoped_refptr i420_buffer = + webrtc::I420Buffer::Create(adapted_width, adapted_height); i420_buffer->ScaleFrom(*buffer->ToI420()); buffer = i420_buffer; } diff --git a/src/signal_listener.cpp b/src/signal_listener.cpp index fb987ca1..6e3fb9e9 100644 --- a/src/signal_listener.cpp +++ b/src/signal_listener.cpp @@ -3,39 +3,31 @@ #include #include -SignalListener::SignalListener() -{ +SignalListener::SignalListener() { SignalManager::add(this); } -SignalListener::~SignalListener() -{ +SignalListener::~SignalListener() { SignalManager::remove(this); } -std::vector SignalManager::_listeners; +std::vector SignalManager::_listeners; -void SignalManager::sigintHandler(int signum) -{ - for (SignalListener *listener : _listeners) - { +void SignalManager::sigintHandler(int signum) { + for (SignalListener* listener : _listeners) { listener->OnSignal(signum); } } -void SignalManager::init() -{ +void SignalManager::init() { signal(SIGINT, SignalManager::sigintHandler); } -void SignalManager::add(SignalListener *listener) -{ +void SignalManager::add(SignalListener* listener) { _listeners.push_back(listener); } -void SignalManager::remove(SignalListener *listener) -{ - _listeners.erase( - std::remove(_listeners.begin(), _listeners.end(), listener), - _listeners.end()); +void SignalManager::remove(SignalListener* listener) { + _listeners.erase(std::remove(_listeners.begin(), _listeners.end(), listener), + _listeners.end()); } \ No newline at end of file diff --git a/src/signal_listener.h b/src/signal_listener.h index 406cd2ba..200b8b10 100644 --- a/src/signal_listener.h +++ b/src/signal_listener.h @@ -5,25 +5,23 @@ class SignalManager; -class SignalListener -{ -public: +class SignalListener { + public: SignalListener(); ~SignalListener(); virtual void OnSignal(int signum) = 0; }; -class SignalManager -{ -public: +class SignalManager { + public: static void init(); - static void add(SignalListener *listener); - static void remove(SignalListener *listener); + static void add(SignalListener* listener); + static void remove(SignalListener* listener); -private: + private: static void sigintHandler(int signum); - static std::vector _listeners; + static std::vector _listeners; }; #endif \ No newline at end of file diff --git a/src/sora/sora_server.cpp b/src/sora/sora_server.cpp index 1cd95e5c..4fbfabbf 100644 --- a/src/sora/sora_server.cpp +++ b/src/sora/sora_server.cpp @@ -1,94 +1,81 @@ #include "sora_server.h" -#include "util.h" #include "sora_session.h" +#include "util.h" -SoraServer::SoraServer( - boost::asio::io_context& ioc, - boost::asio::ip::tcp::endpoint endpoint, - RTCManager* rtc_manager, - ConnectionSettings conn_settings) - : acceptor_(ioc) - , socket_(ioc) - , rtc_manager_(rtc_manager) - , conn_settings_(conn_settings) - , ws_client_(std::make_shared(ioc, rtc_manager, conn_settings)) -{ - boost::system::error_code ec; - - // Open the acceptor - acceptor_.open(endpoint.protocol(), ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "open"); - return; - } +SoraServer::SoraServer(boost::asio::io_context& ioc, + boost::asio::ip::tcp::endpoint endpoint, + RTCManager* rtc_manager, + ConnectionSettings conn_settings) + : acceptor_(ioc), + socket_(ioc), + rtc_manager_(rtc_manager), + conn_settings_(conn_settings), + ws_client_(std::make_shared(ioc, + rtc_manager, + conn_settings)) { + boost::system::error_code ec; - // Allow address reuse - acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "set_option"); - return; - } + // Open the acceptor + acceptor_.open(endpoint.protocol(), ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "open"); + return; + } - // Bind to the server address - acceptor_.bind(endpoint, ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "bind"); - return; - } + // Allow address reuse + acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "set_option"); + return; + } - // Start listening for connections - acceptor_.listen( - boost::asio::socket_base::max_listen_connections, ec); - if (ec) - { - MOMO_BOOST_ERROR(ec, "listen"); - return; - } + // Bind to the server address + acceptor_.bind(endpoint, ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "bind"); + return; + } + // Start listening for connections + acceptor_.listen(boost::asio::socket_base::max_listen_connections, ec); + if (ec) { + MOMO_BOOST_ERROR(ec, "listen"); + return; + } } SoraServer::~SoraServer() { - // これをやっておかないと循環参照でリークする - ws_client_->release(); + // これをやっておかないと循環参照でリークする + ws_client_->release(); } -void SoraServer::run() -{ - if (!acceptor_.is_open()) - return; +void SoraServer::run() { + if (!acceptor_.is_open()) + return; - if (conn_settings_.sora_auto_connect) { - ws_client_->connect(); - } - doAccept(); + if (conn_settings_.sora_auto_connect) { + ws_client_->connect(); + } + doAccept(); } -void SoraServer::doAccept() -{ - acceptor_.async_accept( - socket_, - std::bind( - &SoraServer::onAccept, - shared_from_this(), - std::placeholders::_1)); +void SoraServer::doAccept() { + acceptor_.async_accept(socket_, + std::bind(&SoraServer::onAccept, shared_from_this(), + std::placeholders::_1)); } -void SoraServer::onAccept(boost::system::error_code ec) -{ - if (ec) - { - MOMO_BOOST_ERROR(ec, "accept"); - } - else - { - // Create the session and run it - std::make_shared(std::move(socket_), rtc_manager_, ws_client_, conn_settings_)->run(); - } +void SoraServer::onAccept(boost::system::error_code ec) { + if (ec) { + MOMO_BOOST_ERROR(ec, "accept"); + } else { + // Create the session and run it + std::make_shared(std::move(socket_), rtc_manager_, ws_client_, + conn_settings_) + ->run(); + } - // Accept another connection - doAccept(); + // Accept another connection + doAccept(); } diff --git a/src/sora/sora_server.h b/src/sora/sora_server.h index fbc79869..3b4e7ff0 100644 --- a/src/sora/sora_server.h +++ b/src/sora/sora_server.h @@ -2,15 +2,15 @@ #define SORA_SERVER_H_ #include +#include +#include #include #include #include #include -#include -#include -#include "rtc/manager.h" #include "connection_settings.h" +#include "rtc/manager.h" #include "sora_websocket_client.h" /* @@ -21,28 +21,26 @@ SoraServer には２種類の役割がある。 - 指定されたエンドポイントを listen して、HTTP 経由でリクエストされた処理に対応する - 指定されたシグナリングサーバに WebSocket で接続し、シグナリングサーバからの offer に答えたりする */ -class SoraServer : public std::enable_shared_from_this -{ - boost::asio::ip::tcp::acceptor acceptor_; - boost::asio::ip::tcp::socket socket_; - - RTCManager* rtc_manager_; - ConnectionSettings conn_settings_; - std::shared_ptr ws_client_; - -public: - SoraServer( - boost::asio::io_context& ioc, - boost::asio::ip::tcp::endpoint endpoint, - RTCManager* rtc_manager, - ConnectionSettings conn_settings); - ~SoraServer(); - - void run(); - -private: - void doAccept(); - void onAccept(boost::system::error_code ec); +class SoraServer : public std::enable_shared_from_this { + boost::asio::ip::tcp::acceptor acceptor_; + boost::asio::ip::tcp::socket socket_; + + RTCManager* rtc_manager_; + ConnectionSettings conn_settings_; + std::shared_ptr ws_client_; + + public: + SoraServer(boost::asio::io_context& ioc, + boost::asio::ip::tcp::endpoint endpoint, + RTCManager* rtc_manager, + ConnectionSettings conn_settings); + ~SoraServer(); + + void run(); + + private: + void doAccept(); + void onAccept(boost::system::error_code ec); }; #endif diff --git a/src/sora/sora_session.cpp b/src/sora/sora_session.cpp index 5306bd55..30451983 100644 --- a/src/sora/sora_session.cpp +++ b/src/sora/sora_session.cpp @@ -1,185 +1,153 @@ #include "sora_session.h" -#include -#include #include +#include +#include #include "util.h" using json = nlohmann::json; -SoraSession::SoraSession( - boost::asio::ip::tcp::socket socket, - RTCManager* rtc_manager, - std::shared_ptr ws_client, - ConnectionSettings conn_settings) - : socket_(std::move(socket)) - , strand_(socket_.get_executor()) - , rtc_manager_(rtc_manager) - , ws_client_(ws_client) - , conn_settings_(conn_settings) -{ -} - -void SoraSession::run() -{ - doRead(); +SoraSession::SoraSession(boost::asio::ip::tcp::socket socket, + RTCManager* rtc_manager, + std::shared_ptr ws_client, + ConnectionSettings conn_settings) + : socket_(std::move(socket)), + strand_(socket_.get_executor()), + rtc_manager_(rtc_manager), + ws_client_(ws_client), + conn_settings_(conn_settings) {} + +void SoraSession::run() { + doRead(); } -void SoraSession::doRead() -{ - // Make the request empty before reading, - // otherwise the operation behavior is undefined. - req_ = {}; - - // Read a request - boost::beast::http::async_read(socket_, buffer_, req_, - boost::asio::bind_executor( - strand_, - std::bind( - &SoraSession::onRead, - shared_from_this(), - std::placeholders::_1, - std::placeholders::_2))); +void SoraSession::doRead() { + // Make the request empty before reading, + // otherwise the operation behavior is undefined. + req_ = {}; + + // Read a request + boost::beast::http::async_read( + socket_, buffer_, req_, + boost::asio::bind_executor( + strand_, std::bind(&SoraSession::onRead, shared_from_this(), + std::placeholders::_1, std::placeholders::_2))); } -void SoraSession::onRead( - boost::system::error_code ec, - std::size_t bytes_transferred) -{ - boost::ignore_unused(bytes_transferred); - - // This means they closed the connection - if (ec == boost::beast::http::error::end_of_stream) - return doClose(); - - if (ec) - return MOMO_BOOST_ERROR(ec, "read"); - - if (req_.method() == boost::beast::http::verb::get) - { - if (req_.target() == "/connect/status") - { - std::string state = Util::iceConnectionStateToString(ws_client_->getRTCConnectionState()); - json json_message = { - {"state", state} - }; - sendResponse(createOKwithJson(req_, std::move(json_message))); - } - else if (req_.target() == "/mute/status") - { - std::shared_ptr rtc_conn =ws_client_->getRTCConnection(); - if (rtc_conn) { - json json_message = { - {"audio", !rtc_conn->isAudioEnabled()}, - {"video", !rtc_conn->isVideoEnabled()} - }; - sendResponse(createOKwithJson(req_, std::move(json_message))); - } else { - sendResponse(Util::serverError(req_, "Invalid RTC Connection")); - } - } - else - { - sendResponse(Util::badRequest(req_, "Invalid Request")); - } +void SoraSession::onRead(boost::system::error_code ec, + std::size_t bytes_transferred) { + boost::ignore_unused(bytes_transferred); + + // This means they closed the connection + if (ec == boost::beast::http::error::end_of_stream) + return doClose(); + + if (ec) + return MOMO_BOOST_ERROR(ec, "read"); + + if (req_.method() == boost::beast::http::verb::get) { + if (req_.target() == "/connect/status") { + std::string state = + Util::iceConnectionStateToString(ws_client_->getRTCConnectionState()); + json json_message = {{"state", state}}; + sendResponse(createOKwithJson(req_, std::move(json_message))); + } else if (req_.target() == "/mute/status") { + std::shared_ptr rtc_conn = ws_client_->getRTCConnection(); + if (rtc_conn) { + json json_message = {{"audio", !rtc_conn->isAudioEnabled()}, + {"video", !rtc_conn->isVideoEnabled()}}; + sendResponse(createOKwithJson(req_, std::move(json_message))); + } else { + sendResponse(Util::serverError(req_, "Invalid RTC Connection")); + } + } else { + sendResponse(Util::badRequest(req_, "Invalid Request")); } - else if (req_.method() == boost::beast::http::verb::post) - { - if (req_.target() == "/connect") - { - json json_message = { - {"result", ws_client_->connect()} - }; - sendResponse(createOKwithJson(req_, std::move(json_message))); - } - else if (req_.target() == "/close") - { - json json_message = { - {"result", true} - }; - sendResponse(createOKwithJson(req_, std::move(json_message))); - } - else if (req_.target() == "/mute") - { - json recv_json; - try { - recv_json = json::parse(req_.body()); - } catch(json::parse_error& e) { - sendResponse(Util::badRequest(req_, "Invalid JSON")); - return; - } - - std::shared_ptr rtc_conn = ws_client_->getRTCConnection(); - if (!rtc_conn) { - sendResponse(Util::serverError(req_, "Create RTC Connection Failed")); - return; - } - try { - bool audioMute = recv_json["audio"]; - rtc_conn->setAudioEnabled(!audioMute); - } catch(json::type_error& e) {} - - try { - bool videoMute = recv_json["video"]; - rtc_conn->setVideoEnabled(!videoMute); - } catch(json::type_error& e) {} - - json json_message = { - {"audio", !rtc_conn->isAudioEnabled()}, - {"video", !rtc_conn->isVideoEnabled()} - }; - sendResponse(createOKwithJson(req_, std::move(json_message))); - } - else - { - sendResponse(Util::badRequest(req_, "Invalid Request")); - } + } else if (req_.method() == boost::beast::http::verb::post) { + if (req_.target() == "/connect") { + json json_message = {{"result", ws_client_->connect()}}; + sendResponse(createOKwithJson(req_, std::move(json_message))); + } else if (req_.target() == "/close") { + json json_message = {{"result", true}}; + sendResponse(createOKwithJson(req_, std::move(json_message))); + } else if (req_.target() == "/mute") { + json recv_json; + try { + recv_json = json::parse(req_.body()); + } catch (json::parse_error& e) { + sendResponse(Util::badRequest(req_, "Invalid JSON")); + return; + } + + std::shared_ptr rtc_conn = ws_client_->getRTCConnection(); + if (!rtc_conn) { + sendResponse(Util::serverError(req_, "Create RTC Connection Failed")); + return; + } + try { + bool audioMute = recv_json["audio"]; + rtc_conn->setAudioEnabled(!audioMute); + } catch (json::type_error& e) { + } + + try { + bool videoMute = recv_json["video"]; + rtc_conn->setVideoEnabled(!videoMute); + } catch (json::type_error& e) { + } + + json json_message = {{"audio", !rtc_conn->isAudioEnabled()}, + {"video", !rtc_conn->isVideoEnabled()}}; + sendResponse(createOKwithJson(req_, std::move(json_message))); } else { - sendResponse(Util::badRequest(req_, "Invalid Method")); + sendResponse(Util::badRequest(req_, "Invalid Request")); } + } else { + sendResponse(Util::badRequest(req_, "Invalid Method")); + } } -void SoraSession::onWrite( - boost::system::error_code ec, - std::size_t bytes_transferred, - bool close) -{ - boost::ignore_unused(bytes_transferred); - - if (ec) - return MOMO_BOOST_ERROR(ec, "write"); - - if (close) - { - // This means we should close the connection, usually because - // the response indicated the "Connection: close" semantic. - return doClose(); - } +void SoraSession::onWrite(boost::system::error_code ec, + std::size_t bytes_transferred, + bool close) { + boost::ignore_unused(bytes_transferred); - // We're done with the response so delete it - res_ = nullptr; + if (ec) + return MOMO_BOOST_ERROR(ec, "write"); - // Read another request - doRead(); -} + if (close) { + // This means we should close the connection, usually because + // the response indicated the "Connection: close" semantic. + return doClose(); + } -void SoraSession::doClose() -{ - // Send a TCP shutdown - boost::system::error_code ec; - socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_send, ec); + // We're done with the response so delete it + res_ = nullptr; - // At this point the connection is closed gracefully + // Read another request + doRead(); } -boost::beast::http::response SoraSession::createOKwithJson(const boost::beast::http::request& req, json json_message) { - boost::beast::http::response res{boost::beast::http::status::ok, 11}; - res.set(boost::beast::http::field::server, BOOST_BEAST_VERSION_STRING); - res.set(boost::beast::http::field::content_type, "application/json"); - res.keep_alive(req.keep_alive()); - res.body() = json_message.dump(); - res.prepare_payload(); +void SoraSession::doClose() { + // Send a TCP shutdown + boost::system::error_code ec; + socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_send, ec); + + // At this point the connection is closed gracefully +} - return res; +boost::beast::http::response +SoraSession::createOKwithJson( + const boost::beast::http::request& req, + json json_message) { + boost::beast::http::response res{ + boost::beast::http::status::ok, 11}; + res.set(boost::beast::http::field::server, BOOST_BEAST_VERSION_STRING); + res.set(boost::beast::http::field::content_type, "application/json"); + res.keep_alive(req.keep_alive()); + res.body() = json_message.dump(); + res.prepare_payload(); + + return res; } diff --git a/src/sora/sora_session.h b/src/sora/sora_session.h index 263033e2..5e8209bc 100644 --- a/src/sora/sora_session.h +++ b/src/sora/sora_session.h @@ -2,78 +2,76 @@ #define SORA_SESSION_H_ #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include #include -#include -#include -#include -#include +#include +#include +#include #include +#include -#include "rtc/manager.h" #include "connection_settings.h" +#include "rtc/manager.h" #include "sora_websocket_client.h" // HTTP の１回のリクエストに対して答えるためのクラス -class SoraSession : public std::enable_shared_from_this -{ - boost::asio::ip::tcp::socket socket_; - boost::asio::strand strand_; - boost::beast::flat_buffer buffer_; - boost::beast::http::request req_; - std::shared_ptr res_; - - RTCManager* rtc_manager_; - std::shared_ptr ws_client_; - ConnectionSettings conn_settings_; +class SoraSession : public std::enable_shared_from_this { + boost::asio::ip::tcp::socket socket_; + boost::asio::strand strand_; + boost::beast::flat_buffer buffer_; + boost::beast::http::request req_; + std::shared_ptr res_; -public: - SoraSession( - boost::asio::ip::tcp::socket socket, - RTCManager* rtc_manager, - std::shared_ptr ws_client, - ConnectionSettings conn_settings); + RTCManager* rtc_manager_; + std::shared_ptr ws_client_; + ConnectionSettings conn_settings_; - void run(); + public: + SoraSession(boost::asio::ip::tcp::socket socket, + RTCManager* rtc_manager, + std::shared_ptr ws_client, + ConnectionSettings conn_settings); -private: - void doRead(); + void run(); - void onRead(boost::system::error_code ec, std::size_t bytes_transferred); - void onWrite(boost::system::error_code ec, std::size_t bytes_transferred, bool close); - void doClose(); + private: + void doRead(); -private: - static boost::beast::http::response createOKwithJson(const boost::beast::http::request& req, nlohmann::json json_message); + void onRead(boost::system::error_code ec, std::size_t bytes_transferred); + void onWrite(boost::system::error_code ec, + std::size_t bytes_transferred, + bool close); + void doClose(); - template - void sendResponse(boost::beast::http::response msg) { - auto sp = std::make_shared>(std::move(msg)); + private: + static boost::beast::http::response + createOKwithJson( + const boost::beast::http::request& req, + nlohmann::json json_message); - // msg オブジェクトは書き込みが完了するまで生きている必要があるので、 - // メンバに入れてライフタイムを延ばしてやる - res_ = sp; + template + void sendResponse(boost::beast::http::response msg) { + auto sp = std::make_shared>( + std::move(msg)); - // Write the response - boost::beast::http::async_write( - socket_, - *sp, - boost::asio::bind_executor( - strand_, - std::bind( - &SoraSession::onWrite, - shared_from_this(), - std::placeholders::_1, - std::placeholders::_2, - sp->need_eof()))); - } + // msg オブジェクトは書き込みが完了するまで生きている必要があるので、 + // メンバに入れてライフタイムを延ばしてやる + res_ = sp; + // Write the response + boost::beast::http::async_write( + socket_, *sp, + boost::asio::bind_executor( + strand_, std::bind(&SoraSession::onWrite, shared_from_this(), + std::placeholders::_1, std::placeholders::_2, + sp->need_eof()))); + } }; -#endif // SORA_SESSION_H_ +#endif // SORA_SESSION_H_ diff --git a/src/sora/sora_websocket_client.cpp b/src/sora/sora_websocket_client.cpp index 1d9b2204..814d6cd1 100644 --- a/src/sora/sora_websocket_client.cpp +++ b/src/sora/sora_websocket_client.cpp @@ -9,270 +9,251 @@ using json = nlohmann::json; bool SoraWebsocketClient::parseURL(URLParts& parts) const { - std::string url = conn_settings_.sora_signaling_host; + std::string url = conn_settings_.sora_signaling_host; - if (!URLParts::parse(url, parts)) { - throw std::exception(); - } + if (!URLParts::parse(url, parts)) { + throw std::exception(); + } - std::string default_port; - if (parts.scheme == "wss") { - return true; - } else if (parts.scheme == "ws") { - return false; - } else { - throw std::exception(); - } + std::string default_port; + if (parts.scheme == "wss") { + return true; + } else if (parts.scheme == "ws") { + return false; + } else { + throw std::exception(); + } } boost::asio::ssl::context SoraWebsocketClient::createSSLContext() const { - boost::asio::ssl::context ctx(boost::asio::ssl::context::tlsv12); - ctx.set_default_verify_paths(); - ctx.set_options(boost::asio::ssl::context::default_workarounds | - boost::asio::ssl::context::no_sslv2 | - boost::asio::ssl::context::no_sslv3 | - boost::asio::ssl::context::single_dh_use); - return ctx; + boost::asio::ssl::context ctx(boost::asio::ssl::context::tlsv12); + ctx.set_default_verify_paths(); + ctx.set_options(boost::asio::ssl::context::default_workarounds | + boost::asio::ssl::context::no_sslv2 | + boost::asio::ssl::context::no_sslv3 | + boost::asio::ssl::context::single_dh_use); + return ctx; } -webrtc::PeerConnectionInterface::IceConnectionState SoraWebsocketClient::getRTCConnectionState() const { return rtc_state_; } +webrtc::PeerConnectionInterface::IceConnectionState +SoraWebsocketClient::getRTCConnectionState() const { + return rtc_state_; +} std::shared_ptr SoraWebsocketClient::getRTCConnection() const { - if (rtc_state_ == webrtc::PeerConnectionInterface::IceConnectionState::kIceConnectionConnected) { + if (rtc_state_ == webrtc::PeerConnectionInterface::IceConnectionState:: + kIceConnectionConnected) { return connection_; } else { return nullptr; } } -SoraWebsocketClient::SoraWebsocketClient(boost::asio::io_context& ioc, RTCManager* manager, ConnectionSettings conn_settings) - : ioc_(ioc) - , resolver_(ioc) - , manager_(manager) - , retry_count_(0) - , conn_settings_(conn_settings) - , watchdog_(ioc, std::bind(&SoraWebsocketClient::onWatchdogExpired, this)) -{ - reset(); +SoraWebsocketClient::SoraWebsocketClient(boost::asio::io_context& ioc, + RTCManager* manager, + ConnectionSettings conn_settings) + : ioc_(ioc), + resolver_(ioc), + manager_(manager), + retry_count_(0), + conn_settings_(conn_settings), + watchdog_(ioc, std::bind(&SoraWebsocketClient::onWatchdogExpired, this)) { + reset(); } void SoraWebsocketClient::reset() { - connection_ = nullptr; - connected_ = false; - - if (parseURL(parts_)) { - auto ssl_ctx = createSSLContext(); - boost::beast::websocket::stream> wss(ioc_, ssl_ctx); - ws_.reset(new Websocket(ioc_, std::move(ssl_ctx))); - // SNI の設定を行う - if (!SSL_set_tlsext_host_name(ws_->nativeSecureSocket().next_layer().native_handle(), parts_.host.c_str())) - { - boost::system::error_code ec{static_cast(::ERR_get_error()), boost::asio::error::get_ssl_category()}; - MOMO_BOOST_ERROR(ec, "SSL_set_tlsext_host_name"); - } - } else { - boost::beast::websocket::stream ws(ioc_); - ws_.reset(new Websocket(ioc_)); + connection_ = nullptr; + connected_ = false; + + if (parseURL(parts_)) { + auto ssl_ctx = createSSLContext(); + boost::beast::websocket::stream< + boost::asio::ssl::stream> + wss(ioc_, ssl_ctx); + ws_.reset(new Websocket(ioc_, std::move(ssl_ctx))); + // SNI の設定を行う + if (!SSL_set_tlsext_host_name( + ws_->nativeSecureSocket().next_layer().native_handle(), + parts_.host.c_str())) { + boost::system::error_code ec{static_cast(::ERR_get_error()), + boost::asio::error::get_ssl_category()}; + MOMO_BOOST_ERROR(ec, "SSL_set_tlsext_host_name"); } + } else { + boost::beast::websocket::stream ws(ioc_); + ws_.reset(new Websocket(ioc_)); + } } void SoraWebsocketClient::release() { - connection_ = nullptr; + connection_ = nullptr; } -bool SoraWebsocketClient::connect() -{ - RTC_LOG(LS_INFO) << __FUNCTION__; +bool SoraWebsocketClient::connect() { + RTC_LOG(LS_INFO) << __FUNCTION__; - if (connected_) { - return false; - } + if (connected_) { + return false; + } - std::string port; - if (parts_.port.empty()) { - port = ws_->isSSL() ? "443" : "80"; - } else { - port = parts_.port; - } + std::string port; + if (parts_.port.empty()) { + port = ws_->isSSL() ? "443" : "80"; + } else { + port = parts_.port; + } - // DNS ルックアップ - resolver_.async_resolve( - parts_.host, - port, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &SoraWebsocketClient::onResolve, - shared_from_this(), - std::placeholders::_1, - std::placeholders::_2))); + // DNS ルックアップ + resolver_.async_resolve( + parts_.host, port, + boost::asio::bind_executor( + ws_->strand(), + std::bind(&SoraWebsocketClient::onResolve, shared_from_this(), + std::placeholders::_1, std::placeholders::_2))); - watchdog_.enable(30); + watchdog_.enable(30); - return true; + return true; } -void SoraWebsocketClient::reconnectAfter() -{ +void SoraWebsocketClient::reconnectAfter() { int interval = 5 * (2 * retry_count_ + 1); - RTC_LOG(LS_INFO) << __FUNCTION__ << " reconnect after " << interval <<" sec"; + RTC_LOG(LS_INFO) << __FUNCTION__ << " reconnect after " << interval << " sec"; watchdog_.enable(interval); retry_count_++; } void SoraWebsocketClient::onWatchdogExpired() { - RTC_LOG(LS_WARNING) << __FUNCTION__; + RTC_LOG(LS_WARNING) << __FUNCTION__; - RTC_LOG(LS_INFO) << __FUNCTION__ << " reconnecting...:"; - reset(); - connect(); + RTC_LOG(LS_INFO) << __FUNCTION__ << " reconnecting...:"; + reset(); + connect(); } -void SoraWebsocketClient::onResolve(boost::system::error_code ec, boost::asio::ip::tcp::resolver::results_type results) { - if (ec) { - reconnectAfter(); - return MOMO_BOOST_ERROR(ec, "resolve"); - } +void SoraWebsocketClient::onResolve( + boost::system::error_code ec, + boost::asio::ip::tcp::resolver::results_type results) { + if (ec) { + reconnectAfter(); + return MOMO_BOOST_ERROR(ec, "resolve"); + } - // DNS ルックアップで得られたエンドポイントに対して接続する - if (ws_->isSSL()) { - boost::asio::async_connect( - ws_->nativeSecureSocket().next_layer().next_layer(), - results.begin(), - results.end(), - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &SoraWebsocketClient::onSSLConnect, - shared_from_this(), - std::placeholders::_1))); - } else { - boost::asio::async_connect( - ws_->nativeSocket().next_layer(), - results.begin(), - results.end(), - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &SoraWebsocketClient::onConnect, - shared_from_this(), - std::placeholders::_1))); - } + // DNS ルックアップで得られたエンドポイントに対して接続する + if (ws_->isSSL()) { + boost::asio::async_connect( + ws_->nativeSecureSocket().next_layer().next_layer(), results.begin(), + results.end(), + boost::asio::bind_executor( + ws_->strand(), + std::bind(&SoraWebsocketClient::onSSLConnect, shared_from_this(), + std::placeholders::_1))); + } else { + boost::asio::async_connect( + ws_->nativeSocket().next_layer(), results.begin(), results.end(), + boost::asio::bind_executor( + ws_->strand(), + std::bind(&SoraWebsocketClient::onConnect, shared_from_this(), + std::placeholders::_1))); + } } void SoraWebsocketClient::onSSLConnect(boost::system::error_code ec) { - if (ec) { - reconnectAfter(); - return MOMO_BOOST_ERROR(ec, "SSLConnect"); - } + if (ec) { + reconnectAfter(); + return MOMO_BOOST_ERROR(ec, "SSLConnect"); + } - // SSL のハンドシェイク - ws_->nativeSecureSocket().next_layer().async_handshake( - boost::asio::ssl::stream_base::client, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &SoraWebsocketClient::onSSLHandshake, - shared_from_this(), - std::placeholders::_1))); + // SSL のハンドシェイク + ws_->nativeSecureSocket().next_layer().async_handshake( + boost::asio::ssl::stream_base::client, + boost::asio::bind_executor( + ws_->strand(), std::bind(&SoraWebsocketClient::onSSLHandshake, + shared_from_this(), std::placeholders::_1))); } void SoraWebsocketClient::onSSLHandshake(boost::system::error_code ec) { - if (ec) { - reconnectAfter(); - return MOMO_BOOST_ERROR(ec, "SSLHandshake"); - } + if (ec) { + reconnectAfter(); + return MOMO_BOOST_ERROR(ec, "SSLHandshake"); + } - // Websocket のハンドシェイク - ws_->nativeSecureSocket().async_handshake(parts_.host, parts_.path_query_fragment, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &SoraWebsocketClient::onHandshake, - shared_from_this(), - std::placeholders::_1))); + // Websocket のハンドシェイク + ws_->nativeSecureSocket().async_handshake( + parts_.host, parts_.path_query_fragment, + boost::asio::bind_executor( + ws_->strand(), std::bind(&SoraWebsocketClient::onHandshake, + shared_from_this(), std::placeholders::_1))); } void SoraWebsocketClient::onConnect(boost::system::error_code ec) { - if (ec) { - reconnectAfter(); - return MOMO_BOOST_ERROR(ec, "connect"); - } + if (ec) { + reconnectAfter(); + return MOMO_BOOST_ERROR(ec, "connect"); + } - // Websocket のハンドシェイク - ws_->nativeSocket().async_handshake(parts_.host, parts_.path_query_fragment, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &SoraWebsocketClient::onHandshake, - shared_from_this(), - std::placeholders::_1))); + // Websocket のハンドシェイク + ws_->nativeSocket().async_handshake( + parts_.host, parts_.path_query_fragment, + boost::asio::bind_executor( + ws_->strand(), std::bind(&SoraWebsocketClient::onHandshake, + shared_from_this(), std::placeholders::_1))); } -void SoraWebsocketClient::onHandshake(boost::system::error_code ec) -{ - if (ec) { - reconnectAfter(); - return MOMO_BOOST_ERROR(ec, "Handshake"); - } +void SoraWebsocketClient::onHandshake(boost::system::error_code ec) { + if (ec) { + reconnectAfter(); + return MOMO_BOOST_ERROR(ec, "Handshake"); + } - connected_ = true; + connected_ = true; - ws_->startToRead(std::bind( - &SoraWebsocketClient::onRead, - this, - std::placeholders::_1, - std::placeholders::_2, - std::placeholders::_3)); + ws_->startToRead(std::bind(&SoraWebsocketClient::onRead, this, + std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3)); - doSendConnect(); + doSendConnect(); } -void SoraWebsocketClient::doSendConnect() -{ - json json_message = { +void SoraWebsocketClient::doSendConnect() { + json json_message = { {"type", "connect"}, {"role", "upstream"}, {"channel_id", conn_settings_.sora_channel_id}, - }; + }; - if (!conn_settings_.sora_metadata.is_null()) { - json_message["metadata"] = conn_settings_.sora_metadata; - } + if (!conn_settings_.sora_metadata.is_null()) { + json_message["metadata"] = conn_settings_.sora_metadata; + } - json_message["video"]["codec_type"] = conn_settings_.video_codec; - if (conn_settings_.video_bitrate != 0) { - json_message["video"]["bit_rate"] = conn_settings_.video_bitrate; - } + json_message["video"]["codec_type"] = conn_settings_.video_codec; + if (conn_settings_.video_bitrate != 0) { + json_message["video"]["bit_rate"] = conn_settings_.video_bitrate; + } - json_message["audio"]["codec_type"] = conn_settings_.audio_codec; - if (conn_settings_.audio_bitrate != 0) { - json_message["audio"]["bit_rate"] = conn_settings_.audio_bitrate; - } + json_message["audio"]["codec_type"] = conn_settings_.audio_codec; + if (conn_settings_.audio_bitrate != 0) { + json_message["audio"]["bit_rate"] = conn_settings_.audio_bitrate; + } - ws_->sendText(json_message.dump()); + ws_->sendText(json_message.dump()); } void SoraWebsocketClient::doSendPong() { - json json_message = { - {"type", "pong"} - }; - ws_->sendText(json_message.dump()); + json json_message = {{"type", "pong"}}; + ws_->sendText(json_message.dump()); } -void SoraWebsocketClient::createPeerFromConfig(json jconfig) -{ +void SoraWebsocketClient::createPeerFromConfig(json jconfig) { webrtc::PeerConnectionInterface::RTCConfiguration rtc_config; webrtc::PeerConnectionInterface::IceServers ice_servers; auto jservers = jconfig["iceServers"]; - for (auto jserver : jservers) - { + for (auto jserver : jservers) { const std::string username = jserver["username"]; const std::string credential = jserver["credential"]; auto jurls = jserver["urls"]; - for (const std::string url : jurls) - { + for (const std::string url : jurls) { webrtc::PeerConnectionInterface::IceServer ice_server; ice_server.uri = url; ice_server.username = username; @@ -286,110 +267,107 @@ void SoraWebsocketClient::createPeerFromConfig(json jconfig) connection_ = manager_->createConnection(rtc_config, this); } -void SoraWebsocketClient::close() -{ - if (ws_->isSSL()) { - ws_->nativeSecureSocket().async_close(boost::beast::websocket::close_code::normal, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &SoraWebsocketClient::onClose, - shared_from_this(), - std::placeholders::_1))); - } else { - ws_->nativeSocket().async_close(boost::beast::websocket::close_code::normal, - boost::asio::bind_executor( - ws_->strand(), - std::bind( - &SoraWebsocketClient::onClose, - shared_from_this(), - std::placeholders::_1))); - } +void SoraWebsocketClient::close() { + if (ws_->isSSL()) { + ws_->nativeSecureSocket().async_close( + boost::beast::websocket::close_code::normal, + boost::asio::bind_executor( + ws_->strand(), + std::bind(&SoraWebsocketClient::onClose, shared_from_this(), + std::placeholders::_1))); + } else { + ws_->nativeSocket().async_close( + boost::beast::websocket::close_code::normal, + boost::asio::bind_executor( + ws_->strand(), + std::bind(&SoraWebsocketClient::onClose, shared_from_this(), + std::placeholders::_1))); + } } -void SoraWebsocketClient::onClose(boost::system::error_code ec) -{ - if (ec) - return MOMO_BOOST_ERROR(ec, "close"); +void SoraWebsocketClient::onClose(boost::system::error_code ec) { + if (ec) + return MOMO_BOOST_ERROR(ec, "close"); } -void SoraWebsocketClient::onRead(boost::system::error_code ec, std::size_t bytes_transferred, std::string text) -{ - RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec; - - boost::ignore_unused(bytes_transferred); - - // 書き込みのために読み込み処理がキャンセルされた時にこのエラーになるので、これはエラーとして扱わない - if (ec == boost::asio::error::operation_aborted) - return; - - if (ec) - return MOMO_BOOST_ERROR(ec, "Read"); - - RTC_LOG(LS_INFO) << __FUNCTION__ << ": text=" << text; - - auto json_message = json::parse(text); - const std::string type = json_message["type"]; - if (type == "offer") { - createPeerFromConfig(json_message["config"]); - const std::string sdp = json_message["sdp"]; - connection_->setOffer(sdp); - } else if (type == "notify") { - } else if (type == "ping") { - if (rtc_state_ != webrtc::PeerConnectionInterface::IceConnectionState::kIceConnectionConnected) - { - return; - } - watchdog_.reset(); - doSendPong(); +void SoraWebsocketClient::onRead(boost::system::error_code ec, + std::size_t bytes_transferred, + std::string text) { + RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec; + + boost::ignore_unused(bytes_transferred); + + // 書き込みのために読み込み処理がキャンセルされた時にこのエラーになるので、これはエラーとして扱わない + if (ec == boost::asio::error::operation_aborted) + return; + + if (ec) + return MOMO_BOOST_ERROR(ec, "Read"); + + RTC_LOG(LS_INFO) << __FUNCTION__ << ": text=" << text; + + auto json_message = json::parse(text); + const std::string type = json_message["type"]; + if (type == "offer") { + createPeerFromConfig(json_message["config"]); + const std::string sdp = json_message["sdp"]; + connection_->setOffer(sdp); + } else if (type == "notify") { + } else if (type == "ping") { + if (rtc_state_ != webrtc::PeerConnectionInterface::IceConnectionState:: + kIceConnectionConnected) { + return; } + watchdog_.reset(); + doSendPong(); + } } // WebRTC からのコールバック // これらは別スレッドからやってくるので取り扱い注意 -void SoraWebsocketClient::onIceConnectionStateChange(webrtc::PeerConnectionInterface::IceConnectionState new_state) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " state:" << new_state; - boost::asio::post( - ws_->strand(), - std::bind( - &SoraWebsocketClient::doIceConnectionStateChange, - shared_from_this(), - new_state)); +void SoraWebsocketClient::onIceConnectionStateChange( + webrtc::PeerConnectionInterface::IceConnectionState new_state) { + RTC_LOG(LS_INFO) << __FUNCTION__ << " state:" << new_state; + boost::asio::post(ws_->strand(), + std::bind(&SoraWebsocketClient::doIceConnectionStateChange, + shared_from_this(), new_state)); } -void SoraWebsocketClient::onIceCandidate(const std::string sdp_mid, const int sdp_mlineindex, const std::string sdp) { - json json_message = { - {"type", "candidate"}, - {"candidate", sdp} - }; - ws_->sendText(json_message.dump()); +void SoraWebsocketClient::onIceCandidate(const std::string sdp_mid, + const int sdp_mlineindex, + const std::string sdp) { + json json_message = {{"type", "candidate"}, {"candidate", sdp}}; + ws_->sendText(json_message.dump()); } -void SoraWebsocketClient::onCreateDescription(webrtc::SdpType type, const std::string sdp) { - json json_message = { - {"type", "answer"}, - {"sdp", sdp} - }; - ws_->sendText(json_message.dump()); +void SoraWebsocketClient::onCreateDescription(webrtc::SdpType type, + const std::string sdp) { + json json_message = {{"type", "answer"}, {"sdp", sdp}}; + ws_->sendText(json_message.dump()); } void SoraWebsocketClient::onSetDescription(webrtc::SdpType type) { - RTC_LOG(LS_INFO) << __FUNCTION__ << " SdpType: " << webrtc::SdpTypeToString(type); + RTC_LOG(LS_INFO) << __FUNCTION__ + << " SdpType: " << webrtc::SdpTypeToString(type); if (type == webrtc::SdpType::kOffer) { connection_->createAnswer(); } } -void SoraWebsocketClient::doIceConnectionStateChange(webrtc::PeerConnectionInterface::IceConnectionState new_state) { - RTC_LOG(LS_INFO) << __FUNCTION__ << ": newState=" << Util::iceConnectionStateToString(new_state); - - switch (new_state) { - case webrtc::PeerConnectionInterface::IceConnectionState::kIceConnectionConnected: - retry_count_ = 0; - watchdog_.enable(60); - break; - case webrtc::PeerConnectionInterface::IceConnectionState::kIceConnectionFailed: - reconnectAfter(); - break; - default: - break; - } - rtc_state_ = new_state; +void SoraWebsocketClient::doIceConnectionStateChange( + webrtc::PeerConnectionInterface::IceConnectionState new_state) { + RTC_LOG(LS_INFO) << __FUNCTION__ << ": newState=" + << Util::iceConnectionStateToString(new_state); + + switch (new_state) { + case webrtc::PeerConnectionInterface::IceConnectionState:: + kIceConnectionConnected: + retry_count_ = 0; + watchdog_.enable(60); + break; + case webrtc::PeerConnectionInterface::IceConnectionState:: + kIceConnectionFailed: + reconnectAfter(); + break; + default: + break; + } + rtc_state_ = new_state; } diff --git a/src/sora/sora_websocket_client.h b/src/sora/sora_websocket_client.h index 5afc375b..fc97e441 100644 --- a/src/sora/sora_websocket_client.h +++ b/src/sora/sora_websocket_client.h @@ -2,97 +2,109 @@ #define SORA_WEBSOCKET_CLIENT_ #include -#include -#include -#include -#include #include #include #include +#include +#include +#include #include +#include -#include "ws/websocket.h" -#include "rtc/messagesender.h" -#include "rtc/manager.h" #include "connection_settings.h" +#include "rtc/manager.h" +#include "rtc/messagesender.h" #include "url_parts.h" #include "watchdog.h" +#include "ws/websocket.h" -class SoraWebsocketClient : public std::enable_shared_from_this, public RTCMessageSender -{ - boost::asio::io_context& ioc_; - - boost::asio::ip::tcp::resolver resolver_; - - std::unique_ptr ws_; - - URLParts parts_; - - RTCManager* manager_; - std::shared_ptr connection_; - ConnectionSettings conn_settings_; - - int retry_count_; - webrtc::PeerConnectionInterface::IceConnectionState rtc_state_; - - WatchDog watchdog_; - - bool connected_; - -private: - bool parseURL(URLParts& parts) const; - boost::asio::ssl::context createSSLContext() const; - -public: - webrtc::PeerConnectionInterface::IceConnectionState getRTCConnectionState() const; - std::shared_ptr getRTCConnection() const; - -public: - SoraWebsocketClient(boost::asio::io_context& ioc, RTCManager* manager, ConnectionSettings conn_settings); - void reset(); - -public: - // connection_ = nullptr すると直ちに onIceConnectionStateChange コールバックが呼ばれるが、 - // この中で使っている shared_from_this() がデストラクタ内で使えないため、デストラクタで connection_ = nullptr すると実行時エラーになる。 - // なのでこのクラスを解放する前に明示的に release() 関数を呼んでもらうことにする。 - void release(); - -public: - bool connect(); - -private: - void reconnectAfter(); - void onWatchdogExpired(); - -private: - void onResolve(boost::system::error_code ec, boost::asio::ip::tcp::resolver::results_type results); - void onSSLConnect(boost::system::error_code ec); - void onSSLHandshake(boost::system::error_code ec); - void onConnect(boost::system::error_code ec); - void onHandshake(boost::system::error_code ec); - void doSendConnect(); - void doSendPong(); - void createPeerFromConfig(nlohmann::json jconfig); - -public: - void close(); - -private: - void onClose(boost::system::error_code ec); - -private: - void onRead(boost::system::error_code ec, std::size_t bytes_transferred, std::string text); - -private: - // WebRTC からのコールバック - // これらは別スレッドからやってくるので取り扱い注意 - void onIceConnectionStateChange(webrtc::PeerConnectionInterface::IceConnectionState new_state) override; - void onIceCandidate(const std::string sdp_mid, const int sdp_mlineindex, const std::string sdp) override; - void onCreateDescription(webrtc::SdpType type, const std::string sdp) override; - void onSetDescription(webrtc::SdpType type) override; - -private: - void doIceConnectionStateChange(webrtc::PeerConnectionInterface::IceConnectionState new_state); +class SoraWebsocketClient + : public std::enable_shared_from_this, + public RTCMessageSender { + boost::asio::io_context& ioc_; + + boost::asio::ip::tcp::resolver resolver_; + + std::unique_ptr ws_; + + URLParts parts_; + + RTCManager* manager_; + std::shared_ptr connection_; + ConnectionSettings conn_settings_; + + int retry_count_; + webrtc::PeerConnectionInterface::IceConnectionState rtc_state_; + + WatchDog watchdog_; + + bool connected_; + + private: + bool parseURL(URLParts& parts) const; + boost::asio::ssl::context createSSLContext() const; + + public: + webrtc::PeerConnectionInterface::IceConnectionState getRTCConnectionState() + const; + std::shared_ptr getRTCConnection() const; + + public: + SoraWebsocketClient(boost::asio::io_context& ioc, + RTCManager* manager, + ConnectionSettings conn_settings); + void reset(); + + public: + // connection_ = nullptr すると直ちに onIceConnectionStateChange コールバックが呼ばれるが、 + // この中で使っている shared_from_this() がデストラクタ内で使えないため、デストラクタで connection_ = nullptr すると実行時エラーになる。 + // なのでこのクラスを解放する前に明示的に release() 関数を呼んでもらうことにする。 + void release(); + + public: + bool connect(); + + private: + void reconnectAfter(); + void onWatchdogExpired(); + + private: + void onResolve(boost::system::error_code ec, + boost::asio::ip::tcp::resolver::results_type results); + void onSSLConnect(boost::system::error_code ec); + void onSSLHandshake(boost::system::error_code ec); + void onConnect(boost::system::error_code ec); + void onHandshake(boost::system::error_code ec); + void doSendConnect(); + void doSendPong(); + void createPeerFromConfig(nlohmann::json jconfig); + + public: + void close(); + + private: + void onClose(boost::system::error_code ec); + + private: + void onRead(boost::system::error_code ec, + std::size_t bytes_transferred, + std::string text); + + private: + // WebRTC からのコールバック + // これらは別スレッドからやってくるので取り扱い注意 + void onIceConnectionStateChange( + webrtc::PeerConnectionInterface::IceConnectionState new_state) override; + void onIceCandidate(const std::string sdp_mid, + const int sdp_mlineindex, + const std::string sdp) override; + void onCreateDescription(webrtc::SdpType type, + const std::string sdp) override; + void onSetDescription(webrtc::SdpType type) override; + + private: + void doIceConnectionStateChange( + webrtc::PeerConnectionInterface::IceConnectionState new_state); }; -#endif // SORA_WEBSOCKET_CLIENT_ +#endif // SORA_WEBSOCKET_CLIENT_ diff --git a/src/url_parts.h b/src/url_parts.h index 3d4ecc0a..cafe0c1b 100644 --- a/src/url_parts.h +++ b/src/url_parts.h @@ -9,57 +9,55 @@ Boost.Beast の URL パーサはまだ [PR の段階](https://github.com/boostorg/beast/pull/1231) なので、自前で適当に作っておく。 */ struct URLParts { - std::string scheme; - std::string user_pass; - std::string host; - std::string port; - std::string path_query_fragment; - - // 適当 URL パース - // scheme://[user_pass@]host[:port][/path_query_fragment] - static bool parse(std::string url, URLParts& parts) - { - auto n = url.find("://"); - if (n == std::string::npos) { - return false; - } - parts.scheme = url.substr(0, n); - - n += 3; - auto m = url.find('/', n); - std::string user_pass_host_port; - if (m == std::string::npos) { - user_pass_host_port = url.substr(n); - parts.path_query_fragment = ""; - } else { - user_pass_host_port = url.substr(n, m - n); - parts.path_query_fragment = url.substr(m); - } - - n = 0; - m = user_pass_host_port.find('@'); - std::string host_port; - if (m == std::string::npos) { - parts.user_pass = ""; - host_port = std::move(user_pass_host_port); - } else { - parts.user_pass = user_pass_host_port.substr(n, m - n); - host_port = user_pass_host_port.substr(m + 1); - } + std::string scheme; + std::string user_pass; + std::string host; + std::string port; + std::string path_query_fragment; + + // 適当 URL パース + // scheme://[user_pass@]host[:port][/path_query_fragment] + static bool parse(std::string url, URLParts& parts) { + auto n = url.find("://"); + if (n == std::string::npos) { + return false; + } + parts.scheme = url.substr(0, n); + + n += 3; + auto m = url.find('/', n); + std::string user_pass_host_port; + if (m == std::string::npos) { + user_pass_host_port = url.substr(n); + parts.path_query_fragment = ""; + } else { + user_pass_host_port = url.substr(n, m - n); + parts.path_query_fragment = url.substr(m); + } - n = 0; - m = host_port.find(':'); - if (m == std::string::npos) { - parts.host = std::move(host_port); - parts.port = ""; - } else { - parts.host = host_port.substr(n, m - n); - parts.port = host_port.substr(m + 1); - } + n = 0; + m = user_pass_host_port.find('@'); + std::string host_port; + if (m == std::string::npos) { + parts.user_pass = ""; + host_port = std::move(user_pass_host_port); + } else { + parts.user_pass = user_pass_host_port.substr(n, m - n); + host_port = user_pass_host_port.substr(m + 1); + } - return true; + n = 0; + m = host_port.find(':'); + if (m == std::string::npos) { + parts.host = std::move(host_port); + parts.port = ""; + } else { + parts.host = host_port.substr(n, m - n); + parts.port = host_port.substr(m + 1); } -}; + return true; + } +}; -#endif // URL_PARTS_H_ +#endif // URL_PARTS_H_ diff --git a/src/util.cpp b/src/util.cpp index e0dedec6..77c2eca1 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -1,12 +1,12 @@ #include "util.h" -#include "rtc_base/helpers.h" #include -#include #include -#include -#include #include +#include +#include +#include +#include "rtc_base/helpers.h" #if USE_ROS #include "ros/ros.h" #endif @@ -19,26 +19,30 @@ // HWA を効かせる場合は 1 になる #if USE_MMAL_ENCODER - #define MOMO_USE_MMAL_ENCODER 1 +#define MOMO_USE_MMAL_ENCODER 1 #else - #define MOMO_USE_MMAL_ENCODER 0 +#define MOMO_USE_MMAL_ENCODER 0 #endif // H264 を有効にする場合は 1 になる #if USE_H264 - #define MOMO_USE_H264 1 +#define MOMO_USE_H264 1 #else - #define MOMO_USE_H264 0 +#define MOMO_USE_H264 0 #endif using json = nlohmann::json; #if USE_ROS -void Util::parseArgs(int argc, char *argv[], bool &is_daemon, - bool &use_test, bool &use_ayame, bool &use_sora, - int &log_level, ConnectionSettings &cs) -{ +void Util::parseArgs(int argc, + char* argv[], + bool& is_daemon, + bool& use_test, + bool& use_ayame, + bool& use_sora, + int& log_level, + ConnectionSettings& cs) { ros::init(argc, argv, "momo", ros::init_options::AnonymousName); ros::NodeHandle nh; @@ -64,7 +68,8 @@ void Util::parseArgs(int argc, char *argv[], bool &is_daemon, local_nh.param("audio_bitrate", cs.audio_bitrate, cs.audio_bitrate); local_nh.param("resolution", cs.resolution, cs.resolution); local_nh.param("framerate", cs.framerate, cs.framerate); - local_nh.param("audio_topic_rate", cs.audio_topic_rate, cs.audio_topic_rate); + local_nh.param("audio_topic_rate", cs.audio_topic_rate, + cs.audio_topic_rate); local_nh.param("audio_topic_ch", cs.audio_topic_ch, cs.audio_topic_ch); local_nh.param("priority", cs.priority, cs.priority); local_nh.param("port", cs.port, cs.port); @@ -85,7 +90,8 @@ void Util::parseArgs(int argc, char *argv[], bool &is_daemon, local_nh.param("disable_typing_detection", cs.disable_typing_detection, cs.disable_typing_detection); - if (use_sora && local_nh.hasParam("SIGNALING_URL") && local_nh.hasParam("CHANNEL_ID")) { + if (use_sora && local_nh.hasParam("SIGNALING_URL") && + local_nh.hasParam("CHANNEL_ID")) { local_nh.getParam("SIGNALING_URL", cs.sora_signaling_host); local_nh.getParam("CHANNEL_ID", cs.sora_channel_id); local_nh.param("auto", cs.sora_auto_connect, cs.sora_auto_connect); @@ -94,19 +100,22 @@ void Util::parseArgs(int argc, char *argv[], bool &is_daemon, local_nh.param("metadata", sora_metadata, ""); // メタデータのパース - if (!sora_metadata.empty()) - { + if (!sora_metadata.empty()) { cs.sora_metadata = json::parse(sora_metadata); } } else if (use_test) { - local_nh.param("document_root", cs.test_document_root, get_current_dir_name()); - } else if (use_ayame && local_nh.hasParam("SIGNALING_URL") && local_nh.hasParam("ROOM_ID")) { + local_nh.param("document_root", cs.test_document_root, + get_current_dir_name()); + } else if (use_ayame && local_nh.hasParam("SIGNALING_URL") && + local_nh.hasParam("ROOM_ID")) { local_nh.getParam("SIGNALING_URL", cs.ayame_signaling_host); local_nh.getParam("ROOM_ID", cs.ayame_room_id); // デフォルトはランダムな数値 17 桁 std::string default_ayame_client_id = generateRandomNumericChars(17); - local_nh.param("client_id", cs.ayame_client_id, default_ayame_client_id); - local_nh.param("signaling_key", cs.ayame_signaling_key, cs.ayame_signaling_key); + local_nh.param("client_id", cs.ayame_client_id, + default_ayame_client_id); + local_nh.param("signaling_key", cs.ayame_signaling_key, + cs.ayame_signaling_key); } else { exit(1); } @@ -114,10 +123,14 @@ void Util::parseArgs(int argc, char *argv[], bool &is_daemon, #else -void Util::parseArgs(int argc, char *argv[], bool &is_daemon, - bool &use_test, bool &use_ayame, bool &use_sora, - int &log_level, ConnectionSettings &cs) -{ +void Util::parseArgs(int argc, + char* argv[], + bool& is_daemon, + bool& use_test, + bool& use_ayame, + bool& use_sora, + int& log_level, + ConnectionSettings& cs) { CLI::App app("Momo - WebRTC ネイティブクライアント"); bool version = false; @@ -126,12 +139,16 @@ void Util::parseArgs(int argc, char *argv[], bool &is_daemon, app.add_flag("--no-audio", cs.no_audio, "オーディオを出さない"); #if USE_MMAL_ENCODER app.add_flag("--force-i420", cs.force_i420, "強制的にI420にする"); - app.add_flag("--use-native", cs.use_native, "MJPEGのデコードとビデオのリサイズをハードウェアで行う"); - app.add_option("--video-device", cs.video_device, "デバイスファイル名。省略時はどれかのビデオデバイスを自動検出")->check(CLI::ExistingFile); + app.add_flag("--use-native", cs.use_native, + "MJPEGのデコードとビデオのリサイズをハードウェアで行う"); + app.add_option("--video-device", cs.video_device, + "デバイスファイル名。省略時はどれかのビデオデバイスを自動検出") + ->check(CLI::ExistingFile); #endif app.add_set("--resolution", cs.resolution, {"QVGA", "VGA", "HD", "FHD", "4K"}, "解像度"); - app.add_option("--framerate", cs.framerate, "フレームレート")->check(CLI::Range(1, 60)); + app.add_option("--framerate", cs.framerate, "フレームレート") + ->check(CLI::Range(1, 60)); app.add_flag("--fixed-resolution", cs.fixed_resolution, "固定解像度"); app.add_set("--priority", cs.priority, {"BALANCE", "FRAMERATE", "RESOLUTION"}, "優先設定 (Experimental)"); @@ -139,7 +156,7 @@ void Util::parseArgs(int argc, char *argv[], bool &is_daemon, ->check(CLI::Range(0, 65535)); app.add_flag("--daemon", is_daemon, "デーモン化する"); app.add_flag("--version", version, "バージョン情報の表示"); - auto log_level_map = std::vector>( + auto log_level_map = std::vector >( {{"verbose", 0}, {"info", 1}, {"warning", 2}, {"error", 3}, {"none", 4}}); app.add_option("--log-level", log_level, "ログレベル") ->transform(CLI::CheckedTransformer(log_level_map, CLI::ignore_case)); @@ -164,15 +181,23 @@ void Util::parseArgs(int argc, char *argv[], bool &is_daemon, ->add_option("--document-root", cs.test_document_root, "配信ディレクトリ") ->check(CLI::ExistingDirectory); - ayame_app->add_option("SIGNALING-URL", cs.ayame_signaling_host, "シグナリングホスト")->required(); + ayame_app + ->add_option("SIGNALING-URL", cs.ayame_signaling_host, + "シグナリングホスト") + ->required(); ayame_app->add_option("ROOM-ID", cs.ayame_room_id, "ルームID")->required(); // デフォルトはランダムな数値 17 桁 cs.ayame_client_id = generateRandomNumericChars(17); - ayame_app ->add_option("--client-id", cs.ayame_client_id, "クライアントID"); - ayame_app ->add_option("--signaling-key", cs.ayame_signaling_key, "シグナリングキー"); + ayame_app->add_option("--client-id", cs.ayame_client_id, "クライアントID"); + ayame_app->add_option("--signaling-key", cs.ayame_signaling_key, + "シグナリングキー"); - sora_app->add_option("SIGNALING-URL", cs.sora_signaling_host, "シグナリングホスト")->required(); - sora_app->add_option("CHANNEL-ID", cs.sora_channel_id, "チャンネルID")->required(); + sora_app + ->add_option("SIGNALING-URL", cs.sora_signaling_host, + "シグナリングホスト") + ->required(); + sora_app->add_option("CHANNEL-ID", cs.sora_channel_id, "チャンネルID") + ->required(); sora_app->add_flag("--auto", cs.sora_auto_connect, "自動接続する"); #if MOMO_USE_H264 sora_app->add_set("--video-codec", cs.video_codec, {"VP8", "VP9", "H264"}, @@ -196,7 +221,7 @@ void Util::parseArgs(int argc, char *argv[], bool &is_daemon, try { auto _ = json::parse(input); return std::string(); - } catch (json::parse_error &e) { + } catch (json::parse_error& e) { return "Value " + input + " is not JSON Value"; } }, @@ -205,12 +230,9 @@ void Util::parseArgs(int argc, char *argv[], bool &is_daemon, sora_app->add_option("--metadata", sora_metadata, "メタデータ") ->check(is_json); - try - { + try { app.parse(argc, argv); - } - catch (const CLI::ParseError &e) - { + } catch (const CLI::ParseError& e) { exit(app.exit(e)); } @@ -223,14 +245,14 @@ void Util::parseArgs(int argc, char *argv[], bool &is_daemon, cs.test_document_root = boost::filesystem::current_path().string(); } - if (version) - { - std::cout << "WebRTC Native Client Momo version " MOMO_VERSION " USE_MMAL_ENCODER=" BOOST_PP_STRINGIZE(MOMO_USE_MMAL_ENCODER) << std::endl; + if (version) { + std::cout << "WebRTC Native Client Momo version " MOMO_VERSION + " USE_MMAL_ENCODER=" BOOST_PP_STRINGIZE(MOMO_USE_MMAL_ENCODER) + << std::endl; exit(0); } - if (!test_app->parsed() && !sora_app->parsed() && !ayame_app->parsed()) - { + if (!test_app->parsed() && !sora_app->parsed() && !ayame_app->parsed()) { std::cout << app.help() << std::endl; exit(1); } @@ -250,32 +272,29 @@ void Util::parseArgs(int argc, char *argv[], bool &is_daemon, #endif -std::string Util::generateRandomChars() -{ +std::string Util::generateRandomChars() { return generateRandomChars(32); } -std::string Util::generateRandomChars(size_t length) -{ +std::string Util::generateRandomChars(size_t length) { std::string result; rtc::CreateRandomString(length, &result); return result; } std::string Util::generateRandomNumericChars(size_t length) { - auto randomNumerics = []() -> char - { - const char charset[] = "0123456789"; - const size_t max_index = (sizeof(charset) - 1); - return charset[ rand() % max_index ]; - }; - std::string result(length, 0); - std::generate_n(result.begin(), length, randomNumerics); - return result; + auto randomNumerics = []() -> char { + const char charset[] = "0123456789"; + const size_t max_index = (sizeof(charset) - 1); + return charset[rand() % max_index]; + }; + std::string result(length, 0); + std::generate_n(result.begin(), length, randomNumerics); + return result; } std::string Util::iceConnectionStateToString( - webrtc::PeerConnectionInterface::IceConnectionState state) { + webrtc::PeerConnectionInterface::IceConnectionState state) { switch (state) { case webrtc::PeerConnectionInterface::kIceConnectionNew: return "new"; @@ -301,65 +320,93 @@ namespace http = boost::beast::http; using string_view = boost::beast::string_view; string_view Util::mimeType(string_view path) { - using boost::beast::iequals; - auto const ext = [&path] - { - auto const pos = path.rfind("."); - if(pos == string_view::npos) - return string_view{}; - return path.substr(pos); - }(); - - if (iequals(ext, ".htm")) return "text/html"; - if (iequals(ext, ".html")) return "text/html"; - if (iequals(ext, ".php")) return "text/html"; - if (iequals(ext, ".css")) return "text/css"; - if (iequals(ext, ".txt")) return "text/plain"; - if (iequals(ext, ".js")) return "application/javascript"; - if (iequals(ext, ".json")) return "application/json"; - if (iequals(ext, ".xml")) return "application/xml"; - if (iequals(ext, ".swf")) return "application/x-shockwave-flash"; - if (iequals(ext, ".flv")) return "video/x-flv"; - if (iequals(ext, ".png")) return "image/png"; - if (iequals(ext, ".jpe")) return "image/jpeg"; - if (iequals(ext, ".jpeg")) return "image/jpeg"; - if (iequals(ext, ".jpg")) return "image/jpeg"; - if (iequals(ext, ".gif")) return "image/gif"; - if (iequals(ext, ".bmp")) return "image/bmp"; - if (iequals(ext, ".ico")) return "image/vnd.microsoft.icon"; - if (iequals(ext, ".tiff")) return "image/tiff"; - if (iequals(ext, ".tif")) return "image/tiff"; - if (iequals(ext, ".svg")) return "image/svg+xml"; - if (iequals(ext, ".svgz")) return "image/svg+xml"; - return "application/text"; + using boost::beast::iequals; + auto const ext = [&path] { + auto const pos = path.rfind("."); + if (pos == string_view::npos) + return string_view{}; + return path.substr(pos); + }(); + + if (iequals(ext, ".htm")) + return "text/html"; + if (iequals(ext, ".html")) + return "text/html"; + if (iequals(ext, ".php")) + return "text/html"; + if (iequals(ext, ".css")) + return "text/css"; + if (iequals(ext, ".txt")) + return "text/plain"; + if (iequals(ext, ".js")) + return "application/javascript"; + if (iequals(ext, ".json")) + return "application/json"; + if (iequals(ext, ".xml")) + return "application/xml"; + if (iequals(ext, ".swf")) + return "application/x-shockwave-flash"; + if (iequals(ext, ".flv")) + return "video/x-flv"; + if (iequals(ext, ".png")) + return "image/png"; + if (iequals(ext, ".jpe")) + return "image/jpeg"; + if (iequals(ext, ".jpeg")) + return "image/jpeg"; + if (iequals(ext, ".jpg")) + return "image/jpeg"; + if (iequals(ext, ".gif")) + return "image/gif"; + if (iequals(ext, ".bmp")) + return "image/bmp"; + if (iequals(ext, ".ico")) + return "image/vnd.microsoft.icon"; + if (iequals(ext, ".tiff")) + return "image/tiff"; + if (iequals(ext, ".tif")) + return "image/tiff"; + if (iequals(ext, ".svg")) + return "image/svg+xml"; + if (iequals(ext, ".svgz")) + return "image/svg+xml"; + return "application/text"; } -http::response Util::badRequest(const http::request& req, string_view why) { - http::response res{http::status::bad_request, req.version()}; - res.set(http::field::server, BOOST_BEAST_VERSION_STRING); - res.set(http::field::content_type, "text/html"); - res.keep_alive(req.keep_alive()); - res.body() = why.to_string(); - res.prepare_payload(); - return res; +http::response Util::badRequest( + const http::request& req, + string_view why) { + http::response res{http::status::bad_request, + req.version()}; + res.set(http::field::server, BOOST_BEAST_VERSION_STRING); + res.set(http::field::content_type, "text/html"); + res.keep_alive(req.keep_alive()); + res.body() = why.to_string(); + res.prepare_payload(); + return res; } -http::response Util::notFound(const http::request& req, string_view target) { - http::response res{http::status::not_found, req.version()}; - res.set(http::field::server, BOOST_BEAST_VERSION_STRING); - res.set(http::field::content_type, "text/html"); - res.keep_alive(req.keep_alive()); - res.body() = "The resource '" + target.to_string() + "' was not found."; - res.prepare_payload(); - return res; +http::response Util::notFound( + const http::request& req, + string_view target) { + http::response res{http::status::not_found, req.version()}; + res.set(http::field::server, BOOST_BEAST_VERSION_STRING); + res.set(http::field::content_type, "text/html"); + res.keep_alive(req.keep_alive()); + res.body() = "The resource '" + target.to_string() + "' was not found."; + res.prepare_payload(); + return res; } -http::response Util::serverError(const http::request& req, string_view what) { - http::response res{http::status::internal_server_error, req.version()}; - res.set(http::field::server, BOOST_BEAST_VERSION_STRING); - res.set(http::field::content_type, "text/html"); - res.keep_alive(req.keep_alive()); - res.body() = "An error occurred: '" + what.to_string() + "'"; - res.prepare_payload(); - return res; +http::response Util::serverError( + const http::request& req, + string_view what) { + http::response res{http::status::internal_server_error, + req.version()}; + res.set(http::field::server, BOOST_BEAST_VERSION_STRING); + res.set(http::field::content_type, "text/html"); + res.keep_alive(req.keep_alive()); + res.body() = "An error occurred: '" + what.to_string() + "'"; + res.prepare_payload(); + return res; } diff --git a/src/util.h b/src/util.h index 3a2a44a1..98121be8 100644 --- a/src/util.h +++ b/src/util.h @@ -2,30 +2,42 @@ #define RTC_UTIL_H_ #include -#include #include +#include #include "api/peer_connection_interface.h" #include "connection_settings.h" -class Util -{ - public: - static void parseArgs(int argc, char *argv[], bool &is_daemon, - bool &use_test, bool &use_ayame, bool &use_sora, - int &log_level, ConnectionSettings &cs); - static std::string generateRandomChars(); - static std::string generateRandomChars(size_t length); - static std::string generateRandomNumericChars(size_t length); - static std::string iceConnectionStateToString( - webrtc::PeerConnectionInterface::IceConnectionState state) ; +class Util { + public: + static void parseArgs(int argc, + char* argv[], + bool& is_daemon, + bool& use_test, + bool& use_ayame, + bool& use_sora, + int& log_level, + ConnectionSettings& cs); + static std::string generateRandomChars(); + static std::string generateRandomChars(size_t length); + static std::string generateRandomNumericChars(size_t length); + static std::string iceConnectionStateToString( + webrtc::PeerConnectionInterface::IceConnectionState state); - // MIME type をファイル名の拡張子から調べる - static boost::beast::string_view mimeType(boost::beast::string_view path); + // MIME type をファイル名の拡張子から調べる + static boost::beast::string_view mimeType(boost::beast::string_view path); - // エラーレスポンスをいい感じに作る便利関数 - static boost::beast::http::response badRequest(const boost::beast::http::request& req, boost::beast::string_view why); - static boost::beast::http::response notFound(const boost::beast::http::request& req, boost::beast::string_view target); - static boost::beast::http::response serverError(const boost::beast::http::request& req, boost::beast::string_view what); + // エラーレスポンスをいい感じに作る便利関数 + static boost::beast::http::response + badRequest( + const boost::beast::http::request& req, + boost::beast::string_view why); + static boost::beast::http::response notFound( + const boost::beast::http::request& req, + boost::beast::string_view target); + static boost::beast::http::response + serverError( + const boost::beast::http::request& req, + boost::beast::string_view what); }; // boost::system::error_code のエラーをいい感じに出力するマクロ @@ -34,6 +46,9 @@ class Util // return MOMO_BOOST_ERROR(ec, "onRead") // // のように、return と組み合わせて使える。 -#define MOMO_BOOST_ERROR(ec, what) ([&ec]{ RTC_LOG(LS_ERROR) << __FUNCTION__ << " " what ": " << ec.message(); }()) +#define MOMO_BOOST_ERROR(ec, what) \ + ([&ec] { \ + RTC_LOG(LS_ERROR) << __FUNCTION__ << " " what ": " << ec.message(); \ + }()) #endif diff --git a/src/v4l2_video_capturer/v4l2_video_capturer.cpp b/src/v4l2_video_capturer/v4l2_video_capturer.cpp index c24ccfc7..667e7a81 100644 --- a/src/v4l2_video_capturer/v4l2_video_capturer.cpp +++ b/src/v4l2_video_capturer/v4l2_video_capturer.cpp @@ -34,7 +34,8 @@ #include "rtc/native_buffer.h" -rtc::scoped_refptr V4L2VideoCapture::Create(ConnectionSettings cs) { +rtc::scoped_refptr V4L2VideoCapture::Create( + ConnectionSettings cs) { rtc::scoped_refptr capturer; std::unique_ptr device_info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); @@ -48,14 +49,15 @@ rtc::scoped_refptr V4L2VideoCapture::Create(ConnectionSettings for (int i = 0; i < num_devices; ++i) { char device_name[256]; char unique_name[256]; - if (device_info->GetDeviceName(static_cast(i), - device_name, sizeof(device_name), - unique_name, sizeof(unique_name)) != 0) - { + if (device_info->GetDeviceName(static_cast(i), device_name, + sizeof(device_name), unique_name, + sizeof(unique_name)) != 0) { RTC_LOG(LS_WARNING) << "Failed to GetDeviceName(" << i << ")"; continue; } - RTC_LOG(LS_INFO) << "GetDeviceName(" << i << "): device_name=" << device_name << ", unique_name=" << unique_name; + RTC_LOG(LS_INFO) << "GetDeviceName(" << i + << "): device_name=" << device_name + << ", unique_name=" << unique_name; } for (int i = 0; i < num_devices; ++i) { @@ -77,22 +79,20 @@ rtc::scoped_refptr V4L2VideoCapture::Create( char unique_name[256]; if (device_info->GetDeviceName(static_cast(capture_device_index), device_name, sizeof(device_name), unique_name, - sizeof(unique_name)) != 0) - { + sizeof(unique_name)) != 0) { RTC_LOG(LS_WARNING) << "Failed to GetDeviceName"; return nullptr; - } + } rtc::scoped_refptr v4l2_capturer( - new rtc::RefCountedObject()); + new rtc::RefCountedObject()); if (v4l2_capturer->Init((const char*)&unique_name, cs.video_device) < 0) { - RTC_LOG(LS_WARNING) << "Failed to create V4L2VideoCapture(" - << unique_name + RTC_LOG(LS_WARNING) << "Failed to create V4L2VideoCapture(" << unique_name << ")"; return nullptr; } if (v4l2_capturer->StartCapture(cs) < 0) { - RTC_LOG(LS_WARNING) << "Failed to start V4L2VideoCapture(w = " << cs.getWidth() - << ", h = " << cs.getHeight() + RTC_LOG(LS_WARNING) << "Failed to start V4L2VideoCapture(w = " + << cs.getWidth() << ", h = " << cs.getHeight() << ", fps = " << cs.framerate << ")"; return nullptr; } @@ -110,15 +110,15 @@ V4L2VideoCapture::V4L2VideoCapture() _captureVideoType(webrtc::VideoType::kI420), _pool(NULL) {} -bool V4L2VideoCapture::FindDevice(const char* deviceUniqueIdUTF8, const std::string& device) { +bool V4L2VideoCapture::FindDevice(const char* deviceUniqueIdUTF8, + const std::string& device) { int fd; if ((fd = open(device.c_str(), O_RDONLY)) != -1) { // query device capabilities struct v4l2_capability cap; if (ioctl(fd, VIDIOC_QUERYCAP, &cap) == 0) { if (cap.bus_info[0] != 0) { - if (strncmp((const char*)cap.bus_info, - (const char*)deviceUniqueIdUTF8, + if (strncmp((const char*)cap.bus_info, (const char*)deviceUniqueIdUTF8, strlen((const char*)deviceUniqueIdUTF8)) == 0) // match with device id { @@ -132,7 +132,8 @@ bool V4L2VideoCapture::FindDevice(const char* deviceUniqueIdUTF8, const std::str return false; } -int32_t V4L2VideoCapture::Init(const char* deviceUniqueIdUTF8, const std::string& specifiedVideoDevice) { +int32_t V4L2VideoCapture::Init(const char* deviceUniqueIdUTF8, + const std::string& specifiedVideoDevice) { int fd; bool found = false; @@ -172,8 +173,7 @@ V4L2VideoCapture::~V4L2VideoCapture() { int32_t V4L2VideoCapture::StartCapture(ConnectionSettings cs) { if (_captureStarted) { - if (cs.getWidth() == _currentWidth && - cs.getHeight() == _currentHeight) { + if (cs.getWidth() == _currentWidth && cs.getHeight() == _currentHeight) { return 0; } else { StopCapture(); @@ -183,7 +183,8 @@ int32_t V4L2VideoCapture::StartCapture(ConnectionSettings cs) { rtc::CritScope critScope(&_captureCritSect); // first open /dev/video device if ((_deviceFd = open(_videoDevice.c_str(), O_RDWR | O_NONBLOCK, 0)) < 0) { - RTC_LOG(LS_INFO) << "error in opening " << _videoDevice << " errono = " << errno; + RTC_LOG(LS_INFO) << "error in opening " << _videoDevice + << " errono = " << errno; return -1; } @@ -290,7 +291,8 @@ int32_t V4L2VideoCapture::StartCapture(ConnectionSettings cs) { // If driver doesn't support framerate control, need to hardcode. // Hardcoding the value based on the frame size. if (!driver_framerate_support) { - if (!_useNative && _currentWidth >= 800 && _captureVideoType != webrtc::VideoType::kMJPEG) { + if (!_useNative && _currentWidth >= 800 && + _captureVideoType != webrtc::VideoType::kMJPEG) { _currentFrameRate = 15; } else { _currentFrameRate = 30; @@ -348,9 +350,8 @@ int32_t V4L2VideoCapture::StopCapture() { } bool V4L2VideoCapture::useNativeBuffer() { - return _useNative && - (_captureVideoType == webrtc::VideoType::kMJPEG || - _captureVideoType == webrtc::VideoType::kI420); + return _useNative && (_captureVideoType == webrtc::VideoType::kMJPEG || + _captureVideoType == webrtc::VideoType::kI420); } // critical section protected by the caller @@ -474,40 +475,32 @@ bool V4L2VideoCapture::CaptureProcess() { } rtc::scoped_refptr dst_buffer = nullptr; - if (useNativeBuffer()) - { - rtc::scoped_refptr native_buffer( - NativeBuffer::Create( - _captureVideoType, _currentWidth, _currentHeight)); + if (useNativeBuffer()) { + rtc::scoped_refptr native_buffer(NativeBuffer::Create( + _captureVideoType, _currentWidth, _currentHeight)); memcpy(native_buffer->MutableData(), - (unsigned char*)_pool[buf.index].start, - buf.bytesused); + (unsigned char*)_pool[buf.index].start, buf.bytesused); native_buffer->SetLength(buf.bytesused); dst_buffer = native_buffer; - } - else - { + } else { rtc::scoped_refptr i420_buffer( webrtc::I420Buffer::Create(_currentWidth, _currentHeight)); i420_buffer->InitializeData(); - if (libyuv::ConvertToI420((unsigned char*)_pool[buf.index].start, buf.bytesused, - i420_buffer.get()->MutableDataY(), i420_buffer.get()->StrideY(), - i420_buffer.get()->MutableDataU(), i420_buffer.get()->StrideU(), - i420_buffer.get()->MutableDataV(), i420_buffer.get()->StrideV(), - 0, 0, _currentWidth, _currentHeight, - _currentWidth, _currentHeight, - libyuv::kRotate0, ConvertVideoType(_captureVideoType)) < 0) - { + if (libyuv::ConvertToI420( + (unsigned char*)_pool[buf.index].start, buf.bytesused, + i420_buffer.get()->MutableDataY(), i420_buffer.get()->StrideY(), + i420_buffer.get()->MutableDataU(), i420_buffer.get()->StrideU(), + i420_buffer.get()->MutableDataV(), i420_buffer.get()->StrideV(), + 0, 0, _currentWidth, _currentHeight, _currentWidth, + _currentHeight, libyuv::kRotate0, + ConvertVideoType(_captureVideoType)) < 0) { RTC_LOG(LS_ERROR) << "ConvertToI420 Failed"; - } - else - { + } else { dst_buffer = i420_buffer; } } - if (dst_buffer) - { + if (dst_buffer) { webrtc::VideoFrame video_frame = webrtc::VideoFrame::Builder() .set_video_frame_buffer(dst_buffer) @@ -518,7 +511,7 @@ bool V4L2VideoCapture::CaptureProcess() { .build(); OnCapturedFrame(video_frame); } - + // enqueue the buffer again if (ioctl(_deviceFd, VIDIOC_QBUF, &buf) == -1) { RTC_LOG(LS_INFO) << "Failed to enqueue capture buffer"; diff --git a/src/v4l2_video_capturer/v4l2_video_capturer.h b/src/v4l2_video_capturer/v4l2_video_capturer.h index 87436026..eaabac3f 100644 --- a/src/v4l2_video_capturer/v4l2_video_capturer.h +++ b/src/v4l2_video_capturer/v4l2_video_capturer.h @@ -32,7 +32,8 @@ class V4L2VideoCapture : public ScalableVideoTrackSource { size_t capture_device_index); V4L2VideoCapture(); ~V4L2VideoCapture(); - int32_t Init(const char* deviceUniqueId, const std::string& specifiedVideoDevice); + int32_t Init(const char* deviceUniqueId, + const std::string& specifiedVideoDevice); int32_t StartCapture(ConnectionSettings cs); bool useNativeBuffer() override; @@ -70,4 +71,3 @@ class V4L2VideoCapture : public ScalableVideoTrackSource { }; #endif // V4L2_VIDEO_CAPTURE_H_ - diff --git a/src/watchdog.cpp b/src/watchdog.cpp index 586bd6d5..0fed47fb 100644 --- a/src/watchdog.cpp +++ b/src/watchdog.cpp @@ -1,15 +1,10 @@ #include "watchdog.h" #include - WatchDog::WatchDog(boost::asio::io_context& ioc, std::function callback) - : timer_(ioc) - , callback_(callback) - , timeout_(0) { -} + : timer_(ioc), callback_(callback), timeout_(0) {} -void WatchDog::enable(int timeout) -{ +void WatchDog::enable(int timeout) { timeout_ = timeout; timer_.cancel(); timer_.expires_from_now(boost::posix_time::seconds(timeout)); @@ -22,12 +17,10 @@ void WatchDog::enable(int timeout) }); } -void WatchDog::disable() -{ +void WatchDog::disable() { timer_.cancel(); } -void WatchDog::reset() -{ +void WatchDog::reset() { enable(timeout_); } diff --git a/src/watchdog.h b/src/watchdog.h index 292680ae..fa30bd4c 100644 --- a/src/watchdog.h +++ b/src/watchdog.h @@ -11,16 +11,14 @@ enable() を呼び出した後、一定時間が経過するとコールバッ コールバックが発生する時 WatchDog は無効になるので、必要であれば再度 enable() や reset() を呼び出すこと。 マルチスレッド下では動作しないので注意。 */ -class WatchDog -{ -public: +class WatchDog { + public: WatchDog(boost::asio::io_context& ioc, std::function callback); void enable(int timeout); void disable(); void reset(); -private: - + private: int timeout_; boost::asio::deadline_timer timer_; std::function callback_; diff --git a/src/ws/websocket.cpp b/src/ws/websocket.cpp index 57d081b1..cd012626 100644 --- a/src/ws/websocket.cpp +++ b/src/ws/websocket.cpp @@ -1,184 +1,172 @@ #include "websocket.h" #include "util.h" -#include #include #include #include #include +#include Websocket::Websocket(boost::asio::io_context& ioc) - : ws_(new websocket_t(ioc)) - , strand_(ws_->get_executor()) { + : ws_(new websocket_t(ioc)), strand_(ws_->get_executor()) { ws_->write_buffer_bytes(8192); } -Websocket::Websocket(boost::asio::io_context& ioc, boost::asio::ssl::context ssl_ctx) - : wss_(new ssl_websocket_t(ioc, ssl_ctx)) - , strand_(wss_->get_executor()) { +Websocket::Websocket(boost::asio::io_context& ioc, + boost::asio::ssl::context ssl_ctx) + : wss_(new ssl_websocket_t(ioc, ssl_ctx)), strand_(wss_->get_executor()) { wss_->write_buffer_bytes(8192); } Websocket::Websocket(boost::asio::ip::tcp::socket socket) - : ws_(new websocket_t(std::move(socket))) - , strand_(ws_->get_executor()) { + : ws_(new websocket_t(std::move(socket))), strand_(ws_->get_executor()) { ws_->write_buffer_bytes(8192); } Websocket::~Websocket() { - RTC_LOG(LS_INFO) << __FUNCTION__; + RTC_LOG(LS_INFO) << __FUNCTION__; } -bool Websocket::isSSL() const { return wss_ != nullptr; } -Websocket::websocket_t& Websocket::nativeSocket() { return *ws_; } -Websocket::ssl_websocket_t& Websocket::nativeSecureSocket() { return *wss_; } +bool Websocket::isSSL() const { + return wss_ != nullptr; +} +Websocket::websocket_t& Websocket::nativeSocket() { + return *ws_; +} +Websocket::ssl_websocket_t& Websocket::nativeSecureSocket() { + return *wss_; +} -boost::asio::strand& Websocket::strand() { return strand_; } +boost::asio::strand& +Websocket::strand() { + return strand_; +} void Websocket::startToRead(read_callback_t on_read) { - boost::asio::post( - strand_, - std::bind( - &Websocket::doRead, - this, - on_read)); + boost::asio::post(strand_, std::bind(&Websocket::doRead, this, on_read)); } void Websocket::doRead(read_callback_t on_read) { - RTC_LOG(LS_INFO) << __FUNCTION__; - - if (isSSL()) { - wss_->async_read( - read_buffer_, - boost::asio::bind_executor( - strand_, - std::bind( - &Websocket::onRead, - this, - on_read, - std::placeholders::_1, - std::placeholders::_2))); - } else { - ws_->async_read( - read_buffer_, - boost::asio::bind_executor( - strand_, - std::bind( - &Websocket::onRead, - this, - on_read, - std::placeholders::_1, - std::placeholders::_2))); - } + RTC_LOG(LS_INFO) << __FUNCTION__; + + if (isSSL()) { + wss_->async_read( + read_buffer_, + boost::asio::bind_executor( + strand_, std::bind(&Websocket::onRead, this, on_read, + std::placeholders::_1, std::placeholders::_2))); + } else { + ws_->async_read( + read_buffer_, + boost::asio::bind_executor( + strand_, std::bind(&Websocket::onRead, this, on_read, + std::placeholders::_1, std::placeholders::_2))); + } } -void Websocket::onRead(read_callback_t on_read, boost::system::error_code ec, std::size_t bytes_transferred) { - RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec.message(); +void Websocket::onRead(read_callback_t on_read, + boost::system::error_code ec, + std::size_t bytes_transferred) { + RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec.message(); - // エラーだろうが何だろうが on_read コールバック関数は必ず呼ぶ + // エラーだろうが何だろうが on_read コールバック関数は必ず呼ぶ - const auto text = boost::beast::buffers_to_string(read_buffer_.data()); - read_buffer_.consume(read_buffer_.size()); + const auto text = boost::beast::buffers_to_string(read_buffer_.data()); + read_buffer_.consume(read_buffer_.size()); - on_read(ec, bytes_transferred, std::move(text)); + on_read(ec, bytes_transferred, std::move(text)); - if (ec == boost::asio::error::operation_aborted) - return; + if (ec == boost::asio::error::operation_aborted) + return; - if (ec) - return MOMO_BOOST_ERROR(ec, "onRead"); + if (ec) + return MOMO_BOOST_ERROR(ec, "onRead"); - doRead(on_read); + doRead(on_read); } void Websocket::sendText(std::string text) { - RTC_LOG(LS_INFO) << __FUNCTION__; - boost::asio::post( - strand_, - std::bind( - &Websocket::doSendText, - this, - std::move(text))); + RTC_LOG(LS_INFO) << __FUNCTION__; + boost::asio::post(strand_, + std::bind(&Websocket::doSendText, this, std::move(text))); } void Websocket::doSendText(std::string text) { - RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << text; + RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << text; - bool empty = write_buffer_.empty(); - boost::beast::flat_buffer buffer; + bool empty = write_buffer_.empty(); + boost::beast::flat_buffer buffer; - const auto n = boost::asio::buffer_copy(buffer.prepare(text.size()), boost::asio::buffer(text)); - RTC_LOG(LS_VERBOSE) << __FUNCTION__ << ": n=" << n; - buffer.commit(n); + const auto n = boost::asio::buffer_copy(buffer.prepare(text.size()), + boost::asio::buffer(text)); + RTC_LOG(LS_VERBOSE) << __FUNCTION__ << ": n=" << n; + buffer.commit(n); - { - std::string tmp = boost::beast::buffers_to_string(buffer.data()); - RTC_LOG(LS_VERBOSE) << __FUNCTION__ << ": size=" << tmp.size() << " text=" << tmp; - } + { + std::string tmp = boost::beast::buffers_to_string(buffer.data()); + RTC_LOG(LS_VERBOSE) << __FUNCTION__ << ": size=" << tmp.size() + << " text=" << tmp; + } - write_buffer_.push_back(std::move(buffer)); + write_buffer_.push_back(std::move(buffer)); - if (empty) { - doWrite(); - } + if (empty) { + doWrite(); + } } void Websocket::doWrite() { - RTC_LOG(LS_INFO) << __FUNCTION__; - - auto& buffer = write_buffer_.front(); - - RTC_LOG(LS_VERBOSE) << __FUNCTION__ << ": " << boost::beast::buffers_to_string(buffer.data()); - - { - std::string tmp = boost::beast::buffers_to_string(buffer.data()); - RTC_LOG(LS_VERBOSE) << __FUNCTION__ << ": size=" << tmp.size() << " text=" << tmp; - } - - if (isSSL()) { - wss_->text(true); - wss_->async_write( - buffer.data(), - boost::asio::bind_executor( - strand_, - std::bind( - &Websocket::onWrite, - this, - std::placeholders::_1, - std::placeholders::_2))); - } else { - ws_->text(true); - ws_->async_write( - buffer.data(), - boost::asio::bind_executor( - strand_, - std::bind( - &Websocket::onWrite, - this, - std::placeholders::_1, - std::placeholders::_2))); - } + RTC_LOG(LS_INFO) << __FUNCTION__; + + auto& buffer = write_buffer_.front(); + + RTC_LOG(LS_VERBOSE) << __FUNCTION__ << ": " + << boost::beast::buffers_to_string(buffer.data()); + + { + std::string tmp = boost::beast::buffers_to_string(buffer.data()); + RTC_LOG(LS_VERBOSE) << __FUNCTION__ << ": size=" << tmp.size() + << " text=" << tmp; + } + + if (isSSL()) { + wss_->text(true); + wss_->async_write( + buffer.data(), + boost::asio::bind_executor( + strand_, std::bind(&Websocket::onWrite, this, std::placeholders::_1, + std::placeholders::_2))); + } else { + ws_->text(true); + ws_->async_write( + buffer.data(), + boost::asio::bind_executor( + strand_, std::bind(&Websocket::onWrite, this, std::placeholders::_1, + std::placeholders::_2))); + } } -void Websocket::onWrite(boost::system::error_code ec, std::size_t bytes_transferred) -{ - RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec.message(); +void Websocket::onWrite(boost::system::error_code ec, + std::size_t bytes_transferred) { + RTC_LOG(LS_INFO) << __FUNCTION__ << ": " << ec.message(); - // エラーだろうが何だろうが on_write_ コールバック関数は必ず呼ぶ - // on_write(ec, bytes_transferred); + // エラーだろうが何だろうが on_write_ コールバック関数は必ず呼ぶ + // on_write(ec, bytes_transferred); - if (ec == boost::asio::error::operation_aborted) - return; + if (ec == boost::asio::error::operation_aborted) + return; - if (ec) - return MOMO_BOOST_ERROR(ec, "onWrite"); + if (ec) + return MOMO_BOOST_ERROR(ec, "onWrite"); - { - std::string tmp = boost::beast::buffers_to_string(write_buffer_.front().data()); - RTC_LOG(LS_VERBOSE) << __FUNCTION__ << ": bytes_transferred=" << bytes_transferred << " size=" << tmp.size() << " text=" << tmp; - } + { + std::string tmp = + boost::beast::buffers_to_string(write_buffer_.front().data()); + RTC_LOG(LS_VERBOSE) << __FUNCTION__ + << ": bytes_transferred=" << bytes_transferred + << " size=" << tmp.size() << " text=" << tmp; + } - write_buffer_.erase(write_buffer_.begin()); + write_buffer_.erase(write_buffer_.begin()); - if (!write_buffer_.empty()) - { - doWrite(); - } + if (!write_buffer_.empty()) { + doWrite(); + } } diff --git a/src/ws/websocket.h b/src/ws/websocket.h index e1af7066..44440f59 100644 --- a/src/ws/websocket.h +++ b/src/ws/websocket.h @@ -1,15 +1,15 @@ #ifndef WS_WEBSOCKET_H_ #define WS_WEBSOCKET_H_ -#include #include #include #include #include +#include #include #include -#include #include +#include #include #include @@ -18,53 +18,60 @@ // 任意のスレッドから sendText を呼ぶことで書き込みができる。 // // 接続の確立に関しては、nativeSocket() および nativeSecureSocket() 関数を使って自前で行うこと。 -class Websocket -{ -public: - typedef boost::beast::websocket::stream websocket_t; - typedef boost::beast::websocket::stream> ssl_websocket_t; - typedef std::function read_callback_t; +class Websocket { + public: + typedef boost::beast::websocket::stream + websocket_t; + typedef boost::beast::websocket::stream< + boost::asio::ssl::stream> + ssl_websocket_t; + typedef std::function + read_callback_t; -private: - std::unique_ptr ws_; - std::unique_ptr wss_; + private: + std::unique_ptr ws_; + std::unique_ptr wss_; - boost::asio::strand strand_; + boost::asio::strand strand_; - boost::beast::multi_buffer read_buffer_; - std::vector write_buffer_; + boost::beast::multi_buffer read_buffer_; + std::vector write_buffer_; -public: - // 非SSL - Websocket(boost::asio::io_context& ioc); - // SSL - Websocket(boost::asio::io_context& ioc, boost::asio::ssl::context ssl_ctx); - // 非SSL + ソケット直接 - Websocket(boost::asio::ip::tcp::socket socket); - ~Websocket(); + public: + // 非SSL + Websocket(boost::asio::io_context& ioc); + // SSL + Websocket(boost::asio::io_context& ioc, boost::asio::ssl::context ssl_ctx); + // 非SSL + ソケット直接 + Websocket(boost::asio::ip::tcp::socket socket); + ~Websocket(); - bool isSSL() const; + bool isSSL() const; - websocket_t& nativeSocket(); - ssl_websocket_t& nativeSecureSocket(); + websocket_t& nativeSocket(); + ssl_websocket_t& nativeSecureSocket(); - boost::asio::strand& strand(); + boost::asio::strand& strand(); -public: - // Websocket からの読み込みを開始する。 - void startToRead(read_callback_t on_read); + public: + // Websocket からの読み込みを開始する。 + void startToRead(read_callback_t on_read); -private: - void doRead(read_callback_t on_read); - void onRead(read_callback_t on_read, boost::system::error_code ec, std::size_t bytes_transferred); + private: + void doRead(read_callback_t on_read); + void onRead(read_callback_t on_read, + boost::system::error_code ec, + std::size_t bytes_transferred); -public: - void sendText(std::string text); + public: + void sendText(std::string text); -private: - void doSendText(std::string text); - void doWrite(); - void onWrite(boost::system::error_code ec, std::size_t bytes_transferred); + private: + void doSendText(std::string text); + void doWrite(); + void onWrite(boost::system::error_code ec, std::size_t bytes_transferred); }; -#endif // WS_WEBSOCKET_H_ +#endif // WS_WEBSOCKET_H_ From 3e4c19438ed71d02c4508efee5fc0e92115c3bf5 Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 18 Sep 2019 22:44:41 +0900 Subject: [PATCH 26/37] =?UTF-8?q?=E5=85=A8=E4=BD=93=E3=81=AB=20clang-forma?= =?UTF-8?q?t=20=E3=81=99=E3=82=8B=E3=82=B9=E3=82=AF=E3=83=AA=E3=83=97?= =?UTF-8?q?=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- format.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 format.sh diff --git a/format.sh b/format.sh new file mode 100755 index 00000000..71bf9ace --- /dev/null +++ b/format.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# src/ 以下の全てのファイルに clang-format を適用する + +set -e + +for file in `find src -type f`; do + echo applying $file + clang-format -i -style=file $file +done From 85732c4c4151c6d2e5959f75133b9e92571cedf0 Mon Sep 17 00:00:00 2001 From: melpon Date: Wed, 18 Sep 2019 23:46:56 +0900 Subject: [PATCH 27/37] =?UTF-8?q?Jetson=20Nano=20=E3=81=A7=E3=82=82?= =?UTF-8?q?=E3=83=A9=E3=82=BA=E3=83=91=E3=82=A4=E7=94=A8=E3=82=AA=E3=83=97?= =?UTF-8?q?=E3=82=B7=E3=83=A7=E3=83=B3=E3=81=8C=E4=BD=BF=E3=81=88=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AA=E3=81=AE=E3=81=A7=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --use-native, --force-i420, --video-device --- src/main.cpp | 2 +- src/util.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index e86f80c8..0c128b13 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -80,7 +80,7 @@ int main(int argc, char* argv[]) { rtc::scoped_refptr capturer = MacCapturer::Create(cs.getWidth(), cs.getHeight(), cs.framerate, 0); #else -#if USE_MMAL_ENCODER | USE_JETSON_ENCODER +#if USE_MMAL_ENCODER || USE_JETSON_ENCODER rtc::scoped_refptr capturer = V4L2VideoCapture::Create(cs); #else rtc::scoped_refptr capturer = diff --git a/src/util.cpp b/src/util.cpp index 77c2eca1..6300419a 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -58,7 +58,7 @@ void Util::parseArgs(int argc, local_nh.param("no_video", cs.no_video, cs.no_video); local_nh.param("no_audio", cs.no_audio, cs.no_audio); -#if USE_MMAL_ENCODER +#if USE_MMAL_ENCODER || USE_JETSON_ENCODER local_nh.param("force_i420", cs.force_i420, cs.force_i420); local_nh.param("use_native", cs.use_native, cs.use_native); #endif @@ -137,7 +137,7 @@ void Util::parseArgs(int argc, app.add_flag("--no-video", cs.no_video, "ビデオを表示しない"); app.add_flag("--no-audio", cs.no_audio, "オーディオを出さない"); -#if USE_MMAL_ENCODER +#if USE_MMAL_ENCODER || USE_JETSON_ENCODER app.add_flag("--force-i420", cs.force_i420, "強制的にI420にする"); app.add_flag("--use-native", cs.use_native, "MJPEGのデコードとビデオのリサイズをハードウェアで行う"); From 0df45aafdec3ba97322d1257ed2512bb023a3b8c Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Wed, 18 Sep 2019 23:54:46 +0900 Subject: [PATCH 28/37] =?UTF-8?q?=E3=83=8F=E3=83=BC=E3=83=89=E3=82=A6?= =?UTF-8?q?=E3=82=A7=E3=82=A2=E3=82=A8=E3=83=B3=E3=82=B3=E3=83=BC=E3=83=80?= =?UTF-8?q?=E3=82=92=E3=82=A2=E3=83=94=E3=83=BC=E3=83=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4d4da558..e1fdbbfb 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,13 @@ WebRTC Native Client Momo は libwebrtc を利用しブラウザなしで様々な環境で動作する WebRTC ネイティブクライアントです。 -Raspberry Pi 環境では Raspberry Pi の GPU に積まれている H.264 ハードウェアエンコーダー機能を利用することが可能です。 +### ハードウェアエンコーダへの対応 -また、macOS 環境でも macOS に積まれている [VideoToolbox](https://developer.apple.com/documentation/videotoolbox) に対応しており、こちらも H.264 ハードウェアエンコーダー機能を利用することが可能です。 +Raspberry Pi の GPU に積まれている H.264 ハードウェアエンコーダー機能を利用することが可能です。 + +macOS に積まれている [VideoToolbox](https://developer.apple.com/documentation/videotoolbox) に対応しており、こちらも H.264 ハードウェアエンコーダー機能を利用することが可能です。 + +Jetson Nano に搭載されている H.264 ハードウェアエンコーダー機能を利用することで 4K@30 での配信が可能です。 [ROS](http://www.ros.org/) ノードとしても利用可能です。 From f2f6fd79ef666bc57ee64c048fe0aa88416ec665 Mon Sep 17 00:00:00 2001 From: melpon Date: Thu, 19 Sep 2019 11:54:43 +0900 Subject: [PATCH 29/37] =?UTF-8?q?=E3=83=87=E3=83=90=E3=82=A4=E3=82=B9?= =?UTF-8?q?=E9=96=93=E3=81=AE=E3=82=AA=E3=83=97=E3=82=B7=E3=83=A7=E3=83=B3?= =?UTF-8?q?=E3=81=AE=E5=B7=AE=E7=95=B0=E3=82=92=E3=81=A7=E3=81=8D=E3=82=8B?= =?UTF-8?q?=E3=81=A0=E3=81=91=E7=84=A1=E3=81=8F=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ただし無効なフラグが指定されてるとエラーになる。 --- src/util.cpp | 53 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/src/util.cpp b/src/util.cpp index 6300419a..dfd1bcf3 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -58,9 +58,10 @@ void Util::parseArgs(int argc, local_nh.param("no_video", cs.no_video, cs.no_video); local_nh.param("no_audio", cs.no_audio, cs.no_audio); -#if USE_MMAL_ENCODER || USE_JETSON_ENCODER local_nh.param("force_i420", cs.force_i420, cs.force_i420); local_nh.param("use_native", cs.use_native, cs.use_native); +#if USE_MMAL_ENCODER || USE_JETSON_ENCODER + local_nh.param("video_device", cs.video_device, cs.video_device); #endif local_nh.param("video_codec", cs.video_codec, cs.video_codec); local_nh.param("audio_codec", cs.audio_codec, cs.audio_codec); @@ -135,12 +136,45 @@ void Util::parseArgs(int argc, bool version = false; + auto is_valid_force_i420 = CLI::Validator( + [](std::string input) -> std::string { +#if USE_MMAL_ENCODER || USE_JETSON_ENCODER + return std::string(); +#else + return "このデバイスは --force-i420 に対応していません。"; +#endif + }, + ""); + auto is_valid_use_native = CLI::Validator( + [](std::string input) -> std::string { +#if USE_MMAL_ENCODER || USE_JETSON_ENCODER + return std::string(); +#else + return "このデバイスは --use-native に対応していません。"; +#endif + }, + ""); + + auto is_valid_h264 = CLI::Validator( + [](std::string input) -> std::string { +#if MOMO_USE_H264 + return std::string(); +#else + return "このデバイスは --video-codec=H264 に対応していません。"; +#endif + }, + ""); + app.add_flag("--no-video", cs.no_video, "ビデオを表示しない"); app.add_flag("--no-audio", cs.no_audio, "オーディオを出さない"); -#if USE_MMAL_ENCODER || USE_JETSON_ENCODER - app.add_flag("--force-i420", cs.force_i420, "強制的にI420にする"); + app.add_flag("--force-i420", cs.force_i420, + "強制的にI420にする（対応デバイスのみ）") + ->check(is_valid_force_i420); app.add_flag("--use-native", cs.use_native, - "MJPEGのデコードとビデオのリサイズをハードウェアで行う"); + "MJPEGのデコードとビデオのリサイズをハードウェアで行う" + "（対応デバイスのみ）") + ->check(is_valid_use_native); +#if USE_MMAL_ENCODER || USE_JETSON_ENCODER app.add_option("--video-device", cs.video_device, "デバイスファイル名。省略時はどれかのビデオデバイスを自動検出") ->check(CLI::ExistingFile); @@ -199,13 +233,10 @@ void Util::parseArgs(int argc, sora_app->add_option("CHANNEL-ID", cs.sora_channel_id, "チャンネルID") ->required(); sora_app->add_flag("--auto", cs.sora_auto_connect, "自動接続する"); -#if MOMO_USE_H264 - sora_app->add_set("--video-codec", cs.video_codec, {"VP8", "VP9", "H264"}, - "ビデオコーデック"); -#else - sora_app->add_set("--video-codec", cs.video_codec, {"VP8", "VP9"}, - "ビデオコーデック"); -#endif + sora_app + ->add_set("--video-codec", cs.video_codec, {"VP8", "VP9", "H264"}, + "ビデオコーデック") + ->check(is_valid_h264); sora_app->add_set("--audio-codec", cs.audio_codec, {"OPUS", "PCMU"}, "オーディオコーデック"); sora_app From 9dc9b1cf0824af8620a187e8bcc5629e382372e7 Mon Sep 17 00:00:00 2001 From: voluntas Date: Thu, 19 Sep 2019 12:22:01 +0900 Subject: [PATCH 30/37] =?UTF-8?q?name=20=E3=82=92=E3=83=8F=E3=82=A4?= =?UTF-8?q?=E3=83=95=E3=83=B3=E3=81=A7=E3=81=A4=E3=81=AA=E3=81=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 16b35819..e05711df 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build workflow +name: build-workflow on: push jobs: @@ -17,4 +17,4 @@ jobs: - uses: actions/checkout@v1 - run: DOCKER_BUILDKIT=1 NOTTY=1 NOMOUNT=1 make raspbian-buster_armv7 working-directory: build - timeout-minutes: 120 \ No newline at end of file + timeout-minutes: 120 From 7e09161960ec052454896e695eb8f134cc2b1377 Mon Sep 17 00:00:00 2001 From: voluntas Date: Thu, 19 Sep 2019 12:22:29 +0900 Subject: [PATCH 31/37] =?UTF-8?q?name=20=E3=82=92=E3=83=8F=E3=82=A4?= =?UTF-8?q?=E3=83=95=E3=83=B3=E3=81=A7=E3=81=A4=E3=81=AA=E3=81=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/daily_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/daily_build.yml b/.github/workflows/daily_build.yml index 1087f293..3ee8eb40 100644 --- a/.github/workflows/daily_build.yml +++ b/.github/workflows/daily_build.yml @@ -1,4 +1,4 @@ -name: Daily build workflow +name: daily-build-workflow on: schedule: # UTCで記述する事、この場合は日本時間 9 時にしたいので -9 して 0 にしてある From 65c066f7a745a602fdb60c27da74d43887c3e7f3 Mon Sep 17 00:00:00 2001 From: voluntas Date: Thu, 19 Sep 2019 12:23:06 +0900 Subject: [PATCH 32/37] =?UTF-8?q?GitHub=20Actions=20Badge=20=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e1fdbbfb..13899356 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![GitHub tag (latest SemVer)](https://img.shields.io/github/tag/shiguredo/momo.svg)](https://github.com/shiguredo/momo) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Actions Status](https://github.com/shiguredo/momo/workflows/build-workflow/badge.svg)](https://github.com/shiguredo/momo/actions) ## WebRTC Native Client Momo について From d43392d953bed765de42958f442df329c3a3e356 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Thu, 19 Sep 2019 12:28:20 +0900 Subject: [PATCH 33/37] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=82=92=E6=9B=B4=E6=96=B0=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index a4da8f8c..a4bc7cf6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,8 @@ - [UPDATE] libwebrtc M78 コミットポジションを 3 にする - libwebrtc のハッシュは 68c715dc01cd8cd0ad2726453e7376b5f353fcd1 - @voluntas +- [UPDATE] コマンドオプションをできるだけ共通化する + - @melpon ## 19.09.0 From c58f3ddfc40d6b07d042f0c9675da46dbf6fe5f2 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Thu, 19 Sep 2019 12:31:03 +0900 Subject: [PATCH 34/37] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=82=92=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index a4bc7cf6..2abc0076 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,11 +18,15 @@ - [ADD] CI を CircleCI から GitHub Actions へ切り替える - macOS の時間制限が OSS の場合はないため Weekly build から Daily build のみにきりかえる - @hakobera +- [ADD] clang-format の追加 + - @melpon - [UPDATE] libwebrtc M78 コミットポジションを 3 にする - libwebrtc のハッシュは 68c715dc01cd8cd0ad2726453e7376b5f353fcd1 - @voluntas - [UPDATE] コマンドオプションをできるだけ共通化する - @melpon +- [UPDATE] Raspberry Pi のビルド OS を Ubuntu 16.04 から 18.04 に上げる + - @melpon ## 19.09.0 From fb0fa901a01f89345555f84d9e3b8195389ca22c Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Thu, 19 Sep 2019 12:31:37 +0900 Subject: [PATCH 35/37] =?UTF-8?q?.clang-format=20=E3=81=AB=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 2abc0076..317f973c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,7 +18,7 @@ - [ADD] CI を CircleCI から GitHub Actions へ切り替える - macOS の時間制限が OSS の場合はないため Weekly build から Daily build のみにきりかえる - @hakobera -- [ADD] clang-format の追加 +- [ADD] .clang-format の追加 - @melpon - [UPDATE] libwebrtc M78 コミットポジションを 3 にする - libwebrtc のハッシュは 68c715dc01cd8cd0ad2726453e7376b5f353fcd1 From 303f99698fd09b62d30f4e01245ee19dcecc6e7b Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sat, 21 Sep 2019 21:03:52 +0900 Subject: [PATCH 36/37] =?UTF-8?q?=E3=83=90=E3=83=BC=E3=82=B8=E3=83=A7?= =?UTF-8?q?=E3=83=B3=E3=82=92=2019.09.1=20=E3=81=AB=E4=B8=8A=E3=81=92?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 15908038..4cc44305 100644 --- a/VERSION +++ b/VERSION @@ -1,6 +1,6 @@ # 各種ライブラリのバージョン情報 # 項目を変更してビルドすれば各環境へ反映される（ように作る） -MOMO_VERSION=19.09.1-rc1 +MOMO_VERSION=19.09.1 WEBRTC_VERSION=78 WEBRTC_COMMIT=68c715dc01cd8cd0ad2726453e7376b5f353fcd1 BOOST_VERSION=1.71.0 From 372f5abfc5e74287ef17a4d43bb009ffef4d3a95 Mon Sep 17 00:00:00 2001 From: NAKAI Ryosuke Date: Sun, 22 Sep 2019 20:37:10 +0900 Subject: [PATCH 37/37] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=82=92=E6=9B=B4=E6=96=B0=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 317f973c..b68c8752 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,8 @@ ## develop +## 19.09.1 + - [ADD] Jetson Nano のハードウェアエンコーダを利用する機能を実装 - @tnoho - [ADD] Jetson Nano のビルドを追加