From 2f0b8dad01e1295b3ab57b83e0455879f06ea5f9 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Mon, 2 Dec 2019 17:38:36 +0100 Subject: [PATCH 1/8] Refactoring of core-plugin callbacks and RTP extensions termination --- ice.c | 190 ++++++++++++++++++++++++++++-------- ice.h | 29 +++--- janus.c | 99 +++++++++++++++---- plugins/janus_audiobridge.c | 24 +++-- plugins/janus_duktape.c | 77 ++++++++------- plugins/janus_echotest.c | 77 +++++++-------- plugins/janus_lua.c | 76 ++++++++------- plugins/janus_nosip.c | 25 +++-- plugins/janus_recordplay.c | 44 +++++---- plugins/janus_sip.c | 32 +++--- plugins/janus_sipre.c | 26 +++-- plugins/janus_streaming.c | 36 ++++--- plugins/janus_textroom.c | 47 +++++---- plugins/janus_videocall.c | 80 +++++++-------- plugins/janus_videoroom.c | 132 ++++++++++++------------- plugins/janus_voicemail.c | 10 +- plugins/plugin.c | 28 +++++- plugins/plugin.h | 180 +++++++++++++++++++++++++++++----- rtp.c | 6 +- rtp.h | 5 +- sdp.c | 16 +++ 21 files changed, 810 insertions(+), 429 deletions(-) diff --git a/ice.c b/ice.c index 60ab4b6b6c..2dfb572813 100644 --- a/ice.c +++ b/ice.c @@ -562,7 +562,7 @@ static int janus_seq_in_range(guint16 seqn, guint16 start, guint16 len) { /* Internal method for relaying RTCP messages, optionally filtering them in case they come from plugins */ -void janus_ice_relay_rtcp_internal(janus_ice_handle *handle, int video, char *buf, int len, gboolean filter_rtcp); +void janus_ice_relay_rtcp_internal(janus_ice_handle *handle, janus_plugin_rtcp *packet, gboolean filter_rtcp); /* Map of active plugin sessions */ @@ -2462,12 +2462,32 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp stream->video_is_keyframe = &janus_h264_is_keyframe; } } - /* Pass the data to the responsible plugin */ + /* Prepare the data to pass to the responsible plugin */ + janus_plugin_rtp rtp = { .video = video, .buffer = buf, .length = buflen }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + /* Parse RTP extensions before involving the plugin */ + if(stream->audiolevel_ext_id != -1) { + gboolean vad = FALSE; + int level = -1; + if(janus_rtp_header_extension_parse_audio_level(buf, buflen, + stream->audiolevel_ext_id, &vad, &level) == 0) { + rtp.extensions.audio_level = level; + rtp.extensions.audio_level_vad = vad; + } + } + if(stream->videoorientation_ext_id != -1) { + gboolean c = FALSE, f = FALSE, r1 = FALSE, r0 = FALSE; + if(janus_rtp_header_extension_parse_video_orientation(buf, buflen, + stream->videoorientation_ext_id, &c, &f, &r1, &r0) == 0) { + rtp.extensions.video_orientation = (c<<3) + (f<<2) + (r1<<1) + r0; + } + } + /* Pass the packet to the plugin */ janus_plugin *plugin = (janus_plugin *)handle->app; if(plugin && plugin->incoming_rtp && handle->app_handle && !g_atomic_int_get(&handle->app_handle->stopped) && !g_atomic_int_get(&handle->destroyed)) - plugin->incoming_rtp(handle->app_handle, video, buf, buflen); + plugin->incoming_rtp(handle->app_handle, &rtp); /* Restore the header for the stats (plugins may have messed with it) */ *header = backup; /* Update stats (overall data received, and data received in the last second) */ @@ -2637,7 +2657,8 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp janus_rtcp_fix_ssrc(NULL, nackbuf, res, 1, video ? stream->video_ssrc : stream->audio_ssrc, video ? stream->video_ssrc_peer[vindex] : stream->audio_ssrc_peer); - janus_ice_relay_rtcp_internal(handle, video, nackbuf, res, FALSE); + janus_plugin_rtcp rtcp = { .video = video, .buffer = nackbuf, .length = res }; + janus_ice_relay_rtcp_internal(handle, &rtcp, FALSE); } /* Update stats */ component->nack_sent_recent_cnt += nacks_count; @@ -2853,11 +2874,12 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp return; } + janus_plugin_rtcp rtcp = { .video = video, .buffer = buf, .length = buflen }; janus_plugin *plugin = (janus_plugin *)handle->app; if(plugin && plugin->incoming_rtcp && handle->app_handle && !g_atomic_int_get(&handle->app_handle->stopped) && !g_atomic_int_get(&handle->destroyed)) - plugin->incoming_rtcp(handle->app_handle, video, buf, buflen); + plugin->incoming_rtcp(handle->app_handle, &rtcp); } } return; @@ -2876,11 +2898,12 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp void janus_ice_incoming_data(janus_ice_handle *handle, char *label, char *buffer, int length) { if(handle == NULL || buffer == NULL || length <= 0) return; + janus_plugin_data data = { .label = label, .buffer = buffer, .length = length }; janus_plugin *plugin = (janus_plugin *)handle->app; if(plugin && plugin->incoming_data && handle->app_handle && !g_atomic_int_get(&handle->app_handle->stopped) && !g_atomic_int_get(&handle->destroyed)) - plugin->incoming_data(handle->app_handle, label, buffer, length); + plugin->incoming_data(handle->app_handle, &data); } @@ -3547,7 +3570,8 @@ static gboolean janus_ice_outgoing_transport_wide_cc_feedback(gpointer user_data int len = janus_rtcp_transport_wide_cc_feedback(rtcpbuf, size, stream->video_ssrc, stream->video_ssrc_peer[0], feedback_packet_count, packets_to_process); /* Enqueue it, we'll send it later */ - janus_ice_relay_rtcp_internal(handle, 1, rtcpbuf, len, FALSE); + janus_plugin_rtcp rtcp = { .video = TRUE, .buffer = rtcpbuf, .length = len }; + janus_ice_relay_rtcp_internal(handle, &rtcp, FALSE); if(packets_to_process != packets) { g_queue_free(packets_to_process); } @@ -3596,7 +3620,8 @@ static gboolean janus_ice_outgoing_rtcp_handle(gpointer user_data) { janus_rtcp_sdes_cname((char *)sdes, sdeslen, "janus", 5); sdes->chunk.ssrc = htonl(stream->audio_ssrc); /* Enqueue it, we'll send it later */ - janus_ice_relay_rtcp_internal(handle, 0, rtcpbuf, srlen+sdeslen, FALSE); + janus_plugin_rtcp rtcp = { .video = FALSE, .buffer = rtcpbuf, .length = srlen+sdeslen }; + janus_ice_relay_rtcp_internal(handle, &rtcp, FALSE); /* Check if we detected too many losses, and send a slowlink event in case */ guint lost = janus_rtcp_context_get_lost_all(rtcp_ctx, TRUE); janus_slow_link_update(stream->component, handle, FALSE, TRUE, lost); @@ -3615,7 +3640,8 @@ static gboolean janus_ice_outgoing_rtcp_handle(gpointer user_data) { janus_rtcp_report_block(stream->audio_rtcp_ctx, &rr->rb[0]); rr->rb[0].ssrc = htonl(stream->audio_ssrc_peer); /* Enqueue it, we'll send it later */ - janus_ice_relay_rtcp_internal(handle, 0, rtcpbuf, 32, FALSE); + janus_plugin_rtcp rtcp = { .video = FALSE, .buffer = rtcpbuf, .length = 32 }; + janus_ice_relay_rtcp_internal(handle, &rtcp, FALSE); /* Check if we detected too many losses, and send a slowlink event in case */ guint lost = janus_rtcp_context_get_lost_all(stream->audio_rtcp_ctx, FALSE); janus_slow_link_update(stream->component, handle, FALSE, FALSE, lost); @@ -3655,7 +3681,8 @@ static gboolean janus_ice_outgoing_rtcp_handle(gpointer user_data) { janus_rtcp_sdes_cname((char *)sdes, sdeslen, "janus", 5); sdes->chunk.ssrc = htonl(stream->video_ssrc); /* Enqueue it, we'll send it later */ - janus_ice_relay_rtcp_internal(handle, 1, rtcpbuf, srlen+sdeslen, FALSE); + janus_plugin_rtcp rtcp = { .video = TRUE, .buffer = rtcpbuf, .length = srlen+sdeslen }; + janus_ice_relay_rtcp_internal(handle, &rtcp, FALSE); /* Check if we detected too many losses, and send a slowlink event in case */ guint lost = janus_rtcp_context_get_lost_all(rtcp_ctx, TRUE); janus_slow_link_update(stream->component, handle, TRUE, TRUE, lost); @@ -3678,7 +3705,8 @@ static gboolean janus_ice_outgoing_rtcp_handle(gpointer user_data) { janus_rtcp_report_block(stream->video_rtcp_ctx[vindex], &rr->rb[0]); rr->rb[0].ssrc = htonl(stream->video_ssrc_peer[vindex]); /* Enqueue it, we'll send it later */ - janus_ice_relay_rtcp_internal(handle, 1, rtcpbuf, 32, FALSE); + janus_plugin_rtcp rtcp = { .video = TRUE, .buffer = rtcpbuf, .length = 32 }; + janus_ice_relay_rtcp_internal(handle, &rtcp, FALSE); } } /* Check if we detected too many losses, and send a slowlink event in case */ @@ -4308,18 +4336,80 @@ static void janus_ice_queue_packet(janus_ice_handle *handle, janus_ice_queued_pa } } -void janus_ice_relay_rtp(janus_ice_handle *handle, int video, char *buf, int len) { - if(!handle || handle->queued_packets == NULL || buf == NULL || len < 1) +void janus_ice_relay_rtp(janus_ice_handle *handle, janus_plugin_rtp *packet) { + if(!handle || handle->queued_packets == NULL || packet == NULL || packet->buffer == NULL || + !janus_is_rtp(packet->buffer, packet->length)) return; - if((!video && !janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_HAS_AUDIO)) - || (video && !janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_HAS_VIDEO))) + if((!packet->video && !janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_HAS_AUDIO)) + || (packet->video && !janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_HAS_VIDEO))) return; + /* We need to strip extensions, here, and add those that need to be there manually */ + uint16_t totlen = RTP_HEADER_SIZE, extlen = 0; + char extensions[50]; + janus_rtp_header *header = (janus_rtp_header *)packet->buffer; + header->extension = 0; + /* Add core and plugin extensions, if any */ + if(handle->stream->mid_ext_id > 0) { + header->extension = 1; + memset(extensions, sizeof(extensions), extlen); + janus_rtp_header_extension *extheader = (janus_rtp_header_extension *)extensions; + extheader->type = htons(0xBEDE); + extheader->length = 0; + /* Iterate on all extensions we need */ + char *index = extensions + 4; + /* Check if we need to add the mid extension */ + if(handle->stream->mid_ext_id > 0) { + char *mid = packet->video ? handle->video_mid : handle->audio_mid; + if(mid != NULL) { + size_t midlen = strlen(mid) & 0x0F; + *index = (handle->stream->mid_ext_id << 4) + (midlen ? midlen-1 : 0); + memcpy(index+1, mid, midlen); + index += (midlen + 1); + extlen += (midlen + 1); + } + } + /* Check if the plugin (or source) included other extensions */ + if(packet->extensions.audio_level != -1 && handle->stream->audiolevel_ext_id > 0) { + /* Add audio-level extension */ + *index = (handle->stream->audiolevel_ext_id << 4); + *(index+1) = (packet->extensions.audio_level_vad << 7) + (packet->extensions.audio_level & 0x7F); + index += 2; + extlen += 2; + } + if(packet->extensions.video_orientation != -1 && handle->stream->videoorientation_ext_id > 0) { + /* Add video-orientation extension */ + *index = (handle->stream->videoorientation_ext_id << 4); + *(index+1) = packet->extensions.video_orientation; + index += 2; + extlen += 2; + } + /* Calculate the whole length */ + uint16_t words = extlen/4; + if(extlen%4 != 0) + words++; + extheader->length = htons(words); + /* Update lengths (taking into account the RFC5285 header) */ + extlen = 4 + (words*4); + totlen += extlen; + } + /* Check how large the payload is */ + int plen = 0; + char *payload = janus_rtp_payload(packet->buffer, packet->length, &plen); + if(payload != NULL) + totlen += plen; /* Queue this packet */ janus_ice_queued_packet *pkt = g_malloc(sizeof(janus_ice_queued_packet)); - pkt->data = g_malloc(len+SRTP_MAX_TAG_LEN); - memcpy(pkt->data, buf, len); - pkt->length = len; - pkt->type = video ? JANUS_ICE_PACKET_VIDEO : JANUS_ICE_PACKET_AUDIO; + pkt->data = g_malloc(totlen + SRTP_MAX_TAG_LEN); + /* RTP header first */ + memcpy(pkt->data, packet->buffer, RTP_HEADER_SIZE); + /* Then RTP extensions, if any */ + if(extlen > 0) + memcpy(pkt->data + RTP_HEADER_SIZE, extensions, extlen); + /* Finally the RTP payload, if available */ + if(payload != NULL && plen > 0) + memcpy(pkt->data + RTP_HEADER_SIZE + extlen, payload, plen); + pkt->length = totlen; + pkt->type = packet->video ? JANUS_ICE_PACKET_VIDEO : JANUS_ICE_PACKET_AUDIO; pkt->control = FALSE; pkt->encrypted = FALSE; pkt->retransmission = FALSE; @@ -4328,54 +4418,55 @@ void janus_ice_relay_rtp(janus_ice_handle *handle, int video, char *buf, int len janus_ice_queue_packet(handle, pkt); } -void janus_ice_relay_rtcp_internal(janus_ice_handle *handle, int video, char *buf, int len, gboolean filter_rtcp) { - if(!handle || handle->queued_packets == NULL || buf == NULL || len < 1) +void janus_ice_relay_rtcp_internal(janus_ice_handle *handle, janus_plugin_rtcp *packet, gboolean filter_rtcp) { + if(!handle || handle->queued_packets == NULL || packet == NULL || packet->buffer == NULL || + !janus_is_rtcp(packet->buffer, packet->length)) return; /* We use this internal method to check whether we need to filter RTCP (e.g., to make * sure we don't just forward any SR/RR from peers/plugins, but use our own) or it has * already been done, and so this is actually a packet added by the ICE send thread */ - char *rtcp_buf = buf; - int rtcp_len = len; + char *rtcp_buf = packet->buffer; + int rtcp_len = packet->length; if(filter_rtcp) { /* FIXME Strip RR/SR/SDES/NACKs/etc. */ janus_ice_stream *stream = handle->stream; if(stream == NULL) return; - rtcp_buf = janus_rtcp_filter(buf, len, &rtcp_len); + rtcp_buf = janus_rtcp_filter(packet->buffer, packet->length, &rtcp_len); if(rtcp_buf == NULL || rtcp_len < 1) return; /* Fix all SSRCs before enqueueing, as we need to use the ones for this media * leg. Note that this is only needed for RTCP packets coming from plugins: the * ones created by the core already have the right SSRCs in the right place */ JANUS_LOG(LOG_HUGE, "[%"SCNu64"] Fixing SSRCs (local %u, peer %u)\n", handle->handle_id, - video ? stream->video_ssrc : stream->audio_ssrc, - video ? stream->video_ssrc_peer[0] : stream->audio_ssrc_peer); + packet->video ? stream->video_ssrc : stream->audio_ssrc, + packet->video ? stream->video_ssrc_peer[0] : stream->audio_ssrc_peer); janus_rtcp_fix_ssrc(NULL, rtcp_buf, rtcp_len, 1, - video ? stream->video_ssrc : stream->audio_ssrc, - video ? stream->video_ssrc_peer[0] : stream->audio_ssrc_peer); + packet->video ? stream->video_ssrc : stream->audio_ssrc, + packet->video ? stream->video_ssrc_peer[0] : stream->audio_ssrc_peer); } /* Queue this packet */ janus_ice_queued_packet *pkt = g_malloc(sizeof(janus_ice_queued_packet)); pkt->data = g_malloc(rtcp_len+SRTP_MAX_TAG_LEN+4); memcpy(pkt->data, rtcp_buf, rtcp_len); pkt->length = rtcp_len; - pkt->type = video ? JANUS_ICE_PACKET_VIDEO : JANUS_ICE_PACKET_AUDIO; + pkt->type = packet->video ? JANUS_ICE_PACKET_VIDEO : JANUS_ICE_PACKET_AUDIO; pkt->control = TRUE; pkt->encrypted = FALSE; pkt->retransmission = FALSE; pkt->label = NULL; pkt->added = janus_get_monotonic_time(); janus_ice_queue_packet(handle, pkt); - if(rtcp_buf != buf) { + if(rtcp_buf != packet->buffer) { /* We filtered the original packet, deallocate it */ g_free(rtcp_buf); } } -void janus_ice_relay_rtcp(janus_ice_handle *handle, int video, char *buf, int len) { - janus_ice_relay_rtcp_internal(handle, video, buf, len, TRUE); +void janus_ice_relay_rtcp(janus_ice_handle *handle, janus_plugin_rtcp *packet) { + janus_ice_relay_rtcp_internal(handle, packet, TRUE); /* If this is a PLI and we're simulcasting, send a PLI on other layers as well */ - if(janus_rtcp_has_pli(buf, len)) { + if(janus_rtcp_has_pli(packet->buffer, packet->length)) { janus_ice_stream *stream = handle->stream; if(stream == NULL) return; @@ -4385,7 +4476,8 @@ void janus_ice_relay_rtcp(janus_ice_handle *handle, int video, char *buf, int le janus_rtcp_pli((char *)&plibuf, 12); janus_rtcp_fix_ssrc(NULL, plibuf, sizeof(plibuf), 1, stream->video_ssrc, stream->video_ssrc_peer[1]); - janus_ice_relay_rtcp_internal(handle, 1, plibuf, sizeof(plibuf), FALSE); + janus_plugin_rtcp rtcp = { .video = TRUE, .buffer = plibuf, .length = sizeof(plibuf) }; + janus_ice_relay_rtcp_internal(handle, &rtcp, FALSE); } if(stream->video_ssrc_peer[2]) { char plibuf[12]; @@ -4393,25 +4485,41 @@ void janus_ice_relay_rtcp(janus_ice_handle *handle, int video, char *buf, int le janus_rtcp_pli((char *)&plibuf, 12); janus_rtcp_fix_ssrc(NULL, plibuf, sizeof(plibuf), 1, stream->video_ssrc, stream->video_ssrc_peer[2]); - janus_ice_relay_rtcp_internal(handle, 1, plibuf, sizeof(plibuf), FALSE); + janus_plugin_rtcp rtcp = { .video = TRUE, .buffer = plibuf, .length = sizeof(plibuf) }; + janus_ice_relay_rtcp_internal(handle, &rtcp, FALSE); } } } +void janus_ice_send_pli(janus_ice_handle *handle) { + char rtcpbuf[12]; + memset(rtcpbuf, 0, 12); + janus_rtcp_pli((char *)&rtcpbuf, 12); + janus_plugin_rtcp rtcp = { .video = TRUE, .buffer = rtcpbuf, .length = 12 }; + janus_ice_relay_rtcp(handle, &rtcp); +} + +void janus_ice_send_remb(janus_ice_handle *handle, uint32_t bitrate) { + char rtcpbuf[24]; + janus_rtcp_remb((char *)&rtcpbuf, 24, bitrate); + janus_plugin_rtcp rtcp = { .video = TRUE, .buffer = rtcpbuf, .length = 24 }; + janus_ice_relay_rtcp(handle, &rtcp); +} + #ifdef HAVE_SCTP -void janus_ice_relay_data(janus_ice_handle *handle, char *label, char *buf, int len) { - if(!handle || handle->queued_packets == NULL || buf == NULL || len < 1) +void janus_ice_relay_data(janus_ice_handle *handle, janus_plugin_data *packet) { + if(!handle || handle->queued_packets == NULL || packet == NULL || packet->buffer == NULL || packet->length < 1) return; /* Queue this packet */ janus_ice_queued_packet *pkt = g_malloc(sizeof(janus_ice_queued_packet)); - pkt->data = g_malloc(len); - memcpy(pkt->data, buf, len); - pkt->length = len; + pkt->data = g_malloc(packet->length); + memcpy(pkt->data, packet->buffer, packet->length); + pkt->length = packet->length; pkt->type = JANUS_ICE_PACKET_DATA; pkt->control = FALSE; pkt->encrypted = FALSE; pkt->retransmission = FALSE; - pkt->label = label ? g_strdup(label) : NULL; + pkt->label = packet->label ? g_strdup(packet->label) : NULL; pkt->added = janus_get_monotonic_time(); janus_ice_queue_packet(handle, pkt); } diff --git a/ice.h b/ice.h index 6b2cd7fe7a..e507f1fcb5 100644 --- a/ice.h +++ b/ice.h @@ -404,6 +404,10 @@ struct janus_ice_stream { gint mid_ext_id; /*! \brief RTP Stream extension ID, and the related rtx one */ gint rid_ext_id, ridrtx_ext_id; + /*! \brief Audio levels extension ID */ + gint audiolevel_ext_id; + /*! \brief Video orientation extension ID */ + gint videoorientation_ext_id; /*! \brief Frame marking extension ID */ gint framemarking_ext_id; /*! \brief Whether we do transport wide cc for video */ @@ -581,22 +585,23 @@ void janus_ice_component_destroy(janus_ice_component *component); ///@{ /*! \brief Core RTP callback, called when a plugin has an RTP packet to send to a peer * @param[in] handle The Janus ICE handle associated with the peer - * @param[in] video Whether this is an audio or a video frame - * @param[in] buf The packet data (buffer) - * @param[in] len The buffer lenght */ -void janus_ice_relay_rtp(janus_ice_handle *handle, int video, char *buf, int len); + * @param[in] packet The RTP packet to send */ +void janus_ice_relay_rtp(janus_ice_handle *handle, janus_plugin_rtp *packet); /*! \brief Core RTCP callback, called when a plugin has an RTCP message to send to a peer * @param[in] handle The Janus ICE handle associated with the peer - * @param[in] video Whether this is related to an audio or a video stream - * @param[in] buf The message data (buffer) - * @param[in] len The buffer lenght */ -void janus_ice_relay_rtcp(janus_ice_handle *handle, int video, char *buf, int len); + * @param[in] packet The RTCP message to send */ +void janus_ice_relay_rtcp(janus_ice_handle *handle, janus_plugin_rtcp *packet); /*! \brief Core SCTP/DataChannel callback, called when a plugin has data to send to a peer * @param[in] handle The Janus ICE handle associated with the peer - * @param[in] label The label of the data channel to use - * @param[in] buf The message data (buffer) - * @param[in] len The buffer lenght */ -void janus_ice_relay_data(janus_ice_handle *handle, char *label, char *buf, int len); + * @param[in] packet The message to send */ +void janus_ice_relay_data(janus_ice_handle *handle, janus_plugin_data *packet); +/*! \brief Helper core callback, called when a plugin wants to send a RTCP PLI to a peer + * @param[in] handle The Janus ICE handle associated with the peer */ +void janus_ice_send_pli(janus_ice_handle *handle); +/*! \brief Helper core callback, called when a plugin wants to send a RTCP REMB to a peer + * @param[in] handle The Janus ICE handle associated with the peer + * @param[in] bitrate The bitrate value to put in the REMB message */ +void janus_ice_send_remb(janus_ice_handle *handle, uint32_t bitrate); /*! \brief Plugin SCTP/DataChannel callback, called by the SCTP stack when when there's data for a plugin * @param[in] handle The Janus ICE handle associated with the peer * @param[in] label The label of the data channel the message is from diff --git a/janus.c b/janus.c index 51ded0b37d..2b7f26a972 100644 --- a/janus.c +++ b/janus.c @@ -520,9 +520,11 @@ void janus_transport_task(gpointer data, gpointer user_data); ///@{ int janus_plugin_push_event(janus_plugin_session *plugin_session, janus_plugin *plugin, const char *transaction, json_t *message, json_t *jsep); json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plugin *plugin, const char *sdp_type, const char *sdp, gboolean restart); -void janus_plugin_relay_rtp(janus_plugin_session *plugin_session, int video, char *buf, int len); -void janus_plugin_relay_rtcp(janus_plugin_session *plugin_session, int video, char *buf, int len); -void janus_plugin_relay_data(janus_plugin_session *plugin_session, char *label, char *buf, int len); +void janus_plugin_relay_rtp(janus_plugin_session *plugin_session, janus_plugin_rtp *packet); +void janus_plugin_relay_rtcp(janus_plugin_session *plugin_session, janus_plugin_rtcp *packet); +void janus_plugin_relay_data(janus_plugin_session *plugin_session, janus_plugin_data *message); +void janus_plugin_send_pli(janus_plugin_session *plugin_session); +void janus_plugin_send_remb(janus_plugin_session *plugin_session, uint32_t bitrate); void janus_plugin_close_pc(janus_plugin_session *plugin_session); void janus_plugin_end_session(janus_plugin_session *plugin_session); void janus_plugin_notify_event(janus_plugin *plugin, janus_plugin_session *plugin_session, json_t *event); @@ -534,6 +536,8 @@ static janus_callbacks janus_handler_plugin = .relay_rtp = janus_plugin_relay_rtp, .relay_rtcp = janus_plugin_relay_rtcp, .relay_data = janus_plugin_relay_data, + .send_pli = janus_plugin_send_pli, + .send_remb = janus_plugin_send_remb, .close_pc = janus_plugin_close_pc, .end_session = janus_plugin_end_session, .events_is_enabled = janus_events_is_enabled, @@ -1341,6 +1345,10 @@ int janus_process_incoming_request(janus_request *request) { /* Check if the RTP Stream ID extension is being negotiated */ handle->stream->rid_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_RID); handle->stream->ridrtx_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_REPAIRED_RID); + /* Check if the audio level ID extension is being negotiated */ + handle->stream->audiolevel_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_AUDIO_LEVEL); + /* Check if the video orientation ID extension is being negotiated */ + handle->stream->videoorientation_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION); /* Check if the frame marking ID extension is being negotiated */ handle->stream->framemarking_ext_id = janus_rtp_header_extension_get_id(jsep_sdp, JANUS_RTP_EXTMAP_FRAME_MARKING); /* Check if transport wide CC is supported */ @@ -2772,6 +2780,20 @@ json_t *janus_admin_stream_summary(janus_ice_stream *stream) { json_object_set_new(sc, "video-codec", json_string(stream->video_codec)); json_object_set_new(s, "codecs", sc); } + json_t *se = json_object(); + if(stream->mid_ext_id > 0) + json_object_set_new(se, JANUS_RTP_EXTMAP_MID, json_integer(stream->mid_ext_id)); + if(stream->rid_ext_id > 0) + json_object_set_new(se, JANUS_RTP_EXTMAP_RID, json_integer(stream->rid_ext_id)); + if(stream->ridrtx_ext_id > 0) + json_object_set_new(se, JANUS_RTP_EXTMAP_REPAIRED_RID, json_integer(stream->ridrtx_ext_id)); + if(stream->transport_wide_cc_ext_id > 0) + json_object_set_new(se, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC, json_integer(stream->transport_wide_cc_ext_id)); + if(stream->audiolevel_ext_id > 0) + json_object_set_new(se, JANUS_RTP_EXTMAP_AUDIO_LEVEL, json_integer(stream->audiolevel_ext_id)); + if(stream->videoorientation_ext_id > 0) + json_object_set_new(se, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION, json_integer(stream->videoorientation_ext_id)); + json_object_set_new(s, "extensions", se); json_t *bwe = json_object(); json_object_set_new(bwe, "twcc", stream->do_transport_wide_cc ? json_true() : json_false()); if(stream->transport_wide_cc_ext_id > 0) @@ -3356,25 +3378,39 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug } } } - /* Make sure we don't send the mid/rid/repaired-rid attributes when offering ourselves */ + /* Make sure we don't send the rid/repaired-rid attributes when offering ourselves */ + int mid_ext_id = 0, audiolevel_ext_id = 0, videoorientation_ext_id = 0; GList *temp = parsed_sdp->m_lines; while(temp) { janus_sdp_mline *m = (janus_sdp_mline *)temp->data; GList *tempA = m->attributes; while(tempA) { janus_sdp_attribute *a = (janus_sdp_attribute *)tempA->data; - if(a->name && a->value && (strstr(a->value, JANUS_RTP_EXTMAP_MID) || - strstr(a->value, JANUS_RTP_EXTMAP_RID) || - strstr(a->value, JANUS_RTP_EXTMAP_REPAIRED_RID))) { - m->attributes = g_list_remove(m->attributes, a); - tempA = m->attributes; - janus_sdp_attribute_destroy(a); - continue; + if(a->name && a->value) { + if(strstr(a->value, JANUS_RTP_EXTMAP_MID)) + mid_ext_id = atoi(a->value); + else if(strstr(a->value, JANUS_RTP_EXTMAP_AUDIO_LEVEL)) + audiolevel_ext_id = atoi(a->value); + else if(strstr(a->value, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION)) + videoorientation_ext_id = atoi(a->value); + else if(strstr(a->value, JANUS_RTP_EXTMAP_RID) || + strstr(a->value, JANUS_RTP_EXTMAP_REPAIRED_RID)) { + m->attributes = g_list_remove(m->attributes, a); + tempA = m->attributes; + janus_sdp_attribute_destroy(a); + continue; + } } tempA = tempA->next; } temp = temp->next; } + if(ice_handle->stream && ice_handle->stream->mid_ext_id != mid_ext_id) + ice_handle->stream->mid_ext_id = mid_ext_id; + if(ice_handle->stream && ice_handle->stream->audiolevel_ext_id != audiolevel_ext_id) + ice_handle->stream->audiolevel_ext_id = audiolevel_ext_id; + if(ice_handle->stream && ice_handle->stream->videoorientation_ext_id != videoorientation_ext_id) + ice_handle->stream->videoorientation_ext_id = videoorientation_ext_id; } else { /* Check if the answer does contain the mid/rid/repaired-rid attributes */ gboolean do_mid = FALSE, do_rid = FALSE, do_repaired_rid = FALSE; @@ -3532,40 +3568,63 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug return jsep; } -void janus_plugin_relay_rtp(janus_plugin_session *plugin_session, int video, char *buf, int len) { - if((plugin_session < (janus_plugin_session *)0x1000) || g_atomic_int_get(&plugin_session->stopped) || buf == NULL || len < 1) +void janus_plugin_relay_rtp(janus_plugin_session *plugin_session, janus_plugin_rtp *packet) { + if((plugin_session < (janus_plugin_session *)0x1000) || g_atomic_int_get(&plugin_session->stopped) || + packet == NULL || packet->buffer == NULL || packet->length < 1) return; janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) return; - janus_ice_relay_rtp(handle, video, buf, len); + janus_ice_relay_rtp(handle, packet); } -void janus_plugin_relay_rtcp(janus_plugin_session *plugin_session, int video, char *buf, int len) { - if((plugin_session < (janus_plugin_session *)0x1000) || g_atomic_int_get(&plugin_session->stopped) || buf == NULL || len < 1) +void janus_plugin_relay_rtcp(janus_plugin_session *plugin_session, janus_plugin_rtcp *packet) { + if((plugin_session < (janus_plugin_session *)0x1000) || g_atomic_int_get(&plugin_session->stopped) || + packet == NULL || packet->buffer == NULL || packet->length < 1) return; janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) return; - janus_ice_relay_rtcp(handle, video, buf, len); + janus_ice_relay_rtcp(handle, packet); } -void janus_plugin_relay_data(janus_plugin_session *plugin_session, char *label, char *buf, int len) { - if((plugin_session < (janus_plugin_session *)0x1000) || g_atomic_int_get(&plugin_session->stopped) || buf == NULL || len < 1) +void janus_plugin_relay_data(janus_plugin_session *plugin_session, janus_plugin_data *packet) { + if((plugin_session < (janus_plugin_session *)0x1000) || g_atomic_int_get(&plugin_session->stopped) || + packet == NULL || packet->buffer == NULL || packet->length < 1) return; janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) return; #ifdef HAVE_SCTP - janus_ice_relay_data(handle, label, buf, len); + janus_ice_relay_data(handle, packet); #else JANUS_LOG(LOG_WARN, "Asked to relay data, but Data Channels support has not been compiled...\n"); #endif } +void janus_plugin_send_pli(janus_plugin_session *plugin_session) { + if((plugin_session < (janus_plugin_session *)0x1000) || g_atomic_int_get(&plugin_session->stopped)) + return; + janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; + if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) + || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) + return; + janus_ice_send_pli(handle); +} + +void janus_plugin_send_remb(janus_plugin_session *plugin_session, uint32_t bitrate) { + if((plugin_session < (janus_plugin_session *)0x1000) || g_atomic_int_get(&plugin_session->stopped)) + return; + janus_ice_handle *handle = (janus_ice_handle *)plugin_session->gateway_handle; + if(!handle || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_STOP) + || janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_ALERT)) + return; + janus_ice_send_remb(handle, bitrate); +} + static gboolean janus_plugin_close_pc_internal(gpointer user_data) { /* We actually enforce the close_pc here */ janus_plugin_session *plugin_session = (janus_plugin_session *) user_data; diff --git a/plugins/janus_audiobridge.c b/plugins/janus_audiobridge.c index 8e184d7474..8eb1a89a02 100644 --- a/plugins/janus_audiobridge.c +++ b/plugins/janus_audiobridge.c @@ -693,8 +693,8 @@ void janus_audiobridge_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_audiobridge_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); json_t *janus_audiobridge_handle_admin_message(json_t *message); void janus_audiobridge_setup_media(janus_plugin_session *handle); -void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_audiobridge_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); +void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_audiobridge_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); void janus_audiobridge_hangup_media(janus_plugin_session *handle); void janus_audiobridge_destroy_session(janus_plugin_session *handle, int *error); json_t *janus_audiobridge_query_session(janus_plugin_session *handle); @@ -3179,7 +3179,7 @@ void janus_audiobridge_setup_media(janus_plugin_session *handle) { janus_mutex_unlock(&rooms_mutex); } -void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; janus_audiobridge_session *session = (janus_audiobridge_session *)handle->plugin_handle; @@ -3188,6 +3188,8 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, int video, cha janus_audiobridge_participant *participant = (janus_audiobridge_participant *)session->participant; if(!g_atomic_int_get(&participant->active) || participant->muted || !participant->decoder || !participant->room) return; + char *buf = packet->buffer; + uint16_t len = packet->length; /* Save the frame if we're recording this leg */ janus_recorder_save_frame(participant->arc, buf, len); if(g_atomic_int_get(&participant->active) && participant->decoder) { @@ -3241,8 +3243,8 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, int video, cha if(participant->extmap_id > 0) { /* Check the audio levels, in case we need to notify participants about who's talking */ - int level = 0; - if(janus_rtp_header_extension_parse_audio_level(buf, len, participant->extmap_id, &level) == 0) { + int level = packet->extensions.audio_level; + if(level != -1) { /* Is this silence? */ pkt->silence = (level == 127); if(participant->room && participant->room->audiolevel_event) { @@ -3420,7 +3422,7 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, int video, cha } } -void janus_audiobridge_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_audiobridge_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; /* FIXME Should we care? */ @@ -4948,9 +4950,13 @@ static void janus_audiobridge_relay_rtp_packet(gpointer data, gpointer user_data packet->data->type = participant->opus_pt; /* Fix sequence number and timestamp (room switching may be involved) */ janus_rtp_header_update(packet->data, &participant->context, FALSE, OPUS_SAMPLES); - if(gateway != NULL) - gateway->relay_rtp(session->handle, 0, (char *)packet->data, packet->length); - /* Restore the timestamp and sequence number to what the publisher set them to */ + if(gateway != NULL) { + janus_plugin_rtp rtp = { .video = FALSE, .buffer = (char *)packet->data, .length = packet->length }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + /* FIXME Should we add our own audio level extension? */ + gateway->relay_rtp(session->handle, &rtp); + } + /* Restore the timestamp and sequence number to what the mixer set them to */ packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); } diff --git a/plugins/janus_duktape.c b/plugins/janus_duktape.c index 5b899adcf6..dc21dfd56c 100644 --- a/plugins/janus_duktape.c +++ b/plugins/janus_duktape.c @@ -213,9 +213,9 @@ void janus_duktape_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_duktape_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); json_t *janus_duktape_handle_admin_message(json_t *message); void janus_duktape_setup_media(janus_plugin_session *handle); -void janus_duktape_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_duktape_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_duktape_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len); +void janus_duktape_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_duktape_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); +void janus_duktape_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet); void janus_duktape_slow_link(janus_plugin_session *handle, int uplink, int video); void janus_duktape_hangup_media(janus_plugin_session *handle); void janus_duktape_destroy_session(janus_plugin_session *handle, int *error); @@ -865,10 +865,9 @@ static duk_ret_t janus_duktape_method_setbitrate(duk_context *ctx) { janus_mutex_unlock(&duktape_sessions_mutex); session->bitrate = bitrate; /* Send a REMB right away too, if the PeerConnection is up */ - if(session->bitrate > 0 && g_atomic_int_get(&session->started)) { - char rtcpbuf[24]; - janus_rtcp_remb((char *)(&rtcpbuf), 24, session->bitrate); - janus_core->relay_rtcp(session->handle, 1, rtcpbuf, 24); + if(g_atomic_int_get(&session->started)) { + /* No limit ~= 10000000 */ + janus_core->send_remb(session->handle, session->bitrate ? session->bitrate : 10000000); } /* Done */ janus_refcount_decrease(&session->ref); @@ -924,10 +923,8 @@ static duk_ret_t janus_duktape_method_sendpli(duk_context *ctx) { janus_mutex_unlock(&duktape_sessions_mutex); /* Send a PLI */ session->pli_latest = janus_get_monotonic_time(); - char rtcpbuf[12]; - janus_rtcp_pli((char *)&rtcpbuf, 12); JANUS_LOG(LOG_HUGE, "Sending PLI to session %"SCNu32"\n", session->id); - janus_core->relay_rtcp(session->handle, 1, rtcpbuf, 12); + janus_core->send_pli(session->handle); /* Done */ janus_refcount_decrease(&session->ref); duk_push_int(ctx, 0); @@ -974,7 +971,9 @@ static duk_ret_t janus_duktape_method_relayrtp(duk_context *ctx) { } janus_mutex_unlock(&duktape_sessions_mutex); /* Send the RTP packet */ - janus_core->relay_rtp(session->handle, is_video, (char *)payload, len); + janus_plugin_rtp rtp = { .video = is_video, .buffer = (char *)payload, .length = len }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + janus_core->relay_rtp(session->handle, &rtp); duk_push_int(ctx, 0); return 1; } @@ -1019,7 +1018,8 @@ static duk_ret_t janus_duktape_method_relayrtcp(duk_context *ctx) { } janus_mutex_unlock(&duktape_sessions_mutex); /* Send the RTCP packet */ - janus_core->relay_rtcp(session->handle, is_video, (char *)payload, len); + janus_plugin_rtcp rtcp = { .video = is_video, .buffer = (char *)payload, .length = len }; + janus_core->relay_rtcp(session->handle, &rtcp); duk_push_int(ctx, 0); return 1; } @@ -1040,6 +1040,7 @@ static duk_ret_t janus_duktape_method_relaydata(duk_context *ctx) { janus_duktape_type_string(DUK_TYPE_NUMBER), janus_duktape_type_string(duk_get_type(ctx, 2))); return duk_throw(ctx); } + /* FIXME We should add support for labels, here */ uint32_t id = (uint32_t)duk_get_number(ctx, 0); const char *payload = duk_get_string(ctx, 1); int len = (int)duk_get_number(ctx, 2); @@ -1058,8 +1059,9 @@ static duk_ret_t janus_duktape_method_relaydata(duk_context *ctx) { } janus_refcount_increase(&session->ref); janus_mutex_unlock(&duktape_sessions_mutex); - /* Send the RTP packet */ - janus_core->relay_data(session->handle, NULL, (char *)payload, len); + /* Send the data */ + janus_plugin_data data = { .label = NULL, .buffer = (char *)payload, .length = len }; + janus_core->relay_data(session->handle, &data); janus_refcount_decrease(&session->ref); duk_push_int(ctx, 0); return 1; @@ -1131,10 +1133,8 @@ static duk_ret_t janus_duktape_method_startrecording(duk_context *ctx) { session->vrc = vrc; /* Also send a keyframe request */ session->pli_latest = janus_get_monotonic_time(); - char rtcpbuf[12]; - janus_rtcp_pli((char *)&rtcpbuf, 12); JANUS_LOG(LOG_HUGE, "Sending PLI to session %"SCNu32"\n", session->id); - janus_core->relay_rtcp(session->handle, 1, rtcpbuf, 12); + janus_core->send_pli(session->handle); } if(drc) { session->drc = drc; @@ -2040,7 +2040,7 @@ void janus_duktape_setup_media(janus_plugin_session *handle) { janus_refcount_decrease(&session->ref); } -void janus_duktape_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_duktape_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *rtp_packet) { if(handle == NULL || handle->stopped || g_atomic_int_get(&duktape_stopping) || !g_atomic_int_get(&duktape_initialized)) return; janus_duktape_session *session = (janus_duktape_session *)handle->plugin_handle; @@ -2050,6 +2050,9 @@ void janus_duktape_incoming_rtp(janus_plugin_session *handle, int video, char *b } if(g_atomic_int_get(&session->destroyed) || g_atomic_int_get(&session->hangingup)) return; + gboolean video = rtp_packet->video; + char *buf = rtp_packet->buffer; + uint16_t len = rtp_packet->length; /* Check if the JS script wants to handle/manipulate RTP packets itself */ if(has_incoming_rtp) { /* Yep, pass the data to the JS script and return */ @@ -2096,15 +2099,12 @@ void janus_duktape_incoming_rtp(janus_plugin_session *handle, int video, char *b gint64 now = janus_get_monotonic_time(); if((now-session->pli_latest) >= ((gint64)session->pli_freq*G_USEC_PER_SEC)) { session->pli_latest = now; - char rtcpbuf[12]; - janus_rtcp_pli((char *)&rtcpbuf, 12); - JANUS_LOG(LOG_HUGE, "Sending PLI to session %"SCNu32"\n", session->id); - janus_core->relay_rtcp(handle, 1, rtcpbuf, 12); + janus_core->send_pli(handle); } } } -void janus_duktape_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_duktape_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(handle == NULL || handle->stopped || g_atomic_int_get(&duktape_stopping) || !g_atomic_int_get(&duktape_initialized)) return; janus_duktape_session *session = (janus_duktape_session *)handle->plugin_handle; @@ -2114,6 +2114,9 @@ void janus_duktape_incoming_rtcp(janus_plugin_session *handle, int video, char * } if(g_atomic_int_get(&session->destroyed) || g_atomic_int_get(&session->hangingup)) return; + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; /* Check if the JS script wants to handle/manipulate RTCP packets itself */ if(has_incoming_rtcp) { /* Yep, pass the data to the JS script and return */ @@ -2138,13 +2141,8 @@ void janus_duktape_incoming_rtcp(janus_plugin_session *handle, int video, char * /* If a REMB arrived, make sure we cap it to our configuration, and send it as a video RTCP */ uint32_t bitrate = janus_rtcp_get_remb(buf, len); if(bitrate > 0) { - if(session->bitrate > 0) { - char rtcpbuf[24]; - janus_rtcp_remb((char *)(&rtcpbuf), 24, session->bitrate); - janus_core->relay_rtcp(handle, 1, rtcpbuf, 24); - } else { - janus_core->relay_rtcp(handle, 1, buf, len); - } + /* No limit ~= 10000000 */ + janus_core->send_remb(handle, session->bitrate ? session->bitrate : 10000000); } /* If there's an incoming PLI, instead, relay it to the source of the media if any */ if(janus_rtcp_has_pli(buf, len)) { @@ -2152,16 +2150,14 @@ void janus_duktape_incoming_rtcp(janus_plugin_session *handle, int video, char * janus_mutex_lock_nodebug(&session->sender->recipients_mutex); /* Send a PLI */ session->sender->pli_latest = janus_get_monotonic_time(); - char rtcpbuf[12]; - janus_rtcp_pli((char *)&rtcpbuf, 12); JANUS_LOG(LOG_HUGE, "Sending PLI to session %"SCNu32"\n", session->sender->id); - janus_core->relay_rtcp(session->sender->handle, 1, rtcpbuf, 12); + janus_core->send_pli(session->sender->handle); janus_mutex_unlock_nodebug(&session->sender->recipients_mutex); } } } -void janus_duktape_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len) { +void janus_duktape_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet) { if(handle == NULL || handle->stopped || g_atomic_int_get(&duktape_stopping) || !g_atomic_int_get(&duktape_initialized)) return; janus_duktape_session *session = (janus_duktape_session *)handle->plugin_handle; @@ -2171,6 +2167,8 @@ void janus_duktape_incoming_data(janus_plugin_session *handle, char *label, char } if(g_atomic_int_get(&session->destroyed) || g_atomic_int_get(&session->hangingup)) return; + char *buf = packet->buffer; + uint16_t len = packet->length; /* Are we recording? */ janus_recorder_save_frame(session->drc, buf, len); /* Check if the JS script wants to handle/manipulate data channel packets itself */ @@ -2207,6 +2205,7 @@ void janus_duktape_incoming_data(janus_plugin_session *handle, char *label, char JANUS_LOG(LOG_VERB, "Got a DataChannel message (%zu bytes) to forward: %s\n", strlen(text), text); /* Relay to all recipients */ janus_mutex_lock_nodebug(&session->recipients_mutex); + /* FIXME We should add support for labels, here */ g_slist_foreach(session->recipients, janus_duktape_relay_data_packet, text); janus_mutex_unlock_nodebug(&session->recipients_mutex); g_free(text); @@ -2331,8 +2330,11 @@ static void janus_duktape_relay_rtp_packet(gpointer data, gpointer user_data) { /* Fix sequence number and timestamp (publisher switching may be involved) */ janus_rtp_header_update(packet->data, &session->rtpctx, packet->is_video, packet->is_video ? 4500 : 960); /* Send the packet */ - if(janus_core != NULL) - janus_core->relay_rtp(session->handle, packet->is_video, (char *)packet->data, packet->length); + if(janus_core != NULL) { + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + janus_core->relay_rtp(session->handle, &rtp); + } /* Restore the timestamp and sequence number to what the publisher set them to */ packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); @@ -2349,7 +2351,8 @@ static void janus_duktape_relay_data_packet(gpointer data, gpointer user_data) { if(janus_core != NULL && text != NULL) { JANUS_LOG(LOG_VERB, "Forwarding DataChannel message (%zu bytes) to session %"SCNu32": %s\n", strlen(text), session->id, text); - janus_core->relay_data(session->handle, NULL, text, strlen(text)); + janus_plugin_data data = { .label = NULL, .buffer = text, .length = strlen(text) }; + janus_core->relay_data(session->handle, &data); } return; } diff --git a/plugins/janus_echotest.c b/plugins/janus_echotest.c index ed8b1c5f7f..6b1efb0da4 100644 --- a/plugins/janus_echotest.c +++ b/plugins/janus_echotest.c @@ -142,9 +142,9 @@ void janus_echotest_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_echotest_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); json_t *janus_echotest_handle_admin_message(json_t *message); void janus_echotest_setup_media(janus_plugin_session *handle); -void janus_echotest_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_echotest_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_echotest_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len); +void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_echotest_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); +void janus_echotest_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet); void janus_echotest_slow_link(janus_plugin_session *handle, int uplink, int video); void janus_echotest_hangup_media(janus_plugin_session *handle); void janus_echotest_destroy_session(janus_plugin_session *handle, int *error); @@ -531,7 +531,7 @@ void janus_echotest_setup_media(janus_plugin_session *handle) { /* We really don't care, as we only send RTP/RTCP we get in the first place back anyway */ } -void janus_echotest_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; /* Simple echo test */ @@ -544,6 +544,9 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, int video, char * } if(g_atomic_int_get(&session->destroyed)) return; + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; if(video && session->video_active && (session->ssrc[0] != 0 || session->rid[0] != NULL)) { /* Handle simulcast: backup the header information first */ janus_rtp_header *header = (janus_rtp_header *)buf; @@ -555,10 +558,7 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, int video, char * buf, len, session->ssrc, session->rid, session->vcodec, &session->context); if(session->sim_context.need_pli) { /* Send a PLI */ - char rtcpbuf[12]; - memset(rtcpbuf, 0, 12); - janus_rtcp_pli((char *)&rtcpbuf, 12); - gateway->relay_rtcp(handle, 1, rtcpbuf, 12); + gateway->send_pli(handle); } /* Do we need to drop this? */ if(!relay) @@ -593,7 +593,7 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, int video, char * header->ssrc = htonl(1); janus_recorder_save_frame(session->vrc, buf, len); /* Send the frame back */ - gateway->relay_rtp(handle, video, buf, len); + gateway->relay_rtp(handle, packet); /* Restore header or core statistics will be messed up */ header->ssrc = htonl(ssrc); header->timestamp = htonl(timestamp); @@ -603,13 +603,13 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, int video, char * /* Save the frame if we're recording */ janus_recorder_save_frame(video ? session->vrc : session->arc, buf, len); /* Send the frame back */ - gateway->relay_rtp(handle, video, buf, len); + gateway->relay_rtp(handle, packet); } } } } -void janus_echotest_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_echotest_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; /* Simple echo test */ @@ -621,22 +621,19 @@ void janus_echotest_incoming_rtcp(janus_plugin_session *handle, int video, char } if(g_atomic_int_get(&session->destroyed)) return; - guint32 bitrate = janus_rtcp_get_remb(buf, len); + guint32 bitrate = janus_rtcp_get_remb(packet->buffer, packet->length); if(bitrate > 0) { /* If a REMB arrived, make sure we cap it to our configuration, and send it as a video RTCP */ session->peer_bitrate = bitrate; - if(session->bitrate == 0) /* No limit ~= 10000000 */ - janus_rtcp_cap_remb(buf, len, 10000000); - else - janus_rtcp_cap_remb(buf, len, session->bitrate); - gateway->relay_rtcp(handle, 1, buf, len); + /* No limit ~= 10000000 */ + gateway->send_remb(handle, session->bitrate ? session->bitrate : 10000000); return; } - gateway->relay_rtcp(handle, video, buf, len); + gateway->relay_rtcp(handle, packet); } } -void janus_echotest_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len) { +void janus_echotest_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; /* Simple echo test */ @@ -648,8 +645,11 @@ void janus_echotest_incoming_data(janus_plugin_session *handle, char *label, cha } if(g_atomic_int_get(&session->destroyed)) return; - if(buf == NULL || len <= 0) + if(packet->buffer == NULL || packet->length == 0) return; + char *label = packet->label; + char *buf = packet->buffer; + uint16_t len = packet->length; char *text = g_malloc(len+1); memcpy(text, buf, len); *(text+len) = '\0'; @@ -661,7 +661,13 @@ void janus_echotest_incoming_data(janus_plugin_session *handle, char *label, cha char *reply = g_malloc(strlen(prefix)+len+1); g_snprintf(reply, strlen(prefix)+len+1, "%s%s", prefix, text); g_free(text); - gateway->relay_data(handle, label, reply, strlen(reply)); + /* Prepare the packet and send it back */ + janus_plugin_data r = { + .label = label, + .buffer = reply, + .length = strlen(reply) + }; + gateway->relay_data(handle, &r); g_free(reply); } } @@ -916,10 +922,7 @@ static void *janus_echotest_handler(void *data) { if(!session->video_active && json_is_true(video)) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "Just (re-)enabled video, sending a PLI to recover it\n"); - char buf[12]; - memset(buf, 0, 12); - janus_rtcp_pli((char *)&buf, 12); - gateway->relay_rtcp(session->handle, 1, buf, 12); + gateway->send_pli(session->handle); } session->video_active = json_is_true(video); JANUS_LOG(LOG_VERB, "Setting video property: %s\n", session->video_active ? "true" : "false"); @@ -927,12 +930,7 @@ static void *janus_echotest_handler(void *data) { if(bitrate) { session->bitrate = json_integer_value(bitrate); JANUS_LOG(LOG_VERB, "Setting video bitrate: %"SCNu32"\n", session->bitrate); - if(session->bitrate > 0) { - char buf[24]; - janus_rtcp_remb((char *)&buf, 24, session->bitrate); - JANUS_LOG(LOG_VERB, "Sending REMB\n"); - gateway->relay_rtcp(session->handle, 1, buf, 24); - } + gateway->send_remb(session->handle, session->bitrate ? session->bitrate : 10000000); } if(substream) { session->sim_context.substream_target = json_integer_value(substream); @@ -949,10 +947,7 @@ static void *janus_echotest_handler(void *data) { } else { /* We need to change substream, send a PLI */ JANUS_LOG(LOG_VERB, "Simulcasting substream change, sending a PLI to kickstart it\n"); - char buf[12]; - memset(buf, 0, 12); - janus_rtcp_pli((char *)&buf, 12); - gateway->relay_rtcp(session->handle, 1, buf, 12); + gateway->send_pli(session->handle); } } if(temporal) { @@ -970,10 +965,7 @@ static void *janus_echotest_handler(void *data) { } else { /* We need to change temporal, send a PLI */ JANUS_LOG(LOG_VERB, "Simulcasting temporal layer change, sending a PLI to kickstart it\n"); - char buf[12]; - memset(buf, 0, 12); - janus_rtcp_pli((char *)&buf, 12); - gateway->relay_rtcp(session->handle, 1, buf, 12); + gateway->send_pli(session->handle); } } @@ -1041,6 +1033,8 @@ static void *janus_echotest_handler(void *data) { JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_MID, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_RID, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_REPAIRED_RID, + JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_AUDIO_LEVEL, + JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_FRAME_MARKING, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC, JANUS_SDP_OA_DONE); @@ -1139,10 +1133,7 @@ static void *janus_echotest_handler(void *data) { } /* Send a PLI */ JANUS_LOG(LOG_VERB, "Recording video, sending a PLI to kickstart it\n"); - char buf[12]; - memset(buf, 0, 12); - janus_rtcp_pli((char *)&buf, 12); - gateway->relay_rtcp(session->handle, 1, buf, 12); + gateway->send_pli(session->handle); } if(session->has_data) { memset(filename, 0, 255); diff --git a/plugins/janus_lua.c b/plugins/janus_lua.c index ef576ede69..9898b26a47 100644 --- a/plugins/janus_lua.c +++ b/plugins/janus_lua.c @@ -214,9 +214,9 @@ void janus_lua_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_lua_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); json_t *janus_lua_handle_admin_message(json_t *message); void janus_lua_setup_media(janus_plugin_session *handle); -void janus_lua_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_lua_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_lua_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len); +void janus_lua_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_lua_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); +void janus_lua_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet); void janus_lua_slow_link(janus_plugin_session *handle, int uplink, int video); void janus_lua_hangup_media(janus_plugin_session *handle); void janus_lua_destroy_session(janus_plugin_session *handle, int *error); @@ -778,10 +778,9 @@ static int janus_lua_method_setbitrate(lua_State *s) { janus_mutex_unlock(&lua_sessions_mutex); session->bitrate = bitrate; /* Send a REMB right away too, if the PeerConnection is up */ - if(session->bitrate > 0 && g_atomic_int_get(&session->started)) { - char rtcpbuf[24]; - janus_rtcp_remb((char *)(&rtcpbuf), 24, session->bitrate); - janus_core->relay_rtcp(session->handle, 1, rtcpbuf, 24); + if(g_atomic_int_get(&session->started)) { + /* No limit ~= 10000000 */ + janus_core->send_remb(session->handle, session->bitrate ? session->bitrate : 10000000); } /* Done */ janus_refcount_decrease(&session->ref); @@ -837,10 +836,8 @@ static int janus_lua_method_sendpli(lua_State *s) { janus_mutex_unlock(&lua_sessions_mutex); /* Send a PLI */ session->pli_latest = janus_get_monotonic_time(); - char rtcpbuf[12]; - janus_rtcp_pli((char *)&rtcpbuf, 12); JANUS_LOG(LOG_HUGE, "Sending PLI to session %"SCNu32"\n", session->id); - janus_core->relay_rtcp(session->handle, 1, rtcpbuf, 12); + janus_core->send_pli(session->handle); /* Done */ janus_refcount_decrease(&session->ref); lua_pushnumber(s, 0); @@ -874,7 +871,9 @@ static int janus_lua_method_relayrtp(lua_State *s) { } janus_mutex_unlock(&lua_sessions_mutex); /* Send the RTP packet */ - janus_core->relay_rtp(session->handle, is_video, (char *)payload, len); + janus_plugin_rtp rtp = { .video = is_video, .buffer = (char *)payload, .length = len }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + janus_core->relay_rtp(session->handle, &rtp); lua_pushnumber(s, 0); return 1; } @@ -906,7 +905,8 @@ static int janus_lua_method_relayrtcp(lua_State *s) { } janus_mutex_unlock(&lua_sessions_mutex); /* Send the RTCP packet */ - janus_core->relay_rtcp(session->handle, is_video, (char *)payload, len); + janus_plugin_rtcp rtcp = { .video = is_video, .buffer = (char *)payload, .length = len }; + janus_core->relay_rtcp(session->handle, &rtcp); lua_pushnumber(s, 0); return 1; } @@ -919,6 +919,7 @@ static int janus_lua_method_relaydata(lua_State *s) { lua_pushnumber(s, -1); return 1; } + /* FIXME We should add support for labels, here */ guint32 id = lua_tonumber(s, 1); const char *payload = lua_tostring(s, 2); int len = lua_tonumber(s, 3); @@ -937,8 +938,9 @@ static int janus_lua_method_relaydata(lua_State *s) { } janus_refcount_increase(&session->ref); janus_mutex_unlock(&lua_sessions_mutex); - /* Send the RTP packet */ - janus_core->relay_data(session->handle, NULL, (char *)payload, len); + /* Send the data */ + janus_plugin_data data = { .label = NULL, .buffer = (char *)payload, .length = len }; + janus_core->relay_data(session->handle, &data); janus_refcount_decrease(&session->ref); lua_pushnumber(s, 0); return 1; @@ -1009,10 +1011,8 @@ static int janus_lua_method_startrecording(lua_State *s) { session->vrc = vrc; /* Also send a keyframe request */ session->pli_latest = janus_get_monotonic_time(); - char rtcpbuf[12]; - janus_rtcp_pli((char *)&rtcpbuf, 12); JANUS_LOG(LOG_HUGE, "Sending PLI to session %"SCNu32"\n", session->id); - janus_core->relay_rtcp(session->handle, 1, rtcpbuf, 12); + janus_core->send_pli(session->handle); } if(drc) { session->drc = drc; @@ -1755,7 +1755,7 @@ void janus_lua_setup_media(janus_plugin_session *handle) { janus_refcount_decrease(&session->ref); } -void janus_lua_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_lua_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *rtp_packet) { if(handle == NULL || handle->stopped || g_atomic_int_get(&lua_stopping) || !g_atomic_int_get(&lua_initialized)) return; janus_lua_session *session = (janus_lua_session *)handle->plugin_handle; @@ -1765,6 +1765,9 @@ void janus_lua_incoming_rtp(janus_plugin_session *handle, int video, char *buf, } if(g_atomic_int_get(&session->destroyed) || g_atomic_int_get(&session->hangingup)) return; + gboolean video = rtp_packet->video; + char *buf = rtp_packet->buffer; + uint16_t len = rtp_packet->length; /* Check if the Lua script wants to handle/manipulate RTP packets itself */ if(has_incoming_rtp) { /* Yep, pass the data to the Lua script and return */ @@ -1805,15 +1808,13 @@ void janus_lua_incoming_rtp(janus_plugin_session *handle, int video, char *buf, gint64 now = janus_get_monotonic_time(); if((now-session->pli_latest) >= ((gint64)session->pli_freq*G_USEC_PER_SEC)) { session->pli_latest = now; - char rtcpbuf[12]; - janus_rtcp_pli((char *)&rtcpbuf, 12); JANUS_LOG(LOG_HUGE, "Sending PLI to session %"SCNu32"\n", session->id); - janus_core->relay_rtcp(handle, 1, rtcpbuf, 12); + janus_core->send_pli(handle); } } } -void janus_lua_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_lua_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(handle == NULL || handle->stopped || g_atomic_int_get(&lua_stopping) || !g_atomic_int_get(&lua_initialized)) return; janus_lua_session *session = (janus_lua_session *)handle->plugin_handle; @@ -1823,6 +1824,9 @@ void janus_lua_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, } if(g_atomic_int_get(&session->destroyed) || g_atomic_int_get(&session->hangingup)) return; + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; /* Check if the Lua script wants to handle/manipulate RTCP packets itself */ if(has_incoming_rtcp) { /* Yep, pass the data to the Lua script and return */ @@ -1841,13 +1845,8 @@ void janus_lua_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, /* If a REMB arrived, make sure we cap it to our configuration, and send it as a video RTCP */ guint32 bitrate = janus_rtcp_get_remb(buf, len); if(bitrate > 0) { - if(session->bitrate > 0) { - char rtcpbuf[24]; - janus_rtcp_remb((char *)(&rtcpbuf), 24, session->bitrate); - janus_core->relay_rtcp(handle, 1, rtcpbuf, 24); - } else { - janus_core->relay_rtcp(handle, 1, buf, len); - } + /* No limit ~= 10000000 */ + janus_core->send_remb(handle, session->bitrate ? session->bitrate : 10000000); } /* If there's an incoming PLI, instead, relay it to the source of the media if any */ if(janus_rtcp_has_pli(buf, len)) { @@ -1855,16 +1854,14 @@ void janus_lua_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, janus_mutex_lock_nodebug(&session->sender->recipients_mutex); /* Send a PLI */ session->sender->pli_latest = janus_get_monotonic_time(); - char rtcpbuf[12]; - janus_rtcp_pli((char *)&rtcpbuf, 12); JANUS_LOG(LOG_HUGE, "Sending PLI to session %"SCNu32"\n", session->sender->id); - janus_core->relay_rtcp(session->sender->handle, 1, rtcpbuf, 12); + janus_core->send_pli(session->sender->handle); janus_mutex_unlock_nodebug(&session->sender->recipients_mutex); } } } -void janus_lua_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len) { +void janus_lua_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet) { if(handle == NULL || handle->stopped || g_atomic_int_get(&lua_stopping) || !g_atomic_int_get(&lua_initialized)) return; janus_lua_session *session = (janus_lua_session *)handle->plugin_handle; @@ -1874,6 +1871,8 @@ void janus_lua_incoming_data(janus_plugin_session *handle, char *label, char *bu } if(g_atomic_int_get(&session->destroyed) || g_atomic_int_get(&session->hangingup)) return; + char *buf = packet->buffer; + uint16_t len = packet->length; /* Are we recording? */ janus_recorder_save_frame(session->drc, buf, len); /* Check if the Lua script wants to handle/manipulate data channel packets itself */ @@ -1904,6 +1903,7 @@ void janus_lua_incoming_data(janus_plugin_session *handle, char *label, char *bu JANUS_LOG(LOG_VERB, "Got a DataChannel message (%zu bytes) to forward: %s\n", strlen(text), text); /* Relay to all recipients */ janus_mutex_lock_nodebug(&session->recipients_mutex); + /* FIXME We should add support for labels, here */ g_slist_foreach(session->recipients, janus_lua_relay_data_packet, text); janus_mutex_unlock_nodebug(&session->recipients_mutex); g_free(text); @@ -2016,8 +2016,11 @@ static void janus_lua_relay_rtp_packet(gpointer data, gpointer user_data) { /* Fix sequence number and timestamp (publisher switching may be involved) */ janus_rtp_header_update(packet->data, &session->rtpctx, packet->is_video, packet->is_video ? 4500 : 960); /* Send the packet */ - if(janus_core != NULL) - janus_core->relay_rtp(session->handle, packet->is_video, (char *)packet->data, packet->length); + if(janus_core != NULL) { + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + janus_core->relay_rtp(session->handle, &rtp); + } /* Restore the timestamp and sequence number to what the publisher set them to */ packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); @@ -2034,7 +2037,8 @@ static void janus_lua_relay_data_packet(gpointer data, gpointer user_data) { if(janus_core != NULL && text != NULL) { JANUS_LOG(LOG_VERB, "Forwarding DataChannel message (%zu bytes) to session %"SCNu32": %s\n", strlen(text), session->id, text); - janus_core->relay_data(session->handle, NULL, text, strlen(text)); + janus_plugin_data data = { .label = NULL, .buffer = text, .length = strlen(text) }; + janus_core->relay_data(session->handle, &data); } return; } diff --git a/plugins/janus_nosip.c b/plugins/janus_nosip.c index f6302920fa..1ca61bcd5e 100644 --- a/plugins/janus_nosip.c +++ b/plugins/janus_nosip.c @@ -194,8 +194,8 @@ const char *janus_nosip_get_package(void); void janus_nosip_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_nosip_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); void janus_nosip_setup_media(janus_plugin_session *handle); -void janus_nosip_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_nosip_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); +void janus_nosip_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_nosip_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); void janus_nosip_hangup_media(janus_plugin_session *handle); void janus_nosip_destroy_session(janus_plugin_session *handle, int *error); json_t *janus_nosip_query_session(janus_plugin_session *handle); @@ -990,7 +990,7 @@ void janus_nosip_setup_media(janus_plugin_session *handle) { janus_mutex_unlock(&sessions_mutex); } -void janus_nosip_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_nosip_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { if(handle == NULL || handle->stopped || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; if(gateway) { @@ -1000,6 +1000,9 @@ void janus_nosip_incoming_rtp(janus_plugin_session *handle, int video, char *buf JANUS_LOG(LOG_ERR, "No session associated with this handle...\n"); return; } + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; /* Forward to our NoSIP peer */ if((video && !session->media.video_send) || (!video && !session->media.audio_send)) { /* Dropping packet, peer doesn't want to receive it */ @@ -1067,7 +1070,7 @@ void janus_nosip_incoming_rtp(janus_plugin_session *handle, int video, char *buf } } -void janus_nosip_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_nosip_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(handle == NULL || handle->stopped || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; if(gateway) { @@ -1076,6 +1079,9 @@ void janus_nosip_incoming_rtcp(janus_plugin_session *handle, int video, char *bu JANUS_LOG(LOG_ERR, "No session associated with this handle...\n"); return; } + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; /* Forward to our NoSIP peer */ if((video && session->media.has_video && session->media.video_rtcp_fd != -1) || (!video && session->media.has_audio && session->media.audio_rtcp_fd != -1)) { @@ -1619,9 +1625,7 @@ static void *janus_nosip_handler(void *data) { } /* Send a PLI */ JANUS_LOG(LOG_VERB, "Recording video, sending a PLI to kickstart it\n"); - char buf[12]; - janus_rtcp_pli((char *)&buf, 12); - gateway->relay_rtcp(session->handle, 1, buf, 12); + gateway->send_pli(session->handle); } } } else { @@ -2368,7 +2372,9 @@ static void *janus_nosip_relay_thread(void *data) { /* Save the frame if we're recording */ janus_recorder_save_frame(video ? session->vrc_peer : session->arc_peer, buffer, bytes); /* Relay to browser */ - gateway->relay_rtp(session->handle, video, buffer, bytes); + janus_plugin_rtp rtp = { .video = video, .buffer = buffer, .length = bytes }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + gateway->relay_rtp(session->handle, &rtp); continue; } else { /* Audio or Video RTCP */ @@ -2389,7 +2395,8 @@ static void *janus_nosip_relay_thread(void *data) { bytes = buflen; } /* Relay to browser */ - gateway->relay_rtcp(session->handle, video, buffer, bytes); + janus_plugin_rtcp rtcp = { .video = video, .buffer = buffer, bytes }; + gateway->relay_rtcp(session->handle, &rtcp); continue; } } diff --git a/plugins/janus_recordplay.c b/plugins/janus_recordplay.c index eda808f57e..b3b954993e 100644 --- a/plugins/janus_recordplay.c +++ b/plugins/janus_recordplay.c @@ -295,8 +295,8 @@ void janus_recordplay_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_recordplay_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); json_t *janus_recordplay_handle_admin_message(json_t *message); void janus_recordplay_setup_media(janus_plugin_session *handle); -void janus_recordplay_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_recordplay_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); +void janus_recordplay_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_recordplay_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); void janus_recordplay_slow_link(janus_plugin_session *handle, int uplink, int video); void janus_recordplay_hangup_media(janus_plugin_session *handle); void janus_recordplay_destroy_session(janus_plugin_session *handle, int *error); @@ -1134,7 +1134,6 @@ void janus_recordplay_send_rtcp_feedback(janus_plugin_session *handle, int video return; /* We just do this for video, for now */ janus_recordplay_session *session = (janus_recordplay_session *)handle->plugin_handle; - char rtcpbuf[24]; /* Send a RR+SDES+REMB every five seconds, or ASAP while we are still * ramping up (first 4 RTP packets) */ @@ -1151,9 +1150,7 @@ void janus_recordplay_send_rtcp_feedback(janus_plugin_session *handle, int video } /* Send a new REMB back */ - char rtcpbuf[24]; - janus_rtcp_remb((char *)(&rtcpbuf), 24, bitrate); - gateway->relay_rtcp(handle, video, rtcpbuf, 24); + gateway->send_remb(handle, bitrate); session->video_remb_last = now; } @@ -1163,16 +1160,13 @@ void janus_recordplay_send_rtcp_feedback(janus_plugin_session *handle, int video gint64 interval = (gint64)(session->video_keyframe_interval / 1000) * G_USEC_PER_SEC; if(elapsed >= interval) { - /* Send both a FIR and a PLI, just to be sure */ - janus_rtcp_fir((char *)&rtcpbuf, 20, &session->video_fir_seq); - gateway->relay_rtcp(handle, video, rtcpbuf, 20); - janus_rtcp_pli((char *)&rtcpbuf, 12); - gateway->relay_rtcp(handle, video, rtcpbuf, 12); + /* Send a PLI */ + gateway->send_pli(handle); session->video_keyframe_request_last = now; } } -void janus_recordplay_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_recordplay_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; janus_recordplay_session *session = (janus_recordplay_session *)handle->plugin_handle; @@ -1184,6 +1178,9 @@ void janus_recordplay_incoming_rtp(janus_plugin_session *handle, int video, char return; if(!session->recorder || !session->recording) return; + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; if(video && (session->ssrc[0] != 0 || session->rid[0] != NULL)) { /* Handle simulcast: backup the header information first */ janus_rtp_header *header = (janus_rtp_header *)buf; @@ -1196,10 +1193,7 @@ void janus_recordplay_incoming_rtp(janus_plugin_session *handle, int video, char if(session->sim_context.need_pli) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n"); - char rtcpbuf[12]; - memset(rtcpbuf, 0, 12); - janus_rtcp_pli((char *)&rtcpbuf, 12); - gateway->relay_rtcp(handle, 1, rtcpbuf, 12); + gateway->send_pli(handle); } /* Do we need to drop this? */ if(!save) @@ -1228,7 +1222,7 @@ void janus_recordplay_incoming_rtp(janus_plugin_session *handle, int video, char janus_recordplay_send_rtcp_feedback(handle, video, buf, len); } -void janus_recordplay_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_recordplay_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; } @@ -2405,7 +2399,9 @@ static void *janus_recordplay_playout_thread(void *data) { /* Update payload type */ janus_rtp_header *rtp = (janus_rtp_header *)buffer; rtp->type = audio_pt; - gateway->relay_rtp(session->handle, 0, (char *)buffer, bytes); + janus_plugin_rtp prtp = { .video = FALSE, .buffer = (char *)buffer, .length = bytes }; + janus_plugin_rtp_extensions_reset(&prtp.extensions); + gateway->relay_rtp(session->handle, &prtp); gettimeofday(&now, NULL); abefore.tv_sec = now.tv_sec; abefore.tv_usec = now.tv_usec; @@ -2445,7 +2441,9 @@ static void *janus_recordplay_playout_thread(void *data) { /* Update payload type */ janus_rtp_header *rtp = (janus_rtp_header *)buffer; rtp->type = audio_pt; - gateway->relay_rtp(session->handle, 0, (char *)buffer, bytes); + janus_plugin_rtp prtp = { .video = FALSE, .buffer = (char *)buffer, .length = bytes }; + janus_plugin_rtp_extensions_reset(&prtp.extensions); + gateway->relay_rtp(session->handle, &prtp); asent = TRUE; audio = audio->next; } @@ -2463,7 +2461,9 @@ static void *janus_recordplay_playout_thread(void *data) { /* Update payload type */ janus_rtp_header *rtp = (janus_rtp_header *)buffer; rtp->type = video_pt; - gateway->relay_rtp(session->handle, 1, (char *)buffer, bytes); + janus_plugin_rtp prtp = { .video = TRUE, .buffer = (char *)buffer, .length = bytes }; + janus_plugin_rtp_extensions_reset(&prtp.extensions); + gateway->relay_rtp(session->handle, &prtp); video = video->next; } vsent = TRUE; @@ -2507,7 +2507,9 @@ static void *janus_recordplay_playout_thread(void *data) { /* Update payload type */ janus_rtp_header *rtp = (janus_rtp_header *)buffer; rtp->type = video_pt; - gateway->relay_rtp(session->handle, 1, (char *)buffer, bytes); + janus_plugin_rtp prtp = { .video = TRUE, .buffer = (char *)buffer, .length = bytes }; + janus_plugin_rtp_extensions_reset(&prtp.extensions); + gateway->relay_rtp(session->handle, &prtp); video = video->next; } vsent = TRUE; diff --git a/plugins/janus_sip.c b/plugins/janus_sip.c index 6d22a7218d..c8ce8dbcf8 100644 --- a/plugins/janus_sip.c +++ b/plugins/janus_sip.c @@ -651,8 +651,8 @@ const char *janus_sip_get_package(void); void janus_sip_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_sip_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); void janus_sip_setup_media(janus_plugin_session *handle); -void janus_sip_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_sip_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); +void janus_sip_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_sip_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); void janus_sip_hangup_media(janus_plugin_session *handle); void janus_sip_destroy_session(janus_plugin_session *handle, int *error); json_t *janus_sip_query_session(janus_plugin_session *handle); @@ -2138,7 +2138,7 @@ void janus_sip_setup_media(janus_plugin_session *handle) { /* TODO Only relay RTP/RTCP when we get this event */ } -void janus_sip_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_sip_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; if(gateway) { @@ -2150,6 +2150,9 @@ void janus_sip_incoming_rtp(janus_plugin_session *handle, int video, char *buf, } if(!janus_sip_call_is_established(session)) return; + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; /* Forward to our SIP peer */ if(video) { if(!session->media.video_send) { @@ -2256,7 +2259,7 @@ void janus_sip_incoming_rtp(janus_plugin_session *handle, int video, char *buf, } } -void janus_sip_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_sip_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; if(gateway) { @@ -2267,6 +2270,9 @@ void janus_sip_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, } if(!janus_sip_call_is_established(session)) return; + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; /* Forward to our SIP peer */ if(video) { if(session->media.has_video && session->media.video_rtcp_fd != -1) { @@ -4037,9 +4043,7 @@ static void *janus_sip_handler(void *data) { } /* Send a PLI */ JANUS_LOG(LOG_VERB, "Recording video, sending a PLI to kickstart it\n"); - char buf[12]; - janus_rtcp_pli((char *)&buf, 12); - gateway->relay_rtcp(session->handle, 1, buf, 12); + gateway->send_pli(session->handle); } } } else { @@ -5999,7 +6003,9 @@ static void *janus_sip_relay_thread(void *data) { /* Save the frame if we're recording */ janus_recorder_save_frame(session->arc_peer, buffer, bytes); /* Relay to application */ - gateway->relay_rtp(session->handle, 0, buffer, bytes); + janus_plugin_rtp rtp = { .video = FALSE, .buffer = buffer, .length = bytes }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + gateway->relay_rtp(session->handle, &rtp); continue; } else if(session->media.audio_rtcp_fd != -1 && fds[i].fd == session->media.audio_rtcp_fd) { /* Got something audio (RTCP) */ @@ -6022,7 +6028,8 @@ static void *janus_sip_relay_thread(void *data) { bytes = buflen; } /* Relay to application */ - gateway->relay_rtcp(session->handle, 0, buffer, bytes); + janus_plugin_rtcp rtcp = { .video = FALSE, .buffer = buffer, bytes }; + gateway->relay_rtcp(session->handle, &rtcp); continue; } else if(session->media.video_rtp_fd != -1 && fds[i].fd == session->media.video_rtp_fd) { /* Got something video (RTP) */ @@ -6064,7 +6071,9 @@ static void *janus_sip_relay_thread(void *data) { /* Save the frame if we're recording */ janus_recorder_save_frame(session->vrc_peer, buffer, bytes); /* Relay to application */ - gateway->relay_rtp(session->handle, 1, buffer, bytes); + janus_plugin_rtp rtp = { .video = TRUE, .buffer = buffer, .length = bytes }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + gateway->relay_rtp(session->handle, &rtp); continue; } else if(session->media.video_rtcp_fd != -1 && fds[i].fd == session->media.video_rtcp_fd) { /* Got something video (RTCP) */ @@ -6087,7 +6096,8 @@ static void *janus_sip_relay_thread(void *data) { bytes = buflen; } /* Relay to application */ - gateway->relay_rtcp(session->handle, 1, buffer, bytes); + janus_plugin_rtcp rtcp = { .video = TRUE, .buffer = buffer, bytes }; + gateway->relay_rtcp(session->handle, &rtcp); continue; } } diff --git a/plugins/janus_sipre.c b/plugins/janus_sipre.c index b57ef776f9..f5fafda5a2 100644 --- a/plugins/janus_sipre.c +++ b/plugins/janus_sipre.c @@ -110,8 +110,8 @@ const char *janus_sipre_get_package(void); void janus_sipre_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_sipre_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); void janus_sipre_setup_media(janus_plugin_session *handle); -void janus_sipre_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_sipre_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); +void janus_sipre_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_sipre_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); void janus_sipre_hangup_media(janus_plugin_session *handle); void janus_sipre_destroy_session(janus_plugin_session *handle, int *error); json_t *janus_sipre_query_session(janus_plugin_session *handle); @@ -1473,7 +1473,7 @@ void janus_sipre_setup_media(janus_plugin_session *handle) { /* TODO Only relay RTP/RTCP when we get this event */ } -void janus_sipre_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_sipre_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { if(handle == NULL || handle->stopped || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; if(gateway) { @@ -1483,6 +1483,9 @@ void janus_sipre_incoming_rtp(janus_plugin_session *handle, int video, char *buf JANUS_LOG(LOG_ERR, "No session associated with this handle...\n"); return; } + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; /* Forward to our SIPre peer */ if((video && !session->media.video_send) || (!video && !session->media.audio_send)) { /* Dropping packet, peer doesn't want to receive it */ @@ -1554,7 +1557,7 @@ void janus_sipre_incoming_rtp(janus_plugin_session *handle, int video, char *buf } } -void janus_sipre_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_sipre_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(handle == NULL || handle->stopped || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; if(gateway) { @@ -1563,6 +1566,9 @@ void janus_sipre_incoming_rtcp(janus_plugin_session *handle, int video, char *bu JANUS_LOG(LOG_ERR, "No session associated with this handle...\n"); return; } + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; /* Forward to our SIPre peer */ if((video && session->media.has_video && session->media.video_rtcp_fd != -1) || (!video && session->media.has_audio && session->media.audio_rtcp_fd != -1)) { @@ -2719,10 +2725,7 @@ static void *janus_sipre_handler(void *data) { } /* Send a PLI */ JANUS_LOG(LOG_VERB, "Recording video, sending a PLI to kickstart it\n"); - char buf[12]; - memset(buf, 0, 12); - janus_rtcp_pli((char *)&buf, 12); - gateway->relay_rtcp(session->handle, 1, buf, 12); + gateway->send_pli(session->handle); } } } else { @@ -3557,7 +3560,9 @@ static void *janus_sipre_relay_thread(void *data) { /* Save the frame if we're recording */ janus_recorder_save_frame(video ? session->vrc_peer : session->arc_peer, buffer, bytes); /* Relay to browser */ - gateway->relay_rtp(session->handle, video, buffer, bytes); + janus_plugin_rtp rtp = { .video = video, .buffer = buffer, .length = bytes }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + gateway->relay_rtp(session->handle, &rtp); continue; } else { /* Audio or Video RTCP */ @@ -3579,7 +3584,8 @@ static void *janus_sipre_relay_thread(void *data) { bytes = buflen; } /* Relay to browser */ - gateway->relay_rtcp(session->handle, video, buffer, bytes); + janus_plugin_rtcp rtcp = { .video = video, .buffer = buffer, bytes }; + gateway->relay_rtcp(session->handle, &rtcp); continue; } } diff --git a/plugins/janus_streaming.c b/plugins/janus_streaming.c index b4b453073f..0a696ae409 100644 --- a/plugins/janus_streaming.c +++ b/plugins/janus_streaming.c @@ -721,8 +721,8 @@ void janus_streaming_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_streaming_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); json_t *janus_streaming_handle_admin_message(json_t *message); void janus_streaming_setup_media(janus_plugin_session *handle); -void janus_streaming_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_streaming_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); +void janus_streaming_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_streaming_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); void janus_streaming_hangup_media(janus_plugin_session *handle); void janus_streaming_destroy_session(janus_plugin_session *handle, int *error); json_t *janus_streaming_query_session(janus_plugin_session *handle); @@ -3878,13 +3878,13 @@ void janus_streaming_setup_media(janus_plugin_session *handle) { janus_refcount_decrease(&session->ref); } -void janus_streaming_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_streaming_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; /* FIXME We don't care about what the browser sends us, we're sendonly */ } -void janus_streaming_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_streaming_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; janus_streaming_session *session = (janus_streaming_session *)handle->plugin_handle; @@ -3894,6 +3894,9 @@ void janus_streaming_incoming_rtcp(janus_plugin_session *handle, int video, char if(mp->streaming_source != janus_streaming_source_rtp) return; janus_streaming_rtp_source *source = (janus_streaming_rtp_source *)mp->source; + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; if(!video && (source->audio_rtcp_fd > -1) && (source->audio_rtcp_addr.ss_family != 0)) { JANUS_LOG(LOG_HUGE, "Got audio RTCP feedback from a viewer: SSRC %"SCNu32"\n", janus_rtcp_get_sender_ssrc(buf, len)); @@ -7326,8 +7329,10 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) if(override_mark_bit && !has_marker_bit) { packet->data->markerbit = 1; } + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); if(gateway != NULL) - gateway->relay_rtp(session->handle, packet->is_video, (char *)packet->data, packet->length); + gateway->relay_rtp(session->handle, &rtp); if(override_mark_bit && !has_marker_bit) { packet->data->markerbit = 0; } @@ -7386,8 +7391,10 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) session->sim_context.changed_substream); } /* Send the packet */ + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); if(gateway != NULL) - gateway->relay_rtp(session->handle, packet->is_video, (char *)packet->data, packet->length); + gateway->relay_rtp(session->handle, &rtp); /* Restore the timestamp and sequence number to what the publisher set them to */ packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); @@ -7398,8 +7405,10 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) } else { /* Fix sequence number and timestamp (switching may be involved) */ janus_rtp_header_update(packet->data, &session->context, TRUE, 0); + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); if(gateway != NULL) - gateway->relay_rtp(session->handle, packet->is_video, (char *)packet->data, packet->length); + gateway->relay_rtp(session->handle, &rtp); /* Restore the timestamp and sequence number to what the video source set them to */ packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); @@ -7409,8 +7418,10 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) return; /* Fix sequence number and timestamp (switching may be involved) */ janus_rtp_header_update(packet->data, &session->context, FALSE, 0); + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); if(gateway != NULL) - gateway->relay_rtp(session->handle, packet->is_video, (char *)packet->data, packet->length); + gateway->relay_rtp(session->handle, &rtp); /* Restore the timestamp and sequence number to what the video source set them to */ packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); @@ -7420,8 +7431,10 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data) if(!session->data) return; char *text = (char *)packet->data; - if(gateway != NULL && text != NULL) - gateway->relay_data(session->handle, NULL, text, strlen(text)); + if(gateway != NULL && text != NULL) { + janus_plugin_data data = { .label = NULL, .buffer = text, .length = strlen(text) }; + gateway->relay_data(session->handle, &data); + } } return; @@ -7444,8 +7457,9 @@ static void janus_streaming_relay_rtcp_packet(gpointer data, gpointer user_data) return; } + janus_plugin_rtcp rtcp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; if(gateway != NULL) - gateway->relay_rtcp(session->handle, (packet->is_video ? 1 : 0), (char*)packet->data, packet->length); + gateway->relay_rtcp(session->handle, &rtcp); return; } diff --git a/plugins/janus_textroom.c b/plugins/janus_textroom.c index cf22676c63..51d42a9e27 100644 --- a/plugins/janus_textroom.c +++ b/plugins/janus_textroom.c @@ -532,9 +532,9 @@ void janus_textroom_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_textroom_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); json_t *janus_textroom_handle_admin_message(json_t *message); void janus_textroom_setup_media(janus_plugin_session *handle); -void janus_textroom_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_textroom_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_textroom_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len); +void janus_textroom_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_textroom_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); +void janus_textroom_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet); void janus_textroom_slow_link(janus_plugin_session *handle, int uplink, int video); void janus_textroom_hangup_media(janus_plugin_session *handle); void janus_textroom_destroy_session(janus_plugin_session *handle, int *error); @@ -1297,15 +1297,15 @@ void janus_textroom_setup_media(janus_plugin_session *handle) { janus_mutex_unlock(&sessions_mutex); } -void janus_textroom_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_textroom_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { /* We don't do audio/video */ } -void janus_textroom_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_textroom_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { /* We don't do audio/video */ } -void janus_textroom_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len) { +void janus_textroom_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet) { if(handle == NULL || handle->stopped || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; /* Incoming request from this user: what should we do? */ @@ -1319,6 +1319,8 @@ void janus_textroom_incoming_data(janus_plugin_session *handle, char *label, cha janus_refcount_decrease(&session->ref); return; } + char *buf = packet->buffer; + uint16_t len = packet->length; if(buf == NULL || len <= 0) { janus_refcount_decrease(&session->ref); return; @@ -1428,7 +1430,8 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session janus_textroom_participant *top = g_hash_table_lookup(textroom->participants, to); if(top) { janus_refcount_increase(&top->ref); - gateway->relay_data(top->session->handle, NULL, msg_text, strlen(msg_text)); + janus_plugin_data data = { .label = NULL, .buffer = msg_text, .length = strlen(msg_text) }; + gateway->relay_data(top->session->handle, &data); janus_refcount_decrease(&top->ref); json_object_set_new(sent, to, json_true()); } else { @@ -1447,7 +1450,8 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session janus_textroom_participant *top = g_hash_table_lookup(textroom->participants, to); if(top) { janus_refcount_increase(&top->ref); - gateway->relay_data(top->session->handle, NULL, msg_text, strlen(msg_text)); + janus_plugin_data data = { .label = NULL, .buffer = msg_text, .length = strlen(msg_text) }; + gateway->relay_data(top->session->handle, &data); janus_refcount_decrease(&top->ref); json_object_set_new(sent, to, json_true()); } else { @@ -1467,7 +1471,8 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session janus_textroom_participant *top = value; JANUS_LOG(LOG_VERB, " >> To %s in %"SCNu64": %s\n", top->username, room_id, message); janus_refcount_increase(&top->ref); - gateway->relay_data(top->session->handle, NULL, msg_text, strlen(msg_text)); + janus_plugin_data data = { .label = NULL, .buffer = msg_text, .length = strlen(msg_text) }; + gateway->relay_data(top->session->handle, &data); janus_refcount_decrease(&top->ref); } } @@ -1593,7 +1598,8 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session json_object_set_new(event, "display", json_string(display_text)); char *event_text = json_dumps(event, json_format); json_decref(event); - gateway->relay_data(handle, NULL, event_text, strlen(event_text)); + janus_plugin_data data = { .label = NULL, .buffer = event_text, .length = strlen(event_text) }; + gateway->relay_data(handle, &data); /* Broadcast */ GHashTableIter iter; gpointer value; @@ -1604,7 +1610,7 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session continue; /* Skip us */ janus_refcount_increase(&top->ref); JANUS_LOG(LOG_VERB, " >> To %s in %"SCNu64"\n", top->username, room_id); - gateway->relay_data(top->session->handle, NULL, event_text, strlen(event_text)); + gateway->relay_data(top->session->handle, &data); /* Take note of this user */ json_t *p = json_object(); json_object_set_new(p, "username", json_string(top->username)); @@ -1680,7 +1686,8 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session json_object_set_new(event, "username", json_string(participant->username)); char *event_text = json_dumps(event, json_format); json_decref(event); - gateway->relay_data(handle, NULL, event_text, strlen(event_text)); + janus_plugin_data data = { .label = NULL, .buffer = event_text, .length = strlen(event_text) }; + gateway->relay_data(handle, &data); /* Broadcast */ GHashTableIter iter; gpointer value; @@ -1691,7 +1698,7 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session continue; /* Skip us */ janus_refcount_increase(&top->ref); JANUS_LOG(LOG_VERB, " >> To %s in %"SCNu64"\n", top->username, room_id); - gateway->relay_data(top->session->handle, NULL, event_text, strlen(event_text)); + gateway->relay_data(top->session->handle, &data); janus_refcount_decrease(&top->ref); } free(event_text); @@ -1946,7 +1953,8 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session while(g_hash_table_iter_next(&iter, NULL, &value)) { janus_textroom_participant *top = value; JANUS_LOG(LOG_VERB, " >> To %s in %"SCNu64"\n", top->username, room_id); - gateway->relay_data(top->session->handle, NULL, event_text, strlen(event_text)); + janus_plugin_data data = { .label = NULL, .buffer = event_text, .length = strlen(event_text) }; + gateway->relay_data(top->session->handle, &data); } free(event_text); } @@ -2027,7 +2035,8 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session janus_textroom_participant *top = value; JANUS_LOG(LOG_VERB, " >> To %s in %"SCNu64": %s\n", top->username, room_id, message); janus_refcount_increase(&top->ref); - gateway->relay_data(top->session->handle, NULL, msg_text, strlen(msg_text)); + janus_plugin_data data = { .label = NULL, .buffer = msg_text, .length = strlen(msg_text) }; + gateway->relay_data(top->session->handle, &data); janus_refcount_decrease(&top->ref); } } @@ -2433,7 +2442,8 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session json_object_set_new(event, "room", json_integer(textroom->room_id)); char *event_text = json_dumps(event, json_format); json_decref(event); - gateway->relay_data(handle, NULL, event_text, strlen(event_text)); + janus_plugin_data data = { .label = NULL, .buffer = event_text, .length = strlen(event_text) }; + gateway->relay_data(handle, &data); /* Broadcast */ GHashTableIter iter; gpointer value; @@ -2442,7 +2452,7 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session janus_textroom_participant *top = value; janus_refcount_increase(&top->ref); JANUS_LOG(LOG_VERB, " >> To %s in %"SCNu64"\n", top->username, room_id); - gateway->relay_data(top->session->handle, NULL, event_text, strlen(event_text)); + gateway->relay_data(top->session->handle, &data); janus_mutex_lock(&top->session->mutex); g_hash_table_remove(top->session->rooms, &room_id); janus_mutex_unlock(&top->session->mutex); @@ -2497,7 +2507,8 @@ janus_plugin_result *janus_textroom_handle_incoming_request(janus_plugin_session /* Reply via data channels */ char *reply_text = json_dumps(reply, json_format); json_decref(reply); - gateway->relay_data(handle, NULL, reply_text, strlen(reply_text)); + janus_plugin_data data = { .label = NULL, .buffer = reply_text, .length = strlen(reply_text) }; + gateway->relay_data(handle, &data); free(reply_text); } else { /* Reply via Janus API */ diff --git a/plugins/janus_videocall.c b/plugins/janus_videocall.c index 9d0fb4cbf7..2d381ef45c 100644 --- a/plugins/janus_videocall.c +++ b/plugins/janus_videocall.c @@ -288,9 +288,9 @@ const char *janus_videocall_get_package(void); void janus_videocall_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_videocall_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); void janus_videocall_setup_media(janus_plugin_session *handle); -void janus_videocall_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_videocall_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_videocall_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len); +void janus_videocall_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_videocall_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); +void janus_videocall_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet); void janus_videocall_slow_link(janus_plugin_session *handle, int uplink, int video); void janus_videocall_hangup_media(janus_plugin_session *handle); void janus_videocall_destroy_session(janus_plugin_session *handle, int *error); @@ -370,7 +370,7 @@ typedef struct janus_videocall_session { gboolean video_active; janus_audiocodec acodec;/* Codec used for audio, if available */ janus_videocodec vcodec;/* Codec used for video, if available */ - uint32_t bitrate; + uint32_t bitrate, peer_bitrate; guint16 slowlink_count; struct janus_videocall_session *peer; janus_rtp_switching_context context; @@ -563,6 +563,7 @@ void janus_videocall_create_session(janus_plugin_session *handle, int *error) { session->audio_active = TRUE; session->video_active = TRUE; session->bitrate = 0; /* No limit */ + session->peer_bitrate = 0; session->peer = NULL; session->username = NULL; janus_rtp_switching_context_reset(&session->context); @@ -627,6 +628,7 @@ json_t *janus_videocall_query_session(janus_plugin_session *handle) { json_object_set_new(info, "video_codec", json_string(janus_videocodec_name(session->vcodec))); json_object_set_new(info, "video_active", session->video_active ? json_true() : json_false()); json_object_set_new(info, "bitrate", json_integer(session->bitrate)); + json_object_set_new(info, "peer-bitrate", json_integer(session->peer_bitrate)); json_object_set_new(info, "slowlink_count", json_integer(session->slowlink_count)); } if(session->ssrc[0] != 0 || session->rid[0] != NULL) { @@ -692,7 +694,7 @@ void janus_videocall_setup_media(janus_plugin_session *handle) { /* We really don't care, as we only relay RTP/RTCP we get in the first place anyway */ } -void janus_videocall_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_videocall_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; if(gateway) { @@ -709,6 +711,9 @@ void janus_videocall_incoming_rtp(janus_plugin_session *handle, int video, char } if(g_atomic_int_get(&session->destroyed) || g_atomic_int_get(&peer->destroyed)) return; + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; if(video && session->video_active && (session->ssrc[0] != 0 || session->rid[0] != NULL)) { /* Handle simulcast: backup the header information first */ janus_rtp_header *header = (janus_rtp_header *)buf; @@ -725,10 +730,7 @@ void janus_videocall_incoming_rtp(janus_plugin_session *handle, int video, char if(peer->sim_context.need_pli) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n"); - char rtcpbuf[12]; - memset(rtcpbuf, 0, 12); - janus_rtcp_pli((char *)&rtcpbuf, 12); - gateway->relay_rtcp(session->handle, 1, rtcpbuf, 12); + gateway->send_pli(session->handle); } /* Any event we should notify? */ if(peer->sim_context.changed_substream) { @@ -766,7 +768,7 @@ void janus_videocall_incoming_rtp(janus_plugin_session *handle, int video, char header->ssrc = htonl(1); janus_recorder_save_frame(session->vrc, buf, len); /* Send the frame back */ - gateway->relay_rtp(peer->handle, video, buf, len); + gateway->relay_rtp(peer->handle, packet); /* Restore header or core statistics will be messed up */ header->ssrc = htonl(ssrc); header->timestamp = htonl(timestamp); @@ -776,13 +778,13 @@ void janus_videocall_incoming_rtp(janus_plugin_session *handle, int video, char /* Save the frame if we're recording */ janus_recorder_save_frame(video ? session->vrc : session->arc, buf, len); /* Forward the packet to the peer */ - gateway->relay_rtp(peer->handle, video, buf, len); + gateway->relay_rtp(peer->handle, packet); } } } } -void janus_videocall_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_videocall_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; if(gateway) { @@ -798,21 +800,19 @@ void janus_videocall_incoming_rtcp(janus_plugin_session *handle, int video, char } if(g_atomic_int_get(&session->destroyed) || g_atomic_int_get(&peer->destroyed)) return; - guint32 bitrate = janus_rtcp_get_remb(buf, len); + guint32 bitrate = janus_rtcp_get_remb(packet->buffer, packet->length); if(bitrate > 0) { /* If a REMB arrived, make sure we cap it to our configuration, and send it as a video RTCP */ - if(session->bitrate == 0) /* No limit ~= 10000000 */ - janus_rtcp_cap_remb(buf, len, 10000000); - else - janus_rtcp_cap_remb(buf, len, session->bitrate); - gateway->relay_rtcp(peer->handle, 1, buf, len); + session->peer_bitrate = bitrate; + /* No limit ~= 10000000 */ + gateway->send_remb(handle, session->bitrate ? session->bitrate : 10000000); return; } - gateway->relay_rtcp(peer->handle, video, buf, len); + gateway->relay_rtcp(peer->handle, packet); } } -void janus_videocall_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len) { +void janus_videocall_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; if(gateway) { @@ -828,8 +828,11 @@ void janus_videocall_incoming_data(janus_plugin_session *handle, char *label, ch } if(g_atomic_int_get(&session->destroyed) || g_atomic_int_get(&peer->destroyed)) return; - if(buf == NULL || len <= 0) + if(packet->buffer == NULL || packet->length == 0) return; + char *label = packet->label; + char *buf = packet->buffer; + uint16_t len = packet->length; char *text = g_malloc(len+1); memcpy(text, buf, len); *(text+len) = '\0'; @@ -837,7 +840,12 @@ void janus_videocall_incoming_data(janus_plugin_session *handle, char *label, ch /* Save the frame if we're recording */ janus_recorder_save_frame(session->drc, buf, len); /* Forward the packet to the peer */ - gateway->relay_data(peer->handle, label, text, strlen(text)); + janus_plugin_data r = { + .label = label, + .buffer = text, + .length = strlen(text) + }; + gateway->relay_data(peer->handle, &r); g_free(text); } } @@ -954,6 +962,7 @@ void janus_videocall_hangup_media(janus_plugin_session *handle) { session->acodec = JANUS_AUDIOCODEC_NONE; session->vcodec = JANUS_VIDEOCODEC_NONE; session->bitrate = 0; + session->peer_bitrate = 0; int i=0; for(i=0; i<3; i++) { session->ssrc[i] = 0; @@ -1372,9 +1381,7 @@ static void *janus_videocall_handler(void *data) { if(!session->video_active && json_is_true(video)) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "Just (re-)enabled video, sending a PLI to recover it\n"); - char buf[12]; - janus_rtcp_pli((char *)&buf, 12); - gateway->relay_rtcp(session->handle, 1, buf, 12); + gateway->send_pli(session->handle); } session->video_active = json_is_true(video); JANUS_LOG(LOG_VERB, "Setting video property: %s\n", session->video_active ? "true" : "false"); @@ -1382,14 +1389,7 @@ static void *janus_videocall_handler(void *data) { if(bitrate) { session->bitrate = json_integer_value(bitrate); JANUS_LOG(LOG_VERB, "Setting video bitrate: %"SCNu32"\n", session->bitrate); - if(session->bitrate > 0) { - /* FIXME Generate a new REMB (especially useful for Firefox, which doesn't send any we can cap later) */ - char buf[24]; - janus_rtcp_remb((char *)&buf, 24, session->bitrate); - JANUS_LOG(LOG_VERB, "Sending REMB\n"); - gateway->relay_rtcp(session->handle, 1, buf, 24); - /* FIXME How should we handle a subsequent "no limit" bitrate? */ - } + gateway->send_remb(session->handle, session->bitrate ? session->bitrate : 10000000); } janus_videocall_session *peer = session->peer; if(substream) { @@ -1410,11 +1410,8 @@ static void *janus_videocall_handler(void *data) { } else { /* We need to change substream, send the peer a PLI */ JANUS_LOG(LOG_VERB, "Simulcasting substream change, sending a PLI to kickstart it\n"); - char buf[12]; - memset(buf, 0, 12); - janus_rtcp_pli((char *)&buf, 12); if(peer && peer->handle) - gateway->relay_rtcp(session->handle, 1, buf, 12); + gateway->send_pli(peer->handle); } } if(temporal) { @@ -1435,11 +1432,8 @@ static void *janus_videocall_handler(void *data) { } else { /* We need to change temporal, send a PLI */ JANUS_LOG(LOG_VERB, "Simulcasting temporal layer change, sending a PLI to kickstart it\n"); - char buf[12]; - memset(buf, 0, 12); - janus_rtcp_pli((char *)&buf, 12); if(peer && peer->handle) - gateway->relay_rtcp(session->handle, 1, buf, 12); + gateway->send_pli(peer->handle); } } if(record) { @@ -1508,9 +1502,7 @@ static void *janus_videocall_handler(void *data) { } /* Send a PLI */ JANUS_LOG(LOG_VERB, "Recording video, sending a PLI to kickstart it\n"); - char buf[12]; - janus_rtcp_pli((char *)&buf, 12); - gateway->relay_rtcp(session->handle, 1, buf, 12); + gateway->send_pli(session->handle); } if(session->has_data) { memset(filename, 0, 255); diff --git a/plugins/janus_videoroom.c b/plugins/janus_videoroom.c index 6f18de349a..6842dd413a 100644 --- a/plugins/janus_videoroom.c +++ b/plugins/janus_videoroom.c @@ -1088,9 +1088,9 @@ void janus_videoroom_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_videoroom_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); json_t *janus_videoroom_handle_admin_message(json_t *message); void janus_videoroom_setup_media(janus_plugin_session *handle); -void janus_videoroom_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_videoroom_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_videoroom_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len); +void janus_videoroom_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_videoroom_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); +void janus_videoroom_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet); void janus_videoroom_slow_link(janus_plugin_session *handle, int uplink, int video); void janus_videoroom_hangup_media(janus_plugin_session *handle); void janus_videoroom_destroy_session(janus_plugin_session *handle, int *error); @@ -1687,16 +1687,10 @@ static void janus_videoroom_codecstr(janus_videoroom *videoroom, char *audio_cod } } -static void janus_videoroom_reqfir(janus_videoroom_publisher *publisher, const char *reason) { - /* Send a FIR */ - char buf[20]; - janus_rtcp_fir((char *)&buf, 20, &publisher->fir_seq); - JANUS_LOG(LOG_VERB, "%s sending FIR to %"SCNu64" (%s)\n", reason, publisher->user_id, publisher->display ? publisher->display : "??"); - gateway->relay_rtcp(publisher->session->handle, 1, buf, 20); - /* Send a PLI too, just in case... */ - janus_rtcp_pli((char *)&buf, 12); +static void janus_videoroom_reqpli(janus_videoroom_publisher *publisher, const char *reason) { + /* Send a PLI */ JANUS_LOG(LOG_VERB, "%s sending PLI to %"SCNu64" (%s)\n", reason, publisher->user_id, publisher->display ? publisher->display : "??"); - gateway->relay_rtcp(publisher->session->handle, 1, buf, 12); + gateway->send_pli(publisher->session->handle); /* Update the time of when we last sent a keyframe request */ publisher->fir_latest = janus_get_monotonic_time(); } @@ -3551,7 +3545,7 @@ static json_t *janus_videoroom_process_synchronous_request(janus_videoroom_sessi } } if(video_handle[0] > 0 || video_handle[1] > 0 || video_handle[2] > 0) { - janus_videoroom_reqfir(publisher, "New RTP forward publisher"); + janus_videoroom_reqpli(publisher, "New RTP forward publisher"); /* Done */ if(video_handle[0] > 0) { json_object_set_new(rtp_stream, "video_stream_id", json_integer(video_handle[0])); @@ -4285,7 +4279,7 @@ void janus_videoroom_setup_media(janus_plugin_session *handle) { if(s && s->feed) { janus_videoroom_publisher *p = s->feed; if(p && p->session) { - janus_videoroom_reqfir(p, "New subscriber available"); + janus_videoroom_reqpli(p, "New subscriber available"); /* Also notify event handlers */ if(notify_events && gateway->events_is_enabled()) { json_t *info = json_object(); @@ -4301,7 +4295,7 @@ void janus_videoroom_setup_media(janus_plugin_session *handle) { janus_mutex_unlock(&sessions_mutex); } -void janus_videoroom_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_videoroom_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized) || !gateway) return; janus_videoroom_session *session = (janus_videoroom_session *)handle->plugin_handle; @@ -4316,10 +4310,13 @@ void janus_videoroom_incoming_rtp(janus_plugin_session *handle, int video, char } janus_videoroom *videoroom = participant->room; + gboolean video = packet->video; + char *buf = packet->buffer; + uint16_t len = packet->length; /* In case this is an audio packet and we're doing talk detection, check the audio level extension */ if(!video && videoroom->audiolevel_event && participant->audio_active) { - int level = 0; - if(janus_rtp_header_extension_parse_audio_level(buf, len, participant->audio_level_extmap_id, &level) == 0) { + int level = packet->extensions.audio_level; + if(level != -1) { participant->audio_dBov_sum += level; participant->audio_active_packets++; participant->audio_dBov_level = level; @@ -4548,9 +4545,7 @@ void janus_videoroom_incoming_rtp(janus_plugin_session *handle, int video, char participant->remb_startup--; } JANUS_LOG(LOG_VERB, "Sending REMB (%s, %"SCNu32")\n", participant->display, bitrate); - char rtcpbuf[24]; - janus_rtcp_remb((char *)(&rtcpbuf), 24, bitrate); - gateway->relay_rtcp(handle, video, rtcpbuf, 24); + gateway->send_remb(handle, bitrate); if(participant->remb_startup == 0) participant->remb_latest = janus_get_monotonic_time(); } @@ -4575,7 +4570,7 @@ void janus_videoroom_incoming_rtp(janus_plugin_session *handle, int video, char } if((now-participant->fir_latest) >= ((gint64)videoroom->fir_freq*G_USEC_PER_SEC)) { /* FIXME We send a FIR every tot seconds */ - janus_videoroom_reqfir(participant, "Regular keyframe request"); + janus_videoroom_reqpli(participant, "Regular keyframe request"); } } } @@ -4583,7 +4578,7 @@ void janus_videoroom_incoming_rtp(janus_plugin_session *handle, int video, char janus_videoroom_publisher_dereference_nodebug(participant); } -void janus_videoroom_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_videoroom_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; janus_videoroom_session *session = (janus_videoroom_session *)handle->plugin_handle; @@ -4593,6 +4588,8 @@ void janus_videoroom_incoming_rtcp(janus_plugin_session *handle, int video, char } if(g_atomic_int_get(&session->destroyed)) return; + char *buf = packet->buffer; + uint16_t len = packet->length; if(session->participant_type == janus_videoroom_p_type_subscriber) { /* A subscriber sent some RTCP, check what it is and if we need to forward it to the publisher */ janus_videoroom_subscriber *s = (janus_videoroom_subscriber *)session->participant; @@ -4600,31 +4597,12 @@ void janus_videoroom_incoming_rtcp(janus_plugin_session *handle, int video, char return; if(!s->video) return; /* The only feedback we handle is video related anyway... */ - if(janus_rtcp_has_fir(buf, len)) { - /* We got a FIR, forward it to the publisher */ + if(janus_rtcp_has_fir(buf, len) || janus_rtcp_has_pli(buf, len)) { + /* We got a FIR or PLI, forward a PLI it to the publisher */ if(s->feed) { janus_videoroom_publisher *p = s->feed; if(p && p->session) { - char rtcpbuf[20]; - janus_rtcp_fir((char *)&rtcpbuf, 20, &p->fir_seq); - JANUS_LOG(LOG_VERB, "Got a FIR from a subscriber, forwarding it to %"SCNu64" (%s)\n", p->user_id, p->display ? p->display : "??"); - gateway->relay_rtcp(p->session->handle, 1, rtcpbuf, 20); - /* Update the time of when we last sent a keyframe request */ - p->fir_latest = janus_get_monotonic_time(); - } - } - } - if(janus_rtcp_has_pli(buf, len)) { - /* We got a PLI, forward it to the publisher */ - if(s->feed) { - janus_videoroom_publisher *p = s->feed; - if(p && p->session) { - char rtcpbuf[12]; - janus_rtcp_pli((char *)&rtcpbuf, 12); - JANUS_LOG(LOG_VERB, "Got a PLI from a subscriber, forwarding it to %"SCNu64" (%s)\n", p->user_id, p->display ? p->display : "??"); - gateway->relay_rtcp(p->session->handle, 1, rtcpbuf, 12); - /* Update the time of when we last sent a keyframe request */ - p->fir_latest = janus_get_monotonic_time(); + janus_videoroom_reqpli(p, "PLI from subscriber"); } } } @@ -4635,10 +4613,10 @@ void janus_videoroom_incoming_rtcp(janus_plugin_session *handle, int video, char } } -void janus_videoroom_incoming_data(janus_plugin_session *handle, char *label, char *buf, int len) { +void janus_videoroom_incoming_data(janus_plugin_session *handle, janus_plugin_data *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized) || !gateway) return; - if(buf == NULL || len <= 0) + if(packet->buffer == NULL || packet->length == 0) return; janus_videoroom_session *session = (janus_videoroom_session *)handle->plugin_handle; if(!session || g_atomic_int_get(&session->destroyed) || session->participant_type != janus_videoroom_p_type_publisher) @@ -4650,6 +4628,8 @@ void janus_videoroom_incoming_data(janus_plugin_session *handle, char *label, ch janus_videoroom_publisher_dereference_nodebug(participant); return; } + char *buf = packet->buffer; + uint16_t len = packet->length; /* Any forwarder involved? */ janus_mutex_lock(&participant->rtp_forwarders_mutex); /* Forward RTP to the appropriate port for the rtp_forwarders associated with this publisher, if there are any */ @@ -5582,13 +5562,11 @@ static void *janus_videoroom_handler(void *data) { /* Send a new REMB */ if(session->started) participant->remb_latest = janus_get_monotonic_time(); - char rtcpbuf[24]; - janus_rtcp_remb((char *)(&rtcpbuf), 24, participant->bitrate); - gateway->relay_rtcp(msg->handle, 1, rtcpbuf, 24); + gateway->send_remb(msg->handle, participant->bitrate); } if(keyframe && json_is_true(keyframe)) { /* Send a FIR */ - janus_videoroom_reqfir(participant, "Keyframe request"); + janus_videoroom_reqpli(participant, "Keyframe request"); } janus_mutex_lock(&participant->rec_mutex); gboolean prev_recording_active = participant->recording_active; @@ -5614,7 +5592,7 @@ static void *janus_videoroom_handler(void *data) { strstr(participant->sdp, "m=application") != NULL); if(strstr(participant->sdp, "m=video")) { /* Send a FIR */ - janus_videoroom_reqfir(participant, "Recording video"); + janus_videoroom_reqpli(participant, "Recording video"); } } } @@ -5795,7 +5773,7 @@ static void *janus_videoroom_handler(void *data) { subscriber->video = newvideo; if(subscriber->video) { /* Send a FIR */ - janus_videoroom_reqfir(publisher, "Restoring video for subscriber"); + janus_videoroom_reqpli(publisher, "Restoring video for subscriber"); } } if(data && publisher->data && subscriber->data_offered) @@ -5817,7 +5795,7 @@ static void *janus_videoroom_handler(void *data) { json_decref(event); } else { /* Send a FIR */ - janus_videoroom_reqfir(publisher, "Simulcasting substream change"); + janus_videoroom_reqpli(publisher, "Simulcasting substream change"); } } if(subscriber->feed && subscriber->feed->vcodec == JANUS_VIDEOCODEC_VP8 && @@ -5835,7 +5813,7 @@ static void *janus_videoroom_handler(void *data) { json_decref(event); } else { /* Send a FIR */ - janus_videoroom_reqfir(publisher, "Simulcasting temporal layer change"); + janus_videoroom_reqpli(publisher, "Simulcasting temporal layer change"); } } } @@ -5856,7 +5834,7 @@ static void *janus_videoroom_handler(void *data) { json_decref(event); } else if(spatial_layer != subscriber->target_spatial_layer) { /* Send a FIR to the new RTP forward publisher */ - janus_videoroom_reqfir(publisher, "Need to downscale spatially"); + janus_videoroom_reqpli(publisher, "Need to downscale spatially"); } subscriber->target_spatial_layer = spatial_layer; } @@ -6068,7 +6046,7 @@ static void *janus_videoroom_handler(void *data) { janus_mutex_unlock(&publisher->subscribers_mutex); subscriber->feed = publisher; /* Send a FIR to the new publisher */ - janus_videoroom_reqfir(publisher, "Switching existing subscriber to new publisher"); + janus_videoroom_reqpli(publisher, "Switching existing subscriber to new publisher"); /* Done */ subscriber->paused = paused; event = json_object(); @@ -6634,8 +6612,12 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) if(override_mark_bit && !has_marker_bit) { packet->data->markerbit = 1; } - if(gateway != NULL) - gateway->relay_rtp(session->handle, packet->is_video, (char *)packet->data, packet->length); + if(gateway != NULL) { + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + /* FIXME Add extensions we parsed, e.g., audio level and video orientation */ + gateway->relay_rtp(session->handle, &rtp); + } if(override_mark_bit && !has_marker_bit) { packet->data->markerbit = 0; } @@ -6655,10 +6637,7 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) subscriber->feed->session->handle) { /* Send a PLI */ JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n"); - char rtcpbuf[12]; - memset(rtcpbuf, 0, 12); - janus_rtcp_pli((char *)&rtcpbuf, 12); - gateway->relay_rtcp(subscriber->feed->session->handle, 1, rtcpbuf, 12); + gateway->send_pli(subscriber->feed->session->handle); } /* Do we need to drop this? */ if(!relay) @@ -6692,8 +6671,12 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) subscriber->sim_context.changed_substream); } /* Send the packet */ - if(gateway != NULL) - gateway->relay_rtp(session->handle, packet->is_video, (char *)packet->data, packet->length); + if(gateway != NULL) { + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + /* FIXME Add extensions we parsed, e.g., audio level and video orientation */ + gateway->relay_rtp(session->handle, &rtp); + } /* Restore the timestamp and sequence number to what the publisher set them to */ packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); @@ -6705,8 +6688,12 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) /* Fix sequence number and timestamp (publisher switching may be involved) */ janus_rtp_header_update(packet->data, &subscriber->context, TRUE, 4500); /* Send the packet */ - if(gateway != NULL) - gateway->relay_rtp(session->handle, packet->is_video, (char *)packet->data, packet->length); + if(gateway != NULL) { + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + /* FIXME Add extensions we parsed, e.g., audio level and video orientation */ + gateway->relay_rtp(session->handle, &rtp); + } /* Restore the timestamp and sequence number to what the publisher set them to */ packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); @@ -6720,8 +6707,12 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) /* Fix sequence number and timestamp (publisher switching may be involved) */ janus_rtp_header_update(packet->data, &subscriber->context, FALSE, 960); /* Send the packet */ - if(gateway != NULL) - gateway->relay_rtp(session->handle, packet->is_video, (char *)packet->data, packet->length); + if(gateway != NULL) { + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; + janus_plugin_rtp_extensions_reset(&rtp.extensions); + /* FIXME Add extensions we parsed, e.g., audio level and video orientation */ + gateway->relay_rtp(session->handle, &rtp); + } /* Restore the timestamp and sequence number to what the publisher set them to */ packet->data->timestamp = htonl(packet->timestamp); packet->data->seq_number = htons(packet->seq_number); @@ -6745,7 +6736,8 @@ static void janus_videoroom_relay_data_packet(gpointer data, gpointer user_data) } if(gateway != NULL && text != NULL) { JANUS_LOG(LOG_VERB, "Forwarding DataChannel message (%zu bytes) to viewer: %s\n", strlen(text), text); - gateway->relay_data(session->handle, NULL, text, strlen(text)); + janus_plugin_data data = { .label = NULL, .buffer = text, .length = strlen(text) }; + gateway->relay_data(session->handle, &data); } return; } @@ -6761,7 +6753,7 @@ static void janus_videoroom_rtp_forwarder_rtcp_receive(janus_videoroom_rtp_forwa /* We only handle incoming video PLIs or FIR at the moment */ if(!janus_rtcp_has_fir(buffer, len) && !janus_rtcp_has_pli(buffer, len)) return; - janus_videoroom_reqfir((janus_videoroom_publisher *)forward->source, "RTCP from forwarder"); + janus_videoroom_reqpli((janus_videoroom_publisher *)forward->source, "RTCP from forwarder"); } } diff --git a/plugins/janus_voicemail.c b/plugins/janus_voicemail.c index 5ce38d7e5e..d888fbc327 100644 --- a/plugins/janus_voicemail.c +++ b/plugins/janus_voicemail.c @@ -136,8 +136,8 @@ const char *janus_voicemail_get_package(void); void janus_voicemail_create_session(janus_plugin_session *handle, int *error); struct janus_plugin_result *janus_voicemail_handle_message(janus_plugin_session *handle, char *transaction, json_t *message, json_t *jsep); void janus_voicemail_setup_media(janus_plugin_session *handle); -void janus_voicemail_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len); -void janus_voicemail_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len); +void janus_voicemail_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet); +void janus_voicemail_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet); void janus_voicemail_hangup_media(janus_plugin_session *handle); void janus_voicemail_destroy_session(janus_plugin_session *handle, int *error); json_t *janus_voicemail_query_session(janus_plugin_session *handle); @@ -561,7 +561,7 @@ void janus_voicemail_setup_media(janus_plugin_session *handle) { janus_refcount_decrease(&session->ref); } -void janus_voicemail_incoming_rtp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_voicemail_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; janus_voicemail_session *session = (janus_voicemail_session *)handle->plugin_handle; @@ -582,6 +582,8 @@ void janus_voicemail_incoming_rtp(janus_plugin_session *handle, int video, char return; } /* Save the frame */ + char *buf = packet->buffer; + uint16_t len = packet->length; janus_rtp_header *rtp = (janus_rtp_header *)buf; uint16_t seq = ntohs(rtp->seq_number); if(session->seq == 0) @@ -600,7 +602,7 @@ void janus_voicemail_incoming_rtp(janus_plugin_session *handle, int video, char ogg_write(session); } -void janus_voicemail_incoming_rtcp(janus_plugin_session *handle, int video, char *buf, int len) { +void janus_voicemail_incoming_rtcp(janus_plugin_session *handle, janus_plugin_rtcp *packet) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized)) return; /* FIXME Should we care? */ diff --git a/plugins/plugin.c b/plugins/plugin.c index f1a9ad2ef5..308af95bf4 100644 --- a/plugins/plugin.c +++ b/plugins/plugin.c @@ -16,6 +16,7 @@ #include "../apierror.h" #include "../debug.h" +/* Plugin results */ janus_plugin_result *janus_plugin_result_new(janus_plugin_result_type type, const char *text, json_t *content) { JANUS_LOG(LOG_HUGE, "Creating plugin result...\n"); janus_plugin_result *result = g_malloc(sizeof(janus_plugin_result)); @@ -25,10 +26,6 @@ janus_plugin_result *janus_plugin_result_new(janus_plugin_result_type type, cons return result; } -/*! \brief Helper to quickly destroy a janus_plugin_result instance - * @param[in] result The janus_plugin_result instance to destroy - * @note Will decrease the reference counter of the JSON content, if available - */ void janus_plugin_result_destroy(janus_plugin_result *result) { JANUS_LOG(LOG_HUGE, "Destroying plugin result...\n"); result->text = NULL; @@ -38,3 +35,26 @@ void janus_plugin_result_destroy(janus_plugin_result *result) { g_free(result); } +/* RTP, RTCP and data packets initialization */ +void janus_plugin_rtp_extensions_reset(janus_plugin_rtp_extensions *extensions) { + if(extensions) { + /* By extensions are not added to packets */ + extensions->audio_level = -1; + extensions->audio_level_vad = FALSE; + extensions->video_orientation = -1; + } +} +void janus_plugin_rtp_reset(janus_plugin_rtp *packet) { + if(packet) { + memset(packet, 0, sizeof(janus_plugin_rtp)); + janus_plugin_rtp_extensions_reset(&packet->extensions); + } +} +void janus_plugin_rtcp_reset(janus_plugin_rtcp *packet) { + if(packet) + memset(packet, 0, sizeof(janus_plugin_rtcp)); +} +void janus_plugin_data_reset(janus_plugin_data *packet) { + if(packet) + memset(packet, 0, sizeof(janus_plugin_data)); +} diff --git a/plugins/plugin.h b/plugins/plugin.h index 9a5820fb6d..9c8bf9fcbe 100644 --- a/plugins/plugin.h +++ b/plugins/plugin.h @@ -170,7 +170,7 @@ janus_plugin *create(void) { * Janus instance or it will crash. * */ -#define JANUS_PLUGIN_API_VERSION 13 +#define JANUS_PLUGIN_API_VERSION 14 /*! \brief Initialization of all plugin properties to NULL * @@ -221,6 +221,15 @@ typedef struct janus_plugin_session janus_plugin_session; /*! \brief Result of individual requests passed to plugins */ typedef struct janus_plugin_result janus_plugin_result; +/*! \brief RTP packet exchanged with the core */ +typedef struct janus_plugin_rtp janus_plugin_rtp; +/*! \brief RTP extensions parsed in an RTP packet */ +typedef struct janus_plugin_rtp_extensions janus_plugin_rtp_extensions; +/*! \brief RTCP message exchanged with the core */ +typedef struct janus_plugin_rtcp janus_plugin_rtcp; +/*! \brief Data message exchanged with the core */ +typedef struct janus_plugin_data janus_plugin_data; + /* Use forward declaration to avoid including jansson.h */ typedef struct json_t json_t; @@ -288,25 +297,19 @@ struct janus_plugin { void (* const setup_media)(janus_plugin_session *handle); /*! \brief Method to handle an incoming RTP packet from a peer * @param[in] handle The plugin/gateway session used for this peer - * @param[in] video Whether this is an audio or a video frame - * @param[in] buf The packet data (buffer) - * @param[in] len The buffer lenght */ - void (* const incoming_rtp)(janus_plugin_session *handle, int video, char *buf, int len); + * @param[in] packet The RTP packet and related data */ + void (* const incoming_rtp)(janus_plugin_session *handle, janus_plugin_rtp *packet); /*! \brief Method to handle an incoming RTCP packet from a peer * @param[in] handle The plugin/gateway session used for this peer - * @param[in] video Whether this is related to an audio or a video stream - * @param[in] buf The message data (buffer) - * @param[in] len The buffer lenght */ - void (* const incoming_rtcp)(janus_plugin_session *handle, int video, char *buf, int len); + * @param[in] packet The RTP packet and related data */ + void (* const incoming_rtcp)(janus_plugin_session *handle, janus_plugin_rtcp *packet); /*! \brief Method to handle incoming SCTP/DataChannel data from a peer (text only, for the moment) * \note We currently only support text data, binary data will follow... please also notice that * DataChannels send unterminated strings, so you'll have to terminate them with a \0 yourself to * use them. * @param[in] handle The plugin/gateway session used for this peer - * @param[in] label The label of the data channel to use - * @param[in] buf The message data (buffer) - * @param[in] len The buffer lenght */ - void (* const incoming_data)(janus_plugin_session *handle, char *label, char *buf, int len); + * @param[in] packet The message data and related info */ + void (* const incoming_data)(janus_plugin_session *handle, janus_plugin_data *packet); /*! \brief Method to be notified by the core when too many NACKs have * been received or sent by Janus, and so a slow or potentially * unreliable network is to be expected for this peer @@ -323,7 +326,7 @@ struct janus_plugin { * @param[in] uplink Whether this is related to the uplink (Janus to peer) * or downlink (peer to Janus) * @param[in] video Whether this is related to an audio or a video stream */ - void (* const slow_link)(janus_plugin_session *handle, int uplink, int video); + void (* const slow_link)(janus_plugin_session *handle, gboolean uplink, gboolean video); /*! \brief Callback to be notified about DTLS alerts from a peer (i.e., the PeerConnection is not valid any more) * @param[in] handle The plugin/gateway session used for this peer */ void (* const hangup_media)(janus_plugin_session *handle); @@ -355,22 +358,28 @@ struct janus_callbacks { /*! \brief Callback to relay RTP packets to a peer * @param[in] handle The plugin/gateway session used for this peer - * @param[in] video Whether this is an audio or a video frame - * @param[in] buf The packet data (buffer) - * @param[in] len The buffer lenght */ - void (* const relay_rtp)(janus_plugin_session *handle, int video, char *buf, int len); + * @param[in] packet The RTP packet and related data */ + void (* const relay_rtp)(janus_plugin_session *handle, janus_plugin_rtp *packet); /*! \brief Callback to relay RTCP messages to a peer * @param[in] handle The plugin/gateway session that will be used for this peer - * @param[in] video Whether this is related to an audio or a video stream - * @param[in] buf The message data (buffer) - * @param[in] len The buffer lenght */ - void (* const relay_rtcp)(janus_plugin_session *handle, int video, char *buf, int len); + * @param[in] packet The RTCP packet and related data */ + void (* const relay_rtcp)(janus_plugin_session *handle, janus_plugin_rtcp *packet); /*! \brief Callback to relay SCTP/DataChannel messages to a peer * @param[in] handle The plugin/gateway session that will be used for this peer - * @param[in] label The label of the data channel to use - * @param[in] buf The message data (buffer) - * @param[in] len The buffer lenght */ - void (* const relay_data)(janus_plugin_session *handle, char *label, char *buf, int len); + * @param[in] packet The message data and related info */ + void (* const relay_data)(janus_plugin_session *handle, janus_plugin_data *packet); + + /*! \brief Helper to ask for a keyframe via a RTCP PLI + * @note This is a shortcut, as it is also possible to do the same by crafting + * an RTCP PLI message manually, and passing it to the core via relay_rtcp + * @param[in] handle The plugin/gateway session that will be used for this peer */ + void (* const send_pli)(janus_plugin_session *handle); + /*! \brief Helper to ask for a keyframe via a RTCP PLI + * @note This is a shortcut, as it is also possible to do the same by crafting + * an RTCP REMB message manually, and passing it to the core via relay_rtcp + * @param[in] handle The plugin/gateway session that will be used for this peer + * @param[in] bitrate The bitrate value to send in the REMB message */ + void (* const send_remb)(janus_plugin_session *handle, guint32 bitrate); /*! \brief Callback to ask the core to close a WebRTC PeerConnection * \note A call to this method will result in the core invoking the hangup_media @@ -474,4 +483,123 @@ void janus_plugin_result_destroy(janus_plugin_result *result); ///@} +/** @name Janus plugin media packets + * @brief The Janus core and plugins exchange different kind of media + * packets, specifically RTP packets, RTCP messages and datachannel data. + * While previously these were exchanged between core and plugins using + * generic pointers and their length, Janus now uses a dedicated structure + * for each of them: this allows metadata and other info to be carried + * along the media data itself, making the exchange process extensible + * as a result (the signature remains the same, the data contained in + * the struct can change). + * + * The janus_plugin_rtp structure represents an RTP packet. When creating + * a new packet, it should be initialized with janus_plugin_rtp_init. Besides + * the data and its length, it also contains info on whether the packet is + * audio or video, and a list of the parsed RTP extensions provided in + * an instance of the janus_plugin_rtp_extensions structure. Notice that, + * while this list of extensions is mostly a commodity when receiving a + * packet, making it easier to access their values (the RTP extensions + * will still be part of the incoming RTP packet, so plugins are still free + * to parse them manually), they're very important when it comes to + * outgoing packets instead: in fact, since the Janus core may needs to + * terminate its own extensions with the peer, all RTP extensions that + * are in an RTP packet sent by a plugin are stripped when relay_rtp is + * called. This means that any attempt to inject an RTP extension in an + * outgoing packet will fail if the RTP extension is added to the payload + * manually, and will need to be set in the janus_plugin_rtp_extensions + * structure instead. It's also important to initialize the extensions + * structure before passing the packet to the core, as for each extension + * there may be a different way of telling the core whether the extension + * needs to be added or not (e.f., \c -1 instead of \c 0 or \c NULL ) . + * If the RTP extension management you need is not supported, it must be + * added to the core to get it working. + * + * The janus_plugin_rtcp, instead, represents an RTCP packet, which may + * contain one or more RTCP compound messages. The only info it contains + * are whether it's related to the audio or video stream, and a pointer + * to the data itself and its length. When creating a new packet, it should + * be initialized with janus_plugin_rtcp_init. To make the generation of + * some of the most common RTCP messages easier, a few helper core + * callbacks are provided: this means that, while you can craft RTCP + * messages yourself using the methods available in rtcp.h, it might be + * easier to send, e.g., a keyframe request using the dedicated method, + * which will leave the RTCP crafting process up tp the core. + * + * Finally, the janus_plugin_data represents a datachannel message. The + * only info it contains are the label the message came from, a pointer + * to the data itself and its length. When creating a new packet, it MUST + * be initialized with janus_plugin_data_init. + * + */ +///@{ +/*! \brief Janus plugin RTP extensions */ +struct janus_plugin_rtp_extensions { + /*! \brief Audio level, in DB (0-127, 127=silence); -1 means no extension */ + int8_t audio_level; + /*! \brief Whether the encoder detected voice activity (part of audio-level extension) + * @note Browsers apparently always set this to 1, so it's unreliable and should be ignored */ + gboolean audio_level_vad; + /*! \brief Video orientation bit-mask (0 0 0 0 C F R1 R0); -1 means no extension */ + int8_t video_orientation; +}; +/*! \brief Helper method to initialise/reset the RTP extensions field + * @note This is important because each of the supported extensions may + * use a different value to specify an "extension missing" state, which + * may be different from a 0 or a NULL (e.g., a -1 instead) + * @param[in] extensions Pointer to the janus_plugin_rtp_extensions instance to reset +*/ +void janus_plugin_rtp_extensions_reset(janus_plugin_rtp_extensions *extensions); + +/*! \brief Janus plugin RTP packet */ +struct janus_plugin_rtp { + /*! \brief Whether this is an audio or video RTP packet */ + gboolean video; + /*! \brief The packet data */ + char *buffer; + /*! \brief The packet length */ + uint16_t length; + /*! \brief RTP extensions */ + janus_plugin_rtp_extensions extensions; +}; +/*! \brief Helper method to initialise/reset the RTP packet + * @note The main motivation for this method comes from the presence of the + * extensions as a janus_plugin_rtp_extensions instance. + * @param[in] packet Pointer to the janus_plugin_rtp packet to reset +*/ +void janus_plugin_rtp_reset(janus_plugin_rtp *packet); + +/*! \brief Janus plugin RTCP packet */ +struct janus_plugin_rtcp { + /*! \brief Whether this is an audio or video RTCP packet */ + gboolean video; + /*! \brief The packet data */ + char *buffer; + /*! \brief The packet length */ + uint16_t length; +}; +/*! \brief Helper method to initialise/reset the RTCP packet + * @param[in] packet Pointer to the janus_plugin_rtcp packet to reset +*/ +void janus_plugin_rtcp_reset(janus_plugin_rtcp *packet); + +/*! \brief Janus plugin data message + * @note At the moment, we only support text based datachannels. In the + * future, once we add support for binary data, this structure may be + * extended to include info on the nature of the data itself */ +struct janus_plugin_data { + /*! \brief The label this message belongs to */ + char *label; + /*! \brief The message data */ + char *buffer; + /*! \brief The message length */ + uint16_t length; +}; +/*! \brief Helper method to initialise/reset the data message + * @param[in] packet Pointer to the janus_plugin_data message to reset +*/ +void janus_plugin_data_reset(janus_plugin_data *packet); +///@} + + #endif diff --git a/rtp.c b/rtp.c index 5861541fb3..825929532d 100644 --- a/rtp.c +++ b/rtp.c @@ -174,14 +174,16 @@ static int janus_rtp_header_extension_find(char *buf, int len, int id, return -1; } -int janus_rtp_header_extension_parse_audio_level(char *buf, int len, int id, int *level) { +int janus_rtp_header_extension_parse_audio_level(char *buf, int len, int id, gboolean *vad, int *level) { uint8_t byte = 0; if(janus_rtp_header_extension_find(buf, len, id, &byte, NULL, NULL) < 0) return -1; /* a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level */ - int v = (byte & 0x80) >> 7; + gboolean v = (byte & 0x80) >> 7; int value = byte & 0x7F; JANUS_LOG(LOG_DBG, "%02x --> v=%d, level=%d\n", byte, v, value); + if(vad) + *vad = v; if(level) *level = value; return 0; diff --git a/rtp.h b/rtp.h index 0f4b445d3c..80b413d5b2 100644 --- a/rtp.h +++ b/rtp.h @@ -142,12 +142,15 @@ int janus_rtp_header_extension_get_id(const char *sdp, const char *extension); const char *janus_rtp_header_extension_get_from_id(const char *sdp, int id); /*! \brief Helper to parse a ssrc-audio-level RTP extension (https://tools.ietf.org/html/rfc6464) + * @note Browsers apparently always set the VAD to 1, so it's unreliable and should be ignored: + * only use this method if you're interested in the audio-level value itself. * @param[in] buf The packet data * @param[in] len The packet data length in bytes * @param[in] id The extension ID to look for + * @param[out] vad Whether the encoder thinks there's voice activity * @param[out] level The level value in dBov (0=max, 127=min) * @returns 0 if found, -1 otherwise */ -int janus_rtp_header_extension_parse_audio_level(char *buf, int len, int id, int *level); +int janus_rtp_header_extension_parse_audio_level(char *buf, int len, int id, gboolean *vad, int *level); /*! \brief Helper to parse a video-orientation RTP extension (http://www.3gpp.org/ftp/Specs/html-info/26114.htm) * @param[in] buf The packet data diff --git a/sdp.c b/sdp.c index 9d708c3537..83af144d38 100644 --- a/sdp.c +++ b/sdp.c @@ -59,12 +59,20 @@ janus_sdp *janus_sdp_preparse(void *ice_handle, const char *jsep_sdp, char *erro /* Found mid attribute */ if(m->type == JANUS_SDP_AUDIO && m->port > 0) { JANUS_LOG(LOG_VERB, "[%"SCNu64"] Audio mid: %s\n", handle->handle_id, a->value); + if(strlen(a->value) > 16) { + JANUS_LOG(LOG_ERR, "[%"SCNu64"] Audio mid too large: (%zu > 16)\n", handle->handle_id, strlen(a->value)); + return NULL; + } if(handle->audio_mid == NULL) handle->audio_mid = g_strdup(a->value); if(handle->stream_mid == NULL) handle->stream_mid = handle->audio_mid; } else if(m->type == JANUS_SDP_VIDEO && m->port > 0) { JANUS_LOG(LOG_VERB, "[%"SCNu64"] Video mid: %s\n", handle->handle_id, a->value); + if(strlen(a->value) > 16) { + JANUS_LOG(LOG_ERR, "[%"SCNu64"] Video mid too large: (%zu > 16)\n", handle->handle_id, strlen(a->value)); + return NULL; + } if(handle->video_mid == NULL) handle->video_mid = g_strdup(a->value); if(handle->stream_mid == NULL) @@ -283,12 +291,20 @@ int janus_sdp_process(void *ice_handle, janus_sdp *remote_sdp, gboolean update) /* Found mid attribute */ if(m->type == JANUS_SDP_AUDIO && m->port > 0) { JANUS_LOG(LOG_VERB, "[%"SCNu64"] Audio mid: %s\n", handle->handle_id, a->value); + if(strlen(a->value) > 16) { + JANUS_LOG(LOG_ERR, "[%"SCNu64"] Audio mid too large: (%zu > 16)\n", handle->handle_id, strlen(a->value)); + return -2; + } if(handle->audio_mid == NULL) handle->audio_mid = g_strdup(a->value); if(handle->stream_mid == NULL) handle->stream_mid = handle->audio_mid; } else if(m->type == JANUS_SDP_VIDEO && m->port > 0) { JANUS_LOG(LOG_VERB, "[%"SCNu64"] Video mid: %s\n", handle->handle_id, a->value); + if(strlen(a->value) > 16) { + JANUS_LOG(LOG_ERR, "[%"SCNu64"] Video mid too large: (%zu > 16)\n", handle->handle_id, strlen(a->value)); + return -2; + } if(handle->video_mid == NULL) handle->video_mid = g_strdup(a->value); if(handle->stream_mid == NULL) From 1918f9686aff9decfb067073f07aca0cc8afb298 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Tue, 3 Dec 2019 19:16:35 +0100 Subject: [PATCH 2/8] Fixed RTP extension insertion in some edge cases --- ice.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ice.c b/ice.c index 2dfb572813..6d207ef1ff 100644 --- a/ice.c +++ b/ice.c @@ -4343,15 +4343,22 @@ void janus_ice_relay_rtp(janus_ice_handle *handle, janus_plugin_rtp *packet) { if((!packet->video && !janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_HAS_AUDIO)) || (packet->video && !janus_flags_is_set(&handle->webrtc_flags, JANUS_ICE_HANDLE_WEBRTC_HAS_VIDEO))) return; + uint16_t totlen = RTP_HEADER_SIZE; + /* Check how large the payload is */ + int plen = 0; + char *payload = janus_rtp_payload(packet->buffer, packet->length, &plen); + if(payload != NULL) + totlen += plen; /* We need to strip extensions, here, and add those that need to be there manually */ - uint16_t totlen = RTP_HEADER_SIZE, extlen = 0; + uint16_t extlen = 0; char extensions[50]; janus_rtp_header *header = (janus_rtp_header *)packet->buffer; + int origext = header->extension; header->extension = 0; /* Add core and plugin extensions, if any */ if(handle->stream->mid_ext_id > 0) { header->extension = 1; - memset(extensions, sizeof(extensions), extlen); + memset(extensions, 0, sizeof(extensions)); janus_rtp_header_extension *extheader = (janus_rtp_header_extension *)extensions; extheader->type = htons(0xBEDE); extheader->length = 0; @@ -4392,11 +4399,6 @@ void janus_ice_relay_rtp(janus_ice_handle *handle, janus_plugin_rtp *packet) { extlen = 4 + (words*4); totlen += extlen; } - /* Check how large the payload is */ - int plen = 0; - char *payload = janus_rtp_payload(packet->buffer, packet->length, &plen); - if(payload != NULL) - totlen += plen; /* Queue this packet */ janus_ice_queued_packet *pkt = g_malloc(sizeof(janus_ice_queued_packet)); pkt->data = g_malloc(totlen + SRTP_MAX_TAG_LEN); @@ -4416,6 +4418,8 @@ void janus_ice_relay_rtp(janus_ice_handle *handle, janus_plugin_rtp *packet) { pkt->label = NULL; pkt->added = janus_get_monotonic_time(); janus_ice_queue_packet(handle, pkt); + /* Restore the extension flag to what the plugin set it to */ + header->extension = origext; } void janus_ice_relay_rtcp_internal(janus_ice_handle *handle, janus_plugin_rtcp *packet, gboolean filter_rtcp) { From b75bf350e068d0286c264bb2fd5eb358938107c6 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 4 Dec 2019 18:02:55 +0100 Subject: [PATCH 3/8] Added SDP utils flag to offer extensions, so plugins don't have to add them manually --- plugins/janus_audiobridge.c | 1 + plugins/janus_recordplay.c | 2 + plugins/janus_streaming.c | 4 ++ plugins/janus_videoroom.c | 52 ++++++++++---------- sdp-utils.c | 96 +++++++++++++++++++++++++++++++++++++ sdp-utils.h | 4 ++ 6 files changed, 134 insertions(+), 25 deletions(-) diff --git a/plugins/janus_audiobridge.c b/plugins/janus_audiobridge.c index 8eb1a89a02..3c43fdd427 100644 --- a/plugins/janus_audiobridge.c +++ b/plugins/janus_audiobridge.c @@ -4467,6 +4467,7 @@ static void *janus_audiobridge_handler(void *data) { /* Reject video and data channels, if offered */ JANUS_SDP_OA_VIDEO, FALSE, JANUS_SDP_OA_DATA, FALSE, + JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_MID, JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_AUDIO_LEVEL, JANUS_SDP_OA_DONE); /* Replace the session name */ diff --git a/plugins/janus_recordplay.c b/plugins/janus_recordplay.c index b3b954993e..0cdb9efa9f 100644 --- a/plugins/janus_recordplay.c +++ b/plugins/janus_recordplay.c @@ -628,10 +628,12 @@ static int janus_recordplay_generate_offer(janus_recordplay_recording *rec) { JANUS_SDP_OA_AUDIO_CODEC, janus_audiocodec_name(rec->acodec), JANUS_SDP_OA_AUDIO_PT, rec->audio_pt, JANUS_SDP_OA_AUDIO_DIRECTION, JANUS_SDP_SENDONLY, + JANUS_SDP_OA_AUDIO_EXTENSION, JANUS_RTP_EXTMAP_MID, 1, JANUS_SDP_OA_VIDEO, offer_video, JANUS_SDP_OA_VIDEO_CODEC, janus_videocodec_name(rec->vcodec), JANUS_SDP_OA_VIDEO_PT, rec->video_pt, JANUS_SDP_OA_VIDEO_DIRECTION, JANUS_SDP_SENDONLY, + JANUS_SDP_OA_VIDEO_EXTENSION, JANUS_RTP_EXTMAP_MID, 1, JANUS_SDP_OA_DATA, FALSE, JANUS_SDP_OA_DONE); g_free(rec->offer); diff --git a/plugins/janus_streaming.c b/plugins/janus_streaming.c index 0a696ae409..5031e6f363 100644 --- a/plugins/janus_streaming.c +++ b/plugins/janus_streaming.c @@ -4244,6 +4244,8 @@ static void *janus_streaming_handler(void *data) { g_strlcat(sdptemp, buffer, 2048); } g_strlcat(sdptemp, "a=sendonly\r\n", 2048); + g_snprintf(buffer, 512, "a=extmap:%d %s\r\n", 1, JANUS_RTP_EXTMAP_MID); + g_strlcat(sdptemp, buffer, 2048); } if(mp->codecs.video_pt > 0 && session->video) { /* Add video line */ @@ -4277,6 +4279,8 @@ static void *janus_streaming_handler(void *data) { mp->codecs.video_pt); g_strlcat(sdptemp, buffer, 2048); g_strlcat(sdptemp, "a=sendonly\r\n", 2048); + g_snprintf(buffer, 512, "a=extmap:%d %s\r\n", 1, JANUS_RTP_EXTMAP_MID); + g_strlcat(sdptemp, buffer, 2048); } #ifdef HAVE_SCTP if(mp->data && session->data) { diff --git a/plugins/janus_videoroom.c b/plugins/janus_videoroom.c index 6842dd413a..3f4ad8d7e8 100644 --- a/plugins/janus_videoroom.c +++ b/plugins/janus_videoroom.c @@ -6308,43 +6308,45 @@ static void *janus_videoroom_handler(void *data) { /* Generate an SDP string we can send back to the publisher */ char *answer_sdp = janus_sdp_write(answer); /* Now turn the SDP into what we'll send subscribers, using the static payload types for making switching easier */ + int mid_ext_id = 1; + while(mid_ext_id < 15) { + if(mid_ext_id != participant->audio_level_extmap_id && + mid_ext_id != participant->video_orient_extmap_id && + mid_ext_id != participant->playout_delay_extmap_id) + break; + mid_ext_id++; + } + int twcc_ext_id = 1; + while(twcc_ext_id < 15) { + if(twcc_ext_id != mid_ext_id && + twcc_ext_id != participant->audio_level_extmap_id && + twcc_ext_id != participant->video_orient_extmap_id && + twcc_ext_id != participant->playout_delay_extmap_id) + break; + twcc_ext_id++; + } offer = janus_sdp_generate_offer(s_name, answer->c_addr, JANUS_SDP_OA_AUDIO, participant->audio, JANUS_SDP_OA_AUDIO_CODEC, janus_audiocodec_name(participant->acodec), JANUS_SDP_OA_AUDIO_PT, janus_audiocodec_pt(participant->acodec), JANUS_SDP_OA_AUDIO_DIRECTION, JANUS_SDP_SENDONLY, JANUS_SDP_OA_AUDIO_FMTP, participant->do_opusfec ? "useinbandfec=1" : NULL, + JANUS_SDP_OA_AUDIO_EXTENSION, JANUS_RTP_EXTMAP_AUDIO_LEVEL, + participant->audio_level_extmap_id > 0 ? participant->audio_level_extmap_id : 0, + JANUS_SDP_OA_AUDIO_EXTENSION, JANUS_RTP_EXTMAP_MID, mid_ext_id, JANUS_SDP_OA_VIDEO, participant->video, JANUS_SDP_OA_VIDEO_CODEC, janus_videocodec_name(participant->vcodec), JANUS_SDP_OA_VIDEO_PT, janus_videocodec_pt(participant->vcodec), JANUS_SDP_OA_VIDEO_DIRECTION, JANUS_SDP_SENDONLY, + JANUS_SDP_OA_VIDEO_EXTENSION, JANUS_RTP_EXTMAP_MID, mid_ext_id, + JANUS_SDP_OA_VIDEO_EXTENSION, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION, + participant->video_orient_extmap_id > 0 ? participant->video_orient_extmap_id : 0, + JANUS_SDP_OA_VIDEO_EXTENSION, JANUS_RTP_EXTMAP_PLAYOUT_DELAY, + participant->playout_delay_extmap_id > 0 ? participant->playout_delay_extmap_id : 0, + JANUS_SDP_OA_VIDEO_EXTENSION, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC, + videoroom->transport_wide_cc_ext ? twcc_ext_id : 0, JANUS_SDP_OA_DATA, participant->data, JANUS_SDP_OA_DONE); - /* Add the extmap attributes, if needed */ - if(participant->audio_level_extmap_id > 0) { - janus_sdp_mline *m = janus_sdp_mline_find(offer, JANUS_SDP_AUDIO); - if(m != NULL) { - janus_sdp_attribute *a = janus_sdp_attribute_create("extmap", - "%d %s\r\n", participant->audio_level_extmap_id, JANUS_RTP_EXTMAP_AUDIO_LEVEL); - janus_sdp_attribute_add_to_mline(m, a); - } - } - if(participant->video_orient_extmap_id > 0) { - janus_sdp_mline *m = janus_sdp_mline_find(offer, JANUS_SDP_VIDEO); - if(m != NULL) { - janus_sdp_attribute *a = janus_sdp_attribute_create("extmap", - "%d %s\r\n", participant->video_orient_extmap_id, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION); - janus_sdp_attribute_add_to_mline(m, a); - } - } - if(participant->playout_delay_extmap_id > 0) { - janus_sdp_mline *m = janus_sdp_mline_find(offer, JANUS_SDP_VIDEO); - if(m != NULL) { - janus_sdp_attribute *a = janus_sdp_attribute_create("extmap", - "%d %s\r\n", participant->playout_delay_extmap_id, JANUS_RTP_EXTMAP_PLAYOUT_DELAY); - janus_sdp_attribute_add_to_mline(m, a); - } - } /* Is this room recorded, or are we recording this publisher already? */ janus_mutex_lock(&participant->rec_mutex); if(videoroom->record || participant->recording_active) { diff --git a/sdp-utils.c b/sdp-utils.c index e5a3b5c09c..00990ccdf7 100644 --- a/sdp-utils.c +++ b/sdp-utils.c @@ -1011,6 +1011,9 @@ janus_sdp *janus_sdp_new(const char *name, const char *address) { return sdp; } +static int janus_sdp_id_compare(gconstpointer a, gconstpointer b) { + return GPOINTER_TO_INT(a) - GPOINTER_TO_INT(b); +} janus_sdp *janus_sdp_generate_offer(const char *name, const char *address, ...) { /* This method has a variable list of arguments, telling us what we should offer */ va_list args; @@ -1022,6 +1025,8 @@ janus_sdp *janus_sdp_generate_offer(const char *name, const char *address, ...) const char *audio_codec = NULL, *video_codec = NULL, *audio_fmtp = NULL; int audio_pt = 111, video_pt = 96; janus_sdp_mdirection audio_dir = JANUS_SDP_SENDRECV, video_dir = JANUS_SDP_SENDRECV; + GHashTable *audio_extmaps = NULL, *audio_extids = NULL, + *video_extmaps = NULL, *video_extids = NULL; int property = va_arg(args, int); while(property != JANUS_SDP_OA_DONE) { if(property == JANUS_SDP_OA_AUDIO) { @@ -1052,6 +1057,45 @@ janus_sdp *janus_sdp_generate_offer(const char *name, const char *address, ...) h264_fmtp = va_arg(args, gboolean); } else if(property == JANUS_SDP_OA_DATA_LEGACY) { data_legacy = va_arg(args, gboolean); + } else if(property == JANUS_SDP_OA_AUDIO_EXTENSION || property == JANUS_SDP_OA_VIDEO_EXTENSION) { + char *extmap = va_arg(args, char *); + int id = va_arg(args, int); + if(extmap != NULL && id > 0 && id < 15) { + if(audio_extmaps == NULL) + audio_extmaps = g_hash_table_new(g_str_hash, g_str_equal); + if(audio_extids == NULL) + audio_extids = g_hash_table_new(NULL, NULL); + if(video_extmaps == NULL) + video_extmaps = g_hash_table_new(g_str_hash, g_str_equal); + if(video_extids == NULL) + video_extids = g_hash_table_new(NULL, NULL); + /* Make sure the extmap and ID have not been added already */ + char *audio_extmap = g_hash_table_lookup(audio_extids, GINT_TO_POINTER(id)); + char *video_extmap = g_hash_table_lookup(video_extids, GINT_TO_POINTER(id)); + if((property == JANUS_SDP_OA_VIDEO_EXTENSION && audio_extmap != NULL && strcasecmp(audio_extmap, extmap)) || + (property == JANUS_SDP_OA_AUDIO_EXTENSION && video_extmap != NULL && strcasecmp(video_extmap, extmap))) { + JANUS_LOG(LOG_WARN, "Ignoring duplicate extension %d (already added: %s)\n", + id, audio_extmap ? audio_extmap : video_extmap); + } else { + if(property == JANUS_SDP_OA_AUDIO_EXTENSION) { + if(g_hash_table_lookup(audio_extmaps, extmap) != NULL) { + JANUS_LOG(LOG_WARN, "Ignoring duplicate audio extension %s (already added: %d)\n", + extmap, GPOINTER_TO_INT(g_hash_table_lookup(audio_extmaps, extmap))); + } else { + g_hash_table_insert(audio_extmaps, extmap, GINT_TO_POINTER(id)); + g_hash_table_insert(audio_extids, GINT_TO_POINTER(id), extmap); + } + } else { + if(g_hash_table_lookup(video_extmaps, extmap) != NULL) { + JANUS_LOG(LOG_WARN, "Ignoring duplicate video extension %s (already added: %d)\n", + extmap, GPOINTER_TO_INT(g_hash_table_lookup(video_extmaps, extmap))); + } else { + g_hash_table_insert(video_extmaps, extmap, GINT_TO_POINTER(id)); + g_hash_table_insert(video_extids, GINT_TO_POINTER(id), extmap); + } + } + } + } } else { JANUS_LOG(LOG_WARN, "Unknown property %d for preparing SDP answer, ignoring...\n", property); } @@ -1063,6 +1107,14 @@ janus_sdp *janus_sdp_generate_offer(const char *name, const char *address, ...) if(do_audio && audio_rtpmap == NULL) { JANUS_LOG(LOG_ERR, "Unsupported audio codec '%s', can't prepare an offer\n", audio_codec); va_end(args); + if(audio_extmaps != NULL) + g_hash_table_destroy(audio_extmaps); + if(audio_extids != NULL) + g_hash_table_destroy(audio_extids); + if(video_extmaps != NULL) + g_hash_table_destroy(video_extmaps); + if(video_extids != NULL) + g_hash_table_destroy(video_extids); return NULL; } if(video_codec == NULL) @@ -1071,6 +1123,14 @@ janus_sdp *janus_sdp_generate_offer(const char *name, const char *address, ...) if(do_video && video_rtpmap == NULL) { JANUS_LOG(LOG_ERR, "Unsupported video codec '%s', can't prepare an offer\n", video_codec); va_end(args); + if(audio_extmaps != NULL) + g_hash_table_destroy(audio_extmaps); + if(audio_extids != NULL) + g_hash_table_destroy(audio_extids); + if(video_extmaps != NULL) + g_hash_table_destroy(video_extmaps); + if(video_extids != NULL) + g_hash_table_destroy(video_extids); return NULL; } #ifndef HAVE_SCTP @@ -1101,6 +1161,20 @@ janus_sdp *janus_sdp_generate_offer(const char *name, const char *address, ...) janus_sdp_attribute *a = janus_sdp_attribute_create("fmtp", "%d %s", audio_pt, audio_fmtp); m->attributes = g_list_append(m->attributes, a); } + /* Check if we need to add audio extensions to the SDP */ + if(audio_extids != NULL) { + GList *ids = g_list_sort(g_hash_table_get_keys(audio_extids), janus_sdp_id_compare), *iter = ids; + while(iter) { + char *extmap = g_hash_table_lookup(audio_extids, iter->data); + if(extmap != NULL) { + janus_sdp_attribute *a = janus_sdp_attribute_create("extmap", + "%d %s\r\n", GPOINTER_TO_INT(iter->data), extmap); + janus_sdp_attribute_add_to_mline(m, a); + } + iter = iter->next; + } + g_list_free(ids); + } offer->m_lines = g_list_append(offer->m_lines, m); } if(do_video) { @@ -1127,6 +1201,20 @@ janus_sdp *janus_sdp_generate_offer(const char *name, const char *address, ...) a = janus_sdp_attribute_create("rtcp-fb", "%d goog-remb", video_pt); m->attributes = g_list_append(m->attributes, a); } + /* Check if we need to add audio extensions to the SDP */ + if(video_extids != NULL) { + GList *ids = g_list_sort(g_hash_table_get_keys(video_extids), janus_sdp_id_compare), *iter = ids; + while(iter) { + char *extmap = g_hash_table_lookup(video_extids, iter->data); + if(extmap != NULL) { + janus_sdp_attribute *a = janus_sdp_attribute_create("extmap", + "%d %s\r\n", GPOINTER_TO_INT(iter->data), extmap); + janus_sdp_attribute_add_to_mline(m, a); + } + iter = iter->next; + } + g_list_free(ids); + } offer->m_lines = g_list_append(offer->m_lines, m); } if(do_data) { @@ -1145,6 +1233,14 @@ janus_sdp *janus_sdp_generate_offer(const char *name, const char *address, ...) } offer->m_lines = g_list_append(offer->m_lines, m); } + if(audio_extmaps != NULL) + g_hash_table_destroy(audio_extmaps); + if(audio_extids != NULL) + g_hash_table_destroy(audio_extids); + if(video_extmaps != NULL) + g_hash_table_destroy(video_extmaps); + if(video_extids != NULL) + g_hash_table_destroy(video_extids); /* Done */ va_end(args); diff --git a/sdp-utils.h b/sdp-utils.h index 34b79a250a..ccb6490f8a 100644 --- a/sdp-utils.h +++ b/sdp-utils.h @@ -274,6 +274,10 @@ JANUS_SDP_OA_VIDEO_RTCPFB_DEFAULTS, JANUS_SDP_OA_VIDEO_H264_FMTP, /*! \brief When generating an offer (this is ignored for answers), use the old "DTLS/SCTP" instead of the new "UDP/DTLS/SCTP (default=TRUE for now, depends on what follows) */ JANUS_SDP_OA_DATA_LEGACY, +/*! \brief When generating an offer (this is ignored for answers), negotiate this audio extension: needs two arguments, extmap value and extension ID; can be used multiple times) */ +JANUS_SDP_OA_AUDIO_EXTENSION, +/*! \brief When generating an offer (this is ignored for answers), negotiate this video extension: needs two arguments, extmap value and extension ID; can be used multiple times) */ +JANUS_SDP_OA_VIDEO_EXTENSION, /*! \brief When generating an answer (this is ignored for offers), accept this extension (by default, we reject them all; can be used multiple times) */ JANUS_SDP_OA_ACCEPT_EXTMAP, /*! \brief MUST be used as the last argument in janus_sdp_generate_offer and janus_sdp_generate_answer */ From ffec3c6c840ec1f868ea88f9eb2935b763995ab9 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 5 Dec 2019 18:39:59 +0100 Subject: [PATCH 4/8] Support for transport-wide CC on outgoing streams --- ice.c | 18 +++++- ice.h | 2 + janus.c | 8 ++- rtcp.c | 165 +++++++++++++++++++++++++++++++++++++++++++--------- rtcp.h | 3 + rtp.c | 17 +++++- rtp.h | 13 ++++- sdp-utils.c | 9 ++- 8 files changed, 199 insertions(+), 36 deletions(-) diff --git a/ice.c b/ice.c index 6d207ef1ff..584aaffb62 100644 --- a/ice.c +++ b/ice.c @@ -3499,7 +3499,7 @@ static gint rtcp_transport_wide_cc_stats_comparator(gconstpointer item1, gconstp static gboolean janus_ice_outgoing_transport_wide_cc_feedback(gpointer user_data) { janus_ice_handle *handle = (janus_ice_handle *)user_data; janus_ice_stream *stream = handle->stream; - if(stream && stream->do_transport_wide_cc) { + if(stream && stream->video_send && stream->do_transport_wide_cc) { /* Create a transport wide feedback message */ size_t size = 1300; char rtcpbuf[1300]; @@ -4087,6 +4087,14 @@ static gboolean janus_ice_outgoing_traffic_handle(janus_ice_handle *handle, janu /* ... but only if this isn't a retransmission (for those we already set it before) */ header->ssrc = htonl(video ? stream->video_ssrc : stream->audio_ssrc); } + /* Set the transport-wide sequence number, if needed */ + if(video && stream->transport_wide_cc_ext_id > 0) { + stream->transport_wide_cc_out_seq_num++; + if(janus_rtp_header_extension_set_transport_wide_cc(pkt->data, pkt->length, + stream->transport_wide_cc_ext_id, stream->transport_wide_cc_out_seq_num) < 0) { + JANUS_LOG(LOG_ERR, "[%"SCNu64"] Error setting transport wide CC sequence number...\n", handle->handle_id); + } + } /* Keep track of payload types too */ if(!video && stream->audio_payload_type < 0) { stream->audio_payload_type = header->type; @@ -4364,6 +4372,14 @@ void janus_ice_relay_rtp(janus_ice_handle *handle, janus_plugin_rtp *packet) { extheader->length = 0; /* Iterate on all extensions we need */ char *index = extensions + 4; + /* Check if we need to add the transport-wide CC extension */ + if(packet->video && handle->stream->transport_wide_cc_ext_id > 0) { + *index = (handle->stream->transport_wide_cc_ext_id << 4) + 1; + /* We'll actually set the sequence number later, when sending the packet */ + memset(index+1, 0, 2); + index += 3; + extlen += 3; + } /* Check if we need to add the mid extension */ if(handle->stream->mid_ext_id > 0) { char *mid = packet->video ? handle->video_mid : handle->audio_mid; diff --git a/ice.h b/ice.h index e507f1fcb5..7f02df94af 100644 --- a/ice.h +++ b/ice.h @@ -414,6 +414,8 @@ struct janus_ice_stream { gboolean do_transport_wide_cc; /*! \brief Transport wide cc rtp ext ID */ gint transport_wide_cc_ext_id; + /*! \brief Last sent transport wide seq num */ + guint16 transport_wide_cc_out_seq_num; /*! \brief Last received transport wide seq num */ guint32 transport_wide_cc_last_seq_num; /*! \brief Last transport wide seq num sent on feedback */ diff --git a/janus.c b/janus.c index 2b7f26a972..54870e22f6 100644 --- a/janus.c +++ b/janus.c @@ -3379,7 +3379,7 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug } } /* Make sure we don't send the rid/repaired-rid attributes when offering ourselves */ - int mid_ext_id = 0, audiolevel_ext_id = 0, videoorientation_ext_id = 0; + int mid_ext_id = 0, transport_wide_cc_ext_id = 0, audiolevel_ext_id = 0, videoorientation_ext_id = 0; GList *temp = parsed_sdp->m_lines; while(temp) { janus_sdp_mline *m = (janus_sdp_mline *)temp->data; @@ -3389,6 +3389,8 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug if(a->name && a->value) { if(strstr(a->value, JANUS_RTP_EXTMAP_MID)) mid_ext_id = atoi(a->value); + else if(strstr(a->value, JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC)) + transport_wide_cc_ext_id = atoi(a->value); else if(strstr(a->value, JANUS_RTP_EXTMAP_AUDIO_LEVEL)) audiolevel_ext_id = atoi(a->value); else if(strstr(a->value, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION)) @@ -3407,6 +3409,10 @@ json_t *janus_plugin_handle_sdp(janus_plugin_session *plugin_session, janus_plug } if(ice_handle->stream && ice_handle->stream->mid_ext_id != mid_ext_id) ice_handle->stream->mid_ext_id = mid_ext_id; + if(ice_handle->stream && ice_handle->stream->transport_wide_cc_ext_id != transport_wide_cc_ext_id) { + ice_handle->stream->do_transport_wide_cc = transport_wide_cc_ext_id > 0 ? TRUE : FALSE; + ice_handle->stream->transport_wide_cc_ext_id = transport_wide_cc_ext_id; + } if(ice_handle->stream && ice_handle->stream->audiolevel_ext_id != audiolevel_ext_id) ice_handle->stream->audiolevel_ext_id = audiolevel_ext_id; if(ice_handle->stream && ice_handle->stream->videoorientation_ext_id != videoorientation_ext_id) diff --git a/rtcp.c b/rtcp.c index 4391e5f7a8..3c0fa1ef24 100644 --- a/rtcp.c +++ b/rtcp.c @@ -22,6 +22,26 @@ #include "rtcp.h" #include "utils.h" + +/* Transport CC statuses */ +typedef enum janus_rtp_packet_status { + janus_rtp_packet_status_notreceived = 0, + janus_rtp_packet_status_smalldelta = 1, + janus_rtp_packet_status_largeornegativedelta = 2, + janus_rtp_packet_status_reserved = 3 +} janus_rtp_packet_status; +static const char *janus_rtp_packet_status_description(janus_rtp_packet_status status) { + switch(status) { + case janus_rtp_packet_status_notreceived: return "notreceived"; + case janus_rtp_packet_status_smalldelta: return "smalldelta"; + case janus_rtp_packet_status_largeornegativedelta: return "largeornegativedelta"; + case janus_rtp_packet_status_reserved: return "reserved"; + default: break; + } + return NULL; +} + + gboolean janus_is_rtcp(char *buf, guint len) { if (len < 8) return FALSE; @@ -134,7 +154,7 @@ guint32 janus_rtcp_get_receiver_ssrc(char *packet, int len) { return 0; } -/* Helper to handle an incoming SR: triggered by a call to janus_rtcp_fix_ssrc with fixssrc=0 */ +/* Helper to handle an incoming SR: triggered by a call to janus_rtcp_fix_ssrc with a valid context pointer */ static void janus_rtcp_incoming_sr(janus_rtcp_context *ctx, janus_rtcp_sr *sr) { if(ctx == NULL) return; @@ -146,6 +166,103 @@ static void janus_rtcp_incoming_sr(janus_rtcp_context *ctx, janus_rtcp_sr *sr) { ctx->lsr = (ntp >> 16); } +/* Helper to handle an incoming transport-cc feedback: triggered by a call to janus_rtcp_fix_ssrc a valid context pointer */ +static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_rtcp_fb *twcc, int total) { + if(ctx == NULL || twcc == NULL || total < 16) + return; + /* Parse the header first */ + uint8_t *data = (uint8_t *)twcc->fci; + uint16_t base_seq = 0, status_count = 0; + uint32_t reference = 0; + uint8_t fb_pkt = 0; + memcpy(&base_seq, data, sizeof(uint16_t)); + base_seq = ntohs(base_seq); + memcpy(&status_count, data+2, sizeof(uint16_t)); + status_count = ntohs(status_count); + memcpy(&reference, data+4, sizeof(uint32_t)); + reference = ntohl(reference) >> 8; + fb_pkt = *(data+7); + JANUS_LOG(LOG_HUGE, "[TWCC] seq=%"SCNu16", psc=%"SCNu16", ref=%"SCNu32", fbpc=%"SCNu8"\n", + base_seq, status_count, reference, fb_pkt); + /* Now traverse the feedback: packet chunks first, and then recv deltas */ + total -= 16; + data += 8; + int psc = status_count; + uint16_t chunk = 0; + uint8_t t = 0, ss = 0, s = 0, length = 0; + /* Iterate on all packet chunks */ + JANUS_LOG(LOG_HUGE, "[TWCC] Chunks:\n"); + int num = 0; + GList *list = NULL; + while(psc > 0 && total > 0) { + num++; + memcpy(&chunk, data, sizeof(uint16_t)); + chunk = ntohs(chunk); + t = (chunk & 0x8000) >> 15; + if(t == 0) { + /* Run length */ + s = (chunk & 0x6000) >> 13; + length = (chunk & 0x1FFF); + JANUS_LOG(LOG_HUGE, " [%d] t=run-length, s=%s, l=%"SCNu16"\n", num, + janus_rtp_packet_status_description(s), length); + while(length > 0 && psc > 0) { + list = g_list_append(list, GUINT_TO_POINTER(s)); + length--; + psc--; + } + } else { + /* Status vector */ + ss = (chunk & 0x4000) >> 14; + length = (s ? 7 : 14); + JANUS_LOG(LOG_HUGE, " [%d] t=status-vector, ss=%s, l=%"SCNu16"\n", num, + s ? "2-bit" : "bit", length); + while(length > 0 && psc > 0) { + if(!ss) + s = (chunk & (1 << (length-1))) ? janus_rtp_packet_status_notreceived : janus_rtp_packet_status_reserved; + else + s = (chunk & (3 << (2*length-2))) >> (2*length-2); + list = g_list_append(list, GUINT_TO_POINTER(s)); + length--; + psc--; + } + } + total -= 2; + data += 2; + } + /* Iterate on all recv deltas */ + JANUS_LOG(LOG_HUGE, "[TWCC] Recv Deltas (%d/%"SCNu16"):\n", g_list_length(list), status_count); + num = 0; + uint16_t delta = 0; + uint32_t delta_us = 0; + GList *iter = list; + while(iter != NULL && total > 0) { + num++; + delta = 0; + s = GPOINTER_TO_UINT(iter->data); + if(s == janus_rtp_packet_status_smalldelta) { + /* Small delta = 1 byte */ + delta = *data; + total--; + data++; + } else if(s == janus_rtp_packet_status_largeornegativedelta) { + /* Large or negative delta = 2 bytes */ + if(total < 2) + break; + memcpy(&delta, data, sizeof(uint16_t)); + delta = ntohs(delta); + total -= 2; + data += 2; + } + delta_us = delta*250; + /* Print summary */ + JANUS_LOG(LOG_HUGE, " [%02d][%"SCNu16"] %s (%"SCNu32"us)\n", num, base_seq+num-1, + janus_rtp_packet_status_description(s), delta_us); + iter = iter->next; + } + /* TODO Update the context with the feedback we got */ + g_list_free(list); +} + /* Link quality estimate filter coefficient */ #define LINK_QUALITY_FILTER_K 3.0 @@ -438,6 +555,9 @@ int janus_rtcp_fix_ssrc(janus_rtcp_context *ctx, char *packet, int len, int fixs uint32_t *ssrc = (uint32_t *)rtcpfb->fci; *ssrc = htonl(newssrcr); } + } else if(fmt == 15) { /* transport-cc */ + /* If an RTCP context was provided, parse this transport-cc feedback */ + janus_rtcp_incoming_transport_cc(ctx, rtcpfb, total); } else { JANUS_LOG(LOG_HUGE, " #%d ??? -- RTPFB (205, fmt=%d)\n", pno, fmt); } @@ -1368,13 +1488,6 @@ int janus_rtcp_nacks(char *packet, int len, GSList *nacks) { return words*4+4; } -typedef enum janus_rtp_packet_status { - janus_rtp_packet_status_notreceived = 0, - janus_rtp_packet_status_smalldelta = 1, - janus_rtp_packet_status_largeornegativedelta = 2, - janus_rtp_packet_status_reserved = 3 -} janus_rtp_packet_status; - int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssrc, guint32 media, guint8 feedback_packet_count, GQueue *transport_wide_cc_stats) { if(packet == NULL || size < sizeof(janus_rtcp_header) || transport_wide_cc_stats == NULL || g_queue_is_empty(transport_wide_cc_stats)) return -1; @@ -1401,11 +1514,11 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr /* 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | base sequence number | packet status count | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | reference time | fb pkt. count | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | base sequence number | packet status count | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | reference time | fb pkt. count | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ */ /* The packet as unsigned */ @@ -1473,7 +1586,7 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr timestamp = stat->timestamp; } - /* Check if all previoues ones were equal and this one the firt different */ + /* Check if all previoues ones were equal and this one the first different */ if (all_same && last_status!=janus_rtp_packet_status_reserved && status!=last_status) { /* How big was the same run */ if (g_queue_get_length(statuses)>7) { @@ -1482,9 +1595,9 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr /* 0 1 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - |T| S | Run Length | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |T| S | Run Length | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ T = 0 */ word = janus_push_bits(word, 1, 0); @@ -1516,7 +1629,7 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr /* Store las status */ last_status = status; - /* Check if we can still be enquing for a run */ + /* Check if we can still be enqueuing for a run */ if (!all_same) { /* Check */ if (!all_same && max_status==janus_rtp_packet_status_largeornegativedelta && g_queue_get_length(statuses)>6) { @@ -1524,9 +1637,9 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr /* 0 1 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - |T|S| Symbols | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |T|S| Symbols | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ T = 1 S = 1 */ @@ -1570,11 +1683,11 @@ int janus_rtcp_transport_wide_cc_feedback(char *packet, size_t size, guint32 ssr /* 0 1 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - |T|S| symbol list | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - T = 1 - S = 0 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |T|S| symbol list | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + T = 1 + S = 0 */ word = janus_push_bits(word, 1, 1); word = janus_push_bits(word, 1, 0); diff --git a/rtcp.h b/rtcp.h index cbda8586e7..96ab8abb40 100644 --- a/rtcp.h +++ b/rtcp.h @@ -276,6 +276,9 @@ typedef struct rtcp_context double in_media_link_quality; double out_link_quality; double out_media_link_quality; + + /* TODO Incoming transport-wide CC feedback*/ + } rtcp_context; typedef rtcp_context janus_rtcp_context; diff --git a/rtp.c b/rtp.c index 825929532d..1e463c8b2c 100644 --- a/rtp.c +++ b/rtp.c @@ -303,14 +303,27 @@ int janus_rtp_header_extension_parse_transport_wide_cc(char *buf, int len, int i if(ext == NULL) return -2; int val_len = (*ext & 0x0F) + 1; - if (val_len < 2 || val_len > len-(ext-buf)-1) { + if (val_len < 2 || val_len > len-(ext-buf)-1) return -3; - } memcpy(transSeqNum, ext+1, sizeof(uint16_t)); *transSeqNum = ntohs(*transSeqNum); return 0; } +int janus_rtp_header_extension_set_transport_wide_cc(char *buf, int len, int id, uint16_t transSeqNum) { + char *ext = NULL; + if(janus_rtp_header_extension_find(buf, len, id, NULL, NULL, &ext) < 0) + return -1; + if(ext == NULL) + return -2; + int val_len = (*ext & 0x0F) + 1; + if (val_len < 2 || val_len > len-(ext-buf)-1) + return -3; + transSeqNum = htons(transSeqNum); + memcpy(ext+1, &transSeqNum, sizeof(uint16_t)); + return 0; +} + int janus_rtp_header_extension_replace_id(char *buf, int len, int id, int new_id) { if(!buf || len < 12) return -1; diff --git a/rtp.h b/rtp.h index 80b413d5b2..1931406b6d 100644 --- a/rtp.h +++ b/rtp.h @@ -208,10 +208,17 @@ int janus_rtp_header_extension_parse_framemarking(char *buf, int len, int id, ja * @param[in] buf The packet data * @param[in] len The packet data length in bytes * @param[in] id The extension ID to look for - * @param[out] transSeqNum transport wide sequence number + * @param[out] transSeqNum Variable to read the transport wide sequence number in * @returns 0 if found, -1 otherwise */ -int janus_rtp_header_extension_parse_transport_wide_cc(char *buf, int len, int id, - uint16_t *transSeqNum); +int janus_rtp_header_extension_parse_transport_wide_cc(char *buf, int len, int id, uint16_t *transSeqNum); + +/*! \brief Helper to set a transport wide sequence number (https://tools.ietf.org/html/draft-holmer-rmcat-transport-wide-cc-extensions-01) + * @param[in] buf The packet data + * @param[in] len The packet data length in bytes + * @param[in] id The extension ID to look for + * @param[out] transSeqNum Transport wide sequence number to set + * @returns 0 if found, -1 otherwise */ +int janus_rtp_header_extension_set_transport_wide_cc(char *buf, int len, int id, uint16_t transSeqNum); /*! \brief Helper to replace the ID of an RTP extension with a different one (e.g., * to turn a repaired-rtp-stream-id into a rtp-stream-id after a successful rtx) diff --git a/sdp-utils.c b/sdp-utils.c index 00990ccdf7..27b179d3f7 100644 --- a/sdp-utils.c +++ b/sdp-utils.c @@ -1175,6 +1175,9 @@ janus_sdp *janus_sdp_generate_offer(const char *name, const char *address, ...) } g_list_free(ids); } + /* It is safe to add transport-wide rtcp feedback mesage here, won't be used unless the header extension is negotiated*/ + a = janus_sdp_attribute_create("rtcp-fb", "%d transport-cc", video_pt); + m->attributes = g_list_append(m->attributes, a); offer->m_lines = g_list_append(offer->m_lines, m); } if(do_video) { @@ -1505,10 +1508,10 @@ janus_sdp *janus_sdp_generate_answer(janus_sdp *offer, ...) { am->attributes = g_list_append(am->attributes, a); a = janus_sdp_attribute_create("rtcp-fb", "%d goog-remb", pt); am->attributes = g_list_append(am->attributes, a); - /* It is safe to add transport-wide rtcp feedback mesage here, won't be used unless the header extension is negotiated*/ - a = janus_sdp_attribute_create("rtcp-fb", "%d transport-cc", pt); - am->attributes = g_list_append(am->attributes, a); } + /* It is safe to add transport-wide rtcp feedback mesage here, won't be used unless the header extension is negotiated*/ + a = janus_sdp_attribute_create("rtcp-fb", "%d transport-cc", pt); + am->attributes = g_list_append(am->attributes, a); } } /* Add the extmap attributes, if needed */ From c8fc34de936951e2da84e00058b42b373aaa3dee Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Mon, 9 Dec 2019 16:59:41 +0100 Subject: [PATCH 5/8] Fixed inverted management of received/not-received vector --- rtcp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rtcp.c b/rtcp.c index 3c0fa1ef24..8364e3352c 100644 --- a/rtcp.c +++ b/rtcp.c @@ -203,7 +203,7 @@ static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_rtcp /* Run length */ s = (chunk & 0x6000) >> 13; length = (chunk & 0x1FFF); - JANUS_LOG(LOG_HUGE, " [%d] t=run-length, s=%s, l=%"SCNu16"\n", num, + JANUS_LOG(LOG_HUGE, " [%d] t=run-length, s=%s, l=%"SCNu16"\n", num, janus_rtp_packet_status_description(s), length); while(length > 0 && psc > 0) { list = g_list_append(list, GUINT_TO_POINTER(s)); @@ -218,7 +218,7 @@ static void janus_rtcp_incoming_transport_cc(janus_rtcp_context *ctx, janus_rtcp s ? "2-bit" : "bit", length); while(length > 0 && psc > 0) { if(!ss) - s = (chunk & (1 << (length-1))) ? janus_rtp_packet_status_notreceived : janus_rtp_packet_status_reserved; + s = (chunk & (1 << (length-1))) ? janus_rtp_packet_status_smalldelta : janus_rtp_packet_status_notreceived; else s = (chunk & (3 << (2*length-2))) >> (2*length-2); list = g_list_append(list, GUINT_TO_POINTER(s)); From 4ac2791d7e5e04e04d67a99ccb0959573a198f7b Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 19 Dec 2019 12:32:32 +0100 Subject: [PATCH 6/8] Made video-orientation extension info more developer friendly --- ice.c | 37 +++++++++++++++++++++++++++++++++---- plugins/plugin.c | 4 +++- plugins/plugin.h | 10 ++++++++-- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/ice.c b/ice.c index 0dbfba351d..9170ae7c40 100644 --- a/ice.c +++ b/ice.c @@ -2480,7 +2480,15 @@ static void janus_ice_cb_nice_recv(NiceAgent *agent, guint stream_id, guint comp gboolean c = FALSE, f = FALSE, r1 = FALSE, r0 = FALSE; if(janus_rtp_header_extension_parse_video_orientation(buf, buflen, stream->videoorientation_ext_id, &c, &f, &r1, &r0) == 0) { - rtp.extensions.video_orientation = (c<<3) + (f<<2) + (r1<<1) + r0; + rtp.extensions.video_rotation = 0; + if(r1 && r0) + rtp.extensions.video_rotation = 270; + else if(r1) + rtp.extensions.video_rotation = 180; + else if(r0) + rtp.extensions.video_rotation = 90; + rtp.extensions.video_back_camera = c; + rtp.extensions.video_flipped = f; } } /* Pass the packet to the plugin */ @@ -4378,17 +4386,38 @@ void janus_ice_relay_rtp(janus_ice_handle *handle, janus_plugin_rtp *packet) { } } /* Check if the plugin (or source) included other extensions */ - if(packet->extensions.audio_level != -1 && handle->stream->audiolevel_ext_id > 0) { + if(!packet->video && packet->extensions.audio_level != -1 && handle->stream->audiolevel_ext_id > 0) { /* Add audio-level extension */ *index = (handle->stream->audiolevel_ext_id << 4); *(index+1) = (packet->extensions.audio_level_vad << 7) + (packet->extensions.audio_level & 0x7F); index += 2; extlen += 2; } - if(packet->extensions.video_orientation != -1 && handle->stream->videoorientation_ext_id > 0) { + if(packet->video && packet->extensions.video_rotation != -1 && handle->stream->videoorientation_ext_id > 0) { /* Add video-orientation extension */ *index = (handle->stream->videoorientation_ext_id << 4); - *(index+1) = packet->extensions.video_orientation; + gboolean c = packet->extensions.video_back_camera, + f = packet->extensions.video_flipped, r1 = FALSE, r0 = FALSE; + switch(packet->extensions.video_rotation) { + case 270: + r1 = TRUE; + r0 = TRUE; + break; + case 180: + r1 = TRUE; + r0 = FALSE; + break; + case 90: + r1 = FALSE; + r0 = TRUE; + break; + case 0: + default: + r1 = FALSE; + r0 = FALSE; + break; + } + *(index+1) = (c<<3) + (f<<2) + (r1<<1) + r0; index += 2; extlen += 2; } diff --git a/plugins/plugin.c b/plugins/plugin.c index 308af95bf4..22a578258e 100644 --- a/plugins/plugin.c +++ b/plugins/plugin.c @@ -41,7 +41,9 @@ void janus_plugin_rtp_extensions_reset(janus_plugin_rtp_extensions *extensions) /* By extensions are not added to packets */ extensions->audio_level = -1; extensions->audio_level_vad = FALSE; - extensions->video_orientation = -1; + extensions->video_rotation = -1; + extensions->video_back_camera = FALSE; + extensions->video_flipped = FALSE; } } void janus_plugin_rtp_reset(janus_plugin_rtp *packet) { diff --git a/plugins/plugin.h b/plugins/plugin.h index 23aef17141..88b5870434 100644 --- a/plugins/plugin.h +++ b/plugins/plugin.h @@ -540,8 +540,14 @@ struct janus_plugin_rtp_extensions { /*! \brief Whether the encoder detected voice activity (part of audio-level extension) * @note Browsers apparently always set this to 1, so it's unreliable and should be ignored */ gboolean audio_level_vad; - /*! \brief Video orientation bit-mask (0 0 0 0 C F R1 R0); -1 means no extension */ - int8_t video_orientation; + /*! \brief Video orientation rotation (0, 90, 180, 270); -1 means no extension */ + int16_t video_rotation; + /*! \brief Whether the video orientation extension says this is the back camera + * @note Will be ignored if no rotation value is set */ + gboolean video_back_camera; + /*! \brief Whether the video orientation extension says it's flipped horizontally + * @note Will be ignored if no rotation value is set */ + gboolean video_flipped; }; /*! \brief Helper method to initialise/reset the RTP extensions field * @note This is important because each of the supported extensions may From f6468810b375bd53f455f1e79b262894cee75629 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 19 Dec 2019 12:32:58 +0100 Subject: [PATCH 7/8] Relay RTP extensions info through VideoRoom as well --- plugins/janus_videoroom.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/plugins/janus_videoroom.c b/plugins/janus_videoroom.c index 90fc51d6f6..615305fe9a 100644 --- a/plugins/janus_videoroom.c +++ b/plugins/janus_videoroom.c @@ -1539,6 +1539,8 @@ typedef struct janus_videoroom_rtp_relay_packet { uint32_t ssrc[3]; uint32_t timestamp; uint16_t seq_number; + /* Extensions to add, if any */ + janus_plugin_rtp_extensions extensions; /* The following are only relevant if we're doing VP9 SVC*/ gboolean svc; janus_vp9_svc_info svc_info; @@ -4298,7 +4300,7 @@ void janus_videoroom_setup_media(janus_plugin_session *handle) { janus_mutex_unlock(&sessions_mutex); } -void janus_videoroom_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *packet) { +void janus_videoroom_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *pkt) { if(handle == NULL || g_atomic_int_get(&handle->stopped) || g_atomic_int_get(&stopping) || !g_atomic_int_get(&initialized) || !gateway) return; janus_videoroom_session *session = (janus_videoroom_session *)handle->plugin_handle; @@ -4313,12 +4315,12 @@ void janus_videoroom_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp } janus_videoroom *videoroom = participant->room; - gboolean video = packet->video; - char *buf = packet->buffer; - uint16_t len = packet->length; + gboolean video = pkt->video; + char *buf = pkt->buffer; + uint16_t len = pkt->length; /* In case this is an audio packet and we're doing talk detection, check the audio level extension */ if(!video && videoroom->audiolevel_event && participant->audio_active) { - int level = packet->extensions.audio_level; + int level = pkt->extensions.audio_level; if(level != -1) { participant->audio_dBov_sum += level; participant->audio_active_packets++; @@ -4503,6 +4505,7 @@ void janus_videoroom_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp janus_videoroom_rtp_relay_packet packet; packet.data = rtp; packet.length = len; + packet.extensions = pkt->extensions; packet.is_rtp = TRUE; packet.is_video = video; packet.svc = FALSE; @@ -6620,9 +6623,8 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) packet->data->markerbit = 1; } if(gateway != NULL) { - janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; - janus_plugin_rtp_extensions_reset(&rtp.extensions); - /* FIXME Add extensions we parsed, e.g., audio level and video orientation */ + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length, + .extensions = packet->extensions }; gateway->relay_rtp(session->handle, &rtp); } if(override_mark_bit && !has_marker_bit) { @@ -6679,9 +6681,8 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) } /* Send the packet */ if(gateway != NULL) { - janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; - janus_plugin_rtp_extensions_reset(&rtp.extensions); - /* FIXME Add extensions we parsed, e.g., audio level and video orientation */ + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length, + .extensions = packet->extensions }; gateway->relay_rtp(session->handle, &rtp); } /* Restore the timestamp and sequence number to what the publisher set them to */ @@ -6696,9 +6697,8 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) janus_rtp_header_update(packet->data, &subscriber->context, TRUE, 4500); /* Send the packet */ if(gateway != NULL) { - janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; - janus_plugin_rtp_extensions_reset(&rtp.extensions); - /* FIXME Add extensions we parsed, e.g., audio level and video orientation */ + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length, + .extensions = packet->extensions }; gateway->relay_rtp(session->handle, &rtp); } /* Restore the timestamp and sequence number to what the publisher set them to */ @@ -6715,9 +6715,8 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data) janus_rtp_header_update(packet->data, &subscriber->context, FALSE, 960); /* Send the packet */ if(gateway != NULL) { - janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length }; - janus_plugin_rtp_extensions_reset(&rtp.extensions); - /* FIXME Add extensions we parsed, e.g., audio level and video orientation */ + janus_plugin_rtp rtp = { .video = packet->is_video, .buffer = (char *)packet->data, .length = packet->length, + .extensions = packet->extensions }; gateway->relay_rtp(session->handle, &rtp); } /* Restore the timestamp and sequence number to what the publisher set them to */ From a367ecf31d7d951921078594ef576f581465868c Mon Sep 17 00:00:00 2001 From: Lejla Date: Mon, 20 Jan 2020 18:50:26 +0100 Subject: [PATCH 8/8] added video orientation and audio level extensions (#1928) --- plugins/janus_sip.c | 51 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/plugins/janus_sip.c b/plugins/janus_sip.c index 139db76dbb..c083ba3fa1 100644 --- a/plugins/janus_sip.c +++ b/plugins/janus_sip.c @@ -943,6 +943,8 @@ typedef struct janus_sip_media { janus_rtp_switching_context context; int pipefd[2]; gboolean updated; + int video_orientation_extension_id; + int audio_level_extension_id; } janus_sip_media; typedef struct janus_sip_session { @@ -1350,6 +1352,8 @@ static void janus_sip_media_reset(janus_sip_session *session) { session->media.video_pt_name = NULL; /* Immutable string, no need to free*/ session->media.video_send = TRUE; session->media.pre_hold_video_dir = JANUS_SDP_DEFAULT; + session->media.video_orientation_extension_id = -1; + session->media.audio_level_extension_id = -1; janus_rtp_switching_context_reset(&session->media.context); } @@ -1940,6 +1944,8 @@ void janus_sip_create_session(janus_plugin_session *handle, int *error) { session->media.video_pt_name = NULL; session->media.video_send = TRUE; session->media.pre_hold_video_dir = JANUS_SDP_DEFAULT; + session->media.video_orientation_extension_id = -1; + session->media.audio_level_extension_id = -1; /* Initialize the RTP context */ janus_rtp_switching_context_reset(&session->media.context); session->media.pipefd[0] = -1; @@ -3166,6 +3172,11 @@ static void *janus_sip_handler(void *data) { if(offer_srtp) { JANUS_LOG(LOG_VERB, "Going to negotiate SDES-SRTP (%s)...\n", require_srtp ? "mandatory" : "optional"); } + + /* Get video-orientation extension id from SDP we got */ + session->media.video_orientation_extension_id = janus_rtp_header_extension_get_id(msg_sdp, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION); + /* Get audio-level extension id from SDP we got */ + session->media.audio_level_extension_id = janus_rtp_header_extension_get_id(msg_sdp, JANUS_RTP_EXTMAP_AUDIO_LEVEL); /* Parse the SDP we got, manipulate some things, and generate a new one */ char sdperror[100]; janus_sdp *parsed_sdp = janus_sdp_parse(msg_sdp, sdperror, sizeof(sdperror)); @@ -3424,6 +3435,11 @@ static void *janus_sip_handler(void *data) { if(answer_srtp) { JANUS_LOG(LOG_VERB, "Going to negotiate SDES-SRTP (%s)...\n", session->media.require_srtp ? "mandatory" : "optional"); } + + /* Get video-orientation extension id from SDP we got */ + session->media.video_orientation_extension_id = janus_rtp_header_extension_get_id(msg_sdp, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION); + /* Get audio-level extension id from SDP we got */ + session->media.audio_level_extension_id = janus_rtp_header_extension_get_id(msg_sdp, JANUS_RTP_EXTMAP_AUDIO_LEVEL); /* Parse the SDP we got, manipulate some things, and generate a new one */ char sdperror[100]; janus_sdp *parsed_sdp = janus_sdp_parse(msg_sdp, sdperror, sizeof(sdperror)); @@ -3566,6 +3582,11 @@ static void *janus_sip_handler(void *data) { g_snprintf(error_cause, 512, "[SIP-%s] SDP type %s is incompatible with session status %s\n", session->account.username, msg_sdp_type, janus_sip_call_status_string(session->status)); goto error; } + + /* Get video-orientation extension id from SDP we got */ + session->media.video_orientation_extension_id = janus_rtp_header_extension_get_id(msg_sdp, JANUS_RTP_EXTMAP_VIDEO_ORIENTATION); + /* Get audio-level extension id from SDP we got */ + session->media.audio_level_extension_id = janus_rtp_header_extension_get_id(msg_sdp, JANUS_RTP_EXTMAP_AUDIO_LEVEL); /* Parse the SDP we got, manipulate some things, and generate a new one */ char sdperror[100]; janus_sdp *parsed_sdp = janus_sdp_parse(msg_sdp, sdperror, sizeof(sdperror)); @@ -6005,6 +6026,18 @@ static void *janus_sip_relay_thread(void *data) { /* Relay to application */ janus_plugin_rtp rtp = { .video = FALSE, .buffer = buffer, .length = bytes }; janus_plugin_rtp_extensions_reset(&rtp.extensions); + + /* Add audio-level extension */ + if(session->media.audio_level_extension_id != -1) { + gboolean vad = FALSE; + int level = -1; + if(janus_rtp_header_extension_parse_audio_level(buffer, bytes, + session->media.audio_level_extension_id, &vad, &level) == 0) { + rtp.extensions.audio_level = level; + rtp.extensions.audio_level_vad = vad; + } + } + gateway->relay_rtp(session->handle, &rtp); continue; } else if(session->media.audio_rtcp_fd != -1 && fds[i].fd == session->media.audio_rtcp_fd) { @@ -6073,6 +6106,24 @@ static void *janus_sip_relay_thread(void *data) { /* Relay to application */ janus_plugin_rtp rtp = { .video = TRUE, .buffer = buffer, .length = bytes }; janus_plugin_rtp_extensions_reset(&rtp.extensions); + + /* Add video-orientation extension */ + if(session->media.video_orientation_extension_id > 0) { + gboolean c = FALSE, f = FALSE, r1 = FALSE, r0 = FALSE; + if (janus_rtp_header_extension_parse_video_orientation(buffer, bytes, + session->media.video_orientation_extension_id, &c, &f, &r1, &r0) == 0) { + rtp.extensions.video_rotation = 0; + if (r1 && r0) + rtp.extensions.video_rotation = 270; + else if (r1) + rtp.extensions.video_rotation = 180; + else if (r0) + rtp.extensions.video_rotation = 90; + rtp.extensions.video_back_camera = c; + rtp.extensions.video_flipped = f; + } + } + gateway->relay_rtp(session->handle, &rtp); continue; } else if(session->media.video_rtcp_fd != -1 && fds[i].fd == session->media.video_rtcp_fd) {