Skip to content

Commit

Permalink
Moving FECEncoder into separate file
Browse files Browse the repository at this point in the history
  • Loading branch information
buldo committed Nov 16, 2023
1 parent 5f48b49 commit e409d92
Show file tree
Hide file tree
Showing 8 changed files with 154 additions and 124 deletions.
1 change: 1 addition & 0 deletions WBLib.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ target_sources(wifibroadcast PRIVATE
${CMAKE_CURRENT_LIST_DIR}/src/external/radiotap/radiotap.c
${CMAKE_CURRENT_LIST_DIR}/src/external/fec/fec_base.cpp
${CMAKE_CURRENT_LIST_DIR}/src/fec/FECDecoder.cpp
${CMAKE_CURRENT_LIST_DIR}/src/fec/FECEncoder.cpp
${CMAKE_CURRENT_LIST_DIR}/src/fec/FECStream.cpp
${CMAKE_CURRENT_LIST_DIR}/src/fec/RxBlock.cpp
${CMAKE_CURRENT_LIST_DIR}/src/WBStreamRx.cpp
Expand Down
2 changes: 2 additions & 0 deletions executables/benchmark.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
#include "../src/HelperSources/RandomBufferPot.hpp"
#include "../src/HelperSources/SchedulingHelper.hpp"
#include "../src/external/fec/fec_base.h"
#include "../src/fec/FECEncoder.hpp"
#include "../src/fec/FECConstants.hpp"

// Test the FEC encoding / decoding and Encryption / Decryption performance
// (throughput) of this system
Expand Down
1 change: 1 addition & 0 deletions executables/unit_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include "../src/Ieee80211Header.hpp"
#include "../src/wifibroadcast_spdlog.hpp"
#include "../src/fec/FECDecoder.hpp"
#include "../src/fec/FECEncoder.hpp"

// Simple unit testing for the FEC lib that doesn't require wifi cards

Expand Down
1 change: 1 addition & 0 deletions src/WBStreamTx.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include "SimpleStream.hpp"
#include "HelperSources/TimeHelper.hpp"
#include "WBTxRx.h"
#include "fec/FECEncoder.hpp"

/**
* Transmitter for a (multiplexed) wifbroadcast stream
Expand Down
102 changes: 102 additions & 0 deletions src/fec/FECEncoder.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
#include "FECEncoder.hpp"

#include <cassert>
#include <chrono>
#include <cstring>
#include <memory>

#include "../external/fec/fec_base.h"

#include "FECConstants.hpp"

void FECEncoder::encode_block(
std::vector<std::shared_ptr<std::vector<uint8_t>>> data_packets,
int n_secondary_fragments) {
assert(data_packets.size() <= MAX_N_P_FRAGMENTS_PER_BLOCK);
assert(n_secondary_fragments <= MAX_N_S_FRAGMENTS_PER_BLOCK);
const auto n_primary_fragments = data_packets.size();
// nice to have statistic
m_block_sizes.add(n_primary_fragments);
if (m_block_sizes.get_delta_since_last_reset() >= std::chrono::seconds(1)) {
// wifibroadcast::log::get_default()->debug("Block sizes:
// {}",m_block_sizes.getAvgReadable());
m_curr_fec_block_sizes = m_block_sizes.getMinMaxAvg();
m_block_sizes.reset();
}
FECPayloadHdr header{};
header.block_idx = m_curr_block_idx;
m_curr_block_idx++;
header.n_primary_fragments = n_primary_fragments;
// write and forward all the data packets first
// also calculate the size of the biggest data packet
size_t max_packet_size = 0;
// Store a pointer where the FEC data begins for performing the FEC step later
// on
std::vector<const uint8_t*> primary_fragments_data_p;
for (int i = 0; i < data_packets.size(); i++) {
const auto& data_fragment = data_packets[i];
// wifibroadcast::log::get_default()->debug("In:{}",(int)data_fragment->size());
assert(!data_fragment->empty());
assert(data_fragment->size() <= FEC_PACKET_MAX_PAYLOAD_SIZE);
header.fragment_idx = i;
header.data_size = data_fragment->size();
auto buffer_p = m_block_buffer[i].data();
// copy over the header
memcpy(buffer_p, (uint8_t*)&header, sizeof(FECPayloadHdr));
// write the actual data
memcpy(buffer_p + sizeof(FECPayloadHdr), data_fragment->data(),
data_fragment->size());
// zero out the remaining bytes such that FEC always sees zeroes
// same is done on the rx. These zero bytes are never transmitted via wifi
const auto writtenDataSize = sizeof(FECPayloadHdr) + data_fragment->size();
memset(buffer_p + writtenDataSize, 0,
MAX_PAYLOAD_BEFORE_FEC - writtenDataSize);
max_packet_size = std::max(max_packet_size, data_fragment->size());
// we can forward the data packet immediately via the callback
if (outputDataCallback) {
outputDataCallback(buffer_p, writtenDataSize);
}
// NOTE: FECPayloadHdr::data_size needs to be included during the fec encode
// step
primary_fragments_data_p.push_back(buffer_p + sizeof(FECPayloadHdr) -
sizeof(uint16_t));
}
// then we create as many FEC packets as needed
if (n_secondary_fragments == 0) {
// wifibroadcast::log::get_default()->debug("No FEC step performed");
// no FEC step is actually performed, usefully for debugging / performance
// evaluation
return;
}
const auto before = std::chrono::steady_clock::now();
// Now we perform the actual FEC encode step
std::vector<uint8_t*> secondary_fragments_data_p;
for (int i = 0; i < n_secondary_fragments; i++) {
auto fragment_index = i + n_primary_fragments;
auto buffer_p = m_block_buffer[fragment_index].data();
header.fragment_idx = fragment_index;
// copy over the header
memcpy(buffer_p, (uint8_t*)&header, sizeof(FECPayloadHdr));
// where the FEC packet correction data is written to
secondary_fragments_data_p.push_back(buffer_p + sizeof(FECPayloadHdr) -
sizeof(uint16_t));
}
fec_encode2(max_packet_size + sizeof(uint16_t), primary_fragments_data_p,
secondary_fragments_data_p);
m_fec_block_encode_time.add(std::chrono::steady_clock::now() - before);
if (m_fec_block_encode_time.get_delta_since_last_reset() >=
std::chrono::seconds(1)) {
// wifibroadcast::log::get_default()->debug("FEC encode
// time:{}",m_fec_block_encode_time.getAvgReadable());
m_curr_fec_block_encode_time = m_fec_block_encode_time.getMinMaxAvg();
m_fec_block_encode_time.reset();
}
// and forward all the FEC correction packets
for (int i = 0; i < n_secondary_fragments; i++) {
auto fragment_index = i + n_primary_fragments;
if (outputDataCallback) {
outputDataCallback(m_block_buffer[fragment_index].data(),
sizeof(FECPayloadHdr) + max_packet_size);
}
}
}
44 changes: 44 additions & 0 deletions src/fec/FECEncoder.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#ifndef FEC_ENCODER_HPP
#define FEC_ENCODER_HPP

#include <functional>
#include <memory>

#include "FECConstants.hpp"
#include "TimeHelper.hpp"

class FECEncoder {
public:
typedef std::function<void(const uint8_t* packet, int packet_len)>
OUTPUT_DATA_CALLBACK;
OUTPUT_DATA_CALLBACK outputDataCallback;
explicit FECEncoder() = default;
FECEncoder(const FECEncoder& other) = delete;

public:
/**
* Encodes a new block and forwards the packets for this block
* forwards data packets first, then generated fec packets
* (if needed) and forwards them after.
* @param data_packets the packets for this block
* @param n_secondary_fragments how many secondary fragments (FEC packets)
* should be created
*/
void encode_block(
std::vector<std::shared_ptr<std::vector<uint8_t>>> data_packets,
int n_secondary_fragments);
// Pre-allocated to have space for storing primary fragments (they are needed
// once the fec step needs to be performed) and creating the wanted amount of
// secondary packets
std::array<std::array<uint8_t, MAX_PAYLOAD_BEFORE_FEC>,
MAX_TOTAL_FRAGMENTS_PER_BLOCK>
m_block_buffer{};
uint32_t m_curr_block_idx = 0;
static_assert(sizeof(m_curr_block_idx) == sizeof(FECPayloadHdr::block_idx));
AvgCalculator m_fec_block_encode_time;
MinMaxAvg<std::chrono::nanoseconds> m_curr_fec_block_encode_time{};
BaseAvgCalculator<uint16_t> m_block_sizes{};
MinMaxAvg<uint16_t> m_curr_fec_block_sizes{};
};

#endif // FEC_ENCODER_HPP
81 changes: 0 additions & 81 deletions src/fec/FECStream.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,87 +10,6 @@
#include "RxBlock.hpp"
#include "../wifibroadcast_spdlog.hpp"
#include <spdlog/spdlog.h>
#include <functional>

void FECEncoder::encode_block(
std::vector<std::shared_ptr<std::vector<uint8_t>>> data_packets,
int n_secondary_fragments) {
assert(data_packets.size()<=MAX_N_P_FRAGMENTS_PER_BLOCK);
assert(n_secondary_fragments<=MAX_N_S_FRAGMENTS_PER_BLOCK);
const auto n_primary_fragments=data_packets.size();
// nice to have statistic
m_block_sizes.add(n_primary_fragments);
if(m_block_sizes.get_delta_since_last_reset()>=std::chrono::seconds(1)){
//wifibroadcast::log::get_default()->debug("Block sizes: {}",m_block_sizes.getAvgReadable());
m_curr_fec_block_sizes=m_block_sizes.getMinMaxAvg();
m_block_sizes.reset();
}
FECPayloadHdr header{};
header.block_idx=m_curr_block_idx;
m_curr_block_idx++;
header.n_primary_fragments=n_primary_fragments;
// write and forward all the data packets first
// also calculate the size of the biggest data packet
size_t max_packet_size=0;
// Store a pointer where the FEC data begins for performing the FEC step later on
std::vector<const uint8_t *> primary_fragments_data_p;
for(int i=0;i<data_packets.size();i++){
const auto& data_fragment=data_packets[i];
//wifibroadcast::log::get_default()->debug("In:{}",(int)data_fragment->size());
assert(!data_fragment->empty());
assert(data_fragment->size()<=FEC_PACKET_MAX_PAYLOAD_SIZE);
header.fragment_idx=i;
header.data_size=data_fragment->size();
auto buffer_p=m_block_buffer[i].data();
// copy over the header
memcpy(buffer_p,(uint8_t*)&header,sizeof(FECPayloadHdr));
// write the actual data
memcpy(buffer_p + sizeof(FECPayloadHdr), data_fragment->data(),data_fragment->size());
// zero out the remaining bytes such that FEC always sees zeroes
// same is done on the rx. These zero bytes are never transmitted via wifi
const auto writtenDataSize = sizeof(FECPayloadHdr) + data_fragment->size();
memset(buffer_p + writtenDataSize, 0, MAX_PAYLOAD_BEFORE_FEC - writtenDataSize);
max_packet_size = std::max(max_packet_size, data_fragment->size());
// we can forward the data packet immediately via the callback
if(outputDataCallback){
outputDataCallback(buffer_p,writtenDataSize);
}
// NOTE: FECPayloadHdr::data_size needs to be included during the fec encode step
primary_fragments_data_p.push_back(buffer_p+sizeof(FECPayloadHdr)-sizeof(uint16_t));
}
// then we create as many FEC packets as needed
if(n_secondary_fragments==0){
//wifibroadcast::log::get_default()->debug("No FEC step performed");
// no FEC step is actually performed, usefully for debugging / performance evaluation
return ;
}
const auto before=std::chrono::steady_clock::now();
// Now we perform the actual FEC encode step
std::vector<uint8_t*> secondary_fragments_data_p;
for(int i=0;i<n_secondary_fragments;i++){
auto fragment_index=i+n_primary_fragments;
auto buffer_p=m_block_buffer[fragment_index].data();
header.fragment_idx=fragment_index;
// copy over the header
memcpy(buffer_p,(uint8_t*)&header,sizeof(FECPayloadHdr));
// where the FEC packet correction data is written to
secondary_fragments_data_p.push_back(buffer_p+sizeof(FECPayloadHdr)-sizeof(uint16_t));
}
fec_encode2(max_packet_size+sizeof(uint16_t),primary_fragments_data_p,secondary_fragments_data_p);
m_fec_block_encode_time.add(std::chrono::steady_clock::now()-before);
if(m_fec_block_encode_time.get_delta_since_last_reset()>=std::chrono::seconds(1)){
//wifibroadcast::log::get_default()->debug("FEC encode time:{}",m_fec_block_encode_time.getAvgReadable());
m_curr_fec_block_encode_time=m_fec_block_encode_time.getMinMaxAvg();
m_fec_block_encode_time.reset();
}
// and forward all the FEC correction packets
for(int i=0;i<n_secondary_fragments;i++){
auto fragment_index=i+n_primary_fragments;
if(outputDataCallback){
outputDataCallback(m_block_buffer[fragment_index].data(),sizeof(FECPayloadHdr)+max_packet_size);
}
}
}

uint32_t calculate_n_secondary_fragments(uint32_t n_primary_fragments,
uint32_t fec_overhead_perc) {
Expand Down
46 changes: 3 additions & 43 deletions src/fec/FECStream.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,63 +5,23 @@
#ifndef WIFIBROADCAST_FECSTREAM_H
#define WIFIBROADCAST_FECSTREAM_H

#include <array>
#include <cassert>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <vector>
#include <functional>


#include "../HelperSources/TimeHelper.hpp"
#include "FECPayloadHdr.hpp"
#include "RxBlock.hpp"
#include "FECConstants.hpp"
#include <cstdint>

/**
* For dynamic block sizes, we switched to a FEC overhead "percentage" value.
* e.g. the final data throughput ~= original data throughput * fec overhead percentage
* Rounds up / down (.5), but always at least 1
*/
uint32_t calculate_n_secondary_fragments(uint32_t n_primary_fragments,uint32_t fec_overhead_perc);
uint32_t calculate_n_secondary_fragments(uint32_t n_primary_fragments, uint32_t fec_overhead_perc);

/**
* calculate n from k and percentage as used in FEC terms
* (k: number of primary fragments, n: primary + secondary fragments)
*/
unsigned int calculateN(unsigned int k, unsigned int percentage);
uint32_t calculateN(uint32_t k, uint32_t percentage);

void fec_stream_print_fec_optimization_method();

class FECEncoder {
public:
typedef std::function<void(const uint8_t* packet,int packet_len)>
OUTPUT_DATA_CALLBACK;
OUTPUT_DATA_CALLBACK outputDataCallback;
explicit FECEncoder()=default;
FECEncoder(const FECEncoder &other) = delete;
public:
/**
* Encodes a new block and forwards the packets for this block
* forwards data packets first, then generated fec packets
* (if needed) and forwards them after.
* @param data_packets the packets for this block
* @param n_secondary_fragments how many secondary fragments (FEC packets) should be created
*/
void encode_block(std::vector<std::shared_ptr<std::vector<uint8_t>>> data_packets,int n_secondary_fragments);
// Pre-allocated to have space for storing primary fragments (they are needed once the fec step needs to be performed)
// and creating the wanted amount of secondary packets
std::array<std::array<uint8_t, MAX_PAYLOAD_BEFORE_FEC>,MAX_TOTAL_FRAGMENTS_PER_BLOCK> m_block_buffer{};
uint32_t m_curr_block_idx=0;
static_assert(sizeof(m_curr_block_idx)==sizeof(FECPayloadHdr::block_idx));
AvgCalculator m_fec_block_encode_time;
MinMaxAvg<std::chrono::nanoseconds> m_curr_fec_block_encode_time{};
BaseAvgCalculator<uint16_t> m_block_sizes{};
MinMaxAvg<uint16_t> m_curr_fec_block_sizes{};
};

// quick math regarding sequence numbers:
//uint32_t holds max 4294967295 . At 10 000 pps (packets per seconds) (which is already completely out of reach) this allows the tx to run for 429496.7295 seconds
// 429496.7295 / 60 / 60 = 119.304647083 hours which is also completely overkill for OpenHD (and after this time span, a "reset" of the sequence number happens anyways)
Expand Down

0 comments on commit e409d92

Please sign in to comment.