transcode to memory

[andreavaresio]

Is there a way to set up a muxer recipient to be memory instead of file?

I need the transcoded and muxed output in a memory buffer rather than written to file.

Thanks

Andrea

[mohamedAlaaK]

i have the same issue. i did the greater part where i could get the data into memory but i can't get the header itself.

I've do some kind of thing for audio decoding of (xbox) XMA2 multistream from and to memory. First, you need to write your own AVIOContext for the input :

static const int kBufferSize = XMA_BYTES_PER_BLOCK;

class MemoryInputAVIOContext
{
public:
	MemoryInputAVIOContext(MemoryInputStream* inputStream)
		: _inputStream(inputStream)
		, _buffer_size(kBufferSize)
		, _buffer(static_cast<unsigned char*>(::av_malloc(_buffer_size)))
	{
		_ctx = ::avio_alloc_context(_buffer, _buffer_size, 0, this,
									&MemoryInputAVIOContext::read,
									NULL, // write
									&MemoryInputAVIOContext::seek);
	}

	void reset_inner_context()
	{
		_ctx = NULL;
		_buffer = NULL;
	}

	static int read(void* opaque, unsigned char* buf, int buf_size)
	{
		auto ptr = static_cast<MemoryInputAVIOContext*>(opaque);
		return ptr->_inputStream->read(buf, buf_size);
	}

	static int64_t seek(void* opaque, int64_t offset, int whence)
	{
		auto ptr = static_cast<MemoryInputAVIOContext*>(opaque);
		if (0x10000 == whence)
			return ptr->_inputStream->getTotalLength();
		else
		{
			// need manage whence
			ptr->_inputStream->setPosition(offset);
			return ptr->_inputStream->getPosition();
		}
	}

	::AVIOContext* get_avio()
	{
		return _ctx;
	}

private:
	MemoryInputStream* _inputStream;
	int _buffer_size;
	unsigned char* _buffer;
	::AVIOContext* _ctx;

	MemoryInputAVIOContext(MemoryInputAVIOContext const&);
	MemoryInputAVIOContext& operator=(MemoryInputAVIOContext const&);
};

Here the context is just a wrapper around a Juce's (C++ Framework) MemoryInputStream instance that is just a stream that read a MemoryBlock that contain my audio file datas. For a simple input, you need to provide two (static) callback functions to the constructor (read & seek), but you can add your own 'write' callback too.

For the output of the decoder, i've juste create a FrameSink instance that record the ffmpeg packet to a raw AudioSampleBuffer (always a Juce class), inspired by the audio decode demo :

class RawAudioMemorySink : public AudioFrameSink, public FrameWriter
{
public:
	RawAudioMemorySink(AudioSampleBuffer& audio)
		: buffer(audio)
		, offset(0)
	{}

	~RawAudioMemorySink()
	{
		buffer.setSize(buffer.getNumChannels(), offset, true);
	}

	FrameSinkStream* CreateStream() override
	{
		stream = new FrameSinkStream(this, 0);
		return stream;
	}

	void WriteFrame(int streamIndex, AVFrame* frame, StreamData* metaData) override
	{
		int data_size = av_get_bytes_per_sample((AVSampleFormat)frame->format);
                // assume that data_size of 4 bytes is my float audio samples

		for (auto channel = 0; channel < frame->channels; ++channel)
			buffer.copyFrom(channel, offset, (const float*)frame->data[channel], frame->nb_samples);

		offset += frame->nb_samples;
	}

	void Close(int streamIndex) override
	{
		delete stream;
	}

	bool IsPrimed() override
	{
		return true;
	}

private:
	AudioSampleBuffer& buffer;
	FrameSinkStream* stream;
	int offset;
};

Now this is the main class that use both of the two previous classes, inspired by the Demuxer class, I just play around the audio stream but you can adapt the idea to a video/audio demuxer in memory.

class XMADecoder : public ffmpegcpp::InputSource
{
public:
	XMADecoder(MemoryInputStream* inputStream, AudioSampleBuffer& output)
		: private_context(inputStream)
		, context(::avformat_alloc_context())
		, frameSink(output)
	{
		context->pb = private_context.get_avio();

		int err = avformat_open_input(&context, "arbitrarytext", NULL, NULL);
		if (err < 0)
		{
			cleanup();
			throw FFmpegException("Failed to open memory input stream");
		}

		err = avformat_find_stream_info(context, NULL);
		if (err < 0)
		{
			cleanup();
			throw FFmpegException("Failed to read memory input stream");
		}

		inputStreams = new ffmpegcpp::InputStream*[context->nb_streams];
		for (int i = 0; i < context->nb_streams; ++i)
		{
			inputStreams[i] = nullptr;
		}

		// initialize packet, set data to NULL, let the demuxer fill it
		pkt = av_packet_alloc();
		if (!pkt)
		{
			cleanup();
			throw FFmpegException("Failed to create packet for input stream");
		}
		av_init_packet(pkt);
		pkt->data = NULL;
		pkt->size = 0;
	}

	~XMADecoder()
	{
		cleanup();
	}

	void init()
	{
		int streamIndex = av_find_best_stream(context, AVMEDIA_TYPE_AUDIO, -1, -1, NULL, 0);
		if (streamIndex < 0)
			throw FFmpegException("That stream is already tied to a frame sink, "
								  "you cannot process the same stream multiple times");

		init(streamIndex, &frameSink);
	}

private:
	AVPacket* pkt = nullptr;
	AVFormatContext* context = nullptr;
	MemoryInputAVIOContext private_context;
	RawAudioMemorySink frameSink;

	bool done = false;
	ffmpegcpp::InputStream** inputStreams = nullptr;

	void cleanup()
	{
		if (inputStreams != nullptr)
		{
			for (int i = 0; i < context->nb_streams; ++i)
			{
				if (inputStreams[i] != nullptr)
				{
					delete inputStreams[i];
				}
			}
			delete inputStreams;
			inputStreams = nullptr;
		}
		if (context != nullptr)
		{
			avformat_close_input(&context);
			context = nullptr;
		}
		if (pkt != nullptr)
		{
			av_packet_free(&pkt);
			pkt = nullptr;
		}
	}

	void init(int streamIndex, FrameSink* frameSink)
	{
		if (inputStreams[streamIndex] != nullptr)
			throw FFmpegException("That stream is already tied to a frame sink, you cannot process the same stream multiple times");
	
		// create the stream
		auto inputStream = getInputStream(streamIndex);
		inputStream->Open(frameSink);

		// remember and return
		inputStreams[streamIndex] = inputStream;
	}

	ffmpegcpp::InputStream* getInputStream(int streamIndex)
	{
		// already exists
		if (inputStreams[streamIndex] != nullptr) 
			return inputStreams[streamIndex];

		// The stream doesn't exist but we already processed all our frames, so it makes no sense
		// to add it anymore.
		if (IsDone()) 
			return nullptr;

		AVStream* stream = context->streams[streamIndex];
		AVCodec* codec = CodecDeducer::DeduceDecoder(stream->codecpar->codec_id);
		
		if (codec == nullptr) 
			return nullptr; // no codec found - we can't really do anything with this stream!

		switch (codec->type)
		{
			case AVMEDIA_TYPE_VIDEO:
				inputStreams[streamIndex] = new VideoInputStream(context, stream);
				break;
			case AVMEDIA_TYPE_AUDIO:
				inputStreams[streamIndex] = new AudioInputStream(context, stream);
				break;
		}

		// return the created stream
		return inputStreams[streamIndex];
	}

public:
	void PreparePipeline() override
	{
		bool allPrimed = false;
		do
		{
			Step();

			// see if all input streams are primed
			allPrimed = true;
			for (int i = 0; i < context->nb_streams; ++i)
			{
				auto stream = inputStreams[i];
				if (stream != nullptr)
				{
					if (!stream->IsPrimed()) 
						allPrimed = false;
				}
			}

		} 
		while (!allPrimed && !IsDone());
	}

	bool IsDone() override
	{
		return done;
	}

	void Step() override
	{
		// read frames from the file
		int ret = av_read_frame(context, pkt);

		// EOF
		if (ret == AVERROR_EOF)
		{
			pkt->data = NULL;
			pkt->size = 0;
			for (int i = 0; i < context->nb_streams; ++i)
			{
				auto stream = inputStreams[i];
				if (stream != nullptr)
				{
					pkt->stream_index = i;
					DecodePacket();
					stream->Close();
				}
			}

			done = true;
			return;
		}

		// not ready yet
		if (ret == AVERROR(EAGAIN)) return;

		// error
		if (ret < 0)
			throw FFmpegException("Error during demuxing", ret);

		// decode the finished packet
		DecodePacket();
	}

private:
	void DecodePacket()
	{
		int streamIndex = pkt->stream_index;
		auto inputStream = inputStreams[streamIndex];

		if (inputStream != nullptr)
			inputStream->DecodePacket(pkt);

		// We need to unref the packet here because packets might pass by here
		// that don't have a stream attached to them. We want to dismiss them!
		av_packet_unref(pkt);
	}
};

The most important part come in the constructor :

XMADecoder(MemoryInputStream* inputStream, AudioSampleBuffer& output)
		: private_context(inputStream)
		, context(::avformat_alloc_context())
		, frameSink(output)
	{
		context->pb = private_context.get_avio();

		int err = avformat_open_input(&context, "arbitrarytext", NULL, NULL);

       [...]

private:
	AVPacket* pkt = nullptr;
	AVFormatContext* context = nullptr;
	MemoryInputAVIOContext private_context;
	RawAudioMemorySink frameSink;

I provide first my Juce's input stream, and a reference to the audio output buffer. You just put you private context to the main allocated context and initialize avformat simply like in this example.

A simple init method is in public to allow to initialize your private FrameSink (RawAudioMemorySink for here), with the right audio stream index.

The usage is more than simple, for me it's simply:

class XMA2
{
public::

    [...]

	void decode(AudioSampleBuffer& uncompressed)
	{
		uncompressed.setSize(numChannels, numSamples, false, true);
		uncompressed.clear();
		try
		{
			auto stream = MemoryInputStream(compressed, true);
			auto decoder = XMADecoder(&stream, uncompressed);
			decoder.init();
			decoder.PreparePipeline();
			while (!decoder.IsDone())
				decoder.Step();
		}
		catch (std::exception& e)
		{
			auto error = e.what();
			std::cout << "FFMPEG: " << error << std::endl;
		}
	}

private:
	MemoryBlock compressed;

    [...]

That's all. This is a draft way to do the trick, but hope that help!