Add tests for video camera video recording and fix subsequent recording

samples/SpectatorView.Example.Unity/ProjectSettings/EditorBuildSettings.asset

@@ -4,5 +4,17 @@
 EditorBuildSettings:
   m_ObjectHideFlags: 0
   serializedVersion: 2
-  m_Scenes: []
+  m_Scenes:
+  - enabled: 1
+    path: Assets/MixedReality-SpectatorView/SpectatorView/Scenes/SpectatorView.HoloLens.unity
+    guid: aaaef5009f203534696f812a2010313e
+  - enabled: 1
+    path: Assets/MixedReality-SpectatorView/SpectatorView/Scenes/SpectatorView.Android.unity
+    guid: 8c5e2534a4ad5f64dbb50a1bad484ef2
+  - enabled: 1
+    path: Assets/MixedReality-SpectatorView/SpectatorView/Scenes/SpectatorView.iOS.unity
+    guid: f8cc70d0d868b754ab89047eb4a6317b
+  - enabled: 1
+    path: Assets/MixedReality-SpectatorView/SpectatorView.Editor/Scenes/SpectatorViewCompositor.unity
+    guid: 4b7532bb75e238f49b6f062cf8a58d9e
   m_configObjects: {}

src/SpectatorView.Native/SpectatorView.Compositor/Compositor/CompositorInterface.cpp

@@ -124,8 +124,6 @@ int CompositorInterface::GetNumQueuedOutputFrames()
     return 0;
 }
 
-
-
 void CompositorInterface::SetCompositeFrameIndex(int index)
 {
     compositeFrameIndex = index;
@@ -159,32 +157,44 @@ bool CompositorInterface::InitializeVideoEncoder(ID3D11Device* device)
     return videoEncoder1080p->Initialize(device) && videoEncoder4K->Initialize(device);
 }
 
-void CompositorInterface::StartRecording(VideoRecordingFrameLayout frameLayout)
+bool CompositorInterface::StartRecording(VideoRecordingFrameLayout frameLayout, LPCWSTR lpcDesiredFileName, const int desiredFileNameLength, const int inputFileNameLength, LPWSTR lpFileName, int* fileNameLength)
 {
-    if (frameLayout == VideoRecordingFrameLayout::Composite)
-    {
-        activeVideoEncoder = videoEncoder1080p;
-    }
-    else
-    {
-        activeVideoEncoder = videoEncoder4K;
-    }
-
-    if (activeVideoEncoder == nullptr)
-    {
-        return;
-    }
-
-    videoIndex++;
-    videoRecordingStartTime = compositeFrameIndex * GetColorDuration();
-    audioRecordingStartTime = -1.0;
-
-    std::wstring videoPath = DirectoryHelper::FindUniqueFileName(outputPath, L"Video", L".mp4", videoIndex);
-    activeVideoEncoder->StartRecording(videoPath.c_str(), ENCODE_AUDIO);
+	*fileNameLength = 0;
+
+	std::wstring desiredFileName(lpcDesiredFileName);
+	std::wstring extension(L".mp4");
+	if (!DirectoryHelper::TestFileExtension(desiredFileName, extension))
+	{
+		return false;
+	}
+
+	std::wstring videoPath = DirectoryHelper::FindUniqueFileName(desiredFileName, extension);
+
+	std::shared_lock<std::shared_mutex> lock(encoderLock);
+	if (frameLayout == VideoRecordingFrameLayout::Composite)
+	{
+		activeVideoEncoder = videoEncoder1080p;
+	}
+	else
+	{
+		activeVideoEncoder = videoEncoder4K;
+	}
+
+	if (activeVideoEncoder == nullptr)
+	{
+		return false;
+	}
+
+	activeVideoEncoder->StartRecording(videoPath.c_str(), ENCODE_AUDIO);
+
+	memcpy_s(lpFileName, inputFileNameLength * _WCHAR_T_SIZE, videoPath.c_str(), videoPath.size() * _WCHAR_T_SIZE);
+	*fileNameLength = videoPath.size();
+	return true;
 }
 
 void CompositorInterface::StopRecording()
 {
+	std::shared_lock<std::shared_mutex> lock(encoderLock);
     if (activeVideoEncoder == nullptr)
     {
         return;
@@ -196,31 +206,50 @@ void CompositorInterface::StopRecording()
 
 void CompositorInterface::RecordFrameAsync(BYTE* videoFrame, LONGLONG frameTime, int numFrames)
 {
+#if _DEBUG
+	std::wstring debugString = L"RecordFrameAsync called, frameTime:" + std::to_wstring(frameTime) + L", numFrames:" + std::to_wstring(numFrames) + L"\n";
+	OutputDebugString(debugString.data());
+#endif
+
+	std::shared_lock<std::shared_mutex> lock(encoderLock);
     if (frameProvider == nullptr || activeVideoEncoder == nullptr)
     {
+		OutputDebugString(L"RecordFrameAsync dropped, no active frame provider or encoder\n");
         return;
     }
     if (numFrames < 1) 
         numFrames = 1;
     else if (numFrames > 5) 
         numFrames = 5;
 
-    activeVideoEncoder->QueueVideoFrame(videoFrame, frameTime, numFrames * frameProvider->GetDurationHNS());
+	// The encoder will update sample times internally based on the first seen sample time when recording.
+	// The encoder, however, does assume that audio and video samples will be based on the same source time.
+	// Providing audio and video samples with different starting times will cause issues in the generated video file.
+	LONGLONG sampleTime = frameTime;
+    activeVideoEncoder->QueueVideoFrame(videoFrame, sampleTime, numFrames * frameProvider->GetDurationHNS());
 }
 
-void CompositorInterface::RecordAudioFrameAsync(BYTE* audioFrame, int audioSize, double audioTime)
+void CompositorInterface::RecordAudioFrameAsync(BYTE* audioFrame, LONGLONG audioTime, int audioSize)
 {
+#if _DEBUG
+	std::wstring debugString = L"RecordAudioFrameAsync called, audioTime:" + std::to_wstring(audioTime) + L", audioSize:" + std::to_wstring(audioSize) + L"\n";
+	OutputDebugString(debugString.data());
+#endif
+
+	std::shared_lock<std::shared_mutex> lock(encoderLock);
     if (activeVideoEncoder == nullptr)
     {
+#if _DEBUG
+		OutputDebugString(L"RecordAudioFrameAsync dropped, no active encoder\n");
+#endif
         return;
     }
 
-    if (audioRecordingStartTime < 0)
-        audioRecordingStartTime = audioTime;
-
-    LONGLONG frameTime = videoRecordingStartTime + (LONGLONG)((audioTime - audioRecordingStartTime) * QPC_MULTIPLIER);
-
-    activeVideoEncoder->QueueAudioFrame(audioFrame, audioSize, frameTime);
+	// The encoder will update sample times internally based on the first seen sample time when recording.
+	// The encoder, however, does assume that audio and video samples will be based on the same source time.
+	// Providing audio and video samples with different starting times will cause issues in the generated video file.
+	LONGLONG sampleTime = audioTime;
+    activeVideoEncoder->QueueAudioFrame(audioFrame, audioSize, sampleTime);
 }
 
 bool CompositorInterface::OutputYUV()

src/SpectatorView.Native/SpectatorView.Compositor/Compositor/CompositorInterface.h

@@ -27,9 +27,6 @@ class CompositorInterface
     VideoEncoder* videoEncoder4K = nullptr;
     VideoEncoder* activeVideoEncoder = nullptr;
 
-    int videoIndex = -1;
-    LONGLONG videoRecordingStartTime;
-    double audioRecordingStartTime;
     int photoIndex = -1;
 
     std::wstring outputPath, channelPath;
@@ -38,6 +35,9 @@ class CompositorInterface
 
     LONGLONG stubVideoTime = 0;
 
+	// Audio write calls may occur off the main thread so we need to lock around encoder access.
+	std::shared_mutex encoderLock;
+
 public:
     DLLEXPORT CompositorInterface();
     DLLEXPORT void SetFrameProvider(IFrameProvider::ProviderType type);
@@ -60,10 +60,14 @@ class CompositorInterface
     DLLEXPORT void TakePicture(ID3D11Device* device, int width, int height, int bpp, BYTE* bytes);
 
     DLLEXPORT bool InitializeVideoEncoder(ID3D11Device* device);
-    DLLEXPORT void StartRecording(VideoRecordingFrameLayout frameLayout);
+    DLLEXPORT bool StartRecording(VideoRecordingFrameLayout frameLayout, LPCWSTR lpcDesiredFileName, const int desiredFileNameLength, const int inputFileNameLength, LPWSTR lpFileName, int* fileNameLength);
     DLLEXPORT void StopRecording();
-    DLLEXPORT void RecordFrameAsync(BYTE* videoFrame, LONGLONG frameTime, int numFrames);
-    DLLEXPORT void RecordAudioFrameAsync(BYTE* audioFrame, int audioSize, double audioTime);
+
+	// frameTime is in hundred nano seconds
+	DLLEXPORT void RecordFrameAsync(BYTE* videoFrame, LONGLONG frameTime, int numFrames);
+
+	// audioTime is in hundrend nano seconds
+    DLLEXPORT void RecordAudioFrameAsync(BYTE* audioFrame, LONGLONG audioTime, int audioSize);
 
     DLLEXPORT void SetAlpha(float newAlpha)
     {

src/SpectatorView.Native/SpectatorView.Compositor/Compositor/DirectoryHelper.cpp

@@ -57,6 +57,26 @@ std::wstring DirectoryHelper::FindUniqueFileName(const std::wstring path, std::w
     return tempPath;
 }
 
+std::wstring DirectoryHelper::FindUniqueFileName(const std::wstring filePath, const std::wstring extension)
+{
+	if (!FileExists(filePath))
+	{
+		return filePath;
+	}
+
+	int index = 1;
+	std::wstring filePathWithoutExtension = filePath.substr(0, filePath.size() - extension.size());
+	std::wstring tempPath = filePathWithoutExtension + L"_" + std::to_wstring(index) + extension;
+
+	while (FileExists(tempPath))
+	{
+		index++;
+		tempPath = filePathWithoutExtension + L"_" + std::to_wstring(index) + extension;
+	}
+
+	return tempPath;
+}
+
 // Get the number of files with the given extension in the input directory.
 int DirectoryHelper::NumFiles(std::wstring root, std::wstring extension)
 {
@@ -100,3 +120,16 @@ void DirectoryHelper::DeleteFiles(std::wstring root, std::wstring extension)
         FindClose(hFind);
     }
 }
+
+BOOL DirectoryHelper::TestFileExtension(std::wstring& file, std::wstring& ext)
+{
+	int lstr = file.length();
+	int lext = ext.length();
+
+	if (lstr < lext)
+	{
+		return 0;
+	}
+
+	return file.compare(lstr - lext, lext, ext) == 0;
+}

src/SpectatorView.Native/SpectatorView.Compositor/Compositor/DirectoryHelper.h

@@ -17,7 +17,9 @@ class DirectoryHelper
     static void MoveFiles(std::wstring srcPath, std::wstring destPath);
     static BOOL FileExists(std::wstring path);
     static std::wstring FindUniqueFileName(const std::wstring path, std::wstring fileName, std::wstring extension, int& startingIndex);
+	static std::wstring FindUniqueFileName(const std::wstring filePath, const std::wstring extension);
     static int NumFiles(std::wstring root, std::wstring extension);
     static void DeleteFiles(std::wstring root, std::wstring extension);
+	static BOOL TestFileExtension(std::wstring& file, std::wstring& ext);
 };