SpeechToText #808

samples/CommunityToolkit.Maui.Sample/AppShell.xaml.cs

@@ -84,6 +84,7 @@ public partial class AppShell : Shell
  // Add Essentials View Models
  CreateViewModelMapping<FileSaverPage, FileSaverViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
  CreateViewModelMapping<FolderPickerPage, FolderPickerViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
+ CreateViewModelMapping<SpeechToTextPage, SpeechToTextViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
 
  // Add Extensions View Models
  CreateViewModelMapping<ColorAnimationExtensionsPage, ColorAnimationExtensionsViewModel, ExtensionsGalleryPage, ExtensionsGalleryViewModel>(),

samples/CommunityToolkit.Maui.Sample/MauiProgram.cs

@@ -1,4 +1,5 @@
 using CommunityToolkit.Maui.Markup;
+using CommunityToolkit.Maui.Media;
 using CommunityToolkit.Maui.Sample.Models;
 using CommunityToolkit.Maui.Sample.Pages;
 using CommunityToolkit.Maui.Sample.Pages.Alerts;
@@ -153,6 +154,7 @@ static void RegisterViewsAndViewModels(in IServiceCollection services)
  // Add Essentials Pages + ViewModels
  services.AddTransientWithShellRoute<FileSaverPage, FileSaverViewModel>();
  services.AddTransientWithShellRoute<FolderPickerPage, FolderPickerViewModel>();
+ services.AddTransientWithShellRoute<SpeechToTextPage, SpeechToTextViewModel>();
 
  // Add Extensions Pages + ViewModels
  services.AddTransientWithShellRoute<ColorAnimationExtensionsPage, ColorAnimationExtensionsViewModel>();
@@ -187,6 +189,8 @@ static void RegisterEssentials(in IServiceCollection services)
  services.AddSingleton<IDeviceDisplay>(DeviceDisplay.Current);
  services.AddSingleton<IFileSaver>(FileSaver.Default);
  services.AddSingleton<IFolderPicker>(FolderPicker.Default);
+ services.AddSingleton<ITextToSpeech>(TextToSpeech.Default);
+ services.AddSingleton<ISpeechToText>(SpeechToText.Default);
  }
 
  static IServiceCollection AddTransientWithShellRoute<TPage, TViewModel>(this IServiceCollection services) where TPage : BasePage<TViewModel>

samples/CommunityToolkit.Maui.Sample/Pages/Essentials/SpeechToTextPage.xaml

@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<pages:BasePage xmlns="http://schemas.microsoft.com/dotnet/2021/maui"
+ xmlns:x="http://schemas.microsoft.com/winfx/2009/xaml"
+ xmlns:pages="clr-namespace:CommunityToolkit.Maui.Sample.Pages"
+ x:Class="CommunityToolkit.Maui.Sample.Pages.Essentials.SpeechToTextPage"
+ xmlns:vm="clr-namespace:CommunityToolkit.Maui.Sample.ViewModels.Essentials"
+ xmlns:essentials="clr-namespace:CommunityToolkit.Maui.Sample.Pages.Essentials"
+ x:TypeArguments="vm:SpeechToTextViewModel"
+ x:DataType="vm:SpeechToTextViewModel"
+ Title="SpeechToText">
+
+ <ContentPage.Resources>
+ <essentials:PickerLocaleDisplayConverter x:Key="PickerLocaleDisplayConverter" />
+ </ContentPage.Resources>
+
+ <ScrollView>
+ <Grid
+ RowDefinitions="64, 24, 64, 24, auto, 60, 60, 60"
+ Padding="30,0">
+
+ <Label
+ Grid.Row="0"
+ Text="SpeechToText allows the user to convert speech to text in real time"/>
+
+ <Label
+ Grid.Row="1"
+ Text="Locale"
+ FontAttributes="Bold"/>
+
+ <Picker
+ Grid.Row="2"
+ ItemsSource="{Binding Locales}"
+ SelectedItem="{Binding Locale}"
+ ItemDisplayBinding="{Binding ., Converter={StaticResource PickerLocaleDisplayConverter}}"
+ Margin="0,0,0,20">
+ </Picker>
+
+ <Label
+ Grid.Row="3"
+ Text="Language Output"
+ FontAttributes="Bold"/>
+
+ <Label
+ Grid.Row="4"
+ Text="{Binding RecognitionText}"
+ FontSize="18"
+ HorizontalOptions="Center"
+ HorizontalTextAlignment="Center"
+ MinimumHeightRequest="100"
+ Margin="0,0,0,20" />
+
+ <Button
+ Grid.Row="5"
+ Text="Play"
+ Command="{Binding PlayCommand}"
+ HorizontalOptions="Center"
+ Margin="0,0,0,20"/>
+
+ <Button
+ Grid.Row="6"
+ Text="Listen"
+ Command="{Binding ListenCommand}"
+ HorizontalOptions="Center"
+ Margin="0,0,0,20"/>
+
+ <Button
+ Grid.Row="7"
+ Text="Stop Listening"
+ Command="{Binding ListenCancelCommand}"
+ HorizontalOptions="Center"/>
+
+ </Grid>
+ </ScrollView>
+
+</pages:BasePage>

samples/CommunityToolkit.Maui.Sample/Pages/Essentials/SpeechToTextPage.xaml.cs

@@ -0,0 +1,30 @@
+using System.Globalization;
+using CommunityToolkit.Maui.Converters;
+using CommunityToolkit.Maui.Sample.ViewModels.Essentials;
+
+namespace CommunityToolkit.Maui.Sample.Pages.Essentials;
+
+public partial class SpeechToTextPage : BasePage<SpeechToTextViewModel>
+{
+ public SpeechToTextPage(SpeechToTextViewModel viewModel) : base(viewModel)
+ {
+ InitializeComponent();
+ }
+
+ protected override async void OnAppearing()
+ {
+ base.OnAppearing();
+
+ await BindingContext.SetLocalesCommand.ExecuteAsync(null);
+ }
+}
+
+class PickerLocaleDisplayConverter : BaseConverterOneWay<Locale, string>
+{
+ public override string DefaultConvertReturnValue { get; set; } = string.Empty;
+
+ public override string ConvertFrom(Locale value, CultureInfo? culture)
+ {
+ return $"{value.Language} {value.Name}";
+ }
+}

samples/CommunityToolkit.Maui.Sample/Pages/Views/LazyView/CustomLazyView.cs

@@ -17,5 +17,4 @@ public override async ValueTask LoadViewAsync()
 
  SetHasLazyViewLoaded(true);
  }
-}
-
+}

samples/CommunityToolkit.Maui.Sample/Pages/Views/LazyView/MyLazyView.cs

@@ -8,5 +8,4 @@ public override async ValueTask LoadViewAsync()
  {
  await base.LoadViewAsync();
  }
-}
-
+}

samples/CommunityToolkit.Maui.Sample/Platforms/Android/AndroidManifest.xml

@@ -4,4 +4,6 @@
  <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
  <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
  <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+
+ <uses-permission android:name="android.permission.RECORD_AUDIO" />
 </manifest>

samples/CommunityToolkit.Maui.Sample/Platforms/MacCatalyst/Info.plist

@@ -26,5 +26,7 @@
  </array>
  <key>XSAppIconAssets</key>
  <string>Assets.xcassets/appicon.appiconset</string>
+ <key>NSSpeechRecognitionUsageDescription</key>
+ <string>ISpeechToText Uses Speech Recognition</string>
 </dict>
 </plist>

samples/CommunityToolkit.Maui.Sample/Platforms/Tizen/tizen-manifest.xml

@@ -12,7 +12,9 @@
  <privilege>http://tizen.org/privilege/internet</privilege>
  <privilege>http://tizen.org/privilege/mediastorage</privilege>
  <privilege>http://tizen.org/privilege/externalstorage</privilege>
+ <privilege>http://tizen.org/privilege/recorder</privilege>
  </privileges>
  <dependencies />
  <provides-appdefined-privileges />
+ <feature name="http://tizen.org/feature/microphone">true</feature>
 </manifest>

samples/CommunityToolkit.Maui.Sample/Platforms/iOS/Info.plist

@@ -28,7 +28,9 @@
  </array>
  <key>XSAppIconAssets</key>
  <string>Assets.xcassets/appicon.appiconset</string>
- <key>UIViewControllerBasedStatusBarAppearance</key>
- <false/>
+ <key>UIViewControllerBasedStatusBarAppearance</key>
+ <false/>
+ <key>NSSpeechRecognitionUsageDescription</key>
+ <string>ISpeechToText Uses Speech Recognition</string>
 </dict>
 </plist>

samples/CommunityToolkit.Maui.Sample/ViewModels/Essentials/EssentialsGalleryViewModel.cs

@@ -8,7 +8,8 @@ public EssentialsGalleryViewModel()
  : base(new[]
  {
  SectionModel.Create<FileSaverViewModel>("FileSaver", "Allows the user to save files to the filesystem"),
- SectionModel.Create<FolderPickerViewModel>("FolderPicker", "Allows picking folders from the file system")
+ SectionModel.Create<FolderPickerViewModel>("FolderPicker", "Allows picking folders from the file system"),
+ SectionModel.Create<SpeechToTextViewModel>("SpeechToText", "Converts speech to text"),
  })
  {
  }

samples/CommunityToolkit.Maui.Sample/ViewModels/Essentials/SpeechToTextViewModel.cs

@@ -0,0 +1,99 @@
+using System.Collections.ObjectModel;
+using System.Collections.Specialized;
+using System.Globalization;
+using CommunityToolkit.Maui.Alerts;
+using CommunityToolkit.Maui.Media;
+using CommunityToolkit.Mvvm.ComponentModel;
+using CommunityToolkit.Mvvm.Input;
+
+namespace CommunityToolkit.Maui.Sample.ViewModels.Essentials;
+
+public partial class SpeechToTextViewModel : BaseViewModel
+{
+ const string defaultLanguage = "en-US";
+
+ readonly ITextToSpeech textToSpeech;
+ readonly ISpeechToText speechToText;
+
+ [ObservableProperty]
+ Locale? locale;
+
+ [ObservableProperty]
+ string recognitionText = "Welcome to .NET MAUI Community Toolkit!";
+
+ public SpeechToTextViewModel(ITextToSpeech textToSpeech, ISpeechToText speechToText)
+ {
+ this.textToSpeech = textToSpeech;
+ this.speechToText = speechToText;
+
+ Locales.CollectionChanged += HandleLocalesCollectionChanged;
+ }
+
+ public ObservableCollection<Locale> Locales { get; } = new();
+
+ [RelayCommand]
+ async Task SetLocales()
+ {
+ Locales.Clear();
+
+ var locales = await textToSpeech.GetLocalesAsync();
+ foreach (var locale in locales.OrderBy(x => x.Language).ThenBy(x => x.Name))
+ {
+ Locales.Add(locale);
+ }
+
+ Locale = Locales.FirstOrDefault(x => x.Language is defaultLanguage) ?? Locales.FirstOrDefault();
+ }
+
+ [RelayCommand]
+ async Task Play(CancellationToken cancellationToken)
+ {
+ await textToSpeech.SpeakAsync(RecognitionText, new()
+ {
+ Locale = Locale,
+ Pitch = 2,
+ Volume = 1
+ }, cancellationToken);
+ }
+
+ [RelayCommand(IncludeCancelCommand = true)]
+ async Task Listen(CancellationToken cancellationToken)
+ {
+ const string beginSpeakingPrompt = "Begin speaking...";
+
+ RecognitionText = beginSpeakingPrompt;
+
+ try
+ {
+ RecognitionText = await speechToText.ListenAsync(CultureInfo.GetCultureInfo(Locale?.Language ?? defaultLanguage), new Progress<string>(partialText =>
+ {
+ if (RecognitionText is beginSpeakingPrompt)
+ {
+ RecognitionText = string.Empty;
+ }
+
+ RecognitionText += partialText + " ";
+ }), cancellationToken);
+ }
+ catch (TaskCanceledException)
+ {
+ await Toast.Make("Listening Stopped by User").Show(CancellationToken.None);
+ }
+ catch (Exception e)
+ {
+ await Toast.Make(e.Message).Show(CancellationToken.None);
+ }
+ finally
+ {
+ if (RecognitionText is beginSpeakingPrompt)
+ {
+ RecognitionText = string.Empty;
+ }
+ }
+ }
+
+ void HandleLocalesCollectionChanged(object? sender, NotifyCollectionChangedEventArgs e)
+ {
+ OnPropertyChanged(nameof(Locale));
+ }
+}

src/CommunityToolkit.Maui.Core/CommunityToolkit.Maui.Core.csproj

@@ -55,5 +55,7 @@
  <PrivateAssets>all</PrivateAssets>
  <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
  </PackageReference>
+
+ <PackageReference Include="System.Speech" Version="7.0.0" Condition="'$(TargetFramework)' == 'net7.0-windows10.0.19041.0'" />
  </ItemGroup>
 </Project>

src/CommunityToolkit.Maui.Core/Essentials/SpeechToText/ISpeechToText.shared.cs

@@ -0,0 +1,18 @@
+using System.Globalization;
+
+namespace CommunityToolkit.Maui.Media;
+
+/// <summary>
+/// Allows the user to convert speech to text in real time.
+/// </summary>
+public interface ISpeechToText : IAsyncDisposable
+{
+ /// <summary>
+ /// Converts speech to text in real time.
+ /// </summary>
+ /// <param name="culture">Speak language</param>
+ /// <param name="recognitionResult">Intermediate convertion result.</param>
+ /// <param name="cancellationToken"><see cref="CancellationToken"/></param>
+ /// <returns>Final convertion result</returns>
+ Task<string> ListenAsync(CultureInfo culture, IProgress<string>? recognitionResult, CancellationToken cancellationToken);
+}

src/CommunityToolkit.Maui.Core/Essentials/SpeechToText/SharedSpeechToTextImplementation.macios.cs

@@ -0,0 +1,45 @@
+using AVFoundation;
+using Speech;
+
+namespace CommunityToolkit.Maui.Media;
+
+/// <summary>
+/// Base class for <see cref="SpeechToTextImplementation"/> on iOS + MacCatalyst
+/// </summary>
+public sealed partial class SpeechToTextImplementation
+{
+ AVAudioEngine? audioEngine;
+ SFSpeechRecognizer? speechRecognizer;
+ SFSpeechRecognitionTask? recognitionTask;
+ SFSpeechAudioBufferRecognitionRequest? liveSpeechRequest;
+
+ /// <inheritdoc />
+ public ValueTask DisposeAsync()
+ {
+ audioEngine?.Dispose();
+ speechRecognizer?.Dispose();
+ liveSpeechRequest?.Dispose();
+ recognitionTask?.Dispose();
+ return ValueTask.CompletedTask;
+ }
+
+
+ static Task<bool> IsSpeechPermissionAuthorized()
+ {
+ var taskResult = new TaskCompletionSource<bool>();
+ SFSpeechRecognizer.RequestAuthorization(status =>
+ {
+ taskResult.SetResult(status is SFSpeechRecognizerAuthorizationStatus.Authorized);
+ });
+
+ return taskResult.Task;
+ }
+
+ void StopRecording()
+ {
+ audioEngine?.InputNode.RemoveTapOnBus(new nuint(0));
+ audioEngine?.Stop();
+ liveSpeechRequest?.EndAudio();
+ recognitionTask?.Cancel();
+ }
+}