Add Clova grpc streaming STT

Cargo.toml

@@ -18,6 +18,7 @@ serde_json = "1"
 time = "0.3.37"
 pin-project = "1"
 sentry = "0.35.0"
+bytes = "1.9.0"
 
 ndarray = "0.16.1"
 ort = "2.0.0-rc.9"

crates/clova/Cargo.toml

@@ -7,7 +7,13 @@ edition = "2021"
 tonic-build = "0.12.3"
 
 [dependencies]
-tokio = { workspace = true }
+anyhow = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+bytes = { workspace = true }
+
+tokio = { workspace = true }
+futures = { workspace = true }
+
 prost = "0.13.4"
 tonic = { version = "0.12.3", features = ["tls"] }

crates/clova/build.rs

@@ -1,4 +1,8 @@
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    tonic_build::compile_protos("proto/nest.proto")?;
+    tonic_build::configure()
+        .build_server(false)
+        .out_dir("./src")
+        .compile_protos(&["proto/nest.proto"], &["proto"])?;
+
     Ok(())
 }

crates/clova/src/com.nbp.cdncp.nest.grpc.proto.v1.rs

@@ -0,0 +1,183 @@
+// This file is @generated by prost-build.
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct NestConfig {
+    #[prost(string, tag = "1")]
+    pub config: ::prost::alloc::string::String,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct NestData {
+    #[prost(bytes = "vec", tag = "1")]
+    pub chunk: ::prost::alloc::vec::Vec<u8>,
+    #[prost(string, tag = "2")]
+    pub extra_contents: ::prost::alloc::string::String,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct NestRequest {
+    #[prost(enumeration = "RequestType", tag = "1")]
+    pub r#type: i32,
+    #[prost(oneof = "nest_request::Part", tags = "2, 3")]
+    pub part: ::core::option::Option<nest_request::Part>,
+}
+/// Nested message and enum types in `NestRequest`.
+pub mod nest_request {
+    #[derive(Clone, PartialEq, ::prost::Oneof)]
+    pub enum Part {
+        #[prost(message, tag = "2")]
+        Config(super::NestConfig),
+        #[prost(message, tag = "3")]
+        Data(super::NestData),
+    }
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct NestResponse {
+    #[prost(string, tag = "1")]
+    pub contents: ::prost::alloc::string::String,
+}
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum RequestType {
+    Config = 0,
+    Data = 1,
+}
+impl RequestType {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            Self::Config => "CONFIG",
+            Self::Data => "DATA",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "CONFIG" => Some(Self::Config),
+            "DATA" => Some(Self::Data),
+            _ => None,
+        }
+    }
+}
+/// Generated client implementations.
+pub mod nest_service_client {
+    #![allow(
+        unused_variables,
+        dead_code,
+        missing_docs,
+        clippy::wildcard_imports,
+        clippy::let_unit_value,
+    )]
+    use tonic::codegen::*;
+    use tonic::codegen::http::Uri;
+    #[derive(Debug, Clone)]
+    pub struct NestServiceClient<T> {
+        inner: tonic::client::Grpc<T>,
+    }
+    impl NestServiceClient<tonic::transport::Channel> {
+        /// Attempt to create a new client by connecting to a given endpoint.
+        pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
+        where
+            D: TryInto<tonic::transport::Endpoint>,
+            D::Error: Into<StdError>,
+        {
+            let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
+            Ok(Self::new(conn))
+        }
+    }
+    impl<T> NestServiceClient<T>
+    where
+        T: tonic::client::GrpcService<tonic::body::BoxBody>,
+        T::Error: Into<StdError>,
+        T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
+        <T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
+    {
+        pub fn new(inner: T) -> Self {
+            let inner = tonic::client::Grpc::new(inner);
+            Self { inner }
+        }
+        pub fn with_origin(inner: T, origin: Uri) -> Self {
+            let inner = tonic::client::Grpc::with_origin(inner, origin);
+            Self { inner }
+        }
+        pub fn with_interceptor<F>(
+            inner: T,
+            interceptor: F,
+        ) -> NestServiceClient<InterceptedService<T, F>>
+        where
+            F: tonic::service::Interceptor,
+            T::ResponseBody: Default,
+            T: tonic::codegen::Service<
+                http::Request<tonic::body::BoxBody>,
+                Response = http::Response<
+                    <T as tonic::client::GrpcService<tonic::body::BoxBody>>::ResponseBody,
+                >,
+            >,
+            <T as tonic::codegen::Service<
+                http::Request<tonic::body::BoxBody>,
+            >>::Error: Into<StdError> + std::marker::Send + std::marker::Sync,
+        {
+            NestServiceClient::new(InterceptedService::new(inner, interceptor))
+        }
+        /// Compress requests with the given encoding.
+        ///
+        /// This requires the server to support it otherwise it might respond with an
+        /// error.
+        #[must_use]
+        pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
+            self.inner = self.inner.send_compressed(encoding);
+            self
+        }
+        /// Enable decompressing responses.
+        #[must_use]
+        pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
+            self.inner = self.inner.accept_compressed(encoding);
+            self
+        }
+        /// Limits the maximum size of a decoded message.
+        ///
+        /// Default: `4MB`
+        #[must_use]
+        pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
+            self.inner = self.inner.max_decoding_message_size(limit);
+            self
+        }
+        /// Limits the maximum size of an encoded message.
+        ///
+        /// Default: `usize::MAX`
+        #[must_use]
+        pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
+            self.inner = self.inner.max_encoding_message_size(limit);
+            self
+        }
+        pub async fn recognize(
+            &mut self,
+            request: impl tonic::IntoStreamingRequest<Message = super::NestRequest>,
+        ) -> std::result::Result<
+            tonic::Response<tonic::codec::Streaming<super::NestResponse>>,
+            tonic::Status,
+        > {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::unknown(
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/com.nbp.cdncp.nest.grpc.proto.v1.NestService/recognize",
+            );
+            let mut req = request.into_streaming_request();
+            req.extensions_mut()
+                .insert(
+                    GrpcMethod::new(
+                        "com.nbp.cdncp.nest.grpc.proto.v1.NestService",
+                        "recognize",
+                    ),
+                );
+            self.inner.streaming(req, path, codec).await
+        }
+    }
+}

crates/clova/src/handle.rs

@@ -0,0 +1,31 @@
+// https://api.ncloud-docs.com/docs/ko/ai-application-service-clovaspeech-grpc
+// https://api.ncloud-docs.com/docs/en/ai-application-service-clovaspeech-grpc
+
+use anyhow::Result;
+
+#[derive(Debug, Clone)]
+enum WsMessage {
+    Audio(Audio),
+    ControlMessage(ControlMessage),
+}
+
+#[derive(Debug, Clone)]
+enum ControlMessage {
+    Finalize,
+    KeepAlive,
+    CloseStream,
+}
+
+#[derive(Debug, Clone)]
+struct Audio(Vec<u8>);
+
+struct StreamResponse {}
+
+#[derive(Debug)]
+pub struct Handle {}
+
+impl Handle {
+    async fn new() -> Result<Handle> {
+        Ok(Handle {})
+    }
+}

crates/clova/src/interface.rs

@@ -1,17 +1,17 @@
 use serde::{Deserialize, Serialize};
 
 // https://api.ncloud-docs.com/docs/en/ai-application-service-clovaspeech-grpc#3-request-config-json
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct ConfigRequest {
     pub transcription: Transcription,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct Transcription {
     pub language: Language,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub enum Language {
     #[serde(rename = "ko")]
     Korean,
@@ -29,14 +29,90 @@ pub struct ConfigResponseInner {
     pub status: ConfigResponseStatus,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Debug, PartialEq, Deserialize, Serialize)]
 pub enum ConfigResponseStatus {
     Success,
     Failure,
 }
 
+// https://api.ncloud-docs.com/docs/ai-application-service-clovaspeech-grpc#%EC%9D%91%EB%8B%B5-%EC%98%88%EC%8B%9C1
+#[derive(Debug, Deserialize, Serialize)]
+pub enum StreamResponse {
+    Success(StreamResponseSuccess),
+    Failure(StreamResponseFailure),
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct StreamResponseSuccess {
+    pub uid: String,
+    pub response_type: Vec<String>,
+    pub transcription: TranscriptionResponse,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct TranscriptionResponse {
+    pub text: String,
+    pub position: i32,
+    pub period_positions: Vec<i32>,
+    pub period_align_indices: Vec<i32>,
+    pub ep_flag: bool,
+    pub seq_id: i32,
+    pub epd_type: EpdType,
+    pub start_timestamp: i64,
+    pub end_timestamp: i64,
+    pub confidence: f64,
+    pub align_infos: Vec<AlignInfo>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct AlignInfo {
+    pub word: String,
+    pub start: i64,
+    pub end: i64,
+    pub confidence: f64,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct StreamResponseFailure {
+    pub uid: String,
+    pub response_type: Vec<String>,
+    pub recognize: RecognizeError,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct RecognizeError {
+    pub status: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ep_flag: Option<StatusInfo>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub seq_id: Option<StatusInfo>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub audio: Option<StatusInfo>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct StatusInfo {
+    pub status: String,
+}
+
 mod nest {
-    tonic::include_proto!("com.nbp.cdncp.nest.grpc.proto.v1");
+    include!("./com.nbp.cdncp.nest.grpc.proto.v1.rs");
 }
 
 pub use nest::*;
+
+#[derive(Debug, Deserialize, Serialize)]
+pub enum EpdType {
+    #[serde(rename = "gap")]
+    Gap,
+    #[serde(rename = "endPoint")]
+    EndPoint,
+    #[serde(rename = "durationThreshold")]
+    DurationThreshold,
+    #[serde(rename = "period")]
+    Period,
+    #[serde(rename = "syllableThreshold")]
+    SyllableThreshold,
+    #[serde(rename = "unvoice")]
+    Unvoice,
+}