From c90785dbf9635cad6125dbc594b2815c3387c89b Mon Sep 17 00:00:00 2001 From: hoslo Date: Mon, 26 Feb 2024 17:36:33 +0800 Subject: [PATCH] feat(services/github): add github contents support --- core/Cargo.toml | 1 + core/src/services/github/backend.rs | 324 ++++++++++++++++++++++++++++ core/src/services/github/core.rs | 297 +++++++++++++++++++++++++ core/src/services/github/docs.md | 54 +++++ core/src/services/github/error.rs | 108 ++++++++++ core/src/services/github/lister.rs | 69 ++++++ core/src/services/github/mod.rs | 25 +++ core/src/services/github/writer.rs | 58 +++++ core/src/services/mod.rs | 7 + core/src/types/operator/builder.rs | 2 + core/src/types/scheme.rs | 4 + 11 files changed, 949 insertions(+) create mode 100644 core/src/services/github/backend.rs create mode 100644 core/src/services/github/core.rs create mode 100644 core/src/services/github/docs.md create mode 100644 core/src/services/github/error.rs create mode 100644 core/src/services/github/lister.rs create mode 100644 core/src/services/github/mod.rs create mode 100644 core/src/services/github/writer.rs diff --git a/core/Cargo.toml b/core/Cargo.toml index dbfc4cd309e..7119772d475 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -156,6 +156,7 @@ services-gcs = [ ] services-gdrive = ["internal-path-cache"] services-ghac = [] +services-github = [] services-gridfs = ["dep:mongodb"] services-hdfs = ["dep:hdrs"] services-http = [] diff --git a/core/src/services/github/backend.rs b/core/src/services/github/backend.rs new file mode 100644 index 00000000000..997e2ae92f0 --- /dev/null +++ b/core/src/services/github/backend.rs @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt::Debug; +use std::fmt::Formatter; +use std::sync::Arc; + +use async_trait::async_trait; +use http::StatusCode; +use log::debug; +use serde::Deserialize; + +use super::core::GithubCore; +use super::error::parse_error; +use super::lister::GithubLister; +use super::writer::GithubWriter; +use super::writer::GithubWriters; +use crate::raw::*; +use crate::*; + +/// Config for backblaze Github services support. +#[derive(Default, Deserialize)] +#[serde(default)] +#[non_exhaustive] +pub struct GithubConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, + /// Github access_token. + /// + /// required. + pub token: String, + /// Github repo owner. + /// + /// required. + pub owner: String, + /// Github repo name. + /// + /// required. + pub repo: String, +} + +impl Debug for GithubConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("GithubConfig"); + + d.field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo); + + d.finish_non_exhaustive() + } +} + +/// [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. +#[doc = include_str!("docs.md")] +#[derive(Default)] +pub struct GithubBuilder { + config: GithubConfig, + + http_client: Option, +} + +impl Debug for GithubBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("GithubBuilder"); + + d.field("config", &self.config); + d.finish_non_exhaustive() + } +} + +impl GithubBuilder { + /// Set root of this backend. + /// + /// All operations will happen under this root. + pub fn root(&mut self, root: &str) -> &mut Self { + self.config.root = if root.is_empty() { + None + } else { + Some(root.to_string()) + }; + + self + } + + /// Github access_token. + /// + /// required. + pub fn token(&mut self, token: &str) -> &mut Self { + self.config.token = token.to_string(); + + self + } + + /// Set Github repo owner. + pub fn owner(&mut self, owner: &str) -> &mut Self { + self.config.owner = owner.to_string(); + + self + } + + /// Set Github repo name. + pub fn repo(&mut self, repo: &str) -> &mut Self { + self.config.repo = repo.to_string(); + + self + } + + /// Specify the http client that used by this service. + /// + /// # Notes + /// + /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed + /// during minor updates. + pub fn http_client(&mut self, client: HttpClient) -> &mut Self { + self.http_client = Some(client); + self + } +} + +impl Builder for GithubBuilder { + const SCHEME: Scheme = Scheme::Github; + type Accessor = GithubBackend; + + /// Converts a HashMap into an GithubBuilder instance. + /// + /// # Arguments + /// + /// * `map` - A HashMap containing the configuration values. + /// + /// # Returns + /// + /// Returns an instance of GithubBuilder. + fn from_map(map: HashMap) -> Self { + // Deserialize the configuration from the HashMap. + let config = GithubConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); + + // Create an GithubBuilder instance with the deserialized config. + GithubBuilder { + config, + http_client: None, + } + } + + /// Builds the backend and returns the result of GithubBackend. + fn build(&mut self) -> Result { + debug!("backend build started: {:?}", &self); + + let root = normalize_root(&self.config.root.clone().unwrap_or_default()); + debug!("backend use root {}", &root); + + // Handle token. + if self.config.token.is_empty() { + return Err(Error::new(ErrorKind::ConfigInvalid, "token is empty") + .with_operation("Builder::build") + .with_context("service", Scheme::Github)); + } + + // Handle owner. + if self.config.owner.is_empty() { + return Err(Error::new(ErrorKind::ConfigInvalid, "owner is empty") + .with_operation("Builder::build") + .with_context("service", Scheme::Github)); + } + + debug!("backend use owner {}", &self.config.owner); + + // Handle repo. + if self.config.repo.is_empty() { + return Err(Error::new(ErrorKind::ConfigInvalid, "repo is empty") + .with_operation("Builder::build") + .with_context("service", Scheme::Github)); + } + + debug!("backend use repo {}", &self.config.repo); + + let client = if let Some(client) = self.http_client.take() { + client + } else { + HttpClient::new().map_err(|err| { + err.with_operation("Builder::build") + .with_context("service", Scheme::Github) + })? + }; + + Ok(GithubBackend { + core: Arc::new(GithubCore { + root, + token: self.config.token.clone(), + owner: self.config.owner.clone(), + repo: self.config.repo.clone(), + client, + }), + }) + } +} + +/// Backend for Github services. +#[derive(Debug, Clone)] +pub struct GithubBackend { + core: Arc, +} + +#[async_trait] +impl Accessor for GithubBackend { + type Reader = IncomingAsyncBody; + + type Writer = GithubWriters; + + type Lister = oio::PageLister; + + type BlockingReader = (); + + type BlockingWriter = (); + + type BlockingLister = (); + + fn info(&self) -> AccessorInfo { + let mut am = AccessorInfo::default(); + am.set_scheme(Scheme::Github) + .set_root(&self.core.root) + .set_native_capability(Capability { + stat: true, + + read: true, + + create_dir: true, + + write: true, + write_can_empty: true, + + delete: true, + + list: true, + + ..Default::default() + }); + + am + } + + async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result { + let empty_bytes = bytes::Bytes::new(); + + let resp = self + .core + .upload(&format!("{}.gitkeep", path), empty_bytes) + .await?; + + let status = resp.status(); + + match status { + StatusCode::OK | StatusCode::CREATED => Ok(RpCreateDir::default()), + _ => Err(parse_error(resp).await?), + } + } + + async fn stat(&self, path: &str, _args: OpStat) -> Result { + let resp = self.core.stat(path).await?; + + let status = resp.status(); + + match status { + StatusCode::OK => parse_into_metadata(path, resp.headers()).map(RpStat::new), + _ => Err(parse_error(resp).await?), + } + } + + async fn read(&self, path: &str, _args: OpRead) -> Result<(RpRead, Self::Reader)> { + let resp = self.core.get(path).await?; + + let status = resp.status(); + + match status { + StatusCode::OK => { + let size = parse_content_length(resp.headers())?; + let range = parse_content_range(resp.headers())?; + Ok(( + RpRead::new().with_size(size).with_range(range), + resp.into_body(), + )) + } + _ => Err(parse_error(resp).await?), + } + } + + async fn write(&self, path: &str, _args: OpWrite) -> Result<(RpWrite, Self::Writer)> { + let writer = GithubWriter::new(self.core.clone(), path.to_string()); + + let w = oio::OneShotWriter::new(writer); + + Ok((RpWrite::default(), w)) + } + + async fn delete(&self, path: &str, _: OpDelete) -> Result { + match self.core.delete(path).await { + Ok(_) => Ok(RpDelete::default()), + Err(err) => Err(err), + } + } + + async fn list(&self, path: &str, _args: OpList) -> Result<(RpList, Self::Lister)> { + let l = GithubLister::new(self.core.clone(), path); + Ok((RpList::default(), oio::PageLister::new(l))) + } +} diff --git a/core/src/services/github/core.rs b/core/src/services/github/core.rs new file mode 100644 index 00000000000..2e90f66e269 --- /dev/null +++ b/core/src/services/github/core.rs @@ -0,0 +1,297 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; +use std::fmt::Formatter; + +use base64::Engine; +use bytes::Bytes; +use http::header; +use http::request; +use http::Request; +use http::Response; +use http::StatusCode; +use serde::Deserialize; +use serde::Serialize; + +use crate::raw::*; +use crate::*; + +use super::error::parse_error; + +/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. +#[derive(Clone)] +pub struct GithubCore { + /// The root of this core. + pub root: String, + /// Github access_token. + pub token: String, + /// Github repo owner. + pub owner: String, + /// Github repo name. + pub repo: String, + + pub client: HttpClient, +} + +impl Debug for GithubCore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Backend") + .field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo) + .finish_non_exhaustive() + } +} + +impl GithubCore { + #[inline] + pub async fn send(&self, req: Request) -> Result> { + self.client.send(req).await + } + + pub fn sign(&self, req: request::Builder) -> Result { + let req = req + .header(header::USER_AGENT, format!("opendal-{}", VERSION)) + .header("X-GitHub-Api-Version", "2022-11-28"); + + Ok(req.header( + header::AUTHORIZATION, + format_authorization_by_bearer(&self.token)?, + )) + } +} + +impl GithubCore { + pub async fn get_file_sha(&self, path: &str) -> Result> { + let resp = self.stat(path).await?; + + match resp.status() { + StatusCode::OK => { + let headers = resp.headers(); + + let sha = parse_etag(headers)?; + + let Some(sha) = sha else { + return Err(Error::new( + ErrorKind::Unexpected, + "No ETag found in response headers", + )); + }; + + Ok(Some(sha.trim_matches('"').to_string())) + } + StatusCode::NOT_FOUND => Ok(None), + _ => Err(parse_error(resp).await?), + } + } + + pub async fn stat(&self, path: &str) -> Result> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}", + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::head(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn get(&self, path: &str) -> Result> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}", + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::get(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.raw+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn upload(&self, path: &str, bs: Bytes) -> Result> { + let sha = self.get_file_sha(path).await?; + + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}", + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::put(url); + + let req = self.sign(req)?; + + let mut req_body = CreateOrUpdateContentsRequest { + message: format!("Write {} at {} via opendal", path, chrono::Local::now()), + content: base64::engine::general_purpose::STANDARD.encode(&bs), + sha: None, + }; + + if let Some(sha) = sha { + req_body.sha = Some(sha); + } + + let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?; + + let req = req + .header("Accept", "application/vnd.github+json") + .body(AsyncBody::Bytes(Bytes::from(req_body))) + .map_err(new_request_build_error)?; + + self.send(req).await + } + + pub async fn delete(&self, path: &str) -> Result<()> { + let Some(sha) = self.get_file_sha(path).await? else { + return Ok(()); + }; + + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}", + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::delete(url); + + let req = self.sign(req)?; + + let req_body = DeleteContentsRequest { + message: format!("Delete {} at {} via opendal", path, chrono::Local::now()), + sha, + }; + + let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?; + + let req = req + .header("Accept", "application/vnd.github.object+json") + .body(AsyncBody::Bytes(Bytes::from(req_body))) + .map_err(new_request_build_error)?; + + let resp = self.send(req).await?; + + match resp.status() { + StatusCode::OK => Ok(()), + _ => Err(parse_error(resp).await?), + } + } + + pub async fn list(&self, path: &str) -> Result> { + let path = build_abs_path(&self.root, path); + + let url = format!( + "https://api.github.com/repos/{}/{}/contents/{}", + self.owner, + self.repo, + percent_encode_path(&path) + ); + + let req = Request::get(url); + + let req = self.sign(req)?; + + let req = req + .header("Accept", "application/vnd.github.object+json") + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + let resp = self.send(req).await?; + + match resp.status() { + StatusCode::OK => { + let body = resp.into_body().bytes().await?; + let resp: ListResponse = + serde_json::from_slice(&body).map_err(new_json_deserialize_error)?; + + Ok(resp.entries) + } + _ => Err(parse_error(resp).await?), + } + } +} + +#[derive(Default, Debug, Clone, Serialize)] +pub struct CreateOrUpdateContentsRequest { + pub message: String, + pub content: String, + pub sha: Option, +} + +#[derive(Default, Debug, Clone, Serialize)] +pub struct DeleteContentsRequest { + pub message: String, + pub sha: String, +} + +#[derive(Default, Debug, Clone, Deserialize)] +pub struct ListResponse { + pub entries: Vec, +} + +#[derive(Default, Debug, Clone, Deserialize)] +pub struct Entry { + pub name: String, + pub path: String, + pub sha: String, + pub size: u64, + pub url: String, + pub html_url: String, + pub git_url: String, + pub download_url: Option, + #[serde(rename = "type")] + pub type_field: String, + pub content: Option, + pub encoding: Option, + #[serde(rename = "_links")] + pub links: Links, +} + +#[derive(Default, Debug, Clone, Deserialize)] +pub struct Links { + #[serde(rename = "self")] + pub self_field: String, + pub git: String, + pub html: String, +} diff --git a/core/src/services/github/docs.md b/core/src/services/github/docs.md new file mode 100644 index 00000000000..c16245cfeb4 --- /dev/null +++ b/core/src/services/github/docs.md @@ -0,0 +1,54 @@ +## Capabilities + +This service can be used to: + +- [x] stat +- [x] read +- [x] write +- [ ] create_dir +- [x] delete +- [ ] copy +- [ ] rename +- [x] list +- [ ] scan +- [ ] presign +- [ ] blocking + +## Configuration + +- `root`: Set the work directory for backend +- `token`: Github access token +- `owner`: Github owner +- `repo`: Github repository + +You can refer to [`GithubBuilder`]'s docs for more information + +## Example + +### Via Builder + +```rust +use anyhow::Result; +use opendal::services::Github; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + // create backend builder + let mut builder = Github::default(); + + // set the storage root for OpenDAL + builder.root("/"); + // set the access token for Github API + builder.token("your_access_token"); + // set the owner for Github + builder.owner("your_owner") + // set the repository for Github + builder.repo("your_repo"); + + + let op: Operator = Operator::new(builder)?.finish(); + + Ok(()) +} +``` diff --git a/core/src/services/github/error.rs b/core/src/services/github/error.rs new file mode 100644 index 00000000000..edd52ce7078 --- /dev/null +++ b/core/src/services/github/error.rs @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Buf; +use http::Response; +use serde::Deserialize; + +use crate::raw::*; +use crate::Error; +use crate::ErrorKind; +use crate::Result; + +#[derive(Default, Debug, Deserialize)] +#[allow(dead_code)] +struct GithubError { + error: GithubSubError, +} + +#[derive(Default, Debug, Deserialize)] +#[allow(dead_code)] +struct GithubSubError { + message: String, + documentation_url: String, +} + +/// Parse error response into Error. +pub async fn parse_error(resp: Response) -> Result { + let (parts, body) = resp.into_parts(); + let bs = body.bytes().await?; + + let (kind, retryable) = match parts.status.as_u16() { + 401 | 403 => (ErrorKind::PermissionDenied, false), + 404 => (ErrorKind::NotFound, false), + 304 | 412 => (ErrorKind::ConditionNotMatch, false), + // https://github.com/apache/opendal/issues/4146 + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/423 + // We should retry it when we get 423 error. + 423 => (ErrorKind::RateLimited, true), + // Service like Upyun could return 499 error with a message like: + // Client Disconnect, we should retry it. + 499 => (ErrorKind::Unexpected, true), + 500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true), + _ => (ErrorKind::Unexpected, false), + }; + + let (message, _github_content_err) = + serde_json::from_reader::<_, GithubError>(bs.clone().reader()) + .map(|github_content_err| (format!("{github_content_err:?}"), Some(github_content_err))) + .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None)); + + let mut err = Error::new(kind, &message); + + err = with_error_response_context(err, parts); + + if retryable { + err = err.set_temporary(); + } + + Ok(err) +} + +#[cfg(test)] +mod test { + use futures::stream; + use http::StatusCode; + + use super::*; + + #[tokio::test] + async fn test_parse_error() { + let err_res = vec![( + r#"{ + "message": "Not Found", + "documentation_url": "https://docs.github.com/rest/repos/contents#get-repository-content" + }"#, + ErrorKind::NotFound, + StatusCode::NOT_FOUND, + )]; + + for res in err_res { + let bs = bytes::Bytes::from(res.0); + let body = IncomingAsyncBody::new( + Box::new(oio::into_stream(stream::iter(vec![Ok(bs.clone())]))), + None, + ); + let resp = Response::builder().status(res.2).body(body).unwrap(); + + let err = parse_error(resp).await; + + assert!(err.is_ok()); + assert_eq!(err.unwrap().kind(), res.1); + } + } +} diff --git a/core/src/services/github/lister.rs b/core/src/services/github/lister.rs new file mode 100644 index 00000000000..320f664a02b --- /dev/null +++ b/core/src/services/github/lister.rs @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use async_trait::async_trait; + +use super::core::GithubCore; +use crate::raw::oio::Entry; +use crate::raw::*; +use crate::*; + +pub struct GithubLister { + core: Arc, + path: String, +} + +impl GithubLister { + pub fn new(core: Arc, path: &str) -> Self { + Self { + core, + + path: path.to_string(), + } + } +} + +#[async_trait] +impl oio::PageList for GithubLister { + async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> { + let entries = self.core.list(&self.path).await?; + + for entry in entries { + let path = build_rel_path(&self.core.root, &entry.path); + let entry = if entry.type_field == "dir" { + let path = format!("{}/", path); + Entry::new(&path, Metadata::new(EntryMode::DIR)) + } else { + if path.ends_with(".gitkeep") { + continue; + } + let m = Metadata::new(EntryMode::FILE) + .with_content_length(entry.size) + .with_etag(entry.sha); + Entry::new(&path, m) + }; + + ctx.entries.push_back(entry); + } + + ctx.done = true; + + Ok(()) + } +} diff --git a/core/src/services/github/mod.rs b/core/src/services/github/mod.rs new file mode 100644 index 00000000000..06418c19347 --- /dev/null +++ b/core/src/services/github/mod.rs @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod backend; +pub use backend::GithubBuilder as Github; +pub use backend::GithubConfig; + +mod core; +mod error; +mod lister; +mod writer; diff --git a/core/src/services/github/writer.rs b/core/src/services/github/writer.rs new file mode 100644 index 00000000000..d490d219ae0 --- /dev/null +++ b/core/src/services/github/writer.rs @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use async_trait::async_trait; +use http::StatusCode; + +use super::core::GithubCore; +use super::error::parse_error; +use crate::raw::*; +use crate::*; + +pub type GithubWriters = oio::OneShotWriter; + +pub struct GithubWriter { + core: Arc, + path: String, +} + +impl GithubWriter { + pub fn new(core: Arc, path: String) -> Self { + GithubWriter { core, path } + } +} + +#[async_trait] +impl oio::OneShotWrite for GithubWriter { + async fn write_once(&self, bs: &dyn oio::WriteBuf) -> Result<()> { + let bs = bs.bytes(bs.remaining()); + + let resp = self.core.upload(&self.path, bs).await?; + + let status = resp.status(); + + match status { + StatusCode::OK | StatusCode::CREATED => { + resp.into_body().consume().await?; + Ok(()) + } + _ => Err(parse_error(resp).await?), + } + } +} diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index de5c0cef022..e98245e1249 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -229,6 +229,13 @@ mod gdrive; #[cfg(feature = "services-gdrive")] pub use gdrive::Gdrive; +#[cfg(feature = "services-github")] +mod github; +#[cfg(feature = "services-github")] +pub use github::Github; +#[cfg(feature = "services-github")] +pub use github::GithubConfig; + #[cfg(feature = "services-dropbox")] mod dropbox; #[cfg(feature = "services-dropbox")] diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs index a5767f59e39..cca37f72959 100644 --- a/core/src/types/operator/builder.rs +++ b/core/src/types/operator/builder.rs @@ -199,6 +199,8 @@ impl Operator { Scheme::Ghac => Self::from_map::(map)?.finish(), #[cfg(feature = "services-gridfs")] Scheme::Gridfs => Self::from_map::(map)?.finish(), + #[cfg(feature = "services-github")] + Scheme::Github => Self::from_map::(map)?.finish(), #[cfg(feature = "services-hdfs")] Scheme::Hdfs => Self::from_map::(map)?.finish(), #[cfg(feature = "services-http")] diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs index 6620fa81454..7ae406bc6b7 100644 --- a/core/src/types/scheme.rs +++ b/core/src/types/scheme.rs @@ -151,6 +151,8 @@ pub enum Scheme { Mongodb, /// [gridfs](crate::services::gridfs): MongoDB Gridfs Services Gridfs, + /// [Github Contents][crate::services::Github]: Github contents support. + Github, /// [Native HDFS](crate::services::hdfs_native): Hdfs Native service, using rust hdfs-native client for hdfs HdfsNative, /// Custom that allow users to implement services outside of OpenDAL. @@ -338,6 +340,7 @@ impl FromStr for Scheme { "gdrive" => Ok(Scheme::Gdrive), "ghac" => Ok(Scheme::Ghac), "gridfs" => Ok(Scheme::Gridfs), + "github" => Ok(Scheme::Github), "hdfs" => Ok(Scheme::Hdfs), "http" | "https" => Ok(Scheme::Http), "huggingface" | "hf" => Ok(Scheme::Huggingface), @@ -422,6 +425,7 @@ impl From for &'static str { Scheme::Postgresql => "postgresql", Scheme::Mysql => "mysql", Scheme::Gdrive => "gdrive", + Scheme::Github => "github", Scheme::Dropbox => "dropbox", Scheme::Redis => "redis", Scheme::Rocksdb => "rocksdb",