feat(services/github): add github contents support

core/Cargo.toml

@@ -156,6 +156,7 @@ services-gcs = [
 ]
 services-gdrive = ["internal-path-cache"]
 services-ghac = []
+services-github = []
 services-gridfs = ["dep:mongodb"]
 services-hdfs = ["dep:hdrs"]
 services-http = []

core/src/services/github/backend.rs

@@ -0,0 +1,324 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+
+use super::core::GithubCore;
+use super::error::parse_error;
+use super::lister::GithubLister;
+use super::writer::GithubWriter;
+use super::writer::GithubWriters;
+use crate::raw::*;
+use crate::*;
+
+/// Config for backblaze Github services support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct GithubConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+ /// Github access_token.
+ ///
+ /// required.
+ pub token: String,
+ /// Github repo owner.
+ ///
+ /// required.
+ pub owner: String,
+ /// Github repo name.
+ ///
+ /// required.
+ pub repo: String,
+}
+
+impl Debug for GithubConfig {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ let mut d = f.debug_struct("GithubConfig");
+
+ d.field("root", &self.root)
+ .field("owner", &self.owner)
+ .field("repo", &self.repo);
+
+ d.finish_non_exhaustive()
+ }
+}
+
+/// [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct GithubBuilder {
+ config: GithubConfig,
+
+ http_client: Option<HttpClient>,
+}
+
+impl Debug for GithubBuilder {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ let mut d = f.debug_struct("GithubBuilder");
+
+ d.field("config", &self.config);
+ d.finish_non_exhaustive()
+ }
+}
+
+impl GithubBuilder {
+ /// Set root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub fn root(&mut self, root: &str) -> &mut Self {
+ self.config.root = if root.is_empty() {
+ None
+ } else {
+ Some(root.to_string())
+ };
+
+ self
+ }
+
+ /// Github access_token.
+ ///
+ /// required.
+ pub fn token(&mut self, token: &str) -> &mut Self {
+ self.config.token = token.to_string();
+
+ self
+ }
+
+ /// Set Github repo owner.
+ pub fn owner(&mut self, owner: &str) -> &mut Self {
+ self.config.owner = owner.to_string();
+
+ self
+ }
+
+ /// Set Github repo name.
+ pub fn repo(&mut self, repo: &str) -> &mut Self {
+ self.config.repo = repo.to_string();
+
+ self
+ }
+
+ /// Specify the http client that used by this service.
+ ///
+ /// # Notes
+ ///
+ /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
+ /// during minor updates.
+ pub fn http_client(&mut self, client: HttpClient) -> &mut Self {
+ self.http_client = Some(client);
+ self
+ }
+}
+
+impl Builder for GithubBuilder {
+ const SCHEME: Scheme = Scheme::Github;
+ type Accessor = GithubBackend;
+
+ /// Converts a HashMap into an GithubBuilder instance.
+ ///
+ /// # Arguments
+ ///
+ /// * `map` - A HashMap containing the configuration values.
+ ///
+ /// # Returns
+ ///
+ /// Returns an instance of GithubBuilder.
+ fn from_map(map: HashMap<String, String>) -> Self {
+ // Deserialize the configuration from the HashMap.
+ let config = GithubConfig::deserialize(ConfigDeserializer::new(map))
+ .expect("config deserialize must succeed");
+
+ // Create an GithubBuilder instance with the deserialized config.
+ GithubBuilder {
+ config,
+ http_client: None,
+ }
+ }
+
+ /// Builds the backend and returns the result of GithubBackend.
+ fn build(&mut self) -> Result<Self::Accessor> {
+ debug!("backend build started: {:?}", &self);
+
+ let root = normalize_root(&self.config.root.clone().unwrap_or_default());
+ debug!("backend use root {}", &root);
+
+ // Handle token.
+ if self.config.token.is_empty() {
+ return Err(Error::new(ErrorKind::ConfigInvalid, "token is empty")
+ .with_operation("Builder::build")
+ .with_context("service", Scheme::Github));
+ }
+
+ // Handle owner.
+ if self.config.owner.is_empty() {
+ return Err(Error::new(ErrorKind::ConfigInvalid, "owner is empty")
+ .with_operation("Builder::build")
+ .with_context("service", Scheme::Github));
+ }
+
+ debug!("backend use owner {}", &self.config.owner);
+
+ // Handle repo.
+ if self.config.repo.is_empty() {
+ return Err(Error::new(ErrorKind::ConfigInvalid, "repo is empty")
+ .with_operation("Builder::build")
+ .with_context("service", Scheme::Github));
+ }
+
+ debug!("backend use repo {}", &self.config.repo);
+
+ let client = if let Some(client) = self.http_client.take() {
+ client
+ } else {
+ HttpClient::new().map_err(|err| {
+ err.with_operation("Builder::build")
+ .with_context("service", Scheme::Github)
+ })?
+ };
+
+ Ok(GithubBackend {
+ core: Arc::new(GithubCore {
+ root,
+ token: self.config.token.clone(),
+ owner: self.config.owner.clone(),
+ repo: self.config.repo.clone(),
+ client,
+ }),
+ })
+ }
+}
+
+/// Backend for Github services.
+#[derive(Debug, Clone)]
+pub struct GithubBackend {
+ core: Arc<GithubCore>,
+}
+
+#[async_trait]
+impl Accessor for GithubBackend {
+ type Reader = IncomingAsyncBody;
+
+ type Writer = GithubWriters;
+
+ type Lister = oio::PageLister<GithubLister>;
+
+ type BlockingReader = ();
+
+ type BlockingWriter = ();
+
+ type BlockingLister = ();
+
+ fn info(&self) -> AccessorInfo {
+ let mut am = AccessorInfo::default();
+ am.set_scheme(Scheme::Github)
+ .set_root(&self.core.root)
+ .set_native_capability(Capability {
+ stat: true,
+
+ read: true,
+
+ create_dir: true,
+
+ write: true,
+ write_can_empty: true,
+
+ delete: true,
+
+ list: true,
+
+ ..Default::default()
+ });
+
+ am
+ }
+
+ async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
+ let empty_bytes = bytes::Bytes::new();
+
+ let resp = self
+ .core
+ .upload(&format!("{}.gitkeep", path), empty_bytes)
+ .await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::CREATED => Ok(RpCreateDir::default()),
+ _ => Err(parse_error(resp).await?),
+ }
+ }
+
+ async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> {
+ let resp = self.core.stat(path).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => parse_into_metadata(path, resp.headers()).map(RpStat::new),
+ _ => Err(parse_error(resp).await?),
+ }
+ }
+
+ async fn read(&self, path: &str, _args: OpRead) -> Result<(RpRead, Self::Reader)> {
+ let resp = self.core.get(path).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let size = parse_content_length(resp.headers())?;
+ let range = parse_content_range(resp.headers())?;
+ Ok((
+ RpRead::new().with_size(size).with_range(range),
+ resp.into_body(),
+ ))
+ }
+ _ => Err(parse_error(resp).await?),
+ }
+ }
+
+ async fn write(&self, path: &str, _args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+ let writer = GithubWriter::new(self.core.clone(), path.to_string());
+
+ let w = oio::OneShotWriter::new(writer);
+
+ Ok((RpWrite::default(), w))
+ }
+
+ async fn delete(&self, path: &str, _: OpDelete) -> Result<RpDelete> {
+ match self.core.delete(path).await {
+ Ok(_) => Ok(RpDelete::default()),
+ Err(err) => Err(err),
+ }
+ }
+
+ async fn list(&self, path: &str, _args: OpList) -> Result<(RpList, Self::Lister)> {
+ let l = GithubLister::new(self.core.clone(), path);
+ Ok((RpList::default(), oio::PageLister::new(l)))
+ }
+}