Add package publish

Cargo.toml

@@ -100,7 +100,7 @@ quote = "1"
 ra_ap_toolchain = "0.0.218"
 rayon = "1.10"
 redb = "2.1.2"
-reqwest = { version = "0.11", features = ["gzip", "brotli", "deflate", "json", "stream"], default-features = false }
+reqwest = { version = "0.11", features = ["gzip", "brotli", "deflate", "json", "stream", "multipart"], default-features = false }
 salsa = { package = "rust-analyzer-salsa", version = "0.17.0-pre.6" }
 semver = { version = "1", features = ["serde"] }
 serde = { version = "1", features = ["serde_derive"] }

extensions/scarb-snforge-test-collector/src/main.rs

@@ -52,7 +52,7 @@ fn main() -> Result<()> {
 
         // artifact saved to `{target_dir}/{profile_name}/{package_name}.sierra.json`
         let output_path =
-            snforge_target_dir.join(&format!("{}.snforge_sierra.json", package_metadata.name));
+            snforge_target_dir.join(format!("{}.snforge_sierra.json", package_metadata.name));
         let output_file = File::options()
             .create(true)
             .write(true)

scarb/src/bin/scarb/args.rs

@@ -484,7 +484,7 @@ pub struct PackageArgs {
 pub struct PublishArgs {
     /// Registry index URL to upload the package to.
     #[arg(long, value_name = "URL")]
-    pub index: Url,
+    pub index: Option<Url>,
 
     #[clap(flatten)]
     pub shared_args: PackageSharedArgs,

scarb/src/bin/scarb/commands/publish.rs

@@ -9,9 +9,13 @@ use crate::args::PublishArgs;
 pub fn run(args: PublishArgs, config: &Config) -> Result<()> {
     let ws = ops::read_workspace(config.manifest_path(), config)?;
     let package = args.packages_filter.match_one(&ws)?;
+    let index = match args.index {
+        Some(index) => index,
+        None => package.id.source_id.url.clone(),
+    };
 
     let ops = PublishOpts {
-        index_url: args.index,
+        index_url: index,
         package_opts: PackageOpts {
             allow_dirty: args.shared_args.allow_dirty,
             verify: !args.shared_args.no_verify,

scarb/src/core/registry/client/http.rs

@@ -1,12 +1,17 @@
+use std::env;
+use std::path::Path;
 use std::str::FromStr;
+use std::time::Duration;
 
-use anyhow::{bail, ensure, Context, Result};
+use anyhow::{anyhow, bail, ensure, Context, Result};
 use async_trait::async_trait;
 use futures::StreamExt;
 use reqwest::header::{
-    HeaderMap, HeaderName, HeaderValue, ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED,
+    HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH,
+    LAST_MODIFIED,
 };
-use reqwest::{Response, StatusCode};
+use reqwest::multipart::{Form, Part};
+use reqwest::{Body, Response, StatusCode};
 use tokio::io;
 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufWriter};
 use tokio::sync::OnceCell;
@@ -15,14 +20,13 @@ use tracing::{debug, trace, warn};
 use scarb_ui::components::Status;
 
 use crate::core::registry::client::{
-    CreateScratchFileCallback, RegistryClient, RegistryDownload, RegistryResource,
+    CreateScratchFileCallback, RegistryClient, RegistryDownload, RegistryResource, RegistryUpload,
 };
 use crate::core::registry::index::{IndexConfig, IndexRecords};
 use crate::core::{Config, Package, PackageId, PackageName, SourceId};
 use crate::flock::{FileLockGuard, Filesystem};
 
 // TODO(mkaput): Progressbar.
-// TODO(mkaput): Request timeout.
 
 /// Remote registry served by the HTTP-based registry API.
 pub struct HttpRegistryClient<'c> {
@@ -146,12 +150,81 @@ impl<'c> RegistryClient for HttpRegistryClient<'c> {
     }
 
     async fn supports_publish(&self) -> Result<bool> {
-        // TODO(mkaput): Publishing to HTTP registries is not implemented yet.
-        Ok(false)
+        Ok(self.index_config.load().await?.upload.is_some())
     }
 
-    async fn publish(&self, _package: Package, _tarball: FileLockGuard) -> Result<()> {
-        todo!("Publishing to HTTP registries is not implemented yet.")
+    async fn publish(&self, package: Package, tarball: FileLockGuard) -> Result<RegistryUpload> {
+        let auth_token = env::var("SCARB_REGISTRY_AUTH_TOKEN").map_err(|_| {
+            anyhow!(
+                "missing authentication token. \
+            help: make sure SCARB_REGISTRY_AUTH_TOKEN environment variable is set"
+            )
+        })?;
+
+        let path = tarball.path();
+        ensure!(
+            Path::new(path).exists(),
+            "cannot upload package - file does not exist at path: {}",
+            path
+        );
+
+        let file = tarball.into_async().into_file();
+        let metadata = file.metadata().await?;
+        ensure!(
+            metadata.len() < 5 * 1024 * 1024,
+            "package cannot be larger than `5` MB: found `{}`",
+            &metadata.len() / 1024 / 1024
+        );
+
+        let index_config = self.index_config.load().await?;
+
+        let file_part = Part::stream(Body::from(file))
+            .file_name(format!("{}_{}", &package.id.name, &package.id.version));
+        let form = Form::new().part("file", file_part);
+
+        let response = self
+            .config
+            .online_http()?
+            .post(
+                index_config
+                    .upload
+                    .clone()
+                    .ok_or_else(|| anyhow!("failed to fetch registry upload url"))?,
+            )
+            .header(AUTHORIZATION, format!("Bearer {}", auth_token))
+            .multipart(form)
+            .timeout(Duration::from_secs(60))
+            .send()
+            .await?;
+
+        match response.status() {
+            StatusCode::UNAUTHORIZED => Err(RegistryUpload::Unauthorized)
+                .map_err(|_| anyhow!("invalid authentication token")),
+            StatusCode::FORBIDDEN => Err(RegistryUpload::CannotPublish)
+                .map_err(|_| anyhow!("missing upload permissions or not the package owner")),
+            StatusCode::BAD_REQUEST => Err(RegistryUpload::VersionExists)
+                .map_err(|_| anyhow!("package `{}` already exists", &package.id)),
+            StatusCode::UNPROCESSABLE_ENTITY => {
+                Err(RegistryUpload::Corrupted).map_err(|_| anyhow!("file corrupted during upload"))
+            }
+            StatusCode::OK => Ok(RegistryUpload::Success),
+            _ => {
+                let trace_id = response
+                    .headers()
+                    .get("x-cloud-trace-context")
+                    .and_then(|v| v.to_str().ok());
+
+                let error_message = match trace_id {
+                    Some(id) => format!(
+                        "upload failed with an unexpected error (trace-id: {:?})",
+                        id
+                    ),
+                    None => "upload failed with an unexpected error".to_string(),
+                };
+
+                Err(RegistryUpload::Failed).map_err(|_| anyhow!(error_message))
+            }
+        }
     }
 }
 

scarb/src/core/registry/client/local.rs

@@ -12,7 +12,7 @@ use tracing::trace;
 use url::Url;
 
 use crate::core::registry::client::{
-    CreateScratchFileCallback, RegistryClient, RegistryDownload, RegistryResource,
+    CreateScratchFileCallback, RegistryClient, RegistryDownload, RegistryResource, RegistryUpload,
 };
 use crate::core::registry::index::{IndexDependency, IndexRecord, IndexRecords, TemplateUrl};
 use crate::core::{Checksum, Config, Digest, Package, PackageId, PackageName, Summary};
@@ -151,7 +151,7 @@ impl RegistryClient for LocalRegistryClient<'_> {
         Ok(true)
     }
 
-    async fn publish(&self, package: Package, tarball: FileLockGuard) -> Result<()> {
+    async fn publish(&self, package: Package, tarball: FileLockGuard) -> Result<RegistryUpload> {
         let summary = package.manifest.summary.clone();
         let records_path = self.records_path(&summary.package_id.name);
         let dl_path = self.dl_path(summary.package_id);
@@ -167,7 +167,7 @@ fn publish_impl(
     tarball: FileLockGuard,
     records_path: PathBuf,
     dl_path: PathBuf,
-) -> Result<(), Error> {
+) -> Result<RegistryUpload, Error> {
     let checksum = Digest::recommended().update_read(tarball.deref())?.finish();
     let tarball_path = tarball.path().to_owned();
 
@@ -189,7 +189,7 @@ fn publish_impl(
     })
     .with_context(|| format!("failed to edit records file: {}", records_path.display()))?;
 
-    Ok(())
+    Ok(RegistryUpload::Success)
 }
 
 fn build_record(summary: Summary, checksum: Checksum) -> IndexRecord {

scarb/src/core/registry/client/mod.rs

@@ -33,6 +33,23 @@ pub enum RegistryDownload<T> {
     Download(T),
 }
 
+/// Result from uploading files to a registry.
+#[derive(Debug)]
+pub enum RegistryUpload {
+    /// Missing or invalid authentication token.
+    Unauthorized,
+    /// Missing upload permissions or not the package owner.
+    CannotPublish,
+    /// Package version already exists.
+    VersionExists,
+    /// File corrupted during upload.
+    Corrupted,
+    /// Upload failed for other reasons.
+    Failed,
+    /// Upload successful.
+    Success,
+}
+
 pub type CreateScratchFileCallback = Box<dyn FnOnce(&Config) -> Result<FileLockGuard> + Send>;
 
 #[async_trait]
@@ -88,10 +105,5 @@ pub trait RegistryClient: Send + Sync {
     /// The `package` argument must correspond to just packaged `tarball` file.
     /// The client is free to use information within `package` to send to the registry.
     /// Package source is not required to match the registry the package is published to.
-    async fn publish(&self, package: Package, tarball: FileLockGuard) -> Result<()> {
-        // Silence clippy warnings without using _ in argument names.
-        let _ = package;
-        let _ = tarball;
-        unreachable!("This registry does not support publishing.")
-    }
+    async fn publish(&self, package: Package, tarball: FileLockGuard) -> Result<RegistryUpload>;
 }

scarb/src/core/registry/index/config.rs

@@ -1,5 +1,6 @@
 use anyhow::{ensure, Result};
 use serde::{Deserialize, Serialize};
+use url::Url;
 
 use crate::core::registry::index::{BaseUrl, TemplateUrl};
 
@@ -12,6 +13,7 @@ use crate::core::registry::index::{BaseUrl, TemplateUrl};
 ///   "version": 1,
 ///   "api": "https://example.com/api/v1",
 ///   "dl": "https://example.com/api/v1/download/{package}/{version}",
+///   "upload": "https://example.com/api/v1/packages/new",
 ///   "index": "https://example.com/index/{prefix}/{package}.json"
 /// }
 /// ```
@@ -41,6 +43,11 @@ pub struct IndexConfig {
     /// Usually, this is a location where `config.json` lies, as the rest of index files resides
     /// alongside config.
     pub index: TemplateUrl,
+
+    /// Upload endpoint for all packages.
+    ///
+    /// If this is `None`, the registry does not support package uploads.
+    pub upload: Option<Url>,
 }
 
 impl IndexConfig {
@@ -77,6 +84,7 @@ mod tests {
         let expected = IndexConfig {
             version: Default::default(),
             api: Some("https://example.com/api/v1/".parse().unwrap()),
+            upload: Some("https://example.com/api/v1/packages/new".parse().unwrap()),
             dl: TemplateUrl::new("https://example.com/api/v1/download/{package}/{version}"),
             index: TemplateUrl::new("https://example.com/index/{prefix}/{package}.json"),
         };
@@ -85,6 +93,7 @@ mod tests {
             r#"{
               "version": 1,
               "api": "https://example.com/api/v1",
+              "upload": "https://example.com/api/v1/packages/new",
               "dl": "https://example.com/api/v1/download/{package}/{version}",
               "index": "https://example.com/index/{prefix}/{package}.json"
             }"#,

scarb/src/flock.rs

@@ -100,6 +100,12 @@ impl AsyncFileLockGuard {
         self.lock_kind
     }
 
+    pub fn into_file(mut self) -> tokio::fs::File {
+        self.file
+            .take()
+            .expect("failed to take ownership of the file")
+    }
+
     pub async fn into_sync(mut self) -> FileLockGuard {
         FileLockGuard {
             file: match self.file.take() {

scarb/src/ops/publish.rs

@@ -4,6 +4,7 @@ use url::Url;
 
 use scarb_ui::components::Status;
 
+use crate::core::registry::client::RegistryUpload;
 use crate::core::{PackageId, SourceId, Workspace};
 use crate::ops;
 use crate::sources::RegistrySource;
@@ -51,10 +52,18 @@ pub fn publish(package_id: PackageId, opts: &PublishOpts, ws: &Workspace<'_>) ->
         .print(Status::new("Uploading", &dest_package_id.to_string()));
 
     ws.config().tokio_handle().block_on(async {
-        registry_client.publish(package, tarball).await
+        let upload = registry_client.publish(package, tarball).await;
+        match upload {
+            Ok(RegistryUpload::Success) => {
+                ws.config().ui().print(Status::new(
+                    "Published",
+                    format!("{}", &dest_package_id).as_str(),
+                ));
+                Ok(())
+            }
+            _ => upload.map(|_| ()),
+        }
 
         // TODO(mkaput): Wait for publish here.
-    })?;
-
-    Ok(())
+    })
 }

scarb/tests/http_registry.rs

@@ -12,7 +12,7 @@ use scarb_test_support::registry::http::HttpRegistry;
 
 #[test]
 fn usage() {
-    let mut registry = HttpRegistry::serve();
+    let mut registry = HttpRegistry::serve(None);
     registry.publish(|t| {
         ProjectBuilder::start()
             .name("bar")
@@ -89,7 +89,7 @@ fn usage() {
 
 #[test]
 fn publish_verified() {
-    let mut registry = HttpRegistry::serve();
+    let mut registry = HttpRegistry::serve(None);
     registry.publish_verified(|t| {
         ProjectBuilder::start()
             .name("bar")
@@ -166,7 +166,7 @@ fn publish_verified() {
 
 #[test]
 fn not_found() {
-    let mut registry = HttpRegistry::serve();
+    let mut registry = HttpRegistry::serve(None);
     registry.publish(|t| {
         // Publish a package so that the directory hierarchy is created.
         // Note, however, that we declare a dependency on baZ.
@@ -228,7 +228,7 @@ fn not_found() {
 
 #[test]
 fn missing_config_json() {
-    let registry = HttpRegistry::serve();
+    let registry = HttpRegistry::serve(None);
     fs::remove_file(registry.child("api/v1/index/config.json")).unwrap();
 
     let t = TempDir::new().unwrap();
@@ -270,7 +270,7 @@ fn missing_config_json() {
 fn caching() {
     let cache_dir = TempDir::new().unwrap();
 
-    let mut registry = HttpRegistry::serve();
+    let mut registry = HttpRegistry::serve(None);
     registry.publish(|t| {
         ProjectBuilder::start()
             .name("bar")

scarb/tests/local_registry.rs

@@ -161,6 +161,7 @@ fn publish() {
             [..]  Finished release target(s) in [..]
             [..]  Packaged [..]
             [..] Uploading {name} v{version} (registry+file://[..]/index/)
+            [..] Published {name} v{version} (registry+file://[..]/index/)
             "#});
     };