Store build logs in S3

src/db/add_package.rs

@@ -175,17 +175,14 @@ pub(crate) fn add_build_into_database(
 ) -> Result<i32> {
     debug!("Adding build into database");
     let rows = conn.query(
-        "INSERT INTO builds (rid, rustc_version,
-                                                    cratesfyi_version,
-                                                    build_status, output)
-                                VALUES ($1, $2, $3, $4, $5)
-                                RETURNING id",
+        "INSERT INTO builds (rid, rustc_version, cratesfyi_version, build_status)
+        VALUES ($1, $2, $3, $4)
+        RETURNING id",
         &[
             &release_id,
             &res.rustc_version,
             &res.docsrs_version,
             &res.successful,
-            &res.build_log,
         ],
     )?;
     Ok(rows[0].get(0))

src/docbuilder/rustwide_builder.rs

@@ -322,7 +322,7 @@ impl RustwideBuilder {
 
         let local_storage = tempfile::Builder::new().prefix("docsrs-docs").tempdir()?;
 
-        let res = build_dir
+        let successful = build_dir
             .build(&self.toolchain, &krate, self.prepare_sandbox(&limits))
             .run(|build| {
                 use docsrs_metadata::BuildTargets;
@@ -410,25 +410,27 @@ impl RustwideBuilder {
                     github_repo,
                 )?;
 
-                if let Some(doc_coverage) = res.result.doc_coverage {
+                if let Some(doc_coverage) = res.doc_coverage {
                     add_doc_coverage(&mut conn, release_id, doc_coverage)?;
                 }
 
-                add_build_into_database(&mut conn, release_id, &res.result)?;
+                let build_id = add_build_into_database(&mut conn, release_id, &res.result)?;
+                let build_log_path = format!("build-logs/{}/{}.txt", build_id, default_target);
+                self.storage.store_one(build_log_path, res.build_log)?;
 
                 // Some crates.io crate data is mutable, so we proactively update it during a release
                 match self.index.api().get_crate_data(name) {
                     Ok(crate_data) => update_crate_data_in_database(&mut conn, name, &crate_data)?,
                     Err(err) => warn!("{:#?}", err),
                 }
 
-                Ok(res)
+                Ok(res.result.successful)
             })?;
 
         build_dir.purge()?;
         krate.purge_from_cache(&self.workspace)?;
         local_storage.close()?;
-        Ok(res.result.successful)
+        Ok(successful)
     }
 
     fn build_target(
@@ -556,13 +558,13 @@ impl RustwideBuilder {
 
         Ok(FullBuildResult {
             result: BuildResult {
-                build_log: storage.to_string(),
                 rustc_version: self.rustc_version.clone(),
                 docsrs_version: format!("docsrs {}", crate::BUILD_VERSION),
                 successful,
-                doc_coverage,
             },
+            doc_coverage,
             cargo_metadata,
+            build_log: storage.to_string(),
             target: target.to_string(),
         })
     }
@@ -705,6 +707,8 @@ struct FullBuildResult {
     result: BuildResult,
     target: String,
     cargo_metadata: CargoMetadata,
+    doc_coverage: Option<DocCoverage>,
+    build_log: String,
 }
 
 #[derive(Clone, Copy)]
@@ -724,7 +728,5 @@ pub(crate) struct DocCoverage {
 pub(crate) struct BuildResult {
     pub(crate) rustc_version: String,
     pub(crate) docsrs_version: String,
-    pub(crate) build_log: String,
     pub(crate) successful: bool,
-    pub(crate) doc_coverage: Option<DocCoverage>,
 }

src/storage/mod.rs

@@ -149,9 +149,6 @@ impl Storage {
 
     // Store all files in `root_dir` into the backend under `prefix`.
     //
-    // If the environment is configured with S3 credentials, this will upload to S3;
-    // otherwise, this will store files in the database.
-    //
     // This returns (map<filename, mime type>, set<compression algorithms>).
     pub(crate) fn store_all(
         &self,
@@ -199,10 +196,36 @@ impl Storage {
         self.store_inner(blobs.into_iter().map(Ok))
     }
 
+    // Store file into the backend at the given path (also used to detect mime type), returns the
+    // chosen compression algorithm
+    pub(crate) fn store_one(
+        &self,
+        path: impl Into<String>,
+        content: impl Into<Vec<u8>>,
+    ) -> Result<CompressionAlgorithm, Error> {
+        let path = path.into();
+        let content = content.into();
+        let alg = CompressionAlgorithm::default();
+        let content = compress(&*content, alg)?;
+        let mime = detect_mime(&path)?.to_owned();
+
+        self.store_inner(std::iter::once(Ok(Blob {
+            path,
+            mime,
+            content,
+            compression: Some(alg),
+            // this field is ignored by the backend
+            date_updated: Utc::now(),
+        })))?;
+
+        Ok(alg)
+    }
+
     fn store_inner(
         &self,
-        mut blobs: impl Iterator<Item = Result<Blob, Error>>,
+        blobs: impl IntoIterator<Item = Result<Blob, Error>>,
     ) -> Result<(), Error> {
+        let mut blobs = blobs.into_iter();
         self.transaction(|trans| {
             loop {
                 let batch: Vec<_> = blobs
@@ -249,13 +272,13 @@ trait StorageTransaction {
     fn complete(self: Box<Self>) -> Result<(), Error>;
 }
 
-fn detect_mime(file_path: &Path) -> Result<&'static str, Error> {
-    let mime = mime_guess::from_path(file_path)
+fn detect_mime(file_path: impl AsRef<Path>) -> Result<&'static str, Error> {
+    let mime = mime_guess::from_path(file_path.as_ref())
         .first_raw()
         .unwrap_or("text/plain");
     Ok(match mime {
         "text/plain" | "text/troff" | "text/x-markdown" | "text/x-rust" | "text/x-toml" => {
-            match file_path.extension().and_then(OsStr::to_str) {
+            match file_path.as_ref().extension().and_then(OsStr::to_str) {
                 Some("md") => "text/markdown",
                 Some("rs") => "text/rust",
                 Some("markdown") => "text/markdown",