44//! via `x.py dist hash-and-sign`; the cmdline arguments are set up
55//! by rustbuild (in `src/bootstrap/dist.rs`).
66
7+ mod manifest;
78mod versions;
89
10+ use crate :: manifest:: { Component , FileHash , Manifest , Package , Rename , Target } ;
911use crate :: versions:: { PkgType , Versions } ;
10- use serde :: Serialize ;
11- use std :: collections :: BTreeMap ;
12- use std:: collections:: HashMap ;
12+ use rayon :: prelude :: * ;
13+ use sha2 :: Digest ;
14+ use std:: collections:: { BTreeMap , HashMap , HashSet } ;
1315use std:: env;
16+ use std:: error:: Error ;
1417use std:: fs:: { self , File } ;
15- use std:: io:: { self , Read , Write } ;
18+ use std:: io:: { self , BufReader , Read , Write } ;
1619use std:: path:: { Path , PathBuf } ;
1720use std:: process:: { Command , Stdio } ;
21+ use std:: sync:: Mutex ;
22+ use std:: time:: Instant ;
1823
1924static HOSTS : & [ & str ] = & [
2025 "aarch64-unknown-linux-gnu" ,
@@ -167,57 +172,6 @@ static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
167172
168173static NIGHTLY_ONLY_COMPONENTS : & [ & str ] = & [ "miri-preview" , "rust-analyzer-preview" ] ;
169174
170- #[ derive( Serialize ) ]
171- #[ serde( rename_all = "kebab-case" ) ]
172- struct Manifest {
173- manifest_version : String ,
174- date : String ,
175- pkg : BTreeMap < String , Package > ,
176- renames : BTreeMap < String , Rename > ,
177- profiles : BTreeMap < String , Vec < String > > ,
178- }
179-
180- #[ derive( Serialize ) ]
181- struct Package {
182- version : String ,
183- git_commit_hash : Option < String > ,
184- target : BTreeMap < String , Target > ,
185- }
186-
187- #[ derive( Serialize ) ]
188- struct Rename {
189- to : String ,
190- }
191-
192- #[ derive( Serialize , Default ) ]
193- struct Target {
194- available : bool ,
195- url : Option < String > ,
196- hash : Option < String > ,
197- xz_url : Option < String > ,
198- xz_hash : Option < String > ,
199- components : Option < Vec < Component > > ,
200- extensions : Option < Vec < Component > > ,
201- }
202-
203- impl Target {
204- fn unavailable ( ) -> Self {
205- Self :: default ( )
206- }
207- }
208-
209- #[ derive( Serialize ) ]
210- struct Component {
211- pkg : String ,
212- target : String ,
213- }
214-
215- impl Component {
216- fn from_str ( pkg : & str , target : & str ) -> Self {
217- Self { pkg : pkg. to_string ( ) , target : target. to_string ( ) }
218- }
219- }
220-
221175macro_rules! t {
222176 ( $e: expr) => {
223177 match $e {
@@ -232,25 +186,33 @@ struct Builder {
232186
233187 input : PathBuf ,
234188 output : PathBuf ,
235- gpg_passphrase : String ,
236- digests : BTreeMap < String , String > ,
237189 s3_address : String ,
238190 date : String ,
239191
240- should_sign : bool ,
192+ legacy : bool ,
193+ legacy_gpg_passphrase : String ,
241194}
242195
243196fn main ( ) {
244- // Avoid signing packages while manually testing
245- // Do NOT set this envvar in CI
246- let should_sign = env:: var ( "BUILD_MANIFEST_DISABLE_SIGNING" ) . is_err ( ) ;
247-
248- // Safety check to ensure signing is always enabled on CI
249- // The CI environment variable is set by both Travis and AppVeyor
250- if !should_sign && env:: var ( "CI" ) . is_ok ( ) {
251- println ! ( "The 'BUILD_MANIFEST_DISABLE_SIGNING' env var can't be enabled on CI." ) ;
252- println ! ( "If you're not running this on CI, unset the 'CI' env var." ) ;
253- panic ! ( ) ;
197+ // Up until Rust 1.48 the release process relied on build-manifest to create the SHA256
198+ // checksums of released files and to sign the tarballs. That was moved over to promote-release
199+ // in time for the branching of Rust 1.48, but the old release process still had to work the
200+ // old way.
201+ //
202+ // When running build-manifest through the old ./x.py dist hash-and-sign the environment
203+ // variable will be set, enabling the legacy behavior of generating the .sha256 files and
204+ // signing the tarballs.
205+ //
206+ // Once the old release process is fully decommissioned, the environment variable, all the
207+ // related code in this tool and ./x.py dist hash-and-sign can be removed.
208+ let legacy = env:: var ( "BUILD_MANIFEST_LEGACY" ) . is_ok ( ) ;
209+
210+ // Avoid overloading the old server in legacy mode.
211+ if legacy {
212+ rayon:: ThreadPoolBuilder :: new ( )
213+ . num_threads ( 1 )
214+ . build_global ( )
215+ . expect ( "failed to initialize Rayon" ) ;
254216 }
255217
256218 let mut args = env:: args ( ) . skip ( 1 ) ;
@@ -263,7 +225,7 @@ fn main() {
263225
264226 // Do not ask for a passphrase while manually testing
265227 let mut passphrase = String :: new ( ) ;
266- if should_sign {
228+ if legacy {
267229 // `x.py` passes the passphrase via stdin.
268230 t ! ( io:: stdin( ) . read_to_string( & mut passphrase) ) ;
269231 }
@@ -273,20 +235,21 @@ fn main() {
273235
274236 input,
275237 output,
276- gpg_passphrase : passphrase,
277- digests : BTreeMap :: new ( ) ,
278238 s3_address,
279239 date,
280240
281- should_sign,
241+ legacy,
242+ legacy_gpg_passphrase : passphrase,
282243 }
283244 . build ( ) ;
284245}
285246
286247impl Builder {
287248 fn build ( & mut self ) {
288249 self . check_toolstate ( ) ;
289- self . digest_and_sign ( ) ;
250+ if self . legacy {
251+ self . digest_and_sign ( ) ;
252+ }
290253 let manifest = self . build_manifest ( ) ;
291254
292255 let rust_version = self . versions . package_version ( & PkgType :: Rust ) . unwrap ( ) ;
@@ -324,10 +287,9 @@ impl Builder {
324287 /// Hash all files, compute their signatures, and collect the hashes in `self.digests`.
325288fn digest_and_sign ( & mut self ) {
326289 for file in t ! ( self . input. read_dir( ) ) . map ( |e| t ! ( e) . path ( ) ) {
327- let filename = file. file_name ( ) . unwrap ( ) . to_str ( ) . unwrap ( ) ;
328- let digest = self . hash ( & file) ;
290+ file. file_name ( ) . unwrap ( ) . to_str ( ) . unwrap ( ) ;
291+ self . hash ( & file) ;
329292 self . sign ( & file) ;
330- assert ! ( self . digests. insert( filename. to_string( ) , digest) . is_none( ) ) ;
331293 }
332294 }
333295
@@ -343,6 +305,9 @@ impl Builder {
343305 self . add_profiles_to ( & mut manifest) ;
344306 self . add_renames_to ( & mut manifest) ;
345307 manifest. pkg . insert ( "rust" . to_string ( ) , self . rust_package ( & manifest) ) ;
308+
309+ self . fill_missing_hashes ( & mut manifest) ;
310+
346311 manifest
347312 }
348313
@@ -438,9 +403,12 @@ impl Builder {
438403
439404 fn target_host_combination ( & mut self , host : & str , manifest : & Manifest ) -> Option < Target > {
440405 let filename = self . versions . tarball_name ( & PkgType :: Rust , host) . unwrap ( ) ;
441- let digest = self . digests . remove ( & filename) ?;
442- let xz_filename = filename. replace ( ".tar.gz" , ".tar.xz" ) ;
443- let xz_digest = self . digests . remove ( & xz_filename) ;
406+
407+ let mut target = Target :: from_compressed_tar ( self , & filename) ;
408+ if !target. available {
409+ return None ;
410+ }
411+
444412 let mut components = Vec :: new ( ) ;
445413 let mut extensions = Vec :: new ( ) ;
446414
@@ -496,15 +464,9 @@ impl Builder {
496464 extensions. retain ( & has_component) ;
497465 components. retain ( & has_component) ;
498466
499- Some ( Target {
500- available : true ,
501- url : Some ( self . url ( & filename) ) ,
502- hash : Some ( digest) ,
503- xz_url : xz_digest. as_ref ( ) . map ( |_| self . url ( & xz_filename) ) ,
504- xz_hash : xz_digest,
505- components : Some ( components) ,
506- extensions : Some ( extensions) ,
507- } )
467+ target. components = Some ( components) ;
468+ target. extensions = Some ( extensions) ;
469+ Some ( target)
508470 }
509471
510472 fn profile (
@@ -542,37 +504,19 @@ impl Builder {
542504 let targets = targets
543505 . iter ( )
544506 . map ( |name| {
545- if is_present {
546- // The component generally exists, but it might still be missing for this target.
507+ let target = if is_present {
547508 let filename = self
548509 . versions
549510 . tarball_name ( & PkgType :: from_component ( pkgname) , name)
550511 . unwrap ( ) ;
551- let digest = match self . digests . remove ( & filename) {
552- Some ( digest) => digest,
553- // This component does not exist for this target -- skip it.
554- None => return ( name. to_string ( ) , Target :: unavailable ( ) ) ,
555- } ;
556- let xz_filename = filename. replace ( ".tar.gz" , ".tar.xz" ) ;
557- let xz_digest = self . digests . remove ( & xz_filename) ;
558-
559- (
560- name. to_string ( ) ,
561- Target {
562- available : true ,
563- url : Some ( self . url ( & filename) ) ,
564- hash : Some ( digest) ,
565- xz_url : xz_digest. as_ref ( ) . map ( |_| self . url ( & xz_filename) ) ,
566- xz_hash : xz_digest,
567- components : None ,
568- extensions : None ,
569- } ,
570- )
512+
513+ Target :: from_compressed_tar ( self , & filename)
571514 } else {
572515 // If the component is not present for this build add it anyway but mark it as
573516 // unavailable -- this way rustup won't allow upgrades without --force
574- ( name. to_string ( ) , Target :: unavailable ( ) )
575- }
517+ Target :: unavailable ( )
518+ } ;
519+ ( name. to_string ( ) , target)
576520 } )
577521 . collect ( ) ;
578522
@@ -586,8 +530,9 @@ impl Builder {
586530 ) ;
587531 }
588532
589- fn url ( & self , filename : & str ) -> String {
590- format ! ( "{}/{}/{}" , self . s3_address, self . date, filename)
533+ fn url ( & self , path : & Path ) -> String {
534+ let file_name = path. file_name ( ) . unwrap ( ) . to_str ( ) . unwrap ( ) ;
535+ format ! ( "{}/{}/{}" , self . s3_address, self . date, file_name)
591536 }
592537
593538 fn hash ( & self , path : & Path ) -> String {
@@ -608,7 +553,7 @@ impl Builder {
608553 }
609554
610555 fn sign ( & self , path : & Path ) {
611- if !self . should_sign {
556+ if !self . legacy {
612557 return ;
613558 }
614559
@@ -631,10 +576,45 @@ impl Builder {
631576 . arg ( path)
632577 . stdin ( Stdio :: piped ( ) ) ;
633578 let mut child = t ! ( cmd. spawn( ) ) ;
634- t ! ( child. stdin. take( ) . unwrap( ) . write_all( self . gpg_passphrase . as_bytes( ) ) ) ;
579+ t ! ( child. stdin. take( ) . unwrap( ) . write_all( self . legacy_gpg_passphrase . as_bytes( ) ) ) ;
635580 assert ! ( t!( child. wait( ) ) . success( ) ) ;
636581 }
637582
583+ fn fill_missing_hashes ( & self , manifest : & mut Manifest ) {
584+ // First collect all files that need hashes
585+ let mut need_hashes = HashSet :: new ( ) ;
586+ crate :: manifest:: visit_file_hashes ( manifest, |file_hash| {
587+ if let FileHash :: Missing ( path) = file_hash {
588+ need_hashes. insert ( path. clone ( ) ) ;
589+ }
590+ } ) ;
591+
592+ let collected = Mutex :: new ( HashMap :: new ( ) ) ;
593+ let collection_start = Instant :: now ( ) ;
594+ println ! (
595+ "collecting hashes for {} tarballs across {} threads" ,
596+ need_hashes. len( ) ,
597+ rayon:: current_num_threads( ) . min( need_hashes. len( ) ) ,
598+ ) ;
599+ need_hashes. par_iter ( ) . for_each ( |path| match fetch_hash ( path) {
600+ Ok ( hash) => {
601+ collected. lock ( ) . unwrap ( ) . insert ( path, hash) ;
602+ }
603+ Err ( err) => eprintln ! ( "error while fetching the hash for {}: {}" , path. display( ) , err) ,
604+ } ) ;
605+ let collected = collected. into_inner ( ) . unwrap ( ) ;
606+ println ! ( "collected {} hashes in {:.2?}" , collected. len( ) , collection_start. elapsed( ) ) ;
607+
608+ crate :: manifest:: visit_file_hashes ( manifest, |file_hash| {
609+ if let FileHash :: Missing ( path) = file_hash {
610+ match collected. get ( path) {
611+ Some ( hash) => * file_hash = FileHash :: Present ( hash. clone ( ) ) ,
612+ None => panic ! ( "missing hash for file {}" , path. display( ) ) ,
613+ }
614+ }
615+ } )
616+ }
617+
638618 fn write_channel_files ( & self , channel_name : & str , manifest : & Manifest ) {
639619 self . write ( & toml:: to_string ( & manifest) . unwrap ( ) , channel_name, ".toml" ) ;
640620 self . write ( & manifest. date , channel_name, "-date.txt" ) ;
@@ -648,7 +628,16 @@ impl Builder {
648628 fn write ( & self , contents : & str , channel_name : & str , suffix : & str ) {
649629 let dst = self . output . join ( format ! ( "channel-rust-{}{}" , channel_name, suffix) ) ;
650630 t ! ( fs:: write( & dst, contents) ) ;
651- self . hash ( & dst) ;
652- self . sign ( & dst) ;
631+ if self . legacy {
632+ self . hash ( & dst) ;
633+ self . sign ( & dst) ;
634+ }
653635 }
654636}
637+
638+ fn fetch_hash ( path : & Path ) -> Result < String , Box < dyn Error > > {
639+ let mut file = BufReader :: new ( File :: open ( path) ?) ;
640+ let mut sha256 = sha2:: Sha256 :: default ( ) ;
641+ std:: io:: copy ( & mut file, & mut sha256) ?;
642+ Ok ( hex:: encode ( sha256. finalize ( ) ) )
643+ }
0 commit comments