-
Notifications
You must be signed in to change notification settings - Fork 599
/
uploaders.rs
201 lines (183 loc) · 7.03 KB
/
uploaders.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
use conduit::Request;
use flate2::read::GzDecoder;
use openssl::hash::{Hasher, MessageDigest};
use crate::util::LimitErrorReader;
use crate::util::{human, internal, CargoResult, ChainError, Maximums};
use std::env;
use std::fs::{self, File};
use std::io::{Cursor, Read};
use std::sync::Arc;
use crate::middleware::app::RequestApp;
use crate::models::Crate;
#[derive(Clone, Debug)]
pub enum Uploader {
/// For production usage, uploads and redirects to s3.
/// For test usage with `TestApp::with_proxy()`, the recording proxy is used.
S3 {
bucket: s3::Bucket,
cdn: Option<String>,
},
/// For development usage only: "uploads" crate files to `dist` and serves them
/// from there as well to enable local publishing and download
Local,
}
impl Uploader {
/// Returns the URL of an uploaded crate's version archive.
///
/// The function doesn't check for the existence of the file.
pub fn crate_location(&self, crate_name: &str, version: &str) -> String {
match *self {
Uploader::S3 {
ref bucket,
ref cdn,
..
} => {
let host = match *cdn {
Some(ref s) => s.clone(),
None => bucket.host(),
};
let path = Uploader::crate_path(crate_name, version);
format!("https://{}/{}", host, path)
}
Uploader::Local => format!("/{}", Uploader::crate_path(crate_name, version)),
}
}
/// Returns the URL of an uploaded crate's version readme.
///
/// The function doesn't check for the existence of the file.
pub fn readme_location(&self, crate_name: &str, version: &str) -> String {
match *self {
Uploader::S3 {
ref bucket,
ref cdn,
..
} => {
let host = match *cdn {
Some(ref s) => s.clone(),
None => bucket.host(),
};
let path = Uploader::readme_path(crate_name, version);
format!("https://{}/{}", host, path)
}
Uploader::Local => format!("/{}", Uploader::readme_path(crate_name, version)),
}
}
/// Returns the interna path of an uploaded crate's version archive.
fn crate_path(name: &str, version: &str) -> String {
// No slash in front so we can use join
format!("crates/{}/{}-{}.crate", name, name, version)
}
/// Returns the interna path of an uploaded crate's version readme.
fn readme_path(name: &str, version: &str) -> String {
format!("readmes/{}/{}-{}.html", name, name, version)
}
/// Uploads a file using the configured uploader (either `S3`, `Local`).
///
/// It returns the path of the uploaded file.
pub fn upload<R: std::io::Read + Send + 'static>(
&self,
client: &reqwest::Client,
path: &str,
mut content: R,
content_length: u64,
content_type: &str,
) -> CargoResult<Option<String>> {
match *self {
Uploader::S3 { ref bucket, .. } => {
bucket
.put(client, path, content, content_length, content_type)
.map_err(|e| internal(&format_args!("failed to upload to S3: {}", e)))?;
Ok(Some(String::from(path)))
}
Uploader::Local => {
let filename = env::current_dir().unwrap().join("local_uploads").join(path);
let dir = filename.parent().unwrap();
fs::create_dir_all(dir)?;
let mut file = File::create(&filename)?;
std::io::copy(&mut content, &mut file)?;
Ok(filename.to_str().map(String::from))
}
}
}
/// Uploads a crate and returns the checksum of the uploaded crate file.
pub fn upload_crate(
&self,
req: &mut dyn Request,
krate: &Crate,
maximums: Maximums,
vers: &semver::Version,
) -> CargoResult<Vec<u8>> {
let app = Arc::clone(req.app());
let path = Uploader::crate_path(&krate.name, &vers.to_string());
let mut body = Vec::new();
LimitErrorReader::new(req.body(), maximums.max_upload_size).read_to_end(&mut body)?;
verify_tarball(krate, vers, &body, maximums.max_unpack_size)?;
let checksum = hash(&body);
let content_length = body.len() as u64;
let content = Cursor::new(body);
self.upload(
app.http_client(),
&path,
content,
content_length,
"application/x-tar",
)?;
Ok(checksum)
}
pub(crate) fn upload_readme(
&self,
http_client: &reqwest::Client,
crate_name: &str,
vers: &str,
readme: String,
) -> CargoResult<()> {
let path = Uploader::readme_path(crate_name, vers);
let content_length = readme.len() as u64;
let content = Cursor::new(readme);
self.upload(http_client, &path, content, content_length, "text/html")?;
Ok(())
}
}
fn verify_tarball(
krate: &Crate,
vers: &semver::Version,
tarball: &[u8],
max_unpack: u64,
) -> CargoResult<()> {
// All our data is currently encoded with gzip
let decoder = GzDecoder::new(tarball);
// Don't let gzip decompression go into the weeeds, apply a fixed cap after
// which point we say the decompressed source is "too large".
let decoder = LimitErrorReader::new(decoder, max_unpack);
// Use this I/O object now to take a peek inside
let mut archive = tar::Archive::new(decoder);
let prefix = format!("{}-{}", krate.name, vers);
for entry in archive.entries()? {
let entry = entry.chain_error(|| {
human("uploaded tarball is malformed or too large when decompressed")
})?;
// Verify that all entries actually start with `$name-$vers/`.
// Historically Cargo didn't verify this on extraction so you could
// upload a tarball that contains both `foo-0.1.0/` source code as well
// as `bar-0.1.0/` source code, and this could overwrite other crates in
// the registry!
if !entry.path()?.starts_with(&prefix) {
return Err(human("invalid tarball uploaded"));
}
// Historical versions of the `tar` crate which Cargo uses internally
// don't properly prevent hard links and symlinks from overwriting
// arbitrary files on the filesystem. As a bit of a hammer we reject any
// tarball with these sorts of links. Cargo doesn't currently ever
// generate a tarball with these file types so this should work for now.
let entry_type = entry.header().entry_type();
if entry_type.is_hard_link() || entry_type.is_symlink() {
return Err(human("invalid tarball uploaded"));
}
}
Ok(())
}
fn hash(data: &[u8]) -> Vec<u8> {
let mut hasher = Hasher::new(MessageDigest::sha256()).unwrap();
hasher.update(data).unwrap();
hasher.finish().unwrap().to_vec()
}