Refactor storage

.env.sample

@@ -3,3 +3,6 @@ CRATESFYI_GITHUB_ACCESSTOKEN=
 CRATESFYI_PREFIX=ignored/cratesfyi-prefix
 CRATESFYI_DATABASE_URL=postgresql://cratesfyi:password@localhost
 RUST_LOG=cratesfyi,rustwide=info
+AWS_ACCESS_KEY_ID=cratesfyi
+AWS_SECRET_ACCESS_KEY=secret_key
+S3_ENDPOINT=http://localhost:9000

.github/workflows/ci.yml

@@ -49,12 +49,12 @@ jobs:
           restore-key: |
             ${{ runner.os }}-cargo-build-target-${{ env.CURRENT_RUSTC_VERSION }}-
 
-      - name: Launch the postgres image
+      - name: Launch the postgres and min.io images
         run: |
           cp .env.sample .env
           . .env
           mkdir -p ${CRATESFYI_PREFIX}/public-html
-          docker-compose up -d db
+          docker-compose up -d db s3
           # Give the database enough time to start up
           sleep 5
           # Make sure the database is actually working

README.md

@@ -86,16 +86,22 @@ cargo test
 ```
 
 Some tests require access to the database. To run them, set the
-`CRATESFYI_DATABASE_URL` to the url of a PostgreSQL database. If you are using
-the `docker-compose` environment to run tests against, see the
-[Docker-Compose][docker-compose-section] section for the default PostgreSQL URL.
-You don't have to run the migrations on it or ensure it's empty, as all the
-tests use temporary tables to prevent conflicts with each other or existing
-data. See the [wiki page on developing outside docker-compose][wiki-no-compose]
+`CRATESFYI_DATABASE_URL` in `.env` to the url of a PostgreSQL database,
+and set the `AWS_ACCESS_KEY_ID`, `S3_ENDPOINT`, and `AWS_SECRET_ACCESS_KEY` variables.
+We have some reasonable default parameters in `.env.sample`.
+
+For example, if you are using the `docker-compose` environment to run tests against, you can launch only the database and s3 server like so:
+
+```console
+docker-compose up -d db s3
+```
+
+If you don't want to use docker-compose, see the
+[wiki page on developing outside docker-compose][wiki-no-compose]
 for more information on how to setup this environment.
+Note that either way, you will need docker installed for sandboxing with Rustwide.
 
 [wiki-no-compose]: https://github.com/rust-lang/docs.rs/wiki/Developing-without-docker-compose
-[docker-compose-section]: #Docker-Compose
 
 ### Docker-Compose
 

src/bin/cratesfyi.rs

@@ -83,7 +83,6 @@ pub fn main() {
         .subcommand(SubCommand::with_name("database")
             .about("Database operations")
             .setting(AppSettings::ArgRequiredElseHelp)
-            .subcommand(SubCommand::with_name("move-to-s3"))
             .subcommand(SubCommand::with_name("migrate")
                 .about("Run database migrations")
                 .arg(Arg::with_name("VERSION")))
@@ -263,18 +262,6 @@ pub fn main() {
         } else if matches.subcommand_matches("update-search-index").is_some() {
             let conn = db::connect_db().unwrap();
             db::update_search_index(&conn).expect("Failed to update search index");
-        } else if matches.subcommand_matches("move-to-s3").is_some() {
-            let conn = db::connect_db().unwrap();
-            let mut count = 1;
-            let mut total = 0;
-            while count != 0 {
-                count = db::move_to_s3(&conn, 5_000).expect("Failed to upload batch to S3");
-                total += count;
-                eprintln!(
-                    "moved {} rows to s3 in this batch, total moved so far: {}",
-                    count, total
-                );
-            }
         } else if let Some(matches) = matches.subcommand_matches("delete-crate") {
             let name = matches.value_of("CRATE_NAME").expect("missing crate name");
             let conn = db::connect_db().expect("failed to connect to the database");

src/db/delete_crate.rs

@@ -1,4 +1,4 @@
-use super::file::{s3_client, S3_BUCKET_NAME};
+use crate::storage::s3::{s3_client, S3_BUCKET_NAME};
 use failure::{Error, Fail};
 use postgres::Connection;
 use rusoto_s3::{DeleteObjectsRequest, ListObjectsV2Request, ObjectIdentifier, S3Client, S3};