diff --git a/Cargo.lock b/Cargo.lock index df675240a..c71ab66b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -388,6 +388,8 @@ dependencies = [ "router 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", "sass-rs 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "schemamama 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "schemamama_postgres 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "slug 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "staticfile 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1394,7 +1396,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2088,6 +2090,23 @@ dependencies = [ "antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "schemamama" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "schemamama_postgres" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "postgres 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "schemamama 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "scopeguard" version = "0.3.3" @@ -3120,6 +3139,8 @@ dependencies = [ "checksum sass-sys 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "173ac202b4585ecfb1521159491175a787584fcc346457d53a099b240c69cd41" "checksum schannel 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "0e1a231dc10abf6749cfa5d7767f25888d484201accbd919b66ab5413c502d56" "checksum scheduled-thread-pool 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a2ff3fc5223829be817806c6441279c676e454cc7da608faf03b0ccc09d3889" +"checksum schemamama 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1f726d3b10198a91b545c12e55775ddf4abb681056aa62adf75ed00b68855ef9" +"checksum schemamama_postgres 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9a69defe7b625fa5c4bfda0a1525c9729baef68f620e505464b7bf0a4d1697f6" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum security-framework 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "697d3f3c23a618272ead9e1fb259c1411102b31c6af8b93f1d64cca9c3b0e8e0" "checksum security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab01dfbe5756785b5b4d46e0289e5a18071dfa9a7c2b24213ea00b9ef9b665bf" diff --git a/Cargo.toml b/Cargo.toml index f5c586f9c..2e22748df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,9 @@ comrak = { version = "0.3", default-features = false } toml = "0.4" html5ever = "0.22" cargo = { git = "https://github.com/rust-lang/cargo.git" } +schemamama = "0.3" +schemamama_postgres = "0.2" + # iron dependencies iron = "0.5" diff --git a/README.md b/README.md index 0415725ff..09acf3d43 100644 --- a/README.md +++ b/README.md @@ -108,8 +108,8 @@ cargo run -- build world #### `database` subcommand ```sh -# Initializes database. Currently, only creates tables in database. -cargo run -- database init +# Migrate database to recent version +cargo run -- database migrate # Adds a directory into database to serve with `staticfile` crate. diff --git a/Vagrantfile b/Vagrantfile index f86ed9644..b68903451 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -120,9 +120,9 @@ EOF echo 'DROP DATABASE cratesfyi; CREATE DATABASE cratesfyi OWNER cratesfyi' | sudo -u postgres psql ############################################################ - # Initializing database scheme # + # Migrate database to recent version # ############################################################ - su - cratesfyi -c "cd /vagrant && cargo run -- database init" + su - cratesfyi -c "cd /vagrant && cargo run -- database migrate" ############################################################ # Add essential files for downloaded nigthly # diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 8caa05848..feb518b0f 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -109,8 +109,9 @@ pub fn main() { .subcommand(SubCommand::with_name("daemon").about("Starts cratesfyi daemon")) .subcommand(SubCommand::with_name("database") .about("Database operations") - .subcommand(SubCommand::with_name("init").about("Initialize database. Currently \ - only creates tables in database.")) + .subcommand(SubCommand::with_name("migrate") + .about("Run database migrations") + .arg(Arg::with_name("VERSION"))) .subcommand(SubCommand::with_name("update-github-fields") .about("Updates github stats for crates.")) .subcommand(SubCommand::with_name("add-directory") @@ -204,9 +205,10 @@ pub fn main() { } } else if let Some(matches) = matches.subcommand_matches("database") { - if let Some(_) = matches.subcommand_matches("init") { - let conn = db::connect_db().unwrap(); - db::create_tables(&conn).expect("Failed to initialize database"); + if let Some(matches) = matches.subcommand_matches("migrate") { + let version = matches.value_of("VERSION").map(|v| v.parse::() + .expect("Version should be an integer")); + db::migrate(version).expect("Failed to run database migrations"); } else if let Some(_) = matches.subcommand_matches("update-github-fields") { cratesfyi::utils::github_updater().expect("Failed to update github fields"); } else if let Some(matches) = matches.subcommand_matches("add-directory") { diff --git a/src/db/migrate.rs b/src/db/migrate.rs new file mode 100644 index 000000000..073d4d004 --- /dev/null +++ b/src/db/migrate.rs @@ -0,0 +1,189 @@ +//! Database migrations + +use db::connect_db; +use error::Result as CratesfyiResult; +use postgres::error::Error as PostgresError; +use postgres::transaction::Transaction; +use schemamama::{Migration, Migrator, Version}; +use schemamama_postgres::{PostgresAdapter, PostgresMigration}; + + +/// Creates a new PostgresMigration from upgrade and downgrade queries. +/// Downgrade query should return database to previous state. +/// +/// Example: +/// +/// ``` +/// let my_migration = migration!(100, +/// "Create test table", +/// "CREATE TABLE test ( id SERIAL);", +/// "DROP TABLE test;"); +/// ``` +macro_rules! migration { + ($version:expr, $description:expr, $up:expr, $down:expr) => {{ + struct Amigration; + impl Migration for Amigration { + fn version(&self) -> Version { + $version + } + fn description(&self) -> String { + $description.to_owned() + } + } + impl PostgresMigration for Amigration { + fn up(&self, transaction: &Transaction) -> Result<(), PostgresError> { + info!("Applying migration {}: {}", self.version(), self.description()); + transaction.batch_execute($up).map(|_| ()) + } + fn down(&self, transaction: &Transaction) -> Result<(), PostgresError> { + info!("Removing migration {}: {}", self.version(), self.description()); + transaction.batch_execute($down).map(|_| ()) + } + } + Box::new(Amigration) + }}; +} + + +pub fn migrate(version: Option) -> CratesfyiResult<()> { + let conn = connect_db()?; + let adapter = PostgresAdapter::with_metadata_table(&conn, "database_versions"); + adapter.setup_schema()?; + + let mut migrator = Migrator::new(adapter); + + let migrations: Vec> = vec![ + migration!( + // version + 1, + // description + "Initial database schema", + // upgrade query + "CREATE TABLE crates ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) UNIQUE NOT NULL, + latest_version_id INT DEFAULT 0, + versions JSON DEFAULT '[]', + downloads_total INT DEFAULT 0, + github_description VARCHAR(1024), + github_stars INT DEFAULT 0, + github_forks INT DEFAULT 0, + github_issues INT DEFAULT 0, + github_last_commit TIMESTAMP, + github_last_update TIMESTAMP, + content tsvector + ); + CREATE TABLE releases ( + id SERIAL PRIMARY KEY, + crate_id INT NOT NULL REFERENCES crates(id), + version VARCHAR(100), + release_time TIMESTAMP, + dependencies JSON, + target_name VARCHAR(255), + yanked BOOL DEFAULT FALSE, + is_library BOOL DEFAULT TRUE, + build_status BOOL DEFAULT FALSE, + rustdoc_status BOOL DEFAULT FALSE, + test_status BOOL DEFAULT FALSE, + license VARCHAR(100), + repository_url VARCHAR(255), + homepage_url VARCHAR(255), + documentation_url VARCHAR(255), + description VARCHAR(1024), + description_long VARCHAR(51200), + readme VARCHAR(51200), + authors JSON, + keywords JSON, + have_examples BOOL DEFAULT FALSE, + downloads INT DEFAULT 0, + files JSON, + doc_targets JSON DEFAULT '[]', + doc_rustc_version VARCHAR(100) NOT NULL, + default_target VARCHAR(100), + UNIQUE (crate_id, version) + ); + CREATE TABLE authors ( + id SERIAL PRIMARY KEY, + name VARCHAR(255), + email VARCHAR(255), + slug VARCHAR(255) UNIQUE NOT NULL + ); + CREATE TABLE author_rels ( + rid INT REFERENCES releases(id), + aid INT REFERENCES authors(id), + UNIQUE(rid, aid) + ); + CREATE TABLE keywords ( + id SERIAL PRIMARY KEY, + name VARCHAR(255), + slug VARCHAR(255) NOT NULL UNIQUE + ); + CREATE TABLE keyword_rels ( + rid INT REFERENCES releases(id), + kid INT REFERENCES keywords(id), + UNIQUE(rid, kid) + ); + CREATE TABLE owners ( + id SERIAL PRIMARY KEY, + login VARCHAR(255) NOT NULL UNIQUE, + avatar VARCHAR(255), + name VARCHAR(255), + email VARCHAR(255) + ); + CREATE TABLE owner_rels ( + cid INT REFERENCES releases(id), + oid INT REFERENCES owners(id), + UNIQUE(cid, oid) + ); + CREATE TABLE builds ( + id SERIAL, + rid INT NOT NULL REFERENCES releases(id), + rustc_version VARCHAR(100) NOT NULL, + cratesfyi_version VARCHAR(100) NOT NULL, + build_status BOOL NOT NULL, + build_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + output TEXT + ); + CREATE TABLE queue ( + id SERIAL, + name VARCHAR(255), + version VARCHAR(100), + attempt INT DEFAULT 0, + date_added TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(name, version) + ); + CREATE TABLE files ( + path VARCHAR(4096) NOT NULL PRIMARY KEY, + mime VARCHAR(100) NOT NULL, + date_added TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + date_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + content BYTEA + ); + CREATE TABLE config ( + name VARCHAR(100) NOT NULL PRIMARY KEY, + value JSON NOT NULL + ); + CREATE INDEX ON releases (release_time DESC); + CREATE INDEX content_idx ON crates USING gin(content);", + // downgrade query + "DROP TABLE authors, author_rels, keyword_rels, keywords, owner_rels, + owners, releases, crates, builds, queue, files, config;" + ), + ]; + + for migration in migrations { + migrator.register(migration); + } + + if let Some(version) = version { + if version > migrator.current_version()?.unwrap_or(0) { + migrator.up(Some(version))?; + } else { + migrator.down(Some(version))?; + } + } else { + migrator.up(version)?; + } + + Ok(()) +} diff --git a/src/db/mod.rs b/src/db/mod.rs index c725814db..372a35bb5 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -3,6 +3,7 @@ pub use self::add_package::add_package_into_database; pub use self::add_package::add_build_into_database; pub use self::file::add_path_into_database; +pub use self::migrate::migrate; use postgres::{Connection, TlsMode}; use postgres::error::Error; @@ -12,6 +13,7 @@ use r2d2_postgres; mod add_package; mod file; +mod migrate; /// Connects to database @@ -68,123 +70,6 @@ pub fn update_search_index(conn: &Connection) -> Result { } -/// Creates database tables -pub fn create_tables(conn: &Connection) -> Result<(), Error> { - let queries = ["CREATE TABLE crates ( - id SERIAL PRIMARY KEY, - name VARCHAR(255) UNIQUE NOT NULL, - latest_version_id INT DEFAULT 0, - versions JSON DEFAULT '[]', - downloads_total INT DEFAULT 0, - github_description VARCHAR(1024), - github_stars INT DEFAULT 0, - github_forks INT DEFAULT 0, - github_issues INT DEFAULT 0, - github_last_commit TIMESTAMP, - github_last_update TIMESTAMP, - content tsvector - )", - "CREATE TABLE releases ( - id SERIAL PRIMARY KEY, - crate_id INT NOT NULL REFERENCES crates(id), - version VARCHAR(100), - release_time TIMESTAMP, - dependencies JSON, - target_name VARCHAR(255), - yanked BOOL DEFAULT FALSE, - is_library BOOL DEFAULT TRUE, - build_status BOOL DEFAULT FALSE, - rustdoc_status BOOL DEFAULT FALSE, - test_status BOOL DEFAULT FALSE, - license VARCHAR(100), - repository_url VARCHAR(255), - homepage_url VARCHAR(255), - documentation_url VARCHAR(255), - description VARCHAR(1024), - description_long VARCHAR(51200), - readme VARCHAR(51200), - authors JSON, - keywords JSON, - have_examples BOOL DEFAULT FALSE, - downloads INT DEFAULT 0, - files JSON, - doc_targets JSON DEFAULT '[]', - doc_rustc_version VARCHAR(100) NOT NULL, - default_target VARCHAR(100), - UNIQUE (crate_id, version) - )", - "CREATE TABLE authors ( - id SERIAL PRIMARY KEY, - name VARCHAR(255), - email VARCHAR(255), - slug VARCHAR(255) UNIQUE NOT NULL - )", - "CREATE TABLE author_rels ( - rid INT REFERENCES releases(id), - aid INT REFERENCES authors(id), - UNIQUE(rid, aid) - )", - "CREATE TABLE keywords ( - id SERIAL PRIMARY KEY, - name VARCHAR(255), - slug VARCHAR(255) NOT NULL UNIQUE - )", - "CREATE TABLE keyword_rels ( - rid INT REFERENCES releases(id), - kid INT REFERENCES keywords(id), - UNIQUE(rid, kid) - )", - "CREATE TABLE owners ( - id SERIAL PRIMARY KEY, - login VARCHAR(255) NOT NULL UNIQUE, - avatar VARCHAR(255), - name VARCHAR(255), - email VARCHAR(255) - )", - "CREATE TABLE owner_rels ( - cid INT REFERENCES releases(id), - oid INT REFERENCES owners(id), - UNIQUE(cid, oid) - )", - "CREATE TABLE builds ( - id SERIAL, - rid INT NOT NULL REFERENCES releases(id), - rustc_version VARCHAR(100) NOT NULL, - cratesfyi_version VARCHAR(100) NOT NULL, - build_status BOOL NOT NULL, - build_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - output TEXT - )", - "CREATE TABLE queue ( - id SERIAL, - name VARCHAR(255), - version VARCHAR(100), - attempt INT DEFAULT 0, - date_added TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - UNIQUE(name, version) - )", - "CREATE TABLE files ( - path VARCHAR(4096) NOT NULL PRIMARY KEY, - mime VARCHAR(100) NOT NULL, - date_added TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - date_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - content BYTEA - )", - "CREATE INDEX ON releases (release_time DESC)", - "CREATE INDEX content_idx ON crates USING gin(content)", - "CREATE TABLE config ( - name VARCHAR(100) NOT NULL PRIMARY KEY, - value JSON NOT NULL - )", - "INSERT INTO config VALUES ('database_version', '1'::json)"]; - - for query in queries.into_iter() { - try!(conn.execute(query, &[])); - } - - Ok(()) -} - #[cfg(test)] @@ -198,18 +83,4 @@ mod test { let conn = connect_db(); assert!(conn.is_ok()); } - - - #[test] - #[ignore] - fn test_create_tables() { - let _ = env_logger::try_init(); - let conn = connect_db(); - assert!(conn.is_ok()); - - // FIXME: As expected this test always fails if database is already created - let res = create_tables(&conn.unwrap()); - info!("RES: {:#?}", res); - assert!(res.is_ok()); - } } diff --git a/src/lib.rs b/src/lib.rs index 17bc0af96..9e5cd4f68 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,6 +29,8 @@ extern crate crates_index_diff; extern crate git2; extern crate toml; extern crate html5ever; +extern crate schemamama; +extern crate schemamama_postgres; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::ChrootBuilderResult;