From d7144e544ac963cf6e5e3021d5232aa790326b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 16 Mar 2020 23:45:52 +0000 Subject: [PATCH] break refinery into refinery_core to remove code duplication (#70) closes #58 --- .circleci/config.yml | 9 --- Cargo.toml | 1 + refinery/Cargo.toml | 33 +++------- refinery/src/lib.rs | 14 +--- refinery/tests/mysql.rs | 1 + refinery/tests/mysql_async.rs | 3 +- refinery/tests/postgres.rs | 2 +- refinery/tests/rusqlite.rs | 2 +- refinery/tests/tokio_postgres.rs | 3 +- refinery_cli/Cargo.toml | 10 +-- refinery_cli/src/main.rs | 3 +- refinery_cli/src/migrate.rs | 8 +-- refinery_cli/src/setup.rs | 2 +- refinery_cli/src/util.rs | 25 -------- refinery_core/Cargo.toml | 37 +++++++++++ {refinery => refinery_core}/README.md | 0 {refinery => refinery_core}/src/config.rs | 8 ++- .../src/drivers/mod.rs | 0 .../src/drivers/mysql.rs | 0 .../src/drivers/mysql_async.rs | 0 .../src/drivers/postgres.rs | 0 .../src/drivers/rusqlite.rs | 0 .../src/drivers/tokio_postgres.rs | 0 {refinery => refinery_core}/src/error.rs | 0 refinery_core/src/lib.rs | 64 +++++++++++++++++++ {refinery => refinery_core}/src/runner.rs | 0 .../src/traits/async.rs | 0 {refinery => refinery_core}/src/traits/mod.rs | 0 .../src/traits/sync.rs | 0 .../src/util.rs | 14 ++-- refinery_macros/Cargo.toml | 2 +- refinery_macros/src/lib.rs | 12 +++- 32 files changed, 152 insertions(+), 101 deletions(-) delete mode 100644 refinery_cli/src/util.rs create mode 100644 refinery_core/Cargo.toml rename {refinery => refinery_core}/README.md (100%) rename {refinery => refinery_core}/src/config.rs (97%) rename {refinery => refinery_core}/src/drivers/mod.rs (100%) rename {refinery => refinery_core}/src/drivers/mysql.rs (100%) rename {refinery => refinery_core}/src/drivers/mysql_async.rs (100%) rename {refinery => refinery_core}/src/drivers/postgres.rs (100%) rename {refinery => refinery_core}/src/drivers/rusqlite.rs (100%) rename {refinery => refinery_core}/src/drivers/tokio_postgres.rs (100%) rename {refinery => refinery_core}/src/error.rs (100%) create mode 100644 refinery_core/src/lib.rs rename {refinery => refinery_core}/src/runner.rs (100%) rename {refinery => refinery_core}/src/traits/async.rs (100%) rename {refinery => refinery_core}/src/traits/mod.rs (100%) rename {refinery => refinery_core}/src/traits/sync.rs (100%) rename {refinery_macros => refinery_core}/src/util.rs (91%) diff --git a/.circleci/config.yml b/.circleci/config.yml index db8fd726..61540b03 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -185,15 +185,6 @@ jobs: steps: - checkout - run: cd refinery && cargo test --features tokio,mysql_async --test mysql_async -- --test-threads 1 - # test-windows: - # executor: - # name: win/default - # shell: bash.exe - # steps: - # - checkout - # - run: choco install rust --version << pipeline.parameters.stable >> - # - run: choco install mingw - # - run: cd refinery && CARGO_NET_GIT_FETCH_WITH_CLI=true cargo test --lib workflows: version: 2 build_and_test: diff --git a/Cargo.toml b/Cargo.toml index 1191da2c..c23c3fab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,5 +2,6 @@ members = [ "refinery", "refinery_cli", + "refinery_core", "refinery_macros" ] diff --git a/refinery/Cargo.toml b/refinery/Cargo.toml index 4d13b6f7..878e27cb 100644 --- a/refinery/Cargo.toml +++ b/refinery/Cargo.toml @@ -13,28 +13,17 @@ edition = "2018" [features] default = [] -rusqlite-bundled = ["rusqlite", "rusqlite/bundled"] +rusqlite-bundled = ["refinery-core/rusqlite-bundled"] +rusqlite = ["refinery-core/rusqlite"] +postgres = ["refinery-core/postgres"] +mysql = ["refinery-core/mysql"] +tokio-postgres = ["refinery-core/tokio-postgres"] +mysql_async = ["refinery-core/mysql_async"] +tokio = ["refinery-core/tokio"] [dependencies] -refinery-macros = { version = "0.2", path = "../refinery_macros" } -lazy_static = "1" -regex = "1" -log = "0.4" -chrono = "0.4" -serde = { version = "1", features = ["derive"] } -cfg-if = "0.1.10" -thiserror = "1" -async-trait = "0.1" -toml = "0.5" -siphasher = "0.3" - -rusqlite = {version = "0.21", optional = true} -postgres = {version = "0.17", optional = true} -mysql = {version = "17", optional = true} -tokio-postgres = { version = "0.5", optional = true } -mysql_async = { version = "0.21", optional = true } - -tokio = { version = "0.2", features = ["full"], optional = true } +refinery-core= { version = "0.2.0", path = "../refinery_core" } +refinery-macros= { version = "0.2.0", path = "../refinery_macros" } [dev-dependencies] barrel = { version = "0.6", features = ["sqlite3", "pg", "mysql"] } @@ -42,6 +31,4 @@ futures = "0.3" assert_cmd = "0.12" predicates = "1" tempfile = "3" - -[package.metadata.docs.rs] -all-features = true +chrono = "0.4" diff --git a/refinery/src/lib.rs b/refinery/src/lib.rs index a5798360..73cc33f8 100644 --- a/refinery/src/lib.rs +++ b/refinery/src/lib.rs @@ -17,7 +17,7 @@ Currently, [`Postgres`](https://crates.io/crates/postgres), [`Rusqlite`](https:/ [`include_migration_mods!`]: macro.include_migration_mods.html ### Example -```rust,no_run +```rust,ignore use rusqlite::Connection; mod embedded { @@ -32,14 +32,6 @@ embedded::migrations::runner().run(&mut conn).unwrap(); for more examples refer to the [examples](https://github.com/rust-db/refinery/tree/master/examples) */ -pub mod config; -mod drivers; -mod error; -mod runner; -mod traits; - -pub use crate::error::Error; -pub use crate::runner::{AppliedMigration, Migration, Runner}; -pub use crate::traits::r#async::AsyncMigrate; -pub use crate::traits::sync::Migrate; +pub use refinery_core::config; +pub use refinery_core::{AppliedMigration, AsyncMigrate, Error, Migrate, Migration, Runner}; pub use refinery_macros::{embed_migrations, include_migration_mods}; diff --git a/refinery/tests/mysql.rs b/refinery/tests/mysql.rs index 5f57f9d7..b556f8d9 100644 --- a/refinery/tests/mysql.rs +++ b/refinery/tests/mysql.rs @@ -11,6 +11,7 @@ mod mysql { config::{migrate_from_config, Config, ConfigDbType}, Error, Migrate, Migration, }; + use refinery_core::mysql; use std::process::Command; mod embedded { diff --git a/refinery/tests/mysql_async.rs b/refinery/tests/mysql_async.rs index e78bff83..0d705a5a 100644 --- a/refinery/tests/mysql_async.rs +++ b/refinery/tests/mysql_async.rs @@ -6,11 +6,12 @@ mod mysql_async { use super::mod_migrations; use chrono::{DateTime, Local}; use futures::FutureExt; - use mysql_async::prelude::Queryable; use refinery::{ config::{migrate_from_config_async, Config, ConfigDbType}, AsyncMigrate, Error, Migration, }; + use refinery_core::mysql_async::prelude::Queryable; + use refinery_core::{mysql_async, tokio}; use std::panic::AssertUnwindSafe; fn get_migrations() -> Vec { diff --git a/refinery/tests/postgres.rs b/refinery/tests/postgres.rs index f67556e3..32f1d88e 100644 --- a/refinery/tests/postgres.rs +++ b/refinery/tests/postgres.rs @@ -6,12 +6,12 @@ mod postgres { use super::mod_migrations; use assert_cmd::prelude::*; use chrono::{DateTime, Local}; - use postgres::{Client, NoTls}; use predicates::str::contains; use refinery::{ config::{migrate_from_config, Config, ConfigDbType}, Error, Migrate, Migration, }; + use refinery_core::postgres::{Client, NoTls}; use std::process::Command; mod embedded { diff --git a/refinery/tests/rusqlite.rs b/refinery/tests/rusqlite.rs index cb119901..2a60399a 100644 --- a/refinery/tests/rusqlite.rs +++ b/refinery/tests/rusqlite.rs @@ -11,7 +11,7 @@ mod rusqlite { config::{migrate_from_config, Config, ConfigDbType}, Error, Migrate, Migration, }; - use rusqlite::{Connection, OptionalExtension, NO_PARAMS}; + use refinery_core::rusqlite::{Connection, OptionalExtension, NO_PARAMS}; use std::fs::{self, File}; use std::process::Command; diff --git a/refinery/tests/tokio_postgres.rs b/refinery/tests/tokio_postgres.rs index 92a58c5a..ef587794 100644 --- a/refinery/tests/tokio_postgres.rs +++ b/refinery/tests/tokio_postgres.rs @@ -10,8 +10,9 @@ mod tokio_postgres { config::{migrate_from_config_async, Config, ConfigDbType}, AsyncMigrate, Error, Migration, }; + use refinery_core::tokio_postgres::NoTls; + use refinery_core::{tokio, tokio_postgres}; use std::panic::AssertUnwindSafe; - use tokio_postgres::NoTls; fn get_migrations() -> Vec { let migration1 = Migration::from_filename( diff --git a/refinery_cli/Cargo.toml b/refinery_cli/Cargo.toml index 43a476f6..537dc435 100644 --- a/refinery_cli/Cargo.toml +++ b/refinery_cli/Cargo.toml @@ -16,13 +16,13 @@ path = "src/main.rs" [features] default = ["mysql", "postgresql", "sqlite-bundled"] -postgresql = ["refinery/postgres"] -mysql = ["refinery/mysql"] -sqlite = ["refinery/rusqlite"] -sqlite-bundled = ["refinery/rusqlite-bundled"] +postgresql = ["refinery-core/postgres"] +mysql = ["refinery-core/mysql"] +sqlite = ["refinery-core/rusqlite"] +sqlite-bundled = ["refinery-core/rusqlite-bundled"] [dependencies] -refinery = { version = "0.2", path = "../refinery", default-features = false } +refinery-core = { version = "0.2", path = "../refinery_core", default-features = false } clap = { version = "2", features = ["wrap_help"] } human-panic = "1" toml = "0.5" diff --git a/refinery_cli/src/main.rs b/refinery_cli/src/main.rs index f94ebd9e..750662ea 100644 --- a/refinery_cli/src/main.rs +++ b/refinery_cli/src/main.rs @@ -3,7 +3,6 @@ mod cli; mod migrate; mod setup; -mod util; use anyhow::Error; use env_logger::{Builder, Target}; @@ -24,7 +23,7 @@ fn main() -> Result<(), Error> { let mut builder = Builder::new(); builder .format(|buf, record| writeln!(buf, "{}", record.args())) - .filter(Some("refinery::traits"), LevelFilter::Info) + .filter(Some("refinery_core::traits"), LevelFilter::Info) .target(Target::Stdout) .init(); diff --git a/refinery_cli/src/migrate.rs b/refinery_cli/src/migrate.rs index 3c3b915f..4c889412 100644 --- a/refinery_cli/src/migrate.rs +++ b/refinery_cli/src/migrate.rs @@ -2,13 +2,11 @@ use std::path::Path; use anyhow::{Context, Result}; use clap::ArgMatches; -use refinery::{ +use refinery_core::{ config::{migrate_from_config, Config}, - Migration, + find_migration_files, Migration, MigrationType, }; -use crate::util::find_migration_files; - pub fn handle_migration_command(args: &ArgMatches) -> Result<()> { //safe to call unwrap as we specified default values let config_location = args.value_of("config").unwrap(); @@ -35,7 +33,7 @@ fn run_files_migrations( //safe to call unwrap as we specified default value let path = arg.value_of("path").unwrap(); let path = Path::new(path); - let migration_files_path = find_migration_files(path)?; + let migration_files_path = find_migration_files(path, MigrationType::Sql)?; let mut migrations = Vec::new(); for path in migration_files_path { let sql = std::fs::read_to_string(path.as_path()) diff --git a/refinery_cli/src/setup.rs b/refinery_cli/src/setup.rs index 949afc6d..f467291c 100644 --- a/refinery_cli/src/setup.rs +++ b/refinery_cli/src/setup.rs @@ -16,7 +16,7 @@ use anyhow::{anyhow, Result}; use clap::ArgMatches; -use refinery::config::{Config, ConfigDbType}; +use refinery_core::config::{Config, ConfigDbType}; use std::fs::File; use std::io::{self, Write}; diff --git a/refinery_cli/src/util.rs b/refinery_cli/src/util.rs deleted file mode 100644 index cfbb9f5c..00000000 --- a/refinery_cli/src/util.rs +++ /dev/null @@ -1,25 +0,0 @@ -use std::ffi::OsStr; -use std::path::{Path, PathBuf}; -use walkdir::{DirEntry, WalkDir}; - -use regex::Regex; - -pub(crate) fn find_migration_files( - location: impl AsRef, -) -> Result, std::io::Error> { - let re = Regex::new(r"^V\d+(\.\d+)?__\w+\.sql$").unwrap(); - - let file_paths = WalkDir::new(location) - .into_iter() - .filter_map(Result::ok) - .map(DirEntry::into_path) - // filter by migration file regex - .filter( - move |entry| match entry.file_name().and_then(OsStr::to_str) { - Some(file_name) => re.is_match(file_name), - None => false, - }, - ); - - Ok(file_paths) -} diff --git a/refinery_core/Cargo.toml b/refinery_core/Cargo.toml new file mode 100644 index 00000000..01538a31 --- /dev/null +++ b/refinery_core/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "refinery-core" +version = "0.2.0" +authors = ["Katharina Fey ", "João Oliveira "] +description = "This crate should not be used directly, it is internaly related to Refinery" +license = "MIT OR Apache-2.0" +documentation = "https://docs.rs/refinery/" +repository = "https://github.com/rust-db/refinery" +edition = "2018" + +[features] +default = [] +rusqlite-bundled = ["rusqlite", "rusqlite/bundled"] + +[dependencies] +lazy_static = "1" +regex = "1" +log = "0.4" +chrono = "0.4" +serde = { version = "1", features = ["derive"] } +cfg-if = "0.1.10" +thiserror = "1" +async-trait = "0.1" +toml = "0.5" +siphasher = "0.3" +walkdir = "2.3.1" + +rusqlite = {version = "0.21", optional = true} +postgres = {version = "0.17", optional = true} +mysql = {version = "17", optional = true} +tokio-postgres = { version = "0.5", optional = true } +mysql_async = { version = "0.21", optional = true } + +tokio = { version = "0.2", features = ["full"], optional = true } + +[package.metadata.docs.rs] +all-features = true diff --git a/refinery/README.md b/refinery_core/README.md similarity index 100% rename from refinery/README.md rename to refinery_core/README.md diff --git a/refinery/src/config.rs b/refinery_core/src/config.rs similarity index 97% rename from refinery/src/config.rs rename to refinery_core/src/config.rs index 690d4636..777fc0ec 100644 --- a/refinery/src/config.rs +++ b/refinery_core/src/config.rs @@ -5,10 +5,11 @@ use serde::{Deserialize, Serialize}; use std::fs; use std::path::{Path, PathBuf}; -//refinery config file used by migrate_from_config +// refinery config file used by migrate_from_config if migration from a Config struct is prefered instead of using the macros +// Config can either be instanced with [`Config::new`] or retrieved from a config file with [`Config::from_file_location`] #[derive(Serialize, Deserialize, Debug)] pub struct Config { - pub main: Main, + main: Main, } #[derive(Clone, Copy, Serialize, Deserialize, PartialEq, Debug)] @@ -19,6 +20,7 @@ pub enum ConfigDbType { } impl Config { + /// create a new config instance pub fn new(db_type: ConfigDbType) -> Config { Config { main: Main { @@ -129,7 +131,7 @@ impl Config { } #[derive(Serialize, Deserialize, Debug)] -pub struct Main { +struct Main { db_type: ConfigDbType, db_path: Option, db_host: Option, diff --git a/refinery/src/drivers/mod.rs b/refinery_core/src/drivers/mod.rs similarity index 100% rename from refinery/src/drivers/mod.rs rename to refinery_core/src/drivers/mod.rs diff --git a/refinery/src/drivers/mysql.rs b/refinery_core/src/drivers/mysql.rs similarity index 100% rename from refinery/src/drivers/mysql.rs rename to refinery_core/src/drivers/mysql.rs diff --git a/refinery/src/drivers/mysql_async.rs b/refinery_core/src/drivers/mysql_async.rs similarity index 100% rename from refinery/src/drivers/mysql_async.rs rename to refinery_core/src/drivers/mysql_async.rs diff --git a/refinery/src/drivers/postgres.rs b/refinery_core/src/drivers/postgres.rs similarity index 100% rename from refinery/src/drivers/postgres.rs rename to refinery_core/src/drivers/postgres.rs diff --git a/refinery/src/drivers/rusqlite.rs b/refinery_core/src/drivers/rusqlite.rs similarity index 100% rename from refinery/src/drivers/rusqlite.rs rename to refinery_core/src/drivers/rusqlite.rs diff --git a/refinery/src/drivers/tokio_postgres.rs b/refinery_core/src/drivers/tokio_postgres.rs similarity index 100% rename from refinery/src/drivers/tokio_postgres.rs rename to refinery_core/src/drivers/tokio_postgres.rs diff --git a/refinery/src/error.rs b/refinery_core/src/error.rs similarity index 100% rename from refinery/src/error.rs rename to refinery_core/src/error.rs diff --git a/refinery_core/src/lib.rs b/refinery_core/src/lib.rs new file mode 100644 index 00000000..ea0b128c --- /dev/null +++ b/refinery_core/src/lib.rs @@ -0,0 +1,64 @@ +/*! +Powerful SQL migration toolkit for Rust. + +`refinery` makes running migrations for different databases as easy as possible. +It works by running your migrations on a provided database connection, either by embedding them on your Rust code, or via `refinery_cli`.\ +Currently, [`Postgres`](https://crates.io/crates/postgres), [`Rusqlite`](https://crates.io/crates/rusqlite), and [`Mysql`](https://crates.io/crates/mysql) are supported.\ + +`refinery` works best with [`Barrel`](https://crates.io/crates/barrel) but you can also have your migrations on .sql files or use any other Rust crate for schema generation. + +## Usage + +- Migrations can be defined in .sql files or Rust modules that must have a function called `migration()` that returns a [`std::string::String`] +- Migrations, both .sql files and Rust modules must be named in the format `V{1}__{2}.rs ` where `{1}` represents the migration version and `{2}` the name. +- Migrations can be run either by embedding them on your Rust code with [`embed_migrations!`] and [`include_migration_mods!`] macros, or via `refinery_cli`. + +[`embed_migrations!`]: macro.embed_migrations.html +[`include_migration_mods!`]: macro.include_migration_mods.html + +### Example +```rust,no_run +use rusqlite::Connection; + +mod embedded { + use refinery::embed_migrations; + embed_migrations!("./tests/sql_migrations"); +} + +let mut conn = Connection::open_in_memory().unwrap(); +embedded::migrations::runner().run(&mut conn).unwrap(); +``` + +for more examples refer to the [examples](https://github.com/rust-db/refinery/tree/master/examples) +*/ + +pub mod config; +mod drivers; +mod error; +mod runner; +mod traits; +mod util; + +pub use crate::error::Error; +pub use crate::runner::{AppliedMigration, Migration, Runner}; +pub use crate::traits::r#async::AsyncMigrate; +pub use crate::traits::sync::Migrate; +pub use crate::util::{find_migration_files, MigrationType}; + +#[cfg(feature = "rusqlite")] +pub use rusqlite; + +#[cfg(feature = "postgres")] +pub use postgres; + +#[cfg(feature = "mysql")] +pub use mysql; + +#[cfg(feature = "tokio-postgres")] +pub use tokio_postgres; + +#[cfg(feature = "mysql_async")] +pub use mysql_async; + +#[cfg(feature = "tokio")] +pub use tokio; diff --git a/refinery/src/runner.rs b/refinery_core/src/runner.rs similarity index 100% rename from refinery/src/runner.rs rename to refinery_core/src/runner.rs diff --git a/refinery/src/traits/async.rs b/refinery_core/src/traits/async.rs similarity index 100% rename from refinery/src/traits/async.rs rename to refinery_core/src/traits/async.rs diff --git a/refinery/src/traits/mod.rs b/refinery_core/src/traits/mod.rs similarity index 100% rename from refinery/src/traits/mod.rs rename to refinery_core/src/traits/mod.rs diff --git a/refinery/src/traits/sync.rs b/refinery_core/src/traits/sync.rs similarity index 100% rename from refinery/src/traits/sync.rs rename to refinery_core/src/traits/sync.rs diff --git a/refinery_macros/src/util.rs b/refinery_core/src/util.rs similarity index 91% rename from refinery_macros/src/util.rs rename to refinery_core/src/util.rs index 548e1f15..d999f333 100644 --- a/refinery_macros/src/util.rs +++ b/refinery_core/src/util.rs @@ -1,4 +1,4 @@ -use std::env; +use crate::Error; use std::ffi::OsStr; use std::path::{Path, PathBuf}; use walkdir::{DirEntry, WalkDir}; @@ -23,20 +23,16 @@ impl MigrationType { } } -pub(crate) fn crate_root() -> PathBuf { - let crate_root = env::var("CARGO_MANIFEST_DIR") - .expect("CARGO_MANIFEST_DIR environment variable not present"); - PathBuf::from(crate_root) -} - /// find migrations on file system recursively across directories given a location and [MigrationType] pub fn find_migration_files( location: impl AsRef, migration_type: MigrationType, -) -> Result, std::io::Error> { +) -> Result, Error> { let re = migration_type.file_match_re(); let location: &Path = location.as_ref(); - let location = location.canonicalize().expect("invalid migrations path"); + let location = location + .canonicalize() + .map_err(|err| Error::InvalidMigrationPath(location.to_path_buf(), err))?; let file_paths = WalkDir::new(location) .into_iter() diff --git a/refinery_macros/Cargo.toml b/refinery_macros/Cargo.toml index ff06441f..4d05825d 100644 --- a/refinery_macros/Cargo.toml +++ b/refinery_macros/Cargo.toml @@ -12,11 +12,11 @@ edition = "2018" proc-macro = true [dependencies] +refinery-core= { version = "0.2.0", path = "../refinery_core" } quote = "1" syn = { version = "1", features=["full"] } proc-macro2 = "1" regex = "1" -walkdir = "2.3.1" [dev-dependencies] tempfile = "3" diff --git a/refinery_macros/src/lib.rs b/refinery_macros/src/lib.rs index e554115e..5e43bfad 100644 --- a/refinery_macros/src/lib.rs +++ b/refinery_macros/src/lib.rs @@ -2,16 +2,22 @@ #![recursion_limit = "128"] extern crate proc_macro; -mod util; - use proc_macro::TokenStream; use proc_macro2::{Span as Span2, TokenStream as TokenStream2}; use quote::quote; use quote::ToTokens; +use std::env; use std::ffi::OsStr; +use std::path::PathBuf; use syn::{parse_macro_input, Ident, LitStr}; -use util::{crate_root, find_migration_files, MigrationType}; +use refinery_core::{find_migration_files, MigrationType}; + +pub(crate) fn crate_root() -> PathBuf { + let crate_root = env::var("CARGO_MANIFEST_DIR") + .expect("CARGO_MANIFEST_DIR environment variable not present"); + PathBuf::from(crate_root) +} fn migration_fn_quoted(_migrations: Vec) -> TokenStream2 { let result = quote! {