From 150fda8062e3bd5c97bb57d759b383b64e43d84b Mon Sep 17 00:00:00 2001 From: arctic_hen7 Date: Thu, 30 Dec 2021 11:56:24 +1030 Subject: [PATCH] feat: add tokio (#102) * docs: added preliminary `define_app!` advanced docs * refactor: made plugins system thread-safe All plugin functions must now implement `Send`. * refactor: made stores use async io * fix: added `io-util` flag to tokio Magical compilation errors occur otherwise for some reason. * refactor: made builder async This should increase performance for templates with a very large number of pages, each generated path is now built concurrently. * refactor: made exporting async * refactor: removed unnecessary leftover `Arc`s * refactor: made `FsTranslationsManager` async I *think* this makes everything async, the main thing is just the file operations. * chore(deps): pinned `actix-http` Actix seems to introduce a breaking change in the deps tree on CI, though this hasn't been reported anywhere else. If people start having issues, I'll release an urgent patch. * ci: add `sudo apt update` This should hopefully fix the consistent breakages of `apt` on CI. --- .github/workflows/ci.yml | 4 + docs/next/en-US/SUMMARY.md | 1 + docs/next/en-US/advanced/define_app.md | 58 ++++ examples/basic/.perseus/builder/Cargo.toml | 1 + .../basic/.perseus/builder/src/bin/build.rs | 17 +- .../basic/.perseus/builder/src/bin/export.rs | 102 +++++--- .../basic/.perseus/builder/src/bin/tinker.rs | 2 + packages/perseus-actix-web/Cargo.toml | 1 + packages/perseus/Cargo.toml | 1 + packages/perseus/src/build.rs | 247 ++++++++++-------- packages/perseus/src/errors.rs | 2 +- packages/perseus/src/export.rs | 221 +++++++++------- packages/perseus/src/plugins/action.rs | 12 +- packages/perseus/src/plugins/control.rs | 10 +- packages/perseus/src/plugins/functional.rs | 10 +- packages/perseus/src/plugins/mod.rs | 7 + packages/perseus/src/plugins/plugin.rs | 4 +- packages/perseus/src/plugins/plugins_list.rs | 14 +- packages/perseus/src/stores/immutable.rs | 64 ++++- packages/perseus/src/stores/mutable.rs | 76 ++++-- packages/perseus/src/translations_manager.rs | 44 ++-- 21 files changed, 579 insertions(+), 319 deletions(-) create mode 100644 docs/next/en-US/advanced/define_app.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01de6c1fa2..4631369442 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,7 @@ jobs: - uses: actions/checkout@v2 - run: cargo install bonnie - run: cargo install wasm-pack + - run: sudo apt update - run: sudo apt install firefox firefox-geckodriver - name: Run Firefox WebDriver run: geckodriver & @@ -43,6 +44,7 @@ jobs: - uses: actions/checkout@v2 - run: cargo install bonnie - run: cargo install wasm-pack + - run: sudo apt update - run: sudo apt install firefox firefox-geckodriver - name: Run Firefox WebDriver run: geckodriver & @@ -56,6 +58,7 @@ jobs: - uses: actions/checkout@v2 - run: cargo install bonnie - run: cargo install wasm-pack + - run: sudo apt update - run: sudo apt install firefox firefox-geckodriver - name: Run Firefox WebDriver run: geckodriver & @@ -71,6 +74,7 @@ jobs: - uses: actions/checkout@v2 - run: cargo install bonnie - run: cargo install wasm-pack + - run: sudo apt update - run: sudo apt install firefox firefox-geckodriver - name: Run Firefox WebDriver run: geckodriver & diff --git a/docs/next/en-US/SUMMARY.md b/docs/next/en-US/SUMMARY.md index 40f9d73ada..16019667cf 100644 --- a/docs/next/en-US/SUMMARY.md +++ b/docs/next/en-US/SUMMARY.md @@ -66,3 +66,4 @@ - [Initial Loads](/docs/advanced/initial-loads) - [Subsequent Loads](/docs/advanced/subsequent-loads) - [Routing](/docs/advanced/routing) +- [`define_app` in Detail](/docs/advanced/define_app) diff --git a/docs/next/en-US/advanced/define_app.md b/docs/next/en-US/advanced/define_app.md new file mode 100644 index 0000000000..f206b21153 --- /dev/null +++ b/docs/next/en-US/advanced/define_app.md @@ -0,0 +1,58 @@ +# The `define_app` Macro in Detail + +Many users of Perseus will be perfectly content to leave the inner workings of their app to `define_app`, but some may be curious as to what this macro actually does, and that's what this section will explain. + +Before we begin on the details of this, it's important to understand one thing: **your code does not make your app**. The Perseus engine (the stuff in `.perseus/`) makes your app, and it _imports_ your code to create the specifics of your app, but the actual Wasm entrypoint in in `.perseus/src/lib.rs`. This architecture can be a little unintuitive at first, but it allows Perseus to abstract a huge amount of work behind the scenes, minimizing the amount of boilerplate code that you need to write. + +The issue that arises from this architecture is making your app interface with the engine, and that's where the `define_app!` macro comes in. It defines a number of functions that are then imported by the engine and called to get information about your app. The very first version of Perseus didn't even have a CLI, and all this interfacing had to be done manually! Today though, the process is _much_ easier. + +Before we get on to exactly what the macro defines, it's worth mentioning that using Perseus without the `define_app!` macro is possible, but is not recommended, even for experiences users. The main reasons are twofold: you will be writing _a lot_ of boilerplate code (e.g. you have to define a dummy translations manager even if you're not using i18n) and your app may break with new minor versions, because Perseus considers changes to the engine and the internals of the `define_app!` macro to be non-breaking. If you're still determined to persevere with going macro-less, you should regularly review the Perseus [`CHANGELOG`](https://github.com/arctic-hen7/perseus/blob/main/CHANGELOG.md) to make any changes that are necessary for minor versions. + +## Functions Defined + +Now that we've got all that out of the way, let's really dig into the weeds of this thing! The `define_app!` macro is defined in `packages/perseus/src/macros.rs` in the repository, and that should be your reference while trying to understand the inner workings of it. + +There are two versions of the macro, one that takes i18n options and one that doesn't. This is just syntactic sugar to make things more convenient for the user, and it doesn't affect anything more. Either way, here are the functions that are defined. (Note that a lot of these are defined with secondary internal macros.) + +- `get_plugins` -- this returns an instance of `perseus::Plugins`, either an empty one if no plugins are provided, or whatever the user provides +- `APP_ROOT` (a `static` `&str`) -- this is the HTML `id` of the element to run Perseus in, which is `root` unless something else is provided by the user +- `get_immutable_store` -- this returns an instance of `perseus::stores::ImmutableStore` with either `./dist` or the user-provided distribution directory as the root (whatever is provided here is relative to `.perseus/`) +- `get_mutable_store` -- this returns an instance of `perseus::stores::FsMutableStore` with `./dist/mutable` as the root (relative to `.perseus`), or a user-given mutable store +- `get_translations_manager` -- see below +- `get_locales` + - With i18n -- this returns an instance of `perseus::internal::i18n::Locales`, literally constructed with the given default locale, the other locales, and with `using_i18n` set to `true` + - Without i18n -- this does the same as with i18n, but sets `using_i18n` to `false`, provides no `other` locales, and sets the default to `xx-XX` (the dummy locale expected throughout Perseus if the user isn't using i18n, anything else here if you're not using i18n will result in runtime errors!) +- `get_static_aliases` -- this creates a `HashMap` of your static aliases, from URL to resource location +- `get_templates_map` -- this creates a `HashMap` out of your templates, mapping the result of `template.get_path()` (what you provide to `Template::new()`) to the templates themselves (wrapped in `Rc`s to avoid literal lifetime hell) +- `get_templates_map_atomic` -- exactly the same as `get_templates_map`, but uses `Arc`s instead of `Rc`s (needed for multithreading on the server) +- `get_error_pages` -- this one's simple, it just returns the instance of `ErrorPages` that you provide to the macro + +Most of these are pretty straightforward, they're just very boilerplate-heavy, which is why Perseus does them for you! However, the translations manager is a little less straightforward, because it does different things if Perseus has been deployed to a server (in which case the `standalone` feature will be enabled on Perseus). + +### `get_translations_manager` + +This function is `async`, and it returns something that implements `perseus::internal::i18n::TranslationsManager`. There are four cases of what the user can provide to the macro, and they'll be gone through individually. + +#### No i18n + +We provide a `perseus::internal::i18n::DummyTranslationsManager`, which is designed for this exact purpose. Perseus always needs a translations manager, so this one provides an API interface and no actual functionality. + +#### A custom translations manager + +We just return whatever the user provided. This is technically two cases, because i18n could be either enabled or disabled (though why someone would provide a custom dummy translations manager is a bit of a mystery). + +#### I18n + +If no custom translations manager is provided, we create a `perseus::internal::i18n::FsTranslationsManager` for them, the `::new()` method for which takes three arguments: a directory to expect translation files in, a vector of the locales to cache, and the file extension of translation files (which will always be named as `[locale].[extension]`). + +The first argument is a little challenging, because it will usually be `../translations/` (relative to `.perseus/`), in the root directory of your project. However, if Perseus has been deployed as a standalone server binary, this directory will be in the same folder as the binary, so we use `./translations/` instead. In the macro, this is controlled by the `standalone` feature flag, but that isn't provided to your app, so the best thing to do here is up to you (you might depend on an environment variable that you remember to provide when you deploy). + +The second argument is probably a little weird to you. Caching translations? Well, they're actually the most requested things for the Perseus server, so the `FsTranslationsManager` caches locales when it's started by default. This uses more memory on the server, but makes requests faster in the longer-term (we do the same thing with your `index.html` file). By default, Perseus runs the `.get_all()` function on the instance of `Locales` generated by the macro's own `get_locales()` function to get all your locales, and then it tells the manager to cache everything. This is customizable in the macro by allowing the user to provide a custom instance of `FsTranslationsManager`. + +The final argument is blissfully simple, because it's defined internally in Perseus at `perseus::internal::i18n::TRANSLATOR_FILE_EXT`. The reason this isn't hardcoded is because it's dependent on the `Translator` being used, which is controlled by feature flags. + +Finally, the reason this whole `get_translations_manager()` function is `async` is because it has to `await` that `FsTranslationsManager::new()` call, because translations managers are fully `async` (in case they need to be working with DBs or the like). + +## Conclusion + +If, after all that, you still want to use Perseus without the `define_app!` macro, there's an example on its way! That said, it is _much_ easier to leave things to the macro, or you'll end up writing a huge amount of boilerplate. In fact, all this is just the tip of the iceberg, and there's more transformation that's done on all this in the engine! diff --git a/examples/basic/.perseus/builder/Cargo.toml b/examples/basic/.perseus/builder/Cargo.toml index c2fa26b80e..5dba361ddf 100644 --- a/examples/basic/.perseus/builder/Cargo.toml +++ b/examples/basic/.perseus/builder/Cargo.toml @@ -15,6 +15,7 @@ perseus-engine = { path = "../" } perseus = { path = "../../../../packages/perseus", features = [ "tinker-plugins", "server-side" ] } futures = "0.3" fs_extra = "1" +tokio = { version = "1", features = [ "macros", "rt-multi-thread" ] } # We define a binary for building, serving, and doing both [[bin]] diff --git a/examples/basic/.perseus/builder/src/bin/build.rs b/examples/basic/.perseus/builder/src/bin/build.rs index a0713cbd0d..4c19bc3627 100644 --- a/examples/basic/.perseus/builder/src/bin/build.rs +++ b/examples/basic/.perseus/builder/src/bin/build.rs @@ -1,16 +1,16 @@ -use futures::executor::block_on; use perseus::{internal::build::build_app, PluginAction, SsrNode}; use perseus_engine::app::{ get_immutable_store, get_locales, get_mutable_store, get_plugins, get_templates_map, get_translations_manager, }; -fn main() { - let exit_code = real_main(); +#[tokio::main] +async fn main() { + let exit_code = real_main().await; std::process::exit(exit_code) } -fn real_main() -> i32 { +async fn real_main() -> i32 { // We want to be working in the root of `.perseus/` std::env::set_current_dir("../").unwrap(); let plugins = get_plugins::(); @@ -23,21 +23,22 @@ fn real_main() -> i32 { let immutable_store = get_immutable_store(&plugins); let mutable_store = get_mutable_store(); - let translations_manager = block_on(get_translations_manager()); + // We can't proceed without a translations manager + let translations_manager = get_translations_manager().await; let locales = get_locales(&plugins); // Build the site for all the common locales (done in parallel) // All these parameters can be modified by `define_app!` and plugins, so there's no point in having a plugin opportunity here let templates_map = get_templates_map::(&plugins); - let fut = build_app( + let res = build_app( &templates_map, &locales, (&immutable_store, &mutable_store), &translations_manager, // We use another binary to handle exporting false, - ); - let res = block_on(fut); + ) + .await; if let Err(err) = res { let err_msg = format!("Static generation failed: '{}'.", &err); plugins diff --git a/examples/basic/.perseus/builder/src/bin/export.rs b/examples/basic/.perseus/builder/src/bin/export.rs index fe2a905b83..57ba58dbec 100644 --- a/examples/basic/.perseus/builder/src/bin/export.rs +++ b/examples/basic/.perseus/builder/src/bin/export.rs @@ -1,5 +1,4 @@ use fs_extra::dir::{copy as copy_dir, CopyOptions}; -use futures::executor::block_on; use perseus::{ internal::{build::build_app, export::export_app, get_path_prefix_server}, PluginAction, SsrNode, @@ -11,17 +10,42 @@ use perseus_engine::app::{ use std::fs; use std::path::PathBuf; -fn main() { - let exit_code = real_main(); +#[tokio::main] +async fn main() { + let exit_code = real_main().await; std::process::exit(exit_code) } -fn real_main() -> i32 { +async fn real_main() -> i32 { // We want to be working in the root of `.perseus/` std::env::set_current_dir("../").unwrap(); let plugins = get_plugins::(); + // Building and exporting must be sequential, but that can be done in parallel with static directory/alias copying + let exit_code = build_and_export().await; + if exit_code != 0 { + return exit_code; + } + // After that's done, we can do two copy operations in parallel at least + let exit_code_1 = tokio::task::spawn_blocking(copy_static_dir); + let exit_code_2 = tokio::task::spawn_blocking(copy_static_aliases); + // These errors come from any panics in the threads, which should be propagated up to a panic in the main thread in this case + exit_code_1.await.unwrap(); + exit_code_2.await.unwrap(); + + plugins + .functional_actions + .export_actions + .after_successful_export + .run((), plugins.get_plugin_data()); + println!("Static exporting successfully completed!"); + 0 +} + +async fn build_and_export() -> i32 { + let plugins = get_plugins::(); + plugins .functional_actions .build_actions @@ -31,20 +55,22 @@ fn real_main() -> i32 { let immutable_store = get_immutable_store(&plugins); // We don't need this in exporting, but the build process does let mutable_store = get_mutable_store(); - let translations_manager = block_on(get_translations_manager()); + let translations_manager = get_translations_manager().await; let locales = get_locales(&plugins); // Build the site for all the common locales (done in parallel), denying any non-exportable features + // We need to build and generate those artifacts before we can proceed on to exporting let templates_map = get_templates_map::(&plugins); - let build_fut = build_app( + let build_res = build_app( &templates_map, &locales, (&immutable_store, &mutable_store), &translations_manager, // We use another binary to handle normal building true, - ); - if let Err(err) = block_on(build_fut) { + ) + .await; + if let Err(err) = build_res { let err_msg = format!("Static exporting failed: '{}'.", &err); plugins .functional_actions @@ -61,7 +87,7 @@ fn real_main() -> i32 { .run((), plugins.get_plugin_data()); // Turn the build artifacts into self-contained static files let app_root = get_app_root(&plugins); - let export_fut = export_app( + let export_res = export_app( &templates_map, // Perseus always uses one HTML file, and there's no point in letting a plugin change that "../index.html", @@ -70,8 +96,9 @@ fn real_main() -> i32 { &immutable_store, &translations_manager, get_path_prefix_server(), - ); - if let Err(err) = block_on(export_fut) { + ) + .await; + if let Err(err) = export_res { let err_msg = format!("Static exporting failed: '{}'.", &err); plugins .functional_actions @@ -82,24 +109,11 @@ fn real_main() -> i32 { return 1; } - // Copy the `static` directory into the export package if it exists - // If the user wants extra, they can use static aliases, plugins are unnecessary here - let static_dir = PathBuf::from("../static"); - if static_dir.exists() { - if let Err(err) = copy_dir(&static_dir, "dist/exported/.perseus/", &CopyOptions::new()) { - let err_msg = format!( - "Static exporting failed: 'couldn't copy static directory: '{}''", - &err - ); - plugins - .functional_actions - .export_actions - .after_failed_static_copy - .run(err.to_string(), plugins.get_plugin_data()); - eprintln!("{}", err_msg); - return 1; - } - } + 0 +} + +fn copy_static_dir() -> i32 { + let plugins = get_plugins::(); // Loop through any static aliases and copy them in too // Unlike with the server, these could override pages! // We'll copy from the alias to the path (it could be a directory or a file) @@ -141,11 +155,29 @@ fn real_main() -> i32 { } } - plugins - .functional_actions - .export_actions - .after_successful_export - .run((), plugins.get_plugin_data()); - println!("Static exporting successfully completed!"); + 0 +} + +fn copy_static_aliases() -> i32 { + let plugins = get_plugins::(); + // Copy the `static` directory into the export package if it exists + // If the user wants extra, they can use static aliases, plugins are unnecessary here + let static_dir = PathBuf::from("../static"); + if static_dir.exists() { + if let Err(err) = copy_dir(&static_dir, "dist/exported/.perseus/", &CopyOptions::new()) { + let err_msg = format!( + "Static exporting failed: 'couldn't copy static directory: '{}''", + &err + ); + plugins + .functional_actions + .export_actions + .after_failed_static_copy + .run(err.to_string(), plugins.get_plugin_data()); + eprintln!("{}", err_msg); + return 1; + } + } + 0 } diff --git a/examples/basic/.perseus/builder/src/bin/tinker.rs b/examples/basic/.perseus/builder/src/bin/tinker.rs index f104ffa569..c51a77ca46 100644 --- a/examples/basic/.perseus/builder/src/bin/tinker.rs +++ b/examples/basic/.perseus/builder/src/bin/tinker.rs @@ -12,6 +12,8 @@ fn real_main() -> i32 { let plugins = get_plugins::(); // Run all the tinker actions + // Note: this is deliberately synchronous, tinker actions that need a multithreaded async runtime should probably + // be making their own engines! plugins .functional_actions .tinker diff --git a/packages/perseus-actix-web/Cargo.toml b/packages/perseus-actix-web/Cargo.toml index f358d5dd02..cc5adc034c 100644 --- a/packages/perseus-actix-web/Cargo.toml +++ b/packages/perseus-actix-web/Cargo.toml @@ -16,6 +16,7 @@ categories = ["wasm", "web-programming::http-server", "development-tools", "asyn [dependencies] perseus = { path = "../perseus", version = "0.3.0" } actix-web = "=4.0.0-beta.15" +actix-http = "=3.0.0-beta.16" # Without this, Actix can introduce breaking changes in a dependency tree actix-files = "=0.6.0-beta.10" urlencoding = "2.1" serde = "1" diff --git a/packages/perseus/Cargo.toml b/packages/perseus/Cargo.toml index 684291b232..044a52278c 100644 --- a/packages/perseus/Cargo.toml +++ b/packages/perseus/Cargo.toml @@ -34,6 +34,7 @@ cfg-if = "1" fluent-bundle = { version = "0.15", optional = true } unic-langid = { version = "0.9", optional = true } intl-memoizer = { version = "0.5", optional = true } +tokio = { version = "1", features = [ "fs", "io-util" ] } [features] default = [] diff --git a/packages/perseus/src/build.rs b/packages/perseus/src/build.rs index 92dda77807..a3a8d99d20 100644 --- a/packages/perseus/src/build.rs +++ b/packages/perseus/src/build.rs @@ -2,12 +2,12 @@ use crate::errors::*; use crate::locales::Locales; +use crate::templates::TemplateMap; use crate::translations_manager::TranslationsManager; use crate::translator::Translator; use crate::{ decode_time_str::decode_time_str, stores::{ImmutableStore, MutableStore}, - template::TemplateMap, Template, }; use futures::future::try_join_all; @@ -63,128 +63,143 @@ pub async fn build_template( // Iterate through the paths to generate initial states if needed // Note that build paths pages on incrementally generable pages will use the immutable store + let mut futs = Vec::new(); for path in paths.iter() { - // If needed, we'll contruct a full path that's URL encoded so we can easily save it as a file - let full_path_without_locale = match template.uses_build_paths() { - true => format!("{}/{}", &template_path, path), - // We don't want to concatenate the name twice if we don't have to - false => template_path.clone(), - }; - // Strip trailing `/`s for the reasons described above - let full_path_without_locale = match full_path_without_locale.strip_suffix('/') { - Some(stripped) => stripped.to_string(), - None => full_path_without_locale, - }; - // Add the current locale to the front of that na dencode it as a URL so we can store a flat series of files - // BUG: insanely nested paths won't work whatsoever if the filename is too long, maybe hash instead? - let full_path_encoded = format!( - "{}-{}", - translator.get_locale(), - urlencoding::encode(&full_path_without_locale) - ); + let fut = gen_state_for_path(path, template, translator, (immutable_store, mutable_store)); + futs.push(fut); + } + try_join_all(futs).await?; - // Handle static initial state generation - // We'll only write a static state if one is explicitly generated - // If the template revalidates, use a mutable store, otherwise use an immutable one - if template.uses_build_state() && template.revalidates() { - // We pass in the path to get a state (including the template path for consistency with the incremental logic) - let initial_state = template - .get_build_state(full_path_without_locale.clone(), translator.get_locale()) - .await?; - // Write that intial state to a static JSON file - mutable_store - .write( - &format!("static/{}.json", full_path_encoded), - &initial_state, - ) - .await?; - // Prerender the template using that state - let prerendered = sycamore::render_to_string(|| { - template.render_for_template(Some(initial_state.clone()), translator, true) - }); - // Write that prerendered HTML to a static file - mutable_store - .write(&format!("static/{}.html", full_path_encoded), &prerendered) - .await?; - // Prerender the document `` with that state - // If the page also uses request state, amalgamation will be applied as for the normal content - let head_str = template.render_head_str(Some(initial_state), translator); - mutable_store - .write( - &format!("static/{}.head.html", full_path_encoded), - &head_str, - ) - .await?; - } else if template.uses_build_state() { - // We pass in the path to get a state (including the template path for consistency with the incremental logic) - let initial_state = template - .get_build_state(full_path_without_locale.clone(), translator.get_locale()) - .await?; - // Write that intial state to a static JSON file - immutable_store - .write( - &format!("static/{}.json", full_path_encoded), - &initial_state, - ) - .await?; - // Prerender the template using that state - let prerendered = sycamore::render_to_string(|| { - template.render_for_template(Some(initial_state.clone()), translator, true) - }); - // Write that prerendered HTML to a static file - immutable_store - .write(&format!("static/{}.html", full_path_encoded), &prerendered) - .await?; - // Prerender the document `` with that state - // If the page also uses request state, amalgamation will be applied as for the normal content - let head_str = template.render_head_str(Some(initial_state), translator); - immutable_store - .write( - &format!("static/{}.head.html", full_path_encoded), - &head_str, - ) - .await?; - } + Ok((paths, single_page)) +} - // Handle revalidation, we need to parse any given time strings into datetimes - // We don't need to worry about revalidation that operates by logic, that's request-time only - if template.revalidates_with_time() { - let datetime_to_revalidate = - decode_time_str(&template.get_revalidate_interval().unwrap())?; - // Write that to a static file, we'll update it every time we revalidate - // Note that this runs for every path generated, so it's fully usable with ISR - // Yes, there's a different revalidation schedule for each locale, but that means we don't have to rebuild every locale simultaneously - mutable_store - .write( - &format!("static/{}.revld.txt", full_path_encoded), - &datetime_to_revalidate.to_string(), - ) - .await?; - } +/// Generates state for a single page within a template. This is broken out into a separate function for concurrency. +async fn gen_state_for_path( + path: &str, + template: &Template, + translator: &Translator, + (immutable_store, mutable_store): (&ImmutableStore, &impl MutableStore), +) -> Result<(), ServerError> { + let template_path = template.get_path(); + // If needed, we'll contruct a full path that's URL encoded so we can easily save it as a file + let full_path_without_locale = match template.uses_build_paths() { + true => format!("{}/{}", &template_path, path), + // We don't want to concatenate the name twice if we don't have to + false => template_path.clone(), + }; + // Strip trailing `/`s for the reasons described above + let full_path_without_locale = match full_path_without_locale.strip_suffix('/') { + Some(stripped) => stripped.to_string(), + None => full_path_without_locale, + }; + // Add the current locale to the front of that na dencode it as a URL so we can store a flat series of files + // BUG: insanely nested paths won't work whatsoever if the filename is too long, maybe hash instead? + let full_path_encoded = format!( + "{}-{}", + translator.get_locale(), + urlencoding::encode(&full_path_without_locale) + ); - // Note that SSR has already been handled by checking for `.uses_request_state()` above, we don't need to do any rendering here - // If a template only uses SSR, it won't get prerendered at build time whatsoever + // Handle static initial state generation + // We'll only write a static state if one is explicitly generated + // If the template revalidates, use a mutable store, otherwise use an immutable one + if template.uses_build_state() && template.revalidates() { + // We pass in the path to get a state (including the template path for consistency with the incremental logic) + let initial_state = template + .get_build_state(full_path_without_locale.clone(), translator.get_locale()) + .await?; + // Write that intial state to a static JSON file + mutable_store + .write( + &format!("static/{}.json", full_path_encoded), + &initial_state, + ) + .await?; + // Prerender the template using that state + let prerendered = sycamore::render_to_string(|| { + template.render_for_template(Some(initial_state.clone()), translator, true) + }); + // Write that prerendered HTML to a static file + mutable_store + .write(&format!("static/{}.html", full_path_encoded), &prerendered) + .await?; + // Prerender the document `` with that state + // If the page also uses request state, amalgamation will be applied as for the normal content + let head_str = template.render_head_str(Some(initial_state), translator); + mutable_store + .write( + &format!("static/{}.head.html", full_path_encoded), + &head_str, + ) + .await?; + } else if template.uses_build_state() { + // We pass in the path to get a state (including the template path for consistency with the incremental logic) + let initial_state = template + .get_build_state(full_path_without_locale.clone(), translator.get_locale()) + .await?; + // Write that intial state to a static JSON file + immutable_store + .write( + &format!("static/{}.json", full_path_encoded), + &initial_state, + ) + .await?; + // Prerender the template using that state + let prerendered = sycamore::render_to_string(|| { + template.render_for_template(Some(initial_state.clone()), translator, true) + }); + // Write that prerendered HTML to a static file + immutable_store + .write(&format!("static/{}.html", full_path_encoded), &prerendered) + .await?; + // Prerender the document `` with that state + // If the page also uses request state, amalgamation will be applied as for the normal content + let head_str = template.render_head_str(Some(initial_state), translator); + immutable_store + .write( + &format!("static/{}.head.html", full_path_encoded), + &head_str, + ) + .await?; + } - // If the template is very basic, prerender without any state - // It's safe to add a property to the render options here because `.is_basic()` will only return true if path generation is not being used (or anything else) - if template.is_basic() { - let prerendered = - sycamore::render_to_string(|| template.render_for_template(None, translator, true)); - let head_str = template.render_head_str(None, translator); - // Write that prerendered HTML to a static file - immutable_store - .write(&format!("static/{}.html", full_path_encoded), &prerendered) - .await?; - immutable_store - .write( - &format!("static/{}.head.html", full_path_encoded), - &head_str, - ) - .await?; - } + // Handle revalidation, we need to parse any given time strings into datetimes + // We don't need to worry about revalidation that operates by logic, that's request-time only + if template.revalidates_with_time() { + let datetime_to_revalidate = decode_time_str(&template.get_revalidate_interval().unwrap())?; + // Write that to a static file, we'll update it every time we revalidate + // Note that this runs for every path generated, so it's fully usable with ISR + // Yes, there's a different revalidation schedule for each locale, but that means we don't have to rebuild every locale simultaneously + mutable_store + .write( + &format!("static/{}.revld.txt", full_path_encoded), + &datetime_to_revalidate.to_string(), + ) + .await?; } - Ok((paths, single_page)) + // Note that SSR has already been handled by checking for `.uses_request_state()` above, we don't need to do any rendering here + // If a template only uses SSR, it won't get prerendered at build time whatsoever + + // If the template is very basic, prerender without any state + // It's safe to add a property to the render options here because `.is_basic()` will only return true if path generation is not being used (or anything else) + if template.is_basic() { + let prerendered = + sycamore::render_to_string(|| template.render_for_template(None, translator, true)); + let head_str = template.render_head_str(None, translator); + // Write that prerendered HTML to a static file + immutable_store + .write(&format!("static/{}.html", full_path_encoded), &prerendered) + .await?; + immutable_store + .write( + &format!("static/{}.head.html", full_path_encoded), + &head_str, + ) + .await?; + } + + Ok(()) } async fn build_template_and_get_cfg( diff --git a/packages/perseus/src/errors.rs b/packages/perseus/src/errors.rs index a71bfda19b..1e94440341 100644 --- a/packages/perseus/src/errors.rs +++ b/packages/perseus/src/errors.rs @@ -62,7 +62,7 @@ pub fn err_to_status_code(err: &ServerError) -> u16 { } } -/// Errors that can occur while reading from or writing to an immutable store. +/// Errors that can occur while reading from or writing to a mutable or immutable store. #[derive(Error, Debug)] pub enum StoreError { #[error("asset '{name}' not found in store")] diff --git a/packages/perseus/src/export.rs b/packages/perseus/src/export.rs index 92bc50996f..0c1ae46c4d 100644 --- a/packages/perseus/src/export.rs +++ b/packages/perseus/src/export.rs @@ -9,6 +9,7 @@ use crate::stores::ImmutableStore; use crate::template::TemplateMap; use crate::translations_manager::TranslationsManager; use crate::SsrNode; +use futures::future::{try_join, try_join_all}; use std::fs; /// Gets the static page data. @@ -62,120 +63,160 @@ pub async fn export_app( })?; let html_shell = prep_html_shell(raw_html_shell, &render_cfg, &path_prefix); + // We can do literally everything concurrently here + let mut export_futs = Vec::new(); // Loop over every partial for (path, template_path) in render_cfg { - // We need the encoded path to reference flattened build artifacts - // But we don't create a flattened system with exporting, everything is properly created in a directory structure - let path_encoded = urlencoding::encode(&path).to_string(); - // All initial load pages should be written into their own folders, which prevents a situation of a template root page outside the directory for the rest of that template's pages (see #73) - // The `.html` file extension is added when this variable is used (for contrast to the `.json`s) - let initial_load_path = if path.ends_with("index") { - // However, if it's already an index page, we dont want `index/index.html` - path.to_string() - } else { - format!("{}/index", &path) - }; - - // Get the template itself - let template = templates.get(&template_path); - let template = match template { - Some(template) => template, - None => { - return Err(ServeError::PageNotFound { - path: template_path, - } - .into()) - } - }; - // Create a locale detection file for it if we're using i18n - // These just send the app shell, which will perform a redirect as necessary - // Notably, these also include fallback redirectors if either Wasm or JS is disabled (or both) - // TODO put everything inside its own folder for initial loads? - if locales.using_i18n { - immutable_store - .write( - &format!("exported/{}.html", &initial_load_path), - &interpolate_locale_redirection_fallback( - &html_shell, - // If we don't include the path prefix, fallback redirection will fail for relative paths - &format!("{}/{}/{}", path_prefix, locales.default, &path), - root_id, - ), - ) - .await?; + let fut = export_path( + (path.to_string(), template_path.to_string()), + templates, + locales, + &html_shell, + root_id, + immutable_store, + path_prefix.to_string(), + ); + export_futs.push(fut); + } + // If we're using i18n, loop through the locales to create translations files + let mut translations_futs = Vec::new(); + if locales.using_i18n { + for locale in locales.get_all() { + let fut = create_translation_file(locale, immutable_store, translations_manager); + translations_futs.push(fut); } - // Check if that template uses build state (in which case it should have a JSON file) - let has_state = template.uses_build_state(); - if locales.using_i18n { - // Loop through all the app's locales - for locale in locales.get_all() { - let page_data = get_static_page_data( - &format!("{}-{}", locale, &path_encoded), - has_state, - immutable_store, - ) - .await?; - // Create a full HTML file from those that can be served for initial loads - // The build process writes these with a dummy default locale even though we're not using i18n - let full_html = interpolate_page_data(&html_shell, &page_data, root_id); - immutable_store - .write( - &format!("exported/{}/{}.html", locale, initial_load_path), - &full_html, - ) - .await?; + } + + try_join(try_join_all(export_futs), try_join_all(translations_futs)).await?; + + // Copying in bundles from the filesystem is left to the CLI command for exporting, so we're done! + + Ok(()) +} + +/// Creates a translation file for exporting. This is broken out for concurrency. +async fn create_translation_file( + locale: &str, + immutable_store: &ImmutableStore, + translations_manager: &impl TranslationsManager, +) -> Result<(), ServerError> { + // Get the translations string for that + let translations_str = translations_manager + .get_translations_str_for_locale(locale.to_string()) + .await?; + // Write it to an asset so that it can be served directly + immutable_store + .write( + &format!("exported/.perseus/translations/{}", locale), + &translations_str, + ) + .await?; + + Ok(()) +} + +async fn export_path( + (path, template_path): (String, String), + templates: &TemplateMap, + locales: &Locales, + html_shell: &str, + root_id: &str, + immutable_store: &ImmutableStore, + path_prefix: String, +) -> Result<(), ServerError> { + // We need the encoded path to reference flattened build artifacts + // But we don't create a flattened system with exporting, everything is properly created in a directory structure + let path_encoded = urlencoding::encode(&path).to_string(); + // All initial load pages should be written into their own folders, which prevents a situation of a template root page outside the directory for the rest of that template's pages (see #73) + // The `.html` file extension is added when this variable is used (for contrast to the `.json`s) + let initial_load_path = if path.ends_with("index") { + // However, if it's already an index page, we dont want `index/index.html` + path.to_string() + } else { + format!("{}/index", &path) + }; - // Serialize the page data to JSON and write it as a partial (fetched by the app shell for subsequent loads) - let partial = serde_json::to_string(&page_data).unwrap(); - immutable_store - .write( - &format!("exported/.perseus/page/{}/{}.json", locale, &path), - &partial, - ) - .await?; + // Get the template itself + let template = templates.get(&template_path); + let template = match template { + Some(template) => template, + None => { + return Err(ServeError::PageNotFound { + path: template_path.to_string(), } - } else { + .into()) + } + }; + // Create a locale detection file for it if we're using i18n + // These just send the app shell, which will perform a redirect as necessary + // Notably, these also include fallback redirectors if either Wasm or JS is disabled (or both) + if locales.using_i18n { + immutable_store + .write( + &format!("exported/{}.html", &initial_load_path), + &interpolate_locale_redirection_fallback( + html_shell, + // If we don't include the path prefix, fallback redirection will fail for relative paths + &format!("{}/{}/{}", path_prefix, locales.default, &path), + root_id, + ), + ) + .await?; + } + // Check if that template uses build state (in which case it should have a JSON file) + let has_state = template.uses_build_state(); + if locales.using_i18n { + // Loop through all the app's locales + for locale in locales.get_all() { let page_data = get_static_page_data( - &format!("{}-{}", locales.default, &path_encoded), + &format!("{}-{}", locale, &path_encoded), has_state, immutable_store, ) .await?; // Create a full HTML file from those that can be served for initial loads // The build process writes these with a dummy default locale even though we're not using i18n - let full_html = interpolate_page_data(&html_shell, &page_data, root_id); - // We don't add an extension because this will be queried directly by the browser + let full_html = interpolate_page_data(html_shell, &page_data, root_id); immutable_store - .write(&format!("exported/{}.html", initial_load_path), &full_html) + .write( + &format!("exported/{}/{}.html", locale, initial_load_path), + &full_html, + ) .await?; // Serialize the page data to JSON and write it as a partial (fetched by the app shell for subsequent loads) let partial = serde_json::to_string(&page_data).unwrap(); immutable_store .write( - &format!("exported/.perseus/page/{}/{}.json", locales.default, &path), + &format!("exported/.perseus/page/{}/{}.json", locale, &path), &partial, ) .await?; } + } else { + let page_data = get_static_page_data( + &format!("{}-{}", locales.default, &path_encoded), + has_state, + immutable_store, + ) + .await?; + // Create a full HTML file from those that can be served for initial loads + // The build process writes these with a dummy default locale even though we're not using i18n + let full_html = interpolate_page_data(html_shell, &page_data, root_id); + // We don't add an extension because this will be queried directly by the browser + immutable_store + .write(&format!("exported/{}.html", initial_load_path), &full_html) + .await?; + + // Serialize the page data to JSON and write it as a partial (fetched by the app shell for subsequent loads) + let partial = serde_json::to_string(&page_data).unwrap(); + immutable_store + .write( + &format!("exported/.perseus/page/{}/{}.json", locales.default, &path), + &partial, + ) + .await?; } - // If we're using i18n, loop through the locales to create translations files - if locales.using_i18n { - for locale in locales.get_all() { - // Get the translations string for that - let translations_str = translations_manager - .get_translations_str_for_locale(locale.to_string()) - .await?; - // Write it to an asset so that it can be served directly - immutable_store - .write( - &format!("exported/.perseus/translations/{}", locale), - &translations_str, - ) - .await?; - } - } - // Copying in bundles from the filesystem is left to the CLI command for exporting, so we're done! Ok(()) } diff --git a/packages/perseus/src/plugins/action.rs b/packages/perseus/src/plugins/action.rs index c59372f379..d9d580e698 100644 --- a/packages/perseus/src/plugins/action.rs +++ b/packages/perseus/src/plugins/action.rs @@ -2,19 +2,23 @@ use std::any::Any; use std::collections::HashMap; /// A runner function, which takes action data and plugin data. -pub type Runner = Box R>; +pub type Runner = Box R + Send>; /// A trait for the interface for a plugin action, which abstracts whether it's a functional or a control action. -pub trait PluginAction { +pub trait PluginAction: Send { /// Runs the action. This takes data that the action should expect, along with a map of plugins to their data. - fn run(&self, action_data: A, plugin_data: &HashMap>) -> R2; + fn run(&self, action_data: A, plugin_data: &HashMap>) -> R2; /// Registers a plugin that takes this action. /// /// # Panics /// If the action type can only be taken by one plugin, and one has already been set, this may panic (e.g. for control actions), /// as this is a critical, unrecoverable error that Perseus doesn't need to do anything after. If a plugin registration fails, /// we have to assume that all work in the engine may be not what the user intended. - fn register_plugin(&mut self, name: &str, runner: impl Fn(&A, &dyn Any) -> R + 'static); + fn register_plugin( + &mut self, + name: &str, + runner: impl Fn(&A, &(dyn Any + Send)) -> R + Send + 'static, + ); /// Same as `.register_plugin()`, but takes a prepared runner in a `Box`. fn register_plugin_box(&mut self, name: &str, runner: Runner); } diff --git a/packages/perseus/src/plugins/control.rs b/packages/perseus/src/plugins/control.rs index 31c135405a..86d8abf6e0 100644 --- a/packages/perseus/src/plugins/control.rs +++ b/packages/perseus/src/plugins/control.rs @@ -1,4 +1,4 @@ -use crate::plugins::{PluginAction, Runner}; +use crate::plugins::*; use std::any::Any; use std::collections::HashMap; @@ -12,7 +12,7 @@ pub struct ControlPluginAction { } impl PluginAction> for ControlPluginAction { /// Runs the single registered runner for the action. - fn run(&self, action_data: A, plugin_data: &HashMap>) -> Option { + fn run(&self, action_data: A, plugin_data: &HashMap>) -> Option { // If no runner is defined, this won't have any effect (same as functional actions with no registered runners) self.runner.as_ref().map(|runner| { runner( @@ -27,7 +27,11 @@ impl PluginAction> for ControlPluginAction { ) }) } - fn register_plugin(&mut self, name: &str, runner: impl Fn(&A, &dyn Any) -> R + 'static) { + fn register_plugin( + &mut self, + name: &str, + runner: impl Fn(&A, &(dyn Any + Send)) -> R + Send + 'static, + ) { self.register_plugin_box(name, Box::new(runner)) } fn register_plugin_box(&mut self, name: &str, runner: Runner) { diff --git a/packages/perseus/src/plugins/functional.rs b/packages/perseus/src/plugins/functional.rs index 9ffa933faa..eaccc5e656 100644 --- a/packages/perseus/src/plugins/functional.rs +++ b/packages/perseus/src/plugins/functional.rs @@ -1,4 +1,4 @@ -use crate::plugins::{PluginAction, Runner}; +use crate::plugins::*; use crate::Html; use std::any::Any; use std::collections::HashMap; @@ -12,7 +12,7 @@ impl PluginAction> for FunctionalPluginAction>, + plugin_data: &HashMap>, ) -> HashMap { let mut returns: HashMap = HashMap::new(); for (plugin_name, runner) in &self.runners { @@ -28,7 +28,11 @@ impl PluginAction> for FunctionalPluginAction R + 'static) { + fn register_plugin( + &mut self, + name: &str, + runner: impl Fn(&A, &(dyn Any + Send)) -> R + Send + 'static, + ) { self.register_plugin_box(name, Box::new(runner)) } fn register_plugin_box(&mut self, name: &str, runner: Runner) { diff --git a/packages/perseus/src/plugins/mod.rs b/packages/perseus/src/plugins/mod.rs index 2f5e797a0c..0b35ce1efd 100644 --- a/packages/perseus/src/plugins/mod.rs +++ b/packages/perseus/src/plugins/mod.rs @@ -10,6 +10,13 @@ pub use functional::*; pub use plugin::*; pub use plugins_list::*; +/// A helper function for plugins that don't take any functional actions. This just inserts and empty registrar. +pub fn empty_functional_actions_registrar( + _: FunctionalPluginActions, +) -> FunctionalPluginActions { + FunctionalPluginActions::default() +} + /// A helper function for plugins that don't take any control actions. This just inserts an empty registrar. pub fn empty_control_actions_registrar(_: ControlPluginActions) -> ControlPluginActions { ControlPluginActions::default() diff --git a/packages/perseus/src/plugins/plugin.rs b/packages/perseus/src/plugins/plugin.rs index d49ae2bc25..aa879935c4 100644 --- a/packages/perseus/src/plugins/plugin.rs +++ b/packages/perseus/src/plugins/plugin.rs @@ -21,7 +21,7 @@ pub enum PluginEnv { } /// A Perseus plugin. This must be exported by all plugin crates so the user can register the plugin easily. -pub struct Plugin { +pub struct Plugin { /// The machine name of the plugin, which will be used as a key in a HashMap with many other plugins. This should be the public /// crate name in all cases. pub name: String, @@ -36,7 +36,7 @@ pub struct Plugin { plugin_data_type: PhantomData, } -impl Plugin { +impl Plugin { /// Creates a new plugin with a name, functional actions, control actions, and whether or not the plugin is tinker-only. pub fn new( name: &str, diff --git a/packages/perseus/src/plugins/plugins_list.rs b/packages/perseus/src/plugins/plugins_list.rs index bfbf5ee10a..ef49692acd 100644 --- a/packages/perseus/src/plugins/plugins_list.rs +++ b/packages/perseus/src/plugins/plugins_list.rs @@ -3,7 +3,7 @@ use crate::Html; use std::any::Any; use std::collections::HashMap; -type PluginDataMap = HashMap>; +type PluginDataMap = HashMap>; /// A representation of all the plugins used by an app. Due to the sheer number and compexity of nested fields, this is best transferred /// in an `Rc`, which unfortunately results in double indirection for runner functions. @@ -33,10 +33,10 @@ impl Plugins { /// Registers a new plugin, consuming `self`. For control actions, this will check if a plugin has already registered on an action, /// and throw an error if one has, noting the conflict explicitly in the error message. This can only register plugins that run /// exclusively on the server-side (including tinker-time and the build process). - pub fn plugin( + pub fn plugin( mut self, // This is a function so that it never gets called if we're compiling for Wasm, which means Rust eliminates it as dead code! - plugin: impl Fn() -> Plugin, + plugin: impl Fn() -> Plugin + Send, plugin_data: D, ) -> Self { // If we're compiling for Wasm, plugins that don't run on the client side shouldn't be added (they'll then be eliminated as dead code) @@ -49,7 +49,7 @@ impl Plugins { panic!("attempted to register plugin that can run on the client with `.plugin()`, this plugin should be registered with `.plugin_with_client_privilege()` (this will increase your final bundle size)") } // Insert the plugin data - let plugin_data: Box = Box::new(plugin_data); + let plugin_data: Box = Box::new(plugin_data); let res = self.plugin_data.insert(plugin.name.clone(), plugin_data); // If there was an old value, there are two plugins with the same name, which is very bad (arbitrarily inconsistent behavior overriding) if res.is_some() { @@ -66,10 +66,10 @@ impl Plugins { /// The same as `.plugin()`, but registers a plugin that can run on the client-side. This is deliberately separated out to make /// conditional compilation feasible and to emphasize to users what's incrasing their bundle sizes. Note that this should also /// be used for plugins that run on both the client and server. - pub fn plugin_with_client_privilege( + pub fn plugin_with_client_privilege( mut self, // This is a function to preserve a similar API interface with `.plugin()` - plugin: impl Fn() -> Plugin, + plugin: impl Fn() -> Plugin + Send, plugin_data: D, ) -> Self { let plugin = plugin(); @@ -78,7 +78,7 @@ impl Plugins { panic!("attempted to register plugin that doesn't ever run on the client with `.plugin_with_client_privilege()`, you should use `.plugin()` instead") } // Insert the plugin data - let plugin_data: Box = Box::new(plugin_data); + let plugin_data: Box = Box::new(plugin_data); let res = self.plugin_data.insert(plugin.name.clone(), plugin_data); // If there was an old value, there are two plugins with the same name, which is very bad (arbitrarily inconsistent behavior overriding) if res.is_some() { diff --git a/packages/perseus/src/stores/immutable.rs b/packages/perseus/src/stores/immutable.rs index bcfa23ae53..e6622fc078 100644 --- a/packages/perseus/src/stores/immutable.rs +++ b/packages/perseus/src/stores/immutable.rs @@ -1,5 +1,8 @@ use crate::errors::*; -use std::fs; +use tokio::{ + fs::{create_dir_all, File}, + io::{AsyncReadExt, AsyncWriteExt}, +}; /// An immutable storage system. This wraps filesystem calls in a sensible asynchronous API, allowing abstraction of the base path /// to a distribution directory or the like. Perseus uses this to store assts created at build time that won't change, which is @@ -18,11 +21,25 @@ impl ImmutableStore { /// Reads the given asset from the filesystem asynchronously. pub async fn read(&self, name: &str) -> Result { let asset_path = format!("{}/{}", self.root_path, name); - match fs::metadata(&asset_path) { - Ok(_) => fs::read_to_string(&asset_path).map_err(|err| StoreError::ReadFailed { - name: asset_path, + let mut file = File::open(&asset_path) + .await + .map_err(|err| StoreError::ReadFailed { + name: asset_path.clone(), source: err.into(), - }), + })?; + let metadata = file.metadata().await; + + match metadata { + Ok(_) => { + let mut contents = String::new(); + file.read_to_string(&mut contents) + .await + .map_err(|err| StoreError::ReadFailed { + name: asset_path, + source: err.into(), + })?; + Ok(contents) + } Err(err) if err.kind() == std::io::ErrorKind::NotFound => { Err(StoreError::NotFound { name: asset_path }) } @@ -39,13 +56,34 @@ impl ImmutableStore { let mut dir_tree: Vec<&str> = asset_path.split('/').collect(); dir_tree.pop(); - fs::create_dir_all(dir_tree.join("/")).map_err(|err| StoreError::WriteFailed { - name: asset_path.clone(), - source: err.into(), - })?; - fs::write(&asset_path, content).map_err(|err| StoreError::WriteFailed { - name: asset_path, - source: err.into(), - }) + create_dir_all(dir_tree.join("/")) + .await + .map_err(|err| StoreError::WriteFailed { + name: asset_path.clone(), + source: err.into(), + })?; + + // This will either create the file or truncate it if it already exists + let mut file = File::create(&asset_path) + .await + .map_err(|err| StoreError::WriteFailed { + name: asset_path.clone(), + source: err.into(), + })?; + file.write_all(content.as_bytes()) + .await + .map_err(|err| StoreError::WriteFailed { + name: asset_path.clone(), + source: err.into(), + })?; + // TODO Can we use `sync_data()` here to reduce I/O? + file.sync_all() + .await + .map_err(|err| StoreError::WriteFailed { + name: asset_path, + source: err.into(), + })?; + + Ok(()) } } diff --git a/packages/perseus/src/stores/mutable.rs b/packages/perseus/src/stores/mutable.rs index 21b2d1317b..3bdfedcec1 100644 --- a/packages/perseus/src/stores/mutable.rs +++ b/packages/perseus/src/stores/mutable.rs @@ -1,5 +1,8 @@ use crate::errors::*; -use std::fs; +use tokio::{ + fs::{create_dir_all, File}, + io::{AsyncReadExt, AsyncWriteExt}, +}; /// A trait for mutable stores. This is abstracted away so that users can implement a non-filesystem mutable store, which is useful /// for read-only filesystem environments, as on many modern hosting providers. See the book for further details on this subject. @@ -31,20 +34,32 @@ impl FsMutableStore { impl MutableStore for FsMutableStore { async fn read(&self, name: &str) -> Result { let asset_path = format!("{}/{}", self.root_path, name); - match fs::metadata(&asset_path) { - Ok(_) => fs::read_to_string(&asset_path).map_err(|err| StoreError::ReadFailed { - name: asset_path, + let mut file = File::open(&asset_path) + .await + .map_err(|err| StoreError::ReadFailed { + name: asset_path.clone(), source: err.into(), - }), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - return Err(StoreError::NotFound { name: asset_path }) + })?; + let metadata = file.metadata().await; + + match metadata { + Ok(_) => { + let mut contents = String::new(); + file.read_to_string(&mut contents) + .await + .map_err(|err| StoreError::ReadFailed { + name: asset_path, + source: err.into(), + })?; + Ok(contents) } - Err(err) => { - return Err(StoreError::ReadFailed { - name: asset_path, - source: err.into(), - }) + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + Err(StoreError::NotFound { name: asset_path }) } + Err(err) => Err(StoreError::ReadFailed { + name: asset_path, + source: err.into(), + }), } } // This creates a directory structure as necessary @@ -53,13 +68,34 @@ impl MutableStore for FsMutableStore { let mut dir_tree: Vec<&str> = asset_path.split('/').collect(); dir_tree.pop(); - fs::create_dir_all(dir_tree.join("/")).map_err(|err| StoreError::WriteFailed { - name: asset_path.clone(), - source: err.into(), - })?; - fs::write(&asset_path, content).map_err(|err| StoreError::WriteFailed { - name: asset_path, - source: err.into(), - }) + create_dir_all(dir_tree.join("/")) + .await + .map_err(|err| StoreError::WriteFailed { + name: asset_path.clone(), + source: err.into(), + })?; + + // This will either create the file or truncate it if it already exists + let mut file = File::create(&asset_path) + .await + .map_err(|err| StoreError::WriteFailed { + name: asset_path.clone(), + source: err.into(), + })?; + file.write_all(content.as_bytes()) + .await + .map_err(|err| StoreError::WriteFailed { + name: asset_path.clone(), + source: err.into(), + })?; + // TODO Can we use `sync_data()` here to reduce I/O? + file.sync_all() + .await + .map_err(|err| StoreError::WriteFailed { + name: asset_path, + source: err.into(), + })?; + + Ok(()) } } diff --git a/packages/perseus/src/translations_manager.rs b/packages/perseus/src/translations_manager.rs index a9b5a465a0..139dcb0d5d 100644 --- a/packages/perseus/src/translations_manager.rs +++ b/packages/perseus/src/translations_manager.rs @@ -27,7 +27,8 @@ pub enum TranslationsManagerError { use crate::translator::Translator; use futures::future::join_all; use std::collections::HashMap; -use std::fs; +use tokio::fs::File; +use tokio::io::AsyncReadExt; /// A trait for systems that manage where to put translations. At simplest, we'll just write them to static files, but they might also /// be stored in a CMS. It is **strongly** advised that any implementations use some form of caching, guided by `FsTranslationsManager`. @@ -117,24 +118,33 @@ impl TranslationsManager for FsTranslationsManager { } else { // The file must be named as the locale it describes let asset_path = format!("{}/{}.{}", self.root_path, locale, self.file_ext); - let translations_str = match fs::metadata(&asset_path) { - Ok(_) => fs::read_to_string(&asset_path).map_err(|err| { - TranslationsManagerError::ReadFailed { - locale: locale.clone(), - source: err.into(), - } - })?, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - return Err(TranslationsManagerError::NotFound { locale }) + let mut file = File::open(&asset_path).await.map_err(|err| { + TranslationsManagerError::ReadFailed { + locale: locale.clone(), + source: err.into(), } - Err(err) => { - return Err(TranslationsManagerError::ReadFailed { - locale, - source: err.into(), - }) + })?; + let metadata = file.metadata().await; + + match metadata { + Ok(_) => { + let mut contents = String::new(); + file.read_to_string(&mut contents).await.map_err(|err| { + TranslationsManagerError::ReadFailed { + locale: locale.clone(), + source: err.into(), + } + })?; + Ok(contents) } - }; - Ok(translations_str) + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + Err(TranslationsManagerError::NotFound { locale }) + } + Err(err) => Err(TranslationsManagerError::ReadFailed { + locale, + source: err.into(), + }), + } } } async fn get_translator_for_locale(