Nightly cronjob CI (#2836)

* Uniform indentation

* wip nightly cronjob CI

* Fix GN task on nightly

* Pull in appropriate toolchain for memory CI

* Skip c-tiny/tinywasm tests on forced nightlies

* Make locale size tests work on forced nightlies

* Add way to skip doctests using CFGs

* fix for windows

* Move to makefile

* remove PR gating

* fix memory

Makefile.toml

@@ -21,6 +21,7 @@ ICU4X_NIGHTLY_TOOLCHAIN = { value = "nightly-2022-04-05", condition = { env_not_
 # https://chrome-infra-packages.appspot.com/p/gn/gn
 # TODO: Choose the correct distribution of GN automatically.
 ICU4X_GN_PACKAGE = { value = "gn/gn/linux-amd64", condition = { env_not_set = ["ICU4X_GN_PACKAGE"] } }
+ICU4X_BUILDING_WITH_FORCED_NIGHTLY = { value = "1", condition = { env_set = ["ICU4X_BUILDING_WITH_FORCED_NIGHTLY"] } }
 
 [tasks.quick]
 description = "Run quick version of all lints and builds (useful before pushing to GitHub)"
@@ -172,3 +173,47 @@ dependencies = [
 
     # benchmarking and coverage jobs not included
 ]
+
+[tasks.set-nightly-version-for-ci]
+description = "Set all nightly version env vars for CI"
+category = "CI"
+script_runner = "@duckscript"
+script = '''
+exit_on_error true
+env_file = get_env GITHUB_ENV
+if not ${env_file}
+    echo "GITHUB_ENV not set!"
+    exit 1
+end
+
+event_name = get_env GITHUB_EVENT_NAME
+
+if not ${event_name}
+    echo "GITHUB_EVENT_NAME not set!"
+    exit 1
+end
+
+is_schedule = eq "${event_name}" "schedule"
+is_dispatch = eq "${event_name}" "workflow_dispatch"
+dispatch_needs_nightly = set false
+
+if is_dispatch
+    event_path = get_env GITHUB_EVENT_PATH
+    event_file = readfile ${event_path}
+    event_json = json_parse ${event_file}
+    if ${event_json.inputs.nightly}
+        dispatch_needs_nightly = set true
+    end
+end
+
+if ${is_schedule} or ${dispatch_needs_nightly}
+    echo "Setting up CI environment for forced-nightly Rust build"
+    appendfile ${env_file} "ICU4X_NIGHTLY_TOOLCHAIN=nightly\n"
+    appendfile ${env_file} "INSTALLED_NIGHTLY_VERSION=nightly\n"
+    appendfile ${env_file} "ICU4X_BUILDING_WITH_FORCED_NIGHTLY=1\n"
+    appendfile ${env_file} "RUSTDOCFLAGS=--cfg ICU4X_BUILDING_WITH_FORCED_NIGHTLY\n"
+    exec rustup override set nightly
+else
+    echo "forced-nightly environment not required"
+end
+'''

components/locid/src/locale.rs

@@ -88,6 +88,8 @@ pub struct Locale {
 
 #[test]
 fn test_sizes() {
+    // Remove when we upgrade to a compiler where the new sizes are default
+    let forced_nightly = std::env::var("ICU4X_BUILDING_WITH_FORCED_NIGHTLY").is_ok();
     assert_eq!(core::mem::size_of::<subtags::Language>(), 3);
     assert_eq!(core::mem::size_of::<subtags::Script>(), 4);
     assert_eq!(core::mem::size_of::<subtags::Region>(), 3);
@@ -100,12 +102,21 @@ fn test_sizes() {
     assert_eq!(core::mem::size_of::<extensions::transform::Fields>(), 24);
 
     assert_eq!(core::mem::size_of::<extensions::unicode::Attributes>(), 24);
-    assert_eq!(core::mem::size_of::<extensions::unicode::Keywords>(), 48);
+    assert_eq!(
+        core::mem::size_of::<extensions::unicode::Keywords>(),
+        if forced_nightly { 40 } else { 48 }
+    );
     assert_eq!(core::mem::size_of::<Vec<extensions::other::Other>>(), 24);
     assert_eq!(core::mem::size_of::<extensions::private::Private>(), 24);
-    assert_eq!(core::mem::size_of::<extensions::Extensions>(), 192);
+    assert_eq!(
+        core::mem::size_of::<extensions::Extensions>(),
+        if forced_nightly { 184 } else { 192 }
+    );
 
-    assert_eq!(core::mem::size_of::<Locale>(), 240);
+    assert_eq!(
+        core::mem::size_of::<Locale>(),
+        if forced_nightly { 232 } else { 240 }
+    );
 }
 
 impl Locale {

ffi/gn/.cargo/config

@@ -3,7 +3,6 @@
 # (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
 # AUTO-GENERATED in ffi.toml
-
 [source.crates-io]
 replace-with = "vendored-sources"
 

provider/core/src/key.rs

@@ -144,6 +144,7 @@ impl DataKeyPath {
         /// ```compile_fail
         /// const unsafe fn canary() { core::slice::from_raw_parts(0 as *const u8, 0); }
         /// ```
+        #[cfg(not(ICU4X_BUILDING_WITH_FORCED_NIGHTLY))]
         const _: () = ();
         unsafe {
             // Safe due to invariant that self.path is tagged correctly

tools/scripts/ffi.toml

@@ -108,12 +108,16 @@ install_crate = { rustup_component_name = "rust-src" }
 script_runner = "@duckscript"
 script = '''
 exit_on_error true
-cd ffi/diplomat/c/examples/fixeddecimal_tiny
-rm -f ../../../../../target/debug/libicu_capi_staticlib.a
-rm -f ../../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi_staticlib.a
-rm -f ../../../../../target/x86_64-unknown-linux-gnu/release-opt-size/libicu_capi_staticlib.a
-exec --fail-on-error make
-exec ls -l
+if "${ICU4X_BUILDING_WITH_FORCED_NIGHTLY}"
+    echo "Skipping test-c-tiny since ICU4X_BUILDING_WITH_FORCED_NIGHTLY is set"
+else
+    cd ffi/diplomat/c/examples/fixeddecimal_tiny
+    rm -f ../../../../../target/debug/libicu_capi_staticlib.a
+    rm -f ../../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi_staticlib.a
+    rm -f ../../../../../target/x86_64-unknown-linux-gnu/release-opt-size/libicu_capi_staticlib.a
+    exec --fail-on-error make
+    exec ls -l
+end
 '''
 
 [tasks.test-cpp]
@@ -262,11 +266,15 @@ category = "ICU4X Development"
 script_runner = "@duckscript"
 script = '''
 exit_on_error true
-cd ffi/tinywasm
+if "${ICU4X_BUILDING_WITH_FORCED_NIGHTLY}"
+    echo "Skipping test-wasm since ICU4X_BUILDING_WITH_FORCED_NIGHTLY is set"
+else
+    cd ffi/tinywasm
 
-exec --fail-on-error ./build.sh
-exec --fail-on-error ls -l
-exec --fail-on-error node tiny.mjs
+    exec --fail-on-error ./build.sh
+    exec --fail-on-error ls -l
+    exec --fail-on-error node tiny.mjs
+end
 '''
 
 [tasks.diplomat-get-rev]

tools/scripts/gn.toml

@@ -60,11 +60,13 @@ cd ffi/gn
 
 rm -rf vendor
 output = exec --fail-on-error cargo vendor
+output_trimmed = trim_start ${output.stdout} # Needed since there are whitespace differences between cargo versions
+                                             # Probably can be removed when we update our CI stable
 writefile .cargo/config "# This file is part of ICU4X. For terms of use, please see the file\n"
 appendfile .cargo/config "# called LICENSE at the top level of the ICU4X source tree\n"
 appendfile .cargo/config "# (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).\n\n"
 appendfile .cargo/config "# AUTO-GENERATED in ffi.toml\n"
-appendfile .cargo/config ${output.stdout}
+appendfile .cargo/config ${output_trimmed}
 '''
 
 [tasks.gn-gen]

tools/scripts/wasm.toml

@@ -20,7 +20,7 @@ exit_on_error true
 
 # Re-run the build command only to generate the JSON output (--message-format=json)
 set_env RUSTFLAGS "-C panic=abort -C opt-level=s"
-output = exec cargo +nightly-2022-04-05 wasm-build-release --message-format=json --examples --workspace --features serde --exclude icu_datagen
+output = exec cargo +${ICU4X_NIGHTLY_TOOLCHAIN} wasm-build-release --message-format=json --examples --workspace --features serde --exclude icu_datagen
 if ${output.code}
     echo ${output.stderr}
     trigger_error "Build failed! See output above."

utils/tinystr/src/ascii.rs

@@ -142,6 +142,7 @@ impl<const N: usize> TinyAsciiStr<N> {
         /// ```compile_fail
         /// const unsafe fn canary() { core::slice::from_raw_parts(0 as *const u8, 0); }
         /// ```
+        #[cfg(not(ICU4X_BUILDING_WITH_FORCED_NIGHTLY))]
         const _: () = ();
         // Safe because `self.bytes.as_slice()` pointer-casts to `&[u8]`,
         // and changing the length of that slice to self.len() < N is safe.

utils/zerovec/src/ule/plain.rs

@@ -101,6 +101,7 @@ macro_rules! impl_const_constructors {
                 /// ```compile_fail
                 /// const unsafe fn canary() { core::slice::from_raw_parts(0 as *const u8, 0); }
                 /// ```
+                #[cfg(not(ICU4X_BUILDING_WITH_FORCED_NIGHTLY))]
                 const _: () = ();
                 let len = bytes.len();
                 #[allow(clippy::modulo_one)]

utils/zerovec/src/zerovec/mod.rs

@@ -356,6 +356,7 @@ where
         /// ```compile_fail
         /// const unsafe fn canary() { core::slice::from_raw_parts(0 as *const u8, 0); }
         /// ```
+        #[cfg(not(ICU4X_BUILDING_WITH_FORCED_NIGHTLY))]
         const _: () = ();
         Self::new_borrowed(core::mem::transmute((
             bytes.as_ptr(),

utils/zerovec/src/zerovec/slice.rs

@@ -69,6 +69,7 @@ where
         /// ```compile_fail
         /// const unsafe fn canary() { core::slice::from_raw_parts(0 as *const u8, 0); }
         /// ```
+        #[cfg(not(ICU4X_BUILDING_WITH_FORCED_NIGHTLY))]
         const _: () = ();
         core::mem::transmute((bytes.as_ptr(), bytes.len() / core::mem::size_of::<T::ULE>()))
     }