Auto merge of #2109 - Thomasdezeeuw:issue_2104-regression_tests, r=Jo…

…hnTitor

Add regression test infrastructure

Please the commit messages for details.

I still need to add lists for the following targets, but I got the major ones I think.

TODO:
* aarch64-unknown-hermit
* x86_64-unknown-hermit
* x86_64-pc-solaris
* x86_64-sun-solaris
* sparcv9-sun-solaris
* x86_64-fortanix-unknown-sgx
* x86_64-unknown-illumos
* asmjs-unknown-emscripten
* wasm32-unknown-emscripten
* wasm32-unknown-unknown
* wasm32-wasi
* Check symbols added after commit ed45c26.

TODO: add a bit to the contributing guide about adding to these lists.

Closes #2104.

CONTRIBUTING.md

@@ -28,14 +28,24 @@ at, fear not! This crate has CI support which tests any binding against all
 platforms supported, so you'll see failures if an API is added at the wrong
 level or has different signatures across platforms.
 
+New symbol(s) (i.e. functions, constants etc.) should also be added to the
+symbols list(s) found in the `libc-test/semver` directory. These lists keep
+track of what symbols are public in the libc crate and ensures they remain
+available between changes to the crate. If the new symbol(s) are available on
+all supported Unixes it should be added to `unix.txt` list<sup>1</sup>,
+otherwise they should be added to the OS specific list(s).
+
 With that in mind, the steps for adding a new API are:
 
 1. Determine where in the module hierarchy your API should be added.
-2. Add the API.
+2. Add the API, including adding new symbol(s) to the semver lists.
 3. Send a PR to this repo.
 4. Wait for CI to pass, fixing errors.
 5. Wait for a merge!
 
+<sup>1</sup>: Note that this list has nothing to do with any Unix or Posix
+standard, it's just a list shared between all OSs that declare `#[cfg(unix)]`.
+
 ## Test before you commit
 
 We have two automated tests running on [GitHub Actions](https://github.com/rust-lang/libc/actions):

libc-test/Cargo.toml

@@ -65,3 +65,8 @@ harness = true
 name = "errqueue"
 path = "test/errqueue.rs"
 harness = true
+
+[[test]]
+name = "semver"
+path = "test/semver.rs"
+harness = false

libc-test/build.rs

@@ -3,7 +3,10 @@
 extern crate cc;
 extern crate ctest2 as ctest;
 
-use std::env;
+use std::fs::File;
+use std::io::{BufRead, BufReader, BufWriter, Write};
+use std::path::{Path, PathBuf};
+use std::{env, io};
 
 fn do_cc() {
     let target = env::var("TARGET").unwrap();
@@ -63,9 +66,85 @@ fn ctest_cfg() -> ctest::TestGenerator {
     cfg
 }
 
+fn do_semver() {
+    let mut out = PathBuf::from(env::var("OUT_DIR").unwrap());
+    out.push("semver.rs");
+    let mut output = BufWriter::new(File::create(&out).unwrap());
+
+    let family = env::var("CARGO_CFG_TARGET_FAMILY").unwrap();
+    let vendor = env::var("CARGO_CFG_TARGET_VENDOR").unwrap();
+    let os = env::var("CARGO_CFG_TARGET_OS").unwrap();
+    let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
+    let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap();
+
+    // `libc-test/semver` dir.
+    let mut semver_root =
+        PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+    semver_root.push("semver");
+
+    // NOTE: Windows has the same `family` as `os`, no point in including it
+    // twice.
+    // NOTE: Android doesn't include the unix file (or the Linux file) because
+    // there are some many definitions missing it's actually easier just to
+    // maintain a file for Android.
+    if family != os && os != "android" {
+        process_semver_file(&mut output, &mut semver_root, &family);
+    }
+    process_semver_file(&mut output, &mut semver_root, &vendor);
+    process_semver_file(&mut output, &mut semver_root, &os);
+    let os_arch = format!("{}-{}", os, arch);
+    process_semver_file(&mut output, &mut semver_root, &os_arch);
+    if target_env != "" {
+        let os_env = format!("{}-{}", os, target_env);
+        process_semver_file(&mut output, &mut semver_root, &os_env);
+
+        let os_env_arch = format!("{}-{}-{}", os, target_env, arch);
+        process_semver_file(&mut output, &mut semver_root, &os_env_arch);
+    }
+}
+
+fn process_semver_file<W: Write, P: AsRef<Path>>(
+    output: &mut W,
+    path: &mut PathBuf,
+    file: P,
+) {
+    // NOTE: `path` is reused between calls, so always remove the file again.
+    path.push(file);
+    path.set_extension("txt");
+
+    println!("cargo:rerun-if-changed={}", path.display());
+    let input_file = match File::open(&*path) {
+        Ok(file) => file,
+        Err(ref err) if err.kind() == io::ErrorKind::NotFound => {
+            path.pop();
+            return;
+        }
+        Err(err) => panic!("unexpected error opening file: {}", err),
+    };
+    let input = BufReader::new(input_file);
+
+    write!(output, "// Source: {}.\n", path.display()).unwrap();
+    output.write(b"use libc::{\n").unwrap();
+    for line in input.lines() {
+        let line = line.unwrap().into_bytes();
+        match line.first() {
+            // Ignore comments and empty lines.
+            Some(b'#') | None => continue,
+            _ => {
+                output.write(b"    ").unwrap();
+                output.write(&line).unwrap();
+                output.write(b",\n").unwrap();
+            }
+        }
+    }
+    output.write(b"};\n\n").unwrap();
+    path.pop();
+}
+
 fn main() {
     do_cc();
     do_ctest();
+    do_semver();
 }
 
 macro_rules! headers {

libc-test/semver/README.md

@@ -0,0 +1,17 @@
+# Supported API by libc
+
+These files are read by [`build.rs`](../build.rs) and turned into tests to
+ensure that APIs aren't removed between libc releases.
+
+## File order
+
+Files are including in the following order:
+ * Family, e.g. `unix.txt`. NOTE: Windows is skipped here and includes as OS
+   name below.
+ * Vendor, e.g. `apple.txt`. This allows us to have a single file with system
+   calls shared between multiple OSs, e.g. `ios.txt`, `macos.txt` share the same
+   kernel.
+ * OS, e.g `linux.txt`, `macos.txt`, `windows.txt`.
+ * Architecture specific system calls, e.g. `linux-x86_64.txt` or
+   `linux-aarch64.txt`.
+ * Target environment, e.g. `windows-mscv.txt` or `windows-gnu.txt`.

libc-test/semver/TODO-linux.txt

@@ -0,0 +1,117 @@
+# The following symbols are not not available in some combinations of
+# musl/gnu/android and/or architecture.
+BOTHER
+HWCAP_AES
+HWCAP_ASIMD
+HWCAP_ASIMDDP
+HWCAP_ASIMDFHM
+HWCAP_ASIMDHP
+HWCAP_ASIMDRDM
+HWCAP_ATOMICS
+HWCAP_CPUID
+HWCAP_CRC32
+HWCAP_DCPOP
+HWCAP_DIT
+HWCAP_EVTSTRM
+HWCAP_FCMA
+HWCAP_FLAGM
+HWCAP_FP
+HWCAP_FPHP
+HWCAP_ILRCPC
+HWCAP_JSCVT
+HWCAP_LRCPC
+HWCAP_PACA
+HWCAP_PACG
+HWCAP_PMULL
+HWCAP_SB
+HWCAP_SHA1
+HWCAP_SHA2
+HWCAP_SHA3
+HWCAP_SHA512
+HWCAP_SM3
+HWCAP_SM4
+HWCAP_SSBS
+HWCAP_SVE
+HWCAP_USCAT
+KEYCTL_CAPABILITIES
+KEYCTL_CAPS0_BIG_KEY
+KEYCTL_CAPS0_CAPABILITIES
+KEYCTL_CAPS0_DIFFIE_HELLMAN
+KEYCTL_CAPS0_INVALIDATE
+KEYCTL_CAPS0_MOVE
+KEYCTL_CAPS0_PERSISTENT_KEYRINGS
+KEYCTL_CAPS0_PUBLIC_KEY
+KEYCTL_CAPS0_RESTRICT_KEYRING
+KEYCTL_CAPS1_NS_KEYRING_NAME
+KEYCTL_CAPS1_NS_KEY_TAG
+KEYCTL_MOVE
+NFT_MSG_DELOBJ
+NFT_MSG_GETOBJ
+NFT_MSG_GETOBJ_RESET
+NFT_MSG_NEWOBJ
+PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP
+PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP
+PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
+SCM_TIMESTAMPING_OPT_STATS
+SCM_TIMESTAMPING_PKTINFO
+SCM_TIMESTAMPNS
+SCM_TXTIME
+SCM_WIFI_STATUS
+SO_ATTACH_BPF
+SO_ATTACH_FILTER
+SO_ATTACH_REUSEPORT_CBPF
+SO_ATTACH_REUSEPORT_EBPF
+SO_BINDTOIFINDEX
+SO_BPF_EXTENSIONS
+SO_BSDCOMPAT
+SO_CNX_ADVICE
+SO_COOKIE
+SO_DETACH_BPF
+SO_DETACH_FILTER
+SO_DETACH_REUSEPORT_BPF
+SO_GET_FILTER
+SO_INCOMING_CPU
+SO_INCOMING_NAPI_ID
+SO_LOCK_FILTER
+SO_MAX_PACING_RATE
+SO_MEMINFO
+SO_NOFCS
+SO_NO_CHECK
+SO_PEERGROUPS
+SO_PEERNAME
+SO_RCVTIMEO_NEW
+SO_SECURITY_AUTHENTICATION
+SO_SECURITY_ENCRYPTION_NETWORK
+SO_SECURITY_ENCRYPTION_TRANSPORT
+SO_SELECT_ERR_QUEUE
+SO_SNDTIMEO_NEW
+SO_STYLE
+SO_TIMESTAMPING_NEW
+SO_TIMESTAMPNS
+SO_TIMESTAMPNS_NEW
+SO_TIMESTAMP_NEW
+SO_TXTIME
+SO_WIFI_STATUS
+SO_ZEROCOPY
+SYS__llseek
+SYS__newselect
+SYS__sysctl
+SYS_create_module
+SYS_fadvise64
+SYS_fstatat64
+SYS_get_kernel_syms
+SYS_get_thread_area
+SYS_getrlimit
+SYS_migrate_pages
+SYS_mmap
+SYS_nfsservctl
+SYS_pread64
+SYS_pwrite64
+SYS_query_module
+SYS_set_thread_area
+SYS_uselib
+fsblkcnt64_t
+fsfilcnt64_t
+getrandom
+sysctl
+termios2

libc-test/semver/TODO-unix.txt

@@ -0,0 +1,5 @@
+# These symbols are missing for the targets:
+# * asmjs-unknown-emscripten
+getpwuid_r
+pthread_atfork
+pthread_sigmask

libc-test/semver/android-aarch64.txt

@@ -0,0 +1,11 @@
+HWCAP2_DCPODP
+HWCAP2_FLAGM2
+HWCAP2_FRINT
+HWCAP2_SVE2
+HWCAP2_SVEAES
+HWCAP2_SVEBITPERM
+HWCAP2_SVEPMULL
+HWCAP2_SVESHA3
+HWCAP2_SVESM4
+SYS_arch_specific_syscall
+SYS_syscalls