0.11.2: Handful of Improvements

- On Linux, a check will be performed on transparent_hupages and issue a warning if it is set to always instead of madvise
- The Ion shell is now a preferred shell, with Dash coming as second to Ion
- The ArgumentSplitter was improved -- copied from my Ion shell implementation
- Invalid arguments will issue warnings instead of crashing the program
- If an empty line is passed to standard input, it will be ignored
- All Rust dependencies have been updated to their latest versions

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "parallel"
-version = "0.11.1"
+version = "0.11.2"
 authors = ["Michael Aaron Murphy <mmstickman@gmail.com>"]
 license = "MIT"
 description = "Command-line CPU load balancer for executing jobs in parallel"
@@ -10,40 +10,12 @@ categories = ["command-line-utilities"]
 readme = "README.md"
 
 [dependencies]
-itoa = "0.1"
+itoa = "0.3"
 numtoa = "0.0"
-num_cpus = "1.2"
-permutate = "0.2"
+num_cpus = "1.5"
+permutate = "0.3"
 arrayvec = "0.3"
 time = "0.1"
-smallvec = "0.3"
+smallvec = "0.4"
 sys-info = "0.4"
 wait-timeout = "0.1"
-
-[profile.release]
-opt-level = 3
-lto = true
-
-[package.metadata.deb]
-maintainer = "Michael Aaron Murphy <mmstickman@gmail.com>"
-copyright = "2016-2017, Michael Aaron Murphy <mmstickman@gmail.com>"
-license_file = ["LICENSE", "4"]
-extended_description = """\
-A CPU load balancer for the command-line, written in Rust and inspired by \
-GNU Parallel. The purpose of the application is to parallelize otherwise \
-serial tasks by evenly distributing tasks to all CPU cores in a system.
-
-The program is supplied with a command template and a list of inputs, \
-where those inputs will be evenly passed to each core in the system. \
-Inputs are processed serially in parallel, as in each job will be assigned \
-the next task in the queue as soon as a job completes. In addition, the \
-standard output/error of each task will be buffered so that they are \
-printed in the order that inputs are received, as if the commands were \
-executed serially in a traditional for loop."""
-depends = "$auto"
-section = "utils"
-priority = "optional"
-assets = [
- ["target/release/parallel", "usr/bin/", "755"],
- ["README.md", "usr/share/doc/parallel/README", "644"],
-]

README.md

@@ -8,28 +8,6 @@ This is an attempt at recreating the functionality of [GNU Parallel](https://www
 
 ## Note
 
-A nightly version of the Rust compiler is required to build this application.
-This is required to disable `jemalloc` because it incurs a performance overhead on Linux systems.
-In addition to using a nightly compiler, it is recommended to compile with a musl target.
-
-```
-rustup install nightly
-rustup target add x86_64-unknown-linux-musl
-rustup run nightly cargo build --release --target x86_64-unknown-linux-musl
-```
-
-Additionally, `transparent_hugepages` causes serious performance issues with the implementation
-of parallel, so it is recommended to ensure that your Linux distribution has the parameter set
-to `madvise`:
-
-```
-sudo sh -c "echo madvise > /sys/kernel/mm/transparent_hugepage/enabled"
-```
-
-It's a good idea to install the `dash` shell as this implementation of Parallel will try to use it by default.
-Dash is basically a superior implementation of `sh` that's many times faster and more secure.
-If `dash` cannot be found, it will default to `sh`. Although on Windows it will default to `cmd`.
-
 See the [to-do list](https://github.com/mmstick/parallel/blob/master/TODO.md) for features and improvements that have yet to be done. If you want to contribute, pull requests are welcome. If you have an idea for improvement which isn't listed in the to-do list, feel free to [email me](mailto:mmstickman@gmail.com) and I will consider implementing that idea.
 
 ## Benchmark Comparison to GNU Parallel

TODO.md

@@ -7,6 +7,11 @@ The list is actively updated with each successful pull request.
 - Eliminate the need to run commands within a shell
 - Implement `progress`
 - Fix `-n` issue when using `{1..}` tokens
+- Compress arguments written to the disk with Brotli
+- Re-implement in-memory argument passing versus disk-exclusive argument iteration
+- Rewrite the arguments module
+- Rewrite the error handling logic
+- Utilize the crossbeam crate so that strings don't need to be leaked
 
 ## May or may not implement
 - Kill the youngest job and add it to the back of the queue if available memory is 50% less than `memfree`'s value.

src/arguments/mod.rs

@@ -35,6 +35,7 @@ pub const SHELL_QUOTE: u16 = 128;
 pub const ETA: u16 = 256;
 pub const JOBLOG: u16 = 512;
 pub const JOBLOG_8601: u16 = 1024;
+pub const ION_EXISTS: u16 = 2048;
 
 /// `Args` is a collection of critical options and arguments that were collected at
 /// startup of the application.
@@ -118,13 +119,13 @@ impl Args {
  println!("{}", man::MAN_PAGE);
  exit(0);
  },
- b'k' => (),
  b'p' => self.flags |= PIPE_IS_ENABLED,
  b'q' => quote_enabled = true,
  b's' => self.flags |= QUIET_MODE,
  b'v' => self.flags |= VERBOSE_MODE,
  _ => {
- return Err(ParseErr::InvalidArgument(index-1))
+ let stderr = io::stderr();
+ let _ = writeln!(stderr.lock(), "parallel: unsupported argument: '-{}'", character as char);
  }
  }
  }
@@ -143,8 +144,6 @@ impl Args {
  println!("{}", man::MAN_PAGE);
  exit(0);
  },
- "keep-order" => (),
- "group" => (),
  "joblog" => {
  let file = arguments.get(index).ok_or(ParseErr::JoblogNoValue)?;
  self.joblog = Some(file.to_owned());
@@ -157,7 +156,6 @@ impl Args {
  if val != 0 { self.ncores = val; }
  index += 1;
  },
- "line-buffer" | "lb" => (),
  "num-cpu-cores" => {
  println!("{}", num_cpus::get());
  exit(0);
@@ -172,7 +170,6 @@ impl Args {
  self.memory = parse_memory(val).map_err(|_| ParseErr::MemInvalid(index))?;
  index += 1;
  },
- "no-notice" => (),
  "pipe" => self.flags |= PIPE_IS_ENABLED,
  "quiet" | "silent" => self.flags |= QUIET_MODE,
  "quote" => quote_enabled = true,
@@ -183,13 +180,11 @@ impl Args {
  self.timeout = Duration::from_millis((seconds * 1000f64) as u64);
  index += 1;
  },
- "ungroup" => (),
  "verbose" => self.flags |= VERBOSE_MODE,
  "version" => {
  println!("MIT/Rust Parallel {}", env!("CARGO_PKG_VERSION"));
  exit(0);
  },
- "will-cite" => (),
  "tmpdir" | "tempdir" => {
  *base_path = PathBuf::from(arguments.get(index).ok_or(ParseErr::WorkDirNoValue)?);
  index += 1;
@@ -207,7 +202,10 @@ impl Args {
  comm.push_str(&argument[10..]);
  break
  },
- _ => return Err(ParseErr::InvalidArgument(index-1)),
+ _ => {
+ let stderr = io::stderr();
+ let _ = writeln!(stderr.lock(), "parallel: unsupported argument: '{}'", argument);
+ }
  }
  }
  } else {
@@ -389,17 +387,19 @@ fn write_stdin_to_disk(max_args: usize, mut unprocessed_path: PathBuf, inputs_ar
 
  let stdin = io::stdin();
  if max_args < 2 {
- for line in stdin.lock().lines() {
+ for line in BufReader::new(stdin.lock()).lines() {
  if let Ok(line) = parse_line(line) {
+ if line.is_empty() { continue }
  disk_buffer.write(line.as_bytes()).and_then(|_| disk_buffer.write(b"\n"))
  .map_err(|why| FileErr::Write(unprocessed_path.clone(), why))?;
  number_of_arguments += 1;
  }
  }
  } else {
  let mut max_args_index = max_args;
- for line in stdin.lock().lines() {
+ for line in BufReader::new(stdin.lock()).lines() {
  if let Ok(line) = parse_line(line) {
+ if line.is_empty() { continue }
  if max_args_index == max_args {
  max_args_index -= 1;
  number_of_arguments += 1;
@@ -466,7 +466,7 @@ fn write_inputs_to_disk(lists: Vec<Vec<String>>, current_inputs: Vec<String>, ma
 
  if max_args < 2 {
  disk_buffer.write(b"\n").map_err(|why| FileErr::Write(unprocessed_path.clone(), why))?;
- while let Ok(true) = permutator.next_with_buffer(&mut permutation_buffer) {
+ while permutator.next_with_buffer(&mut permutation_buffer) {
  let mut iter = permutation_buffer.iter();
  disk_buffer.write(iter.next().unwrap().as_bytes())
  .map_err(|why| FileErr::Write(unprocessed_path.clone(), why))?;
@@ -480,7 +480,7 @@ fn write_inputs_to_disk(lists: Vec<Vec<String>>, current_inputs: Vec<String>, ma
  }
  } else {
  let mut max_args_index = max_args - 1;
- while let Ok(true) = permutator.next_with_buffer(&mut permutation_buffer) {
+ while permutator.next_with_buffer(&mut permutation_buffer) {
  let mut iter = permutation_buffer.iter();
  if max_args_index == max_args {
  max_args_index -= 1;