Feature request: save data directories for each run (run index as format specifier in the command)
stagnation opened this issue · 1 comments
stagnation commented
Hi
I have a benchmark use-case where I want to save extra data files for each run, for a more detailed analysis later.
I want to benchmark the number of garbage collection events in a program profile.
My run-tool can easily take the parameter I'm scanning as one argument and the directory as another.
But it is a little convoluted to do today.
Attempts
$ hyperfine \
--runs 5 \
--parameter-list mem 150,160 \
-- './benchmark-different-memory {mem}m {???}'
One attempt is to forgo --runs and use a --parameter-scan for the runs
$ hyperfine \
--parameter-scan run 0 5 \
--parameter-list mem 150,160 \
-- './benchmark-different-memory {mem}m {run}'
error: the argument '--parameter-scan <VAR> <MIN> <MAX>' cannot be used with '--parameter-list <VAR> <VALUES>'
Working solution as a command executor. Though the time benchmark breaks as we set runs to 1.
Else it does both the default number of runs, and the parameter-list for runs.
Using two parameter-lists instead works:
$ hyperfine \
--parameter-list run $(seq 5 | paste -sd ',') \
--parameter-list mem 150,160 \
--runs 1
-- './benchmark-different-memory {mem}m {run}'
This is okay, as I do not care about warmups today, but ideally a warmup should be used.
Then a two-dimensional parameter list could be --parameter-list run WARMUP{1,2} $(seq 5) or something.
One can also hack the code to save the index
For the reference: I also hacked in the index into the run function,
and the run can be taken from the environment.
If someone wants a stricter API.
$ target/debug/hyperfine --runs 3 --parameter-scan x 1 2 env --show-output | grep HYPERFINE_RUN_INDEX
HYPERFINE_RUN_INDEX=0
HYPERFINE_RUN_INDEX=1
HYPERFINE_RUN_INDEX=2
Warning: Command took less than 5 ms to complete. Note that the results might be inaccurate because hyp
erfine can not calibrate the shell startup time much more precise than this limit. You can try to use the
`-N`/`--shell=none` option to disable the shell completely.
HYPERFINE_RUN_INDEX=0
HYPERFINE_RUN_INDEX=1
HYPERFINE_RUN_INDEX=2
And my use case then looks like this, the environment variable is expanded in the run-shell.
~/gits/hyperfine/target/debug/hyperfine \
--runs 5 \
--parameter-list mem 200,250 \
-- './benchmark-different-memory {mem}m $(echo $HYPERFINE_RUN_INDEX)'
stagnation commented
Hacky patch to expose the run index as an environment variable
diff --git a/src/benchmark/executor.rs b/src/benchmark/executor.rs
index 31db47b..57f09b6 100644
--- a/src/benchmark/executor.rs
+++ b/src/benchmark/executor.rs
@@ -20,6 +20,7 @@ pub trait Executor {
&self,
command: &Command<'_>,
command_failure_action: Option<CmdFailureAction>,
+ index: u64,
) -> Result<(TimingResult, ExitStatus)>;
/// Perform a calibration of this executor. For example,
@@ -41,6 +42,7 @@ fn run_command_and_measure_common(
command_input_policy: &CommandInputPolicy,
command_output_policy: &CommandOutputPolicy,
command_name: &str,
+ index: u64,
) -> Result<TimerResult> {
let stdin = command_input_policy.get_stdin()?;
let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
@@ -50,6 +52,9 @@ fn run_command_and_measure_common(
"HYPERFINE_RANDOMIZED_ENVIRONMENT_OFFSET",
randomized_environment_offset::value(),
);
+ command.env(
+ "HYPERFINE_RUN_INDEX", format!("{}", index),
+ );
let result = execute_and_measure(command)
.with_context(|| format!("Failed to run command '{}'", command_name))?;
@@ -83,6 +88,7 @@ impl<'a> Executor for RawExecutor<'a> {
&self,
command: &Command<'_>,
command_failure_action: Option<CmdFailureAction>,
+ index: u64,
) -> Result<(TimingResult, ExitStatus)> {
let result = run_command_and_measure_common(
command.get_command()?,
@@ -90,6 +96,7 @@ impl<'a> Executor for RawExecutor<'a> {
&self.options.command_input_policy,
&self.options.command_output_policy,
&command.get_command_line(),
+ index,
)?;
Ok((
@@ -132,6 +139,7 @@ impl<'a> Executor for ShellExecutor<'a> {
&self,
command: &Command<'_>,
command_failure_action: Option<CmdFailureAction>,
+ index: u64,
) -> Result<(TimingResult, ExitStatus)> {
let mut command_builder = self.shell.command();
command_builder
@@ -150,6 +158,7 @@ impl<'a> Executor for ShellExecutor<'a> {
&self.options.command_input_policy,
&self.options.command_output_policy,
&command.get_command_line(),
+ index,
)?;
// Subtract shell spawning time
@@ -186,9 +195,9 @@ impl<'a> Executor for ShellExecutor<'a> {
let mut times_user: Vec<Second> = vec![];
let mut times_system: Vec<Second> = vec![];
- for _ in 0..COUNT {
+ for i in 0..COUNT {
// Just run the shell without any command
- let res = self.run_command_and_measure(&Command::new(None, ""), None);
+ let res = self.run_command_and_measure(&Command::new(None, ""), None, i);
match res {
Err(_) => {
@@ -258,6 +267,7 @@ impl Executor for MockExecutor {
&self,
command: &Command<'_>,
_command_failure_action: Option<CmdFailureAction>,
+ index: u64,
) -> Result<(TimingResult, ExitStatus)> {
#[cfg(unix)]
let status = {
diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs
index 0699f7d..17c3a3a 100644
--- a/src/benchmark/mod.rs
+++ b/src/benchmark/mod.rs
@@ -57,7 +57,7 @@ impl<'a> Benchmark<'a> {
error_output: &'static str,
) -> Result<TimingResult> {
self.executor
- .run_command_and_measure(command, Some(CmdFailureAction::RaiseError))
+ .run_command_and_measure(command, Some(CmdFailureAction::RaiseError), 0)
.map(|r| r.0)
.map_err(|_| anyhow!(error_output))
}
@@ -160,9 +160,9 @@ impl<'a> Benchmark<'a> {
None
};
- for _ in 0..self.options.warmup_count {
+ for i in 0..self.options.warmup_count {
let _ = run_preparation_command()?;
- let _ = self.executor.run_command_and_measure(self.command, None)?;
+ let _ = self.executor.run_command_and_measure(self.command, None, i)?;
if let Some(bar) = progress_bar.as_ref() {
bar.inc(1)
}
@@ -188,7 +188,7 @@ impl<'a> Benchmark<'a> {
preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
// Initial timing run
- let (res, status) = self.executor.run_command_and_measure(self.command, None)?;
+ let (res, status) = self.executor.run_command_and_measure(self.command, None, 0)?;
let success = status.success();
// Determine number of benchmark runs
@@ -226,7 +226,7 @@ impl<'a> Benchmark<'a> {
}
// Gather statistics (perform the actual benchmark)
- for _ in 0..count_remaining {
+ for i in 0..count_remaining {
run_preparation_command()?;
let msg = {
@@ -238,7 +238,7 @@ impl<'a> Benchmark<'a> {
bar.set_message(msg.to_owned())
}
- let (res, status) = self.executor.run_command_and_measure(self.command, None)?;
+ let (res, status) = self.executor.run_command_and_measure(self.command, None, i + 1)?;
let success = status.success();
times_real.push(res.time_real);