Issues running verneuil operations within a async runtime
Church- opened this issue · 4 comments
Church- commented
As noted in the title, when invoking verneuil operations other then the main verneuil async replication loop inside a async runtime your program will error at runtime with:
thread 'main' panicked at 'Cannot start a runtime from within a runtime. This happens because a function (like `block_on`) attempted to block the current thread while the thread is being used to drive asynchronous tasks.', /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/tokio-1.17.0/src/runtime/enter.rs:39:9
stack backtrace:
0: 0x5608fe2ebff0 - std::backtrace_rs::backtrace::libunwind::trace::h706b838f5bbd876b
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/../../backtrace/src/backtrace/libunwind.rs:90:5
1: 0x5608fe2ebff0 - std::backtrace_rs::backtrace::trace_unsynchronized::hc15f29ae7822b7b8
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x5608fe2ebff0 - std::sys_common::backtrace::_print_fmt::h7c580c971f91926c
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/sys_common/backtrace.rs:67:5
3: 0x5608fe2ebff0 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h2a772198c4032452
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/sys_common/backtrace.rs:46:22
4: 0x5608fe310b7c - core::fmt::write::h9a6d9c74526a6c1b
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/core/src/fmt/mod.rs:1115:17
5: 0x5608fe2e4c65 - std::io::Write::write_fmt::h00f38d9eb2c32e02
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/io/mod.rs:1663:15
6: 0x5608fe2ee53b - std::sys_common::backtrace::_print::h5b3c171e6f864ae4
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/sys_common/backtrace.rs:49:5
7: 0x5608fe2ee53b - std::sys_common::backtrace::print::h93b9e9ed2a98e611
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/sys_common/backtrace.rs:36:9
8: 0x5608fe2ee53b - std::panicking::default_hook::{{closure}}::hd8da92bb68d520c5
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/panicking.rs:208:50
9: 0x5608fe2ee011 - std::panicking::default_hook::hdbc8b2951c5afbab
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/panicking.rs:225:9
10: 0x5608fe2eec04 - std::panicking::rust_panic_with_hook::h8a4c841655926f4e
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/panicking.rs:622:17
11: 0x5608fe2d02e7 - std::panicking::begin_panic::{{closure}}::hb0d33eb2055508d2
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/panicking.rs:542:9
12: 0x5608fe2d02c0 - std::sys_common::backtrace::__rust_end_short_backtrace::hd3b4c97b5a7cad4c
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/sys_common/backtrace.rs:141:18
13: 0x5608fde1800e - std::panicking::begin_panic::hba99d266457f13fb
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/panicking.rs:541:12
14: 0x5608fe2b7b69 - tokio::runtime::enter::enter::h9f1b3f7a8528ef6d
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/tokio-1.17.0/src/runtime/enter.rs:39:9
15: 0x5608fde81ab9 - tokio::runtime::handle::Handle::block_on::h83d62ebbad7b513c
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/tokio-1.17.0/src/runtime/handle.rs:352:34
16: 0x5608fdee64b1 - verneuil::tracker::Tracker::flush_spooled_data::hb8ee1dce4a49165a
at /home/noah/code/reddit_exporter/rust/verneuil/src/tracker/mod.rs:384:13
17: 0x5608fdecbea7 - verneuil__file_flush_replication_data
at /home/noah/code/reddit_exporter/rust/verneuil/src/vfs_ops.rs:88:11
18: 0x5608fdf085e8 - linux_file_control
at /home/noah/code/reddit_exporter/rust/verneuil/c/vfs.c:2150:36
19: 0x5608fdf085e8 - linux_file_control
at /home/noah/code/reddit_exporter/rust/verneuil/c/vfs.c:2107:1
20: 0x7f458b082da0 - sqlite3_file_control
21: 0x7f458b1161a1 - <unknown>
22: 0x7f458b11a9b3 - <unknown>
23: 0x7f458b11f809 - <unknown>
24: 0x7f458b11fbef - <unknown>
25: 0x7f458b11fcea - sqlite3_prepare_v2
26: 0x5608fe126be0 - rusqlite::inner_connection::InnerConnection::prepare::h8dbdc4b6e92dad97
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/rusqlite-0.26.3/src/inner_connection.rs:247:17
27: 0x5608fde28950 - rusqlite::Connection::prepare::hfcb7f155e657f120
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/rusqlite-0.26.3/src/lib.rs:711:9
28: 0x5608fde28950 - rusqlite::pragma::<impl rusqlite::Connection>::pragma::h7c49312f1339a295
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/rusqlite-0.26.3/src/pragma.rs:234:24
29: 0x5608fde2ac77 - reddit_exporter::run::{{closure}}::h0b08dcc4bebf950f
at /home/noah/code/reddit_exporter/rust/reddit_exporter/src/lib.rs:84:9
30: 0x5608fde2ac77 - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::h23826ff10c1568e1
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/core/src/future/mod.rs:80:19
31: 0x5608fde2ac77 - reddit_exporter::main::{{closure}}::h4a70bb1ad8b599de
at /home/noah/code/reddit_exporter/rust/reddit_exporter/src/main.rs:5:5
32: 0x5608fde2ac77 - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::h0c01af9e689a1fba
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/core/src/future/mod.rs:80:19
33: 0x5608fde271df - tokio::park::thread::CachedParkThread::block_on::{{closure}}::hbeb1622ff3aac55c
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/tokio-1.17.0/src/park/thread.rs:263:54
34: 0x5608fde271df - tokio::coop::with_budget::{{closure}}::h5bfc9a4967bb24f0
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/tokio-1.17.0/src/coop.rs:102:9
35: 0x5608fde271df - std::thread::local::LocalKey<T>::try_with::h0bc854f57be45633
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/thread/local.rs:399:16
36: 0x5608fde271df - std::thread::local::LocalKey<T>::with::hbcf97c3248396ffc
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/thread/local.rs:375:9
37: 0x5608fde29c5e - tokio::coop::with_budget::hae6ca180114e01a9
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/tokio-1.17.0/src/coop.rs:95:5
38: 0x5608fde29c5e - tokio::coop::budget::h2509df568e464e85
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/tokio-1.17.0/src/coop.rs:72:5
39: 0x5608fde29c5e - tokio::park::thread::CachedParkThread::block_on::h12239001b61cbb4f
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/tokio-1.17.0/src/park/thread.rs:263:31
40: 0x5608fde30fdf - tokio::runtime::enter::Enter::block_on::h6fa1fb9e9d8e58e2
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/tokio-1.17.0/src/runtime/enter.rs:151:13
41: 0x5608fde30fdf - tokio::runtime::thread_pool::ThreadPool::block_on::ha300f1a646b9eff9
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/tokio-1.17.0/src/runtime/thread_pool/mod.rs:73:9
42: 0x5608fde27725 - tokio::runtime::Runtime::block_on::h68b28e80d59b6be7
at /home/noah/.cargo/registry/src/localhost-20d513fd1fb2345e/tokio-1.17.0/src/runtime/mod.rs:477:43
43: 0x5608fde27725 - reddit_exporter::main::h1bfb50ff936ba5b0
at /home/noah/code/reddit_exporter/rust/reddit_exporter/src/main.rs:6:5
44: 0x5608fde28400 - core::ops::function::FnOnce::call_once::hbfe679e2efad3469
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/core/src/ops/function.rs:227:5
45: 0x5608fde28400 - std::sys_common::backtrace::__rust_begin_short_backtrace::hd96ca07378af8c4f
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/sys_common/backtrace.rs:125:18
46: 0x5608fde2f7fd - std::rt::lang_start::{{closure}}::h8b17894e59eb2d0d
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/rt.rs:63:18
47: 0x5608fe2ef20a - core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once::hcbe5785d54a49ce2
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/core/src/ops/function.rs:259:13
48: 0x5608fe2ef20a - std::panicking::try::do_call::h1c15e7ee8563c29d
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/panicking.rs:401:40
49: 0x5608fe2ef20a - std::panicking::try::hb8b07cef009cb4fc
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/panicking.rs:365:19
50: 0x5608fe2ef20a - std::panic::catch_unwind::h65a8efbe596bcbb1
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/panic.rs:434:14
51: 0x5608fe2ef20a - std::rt::lang_start_internal::{{closure}}::h66ded7ab0f18e9fb
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/rt.rs:45:48
52: 0x5608fe2ef20a - std::panicking::try::do_call::h29cafa65efb3f3f3
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/panicking.rs:401:40
53: 0x5608fe2ef20a - std::panicking::try::h412fc5378b4dbfc4
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/panicking.rs:365:19
54: 0x5608fe2ef20a - std::panic::catch_unwind::h555def6aac5bf192
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/panic.rs:434:14
55: 0x5608fe2ef20a - std::rt::lang_start_internal::hfff48202d7fee6e3
at /rustc/240ff4c4a0d0936c9eeb783fa9ff5c0507a6ffb4/library/std/src/rt.rs:45:20
56: 0x5608fde2790a - main
57: 0x7f458a99b0b3 - __libc_start_main
58: 0x5608fde1b50e - _start
59: 0x0 - <unknown>
Aborted (core dumped)
We tried an attempted patch earlier to fix this to no effect:
diff --git a/src/tracker/mod.rs b/src/tracker/mod.rs
index 63b6c78..ae63cf4 100644
--- a/src/tracker/mod.rs
+++ b/src/tracker/mod.rs
@@ -372,6 +372,20 @@ impl Tracker {
/// tracked database.
#[instrument(skip(self))]
pub fn flush_spooled_data(&self) -> Result<()> {
- crate::copier::copy_spool_path(self.buffer.spooling_directory())
+ let to_flush = self.buffer.spooling_directory();
+
+ if let Ok(handle) = tokio::runtime::Handle::try_current() {
+ // We use our own Tokio runtime in `copy_spool_path`.
+ // Avoid panics due to nested runtimes by flushing in a
+ // temporary thread. Flushes have to upload data over the
+ // network, so they're not exactly fast anyway.
+ let to_flush = to_flush.to_owned();
+
+ handle
+ .block_on(handle.spawn_blocking(move || crate::copier::copy_spool_path(&to_flush)))
+ .map_err(|e| chain_error!(e, "copy_spool_path thread failed"))?
+ } else {
+ crate::copier::copy_spool_path(to_flush)
+ }
}
}
Church- commented
@pkhuong Gave that patch from earlier a try ^ still errorred out.
I'm assuming there would be some obvious issues with trying to run a sync operation inside an async thread right?
Church- commented
Okay this seems to work, doesn't dump a stacktrace at least going to test this a few more times.