no such field: prune
julienduchesne opened this issue · 7 comments
Got a weird error while trying to evaluate some jsonnet files. It's hard to reproduce because it happens in a very convoluted setup 😬
no such field: prune
There is field with similar name present: prune
environments/quaderno-cd:5:3-13: field <prune> access
environments/quaderno-cd:30:1-17: function <noDataEnv> call
I managed to bypass it with the following code:
--- a/crates/jrsonnet-evaluator/src/evaluate/mod.rs
+++ b/crates/jrsonnet-evaluator/src/evaluate/mod.rs
@@ -483,6 +483,11 @@ pub fn evaluate(ctx: Context, expr: &LocExpr) -> Result<Val> {
if conf < 0.8 {
continue;
}
+
+ if conf == 1.0 {
+ return Ok(v.get(field).unwrap().unwrap());
+ }
+
heap.push((conf, field));
}
heap.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(Ordering::Equal));This is obviously terrible but I'm not really familiar enough with Rust to figure out how this could happen and the best way to fix it
This should not happen; something is wrong with the interner.
The key should be obtained from the hashmap, but it isn't, and during iteration over keys, there is a key with the same contents.
Possible it is the same bug as in #98?
To confirm this theory, can you please run your code with this patch, and tell what it prints right before the error?
From bc7390d78e5f3300aee756fa71ec01baf071243a Mon Sep 17 00:00:00 2001
From: Yaroslav Bolyukin <iam@lach.pw>
Date: Thu, 1 Jun 2023 17:44:51 +0200
Subject: [PATCH] bug: same istrs are failing to compare
---
crates/jrsonnet-evaluator/src/evaluate/mod.rs | 3 +++
crates/jrsonnet-interner/src/lib.rs | 4 ++++
2 files changed, 7 insertions(+)
diff --git a/crates/jrsonnet-evaluator/src/evaluate/mod.rs b/crates/jrsonnet-evaluator/src/evaluate/mod.rs
index dc27ea5..f9126ea 100644
--- a/crates/jrsonnet-evaluator/src/evaluate/mod.rs
+++ b/crates/jrsonnet-evaluator/src/evaluate/mod.rs
@@ -483,6 +483,9 @@ pub fn evaluate(ctx: Context, expr: &LocExpr) -> Result<Val> {
if conf < 0.8 {
continue;
}
+
+ let jkey = key.clone().into_flat();
+ eprintln!("{jkey:?} {field:?} {:?} {:?} {}", IStr::as_ptr(&jkey), IStr::as_ptr(&field), jkey == field);
heap.push((conf, field));
}
heap.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(Ordering::Equal));
diff --git a/crates/jrsonnet-interner/src/lib.rs b/crates/jrsonnet-interner/src/lib.rs
index 7b5e065..9142378 100644
--- a/crates/jrsonnet-interner/src/lib.rs
+++ b/crates/jrsonnet-interner/src/lib.rs
@@ -46,6 +46,10 @@ impl IStr {
pub fn cast_bytes(self) -> IBytes {
IBytes(self.0.clone())
}
+
+ pub fn as_ptr(v: &Self) -> *const u8 {
+ Inner::as_ptr(&v.0)
+ }
}
impl Deref for IStr {
--
2.38.3
This is fixed on the latest master version!
Sorry, I was wrong. It's still happening (and I got linked to this issue through an error message 😄). I added the snippets you wrote above and re-ran:
Error message:
- environments/certificates:
evaluating jsonnet: variable is not defined: std
There is variable with similar name present: std
environments/certificates:5:3-7: variable <std> access
environments/certificates:30:1-17: function <noDataEnv> call
Log:
"std" "std" 0x7f44b40018b8 0x7f44a4001a18 false
"std" "std" 0x7f44740018b8 0x7f447c002eb8 false
"std" "std" 0x7f44400018b8 0x7f4484001a18 false
"std" "std" 0x7f43b0001ac8 0x7f448c001a18 false
"std" "std" 0x7f44000018b8 0x7f4460003518 false
"std" "std" 0x7f4414001a18 0x7f44280018b8 false
"std" "std" 0x7f44300018b8 0x7f4450001a18 false
"std" "std" 0x7f44700018b8 0x7f43b8001f28 false
"std" "std" 0x7f43ec001a18 0x7f43b40018b8 false
"std" "std" 0x7f43b0001ac8 0x7f43ec001a18 false
"std" "std" 0x7f43b40018b8 0x7f44280018b8 false
"std" "std" 0x7f4414001a18 0x7f448c001a18 false
"std" "std" 0x7f448c001a18 0x7f44300018b8 false
"std" "std" 0x7f43dc0018b8 0x7f43e00032f8 false
"std" "std" 0x7f4484001a18 0x7f43ec001a18 false
"std" "std" 0x7f43ec001a18 0x7f4498001a18 false
"std" "std" 0x7f44100023a8 0x7f44280018b8 false
"std" "std" 0x7f4484001a18 0x7f43b40018b8 false
"std" "std" 0x7f44b40018b8 0x7f4414001a18 false
"std" "std" 0x7f43cc001a38 0x7f43dc0018b8 false
"std" "std" 0x7f43d00018b8 0x7f43dc0018b8 false
"std" "std" 0x7f43dc0018b8 0x7f43ec001a18 false
"std" "std" 0x7f44c4001a18 0x7f44000018b8 false
"std" "std" 0x7f43ec001a18 0x7f44b40018b8 false
"std" "std" 0x7f44680017e8 0x7f43b0001ac8 false
"std" "std" 0x7f442c001f48 0x7f440c003858 false
"std" "std" 0x7f43b40018b8 0x7f44ac0018b8 false
"std" "std" 0x7f44280018b8 0x7f44580044a8 false
"std" "std" 0x7f43fc0018b8 0x7f4498001a18 false
"std" "std" 0x7f44b00018b8 0x7f4484001a18 false
"std" "std" 0x7f442c001f48 0x7f43fc0018b8 false
"std" "std" 0x7f4498001a18 0x7f44ac0018b8 false
"std" "std" 0x7f4404001f68 0x7f4484001a18 false
"std" "std" 0x7f444c003328 0x7f43ec001a18 false
"std" "std" 0x7f449c0018b8 0x7f44580044a8 false
"std" "std" 0x7f43f8001808 0x7f43c0001c38 false
"std" "std" 0x7f443c0018b8 0x7f447c002eb8 false
"std" "std" 0x7f440c003858 0x7f43b0001ac8 false
"std" "std" 0x7f43ec001a18 0x7f43fc0018b8 false
"std" "std" 0x7f44cc003e38 0x7f44580044a8 false
"std" "std" 0x7f43dc0018b8 0x7f43d00018b8 false
"std" "std" 0x7f440c003858 0x7f44240019d8 false
"std" "std" 0x7f44380018b8 0x7f44c4001a18 false
"std" "std" 0x7f4408001f58 0x7f44200018b8 false
"std" "std" 0x7f43f00018b8 0x7f442c001f48 false
"std" "std" 0x7f43c80018b8 0x7f44a8001a18 false
"std" "std" 0x7f44380018b8 0x7f44240019d8 false
"std" "std" 0x7f44a8001a18 0x7f444c003328 false
"std" "std" 0x7f4404001f68 0x7f442c001f48 false
"std" "std" 0x7f43e4001f48 0x7f44c4001a18 false
"std" "std" 0x7f43b0001ac8 0x7f44380018b8 false
"std" "std" 0x7f44100023a8 0x7f446c001858 false
"std" "std" 0x7f44b00018b8 0x7f448c001a18 false
"std" "std" 0x7f447c002eb8 0x7f44ac0018b8 false
"std" "std" 0x7f43cc001a38 0x7f44700018b8 false
"std" "std" 0x7f43cc001a38 0x7f44580044a8 false
"std" "std" 0x7f4454001a18 0x7f44ac0018b8 false
"std" "std" 0x7f43f8001808 0x7f44a00018b8 false
"std" "std" 0x7f443c0018b8 0x7f442c001f48 false
"std" "std" 0x7f43b0001ac8 0x7f440c003858 false
"std" "std" 0x7f44cc003e38 0x7f44240019d8 false
"std" "std" 0x7f44480021f8 0x7f43ec001a18 false
"std" "std" 0x7f4404001f68 0x7f43dc0018b8 false
"std" "std" 0x7f440c003858 0x7f443c0018b8 false
"std" "std" 0x7f43cc001a38 0x7f447c002eb8 false
"std" "std" 0x7f440c003858 0x7f44480021f8 false
"std" "std" 0x7f4498001a18 0x7f44780020b8 false
"std" "std" 0x7f4404001f68 0x7f44700018b8 false
"std" "std" 0x7f4460003518 0x7f449c0018b8 false
"std" "std" 0x7f43d00018b8 0x7f44380018b8 false
"std" "std" 0x7f4404001f68 0x7f44380018b8 false
"std" "std" 0x7f44580044a8 0x7f449c0018b8 false
"std" "std" 0x7f443c0018b8 0x7f449c0018b8 false
"std" "std" 0x7f43c80018b8 0x7f43c0001c38 false
"std" "std" 0x7f4454001a18 0x7f44b00018b8 false
"std" "std" 0x7f44200018b8 0x7f44300018b8 false
Note that this only happens when calling the library (libjsonnet.so) in a parallel way
Aha, this explains everything: neither implementation of jsonnet implements libjsonnet in the thread-safe way, even go-jsonnet is unsafe to be called from multiple threads
Even with a different VM on each thread? We've been doing that with go-jsonnet for years and have never had any issues (over hundreds of thousands of evals)
You can have different VMs in different threads, yet you can't transfer objects between VMs.
Here you have object created in one thread, and getting fields from it in another.
Every key in jrsonnet is interned in thread-local string pool, thus here we have the same strings defined in different threads.
Thanks for all your help. It turned out to be golang moving goroutines across threads (VM created in one thread but as the goroutine moves on to eval, it's executed in another thread). By using runtime.LockOSThread, it seems to work!