Failure to compact database generates millions of log lines
olivierboudet opened this issue · 5 comments
olivierboudet commented
Sync Gateway version
Couchbase Sync Gateway/3.0.3(1;8187d9a) CE
Operating system
Ubuntu 20.04.4 LTS nodes in GKE 1.21.10
Config file
{
"adminInterface":"0.0.0.0:4985",
"admin_interface_authentication": false,
"interface":"0.0.0.0:4984",
"metricsInterface": "0.0.0.0:4986",
"metrics_interface_authentication": false,
"use_tls_server": false,
"logging": {
"console": {
"log_level": "debug",
"log_keys": ["HTTP", "WS", "Sync"]
}
},
"databases": {
"cheptel": {
"unsupported": {
"user_views": {
"enabled": true
}
},
"server": "http://${CB_HOST}",
"allow_empty_password": false,
"bucket": "cheptel",
"username":"${SG_USER}",
"password":"${SG_PASSWORD}",
"use_views": false,
"num_index_replicas": ${SG_INDEX_REPLICAS},
"enable_shared_bucket_access": true,
"import_docs": true,
"sync": `
function(doc, oldDoc) {
var idSite = doc.identifiantSite ? doc.identifiantSite : "ALL";
channel(doc.model+'_'+idSite);
}
`,
"users": {
"GUEST": {
"disabled": true,
"admin_channels": [
"*"
],
"all_channels": [
"*"
]
}
},
"cache": {
"rev_cache": {
"size": ${DB_REV_CACHE_SIZE}
}
}
},
"porc": {
"unsupported": {
"user_views": {
"enabled": true
}
},
"server": "http://${CB_HOST}",
"allow_empty_password": false,
"bucket": "porc",
"username":"${SG_USER}",
"password":"${SG_PASSWORD}",
"use_views": false,
"num_index_replicas": ${SG_INDEX_REPLICAS},
"enable_shared_bucket_access": true,
"import_docs": true,
"sync": "function(doc, oldDoc) {channel(doc.model+'_'+doc.identifiantSite)};",
"users": {
"GUEST": {
"disabled": true,
"admin_channels": [
"*"
],
"all_channels": [
"*"
]
}
},
"cache": {
"rev_cache": {
"size": ${DB_REV_CACHE_SIZE}
}
}
},
"parametre": {
"unsupported": {
"user_views": {
"enabled": true
}
},
"server": "http://${CB_HOST}",
"allow_empty_password": false,
"bucket": "parametre",
"username":"${SG_USER}",
"password":"${SG_PASSWORD}",
"use_views": false,
"num_index_replicas": ${SG_INDEX_REPLICAS},
"enable_shared_bucket_access": true,
"import_docs": true,
"sync": `
function(doc, oldDoc) {
var idSite = doc.identifiantSite ? doc.identifiantSite : "ALL";
channel(doc.model+'_'+idSite);
}
`,
"users": {
"GUEST": {
"disabled": true,
"admin_channels": [
"*"
],
"all_channels": [
"*"
]
}
},
"cache": {
"rev_cache": {
"size": ${DB_REV_CACHE_SIZE}
}
}
},
"evenement": {
"unsupported": {
"user_views": {
"enabled": true
}
},
"server": "http://${CB_HOST}",
"allow_empty_password": false,
"bucket": "evenement",
"username":"${SG_USER}",
"password":"${SG_PASSWORD}",
"use_views": false,
"num_index_replicas": ${SG_INDEX_REPLICAS},
"enable_shared_bucket_access": true,
"import_docs": true,
"sync": `function(doc, oldDoc) {
if (doc.type === 'MAINTENANCE') {
if (doc.processed) {
channel(doc.model + '_' + doc.type + '_ARCHIVE');
} else {
channel(doc.model + '_' + doc.type);
}
} else if(doc.enErreur === true) {
channel(doc.model + '_' + (doc.consumer || doc.type) + '_ERREUR_' + doc.identifiantSite);
} else {
channel(doc.model + '_' + (doc.consumer || doc.type));
}
}`,
"users": {
"GUEST": {
"disabled": true,
"admin_channels": [
"*"
],
"all_channels": [
"*"
]
}
},
"cache": {
"rev_cache": {
"size": ${DB_REV_CACHE_SIZE}
}
}
},
"structure": {
"unsupported": {
"user_views": {
"enabled": true
}
},
"server": "http://${CB_HOST}",
"allow_empty_password": false,
"bucket": "structure",
"username": "${SG_USER}",
"password": "${SG_PASSWORD}",
"use_views": false,
"num_index_replicas": ${SG_INDEX_REPLICAS},
"enable_shared_bucket_access": true,
"import_docs": true,
"sync": "function(doc, oldDoc) {channel(doc.model+'_'+doc.identifiantSite)};",
"users": {
"GUEST": {
"disabled": true,
"admin_channels": [
"*"
],
"all_channels": [
"*"
]
}
},
"cache": {
"rev_cache": {
"size": ${DB_REV_CACHE_SIZE}
}
}
}
}
}Log output
https://gist.github.com/olivierboudet/f56efeaf0123e3c83dff34dd928b104c
Actual behavior
Sync gateway logs thousands of lines by second which fills out our elasticsearch cluster :-(
Each line seems to be repeated in loop.
2022-07-15T12:32:44.701Z [WRN] Error compacting key <ud>ANIMAL:44JDR:3400000000000000001900800157:20210830</ud> (purge) - tombstone will not be compacted. SubdocGetBodyAndXattr <ud>ANIMAL:44JDR:3400000000000000001900800157:20210830</ud>: Not Found -- db.(*Database).Compact() at database.go:1195
I have to restart the sync gateway to stop generating too many log lines.
PS : we just upgraded Sync Gateway to 3.x last week, we never had this with Sync Gateway 2.8.x
bbrks commented
Thanks for the report - we think this is an error handling issue related to the SDK upgrades we made in 3.0 and are investigating the appropriate fix.
markspolakovs commented
olivierboudet commented
hello @bbrks, when can we expect a new release of sync gateway, because we are stuck on this issue to upgrade to Sync Gateway 3.x
Thanks
bbrks commented
Hi @olivierboudet - this was backported to 3.0.4 in the 2nd comment mentioned above
olivierboudet commented
Oooh, I did not seen the 3.0.4 was released. It is not mentioned on github (https://github.com/couchbase/sync_gateway/releases).
Thank you :)