Occasionally send the Warcprox status to the server
TheTechRobo opened this issue · 1 comments
TheTechRobo commented
This way we can use it in !status!
It can just be stored in RAM, I guess - no DB necessary.
Example JSON for /status
{
"role": "warcprox",
"version": "2.4.17",
"host": "themandownstairs",
"address": "127.0.0.1",
"port": 4551,
"pid": 213414,
"threads": 100,
"active_requests": 2,
"unaccepted_requests": 0,
"load": 0.0,
"queued_urls": 0,
"queue_max_size": 500,
"urls_processed": 17,
"warc_bytes_written": 321401,
"start_time": "2022-10-10T18:16:04.785443+00:00",
"rates_1min": {
"actual_elapsed": 63.26238942146301,
"urls_per_sec": 0.26872206623027767,
"warc_bytes_per_sec": 5080.443576969263
},
"rates_5min": {
"actual_elapsed": 13.262403011322021,
"urls_per_sec": 1.2818189875158534,
"warc_bytes_per_sec": 24233.994376857812
},
"rates_15min": {
"actual_elapsed": 13.262410402297974,
"urls_per_sec": 1.2818182731741143,
"warc_bytes_per_sec": 24233.980871554915
},
"earliest_still_active_fetch_start": "2022-10-10T18:16:17.963828+00:00",
"seconds_behind": 0.102504,
"postfetch_chain": [
{
"processor": "DedupLoader",
"queued_urls": 0
},
{
"processor": "WarcWriterProcessor",
"queued_urls": 0
},
{
"processor": "DedupDb",
"queued_urls": 0
},
{
"processor": "StatsProcessor",
"queued_urls": 0
},
{
"processor": "CrawlLogger",
"queued_urls": 0
},
{
"processor": "RunningStats",
"queued_urls": 0
}
]
}
TheTechRobo commented
We could also make it ping the client directly and ask it for the details right at that moment, which would be more efficient server-side. Might be tricky to implement though.