Memory allocation error under stress

[jug-dev]

TRACE      | 2023-05-14 19:13:37.730978 | worker.jobs.stable_diffusion:start_job:237 - Traceback (most recent call last):
  File "F:\AI\dev\AI-Horde-Worker\worker\jobs\stable_diffusion.py", line 216, in start_job
    self.image = generator(gen_payload)
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\hordelib\horde.py", line 452, in basic_inference
    images = self.generator.run_image_pipeline(pipeline, params)
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\hordelib\comfy_horde.py", line 542, in run_image_pipeline
    result = self.run_pipeline(pipeline_name, params)
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\hordelib\comfy_horde.py", line 513, in run_pipeline
    cleanup()
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\hordelib\comfy_horde.py", line 60, in cleanup
    unload_model_from_gpu(model_data["model"])
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\hordelib\comfy_horde.py", line 100, in unload_model_from_gpu
    _comfy_model_manager.load_model_gpu(model)
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\hordelib\_comfyui\comfy\model_management.py", line 198, in load_model_gpu
    if self.unload_model(release_model):
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\hordelib\_comfyui\comfy\model_management.py", line 173, in unload_model
    model.model.cpu()
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\torch\nn\modules\module.py", line 954, in cpu
    return self._apply(lambda t: t.cpu())
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\torch\nn\modules\module.py", line 797, in _apply
    module._apply(fn)
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\torch\nn\modules\module.py", line 797, in _apply
    module._apply(fn)
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\torch\nn\modules\module.py", line 797, in _apply
    module._apply(fn)
  [Previous line repeated 7 more times]
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\torch\nn\modules\module.py", line 820, in _apply
    param_applied = fn(param)
  File "F:\AI\dev\AI-Horde-Worker\conda\envs\windows\lib\site-packages\torch\nn\modules\module.py", line 954, in <lambda>
    return self._apply(lambda t: t.cpu())
RuntimeError: [enforce fail at ..\c10\core\impl\alloc_cpu.cpp:72] data. DefaultCPUAllocator: not enough memory: you tried to allocate 3276800 bytes.