proxy has too many fd open (deleted files)
Closed this issue · 3 comments
dejonghb commented
Found with version 1.3.16 on 2 nodes:
Example:
root@ftcmp03:~# ls -l /proc/9827/fd
total 0
lr-x------ 1 root root 64 Jun 13 18:27 0 -> /dev/null
lrwx------ 1 root root 64 Jun 13 18:27 1 -> socket:[92684397]
lrwx------ 1 root root 64 Jun 13 18:27 10 -> socket:[117865667]
lrwx------ 1 root root 64 Jun 13 18:27 100 -> socket:[112063349]
l-wx------ 1 root root 64 Jun 13 18:27 101 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration44R6VNI73cqc (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 102 -> socket:[112057180]
l-wx------ 1 root root 64 Jun 13 18:27 103 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration44R6VNJllZ4D (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 104 -> socket:[112062186]
l-wx------ 1 root root 64 Jun 13 18:27 105 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration44R6VN4YwpLo (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 106 -> socket:[112083376]
l-wx------ 1 root root 64 Jun 13 18:27 107 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration5U4S9ugCwgzx (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 108 -> socket:[112076784]
l-wx------ 1 root root 64 Jun 13 18:27 109 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration5U4S9uzCtzNf (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 11 -> socket:[104390890]
lrwx------ 1 root root 64 Jun 13 18:27 110 -> socket:[112090353]
l-wx------ 1 root root 64 Jun 13 18:27 111 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration5U4S9ui6KSsG (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 112 -> socket:[112090556]
lrwx------ 1 root root 64 Jun 13 18:27 113 -> socket:[112093315]
lrwx------ 1 root root 64 Jun 13 18:27 114 -> socket:[112090717]
l-wx------ 1 root root 64 Jun 13 18:27 115 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration5U4S9udGejIC (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 116 -> socket:[117854962]
lrwx------ 1 root root 64 Jun 13 18:27 117 -> socket:[117857798]
lrwx------ 1 root root 64 Jun 13 18:27 118 -> socket:[112118975]
l-wx------ 1 root root 64 Jun 13 18:27 119 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationaB2wCZvzGFXM (deleted)
l-wx------ 1 root root 64 Jun 13 18:27 12 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration5U4S9uajeYe7 (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 120 -> socket:[112101229]
l-wx------ 1 root root 64 Jun 13 18:27 121 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationaB2wCZuQQH7f (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 122 -> socket:[112115298]
l-wx------ 1 root root 64 Jun 13 18:27 123 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationaB2wCZf6sIIr (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 124 -> socket:[112119209]
l-wx------ 1 root root 64 Jun 13 18:27 125 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationaB2wCZQE6LYq (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 126 -> socket:[112118142]
l-wx------ 1 root root 64 Jun 13 18:27 127 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationaB2wCZbdy7lo (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 128 -> socket:[112121030]
l-wx------ 1 root root 64 Jun 13 18:27 129 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationaB2wCZkxv5KE (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 13 -> socket:[99722373]
lrwx------ 1 root root 64 Jun 13 18:27 130 -> socket:[112128901]
l-wx------ 1 root root 64 Jun 13 18:27 131 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration0QkjlihFq0uQ (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 132 -> socket:[112125934]
l-wx------ 1 root root 64 Jun 13 18:27 133 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration0QkjliaTxBt4 (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 134 -> socket:[112136000]
l-wx------ 1 root root 64 Jun 13 18:27 135 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration0QkjliqhFbT0 (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 136 -> socket:[112136102]
l-wx------ 1 root root 64 Jun 13 18:27 137 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration0QkjlifEHPXK (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 138 -> socket:[112142544]
l-wx------ 1 root root 64 Jun 13 18:27 139 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration0QkjliibLJ9s (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 14 -> socket:[112186353]
lrwx------ 1 root root 64 Jun 13 18:27 140 -> socket:[112145502]
l-wx------ 1 root root 64 Jun 13 18:27 141 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configuration0QkjliXRQenu (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 142 -> socket:[112155259]
l-wx------ 1 root root 64 Jun 13 18:27 143 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationSgkVnofu3llH (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 144 -> socket:[112164018]
l-wx------ 1 root root 64 Jun 13 18:27 145 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationSgkVnoEocH7F (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 146 -> socket:[112155360]
l-wx------ 1 root root 64 Jun 13 18:27 147 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationSgkVno3TfZkn (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 148 -> socket:[112156379]
l-wx------ 1 root root 64 Jun 13 18:27 149 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationSgkVno6nlldS (deleted)
l-wx------ 1 root root 64 Jun 13 18:27 15 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationaKHW9sF7sgWA (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 150 -> socket:[112155593]
l-wx------ 1 root root 64 Jun 13 18:27 151 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationSgkVnoSWAWcl (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 152 -> socket:[112165917]
l-wx------ 1 root root 64 Jun 13 18:27 153 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationSgkVnoKt88d7 (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 154 -> socket:[112174691]
l-wx------ 1 root root 64 Jun 13 18:27 155 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationdj8ozh2qSfgl (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 156 -> socket:[112176663]
l-wx------ 1 root root 64 Jun 13 18:27 157 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationdj8ozhz1F1L4 (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 158 -> socket:[112181708]
l-wx------ 1 root root 64 Jun 13 18:27 159 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationdj8ozhSDlMIw (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 16 -> socket:[117853163]
lrwx------ 1 root root 64 Jun 13 18:27 160 -> socket:[112177973]
l-wx------ 1 root root 64 Jun 13 18:27 161 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationdj8ozhCKBAkM (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 162 -> socket:[112179613]
l-wx------ 1 root root 64 Jun 13 18:27 163 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationdj8ozhLTTD3Z (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 164 -> socket:[112178150]
l-wx------ 1 root root 64 Jun 13 18:27 165 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationdj8ozhmwGkOw (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 166 -> socket:[112190632]
l-wx------ 1 root root 64 Jun 13 18:27 167 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationaKHW9sLWMDvp (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 168 -> socket:[112185792]
l-wx------ 1 root root 64 Jun 13 18:27 169 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationaKHW9skegeG1 (deleted)
l-wx------ 1 root root 64 Jun 13 18:27 17 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationgrLHe0ZFD4hu (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 170 -> socket:[112192680]
l-wx------ 1 root root 64 Jun 13 18:27 171 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationaKHW9sb9MMhm (deleted)
lrwx------ 1 root root 64 Jun 13 18:27 172 -> socket:[112190742]
l-wx------ 1 root root 64 Jun 13 18:27 173 -> /tmp/68d1fa47-1d9d-4510-b24a-3d7fe532ded1-volume_configurationaKHW9sGyZoyu (deleted)
...
root@ftcmp04:~# ls -l /proc/29421/fd | grep -c deleted
127
Causes proxy to deny more connections:
root@ftcmp02:~# /usr/bin/alba proxy-statistics --to-json --host=172.22.186.33 --port=26216
2017-06-16 12:16:34 139666 +0200 - ftcmp02 - 10483/0000 - alba/cli - 0 - info - Connecting to ADDR_INET(172.22.186.33,26216)
{"success":false,"error":{"message":"(Unix.Unix_error \"Connection reset by peer\" read \"\")","exception_type":"unknown","exception_code":0}}
Number of fd's in use:
root@ftcmp03:~# pgrep -f proxy-start | while read p; do echo -n "$p "; ls -l /proc/${p}/fd | wc -l; done
9758 81
9827 322
25916 81
25918 35
25920 85
25928 65
25938 111
25949 114
Deleted files:
root@ftcmp03:~# pgrep -f proxy-start | while read p; do echo -n "$p "; ls -l /proc/${p}/fd | grep -c deleted; done
9758 4
9827 127
25916 0
25918 0
25920 0
25928 0
25938 0
25949 0
wimpers commented
@domsj @toolslive do you know if this is fixed in a later version?