panic in tree_deliver
We have seen panics related to the PESI VMOD in our running instances.
Environment
- Varnish from Docker Image
varnish:7.4.2
-
livdp-pesi
pre 92ec4800 (i.e. before theonerror_continue
patch)
Logs
Here is one of the panic logs attached with sensitive data replaced with **REDACTED**
Debug: Version: varnish-7.4.2 revision cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80
Debug: Platform: Linux,5.15.148,x86_64,-junix,-smalloc,-smalloc,-hcritbit
Debug: Child (44) Started
Child launched OK
Info: Child (44) said Child starts
Error: Child (44) not responding to CLI, killed it.
Error: Child (44) not responding to CLI, killed it.
Error: Unexpected reply from ping: 400 CLI communication error
Error: Child (44) died signal=6
Error: Child (44) Panic at: Fri, 16 Feb 2024 11:33:41 GMT
Assert error in VDP_bytes(), cache/cache_deliver_proc.c line 115:
Condition((vdpe) != NULL) not true.
version = varnish-7.4.2 revision cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, vrt api = 18.0
ident = Linux,5.15.148,x86_64,-junix,-smalloc,-smalloc,-hcritbit,epoll
now = 70.443893 (mono), 1708083220.276110 (real)
Backtrace:
0x5637c612b812: /usr/sbin/varnishd(+0x5b812) [0x5637c612b812]
0x5637c61aa2f5: /usr/sbin/varnishd(VAS_Fail+0x45) [0x5637c61aa2f5]
0x5637c610b50d: /usr/sbin/varnishd(VDP_bytes+0x1bd) [0x5637c610b50d]
0x5637c6186bf5: /usr/sbin/varnishd(+0xb6bf5) [0x5637c6186bf5]
0x5637c610bbb8: /usr/sbin/varnishd(VDP_DeliverObj+0x68) [0x5637c610bbb8]
0x7f794d84e97a: ./vmod_cache/_vmod_pesi.bcd25c71f5b257bebad7d192f787e1fbbd804965b95fd35255901246cecaf09f(tree_deliver+0x3ba) [0x7f794d84e97a]
0x7f794d8525f0: ./vmod_cache/_vmod_pesi.bcd25c71f5b257bebad7d192f787e1fbbd804965b95fd35255901246cecaf09f(+0x95f0) [0x7f794d8525f0]
0x5637c610b3d8: /usr/sbin/varnishd(VDP_bytes+0x88) [0x5637c610b3d8]
0x5637c6186b94: /usr/sbin/varnishd(+0xb6b94) [0x5637c6186b94]
0x5637c610bbb8: /usr/sbin/varnishd(VDP_DeliverObj+0x68) [0x5637c610bbb8]
0x5637c615c6fc: /usr/sbin/varnishd(V1D_Deliver+0x44c) [0x5637c615c6fc]
0x5637c6131ac2: /usr/sbin/varnishd(+0x61ac2) [0x5637c6131ac2]
0x5637c6133dd8: /usr/sbin/varnishd(CNT_Request+0xe8) [0x5637c6133dd8]
0x5637c615e602: /usr/sbin/varnishd(+0x8e602) [0x5637c615e602]
0x5637c6152ec2: /usr/sbin/varnishd(+0x82ec2) [0x5637c6152ec2]
0x5637c6153484: /usr/sbin/varnishd(+0x83484) [0x5637c6153484]
0x7f7a5fbbf134: /lib/x86_64-linux-gnu/libc.so.6(+0x89134) [0x7f7a5fbbf134]
0x7f7a5fc3f7dc: /lib/x86_64-linux-gnu/libc.so.6(+0x1097dc) [0x7f7a5fc3f7dc]
argv = {
[0] = \"/usr/sbin/varnishd\",
[1] = \"-F\",
[2] = \"-P\",
[3] = \"/var/run/varnish.pid\",
[4] = \"-f\",
[5] = \"/etc/varnish/current/main.vcl\",
[6] = \"-a\",
[7] = \"/var/lib/haproxy/varnish_listen.sock,PROXY,user=haproxy,group=haproxy,mode=755\",
[8] = \"-T\",
[9] = \"127.0.0.1:6082\",
[10] = \"-S\",
[11] = \"/etc/varnish/varnish-secret\",
[12] = \"-s\",
[13] = \"malloc,18G\",
[14] = \"-n\",
[15] = \"/var/lib/varnish\",
[16] = \"-l\",
[17] = \"4g\",
[18] = \"-t\",
[19] = \"0\",
[20] = \"-s\",
[21] = \"Transient=malloc,4g\",
[22] = \"-p\",
[23] = \"vcl_path=/etc/varnish/current\",
[24] = \"-p\",
[25] = \"thread_pools=8\",
[26] = \"-p\",
[27] = \"thread_pool_min=625\",
[28] = \"-p\",
[29] = \"vsl_reclen=4084\",
[30] = \"-p\",
[31] = \"thread_pool_max=1250\",
[32] = \"-p\",
[33] = \"feature=+esi_disable_xml_check,+esi_ignore_other_elements,+esi_include_onerror\",
[34] = \"-p\",
[35] = \"thread_pool_stack=128k\",
[36] = \"-p\",
[37] = \"vcc_err_unref=off\",
[38] = \"-p\",
[39] = \"max_esi_depth=15\",
[40] = \"-p\",
[41] = \"workspace_client=128k\",
[42] = \"-p\",
[43] = \"workspace_backend=96k\",
[44] = \"-p\",
[45] = \"send_timeout=20\",
[46] = \"-p\",
[47] = \"first_byte_timeout=3\",
[48] = \"-p\",
[49] = \"between_bytes_timeout=2\",
[50] = \"-p\",
[51] = \"cli_timeout=60\",
[52] = \"-p\",
[53] = \"cli_limit=64k\",
[54] = \"-p\",
[55] = \"pipe_timeout=60\",
[56] = \"-p\",
[57] = \"idle_send_timeout=60\",
[58] = \"-p\",
[59] = \"timeout_idle=65\",
[60] = \"-p\",
[61] = \"thread_pool_timeout=300\",
[62] = \"-p\",
[63] = \"http_max_hdr=128\",
[64] = \"-p\",
[65] = \"vcc_allow_inline_c=off\",
}
pthread.self = 0x7f79225be6c0
pthread.name = (cache-worker)
pthread.attr = {
guard = 4096,
stack_bottom = 0x7f792259f000,
stack_top = 0x7f79225bf000,
stack_size = 131072,
}
thr.req = 0x7f7917d29360 {
vxid = 131074, transport = HTTP/1 {
state = HTTP1::Proc
}
step = Req Step transmit
req_body = none,
restarts = 0, esi_level = 0,
vary_b = (nil), vary_e = (nil),
d_ttl = -1.000000, d_grace = -1.000000,
storage = (nil),
director_hint = 0x7f7a5f41e260 {
cli_name = boot.imberator,
admin_health = probe, changed = 1708083198.848297,
type = backend {
conn_pool = 0x7f7a5f447040 {
ident = 0xabc4beda35bae9568a83bc9067f9831e87a61406117c724c970de8a08005de4b,
vrt_endpoint = 0x7f7a5f4297c0 {
uds_path = /sockets/imberator.sock,
},
},
hosthdr = 0.0.0.0,
n_conn = 0,
},
},
sess = 0x7f7917801b20 {
fd = 39, vxid = 131073,
t_open = 1708083220.221190,
t_idle = 1708083220.276461,
ws = 0x7f7917801b60 {
id = \"ses\",
{s, f, r, e} = {0x7f7917801bb8, +176, (nil), +576},
},
transport = HTTP/1 {
state = HTTP1::Proc
}
client = **REDACTED** 36962 /var/lib/haproxy/varnish_listen.sock,
local.endpoint = /var/lib/haproxy/varnish_listen.sock,
local.socket = a0,
local.ip = 0.0.0.0:0,
remote.ip = 0.0.0.0:0,
server.ip = **REDACTED**:6666,
client.ip = **REDACTED**:36962,
},
worker = 0x7f79225bd370 {
ws = 0x7f79225bd3f8 {
id = \"wrk\",
{s, f, r, e} = {0x7f79225bc8e0, +1128, (nil), +2040},
},
VCL::method = DELIVER,
VCL::methods = {RECV, HASH, MISS, DELIVER},
},
ws = 0x7f7917d294c0 {
id = \"req\",
{s, f, r, e} = {0x7f7917d2f098, +35320, (nil), +107168},
},
http_conn = 0x7f7917d2f010 {
fd = 39 (@0x7f7917801b44),
doclose = null(Not Closing)
ws = 0x7f7917d294c0 {
[Already dumped, see above]
},
{rxbuf_b, rxbuf_e} = {0x7f7917d2f098, 0x7f7917d2f1a7},
{pipeline_b, pipeline_e} = {(nil), (nil)},
content_length = -1,
body_status = none,
first_byte_timeout = 0.000000,
between_bytes_timeout = 0.000000,
},
http[req] = 0x7f7917d29560 {
ws = 0x7f7917d294c0 {
[Already dumped, see above]
},
hdrs {
\"GET\",
\"/dcs/products/orientteppich-perser-taebriz-premium-340-x-246-cm-beige-morgenland-rechteckig-hoehe-5-mm-wohnzimmer-handgeknuepft-einzelstueck-mit-zertifikat-1665429185/\",
\"HTTP/1.1\",
\"Host: **REDACTED**\",
\"User-Agent: Blackbox Exporter/0.24.0\",
\"Connection: close\",
\"X-Forwarded-For: **REDACTED**\",
\"Via: 1.1 **REDACTED** (Varnish/7.4)\",
\"X-Real-IP: **REDACTED**\",
\"X-Country: \",
\"X-Isp: \",
\"X-Origin: **REDACTED**\",
\"X-Breakpoint: XL\",
\"Surrogate-Capability: otto_pa=\"Surrogate/1.0\"\",
\"X-Cache: MISS\",
\"X-Ssid: **REDACTED**\",
},
},
http[resp] = 0x7f7917d2a6d0 {
ws = 0x7f7917d294c0 {
[Already dumped, see above]
},
hdrs {
\"HTTP/1.1\",
\"200\",
\"OK\",
\"date: Fri, 16 Feb 2024 11:33:40 GMT\",
\"content-type: text/html;charset=utf-8\",
\"X-Backend: **REDACTED**\",
\"X-Ssid: **REDACTED**\",
\"X-TTL: ttl=1800.000 grace=300.000 decision=nohdr_hitpass\",
\"Cache-Control: private, no-cache, no-store, max-age=0\",
\"Expires: Sat, 01 Jan 1970 00:00:00 GMT\",
\"Pragma: no-cache\",
\"Vary: Accept-Encoding\",
\"X-Varnish: 131074\",
\"Age: 0\",
\"Via: 1.1 **REDACTED** (Varnish/7.4)\",
\"Accept-Ranges: bytes\",
\"X-Recruiting: Seems you like http headers. To write ours, apply at www.otto.de/jobs/ and mention this header.\",
\"Accept-CH: Sec-CH-UA, Sec-CH-UA-Mobile, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Platform, Sec-CH-UA-Platform-Version, Sec-CH-UA-Arch, Sec-CH-UA-Model, Sec-CH-UA-Bitness\",
\"Strict-Transport-Security: max-age=31536000\",
\"X-XSS-Protection: 1; mode=block\",
\"X-Content-Type-Options: nosniff\",
\"X-Cache: MISS\",
\"X-Origin: **REDACTED**\",
\"X-Genesis-URL: **REDACTED**\",
\"X-Original-URL: **REDACTED**\",
\"Connection: close\",
\"Transfer-Encoding: chunked\",
},
},
vdc = 0x7f7917d2efd0 {
nxt = 0x7f7917d37a20,
retval = 0,
filters = {
pesi = 0x7f7917d33c30 { priv = 0x7f7917d33d30 }
gunzip = 0x7f7917d37a20 { priv = 0x7f7915c30000 }
V1B = 0x7f7917d37a58 { priv = (nil) }
},
},
vcl[vcl] = 0x7f7a5f4461e0 {
name = \"boot\",
busy = 58,
discard = 0,
state = auto,
temp = warm,
conf = 0x7f794d9fa2e0 {
syntax = \"41\",
srcname = {
[0] = \"/etc/varnish/current/main.vcl\",
**REDACTED**
[178] = \"<builtin>\",
},
instances = {
**REDACTED**
},
},
},
vpi = 0x7f79225bc8d0 {
handling (VCL::return) = 0x0 (none),
ref = 505,
vpi_ref = 0x7f794d9f9f60 {
source = 178 (\"<builtin>\"),
offset = 3771,
line = 163,
pos = 1,
src = \"}\"
},
},
objcore[REQ] = 0x7f7915c2d000 {
refcnt = 2,
flags = {hfm, hfp},
exp_flags = {refd},
exp = {1708083220.252433, 1800.000000, 0.000000, 0.000000},
objhead = 0x7f7915c0c1c0,
stevedore = 0x7f7a5f43f380 (malloc Transient) {
Simple = 0x7f7915895000,
Obj = 0x7f7915803058 {priv=0x7f7915803050, ptr=0x7f7915895000, len=643, space=648},
LEN = 0x0000000000005819,
VXID = 0x0000000000020003,
FLAGS = 0x16,
GZIPBITS = 0x0000000000000050000000000002c078000000000002c082000000000001a568,
LASTMODIFIED = 0x41d973d205000000,
VARY = {len=27, ptr=0x7f7915895080},
HEADERS = {len=488, ptr=0x7f791589509b},
ESIDATA = 0x7f79158030f8 {priv=0x7f79158030f0, ptr=0x7f79158c0000, len=1530, space=1530},
Body = 0x7f7915803148 {priv=0x7f7915803140, ptr=0x7f7915848700, len=22553, space=22553},
},
},
flags = {
},
privs = 0x7f7917d29550 {
priv = 0x7f7917d33da8 {{p (nil) l 0 m (nil) t \"\"} vmod 7f794d85baa0},
priv = 0x7f7917d2f208 {{p 0x7f7915c05000 l 0 m 0x7f794e605d00 t \"vmod_cookie_cobj\"} vmod 7f794d9fdb88},
priv = 0x7f7917d2f638 {{p 0x7f7917d2f720 l 0 m 0x7f795cc0dd20 t \"vmod_uuid_priv_task\"} vmod 7f794d9fdee8},
priv = 0x7f7917d2fa98 {{p 0x7f7917d2fad8 l -1 m (nil) t \"\"} vmod 7f795e606480},
priv = 0x7f7917d2f528 {{p 0x7f7917d2f568 l -1 m (nil) t \"\"} vmod 7f795e6065a0},
priv = 0x7f7917d31e50 {{p 0x7f7917d31e90 l 0 m (nil) t \"\"} vmod 7f795e606600},
priv = 0x7f7917d31ef8 {{p 0x7f7917d31f38 l -1 m (nil) t \"\"} vmod 7f795e606630},
priv = 0x7f7917d32018 {{p 0x7f7917d32058 l -1 m (nil) t \"\"} vmod 7f795e606660},
priv = 0x7f7917d320c0 {{p 0x7f7917d32100 l -1 m (nil) t \"\"} vmod 7f795e606690},
priv = 0x7f7917d32168 {{p 0x7f7917d321a8 l -1 m (nil) t \"\"} vmod 7f795e6066c0},
priv = 0x7f7917d32210 {{p 0x7f7917d32250 l -1 m (nil) t \"\"} vmod 7f795e6066f0},
priv = 0x7f7917d322b8 {{p 0x7f7917d322f8 l -1 m (nil) t \"\"} vmod 7f795e606720},
priv = 0x7f7917d32360 {{p 0x7f7917d323a0 l -1 m (nil) t \"\"} vmod 7f795e606750},
priv = 0x7f7917d32408 {{p 0x7f7917d32448 l -1 m (nil) t \"\"} vmod 7f795e606780},
priv = 0x7f7917d324b0 {{p 0x7f7917d324f0 l -1 m (nil) t \"\"} vmod 7f795e6067b0},
priv = 0x7f7917d32558 {{p 0x7f7917d32598 l -1 m (nil) t \"\"} vmod 7f795e6067e0},
priv = 0x7f7917d32600 {{p 0x7f7917d32640 l -1 m (nil) t \"\"} vmod 7f795e606810},
priv = 0x7f7917d339c8 {{p 0x7f7917d33a08 l 104 m (nil) t \"\"} vmod 7f795e606870},
priv = 0x7f7917d30298 {{p 0x7f7917d302d8 l 40 m (nil) t \"\"} vmod 7f795e60cd90},
priv = 0x7f7917d31888 {{p 0x7f7917d318c8 l 40 m (nil) t \"\"} vmod 7f795e60d3d0},
priv = 0x7f7917d315e8 {{p 0x7f7917d31628 l 24 m (nil) t \"\"} vmod 7f795e61a900},
priv = 0x7f7917d2f5d0 {{p 0x7f7917d2f610 l 24 m (nil) t \"\"} vmod 7f7a5f4ce200},
priv = 0x7f7917d316f0 {{p 0x7f7917d31730 l -1 m (nil) t \"\"} vmod 7f7a5f4d8560},
priv = 0x7f7917d31798 {{p 0x7f7917d317d8 l -1 m (nil) t \"\"} vmod 7f7a5f4d8590},
priv = 0x7f7917d2f818 {{p 0x7f7917d2f858 l -1 m (nil) t \"\"} vmod 7f7a5f4d8950},
priv = 0x7f7917d2f678 {{p 0x7f7917d2f6b8 l -1 m (nil) t \"\"} vmod 7f7a5f4d8980},
},
top = 0x7f7917d2f078 {
req = 0x7f7917d29360 {
[Already dumped, see above]
},
privs = 0x7f7917d2f090 {
priv = 0x7f7917d2fa58 {{p 0x7f7917d2fb40 l 0 m (nil) t \"\"} vmod 7f794d9fda48},
},
vcl[vcl0] = NULL
},
},
thr.busyobj = NULL
thr.worker = 0x7f79225bd370 {
[Already dumped, see above]
},
vmods = {
std = {0x7f7a5f4f70e0, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 0.0},
directors = {0x7f7a5f4f7150, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 0.0},
re = {0x7f7a5f4f71c0, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 18.0},
re2 = {0x7f7a5f4f72a0, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 18.0},
uuid = {0x7f7a5f4f7310, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 18.0},
header = {0x7f7a5f4f73f0, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 18.0},
blob = {0x7f7a5f4f7460, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 0.0},
var = {0x7f7a5f4f7540, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 18.0},
selector = {0x7f7a5f4f7620, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 18.0},
cookie = {0x7f7a5f4f7690, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 0.0},
pesi = {0x7f7a5f4f7770, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 0.0},
geoip2 = {0x7f7a5f4f77e0, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 0.0},
**REDACTED** = {0x7f7a5f4f7850, Varnish 7.4.2 cd1d10ab53a6f6115b2b4f3b2a1da94c1f749f80, 0.0},
},
pools = {
pool = 0x7f795c801900 {
nidle = 624,
nthr = 625,
lqueue = 0
},
pool = 0x7f795c801f40 {
nidle = 624,
nthr = 625,
lqueue = 0
},
pool = 0x7f795c802580 {
nidle = 570,
nthr = 625,
lqueue = 0
},
pool = 0x7f795c802bc0 {
nidle = 624,
nthr = 625,
lqueue = 0
},
pool = 0x7f795c800640 {
nidle = 624,
nthr = 625,
lqueue = 0
},
pool = 0x7f795c800000 {
nidle = 624,
nthr = 625,
lqueue = 0
},
pool = 0x7f795c800c80 {
nidle = 624,
nthr = 625,
lqueue = 0
},
pool = 0x7f795c8012c0 {
nidle = 624,
nthr = 625,
lqueue = 0
},
},