i'm seeing this panic sometimes when shutting down the system, after
many (but not all) of the disk buffers have been flushed.

'uname -a'
FreeBSD 8.0-CURRENT #1: Sun Mar 23 15:24:03 CET 2008
root@x:/usr/obj/usr/src/sys/spott_fbsd8_i386 i386

it's a GENERIC with most of the kld'able devices turned off, built
saturday 22nd. the PID is always 1 when the panic happens. alas, i don't
have serial access to the box, so i can't give you but the kgdb output.

i'd appreciate links to some tutorial on what best to do with ddb(4) and
kgdb(1). if i don't type "call doadump", i don't even get a dump. here's
kgdb output:

....
Loaded symbols for /boot/kernel/acpi.ko
Reading in symbols for /usr/src/sys/i386/i386/pmap.c...unsupported tag: 'DW_TAG_const_type'...done.

Unread portion of the kernel message buffer:
<6>pflog0: promiscuous mode disabled
<118>Writing entropy file:
<118>.
<118>.
Waiting (max 60 seconds) for system process `vnlru' to stop...done
Waiting (max 60 seconds) for system process `bufdaemon' to stop...done
Waiting (max 60 seconds) for system process `syncer' to stop...
Syncing disks, vnodes remaining...40 36 2 2 1 1 0 0 0 done
All buffers synced.
lock order reversal:
1st 0xc3ab97d4 vfslock (vfslock) @ /usr/src/sys/kern/vfs_mount.c:1249
2nd 0xc3d30e08 syncer (syncer) @ /usr/src/sys/kern/vfs_subr.c:2162
KDB: stack backtrace:
db_trace_self_wrapper(c07921b4,edf0ea68,c05ae5b6,c 079497e,c3d30e08,...) at db_trace_self_wrapper+0x26
kdb_backtrace(c079497e,c3d30e08,c079ba41,c079ba41, c079afff,...) at kdb_backtrace+0x29
witness_checkorder(c3d30e08,9,c079afff,872,c3d30e0 8,...) at witness_checkorder+0x6d6
_lockmgr_args(c3d30e08,20002,c3d30e38,0,ffffffff,. ..) at _lockmgr_args+0x569
vop_stdlock(edf0eb38,c37f8d00,c079afff,c079afff,c0 5ade87,...) at vop_stdlock+0x51
VOP_LOCK1_APV(c07dd2a0,edf0eb38,40000,c079afff,c3d 30e38,...) at VOP_LOCK1_APV+0xa5
_vn_lock(c3d30db0,20002,c079afff,872,0,...) at _vn_lock+0xf2
vrele(c3d30db0,0,c079a95e,4f7,50,...) at vrele+0x142
dounmount(c3ab97d4,80000,c37f8c60,e3846ee8,0,...) at dounmount+0x38a
vfs_unmountall(c078e999,0,c078ea38,128,c05f05c0,.. .) at vfs_unmountall+0x4e
boot(c081dc10,0,c078ea38,ab,edf0ed2c,...) at boot+0x51a
reboot(c37f8c60,edf0ecfc,3f5,c07b4810,c37f8c60,... ) at reboot+0x4b
syscall(edf0ed38) at syscall+0x313
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (55, FreeBSD ELF32, reboot), eip = 0x8050b43, esp = 0xbfbfe8ec, ebp = 0xbfbfe9b8 ---
panic: fifo_close: no v_fifoinfo
cpuid = 0
KDB: enter: panic
exclusive lockmgr ufs r = 0 (0xc4090058) locked @ /usr/src/sys/kern/vfs_subr.c:2367
exclusive lockmgr vfslock r = 0 (0xc3c0329c) locked @ /usr/src/sys/kern/vfs_mount.c:1249
exclusive lockmgr ufs r = 0 (0xc39432a0) locked @ /usr/src/sys/kern/vfs_mount.c:1214
exclusive sleep mutex Giant r = 0 (0xc081dc10) locked @ /usr/src/sys/kern/kern_shutdown.c:171
Physical memory: 887 MB
Dumping 169 MB: 154 138 122 106 90 74 58 42 26 10

#0 doadump () at pcpu.h:195
195 __asm __volatile("movl %%fs:0,%0" : "=r" (td));
(kgdb) bt
Incomplete CFI data; unspecified registers at 0xc05720f3...
....
Reading in symbols for /usr/src/sys/ddb/db_command.c...unsupported tag: 'DW_TAG_const_type'...done.
Incomplete CFI data; unspecified registers at 0xc048cee8...
....
Incomplete CFI data; unspecified registers at 0xc048d2ac...
....
Incomplete CFI data; unspecified registers at 0xc048d663...
....
Reading in symbols for /usr/src/sys/ddb/db_main.c...done.
Incomplete CFI data; unspecified registers at 0xc048ede7...
....
Reading in symbols for /usr/src/sys/kern/subr_kdb.c...done.
Incomplete CFI data; unspecified registers at 0xc059ce5f...
....
Reading in symbols for /usr/src/sys/i386/i386/trap.c...done.
Reading in symbols for /usr/src/sys/i386/i386/exception.s...done.
Reading in symbols for /usr/src/sys/fs/fifofs/fifo_vnops.c...done.
Reading in symbols for /usr/src/sys/ufs/ufs/ufs_vnops.c...done.
Reading in symbols for vnode_if.c...done.
Reading in symbols for /usr/src/sys/kern/vfs_subr.c...unsupported tag: 'DW_TAG_const_type'...done.
Reading in symbols for /usr/src/sys/ufs/ffs/ffs_vfsops.c...done.
Reading in symbols for /usr/src/sys/ufs/ffs/ffs_softdep.c...done.
Reading in symbols for /usr/src/sys/kern/vfs_mount.c...done.
Previous frame inner to this frame (corrupt stack?)
(kgdb) show witness
Undefined show command: "witness". Try "help show".
(kgdb) bt
#0 doadump () at pcpu.h:195
#1 0xc048d029 in db_fncall (dummy1=1, dummy2=0, dummy3=-1064808672, dummy4=0xedf0e734 "\200\264\203\303")
at /usr/src/sys/ddb/db_command.c:514
#2 0xc048d5ac in db_command (last_cmdp=0xc07fb3b4, cmd_table=0x0, dopager=1)
at /usr/src/sys/ddb/db_command.c:411
#3 0xc048d6ba in db_command_loop () at /usr/src/sys/ddb/db_command.c:464
#4 0xc048eebc in db_trap (type=3, code=0) at /usr/src/sys/ddb/db_main.c:228
#5 0xc059cee5 in kdb_trap (type=3, code=0, tf=0xedf0e8dc) at /usr/src/sys/kern/subr_kdb.c:510
#6 0xc07512eb in trap (frame=0xedf0e8dc) at /usr/src/sys/i386/i386/trap.c:643
#7 0xc0736a8b in calltrap () at /usr/src/sys/i386/i386/exception.s:146
#8 0xc059d06a in kdb_enter (why=0xc078ebf7 "panic", msg=0xc078ebf7 "panic") at cpufunc.h:60
#9 0xc0572d0c in panic (fmt=0xc0785708 "fifo_close: no v_fifoinfo") at /usr/src/sys/kern/kern_shutdown.c:556
#10 0xc0519398 in fifo_close (ap=0xedf0e9fc) at /usr/src/sys/fs/fifofs/fifo_vnops.c:426
#11 0xc06f3a44 in ufsfifo_close (ap=0xedf0e9fc) at /usr/src/sys/ufs/ufs/ufs_vnops.c:2043
#12 0xc075d565 in VOP_CLOSE_APV (vop=0xc07eaf80, a=0xedf0e9fc) at vnode_if.c:424
#13 0xc05f32c7 in vgonel (vp=0xc4090000) at vnode_if.h:225
#14 0xc05f4d27 in vflush (mp=0xc3c0329c, rootrefs=0, flags=3, td=0xc37f8c60)
at /usr/src/sys/kern/vfs_subr.c:2412
#15 0xc06e46ca in ffs_flushfiles (mp=0xc3c0329c, flags=2, td=0xc37f8c60)
at /usr/src/sys/ufs/ffs/ffs_vfsops.c:1107
#16 0xc06dea57 in softdep_flushfiles (oldmnt=0xc3c0329c, flags=2, td=0xc37f8c60)
at /usr/src/sys/ufs/ffs/ffs_softdep.c:1085
#17 0xc06e5183 in ffs_unmount (mp=0xc3c0329c, mntflags=524288, td=0xc37f8c60)
at /usr/src/sys/ufs/ffs/ffs_vfsops.c:1032
#18 0xc05ef1de in dounmount (mp=0xc3c0329c, flags=524288, td=0xc37f8c60) at /usr/src/sys/kern/vfs_mount.c:1293
#19 0xc05f23de in vfs_unmountall () at /usr/src/sys/kern/vfs_subr.c:2942
#20 0xc057267a in boot (howto=Variable "howto" is not available.
) at /usr/src/sys/kern/kern_shutdown.c:400
#21 0xc0572e0b in reboot (td=0xc37f8c60, uap=0xedf0ecfc) at /usr/src/sys/kern/kern_shutdown.c:172
#22 0xc07509e3 in syscall (frame=0xedf0ed38) at /usr/src/sys/i386/i386/trap.c:1026
#23 0xc0736af0 in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:203
#24 0x00000033 in ?? ()
(kgdb) list /usr/src/sys/fs/fifofs/fifo_vnops.c:426
421 {
422 struct vnode *vp = ap->a_vp;
423 struct fifoinfo *fip = vp->v_fifoinfo;
424
425 ASSERT_VOP_LOCKED(vp, "fifo_close");
426 KASSERT(fip != NULL, ("fifo_close: no v_fifoinfo"));
427 if (ap->a_fflag & FREAD) {
428 fip->fi_readers--;
429 if (fip->fi_readers == 0)
430 socantsendmore(fip->fi_writesock);
(kgdb) f 10
#10 0xc0519398 in fifo_close (ap=0xedf0e9fc) at /usr/src/sys/fs/fifofs/fifo_vnops.c:426
426 KASSERT(fip != NULL, ("fifo_close: no v_fifoinfo"));
(kgdb) l 405,433
405
406 return (0);
407 }
408
409 /*
410 * Device close routine
411 */
412 /* ARGSUSED */
413 static int
414 fifo_close(ap)
415 struct vop_close_args /* {
416 struct vnode *a_vp;
417 int a_fflag;
418 struct ucred *a_cred;
419 struct thread *a_td;
420 } */ *ap;
421 {
422 struct vnode *vp = ap->a_vp;
423 struct fifoinfo *fip = vp->v_fifoinfo;
424
425 ASSERT_VOP_LOCKED(vp, "fifo_close");
426 KASSERT(fip != NULL, ("fifo_close: no v_fifoinfo"));
427 if (ap->a_fflag & FREAD) {
428 fip->fi_readers--;
429 if (fip->fi_readers == 0)
430 socantsendmore(fip->fi_writesock);
431 }
432 if (ap->a_fflag & FWRITE) {
433 fip->fi_writers--;
(kgdb) p *ap
$6 = {
a_gen = {
a_desc = 0xc07f86a0
},
a_vp = 0xc4090000,
a_fflag = 4,
a_cred = 0x0,
a_td = 0xc37f8c60
}
(kgdb) p *vp
$5 = {
v_type = VFIFO,
v_tag = 0xc07883ef "ufs",
v_op = 0xc07eaf80,
v_data = 0xc406f8c4,
v_mount = 0xc3c0329c,
v_nmntvnodes = {
tqe_next = 0xc6363200,
tqe_prev = 0xc3c03304
},
v_un = {
vu_mount = 0x0,
vu_socket = 0x0,
vu_cdev = 0x0,
vu_fifoinfo = 0x0
},
v_hashlist = {
le_next = 0x0,
le_prev = 0xc7652144
},
v_hash = 222839,
v_cache_src = {
lh_first = 0x0
},
v_cache_dst = {
tqh_first = 0x0,
tqh_last = 0xc4090030
},
v_dd = 0x0,
v_cstart = 0,
v_lasta = 0,
v_lastw = 0,
v_clen = 0,
v_lock = {
lk_object = {
lo_name = 0xc07883ef "ufs",
lo_type = 0xc07883ef "ufs",
lo_flags = 91947008,
lo_witness_data = {
lod_list = {
stqe_next = 0xc0830ac8
},
lod_witness = 0xc0830ac8
}
},
lk_interlock = 0xc081d2e0,
lk_flags = 4194368,
lk_sharecount = 0,
lk_waitcount = 0,
lk_exclusivecount = 1,
lk_prio = 80,
lk_timo = 51,
lk_lockholder = 0xc37f8c60,
lk_newlock = 0x0
},
v_interlock = {
lock_object = {
lo_name = 0xc0794ae0 "vnode interlock",
lo_type = 0xc0794ae0 "vnode interlock",
lo_flags = 16973824,
lo_witness_data = {
lod_list = {
stqe_next = 0xc0832918
},
lod_witness = 0xc0832918
}
},
mtx_lock = 4,
mtx_recurse = 0
},
v_vnlock = 0xc4090058,
v_holdcnt = 3,
v_usecount = 2,
v_iflag = 128,
v_vflag = 0,
v_writecount = 0,
v_freelist = {
tqe_next = 0x0,
tqe_prev = 0xc406d8b4
},
v_bufobj = {
bo_mtx = {
lock_object = {
lo_name = 0xc079baf8 "bufobj interlock",
lo_type = 0xc079baf8 "bufobj interlock",
lo_flags = 16973824,
lo_witness_data = {
lod_list = {
stqe_next = 0xc0831298
},
lod_witness = 0xc0831298
}
},
mtx_lock = 4,
mtx_recurse = 0
},
bo_clean = {
bv_hd = {
tqh_first = 0x0,
tqh_last = 0xc40900d8
},
bv_root = 0x0,
bv_cnt = 0
},
bo_dirty = {
bv_hd = {
tqh_first = 0x0,
tqh_last = 0xc40900e8
},
bv_root = 0x0,
bv_cnt = 0
},
bo_numoutput = 0,
bo_flag = 0,
bo_ops = 0xc07dbca0,
bo_bsize = 16384,
bo_object = 0x0,
bo_synclist = {
le_next = 0x0,
le_prev = 0x0
},
bo_private = 0xc4090000,
__bo_vnode = 0xc4090000
},
v_pollinfo = 0x0,
v_label = 0x0
}

please advise what to do next time this happens.

regards, clemens

_______________________________________________
freebsd-current@freebsd.org mailing list
http://lists.freebsd.org/mailman/lis...reebsd-current
To unsubscribe, send any mail to "freebsd-current-unsubscribe@freebsd.org"