--- Revision None +++ Revision 643230653934 @@ -0,0 +1,533 @@ +[root@localhost ~]# more /data/core.beam.smp.16438.log +Basic crash dump analysis of /data/core.beam.smp.16438. + +Please send the file to support@couchbase.com + +-------------------------------------------------------------------------------- +File information: +-rwxr-xr-x. 1 bin bin 12181270 Oct 11 15:45 /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp +f76dbd0c5dddd3731f18ee215769087d /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp +-rw-------. 1 couchbase couchbase 453914624 Oct 11 23:42 /data/core.beam.smp.16438 +06894bec152c3eb7f2407bb595d8c135 /data/core.beam.smp.16438 +version1.4.4_598_g8587311 +version1.4.4_598_g8587311 +1.4.4_209_g7b9e75f +version1.4.4_598_g8587311 +version1.4.4_598_g8587311 +-------------------------------------------------------------------------------- +Core file callstacks: +GNU gdb (GDB) Red Hat Enterprise Linux (7.2-56.el6) +Copyright (C) 2010 Free Software Foundation, Inc. +License GPLv3+: GNU GPL version 3 or later +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. Type "show copying" +and "show warranty" for details. +This GDB was configured as "x86_64-redhat-linux-gnu". +For bug reporting instructions, please see: +... +Reading symbols from /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp...done. +[New Thread 16460] +[New Thread 5467] +[New Thread 5561] +[New Thread 16468] +[New Thread 16438] +[New Thread 16442] +[New Thread 5559] +[New Thread 16456] +[New Thread 16443] +[New Thread 16439] +[New Thread 16457] +[New Thread 16444] +[New Thread 16453] +[New Thread 16440] +[New Thread 16462] +[New Thread 16461] +[New Thread 16446] +[New Thread 16450] +[New Thread 16451] +[New Thread 16452] +[New Thread 16454] +[New Thread 16458] +[New Thread 16455] +[New Thread 16449] +[New Thread 16459] +[New Thread 16448] +[New Thread 16447] +[New Thread 16445] +[New Thread 16441] +Reading symbols from /lib64/libutil.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libutil.so.1 +Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/libdl.so.2 +Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done. +Loaded symbols for /lib64/libm.so.6 +Reading symbols from /lib64/libncurses.so.5...(no debugging symbols found)...done. +Loaded symbols for /lib64/libncurses.so.5 +Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done. +[Thread debugging using libthread_db enabled] +Loaded symbols for /lib64/libpthread.so.0 +Reading symbols from /lib64/librt.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/librt.so.1 +Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done. +Loaded symbols for /lib64/libc.so.6 +Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/ld-linux-x86-64.so.2 +Reading symbols from /lib64/libtinfo.so.5...(no debugging symbols found)...done. +Loaded symbols for /lib64/libtinfo.so.5 +Reading symbols from /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so...done. +Loaded symbols for /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so +Reading symbols from /usr/lib64/libcrypto.so.6...(no debugging symbols found)...done. +Loaded symbols for /usr/lib64/libcrypto.so.6 +Reading symbols from /usr/lib64/libssl.so.6...(no debugging symbols found)...done. +Loaded symbols for /usr/lib64/libssl.so.6 +Reading symbols from /lib64/libz.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libz.so.1 +Reading symbols from /lib64/libgssapi_krb5.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/libgssapi_krb5.so.2 +Reading symbols from /lib64/libkrb5.so.3...(no debugging symbols found)...done. +Loaded symbols for /lib64/libkrb5.so.3 +Reading symbols from /lib64/libcom_err.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/libcom_err.so.2 +Reading symbols from /lib64/libk5crypto.so.3...(no debugging symbols found)...done. +Loaded symbols for /lib64/libk5crypto.so.3 +Reading symbols from /lib64/libkrb5support.so.0...(no debugging symbols found)...done. +Loaded symbols for /lib64/libkrb5support.so.0 +Reading symbols from /lib64/libkeyutils.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libkeyutils.so.1 +Reading symbols from /lib64/libresolv.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/libresolv.so.2 +Reading symbols from /lib64/libselinux.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libselinux.so.1 +Reading symbols from /opt/couchbase/lib/libv8.so...done. +Loaded symbols for /opt/couchbase/lib/libv8.so +Reading symbols from /opt/couchbase/lib/libicuuc.so.44...done. +Loaded symbols for /opt/couchbase/lib/libicuuc.so.44 +Reading symbols from /opt/couchbase/lib/libicudata.so.44...(no debugging symbols found)...done. +Loaded symbols for /opt/couchbase/lib/libicudata.so.44 +Reading symbols from /opt/couchbase/lib/libicui18n.so.44...done. +Loaded symbols for /opt/couchbase/lib/libicui18n.so.44 +Reading symbols from /lib64/libcrypt.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libcrypt.so.1 +Reading symbols from /usr/lib64/libstdc++.so.6...(no debugging symbols found)...done. +Loaded symbols for /usr/lib64/libstdc++.so.6 +Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libgcc_s.so.1 +Reading symbols from /lib64/libfreebl3.so...(no debugging symbols found)...done. +Loaded symbols for /lib64/libfreebl3.so +Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so...done. +Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so +Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so...done. +Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so +Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so...done. +Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so +Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_ejson_compare.so...done. +Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_ejson_compare.so +Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/couch_view_parser-1.0/priv/couch_view_parser_nif.so...done. +Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/couch_view_parser-1.0/priv/couch_view_parser_nif.so +Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_icu_driver.so...done. +Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_icu_driver.so +Core was generated by `/opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp -A 16 -sbt u -P 327680 -K tru'. +Program terminated with signal 11, Segmentation fault. +#0 0x000000000043ec6b in tree_delete (allctr=, del=0x264ca48, flags=) at beam/erl_bestfit_alloc.c:435 +435 beam/erl_bestfit_alloc.c: No such file or directory. + in beam/erl_bestfit_alloc.c + +Thread 29 (Thread 0x7fd722101700 (LWP 16441)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x0000000000462ad9 in erts_cnd_wait (unused=) at beam/erl_threads.h:697 +#3 erts_smp_cnd_wait (unused=) at beam/erl_smp.h:472 +#4 sys_msg_dispatcher_func (unused=) at beam/erl_trace.c:3238 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc246320) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 28 (Thread 0x7fd720e99700 (LWP 16445)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd7216701b0) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd7216701b0) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd7216701b0) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 27 (Thread 0x7fd720e55700 (LWP 16447)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd7216702a0) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd7216702a0) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd7216702a0) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 26 (Thread 0x7fd720e33700 (LWP 16448)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd721670318) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd721670318) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd721670318) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 25 (Thread 0x7fd720c99700 (LWP 16459)): +#0 0x00007fd7237db0ad in pthread_join () from /lib64/libpthread.so.0 +#1 0x00007fd6cf5d6887 in v8::internal::ContextSwitcher::StopPreemption () at src/v8threads.cc:445 +#2 0x00007fd6ccd048b3 in destroyContext (ctx=0x2618518) at mapreduce.cc:345 +#3 0x00000000005138fb in nif_resource_dtor (bin=) at beam/erl_nif.c:1217 +#4 0x000000000048904b in erts_bin_free (offheap=) at beam/erl_binary.h:303 +#5 erts_cleanup_offheap (offheap=) at beam/erl_message.c:174 +#6 0x0000000000498a81 in delete_process (p=0x7fd6c74be460, pix_lock=) at beam/erl_process.c:6621 +#7 continue_exit_process (p=0x7fd6c74be460, pix_lock=) at beam/erl_process.c:7663 +#8 0x000000000051db94 in terminate_proc (c_p=0x7fd6c74be460, pc=, reg=, bf=) at beam/beam_emu.c:5633 +#9 handle_error (c_p=0x7fd6c74be460, pc=, reg=, bf=) at beam/beam_emu.c:5508 +#10 0x0000000000520cdf in process_main () at beam/beam_emu.c:3273 +#11 0x000000000048d496 in sched_thread_func (vesdp=0x7fd722382fc0) at beam/erl_process.c:3782 +#12 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462f0) at pthread/ethread.c:106 +#13 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#14 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 24 (Thread 0x7fd720e11700 (LWP 16449)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd721670390) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd721670390) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd721670390) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 23 (Thread 0x7fd720d45700 (LWP 16455)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd721670660) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd721670660) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd721670660) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 22 (Thread 0x7fd720cdf700 (LWP 16458)): +#0 0x00007fd7237e209d in waitpid () from /lib64/libpthread.so.0 +#1 0x000000000053906f in child_waiter (unused=) at sys/unix/sys.c:2821 +#2 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462d0) at pthread/ethread.c:106 +#3 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#4 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 21 (Thread 0x7fd720d67700 (LWP 16454)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd7216705e8) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd7216705e8) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd7216705e8) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 20 (Thread 0x7fd720dab700 (LWP 16452)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd7216704f8) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd7216704f8) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd7216704f8) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 19 (Thread 0x7fd720dcd700 (LWP 16451)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd721670480) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd721670480) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd721670480) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 18 (Thread 0x7fd720def700 (LWP 16450)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd721670408) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd721670408) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd721670408) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 17 (Thread 0x7fd720e77700 (LWP 16446)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd721670228) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd721670228) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd721670228) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 16 (Thread 0x7fd71f897700 (LWP 16461)): +#0 bf_link_free_block (allctr=, block=0x7fd6b64a2018, flags=) at beam/erl_bestfit_alloc.c:719 +#1 0x000000000043833f in do_erts_alcu_free (type=, unused=, p=0x7fd6b64a2028) at beam/erl_alloc_util.c:2898 +#2 erts_alcu_free_thr_pref (type=, unused=, p=0x7fd6b64a2028) at beam/erl_alloc_util.c:2949 +#3 0x0000000000489022 in erts_free (offheap=) at beam/erl_alloc.h:226 +#4 erts_bin_free (offheap=) at beam/erl_binary.h:307 +#5 erts_cleanup_offheap (offheap=) at beam/erl_message.c:174 +#6 0x0000000000498a81 in delete_process (p=0x7fd6cac33938, pix_lock=) at beam/erl_process.c:6621 +#7 continue_exit_process (p=0x7fd6cac33938, pix_lock=) at beam/erl_process.c:7663 +#8 0x000000000051db94 in terminate_proc (c_p=0x7fd6cac33938, pc=, reg=, bf=) at beam/beam_emu.c:5633 +#9 handle_error (c_p=0x7fd6cac33938, pc=, reg=, bf=) at beam/beam_emu.c:5508 +#10 0x0000000000520cdf in process_main () at beam/beam_emu.c:3273 +#11 0x000000000048d496 in sched_thread_func (vesdp=0x7fd72239f140) at beam/erl_process.c:3782 +#12 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462f0) at pthread/ethread.c:106 +#13 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#14 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 15 (Thread 0x7fd71ee96700 (LWP 16462)): +#0 process_main () at x86_64-unknown-linux-gnu/opt/smp/beam_hot.h:876 +#1 0x000000000048d496 in sched_thread_func (vesdp=0x7fd7223ad200) at beam/erl_process.c:3782 +#2 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462f0) at pthread/ethread.c:106 +#3 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#4 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 14 (Thread 0x7fd722414700 (LWP 16440)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x000000000046a1cf in erts_cnd_wait (unused=) at beam/erl_threads.h:697 +#3 erts_smp_cnd_wait (unused=) at beam/erl_smp.h:472 +#4 emergency_watchdog (unused=) at beam/utils.c:3945 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc246320) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 13 (Thread 0x7fd720d89700 (LWP 16453)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd721670570) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd721670570) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd721670570) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 12 (Thread 0x7fd720ebb700 (LWP 16444)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd721670138) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd721670138) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd721670138) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 11 (Thread 0x7fd720d01700 (LWP 16457)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd721670750) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd721670750) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd721670750) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 10 (Thread 0x7fd722e15700 (LWP 16439)): +#0 0x00007fd7237e154d in read () from /lib64/libpthread.so.0 +#1 0x000000000053810e in signal_dispatcher_thread_func (unused=) at sys/unix/sys.c:2906 +#2 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc246210) at pthread/ethread.c:106 +#3 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#4 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 9 (Thread 0x7fd720edd700 (LWP 16443)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd7216700c0) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd7216700c0) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd7216700c0) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 8 (Thread 0x7fd720d23700 (LWP 16456)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd7216706d8) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd7216706d8) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd7216706d8) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 7 (Thread 0x7fd6b26bc700 (LWP 5559)): +#0 0x00007fd7232e43bd in nanosleep () from /lib64/libc.so.6 +#1 0x00007fd723319574 in usleep () from /lib64/libc.so.6 +#2 0x00007fd6cf5d6848 in v8::internal::ContextSwitcher::Run (this=0x7fd6d40cd310) at src/v8threads.cc:457 +#3 0x00007fd6cf556580 in v8::internal::ThreadEntry (arg=0x7fd6d40cd310) at src/platform-linux.cc:745 +#4 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#5 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 6 (Thread 0x7fd720eff700 (LWP 16442)): +#0 0x00007fd7237de43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fd721670048) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fd721670048) at beam/erl_async.c:207 +#4 async_main (arg=0x7fd721670048) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462b0) at pthread/ethread.c:106 +#6 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 5 (Thread 0x7fd7244b2700 (LWP 16438)): +#0 0x00007fd723318d03 in select () from /lib64/libc.so.6 +#1 0x0000000000537ba0 in erts_sys_main_thread () at sys/unix/sys.c:3049 +#2 0x0000000000444761 in erl_start (argc=54, argv=) at beam/erl_init.c:1441 +#3 0x0000000000426219 in main (argc=, argv=) at sys/unix/erl_main.c:29 + +Thread 4 (Thread 0x7fd6cccfd700 (LWP 16468)): +#0 0x00007fd7237e1d2d in nanosleep () from /lib64/libpthread.so.0 +#1 0x00007fd6ccd023da in terminatorLoop (args=) at mapreduce_nif.cc:450 +#2 0x0000000000584d89 in thr_wrapper (vtwd=0x7fd71ee95ae0) at pthread/ethread.c:106 +#3 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#4 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 3 (Thread 0x7fd6c6070700 (LWP 5561)): +#0 0x00007fd7232e43bd in nanosleep () from /lib64/libc.so.6 +#1 0x00007fd723319574 in usleep () from /lib64/libc.so.6 +#2 0x00007fd6cf5d6848 in v8::internal::ContextSwitcher::Run (this=0x7fd6c00f7400) at src/v8threads.cc:457 +#3 0x00007fd6cf556580 in v8::internal::ThreadEntry (arg=0x7fd6c00f7400) at src/platform-linux.cc:745 +#4 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#5 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 2 (Thread 0x7fd71dc50700 (LWP 5467)): +#0 0x00007fd7237e0720 in sem_wait () from /lib64/libpthread.so.0 +#1 0x00007fd6cf55618d in v8::internal::LinuxSemaphore::Wait (this=0x7fd6d4009b10) at src/platform-linux.cc:869 +#2 0x00007fd6cf56d45c in v8::internal::RuntimeProfiler::WaitForSomeIsolateToEnterJS () at src/runtime-profiler.cc:386 +#3 0x00007fd6cf557136 in v8::internal::SignalSender::Run (this=0x7fd6d40726c0) at src/platform-linux.cc:1126 +#4 0x00007fd6cf556580 in v8::internal::ThreadEntry (arg=0x7fd6d40726c0) at src/platform-linux.cc:745 +#5 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#6 0x00007fd72332011d in clone () from /lib64/libc.so.6 + +Thread 1 (Thread 0x7fd720298700 (LWP 16460)): +#0 0x000000000043ec6b in tree_delete (allctr=, del=0x264ca48, flags=) at beam/erl_bestfit_alloc.c:435 +#1 0x0000000000437091 in mbc_free (allctr=0x26412c0, p=) at beam/erl_alloc_util.c:867 +#2 0x000000000043833f in do_erts_alcu_free (type=, unused=, p=0x264cb90) at beam/erl_alloc_util.c:2898 +#3 erts_alcu_free_thr_pref (type=, unused=, p=0x264cb90) at beam/erl_alloc_util.c:2949 +#4 0x000000000053cde2 in destroy (s=0x264c988) at drivers/common/gzio.c:442 +#5 0x000000000054e325 in file_stop (e=) at drivers/common/efile_drv.c:698 +#6 0x000000000047fef3 in terminate_port (prt=0x7fd72112dfd8) at beam/io.c:1849 +#7 0x0000000000482908 in erts_do_exit_port (p=0x7fd72112dfd8, from=, reason=) at beam/io.c:2059 +#8 0x0000000000497143 in doit_exit_link (lnk=0x7fd6cac19038, vpcontext=) at beam/erl_process.c:7256 +#9 0x00000000004fab87 in erts_sweep_links (root=, doit=0x496e90 , context=0x7fd720297bd0) at beam/erl_monitors.c:859 +#10 0x0000000000498a05 in continue_exit_process (p=0x7fd6cb835b58, pix_lock=) at beam/erl_process.c:7646 +#11 0x000000000051db94 in terminate_proc (c_p=0x7fd6cb835b58, pc=, reg=, bf=) at beam/beam_emu.c:5633 +#12 handle_error (c_p=0x7fd6cb835b58, pc=, reg=, bf=) at beam/beam_emu.c:5508 +#13 0x0000000000520cdf in process_main () at beam/beam_emu.c:3273 +#14 0x000000000048d496 in sched_thread_func (vesdp=0x7fd722391080) at beam/erl_process.c:3782 +#15 0x0000000000584d89 in thr_wrapper (vtwd=0x7fffcc2462f0) at pthread/ethread.c:106 +#16 0x00007fd7237da851 in start_thread () from /lib64/libpthread.so.0 +#17 0x00007fd72332011d in clone () from /lib64/libc.so.6 +-------------------------------------------------------------------------------- +Module information: +/lib64/libutil.so.1: +lrwxrwxrwx. 1 root root 15 Oct 5 13:44 /lib64/libutil.so.1 -> libutil-2.12.so +aae98c3065bf167a8e867e830d1c8817 /lib64/libutil.so.1 +/lib64/libdl.so.2: +lrwxrwxrwx. 1 root root 13 Oct 5 13:44 /lib64/libdl.so.2 -> libdl-2.12.so +88e56138964e07d30dfc44e0287a0f69 /lib64/libdl.so.2 +/lib64/libm.so.6: +lrwxrwxrwx. 1 root root 12 Oct 5 13:44 /lib64/libm.so.6 -> libm-2.12.so +e9a818815675dbfd19b291272de00f44 /lib64/libm.so.6 +/lib64/libncurses.so.5: +lrwxrwxrwx. 1 root root 17 Sep 19 17:48 /lib64/libncurses.so.5 -> libncurses.so.5.7 +c7f1fe3252dd7381ab04c84d8889f352 /lib64/libncurses.so.5 +/lib64/libpthread.so.0: +lrwxrwxrwx. 1 root root 18 Oct 5 13:44 /lib64/libpthread.so.0 -> libpthread-2.12.so +fa8af7a3f05f54ac461c40ca500147b7 /lib64/libpthread.so.0 +/lib64/librt.so.1: +lrwxrwxrwx. 1 root root 13 Oct 5 13:44 /lib64/librt.so.1 -> librt-2.12.so +f2cedf89cd9f49580afdb9f91b434b97 /lib64/librt.so.1 +/lib64/libc.so.6: +lrwxrwxrwx. 1 root root 12 Oct 5 13:44 /lib64/libc.so.6 -> libc-2.12.so +8ec284d3097552c0d5e9630b279cfcc4 /lib64/libc.so.6 +/lib64/ld-linux-x86-64.so.2: +lrwxrwxrwx. 1 root root 10 Oct 5 13:44 /lib64/ld-linux-x86-64.so.2 -> ld-2.12.so +ee467ccc70564df4de2d2e0eac104fa5 /lib64/ld-linux-x86-64.so.2 +/lib64/libtinfo.so.5: +lrwxrwxrwx. 1 root root 15 Sep 19 17:48 /lib64/libtinfo.so.5 -> libtinfo.so.5.7 +183199f39b8490ee65db7f12ca553db3 /lib64/libtinfo.so.5 +/opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so: +-rwxr-xr-x. 1 bin bin 105391 Oct 11 15:45 /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so +dc63afef083fc7b813a220cd78794144 /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so +/usr/lib64/libcrypto.so.6: +lrwxrwxrwx. 1 root root 19 Sep 19 19:34 /usr/lib64/libcrypto.so.6 -> libcrypto.so.0.9.8e +78dc020d4c5906ef8d38fa8772745aec /usr/lib64/libcrypto.so.6 +/usr/lib64/libssl.so.6: +lrwxrwxrwx. 1 root root 16 Sep 19 19:34 /usr/lib64/libssl.so.6 -> libssl.so.0.9.8e +a02f2de91ba1e75330034bd0d52ca968 /usr/lib64/libssl.so.6 +/lib64/libz.so.1: +lrwxrwxrwx. 1 root root 13 Sep 19 17:48 /lib64/libz.so.1 -> libz.so.1.2.3 +2437e7ab120ea1ad0b8496301dfd08f1 /lib64/libz.so.1 +/lib64/libgssapi_krb5.so.2: +lrwxrwxrwx. 1 root root 21 Sep 19 17:48 /lib64/libgssapi_krb5.so.2 -> libgssapi_krb5.so.2.2 +4a07896ad5cf7e42559312cc6a645f90 /lib64/libgssapi_krb5.so.2 +/lib64/libkrb5.so.3: +lrwxrwxrwx. 1 root root 14 Sep 19 17:48 /lib64/libkrb5.so.3 -> libkrb5.so.3.3 +bd4bc41781b2ad8160f2ef6f1c6b69c7 /lib64/libkrb5.so.3 +/lib64/libcom_err.so.2: +lrwxrwxrwx. 1 root root 17 Sep 19 17:48 /lib64/libcom_err.so.2 -> libcom_err.so.2.1 +ec4544049cb88f5284f7ba57fe22087e /lib64/libcom_err.so.2 +/lib64/libk5crypto.so.3: +lrwxrwxrwx. 1 root root 18 Sep 19 17:48 /lib64/libk5crypto.so.3 -> libk5crypto.so.3.1 +656a9cc782142aeab1f0f659b3d39734 /lib64/libk5crypto.so.3 +/lib64/libkrb5support.so.0: +lrwxrwxrwx. 1 root root 21 Sep 19 17:48 /lib64/libkrb5support.so.0 -> libkrb5support.so.0.1 +7149fe73335280ee240e5b66920ce013 /lib64/libkrb5support.so.0 +/lib64/libkeyutils.so.1: +lrwxrwxrwx. 1 root root 18 Sep 19 17:48 /lib64/libkeyutils.so.1 -> libkeyutils.so.1.3 +8002611a776e9de0ccf7832721b86d95 /lib64/libkeyutils.so.1 +/lib64/libresolv.so.2: +lrwxrwxrwx. 1 root root 17 Oct 5 13:44 /lib64/libresolv.so.2 -> libresolv-2.12.so +80455b98f518df830373b2db62c4e19c /lib64/libresolv.so.2 +/lib64/libselinux.so.1: +-rwxr-xr-x. 1 root root 122040 Dec 7 2011 /lib64/libselinux.so.1 +d498609d23563a027254e14d299375ab /lib64/libselinux.so.1 +/opt/couchbase/lib/libv8.so: +-rwxr-xr-x. 1 bin bin 85345192 Oct 11 15:50 /opt/couchbase/lib/libv8.so +e061e6d32762ade7a40b4a92af499947 /opt/couchbase/lib/libv8.so +/opt/couchbase/lib/libicuuc.so.44: +lrwxrwxrwx. 1 bin bin 16 Oct 11 16:56 /opt/couchbase/lib/libicuuc.so.44 -> libicuuc.so.44.0 +a0ed6347d68900c78789b5a22591d41c /opt/couchbase/lib/libicuuc.so.44 +/opt/couchbase/lib/libicudata.so.44: +lrwxrwxrwx. 1 bin bin 18 Oct 11 16:56 /opt/couchbase/lib/libicudata.so.44 -> libicudata.so.44.0 +797015967c9d62eb4ce64e861e45f500 /opt/couchbase/lib/libicudata.so.44 +/opt/couchbase/lib/libicui18n.so.44: +lrwxrwxrwx. 1 bin bin 18 Oct 11 16:56 /opt/couchbase/lib/libicui18n.so.44 -> libicui18n.so.44.0 +e237dad82e9fa1d5bcb44d6bb7ecff48 /opt/couchbase/lib/libicui18n.so.44 +/lib64/libcrypt.so.1: +lrwxrwxrwx. 1 root root 16 Oct 5 13:44 /lib64/libcrypt.so.1 -> libcrypt-2.12.so +8100ffde3f3465c0086c61e50ed06c14 /lib64/libcrypt.so.1 +/usr/lib64/libstdc++.so.6: +lrwxrwxrwx. 1 root root 19 Sep 19 17:48 /usr/lib64/libstdc++.so.6 -> libstdc++.so.6.0.13 +5b6a438cd9d564af737b1d3f14271ab3 /usr/lib64/libstdc++.so.6 +/lib64/libgcc_s.so.1: +lrwxrwxrwx. 1 root root 28 Oct 5 13:44 /lib64/libgcc_s.so.1 -> libgcc_s-4.4.6-20120305.so.1 +fa27e0c9fbddbeece6d909b9f2c088e3 /lib64/libgcc_s.so.1 +/lib64/libfreebl3.so: +-rwxr-xr-x. 1 root root 383504 Dec 7 2011 /lib64/libfreebl3.so +9fe8811e93e403a2681f99156170b5f5 /lib64/libfreebl3.so +/opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so: +-rwxr-xr-x. 1 bin bin 411908 Oct 11 15:57 /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so +b3f1040e670bef35c1fbff8822febb6f /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so +/opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so: +-rwxr-xr-x. 1 bin bin 673672 Oct 11 15:56 /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so +e92698fc2627b06207aab2f9e18b9203 /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so +/opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so: +-rwxr-xr-x. 1 bin bin 344616 Oct 11 15:56 /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so +9b36f5e9f8933bb01d7ebc4cd409befc /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so +/opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_ejson_compare.so: +-rwxr-xr-x. 1 bin bin 577043 Oct 11 15:56 /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_ejson_compare.so +ccf4f68c122506b83c4e85afcedef3c3 /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_ejson_compare.so +/opt/couchbase/lib/couchdb/erlang/lib/couch_view_parser-1.0/priv/couch_view_parser_nif.so: +-rwxr-xr-x. 1 bin bin 800853 Oct 11 15:57 /opt/couchbase/lib/couchdb/erlang/lib/couch_view_parser-1.0/priv/couch_view_parser_nif.so +41a4fff32b510e4867843ea91fa70268 /opt/couchbase/lib/couchdb/erlang/lib/couch_view_parser-1.0/priv/couch_view_parser_nif.so +/opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_icu_driver.so: +-rwxr-xr-x. 1 bin bin 191106 Oct 11 15:56 /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_icu_driver.so +d55e6c1c92a89c31d7575a4c5d3ed4ca /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_icu_driver.so