--- Revision None +++ Revision 326466346437 @@ -0,0 +1,523 @@ +[root@localhost ~]# more /data/core.beam.smp.7965.log +Basic crash dump analysis of /data/core.beam.smp.7965. + +Please send the file to support@couchbase.com + +-------------------------------------------------------------------------------- +File information: +-rwxr-xr-x. 1 bin bin 12181134 Oct 10 15:10 /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp +2d761bc628b84d9891c5c0e36875aaa3 /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp +-rw-------. 1 couchbase couchbase 321896448 Oct 11 00:00 /data/core.beam.smp.7965 +0c5a52fdde860c8e1222ba8c474ffccb /data/core.beam.smp.7965 +1.4.4_209_g7b9e75f +version1.4.4_598_g8587311 +version1.4.4_598_g8587311 +-------------------------------------------------------------------------------- +Core file callstacks: +GNU gdb (GDB) Red Hat Enterprise Linux (7.2-56.el6) +Copyright (C) 2010 Free Software Foundation, Inc. +License GPLv3+: GNU GPL version 3 or later +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. Type "show copying" +and "show warranty" for details. +This GDB was configured as "x86_64-redhat-linux-gnu". +For bug reporting instructions, please see: +... +Reading symbols from /opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp...done. +[New Thread 7969] +[New Thread 7971] +[New Thread 7974] +[New Thread 7980] +[New Thread 7972] +[New Thread 7987] +[New Thread 7981] +[New Thread 7989] +[New Thread 7970] +[New Thread 7973] +[New Thread 7982] +[New Thread 7978] +[New Thread 7975] +[New Thread 7979] +[New Thread 27725] +[New Thread 7983] +[New Thread 7976] +[New Thread 7967] +[New Thread 7977] +[New Thread 7968] +[New Thread 27722] +[New Thread 7985] +[New Thread 7986] +[New Thread 7965] +[New Thread 7984] +[New Thread 7966] +[New Thread 7995] +[New Thread 7988] +Reading symbols from /lib64/libutil.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libutil.so.1 +Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/libdl.so.2 +Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done. +Loaded symbols for /lib64/libm.so.6 +Reading symbols from /lib64/libncurses.so.5...(no debugging symbols found)...done. +Loaded symbols for /lib64/libncurses.so.5 +Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done. +[Thread debugging using libthread_db enabled] +Loaded symbols for /lib64/libpthread.so.0 +Reading symbols from /lib64/librt.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/librt.so.1 +Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done. +Loaded symbols for /lib64/libc.so.6 +Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/ld-linux-x86-64.so.2 +Reading symbols from /lib64/libtinfo.so.5...(no debugging symbols found)...done. +Loaded symbols for /lib64/libtinfo.so.5 +Reading symbols from /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so...done. +Loaded symbols for /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so +Reading symbols from /usr/lib64/libcrypto.so.6...(no debugging symbols found)...done. +Loaded symbols for /usr/lib64/libcrypto.so.6 +Reading symbols from /usr/lib64/libssl.so.6...(no debugging symbols found)...done. +Loaded symbols for /usr/lib64/libssl.so.6 +Reading symbols from /lib64/libz.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libz.so.1 +Reading symbols from /lib64/libgssapi_krb5.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/libgssapi_krb5.so.2 +Reading symbols from /lib64/libkrb5.so.3...(no debugging symbols found)...done. +Loaded symbols for /lib64/libkrb5.so.3 +Reading symbols from /lib64/libcom_err.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/libcom_err.so.2 +Reading symbols from /lib64/libk5crypto.so.3...(no debugging symbols found)...done. +Loaded symbols for /lib64/libk5crypto.so.3 +Reading symbols from /lib64/libkrb5support.so.0...(no debugging symbols found)...done. +Loaded symbols for /lib64/libkrb5support.so.0 +Reading symbols from /lib64/libkeyutils.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libkeyutils.so.1 +Reading symbols from /lib64/libresolv.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/libresolv.so.2 +Reading symbols from /lib64/libselinux.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libselinux.so.1 +Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_icu_driver.so...done. +Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_icu_driver.so +Reading symbols from /opt/couchbase/lib/libv8.so...done. +Loaded symbols for /opt/couchbase/lib/libv8.so +Reading symbols from /opt/couchbase/lib/libicuuc.so.44...done. +Loaded symbols for /opt/couchbase/lib/libicuuc.so.44 +Reading symbols from /opt/couchbase/lib/libicudata.so.44...(no debugging symbols found)...done. +Loaded symbols for /opt/couchbase/lib/libicudata.so.44 +Reading symbols from /opt/couchbase/lib/libicui18n.so.44...done. +Loaded symbols for /opt/couchbase/lib/libicui18n.so.44 +Reading symbols from /lib64/libcrypt.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libcrypt.so.1 +Reading symbols from /usr/lib64/libstdc++.so.6...(no debugging symbols found)...done. +Loaded symbols for /usr/lib64/libstdc++.so.6 +Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libgcc_s.so.1 +Reading symbols from /lib64/libfreebl3.so...(no debugging symbols found)...done. +Loaded symbols for /lib64/libfreebl3.so +Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so...done. +Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so +Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so...done. +Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so +Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so...done. +Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so +Reading symbols from /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_ejson_compare.so...done. +Loaded symbols for /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_ejson_compare.so +Core was generated by `/opt/couchbase/lib/erlang/erts-5.8.5/bin/beam.smp -A 16 -sbt u -P 327680 -K tru'. +Program terminated with signal 11, Segmentation fault. +#0 0x00007fe075b6d1ab in memcpy () from /lib64/libc.so.6 + +Thread 28 (Thread 0x7fe072143700 (LWP 7988)): +#0 mseg_dealloc (atype=11, seg=0x7fe01e7b1000, size=262144, opt=0x2207f50) at sys/common/erl_mseg.c:765 +#1 0x000000000054ab21 in erts_mseg_dealloc_opt (atype=11, seg=0x7fe01e7b1000, size=262144, opt=0x2207f50) at sys/common/erl_mseg.c:1384 +#2 0x0000000000436d39 in alcu_mseg_dealloc (allctr=0x2207ec0, blk=) at beam/erl_alloc_util.c:517 +#3 destroy_carrier (allctr=0x2207ec0, blk=) at beam/erl_alloc_util.c:1796 +#4 0x000000000043833f in do_erts_alcu_free (type=, unused=, p=0x7fe01e7b1030) at beam/erl_alloc_util.c:2898 +#5 erts_alcu_free_thr_pref (type=, unused=, p=0x7fe01e7b1030) at beam/erl_alloc_util.c:2949 +#6 0x000000000053cde2 in destroy (s=0x2219b80) at drivers/common/gzio.c:442 +#7 0x000000000054e325 in file_stop (e=) at drivers/common/efile_drv.c:698 +#8 0x000000000047fef3 in terminate_port (prt=0x7fe073d42228) at beam/io.c:1849 +#9 0x0000000000482908 in erts_do_exit_port (p=0x7fe073d42228, from=, reason=) at beam/io.c:2059 +#10 0x0000000000497143 in doit_exit_link (lnk=0x7fe0178bed20, vpcontext=) at beam/erl_process.c:7256 +#11 0x00000000004fab87 in erts_sweep_links (root=, doit=0x496e90 , context=0x7fe072142bd0) at beam/erl_monitors.c:859 +#12 0x0000000000498a05 in continue_exit_process (p=0x7fe0178be9e0, pix_lock=) at beam/erl_process.c:7646 +#13 0x000000000051db94 in terminate_proc (c_p=0x7fe0178be9e0, pc=, reg=, bf=) at beam/beam_emu.c:5633 +#14 handle_error (c_p=0x7fe0178be9e0, pc=, reg=, bf=) at beam/beam_emu.c:5508 +#15 0x0000000000520cdf in process_main () at beam/beam_emu.c:3273 +#16 0x000000000048d496 in sched_thread_func (vesdp=0x7fe074c4b140) at beam/erl_process.c:3782 +#17 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b50) at pthread/ethread.c:106 +#18 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#19 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 27 (Thread 0x7fe01f3ac700 (LWP 7995)): +#0 0x00007fe07608dd2d in nanosleep () from /lib64/libpthread.so.0 +#1 0x00007fe01f3b13da in terminatorLoop (args=) at mapreduce_nif.cc:450 +#2 0x0000000000584d89 in thr_wrapper (vtwd=0x7fe072b43ae0) at pthread/ethread.c:106 +#3 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#4 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 26 (Thread 0x7fe0756c1700 (LWP 7966)): +#0 0x00007fe07608d54d in read () from /lib64/libpthread.so.0 +#1 0x000000000053810e in signal_dispatcher_thread_func (unused=) at sys/unix/sys.c:2906 +#2 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12a70) at pthread/ethread.c:106 +#3 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#4 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 25 (Thread 0x7fe0735ad700 (LWP 7984)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c750) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c750) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c750) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 24 (Thread 0x7fe076d5e700 (LWP 7965)): +#0 0x00007fe075bc4d03 in select () from /lib64/libc.so.6 +#1 0x0000000000537ba0 in erts_sys_main_thread () at sys/unix/sys.c:3049 +#2 0x0000000000444761 in erl_start (argc=54, argv=) at beam/erl_init.c:1441 +#3 0x0000000000426219 in main (argc=, argv=) at sys/unix/erl_main.c:29 + +Thread 23 (Thread 0x7fe073545700 (LWP 7986)): +#0 0x00007fe0760870ad in pthread_join () from /lib64/libpthread.so.0 +#1 0x00007fe021c85887 in v8::internal::ContextSwitcher::StopPreemption () at src/v8threads.cc:445 +#2 0x00007fe01f3b38b3 in destroyContext (ctx=0x7fe013199030) at mapreduce.cc:345 +#3 0x00000000005138fb in nif_resource_dtor (bin=) at beam/erl_nif.c:1217 +#4 0x000000000048904b in erts_bin_free (offheap=) at beam/erl_binary.h:303 +#5 erts_cleanup_offheap (offheap=) at beam/erl_message.c:174 +#6 0x0000000000498a81 in delete_process (p=0x7fe0178b36a0, pix_lock=) at beam/erl_process.c:6621 +#7 continue_exit_process (p=0x7fe0178b36a0, pix_lock=) at beam/erl_process.c:7663 +#8 0x000000000051db94 in terminate_proc (c_p=0x7fe0178b36a0, pc=, reg=, bf=) at beam/beam_emu.c:5633 +#9 handle_error (c_p=0x7fe0178b36a0, pc=, reg=, bf=) at beam/beam_emu.c:5508 +#10 0x0000000000520cdf in process_main () at beam/beam_emu.c:3273 +#11 0x000000000048d496 in sched_thread_func (vesdp=0x7fe074c2efc0) at beam/erl_process.c:3782 +#12 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b50) at pthread/ethread.c:106 +#13 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#14 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 22 (Thread 0x7fe07358b700 (LWP 7985)): +#0 0x00007fe07608e09d in waitpid () from /lib64/libpthread.so.0 +#1 0x000000000053906f in child_waiter (unused=) at sys/unix/sys.c:2821 +#2 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b30) at pthread/ethread.c:106 +#3 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#4 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 21 (Thread 0x7fe076b79700 (LWP 27722)): +#0 0x00007fe07608c720 in sem_wait () from /lib64/libpthread.so.0 +#1 0x00007fe021c0518d in v8::internal::LinuxSemaphore::Wait (this=0x7fe06c109b20) at src/platform-linux.cc:869 +#2 0x00007fe021c1c45c in v8::internal::RuntimeProfiler::WaitForSomeIsolateToEnterJS () at src/runtime-profiler.cc:386 +#3 0x00007fe021c06136 in v8::internal::SignalSender::Run (this=0x7fe028039720) at src/platform-linux.cc:1126 +#4 0x00007fe021c05580 in v8::internal::ThreadEntry (arg=0x7fe028039720) at src/platform-linux.cc:745 +#5 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#6 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 20 (Thread 0x7fe0749ad700 (LWP 7968)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x0000000000462ad9 in erts_cnd_wait (unused=) at beam/erl_threads.h:697 +#3 erts_smp_cnd_wait (unused=) at beam/erl_smp.h:472 +#4 sys_msg_dispatcher_func (unused=) at beam/erl_trace.c:3238 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b80) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 19 (Thread 0x7fe07369b700 (LWP 7977)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c408) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c408) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c408) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 18 (Thread 0x7fe074cc0700 (LWP 7967)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x000000000046a1cf in erts_cnd_wait (unused=) at beam/erl_threads.h:697 +#3 erts_smp_cnd_wait (unused=) at beam/erl_smp.h:472 +#4 emergency_watchdog (unused=) at beam/utils.c:3945 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b80) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 17 (Thread 0x7fe0736bd700 (LWP 7976)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c390) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c390) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c390) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 16 (Thread 0x7fe0735cf700 (LWP 7983)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c6d8) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c6d8) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c6d8) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 15 (Thread 0x7fe014d1e700 (LWP 27725)): +#0 0x00007fe075b903bd in nanosleep () from /lib64/libc.so.6 +#1 0x00007fe075bc5574 in usleep () from /lib64/libc.so.6 +#2 0x00007fe021c85848 in v8::internal::ContextSwitcher::Run (this=0x7fe024134460) at src/v8threads.cc:457 +#3 0x00007fe021c05580 in v8::internal::ThreadEntry (arg=0x7fe024134460) at src/platform-linux.cc:745 +#4 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#5 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 14 (Thread 0x7fe073657700 (LWP 7979)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c4f8) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c4f8) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c4f8) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 13 (Thread 0x7fe0736df700 (LWP 7975)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c318) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c318) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c318) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 12 (Thread 0x7fe073679700 (LWP 7978)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c480) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c480) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c480) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 11 (Thread 0x7fe0735f1700 (LWP 7982)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c660) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c660) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c660) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 10 (Thread 0x7fe073723700 (LWP 7973)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c228) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c228) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c228) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 9 (Thread 0x7fe073789700 (LWP 7970)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c0c0) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c0c0) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c0c0) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 8 (Thread 0x7fe071742700 (LWP 7989)): +#0 0x00007fe07608d1e5 in __lll_unlock_wake () from /lib64/libpthread.so.0 +#1 0x00007fe0760896f7 in _L_unlock_657 () from /lib64/libpthread.so.0 +#2 0x00007fe07608965f in pthread_mutex_unlock () from /lib64/libpthread.so.0 +#3 0x000000000049ceb5 in ethr_event_set (p=, trnsfr_lcks=, pix_lock=0x86fb40, unlock=1) at ../include/internal/pthread/ethr_event.h:111 +#4 erts_tse_set (p=, trnsfr_lcks=, pix_lock=0x86fb40, unlock=1) at beam/erl_threads.h:1795 +#5 transfer_locks (p=, trnsfr_lcks=, pix_lock=0x86fb40, unlock=1) at beam/erl_process_lock.c:417 +#6 0x0000000000497527 in doit_exit_monitor (mon=0x7fe01789c4b8, vpcontext=0x7fe071741c10) at beam/erl_process.c:7192 +#7 0x00000000004faad7 in erts_sweep_monitors (root=, doit=0x497280 , context=0x7fe071741c10) at beam/erl_monitors.c:823 +#8 0x0000000000498a23 in continue_exit_process (p=0x7fe01802d670, pix_lock=) at beam/erl_process.c:7653 +#9 0x000000000051db94 in terminate_proc (c_p=0x7fe01802d670, pc=, reg=, bf=) at beam/beam_emu.c:5633 +#10 handle_error (c_p=0x7fe01802d670, pc=, reg=, bf=) at beam/beam_emu.c:5508 +#11 0x0000000000520cdf in process_main () at beam/beam_emu.c:3273 +#12 0x000000000048d496 in sched_thread_func (vesdp=0x7fe074c59200) at beam/erl_process.c:3782 +#13 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b50) at pthread/ethread.c:106 +#14 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#15 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 7 (Thread 0x7fe073613700 (LWP 7981)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c5e8) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c5e8) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c5e8) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 6 (Thread 0x7fe072b44700 (LWP 7987)): +#0 0x0000000000488feb in erts_bin_free (offheap=) at beam/erl_binary.h:302 +#1 erts_cleanup_offheap (offheap=) at beam/erl_message.c:174 +#2 0x0000000000498a81 in delete_process (p=0x7fe0178b7c90, pix_lock=) at beam/erl_process.c:6621 +#3 continue_exit_process (p=0x7fe0178b7c90, pix_lock=) at beam/erl_process.c:7663 +#4 0x000000000051db94 in terminate_proc (c_p=0x7fe0178b7c90, pc=, reg=, bf=) at beam/beam_emu.c:5633 +#5 handle_error (c_p=0x7fe0178b7c90, pc=, reg=, bf=) at beam/beam_emu.c:5508 +#6 0x0000000000520cdf in process_main () at beam/beam_emu.c:3273 +#7 0x000000000048d496 in sched_thread_func (vesdp=0x7fe074c3d080) at beam/erl_process.c:3782 +#8 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b50) at pthread/ethread.c:106 +#9 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#10 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 5 (Thread 0x7fe073745700 (LWP 7972)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c1b0) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c1b0) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c1b0) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 4 (Thread 0x7fe073635700 (LWP 7980)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c570) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c570) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c570) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 3 (Thread 0x7fe073701700 (LWP 7974)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c2a0) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c2a0) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c2a0) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 2 (Thread 0x7fe073767700 (LWP 7971)): +#0 0x00007fe07608a43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x0000000000582e29 in ethr_cond_wait (cnd=, mtx=) at common/ethr_mutex.c:1350 +#2 0x00000000004e8efb in erts_cnd_wait (arg=0x7fe073f1c138) at beam/erl_threads.h:697 +#3 async_get (arg=0x7fe073f1c138) at beam/erl_async.c:207 +#4 async_main (arg=0x7fe073f1c138) at beam/erl_async.c:274 +#5 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#6 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 + +Thread 1 (Thread 0x7fe0737ab700 (LWP 7969)): +#0 0x00007fe075b6d1ab in memcpy () from /lib64/libc.so.6 +#1 0x000000000053c7fe in erts_gzread (file=0x2225af8, buf=, len=16384) at drivers/common/gzio.c:473 +#2 0x000000000054e208 in invoke_read (data=0x2221ff0) at drivers/common/efile_drv.c:966 +#3 0x00000000004e8f55 in async_main (arg=0x7fe073f1c048) at beam/erl_async.c:281 +#4 0x0000000000584d89 in thr_wrapper (vtwd=0x7fff86e12b10) at pthread/ethread.c:106 +#5 0x00007fe076086851 in start_thread () from /lib64/libpthread.so.0 +#6 0x00007fe075bcc11d in clone () from /lib64/libc.so.6 +-------------------------------------------------------------------------------- +Module information: +/lib64/libutil.so.1: +lrwxrwxrwx. 1 root root 15 Oct 5 13:43 /lib64/libutil.so.1 -> libutil-2.12.so +aae98c3065bf167a8e867e830d1c8817 /lib64/libutil.so.1 +/lib64/libdl.so.2: +lrwxrwxrwx. 1 root root 13 Oct 5 13:43 /lib64/libdl.so.2 -> libdl-2.12.so +88e56138964e07d30dfc44e0287a0f69 /lib64/libdl.so.2 +/lib64/libm.so.6: +lrwxrwxrwx. 1 root root 12 Oct 5 13:43 /lib64/libm.so.6 -> libm-2.12.so +e9a818815675dbfd19b291272de00f44 /lib64/libm.so.6 +/lib64/libncurses.so.5: +lrwxrwxrwx. 1 root root 17 Sep 20 13:34 /lib64/libncurses.so.5 -> libncurses.so.5.7 +c7f1fe3252dd7381ab04c84d8889f352 /lib64/libncurses.so.5 +/lib64/libpthread.so.0: +lrwxrwxrwx. 1 root root 18 Oct 5 13:43 /lib64/libpthread.so.0 -> libpthread-2.12.so +fa8af7a3f05f54ac461c40ca500147b7 /lib64/libpthread.so.0 +/lib64/librt.so.1: +lrwxrwxrwx. 1 root root 13 Oct 5 13:43 /lib64/librt.so.1 -> librt-2.12.so +f2cedf89cd9f49580afdb9f91b434b97 /lib64/librt.so.1 +/lib64/libc.so.6: +lrwxrwxrwx. 1 root root 12 Oct 5 13:43 /lib64/libc.so.6 -> libc-2.12.so +8ec284d3097552c0d5e9630b279cfcc4 /lib64/libc.so.6 +/lib64/ld-linux-x86-64.so.2: +lrwxrwxrwx. 1 root root 10 Oct 5 13:43 /lib64/ld-linux-x86-64.so.2 -> ld-2.12.so +ee467ccc70564df4de2d2e0eac104fa5 /lib64/ld-linux-x86-64.so.2 +/lib64/libtinfo.so.5: +lrwxrwxrwx. 1 root root 15 Sep 20 13:34 /lib64/libtinfo.so.5 -> libtinfo.so.5.7 +183199f39b8490ee65db7f12ca553db3 /lib64/libtinfo.so.5 +/opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so: +-rwxr-xr-x. 1 bin bin 105367 Oct 10 15:10 /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so +6e54a6b6dcaa9d8ad139be11e0d4b6a4 /opt/couchbase/lib/erlang/lib/crypto-2.0.4/priv/lib/crypto.so +/usr/lib64/libcrypto.so.6: +lrwxrwxrwx. 1 root root 19 Sep 20 14:51 /usr/lib64/libcrypto.so.6 -> libcrypto.so.0.9.8e +78dc020d4c5906ef8d38fa8772745aec /usr/lib64/libcrypto.so.6 +/usr/lib64/libssl.so.6: +lrwxrwxrwx. 1 root root 16 Sep 20 14:51 /usr/lib64/libssl.so.6 -> libssl.so.0.9.8e +a02f2de91ba1e75330034bd0d52ca968 /usr/lib64/libssl.so.6 +/lib64/libz.so.1: +lrwxrwxrwx. 1 root root 13 Sep 20 13:34 /lib64/libz.so.1 -> libz.so.1.2.3 +2437e7ab120ea1ad0b8496301dfd08f1 /lib64/libz.so.1 +/lib64/libgssapi_krb5.so.2: +lrwxrwxrwx. 1 root root 21 Sep 20 13:34 /lib64/libgssapi_krb5.so.2 -> libgssapi_krb5.so.2.2 +4a07896ad5cf7e42559312cc6a645f90 /lib64/libgssapi_krb5.so.2 +/lib64/libkrb5.so.3: +lrwxrwxrwx. 1 root root 14 Sep 20 13:34 /lib64/libkrb5.so.3 -> libkrb5.so.3.3 +bd4bc41781b2ad8160f2ef6f1c6b69c7 /lib64/libkrb5.so.3 +/lib64/libcom_err.so.2: +lrwxrwxrwx. 1 root root 17 Sep 20 13:34 /lib64/libcom_err.so.2 -> libcom_err.so.2.1 +ec4544049cb88f5284f7ba57fe22087e /lib64/libcom_err.so.2 +/lib64/libk5crypto.so.3: +lrwxrwxrwx. 1 root root 18 Sep 20 13:34 /lib64/libk5crypto.so.3 -> libk5crypto.so.3.1 +656a9cc782142aeab1f0f659b3d39734 /lib64/libk5crypto.so.3 +/lib64/libkrb5support.so.0: +lrwxrwxrwx. 1 root root 21 Sep 20 13:34 /lib64/libkrb5support.so.0 -> libkrb5support.so.0.1 +7149fe73335280ee240e5b66920ce013 /lib64/libkrb5support.so.0 +/lib64/libkeyutils.so.1: +lrwxrwxrwx. 1 root root 18 Sep 20 13:34 /lib64/libkeyutils.so.1 -> libkeyutils.so.1.3 +8002611a776e9de0ccf7832721b86d95 /lib64/libkeyutils.so.1 +/lib64/libresolv.so.2: +lrwxrwxrwx. 1 root root 17 Oct 5 13:43 /lib64/libresolv.so.2 -> libresolv-2.12.so +80455b98f518df830373b2db62c4e19c /lib64/libresolv.so.2 +/lib64/libselinux.so.1: +-rwxr-xr-x. 1 root root 122040 Dec 7 2011 /lib64/libselinux.so.1 +d498609d23563a027254e14d299375ab /lib64/libselinux.so.1 +/opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_icu_driver.so: +-rwxr-xr-x. 1 bin bin 191090 Oct 10 15:19 /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_icu_driver.so +d558f5aa32038379008c1d9725c68397 /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_icu_driver.so +/opt/couchbase/lib/libv8.so: +-rwxr-xr-x. 1 bin bin 85345184 Oct 10 15:14 /opt/couchbase/lib/libv8.so +b6c7f7cd39d687a268b6b7c5272d36cd /opt/couchbase/lib/libv8.so +/opt/couchbase/lib/libicuuc.so.44: +lrwxrwxrwx. 1 bin bin 16 Oct 10 15:52 /opt/couchbase/lib/libicuuc.so.44 -> libicuuc.so.44.0 +42f69b9a8effcfd45a1c70aea9dcb66a /opt/couchbase/lib/libicuuc.so.44 +/opt/couchbase/lib/libicudata.so.44: +lrwxrwxrwx. 1 bin bin 18 Oct 10 15:52 /opt/couchbase/lib/libicudata.so.44 -> libicudata.so.44.0 +797015967c9d62eb4ce64e861e45f500 /opt/couchbase/lib/libicudata.so.44 +/opt/couchbase/lib/libicui18n.so.44: +lrwxrwxrwx. 1 bin bin 18 Oct 10 15:52 /opt/couchbase/lib/libicui18n.so.44 -> libicui18n.so.44.0 +906d4788c04365237419e085f34cc982 /opt/couchbase/lib/libicui18n.so.44 +/lib64/libcrypt.so.1: +lrwxrwxrwx. 1 root root 16 Oct 5 13:43 /lib64/libcrypt.so.1 -> libcrypt-2.12.so +8100ffde3f3465c0086c61e50ed06c14 /lib64/libcrypt.so.1 +/usr/lib64/libstdc++.so.6: +lrwxrwxrwx. 1 root root 19 Sep 20 13:34 /usr/lib64/libstdc++.so.6 -> libstdc++.so.6.0.13 +5b6a438cd9d564af737b1d3f14271ab3 /usr/lib64/libstdc++.so.6 +/lib64/libgcc_s.so.1: +lrwxrwxrwx. 1 root root 28 Oct 5 13:43 /lib64/libgcc_s.so.1 -> libgcc_s-4.4.6-20120305.so.1 +fa27e0c9fbddbeece6d909b9f2c088e3 /lib64/libgcc_s.so.1 +/lib64/libfreebl3.so: +-rwxr-xr-x. 1 root root 383504 Dec 7 2011 /lib64/libfreebl3.so +9fe8811e93e403a2681f99156170b5f5 /lib64/libfreebl3.so +/opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so: +-rwxr-xr-x. 1 bin bin 411836 Oct 10 15:20 /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so +1b715bac077be8bf2e638d3c1cb26061 /opt/couchbase/lib/couchdb/erlang/lib/snappy-1.0.4/priv/snappy_nif.so +/opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so: +-rwxr-xr-x. 1 bin bin 673632 Oct 10 15:19 /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so +5af42359c199dd26548c0383e70d4933 /opt/couchbase/lib/couchdb/erlang/lib/mapreduce-1.0/priv/mapreduce_nif.so +/opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so: +-rwxr-xr-x. 1 bin bin 344520 Oct 10 15:20 /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so +5c4a39422733f2cefa699dd14bc80500 /opt/couchbase/lib/couchdb/erlang/lib/ejson-0.1.0/priv/ejson.so +/opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_ejson_compare.so: +-rwxr-xr-x. 1 bin bin 577011 Oct 10 15:19 /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_ejson_compare.so +0ed1f41631963e85232351d1ae3d9852 /opt/couchbase/lib/couchdb/erlang/lib/couch-1.2.0a-091f7f4-git/priv/lib/couch_ejson_compare.so