--- Revision None +++ Revision 353132386432 @@ -0,0 +1,267 @@ +GNU gdb (GDB) Red Hat Enterprise Linux (7.2-56.el6) +Copyright (C) 2010 Free Software Foundation, Inc. +License GPLv3+: GNU GPL version 3 or later +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. Type "show copying" +and "show warranty" for details. +This GDB was configured as "x86_64-redhat-linux-gnu". +For bug reporting instructions, please see: +... +Reading symbols from /opt/couchbase/bin/memcached...done. +[New Thread 9957] +[New Thread 9752] +[New Thread 9961] +[New Thread 9959] +[New Thread 9748] +[New Thread 9958] +[New Thread 9740] +[New Thread 9747] +[New Thread 9960] +[New Thread 9753] +[New Thread 9755] +[New Thread 9754] +[New Thread 9956] +[New Thread 9751] +Reading symbols from /opt/couchbase/lib/memcached/libmemcached_utilities.so.0...done. +Loaded symbols for /opt/couchbase/lib/memcached/libmemcached_utilities.so.0 +Reading symbols from /opt/couchbase/lib/libevent-2.0.so.5...done. +Loaded symbols for /opt/couchbase/lib/libevent-2.0.so.5 +Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/libdl.so.2 +Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done. +Loaded symbols for /lib64/libm.so.6 +Reading symbols from /lib64/librt.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/librt.so.1 +Reading symbols from /opt/couchbase/lib/libtcmalloc_minimal.so.4...done. +Loaded symbols for /opt/couchbase/lib/libtcmalloc_minimal.so.4 +Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done. +[Thread debugging using libthread_db enabled] +Loaded symbols for /lib64/libpthread.so.0 +Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done. +Loaded symbols for /lib64/libc.so.6 +Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/ld-linux-x86-64.so.2 +Reading symbols from /usr/lib64/libstdc++.so.6...(no debugging symbols found)...done. +Loaded symbols for /usr/lib64/libstdc++.so.6 +Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done. +Loaded symbols for /lib64/libgcc_s.so.1 +Reading symbols from /opt/couchbase/lib/memcached/stdin_term_handler.so...done. +Loaded symbols for /opt/couchbase/lib/memcached/stdin_term_handler.so +Reading symbols from /opt/couchbase/lib/memcached/file_logger.so...done. +Loaded symbols for /opt/couchbase/lib/memcached/file_logger.so +Reading symbols from /opt/couchbase/lib/memcached/bucket_engine.so...done. +Loaded symbols for /opt/couchbase/lib/memcached/bucket_engine.so +Reading symbols from /opt/couchbase/lib/memcached/ep.so...done. +Loaded symbols for /opt/couchbase/lib/memcached/ep.so +Reading symbols from /opt/couchbase/lib/libcouchstore.so.1...done. +Loaded symbols for /opt/couchbase/lib/libcouchstore.so.1 +Reading symbols from /opt/couchbase/lib/libsnappy.so.1...done. +Loaded symbols for /opt/couchbase/lib/libsnappy.so.1 +Reading symbols from /lib64/libnss_files.so.2...(no debugging symbols found)...done. +Loaded symbols for /lib64/libnss_files.so.2 +Core was generated by `/opt/couchbase/bin/memcached -X /opt/couchbase/lib/memcached/stdin_term_handler'. +Program terminated with signal 11, Segmentation fault. +#0 0x00007fb8600f3d03 in tcmalloc::CentralFreeList::FetchFromSpans() () + from /opt/couchbase/lib/libtcmalloc_minimal.so.4 +Missing separate debuginfos, use: debuginfo-install couchbase-server-2.0.0-1945.x86_64 +(gdb) t a a bt + +Thread 14 (Thread 0x7fb85dbe4700 (LWP 9751)): +#0 0x00007fb85fbad404 in memcpy () from /lib64/libc.so.6 +#1 0x00007fb85f8bb1e6 in std::basic_string, std::allocator >::_Rep::_M_clone(std::allocator const&, unsigned long) () from /usr/lib64/libstdc++.so.6 +#2 0x00007fb85f8bb534 in std::basic_string, std::allocator >::assign(std::basic_string, std::allocator > const&) () + from /usr/lib64/libstdc++.so.6 +#3 0x00007fb859d3eb5f in Dispatcher::getDispatcherState() () + from /opt/couchbase/lib/memcached/ep.so + +#4 0x00007fb859d31e98 in EventuallyPersistentEngine::doDispatcherStats (this=0x6cde000, + cookie=0x6c0f080, add_stat=0x40a5c0 ) at src/ep_engine.cc:3269 +#5 0x00007fb859d39f3e in EventuallyPersistentEngine::getStats (this=0x6cde000, cookie=0x6c0f080, + stat_key=, nkey=10, add_stat=0x40a5c0 ) + at src/ep_engine.cc:3332 +#6 0x00007fb859d3a656 in EvpGetStats (handle=, cookie=0x6c0f080, + stat_key=0x6c1f018 "dispatcherult", nkey=10, add_stat=) + at src/ep_engine.cc:191 +#7 0x00007fb85dbec910 in bucket_get_stats (handle=, cookie=0x6c0f080, + stat_key=, nkey=10, add_stat=0x40a5c0 ) + at bucket_engine.c:1720 +#8 0x0000000000410bb6 in process_bin_stat (c=0x6c0f080) at daemon/memcached.c:2199 +#9 0x00000000004114fa in complete_nread_binary (c=0x6c0f080) at daemon/memcached.c:3708 +---Type to continue, or q to quit--- +#10 complete_nread (c=0x6c0f080) at daemon/memcached.c:3820 +#11 conn_nread (c=0x6c0f080) at daemon/memcached.c:5673 +#12 0x0000000000405e55 in event_handler (fd=, which=, + arg=0x6c0f080) at daemon/memcached.c:5936 +#13 0x00007fb8609bbf3c in event_process_active_single_queue (base=0x6ce2500, + flags=) at event.c:1308 +#14 event_process_active (base=0x6ce2500, flags=) at event.c:1375 +#15 event_base_loop (base=0x6ce2500, flags=) at event.c:1572 +#16 0x0000000000414504 in worker_libevent (arg=0x247d900) at daemon/thread.c:301 +#17 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#18 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +Thread 13 (Thread 0x7fb85a9df700 (LWP 9956)): +#0 0x00007fb85fbd03bd in nanosleep () from /lib64/libc.so.6 +#1 0x00007fb85fc05574 in usleep () from /lib64/libc.so.6 +#2 0x00007fb859d4b255 in updateStatsThread (arg=0x247a4c0) at src/memory_tracker.cc:31 +#3 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#4 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +Thread 12 (Thread 0x7fb85bde1700 (LWP 9754)): +#0 0x00007fb85fc0c713 in epoll_wait () from /lib64/libc.so.6 +#1 0x00007fb8609d0576 in epoll_dispatch (base=0x6ce2a00, tv=) at epoll.c:404 +#2 0x00007fb8609bbe44 in event_base_loop (base=0x6ce2a00, flags=) +---Type to continue, or q to quit--- + at event.c:1558 +#3 0x0000000000414504 in worker_libevent (arg=0x247dbe8) at daemon/thread.c:301 +#4 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#5 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +Thread 11 (Thread 0x7fb85b3e0700 (LWP 9755)): +#0 0x00007fb85fc0c713 in epoll_wait () from /lib64/libc.so.6 +#1 0x00007fb8609d0576 in epoll_dispatch (base=0x6ce2780, tv=) at epoll.c:404 +#2 0x00007fb8609bbe44 in event_base_loop (base=0x6ce2780, flags=) + at event.c:1558 +#3 0x0000000000414504 in worker_libevent (arg=0x247dce0) at daemon/thread.c:301 +#4 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#5 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +Thread 10 (Thread 0x7fb85c7e2700 (LWP 9753)): +#0 0x00007fb85fc0c713 in epoll_wait () from /lib64/libc.so.6 +#1 0x00007fb8609d0576 in epoll_dispatch (base=0x6ce2c80, tv=) at epoll.c:404 +#2 0x00007fb8609bbe44 in event_base_loop (base=0x6ce2c80, flags=) + at event.c:1558 +#3 0x0000000000414504 in worker_libevent (arg=0x247daf0) at daemon/thread.c:301 +#4 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#5 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +---Type to continue, or q to quit--- +Thread 9 (Thread 0x7fb8578bc700 (LWP 9960)): +#0 0x00007fb85fec27bb in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x00007fb859d0bed8 in wait (this=0x6d62240, d=...) at src/syncobject.hh:58 +#2 IdleTask::run (this=0x6d62240, d=...) at src/dispatcher.cc:336 +#3 0x00007fb859d0e59a in Dispatcher::run (this=0x6d1f6c0) at src/dispatcher.cc:173 +#4 0x00007fb859d0ee9b in launch_dispatcher_thread (arg=0x6d1f6c0) at src/dispatcher.cc:28 +#5 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#6 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +Thread 8 (Thread 0x7fb85f406700 (LWP 9747)): +#0 0x00007fb85fbfee1d in read () from /lib64/libc.so.6 +#1 0x00007fb85fb95798 in _IO_new_file_underflow () from /lib64/libc.so.6 +#2 0x00007fb85fb9729e in _IO_default_uflow_internal () from /lib64/libc.so.6 +#3 0x00007fb85fb8ba8a in _IO_getline_info_internal () from /lib64/libc.so.6 +#4 0x00007fb85fb8a8e9 in fgets () from /lib64/libc.so.6 +#5 0x00007fb85f407939 in check_stdin_thread (arg=) + at extensions/daemon/stdin_check.c:37 +#6 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +Thread 7 (Thread 0x7fb86100b720 (LWP 9740)): +#0 0x00007fb85fc0c713 in epoll_wait () from /lib64/libc.so.6 +#1 0x00007fb8609d0576 in epoll_dispatch (base=0x6ce2000, tv=) at epoll.c:404 +---Type to continue, or q to quit--- +#2 0x00007fb8609bbe44 in event_base_loop (base=0x6ce2000, flags=) + at event.c:1558 +#3 0x0000000000409742 in main (argc=, argv=) + at daemon/memcached.c:7918 + +Thread 6 (Thread 0x7fb858cbe700 (LWP 9958)): +#0 0x00007fb85fec5054 in __lll_lock_wait () from /lib64/libpthread.so.0 +#1 0x00007fb85fec0388 in _L_lock_854 () from /lib64/libpthread.so.0 +#2 0x00007fb85fec0257 in pthread_mutex_lock () from /lib64/libpthread.so.0 +#3 0x00007fb859d4c26a in Mutex::acquire (this=0x6d1ea90) at src/mutex.cc:80 +#4 0x00007fb859d0e348 in lock (this=0x6d1ea80) at src/locks.hh:48 +#5 LockHolder (this=0x6d1ea80) at src/locks.hh:26 +#6 Dispatcher::run (this=0x6d1ea80) at src/dispatcher.cc:127 +#7 0x00007fb859d0ee9b in launch_dispatcher_thread (arg=0x6d1ea80) at src/dispatcher.cc:28 +#8 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#9 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +Thread 5 (Thread 0x7fb85e7f1700 (LWP 9748)): +#0 0x00007fb85fec27bb in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x00007fb85e7f54d6 in logger_thead_main (arg=0x247a040) at extensions/loggers/file_logger.c:368 +#2 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#3 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +---Type to continue, or q to quit--- +Thread 4 (Thread 0x7fb8582bd700 (LWP 9959)): +#0 0x00007fb85fec27bb in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x00007fb859d0bed8 in wait (this=0x6d622d0, d=...) at src/syncobject.hh:58 +#2 IdleTask::run (this=0x6d622d0, d=...) at src/dispatcher.cc:336 +#3 0x00007fb859d0e59a in Dispatcher::run (this=0x6d1f880) at src/dispatcher.cc:173 +#4 0x00007fb859d0ee9b in launch_dispatcher_thread (arg=0x6d1f880) at src/dispatcher.cc:28 +#5 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#6 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +Thread 3 (Thread 0x7fb856ebb700 (LWP 9961)): +#0 0x00007fb85fec27bb in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +#1 0x00007fb859d2aa4f in wait (this=0x6cde000) at src/syncobject.hh:58 +#2 wait (this=0x6cde000) at src/syncobject.hh:74 +#3 wait (this=0x6cde000) at src/tapconnmap.hh:169 +#4 EventuallyPersistentEngine::notifyPendingConnections (this=0x6cde000) at src/ep_engine.cc:3404 +#5 0x00007fb859d2ab33 in EvpNotifyPendingConns (arg=0x6cde000) at src/ep_engine.cc:1142 +#6 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#7 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +Thread 2 (Thread 0x7fb85d1e3700 (LWP 9752)): +#0 0x00007fb85fc0c713 in epoll_wait () from /lib64/libc.so.6 +#1 0x00007fb8609d0576 in epoll_dispatch (base=0x6ce2280, tv=) at epoll.c:404 +#2 0x00007fb8609bbe44 in event_base_loop (base=0x6ce2280, flags=) +---Type to continue, or q to quit--- + at event.c:1558 +#3 0x0000000000414504 in worker_libevent (arg=0x247d9f8) at daemon/thread.c:301 +#4 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#5 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6 + +Thread 1 (Thread 0x7fb8596bf700 (LWP 9957)): +#0 0x00007fb8600f3d03 in tcmalloc::CentralFreeList::FetchFromSpans() () + from /opt/couchbase/lib/libtcmalloc_minimal.so.4 +#1 0x00007fb8600f3c05 in tcmalloc::CentralFreeList::RemoveRange(void**, void**, int) () + from /opt/couchbase/lib/libtcmalloc_minimal.so.4 +#2 0x00007fb8600f8cd5 in tcmalloc::ThreadCache::FetchFromCentralCache(unsigned long, unsigned long) () from /opt/couchbase/lib/libtcmalloc_minimal.so.4 +#3 0x00007fb8600ee054 in tcmalloc::ThreadCache::Allocate(unsigned long, unsigned long) () + from /opt/couchbase/lib/libtcmalloc_minimal.so.4 +#4 0x00007fb8600ecb60 in (anonymous namespace)::do_malloc(unsigned long) () + from /opt/couchbase/lib/libtcmalloc_minimal.so.4 +#5 0x00007fb8600ec892 in (anonymous namespace)::do_malloc_or_cpp_alloc(unsigned long) () + from /opt/couchbase/lib/libtcmalloc_minimal.so.4 +#6 0x00007fb8601006d0 in tc_malloc () from /opt/couchbase/lib/libtcmalloc_minimal.so.4 +#7 0x00007fb859adf26d in db_write_buf_compressed (db=0x6ce9c70, buf=0x7fb8596be110, + pos=0x7fb8596be130, disk_size=0x7fb8596be128) at src/couch_file_write.c:129 +#8 0x00007fb859ad9fb7 in flush_mr_partial (res=0x45058b80, mr_quota=) + at src/btree_modify.c:222 +---Type to continue, or q to quit--- +#9 0x00007fb859adacdf in flush_mr (rq=, nptr=0x45058b50, + start=, end=, dst=0x45058998) + at src/btree_modify.c:170 +#10 modify_node (rq=, nptr=0x45058b50, start=, + end=, dst=0x45058998) at src/btree_modify.c:484 +#11 0x00007fb859adab5b in modify_node (rq=0x7fb8596be4d0, nptr=0x45058968, start=0, end=2, + dst=0x45058430) at src/btree_modify.c:458 +#12 0x00007fb859adab5b in modify_node (rq=0x7fb8596be4d0, nptr=0x45058400, start=0, end=2, + dst=0x450580c8) at src/btree_modify.c:458 +#13 0x00007fb859adab5b in modify_node (rq=0x7fb8596be4d0, nptr=0x6d9d3e00, start=0, end=2, + dst=0x45058010) at src/btree_modify.c:458 +#14 0x00007fb859adb317 in modify_btree (rq=0x7fb8596be4d0, root=0x6d9d3e00, errcode=0x7fb8596be5dc) + at src/btree_modify.c:607 +#15 0x00007fb859ade0d5 in update_indexes (db=0x6ce9c70, seqs=0x40bd6790, seqvals=0x40bd67b0, + ids=, idvals=0x40bd67c0, numdocs=1) at src/couch_save.c:204 +#16 0x00007fb859adebb0 in couchstore_save_documents (db=0x6ce9c70, docs=0x3a36c5e38, + infos=, numdocs=1, options=1) at src/couch_save.c:359 + +#17 0x00007fb859d87f97 in CouchKVStore::saveDocs (this=0x6d4e000, vbid=673, rev=2, docs=0x2476758, + docinfos=0x2476400, docCount=1) at src/couch-kvstore/couch-kvstore.cc:1556 +#18 0x00007fb859d886d1 in CouchKVStore::commit2couchstore (this=0x6d4e000) + at src/couch-kvstore/couch-kvstore.cc:1491 +#19 0x00007fb859d888da in CouchKVStore::commit (this=0x6d4e000) + at src/couch-kvstore/couch-kvstore.cc:876 +---Type to continue, or q to quit--- +#20 0x00007fb859d11ef5 in TransactionContext::commit (this=0x6cda788) at src/ep.cc:2767 +#21 0x00007fb859d1c180 in EventuallyPersistentStore::flushOutgoingQueue (this=0x6cda480, + flushQueue=0x6cda748, flushPhase=@0x6cd8570, nextVbid=@0x6cd8578) at src/ep.cc:1965 +#22 0x00007fb859d452dc in Flusher::doFlush (this=0x6cd8480) at src/flusher.cc:245 +#23 0x00007fb859d46115 in Flusher::step (this=0x6cd8480, d=..., tid=...) at src/flusher.cc:158 +#24 0x00007fb859d0e59a in Dispatcher::run (this=0x6d1ec40) at src/dispatcher.cc:173 +#25 0x00007fb859d0ee9b in launch_dispatcher_thread (arg=0x6d1ec40) at src/dispatcher.cc:28 +#26 0x00007fb85febe851 in start_thread () from /lib64/libpthread.so.0 +#27 0x00007fb85fc0c11d in clone () from /lib64/libc.so.6