Skip to content

Commit

Permalink
catalog_manager: fix a TSAN race
Browse files Browse the repository at this point in the history 
Noticed this on the flaky test dashboard for alter_table-randomized-test;
see the end of the commit message for the complete output. I also removed
an unrelated and unnecessary lock acquisition.

To test, I looped alter_table_randomized-test in slow mode with TSAN and the
two failures I saw did not report any data races.

  WARNING: ThreadSanitizer: data race (pid=17016)  Read of size 8 at 0x7b4c000010d0 by thread T68 (mutexes: write M1500):
    #0 std::__1::unique_ptr<kudu::hms::HmsCatalog, std::__1::default_delete<kudu::hms::HmsCatalog> >::operator bool() const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2583:19 (libmaster.so+0xb99b1)
    apache#1 kudu::master::CatalogManager::PrepareForLeadershipTask() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/catalog_manager.cc:1055 (libmaster.so+0xb99b1)
    apache#2 kudu::internal::RunnableAdapter<void (kudu::master::CatalogManager::*)()>::Run(kudu::master::CatalogManager*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:136:12 (libmaster.so+0x102fa9)
    apache#3 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::master::CatalogManager::*)()>, void ()(kudu::master::CatalogManager*)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::master::CatalogManager::*)()>, kudu::master::CatalogManager*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:873:14 (libmaster.so+0x102ec5)
    apache#4 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::master::CatalogManager::*)()>, void ()(kudu::master::CatalogManager*), void ()(kudu::internal::UnretainedWrapper<kudu::master::CatalogManager>)>, void ()(kudu::master::CatalogManager*)>::Run(kudu::internal::BindStateBase*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:1065:12 (libmaster.so+0x102e0a)
    apache#5 kudu::Callback<void ()()>::Run() const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:396:12 (libconsensus.so+0xa6dfd)
    apache#6 kudu::ClosureRunnable::Run() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:76:9 (libkudu_util.so+0x1cc9ad)
    apache#7 kudu::ThreadPool::DispatchThread() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:686:22 (libkudu_util.so+0x1c86d8)
    apache#8 boost::_mfi::mf0<void, kudu::ThreadPool>::operator()(kudu::ThreadPool*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29 (libkudu_util.so+0x1d3649)
    apache#9 void boost::_bi::list1<boost::_bi::value<kudu::ThreadPool*> >::operator()<boost::_mfi::mf0<void, kudu::ThreadPool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::ThreadPool>&, boost::_bi::list0&, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9 (libkudu_util.so+0x1d359a)
    apache#10 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::ThreadPool>, boost::_bi::list1<boost::_bi::value<kudu::ThreadPool*> > >::operator()() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16 (libkudu_util.so+0x1d3523)
    apache#11 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::ThreadPool>, boost::_bi::list1<boost::_bi::value<kudu::ThreadPool*> > >, void>::invoke(boost::detail::function::function_buffer&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11 (libkudu_util.so+0x1d3319)
    apache#12 boost::function0<void>::operator()() const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xb6651)
    apache#13 kudu::Thread::SuperviseThread(void*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:615:3 (libkudu_util.so+0x1bfe34)

  Previous write of size 8 at 0x7b4c000010d0 by thread T59:
    #0 std::__1::unique_ptr<kudu::hms::HmsCatalog, std::__1::default_delete<kudu::hms::HmsCatalog> >::reset(kudu::hms::HmsCatalog*) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2596:20 (libmaster.so+0xb8b6f)
    apache#1 kudu::master::CatalogManager::Init(bool) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/catalog_manager.cc:730 (libmaster.so+0xb8b6f)
    apache#2 kudu::master::Master::InitCatalogManager() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:216:3 (libmaster.so+0x11fa5f)
    apache#3 kudu::master::Master::InitCatalogManagerTask() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:205:14 (libmaster.so+0x11f8b2)
    apache#4 kudu::internal::RunnableAdapter<void (kudu::master::Master::*)()>::Run(kudu::master::Master*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:136:12 (libmaster.so+0x124449)
    apache#5 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::master::Master::*)()>, void ()(kudu::master::Master*)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::master::Master::*)()>, kudu::master::Master*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:873:14 (libmaster.so+0x124365)
    apache#6 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::master::Master::*)()>, void ()(kudu::master::Master*), void ()(kudu::internal::UnretainedWrapper<kudu::master::Master>)>, void ()(kudu::master::Master*)>::Run(kudu::internal::BindStateBase*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:1065:12 (libmaster.so+0x1242aa)
    apache#7 kudu::Callback<void ()()>::Run() const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:396:12 (libconsensus.so+0xa6dfd)
    apache#8 kudu::ClosureRunnable::Run() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:76:9 (libkudu_util.so+0x1cc9ad)
    apache#9 kudu::ThreadPool::DispatchThread() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:686:22 (libkudu_util.so+0x1c86d8)
    apache#10 boost::_mfi::mf0<void, kudu::ThreadPool>::operator()(kudu::ThreadPool*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29 (libkudu_util.so+0x1d3649)
    apache#11 void boost::_bi::list1<boost::_bi::value<kudu::ThreadPool*> >::operator()<boost::_mfi::mf0<void, kudu::ThreadPool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::ThreadPool>&, boost::_bi::list0&, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9 (libkudu_util.so+0x1d359a)
    apache#12 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::ThreadPool>, boost::_bi::list1<boost::_bi::value<kudu::ThreadPool*> > >::operator()() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16 (libkudu_util.so+0x1d3523)
    apache#13 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::ThreadPool>, boost::_bi::list1<boost::_bi::value<kudu::ThreadPool*> > >, void>::invoke(boost::detail::function::function_buffer&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11 (libkudu_util.so+0x1d3319)
    apache#14 boost::function0<void>::operator()() const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xb6651)
    apache#15 kudu::Thread::SuperviseThread(void*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:615:3 (libkudu_util.so+0x1bfe34)

  Location is heap block of size 432 at 0x7b4c00000fc0 allocated by main thread:
    #0 operator new(unsigned long) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:57 (kudu-master+0x4c84a3)
    apache#1 kudu::master::Master::Master(kudu::master::MasterOptions const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:122:22 (libmaster.so+0x11e3d5)
    apache#2 kudu::master::MasterMain(int, char**) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_main.cc:79:10 (kudu-master+0x4cb4de)
    apache#3 main /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_main.cc:98:10 (kudu-master+0x4cb1be)

  Mutex M1500 (0x7b4c00001100) created at:
    #0 pthread_rwlock_init /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1304 (kudu-master+0x4593b4)
    apache#1 kudu::RWMutex::Init(kudu::RWMutex::Priority) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/rw_mutex.cc:78:8 (libkudu_util.so+0x1acad8)
    apache#2 kudu::RWMutex::RWMutex(kudu::RWMutex::Priority) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/rw_mutex.cc:56:3 (libkudu_util.so+0x1acd13)
    apache#3 kudu::master::CatalogManager::CatalogManager(kudu::master::Master*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/catalog_manager.cc:688:7 (libmaster.so+0xb81d7)
    apache#4 kudu::master::Master::Master(kudu::master::MasterOptions const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:122:26 (libmaster.so+0x11e3e3)
    apache#5 kudu::master::MasterMain(int, char**) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_main.cc:79:10 (kudu-master+0x4cb4de)
    apache#6 main /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_main.cc:98:10 (kudu-master+0x4cb1be)

  Thread T68 'leader-initiali' (tid=17094, running) created by thread T65 at:
    #0 pthread_create /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:992 (kudu-master+0x45af0b)
    apache#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:559:15 (libkudu_util.so+0x1bf61b)
    apache#2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(), kudu::ThreadPool*>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(), kudu::ThreadPool* const&, scoped_refptr<kudu::Thread>*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.h:164:12 (libkudu_util.so+0x1ca9f5)
    apache#3 kudu::ThreadPool::CreateThread() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:749:10 (libkudu_util.so+0x1c7ce2)
    apache#4 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>, kudu::ThreadPoolToken*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:556:21 (libkudu_util.so+0x1c64af)
    apache#5 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:458:10 (libkudu_util.so+0x1c7f4f)
    apache#6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()()>) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:450:10 (libkudu_util.so+0x1c7e91)
    apache#7 kudu::master::CatalogManager::ElectedAsLeaderCb() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/catalog_manager.cc:754:33 (libmaster.so+0xb936b)
    apache#8 kudu::internal::RunnableAdapter<kudu::Status (kudu::master::CatalogManager::*)()>::Run(kudu::master::CatalogManager*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:136:12 (libmaster.so+0x104130)
    apache#9 kudu::internal::InvokeHelper<false, kudu::Status, kudu::internal::RunnableAdapter<kudu::Status (kudu::master::CatalogManager::*)()>, void ()(kudu::master::CatalogManager*)>::MakeItSo(kudu::internal::RunnableAdapter<kudu::Status (kudu::master::CatalogManager::*)()>, kudu::master::CatalogManager*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:865:21 (libmaster.so+0x10409d)
    apache#10 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<kudu::Status (kudu::master::CatalogManager::*)()>, kudu::Status ()(kudu::master::CatalogManager*), void ()(kudu::internal::UnretainedWrapper<kudu::master::CatalogManager>)>, kudu::Status ()(kudu::master::CatalogManager*)>::Run(kudu::internal::BindStateBase*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:1065:12 (libmaster.so+0x10400f)
    apache#11 kudu::Callback<kudu::Status ()()>::Run() const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:396:12 (libmaster.so+0x14bed6)
    apache#12 kudu::master::SysCatalogTable::SysCatalogStateChanged(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/sys_catalog.cc:343:27 (libmaster.so+0x145539)
    apache#13 kudu::internal::RunnableAdapter<void (kudu::master::SysCatalogTable::*)(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>::Run(kudu::master::SysCatalogTable*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:250:12 (libmaster.so+0x15269d)
    apache#14 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::master::SysCatalogTable::*)(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>, void ()(kudu::master::SysCatalogTable*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::master::SysCatalogTable::*)(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>, kudu::master::SysCatalogTable*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:907:14 (libmaster.so+0x15256b)
    apache#15 kudu::internal::Invoker<2, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::master::SysCatalogTable::*)(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>, void ()(kudu::master::SysCatalogTable*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&), void ()(kudu::internal::UnretainedWrapper<kudu::master::SysCatalogTable>, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >)>, void ()(kudu::master::SysCatalogTable*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>::Run(kudu::internal::BindStateBase*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:1242:12 (libmaster.so+0x152459)
    apache#16 kudu::Callback<void ()(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>::Run(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:436:12 (libtablet.so+0x151681)
    apache#17 kudu::internal::InvokeHelper<false, void, kudu::Callback<void ()(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>, void ()(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>::MakeItSo(kudu::Callback<void ()(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:873:14 (libconsensus.so+0xea428)
    apache#18 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::Callback<void ()(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>, void ()(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&), void ()(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >)>, void ()(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>::Run(kudu::internal::BindStateBase*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:1065:12 (libconsensus.so+0xea3c3)
    apache#19 kudu::Callback<void ()()>::Run() const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:396:12 (libconsensus.so+0xa6dfd)
    apache#20 kudu::ClosureRunnable::Run() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:76:9 (libkudu_util.so+0x1cc9ad)
    apache#21 kudu::ThreadPool::DispatchThread() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:686:22 (libkudu_util.so+0x1c86d8)
    apache#22 boost::_mfi::mf0<void, kudu::ThreadPool>::operator()(kudu::ThreadPool*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29 (libkudu_util.so+0x1d3649)
    apache#23 void boost::_bi::list1<boost::_bi::value<kudu::ThreadPool*> >::operator()<boost::_mfi::mf0<void, kudu::ThreadPool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::ThreadPool>&, boost::_bi::list0&, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9 (libkudu_util.so+0x1d359a)
    apache#24 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::ThreadPool>, boost::_bi::list1<boost::_bi::value<kudu::ThreadPool*> > >::operator()() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16 (libkudu_util.so+0x1d3523)
    apache#25 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::ThreadPool>, boost::_bi::list1<boost::_bi::value<kudu::ThreadPool*> > >, void>::invoke(boost::detail::function::function_buffer&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11 (libkudu_util.so+0x1d3319)
    apache#26 boost::function0<void>::operator()() const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xb6651)
    apache#27 kudu::Thread::SuperviseThread(void*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:615:3 (libkudu_util.so+0x1bfe34)

  Thread T59 'init [worker]-1' (tid=17081, running) created by main thread at:
    #0 pthread_create /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:992 (kudu-master+0x45af0b)
    apache#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:559:15 (libkudu_util.so+0x1bf61b)
    apache#2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(), kudu::ThreadPool*>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(), kudu::ThreadPool* const&, scoped_refptr<kudu::Thread>*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.h:164:12 (libkudu_util.so+0x1ca9f5)
    apache#3 kudu::ThreadPool::CreateThread() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:749:10 (libkudu_util.so+0x1c7ce2)
    apache#4 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>, kudu::ThreadPoolToken*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:556:21 (libkudu_util.so+0x1c64af)
    apache#5 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:458:10 (libkudu_util.so+0x1c7f4f)
    apache#6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()()>) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:450:10 (libkudu_util.so+0x1c7e91)
    apache#7 kudu::master::Master::StartAsync() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:196:3 (libmaster.so+0x11f260)
    apache#8 kudu::master::Master::Start() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:170:3 (libmaster.so+0x11ef25)
    apache#9 kudu::master::MasterMain(int, char**) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_main.cc:84:3 (kudu-master+0x4cb584)
    apache#10 main /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_main.cc:98:10 (kudu-master+0x4cb1be)

Change-Id: I090a832b7fb25d8cb1e770c025048f73ac997eac
Reviewed-on: http://gerrit.cloudera.org:8080/11818
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <[email protected]>
Reviewed-by: Hao Hao <[email protected]>
  • Loading branch information
@adembo
adembo committed Oct 30, 2018
1 parent f731ea0 commit d0205b9
Showing 1 changed file with 7 additions and 1 deletion.
8 changes: 7 additions & 1 deletion src/kudu/master/catalog_manager.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -727,6 +727,13 @@ Status CatalogManager::Init(bool is_first_run) {
},
",");

// The leader_lock_ isn't really intended for this (it's for serializing
// new leadership initialization against regular catalog manager operations)
// but we need to use something to protect this hms_catalog_ write vis a vis
// the read in PrepareForLeadershipTask(), and that read is performed while
// holding leader_lock_, so this is the path of least resistance.
std::lock_guard<RWMutex> leader_lock_guard(leader_lock_);

hms_catalog_.reset(new hms::HmsCatalog(std::move(master_addresses)));
RETURN_NOT_OK_PREPEND(hms_catalog_->Start(),
"failed to start Hive Metastore catalog");
Expand All @@ -736,7 +743,6 @@ Status CatalogManager::Init(bool is_first_run) {
"failed to initialize Hive Metastore notification log listener task");
}

std::lock_guard<LockType> l(lock_);
background_tasks_.reset(new CatalogManagerBgTasks(this));
RETURN_NOT_OK_PREPEND(background_tasks_->Init(),
"Failed to initialize catalog manager background tasks");
Expand Down

0 comments on commit d0205b9

Please sign in to comment.