Uploaded image for project: 'Mesos'
  1. Mesos
  2. MESOS-9171

Mesos agent crashes in CNI isolator when usage is queried

    XMLWordPrintableJSON

Details

    • Bug
    • Status: Resolved
    • Blocker
    • Resolution: Fixed
    • 1.7.0
    • 1.7.0
    • None
    • None

    Description

      Ran into this with a master branch build:

      ../../3rdparty/stout/include/stout/option.hpp:118: const T& Option<T>::get() const & [with T = std::basic_string<char>]: Assertion `isSome()' failed.
      

      The backtrace:

      Program terminated with signal SIGABRT, Aborted.
      #0  0x00007fd0ab922495 in raise () from /lib64/libc.so.6
      #0  0x00007fd0ab922495 in raise () from /lib64/libc.so.6
      #1  0x00007fd0ab923c75 in abort () from /lib64/libc.so.6
      #2  0x00007fd0ab91b60e in __assert_fail_base () from /lib64/libc.so.6
      #3  0x00007fd0ab91b6d0 in __assert_fail () from /lib64/libc.so.6
      #4  0x00007fd0ae473c33 in Option<std::string>::get() const & (this=0x7fd0a4deb5a8) at ../../3rdparty/stout/include/stout/option.hpp:118
      #5  0x00007fd0ae48ae94 in get (this=0x7fd0a4deb5a8) at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/bits/unordered_set.h:93
      #6  mesos::internal::slave::NetworkCniIsolatorProcess::usage (this=0x7fd0a4dea800, containerId=...) at ../../src/slave/containerizer/mesos/isolators/network/cni/cni.cpp:1516
      #7  0x00007fd0ae1770da in operator() (process=<optimized out>, a0=..., promise=..., __closure=<optimized out>) at ../../3rdparty/libprocess/include/process/dispatch.hpp:354
      #8  invoke<process::dispatch(const process::PID<T>&, process::Future<T> (T::*)(P0), A0&&) [with R = mesos::ResourceStatistics; T = mesos::internal::slave::MesosIsolatorProcess; P0 = const mesos::ContainerID&; A0 = const mesos::ContainerID&]::<lambda(std::unique_ptr<process::Promise<mesos::ResourceStatistics>, std::default_delete<process::Promise<mesos::ResourceStatistics> > >, std::decay<const mesos::ContainerID&>::type&&, process::ProcessBase*)>, std::unique_ptr<process::Promise<mesos::ResourceStatistics>, std::default_delete<process::Promise<mesos::ResourceStatistics> > >, mesos::ContainerID, process::ProcessBase*> (f=...) at ../../3rdparty/stout/include/stout/cpp17.hpp:42
      #9  invoke_expand<process::dispatch(const process::PID<T>&, process::Future<T> (T::*)(P0), A0&&) [with R = mesos::ResourceStatistics; T = mesos::internal::slave::MesosIsolatorProcess; P0 = const mesos::ContainerID&; A0 = const mesos::ContainerID&]::<lambda(std::unique_ptr<process::Promise<mesos::ResourceStatistics>, std::default_delete<process::Promise<mesos::ResourceStatistics> > >, std::decay<const mesos::ContainerID&>::type&&, process::ProcessBase*)>, std::tuple<std::unique_ptr<process::Promise<mesos::ResourceStatistics>, std::default_delete<process::Promise<mesos::ResourceStatistics> > >, mesos::ContainerID, std::_Placeholder<1> >, std::tuple<process::ProcessBase*&&>, 0ul, 1ul, 2ul> (args=..., bound_args=..., f=...) at ../../3rdparty/stout/include/stout/lambda.hpp:292
      #10 operator()<process::ProcessBase*> (this=<optimized out>) at ../../3rdparty/stout/include/stout/lambda.hpp:331
      #11 invoke<lambda::internal::Partial<process::dispatch(const process::PID<T>&, process::Future<T> (T::*)(P0), A0&&) [with R = mesos::ResourceStatistics; T = mesos::internal::slave::MesosIsolatorProcess; P0 = const mesos::ContainerID&; A0 = const mesos::ContainerID&]::<lambda(std::unique_ptr<process::Promise<mesos::ResourceStatistics>, std::default_delete<process::Promise<mesos::ResourceStatistics> > >, std::decay<const mesos::ContainerID&>::type&&, process::ProcessBase*)>, std::unique_ptr<process::Promise<mesos::ResourceStatistics>, std::default_delete<process::Promise<mesos::ResourceStatistics> > >, mesos::ContainerID, std::_Placeholder<1> >, process::ProcessBase*> (f=...) at ../../3rdparty/stout/include/stout/cpp17.hpp:42
      #12 operator()<lambda::internal::Partial<process::dispatch(const process::PID<T>&, process::Future<T> (T::*)(P0), A0&&) [with R = mesos::ResourceStatistics; T = mesos::internal::slave::MesosIsolatorProcess; P0 = const mesos::ContainerID&; A0 = const mesos::ContainerID&]::<lambda(std::unique_ptr<process::Promise<mesos::ResourceStatistics>, std::default_delete<process::Promise<mesos::ResourceStatistics> > >, std::decay<const mesos::ContainerID&>::type&&, process::ProcessBase*)>, std::unique_ptr<process::Promise<mesos::ResourceStatistics>, std::default_delete<process::Promise<mesos::ResourceStatistics> > >, mesos::ContainerID, std::_Placeholder<1> >, process::ProcessBase*> (f=..., this=<optimized out>) at ../../3rdparty/stout/include/stout/lambda.hpp:398
      #13 lambda::CallableOnce<void (process::ProcessBase*)>::CallableFn<lambda::internal::Partial<process::Future<mesos::ResourceStatistics> process::dispatch<mesos::ResourceStatistics, mesos::internal::slave::MesosIsolatorProcess, mesos::ContainerID const&, mesos::ContainerID const&>(process::PID<mesos::internal::slave::MesosIsolatorProcess> const&, process::Future<mesos::ResourceStatistics> (mesos::internal::slave::MesosIsolatorProcess::*)(mesos::ContainerID const&), mesos::ContainerID const&)::{lambda(std::unique_ptr<process::Promise<mesos::ResourceStatistics>, std::default_delete<process::Promise<mesos::ResourceStatistics> > >, mesos::ContainerID&&, process::ProcessBase*)#1}, std::unique_ptr<process::Promise<mesos::ResourceStatistics>, std::default_delete<process::Promise<mesos::ResourceStatistics> > >, mesos::ContainerID, std::_Placeholder<1> > >::operator()(process::ProcessBase*&&) && (this=0x7fd099a2a630, args#0=<optimized out>) at ../../3rdparty/stout/include/stout/lambda.hpp:463
      #14 0x00007fd0aed493a2 in operator() (args#0=0x7fd0a4deb6b8, this=<optimized out>) at ../../../3rdparty/stout/include/stout/lambda.hpp:443
      #15 process::ProcessBase::consume(process::DispatchEvent&&) (this=<optimized out>, event=...) at ../../../3rdparty/libprocess/src/process.cpp:3563
      #16 0x00007fd0aed88609 in serve (event=..., this=0x7fd0a4deb6b8) at ../../../3rdparty/libprocess/include/process/process.hpp:87
      #17 process::ProcessManager::resume (this=<optimized out>, process=0x7fd0a4deb6b8) at ../../../3rdparty/libprocess/src/process.cpp:2988
      #18 0x00007fd0aed8f856 in operator() (__closure=0x7fd0a4d44dd8) at ../../../3rdparty/libprocess/src/process.cpp:2497
      #19 _M_invoke<> (this=0x7fd0a4d44dd8) at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:1700
      #20 operator() (this=0x7fd0a4d44dd8) at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/functional:1688
      #21 std::thread::_Impl<std::_Bind_simple<process::ProcessManager::init_threads()::<lambda()>()> >::_M_run(void) (this=0x7fd0a4d44dc0) at /opt/rh/devtoolset-3/root/usr/include/c++/4.9.2/thread:115
      #22 0x00007fd0abd3a470 in ?? () from /usr/lib64/libstdc++.so.6
      #23 0x00007fd0abf91aa1 in start_thread () from /lib64/libpthread.so.0
      #24 0x00007fd0ab9d8bcd in clone () from /lib64/libc.so.6
      

      Suspecting https://github.com/apache/mesos/commit/7708008a3c0c4b3a1b0384c2c739ea258306d671#diff-9b648d3db5c0cac0e9cc2b7ba946c7c2

      rootDir is not specified in this case? https://github.com/apache/mesos/blob/dbcf3c6607b2c6383e4d6ed77180de26a57edf8a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp#L99-L106

      Attachments

        Activity

          People

            urbanserj Sergey Urbanovich
            xujyan Yan Xu
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: