Crash with release-tequila

[tobrun]

We have been receiving a couple of crash reports with release-tequila aka v8.6.0 on Android.
@tmpsantos can someone from your team look into this?

********** Crash dump: **********
Build fingerprint: 'Suunto/salmon/salmon:9/PXDB.191220.001/6089389:userdebug/dev-keys'
Abort message: '/usr/local/google/buildbot/src/android/ndk-release-r20/external/libcxx/../../external/libcxxabi/src/abort_message.cpp:73: abort_message: assertion "terminating" failed'
#00 0x0001cd82 /system/lib/libc.so (abort+62)
#1 0x0001cfe7 /system/lib/libc.so (__assert2+22)
#2 0x002160f5 /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
abort_message
/usr/local/google/buildbot/src/android/ndk-release-r20/external/libcxx/../../external/libcxxabi/src/abort_message.cpp:73:5
#3 0x0021617b /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
demangling_terminate_handler()
/usr/local/google/buildbot/src/android/ndk-release-r20/external/libcxx/../../external/libcxxabi/src/cxa_default_handlers.cpp:76:5
#4 0x00214941 /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
std::__terminate(void ()())
/usr/local/google/buildbot/src/android/ndk-release-r20/external/libcxx/../../external/libcxxabi/src/cxa_handlers.cpp:60:9
#5 0x002148d9 /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so (std::terminate()+28)
std::terminate()
/usr/local/google/buildbot/src/android/ndk-release-r20/external/libcxx/../../external/libcxxabi/src/cxa_handlers.cpp:91:5
#6 0x00213d0f /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so (std::__ndk1::thread::~thread()+10)
std::__ndk1::thread::~thread()
/usr/local/google/buildbot/src/android/ndk-release-r20/external/libcxx/src/thread.cpp:44:9
#7 0x0008d05d /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
mbgl::ThreadedScheduler<1u>::~ThreadedScheduler()
/src/vendor/mapbox-gl-native/next/../src/mbgl/util/thread_pool.hpp:53:5
#8 0x0020f735 /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
std::__ndk1::__shared_count::__release_shared()
/usr/local/google/buildbot/src/android/ndk-release-r20/external/libcxx/src/memory.cpp:49:9
std::__ndk1::__shared_weak_count::__release_shared()
/usr/local/google/buildbot/src/android/ndk-release-r20/external/libcxx/src/memory.cpp:70:0
#9 0x001ef827 /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
std::__ndk1::shared_ptrmbgl::Mailbox::~shared_ptr()
/android/sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/memory:4522:19
#10 0x000e3e6d /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
mbgl::style::GeoJSONVTData::~GeoJSONVTData()
/src/vendor/mapbox-gl-native/src/mbgl/style/sources/geojson_source_impl.cpp:16:7
#11 0x000e3e8d /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
mbgl::style::GeoJSONVTData::~GeoJSONVTData()
/src/vendor/mapbox-gl-native/src/mbgl/style/sources/geojson_source_impl.cpp:16:7
#12 0x0020f735 /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
std::__ndk1::__shared_count::__release_shared()
/usr/local/google/buildbot/src/android/ndk-release-r20/external/libcxx/src/memory.cpp:49:9
std::__ndk1::__shared_weak_count::__release_shared()
/usr/local/google/buildbot/src/android/ndk-release-r20/external/libcxx/src/memory.cpp:70:0
#13 0x001ef827 /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
std::__ndk1::shared_ptrmbgl::Mailbox::~shared_ptr()
/android/sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/memory:4522:19
#14 0x000e421f /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'()::operator()() const
/src/vendor/mapbox-gl-native/src/mbgl/style/sources/geojson_source_impl.cpp:23:26
void mbgl::Scheduler::scheduleAndReplyValue<mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'(), std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> >(mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'() const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&, mapbox::base::WeakPtrmbgl::Scheduler)::'lambda'()::operator()() const
/src/vendor/mapbox-gl-native/next/../include/mbgl/actor/scheduler.hpp:112:0
decltype(std::__ndk1::forward<mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'()>(fp)()) std::__ndk1::__invoke<void mbgl::Scheduler::scheduleAndReplyValue<mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'(), std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> >(mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'() const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&, mapbox::base::WeakPtrmbgl::Scheduler)::'lambda'()&>(mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'()&&)
/android/sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/type_traits:4353:0
void std::__ndk1::__invoke_void_return_wrapper::__call<void mbgl::Scheduler::scheduleAndReplyValue<mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'(), std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> >(mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'() const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&, mapbox::base::WeakPtrmbgl::Scheduler)::'lambda'()&>(void mbgl::Scheduler::scheduleAndReplyValue<mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'(), std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> >(mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'() const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&, mapbox::base::WeakPtrmbgl::Scheduler)::'lambda'()&)
/android/sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/__functional_base:349:0
std::__ndk1::__function::__alloc_func<void mbgl::Scheduler::scheduleAndReplyValue<mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'(), std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> >(mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'() const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&, mapbox::base::WeakPtrmbgl::Scheduler)::'lambda'(), std::__ndk1::allocator<void mbgl::Scheduler::scheduleAndReplyValue<mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'(), std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> >(mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'() const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&, mapbox::base::WeakPtrmbgl::Scheduler)::'lambda'()>, void ()>::operator()()
/android/sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/functional:1527:0
std::__ndk1::__function::__func<void mbgl::Scheduler::scheduleAndReplyValue<mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'(), std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> >(mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'() const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&, mapbox::base::WeakPtrmbgl::Scheduler)::'lambda'(), std::__ndk1::allocator<void mbgl::Scheduler::scheduleAndReplyValue<mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'(), std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> >(mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&)::'lambda'() const&, std::__ndk1::function<void (mapbox::feature::feature_collection<short, std::__ndk1::vector>)> const&, mapbox::base::WeakPtrmbgl::Scheduler)::'lambda'()>, void ()>::operator()()
/android/sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/functional:1651:0
#15 0x0010a5e7 /data/app/com.soy.android.wear.debug-sxenToRVy7pox987m2jo_Q==/lib/arm/libmapbox-gl.so
std::__ndk1::function<void ()>::operator()() const
/android/sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/functional:2347:12
mbgl::ThreadedSchedulerBase::makeSchedulerThread(unsigned int)::$_0::operator()() const
/src/vendor/mapbox-gl-native/src/mbgl/util/thread_pool.cpp:37:0
decltype(std::__ndk1::forward<mbgl::ThreadedSchedulerBase::makeSchedulerThread(unsigned int)::$_0>(fp)()) std::__ndk1::__invoke<mbgl::ThreadedSchedulerBase::makeSchedulerThread(unsigned int)::$_0>(mbgl::ThreadedSchedulerBase::makeSchedulerThread(unsigned int)::$_0&&)
/android/sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/type_traits:4353:0
void std::__ndk1::__thread_execute<std::__ndk1::unique_ptr<std::__ndk1::__thread_struct, std::__ndk1::default_deletestd::__ndk1::__thread_struct >, mbgl::ThreadedSchedulerBase::makeSchedulerThread(unsigned int)::$_0>(std::__ndk1::tuple<std::__ndk1::unique_ptr<std::__ndk1::__thread_struct, std::__ndk1::default_deletestd::__ndk1::__thread_struct >, mbgl::ThreadedSchedulerBase::makeSchedulerThread(unsigned int)::$_0>&, std::__ndk1::__tuple_indices<>)
/android/sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/thread:342:0
void std::__ndk1::__thread_proxy<std::__ndk1::tuple<std::__ndk1::unique_ptr<std::__ndk1::__thread_struct, std::__ndk1::default_deletestd::__ndk1::__thread_struct >, mbgl::ThreadedSchedulerBase::makeSchedulerThread(unsigned int)::$_0> >(void*)
/android/sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/thread:352:0
#16 0x00063a05 /system/lib/libc.so (__pthread_start(void*)+22)
#17 0x0001df75 /system/lib/libc.so (__start_thread+22)
Crash dump is completed

cc @pengdev (thank you for the symbolicated stacktrace!)

[tmpsantos]

cc @pozdnyakov

[LukasPaczos]

I'm able to consistently reproduce in the Navigation SDK using tequila as well.

[tobrun]

I'm able to consistently reproduce in the Navigation SDK using tequila as well.

In hindsight, we need to improve the Android QA process to include downstream testing (navigation) and automated large pool testing (once a week?)

[Siemian]

Seeing similar crash on iOS:

Crashed: com.mapbox.mbgl.Worker 1
0  libsystem_kernel.dylib         0x1b72f9ec4 __pthread_kill + 8
1  libsystem_pthread.dylib        0x1b7219724 pthread_kill$VARIANT$armv81 + 216
2  libsystem_c.dylib              0x1b7169844 abort + 100
3  libc++abi.dylib                0x1b72c27d4 __cxa_bad_cast + 126
4  libc++abi.dylib                0x1b72c28cc demangling_terminate_handler() + 48
5  XXXXXXXXX                      0x100f1f388 CLSTerminateHandler() + 4336989064
6  libc++abi.dylib                0x1b72cf304 std::__terminate(void (*)()) + 16
7  libc++abi.dylib                0x1b72cf2ac std::terminate() + 60
8  libc++.1.dylib                 0x1b7388ef8 std::__1::thread::join() + 22
9  Mapbox                         0x101858fe0 mbgl::ThreadedScheduler<1ul>::~ThreadedScheduler() + 53 (thread_pool.hpp:53)
10 Mapbox                         0x101a5fcbc mbgl::style::GeoJSONVTData::~GeoJSONVTData() + 3587 (memory:3587)
11 Mapbox                         0x101a71bc8 mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__1::function<void (mapbox::feature::feature_collection<short, std::__1::vector>)> const&)::'lambda'()::operator()() const + 3587 (memory:3587)
12 Mapbox                         0x101a71668 void mbgl::Scheduler::scheduleAndReplyValue<mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__1::function<void (mapbox::feature::feature_collection<short, std::__1::vector>)> const&)::'lambda'(), std::__1::function<void (mapbox::feature::feature_collection<short, std::__1::vector>)> >(mbgl::style::GeoJSONVTData::getTile(mbgl::CanonicalTileID const&, std::__1::function<void (mapbox::feature::feature_collection<short, std::__1::vector>)> const&)::'lambda'() const&, std::__1::function<void (mapbox::feature::feature_collection<short, std::__1::vector>)> const&, mapbox::base::WeakPtr<mbgl::Scheduler>)::'lambda'()::operator()() const + 5274 (memory:5274)
13 Mapbox                         0x101ae3b68 void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, mbgl::ThreadedSchedulerBase::makeSchedulerThread(unsigned long)::$_0> >(void*) + 1757 (functional:1757)
14 libsystem_pthread.dylib        0x1b721dd8c _pthread_start + 156
15 libsystem_pthread.dylib        0x1b722176c thread_start + 8

Version: Mapbox-iOS-SDK (5.6.0)