Info

{"backtrace":[{"a":"55EE10914801","b":"55EE0DAD5000","o":"2E3F801","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1E1"}, {"a":"55EE10915E39","b":"55EE0DAD5000","o":"2E40E39","s":"_ZN5mongo15printStackTraceEv","s+":"29"}, {"a":"55EE109137DC","b":"55EE0DAD5000","o":"2E3E7DC","s":"_ZN5mongo12_GLOBAL__N_124abruptQuitWithAddrSignalEiP9siginfo_tPv","s+":"EC"}, {"a":"7FCE279E9630","b":"7FCE279DA000","o":"F630","s":"_L_unlock_13","s+":"34"}, {"a":"55EE0EE56673","b":"55EE0DAD5000","o":"1381673","s":"__eventv","s+":"3C3"}, {"a":"55EE0E77C307","b":"55EE0DAD5000","o":"CA7307","s":"__wt_verbose_worker","s+":"A7"}, {"a":"55EE0E76FCB0","b":"55EE0DAD5000","o":"C9ACB0","s":"__curfile_reset.cold.13","s+":"3D"}, {"a":"55EE0EDB97C7","b":"55EE0DAD5000","o":"12E47C7","s":"_ZN5mongo17WiredTigerSession13releaseCursorEmP11__wt_cursor","s+":"47"}, {"a":"55EE0EDB01B9","b":"55EE0DAD5000","o":"12DB1B9","s":"_ZN5mongo35WiredTigerRecordStoreStandardCursorD0Ev","s+":"39"}, {"a":"55EE0F5C0A1C","b":"55EE0DAD5000","o":"1AEBA1C","s":"_ZN5mongo14CollectionScanD0Ev","s+":"2C"}, {"a":"55EE0F6228D9","b":"55EE0DAD5000","o":"1B4D8D9","s":"_ZN5mongo16PlanExecutorImplD1Ev","s+":"189"}, {"a":"55EE0F622961","b":"55EE0DAD5000","o":"1B4D961","s":"_ZN5mongo16PlanExecutorImplD0Ev","s+":"11"}, {"a":"55EE0F7F9CDC","b":"55EE0DAD5000","o":"1D24CDC","s":"_ZN5mongo8Pipeline7disposeEPNS_16OperationContextE","s+":"3C"}, {"a":"55EE0F6201F3","b":"55EE0DAD5000","o":"1B4B1F3","s":"_ZN5mongo16PlanExecutorImpl7disposeEPNS_16OperationContextE","s+":"53"}, {"a":"55EE0F5ACA9A","b":"55EE0DAD5000","o":"1AD7A9A","s":"_ZN5mongo15ClientCursorPin16deleteUnderlyingEv","s+":"5A"}, {"a":"55EE0E8164D9","b":"55EE0DAD5000","o":"D414D9","s":"_ZN5mongo12runAggregateEPNS_16OperationContextERKNS_15NamespaceStringERKNS_18AggregationRequestERKNS_18LiteParsedPipelineERKNS_7BSONObjERKSt6vectorINS_9PrivilegeESaISF_EEPNS_3rpc21ReplyBuilderInterfaceE.cold.1394","s+":"2DF"}, {"a":"55EE0F2C94BC","b":"55EE0DAD5000","o":"17F44BC","s":"_ZN5mongo12_GLOBAL__N_115PipelineCommand10Invocation3runEPNS_16OperationContextEPNS_3rpc21ReplyBuilderInterfaceE","s+":"6C"}, {"a":"55EE0F8553AF","b":"55EE0DAD5000","o":"1D803AF","s":"_ZN5mongo14CommandHelpers20runCommandInvocationEPNS_16OperationContextERKNS_12OpMsgRequestEPNS_17CommandInvocationEPNS_3rpc21ReplyBuilderInterfaceE","s+":"7F"}, {"a":"55EE0EFF9A9F","b":"55EE0DAD5000","o":"1524A9F","s":"_ZN5mongo12_GLOBAL__N_114runCommandImplEPNS_16OperationContextEPNS_17CommandInvocationERKNS_12OpMsgRequestEPNS_3rpc21ReplyBuilderInterfaceENS_11LogicalTimeERKNS_23ServiceEntryPointCommon5HooksEPNS_14BSONObjBuilderERKNS_30OperationSessionInfoFromClientE","s+":"9EF"}, {"a":"55EE0EFFC049","b":"55EE0DAD5000","o":"1527049","s":"_ZN5mongo12_GLOBAL__N_119execCommandDatabaseEPNS_16OperationContextEPNS_7CommandERKNS_12OpMsgRequestEPNS_3rpc21ReplyBuilderInterfaceERKNS_23ServiceEntryPointCommon5HooksE","s+":"11B9"}, {"a":"55EE0EFFD330","b":"55EE0DAD5000","o":"1528330","s":"_ZN5mongo12_GLOBAL__N_116receivedCommandsEPNS_16OperationContextERKNS_7MessageERKNS_23ServiceEntryPointCommon5HooksE","s+":"650"}, {"a":"55EE0EFFDFBD","b":"55EE0DAD5000","o":"1528FBD","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE","s+":"50D"}, {"a":"55EE0EFEBBDC","b":"55EE0DAD5000","o":"1516BDC","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE","s+":"3C"}, {"a":"55EE0EFF64F5","b":"55EE0DAD5000","o":"15214F5","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE","s+":"F5"}, {"a":"55EE0EFF4375","b":"55EE0DAD5000","o":"151F375","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE","s+":"125"}, {"a":"55EE0EFF5306","b":"55EE0DAD5000","o":"1520306","s":"_ZNSt17_Function_handlerIFvvEZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS2_11ThreadGuardENS1_9transport15ServiceExecutor13ScheduleFlagsENS4_23ServiceExecutorTaskNameENS2_9OwnershipEEUlvE_E9_M_invokeERKSt9_Any_data","s+":"56"}, {"a":"55EE102DA202","b":"55EE0DAD5000","o":"2805202","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE","s+":"182"}, {"a":"55EE0EFF378B","b":"55EE0DAD5000","o":"151E78B","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE","s+":"DB"}, {"a":"55EE0EFF4AED","b":"55EE0DAD5000","o":"151FAED","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE","s+":"6AD"}, {"a":"55EE0EFF4BD0","b":"55EE0DAD5000","o":"151FBD0","s":"_ZN5mongo14future_details4callIRZNS_19ServiceStateMachine14_sourceMessageENS2_11ThreadGuardEEUlNS_10StatusWithINS_7MessageEEEE0_S6_EEDaOT_OT0_","s+":"60"}, {"a":"55EE0EFF4E75","b":"55EE0DAD5000","o":"151FE75","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE","s+":"145"}, {"a":"55EE0EFF43EA","b":"55EE0DAD5000","o":"151F3EA","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE","s+":"19A"}, {"a":"55EE0EFF5306","b":"55EE0DAD5000","o":"1520306","s":"_ZNSt17_Function_handlerIFvvEZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS2_11ThreadGuardENS1_9transport15ServiceExecutor13ScheduleFlagsENS4_23ServiceExecutorTaskNameENS2_9OwnershipEEUlvE_E9_M_invokeERKSt9_Any_data","s+":"56"}, {"a":"55EE102DA868","b":"55EE0DAD5000","o":"2805868","s":"_ZNSt17_Function_handlerIFvvEZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIS0_ENS2_15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameEEUlvE0_E9_M_invokeERKSt9_Any_data","s+":"B8"}, {"a":"55EE1062BE56","b":"55EE0DAD5000","o":"2B56E56","s":"_ZNSt17_Function_handlerIFvvEZN5mongo25launchServiceWorkerThreadESt8functionIS0_EEUlvE1_E9_M_invokeERKSt9_Any_data","s+":"56"}, {"a":"55EE1062BEC4","b":"55EE0DAD5000","o":"2B56EC4","s":"_ZN5mongo12_GLOBAL__N_17runFuncEPv","s+":"14"}, {"a":"7FCE279E1EA5","b":"7FCE279DA000","o":"7EA5","s":"start_thread","s+":"C5"}, {"a":"7FCE2770A96D","b":"7FCE2760C000","o":"FE96D","s":"clone","s+":"6D"}],"processInfo":{"mongodbVersion":"4.4.4","gitVersion":"8db30a63db1a9d84bdcad0c83369623f708e0397","compiledModules":["enterprise"],"uname":{"sysname":"Linux","release":"3.10.0-1160.11.1.el7.x86_64","version":"#1 SMP Fri Dec 18 16:34:56 UTC 2020","machine":"x86_64"},"somap":[{"b":"55EE0DAD5000","elfType":3,"buildId":"87F025C58225BB56443EE3EEEDAD584F66ABB9E5"}, {"b":"7FCE279DA000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"2B482B3BAE79DEF4E5BC9791BC6BBDAE0E93E359"}, {"b":"7FCE2760C000","path":"/lib64/libc.so.6","elfType":3,"buildId":"F9FAFDE281E0E0E2AF45911AD0FA115B64C2CEA8"}]}}

Top User Comments

xgen-internal-githook commented on Mon, 9 May 2022 15:39:29 +0000: Author: {'name': 'Yuhong Zhang', 'email': 'yuhong.zhang@mongodb.com', 'username': 'YuhongZhang98'} Message: SERVER-55173 Skip releasing cursors during shutdown to avoid segmentation fault (cherry picked from commit bc940d6b0adc9254b62e9daf9c92d2c92f8b083d) Branch: v4.4 https://github.com/mongodb/mongo/commit/7a074786f4048e3ac304dfbab12a5fafd5f14f00 xgen-internal-githook commented on Tue, 19 Apr 2022 03:35:56 +0000: Author: {'name': 'Yuhong Zhang', 'email': 'yuhong.zhang@mongodb.com', 'username': 'YuhongZhang98'} Message: SERVER-55173 Skip releasing cursors during shutdown to avoid segmentation fault (cherry picked from commit bc940d6b0adc9254b62e9daf9c92d2c92f8b083d) Branch: v5.0 https://github.com/mongodb/mongo/commit/71b0968070ba18cb59f285f3e0c007e935599d59 xgen-internal-githook commented on Fri, 15 Apr 2022 20:57:35 +0000: Author: {'name': 'Yuhong Zhang', 'email': 'yuhong.zhang@mongodb.com', 'username': 'YuhongZhang98'} Message: SERVER-55173 Skip releasing cursors during shutdown to avoid segmentation fault (cherry picked from commit bc940d6b0adc9254b62e9daf9c92d2c92f8b083d) Branch: v5.3 https://github.com/mongodb/mongo/commit/1359075515532b245d0b77717b6ee13cbc268d97 xgen-internal-githook commented on Mon, 28 Mar 2022 14:00:59 +0000: Author: {'name': 'Yuhong Zhang', 'email': 'yuhong.zhang@mongodb.com', 'username': 'YuhongZhang98'} Message: SERVER-55173 Skip releasing cursors during shutdown to avoid segmentation fault Branch: master https://github.com/mongodb/mongo/commit/bc940d6b0adc9254b62e9daf9c92d2c92f8b083d kyle.suarez commented on Mon, 27 Sep 2021 21:09:08 +0000: After a good discussion, we are pursuing matthew.saltz's suggestion to wait in the WiredTigerCursorCache for all cursors to be returned to the cache before proceeding to shut down. Out of our several proposed solutions, this one is the simplest, and we will get quick feedback on whether or not this proposal is viable. Sending this to the Storage Execution team backlog as we've agreed it makes the most sense for them to explore this change. CC louis.williams JIRAUSER1262199 commented on Thu, 2 Sep 2021 12:51:23 +0000: We're encountering same error on `4.2.12` & `4.2.15` version Mongod. Signal 11 is raised and mongo service entering failed/crashed state. Segementation Fault is observed on all other related nodes having same mongod `4.2.12` & `4.2.15` 2021-08-13T15:30:01.237+0000 F  - [conn684] Invalid access at address: 0x55650b02a0002021-08-13T15:30:01.285+0000 F  - [conn684] Got signal: 11 (Segmentation fault). 0x55626cc65241 0x55626cc6477e 0x55626cc6495c 0x7f63495c1630 0x7f634535c41c----- BEGIN BACKTRACE -----{"backtrace":[{"b":"55626A306000","o":"295F241","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55626A306000","o":"295E77E"},{"b":"55626A306000","o":"295E95C"},{"b":"7F63495B2000","o":"F630"},{"b":"7F6345310000","o":"4C41C"}],"processInfo":{ "mongodbVersion" : "4.2.15", "gitVersion" : "d7fd78dead621a539c20791a93abec34bb1be385", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-1160.36.2.el7.x86_64", "version" : "#1 SMP Wed Jul 21 11:57:15 UTC 2021", "machine" : "x86_64" }, "somap" : [ { "b" : "55626A306000", "elfType" : 3, "buildId" : "3B94268B834009D9D3B84B8E1A69A6F240DD54C5" }, { "b" : "7FFEE3C8E000", "elfType" : 3, "buildId" : "1D0D0FA3882A1742CF8DE0974C7FC6FC7CAD8185" }, { "b" : "7F634A93F000", "path" : "/lib64/libcurl.so.4", "elfType" : 3, "buildId" : "AA185EEE9B8EE0BCE652F7A6E9AAD07B1B3808C4" }, { "b" : "7F634A725000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "B45C711D26DDD9F612D7814CE83B427927C8BC65" }, { "b" : "7F634A365000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "7829C3727DE1DD529EBA8E477756D10267F5656E" }, { "b" : "7F634A0F2000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "4474A303DA0EA2DF580F0648779A5E8F62F0433A" }, { "b" : "7F6349EEE000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7F2E9CB0769D7E57BD669B485A74B537B63A57C4" }, { "b" : "7F6349CE6000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "3E44DF7055942478D052E40FDD1F5B7862B152B0" }, { "b" : "7F63499E4000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "7615604EAF4A068DFAE5085444D15C0DEE93DFBD" }, { "b" : "7F63497CE000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "EDF51350C7F71496149D064AA8B1441F786DF88A" }, { "b" : "7F63495B2000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "E10CC8F2B932FC3DAEDA22F8DAC5EBB969524E5B" }, { "b" : "7F63491E4000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "A317B42B15368ADCAE21C11107691A03EC91059D" }, { "b" : "7F634ABA9000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "62C449974331341BB08DCCE3859560A22AF1E172" }, { "b" : "7F6348FB1000", "path" : "/lib64/libidn.so.11", "elfType" : 3, "buildId" : "2B77BBEFFF65E94F3E0B71A4E89BEB68C4B476C5" }, { "b" : "7F6348D84000", "path" : "/lib64/libssh2.so.1", "elfType" : 3, "buildId" : "CB0BD6C014F41EC926FAC41322C82FF4A5EB88B9" }, { "b" : "7F6348B27000", "path" : "/lib64/libssl3.so", "elfType" : 3, "buildId" : "3B88957E9F14880EA8331F24EADD1B48C37FF032" }, { "b" : "7F63488FF000", "path" : "/lib64/libsmime3.so", "elfType" : 3, "buildId" : "CED43363B6A38A426D2A5EEECC7A267DD7BFBD60" }, { "b" : "7F63485CA000", "path" : "/lib64/libnss3.so", "elfType" : 3, "buildId" : "3A70799DB1310CAFAE52B163B3046EBA6C7AA10D" }, { "b" : "7F634839A000", "path" : "/lib64/libnssutil3.so", "elfType" : 3, "buildId" : "A61604F9C4E3F975A0A1742174F08D6ECF987A63" }, { "b" : "7F6348196000", "path" : "/lib64/libplds4.so", "elfType" : 3, "buildId" : "E761D8698407F6521F26F579D61D5EC8F7EF04A9" }, { "b" : "7F6347F91000", "path" : "/lib64/libplc4.so", "elfType" : 3, "buildId" : "41E234507D6BB1E4FE56A078127D36C1963460CC" }, { "b" : "7F6347D53000", "path" : "/lib64/libnspr4.so", "elfType" : 3, "buildId" : "051956498509E67F79215B76781C1AA0504EB5D5" }, { "b" : "7F6347B06000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "0CAEC124D97114DA40DDEB0FED1FAD5D14C3D626" }, { "b" : "7F634781D000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "4EBF28968DA5784ED6606BFF7C1915C50AC24502" }, { "b" : "7F63475EA000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "5FF9D1075A8D5D62F77F5CE56C935FCD92C62EFA" }, { "b" : "7F63473E6000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "2ADDB65846A50CE45F0C9B62EA35DDA62C6AD7A2" }, { "b" : "7F63471D7000", "path" : "/lib64/liblber-2.4.so.2", "elfType" : 3, "buildId" : "8804516A3226CFB54589FEC0E27D89C93DAF92FF" }, { "b" : "7F6346F82000", "path" : "/lib64/libldap-2.4.so.2", "elfType" : 3, "buildId" : "82277E588B8B663A0A53F4E9516AEC1802BD4FC3" }, { "b" : "7F6346D6C000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "B04855870B0DE434F354DE3147230F2677200B56" }, { "b" : "7F6346B04000", "path" : "/lib64/libbwrap.so.1", "elfType" : 3, "buildId" : "0B5E54BCDB9970EB56C5F8A7029969ABB6C37B1F" }, { "b" : "7F63468F4000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "779381063DAECC27E8480C8F79F0651162586478" }, { "b" : "7F63466F0000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7F63464D3000", "path" : "/lib64/libsasl2.so.3", "elfType" : 3, "buildId" : "E2F2017F821DD1B9D307DA1A9B8014F2941AEB7B" }, { "b" : "7F63462AC000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "903A0BD0BFB4FEE8C284F41BEB9773DED94CBC52" }, { "b" : "7F6346075000", "path" : "/lib64/libcrypt.so.1", "elfType" : 3, "buildId" : "97BE6F9199FED4491B00AA91F7E6EACC4D5328F7" }, { "b" : "7F6345E13000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "9CA3D11F018BEEB719CDB34BE800BF1641350D0A" }, { "b" : "7F6345C10000", "path" : "/lib64/libfreebl3.so", "elfType" : 3, "buildId" : "020C788B41DCC71AEE66B822D7670BC4347DA006" }, { "b" : "7F63459C2000", "path" : "/usr/lib64/bwrap-1.5/libcryptocme.so", "elfType" : 3, "buildId" : "2DD77A93881EE4CFB883B14BC6D8D10C3184B38C" }, { "b" : "7F63457BC000", "path" : "/usr/lib64/bwrap-1.5/libccme_error_info.so", "elfType" : 3, "buildId" : "DDFAB5F4AB36FD6F8F3D63D980D448871940F310" }, { "b" : "7F63455B6000", "path" : "/usr/lib64/bwrap-1.5/libccme_aux_entropy.so", "elfType" : 3, "buildId" : "72880238B13986785D452F3319EDE010806338E6" }, { "b" : "7F6345310000", "path" : "/usr/lib64/bwrap-1.5/libccme_base.so", "elfType" : 3, "buildId" : "A9269B4C5DAB2A2C323871125515D887340EB64A" }, { "b" : "7F634508D000", "path" : "/usr/lib64/bwrap-1.5/libccme_asym.so", "elfType" : 3, "buildId" : "4F591A87B4AA7D6D7A019BE035C0DAEF19CBF3B2" }, { "b" : "7F6344D99000", "path" : "/usr/lib64/bwrap-1.5/libccme_ecc.so", "elfType" : 3, "buildId" : "C9D87694F8B506588CE5BB4019EEEF78485E09A8" }, { "b" : "7F6344B05000", "path" : "/usr/lib64/bwrap-1.5/libccme_base_non_fips.so", "elfType" : 3, "buildId" : "949E6E036E8FA3697242B222D379FACC58102A0B" }, { "b" : "7F63448E6000", "path" : "/usr/lib64/bwrap-1.5/libccme_ecc_non_fips.so", "elfType" : 3, "buildId" : "B622529118F685678E12F2858980DCE223012434" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55626cc65241] mongod(+0x295E77E) [0x55626cc6477e] mongod(+0x295E95C) [0x55626cc6495c] libpthread.so.0(+0xF630) [0x7f63495c1630] libccme_base.so(+0x4C41C) [0x7f634535c41c]-----  END BACKTRACE  ----- JIRAUSER1257640 commented on Thu, 10 Jun 2021 19:30:12 +0000: Per discussion during my meeting with Dave, sending this to him to investigate and Dave can hand it off with what he finds later. louis.williams commented on Mon, 17 May 2021 13:36:17 +0000: Thanks robert@appsignal.com, this confirms my suspicion in my previous comment. This cursorFreer needs to be run while still holding a lock. robert@appsignal.com commented on Mon, 17 May 2021 12:38:54 +0000: We're seeing the same error when shutting down a `4.2.12` version Mongod. This error happened on multiple servers when shutting down (e.g. all 3 nodes in the same replica set had this segfault). 2021-05-17T14:31:31.827+0200 F - [conn1733223] Invalid access at address: 0 2021-05-17T14:31:31.840+0200 I NETWORK [conn1733166] end connection 84.22.106.252:10949 (291 connections now open) 2021-05-17T14:31:31.853+0200 I NETWORK [conn1731034] end connection 37.252.124.172:24617 (290 connections now open) 2021-05-17T14:31:31.870+0200 I NETWORK [conn1730405] end connection 84.22.112.55:46699 (289 connections now open) 2021-05-17T14:31:31.871+0200 I NETWORK [conn1730249] end connection 84.22.96.76:23181 (288 connections now open) 2021-05-17T14:31:31.874+0200 I NETWORK [conn1733342] end connection 46.19.34.32:30499 (287 connections now open) 2021-05-17T14:31:31.893+0200 I NETWORK [conn1730402] end connection 84.22.96.76:26757 (286 connections now open) 2021-05-17T14:31:31.904+0200 F - [conn1733223] Got signal: 11 (Segmentation fault). 0x55e81fef76e1 0x55e81fef6d0c 0x55e81fef6ef0 0x7f76d75ab8a0 0x55e81e4a9d18 0x55e81e042923 0x55e81e036ef6 0x55e81e40f8c7 0x55e81e404f99 0x55e81ed90f8c 0x55e81ed91041 0x55e81eda880f 0x55e81ed90fe7 0x55e81ed91041 0x55e81eda4a9f 0x55e81edf1ff9 0x55e81edf20c1 0x55e81ed71085 0x55e81ed71544 0x55e81e114272 0x55e81eaab10f 0x55e81e78b5c8 0x55e81e78d754 0x55e81e78e4a3 0x55e81e77afbc 0x55e81e787f75 0x55e81e7854af 0x55e81e786e7c 0x55e81f60b652 0x55e81e7824fd 0x55e81e783cfb 0x55e81e784796 0x55e81e78540b 0x55e81e786e7c 0x55e81f60babb 0x55e81fc67e05 0x55e81fc67e64 0x7f76d75a06db 0x7f76d72c9a3f ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"55E81D5DB000","o":"291C6E1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55E81D5DB000","o":"291BD0C"},{"b":"55E81D5DB000","o":"291BEF0"},{"b":"7F76D7599000","o":"128A0"},{"b":"55E81D5DB000","o":"ECED18"},{"b":"55E81D5DB000","o":"A67923","s":"__wt_err_func"},{"b":"55E81D5DB000","o":"A5BEF6"},{"b":"55E81D5DB000","o":"E348C7","s":"_ZN5mongo17WiredTigerSession13releaseCursorEmP11__wt_cursor"},{"b":"55E81D5DB000","o":"E29F99","s":"_ZN5mongo35WiredTigerRecordStoreStandardCursorD0Ev"},{"b":"55E81D5DB000","o":"17B5F8C","s":"_ZN5mongo10FetchStageD1Ev"},{"b":"55E81D5DB000","o":"17B6041","s":"_ZN5mongo10FetchStageD0Ev"},{"b":"55E81D5DB000","o":"17CD80F","s":"_ZN5mongo7OrStageD0Ev"},{"b":"55E81D5DB000","o":"17B5FE7","s":"_ZN5mongo10FetchStageD1Ev"},{"b":"55E81D5DB000","o":"17B6041","s":"_ZN5mongo10FetchStageD0Ev"},{"b":"55E81D5DB000","o":"17C9A9F","s":"_ZN5mongo14MultiPlanStageD0Ev"},{"b":"55E81D5DB000","o":"1816FF9","s":"_ZN5mongo16PlanExecutorImplD1Ev"},{"b":"55E81D5DB000","o":"18170C1","s":"_ZN5mongo16PlanExecutorImplD0Ev"},{"b":"55E81D5DB000","o":"1796085","s":"_ZN5mongo12ClientCursorD1Ev"},{"b":"55E81D5DB000","o":"1796544","s":"_ZN5mongo15ClientCursorPin16deleteUnderlyingEv"},{"b":"55E81D5DB000","o":"B39272"},{"b":"55E81D5DB000","o":"14D010F"},{"b":"55E81D5DB000","o":"11B05C8"},{"b":"55E81D5DB000","o":"11B2754"},{"b":"55E81D5DB000","o":"11B34A3","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE"},{"b":"55E81D5DB000","o":"119FFBC","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE"},{"b":"55E81D5DB000","o":"11ACF75","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},{"b":"55E81D5DB000","o":"11AA4AF","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"55E81D5DB000","o":"11ABE7C"},{"b":"55E81D5DB000","o":"2030652","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE"},{"b":"55E81D5DB000","o":"11A74FD","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE"},{"b":"55E81D5DB000","o":"11A8CFB","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"55E81D5DB000","o":"11A9796","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},{"b":"55E81D5DB000","o":"11AA40B","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"55E81D5DB000","o":"11ABE7C"},{"b":"55E81D5DB000","o":"2030ABB"},{"b":"55E81D5DB000","o":"268CE05"},{"b":"55E81D5DB000","o":"268CE64"},{"b":"7F76D7599000","o":"76DB"},{"b":"7F76D71A8000","o":"121A3F","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.2.12", "gitVersion" : "5593fd8e33b60c75802edab304e23998fa0ce8a5", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.15.0-142-generic", "version" : "#146-Ubuntu SMP Tue Apr 13 01:11:19 UTC 2021", "machine" : "x86_64" }, "somap" : [ { "b" : "55E81D5DB000", "elfType" : 3, "buildId" : "67F1C1D532D1AF44F4FDE9EAC1BC6EFA05924909" }, { "b" : "7FFDCCFC5000", "path" : "linux-vdso.so.1", "elfType" : 3, "buildId" : "880476BBBE1C1622524EA4155E4E261134A14B20" }, { "b" : "7F76D8AED000", "path" : "/usr/lib/x86_64-linux-gnu/libcurl.so.4", "elfType" : 3, "buildId" : "1ACB5F477F6957894410C316595BCE8DB4976EC4" }, { "b" : "7F76D88D2000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "965581AF9EDD1C7EADF35822F93C1C1D95E52AAC" }, { "b" : "7F76D8407000", "path" : "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1", "elfType" : 3, "buildId" : "E4277CE3A9D34904899488641C2403D6792DA413" }, { "b" : "7F76D817A000", "path" : "/usr/lib/x86_64-linux-gnu/libssl.so.1.1", "elfType" : 3, "buildId" : "CA742A427E7AEF089B39C4D773C20EA9E074CE8E" }, { "b" : "7F76D7F76000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "B22BAF34FB22284EC8E3818961CDF01CCAB3441C" }, { "b" : "7F76D7D6E000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "3F29B196C7C124797473113FD2D0833881BF0DE5" }, { "b" : "7F76D79D0000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "9EF1967ED985A60AC2288C3E1D8C8375F48B841D" }, { "b" : "7F76D77B8000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "679F3AE11120EC7C483BC9295345D836F5C104F7" }, { "b" : "7F76D7599000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "BC3C06107774266C5F7DB3F1F380A3DA68AF90FA" }, { "b" : "7F76D71A8000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "D3CF764B2F97AC3EFE366DDD07AD902FB6928FD7" }, { "b" : "7F76D8D6F000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "C93445FE9506EEE727E6F04F1AC8F460E49EB366" }, { "b" : "7F76D6F83000", "path" : "/usr/lib/x86_64-linux-gnu/libnghttp2.so.14", "elfType" : 3, "buildId" : "4F00E5207693FDC249DA42EC6472ACA6A7B929AE" }, { "b" : "7F76D6D66000", "path" : "/usr/lib/x86_64-linux-gnu/libidn2.so.0", "elfType" : 3, "buildId" : "EE6E9462BA2491F4EE8C4E52C3323274A9366614" }, { "b" : "7F76D6B4A000", "path" : "/usr/lib/x86_64-linux-gnu/librtmp.so.1", "elfType" : 3, "buildId" : "69465D8AA6B19086ABF2455A703F9168BF82A69F" }, { "b" : "7F76D693C000", "path" : "/usr/lib/x86_64-linux-gnu/libpsl.so.5", "elfType" : 3, "buildId" : "CDAF1F1946846941F9D06414EC8C812D131A168E" }, { "b" : "7F76D66F1000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "9FD187FE6579DB3B458E021CCAC0ECCAFC66B1B7" }, { "b" : "7F76D649F000", "path" : "/usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2", "elfType" : 3, "buildId" : "3455E5726038F673FC9C90385079A2113FE4F55E" }, { "b" : "7F76D6291000", "path" : "/usr/lib/x86_64-linux-gnu/liblber-2.4.so.2", "elfType" : 3, "buildId" : "5F842A7CA8F5C84B338CCCB703F88F020057C77E" }, { "b" : "7F76D6074000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "EF3E006DFE3132A41D4D4DC0E407D6EA658E11C4" }, { "b" : "7F76D5CF6000", "path" : "/usr/lib/x86_64-linux-gnu/libunistring.so.2", "elfType" : 3, "buildId" : "0E2784298E7D3F4D894FE130ACEFA77C3E624F72" }, { "b" : "7F76D5990000", "path" : "/usr/lib/x86_64-linux-gnu/libgnutls.so.30", "elfType" : 3, "buildId" : "1954013F9BBF9FF3A328A8E516481A2BBD8658EB" }, { "b" : "7F76D575C000", "path" : "/usr/lib/x86_64-linux-gnu/libhogweed.so.4", "elfType" : 3, "buildId" : "CD1E2E2E2B242BD7E34395153EC38DCC35192B20" }, { "b" : "7F76D5526000", "path" : "/usr/lib/x86_64-linux-gnu/libnettle.so.6", "elfType" : 3, "buildId" : "9114D09EA71726FD7D0E06379DECE57F951CB87E" }, { "b" : "7F76D52A5000", "path" : "/usr/lib/x86_64-linux-gnu/libgmp.so.10", "elfType" : 3, "buildId" : "D40EA9B5EC5BC46799E4A412319617BD38BE9341" }, { "b" : "7F76D4FCF000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "320112EE17C7F1886DBEB108429063B22D996E9E" }, { "b" : "7F76D4D9D000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "729F89DD92E3441FA99F8682533F7B92EE48880D" }, { "b" : "7F76D4B99000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "17107881DF65C66B4C6D38CAB37C285FA44663BD" }, { "b" : "7F76D498E000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "EAE217D2281E12BCD43675C73C59DF29D7167C66" }, { "b" : "7F76D4773000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "AA489C2B43FF4886958E90DFBD9DB3825D8F6A9A" }, { "b" : "7F76D4532000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi.so.3", "elfType" : 3, "buildId" : "A1A98DB481968073636BBAECB561A3EA8ED198AE" }, { "b" : "7F76D4203000", "path" : "/usr/lib/x86_64-linux-gnu/libp11-kit.so.0", "elfType" : 3, "buildId" : "8CFF559D91E3376842A1BA41CCF74EB9DECD159B" }, { "b" : "7F76D3FF0000", "path" : "/usr/lib/x86_64-linux-gnu/libtasn1.so.6", "elfType" : 3, "buildId" : "6036B89A3BB671B32E01464C0C82BFA016186352" }, { "b" : "7F76D3DEC000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "F463E107B099910463BC32E837C73D341A52C27B" }, { "b" : "7F76D3BE3000", "path" : "/usr/lib/x86_64-linux-gnu/libheimntlm.so.0", "elfType" : 3, "buildId" : "C2376C5B831991591F1A67B976758185F86896D8" }, { "b" : "7F76D3956000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.26", "elfType" : 3, "buildId" : "69BDEE5FA0FEEDF317308BE850F78761861D520A" }, { "b" : "7F76D36B4000", "path" : "/usr/lib/x86_64-linux-gnu/libasn1.so.8", "elfType" : 3, "buildId" : "315D74995AAA32DE4D15BA25F335066988B1B230" }, { "b" : "7F76D347E000", "path" : "/usr/lib/x86_64-linux-gnu/libhcrypto.so.4", "elfType" : 3, "buildId" : "6673972A1C24A89EBAFBAE696188A4CB26C6DDEB" }, { "b" : "7F76D3268000", "path" : "/usr/lib/x86_64-linux-gnu/libroken.so.18", "elfType" : 3, "buildId" : "430827C33259C12248CF44B91A9A9821114376F5" }, { "b" : "7F76D3060000", "path" : "/usr/lib/x86_64-linux-gnu/libffi.so.6", "elfType" : 3, "buildId" : "3555B5F599C9787DFDDBF9E8DF6F706B9044D985" }, { "b" : "7F76D2E37000", "path" : "/usr/lib/x86_64-linux-gnu/libwind.so.0", "elfType" : 3, "buildId" : "93A0931B1C2818F0EA224CE6FE5E31E84A9B55BB" }, { "b" : "7F76D2C28000", "path" : "/usr/lib/x86_64-linux-gnu/libheimbase.so.1", "elfType" : 3, "buildId" : "669D4CCE42FA4382796EFFCF0C16F459F4382C4C" }, { "b" : "7F76D29DE000", "path" : "/usr/lib/x86_64-linux-gnu/libhx509.so.5", "elfType" : 3, "buildId" : "4B80C543356EE0AF9039EFE7C9EA1CC1F74C426A" }, { "b" : "7F76D26D5000", "path" : "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "elfType" : 3, "buildId" : "1E52E04B0722C6CFB8E534FB8DA45DDF95B46C19" }, { "b" : "7F76D249D000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "0E6872C952DC832E87A90BD2A50DC9C88AC441EF" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55e81fef76e1] mongod(+0x291BD0C) [0x55e81fef6d0c] mongod(+0x291BEF0) [0x55e81fef6ef0] libpthread.so.0(+0x128A0) [0x7f76d75ab8a0] mongod(+0xECED18) [0x55e81e4a9d18] mongod(__wt_err_func+0x90) [0x55e81e042923] mongod(+0xA5BEF6) [0x55e81e036ef6] mongod(_ZN5mongo17WiredTigerSession13releaseCursorEmP11__wt_cursor+0x47) [0x55e81e40f8c7] mongod(_ZN5mongo35WiredTigerRecordStoreStandardCursorD0Ev+0x39) [0x55e81e404f99] mongod(_ZN5mongo10FetchStageD1Ev+0x2C) [0x55e81ed90f8c] mongod(_ZN5mongo10FetchStageD0Ev+0x11) [0x55e81ed91041] mongod(_ZN5mongo7OrStageD0Ev+0x9F) [0x55e81eda880f] mongod(_ZN5mongo10FetchStageD1Ev+0x87) [0x55e81ed90fe7] mongod(_ZN5mongo10FetchStageD0Ev+0x11) [0x55e81ed91041] mongod(_ZN5mongo14MultiPlanStageD0Ev+0x30F) [0x55e81eda4a9f] mongod(_ZN5mongo16PlanExecutorImplD1Ev+0x159) [0x55e81edf1ff9] mongod(_ZN5mongo16PlanExecutorImplD0Ev+0x11) [0x55e81edf20c1] mongod(_ZN5mongo12ClientCursorD1Ev+0x95) [0x55e81ed71085] mongod(_ZN5mongo15ClientCursorPin16deleteUnderlyingEv+0x84) [0x55e81ed71544] mongod(+0xB39272) [0x55e81e114272] mongod(+0x14D010F) [0x55e81eaab10f] mongod(+0x11B05C8) [0x55e81e78b5c8] mongod(+0x11B2754) [0x55e81e78d754] mongod(_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE+0x413) [0x55e81e78e4a3] mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x3C) [0x55e81e77afbc] mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0xE5) [0x55e81e787f75] mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x17F) [0x55e81e7854af] mongod(+0x11ABE7C) [0x55e81e786e7c] mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE+0x182) [0x55e81f60b652] mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE+0x10D) [0x55e81e7824fd] mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0x6DB) [0x55e81e783cfb] mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x316) [0x55e81e784796] mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0xDB) [0x55e81e78540b] mongod(+0x11ABE7C) [0x55e81e786e7c] mongod(+0x2030ABB) [0x55e81f60babb] mongod(+0x268CE05) [0x55e81fc67e05] mongod(+0x268CE64) [0x55e81fc67e64] libpthread.so.0(+0x76DB) [0x7f76d75a06db] libc.so.6(clone+0x3F) [0x7f76d72c9a3f] ----- END BACKTRACE ----- louis.williams commented on Fri, 19 Mar 2021 15:05:46 +0000: The cursorFreer, calls into the storage engine to reset the cursor without holding locks, which technically isn't allowed. Is this happening during a rollback or shutdown?

Steps to Reproduce