MongoDB - Defect ID: 398095
Access violation due to a bug in internal page splitting in WiredTiger

Info

Issue Status as of Jan 27, 2018 ISSUE DESCRIPTION AND IMPACT A bug in the algorithm to do page splitting in the WiredTiger storage engine may trigger a segmentation fault, causing a node to shut down defensively to protect user data. This bug only affects nodes running the WiredTiger storage engine. MMAPv1 nodes can't be affected by this bug DIAGNOSIS AND AFFECTED VERSIONS The bug manifests itself with a message in the logs similar to the one below: 2017-06-23T19:03:29.043+0000 F - [thread1] Invalid access at address: 0x78 2017-06-23T19:03:29.073+0000 F - [thread1] Got signal: 11 (Segmentation fault). ----- BEGIN BACKTRACE ----- [...] mongod(+0x160C2BB) [0x1a0c2bb] mongod(__wt_split_multi+0x85) [0x1a105e5] mongod(__wt_evict+0xA55) [0x1a5eac5] [...] ----- END BACKTRACE ----- This bug is present in all production versions of MongoDB that run with the WiredTiger storage engine. However, it is very unlikely to hit this bug in versions older than (and including) 3.2.13 and 3.4.4, as well as MongoDB 3.0. Users running MongoDB 3.2.14 or 3.4.5 are more likely to trigger this bug. REMEDIATION AND WORKAROUNDS A fix for this issue is included in the 3.2.15 and 3.4.6 production releases and later. Original description 2017-06-23T19:03:29.043+0000 F - [thread1] Invalid access at address: 0x78 2017-06-23T19:03:29.073+0000 F - [thread1] Got signal: 11 (Segmentation fault). 0x1349b82 0x1348cd9 0x1349058 0x7f5d4e570100 0x7f5d4e56bbb0 0x1a0c2bb 0x1a105e5 0x1a5eac5 0x1a593ba 0x1a59917 0x1a5b19a 0x1ac37e6 0x7f5d4e568dc5 0x7f5d4e295ced ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"F49B82","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"F48CD9"},{"b":"400000","o":"F49058"},{"b":"7F5D4E561000","o":"F100"},{"b":"7F5D4E561000","o":"ABB0","s":"__pthread_mutex_unlock"},{"b":"400000","o":"160C2BB"},{"b":"400000","o":"16105E5","s":"__wt_split_multi"},{"b":"400000","o":"165EAC5","s":"__wt_evict"},{"b":"400000","o":"16593BA"},{"b":"400000","o":"1659917"},{"b":"400000","o":"165B19A","s":"__wt_evict_thread_run"},{"b":"400000","o":"16C37E6","s":"__wt_thread_run"},{"b":"7F5D4E561000","o":"7DC5"},{"b":"7F5D4E19F000","o":"F6CED","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.14", "gitVersion" : "92f6668a768ebf294bd4f494c50f48459198e6a3", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-327.36.3.el7.x86_64", "version" : "#1 SMP Mon Oct 24 16:09:20 UTC 2016", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "9033F908262138155963DFBEDCA6620B91383BC9" }, { "b" : "7FFE7B88B000", "elfType" : 3, "buildId" : "B6FCE9EF394108DF1141CEAF5B364C0921A92247" }, { "b" : "7F5D4F489000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "7A36D093BFFC7FDC70BCB163A06077F1B059ABD4" }, { "b" : "7F5D4F0A1000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "94BE6FDE7025B1ED23BE379A30C1E0E67BCF61B1" }, { "b" : "7F5D4EE99000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "1D2AD4EAA62BAD560685A4B8DCCC8D9AA95E22CE" }, { "b" : "7F5D4EC95000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "091060A163E7EDA25572F3B1BAF2E8F80209C00E" }, { "b" : "7F5D4E993000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "6DADD94D2A0885D50D09C465EA1970F23FB4629D" }, { "b" : "7F5D4E77D000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "6AA1DCC4DE7F1836344949857FC2017278631FFD" }, { "b" : "7F5D4E561000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "DF6CCEE00C9F4C983A9464E43D17CC3311B51A8F" }, { "b" : "7F5D4E19F000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "53C0918C85FA9CC08D2B57E76467631AB07554AE" }, { "b" : "7F5D4F6F6000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F9AF7BA309F063D3BF9657A21436B4DCAC03CF07" }, { "b" : "7F5D4DF53000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "D46A230FFF4A7B808B3CFC213D31FCAC542FB504" }, { "b" : "7F5D4DC6E000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "6D6136A0E795420B05854DEF13A10C226FE9CCB2" }, { "b" : "7F5D4DA6A000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "3A1166709F88740C49E060731832E3FAD2DFB66B" }, { "b" : "7F5D4D838000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "AA97A848DD7C9E57B06EC913E10D420AEBBCE027" }, { "b" : "7F5D4D622000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "1982C8CDAE90F898D1AD26DC07E807333B4789D0" }, { "b" : "7F5D4D413000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "AEF6C3D3C5152F339942041519A106FC055DAF71" }, { "b" : "7F5D4D20F000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7F5D4CFF5000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "D587388512371B074E16632C01840EA00F0B1175" }, { "b" : "7F5D4CDD0000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "82FF6B18E1E42825CC2D060F969479AD4AF2F62C" }, { "b" : "7F5D4CB6F000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "AE64AA461A26E01F60408013D361749D56DD0AE1" }, { "b" : "7F5D4C94A000", "path" : "/lib64/liblzma.so.5", "elfType" : 3, "buildId" : "98131C9354279ABD39FD80D4BE5B3EC5678BD9E0" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1349b82] mongod(+0xF48CD9) [0x1348cd9] mongod(+0xF49058) [0x1349058] libpthread.so.0(+0xF100) [0x7f5d4e570100] libpthread.so.0(__pthread_mutex_unlock+0x0) [0x7f5d4e56bbb0] mongod(+0x160C2BB) [0x1a0c2bb] mongod(__wt_split_multi+0x85) [0x1a105e5] mongod(__wt_evict+0xA55) [0x1a5eac5] mongod(+0x16593BA) [0x1a593ba] mongod(+0x1659917) [0x1a59917] mongod(__wt_evict_thread_run+0x6A) [0x1a5b19a] mongod(__wt_thread_run+0x16) [0x1ac37e6] libpthread.so.0(+0x7DC5) [0x7f5d4e568dc5] libc.so.6(clone+0x6D) [0x7f5d4e295ced] ----- END BACKTRACE -----

Top User Comments

kiran_naikwade commented on Wed, 21 Feb 2018 13:51:26 +0000: Ok . .Thanks milkie commented on Wed, 21 Feb 2018 13:15:05 +0000: Kiran_Naikwade your stack trace does not look like the one for this ticket. Instead, it looks like it matches the one in SERVER-32677. Please watch that ticket for updates. kiran_naikwade commented on Wed, 21 Feb 2018 10:22:04 +0000: Above issue is also present in 3.6.2 version of MongoDB. Please have look at below logs : "CentOS Linux release 7.4.1708 (Core) ", architecture: "x86_64", version: "Kernel 3.10.0-693.17.1.el7.x86_64" } } 2018-02-21T18:34:27.926+0900 F - [thread13] Invalid access at address: 0x18 2018-02-21T18:34:27.952+0900 F - [thread13] Got signal: 11 (Segmentation fault). ----- BEGIN BACKTRACE ----- {"backtrace":[ {"b":"565114435000","o":"21FD061","s":"_ZN5mongo15printStackTraceERSo"} , {"b":"565114435000","o":"21FC279"} , {"b":"565114435000","o":"21FC8E6"} , {"b":"7F0CB1E5B000","o":"F5E0"} , {"b":"7F0CB1E5B000","o":"9C30","s":"pthread_mutex_lock"} , {"b":"565114435000","o":"19CE836","s":"_ZN5mongo12CatalogCache27invalidateShardedCollectionERKNS_15NamespaceStringE"} , {"b":"565114435000","o":"19D35C1","s":"_ZN5mongo12CatalogCache42getShardedCollectionRoutingInfoWithRefreshEPNS_16OperationContextERKNS_15NamespaceStringE"} , {"b":"565114435000","o":"105FBFE","s":"_ZN5mongo25SessionsCollectionSharded32_checkCacheForSessionsCollectionEPNS_16OperationContextE"} , {"b":"565114435000","o":"105FF20","s":"_ZN5mongo25SessionsCollectionSharded23setupSessionsCollectionEPNS_16OperationContextE"} , {"b":"565114435000","o":"1AD24CA","s":"_ZN5mongo23LogicalSessionCacheImpl8_refreshEPNS_6ClientE"} , {"b":"565114435000","o":"1AD3978","s":"_ZN5mongo23LogicalSessionCacheImpl16_periodicRefreshEPNS_6ClientE"} , {"b":"565114435000","o":"1046F42"} , {"b":"565114435000","o":"1D0FA7A","s":"_ZN4asio6detail14strand_service8dispatchINS0_7binder1ISt8functionIFvSt10error_codeEES5_EEEEvRPNS1_11strand_implERT_"} , {"b":"565114435000","o":"1D1043C","s":"_ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder1INS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEES9_EESB_EEEEvRPNS1_11strand_implERT_"} , {"b":"565114435000","o":"1D10814","s":"_ZN4asio6detail12wait_handlerINS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEEE11do_completeEPvPNS0_19scheduler_operationERKS6_m"} , {"b":"565114435000","o":"1D92719","s":"_ZN4asio6detail9scheduler10do_run_oneERNS0_27conditionally_enabled_mutex11scoped_lockERNS0_21scheduler_thread_infoERKSt10error_code"} , {"b":"565114435000","o":"1D92961","s":"_ZN4asio6detail9scheduler3runERSt10error_code"} , {"b":"565114435000","o":"1045E4D"} , {"b":"565114435000","o":"230AA40"} , {"b":"7F0CB1E5B000","o":"7E25"} , {"b":"7F0CB1A98000","o":"F834D","s":"clone"} ],"processInfo":{ "mongodbVersion" : "3.6.2", "gitVersion" : "489d177dbd0f0420a8ca04d39fd78d0a2c539420", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-693.17.1.el7.x86_64", "version" : "#1 SMP Thu Jan 25 20:13:58 UTC 2018", "machine" : "x86_64" } , "somap" : [ { "b" : "565114435000", "elfType" : 3, "buildId" : "D02F86992A859CC55616E5547729F7F8C6D7F890" } , { "b" : "7FFD678CA000", "elfType" : 3, "buildId" : "47E1DE363A68C3E5970550C87DAFA3CCF9713953" } , { "b" : "7F0CB306E000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "FF4E72F4E574E143330FB3C66DB51613B0EC65EA" } , { "b" : "7F0CB2DFC000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "ED0AC7DEB91A242C194B3DEF27A215F41CE43116" } , { "b" : "7F0CB299B000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "BC0AE9CA0705BEC1F0C0375AAD839843BB219CB1" } , { "b" : "7F0CB2793000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "6D322588B36D2617C03C0F3B93677E62FCFFDA81" } , { "b" : "7F0CB258F000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "1E42EBFB272D37B726F457D6FE3C33D2B094BB69" } , { "b" : "7F0CB228D000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "808BD35686C193F218A5AAAC6194C49214CFF379" } , { "b" : "7F0CB2077000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "3E85E6D20D2CE9CDAD535084BEA56620BAAD687C" } , { "b" : "7F0CB1E5B000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "A48D21B2578A8381FBD8857802EAA660504248DC" } , { "b" : "7F0CB1A98000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "95FF02A4BEBABC573C7827A66D447F7BABDDAA44" } , { "b" : "7F0CB3288000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "22FA66DA7D14C88BF36C69454A357E5F1DEFAE4E" } , { "b" : "7F0CB184B000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "DA322D74F55A0C4293085371A8D0E94B5962F5E7" } , { "b" : "7F0CB1563000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "B69E63024D408E400401EEA6815317BDA38FB7C2" } , { "b" : "7F0CB135F000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "A3832734347DCA522438308C9F08F45524C65C9B" } , { "b" : "7F0CB112C000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "A48639BF901DB554479BFAD114CB354CF63D7D6E" } , { "b" : "7F0CB0F16000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "EA8E45DC8E395CC5E26890470112D97A1F1E0B65" } , { "b" : "7F0CB0D08000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "6FDF5B013FD2739D304CFB9D723DCBC149EE03C9" } , { "b" : "7F0CB0B04000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" } , { "b" : "7F0CB08DD000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "A88379F56A51950A33198890D37F5F8AEE71F8B4" } , { "b" : "7F0CB067B000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "9CA3D11F018BEEB719CDB34BE800BF1641350D0A" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x565116632061] mongod(+0x21FC279) [0x565116631279] mongod(+0x21FC8E6) [0x5651166318e6] libpthread.so.0(+0xF5E0) [0x7f0cb1e6a5e0] libpthread.so.0(pthread_mutex_lock+0x0) [0x7f0cb1e64c30] mongod(_ZN5mongo12CatalogCache27invalidateShardedCollectionERKNS_15NamespaceStringE+0x46) [0x565115e03836] mongod(_ZN5mongo12CatalogCache42getShardedCollectionRoutingInfoWithRefreshEPNS_16OperationContextERKNS_15NamespaceStringE+0x41) [0x565115e085c1] mongod(_ZN5mongo25SessionsCollectionSharded32_checkCacheForSessionsCollectionEPNS_16OperationContextE+0x10E) [0x565115494bfe] mongod(_ZN5mongo25SessionsCollectionSharded23setupSessionsCollectionEPNS_16OperationContextE+0x20) [0x565115494f20] mongod(_ZN5mongo23LogicalSessionCacheImpl8_refreshEPNS_6ClientE+0x12A) [0x565115f074ca] mongod(_ZN5mongo23LogicalSessionCacheImpl16_periodicRefreshEPNS_6ClientE+0x28) [0x565115f08978] mongod(+0x1046F42) [0x56511547bf42] mongod(ZN4asio6detail14strand_service8dispatchINS0_7binder1ISt8functionIFvSt10error_codeEES5_EEEEvRPNS1_11strand_implERT+0x7A) [0x565116144a7a] mongod(ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder1INS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEES9_EESB_EEEEvRPNS1_11strand_implERT+0x3AC) [0x56511614543c] mongod(_ZN4asio6detail12wait_handlerINS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEEE11do_completeEPvPNS0_19scheduler_operationERKS6_m+0x164) [0x565116145814] mongod(_ZN4asio6detail9scheduler10do_run_oneERNS0_27conditionally_enabled_mutex11scoped_lockERNS0_21scheduler_thread_infoERKSt10error_code+0x389) [0x5651161c7719] mongod(_ZN4asio6detail9scheduler3runERSt10error_code+0xD1) [0x5651161c7961] mongod(+0x1045E4D) [0x56511547ae4d] mongod(+0x230AA40) [0x56511673fa40] libpthread.so.0(+0x7E25) [0x7f0cb1e62e25] libc.so.6(clone+0x6D) [0x7f0cb1b9034d] ----- END BACKTRACE -----

Steps to Reproduce