Info

I am using mongodb 3.0.7, storage engine - wiredTiger My server unexpected power down and the mongod instance stops unexpectedly,now I start cann‘t start mongodb and repair always with error [2016-01-29 08:51:47] mongodb mongodb.conf is dead,restarting.. 2016-01-29T08:51:47.077+0800 I CONTROL ***** SERVER RESTARTED ***** 2016-01-29T08:51:47.116+0800 W - [initandlisten] Detected unclean shutdown - /home/zenic/mongodb/data/mongod.lock is not empty. 2016-01-29T08:51:47.116+0800 W STORAGE [initandlisten] Recovering data from the last clean checkpoint. 2016-01-29T08:51:47.116+0800 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=1G,session_max=20000,eviction=(threads_max=4),statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0), 2016-01-29T08:51:47.132+0800 E STORAGE [initandlisten] WiredTiger (0) [1454028707:132450][17057:0x7f130a908c80], file:WiredTiger.wt, connection: read checksum error for 4096B block at offset 102400: block header checksum of 1600484457 doesn't match expected checksum of 12792921 2016-01-29T08:51:47.132+0800 E STORAGE [initandlisten] WiredTiger (0) [1454028707:132759][17057:0x7f130a908c80], file:WiredTiger.wt, connection: WiredTiger.wt: encountered an illegal file format or internal value 2016-01-29T08:51:47.132+0800 E STORAGE [initandlisten] WiredTiger (-31804) [1454028707:132820][17057:0x7f130a908c80], file:WiredTiger.wt, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic 2016-01-29T08:51:47.132+0800 I - [initandlisten] Fatal Assertion 28558 2016-01-29T08:51:47.147+0800 I CONTROL [initandlisten] 0xf758c9 0xf12561 0xef63e1 0xd94a6a 0x13aab79 0x13aad35 0x13ab1d4 0x12fc0ae 0x12fc603 0x12f93c8 0x12fd328 0x1316da2 0x134070b 0x13a9d28 0x1377adb 0x133ddc3 0xd7ea3f 0xd7c5b8 0xa7e8ed 0x7f5806 0x7fabc9 0x7f1308ec5af5 0x7f378d ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B758C9"},{"b":"400000","o":"B12561"},{"b":"400000","o":"AF63E1"},{"b":"400000","o":"994A6A"},{"b":"400000","o":"FAAB79"},{"b":"400000","o":"FAAD35"},{"b":"400000","o":"FAB1D4"},{"b":"400000","o":"EFC0AE"},{"b":"400000","o":"EFC603"},{"b":"400000","o":"EF93C8"},{"b":"400000","o":"EFD328"},{"b":"400000","o":"F16DA2"},{"b":"400000","o":"F4070B"},{"b":"400000","o":"FA9D28"},{"b":"400000","o":"F77ADB"},{"b":"400000","o":"F3DDC3"},{"b":"400000","o":"97EA3F"},{"b":"400000","o":"97C5B8"},{"b":"400000","o":"67E8ED"},{"b":"400000","o":"3F5806"},{"b":"400000","o":"3FABC9"},{"b":"7F1308EA4000","o":"21AF5"},{"b":"400000","o":"3F378D"}],"processInfo":{ "mongodbVersion" : "3.0.7", "gitVersion" : "6ce7cbe8c6b899552dadd907604559806aa2e9bd", "uname" : { "sysname" : "Linux", "release" : "3.10.0-123.el7.x86_64", "version" : "#1 SMP Mon Jun 30 12:09:22 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E359E52F4A5E2A16804BCA34965CC29081397CDF" }, { "b" : "7FFF912FE000", "elfType" : 3, "buildId" : "D7952DC468957C2B14B6BB79E613D48BA1224706" }, { "b" : "7F130A4E1000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "18562EE0363BC9BD7101610BD86469AA426D0C44" }, { "b" : "7F130A275000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "58FEDFFED1A388AD9E495F9A6C91A851B9537765" }, { "b" : "7F1309E90000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F214E8640FDA5097E7A90CE7974B3FF76C6C42D9" }, { "b" : "7F1309C88000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8832E3070AB0758762836EEC8FCDDEDEF8235340" }, { "b" : "7F1309A84000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "B7C4BC0854BF5DE16B535353B38235CA42349C1E" }, { "b" : "7F130977D000", "path" : "/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "63C62D6263FF98E6DD6896CB3E716E499744A4C9" }, { "b" : "7F130947B000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "D70EAB176DDA46DE292FEB8208A0E8A6718BAF3B" }, { "b" : "7F1309265000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "07120A9AC1BF3BCDD4A3EA1E0C47234A4A5C84F9" }, { "b" : "7F1308EA4000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "78186287BBA77069A056A5CCBEB14B7FD2CA3A4B" }, { "b" : "7F130A6FD000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "4EADCA6CB82E0A85EDB87C15B5E3980742514501" }, { "b" : "7F1308C5A000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "641A441AB91715A7E3AF8AD9AF38EE07F17866FE" }, { "b" : "7F130897A000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "08E8BA638E79EC07F98198ED40F90FA87D5EEEB5" }, { "b" : "7F1308776000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "D2678F5F391BF2877E1BD6FAD16DBC589ED0BBF3" }, { "b" : "7F1308541000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "8269E77C68B707158D2B1BEA356EE0FC2A1C0024" }, { "b" : "7F130832B000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "E45643F27F3B3E960F3691AFC6EC27A98EF7B46B" }, { "b" : "7F130811D000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "577A21CDAA3D662B87D53AFAA12A1E7B34AD513F" }, { "b" : "7F1307F19000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7F1307CFF000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "519F19CF966514EAC9B25BE6FE953750E466D3C1" }, { "b" : "7F1307ADA000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "82FF6B18E1E42825CC2D060F969479AD4AF2F62C" }, { "b" : "7F1307879000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "B19961A753FDFF85BD071340139A7F024BAEFFCA" }, { "b" : "7F1307654000", "path" : "/lib64/liblzma.so.5", "elfType" : 3, "buildId" : "218D03D1F6CF1A099A4D467B5E8ECF4F2BF45750" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf758c9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf12561] mongod(_ZN5mongo13fassertFailedEi+0x61) [0xef63e1] mongod(+0x994A6A) [0xd94a6a] mongod(__wt_eventv+0x489) [0x13aab79] mongod(__wt_err+0x95) [0x13aad35] mongod(__wt_panic+0x24) [0x13ab1d4] mongod(__wt_block_extlist_read+0x6E) [0x12fc0ae] mongod(__wt_block_extlist_read_avail+0x33) [0x12fc603] mongod(__wt_block_checkpoint_load+0x1C8) [0x12f93c8] mongod(+0xEFD328) [0x12fd328] mongod(__wt_btree_open+0xAA2) [0x1316da2] mongod(__wt_conn_btree_get+0x19B) [0x134070b] mongod(__wt_session_get_btree+0x3F8) [0x13a9d28] mongod(__wt_metadata_open+0x2B) [0x1377adb] mongod(wiredtiger_open+0xCD3) [0x133ddc3] mongod(_ZN5mongo18WiredTigerKVEngineC1ERKSsS2_bb+0x52F) [0xd7ea3f] mongod(+0x97C5B8) [0xd7c5b8] mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x30D) [0xa7e8ed] mongod(_ZN5mongo13initAndListenEi+0x246) [0x7f5806] mongod(main+0x139) [0x7fabc9] libc.so.6(__libc_start_main+0xF5) [0x7f1308ec5af5] mongod(+0x3F378D) [0x7f378d] ----- END BACKTRACE ----- 2016-01-29T08:51:47.147+0800 I - [initandlisten] ***aborting after fassert() failure 14 [2016-01-29 08:51:47] mongodb mongodb.conf is running.. when delete the mongod.lock file and running command for repair then the problem remain [root@sdn32 bin]# ./mongod --dbpath /home/zenic/mongodb/data --storageEngine wiredTiger --repair --repairpath /home/zenic/mongodb/data0 2016-01-29T08:59:31.518+0800 I STORAGE [initandlisten] Detected WT journal files. Running recovery from last checkpoint. 2016-01-29T08:59:31.518+0800 I STORAGE [initandlisten] journal to nojournal transition config: create,cache_size=4G,session_max=20000,eviction=(threads_max=4),statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0), 2016-01-29T08:59:31.533+0800 E STORAGE [initandlisten] WiredTiger (0) [1454029171:533420][13575:0x7fd440366c80], file:WiredTiger.wt, connection: read checksum error for 4096B block at offset 102400: block header checksum of 1600484457 doesn't match expected checksum of 12792921 2016-01-29T08:59:31.533+0800 E STORAGE [initandlisten] WiredTiger (0) [1454029171:533611][13575:0x7fd440366c80], file:WiredTiger.wt, connection: WiredTiger.wt: encountered an illegal file format or internal value 2016-01-29T08:59:31.533+0800 E STORAGE [initandlisten] WiredTiger (-31804) [1454029171:533683][13575:0x7fd440366c80], file:WiredTiger.wt, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic 2016-01-29T08:59:31.533+0800 I - [initandlisten] Fatal Assertion 28558 2016-01-29T08:59:31.549+0800 I CONTROL [initandlisten] 0xf758c9 0xf12561 0xef63e1 0xd94a6a 0x13aab79 0x13aad35 0x13ab1d4 0x12fc0ae 0x12fc603 0x12f93c8 0x12fd328 0x1316da2 0x134070b 0x13a9d28 0x1377adb 0x133ddc3 0xd7e90c 0xd7c5b8 0xa7e8ed 0x7f5806 0x7fabc9 0x7fd43e923af5 0x7f378d ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B758C9"},{"b":"400000","o":"B12561"},{"b":"400000","o":"AF63E1"},{"b":"400000","o":"994A6A"},{"b":"400000","o":"FAAB79"},{"b":"400000","o":"FAAD35"},{"b":"400000","o":"FAB1D4"},{"b":"400000","o":"EFC0AE"},{"b":"400000","o":"EFC603"},{"b":"400000","o":"EF93C8"},{"b":"400000","o":"EFD328"},{"b":"400000","o":"F16DA2"},{"b":"400000","o":"F4070B"},{"b":"400000","o":"FA9D28"},{"b":"400000","o":"F77ADB"},{"b":"400000","o":"F3DDC3"},{"b":"400000","o":"97E90C"},{"b":"400000","o":"97C5B8"},{"b":"400000","o":"67E8ED"},{"b":"400000","o":"3F5806"},{"b":"400000","o":"3FABC9"},{"b":"7FD43E902000","o":"21AF5"},{"b":"400000","o":"3F378D"}],"processInfo":{ "mongodbVersion" : "3.0.7", "gitVersion" : "6ce7cbe8c6b899552dadd907604559806aa2e9bd", "uname" : { "sysname" : "Linux", "release" : "3.10.0-123.el7.x86_64", "version" : "#1 SMP Mon Jun 30 12:09:22 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E359E52F4A5E2A16804BCA34965CC29081397CDF" }, { "b" : "7FFF76229000", "elfType" : 3, "buildId" : "D7952DC468957C2B14B6BB79E613D48BA1224706" }, { "b" : "7FD43FF3F000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "18562EE0363BC9BD7101610BD86469AA426D0C44" }, { "b" : "7FD43FCD3000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "58FEDFFED1A388AD9E495F9A6C91A851B9537765" }, { "b" : "7FD43F8EE000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F214E8640FDA5097E7A90CE7974B3FF76C6C42D9" }, { "b" : "7FD43F6E6000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8832E3070AB0758762836EEC8FCDDEDEF8235340" }, { "b" : "7FD43F4E2000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "B7C4BC0854BF5DE16B535353B38235CA42349C1E" }, { "b" : "7FD43F1DB000", "path" : "/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "63C62D6263FF98E6DD6896CB3E716E499744A4C9" }, { "b" : "7FD43EED9000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "D70EAB176DDA46DE292FEB8208A0E8A6718BAF3B" }, { "b" : "7FD43ECC3000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "07120A9AC1BF3BCDD4A3EA1E0C47234A4A5C84F9" }, { "b" : "7FD43E902000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "78186287BBA77069A056A5CCBEB14B7FD2CA3A4B" }, { "b" : "7FD44015B000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "4EADCA6CB82E0A85EDB87C15B5E3980742514501" }, { "b" : "7FD43E6B8000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "641A441AB91715A7E3AF8AD9AF38EE07F17866FE" }, { "b" : "7FD43E3D8000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "08E8BA638E79EC07F98198ED40F90FA87D5EEEB5" }, { "b" : "7FD43E1D4000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "D2678F5F391BF2877E1BD6FAD16DBC589ED0BBF3" }, { "b" : "7FD43DF9F000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "8269E77C68B707158D2B1BEA356EE0FC2A1C0024" }, { "b" : "7FD43DD89000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "E45643F27F3B3E960F3691AFC6EC27A98EF7B46B" }, { "b" : "7FD43DB7B000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "577A21CDAA3D662B87D53AFAA12A1E7B34AD513F" }, { "b" : "7FD43D977000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7FD43D75D000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "519F19CF966514EAC9B25BE6FE953750E466D3C1" }, { "b" : "7FD43D538000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "82FF6B18E1E42825CC2D060F969479AD4AF2F62C" }, { "b" : "7FD43D2D7000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "B19961A753FDFF85BD071340139A7F024BAEFFCA" }, { "b" : "7FD43D0B2000", "path" : "/lib64/liblzma.so.5", "elfType" : 3, "buildId" : "218D03D1F6CF1A099A4D467B5E8ECF4F2BF45750" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf758c9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf12561] mongod(_ZN5mongo13fassertFailedEi+0x61) [0xef63e1] mongod(+0x994A6A) [0xd94a6a] mongod(__wt_eventv+0x489) [0x13aab79] mongod(__wt_err+0x95) [0x13aad35] mongod(__wt_panic+0x24) [0x13ab1d4] mongod(__wt_block_extlist_read+0x6E) [0x12fc0ae] mongod(__wt_block_extlist_read_avail+0x33) [0x12fc603] mongod(__wt_block_checkpoint_load+0x1C8) [0x12f93c8] mongod(+0xEFD328) [0x12fd328] mongod(__wt_btree_open+0xAA2) [0x1316da2] mongod(__wt_conn_btree_get+0x19B) [0x134070b] mongod(__wt_session_get_btree+0x3F8) [0x13a9d28] mongod(__wt_metadata_open+0x2B) [0x1377adb] mongod(wiredtiger_open+0xCD3) [0x133ddc3] mongod(_ZN5mongo18WiredTigerKVEngineC1ERKSsS2_bb+0x3FC) [0xd7e90c] mongod(+0x97C5B8) [0xd7c5b8] mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x30D) [0xa7e8ed] mongod(_ZN5mongo13initAndListenEi+0x246) [0x7f5806] mongod(main+0x139) [0x7fabc9] libc.so.6(__libc_start_main+0xF5) [0x7fd43e923af5] mongod(+0x3F378D) [0x7f378d] ----- END BACKTRACE ----- 2016-01-29T08:59:31.549+0800 I - [initandlisten] ***aborting after fassert() failure

Top User Comments

ramon.fernandez commented on Wed, 10 Feb 2016 20:05:47 +0000: In SERVER-22211 another user experienced corruption in the sizeStorer.wt file, which we tracked down to bad memory. Can you please check your system logs for January and February for system errors related to kernel bugs or memory issues? Note that the shortest recovery path at this stage is to resync from a healthy node. If external corruption happened this or a restore from backups may be the only way to retrieve your data. littleboy commented on Sat, 6 Feb 2016 01:28:41 +0000: Dear Ramon Fernandez ,Thanks for your help. I have extracted them and replaced my files. But there is a new error after my instance is started. Would you please give me some other advice? [2016-02-06 08:20:41] mongodb mongodb.conf is dead,restarting.. 2016-02-06T08:20:41.306+0800 I CONTROL ***** SERVER RESTARTED ***** 2016-02-06T08:20:41.371+0800 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=1G,session_max=20000,eviction=(threads_max=4),statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0), 2016-02-06T08:20:41.454+0800 E STORAGE [initandlisten] WiredTiger (0) [1454718041:454535][14646:0x7fb40c431c20], file:sizeStorer.wt: read checksum error for 4096B block at offset 20480: block header checksum of 2182306285 doesn't match expected checksum of 3244867404 2016-02-06T08:20:41.454+0800 E STORAGE [initandlisten] WiredTiger (0) [1454718041:454653][14646:0x7fb40c431c20], file:sizeStorer.wt: sizeStorer.wt: encountered an illegal file format or internal value 2016-02-06T08:20:41.454+0800 E STORAGE [initandlisten] WiredTiger (-31804) [1454718041:454689][14646:0x7fb40c431c20], file:sizeStorer.wt: the process must exit and restart: WT_PANIC: WiredTiger library panic 2016-02-06T08:20:41.454+0800 I - [initandlisten] Fatal Assertion 28558 2016-02-06T08:20:41.486+0800 I CONTROL [initandlisten] 0xf758c9 0xf12561 0xef63e1 0xd94a6a 0x13aab79 0x13aad35 0x13ab1d4 0x12fe752 0x131842c 0x13161a8 0x1316fed 0x134070b 0x13a9d28 0x13aa1aa 0x134d9a1 0x13a7dea 0x13b621f 0x13b6599 0x136585c 0x13b716c 0x1343da1 0x133e29f 0xd7ea3f 0xd7c5b8 0xa7e8ed 0x7f5806 0x7fabc9 0x3f8a41ecdd 0x7f378d ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B758C9"},{"b":"400000","o":"B12561"},{"b":"400000","o":"AF63E1"},{"b":"400000","o":"994A6A"},{"b":"400000","o":"FAAB79"},{"b":"400000","o":"FAAD35"},{"b":"400000","o":"FAB1D4"},{"b":"400000","o":"EFE752"},{"b":"400000","o":"F1842C"},{"b":"400000","o":"F161A8"},{"b":"400000","o":"F16FED"},{"b":"400000","o":"F4070B"},{"b":"400000","o":"FA9D28"},{"b":"400000","o":"FAA1AA"},{"b":"400000","o":"F4D9A1"},{"b":"400000","o":"FA7DEA"},{"b":"400000","o":"FB621F"},{"b":"400000","o":"FB6599"},{"b":"400000","o":"F6585C"},{"b":"400000","o":"FB716C"},{"b":"400000","o":"F43DA1"},{"b":"400000","o":"F3E29F"},{"b":"400000","o":"97EA3F"},{"b":"400000","o":"97C5B8"},{"b":"400000","o":"67E8ED"},{"b":"400000","o":"3F5806"},{"b":"400000","o":"3FABC9"},{"b":"3F8A400000","o":"1ECDD"},{"b":"400000","o":"3F378D"}],"processInfo":{ "mongodbVersion" : "3.0.7", "gitVersion" : "6ce7cbe8c6b899552dadd907604559806aa2e9bd", "uname" : { "sysname" : "Linux", "release" : "2.6.32-358.el6.x86_64", "version" : "#1 SMP Sun Oct 20 20:02:37 EDT 2013", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E359E52F4A5E2A16804BCA34965CC29081397CDF" }, { "b" : "7FFFBBE70000", "elfType" : 3, "buildId" : "06E4CCE25F95E18BE74E260F0C8EC45F4AE5912C" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "D04F937E9034A0FCF9B2F3774E2308EF3F6244BE" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "5F67D4146E7A0908470ACAD61D365D49A2928D1E" }, { "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "102B1F02F0A96E18EA11141F7BE94CEA81DC952F" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "12E218F80C42E325CB9133887BE9D387AA4A0A3D" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "F7F21994DBB68BF6DE41C1BF86F34BFF488388B3" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "A1B95A762722F8C194732E744D7E675B709756F0" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "3297CD6333EE6C6799FAFC010A1A8983E72E6D17" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "E243AEA761C71C48F7B8C851C04E82B20E728A39" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "1B5A94B8D8B4E3C289401E43FCCA3785AB8EF546" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "4BEEFCFA9DB6982EB57BF15D91399BB0EF233DDF" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "E07E0FE45B730A8EB5D337359D89D599B0322E35" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "B2288408053E82C1420469BE5FCB601D67FDFB45" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "B5C6BCABB9804C1609AE06DF4DB26354B75998A7" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "3521356BE501D8A080BABA8EF7661A81B70FEF0F" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "6173D43E7E7E2A856777F928013894EC02C554CE" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "1D8F7059616680E04F04404F8B3B80543AF7A36D" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "0AC91E08BD23BF25110A45AFD35DA18BA15F1285" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "6A8608A91A0F17B56C67B648CC126A5198A416CA" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "9CCE433B7402733480B7DF2D6B4B45D03F283E8C" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf758c9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf12561] mongod(_ZN5mongo13fassertFailedEi+0x61) [0xef63e1] mongod(+0x994A6A) [0xd94a6a] mongod(__wt_eventv+0x489) [0x13aab79] mongod(__wt_err+0x95) [0x13aad35] mongod(__wt_panic+0x24) [0x13ab1d4] mongod(__wt_bm_read+0x72) [0x12fe752] mongod(__wt_bt_read+0x1AC) [0x131842c] mongod(__wt_btree_tree_open+0x58) [0x13161a8] mongod(__wt_btree_open+0xCED) [0x1316fed] mongod(__wt_conn_btree_get+0x19B) [0x134070b] mongod(__wt_session_get_btree+0x3F8) [0x13a9d28] mongod(__wt_session_get_btree_ckpt+0xAA) [0x13aa1aa] mongod(__wt_curfile_open+0xE1) [0x134d9a1] mongod(__wt_open_cursor+0x26A) [0x13a7dea] mongod(+0xFB621F) [0x13b621f] mongod(+0xFB6599) [0x13b6599] mongod(__wt_log_scan+0x8EC) [0x136585c] mongod(__wt_txn_recover+0x48C) [0x13b716c] mongod(__wt_connection_workers+0x61) [0x1343da1] mongod(wiredtiger_open+0x11AF) [0x133e29f] mongod(_ZN5mongo18WiredTigerKVEngineC1ERKSsS2_bb+0x52F) [0xd7ea3f] mongod(+0x97C5B8) [0xd7c5b8] mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x30D) [0xa7e8ed] mongod(_ZN5mongo13initAndListenEi+0x246) [0x7f5806] mongod(main+0x139) [0x7fabc9] libc.so.6(__libc_start_main+0xFD) [0x3f8a41ecdd] mongod(+0x3F378D) [0x7f378d] ----- END BACKTRACE ----- 2016-02-06T08:20:41.487+0800 I - [initandlisten] ***aborting after fassert() failure 14 [2016-02-06 08:20:41] mongodb mongodb.conf is running.. ramon.fernandez commented on Fri, 5 Feb 2016 14:15:38 +0000: littleboy, I've attached a new file to this ticket, repair_attempt.tgz, containing the WiredTiger.wt and WiredTiger.turtle files after a repair attempt. Can you please extract them in place of your existing ones and try to start this instance again? Thanks, Ramón.

Steps to Reproduce