Symptoms

The NetWorker scanning process receives a segmentation fault on one of the devices during recovery test. Error code: 8792:scanner: done with ssid 503876771 8850:scanner: ssid 2427915217: found beginning of xxxxxxxx.yyyyyyyy.com:index:18377328-00000004-607ef81e-607ef81d-0efc1a00-07f0734a Program received signal SIGSEGV, Segmentation fault. 0x0000000000433cd8 in get_cur_clone (ss=0x2172000, slp=0x216e4d0, volidp=0xb7a150 , is_aftd=0) at /disks/nasbld/nas65/nw/19.9/nsr/mm/scan_it.c:5800 5800/disks/nasbld/nas65/nw/19.9/nsr/mm/scan_it.c: No such file or directory. Missing separate debuginfos, use: debuginfo-install lgtonode-19.8.0.3-1.x86_64

Cause

Code Malfunction. Below is the stack trace of the Core file: (gdb) where #0 0x00000000004330f0 in get_cur_clone (ss=0x2156620, slp=0x214ded0, volidp=0xb623d0 , is_aftd=1) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:5725 #1 0x000000000043b53f in add_to_list (ss=0x7ffffffe35c0, v=, av=0xdb8fe0) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:4578 #2 0x000000000043d1f1 in process_note (cp=, av=0xdb8fe0, files_to_skipp=0x7ffffffee9b0, skip_ss=0x7ffffffee9bc) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:5487 #3 0x000000000043e089 in handle_filesys_device (av=0xdb8fe0, ssid=0x7ffffffeec90, client_id=0x0) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:6764 #4 0x000000000043f6b0 in scan_it (av=0xdb8fe0, fn_p=0x7fffffffe448, rn=0, options=, verbose=6) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:383 #5 0x0000000000431563 in main (argc=0, argv=0x7fffffffe598) at /130gb_ws/code-git-new/19.8/nsr/mm/scanner.c:953 (gdb) info thr Id Target Id Frame * 1 Thread 0x7ffff7fed740 (LWP 21031) 0x00000000004330f0 in get_cur_clone (ss=0x2156620, slp=0x214ded0, volidp=0xb623d0 , is_aftd=1) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:5725 (gdb) bt full #0 0x00000000004330f0 in get_cur_clone (ss=0x2156620, slp=0x214ded0, volidp=0xb623d0 , is_aftd=1) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:5725 i = j = cp = 0x0 fg = #1 0x000000000043b53f in add_to_list (ss=0x7ffffffe35c0, v=, av=0xdb8fe0) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:4578 clone_id = 1695110774 cp = st = 0xb623d1 sst = 0x2156620 stuffit = 1 continued_ss = volname = "\000\000\000\000\000\000\000\000C\335x\000\000\000\000\000 \000\000\000\000\000\000\000\060}`\000\000\000\000\000\030\275\024\002\000\000\000\000\001\000\000\000\000\000\000\000 \275\024\002\000\000\000\000\377\377\377\377\000\000\000\000p6\376\377\377\177\000\000\001\000\000\000\000\000\000\000 \000\000\000\000\000\000\000\264\335x\000\000\000\000\000\000\000\000\000\001\000\000\000p\227`" skip_mdb_update = #2 0x000000000043d1f1 in process_note (cp=, av=0xdb8fe0, files_to_skipp=0x7ffffffee9b0, skip_ss=0x7ffffffee9bc) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:5487 xdrm = {x_op = XDR_DECODE, x_ops = 0xb5dc40 , x_public = 0x0, x_private = 0x214debc " 1697352866q\002", x_base = 0x214d9f0 "\200", x_handy = 0, x_dandy = 34916792, x_priv2 = 0x1 , x_pub2 = 0x214c9c0 ""} sl = trouble = 0 ss_buf = {ss_gen = 2147483652, ss_ssid = {_private = 100663296, _bytes = "\367\tUXe\tUX\024\n\032\000\a\360sJ"}, ss_prev = {_private = 0, _bytes = '\000' }, ss_level = 0, ss_time = 1695110482, ss_create = 1695110488, ss_insert = 1695110488, ss_complete = 1695110495, ss_clientid = {_private = 67108864, _bytes = "TNA\347TNA\346\000\001\032\000\a\360sJ"}, ss_flags = 65793, ss_host = 0x2029d40 "xxxxxx.yyyyyyy.com", ss_name = 0x214c400 "bootstrap", ss_size = 0, ss_nfiles = 521, ss_browse = 2642311, ss_recycle = 2642311, ss_al = 0x131d840, ss_clones = {ss_clones_len = 1, ss_clones_val = 0x214bd00}} notessid = ssidb_buf = {ssi_host = '\000' , ssi_name = '\000' , ssi_time = 0, ssi_expiry = 140737325152176, ssi_size = 18446603336221313601, ssi_nfiles = 140737488238016, ssi_ssid = 4294849983, ssi_flag = 0, ssi_info = 0, ssi_rid = '\000' } ss_expiry = state = 0x0 err = volume = 0x0 #3 0x000000000043e089 in handle_filesys_device (av=0xdb8fe0, ssid=0x7ffffffeec90, client_id=0x0) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:6764 files_to_skipp = 0 ss = 0x214e210 chunks = {{mc_ssid = {_private = 150994944, _bytes = "\000\000\000\035", '\000' }, mc_low = 0, mc_data = { mc_data_len = 1228, mc_data_val = 0x214d9f0 "\200"}}, {mc_ssid = {_private = 150994944, _bytes = "\000\000\000\035", '\000' }, mc_low = 0, mc_data = {mc_data_len = 1228, mc_data_val = 0x2154430 "\200"}}} ---Type to continue, or q to quit--- mkstdin = 0 trouble = ssfilename = 0x4cc skip_ss = 0 is_objstr = 0 pathname = '\000' , "/xxxxxxxxxxxx/xxxxxxxxxxxx_compliance_clone004/active", '\000' ... sa = {av = 0x0, fd = 0, inbuf = '\000' ..., offset = 0} err = sb = {sb = {st_dev = 15448184, st_ino = 140737321422164, st_nlink = 0, st_mode = 20, st_uid = 0, st_gid = 82160, pad0 = 0, st_rdev = 140737488283904, st_size = 34942064, st_blksize = 4108, st_blocks = 82160, st_atim = {tv_sec = 20, tv_nsec = 140737488284000}, st_mtim = {tv_sec = 15366064, tv_nsec = 82160}, st_ctim = {tv_sec = 140737321422944, tv_nsec = 5707792}, __unused = {140737321423208, 4294967420, 0}}, lg_st_ino64 = 0, lg_st_size64 = 12233632, lg_st_blocks64 = 140737488283984} last_bogus_ssid = {_private = 0, _bytes = '\000' } #4 0x000000000043f6b0 in scan_it (av=0xdb8fe0, fn_p=0x7fffffffe448, rn=0, options=, verbose=6) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:383 mrec = { mr_handler = "9\000.e\000\000\000\000\230\345\377\377\377\177\000\000\240\213\262\000\000\000\000\000\020\326\316\000\000\000\000\000\220\354\376\377\377\177\000\000\204\374|\000\000\000\000\000Q\227.e\000\000\000\000\230\345\377\377\377\177\000\000\000\000\000\000\000\000\000\000\220\253\321\000\000\000\000\000Q\227.e\000\000\000\000\310{G", '\000' , mr_version = MREC_VER5, mr_orec = 0, mr_volid = {_private = 0, _bytes = '\000' }, mr_fn = 0, mr_rn = 0, mr_len = 0, mr_chunk = {mr_chunk_len = 0, mr_chunk_val = 0x0}} cp = 0xb62200 clen = 0 mrlen = 2 i = 13555216 j = s = sl = errcnt = bad_volid = 0 bestfn = -1 fn = 4144584024 fn_adjusted = mvolid = {_private = 5, _bytes = "\000\000\000\000\230\345\377\377\377\177\000\000\000\000\000"} currssid = {_private = 100663296, _bytes = "\367\tUXe\tUX\024\n\032\000\a\360sJ"} mfn = 49 mrn = 140737321423570 files_to_skip = 11936256 skip = 0 client_id = 0x0 #5 0x0000000000431563 in main (argc=0, argv=0x7fffffffe598) at /130gb_ws/code-git-new/19.8/nsr/mm/scanner.c:953 c = verbose = 6 sc_RC = ---Type to continue, or q to quit--- error = ssl = act = {__sigaction_handler = {sa_handler = 0x42d460 , sa_sigaction = 0x42d460 }, sa_mask = {__val = {25262595, 0 }}, sa_flags = 268435457, sa_restorer = 0x0} devstat = {sb = {st_dev = 0, st_ino = 0, st_nlink = 0, st_mode = 0, st_uid = 0, st_gid = 0, pad0 = 0, st_rdev = 0, st_size = 0, st_blksize = 0, st_blocks = 0, st_atim = {tv_sec = 0, tv_nsec = 0}, st_mtim = {tv_sec = 0, tv_nsec = 0}, st_ctim = {tv_sec = 0, tv_nsec = 0}, __unused = {0, 0, 0}}, lg_st_ino64 = 0, lg_st_size64 = 0, lg_st_blocks64 = 0} p = 0x0 path_buffer = "xxxxxxxxxxxx_compliance_clone004\000\377\367\377\177\000\000\250\344\377\367\377\177a\000\000P\341\377\367\377\177\000\000\237P\336\367\377\177\000\000\000\000\000\000\000\000\000\000(\002\377\367\377\177\000\000\001\000\000\000\377\177\000\000\000\000\000\000\000\000\000\000\001\000\000\000\377\177\000\000P\341\377\367\377\177\000\000\355\351C+\000\000\000\000't\231\366\377\177\000\000\377\377\377\377", '\000' , "\250\344\377\367\377\177\000\000\360\340\377\377\377\177\000\000\340\340\377\377\377\177\000\000.N=\366\000\000\000\000\336\061A\000\000\000\000\000\377\377\377\377\000\000\000\000"... pathnew = 0xd04650 "/usr/lib/nsr/lib64" type = 0x0 family = 0xe4d170 "disk" in_buf = '\000' ... logical_dev = 0x0 pool = 0x0 ssid_file = 0x0 av = 0xdb8fe0 file_num = 0 record_num = 1 tmp_v = {v_gen = 0, v_volid = {_private = 0, _bytes = '\000' }, v_name = 0x0, v_family = 0x0, v_ftype = 0x0, v_location = 0x0, v_pool = 0x0, v_barcode = 0x0, v_capacity = 0, v_kused = 0, v_krecover = 0, v_nfile = 0, v_nrecord = 0, v_amedia = 0, v_emedia = 0, v_lmedia = 0, v_cmedia = 0, v_rmedia = 0, v_flags = 0, v_mounts = 0, v_recycled = 0, v_avail = 0, v_metric = 0, v_al = 0x0, v_ssid = {ssids_t_len = 0, ssids_t_val = 0x0}} err = ndmp = 0 skip_prompt = 0 mm_t_varp = 0xd1ab90 retention_time = mbs_ssname = '\000' ... mbt_ssname = '\000' devpath = '\000' volume = 0x0 now = 1697552209 hostname = "xxxxxx.yyyyyyy.com", '\000' (gdb) fr 0 #0 0x00000000004330f0 in get_cur_clone (ss=0x2156620, slp=0x214ded0, volidp=0xb623d0 , is_aftd=1) at /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c:5725 5725 in /130gb_ws/code-git-new/19.8/nsr/mm/scan_it.c (gdb) p cp $1 = (ssclone_t *) 0x0 (gdb) p ss $2 = (ss_t *) 0x2156620 (gdb) p *ss $3 = {ss_gen = 2147483649, ss_ssid = {_private = 100663296, _bytes = "\367\tUXe\tUX\024\n\032\000\a\360sJ"}, ss_prev = {_private = 0, _bytes = '\000' }, ss_level = 0, ss_time = 1695110482, ss_create = 1695110488, ss_insert = 1695110488, ss_complete = 0, ss_clientid = {_private = 67108864, _bytes = "TNA\347TNA\346\000\001\032\000\a\360sJ"}, ss_flags = 65796, ss_host = 0x2156870 "xxxxxx.yyyyyyy.com", ss_name = 0x21568d0 "bootstrap", ss_size = 0, ss_nfiles = 0, ss_browse = 2642311, ss_recycle = 2642311, ss_al = 0x21564b0, ss_clones = {ss_clones_len = 0, ss_clones_val = 0x0}} (gdb) Code Malfunction found in NetWorker branch 19.8, 19.9Stack trace

Resolution

Upgrade NetWorker Storage node version to the 19.8.0.4 or 19.9.0.3 version.

Support Cases