forrtl: severe (174): SIGSEGV, segmentation fault occurred Image PC Routine Line Source libcuda.so 00002AAAB45D90D4 Unknown Unknown Unknown libcuda.so 00002AAAB45F9565 Unknown Unknown Unknown libcuda.so 00002AAAB45D9450 Unknown Unknown Unknown libcuda.so 00002AAAB45D9A1C Unknown Unknown Unknown libcuda.so 00002AAAB45D9C30 Unknown Unknown Unknown libcuda.so 00002AAAB45E86D3 Unknown Unknown Unknown libcuda.so 00002AAAB45E9258 Unknown Unknown Unknown libcuda.so 00002AAAB45E98F6 Unknown Unknown Unknown libcuda.so 00002AAAB45C1365 Unknown Unknown Unknown libcuda.so 00002AAAB45C230C Unknown Unknown Unknown libcuda.so 00002AAAB45D2181 Unknown Unknown Unknown libcuda.so 00002AAAB46AF622 Unknown Unknown Unknown libcuda.so 00002AAAB46AF915 Unknown Unknown Unknown libcudart.so.5.5 00002AAAB19E73FD Unknown Unknown Unknown libcudart.so.5.5 00002AAAB19DB280 Unknown Unknown Unknown libcudart.so.5.5 00002AAAB19E5AE6 Unknown Unknown Unknown libcudart.so.5.5 00002AAAB19EA171 Unknown Unknown Unknown libcudart.so.5.5 00002AAAB19D5013 Unknown Unknown Unknown libcudart.so.5.5 00002AAAB19F3BBD Unknown Unknown Unknown pmemd.cuda 0000000000550ED0 Unknown Unknown Unknown pmemd.cuda 00000000004726AF Unknown Unknown Unknown pmemd.cuda 00000000004AD040 Unknown Unknown Unknown pmemd.cuda 0000000000409D1C Unknown Unknown Unknown libc.so.6 00002AAAB370AEAD Unknown Unknown Unknown pmemd.cuda 0000000000409BF9 Unknown Unknown Unknown *** glibc detected *** pmemd.cuda: free(): invalid pointer: 0x0000000002a9b5e0 *** ======= Backtrace: ========= /lib/x86_64-linux-gnu/libc.so.6(+0x76d76)[0x2aaab3762d76] /lib/x86_64-linux-gnu/libc.so.6(cfree+0x6c)[0x2aaab3767b1c] /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcudart.so.5.5(+0x21888)[0x2aaab19e8888] /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcudart.so.5.5(+0x15ab0)[0x2aaab19dcab0] /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcudart.so.5.5(+0x15f75)[0x2aaab19dcf75] /lib/x86_64-linux-gnu/libc.so.6(__cxa_finalize+0xa5)[0x2aaab3723175] /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcudart.so.5.5(+0x6eb6)[0x2aaab19cdeb6] ======= Memory map: ======== 00400000-00e08000 r-xp 00000000 00:13 30021397 /usr/local/programs/common/amber/amber12-pl21-at13-pl22/arch/x86_64-intel_13.1.0.146-cuda_5.5/amber12/bin/pmemd.cuda_SPFP 01007000-0102c000 rw-p 00a07000 00:13 30021397 /usr/local/programs/common/amber/amber12-pl21-at13-pl22/arch/x86_64-intel_13.1.0.146-cuda_5.5/amber12/bin/pmemd.cuda_SPFP 0102c000-0124b000 rw-p 00000000 00:00 0 026b7000-068c3000 rw-p 00000000 00:00 0 [heap] 200000000-200100000 rw-s 00000000 00:04 19662 /dev/zero (deleted) 200100000-1100000000 ---p 00000000 00:00 0 2aaaaaaab000-2aaaaaacb000 r-xp 00000000 00:0c 792717 /lib/x86_64-linux-gnu/ld-2.13.so 2aaaaaacb000-2aaaaaace000 rw-p 00000000 00:00 0 2aaaaaace000-2aaaaaacf000 rw-s 46222d000 00:05 12411 /dev/nvidia2 2aaaaaacf000-2aaaaaad0000 rw-s efd60000 00:05 12411 /dev/nvidia2 2aaaaaad0000-2aaaaaad1000 rw-s 4639ce000 00:05 12411 /dev/nvidia2 2aaaaaad1000-2aaaaaad2000 rw-s efd60000 00:05 12411 /dev/nvidia2 2aaaaaad2000-2aaaaaad3000 rw-s 4675c5000 00:05 12411 /dev/nvidia2 2aaaaaad3000-2aaaaaad4000 rw-s efd60000 00:05 12411 /dev/nvidia2 2aaaaaad4000-2aaaaaad5000 rw-s 46487f000 00:05 12411 /dev/nvidia2 2aaaaaad5000-2aaaaaad6000 rw-s efd60000 00:05 12411 /dev/nvidia2 2aaaaaad6000-2aaaaaad7000 rw-s 44ea16000 00:05 12411 /dev/nvidia2 2aaaaaad7000-2aaaaaad8000 rw-s efd60000 00:05 12411 /dev/nvidia2 2aaaaaad8000-2aaaaaad9000 rw-s 441630000 00:05 12411 /dev/nvidia2 2aaaaaad9000-2aaaaab02000 rw-p 00000000 00:00 0 2aaaaab02000-2aaaaac02000 rw-s 4684c9000 00:05 12411 /dev/nvidia2 2aaaaac02000-2aaaaac23000 rw-p 00000000 00:00 0 2aaaaac23000-2aaaaac43000 rw-s e8000000 00:05 12411 /dev/nvidia2 2aaaaac43000-2aaaaac63000 rw-s e8020000 00:05 12411 /dev/nvidia2 2aaaaac63000-2aaaaac83000 rw-s e8040000 00:05 12411 /dev/nvidia2 2aaaaac83000-2aaaaaca3000 rw-s e8060000 00:05 12411 /dev/nvidia2 2aaaaaca3000-2aaaaacc3000 rw-s e8080000 00:05 12411 /dev/nvidia2 2aaaaacc3000-2aaaaacc4000 rw-s efd60000 00:05 12411 /dev/nvidia2 2aaaaacc4000-2aaaaacc5000 rw-s 464787000 00:05 12411 /dev/nvidia2 2aaaaacc5000-2aaaaacc6000 rw-s efd60000 00:05 12411 /dev/nvidia2 2aaaaacc6000-2aaaaacc7000 rw-s 44f01f000 00:05 12411 /dev/nvidia2 2aaaaacc7000-2aaaaacc8000 rw-s efd61000 00:05 12411 /dev/nvidia2 2aaaaacc8000-2aaaaacc9000 rw-s 4643ba000 00:05 12411 /dev/nvidia2 2aaaaacc9000-2aaaaacca000 rw-s efd61000 00:05 12411 /dev/nvidia2 2aaaaacca000-2aaaaaccb000 r--p 0001f000 00:0c 792717 /lib/x86_64-linux-gnu/ld-2.13.so 2aaaaaccb000-2aaaaaccc000 rw-p 00020000 00:0c 792717 /lib/x86_64-linux-gnu/ld-2.13.so 2aaaaaccc000-2aaaaaccd000 rw-p 00000000 00:00 0 2aaaaaccd000-2aaaab3cd000 r-xp 00000000 00:13 31464014 /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcurand.so.5.5.22 2aaaab3cd000-2aaaab5cc000 ---p 00700000 00:13 31464014 /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcurand.so.5.5.22 2aaaab5cc000-2aaaac99d000 rw-p 006ff000 00:13 31464014 /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcurand.so.5.5.22 2aaaac99d000-2aaaacea6000 rw-p 00000000 00:00 0 2aaaacea6000-2aaab16a1000 r-xp 00000000 00:13 31464004 /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcufft.so.5.5.22 2aaab16a1000-2aaab18a0000 ---p 047fb000 00:13 31464004 /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcufft.so.5.5.22 2aaab18a0000-2aaab1946000 rw-p 047fa000 00:13 31464004 /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcufft.so.5.5.22 2aaab1946000-2aaab19c7000 rw-p 00000000 00:00 0 2aaab19c7000-2aaab1a12000 r-xp 00000000 00:13 31464013 /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcudart.so.5.5.22 2aaab1a12000-2aaab1c11000 ---p 0004b000 00:13 31464013 /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcudart.so.5.5.22 2aaab1c11000-2aaab1c14000 rw-p 0004a000 00:13 31464013 /usr/local/programs/cuda/cuda-5.5/cuda/lib64/libcudart.so.5.5.22 2aaab1c14000-2aaab1c2b000 r-xp 00000000 00:0c 792730 /lib/x86_64-linux-gnu/libpthread-2.13.so 2aaab1c2b000-2aaab1e2a000 ---p 00017000 00:0c 792730 /lib/x86_64-linux-gnu/libpthread-2.13.so 2aaab1e2a000-2aaab1e2b000 r--p 00016000 00:0c 792730 /lib/x86_64-linux-gnu/libpthread-2.13.so 2aaab1e2b000-2aaab1e2c000 rw-p 00017000 00:0c 792730 /lib/x86_64-linux-gnu/libpthread-2.13.so 2aaab1e2c000-2aaab1e30000 rw-p 00000000 00:00 0 2aaab1e30000-2aaab1e57000 r-xp 00000000 00:13 29361719 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libifport.so.5 2aaab1e57000-2aaab2056000 ---p 00027000 00:13 29361719 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libifport.so.5 2aaab2056000-2aaab2059000 rw-p 00026000 00:13 29361719 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libifport.so.5 2aaab2059000-2aaab205f000 rw-p 00000000 00:00 0 2aaab205f000-2aaab217d000 r-xp 00000000 00:13 29361739 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libifcore.so.5 2aaab217d000-2aaab237c000 ---p 0011e000 00:13 29361739 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libifcore.so.5 2aaab237c000-2aaab238e000 rw-p 0011d000 00:13 29361739 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libifcore.so.5 2aaab238e000-2aaab2395000 rw-p 00000000 00:00 0 2aaab2395000-2aaab260f000 r-xp 00000000 00:13 29361677 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libimf.so 2aaab260f000-2aaab280e000 ---p 0027a000 00:13 29361677 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libimf.so 2aaab280e000-2aaab2851000 rw-p 00279000 00:13 29361677 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libimf.so 2aaab2851000-2aaab2ff7000 r-xp 00000000 00:13 29361734 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libsvml.so 2aaab2ff7000-2aaab31f7000 ---p 007a6000 00:13 29361734 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libsvml.so 2aaab31f7000-2aaab321c000 rw-p 007a6000 00:13 29361734 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libsvml.so 2aaab321c000-2aaab329d000 r-xp 00000000 00:0c 792661 /lib/x86_64-linux-gnu/libm-2.13.so 2aaab329d000-2aaab349c000 ---p 00081000 00:0c 792661 /lib/x86_64-linux-gnu/libm-2.13.so 2aaab349c000-2aaab349d000 r--p 00080000 00:0c 792661 /lib/x86_64-linux-gnu/libm-2.13.so 2aaab349d000-2aaab349e000 rw-p 00081000 00:0c 792661 /lib/x86_64-linux-gnu/libm-2.13.so 2aaab349e000-2aaab34e9000 r-xp 00000000 00:13 29361736 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libintlc.so.5 2aaab34e9000-2aaab36e8000 ---p 0004b000 00:13 29361736 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libintlc.so.5 2aaab36e8000-2aaab36eb000 rw-p 0004a000 00:13 29361736 /usr/local/programs/common/intel/compiler/2013.2.146/composer_xe_2013.2.146/compiler/lib/intel64/libintlc.so.5 2aaab36eb000-2aaab36ec000 rw-p 00000000 00:00 0 2aaab36ec000-2aaab386e000 r-xp 00000000 00:0c 792685 /lib/x86_64-linux-gnu/libc-2.13.so 2aaab386e000-2aaab3a6d000 ---p 00182000 00:0c 792685 /lib/x86_64-linux-gnu/libc-2.13.so 2aaab3a6d000-2aaab3a71000 r--p 00181000 00:0c 792685 /lib/x86_64-linux-gnu/libc-2.13.so 2aaab3a71000-2aaab3a72000 rw-p 00185000 00:0c 792685 /lib/x86_64-linux-gnu/libc-2.13.so 2aaab3a72000-2aaab3a77000 rw-p 00000000 00:00 0 2aaab3a77000-2aaab3a8c000 r-xp 00000000 00:0c 792605 /lib/x86_64-linux-gnu/libgcc_s.so.1 2aaab3a8c000-2aaab3c8c000 ---p 00015000 00:0c 792605 /lib/x86_64-linux-gnu/libgcc_s.so.1 2aaab3c8c000-2aaab3c8d000 rw-p 00015000 00:0c 792605 /lib/x86_64-linux-gnu/libgcc_s.so.1 2aaab3c8d000-2aaab3c8f000 r-xp 00000000 00:0c 792555 /lib/x86_64-linux-gnu/libdl-2.13.so 2aaab3c8f000-2aaab3e8f000 ---p 00002000 00:0c 792555 /lib/x86_64-linux-gnu/libdl-2.13.so 2aaab3e8f000-2aaab3e90000 r--p 00002000 00:0c 792555 /lib/x86_64-linux-gnu/libdl-2.13.so 2aaab3e90000-2aaab3e91000 rw-p 00003000 00:0c 792555 /lib/x86_64-linux-gnu/libdl-2.13.so 2aaab3e91000-2aaab3f79000 r-xp 00000000 00:12 2626365 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17 2aaab3f79000-2aaab4179000 ---p 000e8000 00:12 2626365 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17 2aaab4179000-2aaab4181000 r--p 000e8000 00:12 2626365 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17 2aaab4181000-2aaab4183000 rw-p 000f0000 00:12 2626365 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.17 2aaab4183000-2aaab4198000 rw-p 00000000 00:00 0 2aaab4198000-2aaab419f000 r-xp 00000000 00:0c 792654 /lib/x86_64-linux-gnu/librt-2.13.so 2aaab419f000-2aaab439e000 ---p 00007000 00:0c 792654 /lib/x86_64-linux-gnu/librt-2.13.so 2aaab439e000-2aaab439f000 r--p 00006000 00:0c 792654 /lib/x86_64-linux-gnu/librt-2.13.so 2aaab439f000-2aaab43a0000 rw-p 00007000 00:0c 792654 /lib/x86_64-linux-gnu/librt-2.13.so 2aaab43a0000-2aaab4f0b000 r-xp 00000000 00:13 29500473 /usr/local/programs/cuda/driver-331.38/usr/lib/libcuda.so.331.38 2aaab4f0b000-2aaab510a000 ---p 00b6b000 00:13 29500473 /usr/local/programs/cuda/driver-331.38/usr/lib/libcuda.so.331.38 2aaab510a000-2aaab52d6000 rw-p 00b6a000 00:13 29500473 /usr/local/programs/cuda/driver-331.38/usr/lib/libcuda.so.331.38 2aaab52d6000-2aaab52fe000 rw-p 00000000 00:00 0 2aaab52fe000-2aaab5314000 r-xp 00000000 00:0c 792546 /lib/x86_64-linux-gnu/libz.so.1.2.7 2aaab5314000-2aaab5513000 ---p 00016000 00:0c 792546 /lib/x86_64-linux-gnu/libz.so.1.2.7 2aaab5513000-2aaab5514000 r--p 00015000 00:0c 792546 /lib/x86_64-linux-gnu/libz.so.1.2.7 2aaab5514000-2aaab5515000 rw-p 00016000 00:0c 792546 /lib/x86_64-linux-gnu/libz.so.1.2.7 2aaab5515000-2aaab5516000 ---p 00000000 00:00 0 2aaab5516000-2aaab5716000 rwxp 00000000 00:00 0 [stack:4964] 2aaab5716000-2aaab5b16000 rw-s 467627000 00:05 12411 /dev/nvidia2 2aaab5b16000-2aaab5f16000 rw-s 463a60000 00:05 12411 /dev/nvidia2 2aaab5f16000-2aaab6316000 rw-s 46401b000 00:05 12411 /dev/nvidia2 2aaab6316000-2aaab6716000 rw-s 4675ce000 00:05 12411 /dev/nvidia2 2aaab6716000-2aaab6b16000 rw-s 4416c8000 00:05 12411 /dev/nvidia2 2aaab6b16000-2aaab6b36000 rw-s e80a0000 00:05 12411 /dev/nvidia2 2aaab6b36000-2aaab6f36000 rw-s 464800000 00:05 12411 /dev/nvidia2 2aaab6f36000-2aaab6f56000 rw-s e80c0000 00:05 12411 /dev/nvidia2 2aaab6f56000-2aaab7356000 rw-s 441639000 00:05 12411 /dev/nvidia2 2aaab7356000-2aaab7376000 rw-s e80e0000 00:05 12411 /dev/nvidia2 2aaab7376000-2aaab7776000 rw-s 46479c000 00:05 12411 /dev/nvidia2 2aaab7776000-2aaab7796000 rw-s e8100000 00:05 12411 /dev/nvidia2 2aaab7796000-2aaab7b96000 rw-s 44f02a000 00:05 12411 /dev/nvidia2 2aaab7b96000-2aaab7bb6000 rw-s e8120000 00:05 12411 /dev/nvidia2 2aaab7bb6000-2aaab7fb6000 rw-s 4643c3000 00:05 12411 /dev/nvidia2 2aaab7fb6000-2aaab7fb7000 rw-s 463cd1000 00:05 12411 /dev/nvidia2 2aaab7fb7000-2aaab7fb8000 rw-s efd61000 00:05 12411 /dev/nvidia2 2aaab7fb8000-2aaab7fd8000 rw-s e8140000 00:05 12411 /dev/nvidia2 2aaab7fd8000-2aaab7fd9000 rw-s 46a072000 00:05 12411 /dev/nvidia2 2aaab7fd9000-2aaab7fda000 rw-s efd61000 00:05 12411 /dev/nvidia2 2aaab7fda000-2aaab7ffa000 rw-s e8160000 00:05 12411 /dev/nvidia2 2aaab7ffa000-2aaab7ffb000 rw-s 464b8c000 00:05 12411 /dev/nvidia2 2aaab7ffb000-2aaab7ffc000 rw-s efd61000 00:05 12411 /dev/nvidia2 2aaab8000000-2aaab8021000 rw-p 00000000 00:00 0 2aaab8021000-2aaabc000000 ---p 00000000 00:00 0 2aaabc000000-2aaabc400000 rw-s 463cda000 00:05 12411 /dev/nvidia2 2aaabc400000-2aaabc800000 rw-s 46a07c000 00:05 12411 /dev/nvidia2 2aaabc800000-2aaabc900000 rw-s 464b95000 00:05 12411 /dev/nvidia2 2aaabc900000-2aaabca00000 rw-s 4646a8000 00:05 12411 /dev/nvidia2 2aaabca00000-2aaabcb00000 rw-s 44f1aa000 00:05 12411 /dev/nvidia2 2aaabcb00000-2aaabcc00000 rw-s 4415ad000 00:05 12411 /dev/nvidia2 2aaabcc00000-2aaabcd00000 rw-s 46a5af000 00:05 12411 /dev/nvidia2 2aaabcd00000-2aaabcd56000 rw-p 00000000 00:00 0 2aaabcd56000-2aaabcdd7000 rw-s 463eb2000 00:05 12411 /dev/nvidia2 2aaabcdd7000-2aaabd9e5000 rw-p 00000000 00:00 0 7ffffbafd000-7ffffbb58000 rwxp 00000000 00:00 0 [stack] 7ffffbb58000-7ffffbb5a000 rw-p 00000000 00:00 0 7ffffbbea000-7ffffbbeb000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] /var/spool/sge/default/v09/job_scripts/340: line 31: 4960 Aborted (core dumped) pmemd.cuda -O -i NVT.in -o production1_GPU2.out -p $TOP -c 1UAO_1.restart -x production1_GPU2.traj -r production1_GPU2.restart