Hi All,
I am trying to use Intel MKL Extended Eigensolver with intel/17.0.1.132 compiler on nersc/cori supercomputer. I am compiling with:
ftn -qopenmp -mkl diagonalise_sparse.f90 read_write_mtx.f90 -o diagonalise
During compilation I get warning:
/opt/intel/compilers_and_libraries_2017.1.132/linux/mkl/lib/intel64/libmkl_core.a(mkl_memory_patched.o): In function `mkl_serv_set_memory_limit': mkl_memory.c:(.text+0x5a9): warning: Using 'dlopen' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
When I run my job with one process it finishes succesfully, however if I increase number of processes I get error
srun -C haswell -n 4 ./diagonalise
forrtl: severe (174): SIGSEGV, segmentation fault occurred
In any case this is my code:
! diagonalise_sparse.f90 program diagonalise use read_write_mtx implicit none include 'mpif.h' include 'mkl.fi' ! INCLUDE 'mkl_solvers_ee.fi' !------------------------------- ! Declarations !------------------------------- integer, parameter :: PROC_PER_INT = 4 ! Number of cores per interval integer :: nr_of_intervals ! Number of intervals double precision :: interval_lenght ! Lenght of single interval ! Filenames character(len=20) :: filepath = 'test.mtx' character(len=20) :: outfile_evals character(len=20) :: outfile_evects ! Eigensystem real(dp), allocatable :: val(:) ! Eigensystem integer, allocatable :: row_ptr(:) ! Sparse row array of val integer, allocatable :: col_ind(:) ! Sparse column array of val integer :: order ! Order of given matrix integer :: nnz ! Number of nonzero elements ! Solver parameters integer :: M0 ! Initial search subspace (M0 >= M) double precision :: Emin, Emax ! Global search intervals double precision :: Eminl, Emaxl ! Local search intervals integer, dimension(64) :: feastparam ! FEAST parameters ! Results integer :: M ! Nr of eigenvalues double precision, dimension(:,:), allocatable :: X, X0 ! Eigenvectors double precision, dimension(:), allocatable :: E, E0 ! Eigenvalues double precision, dimension(:), allocatable :: res, res0 ! Residuals double precision :: epsout ! Relative error on trace integer :: loop ! Nr of FEAST subspace iterations integer :: info ! Error handling (if info=0: success) character(len=1) :: UPLO='L' ! Matrix storage (Lower/Upper/Full) ! Time measurements double precision :: tottime integer :: count1, count2, count_rate ! Looping integer :: i, j, k ! MPI variables integer :: ierr, rank, numtasks, len ! Global integer :: lrank, lnumtasks, color, key, NEW_COMM_WORLD ! Local character(len=MPI_MAX_PROCESSOR_NAME) :: hostname !------------------------------- ! MPI initialization !------------------------------- ! initialize MPI call MPI_INIT(ierr) ! get number of tasks call MPI_COMM_SIZE(MPI_COMM_WORLD, numtasks, ierr) ! get my rank call MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr) ! Print mpi info from root node if (rank == 0) then print '(A,I0)', 'TOTAL_PROC: ', numtasks endif !------------------------------- ! Read matrix from file !------------------------------- call read_sparse_matrix(filepath, order, nnz, val, row_ptr, col_ind) if (rank == 0) then print '(A,I0)', 'SPARSE_MATRIX_SIZE: ', order print '(A,I0)', 'NNZ: ', nnz endif !------------------------------- ! Call eigensolver !------------------------------- ! Search parameters Emin = -10 Emax = 26 M0 = 6 ! Allocate variables allocate(E(1:M0)) ! Eigenvalues allocate(X(1:order,1:M0)) ! Eigenvectors allocate(res(1:M0)) ! Residuals ! Init default FEAST parameters call feastinit(feastparam) feastparam(1) = 1 ! Print info ! Call eigensolver call system_clock(count1, count_rate) ! d = double ! s = symmetric ! csr = sparse ! ev = eigenvalue call dfeast_scsrev(UPLO, order, & ! which part and order of matrix val, row_ptr, col_ind, & ! matrix stored in csr format feastparam, epsout, loop, & ! feast parameters, trace erro and nr of loops Emin, Emax, M0, & ! search interval and initial subspace E, X, M, res, info) ! diagonalization results call system_clock(count2, count_rate) ! Wait for all processes to finish call MPI_BARRIER(MPI_COMM_WORLD, ierr) tottime = (count2-count1)*1.0d0/count_rate if (rank == 0) then print '(A)', repeat('=', 70) print '(A,F12.2,A)', 'Diagonalization finished in ', tottime, 's' print '(A,I0)', 'Returncode ', info if (info /= 0) then print '(A)', '!!!Diagonalization was unsuccessful! STOPING' stop endif print '(A,I0,A,I0)', 'Number of found eigenvalues ', M, ' expected ', order print '(A)', repeat('=', 70) endif ! Close MPI processes call MPI_FINALIZE(ierr) end program diagonalise
! read_write_mtx.f90 module read_write_mtx ! Module for reading and writing mtx format files created with scipy.mmwrite integer, parameter:: dp=kind(0.d0) ! double precision contains subroutine read_sparse_matrix(filepath, order, nnz, val, row_ptr, col_ind) ! Reads ``.mtx`` file created with ``scipy.mmwrite`` and stores ! information in The Compressed Row Storage (CRS) format. ! The ``val`` vector stores the values of the nonzero elements of ! the matrix, as they are traversed in a row-wise fashion. ! The ``col_ind`` vector stores the column indexes of the elements in ! the ``val`` vector. That is if :math:`\mathrm{val}(k) = a_{i,j}` ! then :math:`\mathrm{col\_ind}(k)=j`. The ``row_ptr`` vector stores ! the locations in the ``val`` vector that start a row, that is, ! if :math:`\mathrm{val}(k)=a_{i,j}` then ! :math:`\mathrm{row\_ptr}(i)\leq k \leq \mathrm{row\_ptr}(i+1)`. ! By convention, we define :math:`\mathrm{row\_ptr}(n+1)=\mathrm{nnz} + 1`, ! where ``nnz`` is the number of nonzeros in the matrix. character(len=20), intent(in) :: filepath ! Path to mtx file integer, intent(out) :: order ! Order of given matrix integer, intent(out) :: nnz ! Number of nonzero elements in sparse matrix real(dp), allocatable, dimension(:), && intent(out) :: val ! Values of nonzero elements integer, allocatable, dimension(:), && intent(out) :: row_ptr ! Location in the val vect. that starts a row integer, allocatable, dimension(:), && intent(out) :: col_ind ! Column indexes of the elements in the val integer, dimension(:), allocatable :: ii, jj ! Indexes of nonzero elements real(dp), dimension(:), allocatable :: direct_values ! Direct_Values corresponding to ii, jj character(len=90) :: cc ! For comment reading integer :: i, c ! Iteration variables !------------------------ ! Count comment lines !------------------------ open(10, file=trim(filepath), status='old') cc = '%' c = -1 do while(cc(1:1) == '%') c = c + 1 read(10,'(A)') cc enddo close(10) !------------------------------ ! Read direct data (3 cols) !------------------------------ open(10, file=trim(filepath), status='old') ! Ignore comments do i=1,c read(10,'(A1)') cc enddo ! Read matrix specifications read(10,*) order, order, nnz allocate(ii(nnz)) allocate(jj(nnz)) allocate(direct_values(nnz)) ! Read columns data do i = 1, nnz read(10,*) ii(i), jj(i), direct_values(i) end do close(10) !------------------------------- ! Convert data to CSR format !------------------------------- call convert2csr(order, nnz, ii, jj, direct_values, row_ptr, col_ind, val) end subroutine read_sparse_matrix subroutine convert2csr(order, nnz, ii, jj, direct_values, row_ptr, col_ind, val) ! Converts mtx data to feast compatible The Compressed Row Storage (CRS) format implicit none integer, intent(in) :: order ! Order of matrix integer, intent(in) :: nnz ! Number of nonzero elements integer, dimension(:), intent(in) :: ii, jj ! Indexes of nonzero elements real(dp), dimension(:), intent(in) :: direct_values ! Direct_Values corresponding to ii, jj real(dp), allocatable, dimension(:), && intent(out) :: val ! Values of nonzero elements integer, allocatable, dimension(:), && intent(out) :: row_ptr ! Location in the val vect. that starts a row integer, allocatable, dimension(:), && intent(out) :: col_ind ! Column indexes of the elements in the val integer :: k, k1, i, j, idum integer, dimension(order) :: iloc double precision :: adum allocate(row_ptr(1:order+1)) row_ptr = 0 allocate(col_ind(1:nnz)) allocate(val(1:nnz)) !----------------------- ! Creation of row_ptr !----------------------- ! finds how many elements in row ! after this loop row_ptr stores nr of ! nonzero elements for each matrix row do k = 1, nnz row_ptr(ii(k)) = row_ptr(ii(k)) + 1 end do ! build row_ptr which maps each row to first nonzero element k = 1 do i = 1, order+1 k1 = row_ptr(i) row_ptr(i) = k k = k + k1 end do !-------------------------------- ! Creation of col_indx and val !-------------------------------- iloc(1:order) = row_ptr(1:order) do k = 1, nnz val(iloc(ii(k))) = direct_values(k) col_ind(iloc(ii(k))) = jj(k) iloc(ii(k)) = iloc(ii(k)) + 1 end do ! Reorder by increasing column do i = 1, order do k = row_ptr(i), row_ptr(i+1) - 1 do k1 = k, row_ptr(i+1) - 1 if (col_ind(k1) < col_ind(k)) then idum = col_ind(k) col_ind(k) = col_ind(k1) col_ind(k1) = idum adum = val(k) val(k) = val(k1) val(k1) = adum endif enddo enddo enddo end subroutine convert2csr end module read_write_mtx
%% test.mtx % 6 6 19 1 1 10 1 5 -2 2 1 3 2 2 9 2 6 3 3 2 7 3 3 8 3 4 7 4 1 3 4 3 8 4 4 7 4 5 5 5 2 8 5 4 9 5 5 9 5 6 13 6 2 4 6 5 2 6 6 -1