! Information on the  job
! This includes  what job is being run
! Thread number OMP/MPI,etc
module wei_run_mod
  use wei_external_mod
  use sep
#ifdef SEP_MPI
  use mpi_sep 
#endif
  implicit none

  type trun
     logical            :: verb    !! verbose
     logical            :: debug   !! Debug
     logical            :: depar   !! debug in the parallel regions
     logical            :: restart !! whether or not we are restarting the job
     logical            :: omp,mpi !! whether or not this is an omp/mpi job 
     logical            :: share_image !! share image file
     integer            :: io_unit !! unit to write out verb info 
     integer            :: m_ith!! mpi thread number
     integer            :: m_nth !! number of mpi threads
     integer            :: o_nth !! number of omp threads
     integer            :: o_ith !! omp thread number
     character(len=4)   :: tag_num
     character(len=128) :: operator
     character(len=128) :: operation,file_out,local_path
     character(len=9)                      :: mach
#ifdef SEP_MPI
     character(len=MPI_MAX_PROCESSOR_NAME) :: name
#endif
  end type trun

   type(trun) :: run

  contains
!----------------------------------------------------------------
!
!
! Function for initializing the run enviro
!
!

  logical function run_init()
    character(len=128) :: operator,operation

#ifdef DEBUG
    character(len=128), parameter :: unit='run_mpi_parallel'
    if(run%debug) call in(unit)
#endif DEBUG
  
    call from_param('verb', run%verb, T)
    call from_param('debug',run%debug,F)
    !! restart was for mpi version that split alon freq. axis
    run%restart=F
!    call from_param('restart',run%restart,F)
    call from_param('depar',run%depar,T)
    if(      run%debug) run%verb =T
    if(.not. run%debug) run%depar=T

    run_init=F
    call from_param('operator',operator,'.R_.D')
    run%operator = operator
    call from_param('operation',operation,'migration.')
    run%operation = operation
    call from_param("io_file",run%file_out,"NONE")
    if(run%file_out(1:4)=="NONE") then
       run%io_unit=0
    else 
       run%io_unit=6
       open(run%io_unit,file=run%file_out)
    end if

    !Initialize parallel environment for run
    if(.not. run_parallel()) then
       write(0,*) "trouble initializing the run_parallel"
       return
    end if
   run_init=T

   if(run%verb) call run_report()
#ifdef DEBUG
    if(run%debug) call out(unit)
#endif
  end function run_init
!----------------------------------------------------------------
!
!  Subroutine for initializing parralel run (omp and mpi)
!
  logical function run_parallel()
#ifdef DEBUG
    character(len=128), parameter :: unit='run_parallel'
    if(run%debug) call in(unit)
#endif
 
    run_parallel=F

    if(.not. run_omp_parallel()) then
      write(0,*) "trouble initializing omp enviro"
      return
    end if
    if(.not. run_mpi_parallel()) then
      write(0,*) "trouble initializing mpi enviro"
      return
    end if
    
    run_parallel=T
#ifdef DEBUG
   if(run%debug) call out(unit)
#endif
  end function run_parallel
!----------------------------------------------------------------
!
!
! Subroutine to initialize OMP run environment
!
! Sets number of omp threads
!

  logical function run_omp_parallel()
#ifdef DEBUG
    character(len=128), parameter :: unit='omp_run_parallel'
    if(run%debug) call in(unit)
#endif

#ifdef SEP_OMP
!$OMP PARALLEL
    run%o_nth = omp_get_num_threads()
!    write(0,*) 'ompn',run%o_nth
!    write(0,*) 'ompi',omp_get_thread_num()
!$OMP END PARALLEL
    run%omp=T
#else
    run%o_nth=1
    run%o_ith=-1
    run%omp=F
#endif

   run_omp_parallel=.true.
#ifdef DEBUG
   if(run%debug) call out(unit)
#endif
  end function  run_omp_parallel
!----------------------------------------------------------------
! Function for setting up mpi parallel enviro for mpi
!
! Turns off verb, debug except for the master thread
! Sets up the local datapath
! Figues out the extra to add for each thread
! records the number of threads and this threads num
!
  logical function run_mpi_parallel()
    character(len=128) :: temp_ch
    integer :: ierr,ilen,iverb,igrab,i
#ifdef DEBUG
    character(len=128), parameter :: unit='run_mpi_parallel'
    if(run%debug) call in(unit)
#endif

    run_mpi_parallel=F

#ifdef SEP_MPI
    call MPI_COMM_SIZE(MPI_COMM_WORLD, run%m_nth,ierr)
    call MPI_COMM_RANK(MPI_COMM_WORLD, run%m_ith,ierr)

    call MPI_Get_processor_name(run%name,ilen,ierr)
    write(run%io_unit,*) "        * ",trim(run%name),&
    " shall be node:",run%m_ith

    call MPI_BARRIER(MPI_COMM_WORLD ,ierr)
    if(run%m_ith==0) write(run%io_unit,'(a,i4)') "         *         Total number nodes =",run%m_nth
    if(run%m_ith==0) write(run%io_unit,'(a,i4)') "         *         Collecting to node:",run%io_unit
    if(run%m_ith==0) write(run%io_unit,'(a,i4)')"         *         Talking to you now is node:",run%m_ith

    call from_param("share_image",run%share_image,.false.)
#else
    run%share_image=.false.
    run%m_nth=1
    run%m_ith=0
#endif

    !TURN OFF VERB AND DEBUG FOR ALL BUT ZERO MPI THREAD
    if(run%m_ith /= 0) then
       run%verb=F
       run%debug=F
    end if

#ifdef SEP_MPI
    if(run%m_nth>1) then
      do i=1,len(run%local_path)-1  ! blanks character string
        run%local_path(ilen:ilen)=" "
      end do
       igrab=getch("local_datapath","s",run%local_path)

       if(0==igrab) run%local_path="/net/"//run%name(1:ilen)//"/scr1/temp/"

       run%mach=trim(run%name(1:8))
       ilen=len(trim(run%local_path));
       if(igrab==1) run%local_path(ilen:ilen)=" "
       if(run%m_ith<10) then
          write(run%tag_num,"(a,i1)") "_",run%m_ith
       else if(run%m_ith <100) then
          write(run%tag_num,"(a,i2)") "_",run%m_ith
       else
          write(run%tag_num,"(a,i3)") "_",run%m_ith
       end if
          if(igrab==0)  then
             write(temp_ch,"(a,a,a,a)")"datapath=",trim(run%local_path),trim(run%tag_num),&
             "DATA_"
          else
             write(temp_ch,"(a,a,a,a)")"datapath=",trim(run%local_path),trim(run%tag_num),&
             "DATA_"
          end if
       if(run%m_ith/=0) call getch_add_string(temp_ch)
    end if
    run%mpi=T
    if(run%m_nth==1) run%mpi=F
#else
    run%mpi=F
#endif
#ifdef DEBUG
    if(run%debug) call out(unit)
#endif
    run_mpi_parallel=T
  end function run_mpi_parallel
!----------------------------------------------------------------
  subroutine run_report()
    write(run%io_unit,*) '        SP *********************************************'
    write(run%io_unit,*) "        *     Operator=",trim(run%operator(1:5)),"      Operation=",trim(run%operation(1:9))
    write(run%io_unit,*) "        *     Verbose:",run%verb,"                  Restart:",run%restart
    write(run%io_unit,*) "        *     Debug:",run%debug,"             Parallel debug:", run%depar
    write(run%io_unit,'(a,i3,a,i3)') "         *     OMP threads=",run%o_nth,"         MPI threads=",run%m_nth
  end subroutine run_report
end module 
