  MODULE tooltiming
! = mode_mnh_timing.f90 routine (MNH-4.10.X)
!
! Modification :
! J.ESCOBAR 13/11/2008 : change (2) in (:) for bug in IBM-SP6 compiler
! J.Escobar 1/09/2011  : reduce 'timing' format
! J.Escobar 12/02/2013 : tribulle to slow on large BG partition , inhib it by a early return in the code 
! V. Garnier 07/04/2014 : Adaption (very few) for an integration into MARS code (Many thanks to Juan and MNH team)
!
#include "toolcpp.h"

   USE parameters, ONLY : rlg
   USE_MPI toolmpi,ONLY : master

   IMPLICIT NONE
   PRIVATE

   !! * Accessibility
   PUBLIC set_iluout_timing, time_stat_ll, time_header_ll, &
          second_mars,timing_legend, timing_separator

   !! * Shared module variables
                                        ! (1)=CPU & (2)=ELAPSED time related to
   REAL(KIND=rlg), DIMENSION(2), PUBLIC :: XT_START,    &  ! Initialization
                                           XT_STORE,    &  ! Outputs
                                           XT_BOUND,    &  ! Boundaries
                                           XT_FORCING,  &  ! External forcing
                                           XT_NUDGING,  &  ! Nudging
                                           XT_DYNAMICS, &  ! State and momentum eq.
                                           XT_TSBUOY,   &  ! Tracers
                                           XT_TURB,     &  ! Turbulence
                                           XT_SUBS,     &  ! Substances
                                           XT_TRAJ,     &  ! Trajectories and ibm
                                           xt_diag,     &  ! Diagnoses
                                           xt_points,   &  ! track_points
                                           xt_misc,     &  ! miscellaneous
                                           xt_oasis,    &  ! oasis coupling
                                           xt_agrif

   !! * Private variables
   INTEGER     :: nluout_timing


CONTAINS

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

SUBROUTINE SECOND_MARS(XT)
!
#ifdef key_MPI_2D
   INCLUDE 'mpif.h'
#endif
!
REAL(KIND=rlg),DIMENSION(2)           :: XT
!
CALL CPU_TIME(XT(1))
#ifdef key_MPI_2D
XT(2) = MPI_Wtime()
#else
XT(2) = XT(1)
#endif
END SUBROUTINE SECOND_MARS

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

!JUAN
      SUBROUTINE SET_ILUOUT_TIMING(KLUOUT)
        IMPLICIT NONE
        INTEGER, INTENT(IN) :: KLUOUT
        NLUOUT_TIMING = KLUOUT
      END SUBROUTINE SET_ILUOUT_TIMING

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

      SUBROUTINE TIMING_SEPARATOR(HSEP)
        IMPLICIT NONE
        CHARACTER(len=1) :: HSEP
        INTEGER   :: J  
        IF_MPI (master) &
        WRITE(NLUOUT_TIMING,FMT= "('|',100(A1),'|')" ) ( HSEP , J=1,100 )
      END SUBROUTINE TIMING_SEPARATOR

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

      SUBROUTINE TIMING_LEGEND()
        CALL  TIMING_SEPARATOR('-')
        IF_MPI (master) &
        WRITE(NLUOUT_TIMING,FMT="( '|', 24X, '| CPUTIM/ELAPSE |   SUM(PROC)   |MEAN(PROC)| MIN(PROC | MAX(PROC)| PERCENT %|')" )
        CALL  TIMING_SEPARATOR('-')
      END SUBROUTINE TIMING_LEGEND

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

!     ########################################
      SUBROUTINE TIME_HEADER_ll
!     ########################################
        USE parameters, ONLY : lchain
        USE_MPI toolmpi,ONLY : nprocs
        IMPLICIT NONE 
        CHARACTER(LEN=lchain) :: modelname
!$   INTEGER  :: OMP_GET_NUM_THREADS

      IF_MPI (master) THEN
        modelname="Root"
        INST_AGRIF modelname="Zoom_"//TRIM(Agrif_Cfixed())
        OPEN(UNIT=NLUOUT_TIMING,FILE="time_analysis.log",status="replace")
        CALL  TIMING_SEPARATOR('-')
        CALL  TIMING_SEPARATOR(' ')
        WRITE(NLUOUT_TIMING,FMT= "('|',32X,'COMPUTING TIME ANALYSIS in MODEL ',A20,15X,'|')" ) modelname
        CALL  TIMING_SEPARATOR(' ')
        CALL  TIMING_SEPARATOR('-')
        CALL  TIMING_SEPARATOR(' ')
#ifdef key_MPI_2D
        WRITE(NLUOUT_TIMING,FMT= "('|',32X,'NUMBER OF MPI CPUS ',I5,44X,'|')" ) nprocs
#endif
!$OMP PARALLEL
!$OMP MASTER
#ifdef key_MPIOMP
!$        WRITE(NLUOUT_TIMING,FMT= "('|',32X,'NUMBER OF OMP THREADS ',I2,3X,'FOR HYBRID JOB',27X,'|')" ) OMP_GET_NUM_THREADS()
#else
!$        WRITE(NLUOUT_TIMING,FMT= "('|',32X,'NUMBER OF OMP THREADS ',I2,3X,'FOR PURE OMP JOB',25X,'|')" ) OMP_GET_NUM_THREADS()
#endif
!$OMP END MASTER
!$OMP END PARALLEL
        CALL  TIMING_SEPARATOR(' ')
        CALL  TIMING_LEGEND()  
      ENDIF_MPI
      END SUBROUTINE TIME_HEADER_ll


!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

!     ########################################
      SUBROUTINE TIME_STAT_ll(PRES, PSUM, HPRINT, HSEP,HFULL)
!     ########################################
!
!*       0.    DECLARATIONS
!
  USE_MPI toolmpi, ONLY : NPROCS,MPI_COMM_MARS

  IMPLICIT NONE

#ifdef key_MPI_2D
   INCLUDE 'mpif.h'
#endif
!
!*       0.1   Declarations of dummy arguments :
! 
  REAL(KIND=rlg),DIMENSION(:), INTENT(IN)         :: PRES ! (1)=CPU & (2)=ELAPSE Processors Timing
!
  REAL(KIND=rlg),DIMENSION(:), INTENT(INOUT)      :: PSUM ! (1)=SUM(CPU) & (2)=SUM(ELAPSE) Timing
!
  CHARACTER(len=*),  INTENT(IN),OPTIONAL :: HPRINT
  CHARACTER       ,  INTENT(IN),OPTIONAL :: HSEP
  CHARACTER(len=*),  INTENT(IN),OPTIONAL :: HFULL
!
!*       0.2   Declarations of local variables :
!
#if ! defined key_MPI_2D
  INTEGER,PARAMETER         :: NPROCS=1
#endif
  INTEGER,PARAMETER         :: NSTAT=5
  REAL(KIND=rlg),DIMENSION(2,NSTAT) :: ZSTAT ! (1)=Sum(proc),(2)=Sum/Nproc,(3)=Min(proc),(4)=Max(proc),(5)=Purcent(1)
  INTEGER                   :: INFO,IROOT,JP
  CHARACTER(len=30)         :: VIDE = ""
  CHARACTER(len=30)         :: FILEC = ""
  INTEGER                   :: IC

  REAL(KIND=rlg),DIMENSION(2,NPROCS)   :: ZSTAT_ALL 
  INTEGER, DIMENSION(NPROCS) :: IND
!
!-------------------------------------------------------------------------------
!
!*       1. CALL THE MPI_ALLREDUCE ROUTINE
!           ------------------------------

#ifdef key_MPI_2D
INFO = -1
! 1.1 Sum(Proc)
  CALL MPI_ALLREDUCE(PRES, ZSTAT(:,1), 2, MPI_REAL8, MPI_SUM, MPI_COMM_MARS, INFO)
! 1.2 Sum/Proc
  ZSTAT(:,2) = ZSTAT(:,1 ) / NPROCS
! 1.3 Min(Proc)
  CALL MPI_ALLREDUCE(PRES, ZSTAT(:,3), 2, MPI_REAL8, MPI_MIN, MPI_COMM_MARS, INFO)
! 1.4 Max(Proc)
  CALL MPI_ALLREDUCE(PRES, ZSTAT(:,4), 2, MPI_REAL8, MPI_MAX, MPI_COMM_MARS, INFO)
#else
  ZSTAT(:,1)=PRES
  ZSTAT(:,2) = ZSTAT(:,1 ) / NPROCS
  ZSTAT(:,3)=PRES
  ZSTAT(:,4)=PRES
#endif

  IF (.NOT.PRESENT(HPRINT)) THEN
   ! return total sum 
   PSUM = ZSTAT(:,1)
  !

  ELSEIF ( ZSTAT(1,1) > 0.0 ) THEN
   ! use Psum , for print stat & pourcent
   ! Purcent
     WHERE ( PSUM /= 0.0 )
        ZSTAT(:,5) = 100.0 * ZSTAT(:,1) / PSUM(:)
     ELSEWHERE
        ZSTAT(:,5) = 0.0
     END WHERE
   ! print stat
   !
   IF_MPI (master) THEN
   IF (PRESENT(HSEP)) CALL  TIMING_SEPARATOR(HSEP)
   WRITE(NLUOUT_TIMING,FMT= "('|',A30,'| CPUTIM ||',F15.3,'|',4(F10.3,'|'),F7.3,'|')" ) HPRINT//VIDE,ZSTAT(1,:)
   WRITE(NLUOUT_TIMING,FMT= "('|',A30,'| ELAPSE ||',F15.3,'|',4(F10.3,'|'),F7.3,'|')" ) HPRINT//VIDE,ZSTAT(2,:)
   ENDIF_MPI

   IF (PRESENT(HFULL)) THEN
      ! gather all data
      !CALL  TIMING_SEPARATOR(HSEP)
      IROOT = 0
      CALL_MPI MPI_GATHER(PRES(:),2,MPI_REAL8,ZSTAT_ALL(:,1),2,MPI_REAL8, IROOT,MPI_COMM_MARS, INFO)
      IF_MPI (master) THEN
         FILEC = trim(adjustl(HPRINT))
         DO IC=1,LEN_trim(FILEC)
            SELECT CASE( ICHAR(FILEC(IC:IC)) )
               CASE(ICHAR(' '),ICHAR('='),ICHAR('>'))
                 FILEC(IC:IC) = '_'
            END SELECT

         END DO
         OPEN (100+NLUOUT_TIMING,file=FILEC)
         WRITE(100+NLUOUT_TIMING,FMT= "(10('#'),A30,10('#'))" ) HPRINT//VIDE
         call  triabulle(ZSTAT_ALL(2,:),ind)
         DO JP=1,NPROCS
            WRITE(100+NLUOUT_TIMING,FMT= "(5(I8,' ',F15.3,' '))" ) &
                 ind(JP),ZSTAT_ALL(2,ind(JP))
         END DO
         CLOSE (100+NLUOUT_TIMING)
      ENDIF_MPI
   END IF
   !
  ENDIF
!
!-------------------------------------------------------------------------------
!
      END SUBROUTINE TIME_STAT_ll
!JUAN

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

subroutine triabulle(vec,ind)
implicit none

real*8   , intent(inout)   :: vec(:)
integer, intent(out)  :: ind(:)


integer               :: n 

logical :: a
integer :: i
integer :: mem

n = size(vec)
a = .true.
do i=1,n 
   ind(i) = i 
enddo

return
!JUAN TO SLOW ON BG !!!

do while (a)
   a=.false.
   do i=1,n-1
      if (vec(ind(i))>vec(ind(i+1))) then
         mem = ind(i)
         ind(i) = ind(i+1)
         ind(i+1) = mem
         a=.true.
      endif
   enddo
enddo

end subroutine triabulle

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

  END MODULE tooltiming
