ecearth3/sources/oasis3-mct/util/lucia/main-lucia.F90

program lucia_analysis
!
! ===================================
! LUCIA post-processing analysis tool
! ===================================
!
! Purpose:
!          - Load balance an OASIS-based coupled system
!          - Assess scalabilities of each model of the coupled system
!          - Estimate coupling cost (interpolations) and models jitter
!
! Getting started:
!
!          - compile your model with OASIS version newer than 
!          - launch coupled sytem with second LOGPRT parameter equal 
!                to -1 (in namcouple file)
!          - process lucia.??.?????? files with lucia-mct script 
!                shell provided with the present FORTRAN program
!
implicit none
!
!   Information related to each coupled field
!
type FIELD_SEQUENCE
    integer*4 ::  namID           ! field rank in namcouple
    integer*4 ::  source_model    ! index of source model
    integer*4 ::  target_model    ! index of target model
    integer*4 ::  source_comm_nb  ! number of send actions from source
    integer*4 ::  target_comm_nb  ! number of receive actions from target
end type FIELD_SEQUENCE

    ! Explicit logical flags
    logical :: l_put, l_before, l_exch_still_valid, l_exch_not_yet_valid, l_add_interp_time
!
    character(len=20) ::  c_verif          ! Lucia identifyer (no more used)
    character(len=3)  ::  c_field_code     ! code for field number or info provided during init
    character(len=24) ::  c_field_name     
    character(len=15) ::  c_comm_type      ! kind of exchange (put or get)

    ! Parameters
    character(len=3)  ::  c_ident_put = "put"
    character(len=3)  ::  c_ident_before = "Before"

    ! Buffer arrays
    character(len=3)   :: c_test_name
    character(len=24)  :: c_dummy
    character(len=300) :: c_argument, log_file_name
!
    character(len=20), dimension(:,:), allocatable ::  field_name
    character(len=10), dimension(:), allocatable   ::  model_name
    character(len=10), dimension(:), allocatable   ::  model_code  ! model rank on OASIS sequence
!
    ! for first static allocation, maximum number of coupling fields
    integer*4  :: max_nb_fields = 300
!
    ! Indexes or temporary arrays
    integer*4  :: i, j, k, l, mi, narg, i_err, newf, tmp_field_code

    integer*4  :: nb_models
    integer*4  :: nb_tot_log_files, log_nb  ! number of log file processed and index

    integer*4  :: cpl_field_nb        ! total number of coupling fields 
                                      ! same as NFIELDS parameter in namcouple
    integer*4  :: i_cpl               ! coupling field index (namcouple order)

    integer*4  :: clk_i               ! event index    1: before send
                                      !                2: after  send
                                      !                3: before receive
                                      !                4: after  receive

    integer*4  ::  before_send = 1
    integer*4  ::  after_send  = 2
    integer*4  ::  before_recv = 3
    integer*4  ::  after_recv  = 4
    integer*4  ::  after_send_or_recv  = 2

    integer*4  :: max_comm_nb         ! maximum nb of coupling tstep among coupling fields
    integer*4  :: i_first_exchg(2)    ! indexes of first coupling field exchanged

    integer*4, dimension(:), allocatable ::  nb_mpi           ! array of mpi partition nb per model
    integer*4, dimension(:), allocatable ::  i_stride         ! stride for log file counting per model
    integer*4, dimension(:), allocatable ::  i_file_count     ! log file nb per model
    integer*4, dimension(:), allocatable ::  valid_comm_nb    ! nb of valid coupling tstep per model
    integer*4, dimension(:), allocatable ::  first_valid_comm ! index of first valid communication

    integer*4, dimension(:), allocatable ::  i_cpl_field      ! index of coupling field in exchange sequence
    integer*4, dimension(:,:), allocatable ::  field_code     ! index of coupling fields in namcouple exchange

    integer*4, dimension(:,:), allocatable ::  comm_type   ! Communication type 
                                                           !  0: get 
                                                           !  1: put

    integer*4, dimension(:,:), allocatable ::  comm_nb        ! nb of coupling tstep per model and per field
!
    real*8 :: r_clock                  ! read clock time
    real*8 :: r_min_time, r_max_time   ! time boundaries
    real*8 :: r_reference = -1.E8      ! reference time
    real*8 :: r_mean                   ! tmp buffer
    real*8 :: temp_t                   ! tmp buffer
    real*8 :: r_impossible_value= 1.E13      ! reference time
    real*8 :: r_test_impossible= 1.E12      ! reference time

    real*8, dimension(:), allocatable ::  calc_time, noncalc_time   ! calculation and non calculation time per model

    !                                     Timing for interpolation time and jitter per model
    real*8, dimension(:), allocatable ::  r_interp_measure, r_interp_time, r_jitter_time

    !                                     Evaluation of variance among log files
    real*8, dimension(:), allocatable ::  send_spread, receive_spread, calc_spread

    real*8, dimension(:,:), allocatable   ::  start_time            ! beginning of first coupling sequence
    real*8, dimension(:,:,:), allocatable ::  min_clock_measure     ! measure of min among log files
    real*8, dimension(:,:,:), allocatable ::  max_clock_measure     ! measure of max among log files
    real*8, dimension(:,:,:), allocatable ::  calc_noncalc_measure  ! calculation and non calculation time for each event
!
!   Informations on coupling fields
    type(FIELD_SEQUENCE), dimension(:), allocatable :: cpl_fields 
!
!   external function
    integer*4 :: iargc
!
!
!   GET THE NUMBER OF COMMAND LINE ARGUMENTS.
!
    narg = iargc()
!
!   CHECK THE NUMBER OF COMMAND LINE ARGUMENTS.
!
    if ( narg==0 ) then
        write (6,*)
        write (6,*) ' Error: Missing arguments'
        stop
    else if ( narg<6 ) then
        write (6,*) ' Wrong number of line arguments '
        write (6,*) ' Coupled models should be 2 at least '
        stop
    end if
!
!   3 argument per model
    nb_models = narg / 3
!
!   ALLOCATIONS of nb_models dimensional arrays
    allocate(nb_mpi(nb_models))
    allocate(i_stride(nb_models))
    allocate(i_file_count(nb_models))
    allocate(i_cpl_field(nb_models))
    allocate(model_name(nb_models))
    allocate(model_code(nb_models))
    allocate(calc_time(nb_models))
    allocate(noncalc_time(nb_models))
    allocate(r_jitter_time(nb_models))
    allocate(send_spread(nb_models))
    allocate(receive_spread(nb_models))
    allocate(calc_spread(nb_models))
    allocate(r_interp_time(nb_models))
    allocate(valid_comm_nb(nb_models))
    allocate(first_valid_comm(nb_models))

!   ALLOCATIONS of nb_models x max_nb_fields dimensional arrays
    allocate(field_name(nb_models, max_nb_fields))
    allocate(field_code(nb_models, max_nb_fields))
    allocate(comm_type(nb_models, max_nb_fields))
    allocate(comm_nb(nb_models, max_nb_fields))
    allocate(start_time(nb_models, max_nb_fields))

    comm_nb(:,:) = 0
    start_time(:,:) = r_impossible_value
!
!   DEFAULT VALUES FOR COMMAND LINE ARGUMENTS.
!
    nb_mpi(:) = 1
    model_code(:) = "none"
    model_name(:) = "none"
    field_name(:,:) = "none"
!
!  1. CHECK THE COMMAND LINE ARGUMENTS.
!
    do i = 1, nb_models
!
       ! GET MODEL RANK ON OASIS SEQUENCE
       call getarg( 3*i-2, model_code(i) )
!
       ! GET LOG FILE NB
       call getarg( 3*i-1, c_argument )
       read(c_argument,'(i6)') i_stride(i)
       ! must be greater than 1
       i_stride(i) = MAX(i_stride(i),1)
!
       ! GET NUMBER OF MPI SUBDOMAINS
       call getarg( 3*i, c_argument )
       read(c_argument,'(i6)') nb_mpi(i)
       ! check that stride still greater than 1
       i_stride(i) = MAX ( nb_mpi(i) / MAX((i_stride(i)-1),1), 1)
!
    end do
!
!   2. READ OASIS-LUCIA LOG FILES CONTAINT
!
!   DATA ARE READ FOR A FIRST TIME TO FIND ARRAYs LENGTH
!   AND COUPLING FIELDS EXCHANGE SEQUENCE
!
    write(6,*) ' '
    write(6,*) ' Processing OASIS LUCIA log files! '
    write(6,*) ' '
!
    i_cpl_field(:)=0

!   Loop on model number
    do i = 1, nb_models
!
       i_file_count(i) = 0
       k = 1

       write(6,*) 'Computed log files for model ', model_code(i), nb_mpi(i), i_stride(i)
       call flush(6)

       ! Loop on log file number
       do j = 0, nb_mpi(i), i_stride(i)

          ! Count number of log file per model
          i_file_count(i) = i_file_count(i) + 1

          write(log_file_name,'("lucia.",a2,".",i6.6)'),model_code(i),j
          write(6,'(TL16,A,1X)', advance='no') TRIM(log_file_name)
          call flush(6)

          OPEN (unit=10, &
                file=TRIM(log_file_name), &
                action="read", &
                status="OLD", &
                form="formatted", &
                iostat=i_err)
          if ( i_err .ne. 0 ) then
             write(6,*) 'Error opening ASCII file ', TRIM(log_file_name)
             stop
          end if

!         write (6,*) ' open ', log_file_name
!
!
!         FIRST GUESS: GET FIELD NAMES AND EXCHANGE TYPE
!
          REWIND(10)
          i_err=0
          ! For each line of the log file
          DO WHILE ( i_err /= -1 )
             READ(10, '(A9,A3,A12,A4,F16.5)', iostat=i_err) &
                                    &      c_verif, &
                                    &      c_field_code, &
                                    &      c_dummy, &
                                    &      c_comm_type, &
                                    &      r_clock
             ! EOF
             IF ( i_err == -1 ) cycle
!
!            if ( i == 2 ) write(6,*) ' file ', TRIM(c_verif), TRIM(c_field_code)

             ! Skip if initial synchro
             IF ( c_field_code(1:3) == "IT " ) cycle

             ! Read model names
             IF ( c_field_code(1:3) == "MD " ) THEN
                IF ( TRIM(model_name(i)) == "none" ) model_name(i) = TRIM(c_dummy)
                cycle
             ENDIF

             IF ( c_field_code(1:3) == "NP " ) cycle
                   
             ! Read field names as declared in namcouple
             IF ( c_field_code(1:3) == "SN " .or. c_field_code(1:3) == "RC " ) THEN
                BACKSPACE(10)
                READ(10, '(A9,A3,A5,A)', iostat=i_err) &
                                         &      c_verif, &
                                         &      c_field_code, &
                                         &      c_dummy, &
                                   &            c_field_name
                IF ( j == 0 ) THEN
                   read(c_dummy(1:4),'(i4)') k
                   field_name(i,k) = TRIM(c_field_name)
                ENDIF
                cycle
             ENDIF
             
             BACKSPACE(10)
             READ(10, '(A20,A3,A16,F16.5)', iostat=i_err) &
                                    &      c_verif, &
                                    &      c_field_code, &
                                    &      c_comm_type, &
                                    &      r_clock

             ! Skip if interpolation time measurement
             IF ( INDEX ( TRIM(c_comm_type),'rpo'  ) /= 0 ) cycle
!
!            PROCESS INFORMATION FROM STANDARD TIMING LINE
! 
             ! When coupling field list is empty (beginning)
             IF ( i_cpl_field(i) == 0 ) THEN
                ! Fill list with first field found
                i_cpl_field(i) = 1
                READ(c_field_code,'(i3)') field_code(i, i_cpl_field(i))
                ! Start counting coupling steps
                comm_nb(i,i_cpl_field(i)) = 1

!                      if ( i == 2 ) write(6,*) ' field ', TRIM(field_code(i, 1)), comm_nb(i,1)

                ! Identify if it is a put or a get communication
                IF ( INDEX ( TRIM(c_comm_type), c_ident_put ) /= 0 ) THEN
                   ! This communication is a put
                   comm_type(i, i_cpl_field(i)) = 0
                   start_time(i,i_cpl_field(i)) = r_clock
                ELSE
                   ! This communication is a get
                   comm_type(i, i_cpl_field(i)) = 1
                   start_time(i,i_cpl_field(i)) = r_clock
                ENDIF

             ! When field list is not empty (loop)
             ELSE 
                ! Coupling field index initialized
                mi = 1
                newf = 0
                ! Check model field number already identified
                DO WHILE ( mi <= i_cpl_field(i) )
                   read(c_field_code,'(i3)') tmp_field_code
                   IF ( field_code(i, mi) == tmp_field_code ) &
                      newf = mi
                   mi = mi + 1
                END DO
                ! Another field found because not identified in the list
                IF ( newf == 0 ) THEN
                   ! Fill list with new field found (same than above)
                   i_cpl_field(i) = i_cpl_field(i) + 1
                   READ(c_field_code,'(i3)') field_code(i, i_cpl_field(i))
                   comm_nb(i,i_cpl_field(i)) = 1

                   ! Identify if it is a put or a get communication
                   IF ( INDEX ( TRIM(c_comm_type), c_ident_put ) /= 0 ) THEN
                      ! This communication is a put
                      comm_type(i, i_cpl_field(i)) = 0
                      start_time(i,i_cpl_field(i)) = r_clock
                   ELSE
                      comm_type(i, i_cpl_field(i)) = 1
                      start_time(i,i_cpl_field(i)) = r_clock
                   ENDIF

                ! Just another coupling step for an already identified field
                ELSE
                   comm_nb(i,newf) = comm_nb(i,newf) + 1
                END IF
             END IF

          ! End loop on read lines
          END DO   

          CLOSE(10)

       ! End loop on log files
       END DO

       write(6,*) ' '
!
!          write(6,*) ' nbFields ', model_name(i) , i_cpl_field(i)
!          do k = 1, i_cpl_field(i)
!            write(6,*) ' Field ', field_code(i, k), comm_type(i,k)
!          ENDDO
!
    ! End loop on models
    END DO
!
!   Coupling fields counterd twice (as sent and received field)
    cpl_field_nb = SUM(i_cpl_field(:))/2

    ALLOCATE ( cpl_fields ( cpl_field_nb ) )
!
!    write(6,*) ' nb fields ', cpl_field_nb
!    call flush(6)
!
!
    DO i = 1, nb_models
       ! Count total send/received fields (divide by proc number)
       comm_nb(i,:) = comm_nb(i,:) / i_file_count(i)
    END DO
!
!   3. FIND EXCHANGE ORDER
!
    j = 1
    ! loop on coupling fields
    DO WHILE ( j <= cpl_field_nb )
       ! Find the earliest exchange
       i_first_exchg = MINLOC(start_time(:,:))
       ! Index of model doing the first exchange
       mi = i_first_exchg(1)
       ! Index of first field exchanged
       i = i_first_exchg(2)

       ! only if it is a send field
       IF ( comm_type(mi,i) == 0 ) THEN
          ! Find the exchange number in OASIS sequence (namcouple)
          cpl_fields(j)%namID = field_code ( mi, i ) 
          ! Set its source model
          cpl_fields(j)%source_model = mi
          ! Set how many times this coupling field has been sent
          cpl_fields(j)%source_comm_nb = comm_nb( mi, i )
          ! 
          DO k = 1, nb_models
             IF ( k == mi ) cycle 
              IF ( ANY ( field_code ( k, 1:i_cpl_field(k) ) == cpl_fields(j)%namID)) THEN
                 ! Set its target model
                 cpl_fields(j)%target_model = k
                 DO l = 1, i_cpl_field(k)
                    IF ( field_code(k,l) == cpl_fields(j)%namID ) &
                       ! Set how many times this coupling field has been received
                       cpl_fields(j)%target_comm_nb = comm_nb( k, l )
                 END DO
              END IF
          END DO
          j = j + 1
       END IF
       start_time(mi,i) = r_impossible_value
    END DO
!
    write (6,*) ' '
    write (6,*) ' "Lucia" analysis '
    write (6,*) ' '
    write (6,*) ' Exchanged fields (based on first exchange): '
!
    write (6,*) ' From model :                  to model '
    DO i = 1, cpl_field_nb
       write (6,*) '         ',  TRIM(model_name(cpl_fields(i)%source_model)), &
                ' ( ', TRIM(field_name(cpl_fields(i)%source_model,cpl_fields(i)%namID)), &
                ' )      ', TRIM(model_name(cpl_fields(i)%target_model)), &
                ' ( ', TRIM(field_name(cpl_fields(i)%target_model,cpl_fields(i)%namID)) , ' )'
    END DO
!    write (6,*) ' '
!    call flush(6)
!
!  4. CHECK COMMUNICATION NUMBER CONCORDANCE
!
    do i = 1, cpl_field_nb
       IF ( cpl_fields(i)%target_comm_nb /= cpl_fields(i)%source_comm_nb ) THEN
          write(6,*) ' WARNING - Coupler exchanges: ' , & 
         TRIM(field_name(cpl_fields(i)%source_model,cpl_fields(i)%namID)) , &
         ' sent ', cpl_fields(i)%source_comm_nb, &
         ' but ', TRIM(field_name(cpl_fields(i)%target_model,cpl_fields(i)%namID)) , &
         ' received ', cpl_fields(i)%target_comm_nb
!
!         In case of unbalanced exchange number (abnormal stop),
!         restrain communication number according to the last valid exchange number
!
          cpl_fields(i)%source_comm_nb = MIN ( cpl_fields(i)%target_comm_nb, cpl_fields(i)%source_comm_nb)
          cpl_fields(i)%target_comm_nb = MIN ( cpl_fields(i)%target_comm_nb, cpl_fields(i)%source_comm_nb)
       ENDIF
    end do
!
!   Find valid number of coupling 
!
    do j = 1, cpl_field_nb
       ! OASIS sends = OASIS receives. Count only before event (/2)
       cpl_fields(j)%target_comm_nb = MIN ( cpl_fields(j)%source_comm_nb , cpl_fields(j)%target_comm_nb ) / 2
       ! Same number of "received" and "send" exchange
       cpl_fields(j)%source_comm_nb = cpl_fields(j)%target_comm_nb
    end do

    !  Substract 1 to number of coupling tstep 
    !  (last exchange ignored to avoid side effect of termination phase)
    max_comm_nb = MAXVAL ( cpl_fields(:)%source_comm_nb ) - 1
    cpl_fields(:)%source_comm_nb = cpl_fields(:)%source_comm_nb - 1
    cpl_fields(:)%target_comm_nb = cpl_fields(:)%source_comm_nb

    ! Set the maximum number of coupling tstep per model
    valid_comm_nb(:) = 0
    do i = 1, nb_models
       do j = 1, cpl_field_nb
          IF ( cpl_fields(j)%source_model == i .or. cpl_fields(j)%target_model == i ) &
             valid_comm_nb(i) = MAX(valid_comm_nb(i),cpl_fields(j)%source_comm_nb)
       end do
    end do
       
! 
!  5. Allocate Timing Arrays 
!    -  to fill with minimum or maximum clock time among log files
!       before and after each coupling communications (put and get)
!       and for each coupling field
!
    ALLOCATE ( min_clock_measure ( cpl_field_nb, max_comm_nb , 4 ) )
    ALLOCATE ( max_clock_measure ( cpl_field_nb, max_comm_nb , 4 ) )

!   initialize min/max counters
    min_clock_measure = r_impossible_value
    max_clock_measure = r_impossible_value * (-1.)
!
!    - to store calculation / non calculation time 
!      for each log file of the same model
!
!   calc_noncalc_measure ( : ,  1 ) : total 'calculation' time
!   calc_noncalc_measure ( : ,  2 ) : total time spent during send operation
!   calc_noncalc_measure ( : ,  3 ) : total time spent during receive operation
!
    nb_tot_log_files = SUM ( i_file_count(:) )
    ALLOCATE ( calc_noncalc_measure ( nb_tot_log_files, max_comm_nb, 3 ) )
    ALLOCATE ( r_interp_measure ( nb_tot_log_files ) )

!   initialize measures total
    calc_noncalc_measure (:,:,:) = 0
!
!   6. READ AGAIN OASIS-LUCIA LOG FILES CONTAINT
!      AND FILL ARRAYS WITH ALL CLOCK TIME MEASURES 
!
    log_nb = 0
    r_max_time = r_impossible_value * (-1.)
    r_min_time = r_impossible_value

    ! Loop on model number
    DO i = 1, nb_models
       ! Loop on log file number
       DO j = 0, nb_mpi(i), i_stride(i)

          ! Count number of log file
          log_nb = log_nb + 1

          write(log_file_name,'("lucia.",a2,".",i6.6)'),model_code(i),j

          OPEN (unit=10, &
                file=TRIM(log_file_name), &
                action="read", &
                status="OLD", &
                form="formatted", &
                iostat=i_err)
          IF ( i_err .ne. 0 ) then
             write(6,*) 'Error opening ASCII file ', TRIM(log_file_name)
             STOP
          END IF

          REWIND(10)

          l_exch_still_valid = .true.
          l_exch_not_yet_valid = .true.

          c_test_name="not"
          mi = 0
          r_interp_measure(log_nb)=0
          l_add_interp_time=.false.

          i_err=0
          ! For each line of the log file
          DO WHILE ( i_err /= -1 .and. l_exch_still_valid )
             READ(10, '(A20,A3,A16,F16.5)', iostat=i_err) &
                       &   c_verif, &
                       &   c_field_code, &
                       &   c_comm_type, &
                       &   r_clock
             IF ( i_err == -1 ) CYCLE

!            Substract first clock measure to store anomaly instead of raw value
!            (to avoid too big values when additionning)
             IF ( c_verif(10:12) == "IT " ) THEN
                BACKSPACE(10)
                READ(10, '(A20,A3,A7,F16.5)', iostat=i_err) &
                          &   c_verif, &
                          &   c_field_code, &
                          &   c_comm_type, &
                          &   r_clock
                r_reference = r_clock
                CYCLE
             ENDIF

             ! Skip model names
             IF ( c_verif(10:12) == "MD " ) cycle

             IF ( c_verif(10:12) == "NP " ) cycle
                   
             ! Skip field names
             IF ( c_verif(10:12) == "SN " .or. c_verif(10:12) == "RC " ) cycle
                   
             r_clock = r_clock - r_reference

             ! Special treatment for interpolation time :cumulated
             IF ( INDEX ( TRIM(c_comm_type), 'interpo' ) /= 0 ) then
                IF ( l_add_interp_time ) then
                   r_interp_measure(log_nb) = r_interp_measure(log_nb) + r_clock
                   l_add_interp_time=.false.
                ELSE
                   r_interp_measure(log_nb) = r_interp_measure(log_nb) - r_clock
                   l_add_interp_time=.true.
                ENDIF
                CYCLE
             ENDIF

!
!            PROCESS INFORMATION FROM STANDARD TIMING LINE
! 

             ! Get the name of the first field exchanged by this model
             IF ( TRIM(c_test_name) == "not" ) c_test_name=TRIM(c_field_code)

!            write(6,*) 'c_comm_type ', c_comm_type
!            write(6,*) 'c_test_name ', c_test_name
!            write(6,*) 'TRIM(c_field_code) ', TRIM(c_field_code)

!
!            Find field name as declared in namcouple
!
             k = 1
             DO WHILE ( k <= cpl_field_nb )
                READ(c_field_code,'(i3)') tmp_field_code
                IF ( cpl_fields(k)%namID == tmp_field_code ) &
                   i_cpl = k
                k = k + 1
             END DO

!            write(6,*) 'field number ', i_cpl
!
!            Determine if the exchange is a put or a get
!                      if the timing is set before or after the exchange

             l_put = .false. 
             l_before = .false. 
!
!            and attribute the corresponding index:
!                                1: Before put
!                                2: After put
!                                3: Before get
!                                4: After get

             IF ( INDEX ( TRIM(c_comm_type), c_ident_put ) /= 0 ) &
                l_put = .true.
             IF ( INDEX ( TRIM(c_comm_type), c_ident_before ) /= 0 ) &
                l_before = .true.

             IF ( l_before ) THEN
                IF ( l_put ) THEN
                   clk_i = before_send
                ELSE
                   clk_i = before_recv
                ENDIF
             ELSE
                IF ( l_put ) THEN
                   clk_i = after_send
                ELSE
                   clk_i = after_recv
                ENDIF
             ENDIF

!            write(6,*) 'index number ', clk_i
!
!            Determine exchange validity
!
!            Measures start (coupler) the first time than a field is received 
!            This excludes restart reading sequence side effect if any
!
!
             IF ( l_exch_not_yet_valid .AND. cpl_fields(i_cpl)%source_comm_nb == valid_comm_nb(i) ) THEN
                l_exch_not_yet_valid = .false.
                ! Get the name of the first field exchanged by this model
                c_test_name = TRIM(c_field_code)
                ! and at what time it is
                r_min_time = MIN ( r_clock, r_min_time )
             END IF

             ! If before exchange of the first coupling field
             ! on a not yet valid exchange
             IF ( TRIM(c_field_code) == TRIM(c_test_name) .and. l_before &
                  .and. .not. l_exch_not_yet_valid ) THEN

                ! Increase exchange number
                mi = mi + 1

                ! Before the last valid exchange
                IF ( mi <= valid_comm_nb(i) ) THEN
                   ! calc/nocalc current index initialization
                   calc_noncalc_measure ( log_nb, mi, 1 ) = &
                   calc_noncalc_measure ( log_nb, mi, 1 ) -  r_clock
                ENDIF

                ! After the first exchange
                IF ( mi > 1 ) THEN
                   ! calc/nocalc previous index finalization
                   calc_noncalc_measure ( log_nb, mi-1, 1 ) = &
                   calc_noncalc_measure ( log_nb, mi-1, 1 ) +  r_clock
                ENDIF

                ! Increase time counter to find timing of last exchange
                r_max_time = MAX ( r_clock, r_max_time )

!                   write(6,*) 'still valid ', l_exch_still_valid

             END IF

             ! Reach maximum number of valid exchanges
             IF ( mi > valid_comm_nb(i) ) &
                l_exch_still_valid = .false.

             ! Do not fill timer arrays if exchange not yet or no more valid
             IF ( l_exch_not_yet_valid .or. .not. l_exch_still_valid ) CYCLE

!                   write(6,*) 'not cycled ', TRIM(c_field_code)

             ! Fill mix/max array compared to previous log file measures
             min_clock_measure(i_cpl ,mi ,clk_i) =  &
                 MIN ( min_clock_measure(i_cpl ,mi ,clk_i), r_clock )

             max_clock_measure(i_cpl ,mi ,clk_i) =  &
                 MAX ( max_clock_measure(i_cpl ,mi ,clk_i), r_clock )

             ! Fill calc/noncalc array for each log file
             ! Sending time
             IF ( clk_i == after_send ) THEN
                 calc_noncalc_measure ( log_nb, mi, 2 ) = &
                 calc_noncalc_measure ( log_nb, mi, 2 ) +  r_clock
             ELSE IF ( clk_i == before_send ) THEN
                 calc_noncalc_measure ( log_nb, mi, 2 ) = &
                 calc_noncalc_measure ( log_nb, mi, 2 ) -  r_clock
             ! Receiving time
             ELSE IF ( clk_i == after_recv ) THEN
                 calc_noncalc_measure ( log_nb, mi, 3 ) = &
                 calc_noncalc_measure ( log_nb, mi, 3 ) +  r_clock
             ELSE IF ( clk_i == before_recv ) THEN
                 calc_noncalc_measure ( log_nb, mi, 3 ) = &
                 calc_noncalc_measure ( log_nb, mi, 3 ) -  r_clock
             ENDIF
             ! Calculation time
             IF ( MOD ( clk_i , after_send_or_recv ) == 1 ) THEN
                 calc_noncalc_measure ( log_nb, mi, 1 ) = &
                 calc_noncalc_measure ( log_nb, mi, 1 ) +  r_clock
             ELSE
                 calc_noncalc_measure ( log_nb, mi, 1 ) = &
                 calc_noncalc_measure ( log_nb, mi, 1 ) -  r_clock
             ENDIF
             
          ! End loop on read lines
          END DO

          CLOSE(10)

       ! End loop on log files
       END DO

    ! End loop on models
    END DO
!
!   7. ANALYSIS
!
    calc_time (:) = 0 ;  noncalc_time (:) = 0
    send_spread (:) = 0 ;  receive_spread (:) = 0; calc_spread (:) = 0
!
!   7.1 ANALYSIS ON MAXIMUM MEAN VALUES AMONG LOG FILES
!
    k = 1

    ! Loop on models
    DO i = 1, nb_models

!       write(6,*), ' Model   : ', i

       ! For most frequently coupled fields
       IF ( valid_comm_nb(i) == max_comm_nb ) THEN
          ! Start analysis on third exchange to avoid side effect
          first_valid_comm(i) = 3
       ELSE
          ! only on second for the others
          first_valid_comm(i) = 2
       END IF

       ! Special treatment for models not involved in coupling (IO servers)
       IF ( valid_comm_nb(i) == 0 ) first_valid_comm(i) = 1

       ! Loop on valid coupling tsteps
       DO j = first_valid_comm(i), valid_comm_nb(i)

          ! Maximum values over log files are added for all valid coupling tsteps
          ! ... for time spent by models on calculations
          calc_time (i) = calc_time (i) + MAXVAL(calc_noncalc_measure (k:k+i_file_count(i)-1, j, 1))
          ! ... for time spent by models on OASIS exchanges (send and receive operations)
          noncalc_time (i) = noncalc_time (i) + &
                             MAXVAL(calc_noncalc_measure (k:k+i_file_count(i)-1, j, 2)) + &
                             MAXVAL(calc_noncalc_measure (k:k+i_file_count(i)-1, j, 3))

          ! Variance among log file is calculated for those 2 values
          r_mean = SUM ( calc_noncalc_measure (k:k+i_file_count(i)-1, j, 2) ) / i_file_count(i)
          send_spread (i) = send_spread (i) + &
                            SQRT ( SUM ( ( calc_noncalc_measure (k:k+i_file_count(i)-1, j, 2) - &
                                           r_mean ) ** 2 ) )
          r_mean = SUM ( calc_noncalc_measure (k:k+i_file_count(i)-1, j, 3) ) / i_file_count(i)
          receive_spread (i) = receive_spread (i) + &
                            SQRT ( SUM ( ( calc_noncalc_measure (k:k+i_file_count(i)-1, j, 3) - &
                                           r_mean ) ** 2 ) ) 
          r_mean = SUM ( calc_noncalc_measure (k:k+i_file_count(i)-1, j, 1) ) / i_file_count(i)
          calc_spread (i) = calc_spread (i) + &
                            SQRT ( SUM ( ( calc_noncalc_measure (k:k+i_file_count(i)-1, j, 1) - &
                                           r_mean ) ** 2 ) )
 !
       END DO

       ! Time spent on OASIS interpolation is a mean value among log files
       r_interp_time(i) = SUM(r_interp_measure(k:k+i_file_count(i)-1))/i_file_count(i)

       ! Counter on log file index among total log file number
       k = k + i_file_count(i)

    END DO
!
!   WRITE INFO ON STANDARD OUTPUT
!
!   Old analysis, no more active
!
!    WRITE(6,*) ' '
!    WRITE(6,*), ' Component -           Computation -       Waiting time (s) -  # cpl step '

!    DO i = 1, nb_models
!       WRITE(6,'(2X, A6, 5X, F10.2, A7, F6.2, A3, 5X, F10.2, A7, F6.2, A4, I4)'), &
!                  model_name(i), &
!                  calc_time(i), &
!                  ' ( +/- ', calc_spread(i) , ' ) ', &
!                  noncalc_time(i), &
!                  ' ( +/- ', send_spread(i)+receive_spread(i), ' )  ', &
!                  valid_comm_nb(i)-first_valid_comm(i)+1
!    END DO
!    WRITE(6,*), ' '
!    call flush(6)
!
!   7.2 ANALYSIS ON BOUNDARY VALUES AMONG LOG FILES
!
!
    r_min_time = 0. ; r_max_time = 0.
!
    calc_time(:) = 0.
    noncalc_time(:) = 0.
    calc_time(:) = 0.
    r_jitter_time(:) = 0.

    ! loop on models
    DO k = 1, nb_models
       ! loop on coupling fields
       DO i = 1, cpl_field_nb

          ! If sent field
          IF ( cpl_fields(i)%source_model == k ) THEN
             ! loop on coupling time steps
                                     ! WARNING valid_comm_nb depends on cpl_field_nb (if different model
                                     ! with diff cpl time step)
             DO j = first_valid_comm(k), valid_comm_nb(k)
                ! If a timing is available for this coupling field at this coupling time step
                IF ( max_clock_measure (i,j,2) < r_test_impossible .and. max_clock_measure (i,j,1) < r_test_impossible ) &
                   ! add sending time to the total of non calculation time
                   noncalc_time(k) = max_clock_measure (i,j,2) - max_clock_measure (i,j,1) + &
                                     noncalc_time(k)
                   ! WARNING : sending time starts when slowest mpi process check on log files is before sending
                   !            and stops when slowest mpi process check on log files is after sending
                IF ( max_clock_measure (i,j,1) < r_test_impossible .and. ABS(min_clock_measure (i,j,1)) < r_test_impossible ) &
                   ! Measure before sending between slowest and fastest mpi process check on log files
                   r_jitter_time(k) = max_clock_measure (i,j,1) - min_clock_measure (i,j,1) + &
                                      r_jitter_time(k)
             ENDDO
          ! If received field
          ELSE IF ( cpl_fields(i)%target_model == k ) THEN
             ! loop on coupling time steps
                                     ! WARNING valid_comm_nb depends on cpl_field_nb (if different model
                                     ! with diff cpl time step)
             DO j = first_valid_comm(k), valid_comm_nb(k)
                ! If a timing is available for this coupling field at this coupling time step
                IF ( max_clock_measure (i,j,4) < r_test_impossible .and. max_clock_measure (i,j,3) < r_test_impossible ) &
                   ! add receiving time to the total of non calculation time
                   noncalc_time(k) = max_clock_measure (i,j,4) - max_clock_measure (i,j,3) + &
                                     noncalc_time(k)
                   ! WARNING : receiving time starts when slowest mpi process check on log files is before receiving
                   !            and stops when slowest mpi process check on log files is after receiving
                IF ( max_clock_measure (i,j,3) < r_test_impossible .and. ABS(min_clock_measure (i,j,3)) < r_test_impossible ) &
                   ! Measure before receiving between slowest and fastest mpi process check on log files
                   r_jitter_time(k) = max_clock_measure (i,j,3) - min_clock_measure (i,j,3) + &
                                      r_jitter_time(k)
             ENDDO
          ENDIF
       ENDDO
!
       r_min_time = r_impossible_value * (-1.)
       r_max_time = r_impossible_value * (-1.)

!      CALCULATE TIME BOUNDS

       l_put = .true.

       ! Loop on coupling fields
       DO i = 1, cpl_field_nb
          ! on target model
          IF ( cpl_fields(i)%target_model == k ) THEN
             ! Measure first valid time when field received (after receiving)
             temp_t = max_clock_measure(i,first_valid_comm(k)-1,4)
             ! If later than reference
             IF ( temp_t > r_min_time .and. temp_t < r_test_impossible ) &
                ! Set it as reference 
                r_min_time = temp_t
             ! Measure last valid time when field received (after receiving)
             temp_t = max_clock_measure(i,valid_comm_nb(k),4)
             ! If later than reference
             IF ( temp_t > r_max_time .and. temp_t < r_test_impossible ) &
                ! Set it as reference 
                r_max_time = temp_t
             l_put = .false.
          ENDIF
       ENDDO

       ! IF NO RECEIVED FIELD ON MODEL DO THE SAME THAN PREVIOUSLY BUT WITH SENT FIELDS
       IF ( l_put ) THEN
          ! Loop on coupling fields
          DO i = 1, cpl_field_nb
             ! on target model
             IF ( cpl_fields(i)%source_model == k ) THEN
                ! Measure first valid time when field received (after receiving)
                temp_t = max_clock_measure(i,first_valid_comm(k)-1,2)
                ! If later than reference
                IF ( temp_t > r_min_time .and. temp_t < r_test_impossible ) &
                   ! Set it as reference 
                   r_min_time = temp_t
                ! Measure last valid time when field received (after receiving)
                temp_t = max_clock_measure(i,valid_comm_nb(k),2)
                ! If later than reference
                IF ( temp_t > r_max_time .and. temp_t < r_test_impossible ) &
                   ! Set it as reference 
                   r_max_time = temp_t
             ENDIF
          ENDDO
       ENDIF
!
!      CALCULATION TIME defined as total time minus OASIS communication time
       calc_time(k) = r_max_time - r_min_time - noncalc_time(k)

    ! End loop on models
    ENDDO
!
    WRITE(6,*) ' '
    WRITE(6,*), ' Load balance analysis '
    WRITE(6,*) ' '
    WRITE(6,*), ' Component -         Calculations   -     Waiting time (s) - # cpl step :'
!
!   WRITE INFO ON DAT FILE FOR GNUPLOT AND STANDARD OUTPUT
!
    WRITE(6,*) ' '
    OPEN (10, file="info.dat")
    DO i = 1, nb_models
       WRITE(10,'(I2, 2X, F10.3, 2X, F10.3, 2X, A6)'), &
                  i, calc_time(i), noncalc_time(i), model_name(i)
       WRITE(6,'(2X, A6, 16X, F10.2, 12X, F10.2, 10X, I4)'), &
                    model_name(i), calc_time(i), noncalc_time(i), valid_comm_nb(i)-first_valid_comm(i)+1
    ENDDO
    CLOSE (10)
          
    WRITE (6,*) ' '
!
    WRITE(6,*), ' Additional informations'
    WRITE(6,*), ' Component -  OASIS mean interpolation time -    Jitter  (s): '
    DO i = 1, nb_models
       WRITE(6,'(2X, A6, 12X, F10.2, 18X, F10.2 )'), &
                  model_name(i), r_interp_time(i), r_jitter_time(i)
    END DO
!
    WRITE (6,*) '  '
    WRITE (6,*) ' lucia completed successfully '
    WRITE (6,*) '  '

end program lucia_analysis