#!/bin/bash -e

# --------------------------------------------------
#
# Script to launch coupled simulations with WRF, WW3 and CROCO 
#
# --------------------------------------------------
#
# Further Information:   
# http://www.croco-ocean.org
#  
# This file is part of CROCOTOOLS
#
# CROCOTOOLS is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published
# by the Free Software Foundation; either version 2 of the License,
# or (at your option) any later version.
#
# CROCOTOOLS is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA  02111-1307  USA
#
# Copyright (c) 2018 S. Jullien
# swen.jullien@ifremer.fr
# --------------------------------------------------

#=========================================================================
#=======================  USER CHANGES  ==================================
#=========================================================================

# source file exporting paths 
# for the configuration
# ----------------------------
source run_env

# Path of working directory 
#(where the simulations are launched)
# ----------------------------
export MYWORKDIR=$wconf/outputs_cpl_ow
# flag for cleaning working dir or not
flag_clean=1

# Simulation type and models 
# (named as in namcouple)
# ----------------------------
RUN=ow
#RUN=aw
#RUN=oa
#RUN=owa
#RUN=FRC
#
# "no" if no model
#mod1=wrfexe
#mod1=toyexe
mod1=no
mod2=wwatch
#mod2=toyexe
#mod2=no
mod3=crocox
#mod3=toyexe
#mod3=no

#
# Number of CPUs for each model
# ----------------------------
NBPROC_1=$1
NBPROC_2=$2
NBPROC_3=$3
#MPI_LAUNCH_CMD="mpirun"
#MPI_LAUNCH_CMD="poe"
MPI_LAUNCH_CMD=$MPI_LAUNCH
#SERIAL_LAUNCH_WW3='./'
SERIAL_LAUNCH_WW3="$MPI_LAUNCH -np 1 "

# Runtime Settings
# ----------------------------
yr1=2009 ; mo1=01 ; dy1=01 ; hr1=00
#yr2=2009 ; mo2=02 ; dy2=01 ; hr2=00
#runtime=$((31*24*3600))
yr2=2009 ; mo2=01 ; dy2=11 ; hr2=00
runtime=$((10*24*3600))

# Time Steps
# ----------------------------
atmdt=150
wavdt=3600 ; wavdtPRO=1200 ; wavdtREF=2400 ; wavdtSRC=60
ocedt=3600 ; ocendtfast=60 ; ocentimes=$((${runtime}/${ocedt}))
cpldt=3600

# Grids sizes
# ----------------------------
atmnx=56 ; atmny=50
wavnx=43 ; wavny=44
ocenx=41 ; oceny=42
hmin=75; # minimum water depth in CROCO, delimiting coastline in WW3 

# Outputs Settings
# ----------------------------
# WRF
wrf_rst_flag=false                 # start from restart ?
wrf_rst_h=24                       # restart interval (h)
wrf_his_h=1                        # output interval (h)
wrf_his_frames=$((31*24))          # nb of outputs per file
wrf_diag_int_m=$((${wrf_his_h}*60))  # diag output interval (m)
wrf_diag_frames=$wrf_his_frames    # nb of diag outputs per file

# WW3 (everything in seconds)
wav_int=3600            # output interval (s)
wav_rst=$((24*3600))    # restart interval (s)

# CROCO
oce_nrst=24    # restart interval (in number of timesteps) 
oce_nhis=1     # history output interval (in number of timesteps) 
oce_navg=1     # average output interval (in number of timesteps) 

# MPI Settings for WRF (see WRF namelist documentation README.namelist in real_in directory)
wrf_nprocX=-1      # -1 for automatic settings
wrf_nprocY=-1      # -1 for automatic settings
wrf_niotaskpg=0    # 0 for default settings
wrf_niogp=1        # 1 for default settings

# Paths for executables
# ----------------------------
export WRF_EXE_DIR=$wrf/exe_coupled
export WW3_EXE_DIR=$ww3/exe_${RUN}_BENGUELA
export CROCO_EXE_DIR=$PWD/croco_in
export TOY_EXE_DIR=$PWD/toy_in

# Namelist files
# ----------------------------
# OASIS namcouple
namcouplename=namcouple.base.${RUN}
#namcouplename=namcouple.base.${RUN}.toyatm

# WRF namelist
wrfnamelist=namelist.input.prep.BENGUELA.${RUN}

# WRF coupled domain(s) (set cplmask to 1 in the following domain(s))
# set to empty if you have already defined your cplmask variable by yourself
wrfcpldom='d01'

# OASIS restart files
# ----------------------------
# flag for creating restart from calm coniditons (1) or not (0)
# if not, atm.nc, wav.nc and oce.nc restart files have to exist in OASIS_FILES_DIR
flag_restart=1
# ww3 file to be used for creating restart file for oasis
ww3file=$wconf/outputs_frc_ww3_CFSR/ww3.200901.nc

# TOY specifications
#-------------------
toyfile=$wconf/outputs_frc_wrf/wrfout_d01_2009-01-01_00\:00\:00
toytype=atm
model_to_toy=wrf
toydt=${atmdt} ; toytimes=$((${runtime}/${toydt}))

# Inputs Settings
# ----------------------------

# WRF
#-----
# date in WRF input files
date='2009_01_2009_12'
# number of WRF domains
nb_dom=1
# is nudging activated?
nudge=0

# CROCO
#-------
# suffix for CROCO executable
crocosuffix=".${RUN}"
# list of CROCO input files
inputlist='ini bry frc'
# suffix for CROCO input files
datesuffix=''
datesuffixcfsr='_Y20??M*'
# if online interpolation of atmospheric fields is used
interponline=0

# WW3
#-----
# suffix for ww3 input setting files
ww3inpsuffix=".${RUN}"
# forcing file(s) list
# name of input file
forcin=(CFSR_wind_Y2009M1.nc)
# name of ww3_prnc.inp extension/input file
forcww3=(wind)
#forcin=(CROCO_current_Y2009M1.nc \
#        CROCO_level_Y2009M1.nc)
#forcww3=(current \
#         level)
#forcin=()
#forcww3=()

#=========================================================================
#=======================  END USER CHANGES  ==============================
#=========================================================================

## ---------------------------- #
## - Create and Clean Workdir - #
## ---------------------------- #

if ! [ -e $MYWORKDIR ] ; then
 echo 'create working directory: '$MYWORKDIR
 mkdir $MYWORKDIR
elif [ $flag_clean == 1 ] ; then
 echo 'clean working directory: '$MYWORKDIR
 rm -Rf $MYWORKDIR/*
fi

## -------------------- #
## - Copy executables - #
## -------------------- #

if [ $mod1 = wrfexe -o $mod2 = wrfexe -o $mod3 = wrfexe ] ; then
 echo 'copy wrf executables'
 cp -f $WRF_EXE_DIR/wrf.exe $MYWORKDIR/wrfexe
fi
if [ $mod1 = wwatch -o $mod2 = wwatch -o $mod3 = wwatch ] ; then
 echo 'copy ww3 executables'
 cp -f $WW3_EXE_DIR/ww3_* $MYWORKDIR/.
 mv -f $MYWORKDIR/ww3_shel $MYWORKDIR/wwatch
fi
if [ $mod1 = crocox -o $mod2 = crocox -o $mod3 = crocox ] ; then
 echo 'copy croco executables'
 cp -f $CROCO_EXE_DIR/croco$crocosuffix $MYWORKDIR/crocox
fi
if [ $mod1 = toyexe -o $mod2 = toyexe -o $mod3 = toyexe ] ; then
 echo 'copy toy executables'
 cp -f ${TOY_EXE_DIR}/toy_model $MYWORKDIR/toyexe
fi

if [ $mod1 = wrfexe -o $mod2 = wrfexe -o $mod3 = wrfexe ] ; then
 ## ------------------------ #
 ## - Copy wrf input files - #
 ## ------------------------ #

 echo ' '
 echo '-- WRF inputs --------------'
 echo 'link wrf input files'
 filelist='wrfinput_d01 wrflowinp_d01 wrfbdy_d01'
 if [ $nb_dom -ge 2 ] ; then
  filelist="$filelist wrfinput_d02 wrflowinp_d02"
  if [ $nb_dom -eq 3 ] ; then
   filelist="$filelist wrfinput_d03 wrflowinp_d03"
  fi
 fi
 if [ $nudge -eq 1 ] ; then
  filelist="$filelist wrffdda_d01"
 fi
 for file in $filelist 
 do
  echo "ln -sf ${WRF_FILES_DIR}/${file}_${date} $MYWORKDIR/$file"
  ln -sf ${WRF_FILES_DIR}/${file}_${date} $MYWORKDIR/$file
 done

 if [ ! -z $wrfcpldom ] ; then
  for dom in $wrfcpldom ; do
   echo 'set CPLMASK to 1 in coupled domain'$dom
   echo "ncap2 -O -s 'CPLMASK(:,0,:,:)=(LANDMASK-1)*(-1)' $MYWORKDIR/wrfinput_$dom $MYWORKDIR/wrfinput_$dom"
   ncap2 -O -s "CPLMASK(:,0,:,:)=(LANDMASK-1)*(-1)" $MYWORKDIR/wrfinput_$dom $MYWORKDIR/wrfinput_$dom
  done
 fi

 # link data files necessary for running wrf in a dedicated directory $wrf/data
 if [ ! -d $wrf/data ] ; then
  mkdir $wrf/data
  ln -s $wrf/run/* $wrf/data/.
  # remove executables that could exist and namelist file
  rm -f $wrf/data/*.exe
  rm -f $wrf/data/namelist.input*
 fi
 echo 'link wrf data files'
 echo "ln -sf ${wrf}/data/* $MYWORKDIR/."
 ln -sf ${wrf}/data/* $MYWORKDIR/.

 echo 'fill wrf namelist file'
 ## Fill WRF namelist file
sed -e "s/<yr1>/$yr1/g"   -e "s/<yr2>/$yr2/g"  \
    -e "s/<mo1>/$mo1/g"   -e "s/<mo2>/$mo2/g"  \
    -e "s/<dy1>/$dy1/g"   -e "s/<dy2>/$dy2/g"  \
    -e "s/<hr1>/$hr1/g"   -e "s/<hr2>/$hr2/g"  \
    -e "s/<rst>/$wrf_rst_flag/g"              -e "s/<rst_int_h>/$wrf_rst_h/g"            \
    -e "s/<his_int_h>/${wrf_his_h}/g"         -e "s/<his_nb_out>/${wrf_his_frames}/g"    \
    -e "s/<xtrm_int_m>/${wrf_diag_int_m}/g"   -e "s/<xtrm_nb_out>/${wrf_diag_frames}/g"  \
    -e "s/<nproc_x>/$wrf_nprocX/g"            -e "s/<nproc_y>/$wrf_nprocY/g"             \
    -e "s/<niotaskpg>/$wrf_niotaskpg/g"       -e "s/<niogp>/$wrf_niogp/g"                \
    -e "s/<dt>/${atmdt}/g"                    \
    $WRF_IN_DIR/${wrfnamelist} > $MYWORKDIR/namelist.input

 if [ -f $WRF_IN_DIR/myoutfields.txt ]; then
  echo 'copy myoutfields.txt file'
  cp -f $WRF_IN_DIR/myoutfields.txt $MYWORKDIR/.
 fi

fi

if [ $mod1 = wwatch -o $mod2 = wwatch -o $mod3 = wwatch ] ; then
 ## ------------------------ #
 ## - Copy ww3 input files - #
 ## ------------------------ #

 WW3_IN_DIR=${WW3_IN_DIR}/inputs_ww3

 echo ' '
 echo '-- WW3 inputs --------------'
 echo 'copy and fill ww3 settings files *.inp'
 cp -f ${WW3_IN_DIR}/*.inp $MYWORKDIR/.

 ## - Fill ww3_grid.inp file -##
sed -e "s/<wavdt>/$wavdt/g" \
    -e "s/<wavdtPRO>/$wavdtPRO/g"  -e "s/<wavdtREF>/$wavdtREF/g"  -e "s/<wavdtSRC>/$wavdtSRC/g"  \
    -e "s/<wavnx>/$wavnx/g"   -e "s/<wavny>/$wavny/g"  \
    -e "s/<hmin>/$hmin/g" \
    $WW3_IN_DIR/ww3_grid.inp.base > $MYWORKDIR/ww3_grid.inp

 ## - Fill ww3_ounf.inp file -##
sed -e "s/<wav_int>/$wav_int/g" \
    -e "s/<yr1>/$yr1/g"  -e "s/<mo1>/$mo1/g"  -e "s/<dy1>/$dy1/g"  -e "s/<hr1>/$hr1/g" \
    $WW3_IN_DIR/ww3_ounf.inp.base > $MYWORKDIR/ww3_ounf.inp

## - Fill ww3_shel.inp file -##
sed -e "s/<yr1>/$yr1/g"  -e "s/<mo1>/$mo1/g"  -e "s/<dy1>/$dy1/g"  -e "s/<hr1>/$hr1/g"  \
    -e "s/<yr2>/$yr2/g"  -e "s/<mo2>/$mo2/g"  -e "s/<dy2>/$dy2/g"  -e "s/<hr2>/$hr2/g" \
    -e "s/<wav_int>/$wav_int/g"  -e "s/<wav_rst>/$wav_rst/g" \
    -e "s/<wavdt>/$wavdt/g" \
    $WW3_IN_DIR/ww3_shel.inp.base$ww3inpsuffix > $MYWORKDIR/ww3_shel.inp

 cp -f ${WW3_FILES_DIR}/*.inp $MYWORKDIR/.

 ls $MYWORKDIR/*.inp

 echo 'link ww3 input files and copy associated settings files'
 lengthforc=${#forcww3[@]}
 for  k in `seq 0 $(( ${lengthforc} - 1))` ; do
  echo "ln -sf ${WW3_FILES_DIR}/${forcin[$k]} $MYWORKDIR/${forcww3[$k]}.nc"
  ln -sf ${WW3_FILES_DIR}/${forcin[$k]} $MYWORKDIR/${forcww3[$k]}.nc
  echo "cp -f ${WW3_IN_DIR}/ww3_prnc.inp.${forcww3[$k]} $MYWORKDIR/."
  cp -f ${WW3_IN_DIR}/ww3_prnc.inp.${forcww3[$k]} $MYWORKDIR/.
 done

fi 

if [ $mod1 = crocox -o $mod2 = crocox -o $mod3 = crocox ] ; then
 ## -------------------------- #
 ## - Copy CROCO input files - #
 ## -------------------------- #

 echo ' '
 echo '-- CROCO inputs --------------'
 echo 'copy croco settings files'
 cp -f $CROCO_EXE_DIR/cppdefs.h${crocosuffix} $MYWORKDIR/.
 cp -f $CROCO_EXE_DIR/param.h $MYWORKDIR/.

 echo 'fill namelist croco.in'
 ## - Fill croco.in file -##
sed -e "s/<ocentimes>/$ocentimes/g" -e "s/<ocedt>/$ocedt/g"       -e "s/<ocendtfast>/$ocendtfast/g" \
    -e "s/<oce_nrst>/$oce_nrst/g"   -e "s/<oce_nhis>/$oce_nhis/g" -e "s/<oce_navg>/$oce_navg/g"     \
    -e "s/<yr1>/$yr1/g"             -e "s/<mo1>/$mo1/g"           \
    -e "s/<yr2>/$yr2/g"             -e "s/<mo2>/$mo2/g"           \
    $CROCO_IN_DIR/croco.in.base > $MYWORKDIR/croco.in

 echo 'link croco input files'
 echo "ln -sf ${CROCO_FILES_DIR}/croco_grd.nc $MYWORKDIR/."
 ln -sf ${CROCO_FILES_DIR}/croco_grd.nc $MYWORKDIR/.
 for file in $inputlist ; do
  echo "ln -sf ${CROCO_FILES_DIR}/croco_${file}${datesuffix}.nc $MYWORKDIR/croco_${file}.nc"
  ln -sf ${CROCO_FILES_DIR}/croco_${file}${datesuffix}.nc $MYWORKDIR/croco_${file}.nc
 done
 if [ $interponline == 1 ] ; then
  for varlist in Downward_Short-Wave_Rad_Flux_surface \
                 Downward_Long-Wave_Rad_Flux \
                 Upward_Long-Wave_Rad_Flux_surface \
                 Upward_Short-Wave_Rad_Flux_surface \
                 U-component_of_wind \
                 V-component_of_wind \
                 Precipitation_rate \
                 Temperature_height_above_ground \
                 Specific_humidity \
                 Temperature_surface
  do
   echo "ln -sf ${CROCO_FILES_DIR}/*/${varlist}${datesuffixcfsr}.nc $MYWORKDIR/."
   ln -sf ${CROCO_FILES_DIR}/*/${varlist}${datesuffixcfsr}.nc $MYWORKDIR/.
  done
 fi

fi

if [ $RUN != FRC ] ; then 

 ## --------------------------------------- #
 ## - Fill namcouple with times and grids - #
 ## --------------------------------------- #
 echo ' '
 echo '-- OASIS inputs --------------'
 echo 'fill oasis namcouple'
sed -e "s/<runtime>/$runtime/g" \
    -e "s/<cpldt>/$cpldt/g"     \
    -e "s/<atmdt>/$atmdt/g"   -e "s/<atmnx>/$atmnx/g"   -e "s/<atmny>/$atmny/g"  \
    -e "s/<wavdt>/$wavdt/g"   -e "s/<wavnx>/$wavnx/g"   -e "s/<wavny>/$wavny/g"  \
    -e "s/<ocedt>/$ocedt/g"   -e "s/<ocenx>/$ocenx/g"   -e "s/<oceny>/$oceny/g"  \
    $OASIS_IN_DIR/inputs_oasis/$namcouplename > $MYWORKDIR/namcouple

 ## ---------------------------------- #
 ## - Create restart files for oasis - #
 ## ---------------------------------- #

 if [ $flag_restart == 1 ] ; then

  echo 'copy oasis scripts'
  cp -f $OASIS_IN_DIR/*.sh $MYWORKDIR/.

  echo 'enter working directory'
  cd $MYWORKDIR

  if [ $mod1 = wrfexe -o $mod2 = wrfexe -o $mod3 = wrfexe ] ; then
   varlist='WRF_d01_EXT_d01_SURF_NET_SOLAR WRF_d01_EXT_d01_EVAP-PRECIP WRF_d01_EXT_d01_SURF_NET_NON-SOLAR WRF_d01_EXT_d01_TAUX WRF_d01_EXT_d01_TAUY WRF_d01_EXT_d01_TAUMOD WRF_d01_EXT_d01_U_01 WRF_d01_EXT_d01_V_01'
   echo 'create restart file for oasis from calm conditions for variables:'$varlist
   ./create_oasis_restart_from_calm_conditions.sh wrfinput_d01 atm.nc wrf "$varlist"
  fi

  if [ $mod1 = wwatch -o $mod2 = wwatch -o $mod3 = wwatch ] ; then 
   varlist='WW3_T0M1 WW3__OHS WW3_CDIR WW3_SDIR WW3__CHA WW3_TAWX WW3_TAWY WW3_TWOX WW3_TWOY'
   echo 'create restart file for oasis from calm conditions for variables:'$varlist
   ./create_oasis_restart_from_calm_conditions.sh $ww3file wav.nc ww3 "$varlist"
  fi

  if [ $mod1 = crocox -o $mod2 = crocox -o $mod3 = crocox ] ; then
   varlist='SRMSSTV0 SRMSSHV0 SRMVOCE0 SRMUOCE0'
   echo 'create restart file for oasis from calm conditions for variables:'$varlist
   ./create_oasis_restart_from_calm_conditions.sh croco_grd.nc oce.nc croco "$varlist"
  fi

  if [ $mod1 = toyexe -o $mod2 = toyexe -o $mod3 = toyexe ] ; then
   varlist='TOY_V_01 TOY_U_01 TOY_TAUX TOY_TAUY TOY_TAUM TOYSRFLX TOYSTFLX TOY__EMP TOY_UOCE TOY_VOCE TOY__SST TOY__SSH TOY_T0M1 TOY___HS TOY_CDIR TOY_SDIR TOY_TWOX TOY_TWOY TOY_TAWX TOY_TAWY TOY__CHA'
   echo 'create restart file for oasis from calm conditions for variables:'$varlist
   ./create_oasis_restart_from_calm_conditions.sh $toyfile ${toytype}.nc $model_to_toy "$varlist"
  fi

  echo 'leave working directory'
  cd -

 else # - flag_restart

  if [ $mod1 = wrfexe -o $mod2 = wrfexe -o $mod3 = wrfexe ] ; then
   echo 'copy restart file for oasis: atm.nc'
   cp -f ${OASIS_FILES_DIR}/atm.nc $MYWORKDIR
  fi
  if [ $mod1 = wwatch -o $mod2 = wwatch -o $mod3 = wwatch ] ; then
   echo 'copy restart file for oasis: wav.nc'
   cp -f ${OASIS_FILES_DIR}/wav.nc $MYWORKDIR
  fi
  if [ $mod1 = crocox -o $mod2 = crocox -o $mod3 = crocox ] ; then
   echo 'copy restart file for oasis: oce.nc'
   cp -f ${OASIS_FILES_DIR}/oce.nc $MYWORKDIR
  fi

 fi # - flag_restart

 if [ $mod1 = toyexe -o $mod2 = toyexe -o $mod3 = toyexe ] ; then
  ## -------------------------- #
  ## - Copy TOY input files -   #
  ## -------------------------- #
  echo "fill TOYNAMELIST.nam.${toytype}.${RUN}"
sed -e "s/<toydt>/$toydt/g" -e "s/<toytimes>/$toytimes/g"   \
    ${TOY_EXE_DIR}/TOYNAMELIST.nam.${toytype}.${RUN} > $MYWORKDIR/TOYNAMELIST.nam

  echo 'create input files for TOY'
  cd ${OASIS_IN_DIR}
  ./create_oasis_toy_files.sh $toyfile $MYWORKDIR/toy_${toytype}.nc $model_to_toy 1,$toytimes
  cd -
 fi

 # grid files from WRF to OASIS (because oasis_grid is not yet implemented in WRF)
 if [ $mod1 = wrfexe -o $mod2 = wrfexe -o $mod3 = wrfexe ] ; then
  echo 'copy oasis scritpts into working dir'
  cp -f $OASIS_IN_DIR/*.sh $MYWORKDIR/.
  cd $MYWORKDIR
  echo 'create grids.nc, masks.nc containing wrf grid for oasis'
  ./create_oasis_grids_for_wrf.sh $MYWORKDIR/wrfinput_d01 ${OASIS_FILES_DIR}
  cd -
  echo 'copy grids.nc, masks.nc containing wrf grid'
  cp -f ${OASIS_FILES_DIR}/grids.wrf.nc $MYWORKDIR/grids.nc
  cp -f ${OASIS_FILES_DIR}/masks.wrf.nc $MYWORKDIR/masks.nc
  #cp -f ${OASIS_FILES_DIR}/areas.wrf.nc $MYWORKDIR/areas.nc
 fi

fi
## ------------- #
## - Execution - #
## ------------- #

echo ' '
echo '-- RUN section --------------'
echo 'enter in the working directory'
cd $MYWORKDIR
pwd

# Prepare MPI run command
if [ $MPI_LAUNCH_CMD == poe ] ; then
 if [ $mod1 != no ] ; then
  for nn in $(seq 1 $NBPROC_1); do
   echo "./$mod1" >> run_file
  done
 fi
 if [ $mod2 != no ] ; then
  for nn in $(seq 1 $NBPROC_2); do
   echo "./$mod2" >> run_file
  done
 fi
 if [ $mod3 != no ] ; then
  for nn in $(seq 1 $NBPROC_3); do
   echo "./$mod3" >> run_file
  done
 fi
 chmod +x run_file
 mpirun_cmd="poe  -pgmmodel MPMD -cmdfile ./run_file"
else
#elif [ $MPI_LAUNCH_CMD == mpirun ] ; then
 if [ $mod1 != no ] ; then
  mpirun_cmd="$MPI_LAUNCH_CMD -np $NBPROC_1 $mod1"
 fi
 if [ $mod2 != no ] ; then
  if [ ! -z $mpirun_cmd ] ; then
    mpirun_cmd="$mpirun_cmd : -np $NBPROC_2 $mod2"
  else
    mpirun_cmd="$MPI_LAUNCH_CMD -np $NBPROC_2 $mod2"
  fi
 fi
 if [ $mod3 != no ] ; then
  mpirun_cmd="$mpirun_cmd : -np $NBPROC_3 $mod3" 
 fi

#else
# echo 'ERROR: '$MPI_LAUNCH_CMD' not implemented yet... Exit'
# exit
fi

if [ $mod1 = wwatch -o $mod2 = wwatch -o $mod3 = wwatch ] ; then

 echo 'WW3 pre-processing before run:'
 # WW3 grid 
 echo "${SERIAL_LAUNCH_WW3}ww3_grid &> grid.out"
 ${SERIAL_LAUNCH_WW3}ww3_grid &> grid.out
 if [ ! -e mod_def.ww3 ]; then
   echo 'ERROR when running ww3_grid, mod_def.ww3 does not exist'
   exit 1
 fi
 # WW3 prnc
 for  k in `seq 0 $(( ${lengthforc} - 1))` ; do
  echo "ln -sf ww3_prnc.inp.${forcww3[$k]} ww3_prnc.inp"
  ln -sf ww3_prnc.inp.${forcww3[$k]} ww3_prnc.inp
  echo "${SERIAL_LAUNCH_WW3}ww3_prnc &> prnc.${forcww3[$k]}.out"
  ${SERIAL_LAUNCH_WW3}ww3_prnc &> prnc.${forcww3[$k]}.out
  if [ ! -e ${forcww3[$k]}.ww3 ]; then
   echo 'ERROR when running ww3_prnc for '${forcww3[$k]}
   exit 1
  fi
 done
 # WW3 strt
 echo "${SERIAL_LAUNCH_WW3}ww3_strt &> strt.out"
 ${SERIAL_LAUNCH_WW3}ww3_strt &> strt.out
 if [ ! -e restart.ww3 ]; then
   echo 'ERROR when running ww3_strt, restart.ww3 does not exist'
   exit 1
 fi

fi

echo 'launch run: '$mpirun_cmd
# RUN
$mpirun_cmd

if [ $mod1 = wwatch -o $mod2 = wwatch -o $mod3 = wwatch ] ; then

 # WW3 format to netcdf
 if [ -e out_grd.ww3 ]; then
  echo 'WW3 post-processing after run:'
  # WW3 format to netcdf
  echo "${SERIAL_LAUNCH_WW3}ww3_ounf &> ounf.out"
  ${SERIAL_LAUNCH_WW3}ww3_ounf &> ounf.out
  #${SERIAL_LAUNCH_WW3}ww3_ounp &> ounp.out
 fi

fi