#
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#                               End user settings
#                        Shell Script for TIEGCM Linux job
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
# Update ESMF module on ys as of 4/14/14:
 set lib_esmf = esmf-6.3.0r-ncdfio-mpi-O
 module load esmf $lib_esmf
#
set mycwd = `pwd`
echo "" ; echo "${0}:"
echo "  Begin execution at `date`"
echo "  Current working directory: $mycwd"
echo "  System: `uname -a`"  
echo ""
#
# Verify directories and make_machine file (make execdir if necessary).
# Get absolute path for dirs that are accessed from the execdir.
#
if (! -d $modeldir) then
  echo ">>> Cannot find model directory $modeldir <<<"
  exit 1
endif
set model = $modeldir:t

if ($model == '') then
  echo "Please remove trailing slash from modeldir $modeldir"
  exit
endif

if (! -d $utildir) then
  echo ">>> Cannot find model scripts directory $utildir <<<" 
  exit 1
endif
set utildir = `perl $utildir/abspath $utildir`

set srcdir  = $modeldir/src
if (! -d $srcdir) then
  echo ">>> Cannot find model source directory $srcdir <<<"
  exit 1
endif
set srcdir = `perl $utildir/abspath $srcdir`

if ($modelres != 5.0 && $modelres != 2.5) then
  echo ">>> Unknown model resolution $modelres <<<"
  exit 1
endif
#
# Copy make files to execdir if necessary:
#
if (! -f $execdir/$make)     cp $utildir/$make $execdir
if (! -f $execdir/Makefile)  cp $utildir/Makefile $execdir
if (! -f $execdir/mkdepends) cp $utildir/mkdepends $execdir
#
# Make default namelist input file if not provided by user:
#
if ($?input) then
  if (! -f $input) then
    echo ">>> Cannot find namelist input file $input <<<"
    exit 1
  endif
else 
  set input = \
    `perl $utildir/mknamelist -model=$model -modelres=$modelres` || \
     echo ">>> Error from mknamelist: fileout = $input" && exit 1
endif

set model  = $execdir/$model
set input  = `perl $utildir/abspath $input`
set output = `perl $utildir/abspath $output`
set mklogs = `perl $utildir/abspath $utildir`
set mklogs = $mklogs/mklogs.ys # special mklogs for yellowstone
#
# Report to stdout:
#
set svnversion = `svnversion $modeldir` || set svnversion = "[none]"
echo -n "  Model directory:   $modeldir" && echo " (SVN revision $svnversion)"
echo "  Exec directory:    $execdir"
echo "  Source directory:  $srcdir"
echo "  Make machine file: $make"
echo "  Namelist input:    $input"
echo "  Stdout Output:     $output"
echo "  Model resolution:  $modelres"
echo "  Debug:             $debug"
echo "  MPI job:           $mpi"
#
# If debug flag has changed from last gmake, clean execdir
# and reset debug file:
#
if (-f $execdir/debug) then
  set lastdebug = `cat $execdir/debug` 
  if ($lastdebug != $debug) then
    echo "Clean execdir $execdir because debug flag switched from $lastdebug to $debug"
    set mycwd = `pwd` ; cd $execdir ; gmake clean ; cd $mycwd
    echo $debug >! $execdir/debug
  endif
else
  echo $debug >! $execdir/debug
  echo "Created file debug with debug flag = $debug"
endif
#
# If mpi flag has changed from last gmake, clean execdir
# and reset mpi file:
#
if (-f $execdir/mpi) then
  set lastmpi = `cat $execdir/mpi` 
  if ($lastmpi != $mpi) then
    echo "Clean execdir $execdir because mpi flag switched from $lastmpi to $mpi"
    set mycwd = `pwd` ; cd $execdir ; gmake clean ; cd $mycwd
    echo $mpi >! $execdir/mpi
  endif
else
  echo $mpi >! $execdir/mpi
  echo "Created file mpi with mpi flag = $mpi"
endif
#
# Copy defs header file to execdir, if necessary, according to 
# requested resolution. This should seamlessly switch between
# resolutions according to $modelres.
#
set defs = $srcdir/defs5.0
if ($modelres == 2.5) set defs = $srcdir/defs2.5
if (-f $execdir/defs.h) then
  cmp -s $execdir/defs.h $defs
  if ($status == 1) then # files differ -> switch resolutions
    echo "Switching defs.h for model resolution $modelres"
    cp $defs $execdir/defs.h
  else
    echo "defs.h already set for model resolution $modelres"
  endif 
else # defs.h does not exist in execdir -> copy appropriate defs file
  echo "Copying $defs to $execdir/defs.h for resolution $modelres"
  cp $defs $execdir/defs.h
endif
#
# cd to execdir and run make:
#
cd $execdir || echo ">>> Cannot cd to execdir $execdir" && exit 1
echo ""
echo "Begin building $model in `pwd`"
#
# Build Make.env file in exec dir, containing needed env vars for Makefile:
#
cat << EOF_env >! Make.env
MAKE_MACHINE = $make
DIRS         = . $srcdir 
MPI          = $mpi
EXECNAME     = $model
NAMELIST     = $input
OUTPUT       = $output
DEBUG        = $debug
SVN_VERSION  = $svnversion
EOF_env
#
# Build the model:
gmake -j8 all || echo ">>> Error return from gmake all" && exit 1
#
if ($exec == "FALSE") then
  echo "Am NOT executing the model because exec = $exec"
  exit 0
endif
#
# MPI/LSF job: append mpirun.lsf command to LSF script 
# (it has #BSUBs from above)
#
if ($mpi == "TRUE") then  # MPI LSF job
  cat << EOF_runscript >> $runscript
  setenv MP_LABELIO YES
  setenv MP_SHARED_MEMORY yes
  mpirun.lsf $model < $input >&! $output
  $mklogs $output
EOF_runscript
  echo " "
  echo "Submitting LSF script $runscript for MPI run"
  bsub < $runscript
#
# Non-mpi job: simple execution statement on the interactive node.
# If you want to submit lsf job, you can use a here-doc like above
# to put the execution statement in the runscript, then submit with
# bsub as above.
#
else
  echo "Executing serial non-MPI run in background on the interactive node.." 
  $model < $input >&! $output &
endif
exit 0