#!/bin/tcsh
#
# This script should be executed on the yellowstone command line. It will 
# compile in execdir on the interactive login node, then create and submit 
# an LSF script (also in execdir), based on the #BSUB resources specified below.
#
# Set shell variables and BSUB settings below:
#
#   modeldir:  Root directory to model source (may be an SVN working dir)
#   execdir:   Directory in which to build and execute (will be created if necessary)
#   input:     Namelist input file (will use default if not provided)
#   output:    Stdout file from model execution (will be created)
#   make:      Build file with platform-specific compile parameters (in scripts dir)
#              Supported compiler files: Make.intel_ys or Make.pgi_ys
#   mpi:       TRUE/FALSE for MPI or non-MPI run
#   modelres:  Model resolution (5.0 or 2.5 degrees)
#   debug:     If TRUE, build and execute a "debug" run
#   exec:      If TRUE, execute the model (build only if FALSE)
#   utildir:   Dir containing supporting scripts (usually $modeldir/scripts)
#   runscript: LSF script with run commands (created in and submitted from execdir)
#
# Use the "modules" command on yellowstone to load/unload compilers, libs, etc:
#   module list     Get list of currently loaded modules
#   module avail    See which modules are available in your current setup
#   module load     Load one or more available modules
#   module switch   Switch between modules (e.g., compilers)
# See also the module man page on yellowstone.
#
# To switch compilers (default is intel):
#   From intel to pgi: type "module switch intel pgi", and use Make.pgi_ys below
#   From pgi to intel: type "module switch pgi intel", and use Make.intel_ys below
#
 set modeldir = vtgcm_trunk
 set execdir  = vtgcm-linux
#set execdir  = /glade/p/work/[user]/vtgcm/vtgcm-linux
#
#set input    = vtgcm.inp
 set output   = vtgcm.out
 set modelres = 5.0
#
# Intel *or* PGI modules must be loaded on yellowstone (see comment above)
 set make     = Make.intel_ys
#set make     = Make.pgi_ys

set mpi       = TRUE
set debug     = FALSE
set exec      = TRUE
set utildir   = $modeldir/scripts
set runscript = $execdir/run.lsf

if (! -d $execdir) mkdir -p $execdir
#
# Set LSF resource usage (create the runscript in execdir):
# (run commands are appended to this script below)
#
cat << EOF >! $runscript
#! /bin/tcsh
#
#BSUB -J vtgcm
#BSUB -P P28100036
#BSUB -q regular
#BSUB -o vtgcm.%J.out
#BSUB -e vtgcm.%J.out
#BSUB -N
#BSUB -u $LOGNAME@ucar.edu
##BSUB -W 0:30
#BSUB -W 0:10
#
# Recommended for single-res (5.0 deg) vtgcm:
# -n16,ptile=16 (1 node, 16 procs/node)
#BSUB -n 16
#BSUB -R "span[ptile=16]"
#
# Recommended for double-res (2.5 deg) vtgcm: 
# -n64,ptile=16 (4 nodes, 16 procs/node)
##BSUB -n 64
##BSUB -R "span[ptile=16]"
#
EOF
#
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#                               End user settings
#                        Shell Script for TIEGCM Linux job
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
set mycwd = `pwd`
echo "" ; echo "${0}:"
echo "  Begin execution at `date`"
echo "  Current working directory: $mycwd"
echo "  System: `uname -a`"  
echo ""
#
# Verify directories and make_machine file (make execdir if necessary).
# Get absolute path for dirs that are accessed from the execdir.
#
if (! -d $modeldir) then
  echo ">>> Cannot find model directory $modeldir <<<"
  exit 1
endif
set model = $modeldir:t

if (! -d $utildir) then
  echo ">>> Cannot find model scripts directory $utildir <<<" 
  exit 1
endif

set srcdir  = $modeldir/src
if (! -d $srcdir) then
  echo ">>> Cannot find model source directory $srcdir <<<"
  exit 1
endif
set srcdir = `perl $utildir/abspath $srcdir`

if ($modelres != 5.0 && $modelres != 2.5) then
  echo ">>> Unknown model resolution $modelres <<<"
  exit 1
endif
#
# Copy make files to execdir if necessary:
#
if (! -f $execdir/$make)     cp $utildir/$make $execdir
if (! -f $execdir/Makefile)  cp $utildir/Makefile $execdir
if (! -f $execdir/mkdepends) cp $utildir/mkdepends $execdir
#
# Make default namelist input file if not provided by user:
#
if ($?input) then
  if (! -f $input) then
    echo ">>> Cannot find namelist input file $input <<<"
    exit 1
  endif
else 
  set input = \
    `perl $utildir/mknamelist -model=$model -modelres=$modelres` || \
     echo ">>> Error from mknamelist: fileout = $input" && exit 1
endif
set input  = `perl $utildir/abspath $input`
set output = `perl $utildir/abspath $output`
set mklogs = `perl $utildir/abspath $utildir`
set mklogs = $mklogs/mklogs.ys # special mklogs for yellowstone
#
# Report to stdout:
#
set svnversion = `svnversion $modeldir` || set svnversion = "[none]"
echo -n "  Model directory:   $modeldir" && echo " (SVN revision $svnversion)"
echo "  Exec directory:    $execdir"
echo "  Source directory:  $srcdir"
echo "  Make machine file: $make"
echo "  Namelist input:    $input"
#
# Copy defs header file to execdir, if necessary, according to 
# requested resolution. This should seamlessly switch between
# resolutions according to $modelres.
#
set defs = $srcdir/defs5.0
if ($modelres == 2.5) set defs = $srcdir/defs2.5
if (-f $execdir/defs.h) then
  cmp -s $execdir/defs.h $defs
  if ($status == 1) then # files differ -> switch resolutions
    echo "Switching defs.h for model resolution $modelres"
    cp $defs $execdir/defs.h
  else
    echo "defs.h already set for model resolution $modelres"
  endif 
else # defs.h does not exist in execdir -> copy appropriate defs file
  echo "Copying $defs to $execdir/defs.h for resolution $modelres"
  cp $defs $execdir/defs.h
endif
#
# If debug flag has changed from last gmake, clean execdir
# and reset debug file:
#
if (-f $execdir/debug) then
  set lastdebug = `cat $execdir/debug` 
  if ($lastdebug != $debug) then
    echo "Clean execdir $execdir because debug flag switched from $lastdebug to $debug"
    set mycwd = `pwd` ; cd $execdir ; gmake clean ; cd $mycwd
    echo $debug >! $execdir/debug
  endif
else
  echo $debug >! $execdir/debug
  echo "Created file debug with debug flag = $debug"
endif
#
# cd to execdir and run make:
#
cd $execdir || echo ">>> Cannot cd to execdir $execdir" && exit 1
echo ""
echo "Begin building $model in `pwd`"
#
# Build Make.env file in exec dir, containing needed env vars for Makefile:
#
cat << EOF >! Make.env
MAKE_MACHINE = $make
DIRS         = . $srcdir 
MPI          = $mpi
EXECNAME     = $model
NAMELIST     = $input
OUTPUT       = $output
DEBUG        = $debug
SVN_VERSION  = $svnversion
EOF
#
# Build the model:
gmake -j8 all || echo ">>> Error return from gmake all" && exit 1
#
if ($exec == "FALSE") then
  echo "Am NOT executing the model because exec = $exec"
  exit 0
endif
set model = $execdir/$model
#
# Append mpi run command to LSF script (it has #BSUBs from above)
#
cat << EOF >> $runscript
setenv MP_LABELIO YES
setenv MP_SHARED_MEMORY yes
mpirun.lsf $model < $input >&! $output
$mklogs $output
EOF
#
# Submit LSF script:
#
echo " "
echo "Submitting LSF script $runscript"
bsub < $runscript
exit 0