#! /bin/csh
#
# This is an example job script to configure, build, and execute a tgcm 
#   model on an IBM system (interactive or batch).
#
# This script can be submitted to either bluesky using llsubmit, or
# to bluevista, using bsub.
#
# To use:
#   If submitting to bluesky:
#     1) Set loadleveler commands (#@) below for batch job, or env vars (setenv)
#   If submitting to bluevista:
#     1) Set LSF commands (#BSUB) below for batch job.
#
#   2) Set script variables below (at least $model and $wrkdir)
#   3) Copy to a large working directory on the target machine.
#   4) Assuming this script is called "jobfile", then:
#
#   To submit to a batch queue on bluesky (loadleveler), use  "llsubmit jobfile"
#   To submit to a batch queue on bluevista (LSF), use  "bsub < jobfile"
#
# See http://download.hao.ucar.edu/pub/tgcm/doc/userguide for more information.
# If you need to customize the configuration beyond the shell vars here, 
#   add or change options to the tgcm_config call below. 
#
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#               USER EDIT LOADLEVELER COMMANDS (for bluesky LoadLeveler)
#
# @ step_name        = tiegcm_ibm_$(jobid)
## @ account_no      = 28103032
#
# @ class            = com_rg8
## @ class            = share
#
# Typical for 8-way nodes on bluesky or bluevista: node=2, total_tasks=16
# @ node             = 2
# @ total_tasks      = 16
# @ node_usage       = not_shared
#
# Wall-clock limit is usually 6 hours:
## @ wall_clock_limit = 02:00:00
# @ wall_clock_limit = 06:00:00
# @ job_type         = parallel
#
# Network on bluesky:   network.MPI = csss,shared,us
# @ network.MPI      = csss,shared,us
#
# Network on bluevista: network.MPI = sn_all,not_shared,us
## @ network.MPI      = sn_all,not_shared,us
#
# @ environment      = COPY_ALL
# @ output           = $(step_name).out
# @ error            = $(step_name).out
# @ notification     = complete
## @ ja_report        = yes
# @ queue
#
set os = `uname -s | tr "[A-Z]" "[a-z]"`
set nodename = `uname -n`
echo os = $os  nodename = $nodename
#
# MP_ and PAL_ env vars are for interactive runs only 
# (they are ignored by loadleveler):
#
setenv MP_LABELIO YES
if ($nodename =~ bs*) then # bluesky
  setenv MP_PROCS 4
  setenv MP_TASKS_PER_NODE 4
  setenv MP_RMPOOL 1
  setenv MP_PGMMODEL SPMD
  setenv MP_SHARED_MEMORY YES
  setenv PAL_ROOT /usr/local/vampir
  setenv PAL_LICENSEFILE /usr/local/vampir/etc/license.dat
endif
#
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
#    USER EDIT BATCH JOB (LSF) RESOURCES (for bluevista LSF)
#
#BSUB -J tiegcm         # job name
#BSUB -P 28103032
#BSUB -a poe            # Parallel Operating Environment
#BSUB -x                # exclusive use of the node
#BSUB -n 16             # number of MPI tasks
#BSUB -q regular        # queue (premium and regular are available)
##BSUB -q debug         # debug queue is available during the day (30 min limit)
#BSUB -o tiegcm.%J.out  # stdout filename
#BSUB -e tiegcm.%J.out  # stderr filename
#BSUB -N                # send mail when job finishes
#BSUB -u $LOGNAME@ucar.edu # email address
##BSUB -W 1:00           # wallclock limit (hh:mm) (6 hrs for prem or reg)
#BSUB -W 6:00           # wallclock limit (hh:mm) (6 hrs for prem or reg)
##BSUB -W 6:00           # wallclock limit (hh:mm) (6 hrs for prem or reg)
##BSUB -W 0:30          # 30 min limit for debug queue
#
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#        USER EDIT SHELL VARIABLES (for batch and interactive jobs)
#
# model:       Desired model name (Required. Use ListModels to see valid models)
# wrkdir:      User's working directory (default is `pwd`, i.e., the cwd)
# usr_srcdirs: List of user source directories (optional)
# input:       Namelist input file (if not given, a default will be created)
# output:      Name of stdout file relative to the wrkdir (default is $model.out)
# tgcmroot:    Root directory of distribution (default env var $TGCMROOT)
# tgcmdata:    Root directory containing input data files (default $TGCMDATA)
# execdir:     Directory in which to build and execute 
#              (default is $wrkdir/$model-$os)
# utildir:     Directory containing config script and needed utilities
#              (default is $tgcmroot/$model/scripts)
# mpi:         Load MPI libraries (can be TRUE, FALSE, or DEFAULT)
# hres:        Horizontal resolution can be either 5.0 (default), or 2.5 degrees
# vres:        Vertical resolution can be either 0.5 (default), or 0.25 (ln(p0/p))
# debug:       If debug=1, configure Makefile for a debug run (default is 0)
#
# Model output history files will be written to $execdir.
#
 set model       = tiegcm
 set wrkdir      = .
#set wrkdir      = $model-$$
 set input       = tiegcm.eqd.inp
#set output      = $model.out     # relative to wrkdir
 set usr_srcdirs = modsrc.eqd
 set tgcmroot    = .
#set tgcmdata    = $TGCMDATA
#set execdir     = $model-ibm
#set utildir     = $TGCMROOT/$model/scripts
#set mpi         = FALSE  # for serial runs (default is TRUE)
 set hres        = 2.5    # horizontal resolution (default is 5.0 deg)
 set vres        = 0.25   # vertical resolution (default is 0.5)
#set zibot       = -10.   # bottom boundary zp (for tiegcm, must be -10 or -7)
#set debug       = 1      # default is 0
#
#             USER SHOULD NOT HAVE TO EDIT BELOW THIS LINE
#             (except maybe to customize tgcm_config below)
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
set mycwd = `pwd`
echo " "
echo "${0}:"
echo "  Begin execution at `date`"
echo "  Current working directory: $mycwd"
echo "  System: `uname -a`"  
echo "  Log name: $LOGNAME"
set userguide = http://download.hao.ucar.edu/pub/tgcm/doc/userguide
#
# Model name is required:
#
if (! $?model) then
  echo "" ; echo ">>> Please set desired model (e.g., 'tiegcm' or 'timegcm')"
  echo "    See $userguide for more information."
  exit 1
endif
echo "  Model: $model"
#
if (! $?wrkdir) set wrkdir = `pwd`     # default working dir is cwd
if (! $?tgcmroot) set tgcmroot = ""
if (! $?tgcmdata) set tgcmdata = ""
if (! $?usr_srcdirs) set usr_srcdirs = ""
if (! $?hres) set hres = 5.0
if (! $?vres) set vres = 0.5
if (! $?zibot) set zibot = -10.
if (! $?mpi) set mpi = ""
if (! $?debug) set debug = 0
#
# Set utility dir from shell var $tgcmroot, or env var $TGCMROOT:
# (must contain Abspath, MakeNamelist, Mklogs, etc) 
#
if (! $?utildir) then
  if ($tgcmroot != "") then    # shell var from above (if set, trumps env var)
    set utildir = $tgcmroot/$model/scripts
  else if ($?TGCMROOT) then    # env var
    set utildir = $TGCMROOT/$model/scripts
  else
    echo "" ; echo ">>> WARNING: cannot determine utility directory."
    echo "" ; echo "Please set either shell var tgcmroot, or env var TGCMROOT"
    echo "  such that the directory tgcmroot/$model/scripts contains"
    echo "  the configuration script tgcm_config, and other needed utilities."
    echo "  See $userguide for more information."
    echo "" ; exit 1
  endif
else     # user provided utildir -- it must exist
  if (! -d $utildir) then
    echo ">>> Cannot find utility directory $utildir" && exit 1
  endif
endif
if (! -x $utildir/Abspath) echo ">>> Cannot find command $utildir/Abspath"
set utildir = `perl $utildir/Abspath $utildir` # full path to utildir
echo "  Utildir = $utildir"
#
set wrkdir = `perl $utildir/Abspath $wrkdir`  # full path to wrkdir
echo "  Wrkdir = $wrkdir"
#
# Set exec directory if necessary (wrkdir/model-os):
#
if (! $?execdir) then
  set os = `uname -s | tr "[A-Z]" "[a-z]"`
  set execdir = $wrkdir/$model-$os # if does not exist, will be created by config
endif
set execdir = `perl $utildir/Abspath $execdir`  # full path to execdir
echo "  Execdir = $execdir"
#
# Configure (pass shell vars to tgcm_config):
# See http://download.hao.ucar.edu/pub/tgcm/doc/userguide for more
#   options to tgcm_config.
#
if (! -x $utildir/tgcm_config) \
  echo ">>> WARNING: Cannot find config script $utildir/tgcm_config"
perl $utildir/tgcm_config -model=$model -wrkdir=$wrkdir -execdir=$execdir \
  -tgcmroot=$tgcmroot -tgcmdata=$tgcmdata -usr_srcdirs=$usr_srcdirs \
  -utildir=$utildir -hres=$hres -vres=$vres -mpi=$mpi -zibot=$zibot \
  -debug=$debug || \
  echo ">>> ${0}: Error from tgcm_config" && exit 1
#
# Copy Mkdepends script to execdir for manual gmake (or default Makefile):
cp $utildir/Mkdepends $execdir || \
  echo "WARNING: Error copying $utildir/Mkdepends to exec dir $execdir"
#
# Make default namelist input if not provided by user:
#
if (! $?input) then
  set input = \
    `perl $utildir/Mknamelist -model=$model -hres=$hres -vres=$vres` || \
     echo "${0}: Error from Mknamelist: fileout = $input" && exit 1
  echo "Made default namelist input file $input" 
else
  echo "Using user provided namelist input file $input"
endif
set namelist = `perl $utildir/Abspath $input`
if (! -e $namelist) then
  echo "${0}: Cannot find namelist input file $namelist" && exit 1
endif
#
# Build model in execdir:
#
cd $execdir || echo ">>> Cannot cd to execdir $execdir" && exit 1
echo "" ; echo "Begin building $model in `pwd` at `date`..."
/usr/local/bin/gmake -j4 || echo ">>> Error return from gmake" && exit 1
echo "End building $model at `date`..."
cd $mycwd  # return to original cwd
#
# Default stdout file is $model.out in the wrkdir:
#
if (! $?output) then
  set output = $wrkdir/$model.out
else
  set output = `perl $utildir/Abspath $wrkdir/$output`
endif
#
# Execute from execdir:
# 
cd $execdir || echo ">>> Error from cd to execdir $execdir" && exit 1
echo "" ; echo "Executing model $model from `pwd` at `date`"
echo "Model output will go to $output"
#
# 1/10/06 btf: bluesky (bs) and bluevista (bv) exec statements are different,
#              depending on batch system LoadLeveler (bs), or LSF (bv).
#
if ($nodename =~ bs*) then # bluesky (LoadLeveler)   # bluesky
  $execdir/$model < $namelist >&! $output || \
    echo ">>> ${0} Execution of $execdir/$model FAILED at `date`" && \
    echo "See output in $output"
else                                                # assume bluevista
  mpirun.lsf $execdir/$model < $namelist >&! $output || \
    echo ">>> ${0} Execution of mpirun.lsf $execdir/$model FAILED at `date`" && \
    echo "See output in $output"
endif          
echo "Run of $model completed at `date`"
#
# Separate output files by MPI task:
# (we are still in $execdir, but $output contains full path to $wrkdir)
#
perl $utildir/Mklogs $output || \
  echo ">>> ${0}: Error from $execdir/Mklogs on output $output"
#
# Prepare and submit mss dispose job (Mkdispose will make and submit 
# dispose.job only if dispose.csh exists in execdir, i.e., model was 
# run with DISPOSE=2):
#
perl $utildir/Mkdispose -submit
#
cd $mycwd # return to original cwd
echo "" ; echo "${0}: Completed execution at `date`"
exit 0