#! /bin/csh
#
# Job script to run "tgcmproc" on IBM system.
#
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# To submit:
#   1. Set env vars below (JOB,EXEC,OUTPUT,REMOTE,POSTCLEAN)
#      (REMOTE might be the only one you have to change)
#   2. Include namelist input file (see "$input" below)
#   3. Optionally change path to source and Makefile (see INCLUDEs below)
#   4. Set class (queue) for loadleveler (see "# @ class" below)
#      (optionally set other loadleveler commands for IBM job)
#   5. Use the "submit" command (/home/tgcm/bld/submit) to submit the job.
#
# JOB   : Job name (will be used in EXECDIR name, input file name, etc)
# EXEC  : Name of executable built by make (see Makefile).
# OUTPUT: File name of stdout output file.
# REMOTE: Machine:path directory where OUTPUT is to be sent via scp.
#         (typically on the local workstation from which this script was submitted)
# EXECDIR: Execution directory (e.g., /ptmp/$user/tgcmproc.$$)
# POSTCLEAN: Set to 1 to remove execution directory after job completion,
#            or set to 0 to save execution directory.
#
setenv JOB tgcmproc
setenv EXEC tgcmproc_f90
setenv REMOTE vishnu.hao:/home/foster/ntwk
#setenv OUTPUT $JOB.$$.out
#setenv OUTPUT gordon.tgcm24rh.vcase.$$.out
#setenv OUTPUT gordon.tgcm24rh.ncase.$$.out
setenv OUTPUT gordon.timegcm1.1.py2b.$$.out
setenv EXECDIR /hao/foster/gordon/$JOB.$$
setenv POSTCLEAN 0
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
# Determine operating system (AIX is only OS for this script):
#
setenv HOST `uname -n`
set os = `uname -s`
if ($os =~ *'AIX'*) then
  set os = AIX
else
  echo " "
  echo ">>> Unknown OS: uname -s = $os"
  echo "Job $JOB NOT submitted."
  exit
endif
setenv OS $os
#
# Make EXECDIR and go there:
#
if (! -e $EXECDIR) then
  echo "Making EXECDIR $EXECDIR"
  set err = 0
  mkdir -p $EXECDIR || set err = 1
  if ($err == 1) then
    echo " "
    echo "WARNING Error making EXECDIR $EXECDIR."
    echo "Job $JOB NOT submitted."
    echo " "
  endif
else
  echo "NOTE: EXECDIR $EXECDIR already exists."
endif
cd $EXECDIR
echo "Moved to exec dir $EXECDIR"
#
# Init output file:
#
touch $OUTPUT
#
echo " "
echo "JOB       = $JOB"
echo "OS        = $os"
echo "HOST      = $HOST"
echo "EXECDIR   = $EXECDIR"
echo "DATE      = `date`"
#
echo " "                      >> $OUTPUT
echo "JOB       = $JOB"       >> $OUTPUT
echo "OS        = $os"        >> $OUTPUT
echo "HOST      = $HOST"      >> $OUTPUT
echo "EXECDIR   = $EXECDIR"   >> $OUTPUT
echo "POSTCLEAN = $POSTCLEAN" >> $OUTPUT
echo "DATE      = `date`"     >> $OUTPUT
echo " "                      >> $OUTPUT
#
# Make loadleveler command file:
#
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
cat << 'EOF_LL' >! loadlev.job
#
# Loadleveler job step to build and execute:
#
# Example: With total_tasks=32, node=8, and DYNAMO=2, a 1-day run at 
#   STEP=180 (480 iters) takes about 9 mins wallclock, NOT including 
#   mss access, and without secondary histories.
#
#
# @ job_name         = tgcmproc
## @ account_no      = 28103032
# @ step_name        = exec
# @ class            = share
## @ class           = dv_reg
# @ wall_clock_limit = 06:00:00
# @ job_type         = serial
# @ environment      = COPY_ALL
# @ executable       = $(step_name).csh
# @ notification     = error
# @ requirements     = (Machine == "$(hostname)")
# @ queue
#
# Loadleveler job step for cleanup:
#
# @ step_name        = cleanup
# @ class            = interactive
# @ wall_clock_limit = 00:00:30
# @ job_type         = serial
# @ executable       = $(step_name).csh
# @ notification     = complete
# @ requirements     = (Machine == "$(hostname)")
# @ dependency       = (exec == 0) || (exec != 0)
# @ queue
#
'EOF_LL'
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
# Build make and exec script:
#
cat << 'EXEC_STEP' >! exec.csh
#! /bin/csh
#
# Shell script for loadleveler execution step:
# Build and execute tgcmproc. 
#
cd $EXECDIR
echo " "
echo "Host = $HOST"
echo "OS   = $OS"
echo "Current working directory = $EXECDIR"
echo " "
#
# Get executable from mss and set permissions:
# Use /TGCM/tgcmproc/tgcmproc_aix to get default version.
#
 set mss_exec = /TGCM/tgcmproc/tgcmproc_aix     # default version
#set mss_exec = /TGCM/tgcmproc/tgcmproc1.2_aix  # specified version
#
set err = 0
msrcp mss:${mss_exec} $EXEC || set err = 1
if ($err == 0) then
  echo "Acquired $mss_exec from NCAR mss"
else
  echo ">>> Error acquiring $mss_exec from NCAR mss"
  exit
endif
chmod u+x $EXEC
#
# Include user's input file:
#
set input = $JOB.inp
cat << 'EOFINP' >! $input
INCLUDE /home/foster/usr/gordon_shepard/gordon.dv.sat.green.inp
'EOFINP'
#
# Execute:
#
set err = 0
echo "Begin execution of $EXEC at `date`" >> $OUTPUT
timex $EXEC < $JOB.inp >>& $OUTPUT || set err = 1
if ($err == 0) then
  echo "Completed execution of $EXEC at `date`" >> $OUTPUT
else
  echo ">>> Error executing $EXEC at `date`" >> $OUTPUT
endif
'EXEC_STEP'
chmod u+x exec.csh
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
# Build cleanup script:
#
cat << 'EOF_CLEANUP' >! cleanup.csh
#! /bin/csh
#
# Remote copy output and remove submitdir.
#
echo " " >> $OUTPUT
echo "Execution directory submitdir = $EXECDIR" >> $OUTPUT
ls -l $OUTPUT >> $OUTPUT
echo "Preparing to scp output $OUTPUT from submitdir to remote $REMOTE" >> $OUTPUT
if ($POSTCLEAN == 1) then
  echo "NOTE: submitdir and its contents will be removed after the scp"
endif
#
set err = 0
scp $OUTPUT $REMOTE >>& $OUTPUT || set err = 1
if ($err != 0) then
  echo " " >> $OUTPUT
  echo ">>> WARNING: error from scp of output $OUTPUT to $REMOTE" >> $OUTPUT
  echo "    Will save $OUTPUT to /home/$user"
  cp $OUTPUT /home/$user || \
    echo ">>> WARNING: error copying $OUTPUT to /home/$user"
endif
#
# Clean exec directory:
#
if ($POSTCLEAN == 1) then
  cd
  rm -r $EXECDIR
endif
exit
'EOF_CLEANUP'
chmod u+x cleanup.csh
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
# Submit the job:
#
llsubmit loadlev.job