#! /bin/csh
#
# Job script to run "gswm".
# This script may be submitted to IBM (AIX), SGI (IRIX), or Sun workstation.
# Use the "submit" command (hao:/home/tgcm/bld/submit) to submit this job.
#
# Resources for SGI (e.g., dataproc):
# QSUB -s /bin/csh
# QSUB -lT 60:00       # time limit
# QSUB -q ded_1        # ded_1
#
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# To submit:
#   1. If qsub job (e.g., SGI dataproc), set resources above.
#   2. Set env vars below (JOB,EXEC,OUTPUT,REMOTE,POSTCLEAN)
#      (REMOTE might be the only one you have to change)
#   3. Include namelist input file (see "$input" below)
#   4. Optionally change path to source and Makefile (see INCLUDEs below)
#   5. If IBM job, set class (queue) for loadleveler (see "# @ class" below)
#      (optionally set other loadleveler commands for IBM job)
#   6. Use the "submit" command (/home/tgcm/bld/submit) to submit the job.
#      (if on Sun workstation, quit submit, then execute the saved .submit file)
#
# JOB   : Job name (will be used in tmpdir name, input file name, etc)
# EXEC  : Name of executable built by make (see Makefile).
# OUTPUT: File name of stdout output file.
# REMOTE: Machine:path directory where OUTPUT is to be sent via rcp.
#         (typically on the local workstation from which this script was submitted)
# POSTCLEAN: Set to 1 to remove execution directory after job completion,
#            or set to 0 to save execution directory.
#
setenv JOB gswm
setenv EXEC gswm
setenv REMOTE vishnu.hao:/home/foster/ntwk
setenv OUTPUT $JOB.$$.out
setenv POSTCLEAN 0
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
# Determine operating system:
#
setenv HOST `uname -n`
set os = `uname -s`
if ($os =~ *'IRIX'*) then
  set os = IRIX
else if ($os =~ *'AIX'*) then
  set os = AIX
else if ($os =~ *'SunOS'*) then
  set os = SUN
else
  echo " "
  echo ">>> Unknown OS: uname -s = $os"
  echo "Job $JOB NOT submitted."
  exit
endif
setenv OS $os
#
# Set up a tmpdir and go there:
#
if ($os == 'AIX') then
  set tmpdir = /ptmp/$user/$JOB.$$
else if ($os == 'IRIX') then
  set tmpdir = /tmp/$user/$JOB.$$
else # sun
  set tmpdir = /e/$user/$JOB.$$
endif
if (! -e $tmpdir) then
  echo "Making tmpdir $tmpdir"
  set err = 0
  mkdir -p $tmpdir || set err = 1
  if ($err == 1) then
    echo "Error making tmpdir $tmpdir."
    echo "Job $JOB NOT submitted."
    exit
  endif
else
  echo "NOTE: tmpdir $tmpdir already exists."
endif
setenv EXECDIR $tmpdir
cd $EXECDIR
echo "Moved to submit dir $EXECDIR"
#
# Init output file:
#
touch $OUTPUT
#
echo " "
echo "JOB       = $JOB"
echo "OS        = $os"
echo "HOST      = $HOST"
echo "EXECDIR   = $EXECDIR"
#
echo " "                     >> $OUTPUT
echo "JOB       = $JOB"      >> $OUTPUT
echo "OS        = $os"       >> $OUTPUT
echo "HOST      = $HOST"     >> $OUTPUT
echo "EXECDIR   = $EXECDIR"  >> $OUTPUT
echo "DATE      = `date`"    >> $OUTPUT
echo " "                     >> $OUTPUT
#
if ($OS == 'AIX') then
#
# Set some env vars for AIX parallel job:
#
setenv MP_LABELIO YES
setenv MP_PGMMODEL SPMD
setenv MP_SHARED_MEMORY YES
#
# Make loadleveler command file:
#
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
cat << 'EOF_LL' >! loadlev.job
#
# Loadleveler job step to build and execute:
#
# @ job_name         = gswm
# @ step_name        = exec
# @ class            = share
## @ class            = com_pr
# @ wall_clock_limit = 00:30:00
## @ job_type         = parallel
# @ job_type         = serial
## @ network.MPI      = css0,shared,us
## @ total_tasks      = 12
## @ node             = 4
# @ environment      = COPY_ALL
# @ executable       = $(step_name).csh
# @ notification     = error
# @ queue
#
# Loadleveler job step for cleanup:
#
# @ step_name        = cleanup
# @ class            = interactive
# @ wall_clock_limit = 00:00:30
# @ job_type         = serial
# @ executable       = $(step_name).csh
# @ notification     = complete
# @ requirements     = (Machine == "$(hostname)")
# @ dependency       = (exec == 0) || (exec != 0)
# @ queue
#
'EOF_LL'
endif # make loadleveler script for AIX system
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
# Build make and exec script:
#
cat << 'EXEC_STEP' >! exec.csh
#! /bin/csh
#
# Shell script for loadleveler execution step:
# Build and execute gswm. 
#
cd $EXECDIR
echo " "
echo "Host = $HOST"
echo "OS   = $OS"
echo "Current working directory = $EXECDIR"
echo " "
#
# Get source and Makefile:
# (INCLUDE statements are resolved by "submit" script (hao:/home/tgcm/bld/submit))
#
INCLUDE -h ~hagan/GSWM/Makefile
INCLUDE -h ~hagan/GSWM/GSWM.inp
INCLUDE -h ~hagan/GSWM/waves.inp
INCLUDE -h /home/gswm/GSWM/GSWM/*.f
INCLUDE -h /home/gswm/GSWM/lib/*.f
#INCLUDE -h /home/gswm/GSWM/dat/ciraO3_*
#INCLUDE -h /home/gswm/GSWM/dat/criso3_*
#INCLUDE -h /home/gswm/GSWM/dat/halmls_*
INCLUDE -h /home/gswm/GSWM/dat/groves_*
#INCLUDE -h /home/gswm/GSWM/dat/h2o_*semi
INCLUDE -h /home/gswm/GSWM/dat/hrdi0*
#INCLUDE -h /home/gswm/GSWM/dat/crista_u*
#INCLUDE -h /home/gswm/GSWM/dat/cris_*
INCLUDE -h /home/gswm/GSWM/dat/revised_gs_50.tables
#INCLUDE -h /home/gswm/GSWM/dat/randel*
#INCLUDE -h /home/gswm/GSWM/dat/khattatov.kzz
#INCLUDE -h /home/gswm/GSWM/dat/gciforce/*apr*.diurn_sm
#INCLUDE -h /home/gswm/GSWM/dat/gciforce/*apr*.semid_sm
INCLUDE -h /home/gswm/GSWM/dat/gciforce/*W2*.semid_sm
#INCLUDE -h /home/gswm/GSWM/dat/7yr-12mos.*
#INCLUDE -h /home/gswm/GSWM/dat/euv_*x.diu
#INCLUDE -h /home/gswm/GSWM/dat/euv_*x.sem
#
# Include user's input file:
#
#set input = $JOB.inp
#cat << 'EOFINP' >! $input
#INCLUDE gswm.inp
#'EOFINP'
#
# Build executable:
#
set err = 0
make >>& $OUTPUT || set err = 1
if ($err == 0) then
  echo "Successful make." >> $OUTPUT
else
  echo "Error from make." >> $OUTPUT
  exit
endif
#
#INCLUDE -h /home/tgcm/bld/mkdep
#perl mkdep >& /dev/null
#set err = 0
#if ($OS == 'SUN') then
#  gnumake $EXEC >>& $OUTPUT || set err = 1
#else
#  gmake $EXEC >>& $OUTPUT || set err = 1
#endif
#if ($err == 0) then
#  echo "Successful gnu make." >> $OUTPUT
#else
#  echo "Error from gnu make." >> $OUTPUT
#  exit
#endif
#
# Execute:
#
set err = 0
#$EXEC < $JOB.inp >>& $OUTPUT || set err = 1
$EXEC >>& $OUTPUT || set err = 1
if ($err == 0) then
  echo "Completed execution of $EXEC." >> $OUTPUT
else
  echo ">>> Error executing $EXEC." >> $OUTPUT
endif
'EXEC_STEP'
chmod u+x exec.csh
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
# Build cleanup script:
#
cat << 'EOF_CLEANUP' >! cleanup.csh
#! /bin/csh
#
# Shell script for loadleveler cleanup step:
#
# Run mklogs to separate tasks stdout, and send output from task 0:
#
set err = 0
mklogs $OUTPUT || set err = 1
if ($err == 1) then
  echo ">>> Error from mklogs -- will return full output $OUTPUT" >> $OUTPUT
else
  mv *task0.out $OUTPUT
  echo "Completed mklogs, sending output from task 0." >> $OUTPUT
endif  
#
# Remote copy output and remove submitdir.
#
echo " " >> $OUTPUT
echo "Execution directory submitdir = $EXECDIR" >> $OUTPUT
echo "Preparing to rcp output $OUTPUT from submitdir to remote $REMOTE" >> $OUTPUT
if ($POSTCLEAN == 1) then
  echo "NOTE: submitdir and its contents will be removed after the rcp"
endif
#
cd $EXECDIR
set err = 0
rcp $OUTPUT $REMOTE || set err = 1
if ($err != 0) then
  echo " " >> $OUTPUT
  echo ">>> WARNING: error from rcp of output $OUTPUT to $REMOTE" >> $OUTPUT
endif
#
# Clean submit directory:
#
if ($POSTCLEAN == 1) then
  cd
  rm -r $EXECDIR
endif
exit
'EOF_CLEANUP'
chmod u+x cleanup.csh
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
# Submit the job:
#
# If AIX, submit to loadleveler:
#
if ($OS == 'AIX') then
  llsubmit loadlev.job
#
# If Sun, assume no batch job, so execute exec.sh interactively in background
# and do not execute cleanup step.
#
else if ($OS == 'SUN') then
  echo " "
  echo "Executing on $OS system in directory $EXECDIR"
  exec.csh &
#
# If not AIX or Sun (i.e., SGI), execute both exec.csh and cleanup.csh:
# (in this case, the "submit" command has submitted the job via qsub)
#
else
  echo " "
  echo "Executing on $OS system in directory $EXECDIR"
  exec.csh
  cleanup.csh
endif