#
# Submit tiegcm1 job to PSC Compaq system lemieux.
# Resources for PBS batch system:
#
#PBS -l walltime=1:00:00
##PBS -l rmsnodes=3:12
#PBS -l rmsnodes=8:32
#PBS -j oe
#PBS -m abe
#PBS -M foster@ucar.edu
#
set JOB = tiegcm1
set EXEC = tiegcm1
set EXECDIR = $SCRATCH/${JOB}_5.0h.$$
set POSTCLEAN = 0
#
# Data files are on PSC archive machine Cray j90 golem.
# These will be copied to EXEC dir prior to execution.
#
set GOLEM = golem:/usr/users1/0/bfoster/$JOB
#
# 5 deg res:
#set  SOURCE = $GOLEM/ROBLE.tiegcm1.py93g18.nc
set SOURCE = $GOLEM/FOSTER.tiegcm1.py93g18.nc
set MAGVOL = $GOLEM/TGCM.data.magfield.nc
set OUTPUT = (FOSTER.tiegcm1.py93g18.nc FOSTER.tiegcm1.py93g19.nc \
              FOSTER.tiegcm1.seqsm41.nc FOSTER.tiegcm1.seqsm42.nc)
#
# Make and move to the EXECDIR directory:
#
if (-d $EXECDIR) then
  echo "Removing existing EXECDIR $EXECDIR"
  /usr/bin/rm -rf $EXECDIR
endif
mkdir -p $EXECDIR
cd $EXECDIR
#
echo " "
echo Begin $JOB job at `date`
echo JOB       = $JOB
echo EXEC      = $EXEC
echo EXECDIR   = $EXECDIR
echo POSTCLEAN = $POSTCLEAN
echo " "
#
# Source code and related files:
#
INCLUDE -h /home/tgcm/tiegcm1/*.F /home/tgcm/tiegcm1/*.h
INCLUDE -h /home/tgcm/bld/mkdep /home/tgcm/bld/mklogs
INCLUDE -h /home/tgcm/tiegcm1/Makefile
INCLUDE -h -ignore_dups /home/foster/tiegcm/tiegcm1/modsrc_dhz/*.F 
INCLUDE -h -ignore_dups /home/foster/tiegcm/tiegcm1/modsrc_dhz/*.h
INCLUDE -h -ignore_dups /home/tgcm/tiegcm1/defs.h
#
#INCLUDE /home/foster/tiegcm/tiegcm1/tiegcm1dhz.inp
cat << 'EOFINP' >! $JOB.inp
INCLUDE /home/foster/tiegcm/tiegcm1/tiegcm1_5.0.inp
'EOFINP'
#
# Load modules:
#
eval `/usr/local/Modules/$MODULE_VERSION/bin/modulecmd csh load netcdf`
#
# Build:
#
set err = 0
perl mkdep || set err = 1
if ($err != 0) then
  echo ">>> Error making dependencies with perl mkdep"
endif
set err = 0
gmake $EXEC || set err = 1
if ($err == 0) then
  echo "Successful gnu make."
else
  echo ">>> Error from gnu make."
  exit $err
endif
#
# Copy data files from golem:
#
echo "Begin remote copy of $SOURCE to $EXECDIR at `date`"
rcp $SOURCE $EXECDIR || echo ">>> Error from rcp of $SOURCE to $EXECDIR"
echo "End remote copy of $SOURCE to $EXECDIR at `date`"
#
echo "Begin remote copy of $MAGVOL to $EXECDIR at `date`"
rcp $MAGVOL $EXECDIR || echo ">>> Error from rcp of $MAGVOL to $EXECDIR"
echo "End remote copy of $MAGVOL to $EXECDIR at `date`"
#
# Execute:
#
set err = 0
echo "Executing prun.."
prun -N ${RMS_NODES} -n ${RMS_PROCS} -v -s -o ./$JOB.%.out -e ./$JOB.%.out \
  ./$EXEC < ./$JOB.inp || set err = 1
if ($err == 0) then
  echo "Completed execution of $EXEC."
else
  echo ">>> Error executing $EXEC."
endif
#
# Save task0 output to golem:
#
if (-e $JOB.0.out) then
  set OUTLIST = $GOLEM/${JOB}lx.$$.task0.out
  set err = 0
  rcp $JOB.0.out $OUTLIST || err = 1
  if ($err == 0) then
    echo "Copied task0 output $JOB.0.out to $OUTLIST"
  else
    echo ">>> Error copying task0 output $JOB.0.out to $OUTLIST"
  endif
else
  echo ">>> WARNING: could not find task0 output file $JOB.0.out"
endif
#
# Save history output files to golem:
#
foreach output ($OUTPUT)
  set err = 0
  if (-e $output) then
    rcp $output $GOLEM/$output || err = 1
    if ($err == 0) then
      echo "Copied history output $output to $GOLEM/$output"
    else
      echo ">>> Error copying history output $output to $GOLEM/$output"
    endif
  else
    echo ">>> WARNING: could not find history output file $output"
  endif
end
#
# Save task0 output to dataproc.ucar.edu:
# This will require authentication key file for scp.
#
#if (-e $JOB.0.out) then
#  set err = 0
#  scp $JOB.0.out dataproc.ucar.edu:$JOB.$$.out || err = 1
#  if ($err == 0) then
#    echo "Copied task0 output to $GOLEM/$JOB.$$.out"
#  else
#    echo ">>> Error copying task0 output to $GOLEM/$JOB.$$.out"
#  endif
#endif
#
# Clean up (remove execdir and its contents):
#
if ($POSTCLEAN == 1) then
  echo "Removing EXECDIR $EXECDIR."
  rm -rf $EXECDIR
endif
exit