#!/usr/bin/perl
use Getopt::Long;
#
# Given an executable, and 1 or more namelist read input files, submit multiple 
# runs under either LSF (e.g., bluevista or lightning), or LL (e.g., bluesky).
#
$progname = $0;
#
my $wallclock = "00:30"; # 30-minute default wall clock limit
my $nosubmit = 0;        # default is to submit the jobs
my $nnodes = 1;          # default number of nodes (LL only)
my $ntasks = 4;          # default total number of MPI tasks
my $interactive = 0;     # default is to submit to batch system
&GetOptions(
   "x|exec=s"        => \$exec,
   "bn|basename=s"   => \$basename,
   "wc|wallclock=s"  => \$wallclock,
   "q|queue=s"       => \$queue,
   "nn|nnodes=i"     => \$nnodes,
   "nt|ntasks=i"     => \$ntasks,
   "ns|nosubmit"     => \$nosubmit,
   "I|interactive"   => \$interactive,
   "h|help"          => \$help,
) or usage();
if ($help) { usage(); } # provide usage if help was requested
#
# Must provide an executable:
#
if (! $exec) {
  print "\n>>> $progname: Please provide an executable file.\n";
  print "(Use the command \"$progname -h\" for a usage message)\n\n";
  exit;
} else {
  $exec = glob($exec);
}
if (! -x $exec) {
  print "\n>>> $progname: Executable $exec must have execution permission.\n";
  print "(Use the command \"$progname -h\" for a usage message)\n\n";
  exit;
}
#
# Remaining args are input files (must provide at least one):
#
if (! @ARGV) {
  print "\n>>> $progname: Please provide at least one namelist input file.\n";
  print "(Use the command \"$progname -h\" for a usage message)\n\n";
  exit;
}
#
# Save input file names in @infiles:
#
while (@ARGV) {
  push @infiles, shift @ARGV;
} # while @ARGV
#
# Get nodename (may be needed to set #BSUB -a):
#
$nodename = `uname -n`; chop $nodename;
#
# Determine batch system type and set submit command accordingly:
#
$LoadShare  = $ENV{"LSF_ENVDIR"};
if ($LoadShare) {                            # LSF system
  $batchsys = "LSF"; 
  $submitcommand = "bsub < ";
  if (! $basename) { $basename = "LSF"; }
  if (! $queue) { $queue = "regular"; }
  print "  Batch system: Load Sharing Facility ($batchsys)\n";  
  if ($interactive) {
    print "\n>>> Can execute interactively only under LoadLeveler.\n\n";
    exit;
  }
} elsif (-d "/usr/lpp/LoadL") {              # LoadLeveler system
  $batchsys = "LL"; 
  $submitcommand = "llsubmit ";
  if (! $basename) { $basename = "LL"; }
  if (! $queue) { $queue = "com_rg8"; }
  print "  Batch system: LoadLeveler ($batchsys)\n";
} else {                                     # unknown batch system
  print "\n>>> $progname: cannot determine batch system:\n";
  print "I can submit to either Load Sharing Facility (LSF), or LoadLeveler (LL):\n";
  print "  Is not Load Share Facility (LSF), because env var LSF_ENVDIR does not exist.\n";
  print "  Is not LoadLeveler because directory /usr/lpp/LoadL does not exist.\n\n";
  exit;
}
#
# Report to stdout before input file loop:
#
print "\n$progname:\n";
print "  Exec file:      $exec\n";
print "  Input files:    @infiles\n";
print "  Submit command: \"$submitcommand\"\n";
print "  Base name:      \"$basename\"\n";
print "  Wallclock:      \"$wallclock\"\n";
print "  Queue:          \"$queue\"\n";
print "  Nnodes:         \"$nnodes\"\n";
print "  Ntasks:         \"$ntasks\"\n";
print "  Nosubmit:       $nosubmit\n";
print "  Interactive:    $interactive\n";
#
# Loop through input files:
#
$i = 1;
print "\n";
foreach $infile (@infiles) {
#
# Make batch job script:
#
  $job_csh = mkjob($batchsys,$exec,$infile,$queue,$wallclock,$nnodes,$ntasks,
                   $interactive,$i);
  print"Input file $i: $infile  Job script: $job_csh\n"; 
#
# Submit current job (unless nosubmit is set):
#
  if (! $nosubmit) {
#
# Submit to batch system:
    if (! $interactive) {
      $command = "$submitcommand $job_csh";
      $stat = system($command);
      if ($stat) { die ">>> Error executing \"$command\"\n"; }
#
# Execute interactively (under LL system only, e.g., bluesky):
    } else {
      my $outfile   = "${basename}_$i.out";  # stdout of model run
      print "Submitting $job_csh interacively in background...\n";
      $stat = system("$job_csh &");
      if ($stat) { die ">>> Error executing $job_csh\n"; }
    }
  }
#
# Increment job counter:
#
  $i++;
} # foreach $infile
exit;
#
#-------------------------------------------------------------------------
sub mkjob {
#
# Build job script for current job:
#
  my ($batchsys,$exec,$infile,$queue,$wallclock,$nnodes,$ntasks,$ia,$ijob) = @_;
#
  my $jobscript = "${basename}_$ijob.job";  # job script returned by this function
  my $outfile   = "${basename}_$ijob.out";  # stdout of model run
  my $jobname   = "${basename}_$ijob";      # step or job name passed to batch sys
#
  open (JOBSCRIPT,"> $jobscript") || die "make_default cannot open file $file";
#
# - - - - - - - - - - - - Build LoadLeveler job script - - - - - - - - - -
if ($batchsys eq "LL") {               # Load Leveler

print JOBSCRIPT <<EOF;
#! /bin/csh
#
# LoadLeveler commands (batch only):
#
# @ step_name        = $jobname
# @ class            = $queue
# @ node             = $nnodes
# @ total_tasks      = $ntasks
# @ node_usage       = not_shared
# @ wall_clock_limit = ${wallclock}:00
# @ job_type         = parallel
# @ network.MPI      = csss,shared,us
# @ queue
#
# Env vars (for interactive runs):
#
setenv MP_PROCS $ntasks
setenv MP_RMPOOL 1
setenv MP_PGMMODEL SPMD
setenv MP_SHARED_MEMORY YES
setenv MP_LABELIO YES
#
$exec < $infile >&! $outfile || \\
echo ">>> Execution of $exec FAILED at `date`" && echo "See output in $outfile"
EOF

# - - - - - - - - - - - - - - Build LSF job script - - - - - - - - - - - -
} elsif ($batchsys eq "LSF") {
  $bsub_a = "poe";
  if ($nodename =~ "ln*") { $bsub_a = "mpich_gm"; }       # lightning

print JOBSCRIPT <<EOF1;
#! /bin/csh
#
#BSUB -J $jobname         # job name
#BSUB -a $bsub_a          # Parallel Operating Environment
#BSUB -x                  # exclusive use of the node
#BSUB -n $ntasks          # number of MPI tasks
#BSUB -q $queue           # queue (e.g., premium or regular)
#BSUB -o $jobname.%J.out  # stdout filename
#BSUB -e $jobname.%J.out  # stderr filename
#BSUB -W $wallclock       # wallclock limit HH:MM
#
setenv MP_LABELIO YES
mpirun.lsf $exec < $infile >&! $outfile || \\
echo ">>> Execution (mpirun.lsf) of $exec FAILED at `date`" && echo "See output in $outfile"
EOF1

# - - - - - - - - - - - - - - - - - - - - - - - - -- - - - - - - - - - - -
  } else {
    print "\n>>> mkjob: unknown batch system: $batchsys\n\n";
    exit;
  }
#
# Return executable csh script:
#
chmod 0755, $jobscript;
return $jobscript;
}
#-------------------------------------------------------------------------
#
sub usage {
  die <<EOF;

SYNOPSIS
  $0 [options] inputfile1, [inputfile2, inputfile3,...]

DESCRIPTION

  Submit multiple jobs of a given executable, using provided namelist
  input files.  The -exec option is required (you must provide an executable),
  and you must provide at least one input file argument.

  This command will submit jobs to either LoadLeveler (LL), or 
    Load Sharing Facility (LSF) batch systems:
  On LL systems, the "llsubmit" command will be used to submit each job.
  On LSF systems, the "bsub" command will be used to submit each job.

OPTIONS       SHORT-FORM  DESCRIPTION
  -exec	        -x        Required executable file (relative or absolute path)
  -basename     -bn       Base name for script and output file names
  -wallclock    -wc       Wall-clock limit HH:MM (default 00:30)
  -queue        -q        Batch queue 
                          (it is the user's responsibility to request a valid queue)
  -ntasks       -nt       Total number of MPI tasks (integer) (default=4)
  -nosubmit     -ns       Set for trial run (jobs are not actually submitted) 
  -help	        -h        Print usage message and quit.

OPTIONS FOR LOADLEVELER SYSTEMS ONLY:
  -interactive  -I        Set to execute interactively instead of to LoadLeveler
                          (each job is started in background)
                          (wallclock and queue are ignored)
  -nnodes       -nn       Number of nodes (integer) (default=1) (LL only)

EXAMPLES
  $0 -exec=/ptmp/foster/tiegcm/tiegcm-aix/tiegcm tiegcm1.inp tiegcm2.inp
  $0 -exec="*aix/tiegcm" -basename=tiegcm *.inp
  $0 -exec="*aix/tiegcm" -queue=premium -wallclock=00:30 -nosubmit *.inp
  $0 -x="*aix/tiegcm" -q=premium -wc=00:30 -ns *.inp

EOF
}
#-------------------------------------------------------------------------