#!/usr/bin/perl
use Getopt::Long;
#
# Given an executable, and 1 or more namelist read input files, submit multiple 
#   runs under either LSF (e.g., bluevista or lightning), or LL (e.g., bluesky).
# If the system is not running LL or LSF, or interactive runs are requested, 
#   this script will attempt simple "command-line" executions of $exec.
# See the "multiinp" command for assistance creating multiple namelist
#   read files for input to this script.
#
my $wallclock = "00:30"; # 30-minute default wall clock limit
my $nosubmit = 0;        # default is to submit the jobs
my $nnodes = 1;          # default number of nodes (LL only)
my $ntasks = 4;          # default total number of MPI tasks
my $interactive = 0;     # default is to submit to batch system
my $serial = 0;          # default is parallel jobs (not serial)
my $sequential = 0;      # default is simultaeneous jobs (not sequential)
&GetOptions(
   "x|exec=s"        => \$exec,
   "bn|basename=s"   => \$basename,
   "wc|wallclock=s"  => \$wallclock,
   "q|queue=s"       => \$queue,
   "nn|nnodes=i"     => \$nnodes,
   "nt|ntasks=i"     => \$ntasks,
   "ns|nosubmit"     => \$nosubmit,
   "ia|interactive"  => \$interactive,
   "ser|serial"      => \$serial,
   "seq|sequential"  => \$sequential,
   "h|help"          => \$help,
) or 
  die "\n>>> $0: Error parsing options.\nPlease execute \"$0 -h\" for a usage message.\n\n";
#
if ($help) { usage(); } # provide usage if help was requested
#
# User must provide an executable:
#
if (! $exec) {
  print "\n>>> $0: Please provide an executable file.\n";
  print "(Use the command \"$0 -h\" for a usage message)\n\n";
  exit;
} else {
  $exec = glob($exec);
}
if (! -x $exec) {
  print "\n>>> $0: Executable $exec must have execution permission.\n";
  print "(Use the command \"$0 -h\" for a usage message)\n\n";
  exit;
}
#
# Remaining args are input files (user must provide at least one):
#
if (! @ARGV) {
  print "\n>>> $0: Please provide at least one namelist input file.\n";
  print "(Use the command \"$0 -h\" for a usage message)\n\n";
  exit;
}
#
# Save input file names in @infiles:
#
$njobs = 0;
while (@ARGV) {
  push @infiles, shift @ARGV;
  $njobs++;
} # while @ARGV
#
# Get nodename (may be needed to set #BSUB -a):
#
$nodename = `uname -n`; chop $nodename;
#
# Determine batch system type and set submit command accordingly:
#
# LSF:
#
$LoadShare  = $ENV{"LSF_ENVDIR"};
if ($LoadShare) {                            # LSF system
  $batchsys = "LSF"; 
  $submitcommand = "bsub < ";
  if (! $basename) { $basename = "LSF"; }
  if (! $queue) { $queue = "regular"; }
  print "  Batch system: Load Sharing Facility ($batchsys)\n";  
#
  if ($interactive and ! $serial) {
    print "\n>>> Sorry, I cannot execute interactive parallel jobs under LSF\n";
    exit;
  }
#
# Loadleveler:
#
} elsif (-d "/usr/lpp/LoadL") {              # LoadLeveler system
  $batchsys = "LL"; 
  $submitcommand = "llsubmit ";
  if (! $basename) { $basename = "LL"; }
  if (! $queue) { $queue = "com_rg8"; }
  print "  Batch system: LoadLeveler ($batchsys)\n";
#
# Unknown batch system -> attempt to execute interactively:
#
} else {                                     # unknown batch system
  if (! $interactive) {
    print "\n$0: cannot determine batch system:\n";
    print "I can submit to either Load Sharing Facility (LSF), or LoadLeveler (LL):\n";
    print "  Is not Load Share Facility (LSF), because env var LSF_ENVDIR is not set\n";
    print "  Is not LoadLeveler because directory /usr/lpp/LoadL does not exist.\n\n";
    print "\nWill attempt direct execution of $exec..\n\n";
  }
  $interactive = 1; # force interactive when batch system is unknown
  if (! $basename) { $basename = "multi"; }
}
if ($ntasks eq 1) { $serial = 1; }
#
# Report options to stdout:
#
print "\n$0:\n";
print "  Exec file:      $exec\n";
print "  Njobs:          $njobs\n";
print "  Input files:    @infiles\n";
print "  Submit command: \"$submitcommand\"\n";
print "  Base name:      \"$basename\"\n";
print "  Wallclock:      \"$wallclock\"\n";
print "  Queue:          \"$queue\"\n";
print "  Nnodes:         \"$nnodes\"\n";
print "  Ntasks:         \"$ntasks\"\n";
print "  Serial:         $serial\n";
print "  Nosubmit:       $nosubmit\n";
print "  Interactive:    $interactive\n";
print "  Sequential:     $sequential\n";
#
# If submitting batch under LL and sequential is requested, build single 
# LoadLeveler job script with multiple foreground executions, submit the
# job, and exit:
#
if ($batchsys eq "LL" and $sequential and ! $interactive) {
  $job_csh = mkjob_LLseq($exec,$queue,$wallclock,$nnodes,$ntasks,
                         $serial,$njobs);
  print "Made LL job script $job_csh with $njobs executions of $exec.\n";
  if (! $nosubmit) {
    $command = "$submitcommand $job_csh";
    $stat = system($command);
    if ($stat) { die ">>> Error executing \"$command\"\n"; }
    print "Submitted job script $job_csh to LoadLeveler at `date`\n";
  }
  exit;
}
#
# Loop through input files:
#
$i = 1;
print "\n";
foreach $infile (@infiles) {
#
# Make batch job script if not interactive:
#
  if (! $interactive) {
    $job_csh = mkjob($batchsys,$exec,$infile,$queue,$wallclock,$nnodes,$ntasks,
                     $serial,$interactive,$sequential,$i);
    print"Input file $i: $infile  Job script: $job_csh\n"; 
  } else {
    print"Input file $i: $infile\n";
  }
#
# Submit current job (unless nosubmit is set):
#
  if (! $nosubmit) {
#
# Submit to batch system:
    if (! $interactive) {                    # submit batch job
      $command = "$submitcommand $job_csh";
      $stat = system($command);
      if ($stat) { die ">>> Error executing \"$command\"\n"; }
#
# Execute interactively. If sequential option is set, run interactive
# jobs in foreground, otherwise run in background.
#
    } else {                                 # make interactive run
      $ii = &mknum($i);
      my $outfile = "${basename}${ii}.out";  # stdout of model run
      sleep 1; # add small time space between interactive runs
#
# Start non-sequential interactive jobs in background:
      if (! $sequential) {                   # non-sequential jobs
        $command = "csh -c \"$exec < $infile >&! $outfile &\"";
        print "Starting interactive exec of $exec in background (job $i)..\n",
        $stat = system($command);
#
# Execute sequential interactive jobs in foreground:
      } else {                               # sequential jobs
        $command = "csh -c \"$exec < $infile >&! $outfile\"";
        print "Executing $exec interactively in foreground",
              " to force sequential execution (job $i)..\n";
        $stat = system($command);
      } # sequential interactive or not
    } # interactive or not
  } # not nosubmit
#
# Increment job counter:
#
  $i++;
} # foreach $infile
exit;
#
#-------------------------------------------------------------------------
sub mkjob {
#
# Build LL or LSF batch job script for current job:
#
  my ($batchsys,$exec,$infile,$queue,$wallclock,$nnodes,$ntasks,$serial,
      $ia,$sequential,$ijob) = @_;
#
  my $jobscript = "${basename}_$ijob.job";  # job script returned by this function
  my $outfile   = "${basename}_$ijob.out";  # stdout of model run
  my $jobname   = "${basename}_$ijob";      # step or job name passed to batch sys
  $prev_job = $ijob - 1;
  $prev_jobname = "${basename}_$prev_job";
#
  open (JOBSCRIPT,"> $jobscript") || die "mkjob: cannot open file $file";
#
# - - - - - - - - - - - - Build LoadLeveler job script - - - - - - - - - -
if ($batchsys eq "LL") {               # Load Leveler
#
# Default parallel job:
#
  my $LL_node      = "# @ node             = $nnodes";
  my $LL_ntasks    = "# @ total_tasks      = $ntasks";
  my $LL_nodeusage = "# @ node_usage       = not_shared";
  my $LL_jobtype   = "# @ job_type         = parallel";
  my $LL_network   = "# @ network.MPI      = csss,shared,us";
#
# Serial job:
#
  if ($serial) {
    $LL_jobtype    = "# @ job_type         = serial";
    $LL_node       = "";
    $LL_ntasks     = "";
    $LL_nodeusage  = "";
    $LL_network    = "";
    if ($queue eq "com_rg8") { $queue = "share"; }
  }
#
print JOBSCRIPT <<EOF;
#! /bin/csh
#
# LoadLeveler commands (batch only):
#
# @ step_name        = $jobname
# @ class            = $queue
$LL_jobtype
$LL_node
$LL_ntasks
$LL_nodeusage
$LL_network
# @ wall_clock_limit = ${wallclock}:00
$LL_dependency
# @ queue
#
# Env vars (for interactive runs):
#
setenv MP_PROCS $ntasks
setenv MP_RMPOOL 1
setenv MP_PGMMODEL SPMD
setenv MP_SHARED_MEMORY YES
setenv MP_LABELIO YES
#
$exec < $infile >&! $outfile || \\
echo ">>> Execution of $exec FAILED at `date`" && echo "See output in $outfile"
EOF
#
# - - - - - - - - - - - - - - Build LSF job script - - - - - - - - - - - -
} elsif ($batchsys eq "LSF") {
#
# Default parallel job:
#
  $bsub_a = "poe";                                        # bluevista
  if ($nodename =~ "ln*") { $bsub_a = "mpich_gm"; }       # lightning
  my $LSF_jobtype   = "#BSUB -a $bsub_a";
  my $LSF_ntasks    = "#BSUB -n $ntasks";
  my $LSF_nodeusage = "#BSUB -x";
#
# Serial job if $ntasks==1:
#
  if ($serial eq 1) {
    $LSF_jobtype    = "";
    $LSF_ntasks     = "";
    $LSF_nodeusage  = "";
    $LSF_dependency = "";
  }
#
# Add dependency on previous job (#BSUB -w) if sequential runs are requested:
#
  my $LSF_dependency= "";
  if ($sequential and $ijob > 1) { $LSF_dependency= "#BSUB -w done($prev_jobname)"; }
#
print JOBSCRIPT <<EOF1;
#! /bin/csh
#
#BSUB -J $jobname         # job name
$LSF_jobtype
$LSF_ntasks
$LSF_nodeusage
$LSF_dependency
#BSUB -q $queue           # queue (e.g., premium or regular)
#BSUB -o $jobname.%J.out  # stdout filename
#BSUB -e $jobname.%J.out  # stderr filename
#BSUB -W $wallclock       # wallclock limit HH:MM
#
setenv MP_LABELIO YES
mpirun.lsf $exec < $infile >&! $outfile || \\
echo ">>> Execution (mpirun.lsf) of $exec FAILED at `date`" && echo "See output in $outfile"
EOF1

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
} else {   # unknown batch system -> main program will attempt
           # interactive "command-line" execution of $exec, so
return;    # return silently

} # batch sys
#
# Return executable csh script:
#
chmod 0755, $jobscript;
return $jobscript;
} # mkjob
#-------------------------------------------------------------------------
sub mkjob_LLseq {
#
# Make batch job script for sequential LL runs.
# This will be a single-step LL job, with multiple foreground executions.
# Note wallclock limit is 6 hours for the *total* job (if 6 hour limit is
# needed for *each* job, you must use an LSF system).
#
  my ($exec,$queue,$wallclock,$nnods,$ntasks,$serial,$njobs) = @_;
  my $jobscript = "${basename}.job";  # job script returned by this function
  my $stepname  = $jobscript;
  my $output    = "${basename}.out";        # script output
  my $LL_node      = "# @ node             = $nnodes";
  my $LL_ntasks    = "# @ total_tasks      = $ntasks";
  my $LL_nodeusage = "# @ node_usage       = not_shared";
  my $LL_jobtype   = "# @ job_type         = parallel";
  my $LL_network   = "# @ network.MPI      = csss,shared,us";
#
# Serial job if $ntasks==1:
#
  if ($serial) {
    $LL_jobtype    = "# @ job_type         = serial";
    $LL_node       = "";
    $LL_ntasks     = "";
    $LL_nodeusage  = "";
    $LL_network    = "";
    if ($queue eq "com_rg8") { $queue = "share"; }
  }
  open (JOBSCRIPT,"> $jobscript") || die "mkjob_LLseq: cannot open file $file";
  print JOBSCRIPT <<EOF;
#! /bin/csh
#
# LoadLeveler commands (batch only):
#
# @ step_name        = $stepname
# @ output           = $output
# @ error            = $output
# @ class            = $queue
# @ wall_clock_limit = ${wallclock}:00
$LL_jobtype
$LL_node
$LL_ntasks
$LL_nodeusage
$LL_network
# @ queue
#
# $exec is executed in foreground for each run to force sequential execution:
EOF
#
# Add foreground execution statements for each input file:
#
my $ijob = 1;
foreach $infile (@infiles) {
  my $outfile   = "${basename}_$ijob.out";  # stdout of model run
  print JOBSCRIPT <<EOF;
#
# Job ${ijob}:
  $exec < $infile >&! $outfile || \\
  echo ">>> Execution of $exec FAILED at `date`" && exit
  echo "Completed execution of job $ijob at `date`"
EOF
  $ijob++;
} # foreach infile
chmod 0755, $jobscript;
return $jobscript;
}
#-------------------------------------------------------------------------
#
sub usage {
  die <<EOF;

SYNOPSIS
  $0 [options] inputfile1, [inputfile2, inputfile3,...]

DESCRIPTION

  Submit multiple jobs of a given executable, using provided namelist
  input files.  The -exec option is required (you must provide an executable),
  and you must provide at least one input file argument.

  This command will submit jobs to either LoadLeveler (LL), or 
    Load Sharing Facility (LSF) batch systems:
  On LL systems, the "llsubmit" command will be used to submit each job.
  On LSF systems, the "bsub" command will be used to submit each job.

  Options that take string arguments: -exec, -basename, -wallclock, -queue
  Options that take integer arguments: -ntasks, -nnodes
  Options that take no arguments: -serial, -sequential, -interactive, -help, -nosubmit

OPTIONS       SHORT-FORM  DESCRIPTION
  -exec	        -x        Required executable file (relative or absolute path)
  -basename     -bn       Base name for script and output file names
                            (default is "LL_" or "LSF_")
  -wallclock    -wc       Wall-clock limit HH:MM (default 00:30)
  -queue        -q        Batch queue (separate defaults for LL vs LSF)
  -ntasks       -nt       Total number of MPI tasks (integer) (default=4)
                            (If ntasks==1, a serial job is assumed).
  -nnodes       -nn       Number of nodes (integer) (default=1) (LL only)
  -serial       -ser      Submit a serial job (same effect as ntasks=1)
  -sequential   -seq      Force sequential execution of each job
                            (if not set, jobs may execute simultaeneously)
  -interactive  -ia       Set to execute interactively instead of batch
                            (parallel interactive jobs not available under LSF)
  -nosubmit     -ns       Set for trial run (jobs are not actually submitted) 
  -help	        -h        Print usage message and quit.

EXAMPLES
  $0 -exec=/ptmp/foster/tiegcm/tiegcm-aix/tiegcm tiegcm1.inp tiegcm2.inp
  $0 -exec="*aix/tiegcm" -basename=tiegcm *.inp
  $0 -exec="*aix/tiegcm" -queue=premium -wallclock=00:30 -nosubmit *.inp
  $0 -x="*aix/tiegcm" -q=premium -wc=00:30 -ns *.inp
  $0 -x glbmean -serial *.inp
  $0 -x glbmean -serial -interactive *.inp
  $0 -x tiegcm -sequential *.inp

CAVEATS
  o Interactive parallel (non-serial) jobs are not available under LSF
  o Total wallclock limit for all sequential jobs under LL is 6 hours
    (under LSF, *each* job has a wallclock limit of 6 hours)
  o String arguments to options containing wildcards must be quoted.
    (e.g., -x="*aix/tiegcm")

NOTES
  o If sequential is set, each job will start only after the previous
    job has completed successfully.
  o For non-sequential LL jobs, a separate job script is made for each
    job, whereas for sequential LL jobs, a single job script is made
    with multiple foreground executions.
  o Sequential interactive jobs are executed in foreground, whereas
    non-sequential interactive jobs are executed in background.
    (job scripts are not made for interactive jobs)
EOF
}
#-------------------------------------------------------------------------
sub mknum(num) {
  my ($num) =  @_;
  my $i;
  if ($num <= 9) {
    $i = "00${num}";
  } elsif ($num <= 99) {
    $i = "0${num}";
  } else {
    $i = "${num}";
  } 
# print "mknum: num=$num -- returning i=$i\n";
  return $i; 
}