#!/opt/local/bin/perl
#
# Usage: submit [-h] [-n] [-host hostname] [-migs] 
#   [-qsub [-o output_spec]] infile
# infile is input file possibly containing INCLUDE statements 
#   (starting in col 1).
# Submit reads infile, resolves "INCLUDE filelist" statements
#   found in infile, and writes output file infile.submit.
#   (see sub usage below for details)
# User is then prompted to submit output file to several cray machines, 
#   unless -n flag is set. (-n means do not prompt for submission 
#   of job, and save output file in cwd).
# If user requests submission to one of the machines, output file is
#   submitted via MIGS (or via cqsub if -qsub option is set), otherwise 
#   the job is not submitted, and output file is saved in cwd, with 
#   chmod u+x.
#
# B. Foster 6/96
#
# 5/98: add -qsub option to submit jobs via cqsub rather than migs
#       (migs remains the default method for now). Also add -o output_spec
#        option, valid only with -qsub)
#
# 9/28/98: changed default submit method from nrnet (migs) to
#          cqsub (nqe). Using -qsub will force cqsub, or using
#          -migs will force nqe.
#
$submitfile = 1;
$submit_host = '';
$submit_method = 'cqsub'; 
$outputspec = '';
while ($ARGV[0] =~ /^-/) {
  $_ = shift;
  if (/^-n/) { 
    $submitfile = 0;
  } elsif (/^-host/) {
    $submit_host = shift;
  } elsif (/^-qsub/) {
    $submit_method = 'cqsub'; # submit via cqsub (nqe)
  } elsif (/^-migs/) {
    $submit_method = 'nrnet'; # submit via migs
  } elsif (/^-o/) {
    $outputspec = shift;
  } elsif (/^-h/) {
    &usage;
    exit 1;
  } else {
    die "\nUnrecognized switch: $_\n Valid options are: ",
        "-n (do not submit job)\n\n";
  }
}
#
if (! $ARGV[0]) {
  print "\n$0: Need input file:\n";
  &usage;
  exit 2;
}
#
# -o output_spec option is valid only under cqsub.
#
if ($outputspec) {
  if ($submit_method ne 'cqsub') {
    print "\nThe -o option (output directory) is valid only with the",
      " -qsub option\n  (i.e., with cqsub method of submission)\n";
    print "  (migs does not allow specification of the directory to ",
      "which output\n  listings are sent)\n\n";
    exit 3;
#
# output_spec may be dir or file path:
#
  } else {
    if (!(-d $outputspec)) {
      $outputdir = &root($outputspec);
      if ($outputdir && !(-d $outputdir)) {
        print "\nCannot find output directory $outputdir\n";
        print "($outputspec was specified with the -o option)\n\n";
        exit 3;
      }
    }
  }
}
$infile = $ARGV[0];
if (! -e $infile) {
  print "Cannot find file $infile\n\n";
  exit 4;
}
if (! ($tgcmroot = @ENV{"TGCMROOT"})) { die
  "\n$0: Your hao TGCMROOT environment variable is not set!",
  "\n        (TGCMROOT should be /home/tgcm) \n\n"; }
push(@INC,"$tgcmroot/bin");
&tgcmlog("submit $infile","$tgcmroot/bin/tgcmlog");
$outfile = $infile . ".submit";
require "ctime.pl";
$date = &ctime(time);
print "\n$date\n";
$cwd = `pwd`; chop $cwd;
print "Submit executing from $cwd\n";
print "Input file:  $infile\n" ;
print "Output file: $outfile\n" ;
if ($submit_host) { print "Will submit to host $submit_host\n"; }
if ($submit_method eq 'nrnet') {
  print "Will submit via MIGS (nrnet)\n";
} else {
  print "Will submit via NQE (cqsub)\n";
}
print "\n" ;
$dotinfile = "." . $infile ;
$stat = `cp $infile $dotinfile` ;
#
# Make multiple passes until INCLUDE statement is not found
# (sub include returns number of files INCLUDEd), thus resolving
# nested INCLUDEs.
#
while (&include($dotinfile,$outfile)) {
  $stat = `cp $outfile $dotinfile` ;
}
unlink($dotinfile);
print "\n" ;
#
# Now we have a complete $outfile with all files included.
# Next prompt for submission to various remote machines unless
#  -n flag was set:
#
if ($submitfile) { 
  &submitjob($submit_host); 
} else {
  $stat = `chmod u+x $outfile` &&  # normal chmod return is 0
    print "WARNING: could not chmod u+x $outfile\n";
  print "Saved output file $outfile\n\n";
}
exit 0;
#
#-------------------------------------------------------------------------
#
sub include {
#
# Search input file for ^INCLUDE statements. When found insert files
# given after the INCLUDE in output file (replace the INCLUDE statement
# with the contents of the file(s)).  If -h flag is set after the
# INCLUDE, then surround the file(s) with here-documents.
#
local ($inputfile, $outputfile) = @_;
local (@incfiles, $incfile, @incspecs, $incspec, $nincfiles);
local (*INFILE, *OUTFILE);
#
open (INFILE,"< $inputfile") || die "sub include can't open $inputfile\n";
open (OUTFILE,"> $outputfile") || die "sub include can't open $outputfile\n";
#
INLOOP:
while (<INFILE>) {			# scan input file
  if (! /^INCLUDE\s+(.*)/ ) {		# echo non-INCLUDE lines to output
    print OUTFILE;
    next INLOOP;
  }
    @incspecs = split(' ') ;		# INCLUDE + list of file specs
    shift(@incspecs) ;			# remove the INCLUDE
#
# If first file spec is "-h", this is an option meaning surround the
# file specs with here-documents, otherwise do not:
#
    $heredocs = '';		# assume no here-docs
    $incspec = $incspecs[0];	# the first arg after INCLUDE
    if ($incspec =~ /^-/) {	# is an option
      if ($incspec ne "-h") {	# -h is only flag arg for the INCLUDEs
        print "\nUnknown option to INCLUDE: $incspec\n\n";
        exit 5;
      }
      shift(@incspecs);		# remove -h option
      $heredocs = 1;		# will do here-docs
    } 
#
# Loop through list of file specs given after the INCLUDE:
#
    foreach $incspec (@incspecs) {	# loop through file specs
#
# If the file spec begins with ~ it must be expanded to include either 
# the user's home (~/) or someone else's home (~who) before going to ls:
#
      if ($incspec =~ /^#/) { next INLOOP; }	# comment
      if ($incspec =~ /^~/) {		# file spec begins with ~
        if ($incspec =~ /^~\//) { 	# ~ refers to user's home (~/)
          $home = $ENV{"HOME"} || $ENV{"LOGDIR"} || (getpwuid($<))[7] ||
            die "include: Can't find your home\n";
          $incspec =~ s/~/$home/ ;
        } else {			# ~ refers to other user (~who)
          $who = $incspec; $who =~ /^~(\w*)/; $who = $1;
          $whoshome = (getpwnam($who))[7];
          $incspec =~ s/~$who/$whoshome/ ;
        }
      }
#
# Use ls to get list of files matching the file spec:
# (Prompt for abort if no files match)
#
      @incfiles = `/bin/ls $incspec` ;	# ls returns list of matching files
      if (! @incfiles) {
        print "\nWARNING: No files match $incspec\: abort? (y/n) ";
        $answer = <STDIN>;
        if ($answer =~ /^y/) { 
          close (OUTFILE);
          $stat = `chmod u+x $outfile` &&  # normal chmod return is 0
            print "WARNING: could not chmod u+x $outfile\n";
          print "\n";
          exit 6;
        }
      }
      chop @incfiles;			# remove newlines returned by ls
#
# Loop through each file that matches the file spec:
#
INCFILES:
      foreach $incfile (@incfiles) {	# loop through matching files
#
# Validate include file (must be non-empty plain text file):
#
        if (! ($fstat = validate_file($incfile))) { next; }
#
# If -h was on the INCLUDE statement, surround the include file with 
#   here-document in output:
# Use tail of $incfile path as file name for here-doc. 
#   (if heredoc name ends with "~", chop it off)
#
        if ($heredocs) {
          $herefile = &tail($incfile);
          if ($herefile =~ /~$/) { chop $herefile; }
#
# If herefile name has been used previously, i.e., 2 paths resulted in 
#  redundant file names, prompt for abort.
#
          if (($offset = &findmember($herefile,@herefiles)) > -1) {
            print 
             "\nWARNING: Two paths result in the same file name: $file\n\n";
            print "  Path 1: $herepaths[$offset]\n";
            print "  Path 2: $incfile\n";
            print "\nIf no abort, then file from path 2 will overwrite",
              " file from path 1\n";
            print "  on the remote machine.\n";
            print "\nabort? (y/n) ";
            $answer = <STDIN>;
            if ($answer =~ /^y/) {
              close(OUTFILE);
              $stat = `chmod u+x $outfile` &&  # normal chmod return is 0
                print "WARNING: could not chmod u+x $outfile\n";
              print "\nJob NOT submitted (file saved as $outfile)\n\n";
              exit 8;
            } else { print "\n"; }
          }
          unshift(@herefiles,$herefile);
          unshift(@herepaths,$incfile);
          $eof = "'EOF$#{herefiles}'";
          print OUTFILE "cat << $eof >! $herefile\n" ;
          if (! open(INCFILE,"<$incfile")) {
            die "submit: cannot open file $incfile\n";
          }
          while (<INCFILE>) { print OUTFILE; }
          print OUTFILE "$eof\n" ;
          if ($incfile =~ /~$/) { 
            printf "Included $incfile (as $herefile)\n";
          } else {
            printf "Included -h $incfile\n";
          }
        } else {	# do not use here-docs
          if ($incfile =~ /\.[fh]$/) {
            print "\nWARNING: $incfile was not included as a here-document\n"; 
            print "Source files (ending in .f or .h) should be included\n";
            print "  as here-documents (i.e., as INCLUDE -h $incfile)\n";
            print "Abort? (y/n) ";
            $answer = <STDIN>; chop $answer;
            if ($answer =~ /^y/) {
              close(OUTFILE);
              $stat = `chmod u+x $outfile` &&  # normal chmod return is 0
                print "WARNING: could not chmod u+x $outfile\n";
              print "\nJob NOT submitted (file saved as $outfile)\n\n";
              exit 8;
            } else { print "\n"; }
          }
          if (! open(INCFILE,"<$incfile")) {
            die "submit: cannot open file $incfile\n";
          }
          while (<INCFILE>) { print OUTFILE }
          printf "Included $incfile\n";
        }		# here-doc or not
      }			# each file matching current filespec
    }			# list of file specs
} # while <INFILE>
close(OUTFILE);
close(INFILE);
$nincfiles = $#incfiles + 1;
return $nincfiles;
} # end sub include
#
#-------------------------------------------------------------------------
#
sub submitjob {
local ($submit_host) = @_;
#
# Get host names, and other info (vendor, model, ncpus, memory, ssd)
# via perl script rdhosts.
#
# It would be nice if hosts table was printed only when prompting
# is necessary (i.e., not print when -host is given on command
# line), but rdhosts must be called here to get the hosts info,
# and rdhosts must print to stdout so that rdhosts.csh will work
# (rdhosts.csh is called by shell scripts getobjsrc, get_tgcmproc, etc)
#
print "Submission is available to the following host machines:\n\n";
#print "Host		Vendor	 Model	 Ncpus	 Memory(Mw)  SSD\n";
print "Host		Vendor	 Model	 OS\n";
print "-------------------------------------------------------\n";
#
# rdhosts prints to stdout (it must, so that rdhosts.csh works)
require("$tgcmroot/dev/rdhosts");
#
print "\nFor more information, please see the following URLs:\n";
print "http://www.scd.ucar.edu/info/SCDdiagram.html\n";
print "http://www.scd.ucar.edu/docs/products/\n\n";
#
if ($submit_host) {
  $validhost = 0;
  foreach $ihost (0..$#hosts) {
    if ($submit_host eq $hosts[$ihost]) { $validhost=1; }
  }
  if ($validhost==0) {
    print "\n>>> host $submit_host is not a valid host name.\n";
    print "    job NOT submitted.\n\n";
    exit 1;
  }
}
#
foreach $ihost (0..$#hosts) {	# loop over host machines
  if ($submit_host) {
    $answer = 'n';
    if ($submit_host eq $hosts[$ihost]) { $answer = 'y'; }
  } else {
    print "Submit to $hosts[$ihost]\? (y/n/q): ";
    $answer = <STDIN>;
    if ($answer =~ /^n/) { next }	# prompt for next machine
  }
#
# Submit from directory ./submit_dir. If this dir already exists,
#   remove its contents before copying outfile and submitting, otherwise
#   create the dir. (assume that ./submit_dir does not exist as a 
#   non-directory file).
# Do chmod u+x on script (not necessary for batch, but convenient for
#   interactive).
#
  if ($answer =~ /^y/) { 	# submit the job
#
# Migs verb consists of the 1st 2 characters of the host name
# plus "qsub":
#
    @chars = split //,$hosts[$ihost];
    $verb = $chars[0] . $chars[1] . qsub;
#
# Make submitdir:
#    
    $submitdir = ".submit_dir";	# dir from which job is submitted
    if (-d $submitdir) { 
      @contents = `/bin/ls $submitdir` ; chop @contents;
      if (@contents) {
        chdir($submitdir) || die "Could not chdir to $submitdir\n";
        unlink @contents ;
        chdir("../") || die "Could not chdir to ../\n";
      }
    } else {
      mkdir($submitdir,00700) || die "Cannot make dir $submitdir\n";
    }
    rename($outfile,"$submitdir/$outfile") || 
      die "Cannot rename $outfile to $submitdir/$outfile\n";
    $stat = `chmod u+x $submitdir/$outfile` &&  # normal chmod return is 0
      print "WARNING: could not chmod u+x $submitdir/$outfile\n";
    print "\nSubmitting to $hosts[$ihost]\:\n" ;
#
# If submitting to LoadLeveler (IBM SP batch manager), must
# rcp the file to the host and execute as a csh script 
# (the csh script will submit via llsubmit)
#
    if ($hosts[$ihost] eq 'blackforest' || $hosts[$ihost] eq 'babyblue') {
      print "Remote copying $submitdir/$outfile to $hosts[$ihost]...\n";
      $err = system("rcp $submitdir/$outfile $hosts[$ihost]:$outfile");
      if ($err == 0) {
        print "Rcp succeeded.\n"; 
      } else {
        print "Error from rcp -- job NOT submitted.\n";
        exit;
      }
      $err = system("rsh $hosts[$ihost] $outfile");
      if ($err == 0) {
        print "Rsh succeeded. Job was submitted.\n\n"; 
      } else {
        print "Error from rsh -- job NOT submitted.\n\n";
      }  
      exit;  
    }
#
# The ftpf migs keyword value tells migs to remove file after received. 
# (directory $submitdir remains)
#
    if ($submit_method eq 'nrnet') {
      $command = 
        "nrnet $verb $submitdir/$outfile l ftpf=unix-rcp-put-binrm";
      print "$command\n";
#
# Submit via cqsub (nqe) rather than migs:
#
    } elsif ($submit_method eq 'cqsub') {
#
# Env vars that should be set for nqe (cqsub method of submission):
#   setenv NQE_HOME /opt/craysoft/nqe/3.2.0.2
#   set path = ($path $NQE_HOME/bin)
#   setenv NQE_DEST_TYPE nqs
#   setenv NQS_SERVER ouray.ucar.edu [$hosts[$ihost] + '.ucar.edu']
#
      $ENV{'NQE_HOME'} = "/opt/craysoft/nqe/3.2.0.2";
      $nqe_home = "/opt/craysoft/nqe/3.2.0.2";
      $ENV{'NQE_DEST_TYPE'} = "nqs";
      $host = $hosts[$ihost];
      if (index($host,'.ucar.edu') == $[ -1) {
        $host = $host . '.ucar.edu';
      }
      $ENV{'NQS_SERVER'} = $host;
#     print "\nSet env var NQS_SERVER = $host\n";
#
# As of 5/98, stupid cqsub does not allow submitting file with 
# names > 15 characters (they are supposed to fix this later by
# truncating, but in the meantime, will truncate here if necessary):
#
      $len = length($outfile);
      if ($len > 15) {
        $new_outfile = substr($outfile,0,15);
        $stat = rename("$submitdir/$outfile","$submitdir/$new_outfile");
        if ($stat != 1) {
          print "\n>>> Error renaming $submitdir/$outfile to ",
                "$submitdir/$new_outfile\n";
          print "stat=$stat Error code: $!\n\n";
          exit;
        }
        print "Renamed $submitdir/$outfile to $submitdir/$new_outfile\n";
        print "  (cqsub file name must be <= 15 characters)\n\n";
        $outfile = $new_outfile;
      }
#
# For now, have cqsub return output to ~/ntwk, like migs. If ntwk does
# not exist, let cqsub return output to the cwd:
#
      if (!($outputspec)) {
        $home = $ENV{"HOME"} || $ENV{"LOGDIR"} || (getpwuid($<))[7];
        if (-d "$home/ntwk") { $outputspec = "$home/ntwk"; }
      }
      if ($outputspec) {
          $command = 
            "$nqe_home/bin/cqsub -mb -me -eo -o $outputspec $submitdir/$outfile";
          print "$command\n";
          print "\nOutput listing for this job will be returned to",
                " $outputspec\n\n"; 
      } else {
        $command = 
          "$nqe_home/bin/cqsub -mb -me -eo $submitdir/$outfile";
        print "$command\n";
        print "\nOutput listing for this job will be returned to",
              " the cwd: $cwd\n\n";
      }
#
# Submit method should be "nrnet" or "qsub" (default is cqsub),
# but user can override with -migs option:
#
    } else {
      print "\nUnknown submission method submit_method = $submit_method\n",
     "Submit method must be nrnet for MIGS, or qsub for NQE (default)).\n",
            "Job NOT submitted.\n\n";
      exit;
    }
# 
# exec executes $command and exits if successful.
# If exec fails, die with warning msg.
#
    exec($command) || 
      die "\n>>> WARNING: error return from the command: $command",
        " -- job NOT submitted <<<\n\n";
  }
  if ($answer =~ /^q/) { 		# exit, but save $outfile
    $stat = `chmod u+x $outfile` &&	# normal chmod return is 0
      print "WARNING: could not chmod u+x $submitdir/$outfile\n";
    print "\nJob NOT submitted (saved file $outfile)\n\n";
    exit 0
  }
}	# foreach host
print "\nJob NOT submitted (saved file $outfile)\n\n";
}	# end sub submitjob
#
#-------------------------------------------------------------------------
#
sub usage {
  print "\n","-" x 72,"\n";
  print "Usage: submit [-h] [-n] [-migs] [-qsub [-o output_spec]]\n"; 
  print "       [-host hostname] script\n\n";
  print "Where: script is original script to submit to a remote machine,\n";
  print "  optionally containing INCLUDE statements (see below).\n\n";
  print "Options:\n";
  print "  -h will print this usage message and exit.\n";
  print "  -n means do not prompt for sumbmission to a host machine.\n";
  print "  -qsub means submit via cqsub (nqe) method (default).\n";
  print "  -migs means submit via nrnet (migs) method.\n";
  print "  -o output_spec means send output listing to output_spec.\n";
  print "     (Note: -o option is valid only with -qsub)\n";
  print "  -host hostname means submit to machine hostname without\n";
  print "    prompting for each host.\n";
  print "\nAn output file named script.submit is created, to which the\n";
  print "  original script is echoed, and in which INCLUDE statements\n";
  print "  are resolved (see below).  The user is then prompted for\n"; 
  print "  submission of the output file to a list of remote machines.\n";
  print "  If the user declines to submit, the output file is saved in\n";
  print "  the working directory, otherwise it is submitted via MIGS from\n";
  print "  the directory ./submit_dir (the original script is always unchanged).\n\n";
  print "INCLUDE statements (in original script):\n\n";
  print "  Must begin in column 1.\n"; 
  print "  (INCLUDE keywords that do not begin in column 1 are ignored)\n\n";
  print "  If an INCLUDE keyword is followed by a list of file specifications,\n";
  print "    then in the output file, the INCLUDE statement will be\n";
  print "    replaced with the contents of each of the files.\n\n"; 
  print "  If an INCLUDE keyword is immediately followed by the -h option\n";
  print "    then the files listed will each be surrounded by 'here-documents'\n";
  print "    in the output file (e.g., cat << 'EOF' >! filename, resulting in\n";
  print "    a separate file called filename on the remote disk).\n\n"; 
  print "  The file list following INCLUDE statements may contain\n";
  print "    wild cards (*), tilde's (~), or relative or full paths.\n\n";
  print "  Comments may begin with '#' anywhere after the INCLUDE keyword.\n\n";
  print "Examples of INCLUDE statements in original script:\n\n";
  print "INCLUDE -h myfile.f\n";
  print "INCLUDE -h ~/dir/myfile.f\n";
  print "INCLUDE -h ~user/dir/theirfile\n";
  print "INCLUDE -h ../dir/anotherfile\n";
  print "INCLUDE -h *.f *.h\n";
  print "INCLUDE -h Makefile timegcm.inp\n";
  print "INCLUDE ~/bin/script.csh  # this is a comment\n";
  print 'INCLUDE $TGCMROOT/bin/tgcm.run'; print "\n";
  print '        ($TGCMROOT is interpreted by the *local* shell)'; 
  print "\n\n";
  print "Submission via MIGS is available to the following hosts:\n\n";
 #print "Host		Vendor	 Model	 Ncpus	 Memory(Mw)  SSD\n";
  print "Host		Vendor	 Model	 OS\n";
  print "-------------------------------------------------------\n";
  push(@INC,"/home/tgcm/bin");
  require("$tgcmroot/dev/rdhosts");
  print "-------------------------------------------------------\n\n";
}
#
#-------------------------------------------------------------------------
#
sub gettime {
  local($file) = @_;
#
# Return create time given in line 1 of $file, e.g., if line1 = 
# c CreateTime: 836886057 Mon Jul  8 22:20:57 US/Mountain 1996
# then return 836886057 (secs since 1/1/70)
#
  open(FILE,"< $file") || die "gettime: can't open $file\n";
  $createtime = <FILE>; chop $createtime;
  $createtime =~ /CreateTime:\s(\d*)/; $createtime = $1;
}
#
#-------------------------------------------------------------------------
#
sub tail {
#
# Given full path, return file name (tail of path)
#
  local ($file) = @_;
  $file =~ /([^\/]*)$/;
  $file = $1;
}
#
#-------------------------------------------------------------------------
#
sub validate_file {
#
# Make sure $file is suitable to be included in output file:
#
  local ($file) = @_;
  local ($tail);
#
  if (! -e $file) {
    print "Warning: could not find $file -- not included\n";
    return 0;
  }
  if (-z $file) {
    print "Warning: $file is empty -- not included\n";
    return 0;
  }
  if (! -f $file || ! -T $file) {
    print "Warning: $file is not a plain text file -- not included\n";
    return 0;
  }
  if (! -r $file) {
    print "Warning: $file does not have read permission -- not included\n";
    return 0;
  }
#
# Do not include files that begin with "," -- these are rand editor backups:
#
  $tail = &tail($file);
  if ($tail =~ /^,/) {
    print "Note: NOT including rand editor backup file $file\n";
    return 0;
  }
  return 1;
}  
#
#-------------------------------------------------------------------------
#
sub findmember {
#
# Given a string $member, and a list @list, search for $member in @list
# and if found return index of $member in @list, otherwise return -1:
#
  local ($member,@list) = @_;
  local ($offset);
  $offset = 0;
  foreach $ele (@list) {
    if ($ele eq $member) {
      return $offset
    }
    $offset++;
  }
  $offset = -1;
  return $offset;
}
#
#-------------------------------------------------------------------------
#
sub tgcmlog {
#
# Write entry to tgcmlog file with command, user, host, and date:
# (this called only if $TGCMROOT is set)
#
  local($command,$logfile) = @_;
  local(*LOGFILE,$login,$date);
  require "ctime.pl";
  ($wday, $mo, $day, $time, $zone, $yr) = split(' ',&ctime(time));
  $date = $wday . ' ' . $mo . ' ' . $day . ' ' . $time . ' ' . $yr . ' ';
  $login = getlogin || (getpwuid($<))[0] || "Kilroy";
  $host = $ENV{"HOST"};
  if (! open(LOGFILE,">> $logfile")) {
    print "WARNING tgcmlog: could not open $logfile\n";
    return;
  }
  print LOGFILE "$command $login $host $date\n";
  close(LOGFILE);
}
#-------------------------------------------------------------------------
sub root {
#
# Given full path, return root dir name (root of path):
#
  local($path) = @_;
  local($root,@parts);
  @parts = split(/\//,$path);
  foreach $i (1..$#parts-1) { $root = ($root . "/") . $parts[$i]; }
  return $root;
}
#-------------------------------------------------------------------------