#!/opt/local/bin/perl
#
# Usage: submit [-h] [-n] infile
# infile is input file possibly containing INCLUDE statements 
#   (starting in col 1).
# Submit reads infile, resolves "INCLUDE filelist" statements
#   found in infile, and writes output file infile.submit.
#   (see sub usage below for details)
# User is then prompted to submit output file to several cray machines, 
#   unless -n flag is set. (-n means do not prompt for submission 
#   of job, and save output file in cwd).
# If user requests submission to one of the machines, output file is
#   submitted via MIGS, otherwise the job is not submitted, and output file
#   is saved in cwd, with chmod u+x.
#
# B. Foster 6/96
#
$submitfile = 1;
while ($ARGV[0] =~ /^-/) {
  $_ = shift;
  if (/^-n/) { 
    $submitfile = 0;
  } elsif (/^-h/) {
    &usage;
    exit 1;
  } else {
    die "\nUnrecognized switch: $_\n Valid options are: -n (do not submit job)\n\n";
  }
}
#
if (! $ARGV[0]) {
  print "\n$0: Need input file:\n";
  &usage;
  exit 2;
}
$infile = $ARGV[0];
if (! -e $infile) {
  print "Cannot find file $infile\n\n";
  exit 3;
}
$tgcmroot = @ENV{"TGCMROOT"};
if ($tgcmroot) { &tgcmlog("submit","$tgcmroot/bin/tgcmlog"); }
$outfile = $infile . ".submit";
require "ctime.pl";
$date = &ctime(time);
print "\n$date\n";
print "Input file:  $infile\n" ;
print "Output file: $outfile\n" ;
print "\n" ;
$dotinfile = "." . $infile ;
$stat = `cp $infile $dotinfile` ;
#
# Make multiple passes until INCLUDE statement is not found
# (sub include returns number of files INCLUDEd), thus resolving
# nested INCLUDEs.
#
while (&include($dotinfile,$outfile)) {
  $stat = `cp $outfile $dotinfile` ;
}
unlink($dotinfile);
print "\n" ;
#
# Now we have a complete $outfile with all files included.
# Next prompt for submission to various remote machines unless
#  -n flag was set:
#
if ($submitfile) { 
  &submitjob(); 
} else {
  $stat = `chmod u+x $outfile` &&  # normal chmod return is 0
    print "WARNING: could not chmod u+x $outfile\n";
  print "Saved output file $outfile\n\n";
}
exit 0;
#
#-------------------------------------------------------------------------
#
sub include {
#
# Search input file for ^INCLUDE statements. When found insert files
# given after the INCLUDE in output file (replace the INCLUDE statement
# with the contents of the file(s)).  If -h flag is set after the
# INCLUDE, then surround the file(s) with here-documents.
#
local ($inputfile, $outputfile) = @_;
local (@incfiles, $incfile, @incspecs, $incspec, $nincfiles);
local (*INFILE, *OUTFILE);
#
open (INFILE,"< $inputfile") || die "sub include can't open $inputfile\n";
open (OUTFILE,"> $outputfile") || die "sub include can't open $outputfile\n";
#
INLOOP:
while (<INFILE>) {			# scan input file
  if (! /^INCLUDE\s+(.*)/ ) {		# echo non-INCLUDE lines to output
    print OUTFILE;
    next INLOOP;
  }
    @incspecs = split(' ') ;		# INCLUDE + list of file specs
    shift(@incspecs) ;			# remove the INCLUDE
#
# If first file spec is "-h", this is an option meaning surround the
# file specs with here-documents, otherwise do not:
#
    $heredocs = '';		# assume no here-docs
    $incspec = $incspecs[0];	# the first arg after INCLUDE
    if ($incspec =~ /^-/) {	# is an option
      if ($incspec ne "-h") {	# -h is only flag arg for the INCLUDEs
        print "\nUnknown option to INCLUDE: $incspec\n\n";
        exit 5;
      }
      shift(@incspecs);		# remove -h option
      $heredocs = 1;		# will do here-docs
    } 
#
# Loop through list of file specs given after the INCLUDE:
#
    foreach $incspec (@incspecs) {	# loop through file specs
#
# If the file spec begins with ~ it must be expanded to include either 
# the user's home (~/) or someone else's home (~who) before going to ls:
#
      if ($incspec =~ /^#/) { next INLOOP; }	# comment
      if ($incspec =~ /^~/) {		# file spec begins with ~
        if ($incspec =~ /^~\//) { 	# ~ refers to user's home (~/)
          $home = $ENV{"HOME"} || $ENV{"LOGDIR"} || (getpwuid($<))[7] ||
            die "include: Can't find your home\n";
          $incspec =~ s/~/$home/ ;
        } else {			# ~ refers to other user (~who)
          $who = $incspec; $who =~ /^~(\w*)/; $who = $1;
          $whoshome = (getpwnam($who))[7];
          $incspec =~ s/~$who/$whoshome/ ;
        }
      }
#
# Use ls to get list of files matching the file spec:
# (Prompt for abort if no files match)
#
      @incfiles = `/bin/ls $incspec` ;	# ls returns list of matching files
      if (! @incfiles) {
        print "\nWARNING: No files match $incspec\: abort? (y/n) ";
        $answer = <STDIN>;
        if ($answer =~ /^y/) { 
          close (OUTFILE);
          $stat = `chmod u+x $outfile` &&  # normal chmod return is 0
            print "WARNING: could not chmod u+x $outfile\n";
          print "\n";
          exit 6;
        }
      }
      chop @incfiles;			# remove newlines returned by ls
#
# Loop through each file that matches the file spec:
#
INCFILES:
      foreach $incfile (@incfiles) {	# loop through matching files
#
# Validate include file (must be non-empty plain text file):
#
        if (! ($fstat = validate_file($incfile))) { next; }
#
# If -h was on the INCLUDE statement, surround the include file with 
#   here-document in output:
# Use tail of $incfile path as file name for here-doc. 
#   (if heredoc name ends with "~", chop it off)
#
        if ($heredocs) {
          $herefile = &tail($incfile);
          if ($herefile =~ /~$/) { chop $herefile; }
#
# If herefile name has been used previously, i.e., 2 paths resulted in 
#  redundant file names, prompt for abort.
#
          if (($offset = &findmember($herefile,@herefiles)) > -1) {
            print 
             "\nWARNING: Two paths resulted in the same file name: $file\n";
            print "  Paths resulting in the redundant file name are:\n";
            print "  $herepaths[$offset] and $incfile\n";
            print "  (File from 2nd path would overwrite file from 1st path\n";
            print "abort? (y/n) ";
            $answer = <STDIN>;
            if ($answer =~ /^y/) {
              close(OUTFILE);
              $stat = `chmod u+x $outfile` &&  # normal chmod return is 0
                print "WARNING: could not chmod u+x $outfile\n";
              print "\nJob NOT submitted (file saved as $outfile)\n\n";
              exit 8;
            } else { print "\n"; }
          }
          unshift(@herefiles,$herefile);
          unshift(@herepaths,$incfile);
          $eof = "'EOF$#{herefiles}'";
          print OUTFILE "cat << $eof >! $herefile\n" ;
          if (! open(INCFILE,"<$incfile")) {
            die "submit: cannot open file $incfile\n";
          }
          while (<INCFILE>) { print OUTFILE; }
          print OUTFILE "$eof\n" ;
          if ($incfile =~ /~$/) { 
            printf "Included $incfile (as $herefile)\n";
          } else {
            printf "Included -h $incfile\n";
          }
        } else {	# do not use here-docs
          if ($incfile =~ /.[fh]$/) {
            print "\nWARNING: $incfile was not included as a here-document\n"; 
            print "Source files (ending in .f or .h) should be included\n";
            print "  as here-documents (i.e., as INCLUDE -h $incfile)\n";
            print "Abort? (y/n) ";
            $answer = <STDIN>; chop $answer;
            if ($answer =~ /^y/) {
              close(OUTFILE);
              $stat = `chmod u+x $outfile` &&  # normal chmod return is 0
                print "WARNING: could not chmod u+x $outfile\n";
              print "\nJob NOT submitted (file saved as $outfile)\n\n";
              exit 8;
            } else { print "\n"; }
          }
          if (! open(INCFILE,"<$incfile")) {
            die "submit: cannot open file $incfile\n";
          }
          while (<INCFILE>) { print OUTFILE }
          printf "Included $incfile\n";
        }		# here-doc or not
      }			# each file matching current filespec
    }			# list of file specs
} # while <INFILE>
close(OUTFILE);
close(INFILE);
$nincfiles = $#incfiles + 1;
return $nincfiles;
} # end sub include
#
#-------------------------------------------------------------------------
#
sub submitjob {
#
# hosts = list of machine names to which output may be submitted.
# verbs = MIGS verbs corresponding to each machine.
# (note perl associative array not used to preserve order of submit options)
#
@hosts = ('antero','shavano','paiute','aztec');
@verbs = ('anjob' ,'shjob',  'pajob' ,'azjob');
#
foreach $i (0..$#hosts) {	# loop over host machines
  print "Submit to $hosts[$i]\? (y/n/q): ";
  $answer = <STDIN>;
  if ($answer =~ /^n/) { next }	# prompt for next machine
#
# Submit from directory ./submit_dir. If this dir already exists,
#   remove its contents before copying outfile and submitting, otherwise
#   create the dir. (assume that ./submit_dir does not exist as a 
#   non-directory file).
# Do chmod u+x on script (not necessary for batch, but convenient for
#   interactive).
#
  if ($answer =~ /^y/) { 	# submit the job
#
# Paiute has only reg queue:
#
    if ($hosts[$i] eq "paiute") {
      &paiute_queue($infile);
    } 
#
# Make submitdir:
#    
    $submitdir = ".submit_dir";	# dir from which job is submitted
    if (-d $submitdir) { 
      @contents = `/bin/ls $submitdir` ; chop @contents;
      if (@contents) {
        chdir($submitdir) || die "Could not chdir to $submitdir\n";
        unlink @contents ;
        chdir("../") || die "Could not chdir to ../\n";
      }
    } else {
      mkdir($submitdir,00700) || die "Cannot make dir $submitdir\n";
    }
    rename($outfile,"$submitdir/$outfile") || 
      die "Cannot rename $outfile to $submitdir/$outfile\n";
    $stat = `chmod u+x $submitdir/$outfile` &&  # normal chmod return is 0
      print "WARNING: could not chmod u+x $submitdir/$outfile\n";
    print "\nSubmitting to $hosts[$i]\:\n" ;
#
# The ftpf migs keyword value tells migs to remove file after received. 
# (directory $submitdir remains)
# 
# 10/1/96: Temporarilly use nrnet.elb, which is modified nrnet script
#          to test new MIGS system which uses elbert.ucar.edu.
#
    $command = 
      "nrnet.elb $verbs[$i] $submitdir/$outfile l ftpf=unix-rcp-put-binrm";
#     "nrnet $verbs[$i] $submitdir/$outfile l ftpf=unix-rcp-put-binrm";
    print "$command\n";
    exec($command);			# exec executes $command and exits
  }
  if ($answer =~ /^q/) { 		# exit, but save $outfile
    $stat = `chmod u+x $outfile` &&	# normal chmod return is 0
      print "WARNING: could not chmod u+x $submitdir/$outfile\n";
    print "\nJob NOT submitted (saved file $outfile)\n\n";
    exit 0
  }
}
print "\nJob NOT submitted (saved file $outfile)\n\n";
}	# end sub submitjob
#
#-------------------------------------------------------------------------
#
sub usage {
  print "\n---------------------------------------------------------------\n";
  print "Usage: submit [-h] [-n] script\n\n";
  print "Where: script is original script to submit to a remote machine,\n";
  print "  optionally containing INCLUDE statements (see below).\n\n";
  print "Options:\n";
  print "  -h will print this usage message and exit.\n";
  print "  -n means do not prompt for sumbmission to a host machine.\n";
  print "\nAn output file named script.submit is created, to which the\n";
  print "  original script is echoed, and in which INCLUDE statements\n";
  print "  are resolved (see below).  The user is then prompted for\n"; 
  print "  submission of the output file to a list of remote machines.\n";
  print "  If the user declines to submit, the output file is saved in\n";
  print "  the working directory, otherwise it is submitted via MIGS from\n";
  print "  the directory ./submit_dir (the original script is always unchanged).\n\n";
  print "INCLUDE statements (in original script):\n\n";
  print "  Must begin in column 1.\n"; 
  print "  (INCLUDE keywords that do not begin in column 1 are ignored)\n\n";
  print "  If an INCLUDE keyword is followed by a list of file specifications,\n";
  print "    then in the output file, the INCLUDE statement will be\n";
  print "    replaced with the contents of each of the files.\n\n"; 
  print "  If an INCLUDE keyword is immediately followed by the -h option\n";
  print "    then the files listed will each be surrounded by 'here-documents'\n";
  print "    in the output file (e.g., cat << 'EOF' >! filename, resulting in\n";
  print "    a separate file called filename on the remote disk).\n\n"; 
  print "  The file list following INCLUDE statements may contain\n";
  print "    wild cards (*), tilde's (~), or relative or full paths.\n\n";
  print "  Comments may begin with '#' anywhere after the INCLUDE keyword.\n\n";
  print "Examples of INCLUDE statements in original script:\n\n";
  print "INCLUDE -h myfile.f\n";
  print "INCLUDE -h ~/dir/myfile.f\n";
  print "INCLUDE -h ~user/dir/theirfile\n";
  print "INCLUDE -h *.f *.h\n";
  print "INCLUDE -h Makefile timegcm.inp\n";
  print "INCLUDE ~/bin/script.csh  # this is a comment\n";
  print "INCLUDE ../timegcm.run\n\n";
  print "\n---------------------------------------------------------------\n";
}
#
#-------------------------------------------------------------------------
#
sub paiute_queue {
#
# Job is begin subitted to paiute -- check that reg queue is being
# used (QSUB -q reg), since reg is the only queue on paiute.
# Prompt for abort if reg queue not being used.
#
  local ($infile) = @_[0];
  local (*INFILE);
  open (INFILE,"< $infile") || 
    die "sub paiute_queue can't open $infile\n";
  while (<INFILE>) {
    if (/QSUB\s*-q\s*(\w*)\s(.*)/) {
      $queue = $1;
      if ($queue ne "reg") {
        print "\nWARNING: paiute has only reg queue, but $infile has the following line:\n\n";
        print;
        print "\nabort? (y/n) ";
        $answer = <STDIN>;
        if ($answer =~ /^y/) {
          close(OUTFILE);
          $stat = `chmod u+x $outfile` &&  # normal chmod return is 0
            print "WARNING: could not chmod u+x $outfile\n";
          print "\nJob NOT submitted (file saved as $outfile)\n\n";
          exit 9;
        }
      }
      last;
    }
  }
  close(INFILE);
}
#
#-------------------------------------------------------------------------
#
sub gettime {
  local($file) = @_;
#
# Return create time given in line 1 of $file, e.g., if line1 = 
# c CreateTime: 836886057 Mon Jul  8 22:20:57 US/Mountain 1996
# then return 836886057 (secs since 1/1/70)
#
  open(FILE,"< $file") || die "gettime: can't open $file\n";
  $createtime = <FILE>; chop $createtime;
  $createtime =~ /CreateTime:\s(\d*)/; $createtime = $1;
}
#
#-------------------------------------------------------------------------
#
sub tail {
#
# Given full path, return file name (tail of path)
#
  local ($file) = @_;
  $file =~ /([^\/]*)$/;
  $file = $1;
}
#
#-------------------------------------------------------------------------
#
sub validate_file {
#
# Make sure $file is suitable to be included in output file:
#
  local ($file) = @_;
  local ($tail);
#
  if (! -e $file) {
    print "Warning: could not find $file -- not included\n";
    return 0;
  }
  if (-z $file) {
    print "Warning: $file is empty -- not included\n";
    return 0;
  }
  if (! -f $file || ! -T $file) {
    print "Warning: $file is not a plain text file -- not included\n";
    return 0;
  }
  if (! -r $file) {
    print "Warning: $file does not have read permission -- not included\n";
    return 0;
  }
#
# Do not include files that begin with "," -- these are rand editor backups:
#
  $tail = &tail($file);
  if ($tail =~ /^,/) {
    print "Note: NOT including rand editor backup file $file\n";
    return 0;
  }
  return 1;
}  
#
#-------------------------------------------------------------------------
#
sub findmember {
#
# Given a string $member, and a list @list, search for $member in @list
# and if found return index of $member in @list, otherwise return -1:
#
  local ($member,@list) = @_;
  local ($offset);
  $offset = 0;
  foreach $ele (@list) {
    if ($ele eq $member) {
      return $offset
    }
    $offset++;
  }
  $offset = -1;
  return $offset;
}
#
#-------------------------------------------------------------------------
#
sub tgcmlog {
#
# Write entry to tgcmlog file with command, user, host, and date:
# (this called only if $TGCMROOT is set)
#
  local($command,$logfile) = @_;
  local(*LOGFILE,$login,$date);
  require "ctime.pl";
  ($wday, $mo, $day, $time, $zone, $yr) = split(' ',&ctime(time));
  $date = $wday . ' ' . $mo . ' ' . $day . ' ' . $time . ' ' . $yr . ' ';
  $login = getlogin || (getpwuid($<))[0] || "Kilroy";
  $host = $ENV{"HOST"};
  if (! open(LOGFILE,">> $logfile")) {
    print "WARNING tgcmlog: could not open $logfile\n";
    return;
  }
  print LOGFILE "$command $login $host $date\n";
  close(LOGFILE);
}