#!/usr/bin/perl

if (scalar(@ARGV) == 0 || $ARGV[0] eq "-h" || $ARGV[0] eq "--help") {
        print "Normal usage is :\n";
        print "run_qchem jobname [num_cpu save mpi]\n";
        print "jobname should be your input .inp or .in file name.\n";
        print "\n";
        print "The default num_cpu (number of cpus) is 1. For parallel jobs in shared memory,\n";
        print "Sensible values are: 1,2,8,16,32,64. \n";
        print "\n";
        print "Use \"save\" to save scratch and/or plot files in a folder called jobname-saved\n"; 
	print "at the end of the calculation. Additional files will be in a folder named for\n";
	print "the jobnumber. Otherwise they are deleted.\n";
        print "In the manual, this is the same as: qchem -save infile outfile savename\n";
        print "http://www.q-chem.com/qchem-website/manual/qchem50_manual/index.html\n";
        print "\n";
        print "QChem has 2 parallel modes: OpenMP (shared memory) or MPI (non-shared memory).\n"; 
        print "For MPI in non-shared memory (over more than one node) use the \"mpi\" option.\n";
        print "Please read the manual link above. Shared memory is required for ADC, CCMAN,\n";
        print "CCMAN2 and Integral Transformation. CIS and TDDFT are not parallel. \n";
	print "You can only use up to 64 core in shared memory. If you ask for more,  \n";
	print "this script will force your job to use MPI which might not be compatible\n";
	print "with the desired job type. \n";
        print "\n";
        print "The order of [num_cpu save mpi] does not matter but must follow  \n";
        print "jobname if they are used.\n";
        print "\n";
        print "Examples:\n";
        print "run_qchem jobname 4\n";
        print "Uses 4 core with all subjobs on the same node in shared memory.\n";
        print "\n";
        print "run_qchem jobname 6 save mpi\n";
        print "Uses 6 core in mpi mode. Saves scratch/restart/plot data.\n";
        print "\n";

        exit;
        }

 $directory = `/bin/pwd`;
 chomp $directory;

	$num_cpu = 1;
        $pe = "";
	$mpi = "shared";
        $thread = "-nt";
        $save = "";
        $time= "";
        $qc = "/usr/software/qchem_500";

        $input = shift(@ARGV);
        $input =~ s/.in$|.inp$//;
        $ext = $&;
	`/usr/bin/dos2unix -q -n $input$ext $input$ext`;
        $jobname = $input . ".job";
        $nodes = $input . ".nodes";

        if ($ext eq "" ) {
                if (-e "$input" . "." . "inp")  {
                        $ext = ".inp";
                }
                elsif (-e "$input" . "." . "in") {
                        $ext = ".in";
                }
                else {
                        print "your input file does not exist", "\n";
                        exit;
                }
        }

###check for ccsd
        $ccsdcheck = 0 ;
        $ccsdcheck = `/bin/grep -c -i "ccsd" $input$ext` ;
        #print $ccsdcheck, "\n";
        chomp $ccsdcheck;

        if ( $ccsdcheck ne 0 ) {
        #print $ccsdcheck, "\n";
print "\n";
        print "This appears to be a CCSD job. You MUST use a cc_memory keyword.\n";
        print "CCSD caclulations overuse memory, often crash nodes and may crash the server.\n";
        print "Most CCSD jobs are hugely disk and memory intensive and should probably use\n";
        print "64 core in shared memory. \n";
        print "mem_static 2000 and cc_memory 30000 are reasonable for a 64 core job.\n";
        print "run_qchem jobname.in 64\n";
        print "is an example command. Do not try more than a few at a time. \n";
        print "Everyone who tries these crashes nodes so please talk to Kathy. \n";
        print "\n";
        print "mem_static 1000 and cc_memory 2500 are reasonable for a single core job \n";
        print "but single core jobs are not advised. Failure to use cc_memory and allocate \n";
	print "adequate core can crash a node, other people's jobs, and may crash tiger.\n";
        print "If your job does not have cc_memory and adequate core, use qdel to stop your job \n";
        print "and then change the settings.\n";
	print "\n";
        }

####end check for ccsd

        foreach $argnum (0 .. $#ARGV) {
		$_ = shift(@ARGV);
                if ( $_ =~ /\d/ ) {
                         $num_cpu = $_;
				if ( $_ >= 128 ) {
                         	$num_cpu = 128;
		print "Your num_cpu request was more than 128 core! This has been replaced by 128.\n";
		print "Talk to Kathy if you really need more.\n";
                	}
		}
	
		if ( $_ =~ /save/ ) {
                         $save = "-save";
                	}
                if ( $_ =~ /restart/ ) {
                         $save = "-save";
                	}
                if ( $_ =~ /mpi/ || $num_cpu > 64  ) {
                         $mpi = "mpi-sm";
                #         print "$mpi \n";
                         $thread = "-np";
                         $qc = "/usr/software/qchem_500_mpich";
			}
        	}

		if ( $num_cpu ne 1 ) {
			$pe = "#\$ -pe $mpi $num_cpu";
		}	


local *FILE;
open (FILE, ">$jobname") || die "Can't open create file named $jobname"; 
print FILE "#\!/bin/tcsh -f","\n";
print FILE "#\$ -cwd","\n";
print FILE "#\$ -o $jobname.o\$JOB_ID","\n";
print FILE "#\$ -j y","\n";
print FILE "#\$ -N q-$input","\n";
print FILE "$pe","\n";
print FILE "$time","\n";
print FILE "\n";
print FILE "setenv QC $qc","\n";
print FILE "source $qc/qcenv.csh","\n";
print FILE "\n";
print FILE "cd \$SGE_O_WORKDIR","\n";
print FILE "/bin/hostname","\n";
print FILE "\n";
print FILE "set echo","\n";
print FILE "setenv PATH /usr/software/nbo6_g16/bin:\${PATH}","\n";
print FILE "setenv NBOEXE /usr/software/nbo6_g16/bin/nbo6.i8.exe","\n";

       if ( $num_cpu eq 1 ) {
#               print FILE "setenv QCMACHINEFILE $\HOST","\n";
       } else {
               print FILE "awk '{ print \$1 }' \$PE_HOSTFILE > $nodes","\n";
               print FILE "setenv QCMACHINEFILE $nodes","\n";
               print FILE "setenv QCSCRATCH \$SGE_O_WORKDIR/\$JOB_ID","\n";
               print FILE "/bin/mkdir -p \$QCSCRATCH","\n";
               print FILE "setenv QCLOCALSCR /scr/\$USER/\$JOB_ID","\n";
	       print FILE "/bin/mkdir -p \$QCLOCALSCR","\n";

        }
               
	if ( $num_cpu > 64 && $mpi eq "mpi-sm" ) {
               print FILE "set node=`cat $nodes`","\n";
               print FILE "echo \$node","\n";
               print FILE "set i = 2","\n";
               print FILE "while ( \$i <= \$#node )","\n";
               print FILE "\t","ssh \$node[\$i] \"mkdir -p \$QCSCRATCH\"","\n";
               print FILE "\t","set i = `expr \$i + 1`","\n";
               print FILE "end","\n";
	       print FILE "setenv QCSCRATCH \$SGE_O_WORKDIR/\$JOB_ID","\n"; 
	       print FILE "/bin/mkdir -p \$QCSCRATCH","\n";
	       print FILE "setenv QCLOCALSCR /scr/\$USER/\$JOB_ID","\n"; 
         }

#print FILE "cat \$TMPDIR/machines","\n";
#print FILE "set node=`cat $nodes`","\n";
#print FILE "echo \$node","\n";

print FILE "/usr/bin/env","\n";
print FILE "\n";
if ( $save eq "" )  { 
	print FILE "$qc/bin/qchem $thread $num_cpu $input$ext $input.out","\n";
	} else {
        print FILE "$qc/bin/qchem $thread $num_cpu -save $input$ext $input.out $input-saved","\n";
        print FILE "mv \$QCSCRATCH/$input-saved \$SGE_O_WORKDIR","\n";
       }

print FILE "\n";
#print FILE "rm -rf \$QCSCRATCH \$QCLOCALSCR" ,"\n";

       if  ( $mpi eq "mpi-sm"  &&  $num_cpu > 64 )  {
#               print "I am here $mpi $num_cpu $openmp\n";
               print FILE "set node=`cat $nodes`","\n";
               print FILE "set i = 2","\n";
               print FILE "while ( \$i <= \$#node )","\n";
               print FILE "\t","ssh \$node[\$i] \"rm -rf /scr/\$USER/\$JOB_ID\"","\n";
               print FILE "\t","set i = `expr \$i + 1`","\n";
               print FILE "end","\n";
       }

$command1 = "cd $directory\; qsub $jobname </dev/null >&/dev/null &";
$command2 = "/usr/bin/ssh tiger \"$command1\"";
#print $command2, "\n";

$hostname = `/bin/hostname`; 
if ( $hostname eq "tiger.cchem.berkeley.edu" ) {
	print "Your job has been submitted on Tiger. Use qstat to follow the progress. \n";
	exec $command1;
	} 
	else {
        #print "$hostname\n";
	print "Your job has been submitted to Tiger. Use qstat to follow the progress. \n";
	exec $command2;
	}