initial work for issue 399: add --force flag

ChangeLog

@@ -1,3 +1,14 @@
+2014-12-17  Jessica Mizzi  <mizzijes@msu.edu>
+
+    * khmer/file.py,sandbox/sweep-reads.py,scripts/{abundance-dist-single,
+    abundance-dist,annotate-partitions,count-median,count-overlap,do-partition,
+    extract-paired-reads,extract-partitions,filter-abund-single,filter-abund,
+    filter-stoptags,interleave-reads,load-graph,load-into-counting,
+    make-initial-stoptags,merge-partitions,normalize-by-median,partition-graph,
+    sample-reads-randomly,split-paired-reads}.py,setup.cfg,
+    tests/{test_script_arguments,test_scripts}.py: Added force option to all 
+    scripts to script IO sanity checks and updated tests to match. 
+
 2014-12-17  Michael R. Crusoe  <mcrusoe@msu.edu>
 
     * scripts/load-graph.py,khmer/_khmermodule.cc: restore threading to

khmer/file.py

@@ -14,7 +14,7 @@
 from stat import S_ISBLK, S_ISFIFO
 
 
-def check_file_status(file_path):
+def check_file_status(file_path, force):
     """Check the status of the file; if the file is empty or doesn't exist
     AND if the file is NOT a fifo/block/named pipe then a warning is printed
     and sys.exit(1) is called
@@ -30,15 +30,17 @@ def check_file_status(file_path):
     if not os.path.exists(file_path):
         print >>sys.stderr, "ERROR: Input file %s does not exist; exiting" % \
                             file_path
-        sys.exit(1)
+        if not force:
+            sys.exit(1)
     else:
         if os.stat(file_path).st_size == 0:
             print >>sys.stderr, "ERROR: Input file %s is empty; exiting." % \
                                 file_path
-            sys.exit(1)
+            if not force:
+                sys.exit(1)
 
 
-def check_space(in_files, _testhook_free_space=None):
+def check_space(in_files, force, _testhook_free_space=None):
     """
     Estimate size of input files passed, then calculate
     disk space available. Exit if insufficient disk space,
@@ -74,10 +76,11 @@ def check_space(in_files, _testhook_free_space=None):
                             % (float(total_size) / 1e9,)
         print >>sys.stderr, "       Free space: %.1f GB" \
                             % (float(free_space) / 1e9,)
-        sys.exit(1)
+        if not force:
+            sys.exit(1)
 
 
-def check_space_for_hashtable(hash_size, _testhook_free_space=None):
+def check_space_for_hashtable(hash_size, force, _testhook_free_space=None):
     """
     Check we have enough size to write a hash table
     """
@@ -99,7 +102,8 @@ def check_space_for_hashtable(hash_size, _testhook_free_space=None):
                             % (float(hash_size) / 1e9,)
         print >>sys.stderr, "       Free space: %.1f GB" \
                             % (float(free_space) / 1e9,)
-        sys.exit(1)
+        if not force:
+            sys.exit(1)
 
 
 def check_valid_file_exists(in_files):

khmer/load_pe.py

@@ -23,7 +23,7 @@ def load_pe(screed_handle):
 
     screed_iter = iter(screed_handle)
 
-    while 1:
+    while True:
         try:
             this_record = screed_iter.next()
         except StopIteration:

sandbox/sweep-reads.py

@@ -191,7 +191,8 @@ def get_parser():
     parser.add_argument(dest='input_fastp', help='Reference fasta or fastp')
     parser.add_argument('input_files', nargs='+',
                         help='Reads to be swept and sorted')
-
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
@@ -224,13 +225,13 @@ def main():
     buf_size = args.buffer_size
     max_reads = args.max_reads
 
-    check_file_status(args.input_fastp)
+    check_file_status(args.input_fastp, args.force)
     check_valid_file_exists(args.input_files)
     all_input_files = [input_fastp]
     all_input_files.extend(args.input_files)
 
     # Check disk space availability
-    check_space(all_input_files)
+    check_space(all_input_files, args.force)
 
     # figure out input file type (FA/FQ) -- based on first file
     ix = iter(screed.open(args.input_files[0]))

scripts/abundance-dist-single.py

@@ -60,6 +60,8 @@ def get_parser():
                         "filename.")
     parser.add_argument('--report-total-kmers', '-t', action='store_true',
                         help="Prints the total number of k-mers to stderr")
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
@@ -68,8 +70,8 @@ def main():  # pylint: disable=too-many-locals,too-many-branches
     args = get_parser().parse_args()
     report_on_config(args)
 
-    check_file_status(args.input_sequence_filename)
-    check_space([args.input_sequence_filename])
+    check_file_status(args.input_sequence_filename, args.force)
+    check_space([args.input_sequence_filename], args.force)
     if args.savetable:
         check_space_for_hashtable(args.n_tables * args.min_tablesize)
 

scripts/abundance-dist.py

@@ -44,6 +44,8 @@ def get_parser():
                         help='Overwrite output file if it exists')
     parser.add_argument('--version', action='version', version='%(prog)s '
                         + khmer.__version__)
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
@@ -53,7 +55,7 @@ def main():
     infiles = [args.input_counting_table_filename,
                args.input_sequence_filename]
     for infile in infiles:
-        check_file_status(infile)
+        check_file_status(infile, args.force)
 
     print ('hashtable from', args.input_counting_table_filename,
            file=sys.stderr)

scripts/annotate-partitions.py

@@ -55,6 +55,8 @@ def get_parser():
                         'annotate.')
     parser.add_argument('--version', action='version', version='%(prog)s '
                         + khmer.__version__)
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
@@ -68,11 +70,11 @@ def main():
 
     partitionmap_file = args.graphbase + '.pmap.merged'
 
-    check_file_status(partitionmap_file)
+    check_file_status(partitionmap_file, args.force)
     for _ in filenames:
-        check_file_status(_)
+        check_file_status(_, args.force)
 
-    check_space(filenames)
+    check_space(filenames, args.force)
 
     print >>sys.stderr, 'loading partition map from:', partitionmap_file
     htable.load_partitionmap(partitionmap_file)

scripts/count-median.py

@@ -51,6 +51,8 @@ def get_parser():
                         help='output summary filename')
     parser.add_argument('--version', action='version', version='%(prog)s '
                         + khmer.__version__)
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
@@ -64,9 +66,9 @@ def main():
 
     infiles = [htfile, input_filename]
     for infile in infiles:
-        check_file_status(infile)
+        check_file_status(infile, args.force)
 
-    check_space(infiles)
+    check_space(infiles, args.force)
 
     print >>sys.stderr, 'loading k-mer counting table from', htfile
     htable = khmer.load_counting_hash(htfile)

scripts/count-overlap.py

@@ -44,7 +44,8 @@ def get_parser():
                         help="input sequence filename")
     parser.add_argument('report_filename', metavar='output_report_filename',
                         help='output report filename')
-
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
@@ -54,9 +55,9 @@ def main():
     report_on_config(args, hashtype='hashbits')
 
     for infile in [args.ptfile, args.fafile]:
-        check_file_status(infile)
+        check_file_status(infile, args.force)
 
-    check_space([args.ptfile, args.fafile])
+    check_space([args.ptfile, args.fafile], args.force)
 
     print >>sys.stderr, 'loading k-mer presence table from', args.ptfile
     ht1 = khmer.load_hashbits(args.ptfile)

scripts/do-partition.py

@@ -45,7 +45,7 @@ def __debug_vm_usage(msg):  # pylint: disable=unused-argument
 
 
 def worker(queue, basename, stop_big_traversals):
-    while 1:
+    while True:
         try:
             (htable, index, start, stop) = queue.get(False)
         except Queue.Empty:
@@ -97,6 +97,8 @@ def get_parser():
     parser.add_argument('graphbase', help="base name for output files")
     parser.add_argument('input_filenames', metavar='input_sequence_filename',
                         nargs='+', help='input FAST[AQ] sequence filenames')
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
@@ -108,9 +110,9 @@ def main():  # pylint: disable=too-many-locals,too-many-statements
     report_on_config(args, hashtype='hashbits')
 
     for infile in args.input_filenames:
-        check_file_status(infile)
+        check_file_status(infile, args.force)
 
-    check_space(args.input_filenames)
+    check_space(args.input_filenames, args.force)
 
     print >>sys.stderr, 'Saving k-mer presence table to %s' % args.graphbase
     print >>sys.stderr, 'Loading kmers from sequences in %s' % \
@@ -137,7 +139,8 @@ def main():  # pylint: disable=too-many-locals,too-many-statements
                               " this data set.  Increase k-mer presence table "
                               "size/num of tables.")
         print >> sys.stderr, "**"
-        sys.exit(1)
+        if not args.force:
+            sys.exit(1)
 
     # partition-graph
 

scripts/extract-paired-reads.py

@@ -74,16 +74,18 @@ def get_parser():
     parser.add_argument('infile')
     parser.add_argument('--version', action='version', version='%(prog)s '
                         + khmer.__version__)
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
 def main():
     info('extract-paired-reads.py')
     args = get_parser().parse_args()
 
-    check_file_status(args.infile)
+    check_file_status(args.infile, args.force)
     infiles = [args.infile]
-    check_space(infiles)
+    check_space(infiles, args.force)
 
     outfile = os.path.basename(args.infile)
     if len(sys.argv) > 2:

scripts/extract-partitions.py

@@ -81,6 +81,8 @@ def get_parser():
                         help='Output unassigned sequences, too')
     parser.add_argument('--version', action='version', version='%(prog)s '
                         + khmer.__version__)
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
@@ -94,9 +96,9 @@ def main():  # pylint: disable=too-many-locals,too-many-branches
     n_unassigned = 0
 
     for infile in args.part_filenames:
-        check_file_status(infile)
+        check_file_status(infile, args.force)
 
-    check_space(args.part_filenames)
+    check_space(args.part_filenames, args.force)
 
     print >>sys.stderr, '---'
     print >>sys.stderr, 'reading partitioned files:', repr(args.part_filenames)

scripts/filter-abund-single.py

@@ -56,16 +56,19 @@ def get_parser():
                         help="FAST[AQ] sequence file to trim")
     parser.add_argument('--report-total-kmers', '-t', action='store_true',
                         help="Prints the total number of k-mers to stderr")
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
 def main():
     info('filter-abund-single.py', ['counting'])
     args = get_parser().parse_args()
-    check_file_status(args.datafile)
-    check_space([args.datafile])
+    check_file_status(args.datafile, args.force)
+    check_space([args.datafile], args.force)
     if args.savetable:
-        check_space_for_hashtable(args.n_tables * args.min_tablesize)
+        check_space_for_hashtable(
+            args.n_tables * args.min_tablesize, args.force)
     report_on_config(args)
 
     print >>sys.stderr, 'making k-mer counting table'

scripts/filter-abund.py

@@ -67,6 +67,8 @@ def get_parser():
                         'file for each input file.')
     parser.add_argument('--version', action='version',
                         version='khmer {v}'.format(v=__version__))
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
@@ -78,9 +80,9 @@ def main():
     infiles = args.input_filename
 
     for _ in infiles:
-        check_file_status(_)
+        check_file_status(_, args.force)
 
-    check_space(infiles)
+    check_space(infiles, args.force)
 
     print >>sys.stderr, 'loading hashtable'
     htable = khmer.load_counting_hash(counting_ht)

scripts/filter-stoptags.py

@@ -45,6 +45,8 @@ def get_parser():
                         nargs='+')
     parser.add_argument('--version', action='version', version='%(prog)s '
                         + khmer.__version__)
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
@@ -55,9 +57,9 @@ def main():
     infiles = args.input_filenames
 
     for _ in infiles:
-        check_file_status(_)
+        check_file_status(_, args.force)
 
-    check_space(infiles)
+    check_space(infiles, args.force)
 
     print >>sys.stderr, 'loading stop tags, with K', args.ksize
     htable = khmer.new_hashbits(args.ksize, 1, 1)

scripts/interleave-reads.py

@@ -63,6 +63,8 @@ def get_parser():
                         default=sys.stdout)
     parser.add_argument('--version', action='version', version='%(prog)s '
                         + khmer.__version__)
+    parser.add_argument('-f', '--force', default=False, action='store_true',
+                        help='Overwrite output file if it exists')
     return parser
 
 
@@ -71,9 +73,9 @@ def main():
     args = get_parser().parse_args()
 
     for _ in args.infiles:
-        check_file_status(_)
+        check_file_status(_, args.force)
 
-    check_space(args.infiles)
+    check_space(args.infiles, args.force)
 
     s1_file = args.infiles[0]
     if len(args.infiles) == 2:
@@ -92,7 +94,7 @@ def main():
         print >> sys.stderr, "Error! R2 file %s does not exist" % s2_file
         fail = True
 
-    if fail:
+    if fail and not args.force:
         sys.exit(1)
 
     print >> sys.stderr, "Interleaving:\n\t%s\n\t%s" % (s1_file, s2_file)