From c6951128779fd1aa0cc13db292e044649b8bd66f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 6 Mar 2018 11:52:34 -0500 Subject: [PATCH] More upload tests and fixes. --- lib/galaxy/datatypes/sniff.py | 15 ++-- lib/galaxy/datatypes/test/1.csv | 1 + lib/galaxy/tools/data_fetch.py | 3 + test-data/1.csv | 1 + test/api/test_tools_upload.py | 82 +++++++++++++++++-- .../tools/sample_datatypes_conf.xml | 1 + 6 files changed, 92 insertions(+), 11 deletions(-) create mode 100644 lib/galaxy/datatypes/test/1.csv create mode 100644 test-data/1.csv diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 9cbe81270b93..c69373b0669f 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -138,7 +138,7 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload"): return (i, temp_name) -def sep2tabs(fname, in_place=True, patt="\\s+"): +def sep2tabs(fname, in_place=True, patt="\\s+", tmp_dir=None, tmp_prefix="gxupload"): """ Transforms in place a 'sep' separated file to a tab separated one @@ -150,13 +150,18 @@ def sep2tabs(fname, in_place=True, patt="\\s+"): '1\\t2\\n3\\t4\\n' """ regexp = re.compile(patt) - fd, temp_name = tempfile.mkstemp() + fd, temp_name = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir) with os.fdopen(fd, "wt") as fp: i = None for i, line in enumerate(open(fname)): - line = line.rstrip('\r\n') - elems = regexp.split(line) - fp.write("%s\n" % '\t'.join(elems)) + if line.endswith("\r"): + line = line.rstrip('\r') + elems = regexp.split(line) + fp.write("%s\r" % '\t'.join(elems)) + else: + line = line.rstrip('\n') + elems = regexp.split(line) + fp.write("%s\n" % '\t'.join(elems)) if i is None: i = 0 else: diff --git a/lib/galaxy/datatypes/test/1.csv b/lib/galaxy/datatypes/test/1.csv new file mode 100644 index 000000000000..80d519fbe2d9 --- /dev/null +++ b/lib/galaxy/datatypes/test/1.csv @@ -0,0 +1 @@ +Transaction_date,Product,Price,Payment_Type,Name,City,State,Country,Account_Created,Last_Login,Latitude,Longitude 1/2/09 6:17,Product1,1200,Mastercard,carolina,Basildon,England,United Kingdom,1/2/09 6:00,1/2/09 6:08,51.5,-1.1166667 1/2/09 4:53,Product1,1200,Visa,Betina,Parkville ,MO,United States,1/2/09 4:42,1/2/09 7:49,39.195,-94.68194 1/2/09 13:08,Product1,1200,Mastercard,Federica e Andrea,Astoria ,OR,United States,1/1/09 16:21,1/3/09 12:32,46.18806,-123.83 1/3/09 14:44,Product1,1200,Visa,Gouya,Echuca,Victoria,Australia,9/25/05 21:13,1/3/09 14:22,-36.1333333,144.75 1/4/09 12:56,Product2,3600,Visa,Gerd W ,Cahaba Heights ,AL,United States,11/15/08 15:47,1/4/09 12:45,33.52056,-86.8025 1/4/09 13:19,Product1,1200,Visa,LAURENCE,Mickleton ,NJ,United States,9/24/08 15:19,1/4/09 13:04,39.79,-75.23806 1/4/09 20:11,Product1,1200,Mastercard,Fleur,Peoria ,IL,United States,1/3/09 9:38,1/4/09 19:45,40.69361,-89.58889 1/2/09 20:09,Product1,1200,Mastercard,adam,Martin ,TN,United States,1/2/09 17:43,1/4/09 20:01,36.34333,-88.85028 1/4/09 13:17,Product1,1200,Mastercard,Renee Elisabeth,Tel Aviv,Tel Aviv,Israel,1/4/09 13:03,1/4/09 22:10,32.0666667,34.7666667 1/4/09 14:11,Product1,1200,Visa,Aidan,Chatou,Ile-de-France,France,6/3/08 4:22,1/5/09 1:17,48.8833333,2.15 1/5/09 2:42,Product1,1200,Diners,Stacy,New York ,NY,United States,1/5/09 2:23,1/5/09 4:59,40.71417,-74.00639 1/5/09 5:39,Product1,1200,Amex,Heidi,Eindhoven,Noord-Brabant,Netherlands,1/5/09 4:55,1/5/09 8:15,51.45,5.4666667 1/2/09 9:16,Product1,1200,Mastercard,Sean ,Shavano Park ,TX,United States,1/2/09 8:32,1/5/09 9:05,29.42389,-98.49333 1/5/09 10:08,Product1,1200,Visa,Georgia,Eagle ,ID,United States,11/11/08 15:53,1/5/09 10:05,43.69556,-116.35306 1/2/09 14:18,Product1,1200,Visa,Richard,Riverside ,NJ,United States,12/9/08 12:07,1/5/09 11:01,40.03222,-74.95778 1/25/09 17:58,Product2,3600,Visa,carol,Ann Arbor ,MI,United States,7/5/08 9:20,2/7/09 18:51,42.27083,-83.72639 1/9/09 14:37,Product1,1200,Visa,Nona,South Jordan ,UT,United States,1/8/09 15:14,2/7/09 19:11,40.56222,-111.92889 1/25/09 2:46,Product2,3600,Visa,Family,Dubai,Dubayy,United Arab Emirates,1/8/09 1:19,2/8/09 2:06,25.2522222,55.28 1/17/09 20:46,Product2,3600,Visa,Michelle,Dubai,Dubayy,United Arab Emirates,4/13/08 2:36,2/8/09 2:12,25.2522222,55.28 1/24/09 7:18,Product2,3600,Visa,Kathryn,Kirriemuir,Scotland,United Kingdom,1/23/09 10:31,2/8/09 2:52,56.6666667,-3 1/11/09 7:09,Product1,1200,Visa,Oswald,Tramore,Waterford,Ireland,10/13/08 16:43,2/8/09 3:02,52.1588889,-7.1463889 1/8/09 4:15,Product1,1200,Visa,Elyssa,Gdansk,Pomorskie,Poland,1/7/09 15:00,2/8/09 3:50,54.35,18.6666667 1/22/09 10:47,Product1,1200,Visa,michelle,Arklow,Wicklow,Ireland,11/18/08 1:32,2/8/09 5:07,52.7930556,-6.1413889 1/26/09 20:47,Product1,1200,Mastercard,Alicia,Lincoln ,NE,United States,6/24/08 8:05,2/8/09 7:29,40.8,-96.66667 1/12/09 12:22,Product1,1200,Mastercard,JP,Tierp,Uppsala,Sweden,1/6/09 11:34,2/8/09 11:15,60.3333333,17.5 1/26/09 1:44,Product2,3600,Visa,Geraldine,Brussels,Brussels (Bruxelles),Belgium,1/31/08 13:28,2/8/09 14:39,50.8333333,4.3333333 1/18/09 12:57,Product1,1200,Mastercard,sandra,Burr Oak ,IA,United States,1/24/08 16:11,2/8/09 15:30,43.45889,-91.86528 1/24/09 21:26,Product1,1200,Visa,Olivia,Wheaton ,IL,United States,5/8/08 16:02,2/8/09 16:00,41.86611,-88.10694 1/26/09 12:26,Product2,3600,Mastercard,Tom,Killeen ,TX,United States,1/26/09 5:23,2/8/09 17:33,31.11694,-97.7275 1/5/09 7:37,Product1,1200,Visa,Annette ,Manhattan ,NY,United States,9/26/08 4:29,2/8/09 18:42,40.71417,-74.00639 1/14/09 12:33,Product1,1200,Visa,SUSAN,Oxford,England,United Kingdom,9/11/08 23:23,2/8/09 23:00,51.75,-1.25 1/14/09 0:15,Product2,3600,Visa,Michael,Paris,Ile-de-France,France,11/28/08 0:07,2/9/09 1:30,48.8666667,2.3333333 1/1/09 12:42,Product1,1200,Visa,ashton,Exeter,England,United Kingdom,12/15/08 1:16,2/9/09 2:52,50.7,-3.5333333 1/6/09 6:07,Product1,1200,Visa,Scott,Rungsted,Frederiksborg,Denmark,12/27/08 14:29,2/9/09 4:20,55.8841667,12.5419444 1/15/09 5:11,Product2,3600,Visa,Pam,London,England,United Kingdom,7/11/06 12:43,2/9/09 4:42,51.52721,0.14559 1/17/09 4:03,Product1,1200,Visa,Lisa ,Borja,Bohol,Philippines,1/17/09 2:45,2/9/09 6:09,9.9136111,124.0927778 1/19/09 10:13,Product2,3600,Mastercard,Pavel,London,England,United Kingdom,2/28/06 5:35,2/9/09 6:57,51.51334,-0.08895 1/18/09 9:42,Product1,1200,Visa,Richard,Jamestown ,RI,United States,1/18/09 9:22,2/9/09 8:30,41.49694,-71.36778 1/9/09 11:14,Product1,1200,Visa,Jasinta Jeanne,Owings Mills ,MD,United States,1/9/09 10:43,2/9/09 9:17,39.41944,-76.78056 1/10/09 13:42,Product1,1200,Visa,Rachel,Hamilton,Ontario,Canada,1/10/09 12:22,2/9/09 9:54,43.25,-79.8333333 1/7/09 7:28,Product1,1200,Amex,Cherish ,Anchorage ,AK,United States,7/28/08 7:31,2/9/09 10:50,61.21806,-149.90028 1/18/09 6:46,Product1,1200,Visa,Shona ,Mornington,Meath,Ireland,1/15/09 9:13,2/9/09 11:55,53.7233333,-6.2825 1/30/09 12:18,Product1,1200,Mastercard,Abikay,Fullerton ,CA,United States,1/26/09 13:34,2/9/09 12:53,33.87028,-117.92444 1/6/09 5:42,Product1,1200,Amex,Abikay,Atlanta ,GA,United States,10/27/08 14:16,2/9/09 13:50,33.74889,-84.38806 1/2/09 10:58,Product2,3600,Visa,Kendra,Toronto,Ontario,Canada,1/2/09 10:38,2/9/09 13:56,43.6666667,-79.4166667 1/8/09 3:29,Product1,1200,Visa,amanda,Liverpool,England,United Kingdom,12/22/08 1:41,2/9/09 14:06,53.4166667,-3 1/12/09 13:23,Product2,3600,Amex,Leila,Ponte San Nicolo,Veneto,Italy,9/13/05 8:42,2/9/09 14:09,45.3666667,11.6166667 1/19/09 9:34,Product1,1200,Amex,amanda,Las Vegas ,NV,United States,5/10/08 8:56,2/9/09 16:44,36.175,-115.13639 1/9/09 7:49,Product1,1200,Visa,Stacy,Rochester Hills ,MI,United States,7/28/08 7:18,2/9/09 17:41,42.68056,-83.13389 1/15/09 5:27,Product2,3600,Visa,Derrick,North Bay,Ontario,Canada,1/6/09 17:42,2/9/09 18:22,46.3,-79.45 1/8/09 23:40,Product1,1200,Visa,Jacob,Lindfield,New South Wales,Australia,1/8/09 17:52,2/9/09 18:31,-33.7833333,151.1666667 1/27/09 11:02,Product1,1200,Mastercard,DOREEN,Madrid,Madrid,Spain,1/24/09 8:21,2/9/09 18:42,40.4,-3.6833333 1/14/09 13:23,Product1,1200,Diners,eugenia,Wisconsin Rapids ,WI,United States,11/15/08 13:57,2/9/09 18:44,44.38361,-89.81722 1/7/09 20:01,Product1,1200,Visa,Karen,Austin ,TX,United States,1/6/09 19:16,2/9/09 19:56,30.26694,-97.74278 1/20/09 12:32,Product1,1200,Visa,Bea,Chicago ,IL,United States,1/16/09 19:08,2/9/09 20:42,41.85,-87.65 1/6/09 14:35,Product1,1200,Diners,Hilde Karin,Las Vegas ,NV,United States,12/17/08 11:59,2/9/09 22:59,36.175,-115.13639 1/4/09 6:51,Product1,1200,Visa,Rima,Mullingar,Westmeath,Ireland,1/3/09 12:34,2/10/09 0:59,53.5333333,-7.35 1/24/09 18:30,Product1,1200,Visa,Ruangrote,Melbourne,Victoria,Australia,7/17/08 5:19,2/10/09 2:12,-37.8166667,144.9666667 1/25/09 5:57,Product1,1200,Amex,pamela,Ayacucho,Buenos Aires,Argentina,1/24/09 9:29,2/10/09 6:38,-37.15,-58.4833333 1/5/09 10:02,Product2,3600,Visa,Emillie,Eagan ,MN,United States,1/5/09 9:03,2/10/09 7:29,44.80417,-93.16667 1/13/09 9:14,Product1,1200,Visa,sangeeta,Vossevangen,Hordaland,Norway,1/9/09 9:31,2/10/09 9:04,60.6333333,6.4333333 1/22/09 7:35,Product1,1200,Visa,Anja,Ferney-Voltaire,Rhone-Alpes,France,1/22/09 6:51,2/10/09 9:18,46.25,6.1166667 1/2/09 11:06,Product1,1200,Mastercard,Andrew,Sevilla,Andalucia,Spain,3/12/06 15:02,2/10/09 10:04,37.3772222,-5.9869444 1/11/09 9:50,Product1,1200,Visa,Bato,Munchengosserstadt,Thuringia,Germany,1/7/09 11:45,2/10/09 10:28,51.05,11.65 1/21/09 20:44,Product1,1200,Mastercard,Ailsa ,Lindenhurst ,NY,United States,1/21/09 7:47,2/10/09 10:51,40.68667,-73.37389 1/5/09 9:09,Product1,1200,Visa,Sophie,Bloomfield ,MI,United States,10/23/06 6:52,2/10/09 10:58,42.53778,-83.23306 1/5/09 12:41,Product1,1200,Visa,Katrin,Calgary,Alberta,Canada,12/3/08 14:49,2/10/09 11:45,51.0833333,-114.0833333 1/28/09 12:54,Product2,3600,Mastercard,Kelly ,Vancouver,British Columbia,Canada,1/27/09 21:04,2/10/09 12:09,49.25,-123.1333333 1/21/09 4:46,Product1,1200,Visa,Tomasz,Klampenborg,Kobenhavn,Denmark,6/10/08 11:25,2/10/09 12:22,55.7666667,12.6 1/7/09 13:28,Product1,1200,Visa,Elizabeth,Calne,England,United Kingdom,1/4/09 13:07,2/10/09 12:39,51.4333333,-2 1/27/09 11:18,Product2,3600,Amex,Michael,Los Angeles ,CA,United States,1/23/09 11:47,2/10/09 13:09,34.05222,-118.24278 1/7/09 12:39,Product2,3600,Visa,Natasha,Milano,Lombardy,Italy,6/2/06 13:01,2/10/09 13:19,45.4666667,9.2 1/24/09 13:54,Product2,3600,Mastercard,Meredith,Kloten,Zurich,Switzerland,1/24/09 12:30,2/10/09 13:47,47.45,8.5833333 1/30/09 6:48,Product1,1200,Mastercard,Nicole,Fayetteville ,NC,United States,1/30/09 4:51,2/10/09 14:41,35.0525,-78.87861 1/22/09 18:07,Product1,1200,Visa,Ryan,Simpsonville ,SC,United States,1/6/09 16:59,2/10/09 15:30,34.73694,-82.25444 1/29/09 15:03,Product1,1200,Visa,Mary ,Auckland,Auckland,New Zealand,2/9/06 11:14,2/10/09 16:31,-36.8666667,174.7666667 1/2/09 14:14,Product1,1200,Diners,Aaron,Reading,England,United Kingdom,11/16/08 15:49,2/10/09 16:38,51.4333333,-1 1/19/09 11:05,Product1,1200,Visa,Bertrand,North Caldwell ,NJ,United States,10/3/08 5:55,2/10/09 18:16,40.83972,-74.27694 \ No newline at end of file diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py index ddde62079256..c6a2b7f6c610 100644 --- a/lib/galaxy/tools/data_fetch.py +++ b/lib/galaxy/tools/data_fetch.py @@ -137,6 +137,9 @@ def _resolve_src(item): line_count, converted_path = sniff.convert_newlines_sep2tabs(path, in_place=in_place, tmp_dir=".") else: line_count, converted_path = sniff.convert_newlines(path, in_place=in_place, tmp_dir=".") + else: + if space_to_tab: + line_count, converted_path = sniff.sep2tabs(path, in_place=in_place, tmp_dir=".") if requested_ext == 'auto': ext = sniff.guess_ext(path, registry.sniff_order) diff --git a/test-data/1.csv b/test-data/1.csv new file mode 100644 index 000000000000..80d519fbe2d9 --- /dev/null +++ b/test-data/1.csv @@ -0,0 +1 @@ +Transaction_date,Product,Price,Payment_Type,Name,City,State,Country,Account_Created,Last_Login,Latitude,Longitude 1/2/09 6:17,Product1,1200,Mastercard,carolina,Basildon,England,United Kingdom,1/2/09 6:00,1/2/09 6:08,51.5,-1.1166667 1/2/09 4:53,Product1,1200,Visa,Betina,Parkville ,MO,United States,1/2/09 4:42,1/2/09 7:49,39.195,-94.68194 1/2/09 13:08,Product1,1200,Mastercard,Federica e Andrea,Astoria ,OR,United States,1/1/09 16:21,1/3/09 12:32,46.18806,-123.83 1/3/09 14:44,Product1,1200,Visa,Gouya,Echuca,Victoria,Australia,9/25/05 21:13,1/3/09 14:22,-36.1333333,144.75 1/4/09 12:56,Product2,3600,Visa,Gerd W ,Cahaba Heights ,AL,United States,11/15/08 15:47,1/4/09 12:45,33.52056,-86.8025 1/4/09 13:19,Product1,1200,Visa,LAURENCE,Mickleton ,NJ,United States,9/24/08 15:19,1/4/09 13:04,39.79,-75.23806 1/4/09 20:11,Product1,1200,Mastercard,Fleur,Peoria ,IL,United States,1/3/09 9:38,1/4/09 19:45,40.69361,-89.58889 1/2/09 20:09,Product1,1200,Mastercard,adam,Martin ,TN,United States,1/2/09 17:43,1/4/09 20:01,36.34333,-88.85028 1/4/09 13:17,Product1,1200,Mastercard,Renee Elisabeth,Tel Aviv,Tel Aviv,Israel,1/4/09 13:03,1/4/09 22:10,32.0666667,34.7666667 1/4/09 14:11,Product1,1200,Visa,Aidan,Chatou,Ile-de-France,France,6/3/08 4:22,1/5/09 1:17,48.8833333,2.15 1/5/09 2:42,Product1,1200,Diners,Stacy,New York ,NY,United States,1/5/09 2:23,1/5/09 4:59,40.71417,-74.00639 1/5/09 5:39,Product1,1200,Amex,Heidi,Eindhoven,Noord-Brabant,Netherlands,1/5/09 4:55,1/5/09 8:15,51.45,5.4666667 1/2/09 9:16,Product1,1200,Mastercard,Sean ,Shavano Park ,TX,United States,1/2/09 8:32,1/5/09 9:05,29.42389,-98.49333 1/5/09 10:08,Product1,1200,Visa,Georgia,Eagle ,ID,United States,11/11/08 15:53,1/5/09 10:05,43.69556,-116.35306 1/2/09 14:18,Product1,1200,Visa,Richard,Riverside ,NJ,United States,12/9/08 12:07,1/5/09 11:01,40.03222,-74.95778 1/25/09 17:58,Product2,3600,Visa,carol,Ann Arbor ,MI,United States,7/5/08 9:20,2/7/09 18:51,42.27083,-83.72639 1/9/09 14:37,Product1,1200,Visa,Nona,South Jordan ,UT,United States,1/8/09 15:14,2/7/09 19:11,40.56222,-111.92889 1/25/09 2:46,Product2,3600,Visa,Family,Dubai,Dubayy,United Arab Emirates,1/8/09 1:19,2/8/09 2:06,25.2522222,55.28 1/17/09 20:46,Product2,3600,Visa,Michelle,Dubai,Dubayy,United Arab Emirates,4/13/08 2:36,2/8/09 2:12,25.2522222,55.28 1/24/09 7:18,Product2,3600,Visa,Kathryn,Kirriemuir,Scotland,United Kingdom,1/23/09 10:31,2/8/09 2:52,56.6666667,-3 1/11/09 7:09,Product1,1200,Visa,Oswald,Tramore,Waterford,Ireland,10/13/08 16:43,2/8/09 3:02,52.1588889,-7.1463889 1/8/09 4:15,Product1,1200,Visa,Elyssa,Gdansk,Pomorskie,Poland,1/7/09 15:00,2/8/09 3:50,54.35,18.6666667 1/22/09 10:47,Product1,1200,Visa,michelle,Arklow,Wicklow,Ireland,11/18/08 1:32,2/8/09 5:07,52.7930556,-6.1413889 1/26/09 20:47,Product1,1200,Mastercard,Alicia,Lincoln ,NE,United States,6/24/08 8:05,2/8/09 7:29,40.8,-96.66667 1/12/09 12:22,Product1,1200,Mastercard,JP,Tierp,Uppsala,Sweden,1/6/09 11:34,2/8/09 11:15,60.3333333,17.5 1/26/09 1:44,Product2,3600,Visa,Geraldine,Brussels,Brussels (Bruxelles),Belgium,1/31/08 13:28,2/8/09 14:39,50.8333333,4.3333333 1/18/09 12:57,Product1,1200,Mastercard,sandra,Burr Oak ,IA,United States,1/24/08 16:11,2/8/09 15:30,43.45889,-91.86528 1/24/09 21:26,Product1,1200,Visa,Olivia,Wheaton ,IL,United States,5/8/08 16:02,2/8/09 16:00,41.86611,-88.10694 1/26/09 12:26,Product2,3600,Mastercard,Tom,Killeen ,TX,United States,1/26/09 5:23,2/8/09 17:33,31.11694,-97.7275 1/5/09 7:37,Product1,1200,Visa,Annette ,Manhattan ,NY,United States,9/26/08 4:29,2/8/09 18:42,40.71417,-74.00639 1/14/09 12:33,Product1,1200,Visa,SUSAN,Oxford,England,United Kingdom,9/11/08 23:23,2/8/09 23:00,51.75,-1.25 1/14/09 0:15,Product2,3600,Visa,Michael,Paris,Ile-de-France,France,11/28/08 0:07,2/9/09 1:30,48.8666667,2.3333333 1/1/09 12:42,Product1,1200,Visa,ashton,Exeter,England,United Kingdom,12/15/08 1:16,2/9/09 2:52,50.7,-3.5333333 1/6/09 6:07,Product1,1200,Visa,Scott,Rungsted,Frederiksborg,Denmark,12/27/08 14:29,2/9/09 4:20,55.8841667,12.5419444 1/15/09 5:11,Product2,3600,Visa,Pam,London,England,United Kingdom,7/11/06 12:43,2/9/09 4:42,51.52721,0.14559 1/17/09 4:03,Product1,1200,Visa,Lisa ,Borja,Bohol,Philippines,1/17/09 2:45,2/9/09 6:09,9.9136111,124.0927778 1/19/09 10:13,Product2,3600,Mastercard,Pavel,London,England,United Kingdom,2/28/06 5:35,2/9/09 6:57,51.51334,-0.08895 1/18/09 9:42,Product1,1200,Visa,Richard,Jamestown ,RI,United States,1/18/09 9:22,2/9/09 8:30,41.49694,-71.36778 1/9/09 11:14,Product1,1200,Visa,Jasinta Jeanne,Owings Mills ,MD,United States,1/9/09 10:43,2/9/09 9:17,39.41944,-76.78056 1/10/09 13:42,Product1,1200,Visa,Rachel,Hamilton,Ontario,Canada,1/10/09 12:22,2/9/09 9:54,43.25,-79.8333333 1/7/09 7:28,Product1,1200,Amex,Cherish ,Anchorage ,AK,United States,7/28/08 7:31,2/9/09 10:50,61.21806,-149.90028 1/18/09 6:46,Product1,1200,Visa,Shona ,Mornington,Meath,Ireland,1/15/09 9:13,2/9/09 11:55,53.7233333,-6.2825 1/30/09 12:18,Product1,1200,Mastercard,Abikay,Fullerton ,CA,United States,1/26/09 13:34,2/9/09 12:53,33.87028,-117.92444 1/6/09 5:42,Product1,1200,Amex,Abikay,Atlanta ,GA,United States,10/27/08 14:16,2/9/09 13:50,33.74889,-84.38806 1/2/09 10:58,Product2,3600,Visa,Kendra,Toronto,Ontario,Canada,1/2/09 10:38,2/9/09 13:56,43.6666667,-79.4166667 1/8/09 3:29,Product1,1200,Visa,amanda,Liverpool,England,United Kingdom,12/22/08 1:41,2/9/09 14:06,53.4166667,-3 1/12/09 13:23,Product2,3600,Amex,Leila,Ponte San Nicolo,Veneto,Italy,9/13/05 8:42,2/9/09 14:09,45.3666667,11.6166667 1/19/09 9:34,Product1,1200,Amex,amanda,Las Vegas ,NV,United States,5/10/08 8:56,2/9/09 16:44,36.175,-115.13639 1/9/09 7:49,Product1,1200,Visa,Stacy,Rochester Hills ,MI,United States,7/28/08 7:18,2/9/09 17:41,42.68056,-83.13389 1/15/09 5:27,Product2,3600,Visa,Derrick,North Bay,Ontario,Canada,1/6/09 17:42,2/9/09 18:22,46.3,-79.45 1/8/09 23:40,Product1,1200,Visa,Jacob,Lindfield,New South Wales,Australia,1/8/09 17:52,2/9/09 18:31,-33.7833333,151.1666667 1/27/09 11:02,Product1,1200,Mastercard,DOREEN,Madrid,Madrid,Spain,1/24/09 8:21,2/9/09 18:42,40.4,-3.6833333 1/14/09 13:23,Product1,1200,Diners,eugenia,Wisconsin Rapids ,WI,United States,11/15/08 13:57,2/9/09 18:44,44.38361,-89.81722 1/7/09 20:01,Product1,1200,Visa,Karen,Austin ,TX,United States,1/6/09 19:16,2/9/09 19:56,30.26694,-97.74278 1/20/09 12:32,Product1,1200,Visa,Bea,Chicago ,IL,United States,1/16/09 19:08,2/9/09 20:42,41.85,-87.65 1/6/09 14:35,Product1,1200,Diners,Hilde Karin,Las Vegas ,NV,United States,12/17/08 11:59,2/9/09 22:59,36.175,-115.13639 1/4/09 6:51,Product1,1200,Visa,Rima,Mullingar,Westmeath,Ireland,1/3/09 12:34,2/10/09 0:59,53.5333333,-7.35 1/24/09 18:30,Product1,1200,Visa,Ruangrote,Melbourne,Victoria,Australia,7/17/08 5:19,2/10/09 2:12,-37.8166667,144.9666667 1/25/09 5:57,Product1,1200,Amex,pamela,Ayacucho,Buenos Aires,Argentina,1/24/09 9:29,2/10/09 6:38,-37.15,-58.4833333 1/5/09 10:02,Product2,3600,Visa,Emillie,Eagan ,MN,United States,1/5/09 9:03,2/10/09 7:29,44.80417,-93.16667 1/13/09 9:14,Product1,1200,Visa,sangeeta,Vossevangen,Hordaland,Norway,1/9/09 9:31,2/10/09 9:04,60.6333333,6.4333333 1/22/09 7:35,Product1,1200,Visa,Anja,Ferney-Voltaire,Rhone-Alpes,France,1/22/09 6:51,2/10/09 9:18,46.25,6.1166667 1/2/09 11:06,Product1,1200,Mastercard,Andrew,Sevilla,Andalucia,Spain,3/12/06 15:02,2/10/09 10:04,37.3772222,-5.9869444 1/11/09 9:50,Product1,1200,Visa,Bato,Munchengosserstadt,Thuringia,Germany,1/7/09 11:45,2/10/09 10:28,51.05,11.65 1/21/09 20:44,Product1,1200,Mastercard,Ailsa ,Lindenhurst ,NY,United States,1/21/09 7:47,2/10/09 10:51,40.68667,-73.37389 1/5/09 9:09,Product1,1200,Visa,Sophie,Bloomfield ,MI,United States,10/23/06 6:52,2/10/09 10:58,42.53778,-83.23306 1/5/09 12:41,Product1,1200,Visa,Katrin,Calgary,Alberta,Canada,12/3/08 14:49,2/10/09 11:45,51.0833333,-114.0833333 1/28/09 12:54,Product2,3600,Mastercard,Kelly ,Vancouver,British Columbia,Canada,1/27/09 21:04,2/10/09 12:09,49.25,-123.1333333 1/21/09 4:46,Product1,1200,Visa,Tomasz,Klampenborg,Kobenhavn,Denmark,6/10/08 11:25,2/10/09 12:22,55.7666667,12.6 1/7/09 13:28,Product1,1200,Visa,Elizabeth,Calne,England,United Kingdom,1/4/09 13:07,2/10/09 12:39,51.4333333,-2 1/27/09 11:18,Product2,3600,Amex,Michael,Los Angeles ,CA,United States,1/23/09 11:47,2/10/09 13:09,34.05222,-118.24278 1/7/09 12:39,Product2,3600,Visa,Natasha,Milano,Lombardy,Italy,6/2/06 13:01,2/10/09 13:19,45.4666667,9.2 1/24/09 13:54,Product2,3600,Mastercard,Meredith,Kloten,Zurich,Switzerland,1/24/09 12:30,2/10/09 13:47,47.45,8.5833333 1/30/09 6:48,Product1,1200,Mastercard,Nicole,Fayetteville ,NC,United States,1/30/09 4:51,2/10/09 14:41,35.0525,-78.87861 1/22/09 18:07,Product1,1200,Visa,Ryan,Simpsonville ,SC,United States,1/6/09 16:59,2/10/09 15:30,34.73694,-82.25444 1/29/09 15:03,Product1,1200,Visa,Mary ,Auckland,Auckland,New Zealand,2/9/06 11:14,2/10/09 16:31,-36.8666667,174.7666667 1/2/09 14:14,Product1,1200,Diners,Aaron,Reading,England,United Kingdom,11/16/08 15:49,2/10/09 16:38,51.4333333,-1 1/19/09 11:05,Product1,1200,Visa,Bertrand,North Caldwell ,NJ,United States,10/3/08 5:55,2/10/09 18:16,40.83972,-74.27694 \ No newline at end of file diff --git a/test/api/test_tools_upload.py b/test/api/test_tools_upload.py index a05fe9dce49f..cb67b84507dd 100644 --- a/test/api/test_tools_upload.py +++ b/test/api/test_tools_upload.py @@ -1,3 +1,5 @@ +import json + from base import api from base.constants import ( ONE_TO_SIX_ON_WINDOWS, @@ -33,19 +35,28 @@ def test_upload1_paste_bad_datatype(self): self._assert_has_keys(create, 'err_msg') assert file_type in create['err_msg'] - def test_upload_posix_newline_fixes(self): + # upload1 rewrites content with posix lines by default but this can be disabled by setting + # to_posix_lines=None in the request. Newer fetch API does not do this by default prefering + # to keep content unaltered if possible but it can be enabled with a simple JSON boolean switch + # of the same name (to_posix_lines). + def test_upload_posix_newline_fixes_by_default(self): windows_content = ONE_TO_SIX_ON_WINDOWS result_content = self._upload_and_get_content(windows_content) self.assertEquals(result_content, ONE_TO_SIX_WITH_TABS) + def test_fetch_posix_unaltered(self): + windows_content = ONE_TO_SIX_ON_WINDOWS + result_content = self._upload_and_get_content(windows_content, api="fetch") + self.assertEquals(result_content, ONE_TO_SIX_ON_WINDOWS) + def test_upload_disable_posix_fix(self): windows_content = ONE_TO_SIX_ON_WINDOWS result_content = self._upload_and_get_content(windows_content, to_posix_lines=None) self.assertEquals(result_content, windows_content) - def test_upload_tab_to_space(self): - table = ONE_TO_SIX_WITH_SPACES - result_content = self._upload_and_get_content(table, space_to_tab="Yes") + def test_fetch_post_lines_option(self): + windows_content = ONE_TO_SIX_ON_WINDOWS + result_content = self._upload_and_get_content(windows_content, api="fetch", to_posix_lines=True) self.assertEquals(result_content, ONE_TO_SIX_WITH_TABS) def test_upload_tab_to_space_off_by_default(self): @@ -53,6 +64,21 @@ def test_upload_tab_to_space_off_by_default(self): result_content = self._upload_and_get_content(table) self.assertEquals(result_content, table) + def test_fetch_tab_to_space_off_by_default(self): + table = ONE_TO_SIX_WITH_SPACES + result_content = self._upload_and_get_content(table, api='fetch') + self.assertEquals(result_content, table) + + def test_upload_tab_to_space(self): + table = ONE_TO_SIX_WITH_SPACES + result_content = self._upload_and_get_content(table, space_to_tab="Yes") + self.assertEquals(result_content, ONE_TO_SIX_WITH_TABS) + + def test_fetch_tab_to_space(self): + table = ONE_TO_SIX_WITH_SPACES + result_content = self._upload_and_get_content(table, api="fetch", space_to_tab=True) + self.assertEquals(result_content, ONE_TO_SIX_WITH_TABS) + @skip_without_datatype("rdata") def test_rdata_not_decompressed(self): # Prevent regression of https://github.com/galaxyproject/galaxy/issues/753 @@ -60,6 +86,30 @@ def test_rdata_not_decompressed(self): rdata_metadata = self._upload_and_get_details(open(rdata_path, "rb"), file_type="auto") self.assertEquals(rdata_metadata["file_ext"], "rdata") + @skip_without_datatype("csv") + def test_csv_upload(self): + csv_path = TestDataResolver().get_filename("1.csv") + csv_metadata = self._upload_and_get_details(open(csv_path, "rb"), file_type="csv") + self.assertEquals(csv_metadata["file_ext"], "csv") + + @skip_without_datatype("csv") + def test_csv_upload_auto(self): + csv_path = TestDataResolver().get_filename("1.csv") + csv_metadata = self._upload_and_get_details(open(csv_path, "rb"), file_type="auto") + self.assertEquals(csv_metadata["file_ext"], "csv") + + @skip_without_datatype("csv") + def test_csv_fetch(self): + csv_path = TestDataResolver().get_filename("1.csv") + csv_metadata = self._upload_and_get_details(open(csv_path, "rb"), api="fetch", ext="csv", to_posix_lines=True) + self.assertEquals(csv_metadata["file_ext"], "csv") + + @skip_without_datatype("csv") + def test_csv_sniff_fetch(self): + csv_path = TestDataResolver().get_filename("1.csv") + csv_metadata = self._upload_and_get_details(open(csv_path, "rb"), api="fetch", ext="auto", to_posix_lines=True) + self.assertEquals(csv_metadata["file_ext"], "csv") + @skip_without_datatype("velvet") def test_composite_datatype(self): with self.dataset_populator.test_history() as history_id: @@ -113,6 +163,11 @@ def test_upload_dbkey(self): datasets = run_response.json()["outputs"] assert datasets[0].get("genome_build") == "hg19", datasets[0] + def test_fetch_dbkey(self): + table = ONE_TO_SIX_WITH_SPACES + details = self._upload_and_get_details(table, api='fetch', dbkey="hg19") + assert details.get("genome_build") == "hg19" + def test_upload_multiple_files_1(self): with self.dataset_populator.test_history() as history_id: payload = self.dataset_populator.upload_payload(history_id, "Test123", @@ -329,8 +384,23 @@ def _upload_and_get_details(self, content, **upload_kwds): history_id, new_dataset = self._upload(content, **upload_kwds) return self.dataset_populator.get_history_dataset_details(history_id, dataset=new_dataset) - def _upload(self, content, **upload_kwds): + def _upload(self, content, api="upload1", **upload_kwds): history_id = self.dataset_populator.new_history() - new_dataset = self.dataset_populator.new_dataset(history_id, content=content, **upload_kwds) + if api == "upload1": + new_dataset = self.dataset_populator.new_dataset(history_id, content=content, **upload_kwds) + else: + assert api == "fetch" + element = dict(src="files", **upload_kwds) + target = { + "destination": {"type": "hdas"}, + "elements": [element], + } + targets = json.dumps([target]) + payload = { + "history_id": history_id, + "targets": targets, + "__files": {"files_0|file_data": content} + } + new_dataset = self.dataset_populator.fetch(payload).json()["outputs"][0] self.dataset_populator.wait_for_history(history_id, assert_ok=upload_kwds.get("assert_ok", True)) return history_id, new_dataset diff --git a/test/functional/tools/sample_datatypes_conf.xml b/test/functional/tools/sample_datatypes_conf.xml index fc08309fd8da..29917074a99c 100644 --- a/test/functional/tools/sample_datatypes_conf.xml +++ b/test/functional/tools/sample_datatypes_conf.xml @@ -4,6 +4,7 @@ +