Warn on DAL overflow (#329)

* Warn on DAL overflow * Re-generated the test data with a different version of `VOTable`
astropy · May 27, 2022 · 82ed24d · 82ed24d
1 parent 949693c
commit 82ed24d
Show file tree

Hide file tree

Showing 19 changed files with 101 additions and 28 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,6 +1,10 @@
 1.4 (unreleased)
 ================
 
+1.3.1 (unreleased)
+==================
+
+- pyvo.dal warns on results with overflow status. [#329]`
 
 1.3 (2022-02-19)
 ==================

diff --git a/pyvo/dal/__init__.py b/pyvo/dal/__init__.py
@@ -6,7 +6,7 @@
 from .scs import search as conesearch
 from .tap import search as tablesearch
 
-from .query import DALService, DALQuery, DALResults, Record
+from .query import DALService, DALQuery, DALResults, DALOverflowWarning, Record
 
 from .sia import SIAService, SIAQuery, SIAResults, SIARecord
 from .ssa import SSAService, SSAQuery, SSAResults, SSARecord
@@ -17,7 +17,7 @@
 
 from .exceptions import (
     DALAccessError, DALProtocolError, DALFormatError, DALServiceError,
-    DALQueryError)
+    DALQueryError, DALOverflowWarning)
 
 __all__ = [
     "imagesearch", "spectrumsearch", "linesearch", "conesearch", "tablesearch",
@@ -30,4 +30,4 @@
     "SIARecord", "SSARecord", "SLARecord", "SCSRecord",
     "AsyncTAPJob",
     "DALAccessError", "DALProtocolError", "DALFormatError", "DALServiceError",
-    "DALQueryError"]
+    "DALQueryError", "DALOverflowWarning"]
diff --git a/pyvo/dal/exceptions.py b/pyvo/dal/exceptions.py
@@ -4,7 +4,7 @@
 
 __all__ = [
     "DALAccessError", "DALProtocolError", "DALFormatError", "DALServiceError",
-    "DALQueryError"]
+    "DALQueryError", "DALOverflowWarning"]
 
 import re
 
@@ -232,3 +232,7 @@ def label(self):
 
 class PyvoUserWarning(AstropyUserWarning):
     pass
+
+
+class DALOverflowWarning(UserWarning):
+    pass
diff --git a/pyvo/dal/query.py b/pyvo/dal/query.py
@@ -40,7 +40,8 @@
 from astropy.utils.exceptions import AstropyDeprecationWarning
 
 from .mimetype import mime_object_maker
-from .exceptions import (DALFormatError, DALServiceError, DALQueryError)
+from .exceptions import (DALFormatError, DALServiceError, DALQueryError,
+                         DALOverflowWarning)
 
 from .. import samp
 
@@ -319,6 +320,10 @@ def __init__(self, votable, url=None, session=None):
         if self._status[0].lower() not in ("ok", "overflow"):
             raise DALQueryError(self._status[1], self._status[0], url)
 
+        if self._status[0].lower() == "overflow":
+            warn("Partial result set. Potential causes MAXREC, async storage space, etc.",
+                 category=DALOverflowWarning);
+
         self._resultstable = self._findresultstable(votable)
         if not self._resultstable:
             raise DALFormatError(
@@ -376,9 +381,13 @@ def _findstatus(self, votable):
 
     def _findstatusinfo(self, infos):
         # this can be overridden to specialize for a particular DAL protocol
+        status = None
+        # return the last status to catch potential overflow or error sent
+        # after the results.
         for info in infos:
             if info.name.lower() == 'query_status':
-                return info
+                status = info
+        return status
 
     def _findinfos(self, votable):
         # this can be overridden to specialize for a particular DAL protocol

diff --git a/pyvo/dal/tests/data/datalink/datalink-obscore.xml b/pyvo/dal/tests/data/datalink/datalink-obscore.xml
@@ -220,7 +220,6 @@
         </TABLEDATA>
       </DATA>
     </TABLE>
-    <INFO name="QUERY_STATUS" value="OVERFLOW" />
   </RESOURCE>
   <RESOURCE type="meta" utype="adhoc:service">
     <PARAM name="resourceIdentifier" datatype="char" arraysize="*" value="ivo://test.com/caom2ops" />

diff --git a/pyvo/dal/tests/data/datalink/datalink-ssa.xml b/pyvo/dal/tests/data/datalink/datalink-ssa.xml
@@ -10,7 +10,6 @@ the name of the instrument after Flash got a second channel in 1995.</DESCRIPTIO
     <INFO name="citation" ucd="meta.bib.bibcode" value="1996A&amp;A...312..539S">This resource contains data associated with the publication 1996A&amp;A...312..539S.</INFO>
     <INFO name="citation" ucd="meta.bib" value="http://dc.zah.uni-heidelberg.de/tableinfo/flashheros.data">For advice on how to cite the resource(s) that contributed to this result, see http://dc.zah.uni-heidelberg.de/tableinfo/flashheros.data</INFO>
     <INFO name="SERVICE_PROTOCOL" value="1.04">SSAP</INFO>
-    <INFO name="QUERY_STATUS" value="OVERFLOW">Exactly 30 rows were returned.  This means your query probably reached the match limit.  Increase MAXREC.</INFO>
     <INFO name="standardID" value="ivo://ivoa.net/std/ssap"/>
     <COOSYS ID="system" system="ICRS"/>
     <COOSYS ID="system0" system="ICRS"/>

diff --git a/pyvo/dal/tests/data/query/basic.xml b/pyvo/dal/tests/data/query/basic.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!-- Produced with astropy.io.votable version 3.1.dev21501
+<!-- Produced with astropy.io.votable version 5.0.1
      http://www.astropy.org/ -->
-<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
  <RESOURCE type="results">
   <INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
   <TABLE>

diff --git a/pyvo/dal/tests/data/query/dataset.xml b/pyvo/dal/tests/data/query/dataset.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!-- Produced with astropy.io.votable version 3.1.dev21501
+<!-- Produced with astropy.io.votable version 5.0.1
      http://www.astropy.org/ -->
-<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
  <RESOURCE type="results">
   <INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
   <TABLE>

diff --git a/pyvo/dal/tests/data/query/errorstatus.xml b/pyvo/dal/tests/data/query/errorstatus.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!-- Produced with astropy.io.votable version 3.1.dev21501
+<!-- Produced with astropy.io.votable version 5.0.1
      http://www.astropy.org/ -->
-<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
  <RESOURCE type="results">
   <INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="ERROR">ERROR</INFO>
   <TABLE>

diff --git a/pyvo/dal/tests/data/query/firstresource.xml b/pyvo/dal/tests/data/query/firstresource.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!-- Produced with astropy.io.votable version 3.1.dev21501
+<!-- Produced with astropy.io.votable version 5.0.1
      http://www.astropy.org/ -->
-<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
  <RESOURCE type="results">
   <INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
   <TABLE>

diff --git a/pyvo/dal/tests/data/query/missingcolumns.xml b/pyvo/dal/tests/data/query/missingcolumns.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!-- Produced with astropy.io.votable version 3.1.dev21501
+<!-- Produced with astropy.io.votable version 5.0.1
      http://www.astropy.org/ -->
-<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
  <RESOURCE type="results">
   <INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
   <TABLE>

diff --git a/pyvo/dal/tests/data/query/missingresource.xml b/pyvo/dal/tests/data/query/missingresource.xml
@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!-- Produced with astropy.io.votable version 3.1.dev21501
+<!-- Produced with astropy.io.votable version 5.0.1
      http://www.astropy.org/ -->
-<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3"/>
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd"/>
diff --git a/pyvo/dal/tests/data/query/missingtable.xml b/pyvo/dal/tests/data/query/missingtable.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!-- Produced with astropy.io.votable version 3.1.dev21501
+<!-- Produced with astropy.io.votable version 5.0.1
      http://www.astropy.org/ -->
-<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
  <RESOURCE type="results">
   <INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
  </RESOURCE>

diff --git a/pyvo/dal/tests/data/query/overflowstatus.xml b/pyvo/dal/tests/data/query/overflowstatus.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Produced with astropy.io.votable version 5.0.1
+     http://www.astropy.org/ -->
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
+ <RESOURCE type="results">
+  <INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK"/>
+  <TABLE>
+   <FIELD ID="_1" datatype="long" name="1" ucd="foo;bar"/>
+   <FIELD ID="_2" arraysize="37" datatype="char" name="2" utype="foobar"/>
+   <DATA>
+    <TABLEDATA>
+     <TR>
+      <TD>23</TD>
+      <TD>Illuminatus</TD>
+     </TR>
+     <TR>
+      <TD>42</TD>
+      <TD>Don't panic, and always carry a towel</TD>
+     </TR>
+     <TR>
+      <TD>1337</TD>
+      <TD>Elite</TD>
+     </TR>
+    </TABLEDATA>
+   </DATA>
+  </TABLE>
+  <INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OVERFLOW"/>
+ </RESOURCE>
+</VOTABLE>
diff --git a/pyvo/dal/tests/data/query/rootinfo.xml b/pyvo/dal/tests/data/query/rootinfo.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!-- Produced with astropy.io.votable version 3.1.dev21501
+<!-- Produced with astropy.io.votable version 5.0.1
      http://www.astropy.org/ -->
-<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
  <INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
  <RESOURCE type="results">
   <TABLE>

diff --git a/pyvo/dal/tests/data/query/tableinfo.xml b/pyvo/dal/tests/data/query/tableinfo.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!-- Produced with astropy.io.votable version 3.1.dev21501
+<!-- Produced with astropy.io.votable version 5.0.1
      http://www.astropy.org/ -->
-<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
  <RESOURCE type="results">
   <TABLE>
    <FIELD ID="_1" datatype="long" name="1" ucd="foo;bar"/>

diff --git a/pyvo/dal/tests/data/querydata/image.fits b/pyvo/dal/tests/data/querydata/image.fits
diff --git a/pyvo/dal/tests/make_testdata.py b/pyvo/dal/tests/make_testdata.py
@@ -41,6 +41,17 @@ def votablefile_errorstatus():
     return votablefile
 
 
+def votablefile_overflowstatus():
+    votablefile = _votablefile()
+
+    info_ok = Info(name='QUERY_STATUS', value='OK')
+    info_overflow = Info(name='QUERY_STATUS', value='OVERFLOW')
+    votablefile.resources[0].infos[0] = info_ok
+    votablefile.resources[0].infos.append(info_overflow)
+
+    return votablefile
+
+
 def votablefile_missingtable():
     votablefile = _votablefile()
     del votablefile.resources[0].tables[0]
@@ -61,7 +72,7 @@ def votablefile_missingcolumns():
 
 def votablefile_firstresource():
     votablefile = _votablefile()
-    votablefile.resources[0]._type = ''
+    votablefile.resources[0]._type = 'results'
     return votablefile
 
 
@@ -126,6 +137,9 @@ def main():
     votablefile_errorstatus().to_xml(
         str(dirname / 'query/errorstatus.xml'), tabledata_format='tabledata')
 
+    votablefile_overflowstatus().to_xml(
+        str(dirname / 'query/overflowstatus.xml'), tabledata_format='tabledata')
+
     votablefile_missingtable().to_xml(
         str(dirname / 'query/missingtable.xml'), tabledata_format='tabledata')
 

diff --git a/pyvo/dal/tests/test_query.py b/pyvo/dal/tests/test_query.py
@@ -17,7 +17,8 @@
 import numpy as np
 
 from pyvo.dal.query import DALService, DALQuery, DALResults, Record
-from pyvo.dal.exceptions import DALServiceError, DALQueryError, DALFormatError
+from pyvo.dal.exceptions import DALServiceError, DALQueryError,\
+    DALFormatError, DALOverflowWarning
 from pyvo.version import version
 from pyvo.utils.compat import ASTROPY_LT_4_1
 
@@ -91,6 +92,10 @@ def register_mocks(mocker):
                 'GET', 'http://example.com/query/errorstatus',
                 content=get_pkg_data_contents('data/query/errorstatus.xml')
             )),
+            stack.enter_context(mocker.register_uri(
+                'GET', 'http://example.com/query/overflowstatus',
+                content=get_pkg_data_contents('data/query/overflowstatus.xml')
+            )),
         ]
 
         def verbosetest_callback(request, context):
@@ -203,6 +208,12 @@ def test_query_exception(self):
         with pytest.raises(DALQueryError):
             service.search()
 
+    def test_query_warning(self):
+        service = DALService('http://example.com/query/overflowstatus')
+
+        with pytest.warns(DALOverflowWarning):
+            service.search()
+
     @pytest.mark.filterwarnings("ignore::astropy.io.votable.exceptions.W53")
     def test_format_exception(self):
         with pytest.raises(DALFormatError):
@@ -283,6 +294,10 @@ def test_init_errorstatus(self):
         with pytest.raises(DALQueryError):
             DALResults.from_result_url('http://example.com/query/errorstatus')
 
+    def test_init_overflowstatus(self):
+        with pytest.warns(DALOverflowWarning):
+            DALResults.from_result_url('http://example.com/query/overflowstatus')
+
     def test_init_missingtable(self):
         with pytest.raises(DALFormatError):
             DALResults.from_result_url('http://example.com/query/missingtable')