PXP-1089 add test + fix acl/authz/size query (#251)

indexd/index/blueprint.py

@@ -169,7 +169,9 @@ def get_urls():
     """
     Returns a list of urls.
     """
-    ids = flask.request.args.getlist("ids")
+    ids = flask.request.args.get("ids")
+    if ids:
+        ids = ids.split(",")
     hashes = flask.request.args.getlist("hash")
     hashes = {h: v for h, v in (x.split(":", 1) for x in hashes)}
     size = flask.request.args.get("size")

indexd/index/drivers/alchemy.py

@@ -370,22 +370,31 @@ def ids(
             if uploader is not None:
                 query = query.filter(IndexRecord.uploader == uploader)
 
+            # filter records that have ALL the URLs
             if urls:
-                query = query.join(IndexRecord.urls)
                 for u in urls:
-                    query = query.filter(IndexRecordUrl.url == u)
+                    sub = session.query(IndexRecordUrl.did).filter(
+                        IndexRecordUrl.url == u
+                    )
+                    query = query.filter(IndexRecord.did.in_(sub.subquery()))
 
+            # filter records that have ALL the ACL elements
             if acl:
-                query = query.join(IndexRecord.acl)
                 for u in acl:
-                    query = query.filter(IndexRecordACE.ace == u)
+                    sub = session.query(IndexRecordACE.did).filter(
+                        IndexRecordACE.ace == u
+                    )
+                    query = query.filter(IndexRecord.did.in_(sub.subquery()))
             elif acl == []:
                 query = query.filter(IndexRecord.acl == None)
 
+            # filter records that have ALL the authz elements
             if authz:
-                query = query.join(IndexRecord.authz)
                 for u in authz:
-                    query = query.filter(IndexRecordAuthz.resource == u)
+                    sub = session.query(IndexRecordAuthz.did).filter(
+                        IndexRecordAuthz.resource == u
+                    )
+                    query = query.filter(IndexRecord.did.in_(sub.subquery()))
             elif authz == []:
                 query = query.filter(IndexRecord.authz == None)
 
@@ -557,7 +566,7 @@ def get_urls(self, size=None, hashes=None, ids=None, start=0, limit=100):
         """
         Returns a list of urls matching supplied size/hashes/dids.
         """
-        if not (size or hashes or ids):
+        if size is None and hashes is None and ids is None:
             raise UserError("Please provide size/hashes/ids to filter")
 
         with self.session as session:

openapis/swagger.yaml

@@ -87,7 +87,7 @@ paths:
         - name: hash
           in: query
           type: string
-          description: hashes specified as algorithm:value
+          description: hashes specified as algorithm:value. Multiple hashes can be specified, for example "?hash=a:xxx&hash=b:yyy"
         - name: ids
           in: query
           type: string
@@ -143,7 +143,7 @@ paths:
           type: string
         - name: metadata
           in: query
-          description: metadata in format key:value
+          description: metadata in format key:value. Multiple metadata values can be specified, for example "?metadata=a:xxx&metadata=b:yyy"
           required: false
           type: string
         - name: size
@@ -153,7 +153,7 @@ paths:
           type: integer
         - name: hash
           in: query
-          description: hash in format hash_type:hash_value
+          description: hash in format hash_type:hash_value. Multiple hashes can be specified, for example "?hash=a:xxx&hash=b:yyy"
           required: false
           type: string
         - name: uploader
@@ -168,19 +168,19 @@ paths:
             pagination params(start, limit) are not supported
           required: false
           type: string
-        - name: urls
+        - name: url
           in: query
-          description: comma delimited urls
+          description: URL to query. Multiple URLs can be specified, for example "?url=url1&url=url2" - in that case, returned records will have ALL URLs
           required: false
           type: string
         - name: acl
           in: query
-          description: comma delimited ace
+          description: comma delimited ACE - if multiple ACE are specified, returned records will have ALL ACEs
           required: false
           type: string
         - name: authz
           in: query
-          description: comma delimited resoures
+          description: comma delimited resources - if multiple resources are specified, returned records will have ALL resources
           required: false
           type: string
         - name: negate_params

tests/test_client.py

@@ -153,6 +153,81 @@ def test_index_list_by_authz(client, user):
     assert rec["records"][0]["authz"] == data["authz"]
 
 
+def test_index_list_by_multiple_authz(client, user):
+    data = get_doc()
+
+    data["authz"] = ["abc"]
+    res = client.post("/index/", json=data, headers=user)
+    assert res.status_code == 200
+
+    data["authz"] = ["abc", "rst", "xyz"]
+    res = client.post("/index/", json=data, headers=user)
+    assert res.status_code == 200
+
+    # query the record with multiple authz elements
+    res = client.get("/index/?authz=" + ",".join(data["authz"]))
+    assert res.status_code == 200
+    rec = res.json
+    assert len(rec["records"]) == 1, "Got records: {}".format(
+        json.dumps(rec["records"], indent=2)
+    )
+    assert rec["records"][0]["authz"] == data["authz"]
+
+
+def test_index_list_by_multiple_acl(client, user):
+    data = get_doc()
+
+    data["acl"] = ["abc"]
+    res = client.post("/index/", json=data, headers=user)
+    assert res.status_code == 200
+
+    data["acl"] = ["abc", "rst", "xyz"]
+    res = client.post("/index/", json=data, headers=user)
+    assert res.status_code == 200
+
+    # query the record with multiple ACL elements
+    res = client.get("/index/?acl=" + ",".join(data["acl"]))
+    assert res.status_code == 200
+    rec = res.json
+    assert len(rec["records"]) == 1, "Got records: {}".format(
+        json.dumps(rec["records"], indent=2)
+    )
+    assert rec["records"][0]["acl"] == data["acl"]
+
+
+def test_index_list_by_urls(client, user):
+    data = get_doc()
+
+    data["urls"] = ["s3://bucket1"]
+    res = client.post("/index/", json=data, headers=user)
+    assert res.status_code == 200
+
+    data["urls"] = ["s3://bucket2"]
+    res = client.post("/index/", json=data, headers=user)
+    assert res.status_code == 200
+
+    data["urls"] = ["s3://bucket2", "s3://bucket3"]
+    res = client.post("/index/", json=data, headers=user)
+    assert res.status_code == 200
+
+    # query the record with a single URL
+    res = client.get("/index/?url=s3://bucket2")
+    assert res.status_code == 200
+    rec = res.json
+    assert len(rec["records"]) == 2, "Got records: {}".format(
+        json.dumps(rec["records"], indent=2)
+    )
+
+    # query the record with multiple URLs
+    res = client.get("/index/?url=s3://bucket2&url=s3://bucket3")
+    assert res.status_code == 200
+    rec = res.json
+    assert len(rec["records"]) == 1, "Got records: {}".format(
+        json.dumps(rec["records"], indent=2)
+    )
+    assert rec["records"][0]["urls"] == data["urls"]
+
+
 def test_index_list_by_version(client, user):
     # post three records of different version
     data = get_doc()
@@ -801,6 +876,21 @@ def test_get_urls(client, user):
     assert record["urls"][0]["metadata"] == data["urls_metadata"][url]
 
 
+def test_get_urls_size_0(client, user):
+    data = get_doc(has_urls_metadata=True)
+    data["size"] = 0
+    response = client.post("/index/", json=data, headers=user)
+    assert response.status_code == 200
+    record = response.json
+
+    response = client.get("/urls/?size={}".format(data["size"]))
+    assert response.status_code == 200
+    record = response.json
+    url = data["urls"][0]
+    assert record["urls"][0]["url"] == url
+    assert record["urls"][0]["metadata"] == data["urls_metadata"][url]
+
+
 def test_index_create(client, user):
     data = get_doc(has_baseid=True)