Skip to content

Fix NOCONTENT + PAYLOAD in search #27

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Fix NOCONTENT + PAYLOAD in search #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e056887
Fix NOCONTENT + PAYLOAD in search
TvoroG Jul 17, 2018
8490c4b
Merge branch 'master' into master
gkorland Mar 13, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 15 additions & 15 deletions redisearch/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class Field(object):
def __init__(self, name, *args):
self.name = name
self.args = args

def redis_args(self):
return [self.name] + list(self.args)

Expand Down Expand Up @@ -80,7 +80,7 @@ def __init__(self, name):

class TagField(Field):
"""
TagField is a tag-indexing field with simpler compression and tokenization.
TagField is a tag-indexing field with simpler compression and tokenization.
See http://redisearch.io/Tags/
"""
def __init__(self, name, separator = ',', no_index=False):
Expand All @@ -91,12 +91,12 @@ def __init__(self, name, separator = ',', no_index=False):

if no_index:
self.args.append(Field.NOINDEX)


class Client(object):
"""
A client for the RediSearch module.
It abstracts the API of the module and lets you just use the engine
A client for the RediSearch module.
It abstracts the API of the module and lets you just use the engine
"""

NUMERIC = 'NUMERIC'
Expand All @@ -117,8 +117,8 @@ class Client(object):

class BatchIndexer(object):
"""
A batch indexer allows you to automatically batch
document indexeing in pipelines, flushing it every N documents.
A batch indexer allows you to automatically batch
document indexeing in pipelines, flushing it every N documents.
"""

def __init__(self, client, chunk_size=1000):
Expand Down Expand Up @@ -193,7 +193,7 @@ def create_index(self, fields, no_term_offsets=False,
args += [self.STOPWORDS, len(stopwords)]
if len(stopwords) > 0:
args += list(stopwords)

args.append('SCHEMA')

args += list(itertools.chain(*(f.redis_args() for f in fields)))
Expand All @@ -205,11 +205,11 @@ def drop_index(self):
Drop the index if it exists
"""
return self.redis.execute_command(self.DROP_CMD, self.index_name)

def _add_document(self, doc_id, conn=None, nosave=False, score=1.0, payload=None,
replace=False, partial=False, language=None, **fields):
"""
Internal add_document used for both batch and single doc indexing
"""
Internal add_document used for both batch and single doc indexing
"""
if conn is None:
conn = self.redis
Expand Down Expand Up @@ -242,17 +242,17 @@ def add_document(self, doc_id, nosave=False, score=1.0, payload=None,

- **doc_id**: the id of the saved document.
- **nosave**: if set to true, we just index the document, and don't save a copy of it. This means that searches will just return ids.
- **score**: the document ranking, between 0.0 and 1.0
- **score**: the document ranking, between 0.0 and 1.0
- **payload**: optional inner-index payload we can save for fast access in scoring functions
- **replace**: if True, and the document already is in the index, we perform an update and reindex the document
- **partial**: if True, the fields specified will be added to the existing document.
This has the added benefit that any fields specified with `no_index`
will not be reindexed again. Implies `replace`
- **language**: Specify the language used for document tokenization.
- **fields** kwargs dictionary of the document fields to be saved and/or indexed.
- **fields** kwargs dictionary of the document fields to be saved and/or indexed.
NOTE: Geo points shoule be encoded as strings of "lon,lat"
"""
return self._add_document(doc_id, conn=None, nosave=nosave, score=score,
return self._add_document(doc_id, conn=None, nosave=nosave, score=score,
payload=payload, replace=replace,
partial=partial, language=language, **fields)

Expand Down Expand Up @@ -367,4 +367,4 @@ def aggregate(self, query):
rows = raw[1:]

res = AggregateResult(rows, cursor, schema)
return res
return res
23 changes: 14 additions & 9 deletions redisearch/result.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,28 @@ def __init__(self, res, hascontent, duration=0, has_payload = False):
self.docs = []

step = 1
if hascontent:
step = 3 if has_payload else 2
else:
# we can't have nocontent and payloads in the same response
has_payload = False
fields_offset = 0
if hascontent and has_payload:
step = 3
fields_offset = 2
elif hascontent and not has_payload:
step = 2
fields_offset = 1
elif not hascontent and has_payload:
step = 2
fields_offset = 0

for i in xrange(1, len(res), step):
id = to_string(res[i])
payload = to_string(res[i+1]) if has_payload else None
fields_offset = 2 if has_payload else 1

fields = {}
if hascontent:
fields = {}
if fields_offset > 0:
fields = dict(
dict(izip(map(to_string, res[i + fields_offset][::2]),
map(to_string, res[i + fields_offset][1::2])))
) if hascontent else {}
)

try:
del fields['id']
except KeyError:
Expand Down
81 changes: 52 additions & 29 deletions test/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ def createIndex(self, client, num_docs = 100):
#conn.flushdb()
#client = Client('test', port=conn.port)
try:
client.create_index((TextField('play', weight=5.0),
TextField('txt'),
client.create_index((TextField('play', weight=5.0),
TextField('txt'),
NumericField('chapter')))
except redis.ResponseError:
client.drop_index()
Expand Down Expand Up @@ -61,7 +61,7 @@ def createIndex(self, client, num_docs = 100):
def testClient(self):

conn = self.redis()

with conn as r:
num_docs = 500
r.flushdb()
Expand Down Expand Up @@ -95,19 +95,19 @@ def testClient(self):
self.assertTrue(doc.id)
self.assertEqual(doc.play, 'Henry IV')
self.assertTrue(len(doc.txt) > 0)

# test no content
res = client.search(Query('king').no_content())
self.assertEqual(194, res.total)
self.assertEqual(10, len(res.docs))
for doc in res.docs:
self.assertNotIn('txt', doc.__dict__)
self.assertNotIn('play', doc.__dict__)

#test verbatim vs no verbatim
total = client.search(Query('kings').no_content()).total
vtotal = client.search(Query('kings').no_content().verbatim()).total
self.assertGreater(total, vtotal)
self.assertGreater(total, vtotal)

# test in fields
txt_total = client.search(Query('henry').no_content().limit_fields('txt')).total
Expand All @@ -124,7 +124,7 @@ def testClient(self):
self.assertEqual(doc.play, 'Henry VI Part 3')
self.assertTrue(len(doc.txt) > 0)



# test inkeys
ids = [x.id for x in client.search(Query('henry')).docs]
Expand All @@ -145,12 +145,12 @@ def testClient(self):
self.assertEqual(52,client.search(Query('king henry').slop(0).in_order()).total)
self.assertEqual(53,client.search(Query('henry king').slop(0)).total)
self.assertEqual(167,client.search(Query('henry king').slop(100)).total)

# test delete document
client.add_document('doc-5ghs2', play = 'Death of a Salesman')
res = client.search(Query('death of a salesman'))
self.assertEqual(1, res.total)

self.assertEqual(1, client.delete_document('doc-5ghs2'))
res = client.search(Query('death of a salesman'))
self.assertEqual(0, res.total)
Expand All @@ -175,7 +175,7 @@ def getCleanClient(self, name):
return client

def testPayloads(self):

conn = self.redis()

with conn as r:
Expand All @@ -191,13 +191,36 @@ def testPayloads(self):
res = client.search(q)
self.assertEqual(2, res.total)
self.assertEqual('doc2', res.docs[0].id)

self.assertEqual('doc1', res.docs[1].id)
self.assertEqual('foo baz', res.docs[1].payload)
self.assertIsNone(res.docs[0].payload)

def testPayloadsWithNoContent(self):

conn = self.redis()

with conn as r:
# Creating a client with a given index name
client = Client('idx', port=conn.port)
client.redis.flushdb()
client.create_index((TextField('txt'),))

client.add_document('doc1', payload = 'foo baz', txt = 'foo bar')
client.add_document('doc2', payload = 'foo baz2', txt = 'foo bar')

q = Query("foo bar").with_payloads().no_content()
res = client.search(q)

self.assertEqual(2, len(res.docs))
self.assertEqual('doc2', res.docs[0].id)
self.assertEqual('foo baz2', res.docs[0].payload)

self.assertEqual('doc1', res.docs[1].id)
self.assertEqual('foo baz', res.docs[1].payload)

def testReplace(self):

conn = self.redis()

with conn as r:
Expand All @@ -212,19 +235,19 @@ def testReplace(self):
res = client.search("foo bar")
self.assertEqual(2, res.total)
client.add_document('doc1', replace = True, txt = 'this is a replaced doc')


res = client.search("foo bar")
self.assertEqual(1, res.total)
self.assertEqual('doc2', res.docs[0].id)


res = client.search("replaced doc")
self.assertEqual(1, res.total)
self.assertEqual('doc1', res.docs[0].id)


def testStopwords(self):
def testStopwords(self):
conn = self.redis()

with conn as r:
Expand All @@ -237,7 +260,7 @@ def testStopwords(self):
client.create_index((TextField('txt'),), stopwords = ['foo', 'bar', 'baz'])
client.add_document('doc1', txt = 'foo bar')
client.add_document('doc2', txt = 'hello world')

q1 = Query("foo bar").no_content()
q2 = Query("foo bar hello world").no_content()
res1, res2 = client.search(q1), client.search(q2)
Expand All @@ -252,7 +275,7 @@ def testFilters(self):
# Creating a client with a given index name
client = Client('idx', port=conn.port)
client.redis.flushdb()

client.create_index((TextField('txt'), NumericField('num'), GeoField('loc')))

client.add_document('doc1', txt = 'foo bar', num = 3.141, loc = '-0.441,51.458')
Expand All @@ -274,7 +297,7 @@ def testFilters(self):
q1 = Query("foo").add_filter(GeoFilter('loc', -0.44, 51.45, 10)).no_content()
q2 = Query("foo").add_filter(GeoFilter('loc', -0.44, 51.45, 100)).no_content()
res1, res2 = client.search(q1), client.search(q2)

self.assertEqual(1, res1.total)
self.assertEqual(2, res2.total)
self.assertEqual('doc1', res1.docs[0].id)
Expand All @@ -289,7 +312,7 @@ def testSortby(self):
# Creating a client with a given index name
client = Client('idx', port=conn.port)
client.redis.flushdb()

client.create_index((TextField('txt'), NumericField('num', sortable=True)))

client.add_document('doc1', txt = 'foo bar', num = 1)
Expand All @@ -300,7 +323,7 @@ def testSortby(self):
q1 = Query("foo").sort_by('num', asc=True).no_content()
q2 = Query("foo").sort_by('num', asc=False).no_content()
res1, res2 = client.search(q1), client.search(q2)

self.assertEqual(3, res1.total)
self.assertEqual('doc1', res1.docs[0].id)
self.assertEqual('doc2', res1.docs[1].id)
Expand All @@ -318,7 +341,7 @@ def testExample(self):
# Creating a client with a given index name
client = Client('myIndex', port=conn.port)
client.redis.flushdb()

# Creating the index definition and schema
client.create_index((TextField('title', weight=5.0), TextField('body')))

Expand All @@ -329,15 +352,15 @@ def testExample(self):
q = Query("search engine").verbatim().no_content().paging(0,5)

res = client.search(q)





self.assertTrue(True)

def testAutoComplete(self):
with self.redis() as r:
self.assertTrue(True)

ac = AutoCompleter('ac', conn=r)
n = 0
with open(TITLES_CSV) as f:
Expand All @@ -349,7 +372,7 @@ def testAutoComplete(self):
#print term, score
self.assertEqual(n,ac.add_suggestions(Suggestion(term,score=score)))


self.assertEqual(n, ac.len())
strs = []
for _ in r.retry_with_rdb_reload():
Expand All @@ -372,7 +395,7 @@ def testAutoComplete(self):
# make sure a second delete returns 0
for sug in strs:
self.assertEqual(0, ac.delete(sug))

# make sure they were actually deleted
ret2 = ac.get_suggestions('bad', fuzzy=True, num=10)
for sug in ret2:
Expand Down Expand Up @@ -473,11 +496,11 @@ def testTags(self):
# Creating a client with a given index name
client = Client('idx', port=conn.port)
client.redis.flushdb()

client.create_index((TextField('txt'), TagField('tags')))

client.add_document('doc1', txt = 'fooz barz', tags = 'foo,foo bar,hello;world')

for i in r.retry_with_rdb_reload():

q = Query("@tags:{foo}")
Expand Down