Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Unicode and Windows support #20

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Add Unicode and Windows support #20

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6272927
Add unicode character support for levenshtein and osa
LEFTazs Feb 3, 2020
9d64fb8
Add Windows and unicode support for damerau
LEFTazs Feb 3, 2020
58a00ef
Remove length arguments from the C methods
LEFTazs Feb 4, 2020
530042b
Fix negative indexing error
LEFTazs Feb 5, 2020
c0d9b33
Add legacy code for backwards compatibility
LEFTazs Feb 6, 2020
454af44
Refractor legacy code
LEFTazs Feb 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions weighted_levenshtein/clev.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,7 @@ cdef enum:

cdef DTYPE_t c_damerau_levenshtein(
int[:] str_a,
LEFTazs marked this conversation as resolved.
Show resolved Hide resolved
Py_ssize_t len_a,
int[:] str_b,
Py_ssize_t len_b,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs,
Expand All @@ -19,9 +17,7 @@ cdef DTYPE_t c_damerau_levenshtein(

cdef DTYPE_t c_optimal_string_alignment(
int[:] word_m,
Py_ssize_t m,
int[:] word_n,
Py_ssize_t n,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs,
Expand All @@ -30,9 +26,7 @@ cdef DTYPE_t c_optimal_string_alignment(

cdef DTYPE_t c_levenshtein(
int[:] word_m,
Py_ssize_t m,
int[:] word_n,
Py_ssize_t n,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs) nogil
Expand Down
31 changes: 18 additions & 13 deletions weighted_levenshtein/clev.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,8 +200,7 @@ def damerau_levenshtein(
intarr2 = convert_string_to_int_array(str2)

return c_damerau_levenshtein(
intarr1, intarr1.size,
intarr2, intarr2.size,
intarr1, intarr2,
insert_costs,
delete_costs,
substitute_costs,
Expand All @@ -212,8 +211,7 @@ dam_lev = damerau_levenshtein


cdef DTYPE_t c_damerau_levenshtein(
int[:] str1, Py_ssize_t len1,
int[:] str2, Py_ssize_t len2,
int[:] str1, int[:] str2,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs,
Expand All @@ -228,8 +226,11 @@ cdef DTYPE_t c_damerau_levenshtein(
unsigned int char_i, char_j
DTYPE_t cost, ret_val
Py_ssize_t db, k, l

Array2D d
Py_ssize_t len1, len2

len1 = str1.shape[0]
LEFTazs marked this conversation as resolved.
Show resolved Hide resolved
len2 = str2.shape[0]

Array2D_init(&d, len1 + 2, len2 + 2)

Expand Down Expand Up @@ -327,8 +328,7 @@ def optimal_string_alignment(
intarr2 = convert_string_to_int_array(str2)

return c_optimal_string_alignment(
intarr1, intarr1.size,
intarr2, intarr2.size,
intarr1, intarr2,
insert_costs,
delete_costs,
substitute_costs,
Expand All @@ -339,8 +339,7 @@ osa = optimal_string_alignment


cdef DTYPE_t c_optimal_string_alignment(
int[:] str1, Py_ssize_t len1,
int[:] str2, Py_ssize_t len2,
int[:] str1, int[:] str2,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs,
Expand All @@ -353,6 +352,10 @@ cdef DTYPE_t c_optimal_string_alignment(
unsigned int char_i, char_j, prev_char_i, prev_char_j
DTYPE_t ret_val
Array2D d
Py_ssize_t len1, len2

len1 = str1.shape[0]
len2 = str2.shape[0]

Array2D_init(&d, len1 + 1, len2 + 1)

Expand Down Expand Up @@ -428,8 +431,7 @@ def levenshtein(
intarr2 = convert_string_to_int_array(str2)

return c_levenshtein(
intarr1, intarr1.size,
intarr2, intarr2.size,
intarr1, intarr2,
insert_costs,
delete_costs,
substitute_costs
Expand All @@ -439,8 +441,7 @@ lev = levenshtein


cdef DTYPE_t c_levenshtein(
int[:] str1, Py_ssize_t len1,
int[:] str2, Py_ssize_t len2,
int[:] str1, int[:] str2,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs) nogil:
Expand All @@ -452,6 +453,10 @@ cdef DTYPE_t c_levenshtein(
unsigned int char_i, char_j
DTYPE_t ret_val
Array2D d
Py_ssize_t len1, len2

len1 = str1.shape[0]
len2 = str2.shape[0]

Array2D_init(&d, len1 + 1, len2 + 1)

Expand Down