Update to use CharSetEncodings

Author: ScottPJones

src/Strs.jl

@@ -75,9 +75,17 @@ So: ASCIIStr would be: Valid, All ASCII, ... i.e. 0 + short/hash bits
 using CharSetEncodings
 
 # From CharSetEncodings
-export CSE, CharSet, Encoding, cse, charset, encoding, basetype, basecse, MaybeSub
+export CSE, CharSet, Encoding, cse, charset, encoding, basetype, basecse, MaybeSub, is_multi
 export @cs_str, @enc_str, @cse
 export BIG_ENDIAN, LITTLE_ENDIAN
+    
+import CharSetEncodings: EncodingStyle, CharSetStyle, CompareStyle
+
+# Convenience aliases
+const SingleCU = SingleCodeUnitEncoding
+const MultiCU  = MultiCodeUnitEncoding
+const SCU = SingleCU()
+const MCU = MultiCU()
 
 # Convenience functions
 export to_ascii, utf8, utf16, utf32

src/compat.jl

@@ -152,7 +152,7 @@ unsafe_crc32c(a, n, crc) = ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_
 function sizeof(str::SubString{T}) where {T<:Str}
     is_multi(str) || return str.endof
     str.endof == 0 && return 0
-    _nextind(CodeUnitMulti(), str.string, str.offset + str.endof) - str.offset - 1
+    _nextind(MultiCU(), str.string, str.offset + str.endof) - str.offset - 1
 end
 
 occurs_in(str::String, hay::String) = contains(hay, str)
@@ -178,7 +178,7 @@ for sym in (:bin, :oct, :dec, :hex)
 end
 
 function repeat(ch::Char, cnt::Integer)
-    cnt > 1 && return String(_repeat(CodeUnitMulti(), UTF8CSE, ch%UInt32, cnt))
+    cnt > 1 && return String(_repeat(MultiCU(), UTF8CSE, ch%UInt32, cnt))
     cnt < 0 && repeaterr(cnt)
     cnt == 0 ? empty_string : string(Char(ch%UInt32))
 end

src/core.jl

@@ -8,44 +8,44 @@ Licensed under MIT License, see LICENSE.md
 Inspired by / derived from code in Julia
 =#
 
-_lastindex(::CodeUnitSingle, str) = (@_inline_meta(); ncodeunits(str))
+_lastindex(::SingleCU, str) = (@_inline_meta(); ncodeunits(str))
 
-@propagate_inbounds function _getindex(::CodeUnitSingle, T, str, pos::Int)
+@propagate_inbounds function _getindex(::SingleCU, T, str, pos::Int)
     @_inline_meta()
     @boundscheck checkbounds(str, pos)
     T(get_codeunit(pointer(str), pos))
 end
 
-@propagate_inbounds function _next(::CodeUnitSingle, T, str, pos)
+@propagate_inbounds function _next(::SingleCU, T, str, pos)
     @_inline_meta()
     @boundscheck 0 < pos <= ncodeunits(str) || boundserr(str, pos)
     T(get_codeunit(str, pos)), pos + 1
 end
 
-_nextcpfun(::CodeUnitSingle, ::Type{S}, pnt::Ptr{T}) where {S,T<:CodeUnitTypes} =
+_nextcpfun(::SingleCU, ::Type{S}, pnt::Ptr{T}) where {S,T<:CodeUnitTypes} =
     get_codeunit(pnt), pnt + sizeof(T)
 _nextcp(::Type{T}, pnt) where {T} = _nextcpfun(CodePointStyle(T), T, pnt)
 
-@propagate_inbounds _getindex(::CodeUnitMulti, T, str, pos::Int) =
-    first(_next(CodeUnitMulti(), T, str, pos))
+@propagate_inbounds _getindex(::MultiCU, T, str, pos::Int) =
+    first(_next(MultiCU(), T, str, pos))
 
-@inline _length(::CodeUnitSingle, str) = ncodeunits(str)
+@inline _length(::SingleCU, str) = ncodeunits(str)
 
 # Use more generic length check
 @inline _length_check(str::SubString{<:Str{C}}, cnt) where {C<:CSE} =
-    _length(CodeUnitMulti(), C, pointer(str), cnt)
+    _length(MultiCU(), C, pointer(str), cnt)
 
 # Go directly to aligned length check
 @inline _length_check(str::Str{C}, cnt) where {C<:CSE} =
-    _length_al(CodeUnitMulti(), C, pointer(str), cnt)
+    _length_al(MultiCU(), C, pointer(str), cnt)
 
-@inline _length(::CodeUnitMulti, str::MaybeSub{T}) where {T<:Str} =
+@inline _length(::MultiCU, str::MaybeSub{T}) where {T<:Str} =
     (cnt = ncodeunits(str); cnt < 2 ? Int(cnt > 0) : @preserve str _length_check(str, cnt))
 
-@inline _length(::CodeUnitSingle, ::Type{<:CSE}, ::Ptr{<:CodeUnitTypes}, cnt::Int) = cnt
+@inline _length(::SingleCU, ::Type{<:CSE}, ::Ptr{<:CodeUnitTypes}, cnt::Int) = cnt
 
-@inline _length(::CodeUnitMulti, str::Str{RawUTF8CSE}) = length(str.data)
-@inline _length(::CodeUnitMulti, str::Str{RawUTF8CSE}, i::Int, j::Int) = length(str.data, i, j)
+@inline _length(::MultiCU, str::Str{RawUTF8CSE}) = length(str.data)
+@inline _length(::MultiCU, str::Str{RawUTF8CSE}, i::Int, j::Int) = length(str.data, i, j)
 
 @propagate_inbounds function _length(cs::CodePointStyle, str, i::Int, j::Int)
     @boundscheck begin
@@ -58,7 +58,7 @@ _nextcp(::Type{T}, pnt) where {T} = _nextcpfun(CodePointStyle(T), T, pnt)
         @preserve str _length(cs, cse(str), bytoff(pointer(str), i - 1), cnt)
 end
 
-@inline _thisind(::CodeUnitSingle, str, len, pnt, pos) = Int(pos)
+@inline _thisind(::SingleCU, str, len, pnt, pos) = Int(pos)
 
 @propagate_inbounds function _thisind(cs::CS, str, pos) where {CS<:CodePointStyle}
     @_inline_meta()
@@ -71,26 +71,26 @@ end
     @preserve str _thisind(cs, str, len, pointer(str), pos)
 end
 
-@propagate_inbounds function _prevind(::CodeUnitSingle, str, i)
+@propagate_inbounds function _prevind(::SingleCU, str, i)
     @_inline_meta()
     @boundscheck 0 < i <= ncodeunits(str)+1 || boundserr(str, i)
     Int(i) - 1
 end
 
-@propagate_inbounds function _prevind(::CodeUnitSingle, str, i, nchar)
+@propagate_inbounds function _prevind(::SingleCU, str, i, nchar)
     @_inline_meta()
     nchar < 0 && ncharerr(nchar)
     @boundscheck 0 < i <= ncodeunits(str)+1 || boundserr(str, i)
     max(Int(i) - nchar, 0)
 end
 
-@propagate_inbounds function _nextind(::CodeUnitSingle, str, i)
+@propagate_inbounds function _nextind(::SingleCU, str, i)
     @_inline_meta()
     @boundscheck 0 <= i <= ncodeunits(str) || boundserr(str, i)
     Int(i) + 1
 end
 
-@propagate_inbounds function _nextind(::CodeUnitSingle, str, i, nchar)
+@propagate_inbounds function _nextind(::SingleCU, str, i, nchar)
     @_inline_meta()
     nchar < 0 && ncharerr(nchar)
     @boundscheck 0 <= i <= ncodeunits(str) || boundserr(str, i)
@@ -149,20 +149,20 @@ end
     _isvalid_char_pos(CodePointStyle(T), cse(T), str, i)
 end
 
-_isvalid_char_pos(::CodeUnitSingle, C, str, i) = true
+_isvalid_char_pos(::SingleCU, C, str, i) = true
 
-@propagate_inbounds function _collectstr(::CodeUnitMulti, ::Type{S},
+@propagate_inbounds function _collectstr(::MultiCU, ::Type{S},
                                          str::MaybeSub{T}) where {S,T<:Str}
-    len = _length(CodeUnitMulti(), str)
+    len = _length(MultiCU(), str)
     vec = create_vector(S, len)
     pos = 1
     @inbounds for i = 1:len
-        vec[i], pos = _next(CodeUnitMulti(), S, str, pos)
+        vec[i], pos = _next(MultiCU(), S, str, pos)
     end
     vec
 end
 
-@propagate_inbounds function _collectstr(::CodeUnitSingle, ::Type{S},
+@propagate_inbounds function _collectstr(::SingleCU, ::Type{S},
                                          str::MaybeSub{T}) where {S,T<:Str}
     len = ncodeunits(str)
     vec = create_vector(S, len)
@@ -219,7 +219,7 @@ function _convert(::Type{C}, ch::T) where {C<:CSE,T<:CodeUnitTypes}
     Str(C, buf)
 end
 
-# Todo: These should be made more generic, work for all CodeUnitSingle types
+# Todo: These should be made more generic, work for all SingleCU types
 
 convert(::Type{<:Str{ASCIICSE}}, ch::Unsigned) =
     is_ascii(ch) ? _convert(ASCIICSE, ch%UInt8) : unierror(UTF_ERR_INVALID_ASCII, 0, ch)
@@ -258,7 +258,7 @@ unsafe_convert(::Type{Ptr{Text4Chr}}, str::MaybeSub{<:Str{<:Quad_CSEs}}) =
 unsafe_convert(::Type{Ptr{Cvoid}},  s::MaybeSub{<:Str{C}}) where {C} =
     reinterpret(Ptr{Cvoid}, unsafe_convert(Ptr{codeunit(C)}, s))
 
-function _reverse(::CodeUnitSingle, ::Type{C}, len, str::Str{C}) where {C<:CSE}
+function _reverse(::SingleCU, ::Type{C}, len, str::Str{C}) where {C<:CSE}
     len < 2 && return str
     @preserve str begin
         pnt = pointer(str)
@@ -273,9 +273,9 @@ function _reverse(::CodeUnitSingle, ::Type{C}, len, str::Str{C}) where {C<:CSE}
     end
 end
 
-function _reverse(::CodeUnitMulti, ::Type{C}, len, str) where {C<:CSE}
+function _reverse(::MultiCU, ::Type{C}, len, str) where {C<:CSE}
     @inbounds ((t = nextind(str, 0)) > len || nextind(str, t) > len) && return str
-    @preserve str _reverse(CodeUnitMulti(), C, len, pointer(str))
+    @preserve str _reverse(MultiCU(), C, len, pointer(str))
 end
 
 reverse(str::MaybeSub{T}) where {C<:CSE,T<:Str{C}} =

src/search.jl

@@ -166,19 +166,18 @@ function find(::Type{D}, ch::AbsChar, str::AbstractString, pos::Integer) where {
         @boundscheck pos > len+1 && boundserr(str, pos)
         return 0
     end
-    # Only check if CodeUnitMulti
-    (cus = CodePointStyle(str)) === CodeUnitMulti() &&
-        (@inbounds is_valid(str, pos) || index_error(str, pos))
+    # Only check if MultiCU
+    is_multi(str) && (@inbounds is_valid(str, pos) || index_error(str, pos))
     # Check here if ch is valid for the type of string
-    is_valid(eltype(str), ch) ? _srch_cp(D(), cus, str, ch, pos, len) : 0
+    is_valid(eltype(str), ch) ? _srch_cp(D(), EncodingStyle(str), str, ch, pos, len) : 0
 end
 
 _get_dir(::Type{First}) = Fwd()
 _get_dir(::Type{Last})  = Rev()
 
 find(::Type{D}, ch::AbsChar, str::AbstractString) where {D<:Union{First,Last}} =
     ((len = ncodeunits(str)) == 0 || !is_valid(eltype(str), ch) ? 0
-     : _srch_cp(_get_dir(D), CodePointStyle(str), str, ch,
+     : _srch_cp(_get_dir(D), EncodingStyle(str), str, ch,
                 D === First ? 1 : lastindex(str), len))
 
 function find(::Type{D}, needle::AbstractString, str::AbstractString,
@@ -198,7 +197,7 @@ function find(::Type{D}, needle::AbstractString, str::AbstractString,
     is_valid(eltype(str), ch) || return _not_found
     # Check if single character
     if nxt > tlen
-        pos = _srch_cp(D(), CodePointStyle(str), str, ch, pos, slen)
+        pos = _srch_cp(D(), EncodingStyle(str), str, ch, pos, slen)
         return pos == 0 ? _not_found : (pos:pos)
     end
     _srch_strings(D(), cmp, str, needle, ch, nxt, pos, slen, tlen)
@@ -214,7 +213,7 @@ function find(::Type{T}, needle::AbstractString, str::AbstractString) where {T<:
     is_valid(eltype(str), ch) || return _not_found
     # Check if single character
     if nxt > tlen
-        pos = _srch_cp(_get_dir(T), CodePointStyle(str), str, ch, pos, slen)
+        pos = _srch_cp(_get_dir(T), EncodingStyle(str), str, ch, pos, slen)
         return pos:(pos - (pos == 0))
     end
     _srch_strings(_get_dir(T), cmp, str, needle, ch, nxt, pos, slen, tlen)
@@ -274,10 +273,10 @@ end
 
 # _srch_cp is only called with values that are valid for that string type,
 # and checking as already been done on the position (pos)
-# These definitions only work for CodeUnitSingle types
-_srch_cp(::Fwd, ::CodeUnitSingle, str::T, cp::AbsChar, pos, len) where {T<:Str} =
+# These definitions only work for SingleCU types
+_srch_cp(::Fwd, ::SingleCU, str::T, cp::AbsChar, pos, len) where {T<:Str} =
     @preserve str _srch_codeunit(Fwd(), pointer(str), cp%codeunit(T), pos, len)
-_srch_cp(::Rev, ::CodeUnitSingle, str::T, cp::AbsChar, pos, len) where {T<:Str} =
+_srch_cp(::Rev, ::SingleCU, str::T, cp::AbsChar, pos, len) where {T<:Str} =
     @preserve str _srch_codeunit(Rev(), pointer(str), cp%codeunit(T), pos)
 
 function _srch_cp(::Fwd, cus, str, cp, pos, len)
@@ -313,7 +312,7 @@ end
 end
 
 function _srch_strings(::Fwd, ::CompareStyle, str, needle, ch, subpos, pos, slen, tlen)
-    cu = CodePointStyle(str)
+    cu = EncodingStyle(str)
     while (pos = _srch_cp(Fwd(), cu, str, ch, pos, slen)) != 0
         nxt = nextind(str, pos)
         res = _cmp_str(str, nxt, slen, needle, subpos, tlen)
@@ -325,7 +324,7 @@ function _srch_strings(::Fwd, ::CompareStyle, str, needle, ch, subpos, pos, slen
 end
 
 function _srch_strings(::Rev, ::CompareStyle, str, needle, ch, nxtsub, pos, slen, tlen)
-    cu = CodePointStyle(str)
+    cu = EncodingStyle(str)
     while (prv = prevind(str, pos)) != 0 &&
         (loc = _srch_cp(Rev(), cu, str, ch, prv, slen)) != 0
         res = _cmp_str(str, nextind(str, loc), slen, needle, nxtsub, tlen)
@@ -359,7 +358,7 @@ function _srch_str_bloom(::Fwd, str, spnt, npnt, ch, pos, slen, tlen)
             end
 
             # match found
-            j == tlen - 1 && return pos:_thisind(CodePointStyle(str), str, slen, spnt, pos + j)
+            j == tlen - 1 && return pos:_thisind(EncodingStyle(str), str, slen, spnt, pos + j)
 
             # no match, try to rule out the next character
             if pos <= slen && _check_bloom_mask(bloom_mask, spnt, pos + tlen)

src/support.jl

@@ -933,10 +933,10 @@ end
     buf
 end
 
-_repeat(::CodeUnitSingle, ::Type{C}, ch::T, cnt) where {T,C<:CSE} =
+_repeat(::SingleCU, ::Type{C}, ch::T, cnt) where {T,C<:CSE} =
     _repeat_chr(basetype(T), ch, cnt)
 
-function _repeat(::CodeUnitMulti, ::Type{UTF8CSE}, ch, cnt)
+function _repeat(::MultiCU, ::Type{UTF8CSE}, ch, cnt)
     if ch <= 0x7f
         _repeat_chr(UInt8, ch, cnt)
     elseif ch <= 0x7ff
@@ -948,7 +948,7 @@ function _repeat(::CodeUnitMulti, ::Type{UTF8CSE}, ch, cnt)
     end
 end
 
-_repeat(::CodeUnitMulti, ::Type{UTF16CSE}, ch, cnt) =
+_repeat(::MultiCU, ::Type{UTF16CSE}, ch, cnt) =
     ch <= 0xffff ? _repeat_chr(UInt16, ch, cnt) : _repeat_chr(UInt32, get_utf16_32(ch), cnt)
 
 function repeat(str::T, cnt::Integer) where {C<:CSE,T<:Str{C}}