Implement +[NSRegularExpression escapedPatternForString:] and -[NSString enumerateSubstringsInRange:options:usingBlock]

.gitignore

@@ -83,3 +83,24 @@ DerivedData/
 **/xcshareddata/WorkspaceSettings.xcsettings
 
 # End of https://www.gitignore.io/api/xcode
+
+.cache
+compile_commands.json
+**.kdev4
+
+# Documentation
+Documentation/Base*
+Documentation/General
+Documentation/manual
+Documentation/ReleaseNotes
+Documentation/ANNOUNCE
+Documentation/*.pdf
+Documentation/README
+Documentation/version.texi
+Documentation/*.aux
+Documentation/*.toc
+Documentation/INSTALL
+Documentation/NEWS
+**/dependencies
+Source/Base.gsdoc
+Source/BaseAdditions.gsdoc

Headers/Foundation/NSRegularExpression.h

@@ -149,6 +149,9 @@ GS_EXPORT_CLASS
                                 inString: (NSString*)string
                                   offset: (NSInteger)offset
                                 template: (NSString*)templat;
+#if OS_API_VERSION(MAC_OS_X_VERSION_10_7, GS_API_LATEST)
++ (NSString *)escapedPatternForString:(NSString *)string;
+#endif
 #if GS_HAS_DECLARED_PROPERTIES
 @property (readonly) NSRegularExpressionOptions options;
 @property (readonly) NSUInteger numberOfCaptureGroups;

Headers/Foundation/NSString.h

@@ -426,6 +426,48 @@ enum {
 typedef NSUInteger NSStringEncodingConversionOptions;
 #endif
 
+#if OS_API_VERSION(MAC_OS_X_VERSION_10_6,GS_API_LATEST) 
+/** For enumerateSubstringsInRange:options:usingBlock: 
+    You must include an substring type (`NSStringEnumerationBy`), and may
+    bitwise or (`|`) with any of the other options. */
+enum {
+    /* Must include one of these 
+       Must fit into 8 bits. */
+    /** Enumerate by lines. Uses lineRangeForRange: */
+    NSStringEnumerationByLines = 0,
+    /** Enumerate by paragraph. Uses paragraphRangeForRange: */
+    NSStringEnumerationByParagraphs = 1,
+    /** Enumerate by composed character sequence. Uses rangeOfComposedCharacterSequencesForRange: */
+    NSStringEnumerationByComposedCharacterSequences = 2,
+    /** Enumerate by word, as specified in Unicode TR 29. 
+        Only supported if GNUstep is compiled with ICU. 
+        Uses UBRK_WORD, with current locale and standard abbreviation lists if 
+        NSStringEnumerationLocalized is passed, otherwise the locale is "en_US_POSIX". */
+    NSStringEnumerationByWords = 3,
+    /** Enumerate by sentence, as specified in Unicode TR 29. 
+        Only supported if GNUstep is compiled with ICU. 
+        Uses UBRK_WORD, with current locale and standard abbreviation lists if 
+        NSStringEnumerationLocalized is passed, otherwise the locale is "en_US_POSIX". */
+    NSStringEnumerationBySentences = 4,
+    #if OS_API_VERSION(MAC_OS_X_VERSION_11,GS_API_LATEST) 
+    /** Undocumented public API on macOS. Not supported by GNUstep. */
+    NSStringEnumerationByCaretPositions = 5,
+    /** Undocumented public API on macOS. Not supported by GNUstep. */
+    NSStringEnumerationByDeletionClusters = 6,
+    #endif
+
+    /* May pass one of these via bitwise or. 
+       Must be a single bit set at an offset >= 8. */
+    NSStringEnumerationReverse = 1UL << 8,
+    NSStringEnumerationSubstringNotRequired = 1UL << 9,
+    NSStringEnumerationLocalized = 1UL << 10
+};
+
+typedef NSUInteger NSStringEnumerationOptions;
+
+DEFINE_BLOCK_TYPE(GSNSStringEnumerationBlock, void, NSString* substring, NSRange substringRange, NSRange enclosingRange, BOOL* stop);
+#endif
+
 /**
  * <p>
  *   <code>NSString</code> objects represent an immutable string of Unicode 3.0
@@ -1050,6 +1092,12 @@ GS_EXPORT_CLASS
 + (Class) constantStringClass;
 #endif	/* GS_API_NONE */
 
+#if OS_API_VERSION(MAC_OS_X_VERSION_10_6,GS_API_LATEST) 
+- (void) enumerateSubstringsInRange: (NSRange)range 
+                            options: (NSStringEnumerationOptions)opts 
+                         usingBlock: (GSNSStringEnumerationBlock)block;
+#endif
+
 @end
 
 GS_EXPORT_CLASS

Source/NSRegularExpression.m

@@ -1064,6 +1064,20 @@ - (NSString*) replacementStringForResult: (NSTextCheckingResult*)result
 }
 #endif
 
++ (NSString *)escapedPatternForString:(NSString *)string {
+  // https://unicode-org.github.io/icu/userguide/strings/regexp.html
+  // Need to escape * ? + [ ( ) { } ^ $ | \ .
+  return [[NSRegularExpression 
+    regularExpressionWithPattern: @"([*?+\\[(){}^$|\\\\.])" 
+                         options: 0 
+                           error: NULL]
+    stringByReplacingMatchesInString: string
+                             options: 0
+                               range: NSMakeRange(0, [string length]) 
+                        withTemplate: @"\\\\$1"
+  ];
+}
+
 - (NSRegularExpressionOptions) options
 {
   return options;

Source/NSString.m

@@ -112,6 +112,14 @@
 # include <icu.h>
 #endif
 
+#import "Foundation/NSObjCRuntime.h"
+#import "GNUstepBase/GSBlocks.h"
+#if GS_USE_ICU
+#include <unicode/ubrk.h>
+#include <unicode/utypes.h>
+#endif
+
+
 /* Create local inline versions of key functions for case-insensitive operations
  */
 #import "Additions/unicode/caseconv.h"
@@ -6248,6 +6256,156 @@ - (BOOL) containsString: (NSString *)string
   return [self rangeOfString: string].location != NSNotFound;
 }
 
+- (void) enumerateSubstringsInRange: (NSRange)range 
+                            options: (NSStringEnumerationOptions)opts 
+                         usingBlock: (GSNSStringEnumerationBlock)block
+{
+  // Get low byte.
+  uint8_t substringType = opts & 0xFF;
+
+  BOOL isReverse = opts & NSStringEnumerationReverse;
+  BOOL substringNotRequired = opts & NSStringEnumerationSubstringNotRequired;
+  BOOL localized = opts & NSStringEnumerationLocalized;
+
+  NSUInteger currentLocation;
+  BOOL stop = NO;
+  if(isReverse) {
+    currentLocation = range.location + range.length;
+  } else {
+    currentLocation = range.location;
+  }
+
+  if (substringType == NSStringEnumerationByLines || substringType == NSStringEnumerationByParagraphs) {
+    BOOL isLineSep = substringType == NSStringEnumerationByLines;
+
+    while(YES) {
+      // contains the index of the first character of the line containing the beginning of aRange.
+      NSUInteger start;
+      // contains the index of the first character past the terminator of the line containing the end of aRange.
+      NSUInteger end;
+      // contains the index of the first character of the terminator of the line containing the end of aRange. 
+      NSUInteger contentsEnd;
+      NSRange currentLocationRange = (NSRange){
+        .location = currentLocation,
+        .length = 0,
+      };
+      [self _getStart: &start 
+                  end: &end 
+          contentsEnd: &contentsEnd 
+             forRange: currentLocationRange 
+              lineSep: isLineSep];
+      // If the enumerated range starts after the line/paragraph, we start at the beginning of the enumerated range
+      NSUInteger substringStart = start > range.location ? start : range.location;
+      NSRange substringRange = (NSRange){
+        .location = substringStart,
+        .length = contentsEnd - substringStart
+      };
+      CALL_BLOCK(block, 
+        substringNotRequired ? nil : [self substringWithRange: substringRange],
+        substringRange,
+        (NSRange){
+          .location = start,
+          .length = end - start
+        },
+        &stop);
+      if(stop) break;
+      if(end == range.location + range.length) break;
+      currentLocation = end;
+    }
+  } else if (substringType == NSStringEnumerationByComposedCharacterSequences) {
+    // We could also use rangeOfComposedCharacterSequenceAtIndex:, but then we would need different logic.
+    while(YES) {
+      // Since all characters are in a composed character sequence, enclosingRange == substringRange
+      NSRange enclosingRange = [self rangeOfComposedCharacterSequenceAtIndex: currentLocation];
+      CALL_BLOCK(block, 
+        substringNotRequired ? nil : [self substringWithRange: enclosingRange],
+        enclosingRange,
+        enclosingRange,
+        &stop);
+      if(stop) break;
+      currentLocation = enclosingRange.location + enclosingRange.length;
+    }
+  } else if (substringType == NSStringEnumerationByWords || substringType == NSStringEnumerationBySentences) {
+    #if GS_USE_ICU
+    // These macros may be useful elsewhere.
+    #define GS_U_HANDLE_ERROR(errorCode, description) do { \
+      if(U_FAILURE(errorCode)) { \
+        NSWarnMLog(@"Error " description ": %s", u_errorName(errorCode)); \
+        return; \
+      } else if(errorCode < U_ZERO_ERROR) { \
+        NSWarnMLog(@"Warning " description ": %s", u_errorName(errorCode)); \
+      } \
+      errorCode = U_ZERO_ERROR; \
+    } while (NO)
+
+    BOOL byWords = substringType == NSStringEnumerationByWords;
+    NSUInteger length = range.length;
+    UChar characters[length];
+    [self getCharacters: characters range: range];
+    UErrorCode errorCode = U_ZERO_ERROR;
+    const char* locale = localized
+      ? [[[[NSLocale currentLocale] 
+                     localeIdentifier] 
+                     // @ss=standard will use lists of common abbreviations, such as Mr., Mrs., etc.
+                     stringByAppendingString: @"@ss=standard"] 
+                     UTF8String]
+      : "en_US_POSIX";
+    UBreakIterator* breakIterator = ubrk_open(byWords ? UBRK_WORD : UBRK_SENTENCE, // type
+                                              locale, // locale
+                                              characters, // text
+                                              length, // textLength
+                                              &errorCode);
+    GS_U_HANDLE_ERROR(errorCode, @"opening ICU break iterator");
+    ubrk_first(breakIterator);
+    while(YES) {
+      // Make sure it's a valid substring.
+      BOOL isValidSubstring = YES;
+
+      if(byWords) {
+        int32_t ruleStatus = ubrk_getRuleStatus(breakIterator);
+        // From ICU User Guide:
+        //   A status value UBRK_WORD_NONE indicates that the boundary does
+        //   not start a word or number.
+        // However, valid words seem to be UBRK_WORD_NONE, and invalid words
+        // seem to be UBRK_WORD_NONE_LIMIT.
+        isValidSubstring = ruleStatus != UBRK_WORD_NONE_LIMIT;
+        NSLog(@"Status for position %d (%d): %d", (int)currentLocation, (int)ubrk_current(breakIterator), (int) ruleStatus);
+      }
+      int32_t nextPosition = ubrk_next(breakIterator);
+      if(nextPosition == UBRK_DONE) break;
+      NSUInteger nextLocation = range.location + nextPosition;
+      // Same as substringRange
+      NSRange enclosingRange = (NSRange){
+        .location = currentLocation,
+        .length = nextLocation - currentLocation
+      };
+
+      if(isValidSubstring) {
+        CALL_BLOCK(block, 
+          substringNotRequired ? nil : [self substringWithRange: enclosingRange],
+          enclosingRange,
+          enclosingRange,
+          &stop);
+        if(stop) break;
+      }
+
+      currentLocation = nextLocation;
+    }
+    #else
+    NSWarnLog(@"NSStringEnumerationByWords and NSStringEnumerationBySentences are not supported when GNUstep-base is compiled without ICU.");
+    return;
+    #endif
+  } else if (substringType == NSStringEnumerationByCaretPositions) {
+    // FIXME - Not documented by Apple.
+    NSWarnLog(@"NSStringEnumerationByCaretPositions is not supported");
+    return;
+  } else if (substringType == NSStringEnumerationByDeletionClusters) {
+    // FIXME - Not documented by Apple.
+    NSWarnLog(@"NSStringEnumerationByDeletionClusters is not supported");
+    return;
+  }
+}
+
 @end
 
 /**

Tests/base/NSString/enumerateSubstringsInRange.m

@@ -0,0 +1,98 @@
+#import "ObjectTesting.h"
+#import <Foundation/NSAutoreleasePool.h>
+#import <Foundation/NSString.h>
+
+#if defined(__has_extension) && __has_extension(blocks)
+int main (int argc, const char * argv[])
+{
+  NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
+  START_SET("Enumerate substrings by lines");
+
+  NSString* s1 = @"Line 1\nLine 2";
+  __block NSUInteger currentIteration = 0;
+  [s1 enumerateSubstringsInRange:(NSRange){
+    .location = 0,
+    .length = [s1 length]
+  }                      options: NSStringEnumerationByLines 
+                      usingBlock: ^(NSString *substring, NSRange substringRange, NSRange enclosingRange, BOOL *stop) {
+    NSLog(@"Substring range: {.location=%ld, .length=%ld}", substringRange.location, substringRange.length);
+    NSLog(@"Enclosing range: {.location=%ld, .length=%ld}", enclosingRange.location, enclosingRange.length);
+    NSLog(@"Substring: %@", substring);
+    // *stop = YES;
+    if(currentIteration == 0) PASS([substring isEqual: @"Line 1"], "First line of \"Line 1\\nLine 2\" is \"Line 1\"");
+    if(currentIteration == 1) PASS([substring isEqual: @"Line 2"], "Second line of \"Line 1\\nLine 2\" is \"Line 2\"");
+    currentIteration++;
+  }]; 
+  PASS(currentIteration == 2, "There are only two lines in \"Line 1\\nLine 2\"");
+  END_SET("Enumerate substrings by lines");
+
+  START_SET("Enumerate substrings by paragraphs");
+
+  NSString* s1 = @"Paragraph 1\nParagraph 2";
+  __block NSUInteger currentIteration = 0;
+  [s1 enumerateSubstringsInRange:(NSRange){
+    .location = 0,
+    .length = [s1 length]
+  }                      options: NSStringEnumerationByParagraphs 
+                      usingBlock: ^(NSString *substring, NSRange substringRange, NSRange enclosingRange, BOOL *stop) {
+    NSLog(@"Substring range: {.location=%ld, .length=%ld}", substringRange.location, substringRange.length);
+    NSLog(@"Enclosing range: {.location=%ld, .length=%ld}", enclosingRange.location, enclosingRange.length);
+    NSLog(@"Substring: %@", substring);
+    // *stop = YES;
+    if(currentIteration == 0) PASS([substring isEqual: @"Paragraph 1"], "First paragraph of \"Paragraph 1\\nParagraph 2\" is \"Paragraph 1\"");
+    if(currentIteration == 1) PASS([substring isEqual: @"Paragraph 2"], "Second paragraph of \"Paragraph 1\\nParagraph 2\" is \"Paragraph 2\"");
+    currentIteration++;
+  }]; 
+  PASS(currentIteration == 2, "There are only two paragraphs in \"Paragraph 1\\nParagraph 2\"");
+  END_SET("Enumerate substrings by paragraphs");
+
+  START_SET("Enumerate substrings by words");
+
+  NSString* s1 = @"Word1 word2.";
+  __block NSUInteger currentIteration = 0;
+  [s1 enumerateSubstringsInRange:(NSRange){
+    .location = 0,
+    .length = [s1 length]
+  }                      options: NSStringEnumerationByWords
+                      usingBlock: ^(NSString *substring, NSRange substringRange, NSRange enclosingRange, BOOL *stop) {
+    NSLog(@"Substring range: {.location=%ld, .length=%ld}", substringRange.location, substringRange.length);
+    NSLog(@"Enclosing range: {.location=%ld, .length=%ld}", enclosingRange.location, enclosingRange.length);
+    NSLog(@"Substring: %@", substring);
+    // *stop = YES;
+    if(currentIteration == 0) PASS([substring isEqual: @"Word1"], "First word of \"Word1 word2.\" is \"Word1\"");
+    if(currentIteration == 1) PASS([substring isEqual: @"word2"], "Second word of \"Word1 word2.\" is \"word2\"");
+    currentIteration++;
+  }]; 
+  PASS(currentIteration == 2, "There are only two words in \"Word1 word2.\"");
+  END_SET("Enumerate substrings by words");
+
+  START_SET("Enumerate substrings by sentences");
+
+  NSString* s1 = @"Sentence 1. Sentence 2.";
+  __block NSUInteger currentIteration = 0;
+  [s1 enumerateSubstringsInRange:(NSRange){
+    .location = 0,
+    .length = [s1 length]
+  }                      options: NSStringEnumerationBySentences
+                      usingBlock: ^(NSString *substring, NSRange substringRange, NSRange enclosingRange, BOOL *stop) {
+    NSLog(@"Substring range: {.location=%ld, .length=%ld}", substringRange.location, substringRange.length);
+    NSLog(@"Enclosing range: {.location=%ld, .length=%ld}", enclosingRange.location, enclosingRange.length);
+    NSLog(@"Substring: %@", substring);
+    // *stop = YES;
+    if(currentIteration == 0) PASS([substring isEqual: @"Sentence 1. "], "First sentence of \"Sentence 1. Sentence 2.\" is \"Sentence 1. \"");
+    if(currentIteration == 1) PASS([substring isEqual: @"Sentence 2."], "Second sentence of \"Sentence 1. Sentence 2.\" is \"Sentence 2.\"");
+    currentIteration++;
+  }]; 
+  PASS(currentIteration == 2, "There are only two sentences in \"Sentence 1. Sentence 2.");
+  END_SET("Enumerate substrings by sentences");
+
+  [pool drain];
+
+  return 0;
+}
+#else
+int main (int argc, const char * argv[])
+{
+  return 0;
+}
+#endif