dlang · wilzbach · Feb 12, 2018
@@ -213,7 +213,7 @@ PACKAGE_std_experimental_allocator_building_blocks = \
 PACKAGE_std_net = curl isemail
 PACKAGE_std_range = interfaces package primitives
 PACKAGE_std_regex = package $(addprefix internal/,generator ir parser \
-  backtracking tests tests2 thompson kickstart)
+  backtracking thompson kickstart)
 
 # Modules in std (including those in packages)
 STD_MODULES=$(call P2MODULES,$(STD_PACKAGES))
@@ -340,7 +340,9 @@ endif
 $(addprefix $(ROOT)/unittest/,$(DISABLED_TESTS)) :
 	@echo Testing $@ - disabled
 
-UT_D_OBJS:=$(addprefix $(ROOT)/unittest/,$(addsuffix .o,$(D_MODULES)))
+UT_D_OBJS:=$(addprefix $(ROOT)/unittest/,$(addsuffix .o,$(D_MODULES) \
+	$(addprefix std/regex/internal/tests/, comon other $(addprefix test, 1 2 3 4  6 \
+		7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28))))
 # need to recompile all unittest objects whenever sth. changes
 $(UT_D_OBJS): $(ALL_D_FILES)
 $(UT_D_OBJS): $(ROOT)/unittest/%.o: %.d

diff --git a/std/regex/internal/tests.d b/std/regex/internal/tests.d
diff --git a/std/regex/internal/tests/common.d b/std/regex/internal/tests/common.d
@@ -0,0 +1,125 @@
+/*
+    Regualar expressions package test suite.
+*/
+module std.regex.internal.common;
+
+package(std.regex):
+
+import std.conv, std.exception, std.meta, std.range,
+    std.typecons, std.regex;
+
+import std.uni : Escapables; // characters that need escaping
+
+struct TestVectors
+{
+    string pattern;
+    string input;
+    string result;
+    string format;
+    string replace;
+    string flags;
+}
+
+
+void runTests(alias tv)()
+{
+
+    string produceExpected(M,String)(auto ref M m, String fmt)
+    {
+        auto app = appender!(String)();
+        replaceFmt(fmt, m.captures, app, true);
+        return app.data;
+    }
+    void run_tests(alias matchFn)()
+    {
+        int i;
+        static foreach (Char; AliasSeq!( char, wchar, dchar))
+        {{
+            alias String = immutable(Char)[];
+            String produceExpected(M,Range)(auto ref M m, Range fmt)
+            {
+                auto app = appender!(String)();
+                replaceFmt(fmt, m.captures, app, true);
+                return app.data;
+            }
+            Regex!(Char) r;
+            foreach (a, tvd; tv)
+            {
+                uint c = tvd.result[0];
+                debug(std_regex_test) writeln(" Test #", a, " pattern: ", tvd.pattern, " with Char = ", Char.stringof);
+                try
+                {
+                    i = 1;
+                    r = regex(to!(String)(tvd.pattern), tvd.flags);
+                }
+                catch (RegexException e)
+                {
+                    i = 0;
+                    debug(std_regex_test) writeln(e.msg);
+                }
+
+                assert((c == 'c') ? !i : i, "failed to compile pattern "~tvd.pattern);
+
+                if (c != 'c')
+                {
+                    auto m = matchFn(to!(String)(tvd.input), r);
+                    i = !m.empty;
+                    assert(
+                        (c == 'y') ? i : !i,
+                        text(matchFn.stringof ~": failed to match pattern #", a ,": ", tvd.pattern)
+                    );
+                    if (c == 'y')
+                    {
+                        auto result = produceExpected(m, to!(String)(tvd.format));
+                        assert(result == to!String(tvd.replace),
+                            text(matchFn.stringof ~": mismatch pattern #", a, ": ", tvd.pattern," expected: ",
+                                    tvd.replace, " vs ", result));
+                    }
+                }
+            }
+        }}
+        debug(std_regex_test) writeln("!!! FReD bulk test done "~matchFn.stringof~" !!!");
+    }
+
+
+    void ct_tests()
+    {
+        import std.algorithm.comparison : equal;
+        static foreach (v; 0 .. tv.length)
+        {{
+            enum tvd = tv[v];
+            static if (tvd.result == "c")
+            {
+                static assert(!__traits(compiles, (){
+                    enum r = regex(tvd.pattern, tvd.flags);
+                }), "errornously compiles regex pattern: " ~ tvd.pattern);
+            }
+            else
+            {
+                //BUG: tv[v] is fine but tvd is not known at compile time?!
+                auto r = ctRegex!(tv[v].pattern, tv[v].flags);
+                auto nr = regex(tvd.pattern, tvd.flags);
+                assert(equal(r.ir, nr.ir),
+                    text("!C-T regex! failed to compile pattern #", v ,": ", tvd.pattern));
+                auto m = match(tvd.input, r);
+                auto c = tvd.result[0];
+                bool ok = (c == 'y') ^ m.empty;
+                assert(ok, text("ctRegex: failed to match pattern #",
+                    v ,": ", tvd.pattern));
+                if (c == 'y')
+                {
+                    import std.stdio : writeln;
+                    auto result = produceExpected(m, tvd.format);
+                    if (result != tvd.replace)
+                        writeln("ctRegex mismatch pattern #", v, ": ", tvd.pattern," expected: ",
+                                tvd.replace, " vs ", result);
+                }
+            }
+        }}
+        debug(std_regex_test) writeln("!!! FReD C-T test done !!!");
+    }
+
+    ct_tests();
+    run_tests!bmatch(); //backtracker
+    run_tests!match(); //thompson VM
+}
diff --git a/std/regex/internal/tests2.d → std/regex/internal/tests/other.d b/std/regex/internal/tests2.d → std/regex/internal/tests/other.d
@@ -9,6 +9,25 @@ import std.conv, std.exception, std.meta, std.range,
 
 import std.uni : Escapables; // characters that need escaping
 
+@safe unittest
+{//sanity checks
+    regex("(a|b)*");
+    regex(`(?:([0-9A-F]+)\.\.([0-9A-F]+)|([0-9A-F]+))\s*;\s*(.*)\s*#`);
+    regex("abc|edf|ighrg");
+    auto r1 = regex("abc");
+    auto r2 = regex("(gylba)");
+    assert(match("abcdef", r1).hit == "abc");
+    assert(!match("wida",r2));
+    assert(bmatch("abcdef", r1).hit == "abc");
+    assert(!bmatch("wida", r2));
+    assert(match("abc", "abc".dup));
+    assert(bmatch("abc", "abc".dup));
+    Regex!char rc;
+    assert(rc.empty);
+    rc = regex("test");
+    assert(!rc.empty);
+}
+
 @safe unittest
 {
     auto cr = ctRegex!("abc");

diff --git a/std/regex/internal/tests/test1.d b/std/regex/internal/tests/test1.d
@@ -0,0 +1,46 @@
+/*
+    Regualar expressions package test suite.
+*/
+module std.regex.internal.tests.tests1;
+
+package(std.regex):
+
+import std.regex.internal.tests.common;
+
+/* The test vectors in this file are altered from Henry Spencer's regexp
+   test code. His copyright notice is:
+
+        Copyright (c) 1986 by University of Toronto.
+        Written by Henry Spencer.  Not derived from licensed software.
+
+        Permission is granted to anyone to use this software for any
+        purpose on any computer system, and to redistribute it freely,
+        subject to the following restrictions:
+
+        1. The author is not responsible for the consequences of use of
+                this software, no matter how awful, even if they arise
+                from defects in it.
+
+        2. The origin of this software must not be misrepresented, either
+                by explicit claim or by omission.
+
+        3. Altered versions must be plainly marked as such, and must not
+                be misrepresented as being the original software.
+ */
+
+@safe unittest
+{
+    static immutable TestVectors[] tv = [
+        TestVectors(  "a\\b",       "a",  "y",    "$&",    "a" ),
+        TestVectors(  "(a)b\\1",   "abaab","y",    "$&",    "aba" ),
+        TestVectors(  "()b\\1",     "aaab", "y",    "$&",    "b" ),
+        TestVectors(  "abc",       "abc",  "y",    "$&",    "abc" ),
+        TestVectors(  "abc",       "xbc",  "n",    "-",    "-" ),
+        TestVectors(  "abc",       "axc",  "n",    "-",    "-" ),
+        TestVectors(  "abc",       "abx",  "n",    "-",    "-" ),
+        TestVectors(  "abc",       "xabcy","y",    "$&",    "abc" ),
+        TestVectors(  "abc",       "ababc","y",    "$&",    "abc" ),
+        TestVectors(  "ab*c",      "abc",  "y",    "$&",    "abc" ),
+    ];
+    runTests!tv;
+}
diff --git a/std/regex/internal/tests/test10.d b/std/regex/internal/tests/test10.d
@@ -0,0 +1,46 @@
+/*
+    Regualar expressions package test suite.
+*/
+module std.regex.internal.tests.tests10;
+
+package(std.regex):
+
+import std.regex.internal.tests.common;
+
+/* The test vectors in this file are altered from Henry Spencer's regexp
+   test code. His copyright notice is:
+
+        Copyright (c) 1986 by University of Toronto.
+        Written by Henry Spencer.  Not derived from licensed software.
+
+        Permission is granted to anyone to use this software for any
+        purpose on any computer system, and to redistribute it freely,
+        subject to the following restrictions:
+
+        1. The author is not responsible for the consequences of use of
+                this software, no matter how awful, even if they arise
+                from defects in it.
+
+        2. The origin of this software must not be misrepresented, either
+                by explicit claim or by omission.
+
+        3. Altered versions must be plainly marked as such, and must not
+                be misrepresented as being the original software.
+ */
+
+@safe unittest
+{
+    static immutable TestVectors[] tv = [
+        TestVectors(  "abcd*efg",  "abcdefg",      "y",    "$&",    "abcdefg" ),
+        TestVectors(  "ab*",       "xabyabbbz",    "y",    "$&",    "ab" ),
+        TestVectors(  "ab*",       "xayabbbz",     "y",    "$&",    "a" ),
+        TestVectors(  "(ab|cd)e",  "abcde",        "y",    "$&-$1",        "cde-cd" ),
+        TestVectors(  "[abhgefdc]ij",      "hij",  "y",    "$&",    "hij" ),
+        TestVectors(  "^(ab|cd)e", "abcde",        "n",    "x$1y",        "xy" ),
+        TestVectors(  "(abc|)ef",  "abcdef",       "y",    "$&-$1",        "ef-" ),
+        TestVectors(  "(a|b)c*d",  "abcd",         "y",    "$&-$1",        "bcd-b" ),
+        TestVectors(  "(ab|ab*)bc",        "abc",  "y",    "$&-$1",        "abc-a" ),
+        TestVectors(  "a([bc]*)c*",        "abc",  "y",    "$&-$1",        "abc-bc" ),
+    ];
+    runTests!tv;
+}
diff --git a/std/regex/internal/tests/test11.d b/std/regex/internal/tests/test11.d
@@ -0,0 +1,46 @@
+/*
+    Regualar expressions package test suite.
+*/
+module std.regex.internal.tests.tests11;
+
+package(std.regex):
+
+import std.regex.internal.tests.common;
+
+/* The test vectors in this file are altered from Henry Spencer's regexp
+   test code. His copyright notice is:
+
+        Copyright (c) 1986 by University of Toronto.
+        Written by Henry Spencer.  Not derived from licensed software.
+
+        Permission is granted to anyone to use this software for any
+        purpose on any computer system, and to redistribute it freely,
+        subject to the following restrictions:
+
+        1. The author is not responsible for the consequences of use of
+                this software, no matter how awful, even if they arise
+                from defects in it.
+
+        2. The origin of this software must not be misrepresented, either
+                by explicit claim or by omission.
+
+        3. Altered versions must be plainly marked as such, and must not
+                be misrepresented as being the original software.
+ */
+
+@safe unittest
+{
+    static immutable TestVectors[] tv = [
+        TestVectors(  "a([bc]*)(c*d)",     "abcd", "y",    "$&-$1-$2",    "abcd-bc-d" ),
+        TestVectors(  "a([bc]+)(c*d)",     "abcd", "y",    "$&-$1-$2",    "abcd-bc-d" ),
+        TestVectors(  "a([bc]*)(c+d)",     "abcd", "y",    "$&-$1-$2",    "abcd-b-cd" ),
+        TestVectors(  "a[bcd]*dcdcde",     "adcdcde",      "y",    "$&",    "adcdcde" ),
+        TestVectors(  "a[bcd]+dcdcde",     "adcdcde",      "n",    "-",    "-" ),
+        TestVectors(  "(ab|a)b*c", "abc",           "y",    "$&-$1",        "abc-ab" ),
+        TestVectors(  "((a)(b)c)(d)",      "abcd",  "y",    "$1-$2-$3-$4",      "abc-a-b-d" ),
+        TestVectors(  "[a-zA-Z_][a-zA-Z0-9_]*",    "alpha",        "y",    "$&",    "alpha" ),
+        TestVectors(  "^a(bc+|b[eh])g|.h$",        "abh",  "y",    "$&-$1",        "bh-" ),
+        TestVectors(  "(bc+d$|ef*g.|h?i(j|k))",    "effgz",        "y",    "$&-$1-$2",    "effgz-effgz-" ),
+    ];
+    runTests!tv;
+}
diff --git a/std/regex/internal/tests/test12.d b/std/regex/internal/tests/test12.d
@@ -0,0 +1,46 @@
+/*
+    Regualar expressions package test suite.
+*/
+module std.regex.internal.tests.tests12;
+
+package(std.regex):
+
+import std.regex.internal.tests.common;
+
+/* The test vectors in this file are altered from Henry Spencer's regexp
+   test code. His copyright notice is:
+
+        Copyright (c) 1986 by University of Toronto.
+        Written by Henry Spencer.  Not derived from licensed software.
+
+        Permission is granted to anyone to use this software for any
+        purpose on any computer system, and to redistribute it freely,
+        subject to the following restrictions:
+
+        1. The author is not responsible for the consequences of use of
+                this software, no matter how awful, even if they arise
+                from defects in it.
+
+        2. The origin of this software must not be misrepresented, either
+                by explicit claim or by omission.
+
+        3. Altered versions must be plainly marked as such, and must not
+                be misrepresented as being the original software.
+ */
+
+@safe unittest
+{
+    static immutable TestVectors[] tv = [
+        TestVectors(  "(bc+d$|ef*g.|h?i(j|k))",    "ij",   "y",    "$&-$1-$2",    "ij-ij-j" ),
+        TestVectors(  "(bc+d$|ef*g.|h?i(j|k))",    "effg", "n",    "-",    "-" ),
+        TestVectors(  "(bc+d$|ef*g.|h?i(j|k))",    "bcdd", "n",    "-",    "-" ),
+        TestVectors(  "(bc+d$|ef*g.|h?i(j|k))",    "reffgz",       "y",    "$&-$1-$2",    "effgz-effgz-" ),
+        TestVectors(  "(((((((((a)))))))))",       "a",    "y",    "$&",    "a" ),
+        TestVectors(  "multiple words of text",    "uh-uh",        "n",    "-",    "-" ),
+        TestVectors(  "multiple words",    "multiple words, yeah", "y",    "$&",    "multiple words" ),
+        TestVectors(  "(.*)c(.*)", "abcde",                "y",    "$&-$1-$2",    "abcde-ab-de" ),
+        TestVectors(  "\\((.*), (.*)\\)",  "(a, b)",       "y",    "($2, $1)",   "(b, a)" ),
+        TestVectors(  "abcd",      "abcd",                   "y",    "$&-&-$$$&",  "abcd-&-$abcd" ),
+    ];
+    runTests!tv;
+}