fix issue 11856 - segfault with circular template constraints

src/dmd/dtemplate.d

@@ -689,6 +689,12 @@ extern (C++) final class TemplateDeclaration : ScopeDsymbol
      */
     extern (D) bool evaluateConstraint(TemplateInstance ti, Scope* sc, Scope* paramscope, Objects* dedargs, FuncDeclaration fd)
     {
+        // circular evaluation of the constraint, see https://issues.dlang.org/show_bug.cgi?id=11856
+        if (constraint.inuse == constraint.inuse.max)
+            return false;
+        constraint.inuse++;
+        scope(exit)
+            constraint.inuse--;
         /* Detect recursive attempts to instantiate this template declaration,
          * https://issues.dlang.org/show_bug.cgi?id=4072
          *  void foo(T)(T x) if (is(typeof(foo(x)))) { }

src/dmd/expression.d

@@ -642,6 +642,7 @@ extern (C++) abstract class Expression : ASTNode
     ubyte parens;   // if this is a parenthesized expression
     Type type;      // !=null means that semantic() has been run
     Loc loc;        // file location
+    ubyte inuse;    // detect expressionSemantic() loops
 
     extern (D) this(const ref Loc loc, TOK op, int size)
     {

src/dmd/expressionsem.d

@@ -11094,6 +11094,12 @@ Expression binSemanticProp(BinExp e, Scope* sc)
 // entrypoint for semantic ExpressionSemanticVisitor
 extern (C++) Expression expressionSemantic(Expression e, Scope* sc)
 {
+    const inConstraint = sc ? (sc.flags & SCOPE.constraint) == SCOPE.constraint : false;
+    if (e.inuse == 2 && inConstraint)
+    {
+        e.error("circular evaluation of constraint `%s`", e.toChars());
+        return new ErrorExp();
+    }
     scope v = new ExpressionSemanticVisitor(sc);
     e.accept(v);
     return v.result;

src/dmd/traits.d

@@ -1533,9 +1533,12 @@ Expression semanticTraits(TraitsExp e, Scope* sc)
             }
             if (ex)
             {
+                const bool inConstraint = (sc2.flags & SCOPE.constraint) == SCOPE.constraint;
+                ex.inuse += inConstraint;
                 ex = ex.expressionSemantic(sc2);
                 ex = resolvePropertiesOnly(sc2, ex);
                 ex = ex.optimize(WANTvalue);
+                ex.inuse -= inConstraint;
                 if (sc2.func && sc2.func.type.ty == Tfunction)
                 {
                     const tf = cast(TypeFunction)sc2.func.type;

test/fail_compilation/ice11856_0.d

@@ -0,0 +1,16 @@
+/*
+TEST_OUTPUT:
+---
+fail_compilation/ice11856_0.d(16): Error: template `ice11856_0.f` cannot deduce function from argument types `!()(int)`, candidates are:
+fail_compilation/ice11856_0.d(10):        `ice11856_0.f(T)(T t) if (!__traits(compiles, .f!T))`
+fail_compilation/ice11856_0.d(13):        `ice11856_0.f(T)(T t) if (!__traits(compiles, .f!T))`
+---
+*/
+
+int f(T)(T t) if(!__traits(compiles,.f!T)) {
+    return 0;
+}
+int f(T)(T t) if(!__traits(compiles,.f!T)) {
+    return 1;
+}
+enum x=f(2);

test/fail_compilation/ice11856_1.d

@@ -0,0 +1,13 @@
+/*
+TEST_OUTPUT:
+---
+fail_compilation/ice11856_1.d(13): Error: template `ice11856_1.g` cannot deduce function from argument types `!()(A)`, candidates are:
+fail_compilation/ice11856_1.d(11):        `ice11856_1.g(T)(T x) if (is(typeof(x.f())))`
+---
+*/
+struct A {}
+
+void f(T)(T x) if (is(typeof(x.g()))) {}
+void g(T)(T x) if (is(typeof(x.f()))) {}
+
+void main() { A().g(); }