Dynamic Schema Generation e.g. for Query Filters

[fridolin-koch]

Expected Behaviour

I'm fairly new in the world of GraphQL so I'm not even sure if this is the right approach.

What I wan't to do is implement powerful filters for list responses, ideally I wan't these filters to be "strongly typed".

Basically I wan't to implement what Prisma is doing: https://www.prisma.io/docs/prisma-graphql-api/reference/queries-qwe1/

Lets imagine the following schema:

type Query {
    categories(limit: Int = 200, offset: Int = 0, filter: CategoriesFilter): CategoryResponse
}

type Category {
    id: ID!
    name: String!
    position: Int!
    enabled: Boolean!
}

Now I would like to have some possibility to generate the type CategoriesFilter dynamically based on some logic so I could generate something like that

input CategoriesFilter {
    id: ID
    id_not: ID
    id_in: [ID]
    id_not_in: [ID]
    name: String
    name_not: String
    name_like: String
    name_not_like: String
    ...etc
}

Internally I would like to let CategoriesFilter resolve to something like map[string]interface{} so I can dynamically evaluate the fields (not sure if this is already possible)

Not sure what the best solution is here my idea would be something like providing stubs:

input CategoriesFilter {}

Then in the .gqlgen.yml I could define a generator function:

models:
  CategoriesFilter:
    generator: github.com/myorg/mytool/generators.MySchemaGenerator

Where MySchemaGenerator is a function like:

func MySchemaGenerator(s *ast.Schema) (*ast.Definition, error)

Actual Behavior

Not possible AFAIK

I did not dug into the code very deep so I'm not sure if this is a viable solution.

I did some research if there is a tool which could prepare the schema in that manner but found nothing... But I you guys are aware of anything like that, I would be glad for any suggestions.

If anything is unclear I'm happy to answer any questions :) Furthermore If you think this is a feasible solution and it makes sense. I'm happy to implement it!

Thanks!

Best Regards,
Frido

[vektah]

I'm not sure this is the right way to think about the problem. The schema is the contract between the server and client, changes to it should be manual and it should be pretty explicit about what filters are supported. You would giving up the ability to link someone your schema as documentation

[fridolin-koch]

Okay, do you have any approach how you would solve this problem?

For small environments with only a hand full of entities you could do it by hand, but for enterprise applications with more than 50 or 100 entities it is really painful...

Thinking further about the problem I think a solution could be to generate a filters.graphql based on the original schema. I could use your parser for that and just generate the file. However this approach would require #383 or some manual merging.

[vektah]

That sounds like a great solution. You can call your schema generator as a second go generate stanza before gqlgen. 👍

I'll take a stab at multiple schemas, shouldn't be too hard

update: #389

[fridolin-koch]

In case anyone else is trying to implement this, and needs a hint..

The tool uses schema directives like

type Category implements Node @generateInputs(where: "CategoryWhereInput", orderBy: "CategoryOrderByInput") {
  id: ID!
  name: String!
  position: Int!
  enabled: Boolean!
}

Here is the go generate code:

// +build ignore

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"text/template"
	"time"

	"github.com/vektah/gqlparser"
	"github.com/vektah/gqlparser/ast"
)

const (
	directiveGenerateInputs = "generateInputs"
)

const (
	typeID           = "ID"
	typeString       = "String"
	typeInt          = "Int"
	typeBoolean      = "Boolean"
	typeDateTime     = "DateTime"
	typeNullDateTime = "NullDateTime"
)

const (
	operationNot           = "not"
	operationIn            = "in"
	operationNotIn         = "not_in"
	operationGt            = "gt"
	operationGte           = "gte"
	operationLt            = "lt"
	operationLte           = "lte"
	operationContains      = "contains"
	operationNotContains   = "not_contains"
	operationStartsWith    = "starts_with"
	operationNotStartsWith = "not_starts_with"
	operationEndsWith      = "ends_with"
	operationNotEndsWith   = "not_ends_with"
	operationLogicAnd      = "AND"
	operationLogicOr       = "OR"
	operationLogicNot      = "NOT"
)

var regexDirective = regexp.MustCompile(`@generateInputs\(where: "([A-Za-z0-9]+)", orderBy: "([A-Za-z0-9]+)"\)`)

type typeEntry struct {
	Where   *inputDefinition
	OrderBy *enumDefinition
}

type inputDefinition struct {
	Name   string
	For    string
	Fields []inputFieldDefinition
}

type enumDefinition struct {
	Name   string
	For    string
	Fields []string
}

type inputFieldDefinition struct {
	Name string
	Type string
}

var filterTemplate = template.Must(template.New("").Parse(`# Code generated by go generate; DO NOT EDIT THIS FILE.
# This file was generated at {{ .Time }}

{{ range .Entries }}
# Filter for {{ .Where.For }}
input {{ .Where.Name }} {
    {{- range .Where.Fields }}
	{{ .Name }}: {{ .Type }}
	{{- end }}
}

# Order By {{ .OrderBy.For }}
enum {{ .OrderBy.Name }} {
	{{- range .OrderBy.Fields }}
	{{ . }}
	{{- end }}
}
{{- end }}

`))

func main() {

	if len(os.Args) != 3 {
		log.Fatalf("Usage: %s <input-schema> <output-schema>", os.Args[0])
	}

	f, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	b, err := ioutil.ReadAll(f)
	if err != nil {
		log.Fatal(err)
	}

	// this is ugly but necessary for the parser to work
	mockSource := &ast.Source{Name: "mock.graphqls", Input: ""}
	matches := regexDirective.FindAllStringSubmatch(string(b), -1)
	for _, m := range matches {
		mockSource.Input += fmt.Sprintf("type %s {} enum %s {}", m[1], m[2])
	}

	// parse schema
	parsedSchema := gqlparser.MustLoadSchema(mockSource, &ast.Source{Name: os.Args[1], Input: string(b)})

	// inputs
	configMap := make(map[string]*typeConfig)
	var typeEntries []typeEntry

	// make config map
	for _, t := range parsedSchema.Types {
		tc, err := newTypeConfig(t)
		if err != nil {
			log.Fatal(err)
		}
		// no directive
		if tc == nil {
			continue
		}
		configMap[t.Name] = tc
	}
	// make filter inputs & order enums
	for _, t := range parsedSchema.Types {

		tc, ok := configMap[t.Name]
		if !ok {
			continue
		}

		// where input
		whereDef := &inputDefinition{
			Name: tc.Where,
			For:  t.Name,
		}

		// order by
		orderByDef := &enumDefinition{
			Name: tc.OrderBy,
			For:  t.Name,
		}

		for _, field := range t.Fields {
			wf, err := whereFields(field, configMap)
			if err != nil {
				log.Fatal(err)
			}
			whereDef.Fields = append(whereDef.Fields, wf...)

			// order by
			of, err := orderByFields(field)
			if err != nil {
				log.Fatal(err)
			}
			orderByDef.Fields = append(orderByDef.Fields, of...)
		}

		// add logic fields to whereDef
		for _, op := range []string{operationLogicAnd, operationLogicOr, operationLogicNot} {
			whereDef.Fields = append(whereDef.Fields, inputFieldDefinition{
				Name: op,
				Type: fmt.Sprintf("[%s!]", tc.Where),
			})
		}

		typeEntries = append(typeEntries, typeEntry{
			Where:   whereDef,
			OrderBy: orderByDef,
		})
	}

	// write to file
	o, err := os.Create(os.Args[2])
	if err != nil {
		log.Fatal(err)
	}
	defer o.Close()

	err = filterTemplate.Execute(o, struct {
		Time    string
		Entries []typeEntry
	}{
		Time:    time.Now().Format(time.RFC3339),
		Entries: typeEntries,
	})
	if err != nil {
		log.Fatal(err)
	}
	// output gql config entries
	fmt.Println("===== ADD THIS TO .gqlgen.yml =====")
	for _, e := range typeEntries {
		fmt.Printf("  %s:\n    model: map[string]interface{}\n  %s:\n    model: github.com/myorg/project/graph.OrderByInput\n", e.Where.Name, e.OrderBy.Name)
	}
}

func whereFields(field *ast.FieldDefinition, configMap map[string]*typeConfig) ([]inputFieldDefinition, error) {
	switch field.Type.NamedType {
	case typeID, typeInt, typeDateTime, typeNullDateTime:
		return expandWhereField(
			field.Name,
			field.Type.NamedType,
			operationNot,
			operationIn,
			operationNotIn,
			operationGt,
			operationGte,
			operationLt,
			operationLte,
		), nil
	case typeString:
		return expandWhereField(
			field.Name,
			field.Type.NamedType,
			operationNot,
			operationIn,
			operationNotIn,
			operationContains,
			operationNotContains,
			operationStartsWith,
			operationNotStartsWith,
			operationEndsWith,
			operationNotEndsWith,
		), nil
	default:
		if tc, ok := configMap[field.Type.NamedType]; ok {
			return []inputFieldDefinition{
				{
					Name: field.Name,
					Type: tc.Where,
				},
			}, nil
		}
	}
	return nil, nil
}

func expandWhereField(field, typeName string, ops ...string) []inputFieldDefinition {
	fields := make([]inputFieldDefinition, 0, len(ops)+1)
	// eq filter
	fields = append(fields, inputFieldDefinition{
		Name: field,
		Type: typeName,
	})
	for _, o := range ops {

		f := inputFieldDefinition{
			Name: fmt.Sprintf("%s_%s", field, o),
			Type: typeName,
		}
		// handle lists
		switch o {
		case operationIn, operationNotIn:
			f.Type = fmt.Sprintf("[%s!]", typeName)
		default:
			f.Type = typeName
		}

		fields = append(fields, f)
	}
	return fields
}

func orderByFields(field *ast.FieldDefinition) ([]string, error) {
	var fields []string

	switch field.Type.NamedType {
	case typeID, typeInt, typeDateTime, typeNullDateTime, typeString:
		fields = append(
			fields,
			fmt.Sprintf("%s_ASC", field.Name),
			fmt.Sprintf("%s_DESC", field.Name),
		)
	}

	return fields, nil
}

type typeConfig struct {
	Where   string
	OrderBy string
}

func newTypeConfig(t *ast.Definition) (*typeConfig, error) {
	// get directive definition
	def := t.Directives.ForName(directiveGenerateInputs)
	if def == nil {
		return nil, nil
	}
	// get where arg
	whereArg := def.Arguments.ForName("where")
	if whereArg == nil {
		return nil, fmt.Errorf("missing where argument for @%s", directiveGenerateInputs)
	}
	// get orderby arg
	orderByArg := def.Arguments.ForName("orderBy")
	if orderByArg == nil {
		return nil, fmt.Errorf("missing orderBy argument for @%s", directiveGenerateInputs)
	}
	// get where value
	whereName, err := whereArg.Value.Value(nil)
	if err != nil {
		return nil, fmt.Errorf("failed to obtain value from where arg: %s", err.Error())
	}
	// get orderBy value
	orderByName, err := orderByArg.Value.Value(nil)
	if err != nil {
		return nil, fmt.Errorf("failed to obtain value from orderBy arg: %s", err.Error())
	}
	return &typeConfig{
		Where:   whereName.(string),
		OrderBy: orderByName.(string),
	}, nil
}

[mia0x75]

In case anyone else is trying to implement this, and needs a hint..

The tool uses schema directives like

type Category implements Node @generateInputs(where: "CategoryWhereInput", orderBy: "CategoryOrderByInput") {
  id: ID!
  name: String!
  position: Int!
  enabled: Boolean!
}

Here is the go generate code:

// +build ignore

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"text/template"
	"time"

	"github.com/vektah/gqlparser"
	"github.com/vektah/gqlparser/ast"
)

const (
	directiveGenerateInputs = "generateInputs"
)

const (
	typeID           = "ID"
	typeString       = "String"
	typeInt          = "Int"
	typeBoolean      = "Boolean"
	typeDateTime     = "DateTime"
	typeNullDateTime = "NullDateTime"
)

const (
	operationNot           = "not"
	operationIn            = "in"
	operationNotIn         = "not_in"
	operationGt            = "gt"
	operationGte           = "gte"
	operationLt            = "lt"
	operationLte           = "lte"
	operationContains      = "contains"
	operationNotContains   = "not_contains"
	operationStartsWith    = "starts_with"
	operationNotStartsWith = "not_starts_with"
	operationEndsWith      = "ends_with"
	operationNotEndsWith   = "not_ends_with"
	operationLogicAnd      = "AND"
	operationLogicOr       = "OR"
	operationLogicNot      = "NOT"
)

var regexDirective = regexp.MustCompile(`@generateInputs\(where: "([A-Za-z0-9]+)", orderBy: "([A-Za-z0-9]+)"\)`)

type typeEntry struct {
	Where   *inputDefinition
	OrderBy *enumDefinition
}

type inputDefinition struct {
	Name   string
	For    string
	Fields []inputFieldDefinition
}

type enumDefinition struct {
	Name   string
	For    string
	Fields []string
}

type inputFieldDefinition struct {
	Name string
	Type string
}

var filterTemplate = template.Must(template.New("").Parse(`# Code generated by go generate; DO NOT EDIT THIS FILE.
# This file was generated at {{ .Time }}

{{ range .Entries }}
# Filter for {{ .Where.For }}
input {{ .Where.Name }} {
    {{- range .Where.Fields }}
	{{ .Name }}: {{ .Type }}
	{{- end }}
}

# Order By {{ .OrderBy.For }}
enum {{ .OrderBy.Name }} {
	{{- range .OrderBy.Fields }}
	{{ . }}
	{{- end }}
}
{{- end }}

`))

func main() {

	if len(os.Args) != 3 {
		log.Fatalf("Usage: %s <input-schema> <output-schema>", os.Args[0])
	}

	f, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	b, err := ioutil.ReadAll(f)
	if err != nil {
		log.Fatal(err)
	}

	// this is ugly but necessary for the parser to work
	mockSource := &ast.Source{Name: "mock.graphqls", Input: ""}
	matches := regexDirective.FindAllStringSubmatch(string(b), -1)
	for _, m := range matches {
		mockSource.Input += fmt.Sprintf("type %s {} enum %s {}", m[1], m[2])
	}

	// parse schema
	parsedSchema := gqlparser.MustLoadSchema(mockSource, &ast.Source{Name: os.Args[1], Input: string(b)})

	// inputs
	configMap := make(map[string]*typeConfig)
	var typeEntries []typeEntry

	// make config map
	for _, t := range parsedSchema.Types {
		tc, err := newTypeConfig(t)
		if err != nil {
			log.Fatal(err)
		}
		// no directive
		if tc == nil {
			continue
		}
		configMap[t.Name] = tc
	}
	// make filter inputs & order enums
	for _, t := range parsedSchema.Types {

		tc, ok := configMap[t.Name]
		if !ok {
			continue
		}

		// where input
		whereDef := &inputDefinition{
			Name: tc.Where,
			For:  t.Name,
		}

		// order by
		orderByDef := &enumDefinition{
			Name: tc.OrderBy,
			For:  t.Name,
		}

		for _, field := range t.Fields {
			wf, err := whereFields(field, configMap)
			if err != nil {
				log.Fatal(err)
			}
			whereDef.Fields = append(whereDef.Fields, wf...)

			// order by
			of, err := orderByFields(field)
			if err != nil {
				log.Fatal(err)
			}
			orderByDef.Fields = append(orderByDef.Fields, of...)
		}

		// add logic fields to whereDef
		for _, op := range []string{operationLogicAnd, operationLogicOr, operationLogicNot} {
			whereDef.Fields = append(whereDef.Fields, inputFieldDefinition{
				Name: op,
				Type: fmt.Sprintf("[%s!]", tc.Where),
			})
		}

		typeEntries = append(typeEntries, typeEntry{
			Where:   whereDef,
			OrderBy: orderByDef,
		})
	}

	// write to file
	o, err := os.Create(os.Args[2])
	if err != nil {
		log.Fatal(err)
	}
	defer o.Close()

	err = filterTemplate.Execute(o, struct {
		Time    string
		Entries []typeEntry
	}{
		Time:    time.Now().Format(time.RFC3339),
		Entries: typeEntries,
	})
	if err != nil {
		log.Fatal(err)
	}
	// output gql config entries
	fmt.Println("===== ADD THIS TO .gqlgen.yml =====")
	for _, e := range typeEntries {
		fmt.Printf("  %s:\n    model: map[string]interface{}\n  %s:\n    model: github.com/myorg/project/graph.OrderByInput\n", e.Where.Name, e.OrderBy.Name)
	}
}

func whereFields(field *ast.FieldDefinition, configMap map[string]*typeConfig) ([]inputFieldDefinition, error) {
	switch field.Type.NamedType {
	case typeID, typeInt, typeDateTime, typeNullDateTime:
		return expandWhereField(
			field.Name,
			field.Type.NamedType,
			operationNot,
			operationIn,
			operationNotIn,
			operationGt,
			operationGte,
			operationLt,
			operationLte,
		), nil
	case typeString:
		return expandWhereField(
			field.Name,
			field.Type.NamedType,
			operationNot,
			operationIn,
			operationNotIn,
			operationContains,
			operationNotContains,
			operationStartsWith,
			operationNotStartsWith,
			operationEndsWith,
			operationNotEndsWith,
		), nil
	default:
		if tc, ok := configMap[field.Type.NamedType]; ok {
			return []inputFieldDefinition{
				{
					Name: field.Name,
					Type: tc.Where,
				},
			}, nil
		}
	}
	return nil, nil
}

func expandWhereField(field, typeName string, ops ...string) []inputFieldDefinition {
	fields := make([]inputFieldDefinition, 0, len(ops)+1)
	// eq filter
	fields = append(fields, inputFieldDefinition{
		Name: field,
		Type: typeName,
	})
	for _, o := range ops {

		f := inputFieldDefinition{
			Name: fmt.Sprintf("%s_%s", field, o),
			Type: typeName,
		}
		// handle lists
		switch o {
		case operationIn, operationNotIn:
			f.Type = fmt.Sprintf("[%s!]", typeName)
		default:
			f.Type = typeName
		}

		fields = append(fields, f)
	}
	return fields
}

func orderByFields(field *ast.FieldDefinition) ([]string, error) {
	var fields []string

	switch field.Type.NamedType {
	case typeID, typeInt, typeDateTime, typeNullDateTime, typeString:
		fields = append(
			fields,
			fmt.Sprintf("%s_ASC", field.Name),
			fmt.Sprintf("%s_DESC", field.Name),
		)
	}

	return fields, nil
}

type typeConfig struct {
	Where   string
	OrderBy string
}

func newTypeConfig(t *ast.Definition) (*typeConfig, error) {
	// get directive definition
	def := t.Directives.ForName(directiveGenerateInputs)
	if def == nil {
		return nil, nil
	}
	// get where arg
	whereArg := def.Arguments.ForName("where")
	if whereArg == nil {
		return nil, fmt.Errorf("missing where argument for @%s", directiveGenerateInputs)
	}
	// get orderby arg
	orderByArg := def.Arguments.ForName("orderBy")
	if orderByArg == nil {
		return nil, fmt.Errorf("missing orderBy argument for @%s", directiveGenerateInputs)
	}
	// get where value
	whereName, err := whereArg.Value.Value(nil)
	if err != nil {
		return nil, fmt.Errorf("failed to obtain value from where arg: %s", err.Error())
	}
	// get orderBy value
	orderByName, err := orderByArg.Value.Value(nil)
	if err != nil {
		return nil, fmt.Errorf("failed to obtain value from orderBy arg: %s", err.Error())
	}
	return &typeConfig{
		Where:   whereName.(string),
		OrderBy: orderByName.(string),
	}, nil
}

could you please paste a simple input schema, thanks.