Remove lang support from outserv

chunker/json_parser.go

@@ -328,9 +328,6 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred
 					Namespace:   namespace,
 					ObjectValue: &pb.Value{Val: &pb.Value_DefaultVal{DefaultVal: x.Star}},
 				}
-				// Here we split predicate and lang directive (ex: "name@en"), if needed. With JSON
-				// mutations that's the only way to send language for a value.
-				nq.Predicate, nq.Lang = x.PredicateLang(nq.Predicate)
 				buf.Push(nq)
 				continue
 			}
@@ -345,10 +342,6 @@ func (buf *NQuadBuffer) mapToNquads(m map[string]interface{}, op int, parentPred
 			Namespace: namespace,
 		}
 
-		// Here we split predicate and lang directive (ex: "name@en"), if needed. With JSON
-		// mutations that's the only way to send language for a value.
-		nq.Predicate, nq.Lang = x.PredicateLang(nq.Predicate)
-
 		switch v := v.(type) {
 		// these int64/float64 cases are needed for FastParseJSON, which doesn't use json.Number
 		case int64, float64:

edgraph/server.go

@@ -686,13 +686,6 @@ func validateDQLSchemaForGraphQL(ctx context.Context,
 				"and is missing @upsert that is expected by the GraphQL API.",
 				x.ParseAttr(gqlPred.Predicate))
 		}
-		// if gqlSchema had @lang, then dqlSchema must have it. dqlSchema can't remove @lang.
-		// if gqlSchema didn't had @lang, it is allowed to dqlSchema to add it.
-		if gqlPred.Lang && !dqlPred.Lang {
-			return errors.Errorf("can't alter predicate %s as it is used by the GraphQL API, "+
-				"and is missing @lang that is expected by the GraphQL API.",
-				x.ParseAttr(gqlPred.Predicate))
-		}
 	}
 
 	// Step-2: validate types

gql/mutation.go

@@ -105,7 +105,6 @@ func (nq NQuad) createEdgePrototype(subjectUid uint64) *pb.DirectedEdge {
 		Entity:    subjectUid,
 		Attr:      nq.Predicate,
 		Namespace: nq.Namespace,
-		Lang:      nq.Lang,
 	}
 }
 
@@ -142,7 +141,6 @@ func (nq NQuad) ToDeletePredEdge() (*pb.DirectedEdge, error) {
 		Entity:    0,
 		Attr:      nq.Predicate,
 		Namespace: nq.Namespace,
-		Lang:      nq.Lang,
 		Op:        pb.DirectedEdge_DEL,
 	}
 

gql/parser.go

@@ -95,7 +95,6 @@ type ShortestPathArgs struct {
 type GroupByAttr struct {
 	Attr  string
 	Alias string
-	Langs []string
 }
 
 // FacetOrder stores ordering for single facet key.
@@ -162,7 +161,6 @@ type Arg struct {
 // Function holds the information about gql functions.
 type Function struct {
 	Attr       string
-	Lang       string // language of the attribute value
 	Name       string // Specifies the name of the function.
 	Args       []Arg  // Contains the arguments of the function.
 	UID        []uint64
@@ -1408,10 +1406,6 @@ func (f *FilterTree) stringHelper(buf *bytes.Buffer) {
 			if f.Func.IsCount || f.Func.IsValueVar || f.Func.IsLenVar {
 				x.Check2(buf.WriteRune(')'))
 			}
-			if len(f.Func.Lang) > 0 {
-				x.Check2(buf.WriteRune('@'))
-				x.Check2(buf.WriteString(f.Func.Lang))
-			}
 
 			for _, arg := range f.Func.Args {
 				if arg.IsValueVar {
@@ -1648,7 +1642,7 @@ func parseRegexArgs(val string) (regexArgs, error) {
 
 func parseFunction(it *lex.ItemIterator, gq *GraphQuery) (*Function, error) {
 	function := &Function{}
-	var expectArg, seenFuncArg, expectLang, isDollar bool
+	var expectArg, seenFuncArg, isDollar bool
 L:
 	for it.Next() {
 		item := it.Item()
@@ -1752,7 +1746,6 @@ L:
 					return nil, itemInFunc.Errorf("Invalid usage of '@' in function " +
 						"argument, must only appear immediately after attr.")
 				}
-				expectLang = true
 				continue
 			case itemMathOp:
 				val = itemInFunc.Val
@@ -1816,9 +1809,8 @@ L:
 				continue
 			}
 
-			if !expectArg && !expectLang {
-				return nil, itemInFunc.Errorf("Expected comma or language but got: %s",
-					itemInFunc.Val)
+			if !expectArg {
+				return nil, itemInFunc.Errorf("Expected comma but got: %s", itemInFunc.Val)
 			}
 
 			vname := collectName(it, itemInFunc.Val)
@@ -1861,13 +1853,6 @@ L:
 				}
 				function.Attr = val
 				attrItemsAgo = 0
-			case expectLang:
-				if val == "*" {
-					return nil, errors.Errorf(
-						"The * symbol cannot be used as a valid language inside functions")
-				}
-				function.Lang = val
-				expectLang = false
 			case function.Name != uidFunc:
 				// For UID function. we set g.UID
 				function.Args = append(function.Args, Arg{Value: val})
@@ -2038,20 +2023,17 @@ loop:
 				continue
 			}
 
-			var langs []string
 			items, err := it.Peek(1)
 			if err == nil && items[0].Typ == itemAt {
 				it.Next() // consume '@'
 				it.Next() // move forward
-				langs, err = parseLanguageList(it)
 				if err != nil {
 					return err
 				}
 			}
 			attrLang := GroupByAttr{
 				Attr:  val,
 				Alias: alias,
-				Langs: langs,
 			}
 			alias = ""
 			gq.GroupbyAttrs = append(gq.GroupbyAttrs, attrLang)
@@ -2342,14 +2324,6 @@ func parseDirective(it *lex.ItemIterator, curp *GraphQuery) error {
 		default:
 			return item.Errorf("Unknown directive [%s]", item.Val)
 		}
-	case len(curp.Attr) > 0 && len(curp.Langs) == 0:
-		// this is language list
-		if curp.Langs, err = parseLanguageList(it); err != nil {
-			return err
-		}
-		if len(curp.Langs) == 0 {
-			return item.Errorf("Expected at least 1 language in list for %s", curp.Attr)
-		}
 	default:
 		return item.Errorf("Expected directive or language list, got @%s", item.Val)
 	}
@@ -2648,8 +2622,7 @@ loop:
 					return nil, it.Errorf("Sorting by an attribute: [%s] "+
 						"can only be done on one language", val)
 				}
-				gq.Order = append(gq.Order,
-					&pb.Order{Attr: attr, Desc: key == "orderdesc", Langs: langs})
+				gq.Order = append(gq.Order, &pb.Order{Attr: attr, Desc: key == "orderdesc"})
 				order[val] = true
 				continue
 			}
@@ -3071,7 +3044,7 @@ func godeep(it *lex.ItemIterator, gq *GraphQuery) error {
 							"can only be done on one language", p.Val)
 					}
 					curp.Order = append(curp.Order,
-						&pb.Order{Attr: attr, Desc: p.Key == "orderdesc", Langs: langs})
+						&pb.Order{Attr: attr, Desc: p.Key == "orderdesc"})
 					order[p.Val] = true
 					continue
 				}

gql/query_test.go

@@ -1067,43 +1067,6 @@ func TestParseQueryWithMultipleVar(t *testing.T) {
 	require.Equal(t, []string{"B"}, res.QueryVars[2].Needs)
 }
 
-func TestParseQueryWithAttrLang(t *testing.T) {
-	query := `
-	{
-		me(func: uid(0x1)) {
-			name
-			friend(first:5, orderasc: name@en) {
-				name@en
-			}
-		}
-	}
-`
-	res, err := Parse(Request{Str: query})
-	require.NoError(t, err)
-	require.NotNil(t, res.Query)
-	require.Equal(t, 1, len(res.Query))
-	require.Equal(t, "name", res.Query[0].Children[1].Order[0].Attr)
-	require.Equal(t, []string{"en"}, res.Query[0].Children[1].Order[0].Langs)
-}
-
-func TestParseQueryWithAttrLang2(t *testing.T) {
-	query := `
-	{
-	  me(func:regexp(name, /^[a-zA-z]*[^Kk ]?[Nn]ight/), orderasc: name@en, first:5) {
-		name@en
-		name@de
-		name@it
-	  }
-	}
-`
-	res, err := Parse(Request{Str: query})
-	require.NoError(t, err)
-	require.NotNil(t, res.Query)
-	require.Equal(t, 1, len(res.Query))
-	require.Equal(t, "name", res.Query[0].Order[0].Attr)
-	require.Equal(t, []string{"en"}, res.Query[0].Order[0].Langs)
-}
-
 func TestParseMutationError(t *testing.T) {
 	query := `
 		mutation {

gql/state_test.go

@@ -222,67 +222,3 @@ func TestIRIRef(t *testing.T) {
 		t.Log(item.String())
 	}
 }
-
-func TestLangSupport(t *testing.T) {
-	input := `
-	query {
-		me(id: test) {
-			name@en
-		}
-	}
-	`
-	l := lex.Lexer{
-		Input: input,
-	}
-	l.Run(lexTopLevel)
-	it := l.NewIterator()
-	for it.Next() {
-		item := it.Item()
-		require.NotEqual(t, item.Typ, lex.ItemError)
-		t.Log(item.String())
-	}
-}
-
-func TestMultiLangSupport(t *testing.T) {
-	input := `
-	query {
-		me(id: test) {
-			name@en, name@en:ru:fr:de
-		}
-	}
-	`
-	l := lex.Lexer{
-		Input: input,
-	}
-	l.Run(lexTopLevel)
-	it := l.NewIterator()
-	for it.Next() {
-		item := it.Item()
-		require.NotEqual(t, item.Typ, lex.ItemError)
-		t.Log(item.String())
-	}
-}
-
-func TestNumberInLang(t *testing.T) {
-	input := `
-	{
-		q(func: eq(name@es-419, "aoeu")) {
-			name@.
-		}
-	}
-	`
-	l := lex.Lexer{
-		Input: input,
-	}
-	l.Run(lexTopLevel)
-	it := l.NewIterator()
-	gotEs := false
-	for it.Next() {
-		item := it.Item()
-		require.NotEqual(t, item.Typ, lex.ItemError)
-		if item.Val == "es-419" {
-			gotEs = true
-		}
-	}
-	require.True(t, gotEs)
-}

outserv/cmd/bulk/mapper.go

@@ -375,10 +375,7 @@ func (m *mapper) createPostings(nq gql.NQuad,
 	p := posting.NewPosting(de)
 	sch := m.schema.getSchema(x.NamespaceAttr(nq.GetNamespace(), nq.GetPredicate()))
 	if nq.GetObjectValue() != nil {
-		lang := de.GetLang()
 		switch {
-		case len(lang) > 0:
-			p.Uid = farm.Fingerprint64([]byte(lang))
 		case sch.List:
 			p.Uid = farm.Fingerprint64(de.Value)
 		default:
@@ -428,7 +425,7 @@ func (m *mapper) addIndexMapEntries(nq gql.NQuad, de *pb.DirectedEdge) {
 		x.Check(err)
 
 		// Extract tokens.
-		toks, err := tok.BuildTokens(schemaVal.Value, tok.GetTokenizerForLang(toker, nq.Lang))
+		toks, err := tok.BuildTokens(schemaVal.Value, toker)
 		x.Check(err)
 
 		attr := x.NamespaceAttr(nq.Namespace, nq.Predicate)

outserv/cmd/live/batch.go

@@ -202,7 +202,6 @@ func createUidEdge(nq *api.NQuad, sid, oid uint64) *pb.DirectedEdge {
 		Entity:    sid,
 		Attr:      nq.Predicate,
 		Namespace: nq.Namespace,
-		Lang:      nq.Lang,
 		ValueId:   oid,
 		ValueType: pb.Posting_UID,
 	}
@@ -213,7 +212,6 @@ func createValueEdge(nq *api.NQuad, sid uint64) (*pb.DirectedEdge, error) {
 		Entity:    sid,
 		Attr:      nq.Predicate,
 		Namespace: nq.Namespace,
-		Lang:      nq.Lang,
 	}
 	val, err := getTypeVal(nq.ObjectValue)
 	if err != nil {
@@ -227,11 +225,7 @@ func createValueEdge(nq *api.NQuad, sid uint64) (*pb.DirectedEdge, error) {
 
 func fingerprintEdge(t *pb.DirectedEdge, pred *predicate) uint64 {
 	var id uint64 = math.MaxUint64
-
-	// Value with a lang type.
-	if len(t.Lang) > 0 {
-		id = farm.Fingerprint64([]byte(t.Lang))
-	} else if pred.List {
+	if pred.List {
 		id = farm.Fingerprint64(t.Value)
 	}
 	return id
@@ -308,7 +302,7 @@ func (l *loader) conflictKeysForNQuad(nq *api.NQuad) ([]uint64, error) {
 		if err != nil {
 			errs = append(errs, err.Error())
 		}
-		toks, err := tok.BuildTokens(schemaVal.Value, tok.GetTokenizerForLang(token, nq.Lang))
+		toks, err := tok.BuildTokens(schemaVal.Value, token)
 		if err != nil {
 			errs = append(errs, err.Error())
 		}

posting/index.go

@@ -43,8 +43,6 @@ type indexMutationInfo struct {
 // index rune, for specific tokenizers.
 func indexTokens(ctx context.Context, info *indexMutationInfo) ([]string, error) {
 	attr := info.edge.Attr
-	lang := info.edge.GetLang()
-
 	schemaType, err := schema.State().TypeOf(attr)
 	if err != nil || !schemaType.IsScalar() {
 		return nil, errors.Errorf("Cannot index attribute %s of type object.", attr)
@@ -60,7 +58,7 @@ func indexTokens(ctx context.Context, info *indexMutationInfo) ([]string, error)
 
 	var tokens []string
 	for _, it := range info.tokenizers {
-		toks, err := tok.BuildTokens(sv.Value, tok.GetTokenizerForLang(it, lang))
+		toks, err := tok.BuildTokens(sv.Value, it)
 		if err != nil {
 			return tokens, err
 		}
@@ -498,11 +496,6 @@ func prefixesToDeleteTokensFor(attr, tokenizerName string, hasLang bool) ([][]by
 	if !ok {
 		return nil, errors.Errorf("Could not find valid tokenizer for %s", tokenizerName)
 	}
-	if hasLang {
-		// We just need the tokenizer identifier for ExactTokenizer having language.
-		// It will be same for all the language.
-		tokenizer = tok.GetTokenizerForLang(tokenizer, "en")
-	}
 	prefix = append(prefix, tokenizer.Identifier())
 	prefixes = append(prefixes, prefix)
 	// All the parts of any list that has been split into multiple parts.
@@ -928,7 +921,6 @@ func rebuildTokIndex(ctx context.Context, rb *IndexRebuild) error {
 				Value: p.Value,
 				Tid:   types.TypeID(p.ValType),
 			}
-			edge.Lang = string(p.LangTag)
 
 			for {
 				err := txn.addIndexMutations(ctx, &indexMutationInfo{