bolt: do not insert duplicate quads found in the same tx; fixes #675

graph/graphtest/graphtest.go

@@ -40,6 +40,9 @@ var graphTests = []struct {
 	test func(t testing.TB, gen testutil.DatabaseFunc, conf *Config)
 }{
 	{"load one quad", TestLoadOneQuad},
+	{"load dup", TestLoadDup},
+	{"load dup single", TestLoadDupSingle},
+	{"load dup raw", TestLoadDupRaw},
 	{"delete quad", TestDeleteQuad},
 	{"sizes", TestSizes},
 	{"iterator", TestIterator},
@@ -217,6 +220,72 @@ func TestLoadOneQuad(t testing.TB, gen testutil.DatabaseFunc, c *Config) {
 	ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), []quad.Quad{q}, false)
 }
 
+func testLoadDup(t testing.TB, gen testutil.DatabaseFunc, c *Config, single bool) {
+	qs, opts, closer := gen(t)
+	defer closer()
+
+	w := testutil.MakeWriter(t, qs, opts)
+
+	q := quad.Make(
+		"Something",
+		"points_to",
+		"Something Else",
+		"context",
+	)
+
+	if single {
+		err := w.AddQuadSet([]quad.Quad{q, q})
+		require.NoError(t, err)
+	} else {
+		err := w.AddQuad(q)
+		require.NoError(t, err)
+		err = w.AddQuad(q)
+		require.NoError(t, err)
+	}
+
+	exp := int64(5)
+	if c.NoPrimitives {
+		exp = 1
+	}
+	require.Equal(t, exp, qs.Size(), "Unexpected quadstore size")
+
+	ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), []quad.Quad{q}, false)
+}
+
+func TestLoadDup(t testing.TB, gen testutil.DatabaseFunc, c *Config) {
+	testLoadDup(t, gen, c, false)
+}
+
+func TestLoadDupSingle(t testing.TB, gen testutil.DatabaseFunc, c *Config) {
+	testLoadDup(t, gen, c, true)
+}
+
+func TestLoadDupRaw(t testing.TB, gen testutil.DatabaseFunc, c *Config) {
+	qs, _, closer := gen(t)
+	defer closer()
+
+	q := quad.Make(
+		"Something",
+		"points_to",
+		"Something Else",
+		"context",
+	)
+
+	err := qs.ApplyDeltas([]graph.Delta{
+		{Quad: q, Action: graph.Add},
+		{Quad: q, Action: graph.Add},
+	}, graph.IgnoreOpts{IgnoreDup: true})
+	require.NoError(t, err)
+
+	exp := int64(5)
+	if c.NoPrimitives {
+		exp = 1
+	}
+	require.Equal(t, exp, qs.Size(), "Unexpected quadstore size")
+
+	ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), []quad.Quad{q}, false)
+}
+
 func TestWriters(t *testing.T, gen testutil.DatabaseFunc, c *Config) {
 	for _, mis := range []bool{false, true} {
 		for _, dup := range []bool{false, true} {

graph/kv/indexing.go

@@ -362,17 +362,24 @@ func (qs *QuadStore) ApplyDeltas(in []graph.Delta, ignoreOpts graph.IgnoreOpts)
 	deltas.IncNode = nil
 	// resolve and insert all new quads
 	links := make([]proto.Primitive, 0, len(deltas.QuadAdd))
+	qadd := make(map[[4]uint64]struct{}, len(deltas.QuadAdd))
 	for _, q := range deltas.QuadAdd {
 		var link proto.Primitive
 		mustBeNew := false
-		for _, dir := range quad.Directions {
+		var qkey [4]uint64
+		for i, dir := range quad.Directions {
 			n, ok := nodes[q.Quad.Get(dir)]
 			if !ok {
 				continue
 			}
 			mustBeNew = mustBeNew || n.New
 			link.SetDirection(dir, n.ID)
+			qkey[i] = n.ID
 		}
+		if _, ok := qadd[qkey]; ok {
+			continue
+		}
+		qadd[qkey] = struct{}{}
 		if !mustBeNew {
 			p, err := qs.hasPrimitive(ctx, tx, &link, false)
 			if err != nil {
@@ -387,6 +394,7 @@ func (qs *QuadStore) ApplyDeltas(in []graph.Delta, ignoreOpts graph.IgnoreOpts)
 		}
 		links = append(links, link)
 	}
+	qadd = nil
 	deltas.QuadAdd = nil
 
 	qstart, err := qs.genIDs(ctx, tx, len(links))

graph/transaction.go

@@ -26,7 +26,12 @@ type Transaction struct {
 
 // NewTransaction initialize a new transaction.
 func NewTransaction() *Transaction {
-	return &Transaction{Deltas: make([]Delta, 0, 10), deltas: make(map[Delta]struct{}, 10)}
+	return NewTransactionN(10)
+}
+
+// NewTransactionN initialize a new transaction with a predefined capacity.
+func NewTransactionN(n int) *Transaction {
+	return &Transaction{Deltas: make([]Delta, 0, n), deltas: make(map[Delta]struct{}, n)}
 }
 
 // AddQuad adds a new quad to the transaction if it is not already present in it.

writer/single.go

@@ -62,14 +62,11 @@ func (s *Single) AddQuad(q quad.Quad) error {
 }
 
 func (s *Single) AddQuadSet(set []quad.Quad) error {
-	deltas := make([]graph.Delta, len(set))
-	for i, q := range set {
-		deltas[i] = graph.Delta{
-			Quad:   q,
-			Action: graph.Add,
-		}
+	tx := graph.NewTransactionN(len(set))
+	for _, q := range set {
+		tx.AddQuad(q)
 	}
-	return s.qs.ApplyDeltas(deltas, s.ignoreOpts)
+	return s.qs.ApplyDeltas(tx.Deltas, s.ignoreOpts)
 }
 
 func (s *Single) RemoveQuad(q quad.Quad) error {