From 1e5c8a445cf57eeb0e211afb0daa37eb7dac5ff4 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 8 May 2014 15:01:02 -0700 Subject: [PATCH 01/30] consul: Adding new session tables --- consul/state_store.go | 74 +++++++++++++++++++++++++++++++++------ consul/structs/structs.go | 8 +++++ 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/consul/state_store.go b/consul/state_store.go index a5e99162eab7..1020cb8826e4 100644 --- a/consul/state_store.go +++ b/consul/state_store.go @@ -17,6 +17,8 @@ const ( dbServices = "services" dbChecks = "checks" dbKVS = "kvs" + dbSessions = "sessions" + dbSessionChecks = "sessionChecks" dbMaxMapSize32bit uint64 = 512 * 1024 * 1024 // 512MB maximum size dbMaxMapSize64bit uint64 = 32 * 1024 * 1024 * 1024 // 32GB maximum size ) @@ -29,16 +31,18 @@ const ( // implementation uses the Lightning Memory-Mapped Database (MDB). // This gives us Multi-Version Concurrency Control for "free" type StateStore struct { - logger *log.Logger - path string - env *mdb.Env - nodeTable *MDBTable - serviceTable *MDBTable - checkTable *MDBTable - kvsTable *MDBTable - tables MDBTables - watch map[*MDBTable]*NotifyGroup - queryTables map[string]MDBTables + logger *log.Logger + path string + env *mdb.Env + nodeTable *MDBTable + serviceTable *MDBTable + checkTable *MDBTable + kvsTable *MDBTable + sessionTable *MDBTable + sessionChecksTable *MDBTable + tables MDBTables + watch map[*MDBTable]*NotifyGroup + queryTables map[string]MDBTables } // StateSnapshot is used to provide a point-in-time snapshot @@ -49,6 +53,15 @@ type StateSnapshot struct { lastIndex uint64 } +// sessionCheck is used to create a many-to-one table such +// that each check registered by a session can be mapped back +// to the session row. +type sessionCheck struct { + Node string + CheckID string + Session string +} + // Close is used to abort the transaction and allow for cleanup func (s *StateSnapshot) Close() error { s.tx.Abort() @@ -219,8 +232,47 @@ func (s *StateStore) initialize() error { }, } + s.sessionTable = &MDBTable{ + Name: dbSessions, + Indexes: map[string]*MDBIndex{ + "id": &MDBIndex{ + Unique: true, + Fields: []string{"ID"}, + }, + "node": &MDBIndex{ + AllowBlank: true, + Fields: []string{"Node"}, + }, + }, + Decoder: func(buf []byte) interface{} { + out := new(structs.Session) + if err := structs.Decode(buf, out); err != nil { + panic(err) + } + return out + }, + } + + s.sessionChecksTable = &MDBTable{ + Name: dbSessionChecks, + Indexes: map[string]*MDBIndex{ + "id": &MDBIndex{ + Unique: true, + Fields: []string{"Node", "CheckID", "Session"}, + }, + }, + Decoder: func(buf []byte) interface{} { + out := new(sessionCheck) + if err := structs.Decode(buf, out); err != nil { + panic(err) + } + return out + }, + } + // Store the set of tables - s.tables = []*MDBTable{s.nodeTable, s.serviceTable, s.checkTable, s.kvsTable} + s.tables = []*MDBTable{s.nodeTable, s.serviceTable, s.checkTable, + s.kvsTable, s.sessionTable, s.sessionChecksTable} for _, table := range s.tables { table.Env = s.env table.Encoder = encoder diff --git a/consul/structs/structs.go b/consul/structs/structs.go index 2a672565f73c..61e575a2572e 100644 --- a/consul/structs/structs.go +++ b/consul/structs/structs.go @@ -335,6 +335,14 @@ type IndexedKeyList struct { QueryMeta } +// Session is used to represent an open session in the KV store. +// This issued to associate node checks with acquired locks. +type Session struct { + ID string + Node string + Checks []string +} + // Decode is used to decode a MsgPack encoded object func Decode(buf []byte, out interface{}) error { var handle codec.MsgpackHandle From 26ee0e3c7642c231a0763c98eb993a8aa7b8b8e4 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 8 May 2014 16:31:03 -0700 Subject: [PATCH 02/30] consul: Adding util method to generate a UUID --- consul/util.go | 16 ++++++++++++++++ consul/util_test.go | 17 +++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/consul/util.go b/consul/util.go index 977d38a04f5c..f830086c04a1 100644 --- a/consul/util.go +++ b/consul/util.go @@ -1,6 +1,7 @@ package consul import ( + crand "crypto/rand" "encoding/binary" "fmt" "github.com/hashicorp/serf/serf" @@ -160,3 +161,18 @@ func runtimeStats() map[string]string { "cpu_count": strconv.FormatInt(int64(runtime.NumCPU()), 10), } } + +// generateUUID is used to generate a random UUID +func generateUUID() string { + buf := make([]byte, 16) + if _, err := crand.Read(buf); err != nil { + panic(fmt.Errorf("failed to read random bytes: %v", err)) + } + + return fmt.Sprintf("%08x-%04x-%04x-%04x-%12x", + buf[0:4], + buf[4:6], + buf[6:8], + buf[8:10], + buf[10:16]) +} diff --git a/consul/util_test.go b/consul/util_test.go index edd73fdf69ad..589404a341e9 100644 --- a/consul/util_test.go +++ b/consul/util_test.go @@ -2,6 +2,7 @@ package consul import ( "github.com/hashicorp/serf/serf" + "regexp" "testing" ) @@ -75,3 +76,19 @@ func TestByteConversion(t *testing.T) { t.Fatalf("no match") } } + +func TestGenerateUUID(t *testing.T) { + prev := generateUUID() + for i := 0; i < 100; i++ { + id := generateUUID() + if prev == id { + t.Fatalf("Should get a new ID!") + } + + matched, err := regexp.MatchString( + "[\\da-f]{8}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{12}", id) + if !matched || err != nil { + t.Fatalf("expected match %s %v %s", id, matched, err) + } + } +} From 836e1b5a10312f4c1b9c4f0539dcda9eeec32146 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 8 May 2014 16:54:04 -0700 Subject: [PATCH 03/30] consul: Adding SessionCreate and SessionRestore --- consul/mdb_table.go | 13 ++++ consul/state_store.go | 149 ++++++++++++++++++++++++++++++++++---- consul/structs/structs.go | 7 +- 3 files changed, 152 insertions(+), 17 deletions(-) diff --git a/consul/mdb_table.go b/consul/mdb_table.go index 96c1fad883f5..f70923cb364f 100644 --- a/consul/mdb_table.go +++ b/consul/mdb_table.go @@ -734,6 +734,19 @@ func (t *MDBTable) SetLastIndexTxn(tx *MDBTxn, index uint64) error { return tx.tx.Put(tx.dbis[t.Name], encRowId, encIndex, 0) } +// SetMaxLastIndexTxn is used to set the last index within a transaction +// if it exceeds the current maximum +func (t *MDBTable) SetMaxLastIndexTxn(tx *MDBTxn, index uint64) error { + current, err := t.LastIndexTxn(tx) + if err != nil { + return err + } + if index > current { + return t.SetLastIndexTxn(tx, index) + } + return nil +} + // StartTxn is used to create a transaction that spans a list of tables func (t MDBTables) StartTxn(readonly bool) (*MDBTxn, error) { var tx *MDBTxn diff --git a/consul/state_store.go b/consul/state_store.go index 1020cb8826e4..3fff27e9afec 100644 --- a/consul/state_store.go +++ b/consul/state_store.go @@ -31,18 +31,18 @@ const ( // implementation uses the Lightning Memory-Mapped Database (MDB). // This gives us Multi-Version Concurrency Control for "free" type StateStore struct { - logger *log.Logger - path string - env *mdb.Env - nodeTable *MDBTable - serviceTable *MDBTable - checkTable *MDBTable - kvsTable *MDBTable - sessionTable *MDBTable - sessionChecksTable *MDBTable - tables MDBTables - watch map[*MDBTable]*NotifyGroup - queryTables map[string]MDBTables + logger *log.Logger + path string + env *mdb.Env + nodeTable *MDBTable + serviceTable *MDBTable + checkTable *MDBTable + kvsTable *MDBTable + sessionTable *MDBTable + sessionCheckTable *MDBTable + tables MDBTables + watch map[*MDBTable]*NotifyGroup + queryTables map[string]MDBTables } // StateSnapshot is used to provide a point-in-time snapshot @@ -253,7 +253,7 @@ func (s *StateStore) initialize() error { }, } - s.sessionChecksTable = &MDBTable{ + s.sessionCheckTable = &MDBTable{ Name: dbSessionChecks, Indexes: map[string]*MDBIndex{ "id": &MDBIndex{ @@ -272,7 +272,7 @@ func (s *StateStore) initialize() error { // Store the set of tables s.tables = []*MDBTable{s.nodeTable, s.serviceTable, s.checkTable, - s.kvsTable, s.sessionTable, s.sessionChecksTable} + s.kvsTable, s.sessionTable, s.sessionCheckTable} for _, table := range s.tables { table.Env = s.env table.Encoder = encoder @@ -1091,6 +1091,127 @@ func (s *StateStore) KVSCheckAndSet(index uint64, d *structs.DirEntry) (bool, er return true, tx.Commit() } +// SessionCreate is used to create a new session. The +// ID will be populated on a successful return +func (s *StateStore) SessionCreate(index uint64, session *structs.Session) error { + // Assign the create index + session.CreateIndex = index + + // Start the transaction + tables := MDBTables{s.nodeTable, s.checkTable, + s.sessionTable, s.sessionCheckTable} + tx, err := tables.StartTxn(false) + if err != nil { + panic(fmt.Errorf("Failed to start txn: %v", err)) + } + defer tx.Abort() + + // Verify that the node exists + res, err := s.nodeTable.GetTxn(tx, "id", session.Node) + if err != nil { + return err + } + if len(res) == 0 { + return fmt.Errorf("Missing node registration") + } + + // Verify that the checks exist and are not critical + for _, checkId := range session.Checks { + res, err := s.checkTable.GetTxn(tx, "id", session.Node, checkId) + if err != nil { + return err + } + if len(res) == 0 { + return fmt.Errorf("Missing check '%s' registration", checkId) + } + chk := res[0].(*structs.HealthCheck) + if chk.Status == structs.HealthCritical { + return fmt.Errorf("Check '%s' is in %s state", checkId, chk.Status) + } + } + + // Generate a new session ID, verify uniqueness + session.ID = generateUUID() + for { + res, err = s.sessionTable.GetTxn(tx, "id", session.ID) + if err != nil { + return err + } + // Quit if this ID is unique + if len(res) == 0 { + break + } + } + + // Insert the session + if err := s.sessionTable.InsertTxn(tx, session); err != nil { + return err + } + + // Insert the check mappings + sCheck := sessionCheck{Node: session.Node, Session: session.ID} + for _, checkID := range session.Checks { + sCheck.CheckID = checkID + if err := s.sessionCheckTable.InsertTxn(tx, &sCheck); err != nil { + return err + } + } + + // Trigger the update notifications + if err := s.sessionTable.SetLastIndexTxn(tx, index); err != nil { + return err + } + defer s.watch[s.sessionTable].Notify() + + if err := s.sessionCheckTable.SetLastIndexTxn(tx, index); err != nil { + return err + } + defer s.watch[s.sessionCheckTable].Notify() + + return tx.Commit() +} + +// SessionRestore is used to restore a session. It should only be used when +// doing a restore, otherwise SessionCreate should be used. +func (s *StateStore) SessionRestore(session *structs.Session) error { + // Start the transaction + tables := MDBTables{s.nodeTable, s.checkTable, + s.sessionTable, s.sessionCheckTable} + tx, err := tables.StartTxn(false) + if err != nil { + panic(fmt.Errorf("Failed to start txn: %v", err)) + } + defer tx.Abort() + + // Insert the session + if err := s.sessionTable.InsertTxn(tx, session); err != nil { + return err + } + + // Insert the check mappings + sCheck := sessionCheck{Node: session.Node, Session: session.ID} + for _, checkID := range session.Checks { + sCheck.CheckID = checkID + if err := s.sessionCheckTable.InsertTxn(tx, &sCheck); err != nil { + return err + } + } + + // Trigger the update notifications + index := session.CreateIndex + if err := s.sessionTable.SetMaxLastIndexTxn(tx, index); err != nil { + return err + } + defer s.watch[s.sessionTable].Notify() + + if err := s.sessionCheckTable.SetMaxLastIndexTxn(tx, index); err != nil { + return err + } + defer s.watch[s.sessionCheckTable].Notify() + + return tx.Commit() +} + // Snapshot is used to create a point in time snapshot func (s *StateStore) Snapshot() (*StateSnapshot, error) { // Begin a new txn on all tables diff --git a/consul/structs/structs.go b/consul/structs/structs.go index 61e575a2572e..f1bbd60bdd12 100644 --- a/consul/structs/structs.go +++ b/consul/structs/structs.go @@ -338,9 +338,10 @@ type IndexedKeyList struct { // Session is used to represent an open session in the KV store. // This issued to associate node checks with acquired locks. type Session struct { - ID string - Node string - Checks []string + ID string + Node string + Checks []string + CreateIndex uint64 } // Decode is used to decode a MsgPack encoded object From 4695da527a82be525ad2d2e3042a2dfd6f981411 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 8 May 2014 17:18:59 -0700 Subject: [PATCH 04/30] consul: Adding session lookup methods --- consul/state_store.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/consul/state_store.go b/consul/state_store.go index 3fff27e9afec..2dfb7ec21e3e 100644 --- a/consul/state_store.go +++ b/consul/state_store.go @@ -299,6 +299,9 @@ func (s *StateStore) initialize() error { "KVSGet": MDBTables{s.kvsTable}, "KVSList": MDBTables{s.kvsTable}, "KVSListKeys": MDBTables{s.kvsTable}, + "SessionGet": MDBTables{s.sessionTable}, + "SessionList": MDBTables{s.sessionTable}, + "NodeSessions": MDBTables{s.sessionTable}, } return nil } @@ -1212,6 +1215,36 @@ func (s *StateStore) SessionRestore(session *structs.Session) error { return tx.Commit() } +// SessionGet is used to get a session entry +func (s *StateStore) SessionGet(id string) (uint64, *structs.Session, error) { + idx, res, err := s.sessionTable.Get("id", id) + var d *structs.Session + if len(res) > 0 { + d = res[0].(*structs.Session) + } + return idx, d, err +} + +// SessionList is used to list all the open sessions +func (s *StateStore) SessionList() (uint64, []*structs.Session, error) { + idx, res, err := s.sessionTable.Get("id") + out := make([]*structs.Session, len(res)) + for i, raw := range res { + out[i] = raw.(*structs.Session) + } + return idx, out, err +} + +// NodeSessions is used to list all the open sessions for a node +func (s *StateStore) NodeSessions(node string) (uint64, []*structs.Session, error) { + idx, res, err := s.sessionTable.Get("node", node) + out := make([]*structs.Session, len(res)) + for i, raw := range res { + out[i] = raw.(*structs.Session) + } + return idx, out, err +} + // Snapshot is used to create a point in time snapshot func (s *StateStore) Snapshot() (*StateSnapshot, error) { // Begin a new txn on all tables From 65ccddcd6b1130edd64867157be73a0b17836314 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 8 May 2014 17:25:22 -0700 Subject: [PATCH 05/30] consul: Adding SessionDestroy --- consul/state_store.go | 55 +++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/consul/state_store.go b/consul/state_store.go index 2dfb7ec21e3e..02567b686856 100644 --- a/consul/state_store.go +++ b/consul/state_store.go @@ -1165,12 +1165,6 @@ func (s *StateStore) SessionCreate(index uint64, session *structs.Session) error return err } defer s.watch[s.sessionTable].Notify() - - if err := s.sessionCheckTable.SetLastIndexTxn(tx, index); err != nil { - return err - } - defer s.watch[s.sessionCheckTable].Notify() - return tx.Commit() } @@ -1206,12 +1200,6 @@ func (s *StateStore) SessionRestore(session *structs.Session) error { return err } defer s.watch[s.sessionTable].Notify() - - if err := s.sessionCheckTable.SetMaxLastIndexTxn(tx, index); err != nil { - return err - } - defer s.watch[s.sessionCheckTable].Notify() - return tx.Commit() } @@ -1245,6 +1233,49 @@ func (s *StateStore) NodeSessions(node string) (uint64, []*structs.Session, erro return idx, out, err } +// SessionDelete is used to destroy a session. +func (s *StateStore) SessionDestroy(index uint64, id string) error { + // Start the transaction + tables := MDBTables{s.sessionTable, s.sessionCheckTable} + tx, err := tables.StartTxn(false) + if err != nil { + panic(fmt.Errorf("Failed to start txn: %v", err)) + } + defer tx.Abort() + + // Get the session + res, err := s.sessionTable.GetTxn(tx, "id", id) + if err != nil { + return err + } + + // Quit if this session does not exist + if len(res) == 0 { + return nil + } + session := res[0].(*structs.Session) + + // Nuke the session + if _, err := s.sessionTable.DeleteTxn(tx, "id", id); err != nil { + return err + } + + // Delete the check mappings + for _, checkID := range session.Checks { + if _, err := s.sessionCheckTable.DeleteTxn(tx, "id", + session.Node, checkID, id); err != nil { + return err + } + } + + // Trigger the update notifications + if err := s.sessionTable.SetLastIndexTxn(tx, index); err != nil { + return err + } + defer s.watch[s.sessionTable].Notify() + return tx.Commit() +} + // Snapshot is used to create a point in time snapshot func (s *StateStore) Snapshot() (*StateSnapshot, error) { // Begin a new txn on all tables From 55f80d01080f28546f4aed7f5e4868ab9bd60e34 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 8 May 2014 20:45:17 -0700 Subject: [PATCH 06/30] bench: minor updates --- bench/bench.json | 10 +++++----- bench/conf/common.json | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bench/bench.json b/bench/bench.json index 492cca5fdb15..5dfb67bd9369 100644 --- a/bench/bench.json +++ b/bench/bench.json @@ -9,7 +9,7 @@ "api_key": "{{ user `do_api_key` }}", "client_id": "{{ user `do_client_id` }}", "region_id": "1", - "size_id": "61", + "size_id": "66", "image_id": "3101045", "snapshot_name": "bench-bootstrap-{{ isotime }}", "name": "bootstrap" @@ -19,7 +19,7 @@ "api_key": "{{ user `do_api_key` }}", "client_id": "{{ user `do_client_id` }}", "region_id": "1", - "size_id": "61", + "size_id": "66", "image_id": "3101045", "snapshot_name": "bench-server-{{ isotime }}", "name": "server" @@ -29,7 +29,7 @@ "api_key": "{{ user `do_api_key` }}", "client_id": "{{ user `do_client_id` }}", "region_id": "1", - "size_id": "61", + "size_id": "66", "image_id": "3101045", "snapshot_name": "bench-worker-{{ isotime }}", "name": "worker" @@ -73,8 +73,8 @@ { "type": "shell", "inline": [ - "curl https://s3.amazonaws.com/hc-ops/boom_linux_amd64 -o /usr/bin/boom", - "chmod +x /usr/bin/boom" + "curl https://s3.amazonaws.com/hc-ops/boom_linux_amd64 -o /usr/local/bin/boom", + "chmod +x /usr/local/bin/boom" ] }, { diff --git a/bench/conf/common.json b/bench/conf/common.json index 2a3fa5d6a852..78ce3bed7041 100644 --- a/bench/conf/common.json +++ b/bench/conf/common.json @@ -1,4 +1,5 @@ { "data_dir": "/var/lib/consul", + "enable_debug": true, "log_level": "info" } From 222996050d492ed6845ce847340a86669babc77f Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Wed, 14 May 2014 17:30:05 -0700 Subject: [PATCH 07/30] consul: Adding session tests --- consul/state_store_test.go | 158 +++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/consul/state_store_test.go b/consul/state_store_test.go index cdf259164e3c..41f9375fdcc3 100644 --- a/consul/state_store_test.go +++ b/consul/state_store_test.go @@ -1561,3 +1561,161 @@ func TestKVSDeleteTree(t *testing.T) { t.Fatalf("bad: %v", ents) } } + +func TestSessionCreate(t *testing.T) { + store, err := testStateStore() + if err != nil { + t.Fatalf("err: %v", err) + } + defer store.Close() + + if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil { + t.Fatalf("err: %v") + } + check := &structs.HealthCheck{ + Node: "foo", + CheckID: "bar", + Status: structs.HealthPassing, + } + if err := store.EnsureCheck(13, check); err != nil { + t.Fatalf("err: %v") + } + + session := &structs.Session{ + Node: "foo", + Checks: []string{"bar"}, + } + + if err := store.SessionCreate(1000, session); err != nil { + t.Fatalf("err: %v", err) + } + + if session.ID == "" { + t.Fatalf("bad: %v", session) + } + + if session.CreateIndex != 1000 { + t.Fatalf("bad: %v", session) + } +} + +func TestSessionCreate_Invalid(t *testing.T) { + store, err := testStateStore() + if err != nil { + t.Fatalf("err: %v", err) + } + defer store.Close() + + // No node registered + session := &structs.Session{ + Node: "foo", + Checks: []string{"bar"}, + } + if err := store.SessionCreate(1000, session); err.Error() != "Missing node registration" { + t.Fatalf("err: %v", err) + } + + // Check not registered + if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil { + t.Fatalf("err: %v") + } + if err := store.SessionCreate(1000, session); err.Error() != "Missing check 'bar' registration" { + t.Fatalf("err: %v", err) + } + + // Unhealthy check + check := &structs.HealthCheck{ + Node: "foo", + CheckID: "bar", + Status: structs.HealthCritical, + } + if err := store.EnsureCheck(13, check); err != nil { + t.Fatalf("err: %v") + } + if err := store.SessionCreate(1000, session); err.Error() != "Check 'bar' is in critical state" { + t.Fatalf("err: %v", err) + } +} + +func TestSession_Lookups(t *testing.T) { + store, err := testStateStore() + if err != nil { + t.Fatalf("err: %v", err) + } + defer store.Close() + + // Create a session + if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil { + t.Fatalf("err: %v") + } + session := &structs.Session{ + Node: "foo", + } + if err := store.SessionCreate(1000, session); err != nil { + t.Fatalf("err: %v", err) + } + + // Lookup by ID + idx, s2, err := store.SessionGet(session.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + if idx != 1000 { + t.Fatalf("bad: %v", idx) + } + if !reflect.DeepEqual(s2, session) { + t.Fatalf("bad: %v", s2) + } + + // Create many sessions + ids := []string{session.ID} + for i := 0; i < 10; i++ { + session := &structs.Session{ + Node: "foo", + } + if err := store.SessionCreate(uint64(1000+i), session); err != nil { + t.Fatalf("err: %v", err) + } + ids = append(ids, session.ID) + } + + // List all + idx, all, err := store.SessionList() + if err != nil { + t.Fatalf("err: %v", err) + } + if idx != 1009 { + t.Fatalf("bad: %v", idx) + } + + // Retrieve the ids + var out []string + for _, s := range all { + out = append(out, s.ID) + } + + sort.Strings(ids) + sort.Strings(out) + if !reflect.DeepEqual(ids, out) { + t.Fatalf("bad: %v %v", ids, out) + } + + // List by node + idx, nodes, err := store.NodeSessions("foo") + if err != nil { + t.Fatalf("err: %v", err) + } + if idx != 1009 { + t.Fatalf("bad: %v", idx) + } + + // Check again for the node list + out = nil + for _, s := range nodes { + out = append(out, s.ID) + } + sort.Strings(out) + if !reflect.DeepEqual(ids, out) { + t.Fatalf("bad: %v %v", ids, out) + } +} From 8fec0c093c4ad60d3f5f78aa4beaddea49a01c16 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 15 May 2014 11:50:30 -0700 Subject: [PATCH 08/30] consul: Adding Defer to MDBTxn --- consul/mdb_table.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/consul/mdb_table.go b/consul/mdb_table.go index f70923cb364f..c4c84b0dc9ce 100644 --- a/consul/mdb_table.go +++ b/consul/mdb_table.go @@ -63,6 +63,7 @@ type MDBTxn struct { readonly bool tx *mdb.Txn dbis map[string]mdb.DBI + after []func() } // Abort is used to close the transaction @@ -74,7 +75,19 @@ func (t *MDBTxn) Abort() { // Commit is used to commit a transaction func (t *MDBTxn) Commit() error { - return t.tx.Commit() + if err := t.tx.Commit(); err != nil { + return err + } + for _, f := range t.after { + f() + } + t.after = nil + return nil +} + +// Defer is used to defer a function call until a successful commit +func (t *MDBTxn) Defer(f func()) { + t.after = append(t.after, f) } type IndexFunc func(*MDBIndex, []string) string From 5e0639a0ebc073c0c21b38b88b84adc8dde7f407 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 15 May 2014 11:51:31 -0700 Subject: [PATCH 09/30] consul: Adding session invalidation --- consul/state_store.go | 102 +++++++++++++++++----- consul/state_store_test.go | 171 ++++++++++++++++++++++++++++++++++++- 2 files changed, 250 insertions(+), 23 deletions(-) diff --git a/consul/state_store.go b/consul/state_store.go index 02567b686856..730ff3e875c6 100644 --- a/consul/state_store.go +++ b/consul/state_store.go @@ -366,8 +366,7 @@ func (s *StateStore) Nodes() (uint64, structs.Nodes) { // EnsureService is used to ensure a given node exposes a service func (s *StateStore) EnsureService(index uint64, node string, ns *structs.NodeService) error { - tables := MDBTables{s.nodeTable, s.serviceTable} - tx, err := tables.StartTxn(false) + tx, err := s.tables.StartTxn(false) if err != nil { panic(fmt.Errorf("Failed to start txn: %v", err)) } @@ -461,8 +460,7 @@ func (s *StateStore) parseNodeServices(tables MDBTables, tx *MDBTxn, name string // DeleteNodeService is used to delete a node service func (s *StateStore) DeleteNodeService(index uint64, node, id string) error { - tables := MDBTables{s.serviceTable, s.checkTable} - tx, err := tables.StartTxn(false) + tx, err := s.tables.StartTxn(false) if err != nil { panic(fmt.Errorf("Failed to start txn: %v", err)) } @@ -476,6 +474,19 @@ func (s *StateStore) DeleteNodeService(index uint64, node, id string) error { } defer s.watch[s.serviceTable].Notify() } + + // Invalidate any sessions using these checks + checks, err := s.checkTable.GetTxn(tx, "node", node, id) + if err != nil { + return err + } + for _, c := range checks { + check := c.(*structs.HealthCheck) + if err := s.invalidateCheck(index, tx, node, check.CheckID); err != nil { + return err + } + } + if n, err := s.checkTable.DeleteTxn(tx, "node", node, id); err != nil { return err } else if n > 0 { @@ -489,13 +500,17 @@ func (s *StateStore) DeleteNodeService(index uint64, node, id string) error { // DeleteNode is used to delete a node and all it's services func (s *StateStore) DeleteNode(index uint64, node string) error { - tables := MDBTables{s.nodeTable, s.serviceTable, s.checkTable} - tx, err := tables.StartTxn(false) + tx, err := s.tables.StartTxn(false) if err != nil { panic(fmt.Errorf("Failed to start txn: %v", err)) } defer tx.Abort() + // Invalidate any sessions held by the node + if err := s.invalidateNode(index, tx, node); err != nil { + return err + } + if n, err := s.serviceTable.DeleteTxn(tx, "id", node); err != nil { return err } else if n > 0 { @@ -633,8 +648,7 @@ func (s *StateStore) EnsureCheck(index uint64, check *structs.HealthCheck) error } // Start the txn - tables := MDBTables{s.nodeTable, s.serviceTable, s.checkTable} - tx, err := tables.StartTxn(false) + tx, err := s.tables.StartTxn(false) if err != nil { panic(fmt.Errorf("Failed to start txn: %v", err)) } @@ -663,6 +677,14 @@ func (s *StateStore) EnsureCheck(index uint64, check *structs.HealthCheck) error check.ServiceName = srv.ServiceName } + // Invalidate any sessions if status is critical + if check.Status == structs.HealthCritical { + err := s.invalidateCheck(index, tx, check.Node, check.CheckID) + if err != nil { + return err + } + } + // Ensure the check is set if err := s.checkTable.InsertTxn(tx, check); err != nil { return err @@ -676,12 +698,17 @@ func (s *StateStore) EnsureCheck(index uint64, check *structs.HealthCheck) error // DeleteNodeCheck is used to delete a node health check func (s *StateStore) DeleteNodeCheck(index uint64, node, id string) error { - tx, err := s.checkTable.StartTxn(false, nil) + tx, err := s.tables.StartTxn(false) if err != nil { return err } defer tx.Abort() + // Invalidate any sessions held by this check + if err := s.invalidateCheck(index, tx, node, id); err != nil { + return err + } + if n, err := s.checkTable.DeleteTxn(tx, "id", node, id); err != nil { return err } else if n > 0 { @@ -1101,9 +1128,7 @@ func (s *StateStore) SessionCreate(index uint64, session *structs.Session) error session.CreateIndex = index // Start the transaction - tables := MDBTables{s.nodeTable, s.checkTable, - s.sessionTable, s.sessionCheckTable} - tx, err := tables.StartTxn(false) + tx, err := s.tables.StartTxn(false) if err != nil { panic(fmt.Errorf("Failed to start txn: %v", err)) } @@ -1172,9 +1197,7 @@ func (s *StateStore) SessionCreate(index uint64, session *structs.Session) error // doing a restore, otherwise SessionCreate should be used. func (s *StateStore) SessionRestore(session *structs.Session) error { // Start the transaction - tables := MDBTables{s.nodeTable, s.checkTable, - s.sessionTable, s.sessionCheckTable} - tx, err := tables.StartTxn(false) + tx, err := s.tables.StartTxn(false) if err != nil { panic(fmt.Errorf("Failed to start txn: %v", err)) } @@ -1235,14 +1258,53 @@ func (s *StateStore) NodeSessions(node string) (uint64, []*structs.Session, erro // SessionDelete is used to destroy a session. func (s *StateStore) SessionDestroy(index uint64, id string) error { - // Start the transaction - tables := MDBTables{s.sessionTable, s.sessionCheckTable} - tx, err := tables.StartTxn(false) + tx, err := s.tables.StartTxn(false) if err != nil { panic(fmt.Errorf("Failed to start txn: %v", err)) } defer tx.Abort() + if err := s.invalidateSession(index, tx, id); err != nil { + return err + } + return tx.Commit() +} + +// invalideNode is used to invalide all sessions belonging to a node +// All tables should be locked in the tx. +func (s *StateStore) invalidateNode(index uint64, tx *MDBTxn, node string) error { + sessions, err := s.sessionTable.GetTxn(tx, "node", node) + if err != nil { + return err + } + for _, sess := range sessions { + session := sess.(*structs.Session).ID + if err := s.invalidateSession(index, tx, session); err != nil { + return err + } + } + return nil +} + +// invalidateCheck is used to invalide all sessions belonging to a check +// All tables should be locked in the tx. +func (s *StateStore) invalidateCheck(index uint64, tx *MDBTxn, node, check string) error { + sessionChecks, err := s.sessionCheckTable.GetTxn(tx, "id", node, check) + if err != nil { + return err + } + for _, sc := range sessionChecks { + session := sc.(*sessionCheck).Session + if err := s.invalidateSession(index, tx, session); err != nil { + return err + } + } + return nil +} + +// invalidateSession is used to invalide a session within a given txn +// All tables should be locked in the tx. +func (s *StateStore) invalidateSession(index uint64, tx *MDBTxn, id string) error { // Get the session res, err := s.sessionTable.GetTxn(tx, "id", id) if err != nil { @@ -1272,8 +1334,8 @@ func (s *StateStore) SessionDestroy(index uint64, id string) error { if err := s.sessionTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.sessionTable].Notify() - return tx.Commit() + tx.Defer(func() { s.watch[s.sessionTable].Notify() }) + return nil } // Snapshot is used to create a point in time snapshot diff --git a/consul/state_store_test.go b/consul/state_store_test.go index 41f9375fdcc3..d048b2fc2236 100644 --- a/consul/state_store_test.go +++ b/consul/state_store_test.go @@ -717,11 +717,11 @@ func TestStoreSnapshot(t *testing.T) { ServiceID: "db", } if err := store.EnsureCheck(17, checkAfter); err != nil { - t.Fatalf("err: %v") + t.Fatalf("err: %v", err) } if err := store.KVSDelete(18, "/web/a"); err != nil { - t.Fatalf("err: %v") + t.Fatalf("err: %v", err) } // Check snapshot has old values @@ -1630,7 +1630,7 @@ func TestSessionCreate_Invalid(t *testing.T) { Status: structs.HealthCritical, } if err := store.EnsureCheck(13, check); err != nil { - t.Fatalf("err: %v") + t.Fatalf("err: %v", err) } if err := store.SessionCreate(1000, session); err.Error() != "Check 'bar' is in critical state" { t.Fatalf("err: %v", err) @@ -1719,3 +1719,168 @@ func TestSession_Lookups(t *testing.T) { t.Fatalf("bad: %v %v", ids, out) } } + +func TestSessionInvalidate_CriticalHealthCheck(t *testing.T) { + store, err := testStateStore() + if err != nil { + t.Fatalf("err: %v", err) + } + defer store.Close() + + if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil { + t.Fatalf("err: %v") + } + check := &structs.HealthCheck{ + Node: "foo", + CheckID: "bar", + Status: structs.HealthPassing, + } + if err := store.EnsureCheck(13, check); err != nil { + t.Fatalf("err: %v") + } + + session := &structs.Session{ + Node: "foo", + Checks: []string{"bar"}, + } + if err := store.SessionCreate(14, session); err != nil { + t.Fatalf("err: %v", err) + } + + // Invalidate the check + check.Status = structs.HealthCritical + if err := store.EnsureCheck(15, check); err != nil { + t.Fatalf("err: %v", err) + } + + // Lookup by ID, should be nil + _, s2, err := store.SessionGet(session.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + if s2 != nil { + t.Fatalf("session should be invalidated") + } +} + +func TestSessionInvalidate_DeleteHealthCheck(t *testing.T) { + store, err := testStateStore() + if err != nil { + t.Fatalf("err: %v", err) + } + defer store.Close() + + if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil { + t.Fatalf("err: %v") + } + check := &structs.HealthCheck{ + Node: "foo", + CheckID: "bar", + Status: structs.HealthPassing, + } + if err := store.EnsureCheck(13, check); err != nil { + t.Fatalf("err: %v") + } + + session := &structs.Session{ + Node: "foo", + Checks: []string{"bar"}, + } + if err := store.SessionCreate(14, session); err != nil { + t.Fatalf("err: %v", err) + } + + // Delete the check + if err := store.DeleteNodeCheck(15, "foo", "bar"); err != nil { + t.Fatalf("err: %v", err) + } + + // Lookup by ID, should be nil + _, s2, err := store.SessionGet(session.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + if s2 != nil { + t.Fatalf("session should be invalidated") + } +} + +func TestSessionInvalidate_DeleteNode(t *testing.T) { + store, err := testStateStore() + if err != nil { + t.Fatalf("err: %v", err) + } + defer store.Close() + + if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil { + t.Fatalf("err: %v") + } + + session := &structs.Session{ + Node: "foo", + } + if err := store.SessionCreate(14, session); err != nil { + t.Fatalf("err: %v", err) + } + + // Delete the node + if err := store.DeleteNode(15, "foo"); err != nil { + t.Fatalf("err: %v") + } + + // Lookup by ID, should be nil + _, s2, err := store.SessionGet(session.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + if s2 != nil { + t.Fatalf("session should be invalidated") + } +} + +func TestSessionInvalidate_DeleteNodeService(t *testing.T) { + store, err := testStateStore() + if err != nil { + t.Fatalf("err: %v", err) + } + defer store.Close() + + if err := store.EnsureNode(11, structs.Node{"foo", "127.0.0.1"}); err != nil { + t.Fatalf("err: %v", err) + } + if err := store.EnsureService(12, "foo", &structs.NodeService{"api", "api", nil, 5000}); err != nil { + t.Fatalf("err: %v", err) + } + check := &structs.HealthCheck{ + Node: "foo", + CheckID: "api", + Name: "Can connect", + Status: structs.HealthPassing, + ServiceID: "api", + } + if err := store.EnsureCheck(13, check); err != nil { + t.Fatalf("err: %v") + } + + session := &structs.Session{ + Node: "foo", + Checks: []string{"api"}, + } + if err := store.SessionCreate(14, session); err != nil { + t.Fatalf("err: %v", err) + } + + // Should invalidate the session + if err := store.DeleteNodeService(15, "foo", "api"); err != nil { + t.Fatalf("err: %v", err) + } + + // Lookup by ID, should be nil + _, s2, err := store.SessionGet(session.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + if s2 != nil { + t.Fatalf("session should be invalidated") + } +} From 1c484e991da6a97dc5d59a8a1c1a1a51629290df Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 15 May 2014 11:57:09 -0700 Subject: [PATCH 10/30] consul: Switch notify to using txn defer --- consul/state_store.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/consul/state_store.go b/consul/state_store.go index 730ff3e875c6..f2b47a69fbc0 100644 --- a/consul/state_store.go +++ b/consul/state_store.go @@ -333,7 +333,7 @@ func (s *StateStore) EnsureNode(index uint64, node structs.Node) error { if err := s.nodeTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.nodeTable].Notify() + tx.Defer(func() { s.watch[s.nodeTable].Notify() }) return tx.Commit() } @@ -397,7 +397,7 @@ func (s *StateStore) EnsureService(index uint64, node string, ns *structs.NodeSe if err := s.serviceTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.serviceTable].Notify() + tx.Defer(func() { s.watch[s.serviceTable].Notify() }) return tx.Commit() } @@ -472,7 +472,7 @@ func (s *StateStore) DeleteNodeService(index uint64, node, id string) error { if err := s.serviceTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.serviceTable].Notify() + tx.Defer(func() { s.watch[s.serviceTable].Notify() }) } // Invalidate any sessions using these checks @@ -493,7 +493,7 @@ func (s *StateStore) DeleteNodeService(index uint64, node, id string) error { if err := s.checkTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.checkTable].Notify() + tx.Defer(func() { s.watch[s.checkTable].Notify() }) } return tx.Commit() } @@ -517,7 +517,7 @@ func (s *StateStore) DeleteNode(index uint64, node string) error { if err := s.serviceTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.serviceTable].Notify() + tx.Defer(func() { s.watch[s.serviceTable].Notify() }) } if n, err := s.checkTable.DeleteTxn(tx, "id", node); err != nil { return err @@ -525,7 +525,7 @@ func (s *StateStore) DeleteNode(index uint64, node string) error { if err := s.checkTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.checkTable].Notify() + tx.Defer(func() { s.watch[s.checkTable].Notify() }) } if n, err := s.nodeTable.DeleteTxn(tx, "id", node); err != nil { return err @@ -533,7 +533,7 @@ func (s *StateStore) DeleteNode(index uint64, node string) error { if err := s.nodeTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.nodeTable].Notify() + tx.Defer(func() { s.watch[s.nodeTable].Notify() }) } return tx.Commit() } @@ -692,7 +692,7 @@ func (s *StateStore) EnsureCheck(index uint64, check *structs.HealthCheck) error if err := s.checkTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.checkTable].Notify() + tx.Defer(func() { s.watch[s.checkTable].Notify() }) return tx.Commit() } @@ -715,7 +715,7 @@ func (s *StateStore) DeleteNodeCheck(index uint64, node, id string) error { if err := s.checkTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.checkTable].Notify() + tx.Defer(func() { s.watch[s.checkTable].Notify() }) } return tx.Commit() } @@ -946,7 +946,7 @@ func (s *StateStore) KVSSet(index uint64, d *structs.DirEntry) error { if err := s.kvsTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.kvsTable].Notify() + tx.Defer(func() { s.watch[s.kvsTable].Notify() }) return tx.Commit() } @@ -1068,7 +1068,7 @@ func (s *StateStore) kvsDeleteWithIndex(index uint64, tableIndex string, parts . if err := s.kvsTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.kvsTable].Notify() + tx.Defer(func() { s.watch[s.kvsTable].Notify() }) } return tx.Commit() } @@ -1117,7 +1117,7 @@ func (s *StateStore) KVSCheckAndSet(index uint64, d *structs.DirEntry) (bool, er if err := s.kvsTable.SetLastIndexTxn(tx, index); err != nil { return false, err } - defer s.watch[s.kvsTable].Notify() + tx.Defer(func() { s.watch[s.kvsTable].Notify() }) return true, tx.Commit() } @@ -1189,7 +1189,7 @@ func (s *StateStore) SessionCreate(index uint64, session *structs.Session) error if err := s.sessionTable.SetLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.sessionTable].Notify() + tx.Defer(func() { s.watch[s.sessionTable].Notify() }) return tx.Commit() } @@ -1222,7 +1222,7 @@ func (s *StateStore) SessionRestore(session *structs.Session) error { if err := s.sessionTable.SetMaxLastIndexTxn(tx, index); err != nil { return err } - defer s.watch[s.sessionTable].Notify() + tx.Defer(func() { s.watch[s.sessionTable].Notify() }) return tx.Commit() } From dc1955526cc74395ecc5acc25473fb40d90c5354 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 15 May 2014 14:56:58 -0700 Subject: [PATCH 11/30] consul: Support KVSLock and KVSUnlock --- consul/state_store.go | 114 ++++++++++++++++++++++---------- consul/state_store_test.go | 131 +++++++++++++++++++++++++++++++++++++ consul/structs/structs.go | 6 +- 3 files changed, 216 insertions(+), 35 deletions(-) diff --git a/consul/state_store.go b/consul/state_store.go index f2b47a69fbc0..92b8f1d439db 100644 --- a/consul/state_store.go +++ b/consul/state_store.go @@ -23,6 +23,17 @@ const ( dbMaxMapSize64bit uint64 = 32 * 1024 * 1024 * 1024 // 32GB maximum size ) +// kvMode is used internally to control which type of set +// operation we are performing +type kvMode int + +const ( + kvSet kvMode = iota + kvCAS + kvLock + kvUnlock +) + // The StateStore is responsible for maintaining all the Consul // state. It is manipulated by the FSM which maintains consistency // through the use of Raft. The goals of the StateStore are to provide @@ -919,35 +930,8 @@ func (s *StateStore) parseNodeInfo(tx *MDBTxn, res []interface{}, err error) str // KVSSet is used to create or update a KV entry func (s *StateStore) KVSSet(index uint64, d *structs.DirEntry) error { - // Start a new txn - tx, err := s.kvsTable.StartTxn(false, nil) - if err != nil { - return err - } - defer tx.Abort() - - // Get the existing node - res, err := s.kvsTable.GetTxn(tx, "id", d.Key) - if err != nil { - return err - } - - // Set the create and modify times - if len(res) == 0 { - d.CreateIndex = index - } else { - d.CreateIndex = res[0].(*structs.DirEntry).CreateIndex - } - d.ModifyIndex = index - - if err := s.kvsTable.InsertTxn(tx, d); err != nil { - return err - } - if err := s.kvsTable.SetLastIndexTxn(tx, index); err != nil { - return err - } - tx.Defer(func() { s.watch[s.kvsTable].Notify() }) - return tx.Commit() + _, err := s.kvsSet(index, d, kvSet) + return err } // KVSRestore is used to restore a DirEntry. It should only be used when @@ -1075,8 +1059,26 @@ func (s *StateStore) kvsDeleteWithIndex(index uint64, tableIndex string, parts . // KVSCheckAndSet is used to perform an atomic check-and-set func (s *StateStore) KVSCheckAndSet(index uint64, d *structs.DirEntry) (bool, error) { + return s.kvsSet(index, d, kvCAS) +} + +// KVSLock works like KVSSet but only writes if the lock can be acquired +func (s *StateStore) KVSLock(index uint64, d *structs.DirEntry) (bool, error) { + return s.kvsSet(index, d, kvLock) +} + +// KVSUnlock works like KVSSet but only writes if the lock can be unlocked +func (s *StateStore) KVSUnlock(index uint64, d *structs.DirEntry) (bool, error) { + return s.kvsSet(index, d, kvUnlock) +} + +// kvsSet is the internal setter +func (s *StateStore) kvsSet( + index uint64, + d *structs.DirEntry, + mode kvMode) (bool, error) { // Start a new txn - tx, err := s.kvsTable.StartTxn(false, nil) + tx, err := s.tables.StartTxn(false) if err != nil { return false, err } @@ -1097,10 +1099,51 @@ func (s *StateStore) KVSCheckAndSet(index uint64, d *structs.DirEntry) (bool, er // Use the ModifyIndex as the constraint. A modify of time of 0 // means we are doing a set-if-not-exists, while any other value // means we expect that modify time. - if d.ModifyIndex == 0 && exist != nil { - return false, nil - } else if d.ModifyIndex > 0 && (exist == nil || exist.ModifyIndex != d.ModifyIndex) { - return false, nil + if mode == kvCAS { + if d.ModifyIndex == 0 && exist != nil { + return false, nil + } else if d.ModifyIndex > 0 && (exist == nil || exist.ModifyIndex != d.ModifyIndex) { + return false, nil + } + } + + // If attempting to lock, check this is possible + if mode == kvLock { + // Verify we have a session + if d.Session == "" { + return false, fmt.Errorf("Missing session") + } + + // Bail if it is already locked + if exist != nil && exist.Session != "" { + return false, nil + } + + // Verify the session exists + res, err := s.sessionTable.GetTxn(tx, "id", d.Session) + if err != nil { + return false, err + } + if len(res) == 0 { + return false, fmt.Errorf("Invalid session") + } + + // Update the lock index + if exist != nil { + exist.LockIndex++ + exist.Session = d.Session + } else { + d.LockIndex = 1 + } + } + + // If attempting to unlock, verify the key exists and is held + if mode == kvUnlock { + if exist == nil || exist.Session != d.Session { + return false, nil + } + // Clear the session to unlock + exist.Session = "" } // Set the create and modify times @@ -1108,6 +1151,9 @@ func (s *StateStore) KVSCheckAndSet(index uint64, d *structs.DirEntry) (bool, er d.CreateIndex = index } else { d.CreateIndex = exist.CreateIndex + d.LockIndex = exist.LockIndex + d.Session = exist.Session + } d.ModifyIndex = index diff --git a/consul/state_store_test.go b/consul/state_store_test.go index d048b2fc2236..d46bd6b4bf1d 100644 --- a/consul/state_store_test.go +++ b/consul/state_store_test.go @@ -1884,3 +1884,134 @@ func TestSessionInvalidate_DeleteNodeService(t *testing.T) { t.Fatalf("session should be invalidated") } } + +func TestKVSLock(t *testing.T) { + store, err := testStateStore() + if err != nil { + t.Fatalf("err: %v", err) + } + defer store.Close() + + if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil { + t.Fatalf("err: %v") + } + session := &structs.Session{Node: "foo"} + if err := store.SessionCreate(4, session); err != nil { + t.Fatalf("err: %v", err) + } + + // Lock with a non-existing keys should work + d := &structs.DirEntry{ + Key: "/foo", + Flags: 42, + Value: []byte("test"), + Session: session.ID, + } + ok, err := store.KVSLock(5, d) + if err != nil { + t.Fatalf("err: %v", err) + } + if !ok { + t.Fatalf("unexpected fail") + } + if d.LockIndex != 1 { + t.Fatalf("bad: %v", d) + } + + // Re-locking should fail + ok, err = store.KVSLock(6, d) + if err != nil { + t.Fatalf("err: %v", err) + } + if ok { + t.Fatalf("expected fail") + } + + // Set a normal key + k1 := &structs.DirEntry{ + Key: "/bar", + Flags: 0, + Value: []byte("asdf"), + } + if err := store.KVSSet(7, k1); err != nil { + t.Fatalf("err: %v", err) + } + + // Should acquire the lock + k1.Session = session.ID + ok, err = store.KVSLock(8, k1) + if err != nil { + t.Fatalf("err: %v", err) + } + if !ok { + t.Fatalf("unexpected fail") + } + + // Re-acquire should fail + ok, err = store.KVSLock(9, k1) + if err != nil { + t.Fatalf("err: %v", err) + } + if ok { + t.Fatalf("expected fail") + } + +} + +func TestKVSUnlock(t *testing.T) { + store, err := testStateStore() + if err != nil { + t.Fatalf("err: %v", err) + } + defer store.Close() + + if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil { + t.Fatalf("err: %v") + } + session := &structs.Session{Node: "foo"} + if err := store.SessionCreate(4, session); err != nil { + t.Fatalf("err: %v", err) + } + + // Unlock with a non-existing keys should fail + d := &structs.DirEntry{ + Key: "/foo", + Flags: 42, + Value: []byte("test"), + Session: session.ID, + } + ok, err := store.KVSUnlock(5, d) + if err != nil { + t.Fatalf("err: %v", err) + } + if ok { + t.Fatalf("expected fail") + } + + // Lock should work + d.Session = session.ID + if ok, _ := store.KVSLock(6, d); !ok { + t.Fatalf("expected lock") + } + + // Unlock should work + d.Session = session.ID + ok, err = store.KVSUnlock(7, d) + if err != nil { + t.Fatalf("err: %v", err) + } + if !ok { + t.Fatalf("unexpected fail") + } + + // Re-lock should work + d.Session = session.ID + if ok, err := store.KVSLock(8, d); err != nil { + t.Fatalf("err: %v", err) + } else if !ok { + t.Fatalf("expected lock") + } + if d.LockIndex != 2 { + t.Fatalf("bad: %v", d) + } +} diff --git a/consul/structs/structs.go b/consul/structs/structs.go index f1bbd60bdd12..9f9116612784 100644 --- a/consul/structs/structs.go +++ b/consul/structs/structs.go @@ -275,9 +275,11 @@ type IndexedNodeDump struct { type DirEntry struct { CreateIndex uint64 ModifyIndex uint64 + LockIndex uint64 Key string Flags uint64 Value []byte + Session string `json:",omitempty"` } type DirEntries []*DirEntry @@ -287,7 +289,9 @@ const ( KVSSet KVSOp = "set" KVSDelete = "delete" KVSDeleteTree = "delete-tree" - KVSCAS = "cas" // Check-and-set + KVSCAS = "cas" // Check-and-set + KVSLock = "lock" // Lock a key + KVSUnlock = "unlock" // Unlock a key ) // KVSRequest is used to operate on the Key-Value store From c19806464b528ffa87e9332fc934ce12f11217b3 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 15 May 2014 15:09:55 -0700 Subject: [PATCH 12/30] consul: Session invalidation releases locks --- consul/state_store.go | 33 ++++++++++++++++++++++++++ consul/state_store_test.go | 48 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/consul/state_store.go b/consul/state_store.go index 92b8f1d439db..ac37a36a5335 100644 --- a/consul/state_store.go +++ b/consul/state_store.go @@ -233,6 +233,10 @@ func (s *StateStore) initialize() error { Fields: []string{"Key"}, IdxFunc: DefaultIndexPrefixFunc, }, + "session": &MDBIndex{ + AllowBlank: true, + Fields: []string{"Session"}, + }, }, Decoder: func(buf []byte) interface{} { out := new(structs.DirEntry) @@ -1363,6 +1367,11 @@ func (s *StateStore) invalidateSession(index uint64, tx *MDBTxn, id string) erro } session := res[0].(*structs.Session) + // Invalidate any held locks + if err := s.invalidateLocks(index, tx, id); err != nil { + return err + } + // Nuke the session if _, err := s.sessionTable.DeleteTxn(tx, "id", id); err != nil { return err @@ -1384,6 +1393,30 @@ func (s *StateStore) invalidateSession(index uint64, tx *MDBTxn, id string) erro return nil } +// invalidateLocks is used to invalidate all the locks held by a session +// within a given txn. All tables should be locked in the tx. +func (s *StateStore) invalidateLocks(index uint64, tx *MDBTxn, id string) error { + pairs, err := s.kvsTable.GetTxn(tx, "session", id) + if err != nil { + return err + } + for _, pair := range pairs { + kv := pair.(*structs.DirEntry) + kv.Session = "" // Clear the lock + kv.ModifyIndex = index // Update the modified time + if err := s.kvsTable.InsertTxn(tx, kv); err != nil { + return err + } + } + if len(pairs) > 0 { + if err := s.kvsTable.SetLastIndexTxn(tx, index); err != nil { + return err + } + tx.Defer(func() { s.watch[s.kvsTable].Notify() }) + } + return nil +} + // Snapshot is used to create a point in time snapshot func (s *StateStore) Snapshot() (*StateSnapshot, error) { // Begin a new txn on all tables diff --git a/consul/state_store_test.go b/consul/state_store_test.go index d46bd6b4bf1d..9000e1690751 100644 --- a/consul/state_store_test.go +++ b/consul/state_store_test.go @@ -2015,3 +2015,51 @@ func TestKVSUnlock(t *testing.T) { t.Fatalf("bad: %v", d) } } + +func TestSessionInvalidate_KeyUnlock(t *testing.T) { + store, err := testStateStore() + if err != nil { + t.Fatalf("err: %v", err) + } + defer store.Close() + + if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil { + t.Fatalf("err: %v") + } + session := &structs.Session{Node: "foo"} + if err := store.SessionCreate(4, session); err != nil { + t.Fatalf("err: %v", err) + } + + // Lock a key with the session + d := &structs.DirEntry{ + Key: "/foo", + Flags: 42, + Value: []byte("test"), + Session: session.ID, + } + ok, err := store.KVSLock(5, d) + if err != nil { + t.Fatalf("err: %v", err) + } + if !ok { + t.Fatalf("unexpected fail") + } + + // Delete the node + if err := store.DeleteNode(6, "foo"); err != nil { + t.Fatalf("err: %v") + } + + // Key should be unlocked + idx, d2, err := store.KVSGet("/foo") + if idx != 6 { + t.Fatalf("bad: %v", idx) + } + if d2.LockIndex != 1 { + t.Fatalf("bad: %v", *d2) + } + if d2.Session != "" { + t.Fatalf("bad: %v", *d2) + } +} From 87720681632bceb6b37062779421890ba66d361e Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 15 May 2014 15:21:34 -0700 Subject: [PATCH 13/30] consul: Adding SessionList to snapshot --- consul/state_store.go | 10 +++++++++ consul/state_store_test.go | 45 +++++++++++++++++++++++++++++++++----- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/consul/state_store.go b/consul/state_store.go index ac37a36a5335..eb27e62fa538 100644 --- a/consul/state_store.go +++ b/consul/state_store.go @@ -1480,3 +1480,13 @@ func (s *StateSnapshot) NodeChecks(node string) structs.HealthChecks { func (s *StateSnapshot) KVSDump(stream chan<- interface{}) error { return s.store.kvsTable.StreamTxn(stream, s.tx, "id") } + +// SessionList is used to list all the open sessions +func (s *StateSnapshot) SessionList() ([]*structs.Session, error) { + res, err := s.store.sessionTable.GetTxn(s.tx, "id") + out := make([]*structs.Session, len(res)) + for i, raw := range res { + out[i] = raw.(*structs.Session) + } + return out, err +} diff --git a/consul/state_store_test.go b/consul/state_store_test.go index 9000e1690751..d3bc1667cdf5 100644 --- a/consul/state_store_test.go +++ b/consul/state_store_test.go @@ -636,6 +636,21 @@ func TestStoreSnapshot(t *testing.T) { t.Fatalf("err: %v", err) } + // Add some sessions + session := &structs.Session{Node: "foo"} + if err := store.SessionCreate(16, session); err != nil { + t.Fatalf("err: %v", err) + } + + session = &structs.Session{Node: "bar"} + if err := store.SessionCreate(17, session); err != nil { + t.Fatalf("err: %v", err) + } + d.Session = session.ID + if ok, err := store.KVSLock(18, d); err != nil || !ok { + t.Fatalf("err: %v", err) + } + // Take a snapshot snap, err := store.Snapshot() if err != nil { @@ -644,7 +659,7 @@ func TestStoreSnapshot(t *testing.T) { defer snap.Close() // Check the last nodes - if idx := snap.LastIndex(); idx != 15 { + if idx := snap.LastIndex(); idx != 18 { t.Fatalf("bad: %v", idx) } @@ -699,14 +714,23 @@ func TestStoreSnapshot(t *testing.T) { t.Fatalf("missing KVS entries!") } + // Check there are 2 sessions + sessions, err := snap.SessionList() + if err != nil { + t.Fatalf("err: %v", err) + } + if len(sessions) != 2 { + t.Fatalf("missing sessions") + } + // Make some changes! - if err := store.EnsureService(14, "foo", &structs.NodeService{"db", "db", []string{"slave"}, 8000}); err != nil { + if err := store.EnsureService(19, "foo", &structs.NodeService{"db", "db", []string{"slave"}, 8000}); err != nil { t.Fatalf("err: %v", err) } - if err := store.EnsureService(15, "bar", &structs.NodeService{"db", "db", []string{"master"}, 8000}); err != nil { + if err := store.EnsureService(20, "bar", &structs.NodeService{"db", "db", []string{"master"}, 8000}); err != nil { t.Fatalf("err: %v", err) } - if err := store.EnsureNode(16, structs.Node{"baz", "127.0.0.3"}); err != nil { + if err := store.EnsureNode(21, structs.Node{"baz", "127.0.0.3"}); err != nil { t.Fatalf("err: %v", err) } checkAfter := &structs.HealthCheck{ @@ -716,11 +740,11 @@ func TestStoreSnapshot(t *testing.T) { Status: structs.HealthCritical, ServiceID: "db", } - if err := store.EnsureCheck(17, checkAfter); err != nil { + if err := store.EnsureCheck(22, checkAfter); err != nil { t.Fatalf("err: %v", err) } - if err := store.KVSDelete(18, "/web/a"); err != nil { + if err := store.KVSDelete(23, "/web/b"); err != nil { t.Fatalf("err: %v", err) } @@ -773,6 +797,15 @@ func TestStoreSnapshot(t *testing.T) { if len(ents) != 2 { t.Fatalf("missing KVS entries!") } + + // Check there are 2 sessions + sessions, err = snap.SessionList() + if err != nil { + t.Fatalf("err: %v", err) + } + if len(sessions) != 2 { + t.Fatalf("missing sessions") + } } func TestEnsureCheck(t *testing.T) { From 137d21fbac86bc7ebfc37f750efff9cb1e23777b Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 15 May 2014 19:22:31 -0700 Subject: [PATCH 14/30] consul: Adding support for sessions to FSM --- consul/fsm.go | 92 ++++++++++++++++++++++++++++++++++----- consul/structs/structs.go | 20 +++++++++ 2 files changed, 101 insertions(+), 11 deletions(-) diff --git a/consul/fsm.go b/consul/fsm.go index 22854729fd3c..c2a1bac2519e 100644 --- a/consul/fsm.go +++ b/consul/fsm.go @@ -67,6 +67,8 @@ func (c *consulFSM) Apply(log *raft.Log) interface{} { return c.applyDeregister(buf[1:], log.Index) case structs.KVSRequestType: return c.applyKVSOperation(buf[1:], log.Index) + case structs.SessionRequestType: + return c.applySessionOperation(buf[1:], log.Index) default: panic(fmt.Errorf("failed to apply request: %#v", buf)) } @@ -152,6 +154,20 @@ func (c *consulFSM) applyKVSOperation(buf []byte, index uint64) interface{} { } else { return act } + case structs.KVSLock: + act, err := c.state.KVSLock(index, &req.DirEnt) + if err != nil { + return err + } else { + return act + } + case structs.KVSUnlock: + act, err := c.state.KVSUnlock(index, &req.DirEnt) + if err != nil { + return err + } else { + return act + } default: c.logger.Printf("[WARN] consul.fsm: Invalid KVS operation '%s'", req.Op) return fmt.Errorf("Invalid KVS operation '%s'", req.Op) @@ -159,6 +175,23 @@ func (c *consulFSM) applyKVSOperation(buf []byte, index uint64) interface{} { return nil } +func (c *consulFSM) applySessionOperation(buf []byte, index uint64) interface{} { + var req structs.SessionRequest + if err := structs.Decode(buf, &req); err != nil { + panic(fmt.Errorf("failed to decode request: %v", err)) + } + switch req.Op { + case structs.SessionCreate: + return c.state.SessionCreate(index, &req.Session) + case structs.SessionDestroy: + return c.state.SessionDestroy(index, req.Session.ID) + default: + c.logger.Printf("[WARN] consul.fsm: Invalid Session operation '%s'", req.Op) + return fmt.Errorf("Invalid Session operation '%s'", req.Op) + } + return nil +} + func (c *consulFSM) Snapshot() (raft.FSMSnapshot, error) { defer func(start time.Time) { c.logger.Printf("[INFO] consul.fsm: snapshot created in %v", time.Now().Sub(start)) @@ -222,6 +255,15 @@ func (c *consulFSM) Restore(old io.ReadCloser) error { return err } + case structs.SessionRequestType: + var req structs.Session + if err := dec.Decode(&req); err != nil { + return err + } + if err := c.state.SessionRestore(&req); err != nil { + return err + } + default: return fmt.Errorf("Unrecognized msg type: %v", msgType) } @@ -244,6 +286,25 @@ func (s *consulSnapshot) Persist(sink raft.SnapshotSink) error { return err } + if err := s.persistNodes(sink, encoder); err != nil { + sink.Cancel() + return err + } + + if err := s.persistSessions(sink, encoder); err != nil { + sink.Cancel() + return err + } + + if err := s.persistKV(sink, encoder); err != nil { + sink.Cancel() + return err + } + return nil +} + +func (s *consulSnapshot) persistNodes(sink raft.SnapshotSink, + encoder *codec.Encoder) error { // Get all the nodes nodes := s.state.Nodes() @@ -258,7 +319,6 @@ func (s *consulSnapshot) Persist(sink raft.SnapshotSink) error { // Register the node itself sink.Write([]byte{byte(structs.RegisterRequestType)}) if err := encoder.Encode(&req); err != nil { - sink.Cancel() return err } @@ -268,7 +328,6 @@ func (s *consulSnapshot) Persist(sink raft.SnapshotSink) error { req.Service = srv sink.Write([]byte{byte(structs.RegisterRequestType)}) if err := encoder.Encode(&req); err != nil { - sink.Cancel() return err } } @@ -280,16 +339,31 @@ func (s *consulSnapshot) Persist(sink raft.SnapshotSink) error { req.Check = check sink.Write([]byte{byte(structs.RegisterRequestType)}) if err := encoder.Encode(&req); err != nil { - sink.Cancel() return err } } } + return nil +} + +func (s *consulSnapshot) persistSessions(sink raft.SnapshotSink, + encoder *codec.Encoder) error { + sessions, err := s.state.SessionList() + if err != nil { + return err + } - // Enable GC of the ndoes - nodes = nil + for _, s := range sessions { + sink.Write([]byte{byte(structs.SessionRequestType)}) + if err := encoder.Encode(s); err != nil { + return err + } + } + return nil +} - // Dump the KVS entries +func (s *consulSnapshot) persistKV(sink raft.SnapshotSink, + encoder *codec.Encoder) error { streamCh := make(chan interface{}, 256) errorCh := make(chan error) go func() { @@ -298,25 +372,21 @@ func (s *consulSnapshot) Persist(sink raft.SnapshotSink) error { } }() -OUTER: for { select { case raw := <-streamCh: if raw == nil { - break OUTER + return nil } sink.Write([]byte{byte(structs.KVSRequestType)}) if err := encoder.Encode(raw); err != nil { - sink.Cancel() return err } case err := <-errorCh: - sink.Cancel() return err } } - return nil } diff --git a/consul/structs/structs.go b/consul/structs/structs.go index 9f9116612784..10c423b0369b 100644 --- a/consul/structs/structs.go +++ b/consul/structs/structs.go @@ -19,6 +19,7 @@ const ( RegisterRequestType MessageType = iota DeregisterRequestType KVSRequestType + SessionRequestType ) const ( @@ -348,6 +349,25 @@ type Session struct { CreateIndex uint64 } +type SessionOp string + +const ( + SessionCreate SessionOp = "create" + SessionDestroy = "destroy" +) + +// SessionRequest is used to operate on sessions +type SessionRequest struct { + Datacenter string + Op SessionOp // Which operation are we performing + Session Session // Which session + WriteRequest +} + +func (r *SessionRequest) RequestDatacenter() string { + return r.Datacenter +} + // Decode is used to decode a MsgPack encoded object func Decode(buf []byte, out interface{}) error { var handle codec.MsgpackHandle From 85656e0f5424f0b5a135852e2e259a045adf040f Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 15 May 2014 19:37:58 -0700 Subject: [PATCH 15/30] consul: FSM tests for session --- consul/fsm.go | 6 +++- consul/fsm_test.go | 79 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/consul/fsm.go b/consul/fsm.go index c2a1bac2519e..41a911a56f1a 100644 --- a/consul/fsm.go +++ b/consul/fsm.go @@ -182,7 +182,11 @@ func (c *consulFSM) applySessionOperation(buf []byte, index uint64) interface{} } switch req.Op { case structs.SessionCreate: - return c.state.SessionCreate(index, &req.Session) + if err := c.state.SessionCreate(index, &req.Session); err != nil { + return err + } else { + return req.Session.ID + } case structs.SessionDestroy: return c.state.SessionDestroy(index, req.Session.ID) default: diff --git a/consul/fsm_test.go b/consul/fsm_test.go index ac8453bec2da..42708a342ec9 100644 --- a/consul/fsm_test.go +++ b/consul/fsm_test.go @@ -569,3 +569,82 @@ func TestFSM_KVSCheckAndSet(t *testing.T) { t.Fatalf("bad: %v", d) } } + +func TestFSM_SessionCreate_Destroy(t *testing.T) { + fsm, err := NewFSM(os.Stderr) + if err != nil { + t.Fatalf("err: %v", err) + } + defer fsm.Close() + + fsm.state.EnsureNode(1, structs.Node{"foo", "127.0.0.1"}) + fsm.state.EnsureCheck(2, &structs.HealthCheck{ + Node: "foo", + CheckID: "web", + Status: structs.HealthPassing, + }) + + // Create a new session + req := structs.SessionRequest{ + Datacenter: "dc1", + Op: structs.SessionCreate, + Session: structs.Session{ + Node: "foo", + Checks: []string{"web"}, + }, + } + buf, err := structs.Encode(structs.SessionRequestType, req) + if err != nil { + t.Fatalf("err: %v", err) + } + resp := fsm.Apply(makeLog(buf)) + if err, ok := resp.(error); ok { + t.Fatalf("resp: %v", err) + } + + // Get the session + id := resp.(string) + _, session, err := fsm.state.SessionGet(id) + if err != nil { + t.Fatalf("err: %v", err) + } + if session == nil { + t.Fatalf("missing") + } + + // Verify the session + if session.ID != id { + t.Fatalf("bad: %v", *session) + } + if session.Node != "foo" { + t.Fatalf("bad: %v", *session) + } + if session.Checks[0] != "web" { + t.Fatalf("bad: %v", *session) + } + + // Try to destroy + destroy := structs.SessionRequest{ + Datacenter: "dc1", + Op: structs.SessionDestroy, + Session: structs.Session{ + ID: id, + }, + } + buf, err = structs.Encode(structs.SessionRequestType, destroy) + if err != nil { + t.Fatalf("err: %v", err) + } + resp = fsm.Apply(makeLog(buf)) + if resp != nil { + t.Fatalf("resp: %v", resp) + } + + _, session, err = fsm.state.SessionGet(id) + if err != nil { + t.Fatalf("err: %v", err) + } + if session != nil { + t.Fatalf("should be destroyed") + } +} From 555533e2f8dcde000693d594ca3ae31b3ca2b839 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 15 May 2014 19:49:45 -0700 Subject: [PATCH 16/30] consul: Testing KVS Lock/Unlock in FSM --- consul/fsm_test.go | 108 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/consul/fsm_test.go b/consul/fsm_test.go index 42708a342ec9..c6a427b7444b 100644 --- a/consul/fsm_test.go +++ b/consul/fsm_test.go @@ -648,3 +648,111 @@ func TestFSM_SessionCreate_Destroy(t *testing.T) { t.Fatalf("should be destroyed") } } + +func TestFSM_KVSLock(t *testing.T) { + fsm, err := NewFSM(os.Stderr) + if err != nil { + t.Fatalf("err: %v", err) + } + defer fsm.Close() + + fsm.state.EnsureNode(1, structs.Node{"foo", "127.0.0.1"}) + session := &structs.Session{Node: "foo"} + fsm.state.SessionCreate(2, session) + + req := structs.KVSRequest{ + Datacenter: "dc1", + Op: structs.KVSLock, + DirEnt: structs.DirEntry{ + Key: "/test/path", + Value: []byte("test"), + Session: session.ID, + }, + } + buf, err := structs.Encode(structs.KVSRequestType, req) + if err != nil { + t.Fatalf("err: %v", err) + } + resp := fsm.Apply(makeLog(buf)) + if resp != true { + t.Fatalf("resp: %v", resp) + } + + // Verify key is locked + _, d, err := fsm.state.KVSGet("/test/path") + if err != nil { + t.Fatalf("err: %v", err) + } + if d == nil { + t.Fatalf("missing") + } + if d.LockIndex != 1 { + t.Fatalf("bad: %v", *d) + } + if d.Session != session.ID { + t.Fatalf("bad: %v", *d) + } +} + +func TestFSM_KVSUnlock(t *testing.T) { + fsm, err := NewFSM(os.Stderr) + if err != nil { + t.Fatalf("err: %v", err) + } + defer fsm.Close() + + fsm.state.EnsureNode(1, structs.Node{"foo", "127.0.0.1"}) + session := &structs.Session{Node: "foo"} + fsm.state.SessionCreate(2, session) + + req := structs.KVSRequest{ + Datacenter: "dc1", + Op: structs.KVSLock, + DirEnt: structs.DirEntry{ + Key: "/test/path", + Value: []byte("test"), + Session: session.ID, + }, + } + buf, err := structs.Encode(structs.KVSRequestType, req) + if err != nil { + t.Fatalf("err: %v", err) + } + resp := fsm.Apply(makeLog(buf)) + if resp != true { + t.Fatalf("resp: %v", resp) + } + + req = structs.KVSRequest{ + Datacenter: "dc1", + Op: structs.KVSUnlock, + DirEnt: structs.DirEntry{ + Key: "/test/path", + Value: []byte("test"), + Session: session.ID, + }, + } + buf, err = structs.Encode(structs.KVSRequestType, req) + if err != nil { + t.Fatalf("err: %v", err) + } + resp = fsm.Apply(makeLog(buf)) + if resp != true { + t.Fatalf("resp: %v", resp) + } + + // Verify key is unlocked + _, d, err := fsm.state.KVSGet("/test/path") + if err != nil { + t.Fatalf("err: %v", err) + } + if d == nil { + t.Fatalf("missing") + } + if d.LockIndex != 1 { + t.Fatalf("bad: %v", *d) + } + if d.Session != "" { + t.Fatalf("bad: %v", *d) + } +} From aa97e24b029a3915c6293cd8f796246ef58db453 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Thu, 15 May 2014 19:57:48 -0700 Subject: [PATCH 17/30] consul: Testing FSM snapshot of sessions --- consul/fsm_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/consul/fsm_test.go b/consul/fsm_test.go index c6a427b7444b..5e5d086d8478 100644 --- a/consul/fsm_test.go +++ b/consul/fsm_test.go @@ -326,6 +326,8 @@ func TestFSM_SnapshotRestore(t *testing.T) { Key: "/test", Value: []byte("foo"), }) + session := &structs.Session{Node: "foo"} + fsm.state.SessionCreate(9, session) // Snapshot snap, err := fsm.Snapshot() @@ -383,6 +385,15 @@ func TestFSM_SnapshotRestore(t *testing.T) { if string(d.Value) != "foo" { t.Fatalf("bad: %v", d) } + + // Verify session is restored + _, s, err := fsm.state.SessionGet(session.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + if s.Node != "foo" { + t.Fatalf("bad: %v", d) + } } func TestFSM_KVSSet(t *testing.T) { From f2b2a68a55815506a7e2bd4d6dcf22c2ad4fb2e7 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Fri, 16 May 2014 14:36:14 -0700 Subject: [PATCH 18/30] consul: First pass at Session RPC endpoints --- consul/server.go | 3 ++ consul/session_endpoint.go | 106 +++++++++++++++++++++++++++++++++++++ consul/structs/structs.go | 17 ++++++ 3 files changed, 126 insertions(+) create mode 100644 consul/session_endpoint.go diff --git a/consul/server.go b/consul/server.go index d873cd2b9b38..598e6bde3e11 100644 --- a/consul/server.go +++ b/consul/server.go @@ -108,6 +108,7 @@ type endpoints struct { Raft *Raft Status *Status KVS *KVS + Session *Session Internal *Internal } @@ -316,6 +317,7 @@ func (s *Server) setupRPC(tlsConfig *tls.Config) error { s.endpoints.Catalog = &Catalog{s} s.endpoints.Health = &Health{s} s.endpoints.KVS = &KVS{s} + s.endpoints.Session = &Session{s} s.endpoints.Internal = &Internal{s} // Register the handlers @@ -324,6 +326,7 @@ func (s *Server) setupRPC(tlsConfig *tls.Config) error { s.rpcServer.Register(s.endpoints.Catalog) s.rpcServer.Register(s.endpoints.Health) s.rpcServer.Register(s.endpoints.KVS) + s.rpcServer.Register(s.endpoints.Session) s.rpcServer.Register(s.endpoints.Internal) list, err := net.ListenTCP("tcp", s.config.RPCAddr) diff --git a/consul/session_endpoint.go b/consul/session_endpoint.go new file mode 100644 index 000000000000..79938fc182fd --- /dev/null +++ b/consul/session_endpoint.go @@ -0,0 +1,106 @@ +package consul + +import ( + "fmt" + "github.com/armon/go-metrics" + "github.com/hashicorp/consul/consul/structs" + "time" +) + +// Session endpoint is used to manipulate sessions for KV +type Session struct { + srv *Server +} + +// Apply is used to apply a modifying request to the data store. This should +// only be used for operations that modify the data +func (s *Session) Apply(args *structs.SessionRequest, reply *string) error { + if done, err := s.srv.forward("Session.Apply", args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"consul", "session", "apply"}, time.Now()) + + // Verify the args + if args.Session.ID == "" && args.Op == structs.SessionDestroy { + return fmt.Errorf("Must provide ID") + } + if args.Session.Node == "" && args.Op == structs.SessionCreate { + return fmt.Errorf("Must provide Node") + } + + // Apply the update + resp, err := s.srv.raftApply(structs.SessionRequestType, args) + if err != nil { + s.srv.logger.Printf("[ERR] consul.session: Apply failed: %v", err) + return err + } + if respErr, ok := resp.(error); ok { + return respErr + } + + // Check if the return type is a string + if respString, ok := resp.(string); ok { + *reply = respString + } + return nil +} + +// Get is used to retrieve a single session +func (s *Session) Get(args *structs.SessionGetRequest, + reply *structs.IndexedSessions) error { + if done, err := s.srv.forward("Session.Get", args, args, reply); done { + return err + } + + // Get the local state + state := s.srv.fsm.State() + return s.srv.blockingRPC(&args.QueryOptions, + &reply.QueryMeta, + state.QueryTables("SessionGet"), + func() error { + index, session, err := state.SessionGet(args.Session) + reply.Index = index + if session != nil { + reply.Sessions = structs.Sessions{session} + } + return err + }) +} + +// List is used to list all the active sessions +func (s *Session) List(args *structs.DCSpecificRequest, + reply *structs.IndexedSessions) error { + if done, err := s.srv.forward("Session.List", args, args, reply); done { + return err + } + + // Get the local state + state := s.srv.fsm.State() + return s.srv.blockingRPC(&args.QueryOptions, + &reply.QueryMeta, + state.QueryTables("SessionList"), + func() error { + var err error + reply.Index, reply.Sessions, err = state.SessionList() + return err + }) +} + +// NodeSessions is used to get all the sessions for a particular node +func (s *Session) NodeSessions(args *structs.NodeSpecificRequest, + reply *structs.IndexedSessions) error { + if done, err := s.srv.forward("Session.NodeSessions", args, args, reply); done { + return err + } + + // Get the local state + state := s.srv.fsm.State() + return s.srv.blockingRPC(&args.QueryOptions, + &reply.QueryMeta, + state.QueryTables("NodeSessions"), + func() error { + var err error + reply.Index, reply.Sessions, err = state.NodeSessions(args.Node) + return err + }) +} diff --git a/consul/structs/structs.go b/consul/structs/structs.go index 10c423b0369b..d2e5aba24e7f 100644 --- a/consul/structs/structs.go +++ b/consul/structs/structs.go @@ -348,6 +348,7 @@ type Session struct { Checks []string CreateIndex uint64 } +type Sessions []*Session type SessionOp string @@ -368,6 +369,22 @@ func (r *SessionRequest) RequestDatacenter() string { return r.Datacenter } +// SessionGetRequest is used to request a session by ID +type SessionGetRequest struct { + Datacenter string + Session string + QueryOptions +} + +func (r *SessionGetRequest) RequestDatacenter() string { + return r.Datacenter +} + +type IndexedSessions struct { + Sessions Sessions + QueryMeta +} + // Decode is used to decode a MsgPack encoded object func Decode(buf []byte, out interface{}) error { var handle codec.MsgpackHandle From b8174dc500ad015c70dec79f2756ad3abe467f08 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Fri, 16 May 2014 15:20:41 -0700 Subject: [PATCH 19/30] consul: Adding tests for session endpoints --- consul/session_endpoint_test.go | 212 ++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 consul/session_endpoint_test.go diff --git a/consul/session_endpoint_test.go b/consul/session_endpoint_test.go new file mode 100644 index 000000000000..20366feda604 --- /dev/null +++ b/consul/session_endpoint_test.go @@ -0,0 +1,212 @@ +package consul + +import ( + "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" + "os" + "testing" +) + +func TestSessionEndpoint_Apply(t *testing.T) { + dir1, s1 := testServer(t) + defer os.RemoveAll(dir1) + defer s1.Shutdown() + client := rpcClient(t, s1) + defer client.Close() + + testutil.WaitForLeader(t, client.Call, "dc1") + + // Just add a node + s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"}) + + arg := structs.SessionRequest{ + Datacenter: "dc1", + Op: structs.SessionCreate, + Session: structs.Session{ + Node: "foo", + }, + } + var out string + if err := client.Call("Session.Apply", &arg, &out); err != nil { + t.Fatalf("err: %v", err) + } + id := out + + // Verify + state := s1.fsm.State() + _, s, err := state.SessionGet(out) + if err != nil { + t.Fatalf("err: %v", err) + } + if s == nil { + t.Fatalf("should not be nil") + } + + // Do a delete + arg.Op = structs.SessionDestroy + arg.Session.ID = out + if err := client.Call("Session.Apply", &arg, &out); err != nil { + t.Fatalf("err: %v", err) + } + + // Verify + _, s, err = state.SessionGet(id) + if err != nil { + t.Fatalf("err: %v", err) + } + if s != nil { + t.Fatalf("bad: %v", s) + } +} + +func TestSessionEndpoint_Get(t *testing.T) { + dir1, s1 := testServer(t) + defer os.RemoveAll(dir1) + defer s1.Shutdown() + client := rpcClient(t, s1) + defer client.Close() + + testutil.WaitForLeader(t, client.Call, "dc1") + + s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"}) + arg := structs.SessionRequest{ + Datacenter: "dc1", + Op: structs.SessionCreate, + Session: structs.Session{ + Node: "foo", + }, + } + var out string + if err := client.Call("Session.Apply", &arg, &out); err != nil { + t.Fatalf("err: %v", err) + } + + getR := structs.SessionGetRequest{ + Datacenter: "dc1", + Session: out, + } + var sessions structs.IndexedSessions + if err := client.Call("Session.Get", &getR, &sessions); err != nil { + t.Fatalf("err: %v", err) + } + + if sessions.Index == 0 { + t.Fatalf("Bad: %v", sessions) + } + if len(sessions.Sessions) != 1 { + t.Fatalf("Bad: %v", sessions) + } + s := sessions.Sessions[0] + if s.ID != out { + t.Fatalf("bad: %v", s) + } +} + +func TestSessionEndpoint_List(t *testing.T) { + dir1, s1 := testServer(t) + defer os.RemoveAll(dir1) + defer s1.Shutdown() + client := rpcClient(t, s1) + defer client.Close() + + testutil.WaitForLeader(t, client.Call, "dc1") + + s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"}) + ids := []string{} + for i := 0; i < 5; i++ { + arg := structs.SessionRequest{ + Datacenter: "dc1", + Op: structs.SessionCreate, + Session: structs.Session{ + Node: "foo", + }, + } + var out string + if err := client.Call("Session.Apply", &arg, &out); err != nil { + t.Fatalf("err: %v", err) + } + ids = append(ids, out) + } + + getR := structs.DCSpecificRequest{ + Datacenter: "dc1", + } + var sessions structs.IndexedSessions + if err := client.Call("Session.List", &getR, &sessions); err != nil { + t.Fatalf("err: %v", err) + } + + if sessions.Index == 0 { + t.Fatalf("Bad: %v", sessions) + } + if len(sessions.Sessions) != 5 { + t.Fatalf("Bad: %v", sessions.Sessions) + } + for i := 0; i < len(sessions.Sessions); i++ { + s := sessions.Sessions[i] + if !strContains(ids, s.ID) { + t.Fatalf("bad: %v", s) + } + if s.Node != "foo" { + t.Fatalf("bad: %v", s) + } + } +} + +func TestSessionEndpoint_NodeSessions(t *testing.T) { + dir1, s1 := testServer(t) + defer os.RemoveAll(dir1) + defer s1.Shutdown() + client := rpcClient(t, s1) + defer client.Close() + + testutil.WaitForLeader(t, client.Call, "dc1") + + s1.fsm.State().EnsureNode(1, structs.Node{"foo", "127.0.0.1"}) + s1.fsm.State().EnsureNode(1, structs.Node{"bar", "127.0.0.1"}) + ids := []string{} + for i := 0; i < 10; i++ { + arg := structs.SessionRequest{ + Datacenter: "dc1", + Op: structs.SessionCreate, + Session: structs.Session{ + Node: "bar", + }, + } + if i < 5 { + arg.Session.Node = "foo" + } + var out string + if err := client.Call("Session.Apply", &arg, &out); err != nil { + t.Fatalf("err: %v", err) + } + if i < 5 { + ids = append(ids, out) + } + } + + getR := structs.NodeSpecificRequest{ + Datacenter: "dc1", + Node: "foo", + } + var sessions structs.IndexedSessions + if err := client.Call("Session.NodeSessions", &getR, &sessions); err != nil { + t.Fatalf("err: %v", err) + } + + if sessions.Index == 0 { + t.Fatalf("Bad: %v", sessions) + } + if len(sessions.Sessions) != 5 { + t.Fatalf("Bad: %v", sessions.Sessions) + } + for i := 0; i < len(sessions.Sessions); i++ { + s := sessions.Sessions[i] + if !strContains(ids, s.ID) { + t.Fatalf("bad: %v", s) + } + if s.Node != "foo" { + t.Fatalf("bad: %v", s) + } + } +} From 0119ec7f0a3def1e3c67ac8288936b177ccad347 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Fri, 16 May 2014 15:49:17 -0700 Subject: [PATCH 20/30] agent: First pass at session endpoints --- command/agent/http.go | 6 ++ command/agent/session_endpoint.go | 126 ++++++++++++++++++++++++++++++ consul/session_endpoint.go | 2 +- consul/session_endpoint_test.go | 2 +- consul/structs/structs.go | 6 +- 5 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 command/agent/session_endpoint.go diff --git a/command/agent/http.go b/command/agent/http.go index 515514ad2eca..c44c0498040b 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -93,6 +93,12 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/kv/", s.wrap(s.KVSEndpoint)) + s.mux.HandleFunc("/v1/session/create", s.wrap(s.SessionCreate)) + s.mux.HandleFunc("/v1/session/destroy/", s.wrap(s.SessionDestroy)) + s.mux.HandleFunc("/v1/session/info/", s.wrap(s.SessionGet)) + s.mux.HandleFunc("/v1/session/node/", s.wrap(s.SessionsForNode)) + s.mux.HandleFunc("/v1/session/list", s.wrap(s.SessionList)) + if enableDebug { s.mux.HandleFunc("/debug/pprof/", pprof.Index) s.mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) diff --git a/command/agent/session_endpoint.go b/command/agent/session_endpoint.go new file mode 100644 index 000000000000..8408baf949c7 --- /dev/null +++ b/command/agent/session_endpoint.go @@ -0,0 +1,126 @@ +package agent + +import ( + "fmt" + "github.com/hashicorp/consul/consul" + "github.com/hashicorp/consul/consul/structs" + "net/http" + "strings" +) + +// SessionCreate is used to create a new session +func (s *HTTPServer) SessionCreate(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + // Default the session to our node + serf check + args := structs.SessionRequest{ + Op: structs.SessionCreate, + Session: structs.Session{ + Node: s.agent.config.NodeName, + Checks: []string{consul.SerfCheckID}, + }, + } + s.parseDC(req, &args.Datacenter) + + // Handle optional request body + if req.ContentLength > 0 { + if err := decodeBody(req, &args.Session, nil); err != nil { + resp.WriteHeader(400) + resp.Write([]byte(fmt.Sprintf("Request decode failed: %v", err))) + return nil, nil + } + } + + // Create the session, get the ID + var out string + if err := s.agent.RPC("Session.Apply", &args, &out); err != nil { + return nil, err + } + + // Format the response as a JSON object + type response struct { + ID string + } + return response{out}, nil +} + +// SessionDestroy is used to destroy an existing session +func (s *HTTPServer) SessionDestroy(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + args := structs.SessionRequest{ + Op: structs.SessionDestroy, + } + s.parseDC(req, &args.Datacenter) + + // Pull out the session id + args.Session.ID = strings.TrimPrefix(req.URL.Path, "/v1/session/destroy/") + if args.Session.ID == "" { + resp.WriteHeader(400) + resp.Write([]byte("Missing session")) + return nil, nil + } + + var out string + if err := s.agent.RPC("Session.Apply", &args, &out); err != nil { + return nil, err + } + return true, nil +} + +// SessionGet is used to get info for a particular session +func (s *HTTPServer) SessionGet(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + args := structs.SessionSpecificRequest{} + if done := s.parse(resp, req, &args.Datacenter, &args.QueryOptions); done { + return nil, nil + } + + // Pull out the session id + args.Session = strings.TrimPrefix(req.URL.Path, "/v1/session/info/") + if args.Session == "" { + resp.WriteHeader(400) + resp.Write([]byte("Missing session")) + return nil, nil + } + + var out structs.IndexedSessions + defer setMeta(resp, &out.QueryMeta) + if err := s.agent.RPC("Session.Get", &args, &out); err != nil { + return nil, err + } + return out.Sessions, nil +} + +// SessionList is used to list all the sessions +func (s *HTTPServer) SessionList(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + args := structs.DCSpecificRequest{} + if done := s.parse(resp, req, &args.Datacenter, &args.QueryOptions); done { + return nil, nil + } + + var out structs.IndexedSessions + defer setMeta(resp, &out.QueryMeta) + if err := s.agent.RPC("Session.List", &args, &out); err != nil { + return nil, err + } + return out.Sessions, nil +} + +// SessionsForNode returns all the nodes belonging to a node +func (s *HTTPServer) SessionsForNode(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + args := structs.NodeSpecificRequest{} + if done := s.parse(resp, req, &args.Datacenter, &args.QueryOptions); done { + return nil, nil + } + + // Pull out the node name + args.Node = strings.TrimPrefix(req.URL.Path, "/v1/session/node/") + if args.Node == "" { + resp.WriteHeader(400) + resp.Write([]byte("Missing node name")) + return nil, nil + } + + var out structs.IndexedSessions + defer setMeta(resp, &out.QueryMeta) + if err := s.agent.RPC("Session.NodeSessions", &args, &out); err != nil { + return nil, err + } + return out.Sessions, nil +} diff --git a/consul/session_endpoint.go b/consul/session_endpoint.go index 79938fc182fd..069f374d1e2d 100644 --- a/consul/session_endpoint.go +++ b/consul/session_endpoint.go @@ -46,7 +46,7 @@ func (s *Session) Apply(args *structs.SessionRequest, reply *string) error { } // Get is used to retrieve a single session -func (s *Session) Get(args *structs.SessionGetRequest, +func (s *Session) Get(args *structs.SessionSpecificRequest, reply *structs.IndexedSessions) error { if done, err := s.srv.forward("Session.Get", args, args, reply); done { return err diff --git a/consul/session_endpoint_test.go b/consul/session_endpoint_test.go index 20366feda604..8b59d7a28c00 100644 --- a/consul/session_endpoint_test.go +++ b/consul/session_endpoint_test.go @@ -81,7 +81,7 @@ func TestSessionEndpoint_Get(t *testing.T) { t.Fatalf("err: %v", err) } - getR := structs.SessionGetRequest{ + getR := structs.SessionSpecificRequest{ Datacenter: "dc1", Session: out, } diff --git a/consul/structs/structs.go b/consul/structs/structs.go index d2e5aba24e7f..0673ba39f7f4 100644 --- a/consul/structs/structs.go +++ b/consul/structs/structs.go @@ -369,14 +369,14 @@ func (r *SessionRequest) RequestDatacenter() string { return r.Datacenter } -// SessionGetRequest is used to request a session by ID -type SessionGetRequest struct { +// SessionSpecificRequest is used to request a session by ID +type SessionSpecificRequest struct { Datacenter string Session string QueryOptions } -func (r *SessionGetRequest) RequestDatacenter() string { +func (r *SessionSpecificRequest) RequestDatacenter() string { return r.Datacenter } From 14be60aaca479d1ebb3b724197149836fcd295cf Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Fri, 16 May 2014 15:49:47 -0700 Subject: [PATCH 21/30] gofmt --- command/agent/agent_endpoint_test.go | 4 ++-- command/agent/catalog_endpoint_test.go | 4 ++-- command/agent/dns_test.go | 2 +- command/agent/health_endpoint_test.go | 2 +- command/agent/kvs_endpoint_test.go | 2 +- command/agent/local_test.go | 2 +- command/agent/rpc_client_test.go | 4 ++-- command/agent/status_endpoint_test.go | 2 +- command/agent/ui_endpoint_test.go | 2 +- command/force_leave_test.go | 2 +- consul/catalog_endpoint_test.go | 4 ++-- consul/client_test.go | 2 +- consul/health_endpoint_test.go | 2 +- consul/internal_endpoint_test.go | 2 +- consul/kvs_endpoint_test.go | 2 +- consul/leader_test.go | 4 ++-- consul/server_test.go | 4 ++-- testutil/wait.go | 6 +++--- 18 files changed, 26 insertions(+), 26 deletions(-) diff --git a/command/agent/agent_endpoint_test.go b/command/agent/agent_endpoint_test.go index fee1b28a624f..c10649672289 100644 --- a/command/agent/agent_endpoint_test.go +++ b/command/agent/agent_endpoint_test.go @@ -1,15 +1,15 @@ package agent import ( + "errors" "fmt" - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "github.com/hashicorp/serf/serf" "net/http" "os" "testing" "time" - "errors" ) func TestHTTPAgentServices(t *testing.T) { diff --git a/command/agent/catalog_endpoint_test.go b/command/agent/catalog_endpoint_test.go index f8b2b6a2c701..bbbeaea8d026 100644 --- a/command/agent/catalog_endpoint_test.go +++ b/command/agent/catalog_endpoint_test.go @@ -2,8 +2,8 @@ package agent import ( "fmt" - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "net/http" "net/http/httptest" "os" @@ -174,7 +174,7 @@ func TestCatalogNodes_Blocking(t *testing.T) { } // Should block for a while - if time.Now().Sub(start) < 50 * time.Millisecond { + if time.Now().Sub(start) < 50*time.Millisecond { // TODO: Failing t.Fatalf("too fast") } diff --git a/command/agent/dns_test.go b/command/agent/dns_test.go index eeab068d9d5e..d4add63548d8 100644 --- a/command/agent/dns_test.go +++ b/command/agent/dns_test.go @@ -2,8 +2,8 @@ package agent import ( "fmt" - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "github.com/miekg/dns" "os" "strings" diff --git a/command/agent/health_endpoint_test.go b/command/agent/health_endpoint_test.go index a4b4b9817e09..be8a993687e8 100644 --- a/command/agent/health_endpoint_test.go +++ b/command/agent/health_endpoint_test.go @@ -2,8 +2,8 @@ package agent import ( "fmt" - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "net/http" "net/http/httptest" "os" diff --git a/command/agent/kvs_endpoint_test.go b/command/agent/kvs_endpoint_test.go index 61216faafca2..24ec51372f7f 100644 --- a/command/agent/kvs_endpoint_test.go +++ b/command/agent/kvs_endpoint_test.go @@ -3,8 +3,8 @@ package agent import ( "bytes" "fmt" - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "net/http" "net/http/httptest" "os" diff --git a/command/agent/local_test.go b/command/agent/local_test.go index 9ce16fed31f6..655b5b2031fc 100644 --- a/command/agent/local_test.go +++ b/command/agent/local_test.go @@ -1,8 +1,8 @@ package agent import ( - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "os" "reflect" "testing" diff --git a/command/agent/rpc_client_test.go b/command/agent/rpc_client_test.go index 77f34632ae4c..66d48065cec3 100644 --- a/command/agent/rpc_client_test.go +++ b/command/agent/rpc_client_test.go @@ -1,15 +1,15 @@ package agent import ( + "errors" "fmt" - "github.com/hashicorp/serf/serf" "github.com/hashicorp/consul/testutil" + "github.com/hashicorp/serf/serf" "io" "net" "os" "strings" "testing" - "errors" "time" ) diff --git a/command/agent/status_endpoint_test.go b/command/agent/status_endpoint_test.go index 4dd1f4e204a9..0e22eaa30f75 100644 --- a/command/agent/status_endpoint_test.go +++ b/command/agent/status_endpoint_test.go @@ -1,9 +1,9 @@ package agent import ( + "github.com/hashicorp/consul/testutil" "os" "testing" - "github.com/hashicorp/consul/testutil" ) func TestStatusLeader(t *testing.T) { diff --git a/command/agent/ui_endpoint_test.go b/command/agent/ui_endpoint_test.go index 510ff973a326..da4f7e590fc1 100644 --- a/command/agent/ui_endpoint_test.go +++ b/command/agent/ui_endpoint_test.go @@ -3,8 +3,8 @@ package agent import ( "bytes" "fmt" - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "io" "io/ioutil" "net/http" diff --git a/command/force_leave_test.go b/command/force_leave_test.go index 8189d41d2aec..d297380a8601 100644 --- a/command/force_leave_test.go +++ b/command/force_leave_test.go @@ -1,13 +1,13 @@ package command import ( + "errors" "fmt" "github.com/hashicorp/consul/testutil" "github.com/hashicorp/serf/serf" "github.com/mitchellh/cli" "strings" "testing" - "errors" ) func TestForceLeaveCommand_implements(t *testing.T) { diff --git a/consul/catalog_endpoint_test.go b/consul/catalog_endpoint_test.go index 9af0b0bd2bef..233c39fd28c2 100644 --- a/consul/catalog_endpoint_test.go +++ b/consul/catalog_endpoint_test.go @@ -498,7 +498,7 @@ func TestCatalogListServices_Blocking(t *testing.T) { } // Should block at least 100ms - if time.Now().Sub(start) < 100 * time.Millisecond { + if time.Now().Sub(start) < 100*time.Millisecond { t.Fatalf("too fast") } @@ -544,7 +544,7 @@ func TestCatalogListServices_Timeout(t *testing.T) { } // Should block at least 100ms - if time.Now().Sub(start) < 100 * time.Millisecond { + if time.Now().Sub(start) < 100*time.Millisecond { // TODO: Failing t.Fatalf("too fast") } diff --git a/consul/client_test.go b/consul/client_test.go index b0b748eef102..10ab0fde7f31 100644 --- a/consul/client_test.go +++ b/consul/client_test.go @@ -2,8 +2,8 @@ package consul import ( "fmt" - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "net" "os" "testing" diff --git a/consul/health_endpoint_test.go b/consul/health_endpoint_test.go index b817f9db0bdd..8f3725ec2ff1 100644 --- a/consul/health_endpoint_test.go +++ b/consul/health_endpoint_test.go @@ -1,8 +1,8 @@ package consul import ( - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "os" "testing" ) diff --git a/consul/internal_endpoint_test.go b/consul/internal_endpoint_test.go index 9b08fe74da8f..e3c33fe92565 100644 --- a/consul/internal_endpoint_test.go +++ b/consul/internal_endpoint_test.go @@ -1,8 +1,8 @@ package consul import ( - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "os" "testing" ) diff --git a/consul/kvs_endpoint_test.go b/consul/kvs_endpoint_test.go index 32ab830ea998..69b8fdfbaa4d 100644 --- a/consul/kvs_endpoint_test.go +++ b/consul/kvs_endpoint_test.go @@ -1,8 +1,8 @@ package consul import ( - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "os" "testing" ) diff --git a/consul/leader_test.go b/consul/leader_test.go index 8e4c4a63b101..d6aec6bd1089 100644 --- a/consul/leader_test.go +++ b/consul/leader_test.go @@ -1,13 +1,13 @@ package consul import ( + "errors" "fmt" - "github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "github.com/hashicorp/serf/serf" "os" "testing" - "errors" "time" ) diff --git a/consul/server_test.go b/consul/server_test.go index a0687bbfdf51..dc984ba35ad2 100644 --- a/consul/server_test.go +++ b/consul/server_test.go @@ -1,6 +1,7 @@ package consul import ( + "errors" "fmt" "github.com/hashicorp/consul/testutil" "io/ioutil" @@ -8,7 +9,6 @@ import ( "os" "testing" "time" - "errors" ) var nextPort = 15000 @@ -293,7 +293,7 @@ func TestServer_JoinLAN_TLS(t *testing.T) { // Verify Raft has established a peer testutil.WaitForResult(func() (bool, error) { - return s1.Stats()["raft"]["num_peers"] == "1", nil + return s1.Stats()["raft"]["num_peers"] == "1", nil }, func(err error) { t.Fatalf("no peer established") }) diff --git a/testutil/wait.go b/testutil/wait.go index e3e0e9149b1a..0bf40937ad41 100644 --- a/testutil/wait.go +++ b/testutil/wait.go @@ -1,9 +1,9 @@ package testutil import ( - "time" - "testing" "github.com/hashicorp/consul/consul/structs" + "testing" + "time" ) type testFn func() (bool, error) @@ -27,7 +27,7 @@ func WaitForResult(test testFn, error errorFn) { } } -type rpcFn func(string, interface {}, interface {}) error +type rpcFn func(string, interface{}, interface{}) error func WaitForLeader(t *testing.T, rpc rpcFn, dc string) structs.IndexedNodes { var out structs.IndexedNodes From 9abd428982781d6c22767c8c639906ca4c288320 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Fri, 16 May 2014 15:58:07 -0700 Subject: [PATCH 22/30] agent: Require PUT to SessionCreate --- command/agent/session_endpoint.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/command/agent/session_endpoint.go b/command/agent/session_endpoint.go index 8408baf949c7..d304312e2447 100644 --- a/command/agent/session_endpoint.go +++ b/command/agent/session_endpoint.go @@ -10,6 +10,12 @@ import ( // SessionCreate is used to create a new session func (s *HTTPServer) SessionCreate(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + // Mandate a PUT request + if req.Method != "PUT" { + resp.WriteHeader(405) + return nil, nil + } + // Default the session to our node + serf check args := structs.SessionRequest{ Op: structs.SessionCreate, From c071932f925a51d9ce3a9219e4c55538b3bf368d Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Mon, 19 May 2014 11:29:50 -0700 Subject: [PATCH 23/30] agent: Session endpoint tests --- command/agent/http_test.go | 10 ++ command/agent/session_endpoint.go | 10 +- command/agent/session_endpoint_test.go | 155 +++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 command/agent/session_endpoint_test.go diff --git a/command/agent/http_test.go b/command/agent/http_test.go index 60d28b6f466f..6b57693f4189 100644 --- a/command/agent/http_test.go +++ b/command/agent/http_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/testutil" "io" "io/ioutil" "net/http" @@ -255,3 +256,12 @@ func getIndex(t *testing.T, resp *httptest.ResponseRecorder) uint64 { } return uint64(val) } + +func httpTest(t *testing.T, f func(srv *HTTPServer)) { + dir, srv := makeHTTPServer(t) + defer os.RemoveAll(dir) + defer srv.Shutdown() + defer srv.agent.Shutdown() + testutil.WaitForLeader(t, srv.agent.RPC, "dc1") + f(srv) +} diff --git a/command/agent/session_endpoint.go b/command/agent/session_endpoint.go index d304312e2447..f8163f6f1139 100644 --- a/command/agent/session_endpoint.go +++ b/command/agent/session_endpoint.go @@ -8,6 +8,11 @@ import ( "strings" ) +// sessionCreateResponse is used to wrap the session ID +type sessionCreateResponse struct { + ID string +} + // SessionCreate is used to create a new session func (s *HTTPServer) SessionCreate(resp http.ResponseWriter, req *http.Request) (interface{}, error) { // Mandate a PUT request @@ -42,10 +47,7 @@ func (s *HTTPServer) SessionCreate(resp http.ResponseWriter, req *http.Request) } // Format the response as a JSON object - type response struct { - ID string - } - return response{out}, nil + return sessionCreateResponse{out}, nil } // SessionDestroy is used to destroy an existing session diff --git a/command/agent/session_endpoint_test.go b/command/agent/session_endpoint_test.go new file mode 100644 index 000000000000..fec4828dd1dc --- /dev/null +++ b/command/agent/session_endpoint_test.go @@ -0,0 +1,155 @@ +package agent + +import ( + "bytes" + "encoding/json" + "github.com/hashicorp/consul/consul" + "github.com/hashicorp/consul/consul/structs" + "net/http" + "net/http/httptest" + "testing" +) + +func TestSessionCreate(t *testing.T) { + httpTest(t, func(srv *HTTPServer) { + // Create a health check + args := &structs.RegisterRequest{ + Datacenter: "dc1", + Node: srv.agent.config.NodeName, + Address: "127.0.0.1", + Check: &structs.HealthCheck{ + CheckID: "consul", + Node: srv.agent.config.NodeName, + Name: "consul", + ServiceID: "consul", + }, + } + var out struct{} + if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil { + t.Fatalf("err: %v", err) + } + + // Associate session with node and 2 health checks + body := bytes.NewBuffer(nil) + enc := json.NewEncoder(body) + raw := map[string]interface{}{ + "Node": srv.agent.config.NodeName, + "Checks": []string{consul.SerfCheckID, "consul"}, + } + enc.Encode(raw) + + req, err := http.NewRequest("PUT", "/v1/session/create", body) + if err != nil { + t.Fatalf("err: %v", err) + } + + resp := httptest.NewRecorder() + obj, err := srv.SessionCreate(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + + if _, ok := obj.(sessionCreateResponse); !ok { + t.Fatalf("should work") + } + + }) +} + +func makeTestSession(t *testing.T, srv *HTTPServer) string { + req, err := http.NewRequest("PUT", "/v1/session/create", nil) + if err != nil { + t.Fatalf("err: %v", err) + } + resp := httptest.NewRecorder() + obj, err := srv.SessionCreate(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + sessResp := obj.(sessionCreateResponse) + return sessResp.ID +} + +func TestSessionDestroy(t *testing.T) { + httpTest(t, func(srv *HTTPServer) { + id := makeTestSession(t, srv) + + req, err := http.NewRequest("PUT", "/v1/session/destroy/"+id, nil) + resp := httptest.NewRecorder() + obj, err := srv.SessionDestroy(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + if resp := obj.(bool); !resp { + t.Fatalf("should work") + } + }) +} + +func TestSessionGet(t *testing.T) { + httpTest(t, func(srv *HTTPServer) { + id := makeTestSession(t, srv) + + req, err := http.NewRequest("GET", + "/v1/session/info/"+id, nil) + resp := httptest.NewRecorder() + obj, err := srv.SessionGet(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + respObj, ok := obj.(structs.Sessions) + if !ok { + t.Fatalf("should work") + } + if len(respObj) != 1 { + t.Fatalf("bad: %v", respObj) + } + }) +} + +func TestSessionList(t *testing.T) { + httpTest(t, func(srv *HTTPServer) { + var ids []string + for i := 0; i < 10; i++ { + ids = append(ids, makeTestSession(t, srv)) + } + + req, err := http.NewRequest("GET", "/v1/session/list", nil) + resp := httptest.NewRecorder() + obj, err := srv.SessionList(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + respObj, ok := obj.(structs.Sessions) + if !ok { + t.Fatalf("should work") + } + if len(respObj) != 10 { + t.Fatalf("bad: %v", respObj) + } + }) +} + +func TestSessionsForNode(t *testing.T) { + httpTest(t, func(srv *HTTPServer) { + var ids []string + for i := 0; i < 10; i++ { + ids = append(ids, makeTestSession(t, srv)) + } + + req, err := http.NewRequest("GET", + "/v1/session/node/"+srv.agent.config.NodeName, nil) + resp := httptest.NewRecorder() + obj, err := srv.SessionsForNode(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + respObj, ok := obj.(structs.Sessions) + if !ok { + t.Fatalf("should work") + } + if len(respObj) != 10 { + t.Fatalf("bad: %v", respObj) + } + }) +} From e0abf2e92cce6404610cc07599e70b83a9603126 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Mon, 19 May 2014 12:50:29 -0700 Subject: [PATCH 24/30] consul: Adding support for lock-delay in sessions --- consul/kvs_endpoint.go | 17 ++++++++++ consul/kvs_endpoint_test.go | 68 +++++++++++++++++++++++++++++++++++++ consul/state_store.go | 67 ++++++++++++++++++++++++++++++++---- consul/state_store_test.go | 9 ++++- consul/structs/structs.go | 9 ++++- 5 files changed, 162 insertions(+), 8 deletions(-) diff --git a/consul/kvs_endpoint.go b/consul/kvs_endpoint.go index fde952432f50..91d8f3bdeaed 100644 --- a/consul/kvs_endpoint.go +++ b/consul/kvs_endpoint.go @@ -25,6 +25,23 @@ func (k *KVS) Apply(args *structs.KVSRequest, reply *bool) error { return fmt.Errorf("Must provide key") } + // If this is a lock, we must check for a lock-delay. Since lock-delay + // is based on wall-time, each peer expire the lock-delay at a slightly + // different time. This means the enforcement of lock-delay cannot be done + // after the raft log is committed as it would lead to inconsistent FSMs. + // Instead, the lock-delay must be enforced before commit. This means that + // only the wall-time of the leader node is used, preventing any inconsistencies. + if args.Op == structs.KVSLock { + state := k.srv.fsm.State() + expires := state.KVSLockDelay(args.DirEnt.Key) + if expires.After(time.Now()) { + k.srv.logger.Printf("[WARN] consul.kvs: Rejecting lock of %s due to lock-delay until %v", + args.DirEnt.Key, expires) + *reply = false + return nil + } + } + // Apply the update resp, err := k.srv.raftApply(structs.KVSRequestType, args) if err != nil { diff --git a/consul/kvs_endpoint_test.go b/consul/kvs_endpoint_test.go index 69b8fdfbaa4d..c4131fdcbdd0 100644 --- a/consul/kvs_endpoint_test.go +++ b/consul/kvs_endpoint_test.go @@ -5,6 +5,7 @@ import ( "github.com/hashicorp/consul/testutil" "os" "testing" + "time" ) func TestKVS_Apply(t *testing.T) { @@ -224,5 +225,72 @@ func TestKVSEndpoint_ListKeys(t *testing.T) { if dirent.Keys[2] != "/test/sub/" { t.Fatalf("Bad: %v", dirent.Keys) } +} + +func TestKVS_Apply_LockDelay(t *testing.T) { + dir1, s1 := testServer(t) + defer os.RemoveAll(dir1) + defer s1.Shutdown() + client := rpcClient(t, s1) + defer client.Close() + + testutil.WaitForLeader(t, client.Call, "dc1") + + // Create and invalidate a session with a lock + state := s1.fsm.State() + if err := state.EnsureNode(1, structs.Node{"foo", "127.0.0.1"}); err != nil { + t.Fatalf("err: %v") + } + session := &structs.Session{ + Node: "foo", + LockDelay: 50 * time.Millisecond, + } + if err := state.SessionCreate(2, session); err != nil { + t.Fatalf("err: %v", err) + } + id := session.ID + d := &structs.DirEntry{ + Key: "test", + Session: id, + } + if ok, err := state.KVSLock(3, d); err != nil || !ok { + t.Fatalf("err: %v", err) + } + if err := state.SessionDestroy(4, id); err != nil { + t.Fatalf("err: %v", err) + } + // Make a new session that is valid + if err := state.SessionCreate(5, session); err != nil { + t.Fatalf("err: %v", err) + } + validId := session.ID + + // Make a lock request + arg := structs.KVSRequest{ + Datacenter: "dc1", + Op: structs.KVSLock, + DirEnt: structs.DirEntry{ + Key: "test", + Session: validId, + }, + } + var out bool + if err := client.Call("KVS.Apply", &arg, &out); err != nil { + t.Fatalf("err: %v", err) + } + if out != false { + t.Fatalf("should not acquire") + } + + // Wait for lock-delay + time.Sleep(50 * time.Millisecond) + + // Should acquire + if err := client.Call("KVS.Apply", &arg, &out); err != nil { + t.Fatalf("err: %v", err) + } + if out != true { + t.Fatalf("should acquire") + } } diff --git a/consul/state_store.go b/consul/state_store.go index eb27e62fa538..1b137f33ad27 100644 --- a/consul/state_store.go +++ b/consul/state_store.go @@ -10,6 +10,8 @@ import ( "os" "runtime" "strings" + "sync" + "time" ) const ( @@ -54,6 +56,22 @@ type StateStore struct { tables MDBTables watch map[*MDBTable]*NotifyGroup queryTables map[string]MDBTables + + // lockDelay is used to mark certain locks as unacquirable. + // When a lock is forcefully released (failing health + // check, destroyed session, etc), it is subject to the LockDelay + // impossed by the session. This prevents another session from + // acquiring the lock for some period of time as a protection against + // split-brains. This is inspired by the lock-delay in Chubby. + // Because this relies on wall-time, we cannot assume all peers + // perceive time as flowing uniformly. This means KVSLock MUST ignore + // lockDelay, since the lockDelay may have expired on the leader, + // but not on the follower. Rejecting the lock could result in + // inconsistencies in the FSMs due to the rate time progresses. Instead, + // only the opinion of the leader is respected, and the Raft log + // is never questioned. + lockDelay map[string]time.Time + lockDelayLock sync.RWMutex } // StateSnapshot is used to provide a point-in-time snapshot @@ -94,10 +112,11 @@ func NewStateStore(logOutput io.Writer) (*StateStore, error) { } s := &StateStore{ - logger: log.New(logOutput, "", log.LstdFlags), - path: path, - env: env, - watch: make(map[*MDBTable]*NotifyGroup), + logger: log.New(logOutput, "", log.LstdFlags), + path: path, + env: env, + watch: make(map[*MDBTable]*NotifyGroup), + lockDelay: make(map[string]time.Time), } // Ensure we can initialize @@ -1076,6 +1095,17 @@ func (s *StateStore) KVSUnlock(index uint64, d *structs.DirEntry) (bool, error) return s.kvsSet(index, d, kvUnlock) } +// KVSLockDelay returns the expiration time of a key lock delay. A key may +// have a lock delay if it was unlocked due to a session invalidation instead +// of a graceful unlock. This must be checked on the leader node, and not in +// KVSLock due to the variability of clocks. +func (s *StateStore) KVSLockDelay(key string) time.Time { + s.lockDelayLock.RLock() + expires := s.lockDelay[key] + s.lockDelayLock.RUnlock() + return expires +} + // kvsSet is the internal setter func (s *StateStore) kvsSet( index uint64, @@ -1367,8 +1397,14 @@ func (s *StateStore) invalidateSession(index uint64, tx *MDBTxn, id string) erro } session := res[0].(*structs.Session) + // Enforce the MaxLockDelay + delay := session.LockDelay + if delay > structs.MaxLockDelay { + delay = structs.MaxLockDelay + } + // Invalidate any held locks - if err := s.invalidateLocks(index, tx, id); err != nil { + if err := s.invalidateLocks(index, tx, delay, id); err != nil { return err } @@ -1395,11 +1431,20 @@ func (s *StateStore) invalidateSession(index uint64, tx *MDBTxn, id string) erro // invalidateLocks is used to invalidate all the locks held by a session // within a given txn. All tables should be locked in the tx. -func (s *StateStore) invalidateLocks(index uint64, tx *MDBTxn, id string) error { +func (s *StateStore) invalidateLocks(index uint64, tx *MDBTxn, + lockDelay time.Duration, id string) error { pairs, err := s.kvsTable.GetTxn(tx, "session", id) if err != nil { return err } + + var expires time.Time + if lockDelay > 0 { + s.lockDelayLock.Lock() + defer s.lockDelayLock.Unlock() + expires = time.Now().Add(lockDelay) + } + for _, pair := range pairs { kv := pair.(*structs.DirEntry) kv.Session = "" // Clear the lock @@ -1407,6 +1452,16 @@ func (s *StateStore) invalidateLocks(index uint64, tx *MDBTxn, id string) error if err := s.kvsTable.InsertTxn(tx, kv); err != nil { return err } + // If there is a lock delay, prevent acquisition + // for at least lockDelay period + if lockDelay > 0 { + s.lockDelay[kv.Key] = expires + time.AfterFunc(lockDelay, func() { + s.lockDelayLock.Lock() + delete(s.lockDelay, kv.Key) + s.lockDelayLock.Unlock() + }) + } } if len(pairs) > 0 { if err := s.kvsTable.SetLastIndexTxn(tx, index); err != nil { diff --git a/consul/state_store_test.go b/consul/state_store_test.go index d3bc1667cdf5..308335826c92 100644 --- a/consul/state_store_test.go +++ b/consul/state_store_test.go @@ -6,6 +6,7 @@ import ( "reflect" "sort" "testing" + "time" ) func testStateStore() (*StateStore, error) { @@ -2059,7 +2060,7 @@ func TestSessionInvalidate_KeyUnlock(t *testing.T) { if err := store.EnsureNode(3, structs.Node{"foo", "127.0.0.1"}); err != nil { t.Fatalf("err: %v") } - session := &structs.Session{Node: "foo"} + session := &structs.Session{Node: "foo", LockDelay: 50 * time.Millisecond} if err := store.SessionCreate(4, session); err != nil { t.Fatalf("err: %v", err) } @@ -2095,4 +2096,10 @@ func TestSessionInvalidate_KeyUnlock(t *testing.T) { if d2.Session != "" { t.Fatalf("bad: %v", *d2) } + + // Key should have a lock delay + expires := store.KVSLockDelay("/foo") + if expires.Before(time.Now().Add(30 * time.Millisecond)) { + t.Fatalf("Bad: %v", expires) + } } diff --git a/consul/structs/structs.go b/consul/structs/structs.go index 0673ba39f7f4..b08a1c8e46e9 100644 --- a/consul/structs/structs.go +++ b/consul/structs/structs.go @@ -29,6 +29,12 @@ const ( HealthCritical = "critical" ) +const ( + // MaxLockDelay provides a maximum LockDelay value for + // a session. Any value above this will not be respected. + MaxLockDelay = 60 * time.Second +) + // RPCInfo is used to describe common information about query type RPCInfo interface { RequestDatacenter() string @@ -343,10 +349,11 @@ type IndexedKeyList struct { // Session is used to represent an open session in the KV store. // This issued to associate node checks with acquired locks. type Session struct { + CreateIndex uint64 ID string Node string Checks []string - CreateIndex uint64 + LockDelay time.Duration } type Sessions []*Session From 00a107dfd91b3f0c3c7ed07fedfed908039410cc Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Mon, 19 May 2014 13:12:15 -0700 Subject: [PATCH 25/30] agent: Adding support for specifying LockDelay, defaults to 15 seconds. --- command/agent/session_endpoint.go | 57 ++++++++++++++++++++++++-- command/agent/session_endpoint_test.go | 39 ++++++++++++++++-- 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/command/agent/session_endpoint.go b/command/agent/session_endpoint.go index f8163f6f1139..a5c02b4df55a 100644 --- a/command/agent/session_endpoint.go +++ b/command/agent/session_endpoint.go @@ -6,6 +6,17 @@ import ( "github.com/hashicorp/consul/consul/structs" "net/http" "strings" + "time" +) + +const ( + // lockDelayMinThreshold is used to convert a numeric lock + // delay value from nanoseconds to seconds if it is below this + // threshold. Users often send a value like 5, which they assume + // is seconds, but because Go uses nanosecond granularity, ends + // up being very small. If we see a value below this threshold, + // we multply by time.Second + lockDelayMinThreshold = 1000 ) // sessionCreateResponse is used to wrap the session ID @@ -25,15 +36,16 @@ func (s *HTTPServer) SessionCreate(resp http.ResponseWriter, req *http.Request) args := structs.SessionRequest{ Op: structs.SessionCreate, Session: structs.Session{ - Node: s.agent.config.NodeName, - Checks: []string{consul.SerfCheckID}, + Node: s.agent.config.NodeName, + Checks: []string{consul.SerfCheckID}, + LockDelay: 15 * time.Second, }, } s.parseDC(req, &args.Datacenter) // Handle optional request body if req.ContentLength > 0 { - if err := decodeBody(req, &args.Session, nil); err != nil { + if err := decodeBody(req, &args.Session, FixupLockDelay); err != nil { resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf("Request decode failed: %v", err))) return nil, nil @@ -50,6 +62,45 @@ func (s *HTTPServer) SessionCreate(resp http.ResponseWriter, req *http.Request) return sessionCreateResponse{out}, nil } +// FixupLockDelay is used to handle parsing the JSON body to session/create +// and properly parsing out the lock delay duration value. +func FixupLockDelay(raw interface{}) error { + rawMap, ok := raw.(map[string]interface{}) + if !ok { + return nil + } + var key string + for k, _ := range rawMap { + if strings.ToLower(k) == "lockdelay" { + key = k + break + } + } + if key != "" { + val := rawMap[key] + // Convert a string value into an integer + if vStr, ok := val.(string); ok { + dur, err := time.ParseDuration(vStr) + if err != nil { + return err + } + if dur < lockDelayMinThreshold { + dur = dur * time.Second + } + rawMap[key] = dur + } + // Convert low value integers into seconds + if vNum, ok := val.(float64); ok { + dur := time.Duration(vNum) + if dur < lockDelayMinThreshold { + dur = dur * time.Second + } + rawMap[key] = dur + } + } + return nil +} + // SessionDestroy is used to destroy an existing session func (s *HTTPServer) SessionDestroy(resp http.ResponseWriter, req *http.Request) (interface{}, error) { args := structs.SessionRequest{ diff --git a/command/agent/session_endpoint_test.go b/command/agent/session_endpoint_test.go index fec4828dd1dc..ad3d58a8225e 100644 --- a/command/agent/session_endpoint_test.go +++ b/command/agent/session_endpoint_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" ) func TestSessionCreate(t *testing.T) { @@ -33,8 +34,9 @@ func TestSessionCreate(t *testing.T) { body := bytes.NewBuffer(nil) enc := json.NewEncoder(body) raw := map[string]interface{}{ - "Node": srv.agent.config.NodeName, - "Checks": []string{consul.SerfCheckID, "consul"}, + "Node": srv.agent.config.NodeName, + "Checks": []string{consul.SerfCheckID, "consul"}, + "LockDelay": "20s", } enc.Encode(raw) @@ -52,10 +54,41 @@ func TestSessionCreate(t *testing.T) { if _, ok := obj.(sessionCreateResponse); !ok { t.Fatalf("should work") } - }) } +func TestFixupLockDelay(t *testing.T) { + inp := map[string]interface{}{ + "lockdelay": float64(15), + } + if err := FixupLockDelay(inp); err != nil { + t.Fatalf("err: %v", err) + } + if inp["lockdelay"] != 15*time.Second { + t.Fatalf("bad: %v", inp) + } + + inp = map[string]interface{}{ + "lockDelay": float64(15 * time.Second), + } + if err := FixupLockDelay(inp); err != nil { + t.Fatalf("err: %v", err) + } + if inp["lockDelay"] != 15*time.Second { + t.Fatalf("bad: %v", inp) + } + + inp = map[string]interface{}{ + "LockDelay": "15s", + } + if err := FixupLockDelay(inp); err != nil { + t.Fatalf("err: %v", err) + } + if inp["LockDelay"] != 15*time.Second { + t.Fatalf("bad: %v", inp) + } +} + func makeTestSession(t *testing.T, srv *HTTPServer) string { req, err := http.NewRequest("PUT", "/v1/session/create", nil) if err != nil { From 6131fad0686a6971308651a77d4e67988bb0f0c4 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Mon, 19 May 2014 16:14:03 -0700 Subject: [PATCH 26/30] agent: Adding locking support to KV store --- command/agent/kvs_endpoint.go | 12 +++++ command/agent/kvs_endpoint_test.go | 70 ++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/command/agent/kvs_endpoint.go b/command/agent/kvs_endpoint.go index 7791d1e9f3c8..2adaae3528a2 100644 --- a/command/agent/kvs_endpoint.go +++ b/command/agent/kvs_endpoint.go @@ -156,6 +156,18 @@ func (s *HTTPServer) KVSPut(resp http.ResponseWriter, req *http.Request, args *s applyReq.Op = structs.KVSCAS } + // Check for lock acquisition + if _, ok := params["acquire"]; ok { + applyReq.DirEnt.Session = params.Get("acquire") + applyReq.Op = structs.KVSLock + } + + // Check for lock release + if _, ok := params["release"]; ok { + applyReq.DirEnt.Session = params.Get("release") + applyReq.Op = structs.KVSUnlock + } + // Check the content-length if req.ContentLength > maxKVSize { resp.WriteHeader(413) diff --git a/command/agent/kvs_endpoint_test.go b/command/agent/kvs_endpoint_test.go index 24ec51372f7f..867b2a01f475 100644 --- a/command/agent/kvs_endpoint_test.go +++ b/command/agent/kvs_endpoint_test.go @@ -339,3 +339,73 @@ func TestKVSEndpoint_ListKeys(t *testing.T) { } } } + +func TestKVSEndpoint_AcquireRelease(t *testing.T) { + httpTest(t, func(srv *HTTPServer) { + // Acquire the lock + id := makeTestSession(t, srv) + req, err := http.NewRequest("PUT", + "/v1/kv/test?acquire="+id, bytes.NewReader(nil)) + if err != nil { + t.Fatalf("err: %v", err) + } + + resp := httptest.NewRecorder() + obj, err := srv.KVSEndpoint(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + if res := obj.(bool); !res { + t.Fatalf("should work") + } + + // Verify we have the lock + req, err = http.NewRequest("GET", "/v1/kv/test", nil) + if err != nil { + t.Fatalf("err: %v", err) + } + resp = httptest.NewRecorder() + obj, err = srv.KVSEndpoint(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + d := obj.(structs.DirEntries)[0] + + // Check the flags + if d.Session != id { + t.Fatalf("bad: %v", d) + } + + // Release the lock + req, err = http.NewRequest("PUT", + "/v1/kv/test?release="+id, bytes.NewReader(nil)) + if err != nil { + t.Fatalf("err: %v", err) + } + resp = httptest.NewRecorder() + obj, err = srv.KVSEndpoint(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + if res := obj.(bool); !res { + t.Fatalf("should work") + } + + // Verify we do not have the lock + req, err = http.NewRequest("GET", "/v1/kv/test", nil) + if err != nil { + t.Fatalf("err: %v", err) + } + resp = httptest.NewRecorder() + obj, err = srv.KVSEndpoint(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + d = obj.(structs.DirEntries)[0] + + // Check the flags + if d.Session != "" { + t.Fatalf("bad: %v", d) + } + }) +} From f91b1c3bf468eb9d3b8b91494bd7fd1ed56d9824 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Mon, 19 May 2014 17:05:35 -0700 Subject: [PATCH 27/30] website: Starting to document sessions --- .../docs/internals/sessions.html.markdown | 107 ++++++++++++++++++ website/source/images/consul-sessions.png | Bin 0 -> 35293 bytes website/source/layouts/docs.erb | 4 + 3 files changed, 111 insertions(+) create mode 100644 website/source/docs/internals/sessions.html.markdown create mode 100644 website/source/images/consul-sessions.png diff --git a/website/source/docs/internals/sessions.html.markdown b/website/source/docs/internals/sessions.html.markdown new file mode 100644 index 000000000000..f45fffeb358e --- /dev/null +++ b/website/source/docs/internals/sessions.html.markdown @@ -0,0 +1,107 @@ +--- +layout: "docs" +page_title: "Sessions" +sidebar_current: "docs-internals-sessions" +--- + +# Consensus Protocol + +Consul provides a session mechansim which can be used to build distributed locks. +Sessions act as a binding layer between nodes, health checks, and key/value data. +They are designed to provide granular locking similar to Chubby. + +
+Advanced Topic! This page covers technical details of +the internals of Consul. You don't need to know these details to effectively +operate and use Consul. These details are documented here for those who wish +to learn about them without having to go spelunking through the source code. +
+ +## Session Design + +A session in Consul represents a contract that has very specific semantics. +When a session is constructed a node name, a list of health checks, and a +`lock-delay` are provided. The newly constructed session is provided with +a named ID which can be used to refer to it. This ID can be used with the KV +store to acquire locks, which are advisory mechanisms for mutual exclusion. +Below is a diagram showing the relationship between these components: + +![Session Architecture](/images/consul-sessions.png) + +The contract that Consul provides is that under any of the folllowing +situations the session will be *invalidated*: + +* Node is deregistered +* Any of the health checks are deregistered +* Any of the health checks go to the critical state +* Session is explicitly destroyed + +When a session is invalidated, any of the locks held in association +with the session are released, and the `ModifyIndex` of the key is +incremented. The session is also destroyed during an invalidation +and can no longer be used to acquire further locks. + +While this is a simple design, it enables a multitude of usage +patterns. By default, the [gossip based failure detector](/docs/internals/gossip.html) +is used as the associated health check. This failure detector allows +Consul to detect when a node that is holding a lock has failed, and +to automatically release the lock. This ability provides **liveness** to +Consul locks, meaning under failure the system can continue to make +progress. However, because there is no perfect failure detector, it's possible +to have a false positive (failure detected) which causes the lock to +be released even though the lock owner is still alive. This means +we are sacrificing some **safety**. + +Conversely, it is possible to create a session with no associated +health checks. This removes the possibility of a false positive, +and trades liveness for safety. You can be absolutely certain Consul +will not release the lock even if the existing owner has failed. +Since Consul APIs allow a session to be force destroyed, this allows +systems to be built that require an operator to intervene in the +case of a failure, but preclude the possibility of a split-brain. + +The final nuance is that sessions may provide a `lock-delay`. This +is a time duration, between 0 and 60 second. When a session invalidation +takes place, Consul prevents any of the previously held locks from +being re-acquired for the `lock-delay` interval; this is a safe guard +inspired by Google's Chubby. The purpose of this delay is to allow +the potentially still live leader to detect the invalidation and stop +processing requests that may lead to inconsistent state. While not a +bulletproof method, it does avoid the need to introduce sleep states +into application logic, and can help mitigate many issues. While the +default is to use a 15 second delay, clients are able to disable this +mechanism by providing a zero delay value. + +## KV Integration + +Integration between the Key/Value store and sessions are the primary +place where sessions are used. A session must be created prior to use, +and is then refered to by it's ID. + +The Key/Value API is extended to support an `acquire` and `release` operation. +The `acquire` operation acts like a Check-And-Set operation, except it +can only succeed if there is no existing lock holder. On success, there +is a normal key update, but there is also an increment to the `LockIndex`, +and the `Session` value is updated to reflect the session holding the lock. + +Once held, the lock can be released using a corresponding `release` operation, +providing the same session. Again, this acts like a Check-And-Set operations, +since the request will fail if given an invalid session. A critical note is +that the session ID can be destroyed without being the creator of the session. +This is by design, as it allows operators to intervene and force terminate +a session if necessary. As mentioned above, a session invalidation will also +cause all held locks to be released. When a lock is released, the `LockIndex`, +does not change, however the `Session` is cleared and the `ModifyIndex` increments. + +These semantics (heavily borrowed from Chubby), allow the tuple of (Key, LockIndex, Session) +to act as a unique "sequencer". This `sequencer` can be passed around and used +to verify if the request belongs to the current lock holder. Because the `LockIndex` +is incremented on each `acquire`, even if the same session re-acquires a lock, +the `sequencer` will be able to detect a stale request. Similarly, if a session is +invalided, the Session corresponding to the given `LockIndex` will be blank. + +To make clear, this locking system is purely *advisory*. There is no enforcement +that clients must acquire a lock to perform any operation. Any client can +read, write, and delete a key without owning the corresponding lock. It is not +the goal of Consul to protect against misbehaving clients. + diff --git a/website/source/images/consul-sessions.png b/website/source/images/consul-sessions.png new file mode 100644 index 0000000000000000000000000000000000000000..babd143f75c1b0777a4bf40b16ab550732762cf1 GIT binary patch literal 35293 zcmZ^~1yG#9(l&~_26uOYh2XFRcM0wg+}+(Bf_rca5Hz@Na9P}gE$;4eH|IOQ+*@_2 z+N#}oyL-Byp6TxCd5Kn0l14`%MuCEYLYI}1_zDFD-3|FrAR$0TL?()hprBB`Sc{9R z$cl?os<=2>SlgLHLCK)!rF*KYE#pUmx_G;$K2THA-4OdAQ_cv=;K8Xz%Lb{yM1;|M zNWe?sVl!!)srN!lM^c+g;-ZFzd|O1m!crORQbz$9v3aasf@6v=rWp#XYdH ztWjRNRX4QUKjFp(St-1l2mGOG4A>*k@TKoz{?^p6V>@Sv~eF zZ;%J5y$TQeJZoIipMZXRiTk0mVRur^-h~f!N2{Ad*4Uk<0o7!WAiBF%t$a=ibp;RA zw4QrS9${yg{AU}D0X6pI#Lg8u-x2359nq}-g*%vy1BTTYoiyl^6YM%AR9KJ< z7Hm2!n+%M#C|tuAktN6yR3NaYE3C)9OWcLPZOb7Db8Zljne5 z54J`Mxxm6J`ttKTehFrl*a8-bE|RzS7ZpBS*a7kA?^wyAk5lv&gv_Bo3&mB0Z)n_z z+mQ@Hp9&DCD7g_{q5Z?KGht>;(3hZ^gO=+ANs*-c$2KLr@Ny9edRMpPTJc3NTKWmL zNG_mQ!;Sh=kVJ=(V5gz_K~jvNUZQPMr+`*MeJ+R3zu;Oh7vnmDj)CY~^p{?p%P>##EoM z^rcgllwK(XS2lu{;43XtR#ULLtPQmRy)by2@P={&N28(kY)Zt~0EMpS`vZ)Gp7Fsgl`eQLqQb()1zTTT7K;j)t1MQPunzGZ)l{ua9Z zc|et}GdVc9<%4pi=waOqT`3i@*lelyOly;3lfa7k3I~}qZ^`p)%HhBb#X;8%vN@f( z-Y(-V=`QhZU&?--@wc&dwQ`#f51G8Tmb(%#Q zbXww11r5h}-1)qp_CJ;9IBe)`Ol+2Ip6A)B;z8vg&mW*XBkuQ|m&l{K_Gq8hC!1T~ zk?>IpaW-)`F%hu_aR&F`XQFI}?2+uw88e{nj zYkr-Uf1DO6`CBDiV_MOuZCR|;IEiDB%%;ctrA5C*%~MQSo?V;Ws8;p7P((GqCcmUd zx<@Z?w0DYf#W?m$PG3b~Okt1750m08Q<|SiRnyGHy2XNne5TFLF_$7>J^H%nJlSN~ z_z^|5rMY!=<91LJ2+elSik`dbbH`9o-I|V0wbViZKTv4cuh={1kp)#aRy#Hi|1Z7~ z>nW?+$75Cs9VQ)kZTqI9x+mbupKMzLQ~#j|8?b5PWZB=n-!+vVH#~Yy{ROiP>i*() zD0P7DFz%?I>7O|fg(JNpyD;vT+&maN!u;p4j}Nw5Coki`hz!0A^|P|&8+kgDOq1x7 z#i(cm7X))330GtTZ@@IcAd4+lPc=~(2 zc(MPq_^b@a2Um@JfKmb{1*eH9g35*b0p1-B*azL$5kkQ}V??VzhYut6BJL3`g!~!J z0Po3_kd2kQn*PMQRl_U5%M5V5VF28px$3VZn8};cp zGn120kZoYSN8|KY?kfDcF>3SNgYzeVk}dX)m*UF(x$@Z%(q|5VJm~ zTA*6jZEV$ezBee=CZ@X8^>I=U!*0M1&+hcEMPrT4Y6i98^zFB??U-$4y0zrv4IO{2 z$G_uuVW(ZCccsssn;uA>Ykv>hueGzj=@>3g{n|9B*&V+sx@zgrI_cIl(Q`J&v`#cj zY~reD=#s0ok)AI>~!v90-H zKVY)9RSg6k7ag_tSOyveN}S4ECV`~GrAvqU6W!+p=N5S@cvE@$_%FAI9w^R#pb0H|$N2v|WO&WLYo5}L(Jfq9TZnEh;j#8r@ZC5J zn$07dB>M=+dH1`CyUWUM9JQ1OB~9Yx=J4lx23&~^hF``;6GRGVxrwwl?abu;s{6G* z7MxLQc-iJsx!U#oqLEQ$`HSB`&_B<=>onpueu2E>N4n8Tfc4?;bNjACr^eTGa7{ut zy#M}_!87~%UAzBo!h0SGY%Idl8C$pSyWyJfq5t~pG-4&DAz<8x@73zn^aNbQfTGO& z<0qizb^c+|qO$Aeq+rn))EH%%7||-u{+{+&s|Td=v z#^N$&!QnkjZ_W2<(^dHap3hyD-$wpi*iIj30p!*zy7`|L?(Fx?4t8cet z0e8W#h0mrpLrI4x$`8tE`SAcxKjfDyFM4~6y^Z^@>t`+jod8HpiR>h!;|c|Ziu3OS z4V9fs2q`z`t<|;Nv=tQuOdajnOw1g=nX`G>J3*+SpoF{xAYbjx-ApLG?Cl&}1-yi* z|3e`F`TlR1otpALByP6C)Y^(Fl;Vyq=9Hh=xY;x{@qcD=aQ&}Y5Chr&Jz?i$<6!^avLUQO z|3(GG9qpZ5%w1g}@hPo-M)`|Kda3;oQHClG>)Ej6P5#mNLWcTtMl z@w@K7tTqqo2>+3(0{0aK`m-O?fWyb?{|Y4*N&7b&gf%!-OcdQnu3itd>Oa#_t7ubQ zUTAA=p>zT%mFpGLcy>*l-Qm)f zmLFe`S|FW5FiiF41S+znICIywC7jqUBA?Z(tkSFV&4H@)x#WG3ut@&pVAh7+7^!K* za6C*)PjidqM7}VMkZAD@G%GG3jY7{gj@9awvsiMZ4AA@6O5n4fNKyO`5;US;hP(DY z?n>}%y%Ff*c^?MDB=Ilo@@7~9m;u28g|~=L)NQws^ZwKAKew*&tfl#MWSlM0^m~<= zFPu9+1t6LDL|O2St;b%f`wW1RFrzA1NSFb-fypv~b!Ot7#b{AzT0`-ch3x$RMt`g3G{YH69D1ba9}Y$0+1Qpng#R>vc?EN*@|Yx0bUTz3 zzB5HpbW-$E908K<9T<tJbGYd~+fsxs8RuLnV!g5vylBw`NE_bbj>l0ZbHunLwe4nRq?`Law}t$v@pC zz!bV#RTg!bbLYLN4YVUbz;D%U>(Sq#T zLbbMA$jkW47K?X_&Ew0sORDlCzt0Zi>kZIKx63b>#+NzptI_W_0T+CtS94vgZl=xe zG&AHz!z#PvSn*;lLqm9jZa15#mbDkZCnV1COzal1?&b1-KH#yd#w3)N&o1V7Dzv=5 zY9PT1(f~AL>}W=Z(RPs7&aYDXSQ2Z**O&u!--}e=tA+HIV;3~g^FcC*XcX#^<}6y& z>C{y!3dzBqSL5g2M=Qvt;&AwVcS2LpXadFpZgI=tDpwsfNyHqIX z=x<{~CK5SJ{y(=FuKK^tJjcl&lrtI4EP)J5%W4Gbt&BDMb%f|#28PJ_SfZ#tNhgw) zIZF4)j^R%fHRb75$lVIeH5hmv2|l1Os}s%(Q3f6;D!n|B>GJI(u5`J6(x_0+%WgTO z-j2!#ceajh=TDlHE4^d}X>-i# zcu<4eae-DQDM?-0@Pe}C#r5XE#AmDi_+SXR_@LxzqGCfwOKqOm-FjQgakfhNuM&r0 zI{AIAh&zcQ@ij&on_h7HJ%1uGkG*v4zT)Rg`A(^~1TsOXT4C36v^IQZ^)Ljqt_Yh& zTdAHmuJ9lJ56l8}f%z%O`sx9{-?F!^E^abdJXE`;$@4)@9hUAZUZ?m7LXBt1Qk>7x z#Nx2aS$mDA&PefH8H}lh##o(Sp#e-aQ7G=h{)CD-H%$J|yCidakYiPaNR%)%rvhfIG?WqG<|YX@D|`e4cL}UDD)_yU#T`mn`Uf&$>r~J8~&l zsB49=fis~k=(w;J-{`X+!CFm6Ep2Wix0q_HO8#_qpf`f4hJnVziT$%O3WfwTQ?HN2 z4-VPPJ-UIw8KW5=*D#S7g44@KBp$EKz6iOyEB^b0=uaQNX{ui(%;E-yVegv1C|2nl zsu2OSNOgEn2$p&xbh^prO*9#QtOX&4zq7ZQ7CF6ej4Vjy&IM3y4*}i$hEJ_5u3F6?`8o2~?fE9jU z{`*$75)ItmI#TqE{21rJ#iy?(ahRhU-D6Nsm#aygQ@aN1M!Z-dW+4}NF3ABx275Qg zaT0^t>D4aiU_fL)VXO(>Wn_-3p8s`l1shmz&h5)P}MP@^1@Xj%(}X7b>fc~%ry_?B^8{g4oE zP0}5DK~2f5@%O|pN{1;V0Mpjiq1|EZhmDsVhfb?fc*Q)9WF-o>wkX7iBuZO&RQw_r zyeid|vl@nBa+v;}uW7nhzo1B_xjH|%p8p`TkeRG+kO7tB0;szsgy=vVC-@ZM)p?z_oj@GH_1-IhF{2BxJ-^WlW&Rj4R!C zn(6m=>5?0omi!=fsD=gD>U_MsqUh=d+J%(`-dD1aXlg=sE_)S#x7LV>jE4jszwG5l zv$&>xKjxKIm6F;@g9-fUQga_V>ZAPX@A>4m#lZ>X($k#9spQAmKOmMKj)bLksPZdY z3&TG4F`Uiq-F)wbr~I@xSe|K*74S8-Gb6&aCAH)+{^ZStf+S;azKwBSUxF<%qUbqA zpP7}i4Tevkm!nT)D{~hSGYp$-Vm{FK+)&_x!grszI~QCJgsM{*VUnf`Sa0CdOVADe@3mU?7HWoX)oO`{X0@8&evuqgs#_q;kn`sc=gm4 z#cHFr?|bh;s@u23Edib1Tn}q!t>rkpb1 zyEHcZtf&i#GArQK8#Uy!9~Kr1zWxY`Hq*@=J70D)ytm?ZGBtx1HdJmiXik3VOCFa&X^~f2Ov`a(&1P}9Gffya z#oq?0%H1)WTa=n=2}J{%NRAMnL!teWjOu_h%T?EMheXf>6a(z?kB`^)f151tdDhoe zV&yWJ9MMv+YZR_R!`W7OYmzv2nKi~V-Y)PiHfEb=lf)4##dfpT)L4I*Kd}W0xl~`V zXb9rG9Z=c4G8`?n3!on?i%HOC1O6=pdv%{ro|ch&#xtZtEms%_Q`WXoeV?w*8ObX# z^r=$G(V;gF1qRq@ADMK@V74$HmfLvyIFNe8;8E4k&wgh)Nv#(|!(X!QV8%DQ;Qf?G zRrCT1lZFjQr|%xoCX)%nRK9#dLi@20nDikjN5Ng+j8On!^WgVK=ccK+x=+`HJwQV3 zLh{#|ZBpQ1j&$9_@5cTVpd4YO<{=p+E&!W8jbb=-A>V9lI%jxYw^rt1pllY{^fqYdnDH_tG_fYQh|%g%o_ev5p;EkglKvUFhU2_8CO z*S|8@bRw6VK_fGqklv^GPvKap7UPdHYCSN6+&Y|!&mc5;s(p>c=@nYIR? zf_9Ru5y*GS%Vf&^Pv59!!cUw5QFBDye`4U$4J@-q38gIIV+lKqVTaDg(XZnEcVi1&V`MvpI@ph3PKHxdz*D)rV87-s>CEKvezQwOMwq3 zx6(Yu+_2@p8U3MfZqon9(vb>)pL!pabZG>;S%ny?ReXMKc3-;UlSe{fX#Tj4v-*se zY2f`4qem(4gzi9MB(!=!)0?OkZT%aPV$~Cc9Ap_}E>xoBB1zFkq1ns60WDC-pmm&hlQ2N`)!^ej+R(S2xq zTdY*Ys7`F3760iU_OjU*8U}D5iR?|LlRwzaFe*dFp`QvTFqFApbH&v7sm??X|9TAq zREkGp+$>w>t5#{&T;)Yh<_XR=f<)vEIy@#k_4}}^^;#0x+IBwtH0Wq9`A4%!`IOs_ zl**_YazS8U9>Km6W$S|+-rKSfAP*9{M*C>co_f{_#%SM)AgR!ATWb3H(dm985MaOD z5WeocPj;ME%YydQWIcNTP6SM92`yt&2>Yb|@Vk=rIcwWLNRlQm%yH~qCc6S9NWJ0( z%TO0YuqPkbtRYADCecVA05Ji$p;{#}*oS`$Ga{e0I}~$xrbig^6;KJ-XnTB=556o^ zs!7NB#uI<0*8HkCh!2=jTM3gg9*iS$o8;eD$zU_6_%;yJWVg)ab6nHHNFGlfG#ElW zp7dycgXKAv{;^Uvh5iQO{%y`}4B>^^p0{jrp93m^=aWYBH@Ekcib6Hnw|_P%b-TGN@SNL%(EGIf%vv` zKZL4(jJ+rKtn*rlGaxzwwwajQVNIK7(v(EJgshp^_;wevKxMmpZe!eBmIncrlceoO zKdb(1_S~)5x=irYb*sSjONjdVnZT6wfk~k4_@qH57n^-9KSx%6+nO(-33p4lB?8`B zZ*GcAZh|m<`j)tY`9a$4YRbZCbuoCrANH%wX=<)JFALfha$yLr6vZ zGDV&aXgq2)=s*{SS+dQb%awpNd-OPh4d1cv_cK8|DQHn z)MZpT2|E_IDumiiR;oT1eTcj4sTb2IYK<0(#GfqU>~@B>U?HJK z(4;CxK$4Oa()0@tO^=8dfbjtG`;ckrJ4^Q4@qsW^rkAS8<=*Okei$lN%oh<(-Bd;Y zKb%&pEfmzR~>@?*6l_HM5UT zGWmX$(K8%^cOG$FC>}}QH4W>!;RQy2w6YZ<6y8BZ4d9pS z23+T})6e@Cm{!1J$YlNmpWz^Ia+XA+?`zj&$+4~E%!?1;GgNe5jj}lN?-1!nw7Kq& z4tkP?#K)gMOcp@UAcQ`>gr_M2sY+5-q3`phaLvb&izePd&?Jgayf|C$%Ar@vC1jQz z@6U4?W#V>R7dl#QWccm4@x##PknZuqew?cWd58|8K&*T-t!G)XJJ36zwb>ag#b0PP zT_hQn7JIoT6;Dh@u{UtxTEu7PH?&3};z0t+&ym9wN@2=}tOI(7L2sR8e+c8|5T^;m z35imeyZ!s2y#T*2RH~#r-R}?J#H&Z98>q{V;wQ@L<^O5oM)AQc+4)^h@T!F(pj-WI zcfWbP-9C?r9B&we;F9u78lHjnwGnU2}r{sNbp*A?kd# zKc4jorxGc95ed>VW``-TNkxD@d2cY{tJQDgGC5$<;mzHT-1pq@Zy^QZMYC7!GZ^(X z@x8k({xS$Vg^k{uN_jh~s)e)6y>nf)Z#f*imoP0#eAIySdNYafuz7MG+xyk;@kQ#QfY_M{)p?BUNT3BcR!*gL_Kf3zY)xy--c%|=JuO7 zZbOonI3yF(L7=V(Yo8N^?3c>E{irmqe-4%fXz=9jBQANZR+HQ4Ktd(kBb<807Wnhp zVR=3q$p$CdN|7Y$W0OYbp2qbqJH)To1fbMtMFx9$og!X?$<|! zo9%@e+7(EcU8`D*(Dn9o4B6l7MCqUx>qPc^s50WmhfA>XL%;V=2gnXPf4|OupMoSZ zMqMLKkg{Q@bu0n967OF(oZ$13?Yd>M^6p%Z73;ZQ27=3t6-Iu__-8+~F-|Y@qN1PR zsDjY5 zV}i-z5p_SsE}cdT^&$ymYd+&BoURzJjg|}&DTtM2(69(oPq)XFPFwwL@XEvYdMyrV zCVwHt#u&jyhudND_uVN4ODTx2vMyx~%Qw;YD^A+Cgx;xxA8A|lF`ha7uerL?DG{NuEDO4*#TrwS+k>>B*yANp<% z3Be6$wYXxfK$x)?ZZ%jNvTa5v&ILSS&_XDmcK-9=hSW>0S!lKXJmM|phz-yl&F9Y6IWQ& z{W3i0?qNHY;&F5Wf_47L0Tq{B|A5&Gqy{+bem>o~}kt0Z~mV5xtysNwNbs?boHMndF^8h`M=9sD$ zj|gZWxpM%?3y?Cf*b#|F;L1cgJ9e$j!-pi>FW!S3sLZ6A zOKBhV(m84)=@uS<(_h_>7t*MvD7pOOU|_T*^&rOy#uD6T+LdN|7U@irtUzV~G*UYJ%~HI#g2ZT6ugwhV0URI)uPIA?(6C( zp2M+zkL396)A_}YzvEdNd8>9@*8|~u=k`Xo4gs@KSy41CMGrH{+x1`rO>O4{yMJp# zaOn%-6_A?Q9RZq9giwrI%pKnSMMhE^@q~=8HL>}Wrv_iA;?ENSGoNtdZ=GXgN}gfIZR-AgNgTGt{Wkt?fM??tCKlw{&gSopqHn-Ca5^P zYL5#@NM)i)ITYNE2%tow@KDm;A311JIrK)}TK`L8Js$?o(|mDoofCjE)#BiW^4n&K zoDO{?-24I3opdcBqK2K^rpriD@2#}>LEwTKcx{x4&L7&tkU3H@gNosyK^PQ|5HJ*8 zPBkKz5UkDydiq<@3{MoisVAxs;frVb$q!^>27b~Ej$8QD&m&%EmPd;ckP6i4!!e46pgkd|> z)9HIU_@&?mDzpz8pL0-|5DZEjl}aA*$o!^SXZ%G+A>384QE#2XA~gw_Edms+dqk-v zxhfjjqQ<^!bDro5mAK9&EKZ_%Xd0itO6|Oqy-LRl+;0ljmdQM&hBM}a3lV%_rPMaC z(p7_oH-1gRZfYzWt~KE zngM8J29+UQieGgcBE7UL55Zd9d~6sGXyu*pnCp95%~#X}4?ABw1Ks<1CaHdLNkgvB z5!$=@>n`P!OuGq49}Nl|vVDHuT!tI*T@nA-a)pCMF0j!H?rn508_i+h{w|?iO}T8@ z&Ft^DY00+>s{QOYy85tL)4~=4T;EqYvM?-x@s+6$1FYw1Y1$ip6j?`LL@T%*!V5I) zvfQA;@U6WGM)2dlgvAtdHz5{rfaeFMQo~tYMS(T3QqyIK{DMtB4>bBgZ?`w;b@LSFRz|T zjw0;w(8d($fs?)^-e%TBV!Co-Jif`{$-wgiw)dQGuxB~CKE-!d>VrvWu(}Tc#o8ZY zz=(oKpgK>h5^bSfBnv)Yoxob@?p9gS#>JH3cddGjD1|VB?jHyh#bf%g(Y-$qIw3v> z!vHdrH>Hu-Sb+?eUt~xveFMFkPz}FB!(kUD`x1%|$I)n?1qEFK1xpjj0?0jxrFQlg zak54ulAa~f?J)(#(@}07?#}*_xIAdXaSI%^7N-_ez0e8AHx^kDIe9>|)_WhnmlA#WNUZovmAk+6rv48|d7!uZ%Z~i9Y--tj3OZ0*n5;&#am@eguSdIW_2i;#ge(19o!IwhS z)@sfuu(NpjD&*Ww@bEfypjOPZH|;0Rm2PtenToHwG3wP4s%Iq(xX#qaqERV2AdfpHfk}3Ub6h7<4YXOPJX0azzoPMv zp4vm#@>qW%;TIkhI6(^yMW%KkjlWUuH7zyISjZ<9BeM-GtgP+TT(YpdF_PFAS~QhNYr&{?gfpK{w@ zvDz%wf7Ylq(CB|m5`m*heF5d%vIGX~aG>)C2IUOBl=OH5HGOW@Ef0=X*~*lJozC>v zFHk?fZ{}ajP57@ns`K=IoLvIWV=iY3-AO+=thvdp{vI+{2o8a-IJPFUdf2su^ydfY z6!kE>4)uCA+wrWIdo80c(!P6nOlgM17B{0@)U$duYvp(jHw~m3#k%HSn!Pp@7>x>? z#Gbe<9gAzW0 z1f->ZG^AU88#8TD1(x0JtnUS`r0U$fRpn3lC$5VrX7i$@aw5KnO#u3dx@D88-XTC@ zvB3S&Bu5H9&efcC?vTv`77hf}EkxJx2T)t-OaAyEWU1G?coX|M7-2_YsJCu zj!p){I(YZ7=e}5O=j&y5ZdBM2c1P#x5nfY~TwAi2eH)0KStdX)A^u!7{VT7-`fNO$ z+jP`UVAy^8se`StvOqHiO{&D9y>cfYl$@el0t=+X;ex+ZxL!2rl{bf~m*3`W6;DR| zo2OdOv!Q_EaUr7`81RiuGZ+E$SpN6X+$KB#(R$Wr!35BV0lpuI6_3nA2~H~ERQw*W z#(f1zvD!De0%k*OG7=X3Mf+EP)7%dYQ0@QHPrcPpXw-{GCRHn$L+|!CFXO9olT{Wu zcb?A&&mqJqUL%FFexV&$fSx)#_$sA4=xm2K^`+yGh8^Q#t1R$hlgj7Ks1_AuNBWg3 zK}i3le**BZU)~6m{)!Pf#7yFy5r17vD#tKiW1Wih38i7a*wON7b>e-Sd{<1M?!DlK zxL2HD-|i?q3|YMSnwccp_@T8SrXe!7sAW@NUvr1xHtRAZd87{URM1`js65n_`I5qJqgV7r8@qxXk zIQ%;O>#@fK%maV6?hWk4A6K4-@SH$`Ktv;gKl%>+yne$xBmhpC1tbA@pd?Co)%9=q zCR!Dy(gLN{>I2S_$B_;r)ubZHgPCIxPH|+X?_n%13w4c=32P4yWgZ9NRqi906moY+ zl2)spFf{HPm3%QndH|a`nU@JCMz=(3ZSURp8{g`LAz{I~oweg8RodT#)8|OtTS?ax( z9S9D`++80e$_?xsM*&0*@6eIHgiqyD>?|*K7P?&YsHerJ9Mw-ZP9{xIOU9_oYR3P$ zI9<<@`h83YwAvleda++I(Z4@gP9codt3DHR1aUgC+MHZ~4_K-?NZC7KCbLM0pr3Av ztAXpzZcFI<&W^&@e&h$J1RoVFjrfKoac${0RY2tp-C6vfWg>645}y!C6ZwIh`F`@p zU65EP9l6}A9|jic;d9?+)x8*_UO^!n>}ptHRWtB)FG#n~k>ga{c|k>ua(h5*DVDCs z06!q+%|;UQ?(^#$J5bn62X^>>Go`|`a9xKHeX{kg3!%@(N<$9TqgrWoW2r9i4wwfg5qVg()-J? zu|Fq_5sA>*Y0AE~782VYT@^N@;jRPT{8>KRoW!1Dr6p>uLxj&Se?X*!HI!X z;mBO1EFbETWkHEXOKjyg{Ut2~nH*A53WY@44hH6ui9ZPSZ;3S?y1o!?wqC**yq%X0 zwepeb((W5MM7UgSZ!i7nvf2d9&7y6D(eyQV8LTnmw7>S ziQPMle3J+FJS6xy9izB16Xr)wllW{AQOh7j2?1N*#?fx~jMn)dUv6G&C5sw2W&hu6b5UB z=v`Fcw9S8P!6@twQ{@Gw#d=@wVRw%ZyiXSQI1wF>s^=Z05;!9>Bvw2#N8DqJg>|Tn@i-0{`q9WFM_vL|K7w(?c8jW ziy!@fx!rVgP*ZxkBJYoQ?-XZ9`P!VP8*Q@I;7u}YEx=AL8hd&Z{MW$efnc|Aft*!a zN*i5R8%d!qS{~coW`XAdR;7F z13d3cC!bN^%1)Fk%R`kK0H*jQLmRWz$hU=}Z*PziWB>Da@x|kpH^jQn+zEkLT?YQB zWD!md0_5v(v+{%p#8s{G8BB)<_(ea-$q@X5oW}5p$rdb~l&4MT*-`?lQNmRl<0V*w z8U)5?idi$|LYz}ws1kFVU#=!HXbOMF76u2bI+QVFaRjwF{h_3cGSOyAs14JmrAz`?~Rl>x-F^lvQ+`4>M@@A zNXQVtdL4%7T9GEtu@my?kDfa>EI(#}3q*Fd2g@7dED>peY;6;Pk!SYSSG;P6dWHux z^}d3rM3&;BN&D3ojnMXwK=}=(sCe28TkQyCltF~SIF1G)lEo1D@8Ls&<&f}jDxOm6C9{EXa=Iu6vZZ7Zsz_^t=^(Dtk z@lh3@h0lRyZs+S&K9Or@oi@V3bx4omtSmE#APs#C6G(C~#J;@LW>3)dH0Rq8bG`Tc ztDqg-W}i8yuK^EzIcUrG?Q#E_ReYK{tlDCMQwkaClOBb{4yvSWo z=%2ccF}ToOY*n5GMJ`3s29e^gQY(Fj;F_?l`$#W%K2pt=Vl8hmL%tqu_&7-(Z7c+` zh^?XAnSFIK{ye-AU`+J-$jA5L#2ib8cPqwRZ$p63@c{OXFs5(OKGT*qfnHvWTj+H@ z#fv>TZ6TyJ!Q~gl?!=0RGOiQp#Nk;|M09!rWq}(E70K1`$%K^n?s`UR*ZDZPRBo!D zU)~zz-6J2n0Ag^nQL$#_b6V4-9dL!^r7jYyOC&+3 zKbRcDq;BsI3BEXD%j+reN`KC&bmt>o?dVICL+3LWc_8~q#8|?*UI0#c3RZbI*Rq>f zDMY&tE2bDbMzu4U9;1mx10A6@!-PFJ@Wv2C;M! zXlyU}H{y3G0l%>oJB~yEtjWv3Mv4Sdi+46SG$chq5Dhy*2cR^^Xx#KO*J&6j@;B%X z1;7D!6OMwP>~IIU;kSlU-XvcN>_una-QX)N$T7jI7PG@L7{;}0{4R%QbYiKPMtkmlX zjsSRyd#*=^NO@kvURGUJS+*t8VITOt)1{1|*e~5@q?b+(yLRux#No&V*t=w>^No70 zE$U@8mbE#2k-Kj4TadUs_5ltB#J_(cpJ!!36AcHq;P>DhOIh7)$qbaprc_vher$dJ z!(mtOrknLU)2FsIquu(rLM^hoCwSc$xhJF=$Wa7G77@x*01P=S1lBFuiyU*vI|d>B z34MxwsBZMLGj?BEs^sDdzh8ejw;=O4*!+?dFO`6y*9|8y9@c2WX%^#qOlkD{c%cBJ z8aS1s-zRX=r4^5XI@vRkQ`(~S$9!Y?`1{tYm(?2wh-5Og1Z$zY(ajL>JZTsVy^grQ z@kH~Ly^Hq4wsNMh$QYKj0AH4i5B?e(NB9@QZp3Ta%tr<62;CC61d1z56ug2VuM5Xc zBzWQ+b-L%j7t4yf^M~A*E8qA z`!{>@QYi;n(r?mTPws_s8l%_NguYg7AU#k?arK9)aL6Y(cr72A(qBX5dQ zypV9EF160$f9d@We!IL8rgz}=-YJ2~<0)V{-R}zC@c*Rf6G&j>i_y}6!qv#RmhJwS z2zhfjcAq^(h{mDy)|N*WmEz$ye`)xXg)}A_9VAa6JC9N@ztMNn=791=SE3j*Tf9ed zTI?$6NuJ-<*yZP(MJ!X&(Y@U_^tT#0MzAlh@J281@;#PE(;*iVn-mFlaU{G62N=DBax@Vy)IpM^yNm~IKU;)C>i%;peverxFd+A9=qX>Q0vY41O( zJL=RTlqPXlZq3`^7dXjw<@@1QKIu{|lR`6@Dl%gnc`cIU!oPP(AdRGMn z6lb~~tTp=iZowt$n#&kB)JviVvsReRRLN64f?z3&xksS6hoV#2na{pO! zqW3{R5c@7Xejo1zmT!-8CnTK#{dyS|oKtdoBjNrvw%SL0$;08AOElA~Z>98L&fEjc zow|@_6ZbSB5-~}@oyls#`yv{=toD$Q(R!(Hbk&8;dbfwf@pjw|3VDmknJhO_HV^|# zD;Gwa2Drh)O8y*(Y~6k+Tj+fJV5}YKb(*iZ(fvsZ(K@BN`46XERTU2MoqMi(Vw>Ful!{V0>7r?qUMBb^zJlFu;MhXZJdWGDqPIBd69z;Kxu++^t1~KB{ zyy@b{I;PW{3#)(SuT^CHRIV3j9Q*JpH4XV zve!7$*=ru>J<_ACHPvh??k8A~3Z3H0*wDP}9zBpCe$5fZsVI44E%a(|`IhCNYJ>(2 zsSS(pQk2s4gq8@uIw)`wn{rLn(5k_9$bpkJlk($;9)5ozFZ`H*{|$A0NE0qQI=fys zE2bLdUQpv9>2581$>|bJh(?0e5BwdjFqq6zkn*GJXdq>|&PI6l+p<4jUP+3)NclQ57(pg7E@x5VOknZl1kZz=53F!tY>F(~Oy9K0?4gmp?lI{*! zN?N)R>0a31@csR_=j=H%J9lU9-1mJx&vS`5jVYoA1~?9Uc!#FLtQv$~8}_S@8g|lS z{~gvPk%ag&7LIxnovIw*A10lTxhJp``RL{o|2CtP);YJ4bGgjaejthLBT!*>uE?~Y z912%rnm|3L-W{5Co-fg3Kog5=3>(kPykbzp9pWs%@Sk;AsK(D*bD~is`ix#R98FRe zrYX^MJhG8`fPiNM1s>q@B1B-cQ`=}t?Q-{mv4 z>Ij+bm*5HX<=-EfO!&qNw4SAVISIhHRKy$5KkxC$(=jvDo1Y_1nxScl@5%N&8FIa1 zrxpKhuNv&09&rby-&64wZXpH4TvExq@YhFc#6wF3B*8Q&HP z@CRrfPSGC7!*DYb1is=BAYETI%8E)t|7X=AnWM!WyWA@mf);Ip&PAqv7FFPP9FdH? zU2@ekU>e1$9xE6mKSEqy4;MtLfK(X3h&{yk#!BRUE3odl7-f{kpnNo)GCLG)wDJ1P zjq=vSJqZahy6$!u}Pwt+decr_L{k@NTXZjfwfrtugjh|2^*lTi-%xkN<;_gw?!@W?`i^8rm7aUI~ z8Q#hMEq>-rKZ=XMs(yn8KJC;XWcz@U5dl$vFJ*L7yt?7* ziYvFU0t^2R=-8&Xa{n;=_V?#riad+z)v-xtp)eY|`tFHsKg1EVFCAK7?i=obB%$PR z--t^^E9r#%N&?_~QYnc5iIHsVx0~*Yz;I*s=E87)wQc`wf}e&VJo_s5a5T321F@D( zTD}_~ye)1K=j%n<-6`+P-Zi&yT-# zBvP;KH9POU#n&5b^*J>@TI=+8mfSuPd+ZPHd!_SD7B%D=3-cWFFXF}|7Ct%} z*P9e>)?WQIqtSVVf_N6k67<?*~kC>wW|kw}qAUGoM?8X8&= z`(2=h4a+o|P}xTP&?2&+e<(SEVOWBNT!C?=brcEg4`~0fbLSLRbZf>^q;;}=#tH|F z?PR1u^Q2FRTDH)MAk*^uf3Hfe&ch3K4JsCXU{JsB@jm@!PN?OoFoBC?@fUtn$+w6% zh%1gFwqa4`P~&`H5CxR#`y8yzuiVd(Z7IT7Dbe|SP?B^2;FG_Y^$feDc~{9qbU|*a z8i*Q{iCDbXctOU&1~en>8dfdaTzA{lWl4lVQP_P6|1F+PYY7R+n{S1B?m}=DU2o#J zjLWa!B+NspeUFl~mgM=DCKdRN<0 za}S5wU0JGH7V;rg`c%h#8gc+?gB^^q)=rfC7|6n?AB!8S@`307MYPQAJBePk-odx_ zSbg9of4zsfQsH*_09_t9E<0bdk#Rv7n_>>ExQl#iU`E;nDEwN#oy$}Mi#lnJ2tx{dx@+sYiGW_t?S;yYSvN2k_t5FbpV#Dw;0fdDzt`o1Zv*0?pp zexXufFrLukIFZH3h2A)^T#+2mbmf`+<4f0F-;C2LNBfbHJAF($JPCV>Y+909Y` z3u&s7FPd#VnakXn_}!>|27sMZfzxaySy9FMQE9z-t=AMkG#rjCF2C~AN$HMOEt@A& zF@vpA7P4vi@gvzE?-i#DT;qTw9Dj2B-S!An&V+lW5VEd-kW8jh0KqL~%=ZoBcUe<@ zVVODs5a&a?w^JK{{Jjvnv6=}Hzii@(*ikx9L{2++&Bjf6pO+nKr!fUqO{ukR;+TOo zaX?BAzY_}yn@Lyh3-0oaC#BaQ#~g1`js9IauWnBwVaB8r{H_xKe6J=&PDeg_GVPi5 zM_Gvwlax5VRjH50qQuK}>mc4U#GgJ{tdriWp8sy7WHp|lGC+Ix+2&Mq%>yDgMA!ACf>4WEkj4XS4BnHVJh?|`e>65D({4fVb2b2A(c5qW zGN{2meBhp1chD2cWp$G1MGS;nPF#2b)#8*Y>QeL9l3&NHs))+osV5cZIRjv+-nyJ> z#^bY{3ZV*yx-B=_;s0?TXXhc-132Qg+ckf?!8pRMveYqLGg}OgYW4is9r61eeEZq2 ziH<1>r2+elwOPadn~}t;ejUJtABq|Wy-K9}_%6V0Ai;MO4IH|k=aavJKETVrSxhYG zz9TOTu$>*2?=~+&JKbQJw=FbR1fEeo?A16u+O3=W_>>@AN>Pj2WkYzS;3(eLkVyGo=9XD_5%Q3fJx_|tu{=Y zbF=oFEX8~H1aLWy_Js=D#=xjRGSrdrGzZ0avUL0ID`aO0WFHfpr@a#W; z3(qi{wB!T#Uiuy^eVZQFw@%%yxw-osnd+vA1&pxaMu*>JDRIoN^P7lx!BOi~qCrKK zSFgmhU$Ou!>GE%H%w_}gguQlt>{>1_R9^bJ(LHmI$K2eRBL%v@bp*!bt#Wd!#1*{% z^$~*y<1)Hv%KmvrGq#D|>0%unL;Q6AewruM>3dRT8S0bsGTMa_L+go>MZzMF{YkHV z_$FdolF(W8AD+(uL{+X+^HC;SWQlPB0K@vmhPnbDt*M#d)(%9C;klT{^yjEd(%{qv zcU)aL_u!||b9rq(Q7l9n(>zaA$zQih{5^SG=zeBck>1*6HRI|#JUm2ox$3&#BliKV z0e*xXQ@gxjB^ma#)(L>XO^6nkL4|AVtD`0G2<3{>!;Nc|2oMz@q{J}3{rBh3nrk(8 zPqZ~)cu;Nwuy2dJbkVXTKhXec>?)1(K+!)r$&4ha8S zAv_We#b!>x`V`Cuv~M#vy;yE?!fJTP!x?KS8ev2O}xLX z08Nn|OO~+~0G26p-=$OFROr^`V_Z9}wxqn6Egm+)u~BRyw&(+Lld@Q_E&~L|*yh(y z{I7rO?po%*=K+&Q`U34WtASVL;}Xtip?dGYm74y2tmQX%ef>((a;R`FoH$iz=jXE@ z4iKqCebtFtHfBLDBtG79s3q5WToS={7EeHrATGt;Ial>kg$5nzyz=kOi}xGa9XlGB zCLOL)mnjVJwxhwor0*__u?NrD{+EXoZP1Q(^vIQ_CsANcp5!6n#^-U(|5WS&K;}@t zv~U0M+du#Dz^4H^BR>^nw0QQc;3aL%d2iLH{#$52m#+Lv&l$iR#64Uc(lHJM+-*ij zsLYS?S=GMRxY^g+8$_jpXaP3~yr$<426_9pxLoY- zwvw51%BpdD3~44m3@D0WIGM2`MD4o5fEA6EN?Kvp+UbqizXqjDjteKj?s#6{1*bm} zHpr)Ivz6<5$-N=q~8juXV2gX!?k zWyC1b5mf7d@n*M0Aomc};-LBQ#bIOE>fY$FuWBc6$}6Vh`H4zw`Ly(cu;5Qpw3%r; zb*)q++I^f>tQIj`-nNp_8l(0oH?g4iU%hPfh?JJA86_)8<>E=iEeB$N_=j^^>&V8U z5Txk{>xBUhnt;69{%9S}jx(tT0GU=xj{eUEk;B*#6Eb|hJ#5u4Iu--r*Wb?@mUiJJ zLJx8@zRD4OF+BMBX3sE$_6>Y>0G{#%1TNwjW163GJAA|_IldkzE6wG=lB`V@`@a7M zo@78?r1Wxv7xMns!O41e!O=pknoK+q!{0X%{i18^&8xKKkBfey@4zX00Y?M1QJh}B zc{~h&n}g#CV1(0Jd;I^k%894N1CL;mGY6cw5{L_94Kao>r_b;lnK zpjF8&4i`(CcGdg93sA6qIILuxT;ASP*bw0)VoZ)wQ!+N+BZAOUxsY2#Ytgfw*QD?4 z=SB|ZYYc{77|Y&%f6`Uni4FUL-W4Psy`^D7EZ99Hm?w^yljB~R>U>zXZ$Z@EMIe0t zj^4R+&;Xw2Ewab}jhJ&~&35sM`eLet3wGdWGb4 zFjZ79NzRz6R}w8Hac}XC4vTNnPL(SMaj5tAcqV7L_sR0;6eCf456Z)#sXD9nxA$6K zm1A?J`{7Hg6R5@_Pgq>K?!Oku%(W1NDg$)=I6|~|DH=S0T7JGS*|_MtGiw#ajvo58 z|M_gGIGfjI3`%?vk8cEZU!;rSu`rMao;YY6|FExV$9W=XWww7f9xxHHu4n=brxO}* zQ%rJW1y-W$Aj44}O<;o+zx+Tk?OspUHD(s0z^GgT8PKrN+X3(KTz$Kzt<5-gBmiGe zjIQ1h!Uh%<{_ z--Me)u}w$^3drKPxMDM4*qo-nB)Fuo;8u_WStpvoa}GM$IX`QhvM2}F`!tkU4^#Wj zNGLF}7qWWStO&S5R^O3@!|CUQ7)LB zY;k1p%tW68?~J2_*uw|MNpB`3ByP^gK<|rzAdoR^UCbKL=yy=zKl4psBxBnwRvl|q zCUS<=qPV$R4s~p^Ix)BtegxkbAF6n%Lkk}E2>EnZs|z>mTh(8A#}Kj4(jh0oHQdS#aD3EFOJkX)EXc~CWY;x z-)&P3@dNTh~2eFwDa_fV5;%4`PMhF*RzOXNIt zC4OuLNN{_@bPfoE0ll%I0xvO1^O*m!3=bpy>7Ji>u~$T{Cw_&OH^v(EYczG{wfeJ# z_P0$Utc0tdat8Pj*9~|`TV9}}lQF3pMc6-%s5VrOqn8B}n<`4yKys_=>k zc@RQdc=4LP@wpI|Y534zLSOSkZu1)u9g|!$J;w1-WWaChaSB1=aJp~=t*RQT`d%pG zj#A7!&HH3c78&cLJZ_gotF@j`%!D?^=^2|^Tt51)*7yALhicn8{7{bo^H}Qo!b8UI z(ESOJTu-qlFlKa9I&W605|^tL-Um-!A#n#IzGEc+9G>Qz5yZ1yb`=|EvYemCXFu`f zs(-@8k4fZD93mmdhpdnMPHIm;+(e$5kP6gd5ay;OUV`22o9^857e)_qetR#wOZ-mf z*o5z>;1y*gU{V$jPr&@EPJn|~HQ?$(3FU&>M6n67fsip6{-uj&pRGK~k%^tnv8VmC zL2h#R9lOjYlFpJ=tNf+8ro4$wl=GD!JDp$(%dG;hi`*5EF?S+QW_ zu4d(jqTo`6vxj2l^s6(09vIng)nYPjHTT`2i(vd)6wu!0^tc8y0Y)wxu zE98cQvJY;dbzizDU~ue4bs;lB>k7MLA4=Pr$Td&Dhac_`QOX3}Kqt+*oiEYxgnU1`hYmr3Pef~rt{JtO|@Cy-J9y=0+pV;+|v*#pf=BRp2 z$}z^ryA@78>lvoDvuX#sldtJlblAtvi}g_~>9tbfj~L}$qPbNUoPb9k#uh|NU#SPH z$&bGPa&(eH`p;r}r7(x7DOiQF>fqf65IT%f-SgL@|JH(?X@3f<|)YsQBI!3RHC$RDOcsN#ZMyahRs-Pusqz+ z`2#2x@;lpYr|X5x4P|77T;x7c?U^lqvE9z_nyb2ebu?WGs#$OOEdID>^s&Q{>S=p& zzXAuM6(;_xXnkbp-&u%J(=P5U(r-QSe#_1OzFj1>IT`Uo1=@O0Gj|@O zBZjwd2dl`$3wonweqaj3l7&qM@1ehks(ySNQv3Kk>1M@|IKxO76b{DTUxWCP!Jcfc zE}_&1Y4d+t%oEjSKsvBMqze@J@+F0 zy*xklIJIL+XVMAVM=qaLHD&;x$lR*0|C2VPbWg*acc~&8yUazLUF8aF%d^`_LcAPF z>2GwrmLjzmNJQ7dEj>g!-x+rK8lXtdUt6?N3r=p!U6$J+xmanDWCh=*65EccZKJq> z4-NS2As=cr0Mm5 zP5wjm1nh==qL#yV!V31-;q`>23e42pVwa<8$<1m!qc!acMPc}Sbt-ui$XP0d*y3}L zQlNUGO6MrD*x{8oqJp{xe%0(Y{!zHkfPu~WC8%YMYG;gmkBPIm?n1XT20!Xh=9ykL z=xpTlYALnCem7@sH*Gdj)iV8Zws{4I#poN`trTPH1?xGFU(%hk95); zWLM*sp1EK zl8UEBVa3}SPygLh)5O)@=t%JBR@q_;HLjT)CGNS-J#h^DUAgs>02vpJ$aI@l|32dk z+u*z{DV^=Wa$KVkyA3_nn~6d!scc4mFce$$5R=80h#ox%81%h2lW#A4$XqB}Tt}|2 z7KUrT3Za+JN;*>Q3w4G^W1+w}$-rXTd--@{E@|N@m?YT`UO5s|74j+OR|6*N#nje` z#B_1@jRpHK`GQ3M%46U`dB=Q-8iF= z+CIbnO^jR8qFiyRN6Aw?#M&!~SH27aK@4%3g*qy#MU@SVIAHL#?_@1ldrfZmyJ0rk z+)oAX?ho)PV&j+*m5T*PV#Uoex29l9-W6aWCLg2V7_t~A3Fyt5I(8ZwIVJaqYvG zjQILDYc-yu7D!jyMdVrM*^9P1)XTgRAIwE^_-xrbtGQEU$EIw5K>wPDdn6}`zElCpC)gG*l#A{()N)df|3nnC?9ZY8GquAieSc^z0EdFA^ z=PY+DP+Q#8@p*QJ4y2?rfxo<$xKi&U5yX&C^1Jhba{%R#1cVyxpB`MewS_JES>-u= zml%?L&zYnF1CqH4l9jAvBl#x=b5U|dw<*{AA&oI7ov6oA4V!sCzIV?s(g*4NzJnTI zYlI7{RND3Pe`vOrUV~(+vd*&=4|3526ptrnJ}08NVWYc;1-kT4wuAxy;9Ie)iC=Tw z3tZnc>x6GRe$IUJQaUVeU7xnN?HUs6$K$h@^to7#CjMfILfD^EH4ht#x?!{*_3$WD zO#51wDyKlgCSN(ust|wf%CeQ+z8JyWd+8!8{>DzdUmakgaq@ug>hP|!7f$!hmK-Xr zuiHAtJl-WfJYceDeWT#)X=O^CA)+O+QF7Tef!-q@RzMw|ZpWfW@(GifjDG_;h(8_r z2Kq=5Lx}jFEwB*MjXz3M8}8&k)oUWj_Opty$QN(>;6k=l3V37pIJ0FO_}L)zPFzvC z1;GMMTbII}W2Wk-mz6xVNKcl34LW*uNi)Ct)_tCX_;C9+#Hg$YF z(CTv7bQ*evzl%A_oAH(s_mY6N=6iBQMAnU58Q0>0 z_tLIpruEfUy`3dHI6cl2gbvYMzAk$m>BbwbTng||#dN0r{v<2qaoM|E5m+7eX=x57 z&N8%_hc3+gz)ZqHr^=!4tk&N#AGw=WMo9U$aN*~xKb_{uGM{(_?pga+=(svi_a{-8 zJ0c`%FCZjea4lqG~T9PShMIV zA#E_C7l-2ip)v|(Aj8PRa@t8Tj&}jA76k4nb=v08JFKAzw;%qlGjuyuOq_gj_Ig#JF5zx zr4n`q(DlXt@ZA?Jv`2qQ_WYz*0WAz8DZgfHy^7Wd%4nZ%2Wk&;2Yx+KiG7w6i7WHT z2#ERi`!DA+*27b%k$23UKz(CjuT*%W_g-D5@tKQEp%|3Q&B<9(;$92XNLkb49{vG5 zF3}?=X9=BsNtqYIFfIqHHF>WL|H>=HQ}4*vBP7@5{{{YVqtLfKn?@4jzl5JHT4BlF zFu&1sovU#4bI!e0-ipzfbJ z&e0gS{M9~Anq~)TUW#;*-|@ih9MNuH~bIjIFD4hA@SBUHz*&foIdr&d#k?GBffuR-&NSNi}&an zjnV!vA4GeJJh44lZo2y00MRU`VmJRZh~3bgA?kS)s4#JPq2Bv=2y|l0sI5Ph4^#Ok z>uxdz_;{%U0SZ-|uZFkWj&AIfBy$*q;wKqp80F8q(dD7tTdgW? z3=+KkU2$mr*OWArfX9>O7i%~mm9jkno#ZP(GSMniO#fV8ox()$Lkfbu2q$fds{CIF z4UPDaIjxamq6zL9F5K0Je_fBxA3MQK6Wf?nqKm_eTAbgA*gsB1pyD3@p)N%t|60O< z2oz*k*IYb~+Lk7`Wg`O0rN0f@d18}xi1jC*ieb3D`OZKU4LK(OafA-t~ z;$?YbBOeTJDvJOP8HgKZL5q{%K=Iu_cXio6=h7{kG(qUN49?<}sa4yDK?n6L=rsGw zH4A|uc@?z*=wsWH+h(eY`{(YjCdzetNIbrRpl9JCFo7VbZ-M!OttKUK%t{WpkI!tl z61C1i?t&+fQZb#XCeG}ko)xc2Y=rO^x<0E0VrP}?#Xpk;f<&l+mf9||Sec42LfHO! z2Ae+Zg{V3-6R=QXVOXYVLj3ICZ=3O2L$*JQ<6rIl<7$i&vjRG;Mt`MMFp=R+ zljy+#!Z2bMdom7I;9ypfaSvq#@ZMA|6ItB#<;wf#TVI}l@7JLU5v!G(N(dZwRCHky zXQFEZvG@x6=d|!%?`bjfVdR3YG%xBdu7VPs3I}i?24YYX-y533OW!vkX%dpa4=1(B zl?2TEGf%;&1MPZN+L$KQ$qSw(pF$?LWhgHy7-H^06dWN`1$i+4^FiWkD5S!TrVdZD zKT{7nv$}^1mp434=ZY%3mj@#kaL&zzRX)P-5@Iz*zLRw^g;XmHV1>SfGt`-TgYqZ0 z2R{){E0bOZ3!W|igU#M|+ZI#9>?$|ogQ@JWE<-}{hL7iK$gfXUbZDt#R4b0-)C0niCyyVUxsyRyZ!ikyFBE#@_XlpR2vZ(vi{UPD40v-N)NbB zVdQr(4tPK{kSY?;rwQSlTDb&q!xp}2v(DUivxO+aaRRy2Jv)}1#gp3%BGFImFBf>T zWV1_8k_WpaUVk5>f@wnQs&JXb030Qgf$fNt7GCg-Y(Ll>M!g67@k+59MojCAN~Mfx z62{9#*Z|yXFVgd4BOt>%Fu&rsTWNC0gl+}_YO6l7s)oral$xgJ7Ul^vxj>f&lD9`q zl-bSdX+r#f-eaXvJHHqZ4K`PAHEzkAQ@eWykO2q4^zoI)YD^srZW}PFP=E6P=oxlE zP6eOmfx5}8QpQLIhmj(Xsv-ci`#*6J#2FX0?MMUq@5MPEvr@zw8o|t1I;+f^!}*BZ zr4GB>yV!%AIl`m77(u+$cJLR=tBnRmA+<>ZK z@^MZN{K>C_PVUo)VLEVBHKG^sp2euGQ7OVqjcr4nm<+!6GB{ag0`|wXA9H5b08s3V zJf~%fWWE+{p#kmi1Xk}-YwMu4ikInq`X9fwzC;fQeT94wpq{;w9u{qHrN1v}fRgj)5Rlr}avxr_v z4&jsqfQBtpeXg{migdm)KoN-N*~l?`1xL~<9{4Z?glpiQdHxfJK`-&4b94a+&TWTp zS@i2mp??A{e{ISY5^%u3=>3l9ZeqFQBivC3#2khsP{o360$@sV8_TkU5D@`jk$5hb zntA5R3XjDg#i!3XLLOhQXB?<-dx~kUZhpYm0rHw(g3M^htG&B#A1bA?9{X`@mg=o` zsZLFysv;+Rhqzn%Mgeqy+;aY_p!65zp3D#RSF+fO2+_#WejloIRCLBSm6owZ&m?9t zKskv1z&n7U6<`*Hb4K4feN5JNMD5L`Rt*C53@rAKjq+H?Er2Y0_#!(l*E1^h0uLiP zBYp+N3j*ixy;U}W3QV6h+%H}^ln4Oy1b8t?qiMhYjAzcIfPxrL?Whn%5hFnPCxo?* zdM+(=q8~LP{3BeAgOzM*eG3RXueBcjp0nzFrO>vYhsW7ns@$|V!1}kQKwchribgy2 zp$Ky%t)}`WPT*%4(zaz~*wZeaUZDShbo1dC=-mui@(aV^pKrVNnh7>@#S{6a5T!rZ2EL6(O>bKEA3YJl&f7T zR&U@eH_tZdJU^?k9E_xt;LW<}Q4gCdXq&EM&|&6>6QT*;nt$CfP9VZ(3fwvGCkufB zVe$zZ=U3ZdC1_4Nkwi`(J$RHX(~^=EF*}N%8n>hr;6X5njb`7Xp=FVc1@ye*x z1Zo6u=O`kmFwGhd)HUkyz-2a)Nn$}0`JzHheCddsu#Q_K?S)tJ?~)6$<`65R{kAS* z-}7@^X%75EBxSogD_tJOO+*w24(0nx^sI1ZEFr4Zxk%5Q2)lD#3|m@ zmgY^Fqt~)H?|Od1r;CTk01g~SwA(l+_23P_*kqS^?LR6OUSTk~NWQ3ZA~CtnE~s?`DA7LI3$qKLf?&;-(}DyTuJ%CDJ$~`GHd1Q>RZo8^2~| zzedWn!Ndx#nn(tcDbK>C_X$dZ!m_VuhIbS?uPi5C^v4!lzbDzIthhzleh|KZ1Sz_N#32M3 zcTjpqyofpyNL)^0JW>ur4>D8t$VfOPq^`vTqF$o9v;ik~fU%|1RH|Li;*7>IvIamvFKGv!GpnS0{5Y!{wt%_aw z6Ga#e`*Y%`)=ExVT#wmCbg-*w=`o~J`dk-`TyWQ;Mv`7mZDwhhE@L&VXVyIFcVwQg zmy4P5hd669;f_Wu0bvvr6M0AeC&VB6+Qk(RnW6!*88Rlx?}#yK;2#hYyGsP05Nogh z>2Ko}PfMl^Wsht;mH`Ql@jptkBAei{Bu*&XXevwbR4gvl?&JUP4?Pk5YNQE%W_J9A zFx$49*_yl;#GjoA|A5r7FSk1WTfQ3zQNEJ9Ps8UUi6i8|3*OD7h=kFx`=&>-9m9XR zUT(ADeCmR|(q=Zv_?E#bhloy)pl@_mvg^Wt;HAsQ$5D+i(TH>vT6sN%e%6eH1>uS$ zftRVpX|%;q;@cWFKGOoD$GY5+8pEx3h`bBH@>ca%$r@-g%x$tuwsHN1;i6|Ipc|3i zMr$?^Xnd&ayrwA72^GdXolgHwy$G?>$ni{mRx(K}ndY^bjqGfje)sx8vFwsJFirED zYs@ZeVFY4(J@0h2*4eCN+*d{nh#NtG-Al+YdPup^%Itoq#`BW3cJ_Ya_PFPkMjw2C ze14VQf9n7_7?+9bV1pN6MK_qenNqSCn(okc@b&1XqkwkF4x)KpX4|OmTdSDU#g7!4 zU25LT?&)?Gy(}pk)Vyies-aN%q1!vf6f0I`;@fk$DPvQ&uTo(Y$cNS*} zJ=<|a06lx2MQ*20n@s0LIw{`O^BZS3bFGmAH#0^EhBOoGGIOhm8|8gS1Bj4NRZtq# zMMbHkX~laAGOjz*$k45!q7xK`z5kBjVd7r)UL7m=3|*DFF6tNu`2wFJGm{vv$3|a5 zT*yj+7~Xa^__UpR-~jVvM&k|(CSO{R%I37v5KUBVtQ5HNvxvycL2C(*!Ip!(_cYi@ zIVs?ZyVG|YP>ys|w8Bu%c|@T7yDm4Ip<)SbyoRTna5R}YBceXK!5*AyVBY|lltjuD z5niLu=$$pkh2GomrIc@hH1|A>g~zPhk(Q*x5?lbrXJE~>EaNG&`#1KEui*aAPqT0d z372RYr_vnbaOcJ6R%@_WJzKL zLGqmauw`~X44SP5j+rsjq zBEe$?{trc4E@vJdr1L6`{mc`$T0OVV)t)rLOp1J#+&%Ha+AOn= zw`;P5<7sZQr_05p+O_JTYq?z6Z-1G{C)(@WwsIiM2*mvxXtO;>i*H>*kTu+c%(=0? z=EvlALFtE0x`p~wSY)rA4b7zxvY<^fHYM=x!IJgaMPx* zcRlLeZ3PA#2v@Vi)v)5q8=hp7-Ggz$NTN3J%lpoc&7~p`DmLyhf5uu{_8@MLw<<=B zY5ac|WvrCaOLe@|iXHrz8fly7H!`Z%-&4CnicB6;53f=Uz6er)MLQmTH8%!APCa3f z`jhGk_+n?)ap0>&h?2fi*x_Dh!GB^@(u2!OcF<#_t z6PfMe)PBm7Sh+>Ib6!KCi(H$JM58G1T7&x7FUw=Bf9mu%@@u+>d?Gdm8w8Q_w_iCe zU#z9VpnT39Zd+6y?L7pOyRE+pw~@sLHXo|y4toWP9)Z!@EDlWl zY_{d{B`$WAu^~!1ZJ&&bYL1lzT#X7Y=&twM*4kBA6{Kg(z&o(u?YBIT+nLwIxpI+c zgcBc#$Vu!*rKoS1w2B2WPuJ49E?!a2c53rxBAsnO_STg+Z*lt|<~FnAb*O#NR>{KB6_T_ySiDvZ=3jM}orvC3>Sh(PH*HpH4B>Iy%zMSQmB~6y1*sRUJw%15Mk85y=QjHO%~0fJSp7J4z}W=#e-bm5c{TQm1Cr-k z?)G}`etN>A@>tr;MC?$4+Hz`r9evA>{rj;}#tb_wF!hE~lxdTk2eKgOC6=s~`n+e< zZqSrRtkao1)Rc5(V!#>VGFVpl1>*oQHgQ+RlWt2Gv`QSvHf08;v;+vMat)Qb&{ zmdxEE;yu0Ek4TjIei2j#6>&Q;MB6*Up~4FVP6a``Uq($i`FDs~PfuZHvg&2& zB=W(lxqgkNzl0yg{iG|2Kl;QJqL0!>Ad8d86nG?U)X(~Br1V7|Vo(X+`!1{T73|wD zyXSs=r*?Jyw^cqR>MQ_0Vj(S6EZ4kU;ImMgV6ynZ2F*@T8mRlu?=WSyLr}{sn@5?A zUaZJ$naipdh1&f~?qjzu3V zQgfTZm||yLHOCVkbJtp}-_>^8KKZpGU8hKnb{i~Y@4pRFeZAC_;&*X6M)6376Y>?-E>mtenr{GehN`p^!TZIap#3;vRshS&XWjO-*g zsjRy03y@a1S}y!VwX2bSG~++_kd%Vix~uKEMsF3C2{g8S5f`}w$eVYC-RM@BSp!8Y1Q1B zT69^88Bwta5p59#g>@(oTdHMS%1hx?ppqA^`EtDp@!RK|C3V#hj}1$aZ=Lw z@`)HO)lLHz-v}akK4_`bwdFj)_BZ*kkajEmHuRQ5RRp_n5SjbwmyFD?+HfQO#V;Zo zl~F&N@}sedxtr=r)>nfg5yo7O7Oi=}ND-6ayo(;@31DG& zxjf>0etgGfnax0)F_^@TzCG+|G5H*mbEEHl?SDGT`!jN36~{m2)Hc^a+>ljP>1o*e zG}LN;wr`fz1od>QRTcXv;3|^1|3Rg5(}MzT#t;3t_C%7fl;@&muvP~~5l<`kS4oJC z8XpZEKK(%dds(Ua*!4`r9yP*KJ#nj=b~n2_twA0BhzDClS)yeTn_36#fOF&&dm6k<*+I_G(0YflFyXV(VK z*aClUlPBsfJLnt7Gl%beaAad`W>HF1Gt^+KUOOBYe+RFGDCv63bvh{i4HFz9W}$7* zAs0X>yf)xwka?L{L21v9@z!H@oT-~aB2oWht#zv*haM>5(LsduAFI5Di1 zU;72)K#YCR8?bZ9u2G7pL0PNGw=q^@BFClA=sl0_XU{zikZ2jkg^R98r`Ofitu5$u z&x2XABFr2;g?~t4jxJwZv+s2$qnic;tMHJ~Be*eLhz`!uFopPE6;UtVqJ*{p+q!ro|NkT<#JDw1SCx}df zL%Kc4R)r0oOT3%T7+_yy5_G2hJM~hS^v~KRrAJZa^CbOr zZ>l@m0DC@?KcTMgFoYY)NNyW#x$Vg?pB040;&6)`!E2L*X-VG@R7EbN_=v`bYeikm z?uI!34e*AnM^Upq&}Jl_vY&W7q^{@Ixp}w<-k6d!R8)ZLs5(tAW3Y$(Xd|+<-j4Gj z?;m1+5oN9h+o;@KFne$#Dn1?Z$vi@IvZrLxA51y|?LIzyT<@yTw(svs>oq@aOO6Nj z8UJS8vHcG^b~y=m>J#CRN#9YDpLedpJOu2#%S|IYnPT%Zs}9deXq$3&+3;LxjWY+7 z`SuOZuSItg*zjh#9`1-Pv7oesA2$o%d;(Pci8#+rTK!x8c+Iwrn3dc&1`aB-R}j1_ zE`NeTo!^qb&{o7nUDv+`gamFI*clnZ#te&o=j0WzpB;QC;!aneUC@3&sfIR54 zuMAL;i-Ydxg>1$ItPK|!iGTVXoW0^b#roqhJ zh(5@*ohoY%bvs5TV~-u#(IwM+j(&fJ)_US8i5m<>gQ>!(I1A@mh5Q<9}$p>o@2R7 zyVTwYADPhgU98}AH`eeBQ{8FKNzkMwcM#4TbbLgaB$pGmj<84ky{oTg&&T^#4Fjk$ zuYM5KG8+0Vh?!5YU6!>Dk2;_%toxg^>~4Z0ZA^l+bzKMQcRa52&QX(a%( zo##cCnVVC+88|18S~~yF^k)F$R{@i|1r~kVCg<6=qQL>0I+zzi^t#y3M|I_WB&DZT_I8~?Gi6yj)Z zmg8lR^F#0E)BQ2d1{LPIQ3s!4;ImLmav~b2!&ak(kh9wTcP6{8HKEpRPKZw!)`x!O ztNj_vHEuTPv#l6)jOLJMXwJ9C=Zg+I=Eq}KIBVg{+2N?mO#-V&6T)sXzv+rQYyU;5 zF_C-I`mW?#?7doS0Si-$jwGa00Ygv87R>b{O~@l$-y>h0(Y%FW!eSZgb|9Q@t)150 zdK>%7t;hE}$Wc$XtR2}J^NIsRjJH!I*df^9PV;))WGyw<$Ia-jO`OvLRYb=`QoN9- z`;GksF*A~c*qG%Rx5)QQ8>&?IH|cR54-a6{i6EEr&aLATx!tbkDG$zwM|^66$vRi| zssgX#uX78dei`c#&upxyB3SB+YN;R;)?fP@f&Jy@O_9pi59DuGd7H>UHBHMz=ma&< zzgdnf4}Y`jM%XxaXX}zTn7qj$FO~)rVM{swla3-AD;`pKo`eg}R-ZFq500n6da`8f zcJij~N*9m2v7}(4ubcXtK9gcf4~*Z2#I1SytAne~zcd_d{WyIyyA&^VxcS8y<9IEg z)50;7xGe3~sMnv4P2g+CHaU2CDTDm5V99DL{?DT-M!6x_?@ds%&Rt3Yl&RY22~0Fc z^5c^0>%uEzmTOUm(WKqa$#-C5RrQRwOXbjjclYc&6zg5}t+mwMpSj=8J+?J9z(HWY z7HS1g|7J8njGZ9Qr(M{7-O7zj?dQNkiK3upB;0(-gMe;zW_@5_Fw3^%TM1@kD>BsmqifiItMGQDzBxCp9A8mcRM5} z-(68xoJ_})EITfplYXvs{alp~p3!$1=h@ZE*bx>A)cNWm9G1BA1@qi59nJL}?K=&S zv~8&|+3$$gt@Jo6PE3#u5vtlYy@sP>TU^-e&Asw`KprG0ds!0p?GE?x+vw{88hIMK z)$=f%;6OuV`yK{zCg9yCV!6ZprFj}f*7CShn%*LJ z`$joV^mBtSW?gR0MrBEjV|nmLY48~q%3^aiCjnzi9RlaqzXf`kcivP?d@bB+fyUv+ zJd3*?AQYunrCvWnp&9H?zJW4LGQ1o6!6cD7t zv~INtUT}HdY{d3*h8wBF?y4x6+ETn98y5Mu@Zvf-pB*{xH2nw1pU75A9{7kK43-aQ zY4{9gmc&@_v3^Pg<}Na_S8+(u3EEqpcY2qc_F1thS%t-Zbddg{tmxYW{@OJYX?m|0 z)E&zf_Qn+WCHU0_ns&BYI3m)iwoLWORm&1hD%PhnA#`!|XWg?Udburl(>;S3wa{mO zs!uz}3+M24>ZJ>dI!pgJyvT6XtFq$X3fevJ(H>~xd=SG61A~{Mt~q*I@(8#PK>Lnm zvcq`b@@;~jT#kgoVjGa`wxIUi?zW6bRgBv~N;9}I@bO<7#Q2%Mu1(1&)S~A!pdgd^ zJSYEMep};b1C&AN95TxTUxbK;i<>?PdfZE3lRQob!lIdI*bI0)jAdC+b6wsrX$$<6 zrovlt>9)`zwj4dLFv*PsFz_`$|DP;i5krs`_>p!YuFoLH&-mj2S!9@*B$tnX5GUZ$ zYWxKJ1n)A39{7M7$+1yBA52T4Bj6x=&L=0$>Lnu@JxWnT3S;N;f)s9@w<~_6qfDaK zZVNz8OufB*p)ALZ?1Fc<{1ZpUpjO;DPW|v6j!A-?9C8CYiN4PBdRot8_S5BoP`22o z#rWyuG8rkKJn#`aGkP)nXtWtw3a*X@#k?ti|LHNy;IqpMSaQ+W&iKidN3fiJUb8a6)Ar(_oo;-@^5=aYijq06VR@IsQO6tna=g|fByWklvmGDI3dA~tum8I&ua|ivC9t!e#Vt@P;Tpi54i1u@4?$+ z-$`P}sZD2rOPnA9*ghxrUsIMv%|Ft;RpIhEK8>(ybdES$Y63rUS|;s2Zi4(0M#dj2 zNjbE(t#bR!JyU(zM&}ClWk8)SFHn=KRwWtm#FgC=;4+MHKyI1GBz(tnJQeT!v`ms8 zi;F?Hd`t1GodF>!#ghj&-n4BbUXaCGeR$ZZpBmkBQO%b zHqffH_l#L#^n`tCtCJv?oU+fhXI6Tg6qUpJJo{{WmoJ51_qcKONEiO>Us7EdG8b?J z9DyATfwlMr^#1<-9j#kCWPiH4x92e4WOa>+1+O0*-(qPzeDKe3jsHVMo9ba0DEIPD9}TF~~|d T41D6|00000NkvXXu0mjf7PQl> literal 0 HcmV?d00001 diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 17fa286a0309..1759149eb00a 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -34,6 +34,10 @@ Gossip Protocol + > + Sessions + + > Security Model From 3fc7b208a5723e8a080d37c7861cf018472b5461 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 20 May 2014 12:22:16 -0700 Subject: [PATCH 28/30] website: Adding guide on leader election --- .../docs/guides/leader-election.html.markdown | 73 +++++++++++++++++++ .../docs/internals/sessions.html.markdown | 13 +++- website/source/layouts/docs.erb | 4 + 3 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 website/source/docs/guides/leader-election.html.markdown diff --git a/website/source/docs/guides/leader-election.html.markdown b/website/source/docs/guides/leader-election.html.markdown new file mode 100644 index 000000000000..6a2a1fc7d5c7 --- /dev/null +++ b/website/source/docs/guides/leader-election.html.markdown @@ -0,0 +1,73 @@ +--- +layout: "docs" +page_title: "Leader Election" +sidebar_current: "docs-guides-leader" +--- + +# Leader Election + +The goal of this guide is to cover how to build client-side leader election using Consul. +If you are interested in the leader election used internally to Consul, you want to +read about the [consensus protocol](/docs/internals/consensus.html) instead. + +There are a number of ways that leader election can be built, so our goal is not to +cover all the possible methods. Instead, we will focus on using Consul's support for +[sessions](/docs/internals/sessions.html), which allow us to build a system that can +gracefully handle failures. + +## Contending Nodes + +The first flow we cover is for nodes who are attempting to acquire leadership +for a given service. All nodes that are participating should agree on a given +key being used to coordinate. A good choice is simply: + + service//leader + +We will refer to this as just `key` for simplicy. + +The first step is to create a session. This is done using the /v1/session/create endpoint. +The session by default makes use of only the gossip failure detector. Additional checks +can be specified if desired. The session ID returned will be refered to as `session`. + +Create `body` to represent the local node. This can be a simple JSON object +that contains the node's name, port or any application specific information +that may be needed. + +Attempt to `acquire` the `key` by doing a `PUT`. This is something like: + + curl -X PUT -d body http://localhost:8500/v1/kv/key?acquire=session + +This will either return `true` or `false`. If `true` is returned, the lock +has been acquired and the local node is now the leader. If `false` is returned, +some other node has acquired the lock. + +All nodes now remain in an idle waiting state. In this state, we watch for changes +on `key`. This is because the lock may be released, the node may fail, etc. +The leader must also watch for changes since it's lock may be released by an operator, +or automatically released due to a false positive in the failure detector. + +Watching for changes is done by doing a blocking query against `key`. If we ever +notice that the `Session` of the `key` is blank, then there is no leader, and we should +retry acquiring the lock. Each attempt to acquire the key should be seperated by a timed +wait. This is because Consul may be enforcing a [`lock-delay`](/docs/internals/sessions.html). + +If the leader ever wishes to step down voluntarily, this should be done by simply +releasing the lock: + + curl -X PUT http://localhost:8500/v1/kv/key?release=session + +## Discovering a Leader + +The second flow is for nodes who are attempting to discover the leader +for a given servie. All nodes that are participating should agree on the key +being used to coordinate, including the contendors. This key will be referred +to as just `key`. + +Clients have a very simple role, they simply read `key` to discover who the current +leader is. If the key has no associated `Session`, then there is no leader. Otherwise, +the value of the key will provide all the application-dependent information required. + +Clients should also watch the key using a blocking query for any changes. If the leader +steps down, or fails, then the `Session` associated with the key will be cleared. When +a new leader is elected, the key value will also be updated. + diff --git a/website/source/docs/internals/sessions.html.markdown b/website/source/docs/internals/sessions.html.markdown index f45fffeb358e..f573a568c533 100644 --- a/website/source/docs/internals/sessions.html.markdown +++ b/website/source/docs/internals/sessions.html.markdown @@ -4,11 +4,12 @@ page_title: "Sessions" sidebar_current: "docs-internals-sessions" --- -# Consensus Protocol +# Sessions Consul provides a session mechansim which can be used to build distributed locks. Sessions act as a binding layer between nodes, health checks, and key/value data. -They are designed to provide granular locking similar to Chubby. +They are designed to provide granular locking, and are heavily inspired +by [The Chubby Lock Service for Loosely-Coupled Distributed Systems](http://research.google.com/archive/chubby.html).
Advanced Topic! This page covers technical details of @@ -87,7 +88,7 @@ and the `Session` value is updated to reflect the session holding the lock. Once held, the lock can be released using a corresponding `release` operation, providing the same session. Again, this acts like a Check-And-Set operations, since the request will fail if given an invalid session. A critical note is -that the session ID can be destroyed without being the creator of the session. +that the lock can be released without being the creator of the session. This is by design, as it allows operators to intervene and force terminate a session if necessary. As mentioned above, a session invalidation will also cause all held locks to be released. When a lock is released, the `LockIndex`, @@ -105,3 +106,9 @@ that clients must acquire a lock to perform any operation. Any client can read, write, and delete a key without owning the corresponding lock. It is not the goal of Consul to protect against misbehaving clients. +## Leader Election + +The primitives provided by sessions and the locking mechanisms of the KV +store can be used to build client-side leader election algorithms. +These are covered in more detail in the [Leader Election guide](/docs/guides/leader-election.html). + diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 1759149eb00a..3d6c84808312 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -144,6 +144,10 @@ External Services + > + Leader Election + + > Multiple Datacenters From a12c52ca2d4347f2f619f8b27f7f3551b936ed3c Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 20 May 2014 15:22:42 -0700 Subject: [PATCH 29/30] website: Documenting the session endpoints --- website/source/docs/agent/http.html.markdown | 132 +++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/website/source/docs/agent/http.html.markdown b/website/source/docs/agent/http.html.markdown index abfc9e04f594..de1210d4ebad 100644 --- a/website/source/docs/agent/http.html.markdown +++ b/website/source/docs/agent/http.html.markdown @@ -16,6 +16,7 @@ All endpoints fall into one of 6 categories: * agent - Agent control * catalog - Manages nodes and services * health - Manages health checks +* session - Session manipulation * status - Consul system status * internal - Internal APIs. Purposely undocumented, subject to change. @@ -805,6 +806,137 @@ It returns a JSON body like this: This endpoint supports blocking queries and all consistency modes. +## Session + +The Session endpoints are used to create, destroy and query sessions. +The following endpoints are supported: + +* /v1/session/create: Creates a new session +* /v1/session/destroy/\: Destroys a given session +* /v1/session/info/\: Queries a given session +* /v1/session/node/\: Lists sessions belonging to a node +* /v1/session/list: Lists all the active sessions + +All of the read session endpoints supports blocking queries and all consistency modes. + +### /v1/session/create + +The create endpoint is used to initialize a new session. +There is more documentation on sessions [here](/docs/internals/sessions.html). +Sessions must be associated with a node, and optionally any number of checks. +By default, the agent uses it's own node name, and provides the "serfHealth" +check, along with a 15 second lock delay. + +By default, the agent's local datacenter is used, but another datacenter +can be specified using the "?dc=" query parameter. It is not recommended +to use cross-region sessions. + +The create endpoint expects a JSON request body to be PUT. The request +body must look like: + + { + "LockDelay": "15s", + "Node": "foobar", + "Checks": ["a", "b", "c"] + } + +None of the fields are mandatory, and in fact no body needs to be PUT +if the defaults are to be used. The `LockDelay` field can be specified +as a duration string using a "s" suffix for seconds. It can also be a numeric +value. Small values are treated as seconds, and otherwise it is provided with +nanosecond granularity. + +The `Node` field must refer to a node that is already registered. By default, +the agent will use it's own name. Lastly, the `Checks` field is used to provide +a list of associated health checks. By default the "serfHealth" check is provided. +It is highly recommended that if you override this list, you include that check. + +The return code is 200 on success, along with a body like: + + {"ID":"adf4238a-882b-9ddc-4a9d-5b6758e4159e"} + +This is used to provide the ID of the newly created session. + +### /v1/session/destroy/\ + +The destroy endpoint is hit with a PUT and destroys the given session. +By default the local datacenter is used, but the "?dc=" query parameter +can be used to specify the datacenter. The session being destroyed must +be provided after the slash. + +The return code is 200 on success. + +### /v1/session/info/\ + +This endpoint is hit with a GET and returns the session information +by ID within a given datacenter. By default the datacenter of the agent is queried, +however the dc can be provided using the "?dc=" query parameter. +The session being queried must be provided after the slash. + +It returns a JSON body like this: + + [ + { + "LockDelay": 1.5e+10, + "Checks": [ + "serfHealth" + ], + "Node": "foobar", + "ID": "adf4238a-882b-9ddc-4a9d-5b6758e4159e", + "CreateIndex": 1086449 + } + ] + +If the session is not found, null is returned instead of a JSON list. +This endpoint supports blocking queries and all consistency modes. + +### /v1/session/node/\ + +This endpoint is hit with a GET and returns the active sessions +for a given node and datacenter. By default the datacenter of the agent is queried, +however the dc can be provided using the "?dc=" query parameter. +The node being queried must be provided after the slash. + +It returns a JSON body like this: + + [ + { + "LockDelay": 1.5e+10, + "Checks": [ + "serfHealth" + ], + "Node": "foobar", + "ID": "adf4238a-882b-9ddc-4a9d-5b6758e4159e", + "CreateIndex": 1086449 + }, + ... + ] + +This endpoint supports blocking queries and all consistency modes. + +### /v1/session/list + +This endpoint is hit with a GET and returns the active sessions +for a given datacenter. By default the datacenter of the agent is queried, +however the dc can be provided using the "?dc=" query parameter. + +It returns a JSON body like this: + + [ + { + "LockDelay": 1.5e+10, + "Checks": [ + "serfHealth" + ], + "Node": "foobar", + "ID": "adf4238a-882b-9ddc-4a9d-5b6758e4159e", + "CreateIndex": 1086449 + }, + ... + ] + +This endpoint supports blocking queries and all consistency modes. + ## Status The Status endpoints are used to get information about the status From 55951c9f879583d835c5d8aa2b99eea40ee20b7d Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 20 May 2014 15:49:36 -0700 Subject: [PATCH 30/30] website: Document KV changes --- website/source/docs/agent/http.html.markdown | 68 +++++++++++++------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/website/source/docs/agent/http.html.markdown b/website/source/docs/agent/http.html.markdown index de1210d4ebad..c220d60bdd1b 100644 --- a/website/source/docs/agent/http.html.markdown +++ b/website/source/docs/agent/http.html.markdown @@ -10,7 +10,7 @@ The main interface to Consul is a RESTful HTTP API. The API can be used for CRUD for nodes, services, checks, and configuration. The endpoints are versioned to enable changes without breaking backwards compatibility. -All endpoints fall into one of 6 categories: +All endpoints fall into one of several categories: * kv - Key/Value store * agent - Agent control @@ -95,6 +95,8 @@ By default the datacenter of the agent is queried, however the dc can be provided using the "?dc=" query parameter. If a client wants to write to all Datacenters, one request per datacenter must be made. +### GET Method + When using the `GET` method, Consul will return the specified key, or if the "?recurse" query parameter is provided, it will return all keys with the given prefix. @@ -105,9 +107,11 @@ Each object will look like: { "CreateIndex": 100, "ModifyIndex": 200, + "LockIndex": 200, "Key": "zip", "Flags": 0, - "Value": "dGVzdA==" + "Value": "dGVzdA==", + "Session": "adf4238a-882b-9ddc-4a9d-5b6758e4159e" } ] @@ -117,15 +121,36 @@ that modified this key. This index corresponds to the `X-Consul-Index` header value that is returned. A blocking query can be used to wait for a value to change. If "?recurse" is used, the `X-Consul-Index` corresponds to the latest `ModifyIndex` and so a blocking query waits until any of the -listed keys are updated. The multiple consistency modes can be used for -`GET` requests as well. +listed keys are updated. The `LockIndex` is the last index of a successful +lock acquisition. If the lock is held, the `Session` key provides the +session that owns the lock. The `Key` is simply the full path of the entry. `Flags` are an opaque unsigned integer that can be attached to each entry. The use of this is left totally to the user. Lastly, the `Value` is a base64 key value. +It is possible to also only list keys without any values by using the +"?keys" query parameter along with a `GET` request. This will return +a list of the keys under the given prefix. The optional "?separator=" +can be used to list only up to a given separator. + +For example, listing "/web/" with a "/" seperator may return: + + [ + "/web/bar", + "/web/foo", + "/web/subdir/" + ] + +Using the key listing method may be suitable when you do not need +the values or flags, or want to implement a key-space explorer. + If no entries are found, a 404 code is returned. +This endpoint supports blocking queries and all consistency modes. + +### PUT method + When using the `PUT` method, Consul expects the request body to be the value corresponding to the key. There are a number of parameters that can be used with a PUT request: @@ -134,36 +159,33 @@ be used with a PUT request: 0 and 2^64-1. It is opaque to the user, but a client application may use it. -* ?cas=\ : This flag is used to turn the `PUT` into a **Check-And-Set** +* ?cas=\ : This flag is used to turn the `PUT` into a Check-And-Set operation. This is very useful as it allows clients to build more complex syncronization primitives on top. If the index is 0, then Consul will only put the key if it does not already exist. If the index is non-zero, then the key is only set if the index matches the `ModifyIndex` of that key. -The return value is simply either `true` or `false`. If the CAS check fails, -then `false` will be returned. +* ?acquire=\ : This flag is used to turn the `PUT` into a lock acquisition + operation. This is useful as it allows leader election to be built on top + of Consul. If the lock is not held and the session is valid, this increments + the `LockIndex` and sets the `Session` value of the key in addition to updating + the key contents. A key does not need to exist to be acquired. + +* ?release=\ : This flag is used to turn the `PUT` into a lock release + operation. This is useful when paired with "?acquire=" as it allows clients to + yield a lock. This will leave the `LockIndex` unmodified but will clear the associated + `Session` of the key. The key must be held by this session to be unlocked. + +The return value is simply either `true` or `false`. If `false` is returned, +then the update has not taken place. + +### DELETE method Lastly, the `DELETE` method can be used to delete a single key or all keys sharing a prefix. If the "?recurse" query parameter is provided, then all keys with the prefix are deleted, otherwise only the specified key. -It is possible to also only list keys without any values by using the -"?keys" query parameter along with a `GET` request. This will return -a list of the keys under the given prefix. The optional "?separator=" -can be used to list only up to a given separator. - -For example, listing "/web/" with a "/" seperator may return: - - [ - "/web/bar", - "/web/foo", - "/web/subdir/" - ] - -Using the key listing method may be suitable when you do not need -the values or flags, or want to implement a key-space explorer. - ## Agent The Agent endpoints are used to interact with a local Consul agent. Usually,