Skip to content

Commit cd323d4

Browse files
yasirfolio3Michael Ng
yasirfolio3
authored and
Michael Ng
committed
feat(PollingConfigManager): Implemented caching headers in PollingConfigManager. (#189)
1 parent ccf2503 commit cd323d4

File tree

6 files changed

+115
-51
lines changed

6 files changed

+115
-51
lines changed

‎pkg/config/polling_manager.go‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package config
1919

2020
import (
2121
"fmt"
22+
"net/http"
2223
"sync"
2324
"time"
2425

@@ -36,6 +37,12 @@ const DefaultPollingInterval = 5 * time.Minute // default to 5 minutes for polli
3637
// DatafileURLTemplate is used to construct the endpoint for retrieving the datafile from the CDN
3738
const DatafileURLTemplate = "https://cdn.optimizely.com/datafiles/%s.json"
3839

40+
// ModifiedSince header key for request
41+
const ModifiedSince = "If-Modified-Since"
42+
43+
// LastModified header key for response
44+
const LastModified = "Last-Modified"
45+
3946
var cmLogger = logging.GetLogger("PollingConfigManager")
4047

4148
// PollingProjectConfigManager maintains a dynamic copy of the project config
@@ -44,6 +51,7 @@ type PollingProjectConfigManager struct {
4451
pollingInterval time.Duration
4552
notificationCenter notification.Center
4653
initDatafile []byte
54+
lastModified string
4755

4856
configLock sync.RWMutex
4957
err error
@@ -89,21 +97,40 @@ func InitialDatafile(datafile []byte) OptionFunc {
8997
func (cm *PollingProjectConfigManager) SyncConfig(datafile []byte) {
9098
var e error
9199
var code int
100+
var respHeaders http.Header
92101

93102
closeMutex := func(e error) {
94103
cm.err = e
95104
cm.configLock.Unlock()
96105
}
97106

98107
if len(datafile) == 0 {
99-
datafile, code, e = cm.requester.Get()
108+
if cm.lastModified != "" {
109+
lastModifiedHeader := utils.Header{Name: ModifiedSince, Value: cm.lastModified}
110+
datafile, respHeaders, code, e = cm.requester.Get(lastModifiedHeader)
111+
} else {
112+
datafile, respHeaders, code, e = cm.requester.Get()
113+
}
100114

101115
if e != nil {
102116
cmLogger.Error(fmt.Sprintf("request returned with http code=%d", code), e)
103117
cm.configLock.Lock()
104118
closeMutex(e)
105119
return
106120
}
121+
122+
if code == http.StatusNotModified {
123+
cmLogger.Debug("The datafile was not modified and won't be downloaded again")
124+
return
125+
}
126+
127+
// Save last-modified date from response header
128+
lastModified := respHeaders.Get(LastModified)
129+
if lastModified != "" {
130+
cm.configLock.Lock()
131+
cm.lastModified = lastModified
132+
cm.configLock.Unlock()
133+
}
107134
}
108135

109136
projectConfig, err := datafileprojectconfig.NewDatafileProjectConfig(datafile)

‎pkg/config/polling_manager_test.go‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package config
1818

1919
import (
20+
"net/http"
2021
"testing"
2122
"time"
2223

@@ -33,17 +34,17 @@ type MockRequester struct {
3334
mock.Mock
3435
}
3536

36-
func (m *MockRequester) Get(headers ...utils.Header) (response []byte, code int, err error) {
37+
func (m *MockRequester) Get(headers ...utils.Header) (response []byte, responseHeaders http.Header, code int, err error) {
3738
args := m.Called(headers)
38-
return args.Get(0).([]byte), args.Int(1), args.Error(2)
39+
return args.Get(0).([]byte), args.Get(1).(http.Header), args.Int(2), args.Error(3)
3940
}
4041

4142
func TestNewPollingProjectConfigManagerWithOptions(t *testing.T) {
4243

4344
mockDatafile := []byte(`{"revision":"42"}`)
4445
projectConfig, _ := datafileprojectconfig.NewDatafileProjectConfig(mockDatafile)
4546
mockRequester := new(MockRequester)
46-
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile, 200, nil)
47+
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile, http.Header{}, http.StatusOK, nil)
4748

4849
// Test we fetch using requester
4950
sdkKey := "test_sdk_key"
@@ -57,13 +58,14 @@ func TestNewPollingProjectConfigManagerWithOptions(t *testing.T) {
5758
assert.Nil(t, err)
5859
assert.NotNil(t, actual)
5960
assert.Equal(t, projectConfig, actual)
61+
6062
exeCtx.TerminateAndWait() // just sending signal and improving coverage
6163
}
6264

6365
func TestNewPollingProjectConfigManagerWithNull(t *testing.T) {
6466
mockDatafile := []byte("NOT-VALID")
6567
mockRequester := new(MockRequester)
66-
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile, 200, nil)
68+
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile, http.Header{}, http.StatusOK, nil)
6769

6870
// Test we fetch using requester
6971
sdkKey := "test_sdk_key"
@@ -82,7 +84,7 @@ func TestNewPollingProjectConfigManagerWithSimilarDatafileRevisions(t *testing.T
8284
mockDatafile2 := []byte(`{"revision":"42","botFiltering":false}`)
8385
projectConfig1, _ := datafileprojectconfig.NewDatafileProjectConfig(mockDatafile1)
8486
mockRequester := new(MockRequester)
85-
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile1, 200, nil)
87+
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile1, http.Header{}, http.StatusOK, nil)
8688

8789
sdkKey := "test_sdk_key"
8890

@@ -101,13 +103,45 @@ func TestNewPollingProjectConfigManagerWithSimilarDatafileRevisions(t *testing.T
101103
assert.Equal(t, projectConfig1, actual)
102104
}
103105

106+
func TestNewPollingProjectConfigManagerWithLastModifiedDates(t *testing.T) {
107+
mockDatafile1 := []byte(`{"revision":"42","botFiltering":true}`)
108+
projectConfig1, _ := datafileprojectconfig.NewDatafileProjectConfig(mockDatafile1)
109+
mockRequester := new(MockRequester)
110+
modifiedDate := "Wed, 16 Oct 2019 20:16:45 GMT"
111+
responseHeaders := http.Header{}
112+
responseHeaders.Set(LastModified, modifiedDate)
113+
114+
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile1, responseHeaders, http.StatusOK, nil)
115+
mockRequester.On("Get", []utils.Header{utils.Header{Name: ModifiedSince, Value: modifiedDate}}).Return([]byte{}, responseHeaders, http.StatusNotModified, nil)
116+
117+
sdkKey := "test_sdk_key"
118+
119+
exeCtx := utils.NewCancelableExecutionCtx()
120+
configManager := NewPollingProjectConfigManager(sdkKey, Requester(mockRequester))
121+
configManager.Start(exeCtx)
122+
123+
// Fetch valid config
124+
actual, err := configManager.GetConfig()
125+
assert.Nil(t, err)
126+
assert.NotNil(t, actual)
127+
assert.Equal(t, projectConfig1, actual)
128+
129+
// Sync and check no changes were made to the previous config because of 304 error code
130+
configManager.SyncConfig([]byte{})
131+
actual, err = configManager.GetConfig()
132+
assert.Nil(t, err)
133+
assert.NotNil(t, actual)
134+
assert.Equal(t, projectConfig1, actual)
135+
mockRequester.AssertExpectations(t)
136+
}
137+
104138
func TestNewPollingProjectConfigManagerWithDifferentDatafileRevisions(t *testing.T) {
105139
mockDatafile1 := []byte(`{"revision":"42","botFiltering":true}`)
106140
mockDatafile2 := []byte(`{"revision":"43","botFiltering":false}`)
107141
projectConfig1, _ := datafileprojectconfig.NewDatafileProjectConfig(mockDatafile1)
108142
projectConfig2, _ := datafileprojectconfig.NewDatafileProjectConfig(mockDatafile2)
109143
mockRequester := new(MockRequester)
110-
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile1, 200, nil)
144+
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile1, http.Header{}, http.StatusOK, nil)
111145

112146
// Test we fetch using requester
113147
sdkKey := "test_sdk_key"
@@ -134,7 +168,7 @@ func TestNewPollingProjectConfigManagerWithErrorHandling(t *testing.T) {
134168
projectConfig1, _ := datafileprojectconfig.NewDatafileProjectConfig(mockDatafile1)
135169
projectConfig2, _ := datafileprojectconfig.NewDatafileProjectConfig(mockDatafile2)
136170
mockRequester := new(MockRequester)
137-
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile1, 200, nil)
171+
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile1, http.Header{}, http.StatusOK, nil)
138172

139173
// Test we fetch using requester
140174
sdkKey := "test_sdk_key"
@@ -165,7 +199,7 @@ func TestNewPollingProjectConfigManagerOnDecision(t *testing.T) {
165199
mockDatafile2 := []byte(`{"revision":"43","botFiltering":false}`)
166200

167201
mockRequester := new(MockRequester)
168-
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile1, 200, nil)
202+
mockRequester.On("Get", []utils.Header(nil)).Return(mockDatafile1, http.Header{}, http.StatusOK, nil)
169203

170204
// Test we fetch using requester
171205
sdkKey := "test_sdk_key"

‎pkg/config/static_manager.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ type StaticProjectConfigManager struct {
3838
func NewStaticProjectConfigManagerFromURL(sdkKey string) (*StaticProjectConfigManager, error) {
3939

4040
requester := utils.NewHTTPRequester(fmt.Sprintf(DatafileURLTemplate, sdkKey))
41-
datafile, code, e := requester.Get()
41+
datafile, _, code, e := requester.Get()
4242
if e != nil {
4343
cmLogger.Error(fmt.Sprintf("request returned with http code=%d", code), e)
4444
return nil, e

‎pkg/event/dispatcher.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package event
2020
import (
2121
"context"
2222
"fmt"
23+
"net/http"
2324
"sync"
2425
"time"
2526

@@ -46,7 +47,7 @@ type HTTPEventDispatcher struct {
4647
func (*HTTPEventDispatcher) DispatchEvent(event LogEvent) (bool, error) {
4748

4849
requester := utils.NewHTTPRequester(event.EndPoint)
49-
_, code, err := requester.Post(event.Event)
50+
_, _, code, err := requester.Post(event.Event)
5051

5152
// also check response codes
5253
// resp.StatusCode == 400 is an error
@@ -55,7 +56,7 @@ func (*HTTPEventDispatcher) DispatchEvent(event LogEvent) (bool, error) {
5556
dispatcherLogger.Error("http.Post failed:", err)
5657
success = false
5758
} else {
58-
if code == 204 {
59+
if code == http.StatusNoContent {
5960
success = true
6061
} else {
6162
dispatcherLogger.Error(fmt.Sprintf("http.Post invalid response %d", code), nil)

‎pkg/utils/requester.go‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,10 @@ var json = jsoniter.ConfigCompatibleWithStandardLibrary
3838

3939
// Requester is used to make outbound requests with
4040
type Requester interface {
41-
Get(...Header) (response []byte, code int, err error)
41+
Get(...Header) (response []byte, responseHeaders http.Header, code int, err error)
4242
GetObj(result interface{}, headers ...Header) error
4343

44-
Post(body interface{}, headers ...Header) (response []byte, code int, err error)
44+
Post(body interface{}, headers ...Header) (response []byte, responseHeaders http.Header, code int, err error)
4545
PostObj(body interface{}, result interface{}, headers ...Header) error
4646

4747
String() string
@@ -101,45 +101,45 @@ func NewHTTPRequester(url string, params ...func(*HTTPRequester)) *HTTPRequester
101101

102102
// Get executes HTTP GET with url and optional extra headers, returns body in []bytes
103103
// url created as url+sdkKey.json
104-
func (r HTTPRequester) Get(headers ...Header) (response []byte, code int, err error) {
104+
func (r HTTPRequester) Get(headers ...Header) (response []byte, responseHeaders http.Header, code int, err error) {
105105
return r.Do("GET", nil, headers)
106106
}
107107

108108
// GetObj executes HTTP GET with url and optional extra headers, returns filled object
109109
func (r HTTPRequester) GetObj(result interface{}, headers ...Header) error {
110-
b, _, err := r.Do("GET", nil, headers)
110+
b, _, _, err := r.Do("GET", nil, headers)
111111
if err != nil {
112112
return err
113113
}
114114
return json.Unmarshal(b, result)
115115
}
116116

117117
// Post executes HTTP POST with url, body and optional extra headers
118-
func (r HTTPRequester) Post(body interface{}, headers ...Header) (response []byte, code int, err error) {
118+
func (r HTTPRequester) Post(body interface{}, headers ...Header) (response []byte, responseHeaders http.Header, code int, err error) {
119119
b, err := json.Marshal(body)
120120
if err != nil {
121-
return nil, 400, err
121+
return nil, nil, http.StatusBadRequest, err
122122
}
123123
return r.Do("POST", bytes.NewBuffer(b), headers)
124124
}
125125

126126
// PostObj executes HTTP POST with uri, body and optional extra headers. Returns filled object
127127
func (r HTTPRequester) PostObj(body, result interface{}, headers ...Header) error {
128-
b, _, err := r.Post(body, headers...)
128+
b, _, _, err := r.Post(body, headers...)
129129
if err != nil {
130130
return err
131131
}
132132
return json.Unmarshal(b, result)
133133
}
134134

135135
// Do executes request and returns response body for requested uri (sdkKey.json).
136-
func (r HTTPRequester) Do(method string, body io.Reader, headers []Header) (response []byte, code int, err error) {
136+
func (r HTTPRequester) Do(method string, body io.Reader, headers []Header) (response []byte, responseHeaders http.Header, code int, err error) {
137137

138-
single := func(request *http.Request) (response []byte, code int, e error) {
138+
single := func(request *http.Request) (response []byte, responseHeaders http.Header, code int, e error) {
139139
resp, doErr := r.client.Do(request)
140140
if doErr != nil {
141141
requesterLogger.Error(fmt.Sprintf("failed to send request %v", request), e)
142-
return nil, 0, doErr
142+
return nil, http.Header{}, 0, doErr
143143
}
144144
defer func() {
145145
if e := resp.Body.Close(); e != nil {
@@ -149,35 +149,35 @@ func (r HTTPRequester) Do(method string, body io.Reader, headers []Header) (resp
149149

150150
if response, err = ioutil.ReadAll(resp.Body); err != nil {
151151
requesterLogger.Error("failed to read body", err)
152-
return nil, resp.StatusCode, err
152+
return nil, resp.Header, resp.StatusCode, err
153153
}
154154

155-
if resp.StatusCode >= 400 {
155+
if resp.StatusCode >= http.StatusBadRequest {
156156
requesterLogger.Warning(fmt.Sprintf("error status code=%d", resp.StatusCode))
157-
return response, resp.StatusCode, errors.New(resp.Status)
157+
return response, resp.Header, resp.StatusCode, errors.New(resp.Status)
158158
}
159159

160-
return response, resp.StatusCode, nil
160+
return response, resp.Header, resp.StatusCode, nil
161161
}
162162

163163
requesterLogger.Debug(fmt.Sprintf("request %s", r.url))
164164
req, err := http.NewRequest(method, r.url, body)
165165
if err != nil {
166166
requesterLogger.Error(fmt.Sprintf("failed to make request %s", r.url), err)
167-
return nil, 0, err
167+
return nil, nil, 0, err
168168
}
169169

170170
r.addHeaders(req, headers)
171171

172172
for i := 0; i < r.retries; i++ {
173173

174-
if response, code, err = single(req); err == nil {
174+
if response, responseHeaders, code, err = single(req); err == nil {
175175
triedMsg := ""
176176
if i > 0 {
177177
triedMsg = fmt.Sprintf(", tried %d time(s)", i+1)
178178
}
179179
requesterLogger.Debug(fmt.Sprintf("completed %s%s", r.url, triedMsg))
180-
return response, code, err
180+
return response, responseHeaders, code, err
181181
}
182182
requesterLogger.Debug(fmt.Sprintf("failed %s with %v", r.url, err))
183183

@@ -187,7 +187,7 @@ func (r HTTPRequester) Do(method string, body io.Reader, headers []Header) (resp
187187
}
188188
}
189189

190-
return response, code, err
190+
return response, responseHeaders, code, err
191191
}
192192

193193
func (r HTTPRequester) addHeaders(req *http.Request, headers []Header) *http.Request {

0 commit comments

Comments
 (0)