Skip to content
This repository was archived by the owner on Aug 15, 2023. It is now read-only.

Commit 23c8c81

Browse files
vecnavecna
authored
feat(backend): implemented originalHref into experiment CSV #384 (#385)
1 parent 0f6451c commit 23c8c81

File tree

3 files changed

+201
-1656
lines changed

3 files changed

+201
-1656
lines changed

‎backend/lib/CSV.js

+166-126
Original file line numberDiff line numberDiff line change
@@ -5,110 +5,142 @@ const moment = require('moment');
55
const utils = require('./utils');
66

77
function produceCSVv1(entries, requestedKeys) {
8-
9-
const keys = requestedKeys || _.keys(entries[0]);
10-
11-
const produced = _.reduce(entries, function(memo, entry, cnt) {
12-
if(!memo.init) {
13-
memo.expect = _.size(keys);
14-
memo.csv = _.trim(JSON.stringify(keys), '][') + "\n";
15-
memo.init = true;
16-
}
17-
18-
if(_.size(keys) !== memo.expect) {
19-
debug("Invalid JSON input: expected %d keys, got %d",
20-
memo.expect, _.size(keys));
21-
throw new Error("CSV fatal issue (JSON input key error)");
8+
const keys = requestedKeys || _.keys(entries[0]);
9+
10+
const produced = _.reduce(
11+
entries,
12+
function (memo, entry, cnt) {
13+
if (!memo.init) {
14+
memo.expect = _.size(keys);
15+
memo.csv = _.trim(JSON.stringify(keys), '][') + '\n';
16+
memo.init = true;
17+
}
18+
19+
if (_.size(keys) !== memo.expect) {
20+
debug(
21+
'Invalid JSON input: expected %d keys, got %d',
22+
memo.expect,
23+
_.size(keys)
24+
);
25+
throw new Error('CSV fatal issue (JSON input key error)');
26+
}
27+
28+
_.each(keys, function (k, i) {
29+
let swap = _.get(entry, k, '');
30+
if (_.endsWith(k, 'Time') || k === 'lastUpdate')
31+
memo.csv += moment(swap).toISOString();
32+
else if (_.isInteger(swap)) {
33+
memo.csv += swap;
34+
} else {
35+
swap = _.replace(swap, /"/g, '〃');
36+
swap = _.replace(swap, /'/g, '’');
37+
memo.csv += '"' + swap + '"';
2238
}
23-
24-
_.each(keys, function(k, i) {
25-
let swap = _.get(entry, k, "");
26-
if(_.endsWith(k,'Time') || k === 'lastUpdate')
27-
memo.csv += moment(swap).toISOString();
28-
else if(_.isInteger(swap)) {
29-
memo.csv += swap;
30-
}
31-
else {
32-
swap = _.replace(swap, /"/g, '〃');
33-
swap = _.replace(swap, /'/g, '’');
34-
memo.csv += '"' + swap + '"';
35-
}
36-
if(!_.eq(i, _.size(keys) - 1))
37-
memo.csv += ',';
38-
});
39-
memo.csv += "\n";
40-
return memo;
41-
42-
}, { init: false, csv: "", expect: 0 });
43-
return produced.csv;
44-
};
39+
if (!_.eq(i, _.size(keys) - 1)) memo.csv += ',';
40+
});
41+
memo.csv += '\n';
42+
return memo;
43+
},
44+
{ init: false, csv: '', expect: 0 }
45+
);
46+
return produced.csv;
47+
}
4548

4649
function flattenSearches(evidence, shared) {
47-
// SEARCH
48-
const fields = [ "position", "title", "authorName",
49-
"authorSource", "sectionName", "href",
50-
"videoId", "views", "isLive", "published",
51-
"secondsAgo" ];
52-
53-
return _.map(evidence.results, function(found, i) {
54-
const keep = _.pick(found, fields);
55-
56-
return {
57-
...shared,
58-
id: shared.metadataId + "S" + i + found.position,
59-
...keep,
60-
query: evidence.query,
61-
}
62-
});
50+
// SEARCH
51+
const fields = [
52+
'position',
53+
'title',
54+
'authorName',
55+
'authorSource',
56+
'sectionName',
57+
'href',
58+
'originalHref',
59+
'videoId',
60+
'views',
61+
'isLive',
62+
'published',
63+
'secondsAgo',
64+
];
65+
66+
return _.map(evidence.results, function (found, i) {
67+
const keep = _.pick(found, fields);
68+
69+
return {
70+
...shared,
71+
id: shared.metadataId + 'S' + i + found.position,
72+
...keep,
73+
query: evidence.query,
74+
};
75+
});
6376
}
6477

6578
function unrollRecommended(evidence, shared) {
66-
// VIDEO
67-
const fields = [ "index", "verified", "videoId",
68-
"recommendedSource", "recommendedTitle",
69-
"recommendedLength", "recommendedRelativeSeconds",
70-
"recommendedViews", "isLive", "publicationTime" ];
71-
72-
return _.map(evidence.related, function(related, i) {
73-
const keep = _.pick(related, fields);
74-
75-
return {
76-
...shared,
77-
id: shared.metadataId + 'V' + i + related.index,
78-
...keep,
79-
watchedVideoId: evidence.videoId,
80-
watchedTitle: evidence.title,
81-
watchedAuthor: evidence.authorName,
82-
watchedChannel: evidence.authorSource,
83-
watchedPubTime: evidence.publicationTime,
84-
forKids: evidence.forKids,
85-
};
86-
});
79+
// VIDEO
80+
const fields = [
81+
'index',
82+
'verified',
83+
'videoId',
84+
'originalHref',
85+
'recommendedSource',
86+
'recommendedTitle',
87+
'recommendedLength',
88+
'recommendedRelativeSeconds',
89+
'recommendedViews',
90+
'isLive',
91+
'publicationTime',
92+
];
93+
94+
return _.map(evidence.related, function (related, i) {
95+
const keep = _.pick(related, fields);
96+
97+
return {
98+
...shared,
99+
id: shared.metadataId + 'V' + i + related.index,
100+
...keep,
101+
watchedVideoId: evidence.videoId,
102+
watchedTitle: evidence.title,
103+
watchedAuthor: evidence.authorName,
104+
watchedChannel: evidence.authorSource,
105+
watchedPubTime: evidence.publicationTime,
106+
forKids: evidence.forKids,
107+
};
108+
});
87109
}
88110

89111
function reconnectAdvertising(evidence, shared) {
90-
throw new Error("Not working ATM! special condition to handle -- test with experiments especially");
112+
throw new Error(
113+
'Not working ATM! special condition to handle -- test with experiments especially'
114+
);
91115
}
92116

93117
function unwindSections(evidence, shared) {
94-
// HOME
95-
const fields = [ "index", "verified",
96-
"videoId", "recommendedSource",
97-
"recommendedHref", "recommendedTitle",
98-
"recommendedLength", "recommendedRelativeSeconds",
99-
"recommendedViews", "isLive", "publicationTime" ];
100-
101-
return _.map(evidence.selected, function(selected, i) {
102-
const keep = _.pick(selected, fields);
103-
104-
return {
105-
...shared,
106-
id: shared.metadataId + 'H' + i + selected.index,
107-
...keep,
108-
};
109-
});
110-
};
111-
118+
// HOME
119+
const fields = [
120+
'index',
121+
'verified',
122+
'originalHref',
123+
'videoId',
124+
'recommendedSource',
125+
'recommendedHref',
126+
'recommendedTitle',
127+
'recommendedLength',
128+
'recommendedRelativeSeconds',
129+
'recommendedViews',
130+
'isLive',
131+
'publicationTime',
132+
];
133+
134+
return _.map(evidence.selected, function (selected, i) {
135+
const keep = _.pick(selected, fields);
136+
137+
return {
138+
...shared,
139+
id: shared.metadataId + 'H' + i + selected.index,
140+
...keep,
141+
};
142+
});
143+
}
112144

113145
/*
114146
- let here to remind self about chardet.
@@ -122,59 +154,67 @@ function unwindSections(evidence, shared) {
122154
*/
123155

124156
function unrollNested(metadata, options) {
125-
/* | options can be: |
157+
/* | options can be: |
126158
| type (home|video|search|adv) |
127159
| private (true|undefined) |
128160
| experiment (true|undefined) | */
129161

130-
return _.flatten(_.compact(_.map(metadata, function(evidence) {
131-
if( options.type !== 'adv' &&
132-
evidence.type !== options.type)
133-
return null;
162+
return _.flatten(
163+
_.compact(
164+
_.map(metadata, function (evidence) {
165+
if (options.type !== 'adv' && evidence.type !== options.type)
166+
return null;
134167

135168
const shared = {
136-
pseudo: utils.string2Food(evidence.publicKey),
137-
metadataId: evidence.id.substr(0, 8),
138-
savingTime: evidence.savingTime,
139-
clientTime: evidence.clientTime,
140-
login: evidence.login,
141-
uxLang: evidence.blang,
169+
pseudo: utils.string2Food(evidence.publicKey),
170+
metadataId: evidence.id.substr(0, 8),
171+
savingTime: evidence.savingTime,
172+
clientTime: evidence.clientTime,
173+
login: evidence.login,
174+
uxLang: evidence.blang,
142175
};
143176

144-
if(options.private) {
145-
const smp = shared.pseudo.split('-');
146-
shared.pseudo = smp[0] + '-' + smp[1];
177+
if (options.private) {
178+
const smp = shared.pseudo.split('-');
179+
shared.pseudo = smp[0] + '-' + smp[1];
147180
}
148-
if(options.experiment) {
149-
shared.experimentId = evidence.experiment.experimentId;
150-
shared.evidencetag = evidence.experiment.evidencetag;
151-
shared.execount = evidence.experiment.execount;
181+
if (options.experiment) {
182+
shared.experimentId = evidence.experiment.experimentId;
183+
shared.evidencetag = evidence.experiment.evidencetag;
184+
shared.execount = evidence.experiment.execount;
185+
// note, this is only present in experiment as sharedDataPull
186+
// in routes/experiment perform an aggregate, but we
187+
// should consider do the same in personal and elsewhere
188+
// and add move this into "shared" initialization above,
189+
// to test it: pick an experiment CSV and then a personal CSV.
190+
shared.originalHref = evidence.originalHref;
152191
}
153192

154193
let entry = null;
155194
/* entry is a list, this piece of code is diving into
156195
* the linked list inside the elements, and based on the
157-
* sub-entry the result is offered back. advertising
196+
* sub-entry the result is offered back. advertising
158197
* might be eterogeneous type */
159-
if(options.type === 'home')
160-
entry = unwindSections(evidence, shared);
161-
else if(options.type === 'video')
162-
entry = unrollRecommended(evidence, shared);
163-
else if(options.type === 'search')
164-
entry = flattenSearches(evidence, shared);
165-
else if(options.type === 'adv')
166-
entry = reconnectAdvertising(evidence, shared);
198+
if (options.type === 'home') entry = unwindSections(evidence, shared);
199+
else if (options.type === 'video')
200+
entry = unrollRecommended(evidence, shared);
201+
else if (options.type === 'search')
202+
entry = flattenSearches(evidence, shared);
203+
else if (options.type === 'adv')
204+
entry = reconnectAdvertising(evidence, shared);
167205

168206
return entry;
169-
})))
207+
})
208+
)
209+
);
170210
}
171211

172-
const allowedTypes = ['video', 'search', 'home', 'adv' ];
212+
const allowedTypes = ['video', 'search', 'home', 'adv'];
173213

174214
module.exports = {
175-
allowedTypes,
176-
// this is the function to call to convert raw metadata
177-
unrollNested,
178-
// this is the one that converts JSON to CSV
179-
produceCSVv1,
215+
allowedTypes,
216+
// this is the function to call to convert raw metadata
217+
unrollNested,
218+
// this is the one that converts JSON to CSV
219+
produceCSVv1,
180220
};

0 commit comments

Comments
 (0)