-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathreadme.html
396 lines (370 loc) · 23.1 KB
/
readme.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
<meta charset="utf-8">
<meta name="generator" content="quarto-1.2.335">
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
<title>Tidyup X: Adding post-processing operations to workflows</title>
<style>
code{white-space: pre-wrap;}
span.smallcaps{font-variant: small-caps;}
div.columns{display: flex; gap: min(4vw, 1.5em);}
div.column{flex: auto; overflow-x: auto;}
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
ul.task-list{list-style: none;}
ul.task-list li input[type="checkbox"] {
width: 0.8em;
margin: 0 0.8em 0.2em -1.6em;
vertical-align: middle;
}
pre > code.sourceCode { white-space: pre; position: relative; }
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
pre > code.sourceCode > span:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode > span { color: inherit; text-decoration: inherit; }
div.sourceCode { margin: 1em 0; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
pre > code.sourceCode { white-space: pre-wrap; }
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
}
pre.numberSource code
{ counter-reset: source-line 0; }
pre.numberSource code > span
{ position: relative; left: -4em; counter-increment: source-line; }
pre.numberSource code > span > a:first-child::before
{ content: counter(source-line);
position: relative; left: -1em; text-align: right; vertical-align: baseline;
border: none; display: inline-block;
-webkit-touch-callout: none; -webkit-user-select: none;
-khtml-user-select: none; -moz-user-select: none;
-ms-user-select: none; user-select: none;
padding: 0 4px; width: 4em;
color: #aaaaaa;
}
pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; }
div.sourceCode
{ }
@media screen {
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
}
code span.al { color: #ff0000; font-weight: bold; } /* Alert */
code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */
code span.at { color: #7d9029; } /* Attribute */
code span.bn { color: #40a070; } /* BaseN */
code span.bu { color: #008000; } /* BuiltIn */
code span.cf { color: #007020; font-weight: bold; } /* ControlFlow */
code span.ch { color: #4070a0; } /* Char */
code span.cn { color: #880000; } /* Constant */
code span.co { color: #60a0b0; font-style: italic; } /* Comment */
code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */
code span.do { color: #ba2121; font-style: italic; } /* Documentation */
code span.dt { color: #902000; } /* DataType */
code span.dv { color: #40a070; } /* DecVal */
code span.er { color: #ff0000; font-weight: bold; } /* Error */
code span.ex { } /* Extension */
code span.fl { color: #40a070; } /* Float */
code span.fu { color: #06287e; } /* Function */
code span.im { color: #008000; font-weight: bold; } /* Import */
code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */
code span.kw { color: #007020; font-weight: bold; } /* Keyword */
code span.op { color: #666666; } /* Operator */
code span.ot { color: #007020; } /* Other */
code span.pp { color: #bc7a00; } /* Preprocessor */
code span.sc { color: #4070a0; } /* SpecialChar */
code span.ss { color: #bb6688; } /* SpecialString */
code span.st { color: #4070a0; } /* String */
code span.va { color: #19177c; } /* Variable */
code span.vs { color: #4070a0; } /* VerbatimString */
code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */
</style>
<script src="readme_files/libs/clipboard/clipboard.min.js"></script>
<script src="readme_files/libs/quarto-html/quarto.js"></script>
<script src="readme_files/libs/quarto-html/popper.min.js"></script>
<script src="readme_files/libs/quarto-html/tippy.umd.min.js"></script>
<script src="readme_files/libs/quarto-html/anchor.min.js"></script>
<link href="readme_files/libs/quarto-html/tippy.css" rel="stylesheet">
<link href="readme_files/libs/quarto-html/quarto-syntax-highlighting.css" rel="stylesheet" id="quarto-text-highlighting-styles">
<script src="readme_files/libs/bootstrap/bootstrap.min.js"></script>
<link href="readme_files/libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
<link href="readme_files/libs/bootstrap/bootstrap.min.css" rel="stylesheet" id="quarto-bootstrap" data-mode="light">
</head>
<body class="fullcontent">
<div id="quarto-content" class="page-columns page-rows-contents page-layout-article">
<main class="content" id="quarto-document-content">
<header id="title-block-header" class="quarto-title-block default">
<div class="quarto-title">
<h1 class="title">Tidyup X: Adding post-processing operations to workflows</h1>
</div>
<div class="quarto-title-meta">
</div>
</header>
<section id="tidyup-x-adding-post-processing-operations-to-workflows" class="level1">
<h1>Tidyup X: Adding post-processing operations to workflows</h1>
<p><strong>Champion</strong>: Max</p>
<p><strong>Co-Champion</strong>: Simon</p>
<p><strong>Status</strong>: Draft</p>
<section id="abstract" class="level2">
<h2 class="anchored" data-anchor-id="abstract">Abstract</h2>
<p>tidymodels workflow objects include pre-processing and modeling steps. Pre-processors are data-oriented operations to prepare the data for the modeling function. We will be including post-processors that can do various things to model outputs (e.g. predictions). This tidyup focuses on the APIs and data structures inside of a workflow object.</p>
</section>
<section id="motivation" class="level2">
<h2 class="anchored" data-anchor-id="motivation">Motivation</h2>
<p>After modeling is complete there are occasions where model outputs, mostly predictions, might requirement modification. For example, calibration of model predictions is an effective technique for improving models. Also, some users may wish to optimize the probability threshold to call a sample an event (in a binary classification problem).</p>
</section>
<section id="solution" class="level2">
<h2 class="anchored" data-anchor-id="solution">Solution</h2>
<section id="user-facing-apis" class="level3">
<h3 class="anchored" data-anchor-id="user-facing-apis">User-facing APIs</h3>
<p>The plan is to have a small set of à la carte functions for specific operations. For example, let’s look at a binary classification model with calibration and threshold optimization:</p>
<div class="cell">
<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode r code-with-copy"><code class="sourceCode r"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>wflow_2 <span class="ot"><-</span> </span>
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a> <span class="fu">workflow</span>() <span class="sc">%>%</span> </span>
<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a> <span class="fu">add_formula</span>(Class <span class="sc">~</span> .) <span class="sc">%>%</span> </span>
<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a> <span class="fu">add_model</span>(<span class="fu">logistic_reg</span>())</span>
<span id="cb1-5"><a href="#cb1-5" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb1-6"><a href="#cb1-6" aria-hidden="true" tabindex="-1"></a>wflow_individ <span class="ot"><-</span> </span>
<span id="cb1-7"><a href="#cb1-7" aria-hidden="true" tabindex="-1"></a> wflow_2 <span class="sc">%>%</span> </span>
<span id="cb1-8"><a href="#cb1-8" aria-hidden="true" tabindex="-1"></a> <span class="fu">add_prob_calibration</span>(cal_object) <span class="sc">%>%</span> </span>
<span id="cb1-9"><a href="#cb1-9" aria-hidden="true" tabindex="-1"></a> <span class="fu">add_prob_threshold</span>(<span class="at">threshold =</span> <span class="fl">0.7</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
</div>
<p>The new <code>add_*</code> functions would have <code>update_*()</code> and <code>remove_*()</code> analogs or overall functions that conduct those operations for all of the different post-processors.</p>
<p>Note that the order of operations matter. In the above example, the system should execute the calibration before threshold the probabilities since the calibration will change the data in a way that invalidates the new threshold.</p>
<p>However, the order that the operations are added to the worklflow should not matter. There are some rules that can be enacted (e.g. calibration before thresholding) but the proposed solution is to have a prioritization scheme that can be, within reason, altered by the user. Each <code>add_*</code> function will have a <code>.priority</code> argument that has reasonable defaults (with one exception, see below).</p>
</section>
<section id="current-list-of-post-processors" class="level3">
<h3 class="anchored" data-anchor-id="current-list-of-post-processors">Current list of post-processors</h3>
<ul>
<li>Calibration of probability estimates to increase their probabilistic fidelity to the observed data</li>
</ul>
<div class="sourceCode" id="cb2"><pre class="sourceCode r code-with-copy"><code class="sourceCode r"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="co"># 'object' is a pre-made calibration object</span></span>
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="co"># Data Inputs: class probabilities</span></span>
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Data Outputs: class probabilities and recomputed class predictions</span></span>
<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a><span class="fu">add_prob_calibration</span>(object, <span class="at">.priority =</span> <span class="fl">1.0</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<ul>
<li>For binary classification outcomes, there may be the need to optimize the probability threshold that produces hard class predictions.</li>
</ul>
<div class="sourceCode" id="cb3"><pre class="sourceCode r code-with-copy"><code class="sourceCode r"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Potentially tunable</span></span>
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="co"># Data Inputs: class probabilities</span></span>
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Data Outputs: class predictions</span></span>
<span id="cb3-4"><a href="#cb3-4" aria-hidden="true" tabindex="-1"></a><span class="fu">add_prob_threshold</span>(<span class="at">threshold =</span> <span class="fu">numeric</span>(), <span class="at">.priority =</span> <span class="fl">2.0</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<ul>
<li>An addition of an equivocal zone where highly uncertain probability estimates where no hard class prediction is reported.</li>
</ul>
<div class="sourceCode" id="cb4"><pre class="sourceCode r code-with-copy"><code class="sourceCode r"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Potentially tunable</span></span>
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a><span class="co"># Data Inputs: class probabilities</span></span>
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Data Outputs: class predictions</span></span>
<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a><span class="fu">add_cls_eq_zone</span>(<span class="at">value =</span> <span class="fu">numeric</span>(), <span class="at">threshold =</span> <span class="fu">numeric</span>(), <span class="at">.priority =</span> <span class="fl">3.0</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<ul>
<li>Calibration of regression predictions to avoid regression to the mean or other systematic issues with models.</li>
</ul>
<div class="sourceCode" id="cb5"><pre class="sourceCode r code-with-copy"><code class="sourceCode r"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="co"># 'object' is a pre-made calibration object</span></span>
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="co"># Data Inputs: regression predictions</span></span>
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Data Outputs: regression predictions</span></span>
<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a><span class="fu">add_reg_calibration</span>(object, <span class="at">.priority =</span> <span class="fl">1.0</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<ul>
<li><p>A general “mutate” operations where users can do on-the-fly operations using self-contained functions. For example,</p>
<ul>
<li>If a user log-transformed their data prior to model, then can exponentiate them here.</li>
<li>For regression models, the predictions can be thresholded to be within a specific range.</li>
</ul></li>
</ul>
<div class="sourceCode" id="cb6"><pre class="sourceCode r code-with-copy"><code class="sourceCode r"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="co"># User will have to always set the priority</span></span>
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a><span class="co"># Data Inputs: all predictions</span></span>
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Data Outputs: all predictions (only these columns are retained)</span></span>
<span id="cb6-4"><a href="#cb6-4" aria-hidden="true" tabindex="-1"></a><span class="fu">add_post_mutate</span>(..., .priority)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<p>This is likely to be a fairly complete menu of possible options. For example, at a later date, nearest-neighbor adjustment of regression predictions (see <a href="https://rviews.rstudio.com/2020/05/21/modern-rule-based-models/">this blog post</a>) would be an interesting option.</p>
</section>
</section>
<section id="implementation" class="level2">
<h2 class="anchored" data-anchor-id="implementation">Implementation</h2>
<p>There are implementation issues within the workflows and tune packages to discuss. There is also the idea that a small side-package will keep the underlying functions and data structures (for now at least). We’ll tentatively call this the <code>dwai</code> package (pronounced “duh-why” and short for Don’t Worry About It).</p>
<section id="dwai" class="level3">
<h3 class="anchored" data-anchor-id="dwai">dwai</h3>
<p>This package will contain</p>
<ul>
<li>A container for the list of possible post-processors specified by the user.</li>
<li>A validation system to resolve conflicts in type or priority.</li>
<li>An interface to apply the operations to the predicted values.</li>
<li>The requisite package dependencies (primarily the probably package)</li>
</ul>
<p>The dwai code may eventually make its way into workflows or probably.</p>
</section>
<section id="workflow-structure" class="level3">
<h3 class="anchored" data-anchor-id="workflow-structure">Workflow structure</h3>
<p>Workflows already contain a placeholder for post-processing:</p>
<div class="cell">
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode r code-with-copy"><code class="sourceCode r"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="fu">library</span>(tidymodels)</span>
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a>wflow_1 <span class="ot"><-</span> </span>
<span id="cb7-4"><a href="#cb7-4" aria-hidden="true" tabindex="-1"></a> <span class="fu">workflow</span>() <span class="sc">%>%</span> </span>
<span id="cb7-5"><a href="#cb7-5" aria-hidden="true" tabindex="-1"></a> <span class="fu">add_model</span>(<span class="fu">linear_reg</span>()) <span class="sc">%>%</span> </span>
<span id="cb7-6"><a href="#cb7-6" aria-hidden="true" tabindex="-1"></a> <span class="fu">add_formula</span>(mpg <span class="sc">~</span> .)</span>
<span id="cb7-7"><a href="#cb7-7" aria-hidden="true" tabindex="-1"></a><span class="fu">names</span>(wflow_1)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<div class="cell-output cell-output-stdout">
<pre><code>[1] "pre" "fit" "post" "trained"</code></pre>
</div>
</div>
<p>There are also existing helper functions that may be relevant:</p>
<ul>
<li><p><code>order_stage_{pre,fit,post}</code> is used to resolve data priorities. For example, in pre-processing, case weights are <a href="https://github.com/tidymodels/workflows/blob/main/R/action.R#L36:L42">evaluated before the pre-processor method</a>.</p></li>
<li><p><code>new_stage_post()</code>, and <code>new_action_post()</code> are existing constructors.</p></li>
</ul>
<p>We will require a <code>.fit_post(workflow, data)</code> that will execute <em>only</em> the post-processing operations; the <code>workflows</code> object will already have trained the <code>pre</code> and <code>fit</code> stages.</p>
</section>
<section id="tune" class="level3">
<h3 class="anchored" data-anchor-id="tune">tune</h3>
<p>The tune package can currently handle pre-processors and models; either stage can be tuned over multiple parameters. tune handles this in a conditional way. It separates out the pre-processing and model tuning parameters and first loops over the pre-processing parameters (if any). Within that loop, another loop tunes over the model parameters (if any).</p>
<p>For post-processing, we will have to add another nested loop that evaluates post-processing tuning parameters in a way that doesn’t require recomputing any model or pre-processing items.</p>
<p>This has the potential to be computationally expensive and adds more complexity to the tune package.</p>
<p>Our current list of post-processors only includes two tuning parameters: threshold for equivocal zones and probability thresholding. These are simple (and fast) operations and should not significant add to the computational burden.</p>
<p>Future operations might be more expensive. For example, see the section below on “Calibration tuning”.</p>
</section>
</section>
<section id="backwards-compatibility" class="level2">
<h2 class="anchored" data-anchor-id="backwards-compatibility">Backwards compatibility</h2>
<p>There should be no issues since we are adding functionality that is independent of the current workflow capabilities.</p>
</section>
<section id="aside-calibration-tuning" class="level2">
<h2 class="anchored" data-anchor-id="aside-calibration-tuning">Aside: Calibration tuning</h2>
</section>
</section>
</main>
<!-- /main column -->
<script id="quarto-html-after-body" type="application/javascript">
window.document.addEventListener("DOMContentLoaded", function (event) {
const toggleBodyColorMode = (bsSheetEl) => {
const mode = bsSheetEl.getAttribute("data-mode");
const bodyEl = window.document.querySelector("body");
if (mode === "dark") {
bodyEl.classList.add("quarto-dark");
bodyEl.classList.remove("quarto-light");
} else {
bodyEl.classList.add("quarto-light");
bodyEl.classList.remove("quarto-dark");
}
}
const toggleBodyColorPrimary = () => {
const bsSheetEl = window.document.querySelector("link#quarto-bootstrap");
if (bsSheetEl) {
toggleBodyColorMode(bsSheetEl);
}
}
toggleBodyColorPrimary();
const icon = "";
const anchorJS = new window.AnchorJS();
anchorJS.options = {
placement: 'right',
icon: icon
};
anchorJS.add('.anchored');
const clipboard = new window.ClipboardJS('.code-copy-button', {
target: function(trigger) {
return trigger.previousElementSibling;
}
});
clipboard.on('success', function(e) {
// button target
const button = e.trigger;
// don't keep focus
button.blur();
// flash "checked"
button.classList.add('code-copy-button-checked');
var currentTitle = button.getAttribute("title");
button.setAttribute("title", "Copied!");
let tooltip;
if (window.bootstrap) {
button.setAttribute("data-bs-toggle", "tooltip");
button.setAttribute("data-bs-placement", "left");
button.setAttribute("data-bs-title", "Copied!");
tooltip = new bootstrap.Tooltip(button,
{ trigger: "manual",
customClass: "code-copy-button-tooltip",
offset: [0, -8]});
tooltip.show();
}
setTimeout(function() {
if (tooltip) {
tooltip.hide();
button.removeAttribute("data-bs-title");
button.removeAttribute("data-bs-toggle");
button.removeAttribute("data-bs-placement");
}
button.setAttribute("title", currentTitle);
button.classList.remove('code-copy-button-checked');
}, 1000);
// clear code selection
e.clearSelection();
});
function tippyHover(el, contentFn) {
const config = {
allowHTML: true,
content: contentFn,
maxWidth: 500,
delay: 100,
arrow: false,
appendTo: function(el) {
return el.parentElement;
},
interactive: true,
interactiveBorder: 10,
theme: 'quarto',
placement: 'bottom-start'
};
window.tippy(el, config);
}
const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
for (var i=0; i<noterefs.length; i++) {
const ref = noterefs[i];
tippyHover(ref, function() {
// use id or data attribute instead here
let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
try { href = new URL(href).hash; } catch {}
const id = href.replace(/^#\/?/, "");
const note = window.document.getElementById(id);
return note.innerHTML;
});
}
const findCites = (el) => {
const parentEl = el.parentElement;
if (parentEl) {
const cites = parentEl.dataset.cites;
if (cites) {
return {
el,
cites: cites.split(' ')
};
} else {
return findCites(el.parentElement)
}
} else {
return undefined;
}
};
var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
for (var i=0; i<bibliorefs.length; i++) {
const ref = bibliorefs[i];
const citeInfo = findCites(ref);
if (citeInfo) {
tippyHover(citeInfo.el, function() {
var popup = window.document.createElement('div');
citeInfo.cites.forEach(function(cite) {
var citeDiv = window.document.createElement('div');
citeDiv.classList.add('hanging-indent');
citeDiv.classList.add('csl-entry');
var biblioDiv = window.document.getElementById('ref-' + cite);
if (biblioDiv) {
citeDiv.innerHTML = biblioDiv.innerHTML;
}
popup.appendChild(citeDiv);
});
return popup.innerHTML;
});
}
}
});
</script>
</div> <!-- /content -->
</body></html>