diff --git a/env/production/config.json b/env/production/config.json index 7c7bc8ff6..585e7409c 100644 --- a/env/production/config.json +++ b/env/production/config.json @@ -110,5 +110,5 @@ "OIDC_GROUPS_CLAIM": "cognito:groups", "SESSION_COOKIE_DOMAIN": "nextstrain.org", "GROUPS_DATA_FILE": "groups.json", - "RESOURCE_INDEX": "s3://nextstrain-inventories/resources.json.gz" + "RESOURCE_INDEX": "./index.files.json.gz" } diff --git a/env/testing/config.json b/env/testing/config.json index d2e8f8c94..aacc81edd 100644 --- a/env/testing/config.json +++ b/env/testing/config.json @@ -108,5 +108,5 @@ "OIDC_USERNAME_CLAIM": "cognito:username", "OIDC_GROUPS_CLAIM": "cognito:groups", "GROUPS_DATA_FILE": "groups.json", - "RESOURCE_INDEX": "s3://nextstrain-inventories/resources.json.gz" + "RESOURCE_INDEX": "./index.files.json.gz" } diff --git a/index.files.json.gz b/index.files.json.gz new file mode 100644 index 000000000..3eebb87ad Binary files /dev/null and b/index.files.json.gz differ diff --git a/resourceIndexer/coreStagingS3.js b/resourceIndexer/coreStagingS3.js index 8fa033eae..5a8316b51 100644 --- a/resourceIndexer/coreStagingS3.js +++ b/resourceIndexer/coreStagingS3.js @@ -30,41 +30,61 @@ function categoriseCoreObjects(item, staging) { || key.startsWith('datasets_') ) return false; - // On the core bucket, directory-like hierarchies are used for intermediate - // files. These intermediate files may include files which auspice can - // display, but nextstrain.org cannot map URLs to directory-like hierarchies. - // There are other resourceTypes here we may consider in the future -- e.g. - // model output JSONs - if (key.includes("/")) { - if (staging===true) return false; - if (key.startsWith('files/')) { - if ( - key.includes('/archive/') - || key.includes('/test/') - || key.includes('/workflows/') - || key.includes('/branch/') - || key.includes('/trial/') - || key.includes('/test-data/') - || key.includes('jen_test/') - || key.match(/\/nextclade-full-run-[\d-]+--UTC\//) - || key.match(/\/\d{4}-\d{2}-\d{2}_results.json/) // forecasts-ncov - || key.endsWith('.png') // forecasts-ncov - ) { - return false; - } - item.resourceType = 'intermediate'; - /* The ID is used for grouping. For a nextstrain.org dataset this would be - combined with the source to form a nextstrain URL, however that's not - applicable here. Instead we use the filepath information without the - leading 'files/' and without the (trailing) filename so that different - files in the same directory structure get grouped together. For instance, - files/ncov/open/x.json -> ncov/open */ - item.resourcePath = key.split('/').slice(1, -1).join('/') - return item; + /* Intermediate files in the core bucket are many and varied, however we expect them + to follow the format specified in + At the moment we only consider "workflows", i.e. files in `/files/workflows/*` + as there are no "datasets" ("build files"?) intermediates. + The file name schema is: + /files + /workflows + {/workflow-repo} (matching github.com/nextstrain{/workflow-repo}) + {/arbitrary-structure*} + /metadata.tsv.zst (etc) + /sequences.fasta.zst (etc) + For the current listing we filter out any files where "/arbitrary-structure*" matches + some hardcoded list in an attempt to filter out test runs which we don't want to surface. + + We also include /files/ncov which predates the above structure design. + + The reported resource ID does not include the "/files/workflows" prefix. + + Redirects aren't considered when constructing the ID, so (e.g.) "monkeypox" and "mpox" are independent. + */ + const intermediateExcludePatterns = [ + /\/branch\//, + /\/test\//, + /\/trial\//, + /\/trials\//, + /\/nextclade-full-run[\d-]+--UTC\//, /* We could detail versions via the datestamped filename if desired */ + /\/\d{4}-\d{2}-\d{2}_results\.json/, // forecasts-ncov + /\.png$/, // forecasts-ncov + ] + if ((key.startsWith("files/workflows/") || key.startsWith("files/ncov/")) && staging===false) { + for (const pattern of intermediateExcludePatterns) { + if (key.match(pattern)) return false; } - return false; + item.resourceType = 'intermediate'; + /* The ID is used for grouping. For a nextstrain.org dataset this would be + combined with the source to form a nextstrain URL, however that's not + applicable here. Instead we use the filepath information without the + leading 'files/' and without the (trailing) filename so that different + files in the same directory structure get grouped together. For instance: + * files/ncov/open/100k/metadata.tsv.xz -> ncov/open/100k + * files/workflows/zika/sequences.fasta.zst -> zika + */ + item.resourcePath = key + .replace(/^files\/ncov\//, "ncov/") + .replace(/^files\/workflows\//, "") + .replace(/\/[^\/]+$/, '') + return item; } + /* All other files with a directory-like structure, including those on the + staging bucket, are ignored. Note that this removes files which don't conform + to the structure described above, as well as some files on the staging bucket. + */ + if (key.includes("/")) return false; + // Some filenames have a double underscore (presumably by mistake) if (key.includes('__')) return false; diff --git a/src/endpoints/listResources.js b/src/endpoints/listResources.js index b871896e5..9cab827f4 100644 --- a/src/endpoints/listResources.js +++ b/src/endpoints/listResources.js @@ -11,7 +11,7 @@ import { contentTypesProvided } from '../negotiate.js'; const listResourcesJson = async (req, res) => { /* API currently only handles a limited range of sources / resource types. ListResources will throw a HTTP error if they do not exist */ - const resourceType = 'dataset'; + const resourceType = req.params.resourceType; const sourceName = req.params.sourceName; const resources = new ListResources([sourceName], [resourceType]); const data = { diff --git a/src/resourceIndex.js b/src/resourceIndex.js index 6e32b5dc5..74519181d 100644 --- a/src/resourceIndex.js +++ b/src/resourceIndex.js @@ -132,6 +132,9 @@ async function updateResourceVersions() { * ListResources is intended to respond to resource listing queries. The current * implementation only handles a single source Id and single resource type, but * this will be extended as needed. + * + * There's definitely an inheritance structure here, but I haven't spent time to + * really draw it out. So instead of polymorphism we use conditionals. */ class ListResources { constructor(sourceIds, resourceTypes) { @@ -148,63 +151,87 @@ class ListResources { this.resourceType = resourceTypes[0]; } - coreDatasetFilter([name, ]) { - /* Consult the manifest to and restrict our listed resources to those whose - _first words_ appear as a top-level key the manifest. Subsequent words - aren't checked, so datasets may be returned which aren't explicitly defined - in the manifest. - - This is very similar to restricting based on the routing rules (e.g. using - `coreBuildPaths`) however the manifest is a subset of those and is used here - as the listed resources should be those for which we have added the pathogen - name to the manifest. - */ - if (!this._coreDatasetFirstWords) { - this._coreDatasetFirstWords = new Set( - global?.availableDatasets?.core?.map((path) => path.split("/")[0]) || [] - ); - } - return this._coreDatasetFirstWords.has(name.split("/")[0]) + filterFn() { + + // TODO XXX + const _coreDatasetFirstWords = new Set( + global?.availableDatasets?.core?.map((path) => path.split("/")[0]) || [] + ); + + const fn = ({ + dataset: { + core([name, ]) { + /* Consult the manifest to and restrict our listed resources to those whose + _first words_ appear as a top-level key the manifest. Subsequent words + aren't checked, so datasets may be returned which aren't explicitly defined + in the manifest. + + This is very similar to restricting based on the routing rules (e.g. using + `coreBuildPaths`) however the manifest is a subset of those and is used here + as the listed resources should be those for which we have added the pathogen + name to the manifest. + */ + return _coreDatasetFirstWords.has(name.split("/")[0]) + }, + staging() {return true;}, + }, + intermediate: { + core() {return true;}, + }, + })[this.resourceType][this.sourceId]; + if (fn!==undefined) return fn; + throw new InternalServerError(`Source "${this.sourceId}" + resource type "${this.resourceType} does not have a corresponding filter function`); } - pathPrefixBySource(name) { + pathPrefix() { /** * We separate out the "source part" from the "pathParts" part in our * routing logic, creating corresponding Source and Resource objects. Here * we go in the other direction. We could link the two approaches in the * future if it's felt this duplication is too brittle. + * + * Returns string | undefined */ - switch (name) { - case "core": - return "" - case "staging": - return "staging/" - default: - throw new InternalServerError(`Source "${name}" does not have a corresponding prefix`) - } + const prefix = ({ + dataset: { + core() {return "";}, + staging() {return "staging/";}, + }, + })?.[this.resourceType]?.[this.sourceId]?.(); + return prefix; } + pathVersions(_resources) { + const fn = ({ + dataset() { + return Object.entries(_resources).map(([name, data]) => { + return [name, data.versions.map((v) => v.date)]; + }) + }, + intermediate() { + return Object.entries(_resources).map(([name, data]) => { + return [name, Object.fromEntries((data.versions).map(({date, fileUrls}) => [date, fileUrls]))] // FIXME XXX + }); + }, + })[this.resourceType]; + if (!fn) throw new InternalServerError(`Resource type "${this.resourceType} does not have a path version extractor`); + return fn(); + } get data() { const _resources = resources?.[this.sourceId]?.[this.resourceType]; if (!_resources) { throw new NotFound(`No resources exist for the provided source-id / resource-type`); } - if (this.resourceType !== 'dataset') { - throw new InternalServerError(`Resource listing is currently only implemented for datasets`); - } - const pathVersions = Object.fromEntries( - Object.entries(_resources).map(([name, data]) => { - return [name, data.versions.map((v) => v.date)]; - }) - .filter((d) => this.sourceId==='core' ? this.coreDatasetFilter(d) : true) - ) - const d = {} - d[this.resourceType] = {} + const d = {}; + d[this.resourceType] = {}; d[this.resourceType][this.sourceId] = { - pathVersions, - pathPrefix: this.pathPrefixBySource(this.sourceId) - } + pathPrefix: this.pathPrefix(), + pathVersions: Object.fromEntries( + this.pathVersions(_resources) + .filter(this.filterFn()) + ) + }; return d; } } diff --git a/src/routing/listResources.js b/src/routing/listResources.js index 596843cb6..a66e59d3c 100644 --- a/src/routing/listResources.js +++ b/src/routing/listResources.js @@ -15,6 +15,6 @@ import {listResources} from '../endpoints/index.js'; * for some discussion about route name choices. */ export function setup(app) { - app.routeAsync("/list-resources/:sourceName") + app.routeAsync("/list-resources/:sourceName/:resourceType") .getAsync(listResources.listResources); } \ No newline at end of file diff --git a/static-site/pages/pathogens/files.jsx b/static-site/pages/pathogens/files.jsx new file mode 100644 index 000000000..0500d3699 --- /dev/null +++ b/static-site/pages/pathogens/files.jsx @@ -0,0 +1,3 @@ +import dynamic from 'next/dynamic' +const Index = dynamic(() => import("../../src/sections/core-files"), {ssr: false}) +export default Index; diff --git a/static-site/src/components/ListResources/IndividualResource.jsx b/static-site/src/components/ListResources/IndividualResource.jsx index 47b4c94af..a4793a567 100644 --- a/static-site/src/components/ListResources/IndividualResource.jsx +++ b/static-site/src/components/ListResources/IndividualResource.jsx @@ -37,7 +37,23 @@ export const ResourceLink = styled.a` text-decoration: none !important; `; +export const ResourceName = styled.span` + font-size: ${resourceFontSize}px; + font-family: monospace; + cursor: pointer; + white-space: pre; /* don't collapse back-to-back spaces */ + color: ${(props) => props.$hovered ? LINK_HOVER_COLOR : LINK_COLOR} !important; + text-decoration: none !important; +`; + function Name({displayName, $hovered, href, topOfColumn}) { + if (!href) { + return ( + + {'• '}{($hovered||topOfColumn) ? displayName.hovered : displayName.default} + + ) + } return ( {'• '}{($hovered||topOfColumn) ? displayName.hovered : displayName.default} @@ -106,24 +122,43 @@ export const IndividualResource = ({data, isMobile}) => { } }, []); + let summaryText; + if (data.versioned && !isMobile) { + summaryText = `${data.updateCadence.summary} (n=${data.nVersions})`; + if (data.fileCounts) { + const {min, max} = data.fileCounts; + if (min===max) { + summaryText += ` (${data.fileCounts.min} files)` + } else { + summaryText += ` (${data.fileCounts.min} - ${data.fileCounts.max} files)` + } + } + } + return ( - setModal(data)}> - - + { data.url ? ( + setModal(data)}> + + + ) : ( + setModal(data)}> + + + )} - {data.versioned && !isMobile && ( + {summaryText && ( Last known update on ${data.lastUpdated}` + `
${data.nVersions} snapshots of this dataset available (click to see them)`}> setModal(data)} /> @@ -140,17 +175,21 @@ export const IndividualResource = ({data, isMobile}) => { * Wrapper component which monitors for mouse-over events and injects a * `hovered: boolean` prop into the child. */ -export const ResourceLinkWrapper = ({children, onShiftClick}) => { +export const ResourceLinkWrapper = ({children, onClick, onShiftClick}) => { const [hovered, setHovered] = useState(false); - const onClick = (e) => { - if (e.shiftKey) { + const _onClick = (e) => { + if (e.shiftKey && onShiftClick) { onShiftClick(); e.preventDefault(); // child elements (e.g. ) shouldn't receive the click } + if (onClick) { + onClick(); + e.preventDefault(); // child elements (e.g. ) shouldn't receive the click + } }; return (
-
setHovered(true)} onMouseOut={() => setHovered(false)} onClick={onClick}> +
setHovered(true)} onMouseOut={() => setHovered(false)} onClick={_onClick}> {React.cloneElement(children, { $hovered: hovered })}
diff --git a/static-site/src/components/ListResources/ModalFiles.jsx b/static-site/src/components/ListResources/ModalFiles.jsx new file mode 100644 index 000000000..00b9117e6 --- /dev/null +++ b/static-site/src/components/ListResources/ModalFiles.jsx @@ -0,0 +1,413 @@ +/* eslint-disable react/prop-types */ +import React, {useEffect, useState, useCallback} from 'react'; +import styled from 'styled-components'; +import * as d3 from "d3"; +import { MdClose } from "react-icons/md"; +import { dodge } from "./dodge"; + +/** + * TODO XXX - this entire file is just Modal.jsx with some changes as I didn't + * want to be constrained and implement the conditional logic just yet. + */ + + +export const RAINBOW20 = ["#511EA8", "#4432BD", "#3F4BCA", "#4065CF", "#447ECC", "#4C91BF", "#56A0AE", "#63AC9A", "#71B486", "#81BA72", "#94BD62", "#A7BE54", "#BABC4A", "#CBB742", "#D9AE3E", "#E29E39", "#E68935", "#E56E30", "#E14F2A", "#DC2F24"]; +const lightGrey = 'rgba(0,0,0,0.1)'; + +export const ResourceModalFiles = ({data, dismissModal}) => { + const [ref, setRef] = useState(null); + const handleRef = useCallback((node) => {setRef(node)}, []) + + const [selectedDates, setSelectedDates] = useState([]) + + useEffect(() => { + const handleEsc = (event) => { + if (event.key === 'Escape') {dismissModal()} + }; + window.addEventListener('keydown', handleEsc); + return () => {window.removeEventListener('keydown', handleEsc);}; + }, [dismissModal]); + + const scrollRef = useCallback((node) => { + /* A CSS-only solution would be to set 'overflow: hidden' on . This + solution works well, but there are still ways to scroll (e.g. via down/up + arrows) */ + node?.addEventListener('wheel', (event) => {event.preventDefault();}, false); + }, []); + + useEffect(() => { + if (!ref || !data) return; + _draw(ref, data, setSelectedDates) + setSelectedDates(data.dates) + }, [ref, data]) + + if (!data) return null; + + console.log("selectedDates", selectedDates) + + const summary = _snapshotSummary(data.dates); + return ( +
+ + + {e.stopPropagation()}}> + + {data.name.replace(/\//g, "│")} + +
+ + {`${data.dates.length} sets of intermediate files spanning ${summary.duration}: ${summary.first} - ${summary.last}`} + +
+
+ {data.updateCadence.description} +
+ +

+ + +
{/* d3 controlled div */} + +
+ Each circle represents a day with a set of uploaded intermediate files. +
+ Click on a circle to filter the table to that day. +
+ Eventually the UI will use a date-slider like UI to select a range of days and filter the table accordingly +
+
setSelectedDates(data.dates)} style={{textDecoration: 'underline', cursor: 'pointer'}}> + Click here to reset the table +
+
+ + +
+ ) +} + +const Bold = styled.span` + font-weight: 500; +` + +const ModalContainer = styled.div` + position: fixed; + min-width: 80%; + max-width: 80%; + min-height: 80%; + left: 10%; + top: 10%; + background-color: #fff; + border: 2px solid black; + font-size: 18px; + padding: 20px; + z-index: 100; +`; + +const Background = styled.div` + position: fixed; + min-width: 100%; + min-height: 100%; + overflow: hidden; + left: 0; + top: 0; + background-color: ${lightGrey}; +` + +const CloseIcon = ({onClick}) => { + const [hovered, setHovered] = useState(false); + return ( +
{setHovered(true)}} onMouseOut={() => setHovered(false)} + > + +
+ ) +} + +const Title = styled.div` + font-size: 20px; + font-weight: 500; + padding-bottom: 15px; +` + +function _snapshotSummary(dates) { + const d = [...dates].sort() + const [d1, d2] = [d[0], d.at(-1)].map((di) => new Date(di)); + const days = (d2-d1)/1000/60/60/24; + let duration = ''; + if (days < 100) duration=`${days} days`; + else if (days < 365*2) duration=`${Math.round(days/(365/12))} months`; + else duration=`${Math.round(days/365)} years`; + return {duration, first: d[0], last:d.at(-1)}; +} + +function _draw(ref, data, setSelectedDates) { + /* Note that _page_ resizes by themselves will not result in this function + rerunning, which isn't great, but for a modal I think it's perfectly + acceptable */ + const sortedDateStrings = [...data.dates].sort(); + const flatData = sortedDateStrings.map((version) => ({version, 'date': new Date(version)})); + + const width = ref.clientWidth; + const graphIndent = 20; + const heights = { + height: 200, + marginTop: 20, + marginAboveAxis: 30, + marginBelowAxis: 50, + hoverAreaAboveAxis: 25, + marginBelowHoverBox: 10, + } + const unselectedOpacity = 0.3; + + + const selection = d3.select(ref); + selection.selectAll('*').remove() + const svg = selection + .append('svg') + .attr('width', width) + .attr('height', heights.height) + + + /* Create the x-scale and draw the x-axis */ + const x = d3.scaleTime() + .domain([flatData[0].date, new Date()]) // the domain extends to the present day + .range([graphIndent, width-graphIndent]) + svg.append('g') + .attr("transform", `translate(0, ${heights.height - heights.marginBelowAxis})`) + .call(d3.axisBottom(x).tickSize(15)) + .call((g) => {g.select(".domain").remove()}) + // .call((g) => {g.select(".domain").clone().attr("transform", `translate(0, -${heights.hoverAreaAboveAxis})`)}) + + + /** Elements which will be made visible on mouse-over interactions */ + const selectedVersionGroup = svg.append('g') + selectedVersionGroup.append("line") + .attr("class", "line") + .attr("x1", 0).attr("x2", 0) + .attr("y1", 100).attr("y2", heights.height - heights.marginBelowAxis) + .style("stroke", "black").style("stroke-width", "2") + .style("opacity", 0) + selectedVersionGroup.append("text") + .attr("class", "message") + .attr("x", 0) + .attr("y", heights.height - heights.marginBelowAxis - 6) // offset to bump text up + .style("font-size", "1.8rem") + .style("opacity", 0) + + + /** + * We use Observable Plot's `dodge` function to apply vertical jitter to the + * snapshot circles. To calculate the most appropriate radius we use a simple + * iterative approach. The current parameters mean the resulting radius will + * be bounded between 4px & 20px + */ + const availBeeswarmHeight = heights.height - heights.marginTop - heights.marginAboveAxis - heights.marginBelowAxis; + let radius = 12; + const padding = 1; + let nextRadius = radius; + let iterCount = 0; + let beeswarmData; + let beeswarmHeight = 0; + let spareHeight = availBeeswarmHeight-beeswarmHeight-radius; + const maxIter = 5; + const radiusJump = 2; + while (iterCount++ 50) { + const nextBeeswarmData = dodge(flatData, { + radius: nextRadius * 2 + padding, + x: (d) => x(d['date']) + }); + const nextBeeswarmHeight = d3.max(nextBeeswarmData.map((d) => d.y)); + const nextSpareHeight = availBeeswarmHeight-nextBeeswarmHeight-nextRadius; + if (nextSpareHeight <= spareHeight && nextSpareHeight > 0) { + beeswarmData = nextBeeswarmData; + beeswarmHeight = nextBeeswarmHeight; + spareHeight = nextSpareHeight; + radius = nextRadius; + nextRadius += radiusJump; + } else { + nextRadius -= radiusJump; + } + } + + /** draw the beeswarm plot */ + const beeswarm = svg.append("g") + .selectAll("circle") + .data(beeswarmData) + .join("circle") + .attr("cx", d => d.x) + .attr("cy", (d) => heights.height - heights.marginBelowAxis - heights.marginAboveAxis - radius - padding - d.y) + .attr("r", radius) + .attr('fill', color) + .on("mouseover", function(e, d) { + /* lower opacity of non-hovered, increase radius of hovered circle */ + beeswarm.join( + (enter) => {}, /* eslint-disable-line */ + (update) => selectSnapshot(update, d) + ) + /* update the vertical line + text which appears on hover */ + const selectedCircleX = d.x; + const textR = selectedCircleX*2 < width; + selectedVersionGroup.select(".line") + .attr("x1", selectedCircleX) + .attr("x2", selectedCircleX) + .attr("y1", heights.height - heights.marginBelowAxis - heights.marginAboveAxis - radius - padding - d.y) + .style("opacity", 1) + selectedVersionGroup.select(".message") + .attr("x", selectedCircleX + (textR ? 5 : -5)) + .style("opacity", 1) + .style("text-anchor", textR ? "start" : "end") + .text(`Intermediate files from ${prettyDate(d.data.version)} (click to filter)`) + }) + .on("mouseleave", function() { + beeswarm.join( + (enter) => {}, /* eslint-disable-line */ + (update) => resetBeeswarm(update) + ) + /* hide the vertical line + text which appeared on mouseover */ + selectedVersionGroup.selectAll("*") + .style("opacity", 0) + }) + .on("click", function(e, d) { + setSelectedDates([d.data.version]) + }) + + /** + * Draw the light-grey bar which doubles as the axis. The mouseover behaviour + * here is to select & show the appropriate snapshot relative to the mouse + * position + */ + svg.append("g") + .append("rect") + .attr('x', x.range()[0]) + .attr('y', heights.height - heights.marginBelowAxis - heights.hoverAreaAboveAxis) + .attr('width', x.range()[1] - x.range()[0]) + .attr('height', heights.hoverAreaAboveAxis) + .attr('stroke', 'black') + .attr('fill', lightGrey) + .on("mousemove", function(e) { + const { datum, hoveredDateStr } = getVersion(e); + beeswarm.join( + (enter) => {}, /* eslint-disable-line */ + (update) => selectSnapshot(update, datum) + ) + /* update the vertical line + text which appears on hover */ + const selectedCircleX = datum.x; + const textR = selectedCircleX*2 < width; + const prettyDates = prettyDate(hoveredDateStr, datum.data.version); + selectedVersionGroup.select(".line") + .attr("x1", selectedCircleX) + .attr("x2", selectedCircleX) + .attr("y1", heights.height - heights.marginBelowAxis - heights.marginAboveAxis - radius - padding - datum.y) + .style("opacity", 1) + selectedVersionGroup.select(".message") + .attr("x", selectedCircleX + (textR ? 5 : -5)) + .style("opacity", 1) + .style("text-anchor", textR ? "start" : "end") + .text(`On ${prettyDates[0]} the latest intermediate files were from ${prettyDates[1]} (click to filter)`) + }) + .on("mouseleave", function() { + beeswarm.join( + (enter) => {}, /* eslint-disable-line */ + (update) => resetBeeswarm(update) + ) + selectedVersionGroup.selectAll("*") + .style("opacity", 0) + }) + .on("click", function(e) { + const { datum } = getVersion(e); + setSelectedDates([datum.data.version]) + }) + + function selectSnapshot(selection, selectedDatum) { + selection // fn is almost identical to beeswarm mouseover + .attr("opacity", (d) => d===selectedDatum ? 1 : unselectedOpacity) + .call((selection) => selection.transition() + .delay(0).duration(150) + .attr("r", (d) => d===selectedDatum ? radius*2 : radius) + ) + } + + function resetBeeswarm(selection) { + selection + .attr("opacity", 1) + .call((update) => update.transition() + .delay(0).duration(150) + .attr("r", radius) + ) + } + + /** + * Given a mouse event in the x-axis' range, find the snapshot which would + * have been the latest at that point in time + */ + function getVersion(mouseEvent) { + const hoveredDate = x.invert(d3.pointer(mouseEvent)[0]); + const hoveredDateStr = _toDateString(hoveredDate); + const versionIdx = d3.bisect(sortedDateStrings, hoveredDateStr)-1; + const datum = beeswarmData[versionIdx]; + return {datum, hoveredDateStr} + } + + + const dateWithYear = d3.utcFormat("%B %d, %Y"); + const dateSameYear = d3.utcFormat("%B %d"); + function prettyDate(mainDate, secondDate) { + const d1 = dateWithYear(new Date(mainDate)); + if (!secondDate) return d1; + const d2 = (mainDate.slice(0,4)===secondDate.slice(0,4) ? dateSameYear : dateWithYear)(new Date(secondDate)); + return [d1, d2]; + } + + + /* Return the appropriate nextstrain rainbow colour of a circle via it's d.x + position relative to the x-axis' range */ + function color(d) { + const _xrange = x.range() + let idx = Math.floor((d.x - _xrange[0]) / (_xrange[1] - _xrange[0]) * RAINBOW20.length) + if (idx===RAINBOW20.length) idx--; + return RAINBOW20[idx]; + } + + function _toDateString(d) { + return `${d.getFullYear()}-${String(d.getMonth()+1).padStart(2,'0')}-${String(d.getDate()).padStart(2,'0')}` + } + +} + +const Table = ({fileUrls, dates}) => { + // fileUrls = date->name->url + const filenames = [... new Set(Object.values(fileUrls).map((o) => Object.keys(o)).flat())].sort(); + const datesReversed = dates.slice().reverse() + + return ( +
+ + + + {datesReversed.map((date) => ( + + ))} + + + + {filenames.map((filename) => ( + + + {datesReversed.map((date) => ( + + ))} + + ))} + +
Filename{date}
{filename} + {fileUrls?.[date]?.[filename] ? ( + link + ) : ( + {`❌`} + )} +
+ ) + +} \ No newline at end of file diff --git a/static-site/src/components/ListResources/index.jsx b/static-site/src/components/ListResources/index.jsx index c85a733ed..01de9e73f 100644 --- a/static-site/src/components/ListResources/index.jsx +++ b/static-site/src/components/ListResources/index.jsx @@ -12,6 +12,7 @@ import { ResourceGroup } from './ResourceGroup'; import { ErrorContainer } from "../../pages/404"; import { TooltipWrapper } from "./IndividualResource"; import {ResourceModal, SetModalContext} from "./Modal"; +import { ResourceModalFiles } from "./ModalFiles"; import { Showcase, useShowcaseCards} from "./Showcase"; /** @@ -24,6 +25,7 @@ import { Showcase, useShowcaseCards} from "./Showcase"; */ function ListResources({ sourceId, + resourceType='dataset', /* matches --resourceTypes for the indexer */ versioned=true, elWidth, quickLinks, @@ -31,7 +33,8 @@ function ListResources({ groupDisplayNames, /* mapping from group name -> display name */ showcase, /* showcase cards */ }) { - const [originalData, dataError] = useDataFetch(sourceId, versioned, defaultGroupLinks, groupDisplayNames); + const [originalData, dataError] = useDataFetch(sourceId, resourceType, versioned, defaultGroupLinks, groupDisplayNames); + console.log(originalData) const showcaseCards = useShowcaseCards(showcase, originalData); const [selectedFilterOptions, setSelectedFilterOptions] = useState([]); const [sortMethod, changeSortMethod] = useState("alphabetical"); @@ -84,8 +87,10 @@ function ListResources({ - { versioned && ( + { versioned && resourceType==='dataset' ? ( setModal(null)}/> + ) : ( + setModal(null)}/> )} diff --git a/static-site/src/components/ListResources/useDataFetch.js b/static-site/src/components/ListResources/useDataFetch.js index 924aaa735..aab39c997 100644 --- a/static-site/src/components/ListResources/useDataFetch.js +++ b/static-site/src/components/ListResources/useDataFetch.js @@ -1,6 +1,5 @@ import { useState, useEffect } from 'react'; - /** * Fetches the datasets for the provided `sourceId` and parses the data into an * array of groups, each representing a "pathogen" and detailing the available @@ -11,11 +10,11 @@ import { useState, useEffect } from 'react'; * response, however in the future we may shift this to the API response and it * may vary across the resources returned. */ -export function useDataFetch(sourceId, versioned, defaultGroupLinks, groupDisplayNames) { +export function useDataFetch(sourceId, resourceType, versioned, defaultGroupLinks, groupDisplayNames) { const [state, setState] = useState(undefined); const [error, setError] = useState(undefined); useEffect(() => { - const url = `/list-resources/${sourceId}`; + const url = `/list-resources/${sourceId}/${resourceType}`; async function fetchAndParse() { let pathVersions, pathPrefix; @@ -25,7 +24,7 @@ export function useDataFetch(sourceId, versioned, defaultGroupLinks, groupDispla console.error(`ERROR: fetching data from "${url}" returned status code ${response.status}`); return setError(true); } - ({ pathVersions, pathPrefix } = (await response.json()).dataset[sourceId]); + ({ pathVersions, pathPrefix } = (await response.json())[resourceType][sourceId]); } catch (err) { console.error(`Error while fetching data from "${url}"`); console.error(err); @@ -36,7 +35,7 @@ export function useDataFetch(sourceId, versioned, defaultGroupLinks, groupDispla resource path). This grouping is constant for all UI options so we do it a single time following the data fetch */ try { - const partitions = partitionByPathogen(pathVersions, pathPrefix, versioned); + const partitions = partitionByPathogen(resourceType, pathVersions, pathPrefix, versioned); setState(groupsFrom(partitions, pathPrefix, defaultGroupLinks, groupDisplayNames)); } catch (err) { console.error(`Error while parsing fetched data`); @@ -46,7 +45,7 @@ export function useDataFetch(sourceId, versioned, defaultGroupLinks, groupDispla } fetchAndParse(); - }, [sourceId, versioned, defaultGroupLinks, groupDisplayNames]); + }, [sourceId, resourceType, versioned, defaultGroupLinks, groupDisplayNames]); return [state, error] } @@ -56,27 +55,37 @@ export function useDataFetch(sourceId, versioned, defaultGroupLinks, groupDispla * representing group names (pathogen names) and values which are arrays of * resource objects. */ -function partitionByPathogen(pathVersions, pathPrefix, versioned) { +function partitionByPathogen(resourceType, pathVersions, pathPrefix, versioned) { return Object.entries(pathVersions).reduce(reduceFn, {}); function reduceFn(store, datum) { const name = datum[0]; const nameParts = name.split('/'); - const sortedDates = [...datum[1]].sort(); + const sortedDates = resourceType==='dataset' ? + [...datum[1]].sort() : + [...Object.keys(datum[1])].sort(); let groupName = nameParts[0]; if (!store[groupName]) store[groupName] = [] + const resourceDetails = { name, groupName, /* decoupled from nameParts */ nameParts, sortingName: _sortableName(nameParts), - url: `/${pathPrefix}${name}`, lastUpdated: sortedDates.at(-1), versioned }; + if (resourceType==='dataset') { + resourceDetails.url = `/${pathPrefix}${name}`; + } else if (resourceType==='intermediate') { + resourceDetails.fileUrls = datum[1]; + const fileCounts = Object.values(resourceDetails.fileUrls) + .map((fileUrls) => Object.keys(fileUrls).length); + resourceDetails.fileCounts = {min: Math.min(...fileCounts), max: Math.max(...fileCounts)}; + } if (versioned) { resourceDetails.firstUpdated = sortedDates[0]; resourceDetails.lastUpdated = sortedDates.at(-1); diff --git a/static-site/src/sections/core-files.jsx b/static-site/src/sections/core-files.jsx new file mode 100644 index 000000000..8d5c20019 --- /dev/null +++ b/static-site/src/sections/core-files.jsx @@ -0,0 +1,39 @@ +import React from "react"; +import { + SmallSpacer, + HugeSpacer, + FlexCenter, +} from "../layouts/generalComponents"; +import ListResources from "../components/ListResources/index"; +import * as splashStyles from "../components/splash/styles"; +import GenericPage from "../layouts/generic-page"; + +const title = "Nextstrain-maintained core files"; +const abstract = ( + <> + WORK IN PROGRESS - core (intermediate) file listing + +); + +class Index extends React.Component { + render() { + return ( + + {title} + + + + + {abstract} + + + + + + + + ); + } +} + +export default Index;