Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create and use a standard utility library for handling zip files in the frontend #11539

Merged
merged 9 commits into from
Jan 2, 2024
Merged

Create and use a standard utility library for handling zip files in the frontend #11539

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
95925d8
Create generic fflate wrapper utility.
rtibbles Nov 3, 2023
2bdcd4b
Simplify zip file loading to only need a URL.
rtibbles Nov 4, 2023
00df0d2
Move blob url creation into kolibri-zip package.
rtibbles Nov 4, 2023
2efb025
Migrate file path substitution logic into kolibri-zip package.
rtibbles Nov 14, 2023
d559cc3
Extend mimetypes.
rtibbles Nov 15, 2023
de14043
Create mapper class.
rtibbles Nov 15, 2023
9bdec4b
Add href substitution to xml and html unzipping.
rtibbles Nov 16, 2023
8336bbb
Add url caching and cleanup behaviour to kolibri-zip utility.
rtibbles Nov 17, 2023
122d1e9
Update perseus renderer to use kolibri-zip functionality.
rtibbles Nov 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
212 changes: 81 additions & 131 deletions kolibri/plugins/perseus_viewer/assets/src/views/PerseusRendererIndex.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@
<script>

import invert from 'lodash/invert';
import JSZip from 'jszip';
import client from 'kolibri.client';
import ZipFile from 'kolibri-zip';
import { Mapper, defaultFilePathMappers } from 'kolibri-zip/src/fileUtils';
import urls from 'kolibri.urls';
import useKResponsiveWindow from 'kolibri-design-system/lib/useKResponsiveWindow';
import { isTouchDevice } from 'kolibri.utils.browserInfo';
Expand Down Expand Up @@ -124,6 +124,67 @@

const blobImageRegex = /blob:[^)^"]+/g;

Khan.imageUrls = {};

function getImagePaths(itemResponse) {
const graphieMatches = {};
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's graphie? Is this a perseus specific image term?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes - it allows loading of either a JSON blob or an SVG file to display graph style images.

const imageMatches = {};
const matches = Array.from(itemResponse.matchAll(allImageRegex));

for (let i = 0; i < matches.length; i++) {
const match = matches[i];
if (match[1]) {
// We have a match for the optional web+graphie matching group
graphieMatches[match[3]] = true;
} else {
imageMatches[match[3]] = true;
}
}
const graphieImages = Object.keys(graphieMatches);
const images = Object.keys(imageMatches);
const svgAndJson = graphieImages.reduce(
(acc, image) => [...acc, `${image}.svg`, `${image}-data.json`],
[]
);
return images.concat(svgAndJson);
}

function replaceImageUrls(itemResponse, packageFiles) {
Object.assign(Khan.imageUrls, packageFiles);
// If the file is not present in the zip file, then fill in a missing image
// file for images, and an empty dummy json file for json
return itemResponse.replace(allImageRegex, (match, g1, g2, image) => {
if (g1) {
// Replace any placeholder values for image URLs with the
// `web+graphie:` prefix separately from any others,
// as they are parsed slightly differently to standard image
// urls (Perseus adds the protocol in place of `web+graphie:`).
if (!Khan.imageUrls[image]) {
Khan.imageUrls[image] = 'data:application/json,';
}
return `web+graphie:${image}`;
} else {
// Replace any placeholder values for image URLs with
// the base URL for the perseus file we are reading from
return packageFiles[image] || imageMissing;
}
});
}

class JSONMapper extends Mapper {
getPaths() {
return getImagePaths(this.file.toString());
}
replacePaths(packageFiles) {
return replaceImageUrls(this.file.toString(), packageFiles);
}
}

const filePathMappers = {
...defaultFilePathMappers,
json: JSONMapper,
};

export default {
name: 'PerseusRendererIndex',
setup() {
Expand Down Expand Up @@ -209,15 +270,14 @@
},

beforeDestroy() {
this.clearItemRenderer();
this.$emit('stopTracking');
this.clearItemRenderer();
if (this.perseusFile) {
this.perseusFile.close();
}
},
created() {
this.perseusFile = null;
this.imageUrls = {};
// Make a global reference for this object
// for access inside perseus.
Khan.imageUrls = this.imageUrls;
const initPromise = mathJaxPromise.then(() =>
perseus.init({ skipMathJax: true, loadExtraWidgets: true })
);
Expand Down Expand Up @@ -313,12 +373,7 @@
} catch (e) {
logging.debug('Error during unmounting of item renderer', e);
}
for (const key in this.imageUrls) {
if (this.imageUrls[key].indexOf('blob:') === 0) {
URL.revokeObjectURL(this.imageUrls[key]);
}
delete this.imageUrls[key];
}
Khan.imageUrls = {};
},
/*
* Special method to extract the current state of a Perseus Sorter widget
Expand Down Expand Up @@ -355,7 +410,7 @@
return this.restoreImageUrls({ hints, question });
},
restoreSerializedState(answerState) {
answerState = this.replaceImageUrls(JSON.stringify(answerState));
answerState = JSON.parse(replaceImageUrls(JSON.stringify(answerState)));
this.itemRenderer.restoreSerializedState(answerState);
this.itemRenderer.getWidgetIds().forEach(id => {
if (sorterWidgetRegex.test(id)) {
Expand Down Expand Up @@ -432,109 +487,21 @@
// dismiss the error message when user click anywhere inside the perseus element.
this.message = null;
},
loadPerseusFile() {
if (this.defaultFile && this.defaultFile.storage_url) {
this.loading = true;
if (!this.perseusFile || this.perseusFileUrl !== this.defaultFile.storage_url) {
return client({
method: 'get',
url: this.defaultFile.storage_url,
responseType: 'arraybuffer',
})
.then(response => {
return JSZip.loadAsync(response.data);
})
.then(perseusFile => {
this.perseusFile = perseusFile;
this.perseusFileUrl = this.defaultFile.storage_url;
})
.catch(err => {
logging.error('Error loading Perseus file', err);
this.reportLoadingError(err);
return Promise.reject(err);
});
} else {
return Promise.resolve();
}
}
},
loadItemData() {
// Only try to do this if itemId is defined.
if (this.itemId && this.defaultFile && this.defaultFile.storage_url) {
this.loading = true;
this.loadPerseusFile()
.then(() => {
const itemDataFile = this.perseusFile.file(`${this.itemId}.json`);
if (itemDataFile) {
return itemDataFile.async('string');
}
return Promise.reject(`item data for ${this.itemId} not found`);
})
.then(itemResponse => {
const graphieMatches = {};
const imageMatches = {};
const matches = Array.from(itemResponse.matchAll(allImageRegex));

for (let i = 0; i < matches.length; i++) {
const match = matches[i];
if (match[1]) {
// We have a match for the optional web+graphie matching group
graphieMatches[match[3]] = true;
} else {
imageMatches[match[3]] = true;
}
}

const graphieImages = Object.keys(graphieMatches);
const images = Object.keys(imageMatches);

const processFile = file => {
if (!this.imageUrls[file]) {
const fileData = this.perseusFile.file(file);
const ext = file.split('.').slice(-1)[0];
if (fileData) {
return fileData.async('arraybuffer').then(buffer => {
let type;
if (ext === 'json') {
type = 'application/json';
} else if (ext === 'svg') {
type = 'image/svg+xml';
} else {
type = `image/${ext}`;
}
const blob = new Blob([buffer], { type });
this.imageUrls[file] = URL.createObjectURL(blob);
});
} else {
// If the file is not present in the zip file, then fill in a missing image
// file for images, and an empty dummy json file for json
let url;
if (ext === 'json') {
url = 'data:application/json,';
} else {
url = imageMissing;
}
this.imageUrls[file] = url;
}
}
return Promise.resolve();
};

const promises = images.map(processFile).concat(
graphieImages.map(image => {
const svgFile = `${image}.svg`;
const jsonFile = `${image}-data.json`;
return Promise.all([processFile(svgFile), processFile(jsonFile)]);
})
);

return Promise.all(promises)
.catch(() => {
return Promise.reject('error loading assessment item images');
})
.then(() => {
this.setItemData(this.replaceImageUrls(itemResponse));
});
if (!this.perseusFile || this.perseusFileUrl !== this.defaultFile.storage_url) {
this.perseusFile = new ZipFile(this.defaultFile.storage_url, {
filePathMappers,
});
this.perseusFileUrl = this.defaultFile.storage_url;
}
this.perseusFile
.file(`${this.itemId}.json`)
.then(itemFile => {
const itemResponse = itemFile.toString();
this.setItemData(JSON.parse(itemResponse));
})
.catch(reason => {
logging.debug('There was an error loading the assessment item data: ', reason);
Expand All @@ -543,25 +510,8 @@
});
}
},
replaceImageUrls(itemResponse) {
return JSON.parse(
itemResponse.replace(allImageRegex, (match, g1, g2, image) => {
if (g1) {
// Replace any placeholder values for image URLs with the
// `web+graphie:` prefix separately from any others,
// as they are parsed slightly differently to standard image
// urls (Perseus adds the protocol in place of `web+graphie:`).
return `web+graphie:${image}`;
} else {
// Replace any placeholder values for image URLs with
// the base URL for the perseus file we are reading from
return this.imageUrls[image] || imageMissing;
}
})
);
},
restoreImageUrls(itemResponse) {
const lookup = invert(this.imageUrls);
const lookup = invert(Khan.imageUrls);
return JSON.parse(
JSON.stringify(itemResponse).replace(blobImageRegex, match => {
// Make sure to add our prefix back in
Expand Down
1 change: 0 additions & 1 deletion kolibri/plugins/perseus_viewer/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
"fractional": "^1.0.0",
"immutable": "^3.8.1",
"jquery": "2.2.4",
"jszip": "^3.10.1",
"kmath": "^0.0.1",
"qtip2": "2.2.0",
"raphael": "^2.2.7",
Expand Down
3 changes: 0 additions & 3 deletions packages/hashi/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
"build": "yarn run build-base --mode=production",
"dev": "yarn run build-base --mode=development --watch",
"compat": "eslint -c ./compat.js ./src/*.js",
"mimetypes": "node ./generateH5PMimeTypeDB.js",
"build-h5p": "node ./downloadH5PVendor.js && webpack --config ./webpack.config.h5p.js --mode=production"
},
"author": "Learning Equality",
Expand All @@ -17,14 +16,12 @@
"eslint-plugin-compat": "^4.2.0",
"html-webpack-plugin": "5.5.3",
"jquery": "3.5.1",
"mime-db": "^1.52.0",
"mutationobserver-shim": "^0.3.7",
"purgecss": "^5.0.0"
},
"dependencies": {
"core-js": "3.33",
"dayjs": "^1.11.10",
"fflate": "^0.8.1",
"iri": "^1.3.1",
"is-language-code": "^3.1.0",
"iso8601-duration": "^2.1.1",
Expand Down
Loading