Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use TinySDF to render CJK glyphs locally #4895

Merged
merged 5 commits into from
Jul 5, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions debug/tinysdf.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
<!DOCTYPE html>
<html>
<head>
<title>Mapbox GL JS debug page</title>
<meta charset='utf-8'>
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
<link rel='stylesheet' href='/dist/mapbox-gl.css' />
<style>
body { margin: 0; padding: 0; }
html, body, #originalMap, #newMap { height: 100%; }
#originalMap, #newMap { width: 49%; }

#originalLabel {
position: absolute;
background: #fff;
top:0;
left:0;
padding:10px;
}

#newLabel {
position: absolute;
background: #fff;
top:0;
right:0;
padding:10px;
}

#checkboxes {
position: absolute;
background: #fff;
bottom:35px;
left:0;
padding:10px;
}

</style>
</head>

<body>

<div id='originalMap' style="float: left"></div>
<div id='newMap' style="float: right"></div>
<label id='originalLabel'>Original Map</label>
<label id='newLabel'>New Map</label>
<div id='checkboxes'>
<label><input id='show-overdraw-checkbox' type='checkbox'> overdraw debug</label><br />
</div>

<script src='/dist/mapbox-gl-dev.js'></script>
<script src='/debug/access_token_generated.js'></script>
<script>

function localizeLayers(map) {
map.on('load', function() {
var style = map.getStyle();
for (var layerID in style.layers) {
var layer = style.layers[layerID];
if (layer.layout && layer.layout['text-field'] === "{name_en}") {
layer.layout['text-field'] = "{name}";
}
}
map.setStyle(style, map.style.glyphSource.localIdeographFontFamily);
});
}

var originalMap = window.originalMap = new mapboxgl.Map({
container: 'originalMap',
zoom: 8.8,
center: [121.574, 31.1489],
style: 'mapbox://styles/mapbox/streets-v10',
hash: true
});

localizeLayers(originalMap);

var newMap = window.newMap = new mapboxgl.Map({
container: 'newMap',
zoom: 8.8,
center: [121.574, 31.1489],
style: 'mapbox://styles/mapbox/streets-v10',
localIdeographFontFamily: '"Noto Sans", "Noto Sans CJK SC", sans-serif',
hash: true
});

localizeLayers(newMap);

document.getElementById('show-overdraw-checkbox').onclick = function() {
originalMap.showOverdrawInspector = !!this.checked;
newMap.showOverdrawInspector = !!this.checked;
};


</script>
</body>
</html>
21 changes: 21 additions & 0 deletions docs/_posts/examples/3400-02-01-local-ideographs.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
---
layout: example
category: example
title: Use locally generated ideographs
description: Rendering Chinese/Japanese/Korean (CJK) ideographs and precomposed Hangul Syllables requires downloading large amounts of font data, which can significantly slow map load times. Use the 'localIdeographFontFamily' setting to speed up map load times by using locally available fonts instead of font data fetched from the server. This setting defines a CSS 'font-family' for locally overriding generation of glyphs in the 'CJK Unified Ideographs' and 'Hangul Syllables' Unicode ranges. In these ranges, font settings from the map's style will be ignored in favor of the locally available font. Keywords in the fontstack defined in the map's style (light/regular/medium/bold) will be translated into a CSS 'font-weight'. When using this setting, keep in mind that the fonts you select may not be available on all users' devices. It is best to specify at least one broadly available fallback font class such as 'sans-serif'.
tags:
- internationalization
---
<div id='map'></div>

<script>

var map = new mapboxgl.Map({
container: 'map',
style: 'mapbox://styles/mapbox/streets-v9',
center: [120.3049, 31.4751],
zoom: 12,
localIdeographFontFamily: "'Noto Sans', 'Noto Sans CJK SC', sans-serif"
});

</script>
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@
"unflowify": "^1.0.0",
"vector-tile": "^1.3.0",
"vt-pbf": "^2.0.2",
"webworkify": "^1.4.0"
"webworkify": "^1.4.0",
"@mapbox/tiny-sdf": "^1.1.0"
},
"devDependencies": {
"@mapbox/mapbox-gl-rtl-text": "^0.1.1",
Expand Down
2 changes: 1 addition & 1 deletion src/style/style.js
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ class Style extends Evented {
this.sprite = new ImageSprite(stylesheet.sprite, this);
}

this.glyphSource = new GlyphSource(stylesheet.glyphs);
this.glyphSource = new GlyphSource(stylesheet.glyphs, options.localIdeographFontFamily);
this._resolve();
this.fire('data', {dataType: 'style'});
this.fire('style.load');
Expand Down
79 changes: 65 additions & 14 deletions src/symbol/glyph_source.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ const verticalizePunctuation = require('../util/verticalize_punctuation');
const Glyphs = require('../util/glyphs');
const GlyphAtlas = require('../symbol/glyph_atlas');
const Protobuf = require('pbf');
const TinySDF = require('@mapbox/tiny-sdf');
const isChar = require('../util/is_char_in_unicode_block');

// A simplified representation of the glyph containing only the properties needed for shaping.
class SimpleGlyph {
Expand All @@ -28,16 +30,18 @@ class GlyphSource {
/**
* @param {string} url glyph template url
*/
constructor(url) {
constructor(url, localIdeographFontFamily) {
this.url = url && normalizeURL(url);
this.atlases = {};
this.stacks = {};
this.loading = {};
this.localIdeographFontFamily = localIdeographFontFamily;
this.tinySDFs = {};
}

getSimpleGlyphs(fontstack, glyphIDs, uid, callback) {
if (this.stacks[fontstack] === undefined) {
this.stacks[fontstack] = {};
this.stacks[fontstack] = { ranges: {}, cjkGlyphs: {} };
}
if (this.atlases[fontstack] === undefined) {
this.atlases[fontstack] = new GlyphAtlas();
Expand All @@ -50,23 +54,37 @@ class GlyphSource {
// the number of pixels the sdf bitmaps are padded by
const buffer = 3;

const missing = {};
const missingRanges = {};
let remaining = 0;

const getGlyph = (glyphID) => {
const range = Math.floor(glyphID / 256);
if (this.localIdeographFontFamily &&
// eslint-disable-next-line new-cap
(isChar['CJK Unified Ideographs'](glyphID) ||
// eslint-disable-next-line new-cap
isChar['Hangul Syllables'](glyphID))) {
if (!stack.cjkGlyphs[glyphID]) {
stack.cjkGlyphs[glyphID] = this.loadCJKGlyph(fontstack, glyphID);
}

if (stack[range]) {
const glyph = stack[range].glyphs[glyphID];
const glyph = stack.cjkGlyphs[glyphID];
const rect = atlas.addGlyph(uid, fontstack, glyph, buffer);
if (glyph) glyphs[glyphID] = new SimpleGlyph(glyph, rect, buffer);
} else {
if (missing[range] === undefined) {
missing[range] = [];
remaining++;
if (stack.ranges[range]) {
const glyph = stack.ranges[range].glyphs[glyphID];
const rect = atlas.addGlyph(uid, fontstack, glyph, buffer);
if (glyph) glyphs[glyphID] = new SimpleGlyph(glyph, rect, buffer);
} else {
if (missingRanges[range] === undefined) {
missingRanges[range] = [];
remaining++;
}
missingRanges[range].push(glyphID);
}
missing[range].push(glyphID);
}
/* eslint-enable new-cap */
};

for (let i = 0; i < glyphIDs.length; i++) {
Expand All @@ -82,9 +100,9 @@ class GlyphSource {

const onRangeLoaded = (err, range, data) => {
if (!err) {
const stack = this.stacks[fontstack][range] = data.stacks[0];
for (let i = 0; i < missing[range].length; i++) {
const glyphID = missing[range][i];
const stack = this.stacks[fontstack].ranges[range] = data.stacks[0];
for (let i = 0; i < missingRanges[range].length; i++) {
const glyphID = missingRanges[range][i];
const glyph = stack.glyphs[glyphID];
const rect = atlas.addGlyph(uid, fontstack, glyph, buffer);
if (glyph) glyphs[glyphID] = new SimpleGlyph(glyph, rect, buffer);
Expand All @@ -94,11 +112,44 @@ class GlyphSource {
if (!remaining) callback(undefined, glyphs, fontstack);
};

for (const r in missing) {
for (const r in missingRanges) {
this.loadRange(fontstack, r, onRangeLoaded);
}
}

createTinySDF(fontFamily, fontWeight) {
return new TinySDF(24, 3, 8, .25, fontFamily, fontWeight);
}

loadCJKGlyph(fontstack, glyphID) {
let tinySDF = this.tinySDFs[fontstack];
if (!tinySDF) {
let fontWeight = '400';
if (/bold/i.test(fontstack)) {
fontWeight = '900';
} else if (/medium/i.test(fontstack)) {
fontWeight = '500';
} else if (/light/i.test(fontstack)) {
fontWeight = '200';
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would move the fontWeight checks up to getSimpleGlyphs so that they're not repeated unnecessarily. Also, you could move out regexps into top-level constants (const boldRe = /bold/i) so that they're not recreated on every check.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These checks are only done once per unique fontstack, so probably only a handful of times.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, you're right. Missed that.

tinySDF = this.tinySDFs[fontstack] = this.createTinySDF(this.localIdeographFontFamily, fontWeight);
}

return {
id: glyphID,
bitmap: tinySDF.draw(String.fromCharCode(glyphID)),
width: 24,
height: 24,
left: 0,
top: -8,
advance: 24
};
}

loadPBF(url, callback) {
ajax.getArrayBuffer(url, callback);
}

loadRange(fontstack, range, callback) {
if (range * 256 > 65535) return callback('glyphs > 65535 not supported');

Expand All @@ -115,7 +166,7 @@ class GlyphSource {
const rangeName = `${range * 256}-${range * 256 + 255}`;
const url = glyphUrl(fontstack, rangeName, this.url);

ajax.getArrayBuffer(url, (err, response) => {
this.loadPBF(url, (err, response) => {
const glyphs = !err && new Glyphs(new Protobuf(response.data));
for (let i = 0; i < loading[range].length; i++) {
loading[range][i](err, range, glyphs);
Expand Down
17 changes: 12 additions & 5 deletions src/ui/map.js
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,11 @@ const defaultOptions = {
* @param {number} [options.bearing=0] The initial bearing (rotation) of the map, measured in degrees counter-clockwise from north. If `bearing` is not specified in the constructor options, Mapbox GL JS will look for it in the map's style object. If it is not specified in the style, either, it will default to `0`.
* @param {number} [options.pitch=0] The initial pitch (tilt) of the map, measured in degrees away from the plane of the screen (0-60). If `pitch` is not specified in the constructor options, Mapbox GL JS will look for it in the map's style object. If it is not specified in the style, either, it will default to `0`.
* @param {boolean} [options.renderWorldCopies=true] If `true`, multiple copies of the world will be rendered, when zoomed out.
* @param {number} [options.maxTileCacheSize=null] The maxiumum number of tiles stored in the tile cache for a given source. If omitted, the cache will be dynamically sized based on the current viewport.
* @param {number} [options.maxTileCacheSize=null] The maxiumum number of tiles stored in the tile cache for a given source. If omitted, the cache will be dynamically sized based on the current viewport.
* @param {string} [options.localIdeographFontFamily=null] If specified, defines a CSS font-family
* for locally overriding generation of glyphs in the 'CJK Unified Ideographs' and 'Hangul Syllables' ranges.
* In these ranges, font settings from the map's style will be ignored, except for font-weight keywords (light/regular/medium/bold).
* The purpose of this option is to avoid bandwidth-intensive glyph server requests. (see [Use locally generated ideographs](https://www.mapbox.com/mapbox-gl-js/example/local-ideographs))
* @example
* var map = new mapboxgl.Map({
* container: 'map',
Expand Down Expand Up @@ -293,7 +297,7 @@ class Map extends Camera {
this.resize();

if (options.classes) this.setClasses(options.classes);
if (options.style) this.setStyle(options.style);
if (options.style) this.setStyle(options.style, { localIdeographFontFamily: options.localIdeographFontFamily });

if (options.attributionControl) this.addControl(new AttributionControl());
this.addControl(new LogoControl(), options.logoPosition);
Expand Down Expand Up @@ -928,11 +932,14 @@ class Map extends Camera {
* @param {Object} [options]
* @param {boolean} [options.diff=true] If false, force a 'full' update, removing the current style
* and adding building the given one instead of attempting a diff-based update.
* @param {string} [options.localIdeographFontFamily=null] If non-null, defines a css font-family
* for locally overriding generation of glyphs in the 'CJK Unified Ideographs' and 'Hangul Syllables'
* ranges. Forces a full update.
* @returns {Map} `this`
* @see [Change a map's style](https://www.mapbox.com/mapbox-gl-js/example/setstyle/)
*/
setStyle(style: any, options?: {diff: boolean}) {
const shouldTryDiff = (!options || options.diff !== false) && this.style && style &&
setStyle(style: any, options?: {diff?: boolean, localIdeographFontFamily?: string}) {
const shouldTryDiff = (!options || (options.diff !== false && !options.localIdeographFontFamily)) && this.style && style &&
!(style instanceof Style) && typeof style !== 'string';
if (shouldTryDiff) {
try {
Expand All @@ -959,7 +966,7 @@ class Map extends Camera {
} else if (style instanceof Style) {
this.style = style;
} else {
this.style = new Style(style, this);
this.style = new Style(style, this, options);
}

this.style.setEventedParent(this, {style: this.style});
Expand Down
Binary file added test/fixtures/0-255.pbf
Binary file not shown.
59 changes: 59 additions & 0 deletions test/unit/symbol/glyph_source.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
'use strict';

const test = require('mapbox-gl-js-test').test;
const GlyphSource = require('../../../src/symbol/glyph_source');
const fs = require('fs');

const mockTinySDF = {
// Return empty 30x30 bitmap (24 fontsize + 3 * 2 buffer)
draw: function () { return new Uint8ClampedArray(900); }
};

function createSource(t, localIdeographFontFamily) {
const aPBF = fs.readFileSync('./test/fixtures/0-255.pbf');
const source = new GlyphSource("https://localhost/fonts/v1{fontstack}/{range}.pbf", localIdeographFontFamily);
t.stub(source, 'createTinySDF').returns(mockTinySDF);
// It would be better to mock with FakeXMLHttpRequest, but the binary encoding
// doesn't survive the mocking
source.loadPBF = function(url, callback) {
callback(null, { data: aPBF });
};

return source;
}


test('GlyphSource', (t) => {
t.test('requests 0-255 PBF', (t) => {
const source = createSource(t);
source.getSimpleGlyphs("Arial Unicode MS", [55], 0, (err, glyphs, fontName) => {
t.notOk(err);
t.equal(fontName, "Arial Unicode MS");
t.equal(glyphs['55'].advance, 12);
t.end();
});
});

t.test('requests remote CJK PBF', (t) => {
const source = createSource(t);
source.getSimpleGlyphs("Arial Unicode MS", [0x5e73], 0, (err, glyphs, fontName) => {
t.notOk(err);
t.equal(fontName, "Arial Unicode MS");
t.notOk(Object.keys(glyphs).length); // The fixture returns a PBF without the glyph we requested
t.end();
});

});

t.test('locally generates CJK PBF', (t) => {
const source = createSource(t, 'sans-serif');
source.getSimpleGlyphs("Arial Unicode MS", [0x5e73], 0, (err, glyphs, fontName) => {
t.notOk(err);
t.equal(fontName, "Arial Unicode MS");
t.equal(glyphs['24179'].advance, 24);
t.end();
});
});

t.end();
});
4 changes: 4 additions & 0 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@
version "3.0.0"
resolved "https://registry.yarnpkg.com/@mapbox/shelf-pack/-/shelf-pack-3.0.0.tgz#44e284c8336eeda1e9dbbb1d61954c70e26e5766"

"@mapbox/tiny-sdf@^1.1.0":
version "1.1.0"
resolved "https://registry.yarnpkg.com/@mapbox/tiny-sdf/-/tiny-sdf-1.1.0.tgz#b0b8f5c22005e6ddb838f421ffd257c1f74f9a20"

"@mapbox/unitbezier@^0.0.0":
version "0.0.0"
resolved "https://registry.yarnpkg.com/@mapbox/unitbezier/-/unitbezier-0.0.0.tgz#15651bd553a67b8581fb398810c98ad86a34524e"
Expand Down