Skip to content
This repository has been archived by the owner on Aug 15, 2019. It is now read-only.

remove content type checks #1532

Merged
merged 5 commits into from
Feb 5, 2019
Merged

remove content type checks #1532

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
53fa528
remove content type checks
pyu10055 Feb 4, 2019
f709f89
updated the constants name
pyu10055 Feb 4, 2019
ed50d19
removed accept headers
pyu10055 Feb 4, 2019
1d1f1e4
Merge branch 'master' into remove_content_type_check
pyu10055 Feb 5, 2019
86a6d92
Merge branch 'master' into remove_content_type_check
pyu10055 Feb 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 8 additions & 21 deletions src/io/browser_http.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ import {IORouter, IORouterRegistry} from './router_registry';
import {IOHandler, ModelArtifacts, SaveResult, WeightsManifestConfig, WeightsManifestEntry} from './types';
import {loadWeightsAsArrayBuffer} from './weights_loader';

const OCTET_STREAM_TYPE = 'application/octet-stream';
const JSON_TYPE = 'application/json';

export class BrowserHTTPRequest implements IOHandler {
protected readonly path: string|string[];
protected readonly requestInit: RequestInit;
Expand Down Expand Up @@ -109,14 +112,13 @@ export class BrowserHTTPRequest implements IOHandler {
'model.json',
new Blob(
[JSON.stringify(modelTopologyAndWeightManifest)],
{type: 'application/json'}),
{type: JSON_TYPE}),
'model.json');

if (modelArtifacts.weightData != null) {
init.body.append(
'model.weights.bin',
new Blob(
[modelArtifacts.weightData], {type: 'application/octet-stream'}),
new Blob([modelArtifacts.weightData], {type: OCTET_STREAM_TYPE}),
'model.weights.bin');
}

Expand Down Expand Up @@ -152,9 +154,7 @@ export class BrowserHTTPRequest implements IOHandler {
*/
private async loadBinaryTopology(): Promise<ArrayBuffer> {
const response = await this.getFetchFunc()(
this.path[0], this.addAcceptHeader('application/octet-stream'));
this.verifyContentType(
response, 'model topology', 'application/octet-stream');
this.path[0], this.addAcceptHeader(OCTET_STREAM_TYPE));

if (!response.ok) {
throw new Error(`Request to ${this.path[0]} failed with error: ${
Expand All @@ -172,21 +172,10 @@ export class BrowserHTTPRequest implements IOHandler {
return requestOptions;
}

private verifyContentType(response: Response, target: string, type: string) {
const contentType = response.headers.get('content-type');
if (!contentType || contentType.indexOf(type) === -1) {
throw new Error(`Request to ${response.url} for ${
target} failed. Expected content type ${type} but got ${
contentType}.`);
}
}

protected async loadBinaryModel(): Promise<ModelArtifacts> {
const graphPromise = this.loadBinaryTopology();
const manifestPromise = await this.getFetchFunc()(
this.path[1], this.addAcceptHeader('application/json'));
this.verifyContentType(
manifestPromise, 'weights manifest', 'application/json');
this.path[1], this.addAcceptHeader(JSON_TYPE));
if (!manifestPromise.ok) {
throw new Error(`Request to ${this.path[1]} failed with error: ${
manifestPromise.statusText}`);
Expand All @@ -210,9 +199,7 @@ export class BrowserHTTPRequest implements IOHandler {

protected async loadJSONModel(): Promise<ModelArtifacts> {
const modelConfigRequest = await this.getFetchFunc()(
this.path as string, this.addAcceptHeader('application/json'));
this.verifyContentType(
modelConfigRequest, 'model topology', 'application/json');
this.path as string, this.addAcceptHeader(JSON_TYPE));

if (!modelConfigRequest.ok) {
throw new Error(`Request to ${this.path} failed with error: ${
Expand Down
178 changes: 0 additions & 178 deletions src/io/browser_http_test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -765,29 +765,6 @@ describeWithFlags('browserHTTPRequest-load', BROWSER_ENVS, () => {
});
});

it('with wrong content type leads to error', async (done) => {
setupFakeWeightFiles(
{
'path1/model.json':
{data: JSON.stringify({}), contentType: 'text/html'}
},
requestInits);
const handler = tf.io.browserHTTPRequest('path1/model.json');
handler.load()
.then(modelTopology1 => {
done.fail(
'Loading with wrong content-type succeeded unexpectedly.');
})
.catch(err => {
expect(err.message)
.toEqual(
'Request to path1/model.json for model topology failed. ' +
'Expected content type application/json ' +
'but got text/html.');
done();
});
});

it('with fetch rejection leads to error', async (done) => {
setupFakeWeightFiles(
{
Expand Down Expand Up @@ -1159,161 +1136,6 @@ describeWithFlags('browserHTTPRequest-load', BROWSER_ENVS, () => {
expect(() => tf.io.browserHTTPRequest(['path1/model.pb'])).toThrow();
});

it('with wrong model content type leads to error', async (done) => {
const weightsManifest: tf.io.WeightsManifestConfig = [
{
paths: ['weightfile0'],
weights: [{
name: 'fooWeight',
shape: [3, 1],
dtype: 'int32',
}]
},
{
paths: ['weightfile1'],
weights: [{
name: 'barWeight',
shape: [2],
dtype: 'bool',
}],
}
];
const floatData1 = new Int32Array([1, 3, 3]);
const floatData2 = new Uint8Array([7, 4]);
setupFakeWeightFiles(
{
'path1/model.pb': {data: modelData, contentType: 'text/html'},
'path2/weights_manifest.json': {
data: JSON.stringify(weightsManifest),
contentType: 'application/json'
},
'path3/weightfile0':
{data: floatData1, contentType: 'application/octet-stream'},
'path3/weightfile1':
{data: floatData2, contentType: 'application/octet-stream'},
},
requestInits);
const handler = tf.io.browserHTTPRequest(
['path1/model.pb', 'path2/weights_manifest.json'], {}, 'path3/');
handler.load()
.then(modelTopology1 => {
done.fail(
'Loading with wrong content-type succeeded unexpectedly.');
})
.catch(err => {
expect(err.message)
.toEqual(
'Request to path1/model.pb for model topology failed. ' +
'Expected content type application/octet-stream ' +
'but got text/html.');
done();
});
});

it('with wrong manifest content type leads to error', async (done) => {
const weightsManifest: tf.io.WeightsManifestConfig = [
{
paths: ['weightfile0'],
weights: [{
name: 'fooWeight',
shape: [3, 1],
dtype: 'int32',
}]
},
{
paths: ['weightfile1'],
weights: [{
name: 'barWeight',
shape: [2],
dtype: 'bool',
}],
}
];
const floatData1 = new Int32Array([1, 3, 3]);
const floatData2 = new Uint8Array([7, 4]);
setupFakeWeightFiles(
{
'path1/model.pb':
{data: modelData, contentType: 'application/octet-stream'},
'path2/weights_manifest.json': {
data: JSON.stringify(weightsManifest),
contentType: 'application/octet-stream'
},
'path3/weightfile0':
{data: floatData1, contentType: 'application/octet-stream'},
'path3/weightfile1':
{data: floatData2, contentType: 'application/octet-stream'},
},
requestInits);
const handler = tf.io.browserHTTPRequest(
['path1/model.pb', 'path2/weights_manifest.json'], {}, 'path3/');
handler.load()
.then(modelTopology1 => {
done.fail(
'Loading with wrong content-type succeeded unexpectedly.');
})
.catch(err => {
expect(err.message)
.toEqual(
'Request to path2/weights_manifest.json for weights ' +
'manifest failed. Expected content type application/json' +
' but got application/octet-stream.');
done();
});
});

it('with wrong weight content type leads to error', async (done) => {
const weightsManifest: tf.io.WeightsManifestConfig = [
{
paths: ['weightfile0'],
weights: [{
name: 'fooWeight',
shape: [3, 1],
dtype: 'int32',
}]
},
{
paths: ['weightfile1'],
weights: [{
name: 'barWeight',
shape: [2],
dtype: 'bool',
}],
}
];
const floatData1 = new Int32Array([1, 3, 3]);
const floatData2 = new Uint8Array([7, 4]);
setupFakeWeightFiles(
{
'path1/model.pb':
{data: modelData, contentType: 'application/octet-stream'},
'path2/weights_manifest.json': {
data: JSON.stringify(weightsManifest),
contentType: 'application/json'
},
'path3/weightfile0':
{data: floatData1, contentType: 'application/json'},
'path3/weightfile1':
{data: floatData2, contentType: 'application/octet-stream'},
},
requestInits);
const handler = tf.io.browserHTTPRequest(
['path1/model.pb', 'path2/weights_manifest.json'], {}, 'path3/');
handler.load()
.then(modelTopology1 => {
done.fail(
'Loading with wrong content-type succeeded unexpectedly.');
})
.catch(err => {
expect(err.message)
.toEqual(
'Request to path3/weightfile0 for weight file failed. ' +
'Expected content type application/octet-stream but ' +
'got application/json.');
done();
});
});

it('with fetch rejection leads to error', async (done) => {
setupFakeWeightFiles(
{
Expand Down
14 changes: 0 additions & 14 deletions src/io/weights_loader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ type RequestHeader = {
};

const OCTET_STREAM_TYPE = 'application/octet-stream';
const CONTENT_TYPE = 'Content-type';
/**
* Reads binary weights data from a number of URLs.
*
Expand Down Expand Up @@ -63,19 +62,6 @@ export async function loadWeightsAsArrayBuffer(
await util.monitorPromisesProgress(
requests, onProgress, fetchStartFraction, fetchEndFraction);

const badContentType = responses.filter(response => {
const contentType = response.headers.get(CONTENT_TYPE);
return !contentType || contentType.indexOf(OCTET_STREAM_TYPE) === -1;
});
if (badContentType.length > 0) {
throw new Error(
badContentType
.map(
resp => `Request to ${resp.url} for weight file failed.` +
` Expected content type ${OCTET_STREAM_TYPE} but got ${
resp.headers.get(CONTENT_TYPE)}.`)
.join('\n'));
}
const bufferPromises = responses.map(response => response.arrayBuffer());

const bufferStartFraction = 0.5;
Expand Down