From d49d0552db3d73af305c2cd3645359acde98cf8a Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Wed, 1 Jun 2016 16:08:00 -0700 Subject: [PATCH 1/2] spec: describe descriptors and digests Add a definition for content descriptors and digests, which are used as content identifiers in the OCI image specification. Signed-off-by: Stephen J Day --- descriptor.md | 123 +++++++++++++++++++++++++++++++++++++++++++++++++ media-types.md | 1 + 2 files changed, 124 insertions(+) create mode 100644 descriptor.md diff --git a/descriptor.md b/descriptor.md new file mode 100644 index 000000000..c13773368 --- /dev/null +++ b/descriptor.md @@ -0,0 +1,123 @@ + + +# OpenContainers Content Descriptors + +OCI have several components that come to together to describe an image. +References between components form a [Merkle Directed Acyclic Graph (DAG)](https://en.wikipedia.org/wiki/Merkle_tree). +The references in the _Merkle DAG_ are expressed through _Content Descriptors_. +A _Content Descriptor_ or _Descriptor_, describes the disposition of targeted content. +A _Descriptor_ includes the type of content, an independently-verifiable content identifier, known as a "digest" and the byte-size of the raw content. + +Descriptors SHOULD be embedded in other formats to securely reference external content. + +Other formats SHOULD use descriptors to securely reference external content. + +## Properties + +The following describe the primary set of properties that make up a _Descriptor_. + +- **`mediaType`** *string* + + This REQUIRED property contains the MIME type of the referenced object. + +- **`digest`** *string* + + This REQUIRED property is the _digest_ of the targeted content, meeting the requirements outlined in [Digests and Verification](#digests-and—verification). + Retrieved content SHOULD be verified against this digest when consumed via untrusted sources. + +- **`size`** *int* + This REQUIRED property specifies the size in bytes of the blob. + This property exists so that a client will have an expected size for the content before validating. + If the length of the retrieved content does not match the specified length, the content SHOULD NOT be trusted. + +### Reserved + +The following are field keys that MUST NOT be used in descriptors specified in other OCI specifications: + +- **`urls`** *array* + + This key is RESERVED for future versions of the specification. + +- **`data`** *string* + + This key is RESERVED for futures versions of the specification. + +All other fields may be included in other OCI specifications. +Extended _Descriptor_ field additions proposed in other OCI specifications SHOULD first be considered for addition into this specification. + +## Digests and Verification + +The _digest_ component of a _Descriptor_ acts as a content identifier, employing [content addressability](http://en.wikipedia.org/wiki/Content-addressable_storage) for the OCI image format. +It uniquely identifies content by taking a collision-resistant hash of the bytes. +Such an identifier can be independently calculated and verified by selection of a common _algorithm_. +If such an identifier can be communicated in a secure manner, one can retrieve the content from an insecure source, calculate it independently and be certain that the correct content was obtained. +Put simply, the identifier is a property of the content. + +To disambiguate from other concepts, we call this identifier a _digest_. +A _digest_ is a serialized hash result, consisting of a _algorithm_ and _hex_ portion. +The _algorithm_ identifies the methodology used to calculate the digest, which are shared by implementations. +The _hex_ portion is the hex-encoded result of the hash. + +We define a _digest_ string to match the following grammar: + +``` +digest := algorithm ":" hex +algorithm := /[A-Fa-f0-9_+.-]+/ +hex := /[A-Fa-f0-9]+/ +``` + +Some examples of _digests_ include the following: + +digest | description | +----------------------------------------------------------------------------------|------------------------------------------------ +sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b | Common sha256 based digest | + +Before consuming content targeted by a descriptor from untrusted sources, the byte content SHOULD be verified against the _digest_. +The size of the content SHOULD be verified, as well, to protect against [Length Extension Attacks](https://en.wikipedia.org/wiki/Length_extension_attack). +Heavy processing of before calculating a hash SHOULD be avoided. +Implementations MAY employ some canonicalization to ensure stable content identifiers. + +### Algorithms + +While the _algorithm_ does allow one to implement a wide variety of algorithms, compliant implementations SHOULD use [SHA-256](#SHA-256). + +Let's use a simple example in pseudo-code to demonstrate a digest calculation: +A _digest_ is calculated by the following pseudo-code, where `H` is the selected hash algorithm, identified by string ``: +``` +let ID(C) = Descriptor.digest +let C = +let D = ':' + EncodeHex(H(C)) +let verified = ID(C) == D +``` +Above, we define the content identifier as `ID(C)`, extracted from the `Descriptor.digest` field. +Content `C` is a string of bytes. +Function `H` returns a the hashs of `C` in bytes and is passed to function `EncodeHex` to obtain the _digest_. +The result `verified` is true if `ID(C)` is equal to `D`, confirming that `C` is the content identified by `D`. +After verification, the following is true: + +``` +D == ID(C) == ':' + EncodeHex(H(C)) +``` + +The _digest_ is confirmed as the content identifier by independently calculating the _digest_. + +#### SHA-256 + +[SHA-256](https://tools.ietf.org/html/rfc4634#page-7) is a collision-resistant hash function, chosen for ubiquity, reasonable size and secure characteristics. +Implementations MUST implement SHA-256 digest verification for use in descriptors. + +## Examples + +The following example describes a [_Manifest_](manifest.md#image-manifest) with a content identifier of "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270", of size 7682 bytes: + +```json,title=Content%20Descriptor&mediatype=application/vnd.oci.descriptor.v1%2Bjson +{ + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7682, + "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270" +} +``` diff --git a/media-types.md b/media-types.md index 9efa5dfd1..bf28e30bc 100644 --- a/media-types.md +++ b/media-types.md @@ -2,6 +2,7 @@ The following `mediaType` MIME types are used by the formats described here, and the resources they reference: +- `application/vnd.oci.descriptor.v1+json`: [Content Descriptor](descriptor.md) - `application/vnd.oci.image.manifest.list.v1+json`: [Manifest list](manifest.md#manifest-list) - `application/vnd.oci.image.manifest.v1+json`: [Image manifest format](manifest.md#image-manifest) - `application/vnd.oci.image.serialization.rootfs.tar.gzip`: ["Layer", as a gzipped tar archive](serialization.md#creating-an-image-filesystem-changeset) From 0b2b3fd913b034a86b915173634555b9351fc9ce Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Thu, 2 Jun 2016 17:57:29 -0700 Subject: [PATCH 2/2] schema: break out definition of descriptor and verify examples Signed-off-by: Stephen J Day --- descriptor.md | 8 +--- schema/content-descriptor.json | 24 ++++++++++++ schema/defs-image.json | 22 ----------- schema/fs.go | 65 ++++++++++++++++++------------- schema/image-manifest-schema.json | 4 +- schema/schema.go | 2 + schema/spec_test.go | 4 ++ 7 files changed, 72 insertions(+), 57 deletions(-) create mode 100644 schema/content-descriptor.json diff --git a/descriptor.md b/descriptor.md index c13773368..8b7fe61e8 100644 --- a/descriptor.md +++ b/descriptor.md @@ -1,9 +1,3 @@ - - # OpenContainers Content Descriptors OCI have several components that come to together to describe an image. @@ -77,7 +71,7 @@ digest sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b | Common sha256 based digest | Before consuming content targeted by a descriptor from untrusted sources, the byte content SHOULD be verified against the _digest_. -The size of the content SHOULD be verified, as well, to protect against [Length Extension Attacks](https://en.wikipedia.org/wiki/Length_extension_attack). +The size of the content SHOULD be verified to reduce hash collision space. Heavy processing of before calculating a hash SHOULD be avoided. Implementations MAY employ some canonicalization to ensure stable content identifiers. diff --git a/schema/content-descriptor.json b/schema/content-descriptor.json new file mode 100644 index 000000000..c4bdb6bde --- /dev/null +++ b/schema/content-descriptor.json @@ -0,0 +1,24 @@ +{ + "description": "OpenContainer Content Descriptor Specification", + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "https://opencontainers.org/schema/descriptor", + "type": "object", + "properties": { + "mediaType": { + "description": "the mediatype of the referenced object", + "$ref": "defs-image.json#definitions/mediaType" + }, + "size": { + "description": "the size in bytes of the referenced object", + "type": "integer" + }, + "digest": { + "$ref": "defs-image.json#definitions/digest" + } + }, + "required": [ + "mediaType", + "size", + "digest" + ] +} diff --git a/schema/defs-image.json b/schema/defs-image.json index a26d1c936..d0d6b780e 100644 --- a/schema/defs-image.json +++ b/schema/defs-image.json @@ -11,28 +11,6 @@ "type": "string", "pattern": "^[a-z0-9]+:[a-fA-F0-9]+$" }, - "descriptor": { - "id": "https://opencontainers.org/schema/image/descriptor", - "type": "object", - "required": [ - "mediaType", - "size", - "digest" - ], - "properties": { - "mediaType": { - "description": "the mediatype of the referenced object", - "$ref": "#definitions/mediaType" - }, - "size": { - "description": "the size in bytes of the referenced object", - "type": "integer" - }, - "digest": { - "$ref": "#definitions/digest" - } - } - }, "manifestDescriptor": { "id": "https://opencontainers.org/schema/image/manifestDescriptor", "type": "object", diff --git a/schema/fs.go b/schema/fs.go index 53ad3e3ac..e49730a95 100644 --- a/schema/fs.go +++ b/schema/fs.go @@ -203,7 +203,7 @@ var _escData = map[string]*_escFile{ "/config-schema.json": { local: "config-schema.json", size: 707, - modtime: 1463700693, + modtime: 1464139547, compressed: ` H4sIAAAJbogA/5SRPW7DMAyF5/oUhpOxjjp0ytoDdOgJVJmKGcCiQDJDUPju1U/c2kvhLobx+L73JOqr adtuAHGMUZFCd2679wjhjYJaDMBt+vN4aT8iOPTobHE9Z+woboTJZmRUjWdjrkKhr+qJ+GIGtl77l1dT @@ -213,10 +213,23 @@ T4aE9IoTdGU2V0tnbzoS/xG1dbMbUdPhbgx7GZK9zscuVu4jgy+HBy99HZ/yKxxMUjBgfi1ZdrjJYiL1 `, }, + "/content-descriptor.json": { + local: "content-descriptor.json", + size: 616, + modtime: 1464914956, + compressed: ` +H4sIAAAJbogA/4yRMVPDMAyF9/wKXdqR1gxMXWFngI1jcG0lVe9iG1kMhet/x4oTSGGgW/L8vvck+7MB +aD1mx5SEYmh30D4mDPcxiKWADPqFQeBhMkWGp4SOOnJ2JG40Yp3dAQer+EEk7Yw55hg2Vd1G7o1n28nm +9s5UbVU58jOSCxNLs5ub84hVt/Hf7ZWTU0Il4/6ITqqWuPAshLmc6GJFG9CTfa7mKv3dVw4Io09DIXag +AmOHXKZBD4uOEV+XM+U8dnlDg+1xq8uuyj8F0tRsfnpH6lzhNtPHf5OoBSjA/iSYr5hmvgkqz9QjX/Z5 +6jHLsvGa4SeqJjVTWsv49k6M+mAvvy93ud5ldfl5bc7NVwAAAP//Zc2MR2gCAAA= +`, + }, + "/defs-config.json": { local: "defs-config.json", size: 1755, - modtime: 1463700704, + modtime: 1464139547, compressed: ` H4sIAAAJbogA/7xUzc7TMBA8N09hBY6BXhAHri1HVKQIOFZusm63xF5rvQEi1HfHSauS/qRKKV8PVeOx Z2bXY/t3olRaQigYvSC59INK52DQYTsKymsWLOpKsxJSCw9uRk40OmAVvwyuVe6hQIOF7vjZXvCoEAVb @@ -230,27 +243,27 @@ W1SwXppll8oQfUVUgXaDqSTtb5f8CQAA//8Cok052wYAAA== "/defs-image.json": { local: "defs-image.json", - size: 3100, - modtime: 1462965360, + size: 2528, + modtime: 1464914958, compressed: ` -H4sIAAAJbogA/+xWy27bMBC86ysIJUAOfqjXGkGAoEGBnlIgPdVQgY20kphKpLqkgzqB/r3U+x3Xidte -erK55A5nhmPSzxZjto/KI55qLoW9YfYNBlzwfKRYCqS5t4uBmJbsNkXxQQoNXCCxTwmEyO5S9HjAPSja -lyVeA2Dw8i1MMUGfw5d9ik3JFLmfbxhpnaqN40gD79Xwai0pdJQXYQIOz7dyWohlDaBLQFtp4iJs6ylo -jVTI+baF1ZO7cLbvVu/Nt+vV1/XCXZzbxdKs7LB9HqLSXWoDU3SEzKN9qmVIkEbcY4aZ913tElb2Mhmw -fJG8f0BPLxkXxbAiwi4uI1DR1eYywp/gG8sSiKvOq4vj9Rgt7mJjvgXXq4/FYCiooi/p9X53MEYES5lt -nfDHjhPm+Nuq1jv0ZVtU/Kk3rryvCm6rmQxBEz9UHQkzSZo7smJtzrk+HsIAychGnw0kFBDnZj7vPetk -uJO7Zmk21HOYSr4sT8X9XqP6TTq121xoDJGm9x9ld15J32oDY3U/6+wkIHhg1t2cIEMTWH8hS51KGoMO -JCX/8/Uv8jV1EAOg4/LUoEzqmNI4maZiBsiLuDYNO8Jej5mTyu4U3B7iTHDGmMPZV6t1XqA6fjV609lY -2OloGbA3klk/mh3KFJ+OVAP6VnIBQu74aS1rUWfpARHsx9MmAckUlwPC2nt+WugjEAcx/IUf7ddLZv0x -YSMo8P3iMoL4c/dnGkCs0JrzJDvwIoIQUkP/H+3REeiCNI+QFHgb9K6myVvWXLJq/aCkOHN6Lwekd4Uv -dwN3Or48T12UYhfH478BrlWPMiuzfgUAAP//VjUNyBwMAAA= +H4sIAAAJbogA/7RWy27bMBC8+ysIJUAOfqjXGkGAoEGBnlIgPTVQgY20kphKpLqkgzqB/r2k3k+nrt0b +ueQOZ4Zjym8LxpwAlU8801wKZ8ucOwy54HamWAakub9LgJiW7D5D8UkKDVwgsS8pRMgeMvR5yH0o2lcl +XgNg8OwRpphiwOHbPsOmZIo8sAfGWmdq67rSwPs1vNpIilzlx5iCy+1RbguxqgF0CegoTVxEbT0DrZEK +OT8eYf3qLd3HD+uPZnS7/r5ZestLp9ialx1OwCNUukttYIqOkfm0z7SMCLKY+8ww83+qXcrKXiZDZjfJ +p2f09YpxUUwrIuzqOgYV32yvY/wNgbEshaTqvLk6Xo/R4i23ZhTerj8Xk4GgFAQPDfhdJUPSCb6PsUaE +S9ltnfDXjhPacx6rWi8Eq7ao+GtvXt1Fp5IloENJqVOVvNYXMuRNRFF15M2kbe5ai71WR32FhCGSsQQD +NpBVQFyaddt70cl5J5vN1nyo8X0qdptNztNeo/pLOvUNcKExQpo+f5TveSXV1kmY5iIGQMflqUGZ1DGl +cTJNxQqQH3NtGnaEvR6zJpXTKXg9xJngjDGHq/+s1j1AdfzL7y3nY2Hno2XATiSzeTEnlCk+H6kG9FRy +IYJ1/LyWtaiz9IAI9uNlk4B0iss7woy0g0JfgDiI4S/8aL8OmfXfhI2gIAiKxwiSr92faQiJwsWcJ+24 +HuW9LyIIITX0/5UcHYEuSPMRkgLvw97TNPnKmkdWbZ6VFBdu78sB2UPhy8PAnY4vb1MPpdgliTMS7S3q +Wb7IF38CAAD//wKthPngCQAA `, }, "/defs.json": { local: "defs.json", size: 3193, - modtime: 1463700693, + modtime: 1464139547, compressed: ` H4sIAAAJbogA/7RWTXPaMBC98ys8tEfa2PIX9NYp/cghAzOZnjo9uGYBtSCpstxpmuG/VzLGWPZiMKWH JPau9r23T6tYzwPHGS4gSyUVinI2fOMMp7CkjJq3zMkzWDhqLXm+WvNc6UdwZgLYO85UQhlI51FASpc0 @@ -268,23 +281,23 @@ MrVJbn8cB+ZnN/gbAAD//0JyEpx5DAAA "/image-manifest-schema.json": { local: "image-manifest-schema.json", - size: 1064, - modtime: 1462965360, + size: 1032, + modtime: 1464914959, compressed: ` -H4sIAAAJbogA/6RTvVLjMBDu/RQ7TspzdMVVaa+64oaCDA1DIeyVvZlYMlrBTCaTd0c/UZAJBSSlV/v9 -Sj5UAHWH3FqaHBldr6G+m1D/NdpJ0mjh3yh7hP9Sk0J2cD9hS4paGbd/BfiS2wFHGaCDc9NaiC0b3aTp -ythedFYq1/z+I9JskXDUZQh7jPGqbVblCEvbgoIDMZ4cJKzbTxjQ5nmL7Wk2Wc9hHSH7kxDMzxLFg2dM -4dL4MvNmIAZFuOuAU0JkcANCFIcsDokP3hIhSAapgbTDHm10EcmvSybmZs9sOWuWifNjOq5H7Ehu0sbh -Rv0PrrP20qIKXB0qbuL6KlzuQvgBaQr1cYGbWfOaivrS17fY8s2YT0l3cu/tl3S5GGmt3BftOxzLvWuF -vfTMgNTauPju+faymx35xkvKn3VeIqvsNTqtLb68ksVg6/Grv+Di5czva163/3iqjtV7AAAA///++ypf -KAQAAA== +H4sIAAAJbogA/6RSPU/zMBDe8ytOacc39TswdWViQAxULIjBJOfkqsYOPoNUVf3v+KMujsoAdMyTez7u +8R0qgLpDbi1Njoyu11A/TKhvjXaSNFq4G2WPcC81KWQHjxO2pKiVcfpfoC+5HXCUgTo4N62F2LLRTUJX +xvais1K55v+NSNgi8ajLFPYc413b7MqRlqYFhQRiPCVIXLefMLDN6xbbEzZZr2EdIfs/YTGPJYknr5iW +S/DlzpuBGBThrgNOGyKDGxCiOWRzSHrwkQRBMkgNpB32aGOKKP63zcQ87Fkt75ptIn5Mv+sRO5KbNHG4 +0v9L6+y9tKiCVoeKmzi+Co+7EB4gTaE+LnizaN5TUV/mymohDWrX5EcwNqrO6Tu593FLei5CWiv3RdsO +x3Lup0beamYotTYu3jVfX2azI99oKfm7TktmlbPGpLXFt3eyGGI9f3flF5cxf495vf7jpTpWnwEAAP// +X3p8DwgEAAA= `, }, "/manifest-list-schema.json": { local: "manifest-list-schema.json", size: 1010, - modtime: 1462965360, + modtime: 1464139547, compressed: ` H4sIAAAJbogA/6ySMU/7MBDF93yKU9rxn/o/MHWFBQnEQMWCGExyaa5q7OAzSFXV747tS0qiMIDoUqkv fu9+7+xjBpBXyKWjzpM1+Rryhw7NtTVek0EHt63eItxrQzWyhzsKP48dllRTqZPlX8xYctlgq6O/8b5b diff --git a/schema/image-manifest-schema.json b/schema/image-manifest-schema.json index 197eeb11d..f25c85fd2 100644 --- a/schema/image-manifest-schema.json +++ b/schema/image-manifest-schema.json @@ -14,12 +14,12 @@ "$ref": "defs-image.json#/definitions/mediaType" }, "config": { - "$ref": "defs-image.json#/definitions/descriptor" + "$ref": "content-descriptor.json" }, "layers": { "type": "array", "items": { - "$ref": "defs-image.json#/definitions/descriptor" + "$ref": "content-descriptor.json" } }, "annotations": { diff --git a/schema/schema.go b/schema/schema.go index 7f9b9f234..2d104bb09 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -18,6 +18,7 @@ import "net/http" // Media types for the OCI image formats const ( + MediaTypeDescriptor Validator = `application/vnd.oci.descriptor.v1+json` MediaTypeManifest Validator = `application/vnd.oci.image.manifest.v1+json` MediaTypeManifestList Validator = `application/vnd.oci.image.manifest.list.v1+json` MediaTypeImageSerialization unimplemented = `application/vnd.oci.image.serialization.rootfs.tar.gzip` @@ -32,6 +33,7 @@ var ( // specs maps OCI schema media types to schema files. specs = map[Validator]string{ + MediaTypeDescriptor: "content-descriptor.json", MediaTypeManifest: "image-manifest-schema.json", MediaTypeManifestList: "manifest-list-schema.json", MediaTypeImageSerializationConfig: "config-schema.json", diff --git a/schema/spec_test.go b/schema/spec_test.go index b24f45ee6..02c336d43 100644 --- a/schema/spec_test.go +++ b/schema/spec_test.go @@ -33,6 +33,10 @@ var ( errFormatInvalid = errors.New("format: invalid") ) +func TestValidateDescriptor(t *testing.T) { + validate(t, "../descriptor.md") +} + func TestValidateManifest(t *testing.T) { validate(t, "../manifest.md") }