This repository has been archived by the owner on Sep 12, 2018. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 879
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add refcount extension to store image references and cleanup orphans. Based heavily on @wking's work in #409 - this is just an adaptation to extension format. Signed-off-by: Andy Goldstein <agoldste@redhat.com>
- Loading branch information
Andy Goldstein
committed
Oct 3, 2014
1 parent
5a92101
commit c7496dc
Showing
3 changed files
with
315 additions
and
0 deletions.
There are no files selected for viewing
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,283 @@ | ||
# -*- coding: utf-8 -*- | ||
|
||
import json | ||
import logging | ||
import sys | ||
|
||
from docker_registry.core import exceptions | ||
from docker_registry.lib import config | ||
import docker_registry.lib.signals | ||
from docker_registry import storage | ||
|
||
|
||
def tag_created(sender, namespace, repository, tag, image): | ||
"""Update image reference data for the current tag and image | ||
""" | ||
logger.debug("[refcount tag_created] namespace={0}; repository={1} tag={2} image={3}". # nopep8 | ||
format(namespace, repository, tag, image)) | ||
try: | ||
add_references(namespace=namespace, | ||
repository=repository, | ||
tag=tag, | ||
image_id=image) | ||
except Exception: | ||
logger.exception("TODO") | ||
|
||
|
||
def tag_deleted(sender, namespace, repository, tag, image): | ||
"""Update image reference data for the deleted tag and image | ||
Removes any images that are no longer referenced by a tag. | ||
""" | ||
logger.debug("[refcount tag_deleted] namespace={0}; repository={1} tag={2} image={3}". # nopep8 | ||
format(namespace, repository, tag, image)) | ||
try: | ||
remove_references(image) | ||
except Exception: | ||
logger.exception("TODO") | ||
|
||
|
||
def extension_info_path(): | ||
"""Path where information about this extension is stored | ||
""" | ||
return 'extensions/refcount' | ||
|
||
|
||
def get_extension_info(): | ||
"""Get information about this extension | ||
Currently just stores a '1' to indicate that the image references have | ||
been initialized | ||
""" | ||
try: | ||
info = store.get_json(extension_info_path()) | ||
except exceptions.FileNotFoundError: | ||
info = {} | ||
|
||
return info | ||
|
||
|
||
def init_references(): | ||
"""Initialize image reference data | ||
Walks all repositories and tags and initializes the image reference data | ||
for the entire registry. | ||
""" | ||
for namespace, repository in get_repositories(): | ||
for tag, image_id in get_tags(namespace=namespace, | ||
repository=repository): | ||
add_references(namespace=namespace, | ||
repository=repository, | ||
tag=tag, | ||
image_id=image_id) | ||
for image_id in get_images(): | ||
_check_references(image_id=image_id) | ||
|
||
|
||
def get_images(): | ||
"""Iterate through images in storage | ||
This helper is useful for upgrades and other storage | ||
maintenance. Yields image ids. | ||
""" | ||
try: | ||
image_paths = list( | ||
store.list_directory(path=store.images)) | ||
except exceptions.FileNotFoundError: | ||
image_paths = [] | ||
for image_path in image_paths: | ||
image_id = image_path.rsplit('/', 1)[-1] | ||
yield image_id | ||
|
||
|
||
def get_repositories(): | ||
"""Iterate through repositories in storage | ||
This helper is useful for upgrades and other storage | ||
maintenance. Yields tuples: | ||
(namespace, repository) | ||
""" | ||
try: | ||
namespace_paths = list( | ||
store.list_directory(path=store.repositories)) | ||
except exceptions.FileNotFoundError: | ||
namespace_paths = [] | ||
for namespace_path in namespace_paths: | ||
namespace = namespace_path.rsplit('/', 1)[-1] | ||
try: | ||
repository_paths = list( | ||
store.list_directory(path=namespace_path)) | ||
except exceptions.FileNotFoundError: | ||
repository_paths = [] | ||
for path in repository_paths: | ||
repository = path.rsplit('/', 1)[-1] | ||
yield (namespace, repository) | ||
|
||
|
||
def get_tags(namespace, repository): | ||
"""Iterate through a repository's tags | ||
This helper is useful for upgrades and other storage | ||
maintenance. Yields tuples: | ||
(tag_name, image_id) | ||
""" | ||
tag_path = store.tag_path(namespace, repository) | ||
for path in store.list_directory(tag_path): | ||
full_tag_name = path.split('/').pop() | ||
if not full_tag_name.startswith('tag_'): | ||
continue | ||
tag_name = full_tag_name[4:] | ||
tag_content = store.get_content(path=path) | ||
yield (tag_name, tag_content) | ||
|
||
|
||
def image_references_path(image_id): | ||
"""Path where image references are stored | ||
""" | ||
return '{0}/{1}/_references'.format(store.images, image_id) | ||
|
||
|
||
def _add_reference(image_id, descendant_id, namespace, repository, | ||
tag): | ||
"""Increment the refcount for a particular image | ||
Record the fact that we're needed for a particular image | ||
(descendant_id) which is tagged, so we know which images are safe | ||
to remove (e.g. any images that have no referring tags). | ||
""" | ||
references_path = image_references_path(image_id=image_id) | ||
try: | ||
references = store.get_json(path=references_path) | ||
except exceptions.FileNotFoundError: | ||
references = {} | ||
key = json.dumps([namespace, repository, tag]) | ||
references[key] = descendant_id | ||
store.put_json(path=references_path, content=references) | ||
|
||
|
||
def _check_references(image_id): | ||
"""Check for image references. If orphaned, remove the image | ||
Checks the existence of all the descendant images that (at one | ||
point) referenced this image. If any of those descendant | ||
images are gone (or if they are no longer tagged with the | ||
listed tag), then remove that entry from the references list. | ||
If no references remain, remove this image. | ||
""" | ||
references_path = image_references_path(image_id=image_id) | ||
try: | ||
references = store.get_json(path=references_path) | ||
except exceptions.FileNotFoundError: | ||
references = {} | ||
changed = False | ||
for namespace_repository_tag, descendant_id in list( | ||
references.items()): | ||
namespace, repository, tag = json.loads(namespace_repository_tag) | ||
descendant_layer_path = store.image_layer_path( | ||
image_id=descendant_id) | ||
if store.exists(path=descendant_layer_path): | ||
tag_path = store.tag_path( | ||
namespace=namespace, repository=repository, tagname=tag) | ||
try: | ||
tagged_image = store.get_content(path=tag_path) | ||
except exceptions.FileNotFoundError: | ||
tagged_image = None | ||
if tagged_image != descendant_id: | ||
# the listed descendant is no longer tagged with this tag | ||
references.pop(namespace_repository_tag) | ||
changed = True | ||
else: | ||
# the listed descendant no longer exists | ||
references.pop(namespace_repository_tag) | ||
changed = True | ||
if changed and references: | ||
store.put_json(path=references_path, content=references) | ||
if not references: | ||
image_path = '{0}/{1}'.format(store.images, image_id) | ||
logger.info('[refcount] Image {0} no longer referenced - removing' | ||
.format(image_id)) | ||
store.remove(image_path) | ||
|
||
|
||
def add_references(namespace, repository, tag, image_id): | ||
"""Increment ancestor refcounts | ||
""" | ||
ancestry_path = store.image_ancestry_path(image_id=image_id) | ||
ancestry = store.get_json(path=ancestry_path) | ||
for id in ancestry: | ||
_add_reference( | ||
image_id=id, descendant_id=image_id, namespace=namespace, | ||
repository=repository, tag=tag) | ||
|
||
|
||
def remove_references(image_id): | ||
"""Decrement ancestor refcounts and remove orphaned images | ||
""" | ||
ancestry_path = store.image_ancestry_path(image_id=image_id) | ||
ancestry = store.get_json(path=ancestry_path) | ||
for id in ancestry: | ||
_check_references(image_id=id) | ||
|
||
|
||
def _apply_version_1(): | ||
logger.info("[refcount] Initializing image references") | ||
init_references() | ||
|
||
def init_extension(current_version): | ||
"""Initialize the refcount extension | ||
Initializes the image reference data if this is the first time this | ||
extension is loaded. | ||
Wires up the signal handlers for tag_created and tag_deleted. | ||
""" | ||
info = get_extension_info() | ||
version = info.get('version', 0) | ||
if version < current_version: | ||
upgrade_start = version + 1 | ||
upgrade_end = current_version + 1 | ||
for version in range(upgrade_start, upgrade_end): | ||
logger.info("[refcount] Applying version {0}".format(version)) | ||
current_module = sys.modules[__name__] | ||
upgrader = getattr(current_module, '_apply_version_{0}'.format(version)) | ||
upgrader() | ||
|
||
info['version'] = current_version | ||
store.put_json(extension_info_path(), info) | ||
|
||
docker_registry.lib.signals.tag_created.connect(tag_created) | ||
docker_registry.lib.signals.tag_deleted.connect(tag_deleted) | ||
|
||
|
||
# | ||
# main entrypoint | ||
# | ||
CURRENT_VERSION=1 | ||
|
||
logger = logging.getLogger(__name__) | ||
|
||
logger.info("[refcount] Loading config") | ||
cfg = config.load() | ||
|
||
enabled = True | ||
|
||
if cfg.extensions is not None and cfg.extensions.refcount is not None: | ||
cfg = cfg.extensions.refcount | ||
if cfg.disabled: | ||
logger.info("[refcount] extension is disabled") | ||
enabled = False | ||
|
||
if enabled: | ||
logger.info("[refcount] extension is enabled") | ||
|
||
store = storage.load() | ||
|
||
logger.info("[refcount] initializing extension") | ||
init_extension(CURRENT_VERSION) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
#!/usr/bin/env python | ||
# -*- coding: utf-8 -*- | ||
|
||
try: | ||
import setuptools | ||
except ImportError: | ||
import distutils.core as setuptools | ||
|
||
setuptools.setup( | ||
name = "refcount-docker-registry-extension", | ||
version = "0.1", | ||
description = "Refcount Docker Registry Extension", | ||
long_description = open('../../README.md').read(), | ||
author = "TODO", | ||
author_email = "TODO", | ||
url = "TODO", | ||
license = open('../../LICENSE').read(), | ||
|
||
classifiers = [ | ||
'Development Status :: 3 - Alpha', | ||
'Topic :: Utilities', | ||
'License :: OSI Approved :: Apache Software License' | ||
], | ||
|
||
entry_points = { | ||
'docker_registry.extensions': [ | ||
'refcount = refcount' | ||
] | ||
}, | ||
|
||
install_requires=['docker-registry-core>=2,<3'], | ||
) |