Add --docker-registry and --image-tag flags to generator

docker/perapp/Makefile

@@ -84,7 +84,7 @@ hashes/sciond: app/Dockerfile.sciond hashes/app_base hashes/app_builder
 	DOCKER_ARGS="$(DOCKER_ARGS)" ./docker_build $@ $<
 
 # Debug images
-DEBUG_TARGETS = border_debug dispatcher_debug sig_debug path_debug_py path_debug beacon_py_debug cert_py_debug sciond_py_debug sciond_debug
+DEBUG_TARGETS = border_debug dispatcher_debug sig_debug path_py_debug path_debug beacon_py_debug cert_py_debug cert_debug sciond_py_debug sciond_debug
 .PHONY: $(DEBUG_TARGETS)
 debug: $(DEBUG_TARGETS)
 
@@ -100,12 +100,12 @@ sig_debug: hashes/sig_debug
 hashes/sig_debug: debug/Dockerfile.sig hashes/debug_base hashes/sig
 	DOCKER_ARGS="$(DOCKER_ARGS)" ./docker_build $@ $<
 
-path_debug_py: hashes/path_debug_py
-hashes/path_debug_py: debug/Dockerfile.path_py hashes/debug_base hashes/path_py
+path_py_debug: hashes/path_py_debug
+hashes/path_py_debug: debug/Dockerfile.path_py hashes/debug_base hashes/path_py
 	DOCKER_ARGS="$(DOCKER_ARGS)" ./docker_build $@ $<
 
 path_debug: hashes/path_debug
-hashes/path_debug: app/Dockerfile.path hashes/debug_base hashes/path
+hashes/path_debug: debug/Dockerfile.path hashes/debug_base hashes/path
 	DOCKER_ARGS="$(DOCKER_ARGS)" ./docker_build $@ $<
 
 beacon_py_debug: hashes/beacon_py_debug
@@ -116,10 +116,14 @@ cert_py_debug: hashes/cert_py_debug
 hashes/cert_py_debug: debug/Dockerfile.cert_py hashes/debug_base hashes/cert_py
 	DOCKER_ARGS="$(DOCKER_ARGS)" ./docker_build $@ $<
 
+cert_debug: hashes/cert_debug
+hashes/cert_debug: debug/Dockerfile.cert hashes/debug_base hashes/cert
+	DOCKER_ARGS="$(DOCKER_ARGS)" ./docker_build $@ $<
+
 sciond_py_debug: hashes/sciond_py_debug
 hashes/sciond_py_debug: debug/Dockerfile.sciond_py hashes/debug_base hashes/sciond_py
 	DOCKER_ARGS="$(DOCKER_ARGS)" ./docker_build $@ $<
 
 sciond_debug: hashes/sciond_debug
-hashes/sciond_debug: app/Dockerfile.sciond hashes/debug_base hashes/sciond
+hashes/sciond_debug: debug/Dockerfile.sciond hashes/debug_base hashes/sciond
 	DOCKER_ARGS="$(DOCKER_ARGS)" ./docker_build $@ $<

docker/perapp/debug/Dockerfile.cert

@@ -0,0 +1,2 @@
+FROM scion_cert:latest
+COPY --from=scion_debug_base:latest / /

docker/perapp/debug/Dockerfile.path

@@ -0,0 +1,2 @@
+FROM scion_path:latest
+COPY --from=scion_debug_base:latest / /

docker/perapp/debug/Dockerfile.sciond

@@ -0,0 +1,2 @@
+FROM scion_sciond:latest
+COPY --from=scion_debug_base:latest / /

docker/perapp/publish

@@ -7,7 +7,7 @@ REMOTE="scionproto"
 echo "Tag: ${1:-}"
 
 # Tag local apps and push to remote
-for app in border dispatcher sig beacon cert path sciond; do
+for app in border dispatcher sig beacon_py cert cert_py path path_py sciond sciond_py; do
     local_img="scion_$app"
     remote_img="$REMOTE/$app"
     docker tag "$local_img:latest" "$remote_img:$1"

python/topology/common.py

@@ -80,13 +80,13 @@ def __repr__(self):
         return "<TopoID: %s>" % self
 
 
-def _prom_addr_br(br_id, br_ele, port_gen):
+def prom_addr_br(br_id, br_ele, port_gen):
     """Get the prometheus address for a border router"""
     pub = _get_pub(br_ele['InternalAddrs'])
     return "[%s]:%s" % (pub['PublicOverlay']['Addr'].ip, port_gen.register(br_id + "prom"))
 
 
-def _prom_addr_infra(infra_id, infra_ele, port_gen):
+def prom_addr_infra(infra_id, infra_ele, port_gen):
     """Get the prometheus address for an infrastructure element."""
     pub = _get_pub(infra_ele['Addrs'])
     return "[%s]:%s" % (pub['Public']['Addr'].ip, port_gen.register(infra_id + "prom"))
@@ -99,19 +99,21 @@ def _get_pub(topo_addr):
     return topo_addr['IPv4']
 
 
-def _get_pub_ip(topo_addr):
-    return _get_pub(topo_addr)["Public"]["Addr"].ip
-
-
-def _get_l4_port(topo_addr):
-    return _get_pub(topo_addr)["Public"]["L4Port"]
-
-
-def _srv_iter(topo_dicts, out_dir, common=False):
+def srv_iter(topo_dicts, out_dir, common=False):
     for topo_id, as_topo in topo_dicts.items():
         base = topo_id.base_dir(out_dir)
         for service in SCION_SERVICE_NAMES:
             for elem in as_topo[service]:
                 yield topo_id, as_topo, os.path.join(base, elem)
         if common:
             yield topo_id, as_topo, os.path.join(base, COMMON_DIR)
+
+
+def docker_image(args, image):
+    if args.docker_registry:
+        image = '%s/%s' % (args.docker_registry, image)
+    else:
+        image = 'scion_%s' % image
+    if args.image_tag:
+        image = '%s:%s' % (image, args.image_tag)
+    return image

python/topology/config.py

@@ -53,7 +53,7 @@
 )
 from topology.ca import CAGenArgs, CAGenerator
 from topology.cert import CertGenArgs, CertGenerator
-from topology.common import _srv_iter, ArgsBase
+from topology.common import srv_iter, ArgsBase
 from topology.docker import DockerGenArgs, DockerGenerator
 from topology.go import GoGenArgs, GoGenerator
 from topology.net import (
@@ -235,7 +235,7 @@ def _write_ca_files(self, topo_dicts, ca_files):
                 write_file(os.path.join(base, path), value.decode())
 
     def _write_trust_files(self, topo_dicts, cert_files):
-        for topo_id, as_topo, base in _srv_iter(
+        for topo_id, as_topo, base in srv_iter(
                 topo_dicts, self.args.output_dir, common=True):
             for path, value in cert_files[topo_id].items():
                 write_file(os.path.join(base, path), value + '\n')
@@ -252,7 +252,7 @@ def _write_conf_policies(self, topo_dicts):
         Write AS configurations and path policies.
         """
         as_confs = {}
-        for topo_id, as_topo, base in _srv_iter(
+        for topo_id, as_topo, base in srv_iter(
                 topo_dicts, self.args.output_dir, common=True):
             as_confs.setdefault(topo_id, yaml.dump(
                 self._gen_as_conf(as_topo), default_flow_style=False))
@@ -280,7 +280,7 @@ def _write_master_keys(self, topo_dicts):
         Write AS master keys.
         """
         master_keys = {}
-        for topo_id, as_topo, base in _srv_iter(
+        for topo_id, as_topo, base in srv_iter(
                 topo_dicts, self.args.output_dir, common=True):
 
             master_keys.setdefault(topo_id, self._gen_master_keys())

python/topology/docker.py

@@ -27,7 +27,7 @@
     read_file,
     write_file,
 )
-from topology.common import _prom_addr_br, _prom_addr_infra, ArgsTopoDicts
+from topology.common import ArgsTopoDicts, docker_image, prom_addr_br, prom_addr_infra
 from topology.docker_utils import DockerUtilsGenArgs, DockerUtilsGenerator
 
 DOCKER_CONF = 'scion-dc.yml'
@@ -114,7 +114,7 @@ def _create_networks(self):
 
     def _br_conf(self, topo_id, topo, base):
         raw_entry = {
-            'image': 'scion_border',
+            'image': docker_image(self.args, 'border'),
             'depends_on': [
                 'scion_disp_br_%s' % topo_id.file_fmt(),
             ],
@@ -134,7 +134,7 @@ def _br_conf(self, topo_id, topo, base):
             entry['container_name'] = self.prefix + k
             entry['volumes'].append('%s:/share/conf:ro' % os.path.join(base, k))
             entry['command'].append('-id=%s' % k)
-            entry['command'].append('-prom=%s' % _prom_addr_br(k, v, self.args.port_gen))
+            entry['command'].append('-prom=%s' % prom_addr_br(k, v, self.args.port_gen))
             # Set BR IPs
             in_net = self.elem_networks[k + "_internal"][0]
             entry['networks'][self.bridges[in_net['net']]] = {'ipv4_address': str(in_net['ipv4'])}
@@ -143,9 +143,9 @@ def _br_conf(self, topo_id, topo, base):
             self.dc_conf['services']['scion_%s' % k] = entry
 
     def _cs_conf(self, topo_id, topo, base):
-        image = 'scion_cert_py' if self.args.cert_server == 'py' else 'scion_cert'
+        image = 'cert_py' if self.args.cert_server == 'py' else 'cert'
         raw_entry = {
-            'image': image,
+            'image': docker_image(self.args, image),
             'depends_on': [
                 self._sciond_name(topo_id),
                 'scion_disp_%s' % topo_id.file_fmt(),
@@ -164,15 +164,15 @@ def _cs_conf(self, topo_id, topo, base):
             if self.args.cert_server == 'py':
                 sciond = get_default_sciond_path(ISD_AS(topo["ISD_AS"]))
                 entry['command'].append('--spki_cache_dir=cache')
-                entry['command'].append('--prom=%s' % _prom_addr_infra(k, v, self.args.port_gen))
+                entry['command'].append('--prom=%s' % prom_addr_infra(k, v, self.args.port_gen))
                 entry['command'].append('--sciond_path=%s' % sciond)
                 entry['command'].append(k)
                 entry['command'].append('conf')
             self.dc_conf['services']['scion_%s' % k] = entry
 
     def _bs_conf(self, topo_id, topo, base):
         raw_entry = {
-            'image': 'scion_beacon_py',
+            'image': docker_image(self.args, 'beacon_py'),
             'depends_on': [
                 self._sciond_name(topo_id),
                 'scion_disp_%s' % topo_id.file_fmt(),
@@ -191,17 +191,17 @@ def _bs_conf(self, topo_id, topo, base):
             name = self.prefix + k
             entry['container_name'] = name
             entry['volumes'].append('%s:/share/conf:ro' % os.path.join(base, k))
-            entry['command'].append('--prom=%s' % _prom_addr_infra(k, v, self.args.port_gen))
+            entry['command'].append('--prom=%s' % prom_addr_infra(k, v, self.args.port_gen))
             entry['command'].append('--sciond_path=%s' %
                                     get_default_sciond_path(ISD_AS(topo["ISD_AS"])))
             entry['command'].append(k)
             entry['command'].append('conf')
             self.dc_conf['services']['scion_%s' % k] = entry
 
     def _ps_conf(self, topo_id, topo, base):
-        image = 'scion_path_py' if self.args.path_server == 'py' else 'scion_path'
+        image = 'path_py' if self.args.path_server == 'py' else 'path'
         raw_entry = {
-            'image': image,
+            'image': docker_image(self.args, image),
             'depends_on': [
                 self._sciond_name(topo_id),
                 'scion_disp_%s' % topo_id.file_fmt(),
@@ -220,7 +220,7 @@ def _ps_conf(self, topo_id, topo, base):
             entry['volumes'].append('%s:/share/conf:ro' % os.path.join(base, k))
             if self.args.path_server == 'py':
                 entry['command'].append('--spki_cache_dir=cache')
-                entry['command'].append('--prom=%s' % _prom_addr_infra(k, v, self.args.port_gen))
+                entry['command'].append('--prom=%s' % prom_addr_infra(k, v, self.args.port_gen))
                 entry['command'].append('--sciond_path=%s' %
                                         get_default_sciond_path(ISD_AS(topo["ISD_AS"])))
                 entry['command'].append(k)
@@ -256,7 +256,7 @@ def _zookeeper_conf(self):
     def _dispatcher_conf(self, topo_id, topo, base):
         # Create dispatcher config
         entry = {
-            'image': 'scion_dispatcher',
+            'image': docker_image(self.args, 'dispatcher'),
             'environment': {
                 'SU_EXEC_USERSPEC': self.user_spec,
             },
@@ -302,9 +302,9 @@ def _infra_dispatcher(self, entry, topo_id):
 
     def _sciond_conf(self, topo_id, base):
         name = self._sciond_name(topo_id)
-        image = 'scion_sciond_py' if self.args.sciond == 'py' else 'scion_sciond'
+        image = 'sciond_py' if self.args.sciond == 'py' else 'sciond'
         entry = {
-            'image': image,
+            'image': docker_image(self.args, image),
             'container_name': '%ssd%s' % (self.prefix, topo_id.file_fmt()),
             'depends_on': [
                 'scion_disp_%s' % topo_id.file_fmt()

python/topology/docker_utils.py

@@ -15,7 +15,7 @@
 # Stdlib
 import os
 # SCION
-from topology.common import ArgsBase
+from topology.common import ArgsBase, docker_image
 
 
 class DockerUtilsGenArgs(ArgsBase):
@@ -68,7 +68,7 @@ def _test_conf(self, topo_id):
         docker = 'docker_' if self.args.in_docker else ''
         cntr_base = '/home/scion/go/src/github.com/scionproto/scion'
         entry = {
-            'image': 'scion_app_builder',
+            'image': docker_image(self.args, 'app_builder'),
             'volumes': [
                 'vol_scion_%sdisp_%s:/run/shm/dispatcher:rw' % (docker, topo_id.file_fmt()),
                 'vol_scion_%ssciond_%s:/run/shm/sciond:rw' % (docker, topo_id.file_fmt()),

python/topology/generator.py

@@ -73,6 +73,8 @@ def add_arguments(parser):
                         help='Generate random IFIDs')
     parser.add_argument('--in-docker', action='store_true',
                         help='Set if running in a docker container')
+    parser.add_argument('--docker-registry', help='Specify docker registry to pull images from')
+    parser.add_argument('--image-tag', help='Docker image tag')
     return parser
 
 

python/topology/prometheus.py

@@ -26,7 +26,7 @@
 # SCION
 from lib.defines import PROM_FILE
 from lib.util import write_file
-from topology.common import _prom_addr_br, _prom_addr_infra, ArgsTopoDicts
+from topology.common import prom_addr_br, prom_addr_infra, ArgsTopoDicts
 
 
 class PrometheusGenArgs(ArgsTopoDicts):
@@ -59,10 +59,10 @@ def generate(self):
         for topo_id, as_topo in self.args.topo_dicts.items():
             ele_dict = defaultdict(list)
             for br_id, br_ele in as_topo["BorderRouters"].items():
-                ele_dict["BorderRouters"].append(_prom_addr_br(br_id, br_ele, self.args.port_gen))
+                ele_dict["BorderRouters"].append(prom_addr_br(br_id, br_ele, self.args.port_gen))
             for svc_type in ["BeaconService", "PathService", "CertificateService"]:
                 for elem_id, elem in as_topo[svc_type].items():
-                    ele_dict[svc_type].append(_prom_addr_infra(elem_id, elem, self.args.port_gen))
+                    ele_dict[svc_type].append(prom_addr_infra(elem_id, elem, self.args.port_gen))
             config_dict[topo_id] = ele_dict
         self._write_config_files(config_dict)
 

python/topology/supervisor.py

@@ -28,7 +28,7 @@
 from lib.defines import SCIOND_API_SOCKDIR
 from lib.packet.scion_addr import ISD_AS
 from lib.util import read_file, write_file
-from topology.common import _prom_addr_br, _prom_addr_infra, ArgsTopoDicts, COMMON_DIR
+from topology.common import prom_addr_br, prom_addr_infra, ArgsTopoDicts, COMMON_DIR
 
 SUPERVISOR_CONF = 'supervisord.conf'
 
@@ -63,7 +63,7 @@ def _std_entries(self, topo, topo_key, cmd, base):
         entries = []
         for elem_id, elem in topo.get(topo_key, {}).items():
             conf_dir = os.path.join(base, elem_id)
-            prom_addr = _prom_addr_infra(elem_id, elem, self.args.port_gen)
+            prom_addr = prom_addr_infra(elem_id, elem, self.args.port_gen)
             entries.append((elem_id, [cmd, "--prom", prom_addr, "--sciond_path",
                                       get_default_sciond_path(ISD_AS(topo["ISD_AS"])),
                                       elem_id, conf_dir]))
@@ -74,7 +74,7 @@ def _br_entries(self, topo, cmd, base):
         for k, v in topo.get("BorderRouters", {}).items():
             conf_dir = os.path.join(base, k)
             entries.append((k, [cmd, "-id=%s" % k, "-confd=%s" % conf_dir,
-                                "-prom=%s" % _prom_addr_br(k, v, self.args.port_gen)]))
+                                "-prom=%s" % prom_addr_br(k, v, self.args.port_gen)]))
         return entries
 
     def _bs_entries(self, topo, base):

python/topology/topo.py

@@ -40,7 +40,7 @@
 from lib.topology import Topology
 from lib.types import LinkType
 from lib.util import write_file
-from topology.common import _srv_iter, ArgsBase, TopoID, SCION_SERVICE_NAMES
+from topology.common import srv_iter, ArgsBase, TopoID, SCION_SERVICE_NAMES
 from topology.net import AddressProxy
 
 DEFAULT_LINK_BW = 1000
@@ -309,7 +309,7 @@ def _generate_as_list(self, topo_id, as_conf):
         self.as_list[key].append(str(topo_id))
 
     def _write_as_topos(self):
-        for topo_id, as_topo, base in _srv_iter(
+        for topo_id, as_topo, base in srv_iter(
                 self.topo_dicts, self.args.output_dir, common=True):
             path = os.path.join(base, TOPO_FILE)
             contents_json = json.dumps(self.topo_dicts[topo_id],