config_processor

Docker/Dockerfile

@@ -4,6 +4,6 @@ RUN python3 -m pip install pip --upgrade
 
 
 COPY src /hogwild-python/src/
-COPY conda.txt setup.py /hogwild-python/
+COPY setup.py /hogwild-python/
 
 RUN cd /hogwild-python/ && pip install -e .

Kubernetes/configs_processor.py

@@ -0,0 +1,42 @@
+import os
+import yaml
+import argparse
+
+
+WORKER_PATH = os.path.join(os.path.abspath(os.path.join(__file__, '..')), 'workers.yaml')
+COORDINATOR_PATH = os.path.join(os.path.abspath(os.path.join(__file__, '..')), 'coordinator.yaml')
+
+parser = argparse.ArgumentParser(add_help=True)
+parser.add_argument('--n-workers', type=int, default=5)
+parser.add_argument('--data-path', type=str, default='data/dataset')
+parser.add_argument('--asynchronous', action="store_true", default=False)
+args = vars(parser.parse_args())
+
+
+if __name__ == '__main__':
+    with open(WORKER_PATH) as f:
+        workers = yaml.load(f)
+
+    with open(COORDINATOR_PATH) as f:
+        coordinator = yaml.load(f)
+
+    n_workers = args['n_workers']
+    data_path = args['data_path']
+    running_mode = 'asynchronous' if args['asynchronous'] else 'synchronous'
+
+    workers['spec']['replicas'] = n_workers
+    env = workers['spec']['template']['spec']['containers'][0]['env']
+    env_mod = [{'name': x['name'], 'value': n_workers} if x['name'] == 'N_WORKERS' else x for x in env]
+    env_mod = [{'name': x['name'], 'value': running_mode} if x['name'] == 'RUNNING_MODE' else x for x in env_mod]
+    env_mod = [{'name': x['name'], 'value': data_path} if x['name'] == 'DATA_PATH' else x for x in env_mod]
+    workers['spec']['template']['spec']['containers'][0]['env'] = env_mod
+
+    env = coordinator['spec']['template']['spec']['containers'][0]['env']
+    env_mod = [{'name': x['name'], 'value': n_workers} if x['name'] == 'N_WORKERS' else x for x in env]
+    env_mod = [{'name': x['name'], 'value': running_mode} if x['name'] == 'RUNNING_MODE' else x for x in env_mod]
+    env_mod = [{'name': x['name'], 'value': data_path} if x['name'] == 'DATA_PATH' else x for x in env_mod]
+    coordinator['spec']['template']['spec']['containers'][0]['env'] = env_mod
+
+    # print(workers)
+
+    # import pdb; pdb.set_trace()

Kubernetes/coordinator.yaml

@@ -7,12 +7,11 @@ metadata:
     app: coordinator-service
 spec:
   ports:
-  - port: 80
-    name: coordinator
+    - port: 80
+      name: coordinator
   clusterIP: None
   selector:
     app: coordinator-service
----
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -21,6 +20,7 @@ metadata:
     app: coordinator-service
 spec:
   serviceName: "coordinator-service"
+  podManagementPolicy: Parallel
   selector:
     matchLabels:
       app: coordinator-service
@@ -31,19 +31,23 @@ spec:
         app: coordinator-service
     spec:
       containers:
-      - name: coordinator-service
-        image: liabifano/hogwild:latest
-        ports:
-        - containerPort: 80
-          name: coordinator
-        command: ["python", "hogwild-python/src/hogwild/coordinator.py", "80"]
-        env:
-        - name: N_WORKERS
-          value: "5"
-        volumeMounts:
+        - name: coordinator-service
+          image: liabifano/hogwild:latest
+          env:
+            - name: N_WORKERS
+              value: "5"
+            - name: RUNNING_MODE
+              value: "synchronous"
+            - name: DATA_PATH
+              value: "my-path"
+          ports:
+            - containerPort: 80
+              name: coordinator
+          command: ["python", "hogwild-python/src/hogwild/coordinator.py", "80"]
+          volumeMounts:
             - name: shared-volume
               mountPath: /data
       volumes:
         - name: shared-volume
           persistentVolumeClaim:
-            claimName: cs449g9-scratch
+            claimName: cs449g9-scratch

Kubernetes/workers.yaml

@@ -7,12 +7,11 @@ metadata:
     app: workers-service
 spec:
   ports:
-  - port: 80
-    name: worker
+    - port: 80
+      name: worker
   clusterIP: None
   selector:
     app: workers-service
----
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -32,19 +31,23 @@ spec:
         app: workers-service
     spec:
       containers:
-      - name: workers-service
-        image: liabifano/hogwild_node:latest
-        env:
-        - name: N_WORKERS
-          value: "5"
-        ports:
-        - containerPort: 80
-          name: worker
-        command: ["python", "hogwild-python/src/hogwild/node.py", "80"]
-        volumeMounts:
+        - name: workers-service
+          image: liabifano/hogwild:latest
+          env:
+            - name: N_WORKERS
+              value: "5"
+            - name: RUNNING_MODE
+              value: "synchronous"
+            - name: DATA_PATH
+              value: "my-path"
+          ports:
+            - containerPort: 80
+              name: worker
+          command: ["python", "hogwild-python/src/hogwild/node.py", "80"]
+          volumeMounts:
             - name: shared-volume
               mountPath: /data
       volumes:
         - name: shared-volume
           persistentVolumeClaim:
-            claimName: cs449g9-scratch
+            claimName: cs449g9-scratch