Refine auto Dataset, Nets, Losses, Metrics, Optimizers and Pipeline (o…

…pen-mmlab#38)

* add comments

* fix

* refine dataset

autogluon/__init__.py

@@ -40,11 +40,12 @@
 except ImportError:
     raise ImportError(
         "Unable to import dependency dask. "
-        "A quick tip is to install via `conda install dask`. ")
+        "A quick tip is to install via `pip install dask[complete]`. ")
 
 
 __version__ = '0.0.1'
 
+from .core import *
 from .dataset import *
 from .loss import *
 from .metric import *
@@ -62,4 +63,5 @@
           + network.__all__ \
           + optim.__all__ \
           + space.__all__ \
-          + task.__all__
+          + task.__all__ \
+          + core.__all__

autogluon/core/core.py

@@ -1,18 +1,29 @@
-import numpy as np
-import ConfigSpace as CS
-import argparse
+from abc import ABC, abstractmethod
 
-import autogluon as ag
+__all__ = ['fit', 'BaseAutoObject']
 
-from ..task.image_classification import pipeline
-from ..task.image_classification.model_zoo import get_model
-from ..network import Nets
-from ..optim import Optimizers, get_optim
-from ..loss import Losses
-from ..metric import Metrics
-from ..searcher import *
 
-__all__ = ['fit']
+class BaseAutoObject(ABC):
+    def __init__(self):
+        super(BaseAutoObject, self).__init__()
+        self._search_space = None
+
+    @property
+    def search_space(self):
+        return self._search_space
+
+    @search_space.setter
+    def search_space(self, cs):
+        self._search_space = cs
+
+    @abstractmethod
+    def _add_search_space(self):
+        pass
+
+    @abstractmethod
+    def _get_search_space_strs(self):
+        pass
+
 
 
 # TODO (cgraywang): put into class that can be inherited and add readme

autogluon/dataset/dataset.py

@@ -1,28 +1,53 @@
+from abc import abstractmethod
+
+from ..core import *
+
 __all__ = ['Dataset']
 
 
-class Dataset(object):
-    def __init__(self, name, train_path=None, val_path=None, batch_size=None, num_workers=None):
+class Dataset(BaseAutoObject):
+    def __init__(self, name, train_path=None, val_path=None, batch_size=None, num_workers=None,
+                 transform_train_fn=None, transform_val_fn=None,
+                 transform_train_list=None, transform_val_list=None, **kwargs):
         # TODO (cgraywang): add search space, handle batch_size, num_workers
+        super(Dataset, self).__init__()
         self.name = name
         self.train_path = train_path
         self.val_path = val_path
         self.batch_size = batch_size
         self.num_workers = num_workers
-        self.search_space = None
-        self.train = None
-        self.val = None
-        self.train_data = None
-        self.val_data = None
+        self.transform_train_fn = transform_train_fn
+        self.transform_val_fn = transform_val_fn
+        self.transform_train_list = transform_train_list
+        self.transform_val_list = transform_val_list
+        self._train = None
+        self._val = None
+        self._num_classes = None
 
-    def _read_dataset(self):
-        pass
+    @property
+    def train(self):
+        return self._train
 
-    def _set_search_space(self, cs):
-        self.search_space = cs
+    @train.setter
+    def train(self, value):
+        self._train = value
 
-    def add_search_space(self):
-        pass
+    @property
+    def val(self):
+        return self._val
 
-    def get_search_space(self):
-        return self.search_space
+    @val.setter
+    def val(self, value):
+        self._val = value
+
+    @property
+    def num_classes(self):
+        return self._num_classes
+
+    @num_classes.setter
+    def num_classes(self, value):
+        self._num_classes = value
+
+    @abstractmethod
+    def _read_dataset(self):
+        pass

autogluon/loss/losses.py

@@ -1,48 +1,43 @@
 import ConfigSpace as CS
 
-import autogluon as ag
+from ..core import *
 from ..space import *
 from .utils import Loss
 
 __all__ = ['Losses']
 
 
-class Losses(object):
+class Losses(BaseAutoObject):
     def __init__(self, loss_list):
-        # TODO(cgraywang): add model instance, for now, use a list of model names
+        # TODO(cgraywang): add instance, for now, use a list
         assert isinstance(loss_list, list), type(loss_list)
+        super(Losses, self).__init__()
         self.loss_list = loss_list
-        self.search_space = None
-        self.add_search_space()
+        self._add_search_space()
 
-    def _set_search_space(self, cs):
-        self.search_space = cs
-
-    def add_search_space(self):
+    def _add_search_space(self):
         cs = CS.ConfigurationSpace()
+        # TODO (cgraywang): add more hparams for loss, e.g., weight
         loss_list_hyper_param = List('loss',
-                                     choices=self.get_loss_strs()) \
-            .get_hyper_param()
+                                     choices=self._get_search_space_strs()).get_hyper_param()
         cs.add_hyperparameter(loss_list_hyper_param)
-        # TODO (cgraywang): do not add hyper-params for loss
-        self._set_search_space(cs)
-
-    def get_search_space(self):
-        return self.search_space
+        self.search_space = cs
 
-    def get_loss_strs(self):
+    def _get_search_space_strs(self):
         loss_strs = []
         for loss in self.loss_list:
             if isinstance(loss, Loss):
                 loss_strs.append(loss.name)
             elif isinstance(loss, str):
                 loss_strs.append(loss)
             else:
-                pass
+                raise NotImplementedError
         return loss_strs
 
     def __repr__(self):
-        return "AutoGluon Losses %s with %s" % (str(self.get_loss_strs()), str(self.search_space))
+        return "AutoGluon Losses %s with %s" % (
+            str(self._get_search_space_strs()), str(self.search_space))
 
     def __str__(self):
-        return "AutoGluon Losses %s with %s" % (str(self.get_loss_strs()), str(self.search_space))
+        return "AutoGluon Losses %s with %s" % (
+            str(self._get_search_space_strs()), str(self.search_space))

autogluon/loss/utils.py

@@ -6,10 +6,6 @@
 __all__ = ['autogluon_losses', 'Loss']
 
 
-def get_hyper_params(self):
-    return self.hyper_params
-
-
 def autogluon_losses(func):
     @functools.wraps(func)
     def wrapper_decorator(*args, **kwargs):

autogluon/metric/metrics.py

@@ -1,50 +1,44 @@
 import ConfigSpace as CS
 
-import autogluon as ag
+from ..core import *
 from ..space import *
 from .utils import Metric
 
 __all__ = ['Metrics']
 
 
-class Metrics(object):
+class Metrics(BaseAutoObject):
     def __init__(self, metric_list):
-        # TODO(cgraywang): add model instance, for now, use a list of model names
+        # TODO(cgraywang): add instance, for now, use a list
+        # TODO(cgraywang): use all option
         assert isinstance(metric_list, list), type(metric_list)
+        super(Metrics, self).__init__()
         self.metric_list = metric_list
-        self.search_space = None
-        self.add_search_space()
+        self._add_search_space()
 
-    def _set_search_space(self, cs):
-        self.search_space = cs
-
-    def add_search_space(self):
+    def _add_search_space(self):
         cs = CS.ConfigurationSpace()
+        # TODO (cgraywang): add more hparams for metric, e.g., weight
         metric_list_hyper_param = List('metric',
-                                       choices=self.get_metric_strs()) \
-            .get_hyper_param()
+                                       choices=self._get_search_space_strs()).get_hyper_param()
         cs.add_hyperparameter(metric_list_hyper_param)
-        # TODO (cgraywang): do not add hyper-params for metric
-        self._set_search_space(cs)
-
-    def get_search_space(self):
-        return self.search_space
+        self.search_space = cs
 
-    def get_metric_strs(self):
+    def _get_search_space_strs(self):
         metric_strs = []
         for metric in self.metric_list:
             if isinstance(metric, Metric):
                 metric_strs.append(metric.name)
             elif isinstance(metric, str):
                 metric_strs.append(metric)
             else:
-                pass
+                raise NotImplementedError
         return metric_strs
 
     def __repr__(self):
         return "AutoGluon Metrics %s with %s" % (
-        str(self.get_metric_strs()), str(self.search_space))
+            str(self._get_search_space_strs()), str(self.search_space))
 
     def __str__(self):
         return "AutoGluon Metrics %s with %s" % (
-        str(self.get_metric_strs()), str(self.search_space))
+            str(self._get_search_space_strs()), str(self.search_space))

autogluon/metric/utils.py

@@ -6,10 +6,6 @@
 __all__ = ['autogluon_metrics', 'Metric']
 
 
-def get_hyper_params(self):
-    return self.hyper_params
-
-
 def autogluon_metrics(func):
     @functools.wraps(func)
     def wrapper_decorator(*args, **kwargs):

autogluon/network/nets.py

@@ -1,28 +1,26 @@
 import ConfigSpace as CS
 
 import autogluon as ag
+from ..core import *
 from ..space import *
 from .utils import Net
 
 __all__ = ['Nets']
 
 
-class Nets(object):
+class Nets(BaseAutoObject):
     def __init__(self, net_list):
+        #TODO (cgraywang): add net user config
         #TODO(cgraywang): add model instance, for now, use a list of model names
         assert isinstance(net_list, list), type(net_list)
+        super(Nets, self).__init__()
         self.net_list = net_list
-        self.search_space = None
-        self.add_search_space()
+        self._add_search_space()
 
-    def _set_search_space(self, cs):
-        self.search_space = cs
-
-    def add_search_space(self):
+    def _add_search_space(self):
         cs = CS.ConfigurationSpace()
         net_list_hyper_param = List('model',
-                                    choices=self.get_net_strs())\
-            .get_hyper_param()
+                                    choices=self._get_search_space_strs()).get_hyper_param()
         cs.add_hyperparameter(net_list_hyper_param)
         for net in self.net_list:
             #TODO(cgraywang): distinguish between different nets, only support resnet for now
@@ -35,27 +33,24 @@ def add_search_space(self):
                     cs.add_hyperparameter(net_hyper_param)
                 #TODO(cgraywang): put condition in presets? split task settings out
                 cond = CS.InCondition(net_hyper_param, net_list_hyper_param,
-                                      self.get_net_strs())
+                                      self._get_search_space_strs())
                 conds.append(cond)
             cs.add_conditions(conds)
-        self._set_search_space(cs)
-
-    def get_search_space(self):
-        return self.search_space
+        self.search_space = cs
 
-    def get_net_strs(self):
+    def _get_search_space_strs(self):
         net_strs = []
         for net in self.net_list:
             if isinstance(net, Net):
                 net_strs.append(net.name)
             elif isinstance(net, str):
                 net_strs.append(net)
             else:
-                pass
+                raise NotImplementedError
         return net_strs
 
     def __repr__(self):
-        return "AutoGluon Nets %s with %s" % (str(self.get_net_strs()), str(self.search_space))
+        return "AutoGluon Nets %s with %s" % (str(self._get_search_space_strs()), str(self.search_space))
 
     def __str__(self):
-        return "AutoGluon Nets %s with %s" % (str(self.get_net_strs()), str(self.search_space))
+        return "AutoGluon Nets %s with %s" % (str(self._get_search_space_strs()), str(self.search_space))