Merge branch 'master' of github.com:AllenInstitute/sonata

* 'master' of github.com:AllenInstitute/sonata:
  updating setup metadata for pypi
  Fixing so that all examples use model_name instead of pop_name
  Updating pysonata code
  Specify how to represent node types when `target_simulator` is "NEST" or "PyNN" (AllenInstitute#33)

.gitignore

@@ -2,6 +2,12 @@
 **/*.egg-info/
 **/*~
 src/pysonata/sonata/tests/tmp
+src/pysonata/build
+src/pysonata/dist
+src/pysonata/.pytest_cache
+**/mechanisms/x86_64
+examples/**/output/log.txt
+examples/**/output/config.json
 .idea
 .cache
-**/mechanisms/x86_64
+

examples/300_intfire/build_network.py

@@ -7,12 +7,12 @@
 # Step 1: Create a v1 mock network of 14 cells (nodes) with across 7 different cell "types"
 net = NetworkBuilder("v1")
 
-net.add_nodes(N=240, pop_name='LIF_exc', location='VisL4', ei='e',
+net.add_nodes(N=240, model_name='LIF_exc', location='VisL4', ei='e',
               model_type='point_process',  # use point_process to indicate were are using point model cells
               model_template='nrn:IntFire1',  # Tell the simulator to use the NEURON built-in IntFire1 type cell
               dynamics_params='IntFire1_exc_1.json')
 
-net.add_nodes(N=60, pop_name='LIF_inh', location='VisL4', ei='i',
+net.add_nodes(N=60, model_name='LIF_inh', location='VisL4', ei='i',
               model_type='point_process',
               model_template='nrn:IntFire1',
               dynamics_params='IntFire1_inh_1.json')
@@ -46,7 +46,7 @@ def recurrent_connections(src_cells, trg_cell, n_syns):
               dynamics_params='instanteneousInh.json')
 
 
-net.add_edges(source={'ei': 'e'}, target={'pop_name': 'LIF_inh'},
+net.add_edges(source={'ei': 'e'}, target={'model_name': 'LIF_inh'},
               iterator='all_to_one',
               connection_rule=recurrent_connections,
               connection_params={'n_syns': 10},
@@ -56,7 +56,7 @@ def recurrent_connections(src_cells, trg_cell, n_syns):
               dynamics_params='instanteneousExc.json')
 
 
-net.add_edges(source={'ei': 'e'}, target={'pop_name': 'LIF_exc'},
+net.add_edges(source={'ei': 'e'}, target={'model_name': 'LIF_exc'},
               iterator='all_to_one',
               connection_rule=recurrent_connections,
               connection_params={'n_syns': 10},
@@ -78,33 +78,33 @@ def generate_positions(N, x0=0.0, x1=300.0, y0=0.0, y1=100.0):
 
 def select_source_cells(src_cells, trg_cell, n_syns):
     if np.random.random() > 0.1:
-        synapses = [n_syns if src['pop_name'] == 'tON' or src['pop_name'] == 'tOFF' else 0 for src in src_cells]
+        synapses = [n_syns if src['model_name'] == 'tON' or src['model_name'] == 'tOFF' else 0 for src in src_cells]
     else:
-        synapses = [n_syns if src['pop_name'] == 'tONOFF' else 0 for src in src_cells]
+        synapses = [n_syns if src['model_name'] == 'tONOFF' else 0 for src in src_cells]
 
     return synapses
 
 
 lgn = NetworkBuilder("lgn")
 pos_x, pos_y = generate_positions(30)
-lgn.add_nodes(N=30, pop_name='tON', ei='e', location='LGN',
+lgn.add_nodes(N=30, model_name='tON', ei='e', location='LGN',
               x=pos_x, y=pos_y,
               model_type='virtual')
 
 pos_x, pos_y = generate_positions(30)
-lgn.add_nodes(N=30, pop_name='tOFF', ei='e', location='LGN',
+lgn.add_nodes(N=30, model_name='tOFF', ei='e', location='LGN',
               x=pos_x, y=pos_y,
               model_type='virtual')
 
 pos_x, pos_y = generate_positions(30)
-lgn.add_nodes(N=30, pop_name='tONOFF', ei='e', location='LGN',
+lgn.add_nodes(N=30, model_name='tONOFF', ei='e', location='LGN',
               x=pos_x, y=pos_y,
               model_type='virtual')
 
 
 
 
-lgn.add_edges(source=lgn.nodes(), target=net.nodes(pop_name='LIF_exc'),
+lgn.add_edges(source=lgn.nodes(), target=net.nodes(model_name='LIF_exc'),
               iterator='all_to_one',
               connection_rule=select_source_cells,
               connection_params={'n_syns': 10},
@@ -113,7 +113,7 @@ def select_source_cells(src_cells, trg_cell, n_syns):
               delay=2.0,
               dynamics_params='instanteneousExc.json')
 
-lgn.add_edges(source=lgn.nodes(), target=net.nodes(pop_name='LIF_inh'),
+lgn.add_edges(source=lgn.nodes(), target=net.nodes(model_name='LIF_inh'),
               iterator='all_to_one',
               connection_rule=select_source_cells,
               connection_params={'n_syns': 10},
@@ -127,16 +127,16 @@ def select_source_cells(src_cells, trg_cell, n_syns):
 
 
 tw = NetworkBuilder("tw")
-tw.add_nodes(N=30, pop_name='TW', ei='e', location='TW', model_type='virtual')
+tw.add_nodes(N=30, model_name='TW', ei='e', location='TW', model_type='virtual')
 
-tw.add_edges(source=tw.nodes(), target=net.nodes(pop_name='LIF_exc'),
+tw.add_edges(source=tw.nodes(), target=net.nodes(model_name='LIF_exc'),
              connection_rule=5,
              syn_weight=0.01,
              weight_function='wmax',
              delay=2.0,
              dynamics_params='instanteneousExc.json')
 
-tw.add_edges(source=tw.nodes(), target=net.nodes(pop_name='LIF_inh'),
+tw.add_edges(source=tw.nodes(), target=net.nodes(model_name='LIF_inh'),
              connection_rule=5,
              syn_weight=0.02,
              weight_function='wmax',

examples/300_intfire/network/lgn_node_types.csv

@@ -1,4 +1,4 @@
-node_type_id model_type ei location pop_name
+node_type_id model_type ei location model_name
 100 virtual e LGN tON
 101 virtual e LGN tOFF
 102 virtual e LGN tONOFF

examples/300_intfire/network/lgn_v1_edge_types.csv

@@ -1,3 +1,3 @@
 edge_type_id target_query source_query delay weight_function syn_weight dynamics_params
-100 pop_name=='LIF_exc' * 2.0 wmax 0.0045 instanteneousExc.json
-101 pop_name=='LIF_inh' * 2.0 wmax 0.0015 instanteneousExc.json
+100 model_name=='LIF_exc' * 2.0 wmax 0.0045 instanteneousExc.json
+101 model_name=='LIF_inh' * 2.0 wmax 0.0015 instanteneousExc.json

examples/300_intfire/network/tw_node_types.csv

@@ -1,2 +1,2 @@
-node_type_id model_type ei location pop_name
+node_type_id model_type ei location model_name
 100 virtual e TW TW

examples/300_intfire/network/tw_v1_edge_types.csv

@@ -1,3 +1,3 @@
 edge_type_id target_query source_query delay weight_function syn_weight dynamics_params
-100 pop_name=='LIF_exc' * 2.0 wmax 0.01 instanteneousExc.json
-101 pop_name=='LIF_inh' * 2.0 wmax 0.02 instanteneousExc.json
+100 model_name=='LIF_exc' * 2.0 wmax 0.01 instanteneousExc.json
+101 model_name=='LIF_inh' * 2.0 wmax 0.02 instanteneousExc.json

examples/300_intfire/network/v1_node_types.csv

@@ -1,3 +1,3 @@
-node_type_id ei pop_name location model_template model_type dynamics_params
+node_type_id ei model_name location model_template model_type dynamics_params
 100 e LIF_exc VisL4 nrn:IntFire1 point_process IntFire1_exc_1.json
 101 i LIF_inh VisL4 nrn:IntFire1 point_process IntFire1_inh_1.json

examples/300_intfire/network/v1_v1_edge_types.csv

@@ -1,5 +1,5 @@
 edge_type_id target_query source_query delay weight_function syn_weight dynamics_params
 100 model_type=='point_process'&ei=='i' ei=='i' 2.0 wmax 0.01 instanteneousInh.json
 101 model_type=='point_process'&ei=='e' ei=='i' 2.0 wmax 0.15 instanteneousInh.json
-102 pop_name=='LIF_inh' ei=='e' 2.0 wmax 0.3 instanteneousExc.json
-103 pop_name=='LIF_exc' ei=='e' 2.0 wmax 0.002 instanteneousExc.json
+102 model_name=='LIF_inh' ei=='e' 2.0 wmax 0.3 instanteneousExc.json
+103 model_name=='LIF_exc' ei=='e' 2.0 wmax 0.002 instanteneousExc.json

examples/300_intfire/plot_spikes.py

@@ -1,3 +1,3 @@
 from bmtk.analyzer.visualization.spikes import plot_spikes
 
-plot_spikes('network/v1_nodes.h5', 'network/v1_node_types.csv', 'output/spikes.h5', group_key='pop_name')
+plot_spikes('network/v1_nodes.h5', 'network/v1_node_types.csv', 'output/spikes.h5', group_key='model_name')

src/pysonata/setup.py

@@ -15,11 +15,19 @@ def prepend_find_packages(*roots):
 
     return packages
 
+
+with open('README.md', 'r') as fhandle:
+    long_description = fhandle.read()
+
+
 setup(
     name='sonata',
-    version=0.1,
+    version='0.0.1',
     description='SONATA Data Format',
-    package_data={'': ['*.md', '*.txt', '*.cfg', '**/*.json', '**/*.hoc', '**/*.h5', '**/*.csv']},
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    url='https://github.com/AllenInstitute/bmtk',
+    package_data={'': ['*.md', '*.txt', '*.cfg', '**/*.json', '**/*.hoc']},
     tests_require=['pytest'],
     install_requires=[
         'jsonschema',

src/pysonata/sonata/io/edge.py

@@ -41,12 +41,13 @@ def next(self):
 
 
 class Edge(object):
-    def __init__(self, src_node_id, trg_node_id, source_pop, target_pop, group_props, edge_types_props):
+    def __init__(self, src_node_id, trg_node_id, source_pop, target_pop, group_id, group_props, edge_types_props):
         self._src_node_id = src_node_id
         self._trg_node_id = trg_node_id
         self._source_population = source_pop
         self._target_population = target_pop
         self._group_props = group_props
+        self._group_id = group_id
         self._edge_type_props = edge_types_props
 
     @property
@@ -65,6 +66,14 @@ def source_population(self):
     def target_population(self):
         return self._target_population
 
+    @property
+    def group_id(self):
+        return self._group_id
+
+    @property
+    def edge_type_id(self):
+        return self._edge_type_props['edge_type_id']
+
     @property
     def dynamics_params(self):
         raise NotImplementedError
@@ -75,4 +84,7 @@ def __getitem__(self, prop_key):
         elif prop_key in self._edge_type_props:
             return self._edge_type_props[prop_key]
         else:
-            raise KeyError
+            raise KeyError('Property {} not found in edge.'.format(prop_key))
+
+    def __contains__(self, prop_key):
+        return prop_key in self._group_props or prop_key in self._edge_type_props

src/pysonata/sonata/io/file.py

@@ -20,20 +20,34 @@
 # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
-from sonata.io import utils
-from sonata.io.file_root import NodesRoot, EdgesRoot
+from . import utils
+from .file_root import NodesRoot, EdgesRoot
 
 
 class File(object):
-    def __init__(self, data_files, data_type_files, mode='r', gid_table=None):
+    def __init__(self, data_files, data_type_files, mode='r', gid_table=None, require_magic=True):
         if mode != 'r':
             raise Exception('Currently only read mode is supported.')
 
         self._data_files = utils.listify(data_files)
         self._data_type_files = utils.listify(data_type_files)
 
-        # file handles
-        self._h5_file_handles = [(f, utils.load_h5(f, mode)) for f in self._data_files]
+        # Open and check HDF5 file(s)
+        self._h5_file_handles = [utils.load_h5(f, mode) for f in self._data_files]
+        if require_magic:
+            map(utils.check_magic, self._h5_file_handles)  # Check magic attribute in h5 files
+
+        # Check version number
+        avail_versions = set(map(utils.get_version, self._h5_file_handles))
+        if len(avail_versions) == 1:
+            self._version = list(avail_versions)[0]
+        elif len(avail_versions) > 1:
+            # TODO: log as warning
+            print('Warning: Passing in multiple hdf5 files of different version')
+            self._version = ','.join(avail_versions)
+        else:
+            self._version = utils.VERSION_NA
+
         self._csv_file_handles = [(f, utils.load_csv(f)) for f in self._data_type_files]
 
         self._has_nodes = False
@@ -75,6 +89,10 @@ def edges(self):
     def has_edges(self):
         return self._has_edges
 
+    @property
+    def version(self):
+        return self._version
+
     def _sort_types_file(self):
         # TODO: node/edge type_id columnn names should not be hardcoded
         for filename, df in self._csv_file_handles:
@@ -92,11 +110,11 @@ def _sort_types_file(self):
                 print('Warning: Could not determine if file {} was an edge-types or node-types file. Ignoring'.format(filename))
 
     def _sort_h5_files(self):
-        for filename, h5 in self._h5_file_handles:
+        for h5 in self._h5_file_handles:
             has_nodes = '/nodes' in h5
             has_edges = '/edges' in h5
             if not (has_nodes or has_edges):
-                print('File {} contains neither nodes nor edges. Ignoring'.format(filename))
+                print('File {} contains neither nodes nor edges. Ignoring'.format(h5.filename))
             else:
                 if has_nodes:
                     self._nodes_groups.append(h5)

src/pysonata/sonata/io/group.py

@@ -43,6 +43,8 @@ def __init__(self, group_id, h5_group, parent):
         self._types_index_col = self._types_table.index_column_name
 
         self._group_columns = ColumnProperty.from_h5(h5_group)
+        # TODO: combine group_columns, group_column_names and group_columns_map, doesn't need to be 3 structures
+        self._group_column_map = {col.name: col for col in self._group_columns}
         self._group_column_names = set(col.name for col in self._group_columns)
         self._group_table = {prop: h5_group[prop.name] for prop in self._group_columns}
         self._ncolumns = len(self._group_columns)
@@ -74,10 +76,33 @@ def has_dynamics_params(self):
     def columns(self):
         return self._group_columns
 
+    @property
+    def group_columns(self):
+        return self._group_columns
+
     @property
     def all_columns(self):
         return self._all_columns
 
+    @property
+    def has_gids(self):
+        return self._parent.has_gids
+
+    @property
+    def parent(self):
+        return self._parent
+
+    def get_dataset(self, column_name):
+        return self._group_table[column_name]
+
+    def column(self, column_name, group_only=False):
+        if column_name in self._group_column_map:
+            return self._group_column_map[column_name]
+        elif not group_only and column_name in self._types_table.columns:
+            return self._types_table.column(column_name)
+        else:
+            return KeyError
+
     def check_format(self):
         # Check that all the properties have the same number of rows
         col_counts = [col.nrows for col in self._group_columns + self._dynamics_params_columns]
@@ -266,6 +291,7 @@ def filter(self, **filter_props):
 
             if group_filter:
                 # Filter by group property values
+                # TODO: Allow group properties to handle lists
                 src_failed = True
                 for k, v in group_prop_filter.items():
                     if node[k] != v:
@@ -306,6 +332,27 @@ def build_indicies(self, force=False):
     def to_dataframe(self):
         raise NotImplementedError
 
+    def _get_parent_ds(self, parent_ds):
+        self.build_indicies()
+        ds_vals = np.zeros(self._indicies_count, dtype=parent_ds.dtype)
+        c_indx = 0
+        for indx_range in self._parent_indicies:
+            indx_beg, indx_end = indx_range[0], indx_range[1]
+            n_indx = c_indx + (indx_end - indx_beg)
+            ds_vals[c_indx:n_indx] = parent_ds[indx_beg:indx_end]
+            c_indx = n_indx
+
+        return ds_vals
+
+    def src_node_ids(self):
+        return self._get_parent_ds(self.parent._source_node_id_ds)
+
+    def trg_node_ids(self):
+        return self._get_parent_ds(self.parent._target_node_id_ds)
+
+    def node_type_ids(self):
+        return self._get_parent_ds(self.parent._type_id_ds)
+
     def get_values(self, property_name, all_rows=False):
         # TODO: Need to take into account if property_name is in the edge-types
         if property_name not in self.columns: