Add support for PostgreSQL HSTORE

doc/source/mapping.rst

@@ -27,6 +27,9 @@ There are three classes for the base geometries: ``Points``, ``LineStrings``  an
 ``with_type_field``
   Controls if the the *value* of the mapped key/value should be stored in the ``type`` column. Defaults to ``True``.
 
+``use_hstore``
+  Controls if all tags should be stored in a ``tags`` hstore column. Defaults to ``False``
+
 .. _mapping:
 
 mapping
@@ -74,6 +77,14 @@ There is a default field for names if you do not supply a field for the `name` c
 .. autofunction:: set_default_name_type
 
 
+PostgreSQL hstore
+^^^^^^^^^^^^^^^^^
+
+By default each mapping only stores the fields that are specified in the `tags` hstore column. If you want to store all OSM tags can call :func:`set_hstore_all_tags`.
+
+.. autofunction:: set_hstore_all_tags
+
+
 Classes
 ~~~~~~~
 

imposm/db/postgis.py

@@ -18,6 +18,7 @@
 
 import psycopg2
 import psycopg2.extensions
+import psycopg2.extras
 
 import logging
 log = logging.getLogger(__name__)
@@ -76,6 +77,8 @@ def connection(self):
             )
             self._connection.set_isolation_level(
                 psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED)
+            # Register adapter and typecaster for dict conversion.
+            psycopg2.extras.register_hstore(self._connection, unicode = True)
         return self._connection
 
     def commit(self):
@@ -163,6 +166,12 @@ def _insert_stmt(self, mapping):
             extra_arg_names = [n for n, t in mapping.fields]
             extra_args = ', %s' * len(extra_arg_names)
             extra_arg_names = ', ' + ', '.join('"' + name + '"' for name in extra_arg_names)
+
+        # Add tags argument for hstore.
+        if mapping.use_hstore:
+            extra_arg_names += ', tags'
+            extra_args += ', %s'
+
         return """INSERT INTO "%(tablename)s"
             (osm_id, geometry %(extra_arg_names)s)
             VALUES (%%s, ST_Transform(ST_GeomFromWKB(%%s, 4326), %(srid)s)
@@ -191,6 +200,10 @@ def create_table(self, mapping):
         for n, t in mapping.fields:
             extra_fields += ', "%s" %s ' % (n, t.column_type)
 
+        # Add hstore column named tags.
+        if mapping.use_hstore:
+            extra_fields += ', "tags" HSTORE '
+
         if config.imposm_pg_serial_id:
             serial_column = "id SERIAL PRIMARY KEY,"
         else:

imposm/mapping.py

@@ -39,9 +39,11 @@
     'FixInvalidPolygons',
     'UnionView',
     'set_default_name_field',
+    'set_hstore_all_tags'
 ]
 
 default_name_field = None
+hstore_all_tags = False
 
 def set_default_name_type(type, column_name='name'):
     """
@@ -55,6 +57,17 @@ def set_default_name_type(type, column_name='name'):
     global default_name_field
     default_name_field = column_name, type
 
+def set_hstore_all_tags(all_tags):
+    """
+    Set flag that indicates that we should disable the filtering of tags.
+
+    ::
+
+        set_hstore_all_tags(True)
+    """
+    global hstore_all_tags
+    hstore_all_tags = all_tags
+
 # changed by imposm.app if the projection is epsg:4326
 import_srs_is_geographic = False
 
@@ -87,10 +100,12 @@ class Mapping(object):
     _insert_stmt = None
     with_type_field = True
 
-    def __init__(self, name, mapping, fields=None, field_filter=None, with_type_field=None):
+    def __init__(self, name, mapping, fields=None, field_filter=None, with_type_field=None,
+                 use_hstore = False):
         self.name = name
         self.mapping = mapping
         self.fields = fields or tuple(self.fields)
+        self.use_hstore = use_hstore
         self.limit_to_polygon = None
         if with_type_field is not None:
             # allow subclass to define other default by setting it as class variable
@@ -154,12 +169,19 @@ def build_geom(self, osm_elem):
             raise DropElem(ex)
 
     def field_values(self, osm_elem):
-        return [t.value(osm_elem.tags.get(n), osm_elem) for n, t in self.fields]
+        values = [t.value(osm_elem.tags.get(n), osm_elem) for n, t in self.fields]
+        # Add dictionary of all tags which will be converted by psycopg2.
+        if self.use_hstore:
+            values.append(osm_elem.tags)
+        return values
 
     def field_dict(self, osm_elem):
         result = dict((n, t.value(osm_elem.tags.get(n), osm_elem)) for n, t in self.fields)
         if self.with_type_field:
             del result['type']
+        # Add dictionary of all tags which will be converted by psycopg2.
+        if self.use_hstore:
+            result['tags'] = osm_elem.tags
         result[osm_elem.cls] = osm_elem.type
         return result
 
@@ -218,6 +240,10 @@ def for_relations(self, tags):
         return self._mapping_for_tags(self.polygon_mappings, tags)
 
     def _tag_filter(self, filter_tags):
+        # Disable filtering of tags by returning None.
+        if hstore_all_tags:
+            return None
+
         def filter(tags):
             for k in tags.keys():
                 if k not in filter_tags:
@@ -262,7 +288,8 @@ def rel_filter(tags):
                 tags.clear()
                 return
             tag_count = len(tags)
-            _rel_filter(tags)
+            if _rel_filter is not None:
+                _rel_filter(tags)
             if len(tags) < tag_count:
                 # we removed tags...
                 if not set(tags).difference(expected_tags):

imposm/psqldb.py

@@ -24,6 +24,7 @@
 createdb -E UTF8 -O ${user} ${dbname}
 createlang plpgsql ${dbname}
 ${postgis}
+echo "CREATE EXTENSION hstore;" | psql -d ${dbname}
 echo "ALTER TABLE spatial_ref_sys OWNER TO ${user};" | psql -d ${dbname}
 echo "ALTER USER ${user} WITH PASSWORD '${password}';" |psql -d ${dbname}
 echo "host\t${dbname}\t${user}\t127.0.0.1/32\tmd5" >> ${pg_hba}