Merge branch 'dev'

ckanext/geodatagov/fanstatic_library/scripts/sorting.js

@@ -16,7 +16,8 @@ jQuery( document ).ready(function() {
         'dataset_type' : {'sortType': 'count', 'sort' : 'desc'},
         'harvest_source_title' : {'sortType': 'count', 'sort' : 'desc'},
         'frequency' : {'sortType': 'count', 'sort' : 'desc'},
-        'source_type' : {'sortType': 'count', 'sort' : 'desc'}
+        'source_type' : {'sortType': 'count', 'sort' : 'desc'},
+		'publisher' : {'sortType': 'count', 'sort' : 'desc'}
         //'extras_progress' : {'sortType': 'count', 'sort' : 'desc'}
     };
 
@@ -207,6 +208,9 @@ jQuery( document ).ready(function() {
         browserURL = browserURL.replace('_' + id + '_sortCnt=desc', '');
         browserURL = browserURL.replace('?&', '?');
         browserURL = browserURL.replace('&&', '&');
+
+		if(browserURL.slice(-1) == '&') 
+		   browserURL = browserURL.substring(0, browserURL.length - 1);
 
         $(this).parent().parent().find('p.module-footer a.read-more').attr('href', url);
 
@@ -297,6 +301,9 @@ jQuery( document ).ready(function() {
         browserURL = browserURL.replace('_' + id + '_sortAlpha=desc', '');
         browserURL = browserURL.replace('?&', '?');
         browserURL = browserURL.replace('&&', '&');
+
+		if(browserURL.slice(-1) == '&') 
+		   browserURL = browserURL.substring(0, browserURL.length - 1);
 
         $(this).parent().parent().find('p.module-footer a.read-more').attr('href', url);
 
@@ -406,9 +413,24 @@ jQuery( document ).ready(function() {
                 }
             });
 
+			var read_more_url = $("a[name='sm_" + id + "']").attr('href');
+			var flag = false;
+			if(read_more_url.indexOf("_" + id + "_limit=0") != -1)
+			   flag = true;		   
+
             $.each(listitems, function(idx, itm) {
-                if(cnt > 0)
+                if(cnt > 0) {
+				    if(flag == true) {
+					   var li_url = $(itm).find('a').attr('href');
+					   li_url = li_url.replace("_" + id + "_limit=0", "");
+
+					   if(li_url.slice(-1) == '&') 
+					      li_url = li_url.substring(0, li_url.length - 1);  
+
+					   $(itm).find('a').attr('href', li_url);
+					}
                     mylist1.append(itm);
+				}
                 cnt--;
             });
         });

ckanext/geodatagov/helpers.py

@@ -1,11 +1,14 @@
 import urllib, urllib2, json, re, HTMLParser
+from urllib2 import Request, urlopen, URLError, HTTPError
 import os, time
 import logging
 
 from pylons import config
 
 from ckan import plugins as p
 from ckan.lib import helpers as h
+import ckan.lib.datapreview as datapreview
+#from routes import url_for as _routes_default_url_for
 
 log = logging.getLogger(__name__)
 
@@ -25,6 +28,14 @@ def render_datetime_datagov(date_str):
 def get_validation_profiles():
     return VALIDATION_PROFILES
 
+def get_validation_schema():
+    try:
+        from ckanext.datajson.harvester_base import VALIDATION_SCHEMA
+    except ImportError:
+        return None
+
+    return VALIDATION_SCHEMA
+
 def get_harvest_object_formats(harvest_object_id):
     try:
         obj = p.toolkit.get_action('harvest_object_show')({}, {'id': harvest_object_id})
@@ -142,8 +153,8 @@ def resource_preview_custom(resource, pkg_id):
                'resource_url': url,
                'raw_resource_url': resource['url']})
 
-    return h.resource_preview(resource, pkg_id)
-
+    return h.resource_preview(resource, pkg_id)	
+	
 types = {
     'web': ('html', 'data', 'esri rest', 'gov', 'org', ''),
     'preview': ('csv', 'xls', 'txt', 'jpg', 'jpeg', 'png', 'gif'),
@@ -158,7 +169,64 @@ def is_type_format(type, resource):
             return True
     return False
 
+def is_preview_available(resource, pkg_id):
 
+    if not resource.get('url', 'data'):
+        return False
+
+    format_lower = resource.get('format', 'data').lower()
+
+    if (format_lower in ['csv', 'txt']):
+     type = 'csv'
+    elif (format_lower == 'xls'):
+	 type = 'xls'
+    elif (format_lower in ['jpg', 'jpeg', 'png', 'gif']):
+	 type = format_lower
+    else:
+	 return False
+
+    direct_embed = config.get('ckan.preview.direct', '').split()
+    if not direct_embed:
+        direct_embed = datapreview.DEFAULT_DIRECT_EMBED
+
+    loadable_in_iframe = config.get('ckan.preview.loadable', '').split()
+    if not loadable_in_iframe:
+        loadable_in_iframe = datapreview.DEFAULT_LOADABLE_IFRAME
+
+    package = p.toolkit.get_action('package_show')({}, {'id': pkg_id})
+    data_dict = {'resource': resource, 'package': package}	
+
+    if (not datapreview.can_be_previewed(data_dict) and format_lower not in direct_embed and format_lower not in loadable_in_iframe):
+	 return False
+
+    if(format_lower in direct_embed):
+     return True
+
+    url = "http://jsonpdataproxy.appspot.com/?callback=result&url=" + resource.get('url') + "&max-results=1000&type=" + type + "&_=" + str(int(time.time()))
+
+    return preview_response(url)
+
+def preview_response(url):
+
+    req = Request(url)
+    try:
+      response = urlopen(req)
+    except HTTPError as e:
+      log.error("The server couldn\'t fulfill the request. Error code: " + e.code)
+      return False
+    except URLError as e:
+      log.error("We failed to reach a server. Reason: " + e.reason)
+      return False
+    else:
+      f = response.read()
+      data = json.loads(f[7:-1]) 
+      error_exists = data.get('error', 0)
+
+    if (error_exists != 0):
+      return False
+
+    return True
+
 def is_web_format(resource):
     return is_type_format('web', resource)
 

ckanext/geodatagov/plugins.py

@@ -5,6 +5,7 @@
 import mimetypes
 from paste.auth.auth_tkt import maybe_encode, encode_ip_timestamp
 from pylons import request
+import ckan.lib.datapreview as datapreview
 
 mimetypes.add_type('application/vnd.ms-fontobject', '.eot')
 
@@ -45,6 +46,7 @@ def ping_connection(dbapi_connection, connection_record, connection_proxy):
 from ckan.logic.converters import convert_from_extras
 from ckan.lib.navl.validators import ignore_missing
 from sqlalchemy.util import OrderedDict
+import ckanext.geodatagov.commands as cs
 
 log = logging.getLogger(__name__)
 
@@ -156,6 +158,7 @@ def dataset_facets(self, facets_dict, package_type):
                             ('frequency', 'Frequency'),
                             ('source_type','Type'),
                             ('organization', 'Organizations'),
+                            #('publisher', 'Publisher'),							
                            ])
 
     def organization_facets(self, facets_dict, organization_type, package_type):
@@ -165,6 +168,7 @@ def organization_facets(self, facets_dict, organization_type, package_type):
 
         return OrderedDict([('frequency', 'Frequency'),
                             ('source_type','Type'),
+                            #('publisher', 'Publisher'),
                            ])
 
 def get_filename_and_extension(resource):
@@ -382,11 +386,13 @@ def get_helpers(self):
                 'get_harvest_object_formats': geodatagov_helpers.get_harvest_object_formats,
                 'get_harvest_source_link': geodatagov_helpers.get_harvest_source_link,
                 'get_validation_profiles': geodatagov_helpers.get_validation_profiles,
+                'get_validation_schema': geodatagov_helpers.get_validation_schema,
                 'get_collection_package': geodatagov_helpers.get_collection_package,
                 'resource_preview_custom': geodatagov_helpers.resource_preview_custom,
                 'is_web_format': geodatagov_helpers.is_web_format,
                 'saml2_user_edit_url': self.saml2_user_edit_url,
                 'is_preview_format': geodatagov_helpers.is_preview_format,
+                'is_preview_available': geodatagov_helpers.is_preview_available,
                 'is_map_format': geodatagov_helpers.is_map_format,
                 'is_map_viewer_format' : geodatagov_helpers.is_map_viewer_format,
                 'get_map_viewer_params': geodatagov_helpers.get_map_viewer_params,
@@ -438,7 +444,8 @@ def dataset_facets(self, facets_dict, package_type):
                             ('groups', 'Topics'),
                             ('organization_type', 'Organization Types'),
                             ('organization', 'Organizations'),
-                            ('vocab_category_all', 'Topic Categories'),
+                            ('publisher', 'Publisher'),
+                            ('vocab_category_all', 'Topic Categories'),                            
                            ## ('extras_progress', 'Progress'),
                            ])
 
@@ -452,6 +459,7 @@ def organization_facets(self, facets_dict, organization_type, package_type):
                                 ('harvest_source_title', 'Harvest Source'),
                                 ('capacity', 'Visibility'),
                                 ('dataset_type', 'Resource Type'),
+                                ('publisher', 'Publisher'),
                                ])
         else:
             return facets_dict
@@ -468,6 +476,8 @@ def group_facets(self, facets_dict, organization_type, package_type):
                                 ('res_format', 'Formats'),
                                 ('organization', 'Organizations'),
                                 (key, 'Categories'),
+                                #('publisher', 'Publisher'),
                                ])
         else:
             return facets_dict
+

ckanext/geodatagov/templates/package/read.html

@@ -133,7 +133,14 @@ <h3>{{ _('About this Dataset') }}</h3>
           </tr>
           <tr>
             <th scope="row" class="dataset-label">Maintainer Email</th>
-            <td>{{ pkg_dict.maintainer_email }}</td>
+            <td>
+             {% set maintainer_email = pkg_dict.maintainer_email %}
+			 {% if maintainer_email %}
+              {{ h.mail_to(email_address=pkg_dict.maintainer_email) }}
+             {% else %}
+              {{ maintainer_email }} 	
+             {% endif %}
+			</td>
           </tr>
         {% else %}
           {% set exclude_list = ['metadata-date', 'dataset-reference-date', 'responsible-party', 'contact-email', 'resource-type'] %}
@@ -147,7 +154,14 @@ <h3>{{ _('About this Dataset') }}</h3>
           </tr>
           <tr>
             <th scope="row" class="dataset-label">Contact Email</th>
-            <td>{{ h.get_pkg_dict_extra(c.pkg_dict, 'contact-email', 'Unknown') }}</td>
+            <td>			 
+             {% set contact_email = h.get_pkg_dict_extra(c.pkg_dict, 'contact-email', 'Unknown') %}
+             {% if contact_email == 'Unknown'%}
+              {{contact_email}}
+             {% else %}
+              {{ h.mail_to(email_address=contact_email) }}	
+             {% endif %}
+            </td>
           </tr>
         {% endif %}
 

ckanext/geodatagov/templates/package/search.html

@@ -76,7 +76,7 @@
             {% endtrans %}
           {% endif %}
           {% if c.q != '' %}
-            <p class="extra">You are searching in catalog.data.gov. Show results in <a href="http://www.data.gov/search-results/?group=site&q={{c.q}}">entire site</a>.</p>
+            <p class="extra">You are searching in the list of datasets. Show results in <a href="http://www.data.gov/search-results/?group=site&q={{c.q}}">entire site</a>.</p>
           {% endif %}
         </div>
       </form>

ckanext/geodatagov/templates/package/snippets/resource_item.html

@@ -35,7 +35,7 @@
     {% elif is_web_format %}
     <i class="icon-external-link"></i>
     {{ _('Visit page') }}
-    {% elif is_preview_format %}
+    {% elif is_preview_format and h.is_preview_available(res, pkg.id)  %}
     <i class="icon-search"></i>
     {{ _('Preview') }}
     {% else %}

ckanext/geodatagov/templates/source/geodatagov_source_form.html

@@ -28,6 +28,18 @@
            </div>
          </div>
 
+         <div class="control-group harvest-types">
+           <label class="control-label">Validation Schema</label>
+           <div class="controls">
+             {% for key, value in h.get_validation_schema() %}
+               {% set checked = key == (data.validator_schema or '')  %}
+               <label class="radio">
+                 <input type="radio" name="validator_schema" value="{{ key }}" {{ "checked " if checked }}>
+                 {{ value }}
+               </label>
+             {% endfor %}
+           </div>
+         </div>
 
 </fieldset>
 

deployment/etc/ckan/production.ini

@@ -76,7 +76,7 @@ ckan.resource_proxy_enabled = 1
 
 search.facets = groups tags res_format responsible-party frequency source_type
 search.facets.responsible-party.title = Organizations
-ckan.search.solr_commit = false
+ckan.search.solr_commit = true
 
 ckan.spatial.validator.profiles = iso19139ngdc