Merge branch 'develop' into custom-migration

MigrationGuide.md

@@ -1,5 +1,9 @@
 # Migration guide
 
+## 2.14.0
+
+Frontend package.json scripts need to be updated due to webpack-cli upgrade. The scripts like start, build previously passed variables to build process like this `--env.appdef=applications`. These need to be modified by changing the dot to a space like this `--env appdef=applications`. Passing similar env-variables from command line also need to be passed without the dot.
+
 ## 2.13.0
 
 ### GeoServer dependency removed

ReleaseNotes.md

@@ -1,5 +1,18 @@
 # Release Notes
 
+## 2.13.1
+
+For a full list of changes see:
+https://github.com/oskariorg/oskari-server/milestone/50?closed=1
+
+- Improved Redis credentials handling: https://github.com/oskariorg/oskari-server/pull/1065
+- Fixed an issue with csw-metadata fetching
+- Added a flag for server to follow redirects automatically
+- Updated libraries:
+    - Spring framework: 5.3.32 -> 5.3.35
+    - Spring security: 5.7.11 -> 5.7.12
+    - commons-compress: 1.25.0 -> 1.26.1
+
 ## 2.13.0
 
 For a full list of changes see:

control-base/src/main/java/fi/nls/oskari/control/data/GetCSWDataHandler.java

@@ -26,6 +26,7 @@
 import java.util.Map;
 import org.json.JSONArray;
 import org.json.JSONObject;
+import org.oskari.capabilities.ogc.LayerCapabilitiesOGC;
 import org.oskari.service.util.ServiceFactory;
 
 import static fi.nls.oskari.csw.service.CSWService.PROP_SERVICE_URL;
@@ -99,7 +100,7 @@ public void handleAction(ActionParameters params) throws ActionException {
         if (layerId != -1) {
             OskariLayer layer = layerService.find(layerId);
             final JSONObject attributes = layer.getAttributes();
-            uuid = layer.getMetadataId();
+            uuid = getMetadataIdForLayer(layer);
 
             try {
                 if (attributes.has(METADATA_URL_PARAM)) {
@@ -146,6 +147,16 @@ record = service.getRecordById(uuid, lang);
 
         ResponseHelper.writeResponse(params, result);
     }
+
+    private String getMetadataIdForLayer(OskariLayer layer) {
+        String uuid = layer.getMetadataId();
+        if (uuid != null && !uuid.trim().isEmpty()) {
+            // override metadataid
+            return uuid;
+        }
+        // uuid from capabilities
+        return layer.getCapabilities().optString(LayerCapabilitiesOGC.METADATA_UUID, null);
+    }
 
     private void prefixImageFilenames(CSWIsoRecord record, final String uuid, final String locale) {
         // This only works for GN2 for paikkatietohakemisto.fi

control-base/src/main/java/fi/nls/oskari/control/layer/GetLayerTileHandler.java

@@ -154,13 +154,13 @@ public void handleAction(final ActionParameters params)
             // just throw it as is if we already handled it
             throw e;
         } catch (Exception e) {
-            LOG.info("Url in proxy error was:", url);
-            throw new ActionParamsException("Couldn't proxy request to actual service", e.getMessage(), e);
+            LOG.debug("Url in proxy error was:", url);
+            throw new ActionCommonException("Couldn't proxy request to actual service: " +  e.getMessage(), e);
         } finally {
-            if(actionTimer != null) {
+            if (actionTimer != null) {
                 actionTimer.stop();
             }
-            if(con != null) {
+            if (con != null) {
                 con.disconnect();
             }
         }

pom.xml

@@ -49,8 +49,8 @@
         <!-- Value comes from profile when needed -->
         <module.build.config></module.build.config>
 
-        <spring.version>5.3.32</spring.version>
-        <spring-security.version>5.7.11</spring-security.version>
+        <spring.version>5.3.35</spring.version>
+        <spring-security.version>5.7.12</spring-security.version>
         <!--
         session-bom 2021.2.3 uses 2.7.4 from session
         https://github.com/spring-projects/spring-session/blob/2.7.4/gradle/dependency-management.gradle
@@ -83,6 +83,7 @@ The 2.15 doesn't report any breaking changes so its backwards compatible with 2.
         <commons-io.version>2.15.0</commons-io.version>
         <commons-csv.version>1.10.0</commons-csv.version>
         <xmlgraphics-fop.version>2.9</xmlgraphics-fop.version>
+        <!-- Note! Check commons-compress override when updating this -->
         <poi-ooxml.version>5.2.5</poi-ooxml.version>
 
         <jsoup.version>1.17.2</jsoup.version>
@@ -143,6 +144,15 @@ The 2.15 doesn't report any breaking changes so its backwards compatible with 2.
     <!-- Managed dependencies -->
     <dependencyManagement>
         <dependencies>
+            <!-- version 1.25.0 is transitive from org.apache.poi/poi-ooxml 5.2.5
+            It has a vulnerability that has been patched in comporess 1.26.1 -->
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-compress</artifactId>
+                <version>1.26.1</version>
+            </dependency>
+
+
             <!-- Oskari -->
             <dependency>
                 <groupId>org.oskari</groupId>

service-base/src/main/java/fi/nls/oskari/cache/JedisManager.java

@@ -36,6 +36,19 @@ public class JedisManager {
      */
     private JedisManager() {}
 
+    public static HostAndPort getHostAndPort() {
+        return new HostAndPort(JedisManager.getHost(), JedisManager.getPort());
+    }
+
+    public static JedisClientConfig getClientConfig() {
+        return DefaultJedisClientConfig.builder()
+                .connectionTimeoutMillis(getConnectionTimeoutMs())
+                .user(getUser())
+                .password(getPassword())
+                .ssl(getUseSSL())
+                .build();
+    }
+
     public static String getHost() {
         return PropertyUtil.get(KEY_REDIS_HOSTNAME, "localhost");
     }
@@ -51,6 +64,9 @@ private static int getConnectionTimeoutMs() {
     private static String getPassword() {
         return PropertyUtil.get("redis.password", null);
     }
+    private static String getUser() {
+        return PropertyUtil.get("redis.user", null);
+    }
     private static boolean getUseSSL() {
         return PropertyUtil.getOptional("redis.ssl", false);
     }
@@ -80,7 +96,7 @@ public static void connect(final int poolSize, final String host, final int port
         poolConfig.setTestOnBorrow(true);
         poolConfig.setBlockWhenExhausted(getBlockWhenExhausted());
         final JedisPool oldPool = pool;
-        pool = new JedisPool(poolConfig, host, port, getConnectionTimeoutMs(), getPassword(), getUseSSL());
+        pool = new JedisPool(poolConfig, host, port, getConnectionTimeoutMs(), getUser(), getPassword(), getUseSSL());
         // Should we use the long format to have an option to pass "client name" to Redis to help debugging issues with shared Redis instances?
         // pool = new JedisPool(poolConfig, host, port, getConnectionTimeoutMs(), getSocketReadTimeoutMs(), getPassword(), Protocol.DEFAULT_DATABASE, getClientName());
 

service-base/src/main/java/fi/nls/oskari/domain/map/OskariLayer.java

@@ -321,6 +321,13 @@ public void setUrl(String url) {
         this.simplifiedUrl = null;
     }
 
+    /**
+     * Returns the override for metadata id. To get the one the service is referencing use:
+     *   getCapabilities().optString(LayerCapabilitiesOGC.METADATA_UUID, null);
+     * We can't override this to return the value from capabilities since the admin ui
+     * needs to know where the metadata id came from: capabilities or override
+     * @return
+     */
     public String getMetadataId() {
         return metadataId;
     }

service-base/src/main/java/org/oskari/cluster/ClusterClient.java

@@ -4,8 +4,7 @@
 import fi.nls.oskari.log.LogFactory;
 import fi.nls.oskari.log.Logger;
 import fi.nls.oskari.util.OskariRuntimeException;
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.JedisPubSub;
+import redis.clients.jedis.*;
 
 import java.util.*;
 import java.util.concurrent.ExecutorService;
@@ -153,7 +152,7 @@ private void startListening(String prefix) {
     // NOTE!! create a new client for subscriptions instead of using pool to make sure clients don't conflict
     private Jedis createClient() {
         if (client == null) {
-            client = new Jedis(JedisManager.getHost(), JedisManager.getPort());
+            client = new Jedis(JedisManager.getHostAndPort(), JedisManager.getClientConfig());
         }
         return client;
     }

service-map/src/main/java/fi/mml/map/mapwindow/util/OskariLayerWorker.java

@@ -80,7 +80,20 @@ public static JSONObject getListOfMapLayersById(final List<Integer> layerIdList,
      */
     public static JSONObject getListOfMapLayers(final List<OskariLayer> layers, final User user,
             final String lang, final String crs, final boolean isPublished, final boolean isSecure) {
-        List<Resource> resources = permissionService.findResourcesByUser(user, ResourceType.maplayer);
+        List<Resource> resources;
+        if (layers.size() < 20) {
+            // usually the case with loading the default app setup
+            resources = layers.stream()
+                    .map(OskariLayer::getId)
+                    .map(id -> Integer.toString(id))
+                    .map(id -> permissionService.findResource(ResourceType.maplayer, id))
+                    .filter(Optional::isPresent)
+                    .map(Optional::get)
+                    .collect(Collectors.toList());
+        } else {
+            // more than 20 layers, just get all the permissions
+            resources = permissionService.findResourcesByUser(user, ResourceType.maplayer);
+        }
         return getListOfMapLayers(layers, user, lang, isSecure, crs, isPublished, new PermissionSet(resources));
     }
 

service-permissions/src/main/java/org/oskari/permissions/PermissionServiceMybatisImpl.java

@@ -9,8 +9,10 @@
 import fi.nls.oskari.log.Logger;
 import fi.nls.oskari.log.LogFactory;
 import fi.nls.oskari.mybatis.MyBatisHelper;
+import org.apache.ibatis.session.Configuration;
 import org.apache.ibatis.session.SqlSession;
 import org.apache.ibatis.session.SqlSessionFactory;
+import org.apache.ibatis.session.SqlSessionFactoryBuilder;
 import org.oskari.permissions.model.Permission;
 import org.oskari.permissions.model.PermissionExternalType;
 import org.oskari.permissions.model.Resource;
@@ -42,11 +44,18 @@ public PermissionServiceMybatisImpl(DataSource ds) {
             LOG.warn("DataSource was null, all future calls will throw NPEs!");
             factory = null;
         } else {
-            factory = MyBatisHelper.initMyBatis(ds, MAPPER);
+            factory = initializeMyBatis(ds);
         }
         cache = CacheManager.getCache(PermissionServiceMybatisImpl.class.getName());
     }
 
+    private SqlSessionFactory initializeMyBatis(final DataSource dataSource) {
+        final Configuration configuration = MyBatisHelper.getConfig(dataSource);
+        MyBatisHelper.addAliases(configuration, Resource.class, Permission.class);
+        MyBatisHelper.addMappers(configuration, MAPPER);
+        return new SqlSessionFactoryBuilder().build(configuration);
+    }
+
     public List<Resource> findResourcesByUser(User user, ResourceType type) {
         // TODO: add userId and user.getRoles() to query and remove filtering on code
         List<Resource> all = findResourcesByType(type);

service-permissions/src/main/java/org/oskari/permissions/ResourceMapper.java

@@ -3,6 +3,7 @@
 import java.util.List;
 import java.util.Set;
 
+import fi.nls.oskari.domain.map.userlayer.UserLayerData;
 import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Insert;
 import org.apache.ibatis.annotations.Many;
@@ -19,46 +20,15 @@
 
 public interface ResourceMapper {
 
-    // TODO: Join tables in SQL and map the result -> improves performance by ~5x
-    @Results(id = "ResourceResult", value = {
-            @Result(property="id", column="id", id=true),
-            @Result(property="type", column="resource_type"),
-            @Result(property="mapping", column="resource_mapping"),
-            @Result(property="permissions", column="id",
-                    javaType=List.class, many=@Many(select="findPermissionsByResourceId", fetchType= FetchType.EAGER))
-    })
-    @Select("SELECT id,"
-            + "resource_type,"
-            + "resource_mapping "
-            + "FROM oskari_resource "
-            + "WHERE id = #{id}")
     Resource findById(@Param("id") int id);
 
     @Select("SELECT EXISTS (SELECT 1 FROM oskari_resource WHERE id = #{id})")
     boolean existsById(@Param("id") int id);
 
-    @ResultMap("ResourceResult")
-    @Select("SELECT id,"
-            + "resource_type,"
-            + "resource_mapping "
-            + "FROM oskari_resource")
     List<Resource> findAll();
 
-    @ResultMap("ResourceResult")
-    @Select("SELECT id,"
-            + "resource_type,"
-            + "resource_mapping "
-            + "FROM oskari_resource "
-            + "WHERE resource_type = #{type}")
-    List<Resource> findByType(String type);
+    List<Resource> findByType(@Param("type") String type);
 
-    @ResultMap("ResourceResult")
-    @Select("SELECT id,"
-            + "resource_type,"
-            + "resource_mapping "
-            + "FROM oskari_resource "
-            + "WHERE resource_type = #{type} "
-            + "AND resource_mapping = #{mapping}")
     Resource findByTypeAndMapping(@Param("type") String type, @Param("mapping") String mapping);
 
     @Select("SELECT EXISTS (SELECT 1 FROM oskari_resource WHERE resource_type = #{type} AND resource_mapping = #{mapping})")
@@ -79,20 +49,6 @@ Set<String> findMappingsForPermission(@Param("resourceType") String resourceType
                                           @Param("permission") String permission,
                                           @Param("external_id") String external_id);
 
-    @Results({
-        @Result(property="id", column="id", id=true),
-        @Result(property="type", column="permission"),
-        @Result(property="externalType", column="external_type"),
-        @Result(property="externalId", column="external_id")
-    })
-    @Select("SELECT id,"
-            + "external_type,"
-            + "permission,"
-            + "external_id "
-            + "FROM oskari_resource_permission "
-            + "WHERE resource_id = #{resourceId}")
-    List<Permission> findPermissionsByResourceId(@Param("resourceId") int resourceId);
-
     @Insert("INSERT INTO oskari_resource (resource_type, resource_mapping) VALUES (#{type},#{mapping})")
     @Options(useGeneratedKeys=true, keyColumn="id", keyProperty="id")
     void insertResource(Resource resource);

service-permissions/src/main/java/org/oskari/permissions/model/Permission.java

@@ -24,7 +24,6 @@ public String getType() {
 
     public void setType(String type) {
         this.type = type;
-
     }
 
     public void setType(PermissionType type) {
@@ -39,6 +38,10 @@ public void setExternalType(String externalType) {
         setExternalType(PermissionExternalType.valueOf(externalType));
     }
 
+    public void setExternalTypeMybatis(String externalType) {
+        setExternalType(PermissionExternalType.valueOf(externalType));
+    }
+
     public void setExternalType(PermissionExternalType externalType) {
         this.externalType = externalType;
     }
@@ -55,6 +58,11 @@ public void setExternalId(String externalId) {
         setExternalId(Integer.parseInt(externalId));
     }
 
+    public void setExternalIdMybatis(String externalId) {
+        setExternalId(externalId);
+    }
+
+
     public void setRoleId(int roleId) {
         setExternalType(PermissionExternalType.ROLE);
         setExternalId(roleId);

service-permissions/src/main/java/org/oskari/permissions/model/Resource.java

@@ -53,7 +53,9 @@ public void setMapping(String namespace, String name) {
 
     public List<Permission> getPermissions() {
         if (permissions == null) {
-            return Collections.emptyList();
+            // mybatis calls get to fill in the results to
+            // -> we can't use Collections.emptyList() here
+            permissions = new ArrayList<>();
         }
         return permissions;
     }