Skip to content

Commit

Permalink
Extend ElasticSearch proxy to filter out elements defined in the sche…
Browse files Browse the repository at this point in the history
…ma filters configurations (#6869)

* Extend ElasticSearch proxy to filter out fields with the withheld information from the query response

* Extend XmlSerializer to support 'authenticated' operation and filter out the xml elements

* Fix ifNotOperation typo in enumeration element in schema-ident.xsd

* MEF export - withheld elements in additional formats export

* Search / Avoid NPE when filtering withheld elements. We always need the documentStandard field to check the schema config.

* Indexing / Store nilReason attribute value instead of only withheld. Add nilReason for links (and not only contact).

---------

Co-authored-by: Francois Prunayre <fx.prunayre@gmail.com>
  • Loading branch information
josegar74 and fxprunayre authored Aug 30, 2023
1 parent b45a697 commit 9f43df8
Show file tree
Hide file tree
Showing 14 changed files with 431 additions and 202 deletions.
199 changes: 105 additions & 94 deletions core/src/main/java/org/fao/geonet/kernel/SchemaManager.java

Large diffs are not rendered by default.

42 changes: 26 additions & 16 deletions core/src/main/java/org/fao/geonet/kernel/XmlSerializer.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.fao.geonet.kernel.datamanager.IMetadataManager;
import org.fao.geonet.kernel.datamanager.IMetadataUtils;
import org.fao.geonet.kernel.schema.MetadataSchema;
import org.fao.geonet.kernel.schema.MetadataSchemaOperationFilter;
import org.fao.geonet.kernel.setting.SettingManager;
import org.fao.geonet.kernel.setting.Settings;
import org.fao.geonet.utils.Log;
Expand Down Expand Up @@ -70,13 +71,13 @@ public static void clearThreadLocal() {
}

public static void removeFilteredElement(Element metadata,
final Pair<String, Element> xPathAndMarkedElement,
final MetadataSchemaOperationFilter filter,
List<Namespace> namespaces) throws JDOMException {
// xPathAndMarkedElement seem can be null in some schemas like dublin core
if (xPathAndMarkedElement == null) return;
if (filter == null) return;

String xpath = xPathAndMarkedElement.one();
Element mark = xPathAndMarkedElement.two();
String xpath = filter.getXpath();
Element mark = filter.getMarkedElement();

List<?> nodes = Xml.selectNodes(metadata,
xpath,
Expand All @@ -89,13 +90,13 @@ public static void removeFilteredElement(Element metadata,

// Remove attributes
@SuppressWarnings("unchecked")
List<Attribute> atts = new ArrayList<Attribute>(element.getAttributes());
List<Attribute> atts = new ArrayList<>(element.getAttributes());
for (Attribute attribute : atts) {
attribute.detach();
}

// Insert attributes or children element of the mark
List<Attribute> markAtts = new ArrayList<Attribute>(mark.getAttributes());
List<Attribute> markAtts = new ArrayList<>(mark.getAttributes());
for (Attribute attribute : markAtts) {
element.setAttribute((Attribute) attribute.clone());
}
Expand Down Expand Up @@ -182,33 +183,42 @@ public Element removeHiddenElements(boolean isIndexingTask, AbstractMetadata met
// Check if a filter is defined for this schema
// for the editing operation ie. user who can not edit
// will not see those elements.
Pair<String, Element> editXpathFilter = mds.getOperationFilter(ReservedOperation.editing);
boolean filterEditOperationElements = editXpathFilter != null;
MetadataSchemaOperationFilter editFilter = mds.getOperationFilter(ReservedOperation.editing);
boolean filterEditOperationElements = editFilter != null;
List<Namespace> namespaces = mds.getNamespaces();
if (context != null) {
if (editXpathFilter != null) {
if (editFilter != null) {
boolean canEdit = accessManager.canEdit(context, id);
if (canEdit) {
filterEditOperationElements = false;
}
}
Pair<String, Element> downloadXpathFilter = mds.getOperationFilter(ReservedOperation.download);
if (downloadXpathFilter != null) {

MetadataSchemaOperationFilter authenticatedFilter = mds.getOperationFilter("authenticated");
if (authenticatedFilter != null) {
boolean isAuthenticated = context.getUserSession().isAuthenticated();
if (!isAuthenticated) {
removeFilteredElement(metadataXml, authenticatedFilter, namespaces);
}
}

MetadataSchemaOperationFilter downloadFilter = mds.getOperationFilter(ReservedOperation.download);
if (downloadFilter != null) {
boolean canDownload = accessManager.canDownload(context, id);
if (!canDownload) {
removeFilteredElement(metadataXml, downloadXpathFilter, namespaces);
removeFilteredElement(metadataXml, downloadFilter, namespaces);
}
}
Pair<String, Element> dynamicXpathFilter = mds.getOperationFilter(ReservedOperation.dynamic);
if (dynamicXpathFilter != null) {
MetadataSchemaOperationFilter dynamicFilter = mds.getOperationFilter(ReservedOperation.dynamic);
if (dynamicFilter != null) {
boolean canDynamic = accessManager.canDynamic(context, id);
if (!canDynamic) {
removeFilteredElement(metadataXml, dynamicXpathFilter, namespaces);
removeFilteredElement(metadataXml, dynamicFilter, namespaces);
}
}
}
if (filterEditOperationElements || (getThreadLocal(false) != null && getThreadLocal(false).forceFilterEditOperation)) {
removeFilteredElement(metadataXml, editXpathFilter, namespaces);
removeFilteredElement(metadataXml, editFilter, namespaces);
}
}
return metadataXml;
Expand Down
10 changes: 5 additions & 5 deletions core/src/main/java/org/fao/geonet/kernel/mef/ExportFormat.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.util.Set;

import org.fao.geonet.GeonetContext;
import org.fao.geonet.constants.Edit;
import org.fao.geonet.constants.Geonet;
import org.fao.geonet.domain.AbstractMetadata;
import org.fao.geonet.domain.Metadata;
Expand Down Expand Up @@ -72,7 +73,7 @@ public static Iterable<Pair<String, String>> getFormats(ServiceContext context,
String outputFileName = entry.getValue();
Path path = metadataSchema.getSchemaDir().resolve(xslFileName);
if (Files.isRegularFile(path)) {
String outputData = formatData(metadata, true, path);
String outputData = formatData(context, metadata, true, path);
allExports.add(Pair.read(outputFileName, outputData));
} else {
// A conversion that does not exist
Expand All @@ -96,10 +97,9 @@ public static Iterable<Pair<String, String>> getFormats(ServiceContext context,
*
* @return ByteArrayInputStream
*/
public static String formatData(AbstractMetadata metadata, boolean transform, Path stylePath) throws Exception {
String xmlData = metadata.getData();

Element md = Xml.loadString(xmlData, false);
public static String formatData(ServiceContext context, AbstractMetadata metadata, boolean transform, Path stylePath) throws Exception {
Element md = context.getBean(DataManager.class).getMetadata(context, metadata.getId() + "", false, false, true);
md.removeChild("info", Edit.NAMESPACE);

// Apply a stylesheet transformation when schema is ISO profil
if (transform) {
Expand Down
92 changes: 47 additions & 45 deletions core/src/main/java/org/fao/geonet/kernel/schema/MetadataSchema.java
Original file line number Diff line number Diff line change
@@ -1,29 +1,25 @@
//==============================================================================
//===
//=== MetadataSchema
//===
//==============================================================================
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
//=== This program is free software; you can redistribute it and/or modify
//=== it under the terms of the GNU General Public License as published by
//=== the Free Software Foundation; either version 2 of the License, or (at
//=== your option) any later version.
//===
//=== This program is distributed in the hope that it will be useful, but
//=== WITHOUT ANY WARRANTY; without even the implied warranty of
//=== MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
//=== General Public License for more details.
//===
//=== You should have received a copy of the GNU General Public License
//=== along with this program; if not, write to the Free Software
//=== Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
//===
//=== Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
//=== Rome - Italy. email: geonetwork@osgeo.org
//==============================================================================
/*
* Copyright (C) 2001-2023 Food and Agriculture Organization of the
* United Nations (FAO-UN), United Nations World Food Programme (WFP)
* and United Nations Environment Programme (UNEP)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or (at
* your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
*
* Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
* Rome - Italy. email: geonetwork@osgeo.org
*/

package org.fao.geonet.kernel.schema;

Expand All @@ -34,9 +30,9 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;

import org.apache.commons.lang.StringUtils;
import org.fao.geonet.api.exception.ResourceNotFoundException;
import org.fao.geonet.constants.Geonet;
import org.fao.geonet.domain.Pair;
import org.fao.geonet.domain.ReservedOperation;
import org.fao.geonet.domain.Schematron;
import org.fao.geonet.domain.SchematronCriteria;
Expand Down Expand Up @@ -82,15 +78,15 @@ public class MetadataSchema {
private static final String XSL_FILE_EXTENSION = ".xsl";
private static final String SCH_FILE_EXTENSION = ".sch";
private static final String SCHEMATRON_RULE_FILE_PREFIX = "schematron-rules";
private Map<String, List<String>> hmElements = new HashMap<String, List<String>>();
private Map<String, List<List<String>>> hmRestric = new HashMap<String, List<List<String>>>();
private Map<String, MetadataType> hmTypes = new HashMap<String, MetadataType>();
private Map<String, List<String>> hmSubs = new HashMap<String, List<String>>();
private Map<String, String> hmSubsLink = new HashMap<String, String>();
private Map<String, Namespace> hmNameSpaces = new HashMap<String, Namespace>();
private Map<String, Namespace> hmPrefixes = new HashMap<String, Namespace>();
private Map<String, Pair<String, Element>> hmOperationFilters =
new HashMap<String, Pair<String, Element>>();
private Map<String, List<String>> hmElements = new HashMap<>();
private Map<String, List<List<String>>> hmRestric = new HashMap<>();
private Map<String, MetadataType> hmTypes = new HashMap<>();
private Map<String, List<String>> hmSubs = new HashMap<>();
private Map<String, String> hmSubsLink = new HashMap<>();
private Map<String, Namespace> hmNameSpaces = new HashMap<>();
private Map<String, Namespace> hmPrefixes = new HashMap<>();
private Map<String, MetadataSchemaOperationFilter> hmOperationFilters =
new HashMap<>();
private String schemaName;
private Path schemaDir;
private String standardUrl;
Expand Down Expand Up @@ -294,8 +290,8 @@ void addElement(String name, String type, List<String> alValues, List<String> al
// first just add the subs - because these are for global elements we
// never have a clash because global elements are all in the same scope
// and are thus unique
if (alSubs != null && alSubs.size() > 0) hmSubs.put(name, alSubs);
if (subLink != null && subLink.length() > 0) hmSubsLink.put(name, subLink);
if (alSubs != null && !alSubs.isEmpty()) hmSubs.put(name, alSubs);
if (subLink != null && StringUtils.isNotBlank(subLink)) hmSubsLink.put(name, subLink);

List<String> exType = hmElements.get(name);

Expand All @@ -309,7 +305,8 @@ void addElement(String name, String type, List<String> alValues, List<String> al

// it's not there so add a new list
} else {
hmElements.put(name, exType = new ArrayList<String>());
exType = new ArrayList<>();
hmElements.put(name, exType);
}
exType.add(type);

Expand All @@ -323,7 +320,8 @@ void addElement(String name, String type, List<String> alValues, List<String> al

// it's not there so add a new list of lists
} else {
hmRestric.put(restricName, exValues = new ArrayList<List<String>>());
exValues = new ArrayList<>();
hmRestric.put(restricName, exValues);
}
exValues.add(alValues);
}
Expand Down Expand Up @@ -361,7 +359,7 @@ public String getNS(String targetNSPrefix) {
*/
@JsonIgnore
public List<Namespace> getNamespaces() {
List<Namespace> list = new ArrayList<Namespace>(hmNameSpaces.size());
List<Namespace> list = new ArrayList<>(hmNameSpaces.size());
for (Namespace ns : hmNameSpaces.values()) {
list.add(ns);
}
Expand All @@ -382,12 +380,12 @@ public String getPrefix(String theNSUri) {
//---------------------------------------------------------------------------
@JsonIgnore
public List<Namespace> getSchemaNS() {
return new ArrayList<Namespace>(hmPrefixes.values());
return new ArrayList<>(hmPrefixes.values());
}

@JsonProperty(value = "namespaces")
public Map<String, String> getSchemaNSWithPrefix() {
Map<String, String> mapNs = new HashMap<String, String>();
Map<String, String> mapNs = new HashMap<>();
List<Namespace> schemaNsList = getSchemaNS();

for (Namespace ns : schemaNsList) {
Expand Down Expand Up @@ -518,7 +516,7 @@ private void setSchematronPriorities() {
this.schemaRepo.saveAll(updated);
}

public void setOperationFilters(Map<String, Pair<String, Element>> operationFilters) {
public void setOperationFilters(Map<String, MetadataSchemaOperationFilter> operationFilters) {
this.hmOperationFilters = operationFilters;
}

Expand All @@ -527,10 +525,14 @@ public void setOperationFilters(Map<String, Pair<String, Element>> operationFilt
*
* @return The XPath to select element to filter or null
*/
public Pair<String, Element> getOperationFilter(ReservedOperation operation) {
public MetadataSchemaOperationFilter getOperationFilter(ReservedOperation operation) {
return hmOperationFilters.get(operation.name());
}

public MetadataSchemaOperationFilter getOperationFilter(String operation) {
return hmOperationFilters.get(operation);
}

@JsonIgnore
public SchemaPlugin getSchemaPlugin() {
return schemaPlugin;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Copyright (C) 2001-2023 Food and Agriculture Organization of the
* United Nations (FAO-UN), United Nations World Food Programme (WFP)
* and United Nations Environment Programme (UNEP)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or (at
* your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
*
* Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
* Rome - Italy. email: geonetwork@osgeo.org
*/
package org.fao.geonet.kernel.schema;

import org.jdom.Element;

public class MetadataSchemaOperationFilter {
private String xpath;
private String jsonpath;
private String ifNotOperation;
private Element markedElement;


public MetadataSchemaOperationFilter(String xpath, String jsonpath, String ifNotOperation) {
this(xpath, jsonpath, ifNotOperation, null);
}

public MetadataSchemaOperationFilter(String xpath, String jsonpath, String ifNotOperation, Element markedElement) {
this.xpath = xpath;
this.jsonpath = jsonpath;
this.ifNotOperation = ifNotOperation;
this.markedElement = markedElement;

}

public String getXpath() {
return xpath;
}

public String getJsonpath() {
return jsonpath;
}

public String getIfNotOperation() {
return ifNotOperation;
}

public Element getMarkedElement() {
return markedElement;
}
}
Loading

0 comments on commit 9f43df8

Please sign in to comment.