Add configurable user agent for retrieving metadata

OpenConext · Dec 19, 2023 · b450ab8 · b450ab8
1 parent 611d835
commit b450ab8
Show file tree

Hide file tree

Showing 7 changed files with 51 additions and 10 deletions.
diff --git a/manage-server/src/main/java/manage/format/SaveURLResource.java b/manage-server/src/main/java/manage/format/SaveURLResource.java
@@ -2,19 +2,34 @@
 
 import org.springframework.core.io.UrlResource;
 
+import java.io.IOException;
 import java.net.URL;
+import java.net.URLConnection;
 import java.util.Arrays;
 import java.util.List;
 
 public class SaveURLResource extends UrlResource {
 
+    private static final String USER_AGENT_KEY = "User-Agent";
+
     private final List<String> allowedProtocols = Arrays.asList("http", "https");
 
-    public SaveURLResource(URL url, boolean dev) {
+    private final String userAgent;
+
+    public SaveURLResource(URL url, boolean dev, String userAgent) {
         super(url);
+        this.userAgent = userAgent;
         String protocol = url.getProtocol();
         if (!dev && !allowedProtocols.contains(protocol)) {
             throw new IllegalArgumentException(String.format("Not allowed protocol %s - allowed protocols are %s", protocol, allowedProtocols));
         }
     }
+
+    @Override
+    protected void customizeConnection(URLConnection con) throws IOException {
+        super.customizeConnection(con);
+        if (null != userAgent && !userAgent.isEmpty()) {
+            con.setRequestProperty(USER_AGENT_KEY, userAgent);
+        }
+    }
 }
diff --git a/manage-server/src/main/java/manage/service/ImporterService.java b/manage-server/src/main/java/manage/service/ImporterService.java
@@ -46,8 +46,11 @@ public class ImporterService {
 
     private TypeSafetyHook metaDataHook;
 
+    private String autoRefreshUserAgent;
+
     public ImporterService(MetaDataAutoConfiguration metaDataAutoConfiguration, Environment environment,
-                           @Value("${product.supported_languages}") String supportedLanguages) {
+                           @Value("${product.supported_languages}") String supportedLanguages,
+                           @Value("${metadata_import.useragent}") String autoRefreshUserAgent) {
 
         this.metaDataAutoConfiguration = metaDataAutoConfiguration;
         this.environment = environment;
@@ -56,12 +59,13 @@ public ImporterService(MetaDataAutoConfiguration metaDataAutoConfiguration, Envi
                 supportedLanguages.split(","))
                 .map(String::trim)
                 .collect(toList()));
+        this.autoRefreshUserAgent = autoRefreshUserAgent;
     }
 
     public Map<String, Object> importXMLUrl(EntityType type, Import importRequest) {
         try {
             Resource resource = new SaveURLResource(new URL(importRequest.getUrl()),
-                    environment.acceptsProfiles(Profiles.of("dev")));
+                    environment.acceptsProfiles(Profiles.of("dev")), autoRefreshUserAgent);
             Map<String, Object> result = importXML(resource, type, Optional
                     .ofNullable(importRequest.getEntityId()));
             if (result.isEmpty()) {
@@ -99,7 +103,7 @@ public Map<String, Object> importJson(String type, Map<String, Object> json) thr
     public Map<String, Object> importJsonUrl(String type, Import importRequest) {
         try {
             Resource resource = new SaveURLResource(new URL(importRequest.getUrl()),
-                    environment.acceptsProfiles(Profiles.of("dev")));
+                    environment.acceptsProfiles(Profiles.of("dev")), autoRefreshUserAgent);
             String json = IOUtils.toString(resource.getInputStream(), Charset.defaultCharset());
             Map<String, Object> map = objectMapper.readValue(json, Map.class);
             return importJson(type, map);

diff --git a/manage-server/src/main/java/manage/service/MetaDataService.java b/manage-server/src/main/java/manage/service/MetaDataService.java
@@ -107,7 +107,7 @@ public Map<String, List> importFeed(Import importRequest) {
                             .map(ServiceProvider::new)
                             .collect(Collectors.toMap(ServiceProvider::getEntityId, sp -> sp));
             String feedUrl = importRequest.getUrl();
-            Resource resource = new SaveURLResource(new URL(feedUrl), environment.acceptsProfiles(Profiles.of("dev")));
+            Resource resource = new SaveURLResource(new URL(feedUrl), environment.acceptsProfiles(Profiles.of("dev")), null);
 
             List<Map<String, Object>> allImports = importerService.importFeed(resource);
             List<Map<String, Object>> imports =

diff --git a/manage-server/src/main/resources/application.yml b/manage-server/src/main/resources/application.yml
@@ -102,6 +102,7 @@ gui:
     content: LOCAL
 
 metadata_import:
+  useragent: ""
   auto_refresh:
     cronSchedule: "-"
 

diff --git a/manage-server/src/test/java/manage/format/SaveURLResourceTest.java b/manage-server/src/test/java/manage/format/SaveURLResourceTest.java
@@ -1,19 +1,38 @@
 package manage.format;
 
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
 
+import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.net.URLConnection;
 
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+@RunWith(MockitoJUnitRunner.class)
 public class SaveURLResourceTest {
 
+    @Mock
+    URLConnection connection;
+
     @Test(expected = IllegalArgumentException.class)
     public void testNotAllowedProtocol() throws MalformedURLException {
-        SaveURLResource resource = new SaveURLResource(new URL("file://local"), false);
+        SaveURLResource resource = new SaveURLResource(new URL("file://local"), false, null);
     }
 
     @Test
     public void testNotAllowedProtocolInDev() throws MalformedURLException {
-        new SaveURLResource(new URL("file://local"), true);
+        new SaveURLResource(new URL("file://local"), true, null);
+    }
+
+    @Test
+    public void testUseUserAgent() throws IOException {
+        SaveURLResource resource = new SaveURLResource(new URL("https://external"), false, "user agent");
+        resource.customizeConnection(connection);
+        verify(connection, times(1)).setRequestProperty("User-Agent", "user agent");
     }
 }
diff --git a/manage-server/src/test/java/manage/service/ImporterServiceTest.java b/manage-server/src/test/java/manage/service/ImporterServiceTest.java
@@ -32,7 +32,8 @@ public class ImporterServiceTest implements TestUtils {
                     new ClassPathResource("metadata_configuration"),
                     new ClassPathResource("metadata_templates")),
                     new MockEnvironment(),
-                    "nl,pt,en");
+                    "nl,pt,en",
+                    null);
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
@@ -150,7 +151,7 @@ public void multiplicity() throws IOException, XMLStreamException {
         MetaDataAutoConfiguration metaDataAutoConfiguration = new MetaDataAutoConfiguration(objectMapper, new ClassPathResource("metadata_configuration"), new ClassPathResource("metadata_templates"));
         Map<String, Object> spSchema = metaDataAutoConfiguration.schemaRepresentation(EntityType.SP);
         Map.class.cast(Map.class.cast(Map.class.cast(Map.class.cast(spSchema.get("properties")).get("metaDataFields")).get("patternProperties")).get("^AssertionConsumerService:([0-3]{0,1}[0-9]{1}):index$")).put("multiplicity", 15);
-        ImporterService alteredSubject = new ImporterService(metaDataAutoConfiguration, new MockEnvironment(), "nl,en,pt");
+        ImporterService alteredSubject = new ImporterService(metaDataAutoConfiguration, new MockEnvironment(), "nl,en,pt", null);
         Map<String, Object> metaData = alteredSubject.importXML(new ClassPathResource("import_xml/assertion_consumer_service.15.xml"), EntityType.SP, Optional.empty());
         Set<Map.Entry> metaDataFields = Map.class.cast(metaData.get("metaDataFields"))
                 .entrySet();

diff --git a/manage-server/src/test/resources/test.properties b/manage-server/src/test/resources/test.properties
@@ -48,4 +48,5 @@ spring.mail.host=localhost
 spring.mail.port=3025
 spring.data.mongodb.uri=mongodb://localhost:27017/metadata_test
 spring.main.banner-mode=off
-metadata_import.auto_refresh.cronSchedule=0 0 0 30 2 *
+metadata_import.auto_refresh.cronSchedule=0 0 0 30 2 *
+metadata_import.useragent=""