From 17cc4d1e2b2b11c0b9e964366b4d72a724b486af Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 7 Aug 2023 15:50:41 -0400 Subject: [PATCH] Solr package for Splainer. (#97) The Splainer webapp wrapped up as a Solr package. There are still some manual steps on the release process to be worked out. --- .gitignore | 3 + CHANGELOG.md | 5 + solr-splainer-package/README.md | 88 +++++++++++ solr-splainer-package/pom.xml | 112 ++++++++++++++ solr-splainer-package/release.py | 53 +++++++ solr-splainer-package/repo/publickey.der | Bin 0 -> 94 bytes solr-splainer-package/repo/repository.json | 18 +++ .../com/o19s/splainer/SplainerHandler.java | 139 ++++++++++++++++++ .../src/main/resources/manifest.json | 37 +++++ 9 files changed, 455 insertions(+) create mode 100644 CHANGELOG.md create mode 100644 solr-splainer-package/README.md create mode 100644 solr-splainer-package/pom.xml create mode 100755 solr-splainer-package/release.py create mode 100644 solr-splainer-package/repo/publickey.der create mode 100644 solr-splainer-package/repo/repository.json create mode 100644 solr-splainer-package/src/main/java/com/o19s/splainer/SplainerHandler.java create mode 100644 solr-splainer-package/src/main/resources/manifest.json diff --git a/.gitignore b/.gitignore index 9b4fc9b..aec1346 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ deploy.sh *.swp docker-compose.override.yml + +solr-splainer-package/target +solr-splainer-package/repo/*.jar diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..293b4a2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +Version numbers correspond to `package.json` version. Follows the _major.minor.bugfix_ naming pattern. + +# 2.20.1 (2023-08-07) +- Introduce versioning to the Splainer project. The http://splainer.io website has been around for years, and we've just pushed changes as they arrive. We are now introducing a changelog process, and labeling the current state as "2.20.1" to match the commit https://github.com/o19s/splainer/commit/46cded05908e5d06ebee03f2cccaf836c60f9438#diff-7ae45ad102eab3b6d7e7896acd08c427a9b25b346470d7bc6507b6481575d519. +- Splainer.io can NOW be deployed into your Solr environment using [Solr Packages](https://solr.apache.org/guide/solr/latest/configuration-guide/package-manager.html). See the package [README.md](./solr-splainer-package/README.md) for more details. https://github.com/o19s/splainer/pull/97. diff --git a/solr-splainer-package/README.md b/solr-splainer-package/README.md new file mode 100644 index 0000000..85623d1 --- /dev/null +++ b/solr-splainer-package/README.md @@ -0,0 +1,88 @@ +# Splainer Solr Plugin + +This project lets you install Splainer into your Solr as a plugin. +You can access it via http://localhost:8983/v2/splainer and avoids common CORS and other network problems that you might encounter using the hosted http://splainer.io site. + + +## Building and installation + +This plugin requires a manual custom step added after you run the main `grunt dist` process in the Splainer application that builds a complete webapp in the `../dist` directory. _This should be automated someday ;-( _. + +1. Copy the following Javascript and paste it at the bottom of `/dist/scripts/app.js`: + +``` +/* Override default config values for talking to Solr + * JSONP->GET + * */ +angular.module('o19s.splainer-search') + .value('defaultSolrConfig', { + sanitize: true, + highlight: true, + debug: true, + numberOfRows: 10, + escapeQuery: true, + apiMethod: 'GET' + }); +``` + +1. Export the private key: + +``` +export SOLR_PACKAGE_SIGNING_PRIVATE_KEY_PATH=~/ssh/solr-private-key.pem +``` + +1. Build the package: + +``` +mvn package +``` + +1. Now for testing, host the solr-splainer-package/repo locally: + +First copy the generated jar into the repo directory: + +``` +cp target/solr-splainer-package* repo/ +``` + +``` +python -m http.server +``` + +1. In a Run Solr and install the package: + + tar -xf solr-9.3.0.tgz; cd solr-9.3.0/ + bin/solr start -c -e films -Denable.packages=true + bin/solr package add-repo splainer-dev "http://localhost:8000/repo/" + bin/solr package list-available + bin/solr package install solr-splainer + bin/solr package deploy solr-splainer -y -cluster + +1. Navigate to http://localhost:8983/v2/splainer on the browser. + +## Changes to make it work in Solr Admin UI + +See [diff from main project](https://github.com/o19s/splainer/compare/main...softwaredoug:solr-splainer:main#diff-18e01ac6a833fb1b20ffbad54f0ad8834a765e766f72cccda1e56cb942864d25R30) + +* Changes communication with Solr to use GET instead af JSONP, same way the Admin UI communicates with Solr + + + + + + + + + + +## Who? + +Based on Another Hack by [Doug Turnbull](http://softwaredoug.com) https://github.com/softwaredoug/solr-splainer. This works if you can't use Solr Packages in your environment. + +Created by [OpenSource Connections](http://opensourceconnections.com). + +Thanks to all the [community contributors](https://github.com/o19s/splainer/graphs/contributors) for finding bugs and sharing fixes!. + +## License + +Released under [Apache 2](LICENSE.txt) diff --git a/solr-splainer-package/pom.xml b/solr-splainer-package/pom.xml new file mode 100644 index 0000000..5715fe4 --- /dev/null +++ b/solr-splainer-package/pom.xml @@ -0,0 +1,112 @@ + + + + + + 4.0.0 + + com.o19s.splainer + solr-splainer-package + + 2.20.1 + + https://github.com/o19s/splainer/solr-splainer-package + + + + org.apache.solr + solr-core + 9.3.0 + + + + commons-io + commons-io + 2.11.0 + + + + org.projectlombok + lombok + 1.18.10 + provided + + + + + + + + maven-compiler-plugin + 3.8.1 + + 11 + 11 + utf-8 + + + + maven-resources-plugin + 3.1.0 + + utf-8 + ${project.build.directory} + + + ${basedir}/target/classes + ../dist + + + ${basedir}/target/classes + src/main/resources + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + + python + . + + release.py + ${project.artifactId} + ${project.version} + + + + python-build + package + + exec + + + + + + + + diff --git a/solr-splainer-package/release.py b/solr-splainer-package/release.py new file mode 100755 index 0000000..492459f --- /dev/null +++ b/solr-splainer-package/release.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import sys +import os +import json +import datetime + +artifact_name = sys.argv[1] +version = sys.argv[2] +if os.getenv("SOLR_PACKAGE_SIGNING_PRIVATE_KEY_PATH") == None: + raise Exception("Please add an environment variable SOLR_PACKAGE_SIGNING_PRIVATE_KEY_PATH to point to your private key (.pem) file used for signing the package artifacts.") + sys.exit(1) + +private_key_file = os.getenv("SOLR_PACKAGE_SIGNING_PRIVATE_KEY_PATH") + +if (os.path.isfile(private_key_file) == False): + raise Exception("SOLR_PACKAGE_SIGNING_PRIVATE_KEY_PATH points to non-existent private key (.pem) file used for signing the package artifacts.") + sys.exit(1) + +# Generate a publickey.der file +os.popen("openssl rsa -in "+private_key_file+" -outform DER -pubout -out repo/publickey.der") + +print("Signing artifact : target/"+artifact_name+"-"+version+".jar") + +# Sign the artifact using the private key +with os.popen("openssl dgst -sha1 -sign "+private_key_file+" target/"+artifact_name+"-"+version+".jar | openssl enc -base64") as f: + signature = "".join(f.readlines()).replace("\n", "") +print("Signed artifacts with: " + signature) + +# Update the repo/repository.json with the released artifact +repository = json.load(open("repo/repository.json")) + +# FORMAT: +# { +# "version": "1.0.0", +# "date": "2019-12-13", +# "artifacts": [ +# { +# "url": "question-answering-1.0.jar", +# "sig": "Hau46QF4424qUDSMwRYa/sO/L4Hfbdr6jLQDEsbJpXJdR6jPmd9v92mAU8wSMO/riVk/Zc4oovCCu2PRWnz7sA==" +# } +# ] +# } + +release = {} +release["version"] = version +release["date"] = str(datetime.date.today()) +release["artifacts"] = [{"url": artifact_name + "-" + version + ".jar", "sig": signature}] + +repository[0]["versions"].append(release) + +json.dump(repository, open('repo/repository.json', 'w'), indent=2) + +print(repository) diff --git a/solr-splainer-package/repo/publickey.der b/solr-splainer-package/repo/publickey.der new file mode 100644 index 0000000000000000000000000000000000000000..75494aac8967d700fab138a03a117e1a254ec518 GIT binary patch literal 94 zcmV-k0HOadTrdp=2`Yw2hW8Bt0RaU714{rfNCH6s;^NPUcQ~Y9+_Bb#o!6*kuFl}c zIKWb}fYJlNsvN^^2CR;SC0S=1l5m;_*(!@?WxVEzO(DB+llaa~sJp;70s{d60mf7( ATL1t6 literal 0 HcmV?d00001 diff --git a/solr-splainer-package/repo/repository.json b/solr-splainer-package/repo/repository.json new file mode 100644 index 0000000..4c6d73e --- /dev/null +++ b/solr-splainer-package/repo/repository.json @@ -0,0 +1,18 @@ +[ + { + "name": "solr-splainer", + "description": "Splainer for Solr", + "versions": [ + { + "version": "2.20.1", + "date": "2023-08-07", + "artifacts": [ + { + "url": "solr-splainer-package-2.20.1.jar", + "sig": "fKkR6CciTnNpMNBhXbtAdZfYxlmNFc2UuE64ljJ4B9f/Hj3Of5U6LrTj7Y8PfrioO6QF8DqXyd773WmZ9ncnWw==" + } + ] + } + ] + } +] diff --git a/solr-splainer-package/src/main/java/com/o19s/splainer/SplainerHandler.java b/solr-splainer-package/src/main/java/com/o19s/splainer/SplainerHandler.java new file mode 100644 index 0000000..87e7870 --- /dev/null +++ b/solr-splainer-package/src/main/java/com/o19s/splainer/SplainerHandler.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.o19s.splainer; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import org.apache.commons.io.IOUtils; +import org.apache.http.entity.ContentType; +import org.apache.lucene.util.ResourceLoader; +import org.apache.lucene.util.ResourceLoaderAware; +import org.apache.solr.api.Command; +import org.apache.solr.api.EndPoint; +import org.apache.solr.client.solrj.SolrRequest.METHOD; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.ReplicationHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.security.PermissionNameProvider; + +@EndPoint( + method = METHOD.GET, + path = "$path-prefix/*", + permission = PermissionNameProvider.Name.CONFIG_READ_PERM +) +@RequiredArgsConstructor +public class SplainerHandler implements ResourceLoaderAware { + @SuppressWarnings("unused") + private final CoreContainer coreContainer; + + private ResourceLoader loader = null; + + @Override + public void inform(ResourceLoader loader) { + System.out.println("inform"); + this.loader = loader; + } + + @Command + public void call(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { + System.out.println("Splainer: call" + req); + String path = req.getHttpSolrCall().getPath(); + String filepath = resolveFilePath(path); + + final InputStream inputStream = loader.openResource(filepath); + if (inputStream == null) { + throw new SolrException(ErrorCode.NOT_FOUND, "File not found: " + filepath); + } + + final byte[] data; + final String contentType; + + if ("".equals(filepath)) { + String indexPath = path.endsWith("/") ? path + "index.html" : path + "/index.html"; + indexPath = indexPath.replaceAll("____v2", "v2"); + data = ("").getBytes( + StandardCharsets.UTF_8); + contentType = ContentType.TEXT_HTML.getMimeType(); + } else { + data = IOUtils.toByteArray(inputStream); + contentType = contentType(filepath); + } + final ModifiableSolrParams newParams = new ModifiableSolrParams(req.getOriginalParams()); + newParams.set(CommonParams.WT, ReplicationHandler.FILE_STREAM); + req.setParams(newParams); + + final SolrCore.RawWriter writer = new SolrCore.RawWriter() { + + @Override + public void write(OutputStream os) throws IOException { + os.write(data); + } + + @Override + public String getContentType() { + return contentType; + } + }; + + rsp.add(ReplicationHandler.FILE_STREAM, writer); + } + + private String contentType(final String filepath) { + final Map types = new HashMap<>(); + types.put("jpg", ContentType.IMAGE_JPEG.getMimeType()); + types.put("jpeg", ContentType.IMAGE_JPEG.getMimeType()); + types.put("png", ContentType.IMAGE_PNG.getMimeType()); + types.put("gif", ContentType.IMAGE_GIF.getMimeType()); + types.put("svg", ContentType.IMAGE_SVG.getMimeType()); + types.put("htm", ContentType.TEXT_HTML.getMimeType()); + types.put("html", ContentType.TEXT_HTML.getMimeType()); + types.put("json", ContentType.APPLICATION_JSON.getMimeType()); + types.put("xml", ContentType.APPLICATION_XML.getMimeType()); + types.put("js", "text/javascript"); + types.put("css", "text/css"); + + final String extension = filepath.split("\\.")[filepath.split("\\.").length - 1]; + return types.getOrDefault(extension, "text/plain"); + } + + private String resolveFilePath(String path) { + System.out.println("Splainer: resolveFilePath" + path); + // Path can be: /____v2/splainer/index.html + if (path.split("/").length < 3) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Can't parse path: " + path); + } + String basepath = "/" + path.split("/")[1] + "/" + path.split("/")[2]; + String filepath = path.substring(path.indexOf(basepath) + basepath.length()); + + if (filepath.startsWith("/")) { + filepath = filepath.substring(1); + } + return filepath; + } +} diff --git a/solr-splainer-package/src/main/resources/manifest.json b/solr-splainer-package/src/main/resources/manifest.json new file mode 100644 index 0000000..2d1bd57 --- /dev/null +++ b/solr-splainer-package/src/main/resources/manifest.json @@ -0,0 +1,37 @@ +{ + "version-constraint": "9 - 10", + "plugins": [ + { + "name": "splainer", + "type": "cluster", + "setup-command": { + "path": "/api/cluster/plugin", + "method": "POST", + "payload": { + "add": { + "name": "${package-name}:${plugin-name}", + "class": "solr-splainer:com.o19s.splainer.SplainerHandler", + "version": "${package-version}", + "path-prefix": "${SPLAINER-PATH-PREFIX}" + } + } + }, + "verify-command": { + "path": "/api/cluster/plugin", + "method": "GET", + "condition": "$['plugin'].['${package-name}:${plugin-name}'].['version']", + "expected": "${package-version}" + }, + "uninstall-command": { + "path": "/api/cluster/plugin", + "method": "POST", + "payload": { + "remove": "${package-name}:${plugin-name}" + } + } + } + ], + "parameter-defaults": { + "SPLAINER-PATH-PREFIX": "splainer" + } +}